mesh-tools
97a2d841c7
The controller plans config, templates and moved providers' provisions, hands bus accounts to the vault and judges the new manifest words, on the feature's SDK and node-engine validator (feature 57, config and provisioning).
591 lines
26 KiB
Go
591 lines
26 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"flag"
|
|
"fmt"
|
|
"os"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/broker"
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
"github.com/novox/mesh-controller/internal/testbus"
|
|
)
|
|
|
|
// The replays of the controller's incidents (novox/hq to-be 45 §9, M9): each a scripted replay of what
|
|
// happened, asserting the rule's outcome rather than the fix's mechanism, so it can be run against the
|
|
// commit before the fix and fail there, and against the fix and pass. **Written only with what the
|
|
// controller had before each fix** — the stores, register, assign, planFor, declarationFor — so the
|
|
// prover (mesh-lab replays/cmd/prove) can lay this file over the older commit and run it there.
|
|
//
|
|
// Registered in mesh-lab's replays/register.go with the fix each one proves; run by this repository's
|
|
// merge check on every pull request with the rest of the suite.
|
|
|
|
// **R263 — a consumer's identity never refuses its provider's machine.** On 2026-10-06 the network
|
|
// manager came to require the mesh's resolver; on a machine whose name made its identity 23 to 26
|
|
// characters against the one global bound of 20, the anchor — the resolver's holder, carrying every
|
|
// consumer's grant — could not compose, and no push to it could go through. The outcome asserted: the
|
|
// provider's machine composes whatever its consumers are called, and a provision that mints no
|
|
// credential holds no consumer to a key's length.
|
|
func TestReplay263AnIdentityTooLongNeverRefusesItsProvidersMachine(t *testing.T) {
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
register(t, open, catalogue.Manifest{Module: "resolver", Version: "1",
|
|
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}})
|
|
register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1",
|
|
Requires: []string{"wildcard-resolution"}})
|
|
for _, a := range [][2]string{{"anchor", "resolver"}, {"laptop", "networkmanager"}} {
|
|
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
|
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
|
}
|
|
}
|
|
// The consumer's machine composes first, as a push does: what it is sent is what its provider grants.
|
|
for _, node := range []string{"laptop", "anchor"} {
|
|
plan, settings, err := planFor(ctx, open, node)
|
|
if err != nil {
|
|
t.Fatalf("%s does not resolve: %v", node, err)
|
|
}
|
|
if _, err := declarationFor(ctx, open, node, plan, settings); err != nil {
|
|
t.Fatalf("%s cannot be sent anything — the consumer networkmanager on laptop, identified "+
|
|
"mesh_laptop_networkmanager (26 characters), refused it: %v", node, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// **R273 — a binding to a consumer's data does not move by itself.** On 2026-10-05 a rule written for
|
|
// the resolver re-bound every database consumer on the home server — which runs its own store, beside
|
|
// its applications' data — to the store seat's holder on the control node, which made each a new empty
|
|
// database; five applications ran on empty data for twenty hours. The outcome asserted: a consumer on a
|
|
// machine running its own store stays bound to it while another machine holds the store's seat; the
|
|
// resolver, which every holder answers alike, follows its seat.
|
|
func TestReplay273AConsumerBesideItsStoreStaysBoundToIt(t *testing.T) {
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
inv := open.inventory
|
|
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, m := range []catalogue.Manifest{
|
|
{Module: "store", Version: "1",
|
|
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}},
|
|
Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}},
|
|
Serves: map[string]map[string]any{"postgres-database": {"port": 5432}},
|
|
Grants: map[string]string{"postgres-database": "/var/lib/mesh/store/grants"}},
|
|
{Module: "resolver", Version: "1",
|
|
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}},
|
|
Claims: []catalogue.Claim{{Name: "mesh-dns-resolver", Scope: catalogue.ScopeMesh}}},
|
|
{Module: "network", Version: "1", Requires: []string{"wildcard-resolution"}},
|
|
{Module: "board", Version: "1", Requires: []string{"postgres-database"}},
|
|
} {
|
|
register(t, open, m)
|
|
}
|
|
assignAll := func(pairs ...[2]string) {
|
|
for _, a := range pairs {
|
|
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
|
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
|
}
|
|
}
|
|
}
|
|
// The control node holds the mesh's store and resolver seats; the home server runs its own of each.
|
|
assignAll([2]string{"anchor", "store"}, [2]string{"anchor", "resolver"})
|
|
for _, seat := range [][2]string{{"mesh-store", "store"}, {"mesh-dns-resolver", "resolver"}} {
|
|
if err := inv.HoldSeat(ctx, seat[0], catalogue.ScopeMesh, "anchor", seat[1]); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
assignAll([2]string{"laptop", "store"}, [2]string{"laptop", "resolver"}, [2]string{"laptop", "network"},
|
|
[2]string{"laptop", "board"})
|
|
|
|
plan, _, err := planFor(ctx, open, "laptop")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var board, network string
|
|
for _, n := range plan.Needs {
|
|
switch {
|
|
case n.For == "board" && n.Name == "postgres-database":
|
|
board = n.From
|
|
case n.For == "network" && n.Name == "wildcard-resolution":
|
|
network = n.From
|
|
}
|
|
}
|
|
if board != "laptop" {
|
|
t.Fatalf("the consumer beside its store was bound to the store on %q, which would make it a new, empty "+
|
|
"database there (issue 273)", board)
|
|
}
|
|
if network != "anchor" {
|
|
t.Fatalf("the resolver was bound to %q; its seat is held on anchor (issue 258)", network)
|
|
}
|
|
}
|
|
|
|
// **R-crashloop — a module that crash-loops after it applied fails its gate on its first machine** (novox/hq
|
|
// ADR 0240; research 032 §6). On the home server the agent server restarted about a hundred times while the
|
|
// mesh read it applied, its tools served and no condition raised — the gate judged a module by what the
|
|
// mesh saw from outside, and nothing looked at what it ran. The outcome asserted: a build whose container
|
|
// exits at start never passes its gate on the first machine, is put back there at the bound, and never
|
|
// reaches the second.
|
|
//
|
|
// The machine is heard as a node-engine says it: its report of the apply, then the same account said again
|
|
// later, each carrying what the engine states of what it runs — `starting` as the apply ends, then the
|
|
// crash loop. What it states of the crash loop is MESH_REPLAY_STATEMENT when the lab's replay ran the
|
|
// engine against a real container (mesh-lab replays, R-crashloop), and otherwise what the engine said of
|
|
// one, kept below. Heard as bytes, so an older controller reads them as it reads any report — this file is
|
|
// written only with what the controller had before the judging, for the prover to lay over that commit.
|
|
func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) {
|
|
withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks)
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
inv := open.inventory
|
|
|
|
crashLoop := []byte(`{"contract":1,"at":"2026-10-07T00:00:00Z","resources":[{"module":"app","resource":"app.server",` +
|
|
`"kind":"container","target":"app-server","state":"unhealthy","reason":"restarting","since":"2026-10-07T00:00:00Z",` +
|
|
`"streak":5,"restarts":2}]}`)
|
|
if path := os.Getenv("MESH_REPLAY_STATEMENT"); path != "" {
|
|
raw, err := os.ReadFile(path)
|
|
if err != nil {
|
|
t.Fatalf("the engine's statement of the crash loop: %v", err)
|
|
}
|
|
crashLoop = raw
|
|
}
|
|
// `null` is an engine that states nothing — older than the judging — and its reports carry no health.
|
|
var stated map[string]any
|
|
if err := json.Unmarshal(crashLoop, &stated); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
for _, b := range []inventory.Build{
|
|
{ID: "build-1", Module: "app", Commit: "c1", Repository: "novox/mesh-catalog", Path: "modules/app",
|
|
Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)},
|
|
{ID: "build-2", Module: "app", Commit: "c2", Repository: "novox/mesh-catalog", Path: "modules/app",
|
|
Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)},
|
|
} {
|
|
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: b.Commit})
|
|
b.Manifest = manifest
|
|
if err := inv.RecordBuild(ctx, b); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
registerAt := func(commit string, asked time.Time) {
|
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: commit},
|
|
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: commit,
|
|
Head: commit, Asked: asked}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
registerAt("c1", time.Now().Add(-2*time.Hour))
|
|
for _, n := range []string{"anchor", "laptop"} {
|
|
if _, err := inv.Assign(ctx, n, "app"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"app": "c1"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
registerAt("c2", time.Now().Add(-time.Minute))
|
|
|
|
// The machine: what it is sent is applied, and its report says what runs is starting.
|
|
listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}}
|
|
sequence := int64(0)
|
|
say := func(node, digest, state string) {
|
|
t.Helper()
|
|
sequence++
|
|
health := map[string]any{}
|
|
for k, v := range stated {
|
|
health[k] = v
|
|
}
|
|
health["at"] = time.Now().UTC().Format(time.RFC3339Nano)
|
|
if state != "" && stated != nil {
|
|
resources := []any{}
|
|
for _, r := range stated["resources"].([]any) {
|
|
kept := map[string]any{}
|
|
for k, v := range r.(map[string]any) {
|
|
kept[k] = v
|
|
}
|
|
kept["state"], kept["reason"] = state, ""
|
|
resources = append(resources, kept)
|
|
}
|
|
health["resources"] = resources
|
|
}
|
|
said := map[string]any{"node": node, "applied": []string{"app.server"}, "declared": digest,
|
|
"report_sequence": sequence}
|
|
if stated != nil {
|
|
said["health"] = health
|
|
}
|
|
body, _ := json.Marshal(said)
|
|
var report link.Report
|
|
if err := json.Unmarshal(body, &report); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := listener.Heard(ctx, report); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
var sent [][]string
|
|
last := map[string]string{}
|
|
n := 0
|
|
wasSend := sendRollout
|
|
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
|
sent = append(sent, append([]string(nil), names...))
|
|
current, err := open.inventory.CurrentBuilds(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, node := range names {
|
|
n++
|
|
digest := fmt.Sprintf("d-%s-%d", node, n)
|
|
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, map[string]string{"app": current["app"].Commit}); err != nil {
|
|
return nil, err
|
|
}
|
|
last[node] = digest
|
|
say(node, digest, "starting")
|
|
}
|
|
return names, nil
|
|
}
|
|
t.Cleanup(func() { sendRollout = wasSend })
|
|
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
|
|
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
|
|
wasWindow := gateWitnessWindow
|
|
gateWitnessWindow = 0
|
|
t.Cleanup(func() { gateWitnessWindow = wasWindow })
|
|
gateWithin, gateEvery = time.Hour, 0
|
|
t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
|
|
|
|
built := time.Now().UTC()
|
|
plan := inventory.Plan{ID: "plan-crashloop", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
|
|
Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
|
|
Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-2"}}}
|
|
if err := inv.SavePlan(ctx, &plan); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
advancePlans(ctx, open) // the first machine is sent the new build, and starts it
|
|
if len(sent) == 0 || !slices.Equal(sent[0], []string{"anchor"}) {
|
|
t.Fatalf("sent %v, not the first machine first", sent)
|
|
}
|
|
advancePlans(ctx, open) // judged while it starts
|
|
// Its container exits at start, and the runtime restarts it: the machine says so, again and again.
|
|
for i := 0; i < 6 && len(sent) == 1; i++ {
|
|
say("anchor", last["anchor"], "")
|
|
advancePlans(ctx, open)
|
|
}
|
|
gateBound = -time.Second // and the bound passes
|
|
advancePlans(ctx, open)
|
|
|
|
p, err := inv.PlanByID(ctx, "plan-crashloop")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
gate := p.Modules["app"].Gate
|
|
for _, names := range sent {
|
|
if slices.Contains(names, "laptop") {
|
|
t.Fatalf("the crash loop passed its gate on anchor and was sent to laptop: sent %v, the gate %+v", sent, gate)
|
|
}
|
|
}
|
|
if gate == nil || gate.Verdict != inventory.GateFailed || p.State != inventory.PlanFailed {
|
|
t.Fatalf("a crash-looping build did not fail its gate on the first machine: the plan is %s (%s), its gate %+v",
|
|
p.State, p.Note, gate)
|
|
}
|
|
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" {
|
|
t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit)
|
|
}
|
|
}
|
|
|
|
// **R145 — a web application that accepts TCP and answers nothing is raised within two looks** (novox/hq
|
|
// ADR 0240 rule 4 and Phase B, issue 145). For eleven hours a web application's port was open and its
|
|
// program ran while every request hung, and the mesh said its machine was healthy; a person found it.
|
|
// Liveness cannot see it and a TCP check cannot either: the port is open. The module's declared HTTP check
|
|
// can. The engine's half (mesh-host internal/liveness TestReplaySilentWebAppIsSaidUnhealthy) states what it
|
|
// found looking at such a program; here the controller hears that statement on two looks in a row and
|
|
// raises the module's condition — the second, never the first. `null` is an engine older than the
|
|
// readiness check: it states the program alive, and nothing is raised.
|
|
func TestReplaySilentWebAppIsRaisedWithinTwoLooks(t *testing.T) {
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
silent := []byte(`{"contract":2,"at":"2026-10-07T00:00:00Z","resources":[{"module":"app","resource":"app.server",` +
|
|
`"kind":"container","target":"app-server","state":"unhealthy","reason":"http / on web: no answer within 5s",` +
|
|
`"since":"2026-10-07T00:00:00Z","streak":3,"check":"http"}]}`)
|
|
if path := os.Getenv("MESH_REPLAY_STATEMENT"); path != "" {
|
|
raw, err := os.ReadFile(path)
|
|
if err != nil {
|
|
t.Fatalf("the engine's statement of the silent web application: %v", err)
|
|
}
|
|
silent = raw
|
|
}
|
|
var h link.Health
|
|
if err := json.Unmarshal(silent, &h); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if h.Contract == 0 {
|
|
t.Fatal("the engine states nothing of the silent web application: it is older than the judging")
|
|
}
|
|
open1 := func() []conditions.Condition {
|
|
t.Helper()
|
|
list, err := conditionsFrom.Open(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return list
|
|
}
|
|
for look := 1; look <= 2; look++ {
|
|
at := time.Now().Add(time.Duration(look) * time.Second)
|
|
h.At = at
|
|
if err := stateHealth(ctx, open.inventory, conditionsFrom, "anchor", h, at); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
raised := open1()
|
|
switch {
|
|
case look == 1 && len(raised) != 0:
|
|
t.Fatalf("one look raised %v", raised[0].Key)
|
|
case look == 2 && (len(raised) != 1 || raised[0].Key != "module.app.anchor.unhealthy"):
|
|
t.Fatalf("two looks in a row did not raise the module's condition: %v", raised)
|
|
case look == 2:
|
|
t.Logf("raised on the second look: %s", raised[0].Summary)
|
|
}
|
|
}
|
|
}
|
|
|
|
// **R296 — a plan's clock counts from its tier, not from its last save.** On 2026-10-07 a plan for a
|
|
// merge to the catalogue, its one module's build asked at 19:48:41 and queued at the build seat for
|
|
// minutes, read "tier 1 of 1, building for 1s", then 4s, then 9s: the line counted from the plan's last
|
|
// save, and every advance — on the 30-second tick and after every build outcome on the mesh — saved it
|
|
// whether or not anything moved. A clock reset by each save is never LATE, so `plans` and `status` could
|
|
// not say a plan was stuck. The outcome asserted: a plan in its tier for forty minutes, saved seconds ago,
|
|
// says forty minutes and LATE.
|
|
func TestReplay296APlansLineCountsFromWhenItEnteredItsTier(t *testing.T) {
|
|
now := time.Date(2026, 10, 7, 20, 30, 0, 0, time.UTC)
|
|
entered := time.Date(2026, 10, 7, 19, 48, 41, 0, time.UTC)
|
|
asked := entered
|
|
p := inventory.Plan{ID: "plan-296", Repository: "novox/mesh-catalog", Branch: "main", Commit: "3ad22356",
|
|
State: inventory.PlanBuilding, Created: entered, TierEntered: entered, Updated: now.Add(-4 * time.Second),
|
|
Tiers: [][]string{{"app"}}, Modules: map[string]*inventory.PlanModule{"app": {State: "asked", AskedAt: &asked}}}
|
|
line := planLine(p, now)
|
|
if !strings.Contains(line, "for 41m19s") || !strings.Contains(line, "LATE") {
|
|
t.Fatalf("a plan in its tier since 19:48:41, read at 20:30:00 and saved 4s before, reads %q", line)
|
|
}
|
|
}
|
|
|
|
// **R296, the save.** The same live case from the store's side: a plan whose only build was asked and
|
|
// still waits is advanced as the tick and the mesh's build outcomes advance it, and is not written again —
|
|
// its revision and its last save stay where its last change left them. Before the fix each advance saved
|
|
// it, and its revision went from 3 to 6.
|
|
func TestReplay296APlanStandingStillInItsTierIsNotSavedAgain(t *testing.T) {
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
inv := open.inventory
|
|
asked := asksRecorded(t)
|
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"},
|
|
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main",
|
|
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "3ad22356aa",
|
|
Paths: []string{"modules/app/index.ts"}, ModuleDirs: []string{"modules/app"}, ModuleDirsSaid: true}
|
|
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
advanceHeld(ctx, open)
|
|
recent, err := inv.RecentPlans(ctx, 1)
|
|
if err != nil || len(recent) != 1 || len(*asked) != 1 {
|
|
t.Fatalf("no plan waiting on its build: %v %v, asked %v", recent, err, *asked)
|
|
}
|
|
before := recent[0]
|
|
if before.State != inventory.PlanBuilding || before.Modules["app"] == nil || before.Modules["app"].State != "asked" {
|
|
t.Fatalf("the plan is not waiting on its build: %+v", before)
|
|
}
|
|
for range 3 {
|
|
advanceHeld(ctx, open)
|
|
}
|
|
after, err := inv.PlanByID(ctx, before.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if after.Revision != before.Revision || !after.Updated.Equal(before.Updated) {
|
|
t.Fatalf("a plan that did not move was saved again: revision %d → %d, saved %s → %s",
|
|
before.Revision, after.Revision, before.Updated, after.Updated)
|
|
}
|
|
}
|
|
|
|
// **R299 — a holder of a seat whose verbs are all optional is not silent.** On 2026-10-07 the
|
|
// node-uplink seat gained its first verbs, `resolvers` and `links`, both optional while its holders catch
|
|
// up (ADR 0246, step 1); its holders served neither, so they registered nothing on the bus's discovery for
|
|
// it, and the self-check's D3 raised `seat.node-uplink.<machine>.silent` on every machine of the mesh,
|
|
// which the healer then acted on. The outcome asserted: through D3 itself — the seats read back from the
|
|
// store as the controller reads them, a holder on record on every machine heard from, and nothing
|
|
// answering the discovery — no holder of a seat whose verbs are all optional is said silent.
|
|
//
|
|
// The live seat is replayed while its verbs are all optional; beside it, a seat whose verbs are all
|
|
// optional by definition, so the outcome is still held once node-uplink's verbs are required (step 3).
|
|
func TestReplay299AHolderOfASeatWhoseVerbsAreAllOptionalIsNotSilent(t *testing.T) {
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
inv := open.inventory
|
|
catchingUp := catalogue.Seat{Name: "node-catching-up", Scope: catalogue.ScopeNode, Decision: "novox/hq issue 299",
|
|
Serves: []catalogue.Verb{{Name: "first", Optional: true}, {Name: "second", Optional: true}}}
|
|
if _, err := inv.SeedSeats(ctx, append(catalogue.DefaultSeats(), catchingUp)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
rows, err := inv.Seats(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// The working set as the serving controller holds it: the store's rows, the compiled seats' marks.
|
|
// A seat the compiled set does not know keeps its mark only as defined here.
|
|
for i := range rows {
|
|
if rows[i].Name == catchingUp.Name {
|
|
rows[i].Serves = catchingUp.Serves
|
|
}
|
|
}
|
|
catalogue.UseSeats(rows)
|
|
t.Cleanup(func() { catalogue.UseSeats(catalogue.DefaultSeats()) })
|
|
|
|
seats := []string{catchingUp.Name}
|
|
if uplink, ok := catalogue.SeatNamed("node-uplink"); ok && len(uplink.Serves) > 0 {
|
|
allOptional := true
|
|
for _, v := range uplink.Serves {
|
|
allOptional = allOptional && v.Optional
|
|
}
|
|
if allOptional {
|
|
seats = append(seats, "node-uplink")
|
|
} else {
|
|
t.Logf("node-uplink's verbs are required at this commit; the live seat is not replayed, its kind is")
|
|
}
|
|
}
|
|
|
|
register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1"})
|
|
now := time.Now()
|
|
var heard []machineFacts
|
|
for _, node := range []string{"anchor", "laptop"} {
|
|
if _, err := inv.Assign(ctx, node, "networkmanager"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, seat := range seats {
|
|
if err := inv.HoldSeat(ctx, seat, catalogue.ScopeNode, node, "networkmanager"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
heard = append(heard, machineFacts{name: node, lastHeard: now})
|
|
}
|
|
js, err := broker.Dial(testbus.URL(t))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(js.Close)
|
|
d := &doctor{open: open, js: js, watchdogs: &watchdogs{last: &signalFacts{now: now, machines: heard}}}
|
|
|
|
found, err := probeHolders(ctx, d)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, o := range found {
|
|
for _, seat := range seats {
|
|
if strings.HasPrefix(o.ID, seat+".") {
|
|
t.Errorf("a holder serving none of a seat's optional verbs was said silent: %s (%s)", o.Summary, o.Said)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// **R300 — a merge that adds a module builds it.** On 2026-10-07 a pull request to the catalogue added
|
|
// one module, systemd-resolved; its delivery plan said "builds new: modules/systemd-resolved, sent
|
|
// nowhere", and at the merge the controller said it "changed nothing any module the mesh holds is built
|
|
// from" and built nothing, so the module was never registered and `assign` refused it until a person
|
|
// built it by hand. The outcome asserted: the merge asks the build seat for the new module's directory, at
|
|
// the branch merged into, from the repository as the mesh spells it, and opens no plan for it.
|
|
func TestReplay300AMergeAddingAModuleAsksForItsBuild(t *testing.T) {
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
inv := open.inventory
|
|
var asked [][3]string
|
|
was := askABuild
|
|
askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) {
|
|
asked = append(asked, [3]string{source.Repository, path, ref})
|
|
return "build-" + path, nil
|
|
}
|
|
t.Cleanup(func() { askABuild = was })
|
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "networkmanager", Version: "1"},
|
|
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/networkmanager", Ref: "main",
|
|
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "3da80a4b00",
|
|
Paths: []string{"modules/systemd-resolved/module.json", "modules/systemd-resolved/cmd/main.go"},
|
|
ModuleDirs: []string{"modules/systemd-resolved"}, ModuleDirsSaid: true}
|
|
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := [3]string{"novox/mesh-catalog", "modules/systemd-resolved", "main"}
|
|
if len(asked) != 1 || asked[0] != want {
|
|
t.Fatalf("the merge adding modules/systemd-resolved asked the build seat for %v, not %v", asked, want)
|
|
}
|
|
if plans, err := inv.OpenPlans(ctx); err != nil || len(plans) != 0 {
|
|
t.Fatalf("a merge moving nothing the mesh holds opened a plan: %+v %v", plans, err)
|
|
}
|
|
}
|
|
|
|
// **R301 — a person's push of a held recorded build is no repair.** On 2026-10-07 the operator approved,
|
|
// machine by machine, new builds of modules whose upgrade policy records rather than rolls out — the
|
|
// resolver, the FortiClient adapter, the network managers, the packet filter, mail. A recorded build
|
|
// moves only by a person's push (ADR 0242), so each was `push <machine>` through the seat with the
|
|
// operator's yes as its why and a word for the change as its cause. The hand-act log counted every one
|
|
// as a repair, and S15 raised `mesh.hand-acts.split-dns.healer-wanted` ("repaired by hand 4 times in 14
|
|
// days … a healer is wanted"), and the same for `words` and `uplink-verbs`. The outcome asserted: a push
|
|
// that carries only held recorded builds, recorded as the push records it, wants no healer however
|
|
// often the person gives that word — through S15 itself, reading the log as the watchdog reads it.
|
|
func TestReplay301APersonsPushOfAHeldRecordedBuildIsNoRepair(t *testing.T) {
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
inv := open.inventory
|
|
start := time.Now().Add(-time.Hour)
|
|
image := "registry.invalid:5000/resolver/server@sha256:" + strings.Repeat("e", 64)
|
|
if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, aContainerBuild(t, "resolver", "c1111111", catalogue.PolicyRecord, start,
|
|
map[string][2]string{"server": {image, ""}}))); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, node := range []string{"anchor", "laptop"} {
|
|
if _, err := inv.Assign(ctx, node, "resolver"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := inv.RecordSent(ctx, nodeID(t, open, node), "d-"+node, map[string]string{"resolver": "c1111111"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
// The merge: the resolver's new build, registered and held for a person's word.
|
|
if _, _, err := takeIn(ctx, inv, aContainerBuild(t, "resolver", "c2222222", catalogue.PolicyRecord,
|
|
start.Add(time.Minute), map[string][2]string{"server": {image, "runtime"}})); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
conn := onATestBus(t)
|
|
// The operator's yes, machine by machine, as the seat's push records it before it sends.
|
|
for _, node := range []string{"anchor", "laptop"} {
|
|
set := flag.NewFlagSet("push", flag.ContinueOnError)
|
|
why := addHandActFlags(set)
|
|
if err := set.Parse([]string{"--why", "operator's yes: the resolver's new build on " + node,
|
|
"--cause", "split-dns"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
why.record(ctx, "push", []string{node})
|
|
}
|
|
acts, err := link.HandActs(ctx, conn, start)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(acts) != 2 {
|
|
t.Fatalf("the two pushes were recorded as %d act(s): %+v", len(acts), acts)
|
|
}
|
|
f := calm(time.Now())
|
|
f.handActs = acts
|
|
for _, o := range watchHandActs(f) {
|
|
if strings.Contains(o.ID, "split-dns") {
|
|
t.Fatalf("a person's push of a held recorded build was counted as a repair: %s — %s", o.ID, o.Summary)
|
|
}
|
|
}
|
|
}
|