package main import ( "context" "encoding/json" "fmt" "reflect" "strings" "testing" "time" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/lease" "github.com/novox/mesh-controller/internal/link" ) // novox/hq issue 318, replayed with only what the controller had before its fix, so it can be laid over the // older commit: the statements are read from the node-engine's JSON, as the controller hears them. // // 2026-10-08, 11:08 UTC: the walk of the builds waiting for a gate sent the laptop three moves in one send — // docker ba6058ab → c78b5fc9, node-tools 9a4140b4 → d4845f7c, openrazer 88135ad0 → c78b5fc9 — and judged // them together. openrazer's build put the operator's account in the group `openrazer` (ADR 0252); the // laptop said the account "relogin needed", and the daemon's unit failed in the account's own service // manager, which began before the group. The gate read "not yet healthy" for ten minutes, failed at its // bound, put openrazer back (which by ADR 0252 also took the group back), and the walk failed: docker and // node-tools, healthy at every judging, never reached the workstation, and every other walk was refused there. // // The rule's outcome: a wait for a person is not a failure. The send passes with the wait carried along, // nothing is put back, the walk goes on to the workstation, and openrazer's condition says, in one sentence // for the operator, that a new login is needed on the laptop. func TestReplay318(t *testing.T) { withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) open := aMesh(t) ctx := t.Context() inv := open.inventory keeper, _ := withConditionsInMemory(t) was := doctorFrom doctorFrom = nil t.Cleanup(func() { doctorFrom = was }) build := func(module, repository, commit string, asked time.Time) { m := catalogue.Manifest{Module: module, Version: "1"} if module == "openrazer" && commit == "c78b5fc9" { // The build that put the operator's account in the group `openrazer` (ADR 0252). m.Resources = []map[string]any{{"id": "operator", "type": "user", "name": "operator", "groups": []any{"openrazer"}}} } manifest, _ := json.Marshal(m) if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + module + "-" + commit, Module: module, Commit: commit, Repository: repository, Path: "modules/" + module, Manifest: manifest, Asked: asked, At: asked, Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + module + "-" + commit}}}); err != nil { t.Fatal(err) } if err := inv.RegisterModule(ctx, m, inventory.Source{ Repository: repository, Seat: "git", Path: "modules/" + module, BuiltFrom: commit, Head: commit, Asked: asked}); err != nil { t.Fatal(err) } } type move struct{ module, repository, from, to string } moves := []move{ {"docker", "novox/mesh-catalog", "ba6058ab", "c78b5fc9"}, {"node-tools", "novox/mesh-tools", "9a4140b4", "d4845f7c"}, {"openrazer", "novox/mesh-catalog", "88135ad0", "c78b5fc9"}, } machines := []string{"laptop", "workstation"} if _, err := inv.AddNode(ctx, "workstation"); err != nil { t.Fatal(err) } old, recent := time.Now().Add(-3*time.Hour), time.Now().Add(-time.Minute) for _, mv := range moves { build(mv.module, mv.repository, mv.from, old) for _, n := range machines { if _, err := inv.Assign(ctx, n, mv.module); err != nil { t.Fatal(err) } } } for _, n := range machines { sent := map[string]string{} for _, mv := range moves { sent[mv.module] = mv.from } if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n, sent); err != nil { t.Fatal(err) } } for _, mv := range moves { build(mv.module, mv.repository, mv.to, recent) } // What each machine says, as its node-engine words it: openrazer's account waits for a new login, and the // daemon's unit failed in that account's own manager; docker's container is healthy. statement := func(at time.Time) link.Health { raw := fmt.Sprintf(`{"contract": %d, "at": %q, "resources": [ {"module": "docker", "resource": "docker.engine", "kind": "service", "target": "docker.service", "state": "healthy", "since": %q}, {"module": "openrazer", "resource": "openrazer.operator", "kind": "account", "target": "operator", "account": "operator", "state": "unhealthy", "since": %q, "streak": 3, "reason": "relogin needed: operator is in the group openrazer, and its running session began before it was; log out of every session and in again, or reboot"}, {"module": "openrazer", "resource": "openrazer.daemon", "kind": "unit", "target": "openrazer-daemon.service", "account": "operator", "state": "unhealthy", "since": %q, "streak": 3, "reason": "failed in the account's own service manager (exit-code)"}]}`, link.ReadinessContract, at.Format(time.RFC3339Nano), at.Format(time.RFC3339Nano), at.Format(time.RFC3339Nano), at.Format(time.RFC3339Nano)) var h link.Health if err := json.Unmarshal([]byte(raw), &h); err != nil { t.Fatal(err) } return h } wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). wasWindow := gateWitnessWindow gateWitnessWindow = 0 t.Cleanup(func() { gateWitnessWindow = wasWindow }) t.Cleanup(func() { machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { modules, err := open.inventory.Assigned(ctx, node) if err != nil { return nil, err } sent, known, err := open.inventory.SentBuilds(ctx, node) if err != nil { return nil, err } return f.moves(node, modules, sent, known, all), nil } releaseHeard = func(context.Context, *stores) (map[string]bool, error) { return map[string]bool{"laptop": true, "workstation": true}, nil } var sends [][]string n := 0 sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { sends = append(sends, append([]string(nil), names...)) current, err := open.inventory.CurrentBuilds(ctx) if err != nil { return nil, err } for _, node := range names { modules, _ := open.inventory.Assigned(ctx, node) carried := map[string]string{} for _, m := range modules { carried[m] = current[m].Commit } n++ digest := fmt.Sprintf("d-%s-%d", node, n) if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil { return nil, err } if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { return nil, err } } return names, nil } // Every judging hears each machine's newest statement first, as the controller does between judgings: // the account's wait and the failed unit are raised as openrazer's condition, after the send. gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { now := time.Now() f := gateFacts{now: now, reports: map[string]inventory.Reported{}, engines: map[string]string{}, rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}, health: map[string]inventory.NodeHealth{}, judged: true} for _, m := range machines { if err := stateHealth(ctx, open.inventory, keeper, m, statement(now), now); err != nil { return f, err } f.served[m] = served{runtime: true, tools: map[string]bool{}} } reports, err := open.inventory.LastReports(ctx) if err != nil { return f, err } for _, r := range reports { f.reports[r.Node] = r } if f.health, err = open.inventory.Healths(ctx); err != nil { return f, err } f.open, f.openErr = keeper.Open(ctx) return f, nil } // The bound is reached at the first judging that is not a pass: what happened at 11:21 happens at once. gateWithin, gateEvery, gateBound = time.Hour, 0, 0 for i := 0; i < 12; i++ { advancePlans(ctx, open) time.Sleep(5 * time.Millisecond) } var p inventory.Plan plans, err := inv.RecentPlans(ctx, 10) if err != nil { t.Fatal(err) } for _, q := range plans { if q.Release != nil { p = q break } } if p.Release == nil { t.Fatal("no walk of the builds waiting for a gate was opened") } if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Done, machines) { t.Fatalf("the walk is %s on %v: %s; want it done on the laptop and then the workstation", p.State, p.Release.Done, p.Note) } if !reflect.DeepEqual(sends, [][]string{{"laptop"}, {"workstation"}}) { t.Fatalf("sent %v; want the laptop, then the workstation, and nothing put back", sends) } current, err := inv.CurrentBuilds(ctx) if err != nil { t.Fatal(err) } if current["openrazer"].Commit != "c78b5fc9" { t.Fatalf("openrazer is registered at %s: its build was put back for a wait for a person", current["openrazer"].Commit) } for _, mv := range moves { v, found, err := inv.GateOf(ctx, "build-"+mv.module+"-"+mv.to) if err != nil || !found || v.Verdict != inventory.GatePassed { t.Fatalf("%s's build %s: verdict %+v (found %v, %v); want a pass", mv.module, mv.to, v, found, err) } if mv.module == "openrazer" && !strings.Contains(v.Why, "relogin needed on ") { t.Errorf("openrazer's pass does not carry its wait for a person: %q", v.Why) } } // What the operator reads: one condition per machine for openrazer, saying a new login is needed there, // a warning for a person, and no fault of the build. list, err := keeper.Open(ctx) if err != nil { t.Fatal(err) } var said []string for _, c := range list { said = append(said, c.Key+": "+c.Summary) if c.Subject.ID == "openrazer.laptop" { if !strings.HasPrefix(c.Summary, "relogin needed on laptop") || c.Severity == conditions.Urgent || c.Resolver != conditions.ResolverOperator { t.Errorf("openrazer's condition on the laptop: %s %s %s %q", c.Key, c.Severity, c.Resolver, c.Summary) } } } if !strings.Contains(strings.Join(said, "\n"), "relogin needed on laptop") { t.Errorf("nothing says a new login is needed on the laptop:\n%s", strings.Join(said, "\n")) } }