jochen
a63939160e
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery superseded: a newer delivery to the same trunk took over its walk
251 lines
10 KiB
Go
251 lines
10 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/lease"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// novox/hq issue 318, replayed with only what the controller had before its fix, so it can be laid over the
|
|
// older commit: the statements are read from the node-engine's JSON, as the controller hears them.
|
|
//
|
|
// 2026-10-08, 11:08 UTC: the walk of the builds waiting for a gate sent the laptop three moves in one send —
|
|
// docker ba6058ab → c78b5fc9, node-tools 9a4140b4 → d4845f7c, openrazer 88135ad0 → c78b5fc9 — and judged
|
|
// them together. openrazer's build put the operator's account in the group `openrazer` (ADR 0252); the
|
|
// laptop said the account "relogin needed", and the daemon's unit failed in the account's own service
|
|
// manager, which began before the group. The gate read "not yet healthy" for ten minutes, failed at its
|
|
// bound, put openrazer back (which by ADR 0252 also took the group back), and the walk failed: docker and
|
|
// node-tools, healthy at every judging, never reached the workstation, and every other walk was refused there.
|
|
//
|
|
// The rule's outcome: a wait for a person is not a failure. The send passes with the wait carried along,
|
|
// nothing is put back, the walk goes on to the workstation, and openrazer's condition says, in one sentence
|
|
// for the operator, that a new login is needed on the laptop.
|
|
func TestReplay318(t *testing.T) {
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
inv := open.inventory
|
|
keeper, _ := withConditionsInMemory(t)
|
|
was := doctorFrom
|
|
doctorFrom = nil
|
|
t.Cleanup(func() { doctorFrom = was })
|
|
|
|
build := func(module, repository, commit string, asked time.Time) {
|
|
m := catalogue.Manifest{Module: module, Version: "1"}
|
|
if module == "openrazer" && commit == "c78b5fc9" {
|
|
// The build that put the operator's account in the group `openrazer` (ADR 0252).
|
|
m.Resources = []map[string]any{{"id": "operator", "type": "user", "name": "operator",
|
|
"groups": []any{"openrazer"}}}
|
|
}
|
|
manifest, _ := json.Marshal(m)
|
|
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + module + "-" + commit, Module: module, Commit: commit,
|
|
Repository: repository, Path: "modules/" + module, Manifest: manifest, Asked: asked, At: asked,
|
|
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + module + "-" + commit}}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := inv.RegisterModule(ctx, m, inventory.Source{
|
|
Repository: repository, Seat: "git", Path: "modules/" + module, BuiltFrom: commit, Head: commit,
|
|
Asked: asked}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
type move struct{ module, repository, from, to string }
|
|
moves := []move{
|
|
{"docker", "novox/mesh-catalog", "ba6058ab", "c78b5fc9"},
|
|
{"node-tools", "novox/mesh-tools", "9a4140b4", "d4845f7c"},
|
|
{"openrazer", "novox/mesh-catalog", "88135ad0", "c78b5fc9"},
|
|
}
|
|
machines := []string{"laptop", "workstation"}
|
|
if _, err := inv.AddNode(ctx, "workstation"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
old, recent := time.Now().Add(-3*time.Hour), time.Now().Add(-time.Minute)
|
|
for _, mv := range moves {
|
|
build(mv.module, mv.repository, mv.from, old)
|
|
for _, n := range machines {
|
|
if _, err := inv.Assign(ctx, n, mv.module); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
for _, n := range machines {
|
|
sent := map[string]string{}
|
|
for _, mv := range moves {
|
|
sent[mv.module] = mv.from
|
|
}
|
|
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n, sent); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for _, mv := range moves {
|
|
build(mv.module, mv.repository, mv.to, recent)
|
|
}
|
|
|
|
// What each machine says, as its node-engine words it: openrazer's account waits for a new login, and the
|
|
// daemon's unit failed in that account's own manager; docker's container is healthy.
|
|
statement := func(at time.Time) link.Health {
|
|
raw := fmt.Sprintf(`{"contract": %d, "at": %q, "resources": [
|
|
{"module": "docker", "resource": "docker.engine", "kind": "service", "target": "docker.service",
|
|
"state": "healthy", "since": %q},
|
|
{"module": "openrazer", "resource": "openrazer.operator", "kind": "account", "target": "operator",
|
|
"account": "operator", "state": "unhealthy", "since": %q, "streak": 3,
|
|
"reason": "relogin needed: operator is in the group openrazer, and its running session began before it was; log out of every session and in again, or reboot"},
|
|
{"module": "openrazer", "resource": "openrazer.daemon", "kind": "unit", "target": "openrazer-daemon.service",
|
|
"account": "operator", "state": "unhealthy", "since": %q, "streak": 3,
|
|
"reason": "failed in the account's own service manager (exit-code)"}]}`,
|
|
link.ReadinessContract, at.Format(time.RFC3339Nano), at.Format(time.RFC3339Nano), at.Format(time.RFC3339Nano),
|
|
at.Format(time.RFC3339Nano))
|
|
var h link.Health
|
|
if err := json.Unmarshal([]byte(raw), &h); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return h
|
|
}
|
|
|
|
wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts
|
|
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
|
t.Cleanup(func() {
|
|
machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
|
|
gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound
|
|
})
|
|
machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
|
|
modules, err := open.inventory.Assigned(ctx, node)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
sent, known, err := open.inventory.SentBuilds(ctx, node)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return f.moves(node, modules, sent, known, all), nil
|
|
}
|
|
releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
|
|
return map[string]bool{"laptop": true, "workstation": true}, nil
|
|
}
|
|
var sends [][]string
|
|
n := 0
|
|
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
|
sends = append(sends, append([]string(nil), names...))
|
|
current, err := open.inventory.CurrentBuilds(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, node := range names {
|
|
modules, _ := open.inventory.Assigned(ctx, node)
|
|
carried := map[string]string{}
|
|
for _, m := range modules {
|
|
carried[m] = current[m].Commit
|
|
}
|
|
n++
|
|
digest := fmt.Sprintf("d-%s-%d", node, n)
|
|
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
|
|
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return names, nil
|
|
}
|
|
// Every judging hears each machine's newest statement first, as the controller does between judgings:
|
|
// the account's wait and the failed unit are raised as openrazer's condition, after the send.
|
|
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
|
now := time.Now()
|
|
f := gateFacts{now: now, reports: map[string]inventory.Reported{}, engines: map[string]string{},
|
|
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}, health: map[string]inventory.NodeHealth{},
|
|
judged: true}
|
|
for _, m := range machines {
|
|
if err := stateHealth(ctx, open.inventory, keeper, m, statement(now), now); err != nil {
|
|
return f, err
|
|
}
|
|
f.served[m] = served{runtime: true, tools: map[string]bool{}}
|
|
}
|
|
reports, err := open.inventory.LastReports(ctx)
|
|
if err != nil {
|
|
return f, err
|
|
}
|
|
for _, r := range reports {
|
|
f.reports[r.Node] = r
|
|
}
|
|
if f.health, err = open.inventory.Healths(ctx); err != nil {
|
|
return f, err
|
|
}
|
|
f.open, f.openErr = keeper.Open(ctx)
|
|
return f, nil
|
|
}
|
|
// The bound is reached at the first judging that is not a pass: what happened at 11:21 happens at once.
|
|
gateSettle, gateEvery, gateBound = 0, 0, 0
|
|
|
|
for i := 0; i < 12; i++ {
|
|
advancePlans(ctx, open)
|
|
time.Sleep(5 * time.Millisecond)
|
|
}
|
|
|
|
var p inventory.Plan
|
|
plans, err := inv.RecentPlans(ctx, 10)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, q := range plans {
|
|
if q.Release != nil {
|
|
p = q
|
|
break
|
|
}
|
|
}
|
|
if p.Release == nil {
|
|
t.Fatal("no walk of the builds waiting for a gate was opened")
|
|
}
|
|
if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Done, machines) {
|
|
t.Fatalf("the walk is %s on %v: %s; want it done on the laptop and then the workstation", p.State,
|
|
p.Release.Done, p.Note)
|
|
}
|
|
if !reflect.DeepEqual(sends, [][]string{{"laptop"}, {"workstation"}}) {
|
|
t.Fatalf("sent %v; want the laptop, then the workstation, and nothing put back", sends)
|
|
}
|
|
current, err := inv.CurrentBuilds(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if current["openrazer"].Commit != "c78b5fc9" {
|
|
t.Fatalf("openrazer is registered at %s: its build was put back for a wait for a person", current["openrazer"].Commit)
|
|
}
|
|
for _, mv := range moves {
|
|
v, found, err := inv.GateOf(ctx, "build-"+mv.module+"-"+mv.to)
|
|
if err != nil || !found || v.Verdict != inventory.GatePassed {
|
|
t.Fatalf("%s's build %s: verdict %+v (found %v, %v); want a pass", mv.module, mv.to, v, found, err)
|
|
}
|
|
if mv.module == "openrazer" && !strings.Contains(v.Why, "relogin needed on ") {
|
|
t.Errorf("openrazer's pass does not carry its wait for a person: %q", v.Why)
|
|
}
|
|
}
|
|
// What the operator reads: one condition per machine for openrazer, saying a new login is needed there,
|
|
// a warning for a person, and no fault of the build.
|
|
list, err := keeper.Open(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var said []string
|
|
for _, c := range list {
|
|
said = append(said, c.Key+": "+c.Summary)
|
|
if c.Subject.ID == "openrazer.laptop" {
|
|
if !strings.HasPrefix(c.Summary, "relogin needed on laptop") || c.Severity == conditions.Urgent ||
|
|
c.Resolver != conditions.ResolverOperator {
|
|
t.Errorf("openrazer's condition on the laptop: %s %s %s %q", c.Key, c.Severity, c.Resolver, c.Summary)
|
|
}
|
|
}
|
|
}
|
|
if !strings.Contains(strings.Join(said, "\n"), "relogin needed on laptop") {
|
|
t.Errorf("nothing says a new login is needed on the laptop:\n%s", strings.Join(said, "\n"))
|
|
}
|
|
}
|