From b1546ed28a880c3109e66cc6f95ce630cc1e5b20 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 13 Oct 2026 00:53:10 +0200 Subject: [PATCH] Hold a walk's first-machine gate while its busy container finishes its build A merge into mesh-controller replaced every build agent and threw away the checks in flight (issue 594). The manifest's container busy is checked and sent only to an engine that reads it; a report's busy waits are kept, the gate starts at the report that says the container replaced, and the wait is a phase of the delivery time. mesh-builder busy drains the agent and answers while a build is in hand (novox/hq ADR 0335). --- cmd/mesh-builder/busy.go | 101 +++++ cmd/mesh-builder/busy_test.go | 77 ++++ cmd/mesh-builder/main.go | 29 +- cmd/mesh-builder/stdout_test.go | 1 + cmd/mesh-controller/busy.go | 84 ++++ cmd/mesh-controller/busy_test.go | 111 ++++++ cmd/mesh-controller/gate.go | 11 + cmd/mesh-controller/plan.go | 15 + go.mod | 2 +- go.sum | 4 +- internal/catalogue/busy.go | 124 ++++++ internal/catalogue/busy_test.go | 94 +++++ internal/catalogue/declaration.go | 7 + internal/catalogue/manifest.go | 2 + internal/inventory/busy_test.go | 142 +++++++ .../0096-a-report-says-a-busy-container.sql | 11 + internal/inventory/nodes.go | 95 ++++- internal/inventory/plans.go | 33 ++ internal/inventory/walkphases.go | 24 +- internal/link/builds_nats.go | 11 + internal/link/enrolment.go | 28 +- internal/link/protocol.go | 31 ++ testdata/store-tests | 1 + .../mesh-host/internal/declaration/busy.go | 87 +++++ .../internal/declaration/declaration.go | 359 +++++++++++++++++- vendor/modules.txt | 4 +- 26 files changed, 1463 insertions(+), 25 deletions(-) create mode 100644 cmd/mesh-builder/busy.go create mode 100644 cmd/mesh-builder/busy_test.go create mode 100644 cmd/mesh-controller/busy.go create mode 100644 cmd/mesh-controller/busy_test.go create mode 100644 internal/catalogue/busy.go create mode 100644 internal/catalogue/busy_test.go create mode 100644 internal/inventory/busy_test.go create mode 100644 internal/inventory/migrations/0096-a-report-says-a-busy-container.sql create mode 100644 vendor/github.com/novox/mesh-host/internal/declaration/busy.go diff --git a/cmd/mesh-builder/busy.go b/cmd/mesh-builder/busy.go new file mode 100644 index 00000000..d3cfdbc3 --- /dev/null +++ b/cmd/mesh-builder/busy.go @@ -0,0 +1,101 @@ +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +// **The build agent says whether it is busy, and being asked drains it** (novox/hq ADR 0335 §1, §5). Its +// container declares `busy` as `mesh-builder busy`: the node-engine runs it inside the container before it +// replaces it. It exits 0 while a build is in hand and 1 when none is; and from the first ask the agent takes +// no new build request, so its answer ends once the build in hand ends — a replaced builder finishes the check +// it runs, instead of the build queue handing it out again from the start (issue 594). +// +// Two marks in the container's own temporary directory, as the ready mark (ready.go): a new container starts +// without them, and a restarted one removes them before it takes work again. +// +// - in hand: written by the agent when it claims a request, before it reads whether it is drained; removed +// when the build ends. +// - drained: written by the ask, before it reads whether a build is in hand. +// +// Each side writes its own mark before it reads the other's, so an ask racing a fetch sees the build in hand, +// or the agent sees the drain and hands the request back; never neither. + +func inHandMark() string { return filepath.Join(os.TempDir(), "mesh-builder.in-hand") } +func drainMark() string { return filepath.Join(os.TempDir(), "mesh-builder.drained") } + +// drained says the agent was asked whether it is busy: it takes no new build request. +func drained() bool { + _, err := os.Stat(drainMark()) + return err == nil +} + +// claim marks a request in hand, unless the agent is drained: then nothing is in hand, and the request goes +// back to the queue. +func claim() bool { + // Made, never written over: one build at a time, and a mark already there is a build in hand. + f, err := os.OpenFile(inHandMark(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) + if err != nil { + // Unmarked, an ask would say idle while this build runs: the build is not taken. + fmt.Fprintf(os.Stderr, "cannot mark a build in hand (%s), so it is handed back: %v\n", inHandMark(), err) + return false + } + _, err = f.WriteString(time.Now().UTC().Format(time.RFC3339) + "\n") + if closeErr := f.Close(); err == nil { + err = closeErr + } + if err != nil { + _ = os.Remove(inHandMark()) + fmt.Fprintf(os.Stderr, "cannot mark a build in hand (%s), so it is handed back: %v\n", inHandMark(), err) + return false + } + if drained() { + _ = os.Remove(inHandMark()) + fmt.Fprintln(os.Stderr, "asked whether busy, so this agent takes no new build: handed back") + return false + } + return true +} + +// holding says what is in hand, for the ask to print: written once the request is read. +func holding(what string) { + _ = os.WriteFile(inHandMark(), []byte(time.Now().UTC().Format(time.RFC3339)+" "+what+"\n"), 0o644) +} + +// released ends the build in hand. +func released() { + if err := os.Remove(inHandMark()); err != nil && !errors.Is(err, os.ErrNotExist) { + fmt.Fprintf(os.Stderr, "cannot remove the mark of the build in hand (%s): %v\n", inHandMark(), err) + } +} + +// undrained clears both marks at the start of a holder: a container started again is asked afresh. +func undrained() error { + for _, mark := range []string{drainMark(), inHandMark()} { + if err := os.Remove(mark); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("cannot remove %s: %w", mark, err) + } + } + return nil +} + +// busy is the ask: it drains the agent, then says whether a build is in hand. Nil — exit 0 — while one is. +func busy() (string, error) { + if err := os.WriteFile(drainMark(), []byte(time.Now().UTC().Format(time.RFC3339)+"\n"), 0o644); err != nil { + // Not drained, it could take the next build: said, and answered busy only for what is in hand now. + fmt.Fprintf(os.Stderr, "cannot mark this agent drained (%s): %v\n", drainMark(), err) + } + held, err := os.ReadFile(inHandMark()) + if err != nil { + return "", errors.New("idle: no build in hand, and this agent takes no new one") + } + since, what, _ := strings.Cut(strings.TrimSpace(string(held)), " ") + if what == "" { + what = "a build" + } + return fmt.Sprintf("busy: %s, in hand since %s; taking no new build", what, since), nil +} diff --git a/cmd/mesh-builder/busy_test.go b/cmd/mesh-builder/busy_test.go new file mode 100644 index 00000000..356074aa --- /dev/null +++ b/cmd/mesh-builder/busy_test.go @@ -0,0 +1,77 @@ +package main + +import ( + "strings" + "sync" + "testing" +) + +// `mesh-builder busy` answers 0 while a build is in hand and non-zero when idle, and after its first ask the +// agent takes no new build request (novox/hq ADR 0335 §5). +func TestBusyAnswersWhileABuildIsInHandAndDrainsTheAgent(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + if err := undrained(); err != nil { + t.Fatal(err) + } + if drained() { + t.Fatal("an agent never asked is drained") + } + // A build is claimed and runs. + if !claim() { + t.Fatal("an agent never asked refused a build") + } + holding("novox/mesh-controller (check-280)") + said, err := busy() + if err != nil || !strings.Contains(said, "novox/mesh-controller (check-280)") { + t.Fatalf("an agent with a build in hand did not answer busy with what it holds: %q %v", said, err) + } + // From that ask on it takes nothing new: the fetch is not made, and one fetched is handed back. + if !drained() { + t.Fatal("the ask did not drain the agent") + } + if claim() { + t.Fatal("a drained agent took a new build") + } + // The build in hand is still in hand after the refused claim, and busy goes on saying so. + if _, err := busy(); err != nil { + t.Fatalf("a refused claim ended the build in hand: %v", err) + } + // It ends: idle, and still drained. + released() + if _, err := busy(); err == nil { + t.Fatal("an agent with nothing in hand answered busy") + } + if claim() { + t.Fatal("a drained agent took a build after its last one ended") + } + // Started again, it is asked afresh. + if err := undrained(); err != nil { + t.Fatal(err) + } + if drained() || !claim() { + t.Fatal("a holder started again is still drained") + } + released() +} + +// An ask racing a fetch sees the build in hand, or the agent sees the drain and hands the request back: never +// neither, which would replace the container under a build just taken. +func TestAnAskRacingAFetchNeverMissesTheBuild(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + for i := 0; i < 200; i++ { + if err := undrained(); err != nil { + t.Fatal(err) + } + var claimed bool + var askErr error + var wg sync.WaitGroup + wg.Add(2) + go func() { defer wg.Done(); claimed = claim() }() + go func() { defer wg.Done(); _, askErr = busy() }() + wg.Wait() + if claimed && askErr != nil { + t.Fatalf("round %d: the agent took a build and the ask said idle", i) + } + released() + } +} diff --git a/cmd/mesh-builder/main.go b/cmd/mesh-builder/main.go index e8ece0d8..7dedef61 100644 --- a/cmd/mesh-builder/main.go +++ b/cmd/mesh-builder/main.go @@ -71,6 +71,12 @@ digest of their own configuration. The result is printed as JSON. exits 0 once the agent running in this container is bound to its build work and pulls it, and 1 before: its container's health check. + + mesh-builder busy + +exits 0 while the agent running in this container has a build in hand, and 1 when it has none; +from the first ask the agent takes no new build: its container's busy command, asked before the +container is replaced (novox/hq ADR 0335). ` func run() error { @@ -84,6 +90,15 @@ func run() error { case "ready": // The container's health check (novox/hq issue 544): exits non-zero until the agent takes work. return ready() + case "busy": + // The container's busy command (novox/hq ADR 0335): drains the agent, and exits non-zero once + // nothing is in hand. + said, err := busy() + if err != nil { + return err + } + fmt.Println(said) + return nil default: fmt.Print(usage) return nil @@ -93,6 +108,9 @@ func run() error { if err := notReady(); err != nil { return err } + if err := undrained(); err != nil { + return err + } credential, err := brokerFrom() if err != nil { return err @@ -145,8 +163,13 @@ func run() error { // **What an earlier holder here left goes before anything is taken** (novox/hq issue 306): a holder // recreated mid-check left the check's store and bus running, and the ask went to another machine. h.removeLeftBehind() - machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused, - Taking: func() { taking(seat) }}) + // Paused or drained, it takes nothing new (novox/hq ADR 0219, ADR 0335): a drained agent finishes the build + // in hand and then says idle, and its container is replaced between builds. + machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{ + Paused: func() bool { return h.Paused() || drained() }, + Taking: func() { taking(seat) }, + Claim: claim, + }) defer machine.Close() // The seat's verbs, on this machine's subjects. Only the seat that declares them: the retired @@ -172,6 +195,8 @@ func run() error { publisher := builder.Registry{Address: registry, Run: builder.Command} return machine.Take(ctx, func(ctx context.Context, work link.Build) { + defer released() + holding(work.Request().Repository + " (" + work.Request().ID + ")") answer(ctx, publisher, on, workspace, work, h) }) } diff --git a/cmd/mesh-builder/stdout_test.go b/cmd/mesh-builder/stdout_test.go index ccba9169..f33e9401 100644 --- a/cmd/mesh-builder/stdout_test.go +++ b/cmd/mesh-builder/stdout_test.go @@ -16,6 +16,7 @@ func TestBuilderDiagnosticsStayOffStdout(t *testing.T) { "version)": true, // --version `"stopping")`: true, // the loop.s shutdown line "usage)": true, // --help text, for a human + "said)": true, // busy: the answer its node-engine reads (novox/hq ADR 0335) } for _, file := range []string{"once.go", "main.go"} { src, err := os.ReadFile(file) diff --git a/cmd/mesh-controller/busy.go b/cmd/mesh-controller/busy.go new file mode 100644 index 00000000..08eee531 --- /dev/null +++ b/cmd/mesh-controller/busy.go @@ -0,0 +1,84 @@ +package main + +import ( + "fmt" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// A first-node gate waits for a busy container (novox/hq ADR 0335 §4). +// +// When a machine reports, on what the gate sent it, that a container of a judged module was left running +// because it said busy, the judging has not started: the build is not yet what runs there. The walk says it +// waits for a busy container on that machine, counts no pass and no bound, and starts the judging — its +// Since — at the report that no longer lists the container waiting. The wait is the node-engine's, bounded +// at 15 minutes; the controller's own bound (busyGateBound past the engine's) is for an engine that stopped +// reporting while it waited, so no walk waits for ever. An engine older than the field never lists one, and +// its gate starts as before. + +// busyGateSlack is how long past the engine's bound the controller still waits for the report that says the +// container replaced: a pass of the engine (its asks every 20 seconds) and the report's way here. +var busyGateSlack = 5 * time.Minute + +// busyWait decides, before a judging, whether the gate waits for a busy container on a machine it judges: +// true while it waits, the gate saying so. False when nothing waits, or no longer: a wait that ended moves the +// gate's Since to its end. Pure but for the gate it is handed. +func busyWait(g *inventory.PlanGate, pairs []judged, f gateFacts, now time.Time) bool { + if g.Busy != nil && g.Busy.Ended != nil { + return false + } + for _, j := range pairs { + b, waits := busyOn(j, f) + if !waits { + continue + } + if g.Busy == nil { + g.Busy = &inventory.GateBusy{Machine: j.node, Module: j.module, Container: b.Container, Since: b.Since} + } + g.Busy.Until, g.Busy.Said = b.Until, b.Said + if now.After(g.Busy.Until.Add(busyGateSlack)) { + // The engine said it would have replaced it by now: the judging starts, and finds the module not + // yet healthy while the machine still lists it waiting (busyStill), until the gate's own bound. + end := now + g.Busy.Ended, g.Busy.PastBound = &end, true + g.Since, g.Passes, g.LastPass, g.Healthy, g.HealthyAt = &end, 0, nil, nil, nil + return false + } + g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, g.Busy.Words(), nil + g.Read(now, false, g.Busy.Words()) + return true + } + if g.Busy == nil { + return false + } + // Replaced: the judging starts at the report that said so. + end := now + if r, said := f.reports[g.Busy.Machine]; said && r.At != nil && r.At.After(g.Busy.Since) && r.At.Before(now) { + end = r.At.UTC() + } + g.Busy.Ended = &end + g.Since, g.Passes, g.LastPass, g.Last, g.Healthy, g.HealthyAt = &end, 0, nil, "", nil, nil + return false +} + +// busyOn is the container of a judged module its machine last reported, on what the gate sent it, as left +// running because it said busy. +func busyOn(j judged, f gateFacts) (inventory.BusyWait, bool) { + r, said := f.reports[j.node] + if !said || r.At == nil || !f.reportedOn(j.node, r) { + return inventory.BusyWait{}, false + } + return inventory.WaitingFor(r.Busy, j.module) +} + +// busyStill is a judged module whose container its machine still lists waiting once the gate stopped waiting +// for it: not yet what was sent, so not healthy, whatever its health says of the container still running. +func busyStill(j judged, f gateFacts) (string, bool) { + b, waits := busyOn(j, f) + if !waits { + return "", false + } + return fmt.Sprintf("%s still runs its busy container %s, past its node-engine's bound of %s", + j.node, b.Container, b.Until.UTC().Format(time.RFC3339)), true +} diff --git a/cmd/mesh-controller/busy_test.go b/cmd/mesh-controller/busy_test.go new file mode 100644 index 00000000..c26a6ace --- /dev/null +++ b/cmd/mesh-controller/busy_test.go @@ -0,0 +1,111 @@ +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// A walk whose module's container waits as busy on its first machine says so, and starts that machine's +// first-machine gate only once the machine reports the container replaced (novox/hq ADR 0335 §4). +func TestAGateWaitsForABusyContainerAndStartsWhenItIsReplaced(t *testing.T) { + sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC) + g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent, + Sent: map[string]inventory.SentDeclaration{"novox": {Digest: "d1", Sequence: 7}}} + pairs := []judged{{module: "build-agent", node: "novox"}, {module: "mesh-controller", node: "novox"}} + reported := sent.Add(20 * time.Second) + waiting := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", + State: inventory.BusyWaiting, Since: reported, Until: reported.Add(15 * time.Minute), + Said: "busy: novox/mesh-controller (check of #280), in hand since 00:58:00Z"}} + facts := func(at time.Time, busy []inventory.BusyWait) gateFacts { + // As judgeMoves hands it: the report held against the gate's own send. + return gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"novox": {Node: "novox", + Outcome: inventory.OutcomeApplied, At: &at, Declared: "d1", ReportedSequence: 7, Busy: busy}}} + } + + now := reported.Add(time.Minute) + if !busyWait(g, pairs, facts(reported, waiting), now) { + t.Fatal("a gate whose module's container waits as busy judged it") + } + if !g.Since.Equal(sent) || g.Busy == nil || g.Busy.Machine != "novox" || g.Busy.Ended != nil { + t.Fatalf("the gate does not keep the wait: %+v %+v", g.Since, g.Busy) + } + for _, said := range []string{gateLine(g), g.Last} { + if !strings.Contains(said, "waits for a busy container on novox") { + t.Errorf("the walk does not say it waits for a busy container there: %s", said) + } + } + if !strings.Contains(g.Last, "check of #280") { + t.Errorf("the wait does not say what the container holds: %s", g.Last) + } + + // Ten minutes on, still busy, within the engine's bound: still waiting, nothing judged, no bound run. + now = reported.Add(10 * time.Minute) + if !busyWait(g, pairs, facts(reported.Add(9*time.Minute), waiting), now) || g.Passes != 0 { + t.Fatalf("the gate stopped waiting within the bound: %+v", g) + } + + // The machine reports it replaced: the judging starts at that report. + replacedAt := reported.Add(12 * time.Minute) + now = replacedAt.Add(5 * time.Second) + replaced := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", + State: "replaced-when-idle", Since: reported, Until: reported.Add(15 * time.Minute)}} + if busyWait(g, pairs, facts(replacedAt, replaced), now) { + t.Fatal("the gate still waits after the container was replaced") + } + if g.Since == nil || !g.Since.Equal(replacedAt) || g.Busy.Ended == nil || !g.Busy.Ended.Equal(replacedAt) { + t.Fatalf("the gate does not start at the report that said it replaced: since %v, %+v", g.Since, g.Busy) + } + if strings.Contains(gateLine(g), "waits for a busy container") { + t.Errorf("a gate judging still says it waits: %s", gateLine(g)) + } + // And a later report listing it waiting again does not hold a gate that started. + if busyWait(g, pairs, facts(now, waiting), now.Add(time.Minute)) { + t.Error("a gate that started waits again") + } +} + +// An engine that stops reporting while it waits holds no walk for ever: past the engine's bound and the slack, +// the judging starts, and finds the module not yet healthy while its container is still listed waiting. +func TestAGateWaitsNoLongerThanTheEnginesBound(t *testing.T) { + sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC) + g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent} + pairs := []judged{{module: "build-agent", node: "novox"}} + waiting := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", + State: inventory.BusyWaiting, Since: sent, Until: sent.Add(15 * time.Minute)}} + f := gateFacts{reports: map[string]inventory.Reported{"novox": {Node: "novox", Outcome: inventory.OutcomeApplied, + At: &sent, Current: true, Busy: waiting}}} + now := sent.Add(15*time.Minute + busyGateSlack + time.Second) + if busyWait(g, pairs, f, now) { + t.Fatal("a gate waited past the engine's bound and the slack") + } + if !g.Busy.PastBound || !g.Since.Equal(now) { + t.Fatalf("the gate does not start, said past the bound: %+v %v", g.Busy, g.Since) + } + if why, still := busyStill(pairs[0], f); !still || !strings.Contains(why, "past its node-engine's bound") { + t.Fatalf("a module still running its busy container reads as what was sent: %q", why) + } +} + +// A report from an engine older than the field lists nothing: the gate starts at the send, as before; and a +// report not about the gate's send says nothing of it. +func TestAGateOfAnEngineThatNeverWaitsStartsAsBefore(t *testing.T) { + sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC) + g := &inventory.PlanGate{Machines: []string{"ace"}, Since: &sent, + Sent: map[string]inventory.SentDeclaration{"ace": {Digest: "d2", Sequence: 9}}} + pairs := []judged{{module: "build-agent", node: "ace"}} + at := sent.Add(time.Minute) + older := gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"ace": {Node: "ace", Outcome: inventory.OutcomeApplied, + At: &at, Declared: "d2", ReportedSequence: 9}}} + if busyWait(g, pairs, older, at) || g.Busy != nil || !g.Since.Equal(sent) { + t.Fatalf("a gate on an engine that never waits did not start at the send: %+v %v", g.Busy, g.Since) + } + elsewhere := gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"ace": {Node: "ace", Outcome: inventory.OutcomeApplied, + At: &at, Declared: "d1", ReportedSequence: 8, Busy: []inventory.BusyWait{{ID: "build-agent.server", + State: inventory.BusyWaiting, Until: at.Add(time.Hour)}}}}} + if busyWait(g, pairs, elsewhere, at) { + t.Fatal("a wait reported about an earlier declaration held the gate") + } +} diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index f5aa1689..c89fb984 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -544,6 +544,11 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs facts.sentBuilds[j.node] = builds } } + // **A busy container on a judged machine holds the judging before it starts** (novox/hq ADR 0335 §4): the + // walk says it waits there, and its gate starts at the report that says the container replaced. + if busyWait(g, pairs, facts, now) { + return "", nil + } // **What is wrong with a machine itself is the machine's** (novox/hq issue 281): read once for each // machine judged, apart from what is wrong with a module there, and never pinned on the module the // gate happens to be kept on. @@ -568,6 +573,9 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs for _, j := range pairs { w := words[j.node] h, said := judgeHealth(j.module, coreComponent(j.module), shelf[j.module], j.node, *g.Since, w.facts) + if why, still := busyStill(j, facts); still && (h == healthGood || h == healthPerson) { + h, said = healthNotYet, why + } if h == healthGood || h == healthPerson { if on, named := w.on[j.module]; named { h, said = healthNotYet, on @@ -1372,6 +1380,9 @@ func gateLine(g *inventory.PlanGate) string { case inventory.GateFailed: what = "FAILED" } + if g.Verdict == "" && g.Waiting() { + what = "waits for a busy container on " + g.Busy.Machine + } line := fmt.Sprintf("gate on %s: %s", strings.Join(g.Machines, ", "), what) if g.Component != "" { line += " (core: " + g.Component + ")" diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index 3a6a0a50..5418d142 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -975,9 +975,14 @@ func renderingFor(ctx context.Context, open *stores, node string, if err != nil { return catalogue.Rendering{}, inventory.Node{}, err } + readsBusy, err := engineReadsBusy(ctx, inv, node) + if err != nil { + return catalogue.Rendering{}, inventory.Node{}, err + } return catalogue.Rendering{ ReadsHealth: readsHealth, JudgesRoot: judgesRoot, + ReadsBusy: readsBusy, BusMembership: memberships[node], Settings: settings, Generators: gens, Grants: grants, Needed: needed, Foreseen: foreseen, Ports: ports, Certificate: certificate, Authority: authority, Mesh: private, Names: names, @@ -1010,6 +1015,16 @@ func engineJudgesRoot(ctx context.Context, inv *inventory.Inventory, node string return had && stated.Contract >= link.RootContract, nil } +// engineReadsBusy says whether a machine's node-engine reads a container's `busy` (novox/hq ADR 0335), by its +// own newest statement, for the same reason as engineReadsHealth: an older engine parses strictly. +func engineReadsBusy(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) { + stated, had, err := inv.HealthOf(ctx, node) + if err != nil { + return false, err + } + return had && stated.Contract >= link.BusyContract, nil +} + // zonesInTheMesh is every zone a module in the mesh declares, where the mesh placed it (novox/hq ADR // 0199): the zone settled from that node's settings, the node's private address, the port the // answering listen is published on there. diff --git a/go.mod b/go.mod index 60ad6d19..be79678a 100644 --- a/go.mod +++ b/go.mod @@ -36,4 +36,4 @@ require ( // committed. Every build (the build agent's `go build`, the Dockerfile) compiles from vendor/ and // fetches nothing; go refuses to build when vendor/ and this file disagree, so a pin moved without // `go mod vendor` fails loudly, at once, everywhere. -replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f +replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261012224132-bb60e7b584d8 diff --git a/go.sum b/go.sum index 96650c70..f00b9ca3 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f h1:8N5OW2mdTNIck2pe4EciTYX5NsrPsHrTLENGNIWYNTU= -git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f/go.mod h1:tcTK4LMs1d6JpZUwy3hSHMFG/MIuDr/XFs7/nbeaW/c= +git.novox.be/novox/mesh-host v0.0.0-20261012224132-bb60e7b584d8 h1:a+SH9NroLc3FkNYcbg0wjMEWGjWp6SCl32oVH2jxh9s= +git.novox.be/novox/mesh-host v0.0.0-20261012224132-bb60e7b584d8/go.mod h1:tcTK4LMs1d6JpZUwy3hSHMFG/MIuDr/XFs7/nbeaW/c= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= diff --git a/internal/catalogue/busy.go b/internal/catalogue/busy.go new file mode 100644 index 00000000..18cbbf02 --- /dev/null +++ b/internal/catalogue/busy.go @@ -0,0 +1,124 @@ +package catalogue + +import ( + "fmt" + "sort" + "strings" + "time" +) + +// A long-running container may say it is busy (novox/hq ADR 0335 §1). +// +// **Beside its health, on the container**: a command run inside it, by its shell, as an exec health check +// is, that exits 0 while the container holds work a replacement would lose and non-zero when it is idle; +// and how long one ask may take. Being asked is also its notice that it is about to be replaced: from the +// first ask it takes no new work. The node-engine asks before it replaces the container, leaves it running +// while it says busy, and replaces it on the first pass it says idle, or at 15 minutes — the engine's bound, +// never the module's. +// +// Sent only to a node-engine that reads it (link.BusyContract): an older, strict engine refuses a field it +// does not know, whole, and to it the container is replaced at once, as before. + +// BusyField is the resource field. +const BusyField = "busy" + +// The bounds of one ask (ADR 0335), the node-engine's own, held here near the author. +const ( + BusyTimeoutDefault = 10 * time.Second + BusyTimeoutCeiling = time.Minute +) + +// Busy is one container's busy, read. +type Busy struct { + Command string + Timeout time.Duration +} + +// ReadBusy reads a resource's `busy` field, its default applied. False when it carries none. +func ReadBusy(r map[string]any) (Busy, bool, []string) { + raw, present := r[BusyField] + if !present { + return Busy{}, false, nil + } + id := fmt.Sprint(r["id"]) + fields, ok := raw.(map[string]any) + if !ok { + return Busy{}, true, []string{fmt.Sprintf("%s: busy is %T; it is an object with a command", id, raw)} + } + var problems []string + keys := make([]string, 0, len(fields)) + for k := range fields { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + if k != "command" && k != "timeout" { + problems = append(problems, fmt.Sprintf("%s: its busy says %q; busy has a command and a timeout", id, k)) + } + } + b := Busy{Timeout: BusyTimeoutDefault} + if v, has := fields["command"]; has { + s, ok := v.(string) + if !ok { + problems = append(problems, fmt.Sprintf("%s: its busy's command is %T; it is text", id, v)) + } + b.Command = strings.TrimSpace(s) + } + if v, has := fields["timeout"]; has { + s, _ := v.(string) + d, err := time.ParseDuration(strings.TrimSpace(s)) + switch { + case err != nil: + problems = append(problems, fmt.Sprintf("%s: its busy's timeout is %v; it is a duration such as \"10s\"", id, v)) + case d <= 0 || d > BusyTimeoutCeiling: + problems = append(problems, fmt.Sprintf("%s: an ask of its busy takes more than nothing and at most %s, not %s", + id, BusyTimeoutCeiling, d)) + default: + b.Timeout = d + } + } + return b, true, problems +} + +// busyProblems is everything wrong with a manifest's `busy` fields, in the manifest's words: a command on +// one line, a timeout within its bounds, and only on a container that stays up. +func busyProblems(m Manifest) []string { + var problems []string + for _, r := range m.Resources { + b, has, read := ReadBusy(r) + if !has { + continue + } + where := m.Module + ": " + fmt.Sprint(r["id"]) + for _, p := range read { + problems = append(problems, m.Module+": "+p) + } + if fmt.Sprint(r["type"]) != "container" || !LongRunning(r) { + problems = append(problems, fmt.Sprintf("%s declares busy and is not a container that stays up: busy is "+ + "asked of a running container before it is replaced (ADR 0335)", where)) + } + if b.Command == "" { + problems = append(problems, where+"'s busy names no command: one run inside the container, exiting 0 "+ + "while it is busy") + } + if strings.ContainsAny(b.Command, "\n\r") { + problems = append(problems, where+"'s busy command is one line") + } + } + return problems +} + +// busyInto composes a container's `busy` into the node-engine's words, or takes it away: for an engine that +// reads it, the command and its timeout written out; for one that does not, the field is not sent, and the +// container is replaced at once, as before. +func busyInto(resource map[string]any, with Rendering) { + if _, has := resource[BusyField]; !has { + return + } + if !with.ReadsBusy { + delete(resource, BusyField) + return + } + b, _, _ := ReadBusy(resource) + resource[BusyField] = map[string]any{"command": b.Command, "timeout": b.Timeout.String()} +} diff --git a/internal/catalogue/busy_test.go b/internal/catalogue/busy_test.go new file mode 100644 index 00000000..871817d2 --- /dev/null +++ b/internal/catalogue/busy_test.go @@ -0,0 +1,94 @@ +package catalogue + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/novox/mesh-host/validate" +) + +// A long-running container may say it is busy (novox/hq ADR 0335 §1): `module check` holds the field to its +// shape near its author, and it is sent only to a node-engine that reads it, in words the engine's own +// validator accepts. + +func busyManifest(t *testing.T, on string, busy any) []byte { + t.Helper() + extra := map[string]any{"id": "agent", "type": "container", "name": "app-agent", + "image": "registry.example/app" + healthDigest} + var m map[string]any + if err := json.Unmarshal(healthManifest(t, "", nil, extra), &m); err != nil { + t.Fatal(err) + } + for _, r := range m["resources"].([]any) { + if r := r.(map[string]any); r["id"] == on { + r[BusyField] = busy + } + } + out, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + return out +} + +func TestABusyIsHeldToItsShape(t *testing.T) { + if _, err := ParseManifest(busyManifest(t, "agent", map[string]any{"command": "mesh-builder busy"})); err != nil { + t.Fatalf("a well-formed busy was refused: %v", err) + } + if _, err := ParseManifest(busyManifest(t, "agent", map[string]any{"command": "x", "timeout": "30s"})); err != nil { + t.Fatalf("a busy with its timeout was refused: %v", err) + } + for name, c := range map[string]struct { + on string + busy any + says string + }{ + "text": {"agent", "mesh-builder busy", "it is an object"}, + "no command": {"agent", map[string]any{"timeout": "5s"}, "names no command"}, + "two lines": {"agent", map[string]any{"command": "a\nb"}, "one line"}, + "a long timeout": {"agent", map[string]any{"command": "x", "timeout": "5m"}, "at most 1m0s"}, + "an unknown key": {"agent", map[string]any{"command": "x", "every": "1m"}, `says "every"`}, + "on a step": {"seed", map[string]any{"command": "x"}, "not a container that stays up"}, + "on a service": {"daemon", map[string]any{"command": "x"}, "not a container that stays up"}, + } { + _, err := ParseManifest(busyManifest(t, c.on, c.busy)) + if err == nil || !strings.Contains(err.Error(), c.says) { + t.Errorf("%s: want a refusal saying %q, got %v", name, c.says, err) + } + } +} + +func TestABusyIsSentOnlyToAnEngineThatReadsIt(t *testing.T) { + m, err := ParseManifest(busyManifest(t, "agent", map[string]any{"command": "mesh-builder busy"})) + if err != nil { + t.Fatal(err) + } + compose := func(with Rendering) []map[string]any { + out, err := Resolution{Node: "laptop", Modules: []Manifest{m}}.Declaration(with) + if err != nil { + t.Fatal(err) + } + return out + } + older := compose(Rendering{}) + if b, sent := older[indexOfID(older, "app.agent")][BusyField]; sent { + t.Fatalf("busy was sent to an engine that does not read it: %v", b) + } + newer := compose(Rendering{ReadsBusy: true}) + got, _ := newer[indexOfID(newer, "app.agent")][BusyField].(map[string]any) + if got["command"] != "mesh-builder busy" || got["timeout"] != "10s" { + t.Fatalf("busy composed as %v", got) + } + // And the engine's own validator takes the container with it. + body, err := json.Marshal(map[string]any{"declaration": validate.Version, "resources": newer}) + if err != nil { + t.Fatal(err) + } + if problems := validate.Declaration(body); len(problems) > 0 { + t.Fatalf("the node-engine's validator refuses the composed busy: %v", problems) + } + if _, has := m.Resources[len(m.Resources)-1][BusyField].(map[string]any)["timeout"]; has { + t.Error("composing wrote the timeout into the catalogue's own manifest") + } +} diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index 4cbdde6b..01a24655 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -246,6 +246,11 @@ type Rendering struct { // statement's contract is link.RootContract or later). To an older, strict engine the field is not // sent, and the account it names is not judged — which the self-check says, as not judged. JudgesRoot bool + + // ReadsBusy says this machine's node-engine reads a container's `busy` (novox/hq ADR 0335: its statement's + // contract is link.BusyContract or later). To an older, strict engine the field is not sent, and the + // container is replaced at once, as before. + ReadsBusy bool } // RootField is a user resource's field saying the account must never become root without a person @@ -1051,6 +1056,8 @@ func (r Resolution) compose(with Rendering, owner map[string]string, // How it is ready, in the node-engine's words: its endpoint as the port this machine // published it on — or not sent at all to an engine older than the field (ADR 0240). healthInto(copied, m, with) + // And whether it is busy (novox/hq ADR 0335): sent only to an engine that asks it. + busyInto(copied, with) // And a user's `root` (novox/hq ADR 0266): sent only when it asserts something, to an engine // that judges it. rootInto(copied, with) diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index 6e8b398f..de8cd23a 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -2242,6 +2242,8 @@ func ParseManifest(raw []byte) (Manifest, error) { // How each long-running resource is ready (novox/hq ADR 0240 rule 2): said near its author, in the // words the node-engine would refuse it in far away. problems = append(problems, healthProblems(m)...) + // And whether a container is busy (novox/hq ADR 0335), likewise. + problems = append(problems, busyProblems(m)...) if len(problems) > 0 { sort.Strings(problems) return Manifest{}, fmt.Errorf("this manifest cannot be used:\n - %s", diff --git a/internal/inventory/busy_test.go b/internal/inventory/busy_test.go new file mode 100644 index 00000000..80ee9684 --- /dev/null +++ b/internal/inventory/busy_test.go @@ -0,0 +1,142 @@ +package inventory + +import ( + "strings" + "testing" + "time" +) + +// A walk whose first machine waited for a busy container counts the wait as a phase of its own, between the +// send to the first machine and the judging, and its delivery time holds it (novox/hq ADR 0335 §4): the +// recorded walk again, its judging started only at 17:54:07, a minute after the send, when the machine said +// the container replaced. +func TestABusyWaitIsAPhaseOfTheDeliveryTime(t *testing.T) { + raw := strings.Replace(recordedWalk, `"gate": {"machines": ["novox"], "since": "2026-10-10T17:53:07.287136827Z",`, + `"gate": {"machines": ["novox"], "since": "2026-10-10T17:54:07.287136827Z", + "busy": {"machine": "novox", "module": "mesh-delivery", "container": "mesh-delivery", + "since": "2026-10-10T17:53:07Z", "until": "2026-10-10T18:08:07Z", "ended": "2026-10-10T17:54:07.287136827Z"},`, 1) + w := walkOf(t, raw).WalkPhases(time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC), nil) + sumsUp(t, w) + if w.TotalMS != 282894 { + t.Fatalf("the wait changed the delivery's total: %dms", w.TotalMS) + } + busy := phase(w, PhaseBusy, 0) + if busy.State != PhaseMeasured || busy.TookMS != 60000 { + t.Fatalf("the busy wait is not a measured phase of a minute: %+v", busy) + } + if judged := phase(w, PhaseJudge, 0); judged.TookMS != 148607-60000 { + t.Fatalf("the judging still counts from the send, not from the replacement: %+v", judged) + } + var names []string + for _, ph := range w.Phases { + names = append(names, ph.Name) + } + if !strings.Contains(strings.Join(names, " "), PhaseSend+" "+PhaseBusy+" "+PhaseJudge) { + t.Fatalf("the busy wait is not between the send and the judging: %v", names) + } +} + +// While the wait goes on, the walk says it waits for a busy container there; a walk where nothing waited has no +// such phase. +func TestAnOpenBusyWaitIsSaid(t *testing.T) { + raw := strings.Replace(recordedWalk, `"gate": {"machines": ["novox"], "since": "2026-10-10T17:53:07.287136827Z", "passes": 3, + "verdict": "passed", "judged_at": "2026-10-10T17:55:35.894985053Z"}`, + `"gate": {"machines": ["novox"], "since": "2026-10-10T17:53:07.287136827Z", + "busy": {"machine": "novox", "module": "mesh-delivery", "container": "mesh-delivery", + "since": "2026-10-10T17:53:07Z", "until": "2026-10-10T18:08:07Z"}}`, 1) + raw = strings.Replace(raw, `"state": "done"`, `"state": "rolling"`, 1) + p := walkOf(t, raw) + if !p.Modules["mesh-delivery"].Gate.Waiting() { + t.Fatal("a gate whose busy wait has not ended does not say it waits") + } + w := p.WalkPhases(time.Date(2026, 10, 10, 17, 58, 0, 0, time.UTC), nil) + busy := phase(w, PhaseBusy, 0) + if busy.State != PhaseUnknown || !strings.Contains(busy.Said, "waits for a busy container on novox") { + t.Fatalf("an open busy wait is not said: %+v", busy) + } + if plain := walkOf(t, recordedWalk).WalkPhases(time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC), nil); phase(plain, PhaseBusy, 0).Name != "" { + t.Fatalf("a walk where nothing waited has a busy phase: %+v", plain.Phases) + } +} + +// A container a report lists waiting is its module's by its id; its output alone is not a new account. +func TestABusyContainerIsItsModules(t *testing.T) { + list := []BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: BusyWaiting, Said: "a"}, + {ID: "web.server", State: "replaced-when-idle"}} + if b, ok := WaitingFor(list, "build-agent"); !ok || b.Container != "mesh-build-agent" { + t.Fatalf("build-agent's container is not found waiting: %+v", b) + } + if _, ok := WaitingFor(list, "web"); ok { + t.Error("a container replaced reads as waiting") + } + if _, ok := WaitingFor(list, "build"); ok { + t.Error("another module's container is found by a prefix of its name") + } + again := []BusyWait{{ID: "build-agent.server", State: BusyWaiting, Said: "b, a minute on"}, list[1]} + if !sameBusy(list, again) { + t.Error("a wait whose output changed is a new account") + } + if sameBusy(list, list[1:]) || !(Doing{Busy: list}).Waiting() || (Doing{Busy: list[1:]}).Waiting() { + t.Error("a wait that ended is not a new account") + } +} + +// A machine's report of a busy container is kept with its last account and read back beside it, and its +// starting and its end are news; its output alone is not (novox/hq ADR 0335). Needs the store. +func TestAReportOfABusyContainerIsKeptAndReadBack(t *testing.T) { + inv, node := aNodeWithModules(t) + ctx := t.Context() + record, err := inv.NodeByName(ctx, node) + if err != nil { + t.Fatal(err) + } + since := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC) + waiting := []BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: BusyWaiting, + Since: since, Until: since.Add(15 * time.Minute), Said: "busy: a check"}} + keep := func(busy []BusyWait) bool { + t.Helper() + news, err := inv.RecordOrderedDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: "d1", Applied: 1, + Busy: busy}, ReportOrder{Epoch: 1, Sequence: 1}, func(ReportOrder) bool { return false }) + if err != nil { + t.Fatal(err) + } + return news + } + read := func() []BusyWait { + t.Helper() + reports, err := inv.LastReports(ctx) + if err != nil { + t.Fatal(err) + } + for _, r := range reports { + if r.Node == node { + return r.Busy + } + } + t.Fatal("the machine's report is not read back") + return nil + } + keep(nil) + if got := read(); len(got) != 0 { + t.Fatalf("a report with nothing busy reads back %+v", got) + } + if !keep(waiting) { + t.Error("a container starting to wait is not news") + } + got := read() + if b, ok := WaitingFor(got, "build-agent"); !ok || !b.Since.Equal(since) || b.Said != "busy: a check" { + t.Fatalf("the wait is not read back: %+v", got) + } + again := []BusyWait{waiting[0]} + again[0].Said = "busy: a check, a minute on" + if keep(again) { + t.Error("a wait whose output changed is news") + } + if !keep([]BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: "replaced-when-idle", + Since: since, Until: since.Add(15 * time.Minute)}}) { + t.Error("a container replaced after it waited is not news") + } + if _, ok := WaitingFor(read(), "build-agent"); ok { + t.Error("a container replaced still reads as waiting") + } +} diff --git a/internal/inventory/migrations/0096-a-report-says-a-busy-container.sql b/internal/inventory/migrations/0096-a-report-says-a-busy-container.sql new file mode 100644 index 00000000..9089d645 --- /dev/null +++ b/internal/inventory/migrations/0096-a-report-says-a-busy-container.sql @@ -0,0 +1,11 @@ +-- A machine's report says which container it left running because it said busy, and which it replaced +-- after it waited (novox/hq ADR 0335, issue 594). +-- +-- A send that replaced the build agent killed the check it ran, and the queue started it again elsewhere. +-- A container may now declare `busy`; the node-engine asks it before replacing it, leaves it running while it +-- says busy, within 15 minutes, and says so in its report's `busy`: [{"id", "container", "state", "since", +-- "until", "said"}], state `waiting`, `replaced-when-idle` or `replaced-at-bound`. Kept with the machine's +-- last account, so a walk's first-machine gate waits while its module's container is listed waiting — across +-- a restart of the controller, which a merge into mesh-controller makes in the same walk that moves the build +-- agent. Empty from an engine that never waits. +alter table node_report add column busy jsonb not null default '[]'; diff --git a/internal/inventory/nodes.go b/internal/inventory/nodes.go index 8552d1ff..30a92a75 100644 --- a/internal/inventory/nodes.go +++ b/internal/inventory/nodes.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "regexp" + "slices" "sort" "strings" "time" @@ -846,6 +847,46 @@ type Doing struct { // machine doing what it was told. Since *time.Time Times int + + // Busy is every container the apply left running because it said busy, and every one it replaced after + // it waited (novox/hq ADR 0335). + Busy []BusyWait +} + +// BusyWait is one container a busy command held, as the machine reported it (novox/hq ADR 0335): its +// resource id (`.`), its name, its state (link.BusyWaiting, BusyReplacedIdle or +// BusyReplacedAtBound), since when it waits, until when at the latest, and the last output of its command. +type BusyWait struct { + ID string `json:"id"` + Container string `json:"container"` + State string `json:"state"` + Since time.Time `json:"since"` + Until time.Time `json:"until"` + Said string `json:"said,omitempty"` +} + +// BusyWaiting is the state of a container left running because it said busy (link.BusyWaiting). +const BusyWaiting = "waiting" + +// WaitingFor is the container of a module that waits because it said busy, and whether one does: a +// resource whose id is `.`. +func WaitingFor(busy []BusyWait, module string) (BusyWait, bool) { + for _, b := range busy { + if b.State == BusyWaiting && strings.HasPrefix(b.ID, module+".") { + return b, true + } + } + return BusyWait{}, false +} + +// Waiting says whether any container waits because it said busy. +func (d Doing) Waiting() bool { + for _, b := range d.Busy { + if b.State == BusyWaiting { + return true + } + } + return false } // FailedResource is one thing a node could not do. @@ -873,6 +914,20 @@ func (d Doing) Stuck() bool { return d.Wrong() && d.Times >= StuckAfter && d.Sin // the same failed resources BY ID. Not by the host's words: an error that carries a duration, a // counter or a temporary path would read as new on every report, and the resource looping on it — // which is what stuck is for — would never be said to be (novox/hq 04-ISSUES/065). +// sameBusy says two reports hold the same containers in the same state of a busy wait: its output, which +// may change on every ask, is not compared. +func sameBusy(a, b []BusyWait) bool { + key := func(list []BusyWait) []string { + out := make([]string, 0, len(list)) + for _, w := range list { + out = append(out, w.ID+"="+w.State) + } + sort.Strings(out) + return out + } + return slices.Equal(key(a), key(b)) +} + func sameFailure(a, b Doing) bool { if a.Outcome != b.Outcome || a.Refused != b.Refused || len(a.Failed) != len(b.Failed) { return false @@ -937,13 +992,20 @@ func recordDoing(ctx context.Context, q queries, node string, d Doing) (news boo if err != nil { return false, err } + if d.Busy == nil { + d.Busy = []BusyWait{} + } + busy, err := json.Marshal(d.Busy) + if err != nil { + return false, err + } var before Doing - var beforeFailed []byte + var beforeFailed, beforeBusy []byte found := q.QueryRow(ctx, - `select outcome, refused, failed, failing_since, failures, coalesce(declared,'') + `select outcome, refused, failed, failing_since, failures, coalesce(declared,''), busy from node_report where node = $1`, node).Scan(&before.Outcome, &before.Refused, &beforeFailed, &before.Since, &before.Times, - &before.Declared) + &before.Declared, &beforeBusy) switch { case errors.Is(found, pgx.ErrNoRows): news = true @@ -953,7 +1015,13 @@ func recordDoing(ctx context.Context, q queries, node string, d Doing) (news boo if err := json.Unmarshal(beforeFailed, &before.Failed); err != nil { return false, err } - news = before.Outcome != d.Outcome || before.Declared != d.Declared || !sameFailure(before, d) + if err := json.Unmarshal(beforeBusy, &before.Busy); err != nil { + return false, err + } + // A container that starts or ends a wait because it said busy is news (novox/hq ADR 0335): the walk + // waiting on this machine is told by it. + news = before.Outcome != d.Outcome || before.Declared != d.Declared || !sameFailure(before, d) || + !sameBusy(before.Busy, d.Busy) } var since *time.Time times := 0 @@ -966,13 +1034,13 @@ func recordDoing(ctx context.Context, q queries, node string, d Doing) (news boo } _, err = q.Exec(ctx, `insert into node_report (node, outcome, refused, failed, applied, at, declared, - failing_since, failures) - values ($1, $2, $3, $4, $5, now(), $6, $7, $8) + failing_since, failures, busy) + values ($1, $2, $3, $4, $5, now(), $6, $7, $8, $9) on conflict (node) do update set outcome = excluded.outcome, refused = excluded.refused, failed = excluded.failed, applied = excluded.applied, at = excluded.at, declared = excluded.declared, - failing_since = excluded.failing_since, failures = excluded.failures`, - node, d.Outcome, d.Refused, failed, d.Applied, d.Declared, since, times) + failing_since = excluded.failing_since, failures = excluded.failures, busy = excluded.busy`, + node, d.Outcome, d.Refused, failed, d.Applied, d.Declared, since, times, busy) if err != nil { return false, err } @@ -1059,6 +1127,8 @@ type Reported struct { // gate holds against the send it made, rather than against the send made last (novox/hq issue 352). Declared string ReportedSequence int64 + // Busy is what the last report said of containers held by their busy command (novox/hq ADR 0335). + Busy []BusyWait } // SentDeclaration is what one send carried to a machine, as a gate keeps it: the declaration's digest and @@ -1113,7 +1183,7 @@ func (i *Inventory) LastReports(ctx context.Context) ([]Reported, error) { rows, err := i.store.Pool().Query(ctx, `select n.name, coalesce(r.outcome, ''), r.at, n.sent_at, r.declared is not null and r.declared <> '' and r.declared = n.sent, - coalesce(r.declared, ''), coalesce(r.reported_sequence, 0) + coalesce(r.declared, ''), coalesce(r.reported_sequence, 0), coalesce(r.busy, '[]') from node n left join node_report r on r.node = n.id order by n.name`) if err != nil { @@ -1123,7 +1193,12 @@ func (i *Inventory) LastReports(ctx context.Context) ([]Reported, error) { var out []Reported for rows.Next() { var r Reported - if err := rows.Scan(&r.Node, &r.Outcome, &r.At, &r.Sent, &r.Current, &r.Declared, &r.ReportedSequence); err != nil { + var busy []byte + if err := rows.Scan(&r.Node, &r.Outcome, &r.At, &r.Sent, &r.Current, &r.Declared, &r.ReportedSequence, + &busy); err != nil { + return nil, err + } + if err := json.Unmarshal(busy, &r.Busy); err != nil { return nil, err } out = append(out, r) diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go index d6933d04..ef1cf10e 100644 --- a/internal/inventory/plans.go +++ b/internal/inventory/plans.go @@ -367,6 +367,39 @@ type PlanGate struct { // alone to the machine holding the bus, every build there kept, so what the send newly grants is on the // bus before the gate judges it. Nil when the list did not change, or the gate's machine holds the bus. Grants *GrantsStep `json:"grants,omitempty"` + // Busy is the wait for a busy container on a judged machine (novox/hq ADR 0335 §4): while the machine + // reports its module's container left running because it said busy, the judging waits and has not + // started; once it reports the container replaced, Since is moved to then. Nil when nothing waited. + Busy *GateBusy `json:"busy,omitempty"` +} + +// GateBusy is a first-node gate's wait for a busy container (novox/hq ADR 0335 §4). +type GateBusy struct { + Machine string `json:"machine"` + Module string `json:"module"` + Container string `json:"container"` + // Since is when the machine's first apply that would have replaced it ran, Until when its node-engine + // replaces it at the latest; Ended when the machine reported it replaced and the judging began — or when + // the controller stopped waiting past the engine's bound, which PastBound says. + Since time.Time `json:"since"` + Until time.Time `json:"until"` + Ended *time.Time `json:"ended,omitempty"` + PastBound bool `json:"past_bound,omitempty"` + // Said is the last output of its busy command, as the machine reported it. + Said string `json:"said,omitempty"` +} + +// Waiting says the gate waits for a busy container and has not started. +func (g *PlanGate) Waiting() bool { return g != nil && g.Busy != nil && g.Busy.Ended == nil } + +// Words is the wait in a line, as the walk says it. +func (b *GateBusy) Words() string { + said := "" + if b.Said != "" { + said = "; its busy command says: " + b.Said + } + return fmt.Sprintf("waits for a busy container on %s: %s holds work, and is replaced once it says idle, by %s "+ + "at the latest (novox/hq ADR 0335)%s", b.Machine, b.Container, b.Until.UTC().Format(time.RFC3339), said) } // GrantsStep is the bus's user list sent to the machine holding the bus ahead of a gated send (novox/hq diff --git a/internal/inventory/walkphases.go b/internal/inventory/walkphases.go index 971945c3..4da4e760 100644 --- a/internal/inventory/walkphases.go +++ b/internal/inventory/walkphases.go @@ -26,6 +26,7 @@ const ( PhaseBetween = "between-tiers" // one tier's end to the next tier's ask PhaseBuild = "build" // a tier asked to its last module built PhaseSend = "send-first" // built to sent to the first machines + PhaseBusy = "busy-wait" // sent first to a busy container there replaced, when one waited (ADR 0335) PhaseJudge = "judgement" // sent first to the first-node gate's last verdict: its readings PhaseRest = "send-rest" // judged to sent to the rest PhaseApply = "apply" // the last send to every machine of the rest reporting the build applied @@ -300,8 +301,11 @@ func (p Plan) points(now time.Time, applied AppliedLookup, w *WalkPhases) []poin if t > p.Tier || (t == p.Tier && p.Tier < len(p.Tiers) && !askedAnyOf(p, tier)) { break } - var asked, built, first, judged, rest *time.Time + var asked, built, first, judged, rest, busyEnded *time.Time allBuilt, anyFirst, allJudged, allRest := true, false, true, true + // A busy container on a first machine holds the judging until it is replaced (novox/hq ADR 0335 §4): a + // phase of its own, only in a tier where one waited, counted in the delivery time and never exempted. + anyBusy, allBusyEnded, busyOn := false, true, "" for _, m := range tier { s := p.Modules[m] if s == nil { @@ -319,6 +323,14 @@ func (p Plan) points(now time.Time, applied AppliedLookup, w *WalkPhases) []poin if s.FirstAt != nil { anyFirst = true first = earliest(first, s.FirstAt) + if s.Gate != nil && s.Gate.Busy != nil { + anyBusy = true + if s.Gate.Busy.Ended == nil { + allBusyEnded, busyOn = false, s.Gate.Busy.Machine + } else { + busyEnded = latest(busyEnded, s.Gate.Busy.Ended) + } + } if s.Gate != nil { if s.Gate.JudgedAt == nil { allJudged = false @@ -349,7 +361,15 @@ func (p Plan) points(now time.Time, applied AppliedLookup, w *WalkPhases) []poin if !allJudged { judged = nil } - out = append(out, point{phase: PhaseSend, tier: t, at: first}, point{phase: PhaseJudge, tier: t, at: judged}) + out = append(out, point{phase: PhaseSend, tier: t, at: first}) + if anyBusy { + said := "" + if !allBusyEnded { + busyEnded, said = nil, "waits for a busy container on "+busyOn + } + out = append(out, point{phase: PhaseBusy, tier: t, at: busyEnded, said: said}) + } + out = append(out, point{phase: PhaseJudge, tier: t, at: judged}) } else { out = append(out, point{phase: PhaseSend, tier: t, none: true, said: "no first machine: nothing to judge"}, point{phase: PhaseJudge, tier: t, none: true, said: "no first machine: nothing to judge"}) diff --git a/internal/link/builds_nats.go b/internal/link/builds_nats.go index b4e7a324..57053452 100644 --- a/internal/link/builds_nats.go +++ b/internal/link/builds_nats.go @@ -175,6 +175,11 @@ type MachineOptions struct { // can take a build, which is what its container's health check reads (novox/hq issue 544). Nil is // told nothing. Taking func() + // Claim is asked once a request is fetched, just before it is handed over: false hands it back to the + // queue at once, for another holder. A holder being drained says so here (novox/hq ADR 0335): it marks + // the request in hand first and reads its drain after, so an ask of `busy` racing the fetch sees one or + // the other, never neither. Nil claims every one. + Claim func() bool } // MachineOverNATS takes build work from the current build role. @@ -317,6 +322,12 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build)) endCancelled(ctx, build) continue } + // **Drained in the moment it was fetched** (novox/hq ADR 0335): handed back at once, as a pause + // landing then is. + if m.opts.Claim != nil && !m.opts.Claim() { + _ = msg.Nak() + continue + } // A build outlives the acknowledgement window many times over; said while it runs, // as the controller says it for its own long handlers, so the server neither hands // the ask to a second machine nor counts the wait against its deliveries. diff --git a/internal/link/enrolment.go b/internal/link/enrolment.go index de36001f..c41da60f 100644 --- a/internal/link/enrolment.go +++ b/internal/link/enrolment.go @@ -11,6 +11,7 @@ import ( "fmt" "log" "sort" + "strings" "time" "github.com/novox/mesh-controller/internal/broker" @@ -418,6 +419,12 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err for id, why := range report.Failed { doing.Failed = append(doing.Failed, inventory.FailedResource{ID: id, Error: why}) } + // And a container left running because it said busy, or replaced after it waited (novox/hq ADR 0335): + // kept with the account, for the walk whose first-machine gate waits on it. + for _, b := range report.Busy { + doing.Busy = append(doing.Busy, inventory.BusyWait{ID: b.ID, Container: b.Container, State: b.State, + Since: b.Since, Until: b.Until, Said: b.Said}) + } // Ordered, so two readings of one failure are the same reading. sort.Slice(doing.Failed, func(i, j int) bool { return doing.Failed[i].ID < doing.Failed[j].ID }) @@ -492,7 +499,14 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err } // And how long the machine took, from the send to this first account of it (novox/hq to-be 45 // Phase 0): what the sent-not-reported bound will be set from. Said if lost, never a failure. - if err := e.Inventory.RecordApplyDuration(ctx, report.Node, report.Declared, doing.Outcome); err != nil { + // + // **Not while a container waits because it said busy** (novox/hq ADR 0335 §4): the machine is not yet + // running what it was sent, and the account that says the container replaced is the one that ends the + // apply — so a walk's rest counts the wait in its delivery time, never exempted. + if doing.Waiting() { + log.Printf("%s applied declaration %s and leaves a busy container running until it says idle: %s", + report.Node, short(report.Declared), busyWords(doing.Busy)) + } else if err := e.Inventory.RecordApplyDuration(ctx, report.Node, report.Declared, doing.Outcome); err != nil { log.Printf("how long %s took to apply could not be recorded: %v", report.Node, err) } @@ -504,3 +518,15 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err } return news, e.Inventory.RecordOwned(ctx, node.ID, report.Applied) } + +// busyWords is every container waiting because it said busy, in a line (novox/hq ADR 0335). +func busyWords(busy []inventory.BusyWait) string { + var out []string + for _, b := range busy { + if b.State == inventory.BusyWaiting { + out = append(out, fmt.Sprintf("%s since %s, at the latest until %s", b.Container, + b.Since.UTC().Format(time.RFC3339), b.Until.UTC().Format(time.RFC3339))) + } + } + return strings.Join(out, "; ") +} diff --git a/internal/link/protocol.go b/internal/link/protocol.go index 63349ece..b9452da9 100644 --- a/internal/link/protocol.go +++ b/internal/link/protocol.go @@ -272,6 +272,12 @@ type Report struct { // `witness` and `not-reversible` are sent only to a machine whose report carries it. Witness int `json:"witness,omitempty"` + // Busy is every container the reporting apply left running because it said busy, and every one it + // replaced after it had waited (novox/hq ADR 0335; mesh-host internal/link/messages.go): a walk's + // first-machine gate waits while its module's container is listed waiting, and starts at the report + // that no longer lists it so. Absent from an engine older than BusyContract, which never waits. + Busy []Busy `json:"busy,omitempty"` + // Health is the machine's word on every long-running resource it runs for a module (novox/hq ADR // 0240, to-be 48 §4; mesh-host's internal/liveness): its state, since when, its failing streak and // the restarts its node-engine counted. **Absent from an engine older than the judging**, which is @@ -436,6 +442,31 @@ const ReadinessContract = 2 // parses strictly and would refuse the whole declaration for it. const RootContract = 3 +// BusyContract is the statement of an engine that also reads a container's `busy` (novox/hq ADR 0335): it asks +// a container that declares it before replacing it and says the wait in its report. Only to such an engine is +// the field sent: an older one parses strictly and would refuse the whole declaration for it. The same number as +// mesh-host internal/link/messages.go's BusyContract. +const BusyContract = 8 + +// The states of a busy container in a report (novox/hq ADR 0335 §2, §3). +const ( + BusyWaiting = "waiting" + BusyReplacedIdle = "replaced-when-idle" + BusyReplacedAtBound = "replaced-at-bound" +) + +// Busy is one container a busy command held, as a node-engine reports it (novox/hq ADR 0335). ID is the +// resource's id, `.`; Since when the first apply that would have replaced it ran, Until when +// the engine's bound replaces it at the latest, Said the last output of its busy command. +type Busy struct { + ID string `json:"id"` + Container string `json:"container"` + State string `json:"state"` + Since time.Time `json:"since"` + Until time.Time `json:"until"` + Said string `json:"said,omitempty"` +} + // ReasonRoot starts the reason of an account verdict that found a way to root (ADR 0266); the node-engine's // own words (mesh-host internal/accounts ReasonRoot). const ReasonRoot = "can become root without a person" diff --git a/testdata/store-tests b/testdata/store-tests index 14d5ada0..017e35b4 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -37,3 +37,4 @@ ./cmd/mesh-controller TestReplay401 ./cmd/mesh-controller TestReplay201ThePlansSend ./internal/inventory TestAWalksRestIsReadFromEachMachinesOwnSend +./internal/inventory TestAReportOfABusyContainerIsKeptAndReadBack diff --git a/vendor/github.com/novox/mesh-host/internal/declaration/busy.go b/vendor/github.com/novox/mesh-host/internal/declaration/busy.go new file mode 100644 index 00000000..3b734852 --- /dev/null +++ b/vendor/github.com/novox/mesh-host/internal/declaration/busy.go @@ -0,0 +1,87 @@ +package declaration + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + "time" +) + +// Busy is how a long-running container says it holds work that would be lost if it were stopped now +// (novox/hq ADR 0335 §1): a command run inside it, by its shell, as an exec health check is, that exits 0 +// while it is busy and non-zero when it is idle. Being asked is also the container's notice that it is +// about to be replaced: from the first ask it takes no new work, so its answer ends once the work in hand +// ends. +// +// **Not part of what the container is.** Busy is not in the container's spec: a changed command changes +// when the container is replaced, never whether — a container is not recreated for its busy command alone. +type Busy struct { + Command string `json:"command"` + // Timeout is how long one ask may take; empty is BusyTimeoutDefault. + Timeout string `json:"timeout,omitempty"` +} + +// The bounds of an ask (novox/hq ADR 0335). +const ( + BusyTimeoutDefault = 10 * time.Second + // BusyTimeoutCeiling is the longest one ask may take: an ask runs inside an apply, and an apply waits + // for nothing longer than a look. + BusyTimeoutCeiling = time.Minute + // BusyBound is how long a container that says busy is left running after the first apply that would + // have replaced it (ADR 0335 §3). The node-engine's, not the module's: no module holds its own + // replacement for ever. + BusyBound = 15 * time.Minute +) + +// UnmarshalJSON reads a busy strictly, as everything a declaration carries is read. +func (b *Busy) UnmarshalJSON(raw []byte) error { + type plain Busy + var p plain + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(&p); err != nil { + return fmt.Errorf("busy: %w", err) + } + *b = Busy(p) + return nil +} + +// Within is how long one ask may take. Validated on arrival, so an unreadable timeout is the default. +func (b *Busy) Within() time.Duration { + if d, err := time.ParseDuration(b.Timeout); err == nil && d > 0 { + return d + } + return BusyTimeoutDefault +} + +// problems holds a container's busy to its shape: a one-line command, a timeout within its bounds, and only +// on a container that stays up. +func (b *Busy) problems(where string, longRunning bool) []string { + if b == nil { + return nil + } + var problems []string + say := func(format string, args ...any) { + problems = append(problems, where+": "+fmt.Sprintf(format, args...)) + } + if !longRunning { + say("busy is asked of a container that stays up; a step or a scheduled run is never replaced mid-run") + } + if strings.TrimSpace(b.Command) == "" { + say("busy needs a command") + } + if strings.ContainsAny(b.Command, "\n\r") { + say("busy's command is one line") + } + if b.Timeout != "" { + d, err := time.ParseDuration(b.Timeout) + switch { + case err != nil: + say("busy's timeout is a duration, not %q", b.Timeout) + case d <= 0 || d > BusyTimeoutCeiling: + say("an ask of busy takes more than nothing and at most %s, not %s", BusyTimeoutCeiling, d) + } + } + return problems +} diff --git a/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go b/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go index 50db892e..a265afa1 100644 --- a/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go +++ b/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go @@ -12,6 +12,7 @@ import ( "fmt" "io" "net" + "path/filepath" "reflect" "regexp" "slices" @@ -132,7 +133,13 @@ func (d *Directory) validate(where string, _ bool) []string { if d.Path == "" { problems = append(problems, where+": a directory needs a path") } - return append(problems, checkMode(where, d.Mode)...) + // A directory's mode may carry the setgid bit (2xxx), so what is made beneath it keeps its group (novox/hq ADR + // 0334 §1); never setuid or sticky, which mean nothing the mesh asks of a directory it owns. + mode := d.Mode + if len(mode) == 4 && mode[0] == '2' { + mode = "0" + mode[1:] + } + return append(problems, checkMode(where, mode)...) } // File is a file with literal content. The host renders nothing. @@ -456,14 +463,51 @@ const ( // **The distinction 04-ISSUES/036 and 026 turn on.** A `directory` resource is the mesh's own — // it creates it, sets its owner and mode, and removes it when empty ([ADR 0030](novox/hq)). A // media library, a download spool is the operator's: it existed before the mesh, several modules -// read and write it at once, and the mesh must not create, chown, reconcile or remove it. The -// host confirms it is there and mounts it; nothing else. +// read and write it at once, and the mesh must not create, chown, chmod or remove it. The host +// confirms it is there and mounts it; nothing else. Library data is a library's (novox/hq ADR 0328), +// not an access. type Access struct { ID string `json:"id"` Type Type `json:"type"` Path string `json:"path"` // Mode is how this module reaches the path: read or read-write. Absent narrows to read. Mode string `json:"mode,omitempty"` + + // Users and Taken are what a controller speaking ADR 0325 sends (the identities that use the folder, and + // that the operator's warrant was carried out): accepted, so its declaration is not refused whole, and not + // acted on since ADR 0328 superseded ADR 0325 — no ACL entry is kept. + Users []AccessUser `json:"users,omitempty"` + Taken bool `json:"taken,omitempty"` + + // Grant is what a controller older than ADR 0325 still sends (ADR 0312's group grant, superseded): accepted + // so its declaration is not refused whole, and never acted on — no account is put in a group for data (§5). + Grant json.RawMessage `json:"grant,omitempty"` +} + +// The uses an access's user has (ADR 0051 §3), and where it runs (ADR 0325 §4). +const ( + FromContainer = "container" + FromToolRunner = "tool-runner" +) + +// AccessUser is one identity that uses an accessed folder: its name and uid as the controller allocated them, its +// use, and where it runs. A tool runner's account may come with no uid (0 or absent): the engine reads it by name. +type AccessUser struct { + Identity string `json:"identity"` + UID int `json:"uid,omitempty"` + Use string `json:"use"` + From string `json:"from"` +} + +// UnmarshalJSON reads a user as strictly as a resource is read. +func (u *AccessUser) UnmarshalJSON(raw []byte) error { + type plain AccessUser + var p plain + if err := strictDecode(raw, &p); err != nil { + return fmt.Errorf("an access's user: %w", err) + } + *u = AccessUser(p) + return nil } func (a *Access) Identity() string { return a.ID } @@ -482,9 +526,43 @@ func (a *Access) validate(where string, _ bool) []string { problems = append(problems, fmt.Sprintf( "%s: an access is %q or %q, not %q", where, AccessRead, AccessReadWrite, a.Mode)) } + for _, u := range a.Users { + switch { + case u.Identity == "" || !accountName.MatchString(u.Identity): + problems = append(problems, fmt.Sprintf("%s: a user of an access is an account's name, and %q is not one", + where, u.Identity)) + case u.UID < 0 || (u.UID == 0 && u.From != FromToolRunner): + // Never root (novox/hq ADR 0325 §11): an entry for uid 0 would be a right root has anyway, and a + // number below it is no account. A tool runner's account alone may come without its number: the + // controller cannot know the operator's account's, and the engine reads it by name + // (apply.ResolveRunners), refusing one that is root's. + problems = append(problems, fmt.Sprintf("%s: the user %s has uid %d, and a module's identity is never "+ + "root or below it (novox/hq ADR 0325 §11)", where, u.Identity, u.UID)) + } + if u.Use != AccessRead && u.Use != AccessReadWrite { + problems = append(problems, fmt.Sprintf("%s: the user %s uses it %q, and a use is %q or %q", where, + u.Identity, u.Use, AccessRead, AccessReadWrite)) + } + if u.Use == AccessReadWrite && a.Mode != AccessReadWrite { + problems = append(problems, fmt.Sprintf("%s: the user %s writes an access declared %s", where, + u.Identity, orRead(a.Mode))) + } + if u.From != FromContainer && u.From != FromToolRunner { + problems = append(problems, fmt.Sprintf("%s: the user %s runs from %q, and that is %q or %q", where, + u.Identity, u.From, FromContainer, FromToolRunner)) + } + } return problems } +// orRead is an access's mode as said: absent narrows to read. +func orRead(mode string) string { + if mode == "" { + return AccessRead + } + return mode +} + func (u *User) Identity() string { return u.ID } func (u *User) Kind() Type { return TypeUser } func (u *User) Target() string { return u.Name } @@ -1160,6 +1238,83 @@ type Container struct { // Health is how this resource is ready (novox/hq ADR 0240 rule 2, Phase B): one kind and its // timing, judged by the node-engine beside liveness. Absent: judged alive or not, and nothing more. Health *Health `json:"health,omitempty"` + + // Busy is how this container says it holds work a replacement would lose (novox/hq ADR 0335): asked + // before it is replaced, and while it says busy it is left running, within the bound. Absent: never + // busy, and replaced at once, as before. + Busy *Busy `json:"busy,omitempty"` + + // RunAs is who the container runs as, resolved by the controller from the module's identity and the + // image's declared interface (novox/hq ADR 0325 §2, §3): told by environment (the image reads PUID and + // PGID, say) or by the runtime's own user option. Part of the spec, so a change recreates it. Absent: the + // container runs as its image says. Its key is `identity`, the manifest's word; the field is not, because + // a resource's Identity is its id. + RunAs *ContainerIdentity `json:"identity,omitempty"` +} + +// The ways an image is told who to run as (novox/hq ADR 0325 §2). +const ( + IdentityByEnv = "env" + IdentityByRun = "run" +) + +// ContainerIdentity is who a container runs as, and how its image is told. +type ContainerIdentity struct { + // Mode is IdentityByEnv or IdentityByRun. + Mode string `json:"mode"` + // UserEnv and GroupEnv are the variables the image reads the user's and the group's number from, for + // IdentityByEnv: "PUID", "PGID". + UserEnv string `json:"user_env,omitempty"` + GroupEnv string `json:"group_env,omitempty"` + UID int `json:"uid"` + GID int `json:"gid"` +} + +// UnmarshalJSON reads an identity as strictly as a resource is read. +func (i *ContainerIdentity) UnmarshalJSON(raw []byte) error { + type plain ContainerIdentity + var p plain + if err := strictDecode(raw, &p); err != nil { + return fmt.Errorf("a container's identity: %w", err) + } + *i = ContainerIdentity(p) + return nil +} + +// envName is a variable's name as a runtime takes it. +var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + +func (i *ContainerIdentity) problems(where string, env map[string]string) []string { + if i == nil { + return nil + } + var problems []string + switch i.Mode { + case IdentityByEnv: + if !envName.MatchString(i.UserEnv) || (i.GroupEnv != "" && !envName.MatchString(i.GroupEnv)) { + problems = append(problems, fmt.Sprintf("%s: an identity told by environment names the variables "+ + "it is told in, and %q and %q are not both names", where, i.UserEnv, i.GroupEnv)) + } + for _, name := range []string{i.UserEnv, i.GroupEnv} { + if _, set := env[name]; set && name != "" { + problems = append(problems, fmt.Sprintf("%s: env sets %s, which the identity sets; one value "+ + "for one variable", where, name)) + } + } + case IdentityByRun: + if i.UserEnv != "" || i.GroupEnv != "" { + problems = append(problems, where+": an identity told by the runtime's user option names no variable") + } + default: + problems = append(problems, fmt.Sprintf("%s: an identity is told by %q or %q, not %q", where, + IdentityByEnv, IdentityByRun, i.Mode)) + } + // Never root (novox/hq ADR 0325 §11): refused here, so nothing of the declaration is made or started. + if i.UID <= 0 || i.GID <= 0 { + problems = append(problems, fmt.Sprintf("%s: the identity is uid %d, gid %d, and a module's identity is "+ + "never root, nor a number below it (novox/hq ADR 0325 §11)", where, i.UID, i.GID)) + } + return problems } func (c *Container) Identity() string { return c.ID } @@ -1168,6 +1323,7 @@ func (c *Container) Target() string { return c.Name } func (c *Container) validate(where string, _ bool) []string { problems := c.Health.problems(where, true, !c.RunOnce && c.Schedule == "") + problems = append(problems, c.Busy.problems(where, !c.RunOnce && c.Schedule == "")...) if c.Name == "" { problems = append(problems, where+": a container needs a name") } @@ -1242,6 +1398,7 @@ func (c *Container) validate(where string, _ bool) []string { problems = append(problems, where+": networks is for a container that keeps running; a step "+ "runs and exits, and joins nothing afterwards") } + problems = append(problems, c.RunAs.problems(where+": container "+c.Name, c.Env)...) return append(problems, checkImage(where, c.Image)...) } @@ -1392,6 +1549,185 @@ type Declaration struct { // the generation of what it puts back, which may be older than what the machine applied, and is // not refused for it. PutBack bool + + // Identities is the identity of every module assigned to this machine (novox/hq ADR 0328 §4): an account + // and a group `mesh-` with the numbers the controller allocated once for the module, the same on + // every machine. The engine makes each before any resource is applied, and removes what it made of one no + // longer declared. Absent is a controller older than ADR 0325. + Identities []Identity + + // Groups is the group of every library this machine is reached by (novox/hq ADR 0328 §3): `mesh-` + // with the number the controller allocated once, mesh-wide. The engine makes each, and removes what it made of + // one no longer declared. Absent is a controller older than ADR 0328. + Groups []Group + + // Libraries is every library this machine is reached by (ADR 0328 §1, §6, §9): its root here, its group, whether + // this is its home, whether the operator's warrant to take it was carried out, the sub-paths its modules ask for, + // and the tool runner's account to put in its group (§11). Absent is a controller older than ADR 0328. + Libraries []Library +} + +// Identity is one module's identity, as the controller allocated it (ADR 0328 §4): its user `mesh-` with +// uid, its own group of the same name and number, its primary group gid (its own, or its first library's group), +// and the groups of its other libraries as added groups. +type Identity struct { + Module string `json:"module"` + Name string `json:"name"` + UID int `json:"uid"` + GID int `json:"gid"` + Groups []int `json:"groups,omitempty"` + // Runner is the tool runner's account, put in the module's own group (novox/hq ADR 0334 §2) when its tools run + // on this machine. Absent for a module that declares no tools here, and from a controller older than ADR 0334. + Runner string `json:"runner,omitempty"` +} + +// Group is one library's group, as the controller allocated it (ADR 0328 §3). +type Group struct { + Name string `json:"name"` + GID int `json:"gid"` +} + +// GroupPrefix starts every library group's name, as IdentityPrefix starts every module's. +const GroupPrefix = "mesh-" + +// LibraryAccessPrefix starts the access a library's wait and its take act name: `library:`. +const LibraryAccessPrefix = "library:" + +// Library is one library as this machine sees it (ADR 0328). +type Library struct { + Library string `json:"library"` + // Root is the library's root on this machine: on its home the data itself, elsewhere the mount point the mounts + // module mounts it at. + Root string `json:"root"` + Group string `json:"group"` + GID int `json:"gid"` + // Home is that the library's data lives on this machine: only there does the engine keep it. + Home bool `json:"home"` + // Taken is the controller saying the operator's warrant to take it was carried out (§7). + Taken bool `json:"taken,omitempty"` + // Paths are the sub-paths its modules ask for, relative to the root. + Paths []string `json:"paths,omitempty"` + // Runner is the tool runner's account, put in the library's group (§11). Absent when no module's tools use it. + Runner string `json:"runner,omitempty"` +} + +// checkLibraries judges the groups and libraries on arrival: every library's group declared, named for it, never +// root's number; paths that stay below the root; a runner that is an account's name; and no two roots that nest +// (§13). +func checkLibraries(groups []Group, libs []Library) []string { + var problems []string + byName := map[string]int{} + numbers := map[int]string{} + for _, g := range groups { + where := fmt.Sprintf("group %q", g.Name) + if !strings.HasPrefix(g.Name, GroupPrefix) || !accountName.MatchString(g.Name) { + problems = append(problems, fmt.Sprintf("%s: a library's group is named %s", where, GroupPrefix)) + } + if g.GID <= 0 { + problems = append(problems, fmt.Sprintf("%s: gid %d, and a library's group is never root, nor a number "+ + "below it (novox/hq ADR 0328 §15); nothing of this declaration was made", where, g.GID)) + } + if _, twice := byName[g.Name]; twice { + problems = append(problems, where+": declared twice") + } + if other, used := numbers[g.GID]; used { + problems = append(problems, fmt.Sprintf("%s: gid %d is %s's too", where, g.GID, other)) + } + byName[g.Name], numbers[g.GID] = g.GID, g.Name + } + seen := map[string]bool{} + for _, l := range libs { + where := fmt.Sprintf("library %q", l.Library) + switch { + case l.Library == "" || strings.ContainsAny(l.Library, "/ \t\n"): + problems = append(problems, where+": a library is named by one word") + case seen[l.Library]: + problems = append(problems, where+": declared twice") + } + seen[l.Library] = true + if !filepath.IsAbs(l.Root) || filepath.Clean(l.Root) != l.Root || l.Root == "/" { + problems = append(problems, fmt.Sprintf("%s: its root %q is not an absolute, clean path below /", where, + l.Root)) + } + if gid, ok := byName[l.Group]; !ok || gid != l.GID { + problems = append(problems, fmt.Sprintf("%s: its group %s (gid %d) is not one this declaration makes", + where, l.Group, l.GID)) + } + if l.Group != GroupPrefix+l.Library { + problems = append(problems, fmt.Sprintf("%s: its group is %s%s, and %s is not", where, GroupPrefix, + l.Library, l.Group)) + } + for _, p := range l.Paths { + if p == "" || filepath.IsAbs(p) || filepath.Clean(p) != p || p == ".." || strings.HasPrefix(p, "../") { + problems = append(problems, fmt.Sprintf("%s: the path %q is not a clean path below its root", where, p)) + } + } + if l.Runner != "" && !accountName.MatchString(l.Runner) { + problems = append(problems, fmt.Sprintf("%s: its runner %q is not an account's name", where, l.Runner)) + } + } + for i, a := range libs { + for _, b := range libs[i+1:] { + if a.Root != "" && b.Root != "" && (a.Root == b.Root || strings.HasPrefix(a.Root, b.Root+"/") || + strings.HasPrefix(b.Root, a.Root+"/")) { + problems = append(problems, fmt.Sprintf("library %q: its root and library %q's nest, and libraries "+ + "never overlap (novox/hq ADR 0328 §13)", a.Library, b.Library)) + } + } + } + return problems +} + +// IdentityPrefix starts every module identity's name. +const IdentityPrefix = "mesh-" + +// checkIdentities judges the identities on arrival: each named for its module, never root, and no name or number +// twice — two modules with one account would write as each other. +func checkIdentities(ids []Identity, groups []Group) []string { + var problems []string + names, uids := map[string]bool{}, map[int]string{} + libraryGroups := map[int]bool{} + for _, g := range groups { + libraryGroups[g.GID] = true + } + for _, id := range ids { + where := fmt.Sprintf("identity %q", id.Name) + if id.Module == "" || id.Name != IdentityPrefix+id.Module || !accountName.MatchString(id.Name) { + problems = append(problems, fmt.Sprintf("%s: a module's identity is named %s, and this one is "+ + "for the module %q", where, IdentityPrefix, id.Module)) + } + if id.UID <= 0 || id.GID <= 0 { + problems = append(problems, fmt.Sprintf("%s: uid %d, gid %d, and a module's identity is never root, "+ + "nor a number below it (novox/hq ADR 0328 §15); nothing of this declaration was made", where, + id.UID, id.GID)) + } + if names[id.Name] { + problems = append(problems, where+": declared twice") + } + if other, used := uids[id.UID]; used { + problems = append(problems, fmt.Sprintf("%s: uid %d is %s's too", where, id.UID, other)) + } + // Its primary group is its own (gid == uid) or a library's this declaration makes (ADR 0328 §4); so is + // every added group. A module's own number is never a library group's. + if id.GID != id.UID && !libraryGroups[id.GID] { + problems = append(problems, fmt.Sprintf("%s: its primary group gid %d is neither its own (%d) nor a "+ + "library's this declaration makes", where, id.GID, id.UID)) + } + if libraryGroups[id.UID] { + problems = append(problems, fmt.Sprintf("%s: uid %d is a library group's number too", where, id.UID)) + } + for _, g := range id.Groups { + if !libraryGroups[g] { + problems = append(problems, fmt.Sprintf("%s: its added group gid %d is not a library's this "+ + "declaration makes", where, g)) + } + } + if id.Runner != "" && !accountName.MatchString(id.Runner) { + problems = append(problems, fmt.Sprintf("%s: its runner %q is not an account's name", where, id.Runner)) + } + names[id.Name], uids[id.UID] = true, id.Name + } + return problems } // LeftOutModuleOf says which left-out module a recorded resource belongs to, if any: its id is the @@ -1561,6 +1897,13 @@ type envelope struct { // sends them only to a host whose reports say `reads_generation`. Generation int64 `json:"generation,omitempty"` PutBack bool `json:"put_back,omitempty"` + // Identities is optional on the wire too (novox/hq ADR 0325 §3, ADR 0328 §4). **An older host refuses this key**; a + // controller sends it only to a host whose health says link.IdentityContract. + Identities []Identity `json:"identities,omitempty"` + // Groups and Libraries are optional too (novox/hq ADR 0328). **An older host refuses these keys**; a controller + // sends them only to a host whose health says link.LibraryContract. + Groups []Group `json:"groups,omitempty"` + Libraries []Library `json:"libraries,omitempty"` } func parse(raw []byte, allowActions bool) (*Declaration, error) { @@ -1578,8 +1921,10 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) { } d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption, Sequence: env.Sequence, - Epoch: env.Epoch, LeftOut: env.LeftOut, Generation: env.Generation, PutBack: env.PutBack} - var problems []string + Epoch: env.Epoch, LeftOut: env.LeftOut, Generation: env.Generation, PutBack: env.PutBack, + Identities: env.Identities, Groups: env.Groups, Libraries: env.Libraries} + problems := checkIdentities(env.Identities, env.Groups) + problems = append(problems, checkLibraries(env.Groups, env.Libraries)...) if env.Generation < 0 { // Below zero would read as "none claimed" and pass every refusal of an older generation. problems = append(problems, fmt.Sprintf("an assignment generation below zero (%d) is not one the "+ @@ -1732,6 +2077,10 @@ func checkWhileStopped(resources []Resource) []string { return problems } +// accountName is what an identity's or an access user's name may be: the portable account name the user tools +// take, optionally ending in `$`, never starting with a dash — it is handed to them as an argument. +var accountName = regexp.MustCompile(`^[A-Za-z0-9_.][A-Za-z0-9_.-]*\$?$`) + func strictDecode(raw []byte, into any) error { // DisallowUnknownFields is the whole point rather than strictness for its own sake: a // field the host does not know is a thing the control plane believes it asked for. diff --git a/vendor/modules.txt b/vendor/modules.txt index 4a86d3fb..62c046af 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -78,7 +78,7 @@ github.com/nats-io/nkeys # github.com/nats-io/nuid v1.0.1 ## explicit github.com/nats-io/nuid -# github.com/novox/mesh-host v0.0.0 => git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f +# github.com/novox/mesh-host v0.0.0 => git.novox.be/novox/mesh-host v0.0.0-20261012224132-bb60e7b584d8 ## explicit; go 1.26.0 github.com/novox/mesh-host/internal/declaration github.com/novox/mesh-host/rootsearch @@ -138,4 +138,4 @@ golang.org/x/text/width # golang.org/x/time v0.15.0 ## explicit; go 1.25.0 golang.org/x/time/rate -# github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f +# github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261012224132-bb60e7b584d8