A walk's first-machine gate waits for a busy container, and mesh-builder busy drains the agent (issue 594, hq ADR 0335) #283
@@ -0,0 +1,101 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// **The build agent says whether it is busy, and being asked drains it** (novox/hq ADR 0335 §1, §5). Its
|
||||
// container declares `busy` as `mesh-builder busy`: the node-engine runs it inside the container before it
|
||||
// replaces it. It exits 0 while a build is in hand and 1 when none is; and from the first ask the agent takes
|
||||
// no new build request, so its answer ends once the build in hand ends — a replaced builder finishes the check
|
||||
// it runs, instead of the build queue handing it out again from the start (issue 594).
|
||||
//
|
||||
// Two marks in the container's own temporary directory, as the ready mark (ready.go): a new container starts
|
||||
// without them, and a restarted one removes them before it takes work again.
|
||||
//
|
||||
// - in hand: written by the agent when it claims a request, before it reads whether it is drained; removed
|
||||
// when the build ends.
|
||||
// - drained: written by the ask, before it reads whether a build is in hand.
|
||||
//
|
||||
// Each side writes its own mark before it reads the other's, so an ask racing a fetch sees the build in hand,
|
||||
// or the agent sees the drain and hands the request back; never neither.
|
||||
|
||||
func inHandMark() string { return filepath.Join(os.TempDir(), "mesh-builder.in-hand") }
|
||||
func drainMark() string { return filepath.Join(os.TempDir(), "mesh-builder.drained") }
|
||||
|
||||
// drained says the agent was asked whether it is busy: it takes no new build request.
|
||||
func drained() bool {
|
||||
_, err := os.Stat(drainMark())
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// claim marks a request in hand, unless the agent is drained: then nothing is in hand, and the request goes
|
||||
// back to the queue.
|
||||
func claim() bool {
|
||||
// Made, never written over: one build at a time, and a mark already there is a build in hand.
|
||||
f, err := os.OpenFile(inHandMark(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
|
||||
if err != nil {
|
||||
// Unmarked, an ask would say idle while this build runs: the build is not taken.
|
||||
fmt.Fprintf(os.Stderr, "cannot mark a build in hand (%s), so it is handed back: %v\n", inHandMark(), err)
|
||||
return false
|
||||
}
|
||||
_, err = f.WriteString(time.Now().UTC().Format(time.RFC3339) + "\n")
|
||||
if closeErr := f.Close(); err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
if err != nil {
|
||||
_ = os.Remove(inHandMark())
|
||||
fmt.Fprintf(os.Stderr, "cannot mark a build in hand (%s), so it is handed back: %v\n", inHandMark(), err)
|
||||
return false
|
||||
}
|
||||
if drained() {
|
||||
_ = os.Remove(inHandMark())
|
||||
fmt.Fprintln(os.Stderr, "asked whether busy, so this agent takes no new build: handed back")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// holding says what is in hand, for the ask to print: written once the request is read.
|
||||
func holding(what string) {
|
||||
_ = os.WriteFile(inHandMark(), []byte(time.Now().UTC().Format(time.RFC3339)+" "+what+"\n"), 0o644)
|
||||
}
|
||||
|
||||
// released ends the build in hand.
|
||||
func released() {
|
||||
if err := os.Remove(inHandMark()); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
fmt.Fprintf(os.Stderr, "cannot remove the mark of the build in hand (%s): %v\n", inHandMark(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// undrained clears both marks at the start of a holder: a container started again is asked afresh.
|
||||
func undrained() error {
|
||||
for _, mark := range []string{drainMark(), inHandMark()} {
|
||||
if err := os.Remove(mark); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("cannot remove %s: %w", mark, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// busy is the ask: it drains the agent, then says whether a build is in hand. Nil — exit 0 — while one is.
|
||||
func busy() (string, error) {
|
||||
if err := os.WriteFile(drainMark(), []byte(time.Now().UTC().Format(time.RFC3339)+"\n"), 0o644); err != nil {
|
||||
// Not drained, it could take the next build: said, and answered busy only for what is in hand now.
|
||||
fmt.Fprintf(os.Stderr, "cannot mark this agent drained (%s): %v\n", drainMark(), err)
|
||||
}
|
||||
held, err := os.ReadFile(inHandMark())
|
||||
if err != nil {
|
||||
return "", errors.New("idle: no build in hand, and this agent takes no new one")
|
||||
}
|
||||
since, what, _ := strings.Cut(strings.TrimSpace(string(held)), " ")
|
||||
if what == "" {
|
||||
what = "a build"
|
||||
}
|
||||
return fmt.Sprintf("busy: %s, in hand since %s; taking no new build", what, since), nil
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// `mesh-builder busy` answers 0 while a build is in hand and non-zero when idle, and after its first ask the
|
||||
// agent takes no new build request (novox/hq ADR 0335 §5).
|
||||
func TestBusyAnswersWhileABuildIsInHandAndDrainsTheAgent(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
if err := undrained(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if drained() {
|
||||
t.Fatal("an agent never asked is drained")
|
||||
}
|
||||
// A build is claimed and runs.
|
||||
if !claim() {
|
||||
t.Fatal("an agent never asked refused a build")
|
||||
}
|
||||
holding("novox/mesh-controller (check-280)")
|
||||
said, err := busy()
|
||||
if err != nil || !strings.Contains(said, "novox/mesh-controller (check-280)") {
|
||||
t.Fatalf("an agent with a build in hand did not answer busy with what it holds: %q %v", said, err)
|
||||
}
|
||||
// From that ask on it takes nothing new: the fetch is not made, and one fetched is handed back.
|
||||
if !drained() {
|
||||
t.Fatal("the ask did not drain the agent")
|
||||
}
|
||||
if claim() {
|
||||
t.Fatal("a drained agent took a new build")
|
||||
}
|
||||
// The build in hand is still in hand after the refused claim, and busy goes on saying so.
|
||||
if _, err := busy(); err != nil {
|
||||
t.Fatalf("a refused claim ended the build in hand: %v", err)
|
||||
}
|
||||
// It ends: idle, and still drained.
|
||||
released()
|
||||
if _, err := busy(); err == nil {
|
||||
t.Fatal("an agent with nothing in hand answered busy")
|
||||
}
|
||||
if claim() {
|
||||
t.Fatal("a drained agent took a build after its last one ended")
|
||||
}
|
||||
// Started again, it is asked afresh.
|
||||
if err := undrained(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if drained() || !claim() {
|
||||
t.Fatal("a holder started again is still drained")
|
||||
}
|
||||
released()
|
||||
}
|
||||
|
||||
// An ask racing a fetch sees the build in hand, or the agent sees the drain and hands the request back: never
|
||||
// neither, which would replace the container under a build just taken.
|
||||
func TestAnAskRacingAFetchNeverMissesTheBuild(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
for i := 0; i < 200; i++ {
|
||||
if err := undrained(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var claimed bool
|
||||
var askErr error
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() { defer wg.Done(); claimed = claim() }()
|
||||
go func() { defer wg.Done(); _, askErr = busy() }()
|
||||
wg.Wait()
|
||||
if claimed && askErr != nil {
|
||||
t.Fatalf("round %d: the agent took a build and the ask said idle", i)
|
||||
}
|
||||
released()
|
||||
}
|
||||
}
|
||||
@@ -71,6 +71,12 @@ digest of their own configuration. The result is printed as JSON.
|
||||
|
||||
exits 0 once the agent running in this container is bound to its build work and pulls it, and
|
||||
1 before: its container's health check.
|
||||
|
||||
mesh-builder busy
|
||||
|
||||
exits 0 while the agent running in this container has a build in hand, and 1 when it has none;
|
||||
from the first ask the agent takes no new build: its container's busy command, asked before the
|
||||
container is replaced (novox/hq ADR 0335).
|
||||
`
|
||||
|
||||
func run() error {
|
||||
@@ -84,6 +90,15 @@ func run() error {
|
||||
case "ready":
|
||||
// The container's health check (novox/hq issue 544): exits non-zero until the agent takes work.
|
||||
return ready()
|
||||
case "busy":
|
||||
// The container's busy command (novox/hq ADR 0335): drains the agent, and exits non-zero once
|
||||
// nothing is in hand.
|
||||
said, err := busy()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(said)
|
||||
return nil
|
||||
default:
|
||||
fmt.Print(usage)
|
||||
return nil
|
||||
@@ -93,6 +108,9 @@ func run() error {
|
||||
if err := notReady(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := undrained(); err != nil {
|
||||
return err
|
||||
}
|
||||
credential, err := brokerFrom()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -145,8 +163,13 @@ func run() error {
|
||||
// **What an earlier holder here left goes before anything is taken** (novox/hq issue 306): a holder
|
||||
// recreated mid-check left the check's store and bus running, and the ask went to another machine.
|
||||
h.removeLeftBehind()
|
||||
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused,
|
||||
Taking: func() { taking(seat) }})
|
||||
// Paused or drained, it takes nothing new (novox/hq ADR 0219, ADR 0335): a drained agent finishes the build
|
||||
// in hand and then says idle, and its container is replaced between builds.
|
||||
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{
|
||||
Paused: func() bool { return h.Paused() || drained() },
|
||||
Taking: func() { taking(seat) },
|
||||
Claim: claim,
|
||||
})
|
||||
defer machine.Close()
|
||||
|
||||
// The seat's verbs, on this machine's subjects. Only the seat that declares them: the retired
|
||||
@@ -172,6 +195,8 @@ func run() error {
|
||||
publisher := builder.Registry{Address: registry, Run: builder.Command}
|
||||
|
||||
return machine.Take(ctx, func(ctx context.Context, work link.Build) {
|
||||
defer released()
|
||||
holding(work.Request().Repository + " (" + work.Request().ID + ")")
|
||||
answer(ctx, publisher, on, workspace, work, h)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ func TestBuilderDiagnosticsStayOffStdout(t *testing.T) {
|
||||
"version)": true, // --version
|
||||
`"stopping")`: true, // the loop.s shutdown line
|
||||
"usage)": true, // --help text, for a human
|
||||
"said)": true, // busy: the answer its node-engine reads (novox/hq ADR 0335)
|
||||
}
|
||||
for _, file := range []string{"once.go", "main.go"} {
|
||||
src, err := os.ReadFile(file)
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A first-node gate waits for a busy container (novox/hq ADR 0335 §4).
|
||||
//
|
||||
// When a machine reports, on what the gate sent it, that a container of a judged module was left running
|
||||
// because it said busy, the judging has not started: the build is not yet what runs there. The walk says it
|
||||
// waits for a busy container on that machine, counts no pass and no bound, and starts the judging — its
|
||||
// Since — at the report that no longer lists the container waiting. The wait is the node-engine's, bounded
|
||||
// at 15 minutes; the controller's own bound (busyGateBound past the engine's) is for an engine that stopped
|
||||
// reporting while it waited, so no walk waits for ever. An engine older than the field never lists one, and
|
||||
// its gate starts as before.
|
||||
|
||||
// busyGateSlack is how long past the engine's bound the controller still waits for the report that says the
|
||||
// container replaced: a pass of the engine (its asks every 20 seconds) and the report's way here.
|
||||
var busyGateSlack = 5 * time.Minute
|
||||
|
||||
// busyWait decides, before a judging, whether the gate waits for a busy container on a machine it judges:
|
||||
// true while it waits, the gate saying so. False when nothing waits, or no longer: a wait that ended moves the
|
||||
// gate's Since to its end. Pure but for the gate it is handed.
|
||||
func busyWait(g *inventory.PlanGate, pairs []judged, f gateFacts, now time.Time) bool {
|
||||
if g.Busy != nil && g.Busy.Ended != nil {
|
||||
return false
|
||||
}
|
||||
for _, j := range pairs {
|
||||
b, waits := busyOn(j, f)
|
||||
if !waits {
|
||||
continue
|
||||
}
|
||||
if g.Busy == nil {
|
||||
g.Busy = &inventory.GateBusy{Machine: j.node, Module: j.module, Container: b.Container, Since: b.Since}
|
||||
}
|
||||
g.Busy.Until, g.Busy.Said = b.Until, b.Said
|
||||
if now.After(g.Busy.Until.Add(busyGateSlack)) {
|
||||
// The engine said it would have replaced it by now: the judging starts, and finds the module not
|
||||
// yet healthy while the machine still lists it waiting (busyStill), until the gate's own bound.
|
||||
end := now
|
||||
g.Busy.Ended, g.Busy.PastBound = &end, true
|
||||
g.Since, g.Passes, g.LastPass, g.Healthy, g.HealthyAt = &end, 0, nil, nil, nil
|
||||
return false
|
||||
}
|
||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, g.Busy.Words(), nil
|
||||
g.Read(now, false, g.Busy.Words())
|
||||
return true
|
||||
}
|
||||
if g.Busy == nil {
|
||||
return false
|
||||
}
|
||||
// Replaced: the judging starts at the report that said so.
|
||||
end := now
|
||||
if r, said := f.reports[g.Busy.Machine]; said && r.At != nil && r.At.After(g.Busy.Since) && r.At.Before(now) {
|
||||
end = r.At.UTC()
|
||||
}
|
||||
g.Busy.Ended = &end
|
||||
g.Since, g.Passes, g.LastPass, g.Last, g.Healthy, g.HealthyAt = &end, 0, nil, "", nil, nil
|
||||
return false
|
||||
}
|
||||
|
||||
// busyOn is the container of a judged module its machine last reported, on what the gate sent it, as left
|
||||
// running because it said busy.
|
||||
func busyOn(j judged, f gateFacts) (inventory.BusyWait, bool) {
|
||||
r, said := f.reports[j.node]
|
||||
if !said || r.At == nil || !f.reportedOn(j.node, r) {
|
||||
return inventory.BusyWait{}, false
|
||||
}
|
||||
return inventory.WaitingFor(r.Busy, j.module)
|
||||
}
|
||||
|
||||
// busyStill is a judged module whose container its machine still lists waiting once the gate stopped waiting
|
||||
// for it: not yet what was sent, so not healthy, whatever its health says of the container still running.
|
||||
func busyStill(j judged, f gateFacts) (string, bool) {
|
||||
b, waits := busyOn(j, f)
|
||||
if !waits {
|
||||
return "", false
|
||||
}
|
||||
return fmt.Sprintf("%s still runs its busy container %s, past its node-engine's bound of %s",
|
||||
j.node, b.Container, b.Until.UTC().Format(time.RFC3339)), true
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A walk whose module's container waits as busy on its first machine says so, and starts that machine's
|
||||
// first-machine gate only once the machine reports the container replaced (novox/hq ADR 0335 §4).
|
||||
func TestAGateWaitsForABusyContainerAndStartsWhenItIsReplaced(t *testing.T) {
|
||||
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent,
|
||||
Sent: map[string]inventory.SentDeclaration{"novox": {Digest: "d1", Sequence: 7}}}
|
||||
pairs := []judged{{module: "build-agent", node: "novox"}, {module: "mesh-controller", node: "novox"}}
|
||||
reported := sent.Add(20 * time.Second)
|
||||
waiting := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
||||
State: inventory.BusyWaiting, Since: reported, Until: reported.Add(15 * time.Minute),
|
||||
Said: "busy: novox/mesh-controller (check of #280), in hand since 00:58:00Z"}}
|
||||
facts := func(at time.Time, busy []inventory.BusyWait) gateFacts {
|
||||
// As judgeMoves hands it: the report held against the gate's own send.
|
||||
return gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"novox": {Node: "novox",
|
||||
Outcome: inventory.OutcomeApplied, At: &at, Declared: "d1", ReportedSequence: 7, Busy: busy}}}
|
||||
}
|
||||
|
||||
now := reported.Add(time.Minute)
|
||||
if !busyWait(g, pairs, facts(reported, waiting), now) {
|
||||
t.Fatal("a gate whose module's container waits as busy judged it")
|
||||
}
|
||||
if !g.Since.Equal(sent) || g.Busy == nil || g.Busy.Machine != "novox" || g.Busy.Ended != nil {
|
||||
t.Fatalf("the gate does not keep the wait: %+v %+v", g.Since, g.Busy)
|
||||
}
|
||||
for _, said := range []string{gateLine(g), g.Last} {
|
||||
if !strings.Contains(said, "waits for a busy container on novox") {
|
||||
t.Errorf("the walk does not say it waits for a busy container there: %s", said)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(g.Last, "check of #280") {
|
||||
t.Errorf("the wait does not say what the container holds: %s", g.Last)
|
||||
}
|
||||
|
||||
// Ten minutes on, still busy, within the engine's bound: still waiting, nothing judged, no bound run.
|
||||
now = reported.Add(10 * time.Minute)
|
||||
if !busyWait(g, pairs, facts(reported.Add(9*time.Minute), waiting), now) || g.Passes != 0 {
|
||||
t.Fatalf("the gate stopped waiting within the bound: %+v", g)
|
||||
}
|
||||
|
||||
// The machine reports it replaced: the judging starts at that report.
|
||||
replacedAt := reported.Add(12 * time.Minute)
|
||||
now = replacedAt.Add(5 * time.Second)
|
||||
replaced := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
||||
State: "replaced-when-idle", Since: reported, Until: reported.Add(15 * time.Minute)}}
|
||||
if busyWait(g, pairs, facts(replacedAt, replaced), now) {
|
||||
t.Fatal("the gate still waits after the container was replaced")
|
||||
}
|
||||
if g.Since == nil || !g.Since.Equal(replacedAt) || g.Busy.Ended == nil || !g.Busy.Ended.Equal(replacedAt) {
|
||||
t.Fatalf("the gate does not start at the report that said it replaced: since %v, %+v", g.Since, g.Busy)
|
||||
}
|
||||
if strings.Contains(gateLine(g), "waits for a busy container") {
|
||||
t.Errorf("a gate judging still says it waits: %s", gateLine(g))
|
||||
}
|
||||
// And a later report listing it waiting again does not hold a gate that started.
|
||||
if busyWait(g, pairs, facts(now, waiting), now.Add(time.Minute)) {
|
||||
t.Error("a gate that started waits again")
|
||||
}
|
||||
}
|
||||
|
||||
// An engine that stops reporting while it waits holds no walk for ever: past the engine's bound and the slack,
|
||||
// the judging starts, and finds the module not yet healthy while its container is still listed waiting.
|
||||
func TestAGateWaitsNoLongerThanTheEnginesBound(t *testing.T) {
|
||||
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent}
|
||||
pairs := []judged{{module: "build-agent", node: "novox"}}
|
||||
waiting := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
||||
State: inventory.BusyWaiting, Since: sent, Until: sent.Add(15 * time.Minute)}}
|
||||
f := gateFacts{reports: map[string]inventory.Reported{"novox": {Node: "novox", Outcome: inventory.OutcomeApplied,
|
||||
At: &sent, Current: true, Busy: waiting}}}
|
||||
now := sent.Add(15*time.Minute + busyGateSlack + time.Second)
|
||||
if busyWait(g, pairs, f, now) {
|
||||
t.Fatal("a gate waited past the engine's bound and the slack")
|
||||
}
|
||||
if !g.Busy.PastBound || !g.Since.Equal(now) {
|
||||
t.Fatalf("the gate does not start, said past the bound: %+v %v", g.Busy, g.Since)
|
||||
}
|
||||
if why, still := busyStill(pairs[0], f); !still || !strings.Contains(why, "past its node-engine's bound") {
|
||||
t.Fatalf("a module still running its busy container reads as what was sent: %q", why)
|
||||
}
|
||||
}
|
||||
|
||||
// A report from an engine older than the field lists nothing: the gate starts at the send, as before; and a
|
||||
// report not about the gate's send says nothing of it.
|
||||
func TestAGateOfAnEngineThatNeverWaitsStartsAsBefore(t *testing.T) {
|
||||
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
g := &inventory.PlanGate{Machines: []string{"ace"}, Since: &sent,
|
||||
Sent: map[string]inventory.SentDeclaration{"ace": {Digest: "d2", Sequence: 9}}}
|
||||
pairs := []judged{{module: "build-agent", node: "ace"}}
|
||||
at := sent.Add(time.Minute)
|
||||
older := gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"ace": {Node: "ace", Outcome: inventory.OutcomeApplied,
|
||||
At: &at, Declared: "d2", ReportedSequence: 9}}}
|
||||
if busyWait(g, pairs, older, at) || g.Busy != nil || !g.Since.Equal(sent) {
|
||||
t.Fatalf("a gate on an engine that never waits did not start at the send: %+v %v", g.Busy, g.Since)
|
||||
}
|
||||
elsewhere := gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"ace": {Node: "ace", Outcome: inventory.OutcomeApplied,
|
||||
At: &at, Declared: "d1", ReportedSequence: 8, Busy: []inventory.BusyWait{{ID: "build-agent.server",
|
||||
State: inventory.BusyWaiting, Until: at.Add(time.Hour)}}}}}
|
||||
if busyWait(g, pairs, elsewhere, at) {
|
||||
t.Fatal("a wait reported about an earlier declaration held the gate")
|
||||
}
|
||||
}
|
||||
@@ -544,6 +544,11 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
facts.sentBuilds[j.node] = builds
|
||||
}
|
||||
}
|
||||
// **A busy container on a judged machine holds the judging before it starts** (novox/hq ADR 0335 §4): the
|
||||
// walk says it waits there, and its gate starts at the report that says the container replaced.
|
||||
if busyWait(g, pairs, facts, now) {
|
||||
return "", nil
|
||||
}
|
||||
// **What is wrong with a machine itself is the machine's** (novox/hq issue 281): read once for each
|
||||
// machine judged, apart from what is wrong with a module there, and never pinned on the module the
|
||||
// gate happens to be kept on.
|
||||
@@ -568,6 +573,9 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
for _, j := range pairs {
|
||||
w := words[j.node]
|
||||
h, said := judgeHealth(j.module, coreComponent(j.module), shelf[j.module], j.node, *g.Since, w.facts)
|
||||
if why, still := busyStill(j, facts); still && (h == healthGood || h == healthPerson) {
|
||||
h, said = healthNotYet, why
|
||||
}
|
||||
if h == healthGood || h == healthPerson {
|
||||
if on, named := w.on[j.module]; named {
|
||||
h, said = healthNotYet, on
|
||||
@@ -1372,6 +1380,9 @@ func gateLine(g *inventory.PlanGate) string {
|
||||
case inventory.GateFailed:
|
||||
what = "FAILED"
|
||||
}
|
||||
if g.Verdict == "" && g.Waiting() {
|
||||
what = "waits for a busy container on " + g.Busy.Machine
|
||||
}
|
||||
line := fmt.Sprintf("gate on %s: %s", strings.Join(g.Machines, ", "), what)
|
||||
if g.Component != "" {
|
||||
line += " (core: " + g.Component + ")"
|
||||
|
||||
@@ -975,9 +975,14 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
readsBusy, err := engineReadsBusy(ctx, inv, node)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
return catalogue.Rendering{
|
||||
ReadsHealth: readsHealth,
|
||||
JudgesRoot: judgesRoot,
|
||||
ReadsBusy: readsBusy,
|
||||
BusMembership: memberships[node],
|
||||
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Foreseen: foreseen, Ports: ports,
|
||||
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
||||
@@ -1010,6 +1015,16 @@ func engineJudgesRoot(ctx context.Context, inv *inventory.Inventory, node string
|
||||
return had && stated.Contract >= link.RootContract, nil
|
||||
}
|
||||
|
||||
// engineReadsBusy says whether a machine's node-engine reads a container's `busy` (novox/hq ADR 0335), by its
|
||||
// own newest statement, for the same reason as engineReadsHealth: an older engine parses strictly.
|
||||
func engineReadsBusy(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) {
|
||||
stated, had, err := inv.HealthOf(ctx, node)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return had && stated.Contract >= link.BusyContract, nil
|
||||
}
|
||||
|
||||
// zonesInTheMesh is every zone a module in the mesh declares, where the mesh placed it (novox/hq ADR
|
||||
// 0199): the zone settled from that node's settings, the node's private address, the port the
|
||||
// answering listen is published on there.
|
||||
|
||||
@@ -36,4 +36,4 @@ require (
|
||||
// committed. Every build (the build agent's `go build`, the Dockerfile) compiles from vendor/ and
|
||||
// fetches nothing; go refuses to build when vendor/ and this file disagree, so a pin moved without
|
||||
// `go mod vendor` fails loudly, at once, everywhere.
|
||||
replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f
|
||||
replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261012224132-bb60e7b584d8
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f h1:8N5OW2mdTNIck2pe4EciTYX5NsrPsHrTLENGNIWYNTU=
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f/go.mod h1:tcTK4LMs1d6JpZUwy3hSHMFG/MIuDr/XFs7/nbeaW/c=
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261012224132-bb60e7b584d8 h1:a+SH9NroLc3FkNYcbg0wjMEWGjWp6SCl32oVH2jxh9s=
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261012224132-bb60e7b584d8/go.mod h1:tcTK4LMs1d6JpZUwy3hSHMFG/MIuDr/XFs7/nbeaW/c=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A long-running container may say it is busy (novox/hq ADR 0335 §1).
|
||||
//
|
||||
// **Beside its health, on the container**: a command run inside it, by its shell, as an exec health check
|
||||
// is, that exits 0 while the container holds work a replacement would lose and non-zero when it is idle;
|
||||
// and how long one ask may take. Being asked is also its notice that it is about to be replaced: from the
|
||||
// first ask it takes no new work. The node-engine asks before it replaces the container, leaves it running
|
||||
// while it says busy, and replaces it on the first pass it says idle, or at 15 minutes — the engine's bound,
|
||||
// never the module's.
|
||||
//
|
||||
// Sent only to a node-engine that reads it (link.BusyContract): an older, strict engine refuses a field it
|
||||
// does not know, whole, and to it the container is replaced at once, as before.
|
||||
|
||||
// BusyField is the resource field.
|
||||
const BusyField = "busy"
|
||||
|
||||
// The bounds of one ask (ADR 0335), the node-engine's own, held here near the author.
|
||||
const (
|
||||
BusyTimeoutDefault = 10 * time.Second
|
||||
BusyTimeoutCeiling = time.Minute
|
||||
)
|
||||
|
||||
// Busy is one container's busy, read.
|
||||
type Busy struct {
|
||||
Command string
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// ReadBusy reads a resource's `busy` field, its default applied. False when it carries none.
|
||||
func ReadBusy(r map[string]any) (Busy, bool, []string) {
|
||||
raw, present := r[BusyField]
|
||||
if !present {
|
||||
return Busy{}, false, nil
|
||||
}
|
||||
id := fmt.Sprint(r["id"])
|
||||
fields, ok := raw.(map[string]any)
|
||||
if !ok {
|
||||
return Busy{}, true, []string{fmt.Sprintf("%s: busy is %T; it is an object with a command", id, raw)}
|
||||
}
|
||||
var problems []string
|
||||
keys := make([]string, 0, len(fields))
|
||||
for k := range fields {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
if k != "command" && k != "timeout" {
|
||||
problems = append(problems, fmt.Sprintf("%s: its busy says %q; busy has a command and a timeout", id, k))
|
||||
}
|
||||
}
|
||||
b := Busy{Timeout: BusyTimeoutDefault}
|
||||
if v, has := fields["command"]; has {
|
||||
s, ok := v.(string)
|
||||
if !ok {
|
||||
problems = append(problems, fmt.Sprintf("%s: its busy's command is %T; it is text", id, v))
|
||||
}
|
||||
b.Command = strings.TrimSpace(s)
|
||||
}
|
||||
if v, has := fields["timeout"]; has {
|
||||
s, _ := v.(string)
|
||||
d, err := time.ParseDuration(strings.TrimSpace(s))
|
||||
switch {
|
||||
case err != nil:
|
||||
problems = append(problems, fmt.Sprintf("%s: its busy's timeout is %v; it is a duration such as \"10s\"", id, v))
|
||||
case d <= 0 || d > BusyTimeoutCeiling:
|
||||
problems = append(problems, fmt.Sprintf("%s: an ask of its busy takes more than nothing and at most %s, not %s",
|
||||
id, BusyTimeoutCeiling, d))
|
||||
default:
|
||||
b.Timeout = d
|
||||
}
|
||||
}
|
||||
return b, true, problems
|
||||
}
|
||||
|
||||
// busyProblems is everything wrong with a manifest's `busy` fields, in the manifest's words: a command on
|
||||
// one line, a timeout within its bounds, and only on a container that stays up.
|
||||
func busyProblems(m Manifest) []string {
|
||||
var problems []string
|
||||
for _, r := range m.Resources {
|
||||
b, has, read := ReadBusy(r)
|
||||
if !has {
|
||||
continue
|
||||
}
|
||||
where := m.Module + ": " + fmt.Sprint(r["id"])
|
||||
for _, p := range read {
|
||||
problems = append(problems, m.Module+": "+p)
|
||||
}
|
||||
if fmt.Sprint(r["type"]) != "container" || !LongRunning(r) {
|
||||
problems = append(problems, fmt.Sprintf("%s declares busy and is not a container that stays up: busy is "+
|
||||
"asked of a running container before it is replaced (ADR 0335)", where))
|
||||
}
|
||||
if b.Command == "" {
|
||||
problems = append(problems, where+"'s busy names no command: one run inside the container, exiting 0 "+
|
||||
"while it is busy")
|
||||
}
|
||||
if strings.ContainsAny(b.Command, "\n\r") {
|
||||
problems = append(problems, where+"'s busy command is one line")
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// busyInto composes a container's `busy` into the node-engine's words, or takes it away: for an engine that
|
||||
// reads it, the command and its timeout written out; for one that does not, the field is not sent, and the
|
||||
// container is replaced at once, as before.
|
||||
func busyInto(resource map[string]any, with Rendering) {
|
||||
if _, has := resource[BusyField]; !has {
|
||||
return
|
||||
}
|
||||
if !with.ReadsBusy {
|
||||
delete(resource, BusyField)
|
||||
return
|
||||
}
|
||||
b, _, _ := ReadBusy(resource)
|
||||
resource[BusyField] = map[string]any{"command": b.Command, "timeout": b.Timeout.String()}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/validate"
|
||||
)
|
||||
|
||||
// A long-running container may say it is busy (novox/hq ADR 0335 §1): `module check` holds the field to its
|
||||
// shape near its author, and it is sent only to a node-engine that reads it, in words the engine's own
|
||||
// validator accepts.
|
||||
|
||||
func busyManifest(t *testing.T, on string, busy any) []byte {
|
||||
t.Helper()
|
||||
extra := map[string]any{"id": "agent", "type": "container", "name": "app-agent",
|
||||
"image": "registry.example/app" + healthDigest}
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(healthManifest(t, "", nil, extra), &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range m["resources"].([]any) {
|
||||
if r := r.(map[string]any); r["id"] == on {
|
||||
r[BusyField] = busy
|
||||
}
|
||||
}
|
||||
out, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestABusyIsHeldToItsShape(t *testing.T) {
|
||||
if _, err := ParseManifest(busyManifest(t, "agent", map[string]any{"command": "mesh-builder busy"})); err != nil {
|
||||
t.Fatalf("a well-formed busy was refused: %v", err)
|
||||
}
|
||||
if _, err := ParseManifest(busyManifest(t, "agent", map[string]any{"command": "x", "timeout": "30s"})); err != nil {
|
||||
t.Fatalf("a busy with its timeout was refused: %v", err)
|
||||
}
|
||||
for name, c := range map[string]struct {
|
||||
on string
|
||||
busy any
|
||||
says string
|
||||
}{
|
||||
"text": {"agent", "mesh-builder busy", "it is an object"},
|
||||
"no command": {"agent", map[string]any{"timeout": "5s"}, "names no command"},
|
||||
"two lines": {"agent", map[string]any{"command": "a\nb"}, "one line"},
|
||||
"a long timeout": {"agent", map[string]any{"command": "x", "timeout": "5m"}, "at most 1m0s"},
|
||||
"an unknown key": {"agent", map[string]any{"command": "x", "every": "1m"}, `says "every"`},
|
||||
"on a step": {"seed", map[string]any{"command": "x"}, "not a container that stays up"},
|
||||
"on a service": {"daemon", map[string]any{"command": "x"}, "not a container that stays up"},
|
||||
} {
|
||||
_, err := ParseManifest(busyManifest(t, c.on, c.busy))
|
||||
if err == nil || !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("%s: want a refusal saying %q, got %v", name, c.says, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestABusyIsSentOnlyToAnEngineThatReadsIt(t *testing.T) {
|
||||
m, err := ParseManifest(busyManifest(t, "agent", map[string]any{"command": "mesh-builder busy"}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
compose := func(with Rendering) []map[string]any {
|
||||
out, err := Resolution{Node: "laptop", Modules: []Manifest{m}}.Declaration(with)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
older := compose(Rendering{})
|
||||
if b, sent := older[indexOfID(older, "app.agent")][BusyField]; sent {
|
||||
t.Fatalf("busy was sent to an engine that does not read it: %v", b)
|
||||
}
|
||||
newer := compose(Rendering{ReadsBusy: true})
|
||||
got, _ := newer[indexOfID(newer, "app.agent")][BusyField].(map[string]any)
|
||||
if got["command"] != "mesh-builder busy" || got["timeout"] != "10s" {
|
||||
t.Fatalf("busy composed as %v", got)
|
||||
}
|
||||
// And the engine's own validator takes the container with it.
|
||||
body, err := json.Marshal(map[string]any{"declaration": validate.Version, "resources": newer})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if problems := validate.Declaration(body); len(problems) > 0 {
|
||||
t.Fatalf("the node-engine's validator refuses the composed busy: %v", problems)
|
||||
}
|
||||
if _, has := m.Resources[len(m.Resources)-1][BusyField].(map[string]any)["timeout"]; has {
|
||||
t.Error("composing wrote the timeout into the catalogue's own manifest")
|
||||
}
|
||||
}
|
||||
@@ -246,6 +246,11 @@ type Rendering struct {
|
||||
// statement's contract is link.RootContract or later). To an older, strict engine the field is not
|
||||
// sent, and the account it names is not judged — which the self-check says, as not judged.
|
||||
JudgesRoot bool
|
||||
|
||||
// ReadsBusy says this machine's node-engine reads a container's `busy` (novox/hq ADR 0335: its statement's
|
||||
// contract is link.BusyContract or later). To an older, strict engine the field is not sent, and the
|
||||
// container is replaced at once, as before.
|
||||
ReadsBusy bool
|
||||
}
|
||||
|
||||
// RootField is a user resource's field saying the account must never become root without a person
|
||||
@@ -1051,6 +1056,8 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
// How it is ready, in the node-engine's words: its endpoint as the port this machine
|
||||
// published it on — or not sent at all to an engine older than the field (ADR 0240).
|
||||
healthInto(copied, m, with)
|
||||
// And whether it is busy (novox/hq ADR 0335): sent only to an engine that asks it.
|
||||
busyInto(copied, with)
|
||||
// And a user's `root` (novox/hq ADR 0266): sent only when it asserts something, to an engine
|
||||
// that judges it.
|
||||
rootInto(copied, with)
|
||||
|
||||
@@ -2242,6 +2242,8 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
||||
// How each long-running resource is ready (novox/hq ADR 0240 rule 2): said near its author, in the
|
||||
// words the node-engine would refuse it in far away.
|
||||
problems = append(problems, healthProblems(m)...)
|
||||
// And whether a container is busy (novox/hq ADR 0335), likewise.
|
||||
problems = append(problems, busyProblems(m)...)
|
||||
if len(problems) > 0 {
|
||||
sort.Strings(problems)
|
||||
return Manifest{}, fmt.Errorf("this manifest cannot be used:\n - %s",
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A walk whose first machine waited for a busy container counts the wait as a phase of its own, between the
|
||||
// send to the first machine and the judging, and its delivery time holds it (novox/hq ADR 0335 §4): the
|
||||
// recorded walk again, its judging started only at 17:54:07, a minute after the send, when the machine said
|
||||
// the container replaced.
|
||||
func TestABusyWaitIsAPhaseOfTheDeliveryTime(t *testing.T) {
|
||||
raw := strings.Replace(recordedWalk, `"gate": {"machines": ["novox"], "since": "2026-10-10T17:53:07.287136827Z",`,
|
||||
`"gate": {"machines": ["novox"], "since": "2026-10-10T17:54:07.287136827Z",
|
||||
"busy": {"machine": "novox", "module": "mesh-delivery", "container": "mesh-delivery",
|
||||
"since": "2026-10-10T17:53:07Z", "until": "2026-10-10T18:08:07Z", "ended": "2026-10-10T17:54:07.287136827Z"},`, 1)
|
||||
w := walkOf(t, raw).WalkPhases(time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC), nil)
|
||||
sumsUp(t, w)
|
||||
if w.TotalMS != 282894 {
|
||||
t.Fatalf("the wait changed the delivery's total: %dms", w.TotalMS)
|
||||
}
|
||||
busy := phase(w, PhaseBusy, 0)
|
||||
if busy.State != PhaseMeasured || busy.TookMS != 60000 {
|
||||
t.Fatalf("the busy wait is not a measured phase of a minute: %+v", busy)
|
||||
}
|
||||
if judged := phase(w, PhaseJudge, 0); judged.TookMS != 148607-60000 {
|
||||
t.Fatalf("the judging still counts from the send, not from the replacement: %+v", judged)
|
||||
}
|
||||
var names []string
|
||||
for _, ph := range w.Phases {
|
||||
names = append(names, ph.Name)
|
||||
}
|
||||
if !strings.Contains(strings.Join(names, " "), PhaseSend+" "+PhaseBusy+" "+PhaseJudge) {
|
||||
t.Fatalf("the busy wait is not between the send and the judging: %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
// While the wait goes on, the walk says it waits for a busy container there; a walk where nothing waited has no
|
||||
// such phase.
|
||||
func TestAnOpenBusyWaitIsSaid(t *testing.T) {
|
||||
raw := strings.Replace(recordedWalk, `"gate": {"machines": ["novox"], "since": "2026-10-10T17:53:07.287136827Z", "passes": 3,
|
||||
"verdict": "passed", "judged_at": "2026-10-10T17:55:35.894985053Z"}`,
|
||||
`"gate": {"machines": ["novox"], "since": "2026-10-10T17:53:07.287136827Z",
|
||||
"busy": {"machine": "novox", "module": "mesh-delivery", "container": "mesh-delivery",
|
||||
"since": "2026-10-10T17:53:07Z", "until": "2026-10-10T18:08:07Z"}}`, 1)
|
||||
raw = strings.Replace(raw, `"state": "done"`, `"state": "rolling"`, 1)
|
||||
p := walkOf(t, raw)
|
||||
if !p.Modules["mesh-delivery"].Gate.Waiting() {
|
||||
t.Fatal("a gate whose busy wait has not ended does not say it waits")
|
||||
}
|
||||
w := p.WalkPhases(time.Date(2026, 10, 10, 17, 58, 0, 0, time.UTC), nil)
|
||||
busy := phase(w, PhaseBusy, 0)
|
||||
if busy.State != PhaseUnknown || !strings.Contains(busy.Said, "waits for a busy container on novox") {
|
||||
t.Fatalf("an open busy wait is not said: %+v", busy)
|
||||
}
|
||||
if plain := walkOf(t, recordedWalk).WalkPhases(time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC), nil); phase(plain, PhaseBusy, 0).Name != "" {
|
||||
t.Fatalf("a walk where nothing waited has a busy phase: %+v", plain.Phases)
|
||||
}
|
||||
}
|
||||
|
||||
// A container a report lists waiting is its module's by its id; its output alone is not a new account.
|
||||
func TestABusyContainerIsItsModules(t *testing.T) {
|
||||
list := []BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: BusyWaiting, Said: "a"},
|
||||
{ID: "web.server", State: "replaced-when-idle"}}
|
||||
if b, ok := WaitingFor(list, "build-agent"); !ok || b.Container != "mesh-build-agent" {
|
||||
t.Fatalf("build-agent's container is not found waiting: %+v", b)
|
||||
}
|
||||
if _, ok := WaitingFor(list, "web"); ok {
|
||||
t.Error("a container replaced reads as waiting")
|
||||
}
|
||||
if _, ok := WaitingFor(list, "build"); ok {
|
||||
t.Error("another module's container is found by a prefix of its name")
|
||||
}
|
||||
again := []BusyWait{{ID: "build-agent.server", State: BusyWaiting, Said: "b, a minute on"}, list[1]}
|
||||
if !sameBusy(list, again) {
|
||||
t.Error("a wait whose output changed is a new account")
|
||||
}
|
||||
if sameBusy(list, list[1:]) || !(Doing{Busy: list}).Waiting() || (Doing{Busy: list[1:]}).Waiting() {
|
||||
t.Error("a wait that ended is not a new account")
|
||||
}
|
||||
}
|
||||
|
||||
// A machine's report of a busy container is kept with its last account and read back beside it, and its
|
||||
// starting and its end are news; its output alone is not (novox/hq ADR 0335). Needs the store.
|
||||
func TestAReportOfABusyContainerIsKeptAndReadBack(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t)
|
||||
ctx := t.Context()
|
||||
record, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
since := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
waiting := []BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: BusyWaiting,
|
||||
Since: since, Until: since.Add(15 * time.Minute), Said: "busy: a check"}}
|
||||
keep := func(busy []BusyWait) bool {
|
||||
t.Helper()
|
||||
news, err := inv.RecordOrderedDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: "d1", Applied: 1,
|
||||
Busy: busy}, ReportOrder{Epoch: 1, Sequence: 1}, func(ReportOrder) bool { return false })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return news
|
||||
}
|
||||
read := func() []BusyWait {
|
||||
t.Helper()
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range reports {
|
||||
if r.Node == node {
|
||||
return r.Busy
|
||||
}
|
||||
}
|
||||
t.Fatal("the machine's report is not read back")
|
||||
return nil
|
||||
}
|
||||
keep(nil)
|
||||
if got := read(); len(got) != 0 {
|
||||
t.Fatalf("a report with nothing busy reads back %+v", got)
|
||||
}
|
||||
if !keep(waiting) {
|
||||
t.Error("a container starting to wait is not news")
|
||||
}
|
||||
got := read()
|
||||
if b, ok := WaitingFor(got, "build-agent"); !ok || !b.Since.Equal(since) || b.Said != "busy: a check" {
|
||||
t.Fatalf("the wait is not read back: %+v", got)
|
||||
}
|
||||
again := []BusyWait{waiting[0]}
|
||||
again[0].Said = "busy: a check, a minute on"
|
||||
if keep(again) {
|
||||
t.Error("a wait whose output changed is news")
|
||||
}
|
||||
if !keep([]BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: "replaced-when-idle",
|
||||
Since: since, Until: since.Add(15 * time.Minute)}}) {
|
||||
t.Error("a container replaced after it waited is not news")
|
||||
}
|
||||
if _, ok := WaitingFor(read(), "build-agent"); ok {
|
||||
t.Error("a container replaced still reads as waiting")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
-- A machine's report says which container it left running because it said busy, and which it replaced
|
||||
-- after it waited (novox/hq ADR 0335, issue 594).
|
||||
--
|
||||
-- A send that replaced the build agent killed the check it ran, and the queue started it again elsewhere.
|
||||
-- A container may now declare `busy`; the node-engine asks it before replacing it, leaves it running while it
|
||||
-- says busy, within 15 minutes, and says so in its report's `busy`: [{"id", "container", "state", "since",
|
||||
-- "until", "said"}], state `waiting`, `replaced-when-idle` or `replaced-at-bound`. Kept with the machine's
|
||||
-- last account, so a walk's first-machine gate waits while its module's container is listed waiting — across
|
||||
-- a restart of the controller, which a merge into mesh-controller makes in the same walk that moves the build
|
||||
-- agent. Empty from an engine that never waits.
|
||||
alter table node_report add column busy jsonb not null default '[]';
|
||||
+85
-10
@@ -10,6 +10,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -846,6 +847,46 @@ type Doing struct {
|
||||
// machine doing what it was told.
|
||||
Since *time.Time
|
||||
Times int
|
||||
|
||||
// Busy is every container the apply left running because it said busy, and every one it replaced after
|
||||
// it waited (novox/hq ADR 0335).
|
||||
Busy []BusyWait
|
||||
}
|
||||
|
||||
// BusyWait is one container a busy command held, as the machine reported it (novox/hq ADR 0335): its
|
||||
// resource id (`<module>.<resource>`), its name, its state (link.BusyWaiting, BusyReplacedIdle or
|
||||
// BusyReplacedAtBound), since when it waits, until when at the latest, and the last output of its command.
|
||||
type BusyWait struct {
|
||||
ID string `json:"id"`
|
||||
Container string `json:"container"`
|
||||
State string `json:"state"`
|
||||
Since time.Time `json:"since"`
|
||||
Until time.Time `json:"until"`
|
||||
Said string `json:"said,omitempty"`
|
||||
}
|
||||
|
||||
// BusyWaiting is the state of a container left running because it said busy (link.BusyWaiting).
|
||||
const BusyWaiting = "waiting"
|
||||
|
||||
// WaitingFor is the container of a module that waits because it said busy, and whether one does: a
|
||||
// resource whose id is `<module>.<resource>`.
|
||||
func WaitingFor(busy []BusyWait, module string) (BusyWait, bool) {
|
||||
for _, b := range busy {
|
||||
if b.State == BusyWaiting && strings.HasPrefix(b.ID, module+".") {
|
||||
return b, true
|
||||
}
|
||||
}
|
||||
return BusyWait{}, false
|
||||
}
|
||||
|
||||
// Waiting says whether any container waits because it said busy.
|
||||
func (d Doing) Waiting() bool {
|
||||
for _, b := range d.Busy {
|
||||
if b.State == BusyWaiting {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// FailedResource is one thing a node could not do.
|
||||
@@ -873,6 +914,20 @@ func (d Doing) Stuck() bool { return d.Wrong() && d.Times >= StuckAfter && d.Sin
|
||||
// the same failed resources BY ID. Not by the host's words: an error that carries a duration, a
|
||||
// counter or a temporary path would read as new on every report, and the resource looping on it —
|
||||
// which is what stuck is for — would never be said to be (novox/hq 04-ISSUES/065).
|
||||
// sameBusy says two reports hold the same containers in the same state of a busy wait: its output, which
|
||||
// may change on every ask, is not compared.
|
||||
func sameBusy(a, b []BusyWait) bool {
|
||||
key := func(list []BusyWait) []string {
|
||||
out := make([]string, 0, len(list))
|
||||
for _, w := range list {
|
||||
out = append(out, w.ID+"="+w.State)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
return slices.Equal(key(a), key(b))
|
||||
}
|
||||
|
||||
func sameFailure(a, b Doing) bool {
|
||||
if a.Outcome != b.Outcome || a.Refused != b.Refused || len(a.Failed) != len(b.Failed) {
|
||||
return false
|
||||
@@ -937,13 +992,20 @@ func recordDoing(ctx context.Context, q queries, node string, d Doing) (news boo
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if d.Busy == nil {
|
||||
d.Busy = []BusyWait{}
|
||||
}
|
||||
busy, err := json.Marshal(d.Busy)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
var before Doing
|
||||
var beforeFailed []byte
|
||||
var beforeFailed, beforeBusy []byte
|
||||
found := q.QueryRow(ctx,
|
||||
`select outcome, refused, failed, failing_since, failures, coalesce(declared,'')
|
||||
`select outcome, refused, failed, failing_since, failures, coalesce(declared,''), busy
|
||||
from node_report where node = $1`,
|
||||
node).Scan(&before.Outcome, &before.Refused, &beforeFailed, &before.Since, &before.Times,
|
||||
&before.Declared)
|
||||
&before.Declared, &beforeBusy)
|
||||
switch {
|
||||
case errors.Is(found, pgx.ErrNoRows):
|
||||
news = true
|
||||
@@ -953,7 +1015,13 @@ func recordDoing(ctx context.Context, q queries, node string, d Doing) (news boo
|
||||
if err := json.Unmarshal(beforeFailed, &before.Failed); err != nil {
|
||||
return false, err
|
||||
}
|
||||
news = before.Outcome != d.Outcome || before.Declared != d.Declared || !sameFailure(before, d)
|
||||
if err := json.Unmarshal(beforeBusy, &before.Busy); err != nil {
|
||||
return false, err
|
||||
}
|
||||
// A container that starts or ends a wait because it said busy is news (novox/hq ADR 0335): the walk
|
||||
// waiting on this machine is told by it.
|
||||
news = before.Outcome != d.Outcome || before.Declared != d.Declared || !sameFailure(before, d) ||
|
||||
!sameBusy(before.Busy, d.Busy)
|
||||
}
|
||||
var since *time.Time
|
||||
times := 0
|
||||
@@ -966,13 +1034,13 @@ func recordDoing(ctx context.Context, q queries, node string, d Doing) (news boo
|
||||
}
|
||||
_, err = q.Exec(ctx,
|
||||
`insert into node_report (node, outcome, refused, failed, applied, at, declared,
|
||||
failing_since, failures)
|
||||
values ($1, $2, $3, $4, $5, now(), $6, $7, $8)
|
||||
failing_since, failures, busy)
|
||||
values ($1, $2, $3, $4, $5, now(), $6, $7, $8, $9)
|
||||
on conflict (node) do update set outcome = excluded.outcome, refused = excluded.refused,
|
||||
failed = excluded.failed, applied = excluded.applied, at = excluded.at,
|
||||
declared = excluded.declared,
|
||||
failing_since = excluded.failing_since, failures = excluded.failures`,
|
||||
node, d.Outcome, d.Refused, failed, d.Applied, d.Declared, since, times)
|
||||
failing_since = excluded.failing_since, failures = excluded.failures, busy = excluded.busy`,
|
||||
node, d.Outcome, d.Refused, failed, d.Applied, d.Declared, since, times, busy)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
@@ -1059,6 +1127,8 @@ type Reported struct {
|
||||
// gate holds against the send it made, rather than against the send made last (novox/hq issue 352).
|
||||
Declared string
|
||||
ReportedSequence int64
|
||||
// Busy is what the last report said of containers held by their busy command (novox/hq ADR 0335).
|
||||
Busy []BusyWait
|
||||
}
|
||||
|
||||
// SentDeclaration is what one send carried to a machine, as a gate keeps it: the declaration's digest and
|
||||
@@ -1113,7 +1183,7 @@ func (i *Inventory) LastReports(ctx context.Context) ([]Reported, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select n.name, coalesce(r.outcome, ''), r.at, n.sent_at,
|
||||
r.declared is not null and r.declared <> '' and r.declared = n.sent,
|
||||
coalesce(r.declared, ''), coalesce(r.reported_sequence, 0)
|
||||
coalesce(r.declared, ''), coalesce(r.reported_sequence, 0), coalesce(r.busy, '[]')
|
||||
from node n left join node_report r on r.node = n.id
|
||||
order by n.name`)
|
||||
if err != nil {
|
||||
@@ -1123,7 +1193,12 @@ func (i *Inventory) LastReports(ctx context.Context) ([]Reported, error) {
|
||||
var out []Reported
|
||||
for rows.Next() {
|
||||
var r Reported
|
||||
if err := rows.Scan(&r.Node, &r.Outcome, &r.At, &r.Sent, &r.Current, &r.Declared, &r.ReportedSequence); err != nil {
|
||||
var busy []byte
|
||||
if err := rows.Scan(&r.Node, &r.Outcome, &r.At, &r.Sent, &r.Current, &r.Declared, &r.ReportedSequence,
|
||||
&busy); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := json.Unmarshal(busy, &r.Busy); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, r)
|
||||
|
||||
@@ -367,6 +367,39 @@ type PlanGate struct {
|
||||
// alone to the machine holding the bus, every build there kept, so what the send newly grants is on the
|
||||
// bus before the gate judges it. Nil when the list did not change, or the gate's machine holds the bus.
|
||||
Grants *GrantsStep `json:"grants,omitempty"`
|
||||
// Busy is the wait for a busy container on a judged machine (novox/hq ADR 0335 §4): while the machine
|
||||
// reports its module's container left running because it said busy, the judging waits and has not
|
||||
// started; once it reports the container replaced, Since is moved to then. Nil when nothing waited.
|
||||
Busy *GateBusy `json:"busy,omitempty"`
|
||||
}
|
||||
|
||||
// GateBusy is a first-node gate's wait for a busy container (novox/hq ADR 0335 §4).
|
||||
type GateBusy struct {
|
||||
Machine string `json:"machine"`
|
||||
Module string `json:"module"`
|
||||
Container string `json:"container"`
|
||||
// Since is when the machine's first apply that would have replaced it ran, Until when its node-engine
|
||||
// replaces it at the latest; Ended when the machine reported it replaced and the judging began — or when
|
||||
// the controller stopped waiting past the engine's bound, which PastBound says.
|
||||
Since time.Time `json:"since"`
|
||||
Until time.Time `json:"until"`
|
||||
Ended *time.Time `json:"ended,omitempty"`
|
||||
PastBound bool `json:"past_bound,omitempty"`
|
||||
// Said is the last output of its busy command, as the machine reported it.
|
||||
Said string `json:"said,omitempty"`
|
||||
}
|
||||
|
||||
// Waiting says the gate waits for a busy container and has not started.
|
||||
func (g *PlanGate) Waiting() bool { return g != nil && g.Busy != nil && g.Busy.Ended == nil }
|
||||
|
||||
// Words is the wait in a line, as the walk says it.
|
||||
func (b *GateBusy) Words() string {
|
||||
said := ""
|
||||
if b.Said != "" {
|
||||
said = "; its busy command says: " + b.Said
|
||||
}
|
||||
return fmt.Sprintf("waits for a busy container on %s: %s holds work, and is replaced once it says idle, by %s "+
|
||||
"at the latest (novox/hq ADR 0335)%s", b.Machine, b.Container, b.Until.UTC().Format(time.RFC3339), said)
|
||||
}
|
||||
|
||||
// GrantsStep is the bus's user list sent to the machine holding the bus ahead of a gated send (novox/hq
|
||||
|
||||
@@ -26,6 +26,7 @@ const (
|
||||
PhaseBetween = "between-tiers" // one tier's end to the next tier's ask
|
||||
PhaseBuild = "build" // a tier asked to its last module built
|
||||
PhaseSend = "send-first" // built to sent to the first machines
|
||||
PhaseBusy = "busy-wait" // sent first to a busy container there replaced, when one waited (ADR 0335)
|
||||
PhaseJudge = "judgement" // sent first to the first-node gate's last verdict: its readings
|
||||
PhaseRest = "send-rest" // judged to sent to the rest
|
||||
PhaseApply = "apply" // the last send to every machine of the rest reporting the build applied
|
||||
@@ -300,8 +301,11 @@ func (p Plan) points(now time.Time, applied AppliedLookup, w *WalkPhases) []poin
|
||||
if t > p.Tier || (t == p.Tier && p.Tier < len(p.Tiers) && !askedAnyOf(p, tier)) {
|
||||
break
|
||||
}
|
||||
var asked, built, first, judged, rest *time.Time
|
||||
var asked, built, first, judged, rest, busyEnded *time.Time
|
||||
allBuilt, anyFirst, allJudged, allRest := true, false, true, true
|
||||
// A busy container on a first machine holds the judging until it is replaced (novox/hq ADR 0335 §4): a
|
||||
// phase of its own, only in a tier where one waited, counted in the delivery time and never exempted.
|
||||
anyBusy, allBusyEnded, busyOn := false, true, ""
|
||||
for _, m := range tier {
|
||||
s := p.Modules[m]
|
||||
if s == nil {
|
||||
@@ -319,6 +323,14 @@ func (p Plan) points(now time.Time, applied AppliedLookup, w *WalkPhases) []poin
|
||||
if s.FirstAt != nil {
|
||||
anyFirst = true
|
||||
first = earliest(first, s.FirstAt)
|
||||
if s.Gate != nil && s.Gate.Busy != nil {
|
||||
anyBusy = true
|
||||
if s.Gate.Busy.Ended == nil {
|
||||
allBusyEnded, busyOn = false, s.Gate.Busy.Machine
|
||||
} else {
|
||||
busyEnded = latest(busyEnded, s.Gate.Busy.Ended)
|
||||
}
|
||||
}
|
||||
if s.Gate != nil {
|
||||
if s.Gate.JudgedAt == nil {
|
||||
allJudged = false
|
||||
@@ -349,7 +361,15 @@ func (p Plan) points(now time.Time, applied AppliedLookup, w *WalkPhases) []poin
|
||||
if !allJudged {
|
||||
judged = nil
|
||||
}
|
||||
out = append(out, point{phase: PhaseSend, tier: t, at: first}, point{phase: PhaseJudge, tier: t, at: judged})
|
||||
out = append(out, point{phase: PhaseSend, tier: t, at: first})
|
||||
if anyBusy {
|
||||
said := ""
|
||||
if !allBusyEnded {
|
||||
busyEnded, said = nil, "waits for a busy container on "+busyOn
|
||||
}
|
||||
out = append(out, point{phase: PhaseBusy, tier: t, at: busyEnded, said: said})
|
||||
}
|
||||
out = append(out, point{phase: PhaseJudge, tier: t, at: judged})
|
||||
} else {
|
||||
out = append(out, point{phase: PhaseSend, tier: t, none: true, said: "no first machine: nothing to judge"},
|
||||
point{phase: PhaseJudge, tier: t, none: true, said: "no first machine: nothing to judge"})
|
||||
|
||||
@@ -175,6 +175,11 @@ type MachineOptions struct {
|
||||
// can take a build, which is what its container's health check reads (novox/hq issue 544). Nil is
|
||||
// told nothing.
|
||||
Taking func()
|
||||
// Claim is asked once a request is fetched, just before it is handed over: false hands it back to the
|
||||
// queue at once, for another holder. A holder being drained says so here (novox/hq ADR 0335): it marks
|
||||
// the request in hand first and reads its drain after, so an ask of `busy` racing the fetch sees one or
|
||||
// the other, never neither. Nil claims every one.
|
||||
Claim func() bool
|
||||
}
|
||||
|
||||
// MachineOverNATS takes build work from the current build role.
|
||||
@@ -317,6 +322,12 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
|
||||
endCancelled(ctx, build)
|
||||
continue
|
||||
}
|
||||
// **Drained in the moment it was fetched** (novox/hq ADR 0335): handed back at once, as a pause
|
||||
// landing then is.
|
||||
if m.opts.Claim != nil && !m.opts.Claim() {
|
||||
_ = msg.Nak()
|
||||
continue
|
||||
}
|
||||
// A build outlives the acknowledgement window many times over; said while it runs,
|
||||
// as the controller says it for its own long handlers, so the server neither hands
|
||||
// the ask to a second machine nor counts the wait against its deliveries.
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"fmt"
|
||||
"log"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
@@ -418,6 +419,12 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err
|
||||
for id, why := range report.Failed {
|
||||
doing.Failed = append(doing.Failed, inventory.FailedResource{ID: id, Error: why})
|
||||
}
|
||||
// And a container left running because it said busy, or replaced after it waited (novox/hq ADR 0335):
|
||||
// kept with the account, for the walk whose first-machine gate waits on it.
|
||||
for _, b := range report.Busy {
|
||||
doing.Busy = append(doing.Busy, inventory.BusyWait{ID: b.ID, Container: b.Container, State: b.State,
|
||||
Since: b.Since, Until: b.Until, Said: b.Said})
|
||||
}
|
||||
// Ordered, so two readings of one failure are the same reading.
|
||||
sort.Slice(doing.Failed, func(i, j int) bool { return doing.Failed[i].ID < doing.Failed[j].ID })
|
||||
|
||||
@@ -492,7 +499,14 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err
|
||||
}
|
||||
// And how long the machine took, from the send to this first account of it (novox/hq to-be 45
|
||||
// Phase 0): what the sent-not-reported bound will be set from. Said if lost, never a failure.
|
||||
if err := e.Inventory.RecordApplyDuration(ctx, report.Node, report.Declared, doing.Outcome); err != nil {
|
||||
//
|
||||
// **Not while a container waits because it said busy** (novox/hq ADR 0335 §4): the machine is not yet
|
||||
// running what it was sent, and the account that says the container replaced is the one that ends the
|
||||
// apply — so a walk's rest counts the wait in its delivery time, never exempted.
|
||||
if doing.Waiting() {
|
||||
log.Printf("%s applied declaration %s and leaves a busy container running until it says idle: %s",
|
||||
report.Node, short(report.Declared), busyWords(doing.Busy))
|
||||
} else if err := e.Inventory.RecordApplyDuration(ctx, report.Node, report.Declared, doing.Outcome); err != nil {
|
||||
log.Printf("how long %s took to apply could not be recorded: %v", report.Node, err)
|
||||
}
|
||||
|
||||
@@ -504,3 +518,15 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err
|
||||
}
|
||||
return news, e.Inventory.RecordOwned(ctx, node.ID, report.Applied)
|
||||
}
|
||||
|
||||
// busyWords is every container waiting because it said busy, in a line (novox/hq ADR 0335).
|
||||
func busyWords(busy []inventory.BusyWait) string {
|
||||
var out []string
|
||||
for _, b := range busy {
|
||||
if b.State == inventory.BusyWaiting {
|
||||
out = append(out, fmt.Sprintf("%s since %s, at the latest until %s", b.Container,
|
||||
b.Since.UTC().Format(time.RFC3339), b.Until.UTC().Format(time.RFC3339)))
|
||||
}
|
||||
}
|
||||
return strings.Join(out, "; ")
|
||||
}
|
||||
|
||||
@@ -272,6 +272,12 @@ type Report struct {
|
||||
// `witness` and `not-reversible` are sent only to a machine whose report carries it.
|
||||
Witness int `json:"witness,omitempty"`
|
||||
|
||||
// Busy is every container the reporting apply left running because it said busy, and every one it
|
||||
// replaced after it had waited (novox/hq ADR 0335; mesh-host internal/link/messages.go): a walk's
|
||||
// first-machine gate waits while its module's container is listed waiting, and starts at the report
|
||||
// that no longer lists it so. Absent from an engine older than BusyContract, which never waits.
|
||||
Busy []Busy `json:"busy,omitempty"`
|
||||
|
||||
// Health is the machine's word on every long-running resource it runs for a module (novox/hq ADR
|
||||
// 0240, to-be 48 §4; mesh-host's internal/liveness): its state, since when, its failing streak and
|
||||
// the restarts its node-engine counted. **Absent from an engine older than the judging**, which is
|
||||
@@ -436,6 +442,31 @@ const ReadinessContract = 2
|
||||
// parses strictly and would refuse the whole declaration for it.
|
||||
const RootContract = 3
|
||||
|
||||
// BusyContract is the statement of an engine that also reads a container's `busy` (novox/hq ADR 0335): it asks
|
||||
// a container that declares it before replacing it and says the wait in its report. Only to such an engine is
|
||||
// the field sent: an older one parses strictly and would refuse the whole declaration for it. The same number as
|
||||
// mesh-host internal/link/messages.go's BusyContract.
|
||||
const BusyContract = 8
|
||||
|
||||
// The states of a busy container in a report (novox/hq ADR 0335 §2, §3).
|
||||
const (
|
||||
BusyWaiting = "waiting"
|
||||
BusyReplacedIdle = "replaced-when-idle"
|
||||
BusyReplacedAtBound = "replaced-at-bound"
|
||||
)
|
||||
|
||||
// Busy is one container a busy command held, as a node-engine reports it (novox/hq ADR 0335). ID is the
|
||||
// resource's id, `<module>.<resource>`; Since when the first apply that would have replaced it ran, Until when
|
||||
// the engine's bound replaces it at the latest, Said the last output of its busy command.
|
||||
type Busy struct {
|
||||
ID string `json:"id"`
|
||||
Container string `json:"container"`
|
||||
State string `json:"state"`
|
||||
Since time.Time `json:"since"`
|
||||
Until time.Time `json:"until"`
|
||||
Said string `json:"said,omitempty"`
|
||||
}
|
||||
|
||||
// ReasonRoot starts the reason of an account verdict that found a way to root (ADR 0266); the node-engine's
|
||||
// own words (mesh-host internal/accounts ReasonRoot).
|
||||
const ReasonRoot = "can become root without a person"
|
||||
|
||||
Vendored
+1
@@ -37,3 +37,4 @@
|
||||
./cmd/mesh-controller TestReplay401
|
||||
./cmd/mesh-controller TestReplay201ThePlansSend
|
||||
./internal/inventory TestAWalksRestIsReadFromEachMachinesOwnSend
|
||||
./internal/inventory TestAReportOfABusyContainerIsKeptAndReadBack
|
||||
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
package declaration
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Busy is how a long-running container says it holds work that would be lost if it were stopped now
|
||||
// (novox/hq ADR 0335 §1): a command run inside it, by its shell, as an exec health check is, that exits 0
|
||||
// while it is busy and non-zero when it is idle. Being asked is also the container's notice that it is
|
||||
// about to be replaced: from the first ask it takes no new work, so its answer ends once the work in hand
|
||||
// ends.
|
||||
//
|
||||
// **Not part of what the container is.** Busy is not in the container's spec: a changed command changes
|
||||
// when the container is replaced, never whether — a container is not recreated for its busy command alone.
|
||||
type Busy struct {
|
||||
Command string `json:"command"`
|
||||
// Timeout is how long one ask may take; empty is BusyTimeoutDefault.
|
||||
Timeout string `json:"timeout,omitempty"`
|
||||
}
|
||||
|
||||
// The bounds of an ask (novox/hq ADR 0335).
|
||||
const (
|
||||
BusyTimeoutDefault = 10 * time.Second
|
||||
// BusyTimeoutCeiling is the longest one ask may take: an ask runs inside an apply, and an apply waits
|
||||
// for nothing longer than a look.
|
||||
BusyTimeoutCeiling = time.Minute
|
||||
// BusyBound is how long a container that says busy is left running after the first apply that would
|
||||
// have replaced it (ADR 0335 §3). The node-engine's, not the module's: no module holds its own
|
||||
// replacement for ever.
|
||||
BusyBound = 15 * time.Minute
|
||||
)
|
||||
|
||||
// UnmarshalJSON reads a busy strictly, as everything a declaration carries is read.
|
||||
func (b *Busy) UnmarshalJSON(raw []byte) error {
|
||||
type plain Busy
|
||||
var p plain
|
||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&p); err != nil {
|
||||
return fmt.Errorf("busy: %w", err)
|
||||
}
|
||||
*b = Busy(p)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Within is how long one ask may take. Validated on arrival, so an unreadable timeout is the default.
|
||||
func (b *Busy) Within() time.Duration {
|
||||
if d, err := time.ParseDuration(b.Timeout); err == nil && d > 0 {
|
||||
return d
|
||||
}
|
||||
return BusyTimeoutDefault
|
||||
}
|
||||
|
||||
// problems holds a container's busy to its shape: a one-line command, a timeout within its bounds, and only
|
||||
// on a container that stays up.
|
||||
func (b *Busy) problems(where string, longRunning bool) []string {
|
||||
if b == nil {
|
||||
return nil
|
||||
}
|
||||
var problems []string
|
||||
say := func(format string, args ...any) {
|
||||
problems = append(problems, where+": "+fmt.Sprintf(format, args...))
|
||||
}
|
||||
if !longRunning {
|
||||
say("busy is asked of a container that stays up; a step or a scheduled run is never replaced mid-run")
|
||||
}
|
||||
if strings.TrimSpace(b.Command) == "" {
|
||||
say("busy needs a command")
|
||||
}
|
||||
if strings.ContainsAny(b.Command, "\n\r") {
|
||||
say("busy's command is one line")
|
||||
}
|
||||
if b.Timeout != "" {
|
||||
d, err := time.ParseDuration(b.Timeout)
|
||||
switch {
|
||||
case err != nil:
|
||||
say("busy's timeout is a duration, not %q", b.Timeout)
|
||||
case d <= 0 || d > BusyTimeoutCeiling:
|
||||
say("an ask of busy takes more than nothing and at most %s, not %s", BusyTimeoutCeiling, d)
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
+354
-5
@@ -12,6 +12,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"slices"
|
||||
@@ -132,7 +133,13 @@ func (d *Directory) validate(where string, _ bool) []string {
|
||||
if d.Path == "" {
|
||||
problems = append(problems, where+": a directory needs a path")
|
||||
}
|
||||
return append(problems, checkMode(where, d.Mode)...)
|
||||
// A directory's mode may carry the setgid bit (2xxx), so what is made beneath it keeps its group (novox/hq ADR
|
||||
// 0334 §1); never setuid or sticky, which mean nothing the mesh asks of a directory it owns.
|
||||
mode := d.Mode
|
||||
if len(mode) == 4 && mode[0] == '2' {
|
||||
mode = "0" + mode[1:]
|
||||
}
|
||||
return append(problems, checkMode(where, mode)...)
|
||||
}
|
||||
|
||||
// File is a file with literal content. The host renders nothing.
|
||||
@@ -456,14 +463,51 @@ const (
|
||||
// **The distinction 04-ISSUES/036 and 026 turn on.** A `directory` resource is the mesh's own —
|
||||
// it creates it, sets its owner and mode, and removes it when empty ([ADR 0030](novox/hq)). A
|
||||
// media library, a download spool is the operator's: it existed before the mesh, several modules
|
||||
// read and write it at once, and the mesh must not create, chown, reconcile or remove it. The
|
||||
// host confirms it is there and mounts it; nothing else.
|
||||
// read and write it at once, and the mesh must not create, chown, chmod or remove it. The host
|
||||
// confirms it is there and mounts it; nothing else. Library data is a library's (novox/hq ADR 0328),
|
||||
// not an access.
|
||||
type Access struct {
|
||||
ID string `json:"id"`
|
||||
Type Type `json:"type"`
|
||||
Path string `json:"path"`
|
||||
// Mode is how this module reaches the path: read or read-write. Absent narrows to read.
|
||||
Mode string `json:"mode,omitempty"`
|
||||
|
||||
// Users and Taken are what a controller speaking ADR 0325 sends (the identities that use the folder, and
|
||||
// that the operator's warrant was carried out): accepted, so its declaration is not refused whole, and not
|
||||
// acted on since ADR 0328 superseded ADR 0325 — no ACL entry is kept.
|
||||
Users []AccessUser `json:"users,omitempty"`
|
||||
Taken bool `json:"taken,omitempty"`
|
||||
|
||||
// Grant is what a controller older than ADR 0325 still sends (ADR 0312's group grant, superseded): accepted
|
||||
// so its declaration is not refused whole, and never acted on — no account is put in a group for data (§5).
|
||||
Grant json.RawMessage `json:"grant,omitempty"`
|
||||
}
|
||||
|
||||
// The uses an access's user has (ADR 0051 §3), and where it runs (ADR 0325 §4).
|
||||
const (
|
||||
FromContainer = "container"
|
||||
FromToolRunner = "tool-runner"
|
||||
)
|
||||
|
||||
// AccessUser is one identity that uses an accessed folder: its name and uid as the controller allocated them, its
|
||||
// use, and where it runs. A tool runner's account may come with no uid (0 or absent): the engine reads it by name.
|
||||
type AccessUser struct {
|
||||
Identity string `json:"identity"`
|
||||
UID int `json:"uid,omitempty"`
|
||||
Use string `json:"use"`
|
||||
From string `json:"from"`
|
||||
}
|
||||
|
||||
// UnmarshalJSON reads a user as strictly as a resource is read.
|
||||
func (u *AccessUser) UnmarshalJSON(raw []byte) error {
|
||||
type plain AccessUser
|
||||
var p plain
|
||||
if err := strictDecode(raw, &p); err != nil {
|
||||
return fmt.Errorf("an access's user: %w", err)
|
||||
}
|
||||
*u = AccessUser(p)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *Access) Identity() string { return a.ID }
|
||||
@@ -482,9 +526,43 @@ func (a *Access) validate(where string, _ bool) []string {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: an access is %q or %q, not %q", where, AccessRead, AccessReadWrite, a.Mode))
|
||||
}
|
||||
for _, u := range a.Users {
|
||||
switch {
|
||||
case u.Identity == "" || !accountName.MatchString(u.Identity):
|
||||
problems = append(problems, fmt.Sprintf("%s: a user of an access is an account's name, and %q is not one",
|
||||
where, u.Identity))
|
||||
case u.UID < 0 || (u.UID == 0 && u.From != FromToolRunner):
|
||||
// Never root (novox/hq ADR 0325 §11): an entry for uid 0 would be a right root has anyway, and a
|
||||
// number below it is no account. A tool runner's account alone may come without its number: the
|
||||
// controller cannot know the operator's account's, and the engine reads it by name
|
||||
// (apply.ResolveRunners), refusing one that is root's.
|
||||
problems = append(problems, fmt.Sprintf("%s: the user %s has uid %d, and a module's identity is never "+
|
||||
"root or below it (novox/hq ADR 0325 §11)", where, u.Identity, u.UID))
|
||||
}
|
||||
if u.Use != AccessRead && u.Use != AccessReadWrite {
|
||||
problems = append(problems, fmt.Sprintf("%s: the user %s uses it %q, and a use is %q or %q", where,
|
||||
u.Identity, u.Use, AccessRead, AccessReadWrite))
|
||||
}
|
||||
if u.Use == AccessReadWrite && a.Mode != AccessReadWrite {
|
||||
problems = append(problems, fmt.Sprintf("%s: the user %s writes an access declared %s", where,
|
||||
u.Identity, orRead(a.Mode)))
|
||||
}
|
||||
if u.From != FromContainer && u.From != FromToolRunner {
|
||||
problems = append(problems, fmt.Sprintf("%s: the user %s runs from %q, and that is %q or %q", where,
|
||||
u.Identity, u.From, FromContainer, FromToolRunner))
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// orRead is an access's mode as said: absent narrows to read.
|
||||
func orRead(mode string) string {
|
||||
if mode == "" {
|
||||
return AccessRead
|
||||
}
|
||||
return mode
|
||||
}
|
||||
|
||||
func (u *User) Identity() string { return u.ID }
|
||||
func (u *User) Kind() Type { return TypeUser }
|
||||
func (u *User) Target() string { return u.Name }
|
||||
@@ -1160,6 +1238,83 @@ type Container struct {
|
||||
// Health is how this resource is ready (novox/hq ADR 0240 rule 2, Phase B): one kind and its
|
||||
// timing, judged by the node-engine beside liveness. Absent: judged alive or not, and nothing more.
|
||||
Health *Health `json:"health,omitempty"`
|
||||
|
||||
// Busy is how this container says it holds work a replacement would lose (novox/hq ADR 0335): asked
|
||||
// before it is replaced, and while it says busy it is left running, within the bound. Absent: never
|
||||
// busy, and replaced at once, as before.
|
||||
Busy *Busy `json:"busy,omitempty"`
|
||||
|
||||
// RunAs is who the container runs as, resolved by the controller from the module's identity and the
|
||||
// image's declared interface (novox/hq ADR 0325 §2, §3): told by environment (the image reads PUID and
|
||||
// PGID, say) or by the runtime's own user option. Part of the spec, so a change recreates it. Absent: the
|
||||
// container runs as its image says. Its key is `identity`, the manifest's word; the field is not, because
|
||||
// a resource's Identity is its id.
|
||||
RunAs *ContainerIdentity `json:"identity,omitempty"`
|
||||
}
|
||||
|
||||
// The ways an image is told who to run as (novox/hq ADR 0325 §2).
|
||||
const (
|
||||
IdentityByEnv = "env"
|
||||
IdentityByRun = "run"
|
||||
)
|
||||
|
||||
// ContainerIdentity is who a container runs as, and how its image is told.
|
||||
type ContainerIdentity struct {
|
||||
// Mode is IdentityByEnv or IdentityByRun.
|
||||
Mode string `json:"mode"`
|
||||
// UserEnv and GroupEnv are the variables the image reads the user's and the group's number from, for
|
||||
// IdentityByEnv: "PUID", "PGID".
|
||||
UserEnv string `json:"user_env,omitempty"`
|
||||
GroupEnv string `json:"group_env,omitempty"`
|
||||
UID int `json:"uid"`
|
||||
GID int `json:"gid"`
|
||||
}
|
||||
|
||||
// UnmarshalJSON reads an identity as strictly as a resource is read.
|
||||
func (i *ContainerIdentity) UnmarshalJSON(raw []byte) error {
|
||||
type plain ContainerIdentity
|
||||
var p plain
|
||||
if err := strictDecode(raw, &p); err != nil {
|
||||
return fmt.Errorf("a container's identity: %w", err)
|
||||
}
|
||||
*i = ContainerIdentity(p)
|
||||
return nil
|
||||
}
|
||||
|
||||
// envName is a variable's name as a runtime takes it.
|
||||
var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||
|
||||
func (i *ContainerIdentity) problems(where string, env map[string]string) []string {
|
||||
if i == nil {
|
||||
return nil
|
||||
}
|
||||
var problems []string
|
||||
switch i.Mode {
|
||||
case IdentityByEnv:
|
||||
if !envName.MatchString(i.UserEnv) || (i.GroupEnv != "" && !envName.MatchString(i.GroupEnv)) {
|
||||
problems = append(problems, fmt.Sprintf("%s: an identity told by environment names the variables "+
|
||||
"it is told in, and %q and %q are not both names", where, i.UserEnv, i.GroupEnv))
|
||||
}
|
||||
for _, name := range []string{i.UserEnv, i.GroupEnv} {
|
||||
if _, set := env[name]; set && name != "" {
|
||||
problems = append(problems, fmt.Sprintf("%s: env sets %s, which the identity sets; one value "+
|
||||
"for one variable", where, name))
|
||||
}
|
||||
}
|
||||
case IdentityByRun:
|
||||
if i.UserEnv != "" || i.GroupEnv != "" {
|
||||
problems = append(problems, where+": an identity told by the runtime's user option names no variable")
|
||||
}
|
||||
default:
|
||||
problems = append(problems, fmt.Sprintf("%s: an identity is told by %q or %q, not %q", where,
|
||||
IdentityByEnv, IdentityByRun, i.Mode))
|
||||
}
|
||||
// Never root (novox/hq ADR 0325 §11): refused here, so nothing of the declaration is made or started.
|
||||
if i.UID <= 0 || i.GID <= 0 {
|
||||
problems = append(problems, fmt.Sprintf("%s: the identity is uid %d, gid %d, and a module's identity is "+
|
||||
"never root, nor a number below it (novox/hq ADR 0325 §11)", where, i.UID, i.GID))
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
func (c *Container) Identity() string { return c.ID }
|
||||
@@ -1168,6 +1323,7 @@ func (c *Container) Target() string { return c.Name }
|
||||
|
||||
func (c *Container) validate(where string, _ bool) []string {
|
||||
problems := c.Health.problems(where, true, !c.RunOnce && c.Schedule == "")
|
||||
problems = append(problems, c.Busy.problems(where, !c.RunOnce && c.Schedule == "")...)
|
||||
if c.Name == "" {
|
||||
problems = append(problems, where+": a container needs a name")
|
||||
}
|
||||
@@ -1242,6 +1398,7 @@ func (c *Container) validate(where string, _ bool) []string {
|
||||
problems = append(problems, where+": networks is for a container that keeps running; a step "+
|
||||
"runs and exits, and joins nothing afterwards")
|
||||
}
|
||||
problems = append(problems, c.RunAs.problems(where+": container "+c.Name, c.Env)...)
|
||||
return append(problems, checkImage(where, c.Image)...)
|
||||
}
|
||||
|
||||
@@ -1392,6 +1549,185 @@ type Declaration struct {
|
||||
// the generation of what it puts back, which may be older than what the machine applied, and is
|
||||
// not refused for it.
|
||||
PutBack bool
|
||||
|
||||
// Identities is the identity of every module assigned to this machine (novox/hq ADR 0328 §4): an account
|
||||
// and a group `mesh-<module>` with the numbers the controller allocated once for the module, the same on
|
||||
// every machine. The engine makes each before any resource is applied, and removes what it made of one no
|
||||
// longer declared. Absent is a controller older than ADR 0325.
|
||||
Identities []Identity
|
||||
|
||||
// Groups is the group of every library this machine is reached by (novox/hq ADR 0328 §3): `mesh-<library>`
|
||||
// with the number the controller allocated once, mesh-wide. The engine makes each, and removes what it made of
|
||||
// one no longer declared. Absent is a controller older than ADR 0328.
|
||||
Groups []Group
|
||||
|
||||
// Libraries is every library this machine is reached by (ADR 0328 §1, §6, §9): its root here, its group, whether
|
||||
// this is its home, whether the operator's warrant to take it was carried out, the sub-paths its modules ask for,
|
||||
// and the tool runner's account to put in its group (§11). Absent is a controller older than ADR 0328.
|
||||
Libraries []Library
|
||||
}
|
||||
|
||||
// Identity is one module's identity, as the controller allocated it (ADR 0328 §4): its user `mesh-<module>` with
|
||||
// uid, its own group of the same name and number, its primary group gid (its own, or its first library's group),
|
||||
// and the groups of its other libraries as added groups.
|
||||
type Identity struct {
|
||||
Module string `json:"module"`
|
||||
Name string `json:"name"`
|
||||
UID int `json:"uid"`
|
||||
GID int `json:"gid"`
|
||||
Groups []int `json:"groups,omitempty"`
|
||||
// Runner is the tool runner's account, put in the module's own group (novox/hq ADR 0334 §2) when its tools run
|
||||
// on this machine. Absent for a module that declares no tools here, and from a controller older than ADR 0334.
|
||||
Runner string `json:"runner,omitempty"`
|
||||
}
|
||||
|
||||
// Group is one library's group, as the controller allocated it (ADR 0328 §3).
|
||||
type Group struct {
|
||||
Name string `json:"name"`
|
||||
GID int `json:"gid"`
|
||||
}
|
||||
|
||||
// GroupPrefix starts every library group's name, as IdentityPrefix starts every module's.
|
||||
const GroupPrefix = "mesh-"
|
||||
|
||||
// LibraryAccessPrefix starts the access a library's wait and its take act name: `library:<name>`.
|
||||
const LibraryAccessPrefix = "library:"
|
||||
|
||||
// Library is one library as this machine sees it (ADR 0328).
|
||||
type Library struct {
|
||||
Library string `json:"library"`
|
||||
// Root is the library's root on this machine: on its home the data itself, elsewhere the mount point the mounts
|
||||
// module mounts it at.
|
||||
Root string `json:"root"`
|
||||
Group string `json:"group"`
|
||||
GID int `json:"gid"`
|
||||
// Home is that the library's data lives on this machine: only there does the engine keep it.
|
||||
Home bool `json:"home"`
|
||||
// Taken is the controller saying the operator's warrant to take it was carried out (§7).
|
||||
Taken bool `json:"taken,omitempty"`
|
||||
// Paths are the sub-paths its modules ask for, relative to the root.
|
||||
Paths []string `json:"paths,omitempty"`
|
||||
// Runner is the tool runner's account, put in the library's group (§11). Absent when no module's tools use it.
|
||||
Runner string `json:"runner,omitempty"`
|
||||
}
|
||||
|
||||
// checkLibraries judges the groups and libraries on arrival: every library's group declared, named for it, never
|
||||
// root's number; paths that stay below the root; a runner that is an account's name; and no two roots that nest
|
||||
// (§13).
|
||||
func checkLibraries(groups []Group, libs []Library) []string {
|
||||
var problems []string
|
||||
byName := map[string]int{}
|
||||
numbers := map[int]string{}
|
||||
for _, g := range groups {
|
||||
where := fmt.Sprintf("group %q", g.Name)
|
||||
if !strings.HasPrefix(g.Name, GroupPrefix) || !accountName.MatchString(g.Name) {
|
||||
problems = append(problems, fmt.Sprintf("%s: a library's group is named %s<library>", where, GroupPrefix))
|
||||
}
|
||||
if g.GID <= 0 {
|
||||
problems = append(problems, fmt.Sprintf("%s: gid %d, and a library's group is never root, nor a number "+
|
||||
"below it (novox/hq ADR 0328 §15); nothing of this declaration was made", where, g.GID))
|
||||
}
|
||||
if _, twice := byName[g.Name]; twice {
|
||||
problems = append(problems, where+": declared twice")
|
||||
}
|
||||
if other, used := numbers[g.GID]; used {
|
||||
problems = append(problems, fmt.Sprintf("%s: gid %d is %s's too", where, g.GID, other))
|
||||
}
|
||||
byName[g.Name], numbers[g.GID] = g.GID, g.Name
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, l := range libs {
|
||||
where := fmt.Sprintf("library %q", l.Library)
|
||||
switch {
|
||||
case l.Library == "" || strings.ContainsAny(l.Library, "/ \t\n"):
|
||||
problems = append(problems, where+": a library is named by one word")
|
||||
case seen[l.Library]:
|
||||
problems = append(problems, where+": declared twice")
|
||||
}
|
||||
seen[l.Library] = true
|
||||
if !filepath.IsAbs(l.Root) || filepath.Clean(l.Root) != l.Root || l.Root == "/" {
|
||||
problems = append(problems, fmt.Sprintf("%s: its root %q is not an absolute, clean path below /", where,
|
||||
l.Root))
|
||||
}
|
||||
if gid, ok := byName[l.Group]; !ok || gid != l.GID {
|
||||
problems = append(problems, fmt.Sprintf("%s: its group %s (gid %d) is not one this declaration makes",
|
||||
where, l.Group, l.GID))
|
||||
}
|
||||
if l.Group != GroupPrefix+l.Library {
|
||||
problems = append(problems, fmt.Sprintf("%s: its group is %s%s, and %s is not", where, GroupPrefix,
|
||||
l.Library, l.Group))
|
||||
}
|
||||
for _, p := range l.Paths {
|
||||
if p == "" || filepath.IsAbs(p) || filepath.Clean(p) != p || p == ".." || strings.HasPrefix(p, "../") {
|
||||
problems = append(problems, fmt.Sprintf("%s: the path %q is not a clean path below its root", where, p))
|
||||
}
|
||||
}
|
||||
if l.Runner != "" && !accountName.MatchString(l.Runner) {
|
||||
problems = append(problems, fmt.Sprintf("%s: its runner %q is not an account's name", where, l.Runner))
|
||||
}
|
||||
}
|
||||
for i, a := range libs {
|
||||
for _, b := range libs[i+1:] {
|
||||
if a.Root != "" && b.Root != "" && (a.Root == b.Root || strings.HasPrefix(a.Root, b.Root+"/") ||
|
||||
strings.HasPrefix(b.Root, a.Root+"/")) {
|
||||
problems = append(problems, fmt.Sprintf("library %q: its root and library %q's nest, and libraries "+
|
||||
"never overlap (novox/hq ADR 0328 §13)", a.Library, b.Library))
|
||||
}
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// IdentityPrefix starts every module identity's name.
|
||||
const IdentityPrefix = "mesh-"
|
||||
|
||||
// checkIdentities judges the identities on arrival: each named for its module, never root, and no name or number
|
||||
// twice — two modules with one account would write as each other.
|
||||
func checkIdentities(ids []Identity, groups []Group) []string {
|
||||
var problems []string
|
||||
names, uids := map[string]bool{}, map[int]string{}
|
||||
libraryGroups := map[int]bool{}
|
||||
for _, g := range groups {
|
||||
libraryGroups[g.GID] = true
|
||||
}
|
||||
for _, id := range ids {
|
||||
where := fmt.Sprintf("identity %q", id.Name)
|
||||
if id.Module == "" || id.Name != IdentityPrefix+id.Module || !accountName.MatchString(id.Name) {
|
||||
problems = append(problems, fmt.Sprintf("%s: a module's identity is named %s<module>, and this one is "+
|
||||
"for the module %q", where, IdentityPrefix, id.Module))
|
||||
}
|
||||
if id.UID <= 0 || id.GID <= 0 {
|
||||
problems = append(problems, fmt.Sprintf("%s: uid %d, gid %d, and a module's identity is never root, "+
|
||||
"nor a number below it (novox/hq ADR 0328 §15); nothing of this declaration was made", where,
|
||||
id.UID, id.GID))
|
||||
}
|
||||
if names[id.Name] {
|
||||
problems = append(problems, where+": declared twice")
|
||||
}
|
||||
if other, used := uids[id.UID]; used {
|
||||
problems = append(problems, fmt.Sprintf("%s: uid %d is %s's too", where, id.UID, other))
|
||||
}
|
||||
// Its primary group is its own (gid == uid) or a library's this declaration makes (ADR 0328 §4); so is
|
||||
// every added group. A module's own number is never a library group's.
|
||||
if id.GID != id.UID && !libraryGroups[id.GID] {
|
||||
problems = append(problems, fmt.Sprintf("%s: its primary group gid %d is neither its own (%d) nor a "+
|
||||
"library's this declaration makes", where, id.GID, id.UID))
|
||||
}
|
||||
if libraryGroups[id.UID] {
|
||||
problems = append(problems, fmt.Sprintf("%s: uid %d is a library group's number too", where, id.UID))
|
||||
}
|
||||
for _, g := range id.Groups {
|
||||
if !libraryGroups[g] {
|
||||
problems = append(problems, fmt.Sprintf("%s: its added group gid %d is not a library's this "+
|
||||
"declaration makes", where, g))
|
||||
}
|
||||
}
|
||||
if id.Runner != "" && !accountName.MatchString(id.Runner) {
|
||||
problems = append(problems, fmt.Sprintf("%s: its runner %q is not an account's name", where, id.Runner))
|
||||
}
|
||||
names[id.Name], uids[id.UID] = true, id.Name
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// LeftOutModuleOf says which left-out module a recorded resource belongs to, if any: its id is the
|
||||
@@ -1561,6 +1897,13 @@ type envelope struct {
|
||||
// sends them only to a host whose reports say `reads_generation`.
|
||||
Generation int64 `json:"generation,omitempty"`
|
||||
PutBack bool `json:"put_back,omitempty"`
|
||||
// Identities is optional on the wire too (novox/hq ADR 0325 §3, ADR 0328 §4). **An older host refuses this key**; a
|
||||
// controller sends it only to a host whose health says link.IdentityContract.
|
||||
Identities []Identity `json:"identities,omitempty"`
|
||||
// Groups and Libraries are optional too (novox/hq ADR 0328). **An older host refuses these keys**; a controller
|
||||
// sends them only to a host whose health says link.LibraryContract.
|
||||
Groups []Group `json:"groups,omitempty"`
|
||||
Libraries []Library `json:"libraries,omitempty"`
|
||||
}
|
||||
|
||||
func parse(raw []byte, allowActions bool) (*Declaration, error) {
|
||||
@@ -1578,8 +1921,10 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) {
|
||||
}
|
||||
|
||||
d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption, Sequence: env.Sequence,
|
||||
Epoch: env.Epoch, LeftOut: env.LeftOut, Generation: env.Generation, PutBack: env.PutBack}
|
||||
var problems []string
|
||||
Epoch: env.Epoch, LeftOut: env.LeftOut, Generation: env.Generation, PutBack: env.PutBack,
|
||||
Identities: env.Identities, Groups: env.Groups, Libraries: env.Libraries}
|
||||
problems := checkIdentities(env.Identities, env.Groups)
|
||||
problems = append(problems, checkLibraries(env.Groups, env.Libraries)...)
|
||||
if env.Generation < 0 {
|
||||
// Below zero would read as "none claimed" and pass every refusal of an older generation.
|
||||
problems = append(problems, fmt.Sprintf("an assignment generation below zero (%d) is not one the "+
|
||||
@@ -1732,6 +2077,10 @@ func checkWhileStopped(resources []Resource) []string {
|
||||
return problems
|
||||
}
|
||||
|
||||
// accountName is what an identity's or an access user's name may be: the portable account name the user tools
|
||||
// take, optionally ending in `$`, never starting with a dash — it is handed to them as an argument.
|
||||
var accountName = regexp.MustCompile(`^[A-Za-z0-9_.][A-Za-z0-9_.-]*\$?$`)
|
||||
|
||||
func strictDecode(raw []byte, into any) error {
|
||||
// DisallowUnknownFields is the whole point rather than strictness for its own sake: a
|
||||
// field the host does not know is a thing the control plane believes it asked for.
|
||||
|
||||
Vendored
+2
-2
@@ -78,7 +78,7 @@ github.com/nats-io/nkeys
|
||||
# github.com/nats-io/nuid v1.0.1
|
||||
## explicit
|
||||
github.com/nats-io/nuid
|
||||
# github.com/novox/mesh-host v0.0.0 => git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f
|
||||
# github.com/novox/mesh-host v0.0.0 => git.novox.be/novox/mesh-host v0.0.0-20261012224132-bb60e7b584d8
|
||||
## explicit; go 1.26.0
|
||||
github.com/novox/mesh-host/internal/declaration
|
||||
github.com/novox/mesh-host/rootsearch
|
||||
@@ -138,4 +138,4 @@ golang.org/x/text/width
|
||||
# golang.org/x/time v0.15.0
|
||||
## explicit; go 1.25.0
|
||||
golang.org/x/time/rate
|
||||
# github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f
|
||||
# github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261012224132-bb60e7b584d8
|
||||
|
||||
Reference in New Issue
Block a user