Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
81f3c2d01b |
@@ -3,6 +3,3 @@
|
||||
# A built binary. The lab writes one here when pointed at the repository root by mistake;
|
||||
# built artifacts belong in build/, which is already ignored.
|
||||
/mesh-builder
|
||||
/mesh-controller
|
||||
/postgres-provisioner
|
||||
/redis-provisioner
|
||||
|
||||
@@ -99,22 +99,16 @@ proxy-image:
|
||||
@echo
|
||||
@docker image inspect $(PROXY_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||
|
||||
# The whole check. Raises a database, runs the repository's own check against it -- merge-check.sh,
|
||||
# the very script `mesh/repo-check` runs -- and takes the database down again, including when the
|
||||
# check fails, which is why the teardown is not conditional and the script's exit status is the
|
||||
# target's.
|
||||
# The whole gate. Raises a database, runs everything against it, and takes it down again --
|
||||
# including when the tests fail, which is why the teardown is not conditional.
|
||||
#
|
||||
# **It calls the script rather than restating it** (novox/hq issue 431): this target used to list its
|
||||
# own steps, its formatting step walked vendor/ where the script's does not, and it failed on
|
||||
# third-party code no change could fix -- so a developer's check and the gate disagreed, and the steps
|
||||
# after formatting never ran through it. The script is the one list of steps; this target only gives it
|
||||
# a database (MESH_TEST_POSTGRES, exported above).
|
||||
#
|
||||
# **Packages in parallel, under the race detector, each test on a bus of its own** (internal/testbus),
|
||||
# as the script runs them: a bus per test, of the release the mesh runs, makes the suite the same in any
|
||||
# order, and the timeout bounds a hang to a failure with a stack, never a stalled gate.
|
||||
check: postgres
|
||||
@sh merge-check.sh ; status=$$? ; docker rm -f $(PG_CONTAINER) >/dev/null 2>&1 || true ; exit $$status
|
||||
# **Packages in parallel, under the race detector, each test on a bus of its own** (internal/testbus).
|
||||
# It was one package at a time against one shared bus, because the live tests assert, read and remove
|
||||
# the mesh's own objects by their fixed names, and two packages at once deleted what the other read; the
|
||||
# suite was red run as Go runs it and read as noise. A bus per test, of the release the mesh runs, made
|
||||
# it the same in any order. The timeout bounds a hang to a failure with a stack, never a stalled gate.
|
||||
check: fmt vet postgres
|
||||
@go test -race -timeout 15m ./... ; status=$$? ; $(MAKE) postgres-stop ; exit $$status
|
||||
|
||||
# Without a database the store's tests skip rather than fail, so this is the honest subset and not
|
||||
# the gate.
|
||||
@@ -124,10 +118,8 @@ test:
|
||||
vet:
|
||||
go vet ./...
|
||||
|
||||
# Quick steps for a developer, not part of `check`. The directories gofmt reads must be the ones
|
||||
# merge-check.sh lists, never `.`, which walks vendor/ (novox/hq issue 431).
|
||||
fmt:
|
||||
@unformatted=$$(gofmt -l cmd internal examples) ; \
|
||||
@unformatted=$$(gofmt -l . 2>/dev/null) ; \
|
||||
if [ -n "$$unformatted" ] ; then echo "not gofmt'd:" ; echo "$$unformatted" ; exit 1 ; fi
|
||||
|
||||
postgres:
|
||||
|
||||
@@ -16,8 +16,7 @@ RUN CGO_ENABLED=0 go build -trimpath -ldflags '-s -w' -o /mesh-builder ./cmd/mes
|
||||
FROM ${ALPINE_BASE}
|
||||
# git to clone what it is asked to build, and the docker client to build and push it. The daemon
|
||||
# is the machine's, reached through its socket — a build machine shares the runtime it was given
|
||||
# rather than running one inside itself. xz to read an upstream release an archive is pinned to
|
||||
# (novox/hq ADR 0316): Go's standard library reads gzip and bzip2 and not xz.
|
||||
RUN apk add --no-cache git docker-cli xz
|
||||
# rather than running one inside itself.
|
||||
RUN apk add --no-cache git docker-cli
|
||||
COPY --from=build /mesh-builder /usr/local/bin/mesh-builder
|
||||
ENTRYPOINT ["/usr/local/bin/mesh-builder"]
|
||||
|
||||
@@ -306,9 +306,6 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
for _, r := range built.Read {
|
||||
result.Read = append(result.Read, link.ReadRepository{Repository: r.Repository, Ref: r.Ref})
|
||||
}
|
||||
for _, s := range built.Sources {
|
||||
result.Sources = append(result.Sources, link.BuildSource{Repository: s.Repository, Ref: s.Ref, Paths: s.Paths})
|
||||
}
|
||||
say("built", built.Manifest.Module+" from "+short(built.Commit))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -180,21 +179,7 @@ func (a *actor) release() {
|
||||
// holderOf is this process as the lease's holder.
|
||||
func holderOf(instance string) lease.Holder {
|
||||
host, _ := os.Hostname()
|
||||
build := runningBuild()
|
||||
if build == "" {
|
||||
build = version
|
||||
}
|
||||
return lease.Holder{Instance: instance, Host: host, Build: build}
|
||||
}
|
||||
|
||||
// RunningBuildVar is where the declaration tells this process which build it is (module.json, the
|
||||
// controller process's env): the version its bundle is delivered as, `${version}` composed by the
|
||||
// catalogue from the bundle's digest. Empty for a process placed by hand.
|
||||
const RunningBuildVar = "MESH_CONTROLLER_VERSION"
|
||||
|
||||
// runningBuild is the version of the build this process is, or empty when the declaration did not say.
|
||||
func runningBuild() string {
|
||||
return strings.TrimSpace(os.Getenv(RunningBuildVar))
|
||||
return lease.Holder{Instance: instance, Host: host, Build: version}
|
||||
}
|
||||
|
||||
// serveUnderTheLease takes the lease for the serving controller, waiting while another holds it, and
|
||||
|
||||
+15
-67
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"io"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -55,9 +54,6 @@ func assignWith(ctx context.Context, open *stores, node string, opts assignOptio
|
||||
if err := refusedMovingTheController(modules); err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Every line of the answer comes from the one resolution after the write (novox/hq issue 237):
|
||||
// what the reads before it would say about a machine left out is not the answer's.
|
||||
ctx = leftOutSaidBy(ctx, io.Discard)
|
||||
// Held while it is recorded, so it cannot land between a converge's preview and its flip and
|
||||
// be taken without ever having been previewed (novox/hq ADR 0100).
|
||||
ctx, release, err := holdNodes(ctx, open, []string{node})
|
||||
@@ -129,28 +125,22 @@ func assignWith(ctx context.Context, open *stores, node string, opts assignOptio
|
||||
for _, line := range settled {
|
||||
answer += "\n " + line
|
||||
}
|
||||
// What this act changed about this node's unmet seat dependencies, and nothing else (novox/hq
|
||||
// ADR 0207): a dependency of a module just assigned, or one this assignment met. The rest of the
|
||||
// node's list, and every other node's, is `status`'s.
|
||||
for _, line := range unheldChange(shelf, node, before, append(append([]string(nil), before...), added...)) {
|
||||
answer += "\n " + line
|
||||
}
|
||||
// Its bus credential, in the same act (novox/hq issue 203): an assignment pushed before its
|
||||
// credential exists delivers a process that cannot authenticate and crash-loops until somebody
|
||||
// runs a second verb and a second push. Issued here when the module speaks on the bus and has
|
||||
// no credential yet; kept when it has one, so re-assigning rotates nothing.
|
||||
var issued []string
|
||||
for _, module := range added {
|
||||
if line := issueOnAssign(ctx, open, node, module); line != "" {
|
||||
issued = append(issued, line)
|
||||
answer += "\n " + line
|
||||
}
|
||||
}
|
||||
// **The one resolution the rest of the answer is said from** (novox/hq issue 237), taken after
|
||||
// the write: what this act changed about this node's unmet seat dependencies, and nothing else
|
||||
// (novox/hq ADR 0207) — a dependency of a module just assigned, or one this assignment met; the
|
||||
// rest of the node's list, and every other node's, is `status`'s — and whether the node resolves.
|
||||
after := resolvedAfter(ctx, open, shelf, node)
|
||||
for _, line := range unheldChangeTo(shelf, node, before, after.modules, after.unheld) {
|
||||
answer += "\n " + line
|
||||
}
|
||||
for _, line := range issued {
|
||||
answer += "\n " + line
|
||||
}
|
||||
plan, err := after.plan, after.err
|
||||
plan, _, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
// Kept, and still refused. Both halves are the answer, and the rest of the mesh is still
|
||||
// worth reporting: this machine's refusal is rarely the only consequence.
|
||||
@@ -228,8 +218,6 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string)
|
||||
if len(modules) == 0 {
|
||||
return "", fmt.Errorf("unassign %s names no module", node)
|
||||
}
|
||||
// One resolution after the write, as assign's answer (novox/hq issue 237).
|
||||
ctx = leftOutSaidBy(ctx, io.Discard)
|
||||
ctx, release, err := holdNodes(ctx, open, []string{node})
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -274,21 +262,14 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string)
|
||||
}
|
||||
answer := fmt.Sprintf("%s no longer runs %s — run `push %s` to make it so",
|
||||
node, strings.Join(modules, ", "), node)
|
||||
after := resolvedAfter(ctx, open, shelf, node)
|
||||
for _, line := range unheldChangeTo(shelf, node, assigned, after.modules, after.unheld) {
|
||||
answer += "\n " + line
|
||||
var left []string
|
||||
for _, a := range assigned {
|
||||
if !slices.Contains(modules, a) {
|
||||
left = append(left, a)
|
||||
}
|
||||
}
|
||||
// Whether the node itself still resolves, from the same resolution: before, only the rest of the
|
||||
// mesh's reads said it, on standard error and once per other machine.
|
||||
if after.err != nil {
|
||||
if unresolvable(after.err) {
|
||||
answer += fmt.Sprintf("\n but %s does not resolve now, so nothing will be sent to it:", node)
|
||||
} else {
|
||||
answer += fmt.Sprintf("\n but whether %s still resolves could not be read:", node)
|
||||
}
|
||||
for _, line := range strings.Split(strings.TrimRight(after.err.Error(), "\n"), "\n") {
|
||||
answer += "\n " + strings.TrimSpace(line)
|
||||
}
|
||||
for _, line := range unheldChange(shelf, node, assigned, left) {
|
||||
answer += "\n " + line
|
||||
}
|
||||
// What it leaves behind that is irreplaceable is kept and retired, never removed (novox/hq ADR 0233).
|
||||
for _, line := range keptOnUnassign(ctx, open.inventory, node, modules) {
|
||||
@@ -297,39 +278,6 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string)
|
||||
return answer + blockedElsewhere(ctx, open, node), nil
|
||||
}
|
||||
|
||||
// afterAct is the one resolution of a node an act's answer is said from, taken after its write
|
||||
// (novox/hq issue 237): the plan or why there is none, the modules it has on the node, and its unmet
|
||||
// seat dependencies.
|
||||
type afterAct struct {
|
||||
plan catalogue.Resolution
|
||||
err error
|
||||
modules []string
|
||||
unheld []catalogue.Unheld
|
||||
}
|
||||
|
||||
// resolvedAfter resolves the node as `plan` does, once, after the act's write. When the node resolves,
|
||||
// its modules and unmet seat dependencies are the resolution's — the closure, so a holder pulled in by
|
||||
// a requirement counts as `plan` counts it. When it does not, they are judged over the assignments read
|
||||
// after the write, which can only find more unmet than the closure would: a seat said held there is
|
||||
// held in the closure too.
|
||||
func resolvedAfter(ctx context.Context, open *stores, shelf map[string]catalogue.Manifest, node string) afterAct {
|
||||
plan, _, err := planFor(ctx, open, node)
|
||||
if err == nil {
|
||||
out := afterAct{plan: plan, unheld: plan.Unheld}
|
||||
for _, m := range plan.Modules {
|
||||
out.modules = append(out.modules, m.Module)
|
||||
}
|
||||
return out
|
||||
}
|
||||
out := afterAct{err: err}
|
||||
assigned, readErr := open.inventory.Assigned(ctx, node)
|
||||
if readErr != nil {
|
||||
return out
|
||||
}
|
||||
out.modules, out.unheld = assigned, unheldOver(shelf, node, assigned)
|
||||
return out
|
||||
}
|
||||
|
||||
// splitModules is a surface's one `module` field as the modules it names: several, comma-separated,
|
||||
// are one act (novox/hq ADR 0207), so the holders that depend on each other go on together from the
|
||||
// command API and the controller seat's verbs as they do from the command line.
|
||||
|
||||
@@ -408,9 +408,6 @@ func comparisonOf(module string, c comparison, opts takeOptions) (preview string
|
||||
switch {
|
||||
case sec.Origin == inventory.OriginAccepted:
|
||||
fmt.Fprintf(&b, " %s %s: accepted from a person, carried in as it is\n", what, name)
|
||||
case sec.Origin == inventory.OriginIssued:
|
||||
// Taken from the provider's software (novox/hq ADR 0305): the value the running service uses.
|
||||
fmt.Fprintf(&b, " %s %s: issued by the provider's software, carried in as it is\n", what, name)
|
||||
case opts.Mint[sec.Name]:
|
||||
fmt.Fprintf(&b, " %s %s: minted by the mesh; the service takes the new value, as --mint said\n", what, name)
|
||||
case foundData:
|
||||
|
||||
@@ -1,180 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A refusal the bus said once is counted once** (novox/hq issue 402). The bus refused the controller
|
||||
// one publish at 18:52:59 UTC on 2026-10-10 and never again, yet S9 looked at the one remembered
|
||||
// advisory every half minute for the hour it is kept, and the condition said "observed 15 time(s), last
|
||||
// 13s ago" with a refusal in its evidence every 30 seconds: two of us read it as a refusal going on and
|
||||
// went looking for a missing grant. The condition counts what the bus said, with when it last said it,
|
||||
// never how often the controller looked.
|
||||
func TestARefusalSaidOnceIsCountedOnceHoweverOftenItIsLookedAt(t *testing.T) {
|
||||
refused := time.Date(2026, 10, 10, 18, 52, 59, 0, time.UTC)
|
||||
now := refused.Add(10 * time.Second)
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
||||
t.Cleanup(func() { k.Close(t.Context()) })
|
||||
|
||||
// The words the live condition bus.controller.refused carried on 2026-10-10.
|
||||
said := "the bus refused the controller: nats: permissions violation: Permissions Violation for " +
|
||||
`Publish to "mesh.seat.mesh-delivery.tool.times"`
|
||||
advisory := link.Advisory{Kind: link.AdvisoryRefused, ID: "controller", Said: said,
|
||||
First: refused, Last: refused, Count: 1}
|
||||
look := func() conditions.Condition {
|
||||
t.Helper()
|
||||
f := &signalFacts{now: now, host: "novox", advisories: []link.Advisory{advisory}}
|
||||
if err := k.Reconcile(t.Context(), "S9", watchAdvisories(f)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c, found, err := conditions.ReadOne(t.Context(), store, "bus.controller.refused")
|
||||
if err != nil || !found {
|
||||
t.Fatalf("bus.controller.refused: found %v, %v", found, err)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
var c conditions.Condition
|
||||
for range 15 {
|
||||
c = look()
|
||||
now = now.Add(30 * time.Second)
|
||||
}
|
||||
if c.Observations != 1 || len(c.Evidence) != 1 {
|
||||
t.Fatalf("one refusal, looked at 15 times, reads as observed %d time(s) with %d evidence lines: %+v",
|
||||
c.Observations, len(c.Evidence), c.Evidence)
|
||||
}
|
||||
if !c.LastObserved.Equal(refused) || !c.Evidence[0].At.Equal(refused) {
|
||||
t.Fatalf("last observed %s, evidence at %s: the refusal was at %s", c.LastObserved, c.Evidence[0].At, refused)
|
||||
}
|
||||
|
||||
// The bus refuses it three times more between two looks: the condition counts four refusals, says
|
||||
// the newest, and adds one line of evidence for the look that saw them.
|
||||
again := now.Add(-5 * time.Second)
|
||||
advisory.Last, advisory.Count = again, 4
|
||||
c = look()
|
||||
if c.Observations != 4 || len(c.Evidence) != 2 || !c.LastObserved.Equal(again) || !c.Evidence[0].At.Equal(again) {
|
||||
t.Fatalf("four refusals read as observed %d time(s), last %s, evidence %+v", c.Observations, c.LastObserved, c.Evidence)
|
||||
}
|
||||
if !strings.Contains(c.Summary, "(4 times since 18:52 UTC)") {
|
||||
t.Fatalf("summary %q", c.Summary)
|
||||
}
|
||||
now = now.Add(30 * time.Second)
|
||||
if c = look(); c.Observations != 4 || len(c.Evidence) != 2 {
|
||||
t.Fatalf("a look with nothing new counted: observed %d time(s), evidence %+v", c.Observations, c.Evidence)
|
||||
}
|
||||
}
|
||||
|
||||
// **A condition that looks at the present still counts its looks** (novox/hq issue 402 changes only
|
||||
// what reads a record): a machine silent for three looks was observed three times, and says so.
|
||||
func TestAConditionOfThePresentCountsItsLooks(t *testing.T) {
|
||||
now := time.Date(2026, 10, 10, 19, 0, 0, 0, time.UTC)
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
||||
t.Cleanup(func() { k.Close(t.Context()) })
|
||||
var c conditions.Condition
|
||||
for range 3 {
|
||||
var err error
|
||||
if c, err = k.Observe(t.Context(), conditions.Observation{Scope: conditions.ScopeMachine, ID: "ace",
|
||||
Kind: "silent", Machine: "ace", Severity: conditions.Warning, Summary: "ace has not been heard from",
|
||||
Source: "S1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now = now.Add(30 * time.Second)
|
||||
}
|
||||
if c.Observations != 3 || len(c.Evidence) != 3 || !c.LastObserved.Equal(now.Add(-30*time.Second)) {
|
||||
t.Fatalf("observed %d time(s), %d evidence lines, last %s", c.Observations, len(c.Evidence), c.LastObserved)
|
||||
}
|
||||
}
|
||||
|
||||
// **One key, one watcher** (novox/hq issue 440). A consumer's max-deliveries condition is said from
|
||||
// DEAD_LETTERS while it holds a message the consumer gave up on (issue 330). A max-deliveries advisory
|
||||
// without a token names the same key; read beside it, each look added an observation and a line of
|
||||
// evidence through the dead-letter half, and the two summaries overwrote each other on every look. The
|
||||
// dead-letter row alone says that key, and counts the messages given up on, never the looks.
|
||||
func TestAHeldDeadLetterIsCountedByWhatWasGivenUpNotByTheLooks(t *testing.T) {
|
||||
gaveUp := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
|
||||
now := gaveUp.Add(20 * time.Second)
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
||||
t.Cleanup(func() { k.Close(t.Context()) })
|
||||
|
||||
const key = "bus.EVENTS.media_sonarr.max-deliveries"
|
||||
held := map[string]int{"EVENTS.media_sonarr": 1}
|
||||
newest := map[string]time.Time{"EVENTS.media_sonarr": gaveUp}
|
||||
advisory := link.Advisory{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.media_sonarr", Stream: "EVENTS",
|
||||
Consumer: "media_sonarr", Said: "sonarr on media handed message 7 over 5 times and gave up on it",
|
||||
First: gaveUp, Last: gaveUp, Count: 1}
|
||||
look := func() conditions.Condition {
|
||||
t.Helper()
|
||||
f := &signalFacts{now: now, host: "novox", advisories: []link.Advisory{advisory},
|
||||
deadLetters: held, deadLettersNewest: newest}
|
||||
if err := k.Reconcile(t.Context(), "S9", watchAdvisories(f)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c, found, err := conditions.ReadOne(t.Context(), store, key)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("%s: found %v, %v", key, found, err)
|
||||
}
|
||||
if !strings.Contains(c.Summary, "dead-letters verb") {
|
||||
t.Fatalf("the summary is not the dead-letter row's: %q", c.Summary)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
var c conditions.Condition
|
||||
for range 5 {
|
||||
c = look()
|
||||
now = now.Add(30 * time.Second)
|
||||
}
|
||||
if c.Observations != 1 || len(c.Evidence) != 1 || !c.LastObserved.Equal(gaveUp) {
|
||||
t.Fatalf("one message given up on, looked at five times, reads as observed %d time(s), last %s, "+
|
||||
"with %d evidence lines: %+v", c.Observations, c.LastObserved, len(c.Evidence), c.Evidence)
|
||||
}
|
||||
|
||||
// The consumer gives up on a second message between two looks: two given up on, one more line.
|
||||
again := now.Add(-10 * time.Second)
|
||||
held["EVENTS.media_sonarr"], newest["EVENTS.media_sonarr"] = 2, again
|
||||
c = look()
|
||||
if c.Observations != 2 || len(c.Evidence) != 2 || !c.LastObserved.Equal(again) {
|
||||
t.Fatalf("two given up on read as observed %d time(s), last %s, evidence %+v", c.Observations,
|
||||
c.LastObserved, c.Evidence)
|
||||
}
|
||||
now = now.Add(30 * time.Second)
|
||||
if c = look(); c.Observations != 2 || len(c.Evidence) != 2 {
|
||||
t.Fatalf("a look with nothing new counted: observed %d time(s), evidence %+v", c.Observations, c.Evidence)
|
||||
}
|
||||
}
|
||||
|
||||
// The advisory still says the max-deliveries it alone knows: a message given up on that could not be
|
||||
// kept, under a key of its own (issue 330), which issue 440's change leaves as it was.
|
||||
func TestAMessageThatCouldNotBeKeptIsStillSaidFromTheAdvisory(t *testing.T) {
|
||||
at := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
|
||||
f := &signalFacts{now: at, host: "novox", advisories: []link.Advisory{{Kind: link.AdvisoryMaxDeliveries,
|
||||
ID: "EVENTS.media_sonarr", Stream: "EVENTS", Consumer: "media_sonarr", Token: link.AdvisoryNotKept,
|
||||
Said: "sonarr on media gave up on message 7, and it could not be kept", First: at, Last: at, Count: 1}}}
|
||||
said := watchAdvisories(f)
|
||||
if len(said) != 1 || said[0].Key() != "bus.EVENTS.media_sonarr.not-kept" {
|
||||
t.Fatalf("%+v", said)
|
||||
}
|
||||
}
|
||||
|
||||
// A max-deliveries advisory without a token names a consumer's dead-letter key, which only DEAD_LETTERS
|
||||
// says (novox/hq issues 330 and 440): with nothing held, the advisory alone raises nothing.
|
||||
func TestAMaxDeliveriesAdvisoryAloneRaisesNothing(t *testing.T) {
|
||||
at := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
|
||||
f := &signalFacts{now: at, host: "novox", advisories: []link.Advisory{{Kind: link.AdvisoryMaxDeliveries,
|
||||
ID: "EVENTS.media_sonarr", Stream: "EVENTS", Consumer: "media_sonarr",
|
||||
Said: "sonarr on media handed message 7 over 5 times and gave up on it", First: at, Last: at, Count: 1}}}
|
||||
if said := watchAdvisories(f); len(said) != 0 {
|
||||
t.Fatalf("said %+v", said)
|
||||
}
|
||||
}
|
||||
@@ -48,8 +48,7 @@ const agentAccountProbe = "DA"
|
||||
//
|
||||
// The one judgement: `node show`, the self-check, and ADR 0259's router honouring a verified sender read
|
||||
// it here.
|
||||
// now is the judging clock, threaded so a caller judging several things at one instant judges them all at it.
|
||||
func agentConfined(ctx context.Context, inv *inventory.Inventory, node string, now time.Time) (named, confined bool, why string, err error) {
|
||||
func agentConfined(ctx context.Context, inv *inventory.Inventory, node string) (named, confined bool, why string, err error) {
|
||||
n, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return false, false, "", err
|
||||
@@ -62,7 +61,7 @@ func agentConfined(ctx context.Context, inv *inventory.Inventory, node string, n
|
||||
if err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
confined, why = judgedConfined(n.AgentAccount, h, had, now)
|
||||
confined, why = judgedConfined(n.AgentAccount, h, had, time.Now())
|
||||
return true, confined, why, nil
|
||||
}
|
||||
|
||||
@@ -119,11 +118,9 @@ func judgedConfined(agent string, h inventory.NodeHealth, had bool, now time.Tim
|
||||
}
|
||||
|
||||
// searchQuietFor is how long the controller lets an agent account's verdict wait for the node-engine's setuid
|
||||
// search before that is itself the urgent condition: the longest a search is expected to take (link.RootSearchQuiet,
|
||||
// the engine's own value; novox/hq issue 361 — the search runs to completion, with no bound of its own), counted
|
||||
// from when this controller first saw it waiting, never from the engine's start. Quiet raises nothing; it never
|
||||
// makes the agent account confined, which only a healthy verdict from a complete, fresh search does.
|
||||
const searchQuietFor = link.RootSearchQuiet
|
||||
// search before that is itself the urgent condition: the engine's bound on one search (link.RootSearchBound, the
|
||||
// engine's own value), counted from when this controller first saw it waiting, never from the engine's start.
|
||||
const searchQuietFor = link.RootSearchBound
|
||||
|
||||
// The kinds of an agent account's verdict, for the quiet a search earns.
|
||||
const (
|
||||
@@ -209,10 +206,10 @@ func probeAgentAccounts(ctx context.Context, d *doctor) ([]conditions.Observatio
|
||||
if confined {
|
||||
continue
|
||||
}
|
||||
// Not judged yet only because the node-engine's search runs and no complete, fresh one judges: not the
|
||||
// Not judged yet only because the first search since the node-engine started is still running: not the
|
||||
// urgent condition after every restart. The agent is still not confined — ADR 0259's router reads
|
||||
// agentConfined, not this — and `node show` still says not judged. Loud again once the search fails,
|
||||
// waits past searchQuietFor, or the statement goes stale.
|
||||
// runs out its bound, or the statement goes stale.
|
||||
if quiet {
|
||||
continue
|
||||
}
|
||||
@@ -231,7 +228,7 @@ func agentAccountLines(ctx context.Context, inv *inventory.Inventory, n inventor
|
||||
return []string{fmt.Sprintf(" agents run as the operator account (%s); no agent account is named",
|
||||
orNoneKnown(n.Account))}
|
||||
}
|
||||
_, confined, why, err := agentConfined(ctx, inv, n.Name, time.Now())
|
||||
_, confined, why, err := agentConfined(ctx, inv, n.Name)
|
||||
if err != nil {
|
||||
return []string{fmt.Sprintf(" agents run as %s (home %s); whether it can become root could NOT be read: %v",
|
||||
n.AgentAccount, n.AgentHome(), err)}
|
||||
|
||||
@@ -106,10 +106,10 @@ func TestTheSelfCheckSaysAnAgentAccountThatCanBecomeRoot(t *testing.T) {
|
||||
if found, err = probeAgentAccounts(ctx, d); err != nil || len(onlyMachine(found, "anchor")) != 0 {
|
||||
t.Fatalf("a judged agent account still fails: %+v %v", found, err)
|
||||
}
|
||||
if named, confined, why, err := agentConfined(ctx, inv, "anchor", time.Now()); err != nil || !named || !confined {
|
||||
if named, confined, why, err := agentConfined(ctx, inv, "anchor"); err != nil || !named || !confined {
|
||||
t.Fatalf("agentConfined on anchor: %v %v %q %v", named, confined, why, err)
|
||||
}
|
||||
if named, _, why, err := agentConfined(ctx, inv, "laptop", time.Now()); err != nil || named ||
|
||||
if named, _, why, err := agentConfined(ctx, inv, "laptop"); err != nil || named ||
|
||||
!strings.Contains(why, "operator account") {
|
||||
t.Fatalf("agentConfined on a machine naming none: %v %q %v", named, why, err)
|
||||
}
|
||||
@@ -199,20 +199,14 @@ func TestTheNodeVerbOnlyShows(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Until a complete search for setuid programs judges — none since the node-engine's state was kept, or the last
|
||||
// older than the engine lets one judge — the agent account is not judged, and the search runs to its end with no
|
||||
// bound (novox/hq issue 361). DA does not raise that as urgent within searchQuietFor, counted from when this
|
||||
// controller first saw it waiting — never from the engine's own "since", which a restart resets, so an engine
|
||||
// restarted in a loop does not keep it quiet. The account is still not confined, and `node show` still says not
|
||||
// judged; a search that failed, or a way to root found, is urgent at once.
|
||||
// After every node-engine restart its search for setuid programs runs for up to its bound, and the agent account
|
||||
// is not judged until it ends. DA does not raise that as urgent while the search is within its bound, counted from
|
||||
// when this controller first saw it waiting — never from the engine's own "since", which a restart resets, so an
|
||||
// engine restarted in a loop does not keep it quiet. The account is still not confined, and `node show` still
|
||||
// says not judged; a search that failed, or a way to root found, is urgent at once.
|
||||
func TestASearchStillRunningAfterARestartIsNotUrgent(t *testing.T) {
|
||||
if searchQuietFor != rootsearch.Quiet {
|
||||
t.Fatalf("the quiet is %s and the node-engine's %s: they are one value", searchQuietFor, rootsearch.Quiet)
|
||||
}
|
||||
// A daily search that finishes within the quiet never leaves the account unjudged between two of them.
|
||||
if rootsearch.FreshFor < rootsearch.Every+rootsearch.Quiet {
|
||||
t.Fatalf("a complete search judges for %s, less than a day's search (%s) and the quiet (%s)",
|
||||
rootsearch.FreshFor, rootsearch.Every, rootsearch.Quiet)
|
||||
if searchQuietFor != rootsearch.Bound {
|
||||
t.Fatalf("the quiet is %s and the node-engine's bound %s: they are one value", searchQuietFor, rootsearch.Bound)
|
||||
}
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
@@ -252,7 +246,7 @@ func TestASearchStillRunningAfterARestartIsNotUrgent(t *testing.T) {
|
||||
if found := say(link.StateUnknown, running); len(found) != 0 {
|
||||
t.Fatalf("a search first seen now was raised: %+v", found)
|
||||
}
|
||||
if _, confined, why, _ := agentConfined(ctx, inv, "anchor", time.Now()); confined || !strings.Contains(why, "not judged") {
|
||||
if _, confined, why, _ := agentConfined(ctx, inv, "anchor"); confined || !strings.Contains(why, "not judged") {
|
||||
t.Fatalf("an account whose search runs was read as confined: %q", why)
|
||||
}
|
||||
// The engine restarted again and again, each statement's own since fresh: the controller's clock runs on.
|
||||
|
||||
@@ -1,946 +0,0 @@
|
||||
package main
|
||||
|
||||
// The controller asks, and acts on the operator's warrant (novox/hq ADR 0259 §6). It holds no channel, no
|
||||
// identity and no factor: it asks the router like any other module, and performs the answer chosen with its
|
||||
// own grant.
|
||||
//
|
||||
// - **For every open, unsilenced condition that needs the operator and names its answers**, one ask is
|
||||
// published on the `operator-channel` seat under the controller's own name: the condition's words, its
|
||||
// actions as options at their levels (Silence acknowledges; Release, Stop, Start and Restart approve),
|
||||
// answered by the operator, expiring after a day (a week when every option only acknowledges). A
|
||||
// condition that clears, is silenced, or changes its answers has its ask cancelled; an ask that expired
|
||||
// unanswered is asked again while the condition lasts; one the router refused is asked again after a wait
|
||||
// that grows with each refusal in a row, at most half an hour, so a refusal is never believed longer. Each ask is kept in the controller's bucket
|
||||
// `asked`, so a restart neither asks twice nor forgets.
|
||||
// - **On a warrant**, heard on the seat's event under the controller's own name (which only the router may
|
||||
// say), the controller acts once per ask: only for an ask it holds, only for the option it offered at
|
||||
// that option's level, and only while the condition is still open. It performs the action as itself —
|
||||
// a silence through its own conditions, any other through the verb the action names — with the warrant's
|
||||
// words as its why, and records it in the hand-act log as the operator's decision, naming the channel,
|
||||
// the ask and the proofs. An ask that ended without a choice is recorded and nothing is done.
|
||||
// - **A warrant it missed** while away is read from the router's record of its asks, under its own name.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The asker's name on the seat: the controller's module.
|
||||
const askerName = broker.ControllerSeat
|
||||
|
||||
// How long an ask lasts: a day when an answer approves, a week when every answer only acknowledges.
|
||||
const (
|
||||
// askApproveFor is a day less a margin, so an ask is never refused at the router for lasting a day and
|
||||
// a moment (the SDK's bound is a day).
|
||||
askApproveFor = 24*time.Hour - 10*time.Minute
|
||||
askAcknowledgeFor = 7 * 24 * time.Hour
|
||||
// askEvery is how often what is open is asked about again, beside every change.
|
||||
askEvery = time.Minute
|
||||
// askCatchUpAfter is how old an open ask is before the router's record of it is read: a warrant heard
|
||||
// on the event needs no reading.
|
||||
askCatchUpAfter = 2 * time.Minute
|
||||
// askAgainAfterAnswer is how long a condition the operator answered is not asked about again with the
|
||||
// same answers: what was chosen takes a while to clear it, and asking again at once would ask twice.
|
||||
askAgainAfterAnswer = time.Hour
|
||||
// askMostOpen is how many asks the controller holds open at once (the router refuses a fourth): the
|
||||
// most urgent conditions first, then the oldest.
|
||||
askMostOpen = asks.MostOpen
|
||||
// askRefusedRetryMost is the longest a refusal is believed without asking again (novox/hq issue 369): the
|
||||
// router refused while its channels had not yet said they could send, the channels could send twenty
|
||||
// minutes later, and the controller repeated that refusal for eleven hours. A refused ask is asked again
|
||||
// after askEvery, then twice as long after each refusal in a row, never longer than this — and at once when
|
||||
// the channels change.
|
||||
askRefusedRetryMost = 30 * time.Minute
|
||||
// askRefusedCounted is how far back refusals in a row are counted for that wait.
|
||||
askRefusedCounted = 6 * time.Hour
|
||||
// askVerdictWait is how long an ask made again after a refusal waits for the router's word before it is
|
||||
// taken as taken: the router refuses an ask as it reads it, so a refusal comes within seconds. Meanwhile
|
||||
// the condition saying it was not delivered stands as it was, neither cleared nor raised again.
|
||||
askVerdictWait = 2 * time.Minute
|
||||
)
|
||||
|
||||
// refusedRetryAfter is how long a part refused n times in a row waits before it is asked again.
|
||||
func refusedRetryAfter(n int) time.Duration {
|
||||
wait := askEvery
|
||||
for i := 1; i < n && wait < askRefusedRetryMost; i++ {
|
||||
wait *= 2
|
||||
}
|
||||
return min(wait, askRefusedRetryMost)
|
||||
}
|
||||
|
||||
// What became of an ask, as the controller keeps it.
|
||||
const (
|
||||
askOpen = "open"
|
||||
askCancelled = "cancelled"
|
||||
)
|
||||
|
||||
// asked is one ask the controller made, as it keeps it.
|
||||
type asked struct {
|
||||
ID string `json:"id"`
|
||||
Condition string `json:"condition"`
|
||||
// Channels is what the channels were when it was asked (asker.channels): an ask the router refused is asked
|
||||
// again at once when the condition's answers or the channels change, and otherwise after a wait that grows
|
||||
// with each refusal in a row (refusedRetryAfter, novox/hq issue 369).
|
||||
Channels string `json:"channels,omitempty"`
|
||||
Ask asks.Ask `json:"ask"`
|
||||
Actions []conditions.Action `json:"actions"`
|
||||
// Options are the actions by option id.
|
||||
Options map[string]int `json:"options"`
|
||||
State string `json:"state"`
|
||||
Opened time.Time `json:"opened"`
|
||||
Ended time.Time `json:"ended,omitempty"`
|
||||
Warrant *asks.Warrant `json:"warrant,omitempty"`
|
||||
// Acted is what the controller did on the warrant: empty before it did anything, "acting" while it acts,
|
||||
// then "done", "failed: …" or "nothing: …". Anything but empty is never acted on again.
|
||||
Acted string `json:"acted,omitempty"`
|
||||
// Part is which ask of its condition this is (askPart): empty for the one that carries the condition's
|
||||
// answers, or the authorising ones where it has both; "acknowledge" for its acknowledging answers asked
|
||||
// apart (the review of 2026-10-09, M1).
|
||||
Part string `json:"part,omitempty"`
|
||||
// Rehearsal is an ask started at the controller's terminal (rehearse.go): about no condition, its answers
|
||||
// perform nothing, and the reconciling of conditions leaves it alone.
|
||||
Rehearsal bool `json:"rehearsal,omitempty"`
|
||||
// Proposal is a settings layer proposed through a verb (proposals.go, novox/hq ADR 0277): about no
|
||||
// condition, set on Approve by the serving controller itself, and left alone by the reconciling of conditions.
|
||||
Proposal *settingsProposal `json:"proposal,omitempty"`
|
||||
}
|
||||
|
||||
// ofACondition says an ask is one of a condition's: not a rehearsal, not a proposal.
|
||||
func (r asked) ofACondition() bool { return !r.Rehearsal && r.Proposal == nil }
|
||||
|
||||
// partKey is an ask's place among what is asked: its condition and its part.
|
||||
func partKey(condition, part string) string { return condition + "#" + part }
|
||||
|
||||
// partAcknowledge is the part of a condition asked apart for its acknowledging answers.
|
||||
const partAcknowledge = "acknowledge"
|
||||
|
||||
// askPart is one ask a condition is asked with: its part, what it is about, and its answers.
|
||||
type askPart struct {
|
||||
name string
|
||||
about string
|
||||
actions []conditions.Action
|
||||
}
|
||||
|
||||
// levelOf is an action's level as an option offers it: one that says none is never taken for less than
|
||||
// approve.
|
||||
func levelOf(act conditions.Action) asks.Level {
|
||||
if act.Level == "" {
|
||||
return asks.Approve
|
||||
}
|
||||
return asks.Level(act.Level)
|
||||
}
|
||||
|
||||
// partsOf is the asks a condition is asked with (the review of 2026-10-09, M1): one, when its answers are all
|
||||
// of one kind; else its authorising answers (Release, Stop, Restart) in one ask, about the condition, and its
|
||||
// acknowledging ones (Silence) in another. **An acknowledgement never shares an ask with an approval**: a
|
||||
// channel that only acknowledges would otherwise answer the ask, and end the approval with it.
|
||||
func partsOf(c conditions.Condition) []askPart {
|
||||
var ack, auth []conditions.Action
|
||||
for _, act := range c.Actions {
|
||||
if levelOf(act) == asks.Acknowledge {
|
||||
ack = append(ack, act)
|
||||
} else {
|
||||
auth = append(auth, act)
|
||||
}
|
||||
}
|
||||
if len(ack) == 0 || len(auth) == 0 {
|
||||
return []askPart{{about: c.Key, actions: c.Actions}}
|
||||
}
|
||||
return []askPart{{about: c.Key, actions: auth},
|
||||
{name: partAcknowledge, about: c.Key + "." + partAcknowledge, actions: ack}}
|
||||
}
|
||||
|
||||
// askedStore keeps the asks (broker.AskedBucket). **Every write after the first is a compare-and-set** (the
|
||||
// review of 2026-10-09, L2): an ask is created once, and changed only over the revision it was read at, the
|
||||
// change decided again on what is read — so two controllers, or two deliveries of one warrant, never write
|
||||
// over each other, and of two that would act only the one whose write stands does.
|
||||
type askedStore interface {
|
||||
Get(ctx context.Context, id string) (*asked, error)
|
||||
// Create keeps a new ask, and refuses one already kept under its id.
|
||||
Create(ctx context.Context, a asked) error
|
||||
// Change applies change to the ask kept under id, by compare-and-set, and says whether its write stood.
|
||||
// change says whether to write at all; on a write that came between, it is asked again on what is read.
|
||||
Change(ctx context.Context, id string, change func(*asked) bool) (bool, error)
|
||||
All(ctx context.Context) ([]asked, error)
|
||||
}
|
||||
|
||||
// askChangeTries is how often a change is read and tried again when another write came between.
|
||||
const askChangeTries = 5
|
||||
|
||||
// asker is the controller asking the operator and acting on the answer.
|
||||
type asker struct {
|
||||
open func(ctx context.Context) ([]conditions.Condition, error)
|
||||
silence func(ctx context.Context, key string, d time.Duration, by, why string) error
|
||||
store askedStore
|
||||
// publish puts a message on a subject's stream, de-duplicated by id.
|
||||
publish func(ctx context.Context, subject string, body []byte, id string) error
|
||||
// call performs an action's verb with its arguments, as the controller.
|
||||
call func(ctx context.Context, a conditions.Action, args map[string]string) error
|
||||
// setLayer sets a proposed settings layer on the operator's warrant (novox/hq ADR 0277); nil cannot.
|
||||
setLayer setOnWarrant
|
||||
// record writes the hand-act log.
|
||||
record func(ctx context.Context, act link.HandAct) error
|
||||
// routerRecord reads the router's record of an ask for a warrant missed; nil reads nothing.
|
||||
routerRecord func(ctx context.Context, id string) (*asks.Warrant, error)
|
||||
// routerHere says whether a router holds the seat and takes asks under the asker's name; nil is yes.
|
||||
routerHere func(ctx context.Context) (bool, error)
|
||||
// grantHeld says whether the bus holds the controller's grant to ask: the user list the bus's machine was
|
||||
// last sent is the one the mesh composes now (novox/hq issue 353). The grant is composed from the router's
|
||||
// assignment and reaches the bus only when that machine is next pushed, so between `assign` and `push`
|
||||
// the record says a router is here and the bus refuses every ask. why says what to do; nil is yes.
|
||||
grantHeld func(ctx context.Context) (held bool, why string, err error)
|
||||
// channels is what the channels are now, as a fingerprint: who holds which kind, promising what.
|
||||
channels func(ctx context.Context) string
|
||||
// raise keeps the asker's own condition (sourceAsker): which conditions needing the operator could not be
|
||||
// asked, and why. Nil raises nothing (a test that does not look).
|
||||
raise func(ctx context.Context, obs []conditions.Observation) error
|
||||
now func() time.Time
|
||||
logf func(string, ...any)
|
||||
|
||||
saidNoRouter bool
|
||||
saidNoGrant bool
|
||||
// saidGrantCaveat is whether a grant held with a caveat was said, so it is said once.
|
||||
saidGrantCaveat bool
|
||||
|
||||
mu sync.Mutex
|
||||
nudged chan struct{}
|
||||
}
|
||||
|
||||
func (a *asker) nudge() {
|
||||
if a == nil {
|
||||
return
|
||||
}
|
||||
a.mu.Lock()
|
||||
if a.nudged == nil {
|
||||
a.nudged = make(chan struct{}, 1)
|
||||
}
|
||||
ch := a.nudged
|
||||
a.mu.Unlock()
|
||||
select {
|
||||
case ch <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// keep asks until ctx ends: now, on every change of a condition, and every askEvery.
|
||||
func (a *asker) keep(ctx context.Context) {
|
||||
a.nudge()
|
||||
tick := time.NewTicker(askEvery)
|
||||
defer tick.Stop()
|
||||
a.mu.Lock()
|
||||
nudged := a.nudged
|
||||
a.mu.Unlock()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
case <-nudged:
|
||||
}
|
||||
if err := a.reconcile(ctx); err != nil {
|
||||
a.logf("what the operator is asked could not be brought up to date: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// wants says whether a condition is one to ask about now.
|
||||
func wants(c conditions.Condition, now time.Time) bool {
|
||||
return len(c.Actions) > 0 && c.Needs != "" && !c.SilencedAt(now)
|
||||
}
|
||||
|
||||
func sameAsked(a []conditions.Action, b []conditions.Action) bool {
|
||||
x, _ := json.Marshal(a)
|
||||
y, _ := json.Marshal(b)
|
||||
return string(x) == string(y)
|
||||
}
|
||||
|
||||
// reconcile brings what is asked in line with what is open.
|
||||
func (a *asker) reconcile(ctx context.Context) error {
|
||||
now := a.now()
|
||||
if a.routerHere != nil {
|
||||
here, err := a.routerHere(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !here {
|
||||
if !a.saidNoRouter {
|
||||
a.logf("no router takes asks under the controller's name (a module declaring %s with its ask "+
|
||||
"named by its caller, assigned): the operator is asked nothing until one is", broker.AsksSeat)
|
||||
a.saidNoRouter = true
|
||||
}
|
||||
open, err := a.open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var unasked []conditions.Condition
|
||||
for _, c := range open {
|
||||
if wants(c, now) {
|
||||
unasked = append(unasked, c)
|
||||
}
|
||||
}
|
||||
return a.sayUnasked(ctx, unasked, "no router takes the controller's asks: no module holding "+
|
||||
broker.AsksSeat+" that takes an ask under its asker's name is assigned")
|
||||
}
|
||||
a.saidNoRouter = false
|
||||
}
|
||||
if a.grantHeld != nil {
|
||||
held, why, err := a.grantHeld(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !held {
|
||||
if !a.saidNoGrant {
|
||||
a.logf("the bus does not hold the controller's grant to ask yet: %s; the operator is asked nothing until it does", why)
|
||||
a.saidNoGrant = true
|
||||
}
|
||||
open, err := a.open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var unasked []conditions.Condition
|
||||
for _, c := range open {
|
||||
if wants(c, now) {
|
||||
unasked = append(unasked, c)
|
||||
}
|
||||
}
|
||||
return a.sayUnasked(ctx, unasked, "the bus does not hold the controller's grant to ask yet: "+why)
|
||||
}
|
||||
if why != "" && !a.saidGrantCaveat {
|
||||
// Held, with a caveat worth one line (an older bus module cannot confirm the reload).
|
||||
a.logf("the controller asks: %s", why)
|
||||
a.saidGrantCaveat = true
|
||||
} else if why == "" {
|
||||
a.saidGrantCaveat = false
|
||||
}
|
||||
a.saidNoGrant = false
|
||||
}
|
||||
channels := ""
|
||||
if a.channels != nil {
|
||||
channels = a.channels(ctx)
|
||||
}
|
||||
open, err := a.open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
all, err := a.store.All(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
byCondition := map[string]asked{} // by partKey
|
||||
// What is open and not a condition's — a rehearsal, a proposal — still counts toward what the router holds
|
||||
// open for the controller (askMostOpen); expired unanswered, it is kept so, as the router says it too.
|
||||
otherOpen := 0
|
||||
for _, r := range all {
|
||||
if r.State == askOpen && !r.ofACondition() && !now.Before(r.Ask.Expires) {
|
||||
if _, err := a.store.Change(ctx, r.ID, func(x *asked) bool {
|
||||
if x.State != askOpen || x.Acted != "" {
|
||||
return false
|
||||
}
|
||||
x.State, x.Ended, x.Acted = string(asks.OutcomeExpired), now, "nothing: the ask expired unanswered"
|
||||
return true
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if r.State == askOpen && !r.ofACondition() {
|
||||
otherOpen++
|
||||
}
|
||||
if r.State == askOpen && r.ofACondition() {
|
||||
k := partKey(r.Condition, r.Part)
|
||||
if prior, held := byCondition[k]; !held || r.Opened.After(prior.Opened) {
|
||||
byCondition[k] = r
|
||||
}
|
||||
}
|
||||
}
|
||||
// A warrant missed while away, read from the router's record: for a condition's ask, and for a proposal's or a
|
||||
// rehearsal's alike.
|
||||
if a.routerRecord != nil {
|
||||
var lookedUp []asked
|
||||
for _, r := range byCondition {
|
||||
lookedUp = append(lookedUp, r)
|
||||
}
|
||||
for _, r := range all {
|
||||
if r.State == askOpen && !r.ofACondition() {
|
||||
lookedUp = append(lookedUp, r)
|
||||
}
|
||||
}
|
||||
for _, r := range lookedUp {
|
||||
if now.Sub(r.Opened) < askCatchUpAfter {
|
||||
continue
|
||||
}
|
||||
if w, err := a.routerRecord(ctx, r.ID); err == nil && w != nil {
|
||||
body, _ := json.Marshal(w)
|
||||
if err := a.Decided(ctx, body); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if all, err = a.store.All(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
byCondition = map[string]asked{}
|
||||
for _, r := range all {
|
||||
if r.State == askOpen && r.ofACondition() {
|
||||
byCondition[partKey(r.Condition, r.Part)] = r
|
||||
}
|
||||
}
|
||||
}
|
||||
// What the operator answered lately, by condition: not asked again at once; and what the router refused,
|
||||
// newest first: not asked again until the answers or the channels change.
|
||||
answered, refused := map[string]asked{}, map[string]asked{}
|
||||
for _, r := range all {
|
||||
k := partKey(r.Condition, r.Part)
|
||||
if r.State == string(asks.OutcomeChosen) && now.Sub(r.Ended) < askAgainAfterAnswer {
|
||||
answered[k] = r
|
||||
}
|
||||
if r.State == string(asks.OutcomeRefused) {
|
||||
if prior, has := refused[k]; !has || r.Opened.After(prior.Opened) {
|
||||
refused[k] = r
|
||||
}
|
||||
}
|
||||
}
|
||||
// Refusals in a row, by part: those since the last ask that was not refused, within askRefusedCounted.
|
||||
// superseded: a part asked since its last refusal, by an ask the router did not refuse — open, answered,
|
||||
// expired or cancelled. Its refusal is history then, never said again (the review of PR 198: an answered
|
||||
// retry brought the refusal back).
|
||||
inRow, superseded := map[string]int{}, map[string]asked{}
|
||||
for k, last := range refused {
|
||||
for _, r := range all {
|
||||
if partKey(r.Condition, r.Part) == k && r.State != string(asks.OutcomeRefused) && r.State != askUnsent &&
|
||||
r.Opened.After(last.Opened) && r.Opened.After(superseded[k].Opened) {
|
||||
superseded[k] = r
|
||||
}
|
||||
}
|
||||
var since time.Time
|
||||
for _, r := range all {
|
||||
if partKey(r.Condition, r.Part) == k && r.State != string(asks.OutcomeRefused) && r.State != askUnsent &&
|
||||
!r.Opened.After(last.Opened) && r.Opened.After(since) {
|
||||
since = r.Opened
|
||||
}
|
||||
}
|
||||
for _, r := range all {
|
||||
if partKey(r.Condition, r.Part) == k && r.State == string(asks.OutcomeRefused) && r.Opened.After(since) &&
|
||||
now.Sub(r.Opened) < askRefusedCounted {
|
||||
inRow[k]++
|
||||
}
|
||||
}
|
||||
}
|
||||
wanted := map[string]bool{}
|
||||
var unasked []conditions.Condition // refused by the router, and nothing it was refused for changed
|
||||
var refusedWords []string
|
||||
// The most urgent first, then the oldest: those are asked when no more than askMostOpen may be.
|
||||
sort.SliceStable(open, func(i, j int) bool {
|
||||
ui, uj := open[i].Severity == conditions.Urgent, open[j].Severity == conditions.Urgent
|
||||
if ui != uj {
|
||||
return ui
|
||||
}
|
||||
if !open[i].Raised.Equal(open[j].Raised) {
|
||||
return open[i].Raised.Before(open[j].Raised)
|
||||
}
|
||||
return open[i].Key < open[j].Key
|
||||
})
|
||||
openNow := otherOpen
|
||||
for _, c := range open {
|
||||
if !wants(c, now) {
|
||||
continue
|
||||
}
|
||||
for _, p := range partsOf(c) {
|
||||
if r, held := byCondition[partKey(c.Key, p.name)]; held && sameAsked(r.Actions, p.actions) && now.Before(r.Ask.Expires) {
|
||||
openNow++
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, c := range open {
|
||||
if !wants(c, now) {
|
||||
continue
|
||||
}
|
||||
saidUnasked := false
|
||||
for _, p := range partsOf(c) {
|
||||
key := partKey(c.Key, p.name)
|
||||
wanted[key] = true
|
||||
if r, was := refused[key]; was && sameAsked(r.Actions, p.actions) {
|
||||
cur, held := byCondition[key]
|
||||
ended := r.Ended
|
||||
if ended.IsZero() {
|
||||
ended = r.Opened
|
||||
}
|
||||
later, asked := superseded[key]
|
||||
waiting := !held && !asked && r.Channels == channels && now.Sub(ended) < refusedRetryAfter(inRow[key])
|
||||
// Asked again now (the wait over), or lately and the router's word not in yet: said as it was until
|
||||
// that word, so the condition neither clears nor is raised again at each try.
|
||||
retrying := !held && !asked && !waiting
|
||||
verdictDue := held && asked && later.ID == cur.ID && now.Sub(cur.Opened) < askVerdictWait
|
||||
if waiting || retrying || verdictDue {
|
||||
if !saidUnasked {
|
||||
unasked, saidUnasked = append(unasked, c), true
|
||||
}
|
||||
if r.Warrant != nil && r.Warrant.Words != "" {
|
||||
refusedWords = append(refusedWords, r.Warrant.Words)
|
||||
}
|
||||
}
|
||||
if waiting {
|
||||
continue // refused lately, and nothing it was refused for has changed: asked again after the wait
|
||||
}
|
||||
}
|
||||
if r, done := answered[key]; done && sameAsked(r.Actions, p.actions) {
|
||||
if _, held := byCondition[key]; !held {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if r, held := byCondition[key]; held {
|
||||
switch {
|
||||
case !sameAsked(r.Actions, p.actions):
|
||||
if err := a.cancel(ctx, r, "its answers changed"); err != nil {
|
||||
return err
|
||||
}
|
||||
case !now.Before(r.Ask.Expires):
|
||||
// Expired unanswered: the router says so too; asked again below while it lasts.
|
||||
if _, err := a.store.Change(ctx, r.ID, func(x *asked) bool {
|
||||
if x.State != askOpen {
|
||||
return false
|
||||
}
|
||||
x.State, x.Ended = string(asks.OutcomeExpired), now
|
||||
return true
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
openNow--
|
||||
default:
|
||||
continue
|
||||
}
|
||||
}
|
||||
if openNow >= askMostOpen {
|
||||
continue // asked when one of the open ones ends, most urgent first
|
||||
}
|
||||
if err := a.ask(ctx, c, p, channels); err != nil {
|
||||
a.logf("the operator could not be asked about %s: %v", c.Key, err)
|
||||
continue
|
||||
}
|
||||
openNow++
|
||||
}
|
||||
}
|
||||
stillOpen := map[string]conditions.Condition{}
|
||||
for _, c := range open {
|
||||
stillOpen[c.Key] = c
|
||||
}
|
||||
for key, r := range byCondition {
|
||||
if wanted[key] {
|
||||
continue
|
||||
}
|
||||
// **A silence never takes an approval back** (the confirmation review of 2026-10-09, M1). Silence is an
|
||||
// acknowledgement — anyone at the desk may give it — so a condition silenced while its approval is asked
|
||||
// keeps that ask open, unchanged, until it is answered on a channel that proves who answered, or expires.
|
||||
// It is not asked again once it ends, while the silence lasts.
|
||||
if c, open := stillOpen[r.Condition]; open && c.SilencedAt(now) && r.Ask.Highest() != asks.Acknowledge &&
|
||||
now.Before(r.Ask.Expires) && keepsItsAnswers(c, r) {
|
||||
continue
|
||||
}
|
||||
if err := a.cancel(ctx, r, "the condition ended, was silenced or needs nothing now"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
why := "the router refused the ask"
|
||||
if len(refusedWords) > 0 {
|
||||
why += ": " + refusedWords[0]
|
||||
}
|
||||
return a.sayUnasked(ctx, unasked, why)
|
||||
}
|
||||
|
||||
// keepsItsAnswers says a condition still offers the answers an ask kept was asked with.
|
||||
func keepsItsAnswers(c conditions.Condition, r asked) bool {
|
||||
for _, p := range partsOf(c) {
|
||||
if partKey(c.Key, p.name) == partKey(r.Condition, r.Part) {
|
||||
return sameAsked(r.Actions, p.actions)
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// sourceAsker raises the asker's own condition.
|
||||
const sourceAsker = "asker"
|
||||
|
||||
// sayUnasked keeps the asker's one condition: while a condition that needs the operator could not be asked
|
||||
// on any channel, said loudly (failure must be loud), cleared when every one could be.
|
||||
func (a *asker) sayUnasked(ctx context.Context, unasked []conditions.Condition, why string) error {
|
||||
if a.raise == nil {
|
||||
return nil
|
||||
}
|
||||
var obs []conditions.Observation
|
||||
if len(unasked) > 0 {
|
||||
keys := make([]string, 0, len(unasked))
|
||||
severity := conditions.Warning
|
||||
for _, c := range unasked {
|
||||
keys = append(keys, c.Key)
|
||||
if c.Severity == conditions.Urgent {
|
||||
severity = conditions.Urgent
|
||||
}
|
||||
}
|
||||
sort.Strings(keys)
|
||||
obs = append(obs, conditions.Observation{Scope: conditions.ScopeSeat, ID: broker.AsksSeat, Token: "unasked",
|
||||
Kind: "asks-undelivered", Severity: severity, Source: sourceAsker,
|
||||
Summary: fmt.Sprintf("%d condition(s) that need the operator could not be asked on any channel: %s; %s",
|
||||
len(keys), strings.Join(keys, ", "), why),
|
||||
Headline: "Questions for you not delivered",
|
||||
Explanation: "The mesh could not send you its questions on any channel.",
|
||||
Needs: "answer them from the mesh MCP server, and check why no channel carries them.",
|
||||
Resolved: "The mesh can ask you again"})
|
||||
}
|
||||
if err := a.raise(ctx, obs); err != nil {
|
||||
a.logf("whether the operator could be asked could not be kept as a condition: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// optionID is an action's label as an option's id: "Silence for a week" is silence-for-a-week.
|
||||
func optionID(label string) string {
|
||||
var b strings.Builder
|
||||
dash := false
|
||||
for _, r := range strings.ToLower(label) {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
|
||||
b.WriteRune(r)
|
||||
dash = false
|
||||
case !dash && b.Len() > 0:
|
||||
b.WriteByte('-')
|
||||
dash = true
|
||||
}
|
||||
}
|
||||
return strings.TrimSuffix(b.String(), "-")
|
||||
}
|
||||
|
||||
// doesWords is what an action does, in the words an option says it with.
|
||||
func doesWords(act conditions.Action) string {
|
||||
switch {
|
||||
case act.Arguments["silence"] != "":
|
||||
return "nothing more is said of it for a week"
|
||||
case act.Verb == "mesh-delivery.release":
|
||||
return "the delivery goes on"
|
||||
case act.Verb == "mesh-delivery.stop":
|
||||
return "the delivery ends"
|
||||
case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["go"] != "":
|
||||
return "the delivery starts"
|
||||
case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["stop"] != "":
|
||||
return "the delivery is stopped"
|
||||
case strings.HasSuffix(act.Verb, ".restart"):
|
||||
return "its service is restarted on " + act.Machine
|
||||
}
|
||||
return strings.ToLower(act.Label)
|
||||
}
|
||||
|
||||
// askText is a condition's words as an ask says them: without where an answer is given when no channel can
|
||||
// give it (FromMeshMCPServer), since the ask is answered on a channel and the router says where else.
|
||||
func askText(s string) string {
|
||||
for _, with := range []string{", " + FromMeshMCPServer, " " + FromMeshMCPServer} {
|
||||
s = strings.ReplaceAll(s, with, ".")
|
||||
}
|
||||
return strings.ReplaceAll(s, "..", ".")
|
||||
}
|
||||
|
||||
// askOf is the ask one part of a condition is asked with.
|
||||
func askOf(id string, c conditions.Condition, p askPart, now time.Time) (asks.Ask, map[string]int) {
|
||||
q := asks.Ask{ID: id, Headline: c.Headline, Explanation: askText(c.Explanation), Who: asks.Operator,
|
||||
OnExpiry: "nothing is done, and you are asked again while it lasts", About: p.about,
|
||||
Urgent: c.Severity == conditions.Urgent}
|
||||
options := map[string]int{}
|
||||
approves := false
|
||||
for i, act := range p.actions {
|
||||
level := levelOf(act) // an action that says nothing of its level is never taken for less than approve
|
||||
approves = approves || level != asks.Acknowledge
|
||||
oid := optionID(act.Label)
|
||||
options[oid] = i
|
||||
// Every option binds the exact act it stands for (novox/hq ADR 0259 §6): the verb, the machine and
|
||||
// every argument. The warrant then authorises that act and no other.
|
||||
binds, _ := asks.ActDigest(boundAct(act))
|
||||
q.Options = append(q.Options, asks.Option{ID: oid, Label: act.Label, Does: doesWords(act), Level: level,
|
||||
Binds: binds})
|
||||
}
|
||||
q.Expires = now.Add(askAcknowledgeFor)
|
||||
if approves {
|
||||
q.Expires = now.Add(askApproveFor)
|
||||
}
|
||||
return q, options
|
||||
}
|
||||
|
||||
// boundAct is what an option's Binds digests: the act exactly as the controller will perform it — its verb,
|
||||
// machine, level, and each argument as "arg.<name>" — and never its label or words.
|
||||
func boundAct(act conditions.Action) asks.Act {
|
||||
out := asks.Act{"verb": act.Verb, "machine": act.Machine, "level": act.Level}
|
||||
for k, v := range act.Arguments {
|
||||
out["arg."+k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func newAskID() string {
|
||||
var b [8]byte
|
||||
_, _ = rand.Read(b[:])
|
||||
return "c" + hex.EncodeToString(b[:])
|
||||
}
|
||||
|
||||
// askUnsent is an ask kept and never published: asked again at the next look.
|
||||
const askUnsent = "unsent"
|
||||
|
||||
// ask publishes one ask about a part of a condition, kept before it is published (the review of 2026-10-09,
|
||||
// L3): a warrant for it then always finds it, and one whose publishing failed is marked so and asked again.
|
||||
func (a *asker) ask(ctx context.Context, c conditions.Condition, p askPart, channels string) error {
|
||||
now := a.now()
|
||||
id := newAskID()
|
||||
q, options := askOf(id, c, p, now)
|
||||
if err := q.Check(now); err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := json.Marshal(q)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := a.store.Create(ctx, asked{ID: id, Condition: c.Key, Part: p.name, Ask: q, Actions: p.actions,
|
||||
Options: options, State: askOpen, Opened: now, Channels: channels}); err != nil {
|
||||
return fmt.Errorf("the ask could not be kept, so it was not asked: %w", err)
|
||||
}
|
||||
if err := a.publish(ctx, asks.AskSubject(askerName), body, "ask."+id); err != nil {
|
||||
if _, cerr := a.store.Change(ctx, id, func(x *asked) bool {
|
||||
if x.State != askOpen || x.Acted != "" {
|
||||
return false
|
||||
}
|
||||
x.State, x.Ended, x.Acted = askUnsent, a.now(), "nothing: it could not be published: "+err.Error()
|
||||
return true
|
||||
}); cerr != nil {
|
||||
a.logf("the ask %s could not be published, and could not be marked so: %v", id, cerr)
|
||||
}
|
||||
return err
|
||||
}
|
||||
a.logf("asked the operator about %s (%s): %d answer(s)", c.Key, id, len(q.Options))
|
||||
return nil
|
||||
}
|
||||
|
||||
// cancel takes an ask back: kept cancelled first, so a warrant that comes after is refused, then said to the
|
||||
// router; a cancel the router did not hear leaves the ask to expire there, and nothing is done on it here.
|
||||
func (a *asker) cancel(ctx context.Context, r asked, why string) error {
|
||||
stood, err := a.store.Change(ctx, r.ID, func(x *asked) bool {
|
||||
if x.State != askOpen {
|
||||
return false
|
||||
}
|
||||
x.State, x.Ended = askCancelled, a.now()
|
||||
return true
|
||||
})
|
||||
if err != nil || !stood {
|
||||
return err
|
||||
}
|
||||
body, _ := json.Marshal(map[string]string{"id": r.ID})
|
||||
if err := a.publish(ctx, asks.CancelSubject(askerName), body, "cancel."+r.ID); err != nil {
|
||||
a.logf("the ask %s about %s is taken back here, and the router could not be told (%v): it expires there, "+
|
||||
"and no answer to it is acted on", r.ID, r.Condition, err)
|
||||
return nil
|
||||
}
|
||||
a.logf("took back the ask %s about %s: %s", r.ID, r.Condition, why)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Decided takes the router's word on one of the controller's asks (link.Decider). An error is returned only
|
||||
// when what was decided could not be kept, so the word is held and heard again.
|
||||
func (a *asker) Decided(ctx context.Context, body []byte) error {
|
||||
var w asks.Warrant
|
||||
if err := json.Unmarshal(body, &w); err != nil {
|
||||
a.logf("the router's word on an ask could not be read; ignored: %v", err)
|
||||
return nil
|
||||
}
|
||||
if w.Asker != askerName {
|
||||
a.logf("REFUSED a warrant for %s's ask %s: the controller acts only on its own", w.Asker, w.Ask)
|
||||
return nil
|
||||
}
|
||||
r, err := a.store.Get(ctx, w.Ask)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if r == nil {
|
||||
a.logf("REFUSED a warrant for the ask %s, which the controller does not hold", w.Ask)
|
||||
return nil
|
||||
}
|
||||
if r.Acted != "" {
|
||||
return nil // heard again: acted on once
|
||||
}
|
||||
now := a.now()
|
||||
if w.Outcome != asks.OutcomeChosen {
|
||||
acted := "nothing: the ask " + string(w.Outcome)
|
||||
if w.Words != "" {
|
||||
acted += ": " + w.Words
|
||||
}
|
||||
if _, err := a.store.Change(ctx, r.ID, func(x *asked) bool {
|
||||
if x.Acted != "" {
|
||||
return false
|
||||
}
|
||||
x.State, x.Ended, x.Warrant, x.Acted = string(w.Outcome), now, &w, acted
|
||||
return true
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
a.logf("the ask %s about %s ended %s; nothing is done", r.ID, r.Condition, w.Outcome)
|
||||
return nil
|
||||
}
|
||||
if r.State != askOpen {
|
||||
// Cancelled, replaced or expired in the controller's own record: no answer to it is acted on.
|
||||
a.logf("REFUSED a warrant for the ask %s, which is %s in the controller's own record", r.ID, r.State)
|
||||
return nil
|
||||
}
|
||||
option, err := w.For(askerName, r.Ask)
|
||||
if err != nil {
|
||||
a.logf("REFUSED a warrant for the ask %s: %v", r.ID, err)
|
||||
return nil
|
||||
}
|
||||
index, offered := r.Options[option.ID]
|
||||
if !offered || index >= len(r.Actions) {
|
||||
a.logf("REFUSED a warrant for the ask %s: it chose %s, which no action stands for", r.ID, option.ID)
|
||||
return nil
|
||||
}
|
||||
act := r.Actions[index]
|
||||
// The act about to be performed is the one the option bound when the controller asked: a record changed
|
||||
// since is refused, never performed.
|
||||
if err := option.Performs(boundAct(act)); err != nil {
|
||||
a.logf("REFUSED a warrant for the ask %s: %v", r.ID, err)
|
||||
return nil
|
||||
}
|
||||
open, err := a.open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stillOpen := r.Rehearsal || r.Proposal != nil // a rehearsal and a proposal are about no condition
|
||||
for _, c := range open {
|
||||
stillOpen = stillOpen || c.Key == r.Condition
|
||||
}
|
||||
if !stillOpen {
|
||||
// The asker checks the state is still what it asked about before it acts (to-be 46 §10, step 7).
|
||||
if _, err := a.store.Change(ctx, r.ID, func(x *asked) bool {
|
||||
if x.State != askOpen || x.Acted != "" {
|
||||
return false
|
||||
}
|
||||
x.State, x.Warrant, x.Ended, x.Acted = string(asks.OutcomeChosen), &w, now,
|
||||
"nothing: the condition ended before the answer"
|
||||
return true
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
a.logf("%s, for %s, which ended meanwhile: nothing is done", w.Says(), r.Condition)
|
||||
return nil
|
||||
}
|
||||
// Claimed before acting, by compare-and-set: only the delivery whose write stands acts (security review
|
||||
// of 2026-10-08, finding 9). Not by the warrant's message id, which another publisher could take first:
|
||||
// the controller's own record decides.
|
||||
claimed, err := a.store.Change(ctx, r.ID, func(x *asked) bool {
|
||||
if x.State != askOpen || x.Acted != "" {
|
||||
return false
|
||||
}
|
||||
x.State, x.Warrant, x.Acted = string(asks.OutcomeChosen), &w, "acting"
|
||||
return true
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !claimed {
|
||||
a.logf("the warrant for the ask %s was already taken by another delivery; nothing more is done", r.ID)
|
||||
return nil
|
||||
}
|
||||
r.Acted = "acting"
|
||||
|
||||
why := fmt.Sprintf("%s (ask %s)", w.Says(), r.ID)
|
||||
args := map[string]string{}
|
||||
for k, v := range act.Arguments {
|
||||
args[k] = v
|
||||
}
|
||||
if v, takes := args["why"]; takes && v == "" {
|
||||
args["why"] = why
|
||||
}
|
||||
var acted error
|
||||
outcome := "done"
|
||||
switch {
|
||||
case r.Rehearsal && act.Verb == rehearsalVerb:
|
||||
// A rehearsal's answer performs nothing: it is recorded below as the operator's decision.
|
||||
case r.Proposal != nil && act.Verb == proposalVerb:
|
||||
// A proposed settings layer, set by this controller itself on Approve (novox/hq ADR 0277): the act's
|
||||
// digest of the values is held to the record's own values before anything is set.
|
||||
outcome, acted = a.decideProposal(ctx, *r, act, w)
|
||||
if acted == nil && outcome != "" && !strings.HasPrefix(outcome, "nothing") {
|
||||
outcome = "done: " + outcome
|
||||
}
|
||||
case act.Arguments["silence"] != "":
|
||||
acted = a.silence(ctx, act.Arguments["silence"], conditions.MaxSilence, byWords(w), why)
|
||||
default:
|
||||
acted = a.call(ctx, act, args)
|
||||
}
|
||||
ended := a.now()
|
||||
if acted != nil {
|
||||
outcome = "failed: " + acted.Error()
|
||||
}
|
||||
r.Ended, r.Acted = ended, outcome
|
||||
if _, err := a.store.Change(ctx, r.ID, func(x *asked) bool {
|
||||
if x.Acted != "acting" {
|
||||
return false
|
||||
}
|
||||
x.Ended, x.Acted = ended, outcome
|
||||
return true
|
||||
}); err != nil {
|
||||
a.logf("%s was acted on (%s), and how it ended could NOT be kept: %v", r.ID, outcome, err)
|
||||
}
|
||||
verbArgs := []string{act.Verb}
|
||||
if act.Machine != "" {
|
||||
verbArgs = append(verbArgs, "on "+act.Machine)
|
||||
}
|
||||
keys := make([]string, 0, len(args))
|
||||
for k := range args {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
if k != "why" {
|
||||
verbArgs = append(verbArgs, k+"="+args[k])
|
||||
}
|
||||
}
|
||||
if err := a.record(ctx, link.HandAct{Verb: handActWarrant, Args: verbArgs, Why: why, By: byWords(w),
|
||||
Cause: conditions.CauseOperatorAnswer, Condition: r.Condition, Via: viaWords(w), Ask: r.ID,
|
||||
Proofs: w.Proofs, RequestedBy: r.Condition, Outcome: r.Acted}); err != nil {
|
||||
a.logf("%s was done, and could NOT be recorded in the hand-act log: %v", why, err)
|
||||
}
|
||||
a.logf("%s: %s", why, r.Acted)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handActWarrant is the verb an act the operator chose on a warrant is recorded under: a person's decision,
|
||||
// never a repair (handActVerbs).
|
||||
const handActWarrant = "warrant"
|
||||
|
||||
// byWords is who chose, as the hand-act log says it: "the operator, as telegram identity 42".
|
||||
func byWords(w asks.Warrant) string {
|
||||
if w.By == nil {
|
||||
return "the operator"
|
||||
}
|
||||
return fmt.Sprintf("the %s, as %s identity %s", w.By.Who, w.By.Kind, w.By.Identity)
|
||||
}
|
||||
|
||||
// viaWords is the channel an answer came through: its module and kind, and how the sender was known.
|
||||
func viaWords(w asks.Warrant) string {
|
||||
if w.By == nil {
|
||||
return w.Channel
|
||||
}
|
||||
via := w.Channel + " (" + w.By.Kind + ")"
|
||||
if w.By.Verified != "" {
|
||||
via += ", " + w.By.Verified
|
||||
}
|
||||
return via
|
||||
}
|
||||
|
||||
// errNotGranted is an action whose verb the controller's grant does not name.
|
||||
var errNotGranted = errors.New("the controller's grant does not name this verb")
|
||||
@@ -1,151 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// busAsker is an asker on a real bus's `asked` bucket, counting what it performs: two of them are two
|
||||
// controllers sharing one record.
|
||||
type busAskerRig struct {
|
||||
mu sync.Mutex
|
||||
called int
|
||||
acts int
|
||||
open []conditions.Condition
|
||||
sent [][]byte
|
||||
}
|
||||
|
||||
func (rig *busAskerRig) asker(t *testing.T, conn *nats.Conn, now time.Time) *asker {
|
||||
return &asker{
|
||||
open: func(context.Context) ([]conditions.Condition, error) {
|
||||
rig.mu.Lock()
|
||||
defer rig.mu.Unlock()
|
||||
return rig.open, nil
|
||||
},
|
||||
silence: func(context.Context, string, time.Duration, string, string) error { return nil },
|
||||
store: busAsked{conn: conn},
|
||||
publish: func(_ context.Context, subject string, body []byte, _ string) error {
|
||||
rig.mu.Lock()
|
||||
defer rig.mu.Unlock()
|
||||
if subject == asks.AskSubject(askerName) {
|
||||
rig.sent = append(rig.sent, body)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
call: func(context.Context, conditions.Action, map[string]string) error {
|
||||
time.Sleep(20 * time.Millisecond) // long enough for the other delivery to arrive meanwhile
|
||||
rig.mu.Lock()
|
||||
defer rig.mu.Unlock()
|
||||
rig.called++
|
||||
return nil
|
||||
},
|
||||
record: func(context.Context, link.HandAct) error {
|
||||
rig.mu.Lock()
|
||||
defer rig.mu.Unlock()
|
||||
rig.acts++
|
||||
return nil
|
||||
},
|
||||
now: func() time.Time { return now },
|
||||
logf: t.Logf,
|
||||
}
|
||||
}
|
||||
|
||||
func askedBus(t *testing.T) *nats.Conn {
|
||||
t.Helper()
|
||||
conn, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(conn.Close)
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.CreateKeyValue(context.Background(), jetstream.KeyValueConfig{Bucket: broker.AskedBucket}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return conn
|
||||
}
|
||||
|
||||
// The review of 2026-10-09 (L7): two deliveries of one warrant, to two controllers at once, perform its act
|
||||
// exactly once and record it once — the record's compare-and-set decides, never the warrant's message id.
|
||||
func TestTwoAnswersAtOnceActOnce(t *testing.T) {
|
||||
conn := askedBus(t)
|
||||
now := time.Date(2026, 10, 9, 14, 0, 0, 0, time.UTC)
|
||||
rig := &busAskerRig{open: []conditions.Condition{heldCondition()}}
|
||||
first, second := rig.asker(t, conn, now), rig.asker(t, conn, now)
|
||||
if err := first.reconcile(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rig.sent) != 1 {
|
||||
t.Fatalf("asked %d times", len(rig.sent))
|
||||
}
|
||||
var q asks.Ask
|
||||
_ = json.Unmarshal(rig.sent[0], &q)
|
||||
release, _ := q.Option("release")
|
||||
w := asks.Warrant{Ask: q.ID, Asker: askerName, About: q.About, Outcome: asks.OutcomeChosen, Option: release.ID,
|
||||
Label: release.Label, Level: release.Level, Channel: "telegram", Proofs: []string{"P1"}, At: now,
|
||||
AskDigest: q.Digest(), By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}}
|
||||
body, _ := json.Marshal(w)
|
||||
var wg sync.WaitGroup
|
||||
for _, a := range []*asker{first, second, first, second} {
|
||||
wg.Add(1)
|
||||
go func(a *asker) {
|
||||
defer wg.Done()
|
||||
if err := a.Decided(context.Background(), body); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}(a)
|
||||
}
|
||||
wg.Wait()
|
||||
if rig.called != 1 || rig.acts != 1 {
|
||||
t.Fatalf("performed %d time(s), recorded %d time(s)", rig.called, rig.acts)
|
||||
}
|
||||
got, err := busAsked{conn: conn}.Get(context.Background(), q.ID)
|
||||
if err != nil || got == nil || got.Acted != "done" {
|
||||
t.Fatalf("kept as %+v (%v)", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The review of 2026-10-09 (L2): a write decided on a record read earlier never lands over one made since. A
|
||||
// cancel read before the answer was acted on leaves the act's record as it is.
|
||||
func TestAStaleCancelDoesNotWriteOverAnAct(t *testing.T) {
|
||||
conn := askedBus(t)
|
||||
now := time.Date(2026, 10, 9, 14, 0, 0, 0, time.UTC)
|
||||
rig := &busAskerRig{open: []conditions.Condition{heldCondition()}}
|
||||
a := rig.asker(t, conn, now)
|
||||
if err := a.reconcile(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var q asks.Ask
|
||||
_ = json.Unmarshal(rig.sent[0], &q)
|
||||
stale, _ := busAsked{conn: conn}.Get(context.Background(), q.ID)
|
||||
release, _ := q.Option("release")
|
||||
w := asks.Warrant{Ask: q.ID, Asker: askerName, About: q.About, Outcome: asks.OutcomeChosen, Option: release.ID,
|
||||
Label: release.Label, Level: release.Level, Channel: "telegram", At: now, AskDigest: q.Digest(),
|
||||
By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}}
|
||||
body, _ := json.Marshal(w)
|
||||
if err := a.Decided(context.Background(), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := a.cancel(context.Background(), *stale, "the condition ended"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ := busAsked{conn: conn}.Get(context.Background(), q.ID)
|
||||
if got.State != string(asks.OutcomeChosen) || got.Acted != "done" {
|
||||
t.Errorf("a stale cancel wrote over the act: %+v", got)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,589 +0,0 @@
|
||||
package main
|
||||
|
||||
// The asker on the bus: its asks in the controller's bucket `asked`, its asks and cancels published on the
|
||||
// seat under the controller's name, the verbs a warrant chooses called with the controller's grant, and the
|
||||
// router's record of its asks read under its name (novox/hq ADR 0259).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// askerFrom is the serving controller's asker; nil in any other process.
|
||||
var askerFrom *asker
|
||||
|
||||
// askWithin is how long a verb a warrant chose is given to answer.
|
||||
const askWithin = time.Minute
|
||||
|
||||
type busAsked struct{ conn *nats.Conn }
|
||||
|
||||
func (b busAsked) kv(ctx context.Context) (jetstream.KeyValue, error) {
|
||||
js, err := jetstream.New(b.conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return js.KeyValue(ctx, broker.AskedBucket)
|
||||
}
|
||||
|
||||
func (b busAsked) Get(ctx context.Context, id string) (*asked, error) {
|
||||
kv, err := b.kv(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e, err := kv.Get(ctx, id)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var r asked
|
||||
return &r, json.Unmarshal(e.Value(), &r)
|
||||
}
|
||||
|
||||
// Create keeps a new ask under its id, and only where none is kept: never over another.
|
||||
func (b busAsked) Create(ctx context.Context, r asked) error {
|
||||
kv, err := b.kv(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = kv.Create(ctx, r.ID, body)
|
||||
return err
|
||||
}
|
||||
|
||||
// Change applies change to the ask kept under id by compare-and-set on its key's revision (the review of
|
||||
// 2026-10-09, L2): read, changed, and written only over the revision read; when another write came between,
|
||||
// read again and asked again, at most askChangeTries times. change says whether to write at all.
|
||||
func (b busAsked) Change(ctx context.Context, id string, change func(*asked) bool) (bool, error) {
|
||||
kv, err := b.kv(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for try := 0; try < askChangeTries; try++ {
|
||||
e, err := kv.Get(ctx, id)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
var r asked
|
||||
if err := json.Unmarshal(e.Value(), &r); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !change(&r) {
|
||||
return false, nil
|
||||
}
|
||||
body, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if _, err := kv.Update(ctx, id, body, e.Revision()); err != nil {
|
||||
var api *jetstream.APIError
|
||||
if errors.Is(err, jetstream.ErrKeyExists) || (errors.As(err, &api) && api.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence) {
|
||||
continue
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
return false, fmt.Errorf("the ask %s changed under every one of %d tries", id, askChangeTries)
|
||||
}
|
||||
|
||||
func (b busAsked) All(ctx context.Context) ([]asked, error) {
|
||||
kv, err := b.kv(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lister, err := kv.ListKeys(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = lister.Stop() }()
|
||||
var out []asked
|
||||
for k := range lister.Keys() {
|
||||
e, err := kv.Get(ctx, k)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var r asked
|
||||
if json.Unmarshal(e.Value(), &r) == nil {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// callAction performs an action's verb as the controller, through the grant that names it.
|
||||
func callAction(conn *nats.Conn) func(ctx context.Context, a conditions.Action, args map[string]string) error {
|
||||
return func(ctx context.Context, a conditions.Action, args map[string]string) error {
|
||||
seat, verb, ok := strings.Cut(a.Verb, ".")
|
||||
if !ok {
|
||||
return fmt.Errorf("%q names no seat and verb", a.Verb)
|
||||
}
|
||||
body := map[string]any{}
|
||||
for k, v := range args {
|
||||
body[k] = v
|
||||
}
|
||||
if seat == catalogue.DeliverySeat {
|
||||
_, err := askDeliveryOwner(ctx, conn, verb, body)
|
||||
return err
|
||||
}
|
||||
granted := false
|
||||
for _, v := range broker.VerbsTheControllerActsOnAWarrant {
|
||||
granted = granted || (v.Seat == seat && v.Verb == verb)
|
||||
}
|
||||
if !granted {
|
||||
return fmt.Errorf("%s: %w", a.Verb, errNotGranted)
|
||||
}
|
||||
var answer link.Answer
|
||||
var err error
|
||||
if a.Machine != "" {
|
||||
answer, err = link.AskSeatTool(ctx, conn, seat, verb, a.Machine, body, askWithin)
|
||||
} else {
|
||||
answer, err = link.AskMeshSeatTool(ctx, conn, seat, verb, body, askWithin)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return fmt.Errorf("%s refused: %s", a.Verb, answer.Error)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// routerRecordOf reads the router's record of one of the controller's asks, under its name, and answers
|
||||
// how it ended when it did: the bucket is the one the asks seat's declarer names as its records.
|
||||
func routerRecordOf(conn *nats.Conn, inv *inventory.Inventory) func(ctx context.Context, id string) (*asks.Warrant, error) {
|
||||
return func(ctx context.Context, id string) (*asks.Warrant, error) {
|
||||
bucket, err := asksRecords(ctx, inv)
|
||||
if err != nil || bucket == "" {
|
||||
return nil, err
|
||||
}
|
||||
reply, err := conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET.KV_"+bucket+".$KV."+bucket+"."+askerName+"."+id, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if reply.Header.Get("Status") != "" {
|
||||
return nil, nil // none, or not readable: the event says it
|
||||
}
|
||||
var rec struct {
|
||||
State string `json:"state"`
|
||||
Warrant *asks.Warrant `json:"warrant"`
|
||||
}
|
||||
if json.Unmarshal(reply.Data, &rec) != nil || rec.State == "open" || rec.Warrant == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return rec.Warrant, nil
|
||||
}
|
||||
}
|
||||
|
||||
// asksRecords is the bucket the asks seat's declarer keeps its record of asks in.
|
||||
func asksRecords(ctx context.Context, inv *inventory.Inventory) (string, error) {
|
||||
declared, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, m := range declared {
|
||||
for _, s := range m.DefinesSeats {
|
||||
if s.Name == broker.AsksSeat && len(s.Records) > 0 {
|
||||
return broker.BucketName(m.Module, s.Records[0]), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// routerHereIn says whether a module declaring the asks seat, with its ask named by its caller, is assigned:
|
||||
// without it nothing takes an ask, and asking would only fill a queue nobody reads.
|
||||
func routerHereIn(inv *inventory.Inventory) func(ctx context.Context) (bool, error) {
|
||||
return func(ctx context.Context) (bool, error) {
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
for _, s := range e.Manifest.DefinesSeats {
|
||||
if s.Name == broker.AsksSeat && s.NamedByCaller("ask") && len(e.On) > 0 {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
// grantHeldIn says whether the bus holds the controller's grant to ask (novox/hq issue 353), judged on the two
|
||||
// facts that make it so, and failing closed on each:
|
||||
//
|
||||
// 1. **composed**: the user list the mesh composes now grants the controller itself to publish its ask
|
||||
// (controllerMayAsk) — the controller's own grant, not whether any user of the list changed;
|
||||
// 2. **in force**: the bus's own module, on the machine holding the bus, says the user list there allows it
|
||||
// and the server reloaded after that list was written (the nats module's `nats_user_can`, `in_force`) — a
|
||||
// list sent is not a list the bus enforces until it reloads.
|
||||
//
|
||||
// Anything that cannot be worked out is not held, with why, so the conditions that need the operator are said
|
||||
// as undelivered rather than asked into a refusal.
|
||||
func grantHeldIn(open *stores, conn *nats.Conn) func(ctx context.Context) (bool, string, error) {
|
||||
return grantHeldBy(composedGrantIn(open), busGrantIn(open, conn), time.Now)
|
||||
}
|
||||
|
||||
// grantJudge is one of the two facts: whether it holds, and why not.
|
||||
type grantJudge func(ctx context.Context) (bool, string, error)
|
||||
|
||||
// grantHeldBy is the judgement over the two facts, remembered for grantLookEvery: composing the list resolves
|
||||
// the bus's machine whole, and asking the bus's module is a call.
|
||||
func grantHeldBy(composed, inForce grantJudge, now func() time.Time) func(ctx context.Context) (bool, string, error) {
|
||||
var mu sync.Mutex
|
||||
var at time.Time
|
||||
var held bool
|
||||
var why string
|
||||
return func(ctx context.Context) (bool, string, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if !at.IsZero() && now().Sub(at) < grantLookEvery {
|
||||
return held, why, nil
|
||||
}
|
||||
held, why = judgeGrant(ctx, composed, inForce)
|
||||
at = now()
|
||||
return held, why, nil
|
||||
}
|
||||
}
|
||||
|
||||
// judgeGrant is held only when both facts hold; an error is a fact not known, never a yes. A fact that holds
|
||||
// with a caveat (a reload not confirmed) gives that caveat as the why of a yes.
|
||||
func judgeGrant(ctx context.Context, composed, inForce grantJudge) (bool, string) {
|
||||
var caveat []string
|
||||
for _, fact := range []struct {
|
||||
judge grantJudge
|
||||
what string
|
||||
}{{composed, "whether the mesh composes the controller a grant to ask"}, {inForce, "whether the bus enforces it"}} {
|
||||
if fact.judge == nil {
|
||||
return false, fact.what + " is not judged here"
|
||||
}
|
||||
ok, why, err := fact.judge(ctx)
|
||||
if err != nil {
|
||||
return false, fact.what + " could not be worked out: " + err.Error()
|
||||
}
|
||||
if !ok {
|
||||
return false, why
|
||||
}
|
||||
if why != "" {
|
||||
caveat = append(caveat, why)
|
||||
}
|
||||
}
|
||||
return true, strings.Join(caveat, "; ")
|
||||
}
|
||||
|
||||
// controllerMayAsk says whether the controller's own grant, as composed from these records, lets it publish its
|
||||
// ask (ADR 0259 §3: composed while a router that takes asks under its asker's name is assigned).
|
||||
func controllerMayAsk(records broker.Records) (bool, error) {
|
||||
users, err := broker.Users(records)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, u := range users {
|
||||
if u.Kind != broker.KindController {
|
||||
continue
|
||||
}
|
||||
perms, err := broker.PermissionsFor(u)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return broker.MayPublish(perms, asks.AskSubject(askerName)), nil
|
||||
}
|
||||
return false, errors.New("the composition holds no controller")
|
||||
}
|
||||
|
||||
// composedGrantIn is the first fact, from the mesh's records.
|
||||
func composedGrantIn(open *stores) grantJudge {
|
||||
return func(ctx context.Context) (bool, string, error) {
|
||||
if open == nil || open.inventory == nil {
|
||||
return false, "", errors.New("the mesh's records are not open")
|
||||
}
|
||||
records, err := open.inventory.BusRecords(ctx)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
may, err := controllerMayAsk(records)
|
||||
if err != nil || may {
|
||||
return may, "", err
|
||||
}
|
||||
return false, "the mesh composes the controller no grant to ask: no router that takes asks under its " +
|
||||
"asker's name is assigned", nil
|
||||
}
|
||||
}
|
||||
|
||||
// busGrantIn is the second fact, asked of the bus's own module on the machine holding the bus, with the check
|
||||
// main made before it (novox/hq issue 353, PR 186: is the user list sent to the bus's machine the one composed
|
||||
// now) kept for the cases the module's word does not settle.
|
||||
func busGrantIn(open *stores, conn *nats.Conn) grantJudge {
|
||||
ask := func(ctx context.Context) (string, json.RawMessage, error) {
|
||||
if open == nil || open.inventory == nil || conn == nil {
|
||||
return "", nil, errors.New("the mesh's records or the bus are not open")
|
||||
}
|
||||
holders, err := seatHolders(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
h, held := holders[theBrokerSeat]
|
||||
if !held || h.Node == "" || h.Module == "" {
|
||||
return "", nil, nil
|
||||
}
|
||||
a, err := link.AskModuleToolOn(ctx, conn, h.Module, busUserCanTool, h.Node, map[string]any{
|
||||
"user": broker.ControllerName, "action": "publish", "subject": asks.AskSubject(askerName)}, 15*time.Second)
|
||||
if err != nil {
|
||||
return h.Node, nil, err
|
||||
}
|
||||
if a.Error != "" {
|
||||
return h.Node, nil, errors.New(a.Error)
|
||||
}
|
||||
return h.Node, a.Result, nil
|
||||
}
|
||||
list := func(ctx context.Context) (busListState, error) {
|
||||
if open == nil || open.inventory == nil {
|
||||
return busListState{}, errors.New("the mesh's records are not open")
|
||||
}
|
||||
s, err := brokerListState(ctx, open, nil)
|
||||
if err != nil || s.Machine == "" || !s.Behind {
|
||||
return s, err
|
||||
}
|
||||
step, err := busHeld(ctx, open.inventory, []string{s.Machine})
|
||||
if err != nil {
|
||||
return busListState{}, err
|
||||
}
|
||||
s.BusStep = step[s.Machine]
|
||||
return s, nil
|
||||
}
|
||||
return busGrantBy(ask, list)
|
||||
}
|
||||
|
||||
// busListState is the user list on the bus's machine as the controller's record knows it: whether the list
|
||||
// sent there is behind the one composed now, why it could not be composed, and the bus step a send there waits
|
||||
// on (that machine cannot be pushed until a person's `bus upgrade`).
|
||||
type busListState struct {
|
||||
Machine string
|
||||
Behind bool
|
||||
Uncomposed error
|
||||
BusStep string
|
||||
}
|
||||
|
||||
// remedy is the one thing that sends the bus's machine the list composed now: the plan's own reason when it
|
||||
// cannot be composed, the bus upgrade while sends there wait on it, and otherwise a push of that machine.
|
||||
func (s busListState) remedy(node string) string {
|
||||
if s.Machine != "" {
|
||||
node = s.Machine
|
||||
}
|
||||
switch {
|
||||
case s.Uncomposed != nil:
|
||||
return fmt.Sprintf("the user list for %s cannot be composed (%v); `plan %s` says what to fix first",
|
||||
node, s.Uncomposed, node)
|
||||
case s.BusStep != "":
|
||||
return fmt.Sprintf("sends to %s wait on the bus's planned step, so a push will not carry it: %s", node, s.BusStep)
|
||||
}
|
||||
return fmt.Sprintf("`push %s` carries it", node)
|
||||
}
|
||||
|
||||
// busGrantBy judges the bus's word, and names for each way it is not held the one thing that makes it held:
|
||||
//
|
||||
// - **in force**: held;
|
||||
// - **allowed, from a module older than `in_force`**: held, with the caveat that the reload is not confirmed.
|
||||
// The file allows the publish, and the bus reloads within about 5 s of a change to it, so the worst case is
|
||||
// one ask refused in those seconds, which is asked again (issue 369). The list check is not consulted:
|
||||
// while a bus step is pending the bus's machine cannot be pushed, so any change to the composed list would
|
||||
// read as behind for good and silence every ask until a person's bus upgrade (the review of PR 188);
|
||||
// - **allowed, written, not reloaded** (the module gives both times): wait about 5 s;
|
||||
// - **allowed, in force not known** (the module could not read a time): what is not known, said;
|
||||
// - **not allowed, or the module cannot answer**: the list check decides — the remedy that sends the list
|
||||
// when it is behind (busListState.remedy), and, for a module that cannot answer, held while the list sent
|
||||
// is current, as main judged it.
|
||||
//
|
||||
// ask answers the machine holding the bus and its module's answer ("" when nothing holds the bus); list is the
|
||||
// record's word on the list sent to that machine.
|
||||
func busGrantBy(ask func(ctx context.Context) (string, json.RawMessage, error),
|
||||
list func(ctx context.Context) (busListState, error)) grantJudge {
|
||||
return func(ctx context.Context) (bool, string, error) {
|
||||
node, raw, askErr := ask(ctx)
|
||||
if askErr == nil && node == "" {
|
||||
return false, "no module holds the bus, so nothing says what it enforces", nil
|
||||
}
|
||||
var answer busAnswer
|
||||
if askErr == nil {
|
||||
if answer, askErr = readBusAnswer(raw); askErr != nil {
|
||||
askErr = fmt.Errorf("its answer could not be read: %w", askErr)
|
||||
}
|
||||
}
|
||||
if askErr != nil {
|
||||
if node == "" {
|
||||
node = "the bus's machine"
|
||||
}
|
||||
s, err := list(ctx)
|
||||
if err != nil {
|
||||
return false, "", fmt.Errorf("the bus module on %s could not answer (%v), and whether the list sent "+
|
||||
"there is the one composed now could not be worked out: %w", node, askErr, err)
|
||||
}
|
||||
if s.Behind {
|
||||
return false, fmt.Sprintf("the bus module on %s could not answer (%v), and the user list sent there is "+
|
||||
"behind what the mesh composes; %s", node, askErr, s.remedy(node)), nil
|
||||
}
|
||||
return true, fmt.Sprintf("the bus module on %s could not answer (%v); judged by the user list sent there, "+
|
||||
"which is the one composed now", node, askErr), nil
|
||||
}
|
||||
notAllowed := func(older string) (bool, string, error) {
|
||||
s, err := list(ctx)
|
||||
if err != nil {
|
||||
return false, "", fmt.Errorf("the user list on %s does not allow the controller to ask%s, and whether "+
|
||||
"the list sent there is the one composed now could not be worked out: %w", node, older, err)
|
||||
}
|
||||
if s.Behind {
|
||||
return false, fmt.Sprintf("the user list on %s does not allow the controller to ask%s: it is behind "+
|
||||
"what the mesh composes; %s", node, older, s.remedy(node)), nil
|
||||
}
|
||||
return false, fmt.Sprintf("the user list on %s does not allow the controller to ask%s, though the list "+
|
||||
"sent there is the one composed now; %s again", node, older, s.remedy(node)), nil
|
||||
}
|
||||
switch {
|
||||
case answer.InForce == nil && answer.Allowed:
|
||||
return true, fmt.Sprintf("the user list on %s allows the controller to ask, but its reload is not "+
|
||||
"confirmed: the bus module there is older than the answer whether a grant is in force, and the bus "+
|
||||
"upgrade (`bus upgrade`) carries one that confirms it", node), nil
|
||||
case answer.InForce == nil:
|
||||
return notAllowed(" (the bus module there is older than the answer whether a grant is in force)")
|
||||
case !answer.Allowed:
|
||||
return notAllowed("")
|
||||
case *answer.InForce:
|
||||
return true, "", nil
|
||||
case answer.FileWritten != "" && answer.ServerLoaded != "":
|
||||
return false, fmt.Sprintf("the bus on %s has the controller's grant to ask written (%s) but has not "+
|
||||
"reloaded since (last load %s); it reloads in place within about 5 s, so nothing is to be done but wait",
|
||||
node, answer.FileWritten, answer.ServerLoaded), nil
|
||||
}
|
||||
why := answer.Why
|
||||
if why == "" {
|
||||
why = "it gives no reason"
|
||||
}
|
||||
return false, fmt.Sprintf("the user list on %s allows the controller to ask, but whether the bus enforces it "+
|
||||
"is not known: %s", node, why), nil
|
||||
}
|
||||
}
|
||||
|
||||
// busUserCanTool is the bus module's answer to whether a user may do something, and whether it is in force.
|
||||
const busUserCanTool = "nats_user_can"
|
||||
|
||||
// busAnswer is the bus module's answer to busUserCanTool; InForce is nil from a module older than that field,
|
||||
// and FileWritten and ServerLoaded are given only when it could read both times (mesh-catalog's nats InForce).
|
||||
type busAnswer struct {
|
||||
Allowed bool `json:"allowed"`
|
||||
InForce *bool `json:"in_force"`
|
||||
Why string `json:"why"`
|
||||
FileWritten string `json:"file_written"`
|
||||
ServerLoaded string `json:"server_loaded"`
|
||||
}
|
||||
|
||||
func readBusAnswer(raw json.RawMessage) (busAnswer, error) {
|
||||
var answer busAnswer
|
||||
err := json.Unmarshal(raw, &answer)
|
||||
return answer, err
|
||||
}
|
||||
|
||||
// grantLookEvery is how often the bus's user list is judged against the one its machine was last sent.
|
||||
const grantLookEvery = 30 * time.Second
|
||||
|
||||
// channelsIn is what the channels are now, as a fingerprint: each module claiming a kind of the channel
|
||||
// bench, where, promising what, and whether of its own account. An ask the router refused is asked again
|
||||
// once this changes.
|
||||
func channelsIn(inv *inventory.Inventory) func(ctx context.Context) string {
|
||||
return func(ctx context.Context) string {
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
var parts []string
|
||||
for _, e := range entries {
|
||||
for _, c := range e.Manifest.Claims {
|
||||
if c.Kind == "" || !catalogue.KindedBenches[c.Name] {
|
||||
continue
|
||||
}
|
||||
on := append([]string(nil), e.On...)
|
||||
sort.Strings(on)
|
||||
caps := append([]string(nil), c.Capabilities...)
|
||||
sort.Strings(caps)
|
||||
parts = append(parts, fmt.Sprintf("%s/%s=%s@%s[%s]own:%t", c.Name, c.Kind, e.Manifest.Module,
|
||||
strings.Join(on, ","), strings.Join(caps, ","), e.Manifest.RunsAs != ""))
|
||||
}
|
||||
}
|
||||
sort.Strings(parts)
|
||||
return strings.Join(parts, ";")
|
||||
}
|
||||
}
|
||||
|
||||
// startAsking makes the serving controller's asker and hands it the router's words.
|
||||
func startAsking(ctx context.Context, open *stores, server *link.Server, conn *nats.Conn, keeper *conditions.Keeper) {
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
fmt.Printf("the operator cannot be asked: %v\n", err)
|
||||
return
|
||||
}
|
||||
a := servingAsker(open, conn, js, keeper)
|
||||
if err := server.Decides(a); err != nil {
|
||||
fmt.Printf("the operator's answers cannot be heard, so nothing is asked: %v\n", err)
|
||||
return
|
||||
}
|
||||
askerFrom = a
|
||||
go a.keep(ctx)
|
||||
}
|
||||
|
||||
// servingAsker is the serving controller's asker, wired to the mesh: what it reads, publishes, performs and
|
||||
// records, and the judgements it asks by — the bus's grant to ask among them (novox/hq issue 353).
|
||||
func servingAsker(open *stores, conn *nats.Conn, js jetstream.JetStream, keeper *conditions.Keeper) *asker {
|
||||
return &asker{
|
||||
open: keeper.Open,
|
||||
silence: func(ctx context.Context, key string, d time.Duration, by, why string) error {
|
||||
_, err := keeper.Silence(ctx, key, d, by, why)
|
||||
return err
|
||||
},
|
||||
store: busAsked{conn: conn},
|
||||
publish: func(ctx context.Context, subject string, body []byte, id string) error {
|
||||
_, err := js.Publish(ctx, subject, body, jetstream.WithMsgID(id))
|
||||
return err
|
||||
},
|
||||
call: callAction(conn),
|
||||
// A proposed settings layer is set by this controller itself on the warrant (novox/hq ADR 0277).
|
||||
setLayer: setLayerIn(open),
|
||||
record: func(ctx context.Context, act link.HandAct) error {
|
||||
_, err := link.RecordHandAct(ctx, conn, act)
|
||||
return err
|
||||
},
|
||||
routerRecord: routerRecordOf(conn, open.inventory),
|
||||
routerHere: routerHereIn(open.inventory),
|
||||
grantHeld: grantHeldIn(open, conn),
|
||||
channels: channelsIn(open.inventory),
|
||||
raise: func(ctx context.Context, obs []conditions.Observation) error {
|
||||
return keeper.Reconcile(ctx, sourceAsker, obs)
|
||||
},
|
||||
now: time.Now,
|
||||
logf: func(format string, args ...any) { fmt.Printf(format+"\n", args...) },
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,966 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The suite's tests of a merge, but for the merge window's own (this file's), plan the merge as it is heard: a
|
||||
// window of nothing closes at once, so a merge with no walk open is cut into its walk at once, as every merge
|
||||
// was planned before novox/hq ADR 0276. The window's tests set it through the controller's settings.
|
||||
func init() { mergeWindowDefault = 0 }
|
||||
|
||||
// windowed is a mesh whose controller's settings give a merge window of 90 seconds and at most 10 minutes, as its
|
||||
// settings file says them, and
|
||||
// modules built from the catalogue (app, notes) and from three repositories of their own.
|
||||
func windowed(t *testing.T) *stores {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
settings := t.TempDir() + "/merge-window.json"
|
||||
if err := os.WriteFile(settings, []byte(`{"merge-window": "90s", "merge-window-at-most": "10m"}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv(mergeWindowFileVar, settings)
|
||||
noGroups(t)
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"},
|
||||
inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Ref: "main", BuiltFrom: "c0",
|
||||
Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, m := range []struct{ module, repository, path string }{
|
||||
{"app", "novox/mesh-catalog", "modules/app"},
|
||||
{"notes", "novox/mesh-catalog", "modules/notes"},
|
||||
{"one", "novox/one", ""},
|
||||
{"two", "novox/two", ""},
|
||||
{"three", "novox/three", ""},
|
||||
} {
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1"},
|
||||
inventory.Source{Repository: m.repository, Seat: "git", Path: m.path, Ref: "main", BuiltFrom: "c0",
|
||||
Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return open
|
||||
}
|
||||
|
||||
// t0 is the moment a test's clock starts: merges are dated by it, and heard after it.
|
||||
var t0 = time.Now().UTC().Truncate(time.Second)
|
||||
|
||||
// catalogueMerge is a merge of the catalogue changing one module's directory, made at a moment, from a pull
|
||||
// request's head branch of its own, as the forge announces every merge. Whether its head names a delivery group
|
||||
// with a member still to merge is mesh-delivery's word, which windowed stubs as no group at all (noGroups); a test
|
||||
// whose merges wait for the window as a group's says so (groupsWaiting).
|
||||
func catalogueMerge(commit, module string, made time.Time) link.SourceMoved {
|
||||
return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Head: "pr/" + commit, Commit: commit,
|
||||
MergedAt: made.Format(time.RFC3339Nano), Paths: []string{"modules/" + module + "/module.json"},
|
||||
ModuleDirs: []string{"modules/" + module}, ModuleDirsSaid: true}
|
||||
}
|
||||
|
||||
// repoMerge is a merge of a repository of one module's own, from a head branch of its own (catalogueMerge).
|
||||
func repoMerge(repo, commit string, made time.Time) link.SourceMoved {
|
||||
return link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Head: "pr/" + commit, Commit: commit,
|
||||
MergedAt: made.Format(time.RFC3339Nano), Paths: []string{"main.go"}}
|
||||
}
|
||||
|
||||
// noGroups stubs mesh-delivery's word on groups for a test: no head names a group.
|
||||
func noGroups(t *testing.T) {
|
||||
t.Helper()
|
||||
was := deliveryGroupOf
|
||||
deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) { return nil, false, nil }
|
||||
t.Cleanup(func() { deliveryGroupOf = was })
|
||||
}
|
||||
|
||||
// groupsWaiting stubs mesh-delivery's word on groups for a test: each head named is a group with a member of another
|
||||
// repository whose pull request is open (novox/elsewhere, ready), so a merge from it waits for the window (ADR 0306
|
||||
// decision 2); every other head names no group.
|
||||
func groupsWaiting(t *testing.T, heads ...string) {
|
||||
t.Helper()
|
||||
was := deliveryGroupOf
|
||||
deliveryGroupOf = func(_ context.Context, _ bool, head string) ([]groupMember, bool, error) {
|
||||
if slices.Contains(heads, head) {
|
||||
return []groupMember{{Repository: "novox/elsewhere", State: "ready"}}, true, nil
|
||||
}
|
||||
return nil, false, nil
|
||||
}
|
||||
t.Cleanup(func() { deliveryGroupOf = was })
|
||||
}
|
||||
|
||||
// everyHeadWaits stubs mesh-delivery's word on groups for a test whose merges each wait for the window as they did
|
||||
// before novox/hq ADR 0304: every head is a group with a member of another repository still to merge.
|
||||
func everyHeadWaits(t *testing.T) {
|
||||
t.Helper()
|
||||
was := deliveryGroupOf
|
||||
deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) {
|
||||
return []groupMember{{Repository: "novox/elsewhere", State: "ready"}}, true, nil
|
||||
}
|
||||
t.Cleanup(func() { deliveryGroupOf = was })
|
||||
}
|
||||
|
||||
// fromHead is a merge from a head branch a test names: merges from one head are members of one delivery group,
|
||||
// related by it (novox/hq ADR 0304 decision 1), so they share a batch whatever else they share.
|
||||
func fromHead(m link.SourceMoved, head string) link.SourceMoved {
|
||||
m.Head = head
|
||||
return m
|
||||
}
|
||||
|
||||
// lone is a merge that names no head: known to be no delivery group's, it waits for no window unless it is
|
||||
// related to a batch assembling (ADR 0304 decision 1).
|
||||
func lone(m link.SourceMoved) link.SourceMoved {
|
||||
m.Head = ""
|
||||
return m
|
||||
}
|
||||
|
||||
// hear is a merge heard at a moment.
|
||||
func hear(t *testing.T, open *stores, m link.SourceMoved, now time.Time) {
|
||||
t.Helper()
|
||||
if err := (following{open: open}).hearMerge(t.Context(), m, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// cutAt is the cutter looking at a moment.
|
||||
func cutAt(t *testing.T, open *stores, now time.Time) {
|
||||
t.Helper()
|
||||
if err := cutBatchesHeld(t.Context(), open, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// walks is every plan record that is a walk, newest first, and the batches not yet cut.
|
||||
func walks(t *testing.T, open *stores) (walks, batches []inventory.Plan) {
|
||||
t.Helper()
|
||||
recent, err := open.inventory.RecentPlans(t.Context(), 50)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, p := range recent {
|
||||
if p.Batch() {
|
||||
batches = append(batches, p)
|
||||
} else {
|
||||
walks = append(walks, p)
|
||||
}
|
||||
}
|
||||
return walks, batches
|
||||
}
|
||||
|
||||
// The replay of issue 362: two catalogue merges 18 seconds apart are one batch, cut once into one walk at the
|
||||
// later commit, which names both merges — the earlier carried by the later.
|
||||
func TestTwoMergesSecondsApartAreOneWalkAtTheLaterCommit(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asked := asksWithPaths(t)
|
||||
claude := catalogueMerge("553b7191claude", "app", t0)
|
||||
dunst := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second))
|
||||
hear(t, open, claude, t0.Add(time.Second))
|
||||
hear(t, open, dunst, t0.Add(18*time.Second))
|
||||
if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 1 {
|
||||
t.Fatalf("within the window: %d walk(s), %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
cutAt(t, open, t0.Add(18*time.Second+89*time.Second))
|
||||
if ws, _ := walks(t, open); len(ws) != 0 {
|
||||
t.Fatalf("cut before the window closed: %+v", ws)
|
||||
}
|
||||
cutAt(t, open, t0.Add(18*time.Second+90*time.Second))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 0 {
|
||||
t.Fatalf("after the window: %d walk(s), %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
w := ws[0]
|
||||
if w.Commit != dunst.Commit || len(w.Commits) != 1 || w.Commits[0].Commit != dunst.Commit {
|
||||
t.Fatalf("the walk is at %s %+v, not the later commit", w.Commit, w.Commits)
|
||||
}
|
||||
for _, m := range []string{"app", "notes"} {
|
||||
if _, in := w.Modules[m]; !in {
|
||||
t.Fatalf("the walk does not build %s, which one of its merges moved: %v", m, w.Modules)
|
||||
}
|
||||
}
|
||||
want := []inventory.PlanMerge{{Repository: "novox/mesh-catalog", Commit: claude.Commit, Carried: dunst.Commit},
|
||||
{Repository: "novox/mesh-catalog", Commit: dunst.Commit}}
|
||||
if w.Delivery == nil || !slices.EqualFunc(w.Delivery.Merges, want, sameMerge) {
|
||||
t.Fatalf("the walk answers %+v, want %+v", w.Delivery, want)
|
||||
}
|
||||
if len(*asked) != 2 || (*asked)[0][2] != "main" || (*asked)[1][2] != "main" {
|
||||
t.Fatalf("the walk did not ask its modules at the branch, which holds the commit it carries: %v", *asked)
|
||||
}
|
||||
// Heard again, as the bus may hand it over twice: never a second merge, nor a second walk.
|
||||
hear(t, open, claude, t0.Add(5*time.Minute))
|
||||
if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 {
|
||||
t.Fatalf("a merge heard twice made %d walk(s) and %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
}
|
||||
|
||||
// Merges of three repositories in one window are one walk, one commit for each.
|
||||
func TestThreeRepositoriesInOneWindowAreOneWalk(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "feat/three")
|
||||
asksRecorded(t)
|
||||
for i, r := range []string{"one", "two", "three"} {
|
||||
hear(t, open, fromHead(repoMerge(r, "c-"+r, t0.Add(time.Duration(i)*20*time.Second)), "feat/three"),
|
||||
t0.Add(time.Duration(i)*20*time.Second+time.Second))
|
||||
}
|
||||
cutAt(t, open, t0.Add(41*time.Second+90*time.Second))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 0 {
|
||||
t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
w := ws[0]
|
||||
if len(w.Commits) != 3 || len(w.Delivery.Merges) != 3 {
|
||||
t.Fatalf("the walk carries %+v and answers %+v", w.Commits, w.Delivery.Merges)
|
||||
}
|
||||
for _, r := range []string{"one", "two", "three"} {
|
||||
if w.CommitOf("novox/"+r) != "c-"+r {
|
||||
t.Fatalf("the walk carries %s at %q", r, w.CommitOf("novox/"+r))
|
||||
}
|
||||
if _, in := w.Modules[r]; !in {
|
||||
t.Fatalf("the walk does not build %s: %v", r, w.Modules)
|
||||
}
|
||||
}
|
||||
// Each module's ask is recorded at its own repository's commit.
|
||||
for _, r := range []string{"one", "two", "three"} {
|
||||
q, found, err := open.inventory.BuildRequestByID(t.Context(), w.Modules[r].Build)
|
||||
if err != nil || !found || q.Commit != "c-"+r {
|
||||
t.Fatalf("%s's build was not recorded at its own commit: %+v %v %v", r, q, found, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Each merge restarts the window: a merge at second 80 keeps the batch open until second 170; the window
|
||||
// closes at most ten minutes after its first merge however busy.
|
||||
func TestAMergeRestartsTheWindowAndTheMaximumClosesIt(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "feat/both", "feat/busy")
|
||||
asksWithPaths(t)
|
||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0)
|
||||
hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(80*time.Second)), "feat/both"), t0.Add(80*time.Second))
|
||||
cutAt(t, open, t0.Add(100*time.Second))
|
||||
if ws, _ := walks(t, open); len(ws) != 0 {
|
||||
t.Fatal("the window closed 90 seconds after its first merge, not after its last")
|
||||
}
|
||||
cutAt(t, open, t0.Add(170*time.Second))
|
||||
if ws, _ := walks(t, open); len(ws) != 1 || len(ws[0].Commits) != 2 {
|
||||
t.Fatalf("the window did not close 90 seconds after its last merge: %+v", ws)
|
||||
}
|
||||
|
||||
busy := windowed(t)
|
||||
groupsWaiting(t, "feat/both", "feat/busy")
|
||||
asksWithPaths(t)
|
||||
var last time.Time
|
||||
for i := 0; i < 12; i++ {
|
||||
last = t0.Add(time.Duration(i) * time.Minute)
|
||||
repo := []string{"one", "two", "three"}[i%3]
|
||||
hear(t, busy, fromHead(repoMerge(repo, "c"+string(rune('a'+i)), last), "feat/busy"), last)
|
||||
if ws, _ := walks(t, busy); len(ws) > 0 {
|
||||
if i != 10 {
|
||||
t.Fatalf("a busy window closed at merge %d (minute %d), not at its maximum", i, i)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
ws, _ := walks(t, busy)
|
||||
if len(ws) != 1 {
|
||||
t.Fatalf("ten minutes of merges a minute apart were never cut: %d walk(s)", len(ws))
|
||||
}
|
||||
}
|
||||
|
||||
// `plans` lists the batch being assembled first, in the operator's words: how long is left, when at the latest,
|
||||
// what is grouped, and that the plan is not yet calculated. It says so on the bus as plan-moved, too.
|
||||
func TestTheAssemblingBatchIsShown(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "pr/553b7191claude")
|
||||
asksWithPaths(t)
|
||||
var said []inventory.Plan
|
||||
was := inventory.PlanSaved
|
||||
inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { said = append(said, p) }
|
||||
t.Cleanup(func() { inventory.PlanSaved = was })
|
||||
now := time.Now().UTC()
|
||||
claude := catalogueMerge("553b7191claude", "app", now.Add(-20*time.Second))
|
||||
claude.Number, claude.Title = 174, "claude-code: an agent proposes a section"
|
||||
dunst := catalogueMerge("48bda475dunst", "notes", now.Add(-2*time.Second))
|
||||
dunst.Number, dunst.Title = 175, "dunst: the font the operator chose"
|
||||
hear(t, open, claude, now.Add(-19*time.Second))
|
||||
hear(t, open, dunst, now.Add(-time.Second))
|
||||
hear(t, open, fromHead(repoMerge("one", "a6bc0931one", now), dunst.Head), now)
|
||||
out := captured(t, func() error { return plansCommand(t.Context(), nil) })
|
||||
lines := strings.Split(out, "\n")
|
||||
first := lines[0]
|
||||
for _, want := range []string{"assembling: ", " s left (at the latest ", "grouped: novox/mesh-catalog@48bda475 " +
|
||||
"(answers 553b7191), novox/one@a6bc0931; plan not yet calculated"} {
|
||||
if !strings.Contains(first, want) {
|
||||
t.Fatalf("plans' first line %q does not say %q", first, want)
|
||||
}
|
||||
}
|
||||
// Under it, one line per repository: the pull request, what it answers, what it moves.
|
||||
if len(lines) < 3 || strings.TrimSpace(lines[1]) != "mesh-catalog #175 dunst: the font the operator chose (answers "+
|
||||
"#174 claude-code: an agent proposes a section) · app, notes" ||
|
||||
strings.TrimSpace(lines[2]) != "one a6bc0931 · one" {
|
||||
t.Fatalf("the grouped list reads %q", lines[1:4])
|
||||
}
|
||||
if len(said) == 0 || said[len(said)-1].State != inventory.PlanAssembling || said[len(said)-1].Delivery.Batch == nil ||
|
||||
len(said[len(said)-1].Delivery.Merges) != 3 {
|
||||
t.Fatalf("the batch was not said as assembling with its merges: %+v", said)
|
||||
}
|
||||
}
|
||||
|
||||
// captured is what a command printed.
|
||||
func captured(t *testing.T, run func() error) string {
|
||||
t.Helper()
|
||||
r, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
was := os.Stdout
|
||||
os.Stdout = w
|
||||
runErr := run()
|
||||
os.Stdout = was
|
||||
_ = w.Close()
|
||||
var b bytes.Buffer
|
||||
_, _ = io.Copy(&b, r)
|
||||
if runErr != nil {
|
||||
t.Fatal(runErr)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// One walk at a time: a merge heard while a walk runs joins the next batch, which is cut when the walk ends;
|
||||
// the walk that started is never superseded.
|
||||
func TestAMergeDuringAWalkJoinsTheNextBatch(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(90*time.Second))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 || !ws[0].Open() {
|
||||
t.Fatalf("the first batch was not cut: %+v", ws)
|
||||
}
|
||||
first := ws[0]
|
||||
hear(t, open, repoMerge("two", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
|
||||
cutAt(t, open, t0.Add(5*time.Minute))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != first.ID {
|
||||
t.Fatalf("the merge during the walk: %d walk(s), batches %+v", len(ws), bs)
|
||||
}
|
||||
if got, _ := open.inventory.PlanByID(t.Context(), first.ID); !got.Open() {
|
||||
t.Fatalf("the started walk was %s by the next merge", got.State)
|
||||
}
|
||||
if !strings.HasPrefix(batchWords(bs[0], t0.Add(5*time.Minute)), "queued behind "+first.ID) {
|
||||
t.Fatalf("a queued batch reads %q", batchWords(bs[0], t0.Add(5*time.Minute)))
|
||||
}
|
||||
first.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(t.Context(), &first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(6*time.Minute))
|
||||
ws, bs = walks(t, open)
|
||||
if len(ws) != 2 || len(bs) != 0 || ws[0].Commit != "c2" {
|
||||
t.Fatalf("the queued batch was not cut when the walk ended: %+v %+v", ws, bs)
|
||||
}
|
||||
}
|
||||
|
||||
// A walk waiting for its delivery's word is folded into the next batch's walk, which answers its merges; it
|
||||
// names the walk that took it over.
|
||||
func TestAWaitingWalkIsFoldedIntoTheNextBatch(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(90*time.Second))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 || !ws[0].Waiting() {
|
||||
t.Fatalf("the walk does not wait for its word: %+v", ws)
|
||||
}
|
||||
waiting := ws[0]
|
||||
hear(t, open, repoMerge("two", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
|
||||
cutAt(t, open, t0.Add(2*time.Minute+90*time.Second))
|
||||
ws, bs := walks(t, open)
|
||||
if len(bs) != 0 || len(ws) != 2 {
|
||||
t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
folded, err := open.inventory.PlanByID(ctx, waiting.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
newer := ws[0]
|
||||
if folded.State != inventory.PlanSuperseded || folded.Delivery.TakenOverBy != newer.ID {
|
||||
t.Fatalf("the waiting walk is %s, taken over by %q", folded.State, folded.Delivery.TakenOverBy)
|
||||
}
|
||||
if len(newer.Delivery.Merges) != 2 || newer.CommitOf("novox/one") != "c1" || newer.CommitOf("novox/two") != "c2" {
|
||||
t.Fatalf("the newer walk answers %+v", newer.Delivery.Merges)
|
||||
}
|
||||
if _, in := newer.Modules["one"]; !in {
|
||||
t.Fatalf("what the folded walk was to build is not built: %v", newer.Modules)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge heard after a later merge of its branch was walked is answered by that walk when it builds all it
|
||||
// moves — named at once on a walk done — and joins the next batch when it does not.
|
||||
func TestALateMergeIsAnsweredByTheWalkOfTheLaterOne(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
later := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second))
|
||||
hear(t, open, later, t0.Add(18*time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
w := ws[0]
|
||||
w.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Moves only notes, which the done walk built from the branch after it: answered there, at once.
|
||||
hear(t, open, catalogueMerge("553b7191early", "notes", t0), t0.Add(15*time.Minute))
|
||||
got, _ := open.inventory.PlanByID(ctx, w.ID)
|
||||
if got.State != inventory.PlanDone || len(got.Delivery.Merges) != 2 ||
|
||||
!sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: "553b7191early",
|
||||
Carried: later.Commit}) {
|
||||
t.Fatalf("the late merge is not named by the walk that carried it: %+v", got.Delivery.Merges)
|
||||
}
|
||||
// Moves app, which that walk never built: the next batch walks it.
|
||||
hear(t, open, catalogueMerge("1111aaaaapp", "app", t0.Add(time.Second)), t0.Add(16*time.Minute))
|
||||
_, bs := walks(t, open)
|
||||
if len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "1111aaaaapp" {
|
||||
t.Fatalf("a late merge moving what the walk never built did not join the next batch: %+v", bs)
|
||||
}
|
||||
}
|
||||
|
||||
// A failed walk marks nothing delivered: its earlier merges are walked alone, on their own commit, the newest
|
||||
// first; the first delivered answers the older ones, and the search stops.
|
||||
func TestAFailedWalkWalksItsEarlierMergesAlone(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asked := asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
// Made before the cut marks their modules seen, as merges are: walked again, they are news all the same.
|
||||
oldest := catalogueMerge("aaaa0001", "app", t0.Add(-3*time.Minute))
|
||||
middle := catalogueMerge("bbbb0002", "app", t0.Add(-3*time.Minute+10*time.Second))
|
||||
newest := catalogueMerge("cccc0003", "notes", t0.Add(-3*time.Minute+20*time.Second))
|
||||
for i, m := range []link.SourceMoved{oldest, middle, newest} {
|
||||
hear(t, open, m, t0.Add(time.Duration(i)*10*time.Second+time.Second))
|
||||
}
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
failed := ws[0]
|
||||
failed.State, failed.Note = inventory.PlanFailed, "notes failed to build in tier 0"
|
||||
if err := open.inventory.SavePlan(ctx, &failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(3*time.Minute))
|
||||
got, _ := open.inventory.PlanByID(ctx, failed.ID)
|
||||
if got.State != inventory.PlanFailed || len(got.Delivery.Merges) != 3 {
|
||||
t.Fatalf("the failed walk's record changed: %s %+v", got.State, got.Delivery.Merges)
|
||||
}
|
||||
ws, _ = walks(t, open)
|
||||
alone := ws[0]
|
||||
if alone.ID == failed.ID || alone.Commit != middle.Commit || len(alone.Delivery.Merges) != 1 ||
|
||||
!sameMerge(alone.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: middle.Commit}) {
|
||||
t.Fatalf("the newest earlier merge was not walked alone on its own commit: %+v", alone)
|
||||
}
|
||||
if _, in := alone.Modules["app"]; !in || (*asked)[len(*asked)-1] != [3]string{"novox/mesh-catalog", "modules/app",
|
||||
middle.Commit} {
|
||||
t.Fatalf("the merge walked alone does not build app at its own commit: %v, asked %v", alone.Modules, *asked)
|
||||
}
|
||||
// Retried, the failed walk would name merges the search answers now.
|
||||
if _, err := retryPlan(ctx, open, failed.ID); err == nil || !strings.Contains(err.Error(), "walked alone") {
|
||||
t.Fatalf("a failed walk whose merges are searched was retried: %v", err)
|
||||
}
|
||||
alone.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &alone); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(4*time.Minute))
|
||||
ws, bs := walks(t, open)
|
||||
if ws[0].ID != alone.ID || len(bs) != 0 {
|
||||
t.Fatalf("the search went on after a merge was delivered: %+v", ws[0])
|
||||
}
|
||||
done, _ := open.inventory.PlanByID(ctx, alone.ID)
|
||||
if len(done.Delivery.Merges) != 2 || !sameMerge(done.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog",
|
||||
Commit: oldest.Commit, Carried: middle.Commit}) {
|
||||
t.Fatalf("the oldest merge is not answered by the walk that delivered the one after it: %+v", done.Delivery.Merges)
|
||||
}
|
||||
// A stopped walk starts no search: a person ended it.
|
||||
hear(t, open, catalogueMerge("dddd0004", "app", t0.Add(5*time.Minute)), t0.Add(5*time.Minute))
|
||||
hear(t, open, catalogueMerge("eeee0005", "notes", t0.Add(5*time.Minute+time.Second)), t0.Add(5*time.Minute+time.Second))
|
||||
cutAt(t, open, t0.Add(8*time.Minute))
|
||||
ws, _ = walks(t, open)
|
||||
if _, err := stopWalk(ctx, open.inventory, ws[0].ID, "mesh-delivery for jochen", "not now"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(9*time.Minute))
|
||||
if again, _ := walks(t, open); again[0].ID != ws[0].ID {
|
||||
t.Fatalf("a stopped walk's earlier merge was walked alone: %+v", again[0])
|
||||
}
|
||||
}
|
||||
|
||||
// A delivery group's order holds inside the walk (ADR 0249 unchanged): merges of two repositories from one head
|
||||
// branch name, the node-engine's before the controller's, are tiered so; without the group, one tier.
|
||||
func TestAGroupsOrderBecomesTiersInsideTheWalk(t *testing.T) {
|
||||
for _, grouped := range []bool{true, false} {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
for _, m := range []string{"mesh-host"} {
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"},
|
||||
inventory.Source{Repository: "novox/" + m, Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
host := repoMerge("mesh-host", "h1", t0)
|
||||
ctl := repoMerge("mesh-controller", "k1", t0.Add(time.Second))
|
||||
host.Head, ctl.Head = "feat/together", "feat/together"
|
||||
if !grouped {
|
||||
ctl.Head = "feat/alone"
|
||||
}
|
||||
hear(t, open, ctl, t0.Add(2*time.Second))
|
||||
hear(t, open, host, t0.Add(3*time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 {
|
||||
t.Fatalf("%d walks", len(ws))
|
||||
}
|
||||
tiers := ws[0].Tiers
|
||||
if grouped {
|
||||
if len(tiers) != 2 || !slices.Equal(tiers[0], []string{"mesh-host"}) ||
|
||||
!slices.Equal(tiers[1], []string{"mesh-controller"}) {
|
||||
t.Fatalf("the group's order is not the walk's tiers: %v", tiers)
|
||||
}
|
||||
} else if len(tiers) != 1 {
|
||||
t.Fatalf("two merges of no group were ordered: %v", tiers)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A restarted controller resumes the window where it stood: the batch, its merges and their times are in the
|
||||
// store, a merge handed over again is not doubled, and the batch is cut when its window closes.
|
||||
func TestABatchSurvivesARestart(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "feat/both")
|
||||
asksWithPaths(t)
|
||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0)
|
||||
hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(30*time.Second)), "feat/both"), t0.Add(30*time.Second))
|
||||
|
||||
// A new controller: nothing of the first one's but the store.
|
||||
again, err := openStores(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(again.Close)
|
||||
hear(t, again, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0.Add(60*time.Second)) // the bus hands it over again
|
||||
cutAt(t, again, t0.Add(119*time.Second))
|
||||
if ws, bs := walks(t, again); len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 {
|
||||
t.Fatalf("the restarted controller's batch: %d walk(s), %+v", len(ws), bs)
|
||||
}
|
||||
cutAt(t, again, t0.Add(120*time.Second))
|
||||
ws, bs := walks(t, again)
|
||||
if len(ws) != 1 || len(bs) != 0 || len(ws[0].Delivery.Merges) != 2 {
|
||||
t.Fatalf("the restarted controller did not cut the batch at its window: %+v %+v", ws, bs)
|
||||
}
|
||||
}
|
||||
|
||||
// The window's settings reach the controller in its settings file, read at every look; one that says no
|
||||
// duration, or no file, is the default.
|
||||
func TestTheWindowIsTheControllersSetting(t *testing.T) {
|
||||
file := t.TempDir() + "/merge-window.json"
|
||||
t.Setenv(mergeWindowFileVar, file)
|
||||
for _, c := range []struct {
|
||||
content string
|
||||
window, atMost time.Duration
|
||||
}{
|
||||
{`{"merge-window": "60s", "merge-window-at-most": "5m"}`, time.Minute, 5 * time.Minute},
|
||||
{`{"merge-window": 30, "merge-window-at-most": ""}`, 30 * time.Second, mergeWindowAtMostDefault},
|
||||
{`{"merge-window": "soon"}`, mergeWindowDefault, mergeWindowAtMostDefault},
|
||||
} {
|
||||
if err := os.WriteFile(file, []byte(c.content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if w, m := mergeWindowOf(); w != c.window || m != c.atMost {
|
||||
t.Errorf("%s reads as %s and %s, want %s and %s", c.content, w, m, c.window, c.atMost)
|
||||
}
|
||||
}
|
||||
t.Setenv(mergeWindowFileVar, file+".gone")
|
||||
if w, m := mergeWindowOf(); w != mergeWindowDefault || m != mergeWindowAtMostDefault {
|
||||
t.Errorf("no file reads as %s and %s", w, m)
|
||||
}
|
||||
}
|
||||
|
||||
// S16 names every merge a waiting walk answers, never one commit a supersession may have ended (issue 362).
|
||||
func TestAWaitingWalkNamesTheMergesItAnswers(t *testing.T) {
|
||||
f := calm(t0)
|
||||
f.waits = []waitFacts{{id: "plan-2", repository: "novox/mesh-catalog", commit: "48bda475dunst", awaits: "mesh-delivery",
|
||||
since: t0.Add(-31 * time.Minute), merges: []inventory.PlanMerge{
|
||||
{Repository: "novox/mesh-catalog", Commit: "553b7191claude", Carried: "48bda475dunst"},
|
||||
{Repository: "novox/mesh-catalog", Commit: "48bda475dunst"}}}}
|
||||
got := watchWaits(f)
|
||||
if len(got) != 1 || !strings.Contains(got[0].Summary, "novox/mesh-catalog@553b7191") ||
|
||||
!strings.Contains(got[0].Summary, "novox/mesh-catalog@48bda475") {
|
||||
t.Fatalf("the waiting walk does not name both merges: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge heard after a later merge of its branch was cut is named by that walk however its modules read since
|
||||
// the cut, in a repository of one module (review of this change); a walk that never built what it moves names
|
||||
// nothing, and the merge joins the next batch.
|
||||
func TestALateMergeOfAOneModuleRepositoryIsNamed(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
c2 := repoMerge("one", "c2", t0.Add(-50*time.Second))
|
||||
hear(t, open, c2, t0)
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
w := ws[0]
|
||||
w.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, repoMerge("one", "c1", t0.Add(-60*time.Second)), t0.Add(15*time.Minute))
|
||||
got, _ := open.inventory.PlanByID(ctx, w.ID)
|
||||
if len(got.Delivery.Merges) != 2 || !sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/one",
|
||||
Commit: "c1", Carried: "c2"}) {
|
||||
t.Fatalf("the late merge was not named by the walk that carried it: %+v", got.Delivery.Merges)
|
||||
}
|
||||
|
||||
// A walk of two that built only two: a late merge of one is not its to name.
|
||||
hear(t, open, repoMerge("two", "d2", t0.Add(16*time.Minute)), t0.Add(16*time.Minute))
|
||||
cutAt(t, open, t0.Add(18*time.Minute))
|
||||
ws, _ = walks(t, open)
|
||||
w = ws[0]
|
||||
delete(w.Modules, "two")
|
||||
w.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, repoMerge("two", "d1", t0.Add(15*time.Minute)), t0.Add(20*time.Minute))
|
||||
if got, _ := open.inventory.PlanByID(ctx, w.ID); len(got.Delivery.Merges) != 1 {
|
||||
t.Fatalf("a walk that never built what the late merge moves named it: %+v", got.Delivery.Merges)
|
||||
}
|
||||
if _, bs := walks(t, open); len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "d1" {
|
||||
t.Fatalf("the late merge did not join the next batch: %+v", bs)
|
||||
}
|
||||
}
|
||||
|
||||
// One walk at a time holds against the delivery's word too: a waiting walk let go beside a started one takes the
|
||||
// word and starts once that one ended, asking nothing before.
|
||||
func TestAWalkLetGoBesideAStartedOneStartsOnceItEnded(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asked := asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
started, _ := walks(t, open)
|
||||
waiting := inventory.Plan{ID: "plan-waiting", Repository: "novox/two", Branch: "main", Commit: "d1", Created: t0,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"two"}}, Modules: map[string]*inventory.PlanModule{"two": {}},
|
||||
Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}}
|
||||
if err := open.inventory.SavePlan(ctx, &waiting); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := letGo(ctx, open.inventory, waiting.ID, catalogue.DeliverySeat, "its turn"); err != nil {
|
||||
t.Fatalf("the word was refused beside %s: %v", started[0].ID, err)
|
||||
}
|
||||
before := len(*asked)
|
||||
advanceHeld(ctx, open)
|
||||
got, _ := open.inventory.PlanByID(ctx, waiting.ID)
|
||||
if len(*asked) != before || !strings.Contains(got.Note, "starts once "+started[0].ID) {
|
||||
t.Fatalf("a walk let go beside a started one asked (%d → %d) or does not say it waits: %q", before, len(*asked), got.Note)
|
||||
}
|
||||
// Read as a wait, an hour on: its note on the line, never LATE, and no tier of it late for S3.
|
||||
later := time.Now().Add(time.Hour)
|
||||
if line := planLine(got, later); !strings.Contains(line, "starts once "+started[0].ID) || strings.Contains(line, "LATE") {
|
||||
t.Fatalf("a deferred walk reads %q", line)
|
||||
}
|
||||
facts, _, err := gatherPlans(ctx, open.inventory, later, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range facts {
|
||||
if f.id == got.ID {
|
||||
t.Fatalf("a deferred walk is watched as a tier running late: %+v", f)
|
||||
}
|
||||
}
|
||||
done := started[0]
|
||||
done.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) != before+1 {
|
||||
t.Fatalf("the walk did not start once the started one ended: asked %v", *asked)
|
||||
}
|
||||
}
|
||||
|
||||
// A file a merge of the batch removed and a later one brought back is not removed; one removed last is.
|
||||
func TestARemovedFileIsTheLastMergesWord(t *testing.T) {
|
||||
ev := func(commit string, paths, removed []string) inventory.BatchedMerge {
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "one", Base: "main", Commit: commit, Paths: paths, Removed: removed}
|
||||
b, _ := json.Marshal(m)
|
||||
return inventory.BatchedMerge{Repository: "novox/one", Branch: "main", Commit: commit, Event: b,
|
||||
Merged: t0.Add(time.Duration(len(commit)) * time.Second)}
|
||||
}
|
||||
got := combinedMerges([]inventory.BatchedMerge{
|
||||
ev("a", []string{"x/module.json", "y/module.json"}, []string{"x/module.json", "y/module.json"}),
|
||||
ev("bb", []string{"x/module.json"}, nil),
|
||||
ev("ccc", []string{"z.go"}, nil)})
|
||||
if len(got) != 1 || got[0].Commit != "ccc" || !slices.Equal(got[0].Removed, []string{"y/module.json"}) ||
|
||||
len(got[0].Paths) != 3 {
|
||||
t.Fatalf("combined as %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The catch-up hands over what the bus lost after its branch's later merges were cut, and the record keeps it:
|
||||
// an earlier merge is named by the walk that carried it; a merge made before a cut and heard after it, which
|
||||
// no later merge carries, joins the next batch — neither reads as history and is dropped (review of this change).
|
||||
func TestTheCatchUpKeepsWhatACutMadeHistory(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
base := time.Now().UTC().Add(-time.Hour).Truncate(time.Second)
|
||||
hear(t, open, repoMerge("one", "c2", base.Add(10*time.Second)), base.Add(11*time.Second))
|
||||
cutAt(t, open, base.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
w := ws[0]
|
||||
w.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lost := []link.AnnouncedMerge{
|
||||
{SourceMoved: repoMerge("one", "c1", base), At: base.Add(time.Second)},
|
||||
{SourceMoved: repoMerge("one", "c3lost", base.Add(20*time.Second)), At: base.Add(21 * time.Second)},
|
||||
}
|
||||
catalogued := func(ctx context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
|
||||
entries, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
read, err := readForPlanning(ctx, open.inventory)
|
||||
return entries, read, err
|
||||
}
|
||||
if err := catchUpOnMerges(ctx, time.Now(), unheardMerges{announcedList(lost), open.inventory}, catalogued,
|
||||
(following{open}).SourceMoved, t.Logf); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ := open.inventory.PlanByID(ctx, w.ID)
|
||||
if len(got.Delivery.Merges) != 2 || !sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/one",
|
||||
Commit: "c1", Carried: "c2"}) {
|
||||
t.Fatalf("the earlier merge the catch-up handed over is not named by the walk that carried it: %+v",
|
||||
got.Delivery.Merges)
|
||||
}
|
||||
if _, kept, err := open.inventory.MergeOf(ctx, "novox/one", "c3lost"); err != nil || !kept {
|
||||
t.Fatalf("the merge made before the cut and heard after it was dropped: %v %v", kept, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge on the controller's own path never shares a batch with one that waits for mesh-delivery's word
|
||||
// (decided during the build, 2026-10-10): the two are batches of their own, cut one after the other; a walk on
|
||||
// the own path folds no waiting catalogue walk but batches its merges again behind it; the catalogue's walk
|
||||
// still waits for the word, so no catalogue delivery skips its turn.
|
||||
func TestAMergeOnTheControllersPathNeverSharesABatch(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asked := asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, catalogueMerge("aaaa0001", "app", t0), t0)
|
||||
hear(t, open, repoMerge("mesh-controller", "k1", t0.Add(10*time.Second)), t0.Add(10*time.Second))
|
||||
hear(t, open, catalogueMerge("bbbb0002", "notes", t0.Add(20*time.Second)), t0.Add(20*time.Second))
|
||||
_, bs := walks(t, open)
|
||||
var ownBatch, catalogueBatch inventory.Plan
|
||||
for _, b := range bs {
|
||||
if b.OwnPath() {
|
||||
ownBatch = b
|
||||
} else {
|
||||
catalogueBatch = b
|
||||
}
|
||||
}
|
||||
if len(bs) != 2 || ownBatch.ID == "" || catalogueBatch.ID == "" || len(catalogueBatch.Delivery.Merges) != 2 ||
|
||||
len(ownBatch.Delivery.Merges) != 1 {
|
||||
t.Fatalf("a controller merge and two catalogue merges in one window: %+v", bs)
|
||||
}
|
||||
if !strings.Contains(batchWords(ownBatch, t0.Add(30*time.Second)), "own path") {
|
||||
t.Fatalf("the own-path batch does not say so: %q", batchWords(ownBatch, t0.Add(30*time.Second)))
|
||||
}
|
||||
// Both windows closed, the controller's batch is cut first and starts; the catalogue batch queues behind it,
|
||||
// is cut when the controller's walk ended, and waits for its word with both merges.
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || ws[0].Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k1" {
|
||||
t.Fatalf("the controller's batch was not cut first into a started walk: %+v", ws)
|
||||
}
|
||||
ownWalk := ws[0]
|
||||
for _, m := range []string{"app", "notes"} {
|
||||
if _, in := ownWalk.Modules[m]; in {
|
||||
t.Fatalf("the controller's walk builds %s, a catalogue module: it skipped its turn", m)
|
||||
}
|
||||
}
|
||||
if len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != ownWalk.ID || bs[0].OwnPath() {
|
||||
t.Fatalf("the catalogue batch does not queue behind the controller's walk: %+v", bs)
|
||||
}
|
||||
ownWalk.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &ownWalk); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(3*time.Minute))
|
||||
ws, bs = walks(t, open)
|
||||
if len(bs) != 0 || ws[0].ID != catalogueBatch.ID || !ws[0].Waiting() || len(ws[0].Delivery.Merges) != 2 {
|
||||
t.Fatalf("the catalogue batch was not cut into a waiting walk once the controller's ended: %+v %+v", ws, bs)
|
||||
}
|
||||
for _, m := range []string{"app", "notes"} {
|
||||
if _, in := ws[0].Modules[m]; !in {
|
||||
t.Fatalf("the catalogue walk does not build %s: %v", m, ws[0].Modules)
|
||||
}
|
||||
}
|
||||
for _, a := range *asked {
|
||||
if a[1] == "modules/app" || a[1] == "modules/notes" {
|
||||
t.Fatalf("a catalogue module was asked without the word: %v", *asked)
|
||||
}
|
||||
}
|
||||
|
||||
// A catalogue walk waiting for its word when an own-path batch is cut keeps waiting beside the own-path
|
||||
// walk, is not folded into it, and its word is taken meanwhile: it starts once the own-path walk ended.
|
||||
catalogueWalk := ws[0]
|
||||
hear(t, open, repoMerge("mesh-controller", "k2", t0.Add(4*time.Minute)), t0.Add(4*time.Minute))
|
||||
cutAt(t, open, t0.Add(6*time.Minute))
|
||||
ws, _ = walks(t, open)
|
||||
kept, _ := open.inventory.PlanByID(ctx, catalogueWalk.ID)
|
||||
if !kept.Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k2" || ws[0].Waiting() {
|
||||
t.Fatalf("the waiting catalogue walk was not kept waiting beside the controller's walk: %s %+v", kept.State, ws)
|
||||
}
|
||||
if _, in := ws[0].Modules["app"]; in {
|
||||
t.Fatalf("the controller's walk folded the waiting catalogue walk in: %v", ws[0].Modules)
|
||||
}
|
||||
if _, err := letGo(ctx, open.inventory, catalogueWalk.ID, catalogue.DeliverySeat, "its turn"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := len(*asked)
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) != before {
|
||||
t.Fatalf("the catalogue walk started beside the controller's: asked %v", (*asked)[before:])
|
||||
}
|
||||
own2 := ws[0]
|
||||
own2.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &own2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) < before+1 || (*asked)[before][1] != "modules/app" {
|
||||
t.Fatalf("the catalogue walk did not start once the controller's ended: %v", (*asked)[before:])
|
||||
}
|
||||
}
|
||||
|
||||
// `plans` names a walk by what it moves (asked by the operator, 2026-10-10): the repository's pull request and
|
||||
// title, the modules it moves and the machines running them, then the state words; a record naming no pull
|
||||
// request is named by its commit, and a title is cut at fifty runes.
|
||||
func TestAPlanLineNamesWhatItMoves(t *testing.T) {
|
||||
now := time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)
|
||||
p := inventory.Plan{ID: "plan-1", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c1c1c1c1c1",
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"messenger", "telegram"}}, TierEntered: now.Add(-2 * time.Minute),
|
||||
Modules: map[string]*inventory.PlanModule{"messenger": {State: "asked"}, "telegram": {State: "asked"}},
|
||||
Commits: []inventory.PlanCommit{{Repository: "novox/mesh-catalog", Branch: "main", Commit: "c1c1c1c1c1"}},
|
||||
Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{{Repository: "novox/mesh-catalog",
|
||||
Commit: "c1c1c1c1c1", Number: 175, Title: "a tap shows its outcome", Moves: []string{"telegram", "messenger"}}}}}
|
||||
running := func(module string) []string {
|
||||
if module == "messenger" || module == "telegram" {
|
||||
return []string{"novox"}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if got, want := planLineOn(p, now, pauseView{}, tierAtLeast, running),
|
||||
"mesh-catalog #175 a tap shows its outcome · messenger, telegram → novox · tier 1 of 1, building for 2m0s"; got != want {
|
||||
t.Fatalf("the line reads %q, want %q", got, want)
|
||||
}
|
||||
if got := planLine(p, now); !strings.HasPrefix(got, "mesh-catalog #175 a tap shows its outcome · messenger, telegram · tier") {
|
||||
t.Fatalf("without the machines the line reads %q", got)
|
||||
}
|
||||
old := p
|
||||
old.Commits, old.Delivery = nil, nil
|
||||
if got := planLine(old, now); !strings.HasPrefix(got, "mesh-catalog c1c1c1c1 · tier 1 of 1") {
|
||||
t.Fatalf("a record naming no pull request reads %q", got)
|
||||
}
|
||||
long := p
|
||||
long.Delivery.Merges[0].Title = strings.Repeat("abcdefghij", 6)
|
||||
if got := planHeadline(long, nil); !strings.Contains(got, "#175 "+strings.Repeat("abcdefghij", 4)+"abcdefghi…") {
|
||||
t.Fatalf("a long title is not cut at fifty runes: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// sameMerge compares what a walk names of a merge: its repository, commit and the commit carrying it.
|
||||
func sameMerge(a, b inventory.PlanMerge) bool {
|
||||
return a.Repository == b.Repository && a.Commit == b.Commit && a.Carried == b.Carried
|
||||
}
|
||||
|
||||
// A catalogue merge heard after a later merge of its branch was walked without the word (a merge that touched
|
||||
// the bus too, on the controller's own path) is not answered by that walk while the delivery seat has a holder:
|
||||
// its delivery would skip its turn. It joins the next catalogue batch.
|
||||
func TestALateCatalogueMergeIsNotAnsweredByAnOwnPathWalk(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
for _, m := range []struct {
|
||||
module string
|
||||
claims []catalogue.Claim
|
||||
}{
|
||||
{"nats", nil},
|
||||
{"mesh-delivery", []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
|
||||
} {
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1", Claims: m.claims},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + m.module, Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mixed := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "m1xed", MergedAt: t0.Format(time.RFC3339Nano),
|
||||
Paths: []string{"modules/nats/module.json", "modules/app/module.json"}, ModuleDirs: []string{"modules/nats", "modules/app"},
|
||||
ModuleDirsSaid: true}
|
||||
hear(t, open, mixed, t0.Add(time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 || ws[0].Waiting() {
|
||||
t.Fatalf("the mixed merge's walk waited, or was not cut: %+v", ws)
|
||||
}
|
||||
done := ws[0]
|
||||
done.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, catalogueMerge("ear1ier", "app", t0.Add(-30*time.Second)), t0.Add(3*time.Minute))
|
||||
got, _ := open.inventory.PlanByID(ctx, done.ID)
|
||||
_, bs := walks(t, open)
|
||||
if len(got.Delivery.Merges) != 1 || len(bs) != 1 || bs[0].OwnPath() || bs[0].Delivery.Merges[0].Commit != "ear1ier" {
|
||||
t.Fatalf("the late catalogue merge was answered by the own-path walk (%+v) rather than the next catalogue batch (%+v)",
|
||||
got.Delivery.Merges, bs)
|
||||
}
|
||||
}
|
||||
@@ -1,107 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// **Every wait of a batch has a bound and a condition** (novox/hq ADR 0276 decision 9): a batch still
|
||||
// assembling a minute past its maximum while no walk is open is a cut the controller failed to make (S18),
|
||||
// and a batch waiting behind an open walk longer than that walk's bound waits on a walk gone wrong (S19).
|
||||
|
||||
// The kinds S18 and S19 raise.
|
||||
const (
|
||||
kindBatchNotCut = "batch-not-cut"
|
||||
kindBatchBehindWalk = "batch-behind-walk"
|
||||
)
|
||||
|
||||
// batchFacts is one batch not yet cut, as the watchdogs read it.
|
||||
type batchFacts struct {
|
||||
id, state, grouped string
|
||||
// atMost is when its window closes at the latest; closed when it closed.
|
||||
atMost, closed time.Time
|
||||
// behind is the open walk it waits behind, and walkBound that walk's bound: a tier's bound for each of its
|
||||
// tiers.
|
||||
behind string
|
||||
walkBound time.Duration
|
||||
}
|
||||
|
||||
// gatherBatches is every batch not yet cut, with the bound of the walk it waits behind.
|
||||
func gatherBatches(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]batchFacts, error) {
|
||||
batches, err := inv.Batches(ctx)
|
||||
if err != nil || len(batches) == 0 {
|
||||
return nil, err
|
||||
}
|
||||
bounds, err := measuredTierBounds(ctx, inv, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []batchFacts
|
||||
for _, b := range batches {
|
||||
f := batchFacts{id: b.ID, state: b.State, grouped: groupedWords(b)}
|
||||
if w := b.Delivery; w != nil && w.Batch != nil {
|
||||
f.atMost, f.closed, f.behind = w.Batch.AtMost, w.Batch.ClosesAt, w.Batch.Behind
|
||||
if f.atMost.Before(f.closed) {
|
||||
f.closed = f.atMost
|
||||
}
|
||||
if w.Batch.CutAtOnce && !b.Created.IsZero() {
|
||||
// Its window closed when it was made, which is its latest too (ADR 0304 decision 1).
|
||||
f.atMost, f.closed = b.Created, b.Created
|
||||
}
|
||||
}
|
||||
if f.behind != "" {
|
||||
if walk, err := inv.PlanByID(ctx, f.behind); err == nil {
|
||||
f.walkBound = bounds.of(walk.Repository) * time.Duration(max(1, len(walk.Tiers)))
|
||||
}
|
||||
}
|
||||
out = append(out, f)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// watchBatchesNotCut is S18: a batch still assembling a minute past its maximum, while no walk is open.
|
||||
func watchBatchesNotCut(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, b := range f.batches {
|
||||
if b.state != inventory.PlanAssembling || b.atMost.IsZero() || f.now.Sub(b.atMost) <= batchLateAfter {
|
||||
continue
|
||||
}
|
||||
late := f.now.Sub(b.atMost)
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id, Kind: kindBatchNotCut,
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the batch %s is still assembling %s past its latest close (%s), with no walk open: "+
|
||||
"the controller did not cut it; grouped: %s", b.id, ago(late), b.atMost.UTC().Format(time.RFC3339), b.grouped),
|
||||
Said: fmt.Sprintf("assembling since %s past its maximum", ago(late)),
|
||||
Headline: "Merged changes are not being delivered",
|
||||
Explanation: fmt.Sprintf("Merges collected for one delivery should have been planned %s ago and were not. "+
|
||||
"Nothing is lost; the mesh keeps them until it plans them.", humanDuration(late)),
|
||||
Resolved: "Merged changes are being delivered again"})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// watchBatchesBehind is S19: a batch waiting behind an open walk longer than that walk's bound, naming it.
|
||||
func watchBatchesBehind(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, b := range f.batches {
|
||||
if b.state != inventory.PlanQueued || b.behind == "" || b.walkBound <= 0 || f.now.Sub(b.closed) <= b.walkBound {
|
||||
continue
|
||||
}
|
||||
in := f.now.Sub(b.closed)
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id, Kind: kindBatchBehindWalk,
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the batch %s has waited %s behind the walk %s, longer than that walk's bound (%s); "+
|
||||
"grouped: %s; `plans %s` says where that walk stands", b.id, ago(in), b.behind, ago(b.walkBound),
|
||||
b.grouped, b.behind),
|
||||
Said: fmt.Sprintf("queued behind %s for %s", b.behind, ago(in)),
|
||||
Headline: "Merged changes wait behind a slow delivery",
|
||||
Explanation: fmt.Sprintf("Merges collected for the next delivery have waited %s for the delivery before "+
|
||||
"them, which is taking longer than it should. Nothing is lost.", humanDuration(in)),
|
||||
Resolved: "Merged changes no longer wait behind a slow delivery"})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1,207 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// **A filling bucket or log is said before it is full** (novox/hq ADR 0297 §6, issue 501).
|
||||
//
|
||||
// A module's bucket and its log each have a cap on the bus, and when either is full the bus refuses
|
||||
// the next write: the module stops doing what it writes for, and nothing said so before. On
|
||||
// 2026-10-11 the issue tracker's bucket held a sixth of its cap after two days, growing toward a stop
|
||||
// nobody would have heard coming. So the self-check reads every module's bucket and log, and one at
|
||||
// fillRaiseAt of its cap or more raises a condition naming the module, the bucket or log, and how full
|
||||
// it is.
|
||||
//
|
||||
// **Cleared by observation below fillClearBelow, not below fillRaiseAt**: a bucket or log hovering at
|
||||
// its threshold would otherwise be raised and cleared on every run. Between the two, an open condition
|
||||
// is kept and none is raised.
|
||||
//
|
||||
// **One that cannot be read is said, and the rest are still judged.** An unanswered question about one
|
||||
// stream is no reason to know nothing of the others: it is a finding of its own, naming the bucket or
|
||||
// log and why, and a fill condition already open for it is kept, because not knowing is not a pass.
|
||||
|
||||
// The fill thresholds, in percent of a bucket's or log's cap.
|
||||
const (
|
||||
fillRaiseAt = 75
|
||||
fillClearBelow = 70
|
||||
)
|
||||
|
||||
// The conditions the fill probe raises: one filling, and one that could not be read.
|
||||
const (
|
||||
kindBucketOrLogFilling = "bucket-or-log-filling"
|
||||
kindBucketOrLogUnread = "bucket-or-log-unread"
|
||||
)
|
||||
|
||||
// probeFillID is the probe's id in the registry.
|
||||
const probeFillID = "D-fill"
|
||||
|
||||
// bucketOrLog is one module's bucket or log as the fill probe reads it.
|
||||
type bucketOrLog struct {
|
||||
Module string
|
||||
// Kind is `bucket` or `log`; Name its local name; Stream the stream it is on the bus.
|
||||
Kind, Name, Stream string
|
||||
}
|
||||
|
||||
// filled is how full one bucket or log is, read from the bus.
|
||||
type filled struct {
|
||||
Bytes, Max uint64
|
||||
}
|
||||
|
||||
// percent is how full, in whole percent, rounded down.
|
||||
func (f filled) percent() uint64 {
|
||||
if f.Max == 0 {
|
||||
return 0
|
||||
}
|
||||
return f.Bytes * 100 / f.Max
|
||||
}
|
||||
|
||||
// fillKey is where a bucket's or log's fill condition is kept.
|
||||
func fillKey(s bucketOrLog) string { return conditions.Key(conditions.ScopeBus, s.Stream, "filling") }
|
||||
|
||||
// fillObservation is what one bucket's or log's fill says: a finding at fillRaiseAt or above, and,
|
||||
// while its condition is open, at fillClearBelow or above too; nothing otherwise, which is what clears
|
||||
// it. The operator's words are the kind's (plain_words.go); the summary and the evidence name the
|
||||
// module, the bucket or log, and the fill.
|
||||
func fillObservation(s bucketOrLog, f filled, open bool) (conditions.Observation, bool) {
|
||||
if f.Max == 0 {
|
||||
return conditions.Observation{}, false // one with no cap cannot fill
|
||||
}
|
||||
// Compared in bytes, not rounded percent: 74.9% is not 75%.
|
||||
atRaise := f.Bytes*100 >= f.Max*fillRaiseAt
|
||||
aboveClear := f.Bytes*100 >= f.Max*fillClearBelow
|
||||
if !atRaise && !(open && aboveClear) {
|
||||
return conditions.Observation{}, false
|
||||
}
|
||||
pct := f.percent()
|
||||
return conditions.Observation{
|
||||
Scope: conditions.ScopeBus, ID: s.Stream, Token: "filling", Kind: kindBucketOrLogFilling,
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s's %s %s holds %s of %s, %d%%: at its cap the bus refuses the module's next write",
|
||||
s.Module, s.Kind, s.Name, mibWords(f.Bytes), mibWords(f.Max), pct),
|
||||
Said: fmt.Sprintf("module %s, %s %s (stream %s): %d of %d bytes, %d%%", s.Module, s.Kind, s.Name,
|
||||
s.Stream, f.Bytes, f.Max, pct),
|
||||
}, true
|
||||
}
|
||||
|
||||
// unreadObservation says one bucket or log could not be read, and why. Asked over the network, so one
|
||||
// look can be wrong: raised on the second look in a row (confirm.go).
|
||||
func unreadObservation(s bucketOrLog, why error) conditions.Observation {
|
||||
return conditions.Observation{
|
||||
Scope: conditions.ScopeBus, ID: s.Stream, Token: "unread", Kind: kindBucketOrLogUnread,
|
||||
Severity: conditions.Warning, Confirm: true,
|
||||
Summary: fmt.Sprintf("%s's %s %s could not be read, so how full it is is not known", s.Module, s.Kind, s.Name),
|
||||
Said: fmt.Sprintf("module %s, %s %s (stream %s): %v", s.Module, s.Kind, s.Name, s.Stream, why),
|
||||
}
|
||||
}
|
||||
|
||||
// keptFilling is an open fill condition said again for a bucket or log that could not be read this
|
||||
// time: not knowing how full it is now is no reason to say it has room. Only ever made from a condition
|
||||
// read back as open — its own summary and severity, never words made up for it.
|
||||
func keptFilling(s bucketOrLog, open conditions.Condition, why error) conditions.Observation {
|
||||
return conditions.Observation{
|
||||
Scope: conditions.ScopeBus, ID: s.Stream, Token: "filling", Kind: kindBucketOrLogFilling,
|
||||
Severity: open.Severity, Summary: open.Summary,
|
||||
Said: fmt.Sprintf("module %s, %s %s (stream %s): not read this time (%v); kept open as it was",
|
||||
s.Module, s.Kind, s.Name, s.Stream, why),
|
||||
}
|
||||
}
|
||||
|
||||
// mibWords is a size as a person reads it, in MiB with one decimal.
|
||||
func mibWords(b uint64) string {
|
||||
return fmt.Sprintf("%.1f MiB", float64(b)/(1024*1024))
|
||||
}
|
||||
|
||||
// readFill is how full one bucket or log is on the bus; found false when it is not on the bus.
|
||||
type readFill func(ctx context.Context, s bucketOrLog) (f filled, found bool, err error)
|
||||
|
||||
// openFill is the fill condition open under a key, if one is, or why it could not be read.
|
||||
type openFill func(ctx context.Context, key string) (conditions.Condition, bool, error)
|
||||
|
||||
// judgeFills judges every bucket and log on its own: one that cannot be read is said as unread, with
|
||||
// its fill condition kept only when that condition was read back and is open — when it cannot be read
|
||||
// either, nothing is said of its fill, which the unread finding already covers. Every other is judged
|
||||
// by its fill; for one of those, a condition that cannot be read is taken as open, so an unknown never
|
||||
// clears a fill measured between fillClearBelow and fillRaiseAt.
|
||||
func judgeFills(ctx context.Context, all []bucketOrLog, read readFill, open openFill) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, s := range all {
|
||||
f, found, err := read(ctx, s)
|
||||
if err != nil {
|
||||
out = append(out, unreadObservation(s, err))
|
||||
if c, isOpen, cerr := open(ctx, fillKey(s)); cerr == nil && isOpen {
|
||||
out = append(out, keptFilling(s, c, err))
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !found {
|
||||
continue // not on the bus: created on the controller's next raise, and nothing there can fill
|
||||
}
|
||||
_, isOpen, cerr := open(ctx, fillKey(s))
|
||||
isOpen = isOpen || cerr != nil
|
||||
if o, said := fillObservation(s, f, isOpen); said {
|
||||
out = append(out, o)
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].Key() < out[j].Key() })
|
||||
return out
|
||||
}
|
||||
|
||||
// declaredBucketsAndLogs is every module's bucket and log the catalogue declares, by its stream.
|
||||
func declaredBucketsAndLogs(ctx context.Context, d *doctor) ([]bucketOrLog, error) {
|
||||
buckets, err := d.open.inventory.DeclaredBuckets(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logs, err := d.open.inventory.DeclaredLogs(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []bucketOrLog
|
||||
for _, b := range buckets {
|
||||
out = append(out, bucketOrLog{Module: b.Module, Kind: "bucket", Name: b.Name, Stream: "KV_" + b.Bucket()})
|
||||
}
|
||||
for _, l := range logs {
|
||||
out = append(out, bucketOrLog{Module: l.Module, Kind: "log", Name: l.Name, Stream: l.Stream()})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// probeFill reads every module's bucket and log on the bus and says each one filling toward its cap,
|
||||
// and each one that could not be read.
|
||||
func probeFill(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
all, err := declaredBucketsAndLogs(ctx, d)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js := d.js.Context()
|
||||
read := func(ctx context.Context, s bucketOrLog) (filled, bool, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return filled{}, false, err
|
||||
}
|
||||
info, err := js.StreamInfo(s.Stream, nats.Context(ctx))
|
||||
switch {
|
||||
case errors.Is(err, nats.ErrStreamNotFound):
|
||||
return filled{}, false, nil
|
||||
case err != nil:
|
||||
return filled{}, false, err
|
||||
case info.Config.MaxBytes <= 0:
|
||||
return filled{}, true, nil
|
||||
}
|
||||
return filled{Bytes: info.State.Bytes, Max: uint64(info.Config.MaxBytes)}, true, nil
|
||||
}
|
||||
open := func(ctx context.Context, key string) (conditions.Condition, bool, error) {
|
||||
if d.keeper == nil {
|
||||
return conditions.Condition{}, false, nil
|
||||
}
|
||||
return d.keeper.Get(ctx, key)
|
||||
}
|
||||
return judgeFills(ctx, all, read, open), nil
|
||||
}
|
||||
@@ -1,165 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// **A filling log or bucket is said at three quarters of its cap and cleared below seven tenths**
|
||||
// (novox/hq ADR 0297 §6): raised at 75% naming the module, the bucket or log and its fill; not at 74.9%;
|
||||
// kept between 70% and 75% while it is open, and not raised there when it is not; and cleared — said no
|
||||
// more — once it reads below 70%, open or not.
|
||||
func TestAFillingBucketOrLogIsRaisedAtThreeQuartersAndClearedBelowSevenTenths(t *testing.T) {
|
||||
const mib = 1024 * 1024
|
||||
log := bucketOrLog{Module: "mesh-issues", Kind: "log", Name: "changes", Stream: "LOG_mesh-issues_changes"}
|
||||
bucket := bucketOrLog{Module: "mesh-issues", Kind: "bucket", Name: "issues", Stream: "KV_mesh-issues_issues"}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
of bucketOrLog
|
||||
bytes uint64
|
||||
max uint64
|
||||
open bool
|
||||
said bool
|
||||
}{
|
||||
{"a log at exactly 75%", log, 768 * mib, 1024 * mib, false, true},
|
||||
{"a bucket at exactly 75%", bucket, 48 * mib, 64 * mib, false, true},
|
||||
{"a bucket full", bucket, 64 * mib, 64 * mib, false, true},
|
||||
{"a log just under 75%", log, 768*mib - 1, 1024 * mib, false, false},
|
||||
{"a bucket at 72%, not open", bucket, 64 * mib * 72 / 100, 64 * mib, false, false},
|
||||
{"a bucket at 72%, open: kept", bucket, 64 * mib * 72 / 100, 64 * mib, true, true},
|
||||
{"a log at exactly 70%, open: kept", log, 700 * mib, 1000 * mib, true, true},
|
||||
{"a log just under 70%, open: cleared", log, 700*mib - 1, 1000 * mib, true, false},
|
||||
{"a bucket at 10%, open: cleared", bucket, 64 * mib / 10, 64 * mib, true, false},
|
||||
{"a bucket with no cap", bucket, 100, 0, true, false},
|
||||
} {
|
||||
o, said := fillObservation(c.of, filled{Bytes: c.bytes, Max: c.max}, c.open)
|
||||
if said != c.said {
|
||||
t.Errorf("%s: said %v, want %v", c.name, said, c.said)
|
||||
continue
|
||||
}
|
||||
if !said {
|
||||
continue
|
||||
}
|
||||
if o.Key() != fillKey(c.of) || o.Kind != kindBucketOrLogFilling || o.Severity != conditions.Warning {
|
||||
t.Errorf("%s: raised as %s (%s, %s)", c.name, o.Key(), o.Kind, o.Severity)
|
||||
}
|
||||
for _, part := range []string{c.of.Module, c.of.Kind + " " + c.of.Name, "%", " of "} {
|
||||
if !strings.Contains(o.Summary, part) {
|
||||
t.Errorf("%s: does not name %q: %q", c.name, part, o.Summary)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(o.Said, "bytes") {
|
||||
t.Errorf("%s: the evidence does not say the fill in bytes: %q", c.name, o.Said)
|
||||
}
|
||||
}
|
||||
o, _ := fillObservation(log, filled{Bytes: 800 * mib, Max: 1024 * mib}, false)
|
||||
if !strings.Contains(o.Summary, "800.0 MiB of 1024.0 MiB, 78%") {
|
||||
t.Errorf("the fill is said as %q", o.Summary)
|
||||
}
|
||||
}
|
||||
|
||||
// **One bucket or log that cannot be read does not stop the others being judged** (review of #231): it is
|
||||
// said as unread with its reason, a fill condition open for it is kept, and every other bucket and log
|
||||
// is judged by its own fill.
|
||||
func TestAnUnreadableBucketOrLogIsSaidAndTheRestAreStillJudged(t *testing.T) {
|
||||
const mib = 1024 * 1024
|
||||
broken := bucketOrLog{Module: "a", Kind: "bucket", Name: "broken", Stream: "KV_a_broken"}
|
||||
brokenOpen := bucketOrLog{Module: "a", Kind: "log", Name: "was-filling", Stream: "LOG_a_was-filling"}
|
||||
full := bucketOrLog{Module: "b", Kind: "log", Name: "changes", Stream: "LOG_b_changes"}
|
||||
empty := bucketOrLog{Module: "c", Kind: "bucket", Name: "quiet", Stream: "KV_c_quiet"}
|
||||
absent := bucketOrLog{Module: "d", Kind: "bucket", Name: "not-yet", Stream: "KV_d_not-yet"}
|
||||
refusal := errors.New("the bus did not answer")
|
||||
read := func(_ context.Context, s bucketOrLog) (filled, bool, error) {
|
||||
switch s {
|
||||
case broken, brokenOpen:
|
||||
return filled{}, false, refusal
|
||||
case full:
|
||||
return filled{Bytes: 900 * mib, Max: 1000 * mib}, true, nil
|
||||
case empty:
|
||||
return filled{Bytes: 1, Max: 64 * mib}, true, nil
|
||||
}
|
||||
return filled{}, false, nil
|
||||
}
|
||||
open := func(_ context.Context, key string) (conditions.Condition, bool, error) {
|
||||
if key == fillKey(brokenOpen) {
|
||||
return conditions.Condition{Key: key, Severity: conditions.Warning,
|
||||
Summary: "a's log was-filling holds 800.0 MiB of 1024.0 MiB, 78%"}, true, nil
|
||||
}
|
||||
return conditions.Condition{}, false, nil
|
||||
}
|
||||
got := map[string]conditions.Observation{}
|
||||
for _, o := range judgeFills(context.Background(), []bucketOrLog{broken, brokenOpen, full, empty, absent}, read, open) {
|
||||
got[o.Key()] = o
|
||||
}
|
||||
for _, s := range []bucketOrLog{broken, brokenOpen} {
|
||||
o, said := got[conditions.Key(conditions.ScopeBus, s.Stream, "unread")]
|
||||
if !said || o.Kind != kindBucketOrLogUnread || !o.Confirm || !strings.Contains(o.Said, refusal.Error()) {
|
||||
t.Errorf("%s could not be read and was said as %+v", s.Stream, o)
|
||||
}
|
||||
}
|
||||
if o, kept := got[fillKey(brokenOpen)]; !kept || !strings.Contains(o.Said, "kept open") {
|
||||
t.Errorf("an open fill condition of a log not read this time was not kept: %+v", o)
|
||||
}
|
||||
if _, said := got[fillKey(broken)]; said {
|
||||
t.Error("a bucket not read and not filling was said filling")
|
||||
}
|
||||
if o, said := got[fillKey(full)]; !said || o.Kind != kindBucketOrLogFilling {
|
||||
t.Errorf("a log at 90%% beside an unreadable one was not judged: %+v", got)
|
||||
}
|
||||
if len(got) != 4 {
|
||||
t.Errorf("said %d findings, want 4 (two unread, one kept, one filling): %v", len(got), got)
|
||||
}
|
||||
}
|
||||
|
||||
// Both kinds have plain words of their own, which hold to the plain rule (novox/hq ADR 0253).
|
||||
func TestAFillingOrUnreadBucketOrLogIsSaidInPlainWords(t *testing.T) {
|
||||
for _, kind := range []string{kindBucketOrLogFilling, kindBucketOrLogUnread} {
|
||||
wording, has := plainWordings[kind]
|
||||
if !has {
|
||||
t.Errorf("%s has no plain words", kind)
|
||||
continue
|
||||
}
|
||||
if why, ok := conditions.PlainWords(wording(conditions.Observation{Kind: kind})); !ok {
|
||||
t.Errorf("%s: %s", kind, why)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **When neither the bucket or log nor its condition can be read, nothing is said of its fill** (second
|
||||
// review of #231): the unread finding covers it, and no fill is invented for it. For one that is read and
|
||||
// measured between seven tenths and three quarters, a condition that cannot be read is taken as open, so
|
||||
// an unknown never clears it.
|
||||
func TestNothingIsSaidOfAFillWhenNeitherItNorItsConditionCanBeRead(t *testing.T) {
|
||||
const mib = 1024 * 1024
|
||||
broken := bucketOrLog{Module: "a", Kind: "log", Name: "changes", Stream: "LOG_a_changes"}
|
||||
between := bucketOrLog{Module: "b", Kind: "bucket", Name: "issues", Stream: "KV_b_issues"}
|
||||
read := func(_ context.Context, s bucketOrLog) (filled, bool, error) {
|
||||
if s == broken {
|
||||
return filled{}, false, errors.New("the bus did not answer")
|
||||
}
|
||||
return filled{Bytes: 72 * mib, Max: 100 * mib}, true, nil
|
||||
}
|
||||
open := func(context.Context, string) (conditions.Condition, bool, error) {
|
||||
return conditions.Condition{}, false, errors.New("the conditions bucket did not answer")
|
||||
}
|
||||
got := map[string]conditions.Observation{}
|
||||
for _, o := range judgeFills(context.Background(), []bucketOrLog{broken, between}, read, open) {
|
||||
got[o.Key()] = o
|
||||
}
|
||||
if o, said := got[fillKey(broken)]; said {
|
||||
t.Fatalf("a fill was said for a log whose fill and condition could not be read: %+v", o)
|
||||
}
|
||||
if _, said := got[conditions.Key(conditions.ScopeBus, broken.Stream, "unread")]; !said {
|
||||
t.Error("the log that could not be read was not said as unread")
|
||||
}
|
||||
if _, kept := got[fillKey(between)]; !kept {
|
||||
t.Error("a bucket at 72% whose condition could not be read was cleared")
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Errorf("said %d findings, want 2: %v", len(got), got)
|
||||
}
|
||||
}
|
||||
@@ -166,10 +166,6 @@ func buildFrom(result link.BuildResult) inventory.Build {
|
||||
for _, r := range result.Read {
|
||||
kept.Read = append(kept.Read, inventory.ReadRepository{Repository: r.Repository, Ref: r.Ref})
|
||||
}
|
||||
// What it was made from, as files (novox/hq ADR 0267): the planner maps the next merge onto it.
|
||||
for _, s := range result.Sources {
|
||||
kept.Sources = append(kept.Sources, inventory.BuildSource{Repository: s.Repository, Ref: s.Ref, Paths: s.Paths})
|
||||
}
|
||||
var announced []inventory.Artifact
|
||||
for _, made := range result.Made {
|
||||
announced = append(announced, inventory.Artifact{
|
||||
@@ -835,9 +831,6 @@ type answers struct {
|
||||
// public name on the machine went dark. The holds were correct; they were recorded only in the
|
||||
// machine's own state file, and the one visible symptom was a count that did not add up.
|
||||
untaken map[string]map[string]int
|
||||
// leftOut is, per machine, every module of its set its composition leaves out, and why (novox/hq issue
|
||||
// 380): assigned and not applied, which every push said only in passing. Not well while there is any.
|
||||
leftOut map[string][]leftOutModule
|
||||
// unheld is every module on a machine whose resources are applied through a seat nothing on
|
||||
// that machine holds (novox/hq ADR 0207), with the modules that could hold it. Reported, not
|
||||
// refused, until the switch — and while there is any, the mesh is not all well: the order the
|
||||
@@ -866,10 +859,6 @@ type answers struct {
|
||||
// applied, is not well: an assignment somebody made is not made yet, or will not be.
|
||||
pending []inventory.PendingAssignment
|
||||
pendingUnread string
|
||||
// lastSent is every machine's last recorded send, by name, with what the machine answered of it (novox/hq
|
||||
// issue 485); lastSentUnread why it could not be read.
|
||||
lastSent map[string]inventory.Send
|
||||
lastSentUnread string
|
||||
}
|
||||
|
||||
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
|
||||
|
||||
@@ -153,97 +153,8 @@ type forgeFacts struct {
|
||||
}
|
||||
|
||||
// askTheForge asks the forge, through its module's tools on the bus, for a repository's id and its branch's
|
||||
// protection, waiting out a forge not served for a moment (novox/hq issue 486). A variable so a test needs no
|
||||
// forge.
|
||||
// protection. A variable so a test needs no forge.
|
||||
var askTheForge = func(ctx context.Context, owner, repo, branch string) (forgeFacts, error) {
|
||||
return forgeWaitedOut(ctx, forgeWait, owner+"/"+repo, func(ctx context.Context) (forgeFacts, error) {
|
||||
return askTheForgeOnce(ctx, owner, repo, branch)
|
||||
})
|
||||
}
|
||||
|
||||
// waitOut bounds how long a question is asked again while its answer is "not now": a pause of first, doubling
|
||||
// to most, for at most bound in all. now and sleep are the clock, so a test waits in no real time.
|
||||
type waitOut struct {
|
||||
bound, first, most time.Duration
|
||||
now func() time.Time
|
||||
sleep func(context.Context, time.Duration) error
|
||||
}
|
||||
|
||||
// forgeWait is the registration's wait for the forge (novox/hq issue 486). On 2026-10-11 a build's registration
|
||||
// asked the forge while node-tools on the control-node re-served after a rollout of the tool runner, heard
|
||||
// "nothing serves gitea.gitea_api", and failed its plan; the forge answered again seconds later. A re-serve takes
|
||||
// about 10 s, so 60 s waits one out with room to spare. It is a cap on the whole wait, an ask in flight included:
|
||||
// the registration holds the controller's one receive loop while it waits, and a loop held two minutes reads as
|
||||
// deaf. The pauses are those the build machine waits the registry with (novox/hq issue 457).
|
||||
var forgeWait = waitOut{bound: 60 * time.Second, first: time.Second, most: 10 * time.Second,
|
||||
now: time.Now, sleep: sleepOrDone}
|
||||
|
||||
// sleepOrDone pauses for d, or less when ctx ends first.
|
||||
func sleepOrDone(ctx context.Context, d time.Duration) error {
|
||||
t := time.NewTimer(d)
|
||||
defer t.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-t.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// forgeNotNow is an answer that says the forge's tool is not served now — the tool runner re-serving — not that
|
||||
// the forge refused. Only that is asked again: a refusal, an unreadable answer or a repository with no id is the
|
||||
// forge's word, and a tool that did not answer in time has already held the receive loop its whole wait.
|
||||
func forgeNotNow(err error) bool { return errors.Is(err, link.ErrNothingServes) }
|
||||
|
||||
// forgeWaitedOut asks the forge about repository, asking again while it answers "not served now", all of it
|
||||
// within w's bound, an ask in flight included (novox/hq issue 486). Past the bound it fails, saying it waited,
|
||||
// for how long, and what the forge last said; when the caller's own context ends, it says that instead.
|
||||
func forgeWaitedOut(ctx context.Context, w waitOut, repository string,
|
||||
ask func(context.Context) (forgeFacts, error)) (forgeFacts, error) {
|
||||
capped, cancel := context.WithTimeout(ctx, w.bound)
|
||||
defer cancel()
|
||||
began := w.now()
|
||||
pause := w.first
|
||||
for asked := 1; ; asked++ {
|
||||
facts, err := ask(capped)
|
||||
if err == nil {
|
||||
if asked > 1 {
|
||||
fmt.Printf("the forge answered about %s again, after %s (novox/hq issue 486)\n", repository,
|
||||
w.now().Sub(began).Round(time.Second))
|
||||
}
|
||||
return facts, nil
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return forgeFacts{}, err // the caller's own context ended: no wait of ours to speak of
|
||||
}
|
||||
gaveUp := func() (forgeFacts, error) {
|
||||
return forgeFacts{}, fmt.Errorf("waited %s for the forge to answer about %s, asking %d times, and it "+
|
||||
"did not (novox/hq issue 486): %w", w.now().Sub(began).Round(time.Second), repository, asked, err)
|
||||
}
|
||||
if capped.Err() != nil {
|
||||
return gaveUp() // the bound ended an ask in flight
|
||||
}
|
||||
if !forgeNotNow(err) {
|
||||
return forgeFacts{}, err
|
||||
}
|
||||
if w.now().Sub(began)+pause > w.bound {
|
||||
return gaveUp()
|
||||
}
|
||||
fmt.Printf("the forge cannot be asked about %s now (%v); asking again in %s, for at most %s in all "+
|
||||
"(novox/hq issue 486)\n", repository, err, pause, w.bound)
|
||||
if serr := w.sleep(capped, pause); serr != nil {
|
||||
if ctx.Err() != nil {
|
||||
return forgeFacts{}, fmt.Errorf("stopped waiting for the forge to answer about %s: %w", repository,
|
||||
ctx.Err())
|
||||
}
|
||||
return gaveUp()
|
||||
}
|
||||
pause = min(2*pause, w.most)
|
||||
}
|
||||
}
|
||||
|
||||
// askTheForgeOnce asks the forge once, through its module's tools on the bus.
|
||||
func askTheForgeOnce(ctx context.Context, owner, repo, branch string) (forgeFacts, error) {
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
return forgeFacts{}, err
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"os"
|
||||
"os/exec"
|
||||
@@ -12,7 +11,6 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
@@ -210,9 +208,7 @@ func TestARollbackNeverPutsBackABuildFromAnotherRepository(t *testing.T) {
|
||||
if _, _, err := takeIn(ctx, inv, fork); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("the fork's build was taken in: %v", err)
|
||||
}
|
||||
// Asked after the registered build, whenever the test runs: an id naming a fixed moment read as older
|
||||
// than the registered build once the clock passed it (2026-10-10 02:40 UTC), and the test failed on main.
|
||||
failed := onTrunk(fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano()), "novox/mesh-catalog", "git", "modules/sudo",
|
||||
failed := onTrunk("build-1791600000000000000", "novox/mesh-catalog", "git", "modules/sudo",
|
||||
map[string]any{"module": "sudo", "version": "2"})
|
||||
failed.Commit = "badbadbad0123456"
|
||||
if _, _, err := takeIn(ctx, inv, failed); err != nil {
|
||||
|
||||
@@ -53,26 +53,9 @@ func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Ra
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// And the work queues of seats that name their caller or their kind, with each holder's worker
|
||||
// (novox/hq ADR 0259 §3): an ask queues until the router takes it, a channel's work until that kind
|
||||
// takes it.
|
||||
trafficStreams, trafficWorkers, err := seatTrafficObjects(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Every one tried, and every failure named: one module's consumer the bus refuses is no reason
|
||||
// the modules after it in the list hear nothing (novox/hq issue 208, where this runs on each send).
|
||||
var failed []error
|
||||
for _, s := range trafficStreams {
|
||||
if err := r.EnsureStream(s); err != nil {
|
||||
failed = append(failed, fmt.Errorf("the work queue %s: %w", s.Name, err))
|
||||
}
|
||||
}
|
||||
for _, c := range trafficWorkers {
|
||||
if err := r.EnsureConsumer(c); err != nil {
|
||||
failed = append(failed, fmt.Errorf("the worker %s on %s: %w", c.Name, c.Stream, err))
|
||||
}
|
||||
}
|
||||
for _, c := range consumers {
|
||||
if err := r.EnsureConsumer(c.Consumer); err != nil {
|
||||
failed = append(failed, fmt.Errorf("how %s on %s hears what it consumes: %w", c.Module, c.Node, err))
|
||||
@@ -147,37 +130,6 @@ func moduleConsumers(ctx context.Context, inv *inventory.Inventory) ([]broker.Mo
|
||||
return broker.ConsumersOf(users), nil
|
||||
}
|
||||
|
||||
// seatTrafficObjects is the work queues and workers of seats that name their caller or their kind, from
|
||||
// the records the user list is composed from.
|
||||
func seatTrafficObjects(ctx context.Context, inv *inventory.Inventory) ([]broker.Stream, []broker.Consumer, error) {
|
||||
records, err := inv.BusRecords(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
users, err := broker.Users(records)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
streams, workers := broker.SeatTrafficObjects(users)
|
||||
// And the queue of every such seat the catalogue declares, held or not: work queues from registration,
|
||||
// so what is submitted before a holder is assigned waits for it (the correctness review of 2026-10-08).
|
||||
declared, err := inv.DeclaredTrafficSeats(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
have := map[string]bool{}
|
||||
for _, s := range streams {
|
||||
have[s.Name] = true
|
||||
}
|
||||
for _, s := range broker.TrafficQueues(declared) {
|
||||
if !have[s.Name] {
|
||||
streams = append(streams, s)
|
||||
have[s.Name] = true
|
||||
}
|
||||
}
|
||||
return streams, workers, nil
|
||||
}
|
||||
|
||||
// moduleConsumerCount is how many modules hear what they consume, for the raise's one line.
|
||||
func moduleConsumerCount(ctx context.Context, inv *inventory.Inventory) (int, error) {
|
||||
consumers, err := moduleConsumers(ctx, inv)
|
||||
|
||||
@@ -2,7 +2,6 @@ package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
@@ -29,14 +28,6 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
|
||||
for _, e := range entries {
|
||||
byName[e.Manifest.Module] = e
|
||||
}
|
||||
// The machine holding the bus, whose declaration carries every module's grants (novox/hq issue 490).
|
||||
holder := ""
|
||||
for _, e := range entries {
|
||||
if e.Manifest.BusUsers != "" && e.Manifest.ClaimsSeat(catalogue.BrokerSeat) && len(e.On) > 0 {
|
||||
holder = e.On[0]
|
||||
}
|
||||
}
|
||||
var granting []string
|
||||
machines := map[string]*link.MachinePlan{}
|
||||
machine := func(name string) *link.MachinePlan {
|
||||
if machines[name] == nil {
|
||||
@@ -59,10 +50,6 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
|
||||
machine(on).Receives = append(machine(on).Receives, name)
|
||||
}
|
||||
}
|
||||
if !waits && holder != "" && !(len(e.On) == 1 && e.On[0] == holder) && len(e.On) > 0 &&
|
||||
!slices.Contains(granting, name) {
|
||||
granting = append(granting, name)
|
||||
}
|
||||
switch {
|
||||
case (name == "nats" || catalogue.ProvidesBus(e.Manifest)) && len(e.On) > 0:
|
||||
// Said before the merge (novox/hq issue 336): a new bus build holds every send to the bus's machine
|
||||
@@ -89,13 +76,6 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(granting) > 0 {
|
||||
// Said before the merge (novox/hq issue 490): what the walk does when the change alters the bus's
|
||||
// user list. Whether it does is known only once the builds are registered, so it is said as a rule.
|
||||
p.Steps = append(p.Steps, fmt.Sprintf("%s: if this changes what the bus's user list says (a new tool, "+
|
||||
"subject or user), %s is sent that list alone, every build there kept, before %s is judged on its first "+
|
||||
"machine", grantsStepWord, holder, strings.Join(granting, ", ")))
|
||||
}
|
||||
var names []string
|
||||
for name := range machines {
|
||||
names = append(names, name)
|
||||
@@ -108,10 +88,6 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
|
||||
return p
|
||||
}
|
||||
|
||||
// grantsStepWord names the grants step (novox/hq issue 490): the bus's user list sent alone to the machine
|
||||
// holding the bus ahead of a walk's gate. Not the bus step, which replaces the bus itself.
|
||||
const grantsStepWord = "grants step"
|
||||
|
||||
// busUpgradeNeeded is how a change plan says that merging it holds the bus's machine for a person's step.
|
||||
const busUpgradeNeeded = "merging this needs a person's bus upgrade"
|
||||
|
||||
|
||||
@@ -32,9 +32,6 @@ import (
|
||||
// provision it wants that a manifest given here offers. An overflow is refused in the pull request
|
||||
// that introduces it — a new requirement, a lowered bound, a longer slug — instead of on the
|
||||
// provider's machine when a real machine's name first meets the module's.
|
||||
// SomeManifestsVar, set by the merge gate, says the manifests given are only some of their repository's.
|
||||
const SomeManifestsVar = "MESH_MODULE_CHECK_SOME"
|
||||
|
||||
func moduleCheck(paths []string, out io.Writer) error {
|
||||
return moduleCheckFor(paths, catalogue.DefaultLongestMachine, out)
|
||||
}
|
||||
@@ -88,24 +85,12 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
// Between the manifests: a seat declared twice, a use of a seat nothing declares, a claim on
|
||||
// a seat that does not exist. Run only over what parsed, because a problem inside one manifest
|
||||
// has already been said and would be said again here in a worse form.
|
||||
//
|
||||
// **Over some of a repository's manifests, a seat none of them declares is a note** (novox/hq issue 364), as
|
||||
// this command's own word says above: the merge gate passes only the manifests a change touches, and says so
|
||||
// with SomeManifestsVar, so a module that uses or claims a seat another module declares (the operator
|
||||
// channel's, a channel bench) was refused there for a manifest it was not given. Given every manifest — the
|
||||
// catalogue's own check, and registration — it stays a refusal.
|
||||
some := os.Getenv(SomeManifestsVar) != ""
|
||||
problems := catalogue.CatalogueProblems(shelf)
|
||||
sort.Strings(problems)
|
||||
for _, p := range problems {
|
||||
if some && catalogue.IsUndeclaredSeat(p) {
|
||||
fmt.Fprintf(out, "note: %s among the manifests given; registration judges it against the whole catalogue, "+
|
||||
"and passing the declaring module's manifest too judges it here\n", p)
|
||||
continue
|
||||
}
|
||||
fmt.Fprintln(out, p)
|
||||
failed++
|
||||
}
|
||||
failed += len(problems)
|
||||
|
||||
// Between the manifests too: an identity against the bounds of the provisions it wants, which
|
||||
// only the provider's manifest states.
|
||||
@@ -179,14 +164,6 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
}
|
||||
fmt.Fprintf(out, ", keeps state %s", strings.Join(kept, ", "))
|
||||
}
|
||||
// And the logs it keeps, with their caps (novox/hq ADR 0297).
|
||||
if len(m.Logs) > 0 {
|
||||
kept := make([]string, 0, len(m.Logs))
|
||||
for _, l := range m.Logs {
|
||||
kept = append(kept, fmt.Sprintf("%s (%d MiB)", l.Name, l.Cap()))
|
||||
}
|
||||
fmt.Fprintf(out, ", keeps log %s", strings.Join(kept, ", "))
|
||||
}
|
||||
if len(m.Reads) > 0 {
|
||||
fmt.Fprintf(out, ", reads %s", strings.Join(m.Reads, ", "))
|
||||
}
|
||||
|
||||
@@ -82,9 +82,7 @@ func checkHereCommand(ctx context.Context, args []string) error {
|
||||
if _, err := git("fetch", "--quiet", "origin", *base); err != nil {
|
||||
return fmt.Errorf("cannot fetch %s to say what the change touches: %w", *base, err)
|
||||
}
|
||||
// Without rename detection, so a file moved out of a build source is said under its old name as well:
|
||||
// its going is a change to the build that held it (novox/hq ADR 0267).
|
||||
changedText, err := git("diff", "--name-only", "--no-renames", "origin/"+*base+"...HEAD")
|
||||
changedText, err := git("diff", "--name-only", "origin/"+*base+"...HEAD")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -6,10 +6,8 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/beside"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The check anybody can run is the check registration runs (novox/hq issue 148, ADR 0037): a manifest
|
||||
@@ -58,7 +56,10 @@ func TestModuleCheckJudgesBetweenTheManifestsGiven(t *testing.T) {
|
||||
|
||||
// The real catalogue passes the command, the way it passes the test that used to be the only check.
|
||||
func TestModuleCheckPassesTheCatalogue(t *testing.T) {
|
||||
root := beside.Catalogue(t)
|
||||
root := filepath.Join("..", "..", "..", "mesh-catalog", "modules")
|
||||
if _, err := os.Stat(root); err != nil {
|
||||
t.Skipf("catalogue sibling not present: %v", err)
|
||||
}
|
||||
paths, err := manifestsUnder(root)
|
||||
if err != nil || len(paths) == 0 {
|
||||
t.Fatalf("no manifests under %s: %v", root, err)
|
||||
@@ -102,23 +103,3 @@ func TestTheControllersManifestServesEveryVerbOfItsSeat(t *testing.T) {
|
||||
t.Fatalf("the controller's own module.json fails module check: %v\n%s", err, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A module that uses a seat another module declares (novox/hq issue 364): refused over the whole catalogue when the
|
||||
// declarer is missing, a note when the gate says it gives only the manifests a change touches.
|
||||
func TestASeatAnotherModuleDeclaresIsANoteOverSomeManifests(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
user := filepath.Join(dir, "user.json")
|
||||
os.WriteFile(user, []byte(`{"module":"asker","version":"1","uses":["operator-channel"]}`), 0o600)
|
||||
var out bytes.Buffer
|
||||
if err := moduleCheck([]string{user}, &out); err == nil {
|
||||
t.Fatalf("a use of a seat nothing given declares passed the whole-catalogue check:\n%s", out.String())
|
||||
}
|
||||
t.Setenv(SomeManifestsVar, "1")
|
||||
out.Reset()
|
||||
if err := moduleCheck([]string{user}, &out); err != nil {
|
||||
t.Fatalf("over some manifests the use was refused:\n%s", out.String())
|
||||
}
|
||||
if !strings.Contains(out.String(), "note: asker uses the seat \"operator-channel\"") {
|
||||
t.Fatalf("the note was not said:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,8 +5,6 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path"
|
||||
"regexp"
|
||||
"runtime/debug"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -145,7 +143,7 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
read, err := readForPlanning(ctx, inv)
|
||||
read, err := inv.ReadRepositories(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -248,11 +246,9 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop
|
||||
if dir == "mesh-controller" {
|
||||
beside["mesh-controller-main"] = link.CheckedOut{Repository: url, Ref: refs["mesh-controller-main"]}
|
||||
if e.Source.Seat != "" {
|
||||
for _, sibling := range []string{"mesh-lab", "mesh-sdk"} {
|
||||
if url, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository,
|
||||
sibling)}); err == nil {
|
||||
beside[sibling] = link.CheckedOut{Repository: url, Ref: refs[sibling]}
|
||||
}
|
||||
if lab, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository,
|
||||
"mesh-lab")}); err == nil {
|
||||
beside["mesh-lab"] = link.CheckedOut{Repository: lab, Ref: refs["mesh-lab"]}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -275,54 +271,17 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop
|
||||
// catalogue, whose checkout beside is what tests read its files from, at its main, what the next merge
|
||||
// builds from; and beside the controller its main, for a judge the running controller predates, and the
|
||||
// lab's main, whose replays every check runs. **One rule, read by the check the controller asks for and by
|
||||
// the facts snapshot** (Facts.Beside), so a check run by hand clones what the build seat clones. Beside the
|
||||
// controller also the SDK, checked out at the commit the running controller's go.mod pins (sdkPinned), not
|
||||
// one a desktop holds (novox/hq issue 449). The checkout only places the clone: the conformance test reads the
|
||||
// fixtures at the pin of the tree under check, from the clone's history, so a pull request moving the SDK is
|
||||
// judged against the SDK it moves to (internal/link/conformance_test.go).
|
||||
// the facts snapshot** (Facts.Beside), so a check run by hand clones what the build seat clones.
|
||||
func besideRefs(dir, running string) map[string]string {
|
||||
switch dir {
|
||||
case "mesh-catalog":
|
||||
return map[string]string{dir: "main"}
|
||||
case "mesh-controller":
|
||||
return map[string]string{dir: running, "mesh-controller-main": "main", "mesh-lab": "main",
|
||||
"mesh-sdk": sdkPinned()}
|
||||
return map[string]string{dir: running, "mesh-controller-main": "main", "mesh-lab": "main"}
|
||||
}
|
||||
return map[string]string{dir: running}
|
||||
}
|
||||
|
||||
// sdkModule is the Go module of the SDK the controller is built against.
|
||||
const sdkModule = "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
// pseudoCommit is the commit a Go pseudo-version names: v0.1.11-0.20261009143344-f047d0a4a970 → f047d0a4a970.
|
||||
var pseudoCommit = regexp.MustCompile(`-([0-9a-f]{12})$`)
|
||||
|
||||
// sdkPinned is the ref of the SDK repository this controller was built from, as its go.mod pins it and its
|
||||
// build records it: a pseudo-version's commit, or a release's tag (the SDK tags its Go module under go/).
|
||||
// "main" only for a binary that records no SDK version — a local replace — which a running controller is
|
||||
// not (novox/hq issue 449).
|
||||
func sdkPinned() string {
|
||||
info, ok := debug.ReadBuildInfo()
|
||||
if !ok {
|
||||
return "main"
|
||||
}
|
||||
for _, dep := range info.Deps {
|
||||
if dep.Path != sdkModule {
|
||||
continue
|
||||
}
|
||||
if dep.Replace != nil {
|
||||
dep = dep.Replace
|
||||
}
|
||||
if m := pseudoCommit.FindStringSubmatch(dep.Version); m != nil {
|
||||
return m[1]
|
||||
}
|
||||
if strings.HasPrefix(dep.Version, "v") {
|
||||
return "go/" + dep.Version
|
||||
}
|
||||
}
|
||||
return "main"
|
||||
}
|
||||
|
||||
// siblingOf is another repository of the same owner: novox/mesh-controller → novox/mesh-lab.
|
||||
func siblingOf(repository, name string) string {
|
||||
if cut := strings.LastIndex(repository, "/"); cut >= 0 {
|
||||
|
||||
@@ -1,76 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The looks of the open conditions — how often each was observed, when last, its newest evidence — are
|
||||
// kept in the controller's store, not the conditions bucket (novox/hq issue 420, ADR 0289 §1): the bucket
|
||||
// is written when what a condition says changes, never for a look that saw it as it was.
|
||||
|
||||
// storeLooks keeps the conditions' looks in the inventory.
|
||||
type storeLooks struct{ inv *inventory.Inventory }
|
||||
|
||||
func (s storeLooks) KeepLooks(ctx context.Context, key string, l conditions.Looks) error {
|
||||
evidence, err := json.Marshal(l.Evidence)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.inv.KeepConditionLooks(ctx, inventory.ConditionLooks{Key: key, Observations: l.Observations,
|
||||
LastObserved: l.LastObserved, Evidence: evidence})
|
||||
}
|
||||
|
||||
func (s storeLooks) Looked(ctx context.Context, key string) (conditions.Looks, bool, error) {
|
||||
row, kept, err := s.inv.ConditionLooksOf(ctx, key)
|
||||
if err != nil || !kept {
|
||||
return conditions.Looks{}, kept, err
|
||||
}
|
||||
l, err := looksOf(row)
|
||||
return l, err == nil, err
|
||||
}
|
||||
|
||||
func (s storeLooks) AllLooks(ctx context.Context) (map[string]conditions.Looks, error) {
|
||||
rows, err := s.inv.AllConditionLooks(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make(map[string]conditions.Looks, len(rows))
|
||||
for key, row := range rows {
|
||||
l, err := looksOf(row)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[key] = l
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s storeLooks) ForgetLooks(ctx context.Context, key string) error {
|
||||
return s.inv.ForgetConditionLooks(ctx, key)
|
||||
}
|
||||
|
||||
func looksOf(row inventory.ConditionLooks) (conditions.Looks, error) {
|
||||
l := conditions.Looks{Observations: row.Observations, LastObserved: row.LastObserved.UTC()}
|
||||
if len(row.Evidence) > 0 {
|
||||
if err := json.Unmarshal(row.Evidence, &l.Evidence); err != nil {
|
||||
return conditions.Looks{}, fmt.Errorf("the evidence kept for the condition %s cannot be read: %w", row.Key, err)
|
||||
}
|
||||
}
|
||||
return l, nil
|
||||
}
|
||||
|
||||
// looksHere is where a process that is not the serving controller reads the conditions' looks: the store,
|
||||
// opened for it, closed by the function returned. A store that cannot be opened is said on every condition
|
||||
// read through it, never read as none.
|
||||
func looksHere(ctx context.Context) (conditions.LookStore, func()) {
|
||||
inv, err := openInventory(ctx)
|
||||
if err != nil {
|
||||
return conditions.NoLooks{Err: err}, func() {}
|
||||
}
|
||||
return storeLooks{inv}, inv.Close
|
||||
}
|
||||
@@ -1,87 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// **Looks at an unchanged condition write the bucket once, and `conditions` still says how often and when
|
||||
// it was seen** (novox/hq issue 420): the looks are in the controller's store, and the verb, run as a
|
||||
// process of its own, reads them from there. On 2026-10-10 one open condition had rewritten the bucket 7 003
|
||||
// times in two days.
|
||||
func TestTheConditionsVerbReadsTheLooksTheBucketNoLongerHolds(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
bus := testbus.Start(t)
|
||||
serving, err := broker.Dial(bus.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer serving.Close()
|
||||
if err := serving.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := t.Context()
|
||||
store, history, err := conditions.OnTheBus(ctx, serving.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
k := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history, Looks: storeLooks{inv}})
|
||||
defer k.Close(context.Background())
|
||||
look := conditions.Observation{Scope: conditions.ScopeMachine, ID: "anchor", Kind: "silent", Machine: "anchor",
|
||||
Severity: conditions.Warning, Summary: "anchor has not been heard from", Source: "S1"}
|
||||
for range 4 {
|
||||
if _, err := k.Observe(ctx, look); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
api, err := jetstream.New(serving.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stream, err := api.Stream(ctx, "KV_"+broker.ConditionsBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := stream.Info(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.State.LastSeq != 1 {
|
||||
t.Fatalf("four looks at one unchanged condition wrote the bucket %d times, want 1", info.State.LastSeq)
|
||||
}
|
||||
|
||||
// The verb as a process other than the serving controller: it reads the bucket and the store.
|
||||
keptBefore := conditionsFrom
|
||||
conditionsFrom = nil
|
||||
defer func() { conditionsFrom = keptBefore }()
|
||||
t.Setenv(broker.NATSVar, bus.ClientURL())
|
||||
t.Setenv(broker.CertificateVar, "")
|
||||
var shown bytes.Buffer
|
||||
if err := showCondition(ctx, []string{"machine.anchor.silent", "--json"}, &shown); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var c conditions.Condition
|
||||
if err := json.Unmarshal(shown.Bytes(), &c); err != nil {
|
||||
t.Fatalf("%v: %s", err, shown.String())
|
||||
}
|
||||
if c.Observations != 4 || len(c.Evidence) != 4 || c.LastObserved.IsZero() || c.LooksUnread != "" ||
|
||||
time.Since(c.LastObserved) > time.Minute {
|
||||
t.Fatalf("conditions show reads %d observation(s), %d evidence line(s), last %s (%q)", c.Observations,
|
||||
len(c.Evidence), c.LastObserved, c.LooksUnread)
|
||||
}
|
||||
listed, err := openConditions(ctx)
|
||||
if err != nil || len(listed) != 1 || listed[0].Observations != 4 {
|
||||
t.Fatalf("conditions lists %+v: %v", listed, err)
|
||||
}
|
||||
}
|
||||
@@ -24,18 +24,16 @@ import (
|
||||
// **`status` used to be the only place the mesh said it was wrong, and only to whoever asked.** Every
|
||||
// core failure of research 031 was found by a person looking. A condition is the mesh saying it: raised
|
||||
// by a watchdog when a signal is late (signals.go), by a probe when an invariant does not hold
|
||||
// (doctor.go), or by an event a provider sends (standing.go); kept on the bus with since-when, and its
|
||||
// looks — how often, when last, the newest evidence — in the controller's store (condition_looks.go,
|
||||
// novox/hq issue 420); said on the bus as it changes, for the operator's channel to carry; and cleared when an
|
||||
// (doctor.go), or by an event a provider sends (standing.go); kept on the bus with since-when and
|
||||
// evidence; said on the bus as it changes, for the operator's channel to carry; and cleared when an
|
||||
// observation says it is resolved. Nobody resolves one by hand. A person who knows silences it, for a
|
||||
// while, with a reason, and that is recorded as a hand act.
|
||||
|
||||
// conditionsFrom is the serving controller's keeper; nil in any other process, which opens its own.
|
||||
var conditionsFrom *conditions.Keeper
|
||||
|
||||
// keeperOn is a keeper over the store on a connection, saying its transitions on that connection, with
|
||||
// the conditions' looks kept in looks (condition_looks.go).
|
||||
func keeperOn(ctx context.Context, conn *nats.Conn, looks conditions.LookStore) (*conditions.Keeper, error) {
|
||||
// keeperOn is a keeper over the store on a connection, saying its transitions on that connection.
|
||||
func keeperOn(ctx context.Context, conn *nats.Conn) (*conditions.Keeper, error) {
|
||||
store, history, err := conditions.OnTheBus(ctx, conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -44,12 +42,12 @@ func keeperOn(ctx context.Context, conn *nats.Conn, looks conditions.LookStore)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history, Looks: looks,
|
||||
return conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history,
|
||||
Teller: link.OverNATS{Conn: conn, JS: js},
|
||||
Say: func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) },
|
||||
// What status leads with changed: composed again soon (a nudge outside the serving controller
|
||||
// does nothing).
|
||||
Changed: func() { statusFrom.nudge(); askerFrom.nudge() },
|
||||
Changed: statusFrom.nudge,
|
||||
// Written under the lease, carrying its epoch (novox/hq to-be 45 §6).
|
||||
Epoch: func() (uint64, error) { return theLease.epoch(context.WithoutCancel(ctx)) }}), nil
|
||||
}
|
||||
@@ -60,10 +58,8 @@ func withKeeper(ctx context.Context, f func(*conditions.Keeper) error) error {
|
||||
if conditionsFrom != nil {
|
||||
return f(conditionsFrom)
|
||||
}
|
||||
looks, closeLooks := looksHere(ctx)
|
||||
defer closeLooks()
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
k, err := keeperOn(ctx, conn, looks)
|
||||
k, err := keeperOn(ctx, conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -86,8 +82,6 @@ func openConditions(ctx context.Context) ([]conditions.Condition, error) {
|
||||
return nil, fmt.Errorf("this process has no bus to read the conditions from: %w", err)
|
||||
}
|
||||
var out []conditions.Condition
|
||||
looks, closeLooks := looksHere(ctx)
|
||||
defer closeLooks()
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
store, _, err := conditions.OnTheBus(ctx, conn)
|
||||
if err != nil {
|
||||
@@ -95,7 +89,7 @@ func openConditions(ctx context.Context) ([]conditions.Condition, error) {
|
||||
}
|
||||
reading, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
out, err = conditions.ReadWith(reading, store, looks)
|
||||
out, err = conditions.Read(reading, store)
|
||||
return err
|
||||
})
|
||||
return out, err
|
||||
@@ -258,14 +252,12 @@ func showCondition(ctx context.Context, args []string, w io.Writer) error {
|
||||
if conditionsFrom != nil {
|
||||
c, found, err = conditionsFrom.Get(ctx, key)
|
||||
} else {
|
||||
looks, closeLooks := looksHere(ctx)
|
||||
defer closeLooks()
|
||||
err = onTheBus(func(conn *nats.Conn) error {
|
||||
store, _, err := conditions.OnTheBus(ctx, conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c, found, err = conditions.ReadOneWith(ctx, store, looks, key)
|
||||
c, found, err = conditions.ReadOne(ctx, store, key)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
@@ -169,44 +169,6 @@ func TestAShrinkOfMoreThanHalfIsUrgent(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// THE FALSE ALARM (issue 368), replayed through the store D13 reads: an agent's home moved from the
|
||||
// operator's own home (94.7 MB) to the agent account's fresh one (490 B), and `data-shrank` was raised
|
||||
// for data that was never lost. A moved item is read against its new path only, so nothing is raised —
|
||||
// and a genuine shrink at the new path, a week of history later, still is.
|
||||
func TestAMovedPathIsNoShrinkAndAShrinkThereStillIs(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
ctx := t.Context()
|
||||
shelf := shelfFor(t, houseManifest)
|
||||
declared := []inventory.DeclaredData{{Module: "house", Item: "config", Class: "irreplaceable", Owned: true}}
|
||||
start := time.Now().Add(-6 * time.Hour)
|
||||
measure := func(at time.Time, path string, size int64) []conditions.Observation {
|
||||
t.Helper()
|
||||
if _, err := inv.RecordData(ctx, "home", declared, map[string]map[string]inventory.Measurement{"house": {
|
||||
"config": {Path: path, Size: bytesOf(size), MeasuredAt: when(at), LastWrite: when(at),
|
||||
LastBackup: when(at)}}}, "", at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
records, err := inv.Data(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
peaks, err := inv.DataPeaks(ctx, at.Add(-shrinkWindow))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return dataFindings(records, peaks, shelf, nil, nil, at)
|
||||
}
|
||||
measure(start, "/home/operator/.claude", 94_700_000)
|
||||
if got := findingsByKind(measure(start.Add(10*time.Minute), "/home/agent/.claude", 490)); got[kindDataShrank].Kind != "" {
|
||||
t.Fatalf("a moved path raised a shrink: %+v", got[kindDataShrank])
|
||||
}
|
||||
measure(start.Add(2*time.Hour), "/home/agent/.claude", 300<<20)
|
||||
got := findingsByKind(measure(start.Add(4*time.Hour), "/home/agent/.claude", 1<<20))[kindDataShrank]
|
||||
if got.Severity != conditions.Urgent || !strings.Contains(got.Summary, "shrank") {
|
||||
t.Fatalf("a genuine shrink at the new path was not raised: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Data said to be written all the time and not written; data with no backup or an old one — urgent when
|
||||
// irreplaceable, a warning when valuable; and a new item given its bound before it is said.
|
||||
func TestQuietDataAndMissingBackupsAreSaidByClass(t *testing.T) {
|
||||
|
||||
@@ -144,15 +144,11 @@ func actOnDeadLetter(ctx context.Context, on *busHandles, act string, id uint64,
|
||||
}
|
||||
switch act {
|
||||
case "deliver":
|
||||
delivered, to, err := link.DeliverAgain(on.js, id)
|
||||
_, to, err := link.DeliverAgain(on.js, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
answer["delivered_on"] = to
|
||||
if delivered.Original != "" {
|
||||
// What became of the ask in its seat's queue (novox/hq issue 334): removed, or left, and why.
|
||||
answer["original"] = delivered.Original
|
||||
}
|
||||
answer["done"] = fmt.Sprintf("dead letter %d was delivered again to %s, and nobody else; it is no longer kept",
|
||||
id, consumerWho(d.Stream, d.Consumer))
|
||||
case "drop":
|
||||
|
||||
+25
-604
@@ -10,7 +10,6 @@ import (
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
@@ -83,38 +82,6 @@ func waitingNote(p inventory.Plan) string {
|
||||
p.Delivery.Awaits, p.ID)
|
||||
}
|
||||
|
||||
// waitingSaid is what a walk waiting for its word says when the delivery's owner said what holds it (novox/hq
|
||||
// issue 529): a delivery held for a person, its reason and the verb that releases it — the person's decision, not
|
||||
// a hand start — or the deliveries not at their turn. Told nothing, it is waitingNote.
|
||||
func waitingSaid(p inventory.Plan, lines []stalledLine) string {
|
||||
if len(lines) == 0 {
|
||||
return waitingNote(p)
|
||||
}
|
||||
var held, unheld, other []string
|
||||
for _, l := range lines {
|
||||
switch l.State {
|
||||
case "held":
|
||||
held = append(held, l.Says)
|
||||
case "unheld":
|
||||
unheld = append(unheld, l.Says)
|
||||
default:
|
||||
other = append(other, l.Says)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case len(held) > 0:
|
||||
return fmt.Sprintf("published; its walk waits for %s's word, and a delivery it carries is held for a person: %s",
|
||||
p.Delivery.Awaits, strings.Join(held, "; "))
|
||||
case len(other) > 0:
|
||||
// Not at its turn: it goes when its turn comes. A hand start here would skip its group's order (review of
|
||||
// novox/hq issue 529), so none is offered.
|
||||
return fmt.Sprintf("published; its walk waits for %s's word: %s", p.Delivery.Awaits, strings.Join(other, "; "))
|
||||
}
|
||||
// Held by nothing: nothing will give it its word, so the hand start is the way on.
|
||||
return fmt.Sprintf("published; its walk waits for %s's word: %s — `plans go %s --why …` starts it by hand",
|
||||
p.Delivery.Awaits, strings.Join(unheld, "; "), p.ID)
|
||||
}
|
||||
|
||||
// letGo gives a waiting walk its word: by the delivery's owner, or a person. The next advance asks its
|
||||
// first tier. Refused for a walk that does not wait.
|
||||
func letGo(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, error) {
|
||||
@@ -136,8 +103,6 @@ func letGo(ctx context.Context, inv *inventory.Inventory, id, by, why string) (i
|
||||
return p, fmt.Errorf("%s was let go by %s at %s already", p.ID, p.Delivery.By,
|
||||
p.Delivery.Go.Local().Format("15:04:05"))
|
||||
}
|
||||
// **One walk at a time** (novox/hq ADR 0276): the word is taken, and the walk starts once no other walk is
|
||||
// started (advanceOnce), so the delivery's owner says it once and is not refused.
|
||||
now := time.Now().UTC()
|
||||
p.Delivery.Go, p.Delivery.By, p.Delivery.Why = &now, by, why
|
||||
p.Note = "let go by " + by + "; its first tier is asked next"
|
||||
@@ -150,419 +115,28 @@ func letGo(ctx context.Context, inv *inventory.Inventory, id, by, why string) (i
|
||||
// stopWalk ends a walk on its delivery's word: failed, said as stopped by whom, why. What it asked still
|
||||
// builds and registers; nothing further is asked or sent.
|
||||
func stopWalk(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, error) {
|
||||
p, _, err := stopWalkWithout(ctx, inv, id, by, why, nil, time.Now().UTC())
|
||||
return p, err
|
||||
}
|
||||
|
||||
// stopAnswer is what a stop that leaves merges out says, as data (novox/hq ADR 0299): the record stopped, the
|
||||
// batch that walks its other merges again with those merges, every merge left out, the modules the stopped walk
|
||||
// already sent that stay on the machines named, the modules the batch's walk withholds, and every merge elsewhere
|
||||
// that will deliver a stopped change when it is walked — a later merge of its repository, or one moving its modules.
|
||||
type stopAnswer struct {
|
||||
Stopped string `json:"stopped"`
|
||||
WalkedAgainBy string `json:"walked_again_by,omitempty"`
|
||||
WalkedAgain []string `json:"walked_again,omitempty"`
|
||||
LeftOut []inventory.PlanLeftOut `json:"left_out,omitempty"`
|
||||
Kept []inventory.PlanKept `json:"kept,omitempty"`
|
||||
Withheld []string `json:"withheld,omitempty"`
|
||||
StillCarriedBy []string `json:"still_carried_by,omitempty"`
|
||||
}
|
||||
|
||||
// stopWalkWithout ends a walk on its delivery's word and, given merges to leave out, walks every other merge it
|
||||
// answered again without them (novox/hq issue 459, ADR 0299): one walk carries every merge of its batch, so ending
|
||||
// it for one group member ended every other delivery on it.
|
||||
//
|
||||
// - Left out are the merges named, each later merge of their repository's branch, which contains one, and each
|
||||
// merge moving a module a left-out merge moves: a module is built at its branch, which holds the stopped merge.
|
||||
// - The others join the open batch of their kind — merges heard while the walk ran wait there, and an older
|
||||
// commit never walks after a newer one — or a new batch when none is open; its window is closed, and its walk
|
||||
// withholds the left-out merges' modules where it would build them only as dependents.
|
||||
// - Given a batch not yet cut, the merges left out leave it for a stopped record of their own, and the batch
|
||||
// goes on without them; with nothing left in it, the batch itself is stopped.
|
||||
// - The stopped record, the batch and every merge moved are written in one transaction (decision 7).
|
||||
//
|
||||
// With nothing left to walk, or no merge named, the walk is only ended. A merge named that the record does not
|
||||
// answer refuses the stop whole.
|
||||
func stopWalkWithout(ctx context.Context, inv *inventory.Inventory, id, by, why string, without []string,
|
||||
now time.Time) (inventory.Plan, stopAnswer, error) {
|
||||
said := stopAnswer{Stopped: id}
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, said, err
|
||||
return inventory.Plan{}, err
|
||||
}
|
||||
defer release()
|
||||
p, err := inv.PlanByID(ctx, id)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, said, err
|
||||
return inventory.Plan{}, err
|
||||
}
|
||||
batch := p.Batch()
|
||||
if !p.Open() && !(batch && len(without) > 0) {
|
||||
return p, said, fmt.Errorf("%s is already %s", p.ID, p.State)
|
||||
if !p.Open() {
|
||||
return p, fmt.Errorf("%s is already %s", p.ID, p.State)
|
||||
}
|
||||
var merges, again []inventory.BatchedMerge
|
||||
if len(without) > 0 {
|
||||
if merges, err = inv.MergesOf(ctx, p.ID); err != nil {
|
||||
return p, said, err
|
||||
}
|
||||
if again, said.LeftOut, err = leaveOut(merges, without); err != nil {
|
||||
return p, said, fmt.Errorf("%s is not stopped: %w", p.ID, err)
|
||||
}
|
||||
if p.Delivery == nil {
|
||||
p.Delivery = &inventory.PlanDelivery{}
|
||||
}
|
||||
stopped := func(r *inventory.Plan) {
|
||||
if r.Delivery == nil {
|
||||
r.Delivery = &inventory.PlanDelivery{}
|
||||
}
|
||||
r.Delivery.Stopped, r.Delivery.StoppedWhy, r.Delivery.LeftOut = by, why, said.LeftOut
|
||||
r.State = inventory.PlanFailed
|
||||
r.Note = fmt.Sprintf("stopped by %s at tier %d: %s", by, r.Tier, why)
|
||||
p.Delivery.Stopped, p.Delivery.StoppedWhy = by, why
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = fmt.Sprintf("stopped by %s at tier %d: %s", by, p.Tier, why)
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
return p, err
|
||||
}
|
||||
withheld := movesOf(leftOutMerges(merges, said.LeftOut))
|
||||
said.Withheld = withheld
|
||||
if said.StillCarriedBy, err = stillCarried(ctx, inv, p.ID, merges, said.LeftOut, withheld); err != nil {
|
||||
return p, said, err
|
||||
}
|
||||
if batch {
|
||||
return p, said, stopInBatch(ctx, inv, &p, merges, again, &said, stopped, now)
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return p, said, err
|
||||
}
|
||||
running := map[string][]string{}
|
||||
for _, e := range entries {
|
||||
running[e.Manifest.Module] = e.On
|
||||
}
|
||||
said.Kept = keptOf(p, movesOf(again), running)
|
||||
stopped(&p)
|
||||
p.Delivery.Kept = said.Kept
|
||||
if len(again) == 0 {
|
||||
if len(said.LeftOut) > 0 {
|
||||
p.Note += fmt.Sprintf("; left out %s, nothing else to walk", mergesLeftOutWords(said.LeftOut))
|
||||
}
|
||||
return p, said, inv.SavePlan(ctx, &p)
|
||||
}
|
||||
target, folded, foldMoves, err := batchToWalkAgain(ctx, inv, p.ID, again, now)
|
||||
if err != nil {
|
||||
return p, said, fmt.Errorf("%s is not stopped: %w", p.ID, err)
|
||||
}
|
||||
target.Delivery.Batch.Closed = true
|
||||
target.Delivery.Batch.Withheld = union(target.Delivery.Batch.Withheld, withheld)
|
||||
sort.Strings(target.Delivery.Batch.Withheld)
|
||||
said.WalkedAgainBy = target.ID
|
||||
var moves []inventory.MergeMove
|
||||
for _, m := range again {
|
||||
said.WalkedAgain = append(said.WalkedAgain, m.Repository+"@"+m.Commit)
|
||||
moves = append(moves, inventory.MergeMove{Repository: m.Repository, Commit: m.Commit, Plan: target.ID, Alone: m.Alone})
|
||||
}
|
||||
p.Delivery.WalkedAgainBy = target.ID
|
||||
p.Note += fmt.Sprintf("; its other merges are walked again by %s: %s", target.ID, strings.Join(said.WalkedAgain, ", "))
|
||||
if len(said.LeftOut) > 0 {
|
||||
p.Note += "; left out " + mergesLeftOutWords(said.LeftOut)
|
||||
}
|
||||
if len(said.Kept) > 0 {
|
||||
p.Note += "; " + keptWords(said.Kept)
|
||||
}
|
||||
// The batch first, so no merge names a plan the store does not hold; all of it, or nothing (decision 7) — the
|
||||
// batches folded into it too.
|
||||
if err := inv.SavePlansMoving(ctx, append([]*inventory.Plan{&target, &p}, folded...),
|
||||
append(moves, foldMoves...)); err != nil {
|
||||
return p, said, fmt.Errorf("%s is not stopped, and none of its merges moved: %w", p.ID, err)
|
||||
}
|
||||
// What the batch holds and names now: written again by the cutter's next look should this fail.
|
||||
if err := keepBatch(ctx, inv, &target, now, ""); err != nil {
|
||||
fmt.Printf(" %s holds its merges; its record is written again at the next look: %v\n", target.ID, err)
|
||||
}
|
||||
fmt.Printf("%s stopped by %s; %s walks its other merges again: %s\n", p.ID, by, target.ID,
|
||||
strings.Join(said.WalkedAgain, ", "))
|
||||
return p, said, nil
|
||||
}
|
||||
|
||||
// stopInBatch leaves merges out of a batch not yet cut (ADR 0299): they go to a stopped record of their own, which
|
||||
// names the batch as walking the others, and the batch withholds their modules; a batch left with none is stopped
|
||||
// itself.
|
||||
func stopInBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan, merges, again []inventory.BatchedMerge,
|
||||
said *stopAnswer, stopped func(*inventory.Plan), now time.Time) error {
|
||||
if len(again) == 0 {
|
||||
stopped(b)
|
||||
b.Delivery.Batch = nil
|
||||
b.Note = "a batch whose every merge was left out by a stop: " + b.Note
|
||||
return inv.SavePlan(ctx, b)
|
||||
}
|
||||
left := leftOutMerges(merges, said.LeftOut)
|
||||
first := left[0]
|
||||
r := inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch,
|
||||
Commit: first.Commit, Merged: first.Merged, Created: now, Tiers: [][]string{},
|
||||
Modules: map[string]*inventory.PlanModule{}}
|
||||
if r.ID == b.ID {
|
||||
r.ID += "-left"
|
||||
}
|
||||
stopped(&r)
|
||||
r.Delivery.WalkedAgainBy = b.ID
|
||||
r.Note += fmt.Sprintf("; left out of %s before it was walked: %s; %s walks the others", b.ID,
|
||||
mergesLeftOutWords(said.LeftOut), b.ID)
|
||||
b.Delivery.Batch.Withheld = union(b.Delivery.Batch.Withheld, said.Withheld)
|
||||
sort.Strings(b.Delivery.Batch.Withheld)
|
||||
var moves []inventory.MergeMove
|
||||
for _, m := range left {
|
||||
moves = append(moves, inventory.MergeMove{Repository: m.Repository, Commit: m.Commit, Plan: r.ID, Alone: m.Alone})
|
||||
}
|
||||
for _, m := range again {
|
||||
said.WalkedAgain = append(said.WalkedAgain, m.Repository+"@"+m.Commit)
|
||||
}
|
||||
said.Stopped, said.WalkedAgainBy = r.ID, b.ID
|
||||
if err := inv.SavePlansMoving(ctx, []*inventory.Plan{&r, b}, moves); err != nil {
|
||||
return fmt.Errorf("%s is not changed, and none of its merges moved: %w", b.ID, err)
|
||||
}
|
||||
if err := keepBatch(ctx, inv, b, now, ""); err != nil {
|
||||
fmt.Printf(" %s holds its merges; its record is written again at the next look: %v\n", b.ID, err)
|
||||
}
|
||||
fmt.Printf("%s: %s left out by a stop, kept by %s\n", b.ID, mergesLeftOutWords(said.LeftOut), r.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// batchToWalkAgain is the batch a stopped walk's merges join (novox/hq ADR 0299): a batch of their kind not yet cut
|
||||
// that they are related to (ADR 0304 decision 1, ADR 0306 decision 3), or a new one, not yet written. Related to
|
||||
// two or more, the newer are folded into the oldest: the plans and merge moves that fold them, to be written in
|
||||
// the stop's one act. The kind is read from what the merges moved; the stopped walk itself is no walk a cut folds.
|
||||
func batchToWalkAgain(ctx context.Context, inv *inventory.Inventory, stopping string, again []inventory.BatchedMerge,
|
||||
now time.Time) (inventory.Plan, []*inventory.Plan, []inventory.MergeMove, error) {
|
||||
own := false
|
||||
for _, n := range movesOf(again) {
|
||||
if _, on := onTheControllersPath[n]; on {
|
||||
own = true
|
||||
}
|
||||
}
|
||||
entries, read, edges, err := theGraph(ctx, inv)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, nil, nil, err
|
||||
}
|
||||
joins, _, _, err := relatedBatches(ctx, inv, again, own, stopping, entries, read, edges)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, nil, nil, err
|
||||
}
|
||||
if len(joins) > 0 {
|
||||
into := joins[0]
|
||||
if len(joins) == 1 {
|
||||
return into, nil, nil, nil
|
||||
}
|
||||
plans, moves, err := foldInto(ctx, inv, &into, joins[1:])
|
||||
if err != nil {
|
||||
return inventory.Plan{}, nil, nil, err
|
||||
}
|
||||
return into, plans[1:], moves, nil
|
||||
}
|
||||
first := again[0]
|
||||
return inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch,
|
||||
Commit: first.Commit, Merged: first.Merged, Created: now, State: inventory.PlanAssembling,
|
||||
Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{},
|
||||
Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: own}}}, nil, nil, nil
|
||||
}
|
||||
|
||||
// leaveOut splits a walk's merges by the merges named to leave out, each `<repository>@<commit>` (a commit's
|
||||
// first characters will do): the merges walked again, and those left out — each named one; each later merge of
|
||||
// its repository's branch, which contains it; and each merge moving a module a left-out merge moves, which is
|
||||
// built at its branch and so holds the stopped merge — until none more is. A name the walk does not answer is
|
||||
// refused.
|
||||
func leaveOut(merges []inventory.BatchedMerge, without []string) ([]inventory.BatchedMerge, []inventory.PlanLeftOut, error) {
|
||||
var stopped []inventory.BatchedMerge
|
||||
for _, w := range without {
|
||||
repository, commit, ok := strings.Cut(strings.TrimSpace(w), "@")
|
||||
if !ok || repository == "" || len(commit) < 7 {
|
||||
return nil, nil, fmt.Errorf("%q names no merge: <repository>@<commit>", w)
|
||||
}
|
||||
i := slices.IndexFunc(merges, func(m inventory.BatchedMerge) bool {
|
||||
return strings.EqualFold(m.Repository, repository) && strings.HasPrefix(m.Commit, commit)
|
||||
})
|
||||
if i < 0 {
|
||||
return nil, nil, fmt.Errorf("it answers no merge %s@%s", repository, commit)
|
||||
}
|
||||
stopped = append(stopped, merges[i])
|
||||
}
|
||||
same := func(a, b inventory.BatchedMerge) bool {
|
||||
return strings.EqualFold(a.Repository, b.Repository) && a.Commit == b.Commit
|
||||
}
|
||||
out := map[int]inventory.PlanLeftOut{}
|
||||
for i, m := range merges {
|
||||
if slices.ContainsFunc(stopped, func(s inventory.BatchedMerge) bool { return same(s, m) }) {
|
||||
out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit}
|
||||
}
|
||||
}
|
||||
for grew := true; grew; {
|
||||
grew = false
|
||||
for i, m := range merges {
|
||||
if _, left := out[i]; left {
|
||||
continue
|
||||
}
|
||||
for j, o := range out {
|
||||
l := merges[j]
|
||||
contains := o.Contains
|
||||
if contains == "" {
|
||||
contains = l.Commit
|
||||
}
|
||||
if strings.EqualFold(l.Repository, m.Repository) && l.Branch == m.Branch && laterMerge(m, l) {
|
||||
out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit, Contains: contains}
|
||||
} else if shared := sharedMoves(m, l); len(shared) > 0 {
|
||||
out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit, Contains: contains,
|
||||
Shares: shared}
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
grew = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
var again []inventory.BatchedMerge
|
||||
var left []inventory.PlanLeftOut
|
||||
for i, m := range merges {
|
||||
if o, is := out[i]; is {
|
||||
left = append(left, o)
|
||||
} else {
|
||||
again = append(again, m)
|
||||
}
|
||||
}
|
||||
sort.SliceStable(left, func(i, j int) bool { return left[i].Contains == "" && left[j].Contains != "" })
|
||||
return again, left, nil
|
||||
}
|
||||
|
||||
// sharedMoves are the modules two kept merges both moved when they were heard.
|
||||
func sharedMoves(a, b inventory.BatchedMerge) []string {
|
||||
var out []string
|
||||
theirs := announcedOf(b).Moves
|
||||
for _, n := range announcedOf(a).Moves {
|
||||
if slices.Contains(theirs, n) && !slices.Contains(out, n) {
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// leftOutMerges are the kept merges a stop left out.
|
||||
func leftOutMerges(merges []inventory.BatchedMerge, out []inventory.PlanLeftOut) []inventory.BatchedMerge {
|
||||
var left []inventory.BatchedMerge
|
||||
for _, m := range merges {
|
||||
if slices.ContainsFunc(out, func(o inventory.PlanLeftOut) bool {
|
||||
return strings.EqualFold(o.Repository, m.Repository) && o.Commit == m.Commit
|
||||
}) {
|
||||
left = append(left, m)
|
||||
}
|
||||
}
|
||||
return left
|
||||
}
|
||||
|
||||
// movesOf is every module kept merges moved when they were heard, sorted.
|
||||
func movesOf(merges []inventory.BatchedMerge) []string {
|
||||
var out []string
|
||||
for _, m := range merges {
|
||||
out = union(out, announcedOf(m).Moves)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// keptOf are the modules a stopped walk sent that no merge walked again moves, and the machines each reached:
|
||||
// those keep running the stopped walk's build. A module sent to its first machines reached those, and the rest it
|
||||
// was sent to after them; one sent with no first machine (its policy sends it to all together, or a later tier is
|
||||
// built by it) reached every machine running it.
|
||||
func keptOf(p inventory.Plan, walkedAgain []string, running map[string][]string) []inventory.PlanKept {
|
||||
var out []inventory.PlanKept
|
||||
for name, m := range p.Modules {
|
||||
if m == nil || slices.Contains(walkedAgain, name) {
|
||||
continue
|
||||
}
|
||||
var machines []string
|
||||
if m.FirstAt != nil || m.SentAt != nil {
|
||||
machines = union(machines, m.First)
|
||||
}
|
||||
if m.SentAt != nil && len(m.First) == 0 {
|
||||
machines = union(machines, running[name])
|
||||
}
|
||||
if m.Gate != nil && m.Gate.Since != nil {
|
||||
machines = union(machines, m.Gate.Machines)
|
||||
}
|
||||
for machine := range m.Rest {
|
||||
machines = union(machines, []string{machine})
|
||||
}
|
||||
if len(machines) == 0 {
|
||||
continue
|
||||
}
|
||||
sort.Strings(machines)
|
||||
out = append(out, inventory.PlanKept{Module: name, Machines: machines})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module })
|
||||
return out
|
||||
}
|
||||
|
||||
// keptWords is what a stopped walk leaves on the machines, as a person reads it.
|
||||
func keptWords(kept []inventory.PlanKept) string {
|
||||
var words []string
|
||||
for _, k := range kept {
|
||||
words = append(words, k.Module+" on "+strings.Join(k.Machines, ", "))
|
||||
}
|
||||
return "left running the stopped walk's build, which nothing walks again: " + strings.Join(words, "; ")
|
||||
}
|
||||
|
||||
// stillCarried are the merges outside the stopped record, in a batch or an open walk, that will deliver a stopped
|
||||
// change when they are walked: a later merge of its repository's branch, which contains it, or one that moves a
|
||||
// module it moves. Holding a merged change back for good is a revert's (ADR 0299).
|
||||
func stillCarried(ctx context.Context, inv *inventory.Inventory, id string, merges []inventory.BatchedMerge,
|
||||
out []inventory.PlanLeftOut, withheld []string) ([]string, error) {
|
||||
var named []inventory.BatchedMerge
|
||||
for _, m := range leftOutMerges(merges, out) {
|
||||
if slices.ContainsFunc(out, func(o inventory.PlanLeftOut) bool { return o.Commit == m.Commit && o.Contains == "" }) {
|
||||
named = append(named, m)
|
||||
}
|
||||
}
|
||||
if len(named) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
open, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
batches, err := inv.Batches(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var carried []string
|
||||
for _, p := range append(open, batches...) {
|
||||
if p.ID == id {
|
||||
continue
|
||||
}
|
||||
ms, err := inv.MergesOf(ctx, p.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, m := range ms {
|
||||
later := slices.ContainsFunc(named, func(s inventory.BatchedMerge) bool {
|
||||
return strings.EqualFold(s.Repository, m.Repository) && s.Branch == m.Branch && laterMerge(m, s)
|
||||
})
|
||||
shares := slices.ContainsFunc(announcedOf(m).Moves, func(n string) bool { return slices.Contains(withheld, n) })
|
||||
if later || shares {
|
||||
carried = append(carried, fmt.Sprintf("%s@%s (in %s)", m.Repository, m.Commit, p.ID))
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(carried)
|
||||
return carried, nil
|
||||
}
|
||||
|
||||
// mergesLeftOutWords is the merges a stop left out, as a person reads them: each that contains a stopped one, or
|
||||
// shares its modules, says so.
|
||||
func mergesLeftOutWords(out []inventory.PlanLeftOut) string {
|
||||
var words []string
|
||||
for _, o := range out {
|
||||
s := o.Repository + "@" + short(o.Commit)
|
||||
switch {
|
||||
case len(o.Shares) > 0:
|
||||
s += " (which moves " + strings.Join(o.Shares, ", ") + " as " + short(o.Contains) + " does)"
|
||||
case o.Contains != "":
|
||||
s += " (which contains " + short(o.Contains) + ")"
|
||||
}
|
||||
words = append(words, s)
|
||||
}
|
||||
return strings.Join(words, ", ")
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// orderMember is one member of a group, as mesh-delivery says it.
|
||||
@@ -890,8 +464,6 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
group := set.String("group", "", "a delivery group's id")
|
||||
by := set.String("by", catalogue.DeliverySeat, "who says it")
|
||||
why := set.String("why", "", "why")
|
||||
without := set.String("without", "", "merges a stop leaves out, <repository>@<commit>, comma-separated: the walk's "+
|
||||
"other merges are walked again without them (novox/hq ADR 0299)")
|
||||
limit := set.Int("n", 50, "how many ended walks to answer beside the open ones")
|
||||
planID := set.String("plan", "", "one walk")
|
||||
positionals, err := parseAround(set, rest)
|
||||
@@ -949,7 +521,7 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
if *group == "" && len(members) == 1 {
|
||||
// One head, checked again as the forge's announcement would have it (a recheck): the controller's
|
||||
// own path for a pull request, run because the delivery's owner asked.
|
||||
if err := sayingOnTheBus(ctx, inv, func() error { return (following{open}).PullUpdated(ctx, members[0].pull()) }); err != nil {
|
||||
if err := sayingOnTheBus(ctx, func() error { return (following{open}).PullUpdated(ctx, members[0].pull()) }); err != nil {
|
||||
return err
|
||||
}
|
||||
return answer(map[string]any{"rechecked": members[0].ID})
|
||||
@@ -967,7 +539,7 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
return errors.New("delivery go <plan> [--by <who>] [--why <text>]")
|
||||
}
|
||||
var p inventory.Plan
|
||||
if err := sayingOnTheBus(ctx, inv, func() (err error) {
|
||||
if err := sayingOnTheBus(ctx, func() (err error) {
|
||||
p, err = letGo(ctx, inv, positionals[0], *by, strings.TrimSpace(*why))
|
||||
return err
|
||||
}); err != nil {
|
||||
@@ -978,21 +550,15 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
return nil
|
||||
case "stop":
|
||||
if len(positionals) != 1 || strings.TrimSpace(*why) == "" {
|
||||
return errors.New("delivery stop <plan> --why <text> [--by <who>] [--without <repository>@<commit>,…]")
|
||||
return errors.New("delivery stop <plan> --why <text> [--by <who>]")
|
||||
}
|
||||
var p inventory.Plan
|
||||
var said stopAnswer
|
||||
if err := sayingOnTheBus(ctx, inv, func() (err error) {
|
||||
p, said, err = stopWalkWithout(ctx, inv, positionals[0], *by, strings.TrimSpace(*why), splitList(*without),
|
||||
time.Now().UTC())
|
||||
if err := sayingOnTheBus(ctx, func() (err error) {
|
||||
p, err = stopWalk(ctx, inv, positionals[0], *by, strings.TrimSpace(*why))
|
||||
return err
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if *without != "" {
|
||||
// What it left out and what it walks again, as data: the delivery's owner reads it (ADR 0299).
|
||||
return answer(said)
|
||||
}
|
||||
fmt.Printf("%s stopped by %s at tier %d of %d; what was asked still builds and registers, nothing further "+
|
||||
"is asked or sent\n", p.ID, *by, p.Tier, len(p.Tiers))
|
||||
return nil
|
||||
@@ -1008,12 +574,6 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
if walks, err = inv.OpenPlans(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
// And the batch being assembled (novox/hq ADR 0276): never a walk to link or let go, shown.
|
||||
batches, err := inv.Batches(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
walks = append(walks, batches...)
|
||||
recent, err := inv.RecentPlans(ctx, *limit)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1028,81 +588,12 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Each walk's phases, with the rest's reports (novox/hq ADR 0282 decision 6).
|
||||
now := time.Now()
|
||||
for i := range walks {
|
||||
walks[i].Phases = walks[i].WalkPhases(now, appliedFrom(ctx, inv))
|
||||
}
|
||||
return answer(map[string]any{"held": deliverySeatHeld(entries), "walks": walks,
|
||||
"own-path": sortedKeysOf(ownPathWords())})
|
||||
}
|
||||
return fmt.Errorf("delivery %s: plan, order, check, go, stop or walks", sub)
|
||||
}
|
||||
|
||||
// appliedFrom answers a machine's first report after a send from the controller's `apply` durations; a lookup
|
||||
// that fails is a report not read, which leaves the walk's end unknown rather than wrong.
|
||||
func appliedFrom(ctx context.Context, inv *inventory.Inventory) inventory.AppliedLookup {
|
||||
return func(node string, sent time.Time) (inventory.AppliedReport, bool) {
|
||||
r, ok, err := inv.FirstAppliedAfter(ctx, node, sent)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "the report of %s after %s could not be read: %v\n", node, sent.Format(time.RFC3339), err)
|
||||
return inventory.AppliedReport{}, false
|
||||
}
|
||||
return r, ok
|
||||
}
|
||||
}
|
||||
|
||||
// recordWalkPhases keeps, once, each phase of every walk ended lately whose end is known, as a duration of kind
|
||||
// walk-phase per class (novox/hq ADR 0282 decision 6): what `durations` summarises. A walk whose end is unknown
|
||||
// past ApplySilentAfter keeps its measured phases without its total.
|
||||
func recordWalkPhases(ctx context.Context, inv *inventory.Inventory, now time.Time) error {
|
||||
recent, err := inv.RecentPlans(ctx, 30)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, p := range recent {
|
||||
if p.State != inventory.PlanDone || p.Release != nil || now.Sub(p.Updated) > 2*time.Hour {
|
||||
continue
|
||||
}
|
||||
anyKept, totalKept, err := inv.WalkPhasesKept(ctx, p.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if totalKept {
|
||||
continue
|
||||
}
|
||||
ph := p.WalkPhases(now, appliedFrom(ctx, inv))
|
||||
if ph != nil && ph.End == nil && anyKept {
|
||||
continue // kept without its end; kept again only once its end is known
|
||||
}
|
||||
if ph == nil || (ph.End == nil && now.Sub(p.Updated) < inventory.ApplySilentAfter+time.Minute) {
|
||||
continue
|
||||
}
|
||||
class := ph.Class
|
||||
if class == "" {
|
||||
class = "unclassed"
|
||||
}
|
||||
for _, x := range ph.Phases {
|
||||
if x.State != inventory.PhaseMeasured || x.Start == nil {
|
||||
continue
|
||||
}
|
||||
if err := inv.RecordDuration(ctx, inventory.Duration{Kind: inventory.DurationWalkPhase,
|
||||
Subject: class + "/" + x.Name, Ref: fmt.Sprintf("%s/%s/%d", p.ID, x.Name, x.Tier), Started: *x.Start,
|
||||
Took: time.Duration(x.TookMS) * time.Millisecond, Detail: p.Named()}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if ph.End != nil && ph.From != nil {
|
||||
if err := inv.RecordDuration(ctx, inventory.Duration{Kind: inventory.DurationWalkPhase,
|
||||
Subject: class + "/total", Ref: p.ID + "/total", Started: *ph.From,
|
||||
Took: time.Duration(ph.TotalMS) * time.Millisecond, Detail: ph.Said}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ownPathWords is the controller's own path as words, for an answer.
|
||||
func ownPathWords() map[string]string { return onTheControllersPath }
|
||||
|
||||
@@ -1113,7 +604,7 @@ func theGraph(ctx context.Context, inv *inventory.Inventory) ([]inventory.Entry,
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
read, err := readForPlanning(ctx, inv)
|
||||
read, err := inv.ReadRepositories(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
@@ -1224,19 +715,12 @@ func shelfOf(entries []inventory.Entry) map[string]catalogue.Manifest {
|
||||
// delivery it walks reads its steps from. Never in the way of the walk: said beside it, and a save that could
|
||||
// not be said is logged — the delivery's owner, which also asks for the walks it follows, finds it by
|
||||
// comparison. Records arriving out of order are told apart by the plan's revision.
|
||||
//
|
||||
// On the save that starts the walk and the one that ends it, the walk also says what it touches, its bound and,
|
||||
// at the end, its outcome (novox/hq ADR 0308 decision 1, walk_said.go), read from facts; nil facts say only what
|
||||
// the walk recorded.
|
||||
func sayPlanMoved(ctx context.Context, bus link.Bus, facts walkFacts, p inventory.Plan) {
|
||||
go publishPlanMoved(ctx, bus, facts, p)
|
||||
func sayPlanMoved(ctx context.Context, bus link.Bus, p inventory.Plan) {
|
||||
go publishPlanMoved(ctx, bus, p)
|
||||
}
|
||||
|
||||
func publishPlanMoved(ctx context.Context, bus link.Bus, facts walkFacts, p inventory.Plan) {
|
||||
// Its phases so far (novox/hq ADR 0282 decision 6): the rest's reports come after the walk ends, and are
|
||||
// read by whoever asks for the walk (`delivery walks`).
|
||||
p.Phases = p.WalkPhases(time.Now(), nil)
|
||||
body, err := json.Marshal(planMovedBody{Plan: p, walkSaid: sayWalk(ctx, facts, p)})
|
||||
func publishPlanMoved(ctx context.Context, bus link.Bus, p inventory.Plan) {
|
||||
body, err := json.Marshal(p)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -1251,7 +735,7 @@ func publishPlanMoved(ctx context.Context, bus link.Bus, facts walkFacts, p inve
|
||||
// controller's, or a connection of the command's own — a verb runs as a command of its own, and what it kept
|
||||
// would otherwise be said by nobody until the delivery's owner read the walks back. Without a bus the command
|
||||
// still runs; what it did is found by comparison.
|
||||
func sayingOnTheBus(ctx context.Context, facts walkFacts, f func() error) error {
|
||||
func sayingOnTheBus(ctx context.Context, f func() error) error {
|
||||
if checkEvents != nil {
|
||||
return f()
|
||||
}
|
||||
@@ -1262,10 +746,9 @@ func sayingOnTheBus(ctx context.Context, facts walkFacts, f func() error) error
|
||||
return err
|
||||
}
|
||||
bus := link.OverNATS{Conn: conn, JS: js}
|
||||
was := inventory.PlanSaved
|
||||
checkEvents = bus
|
||||
inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { publishPlanMoved(ctx, bus, facts, p) }
|
||||
defer func() { checkEvents, inventory.PlanSaved = nil, was }()
|
||||
inventory.PlanSaved = func(p inventory.Plan) { publishPlanMoved(ctx, bus, p) }
|
||||
defer func() { checkEvents, inventory.PlanSaved = nil, nil }()
|
||||
ran = true
|
||||
return f()
|
||||
})
|
||||
@@ -1275,65 +758,3 @@ func sayingOnTheBus(ctx context.Context, facts walkFacts, f func() error) error
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// sayingWhatAVerbKept is PlanSaved for a process that is not the serving controller: each walk the verb keeps is
|
||||
// said as `plan-moved` before the save returns, with the facts of the inventory that kept it, on the bus the verb
|
||||
// reaches. It reaches the bus on the first save and never for a verb that keeps no walk. When the bus cannot be
|
||||
// reached, a save that started or ended a walk says so on the verb's error output: its mark is kept and no later
|
||||
// save says it, so the router holds what the walk touched until its bound, or not at all for a start. done lets
|
||||
// the bus go and takes the hook back.
|
||||
func sayingWhatAVerbKept(ctx context.Context) (said func(*inventory.Inventory, inventory.Plan), done func()) {
|
||||
var (
|
||||
mu sync.Mutex
|
||||
reached bool
|
||||
bus link.Bus
|
||||
closing func()
|
||||
failed error
|
||||
)
|
||||
said = func(inv *inventory.Inventory, p inventory.Plan) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if !reached {
|
||||
reached = true
|
||||
if handActConn != nil {
|
||||
bus, failed = overNATS(handActConn)
|
||||
} else if js, err := aBus(); err != nil {
|
||||
failed = err
|
||||
} else if bus, failed = overNATS(js.Conn()); failed == nil {
|
||||
closing = js.Close
|
||||
} else {
|
||||
js.Close()
|
||||
}
|
||||
}
|
||||
if bus == nil {
|
||||
switch p.WalkSaid {
|
||||
case inventory.WalkStarted:
|
||||
fmt.Fprintf(os.Stderr, "%s: its start is kept and could NOT be said as plan-moved (%v): the router "+
|
||||
"does not hold what it touches\n", p.ID, failed)
|
||||
case inventory.WalkEnded:
|
||||
fmt.Fprintf(os.Stderr, "%s: its end is kept and could NOT be said as plan-moved (%v): the router holds "+
|
||||
"what it touched until its bound\n", p.ID, failed)
|
||||
}
|
||||
return
|
||||
}
|
||||
publishPlanMoved(ctx, bus, inv, p)
|
||||
}
|
||||
done = func() {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
inventory.PlanSaved = nil
|
||||
if closing != nil {
|
||||
closing()
|
||||
}
|
||||
}
|
||||
return said, done
|
||||
}
|
||||
|
||||
// overNATS is the bus a connection carries, with its JetStream.
|
||||
func overNATS(conn *nats.Conn) (link.Bus, error) {
|
||||
js, err := conn.JetStream()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return link.OverNATS{Conn: conn, JS: js}, nil
|
||||
}
|
||||
|
||||
@@ -37,41 +37,12 @@ type stalledLine struct {
|
||||
// Number is the pull request's, for a line of a head the forge never announced (novox/hq issue 347).
|
||||
Number int `json:"number,omitempty"`
|
||||
State string `json:"state"`
|
||||
// Walk is the walk a line is about: a waiting walk no live delivery holds, or, asked with `walks`, each walk
|
||||
// waiting for the owner's word and what holds it (novox/hq issues 362, 529).
|
||||
Walk string `json:"walk,omitempty"`
|
||||
// Waiting are the merge checks a line of the state `unanswered` waits on, as mesh-delivery says each:
|
||||
// "mesh/merge-gate pending since <UTC time>", "mesh/repo-check never set" (novox/hq issue 438).
|
||||
Waiting []string `json:"waiting,omitempty"`
|
||||
// Checks are the same merge checks as data, which the controller words in the operator's own time (novox/hq
|
||||
// issue 443): a time inside a finished sentence cannot be said again in another zone. A mesh-delivery from before
|
||||
// says none, and Waiting is said as it reads.
|
||||
Checks []waitingCheck `json:"checks,omitempty"`
|
||||
// Held is the state of the delivery of a line of the state `unanswered`, ready or rejected, when one is held
|
||||
// (novox/hq issues 441, 451): its check answered once, so a recheck is the step. Empty for a head with no
|
||||
// delivery, and from a mesh-delivery that does not say it yet; the words are never read for it.
|
||||
Held string `json:"held,omitempty"`
|
||||
For string `json:"for"`
|
||||
Bound string `json:"bound"`
|
||||
H2 string `json:"h2"`
|
||||
Says string `json:"says"`
|
||||
For string `json:"for"`
|
||||
Bound string `json:"bound"`
|
||||
H2 string `json:"h2"`
|
||||
Says string `json:"says"`
|
||||
}
|
||||
|
||||
// waitingCheck is one merge check a stalled line waits on, as mesh-delivery gives it: its context, its state —
|
||||
// "pending", or "never-set" — and, for a pending one, since when in RFC 3339, empty when the forge did not say.
|
||||
type waitingCheck struct {
|
||||
Context string `json:"context"`
|
||||
State string `json:"state"`
|
||||
Since string `json:"since,omitempty"`
|
||||
}
|
||||
|
||||
// wordsNow is the time the words are said at, which says whether a time needs its day; a seam a test replaces.
|
||||
var wordsNow = time.Now
|
||||
|
||||
// wordsZone is the zone a stalled line's times are said in: the controller's local zone, as every other time in its
|
||||
// messages. A seam a test replaces, so that no test writes time.Local, which every goroutine of the package reads.
|
||||
var wordsZone = func() *time.Location { return time.Local }
|
||||
|
||||
// operatorsOnly is whether the table leaves H2 nothing to do for the line: the state is the operator's.
|
||||
func (l stalledLine) operatorsOnly() bool { return l.H2 == "" || strings.HasPrefix(l.H2, "none") }
|
||||
|
||||
@@ -132,34 +103,6 @@ func deliveriesStalled(ctx context.Context, conn *nats.Conn, held bool) ([]stall
|
||||
return lines, nil
|
||||
}
|
||||
|
||||
// walkWaits are, by walk, the lines mesh-delivery says of a walk waiting for its word: what holds it.
|
||||
type walkWaits map[string][]stalledLine
|
||||
|
||||
// walksWaiting asks the delivery's owner what holds each walk waiting for its word (novox/hq issue 529): its
|
||||
// `stalled` asked with `walks`, so a line of `plans` names a held delivery, its reason and the verb that releases
|
||||
// it, instead of pointing at a hand start. Nil when no holder is on record or none answers: the line then reads as
|
||||
// before. An owner from before this answers its stalled lines, of which only those naming a walk are kept.
|
||||
func walksWaiting(ctx context.Context, conn *nats.Conn, held bool) walkWaits {
|
||||
if !held {
|
||||
return nil
|
||||
}
|
||||
raw, err := askDeliveryOwner(ctx, conn, "stalled", map[string]any{"walks": true})
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var lines []stalledLine
|
||||
if json.Unmarshal(raw, &lines) != nil {
|
||||
return nil
|
||||
}
|
||||
out := walkWaits{}
|
||||
for _, l := range lines {
|
||||
if l.Walk != "" {
|
||||
out[l.Walk] = append(out[l.Walk], l)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// stalledObservations are the conditions of what is stalled.
|
||||
func stalledObservations(lines []stalledLine) []conditions.Observation {
|
||||
out := make([]conditions.Observation, 0, len(lines))
|
||||
|
||||
@@ -136,13 +136,12 @@ func TestTheDeliveryOwnerIsAskedOverTheBus(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// And release and stop, which the operator's warrant chooses (novox/hq ADR 0259).
|
||||
for _, verb := range []string{"stalled", "close", "release", "stop"} {
|
||||
for _, verb := range []string{"stalled", "close"} {
|
||||
if !slices.Contains(granted.Publish, link.SeatToolSubject(catalogue.DeliverySeat, verb)) {
|
||||
t.Errorf("the controller may not ask %s.%s", catalogue.DeliverySeat, verb)
|
||||
}
|
||||
}
|
||||
if _, err := askDeliveryOwner(t.Context(), nil, "retire-history", nil); err == nil || !strings.Contains(err.Error(), "grant") {
|
||||
if _, err := askDeliveryOwner(t.Context(), nil, "stop", nil); err == nil || !strings.Contains(err.Error(), "grant") {
|
||||
t.Fatalf("a verb the grant does not name was asked: %v", err)
|
||||
}
|
||||
conn, err := nats.Connect(testbus.URL(t))
|
||||
|
||||
@@ -119,13 +119,6 @@ func TestTheBusMachineIsBehindByItsUserListAlone(t *testing.T) {
|
||||
if userListBehind("", "") {
|
||||
t.Fatal("a machine sent no list reads as behind")
|
||||
}
|
||||
// A list that could not be composed is behind: not known is never "not behind" (the review of hq issue 353).
|
||||
if !listBehind("", errors.New("the plan cannot be worked out"), digestOf([]byte(list))) {
|
||||
t.Fatal("a list nobody could compose reads as current")
|
||||
}
|
||||
if listBehind(list, nil, digestOf([]byte(list))) || !listBehind(list, nil, "") {
|
||||
t.Fatal("listBehind does not read the digest as userListBehind does")
|
||||
}
|
||||
}
|
||||
|
||||
// The machine holding the bus goes first: its declaration carries the user list the new grants are
|
||||
|
||||
@@ -227,9 +227,7 @@ func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testin
|
||||
t.Fatalf("with no holder on record the walk waited (%v) or asked %v", p.Waiting(), *asked)
|
||||
}
|
||||
|
||||
// The holder on record: the next merge waits, asking nothing, and an advance asks nothing either. One walk
|
||||
// is open at a time (novox/hq ADR 0276): the first ends before the next merge's batch is cut.
|
||||
finish(p.ID)
|
||||
// The holder on record: the next merge waits, asking nothing, and an advance asks nothing either.
|
||||
if _, err := inv.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -261,11 +259,10 @@ func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testin
|
||||
t.Fatalf("the word was not kept: %+v %v", got.Delivery, err)
|
||||
}
|
||||
|
||||
// The delivery's owner's own merge never waits for it. A walk that started is never taken over (novox/hq
|
||||
// ADR 0276): the merge's batch is cut once it ended.
|
||||
finish(p.ID)
|
||||
// The delivery's owner's own merge never waits for it — and takes over what the older walk had not
|
||||
// built (app, folded in: ADR 0218), which goes with it on the controller's own path.
|
||||
p = merge("c3cccccccc", "modules/mesh-delivery/main.go")
|
||||
if p.Waiting() || len(*asked) != 3 {
|
||||
if p.Waiting() || len(*asked) != 4 {
|
||||
t.Fatalf("mesh-delivery's own walk waited for mesh-delivery: %v %v", p.Waiting(), *asked)
|
||||
}
|
||||
|
||||
@@ -283,7 +280,7 @@ func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testin
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
got, _ = inv.PlanByID(ctx, p.ID)
|
||||
if got.Waiting() || !strings.HasPrefix(got.Delivery.By, "a person") || len(*asked) < 4 {
|
||||
if got.Waiting() || !strings.HasPrefix(got.Delivery.By, "a person") || len(*asked) < 5 {
|
||||
t.Fatalf("a person's word did not start the walk: %+v, asked %v", got.Delivery, *asked)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,333 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/secrets"
|
||||
)
|
||||
|
||||
// A module's own secret given at the operator's desk (novox/hq ADR 0259 §10, ADR 0277).
|
||||
//
|
||||
// mesh-controller secret ask <node> <module> <name> [--at <desk>]
|
||||
//
|
||||
// **The value never passes through whoever asked for it.** An agent, or the operator at the mesh MCP
|
||||
// server, calls `secret-ask` (or `give`) with the machine, the module, the secret's name and the desk — never
|
||||
// a value. The controller makes a sealing keypair for this one call, asks the desk's `node-launcher.secret` to
|
||||
// prompt the operator without showing what is typed, and is answered with what was typed **sealed to that
|
||||
// key**: no plaintext on the bus, in a runtime's log or in any call's record. It opens it here, seals it to the
|
||||
// module's machine exactly as `secret accept` does, and forgets it. What it answers says only that the
|
||||
// value was taken, or why not.
|
||||
//
|
||||
// **Bounded, and the prompt says who asked** (ADR 0277): one open prompt per secret and few an hour, read from
|
||||
// the store before the prompt opens (inventory.OpenSecretAsk), so an agent cannot keep a prompt in front of the
|
||||
// operator until they type. The prompt names the module, the secret, the machine and who asked — the caller as
|
||||
// the bus named it, never a word the caller chose — written by the desk's launcher from those names alone.
|
||||
//
|
||||
// **What remains** (ADR 0234's accepted residual risk): on an X11 desk any program of the operator's
|
||||
// account can read the keys as they are typed. And a program that calls the desk's prompt itself, with a
|
||||
// key of its own, is answered with what the operator typed into a prompt they did not ask for — as it could
|
||||
// draw a window of its own. The prompt says who asks and for what, so the operator types only into a
|
||||
// prompt they started.
|
||||
|
||||
// deskPromptWithin is how long the prompt waits for the operator: below the runtime's thirty seconds for
|
||||
// one call, as the launcher's menu is.
|
||||
const deskPromptWithin = 25
|
||||
|
||||
// deskGive is the desk path, its four reaches given so a test needs no store and no bus.
|
||||
type deskGive struct {
|
||||
// declares refuses a module or a secret the mesh would refuse, before anybody is asked to type.
|
||||
declares func(module, name string) error
|
||||
// known refuses a machine the mesh does not know, before anybody is asked to type; nil knows every one
|
||||
// (a test that does not look).
|
||||
known func(machine string) error
|
||||
// trusted says a module runs as an account of its own: its secret is never taken at a desk (below). Nil is
|
||||
// never (a test that does not look).
|
||||
trusted func(module string) (bool, error)
|
||||
// ask asks one machine's node-launcher.secret and answers its result, or the holder's refusal.
|
||||
ask func(machine string, args map[string]any) (json.RawMessage, error)
|
||||
// accept seals the value as `secret accept` does, and says whether it lives until the module's start.
|
||||
accept func(value string) (untilStart bool, err error)
|
||||
// record writes the act in the hand-act log.
|
||||
record func(link.HandAct) error
|
||||
// announce raises the condition that says a module's own secret was given (secretGivenObservation), on
|
||||
// every channel; nil announces nothing (a test that does not look).
|
||||
announce func(node, module, name, how string) error
|
||||
// askedBy is who asked, as the bus named the caller: said in the prompt and recorded.
|
||||
askedBy string
|
||||
// open records the ask and holds the bounds (one open per secret, few an hour), answering the record's id;
|
||||
// nil keeps no record (a test that does not look). end closes it with how it ended.
|
||||
open func(node, module, name, desk string) (int64, error)
|
||||
end func(id int64, outcome string) error
|
||||
}
|
||||
|
||||
// askedByName is the caller as the prompt names it: the first clause of what the bus said, in the characters
|
||||
// a name has, at most 80 of them. The desk's launcher refuses anything else, so no words of the caller's own
|
||||
// reach the prompt.
|
||||
func askedByName(caller string) string {
|
||||
first, _, _ := strings.Cut(caller, ",")
|
||||
var b strings.Builder
|
||||
for _, r := range strings.TrimSpace(first) {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '.', r == '_', r == '/', r == '@',
|
||||
r == '-', r == ' ':
|
||||
b.WriteRune(r)
|
||||
default:
|
||||
b.WriteRune('-')
|
||||
}
|
||||
if b.Len() >= 80 {
|
||||
break
|
||||
}
|
||||
}
|
||||
name := strings.TrimSpace(b.String())
|
||||
if name == "" || strings.HasPrefix(name, "-") {
|
||||
return "an unnamed caller"
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// errNothingGiven is a prompt dismissed, or not answered in time: nothing changes.
|
||||
var errNothingGiven = errors.New("nothing was given")
|
||||
|
||||
// give asks the desk for the value and seals it; it answers the words said to the caller.
|
||||
func (d deskGive) give(node, module, name, desk string) (string, error) {
|
||||
for what, v := range map[string]string{"the machine": node, "the module": module, "the secret": name, "the desk": desk} {
|
||||
if strings.TrimSpace(v) == "" {
|
||||
return "", fmt.Errorf("%s is not named", what)
|
||||
}
|
||||
}
|
||||
if d.known != nil {
|
||||
for what, machine := range map[string]string{"the machine the secret is for": node, "the desk": desk} {
|
||||
if err := d.known(machine); err != nil {
|
||||
return "", fmt.Errorf("nobody was asked to type anything: %s, %s, is not a machine this mesh knows: %w",
|
||||
what, machine, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := d.declares(module, name); err != nil {
|
||||
return "", fmt.Errorf("nobody was asked to type anything: %w", err)
|
||||
}
|
||||
// **A trusted party's secret is not taken at a desk** (the confirmation review of 2026-10-09, N1-give). The
|
||||
// prompt's answer comes back over the desk machine's bus, on a subject that machine's runtime answers — and
|
||||
// on a desk machine agents run as the operator, who holds that runtime's credential. An agent could answer
|
||||
// first, with a bot token of its own sealed to this call's key, and the channel the operator's answers are
|
||||
// proven on would be the agent's. So the value of a module running as its own account is typed at the
|
||||
// controller's terminal, where no bus carries it.
|
||||
if d.trusted != nil {
|
||||
trusted, err := d.trusted(module)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("nobody was asked to type anything: whether %s is a trusted party could not be read: %w", module, err)
|
||||
}
|
||||
if trusted {
|
||||
return "", fmt.Errorf("nobody was asked to type anything: %s runs as an account of its own and proves the "+
|
||||
"operator's answers, so its %s is given at the controller's terminal alone — there, run `mesh-controller "+
|
||||
"secret accept %s %s %s` and type it at its prompt. A desk's prompt is answered over the desk machine's "+
|
||||
"bus, where an agent may answer first (novox/hq ADR 0259 §10)", module, name, node, module, name)
|
||||
}
|
||||
}
|
||||
// The bounds, read and kept before anybody is asked to type (novox/hq ADR 0277): one open prompt per secret,
|
||||
// few an hour. How the ask ends is recorded whatever happens below.
|
||||
outcome := "failed"
|
||||
if d.open != nil {
|
||||
id, err := d.open(node, module, name, desk)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("nobody was asked to type anything: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if d.end != nil {
|
||||
_ = d.end(id, outcome)
|
||||
}
|
||||
}()
|
||||
}
|
||||
public, private, err := secrets.Keypair()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("no key could be made to take the value: %w", err)
|
||||
}
|
||||
// By name, never by words: the holder writes the prompt from these, and says the controller asks, which
|
||||
// the bus alone makes true (broker.ControllerOnly). Who asked is the bus's word on the caller, cut to a
|
||||
// name's characters — never an argument of the call.
|
||||
raw, err := d.ask(desk, map[string]any{
|
||||
"module": module,
|
||||
"secret": name,
|
||||
"node": node,
|
||||
"asked_by": askedByName(d.askedBy),
|
||||
"seal_to": public,
|
||||
"timeout_seconds": deskPromptWithin,
|
||||
})
|
||||
if err != nil {
|
||||
outcome = "refused"
|
||||
return "", fmt.Errorf("the desk on %s could not be asked: %w", desk, err)
|
||||
}
|
||||
var answer struct {
|
||||
Sealed string `json:"sealed"`
|
||||
Cancelled bool `json:"cancelled"`
|
||||
TimedOut bool `json:"timed_out"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &answer); err != nil {
|
||||
return "", fmt.Errorf("the desk on %s answered something that is not the prompt's answer", desk)
|
||||
}
|
||||
switch {
|
||||
case answer.TimedOut:
|
||||
outcome = "timed-out"
|
||||
return "", fmt.Errorf("%w: the prompt on %s was not answered within %d seconds", errNothingGiven, desk, deskPromptWithin)
|
||||
case answer.Cancelled:
|
||||
outcome = "dismissed"
|
||||
return "", fmt.Errorf("%w: the prompt on %s was dismissed", errNothingGiven, desk)
|
||||
case answer.Sealed == "":
|
||||
return "", fmt.Errorf("the desk on %s answered no sealed value", desk)
|
||||
}
|
||||
opened, err := secrets.Open(private, answer.Sealed)
|
||||
if err != nil {
|
||||
// Never the value, never what failed to open: only that it was not sealed to this call.
|
||||
return "", fmt.Errorf("the desk on %s answered a value not sealed to this call; nothing was taken", desk)
|
||||
}
|
||||
value := asSupplied(string(opened))
|
||||
for i := range opened {
|
||||
opened[i] = 0
|
||||
}
|
||||
if strings.TrimSpace(value) == "" {
|
||||
outcome = "empty"
|
||||
return "", fmt.Errorf("%w: the prompt on %s was answered empty", errNothingGiven, desk)
|
||||
}
|
||||
untilStart, err := d.accept(value)
|
||||
value = ""
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
outcome = "given"
|
||||
act := link.HandAct{Verb: "secret accept", Args: []string{node, module, name, "--at-desk", desk},
|
||||
Why: fmt.Sprintf("the operator gave %s for %s on %s at the desk on %s, asked by %s", name, module, node, desk,
|
||||
askedByName(d.askedBy)),
|
||||
Cause: "given-at-the-desk"}
|
||||
recorded := ""
|
||||
if err := d.record(act); err != nil {
|
||||
recorded = fmt.Sprintf("\n this act could NOT be recorded in the hand-act log, and is done anyway: %v", err)
|
||||
}
|
||||
if d.announce != nil {
|
||||
if err := d.announce(node, module, name, "at the desk on "+desk); err != nil {
|
||||
recorded += fmt.Sprintf("\n this change could NOT be announced on the operator's channels: %v", err)
|
||||
}
|
||||
}
|
||||
words := fmt.Sprintf("%s on %s now holds %q, given at the desk on %s and sealed to %s; the mesh cannot read it "+
|
||||
"back.\n run `push %s` to send it", module, node, name, desk, node, node)
|
||||
if untilStart {
|
||||
words += fmt.Sprintf("\n it lives until %s next starts well under the mesh, and is then replaced with a value "+
|
||||
"the mesh makes (ADR 0228)", module)
|
||||
}
|
||||
return words + recorded, nil
|
||||
}
|
||||
|
||||
// askAtDesk is `secret ask <node> <module> <name> [--at <machine>]`, and the terminal's `secret accept … --at-desk
|
||||
// <machine>`: the desk path, on this controller's stores and bus. The desk is the module's machine unless named.
|
||||
func askAtDesk(ctx context.Context, node, module, name, desk string) error {
|
||||
if desk == "" {
|
||||
desk = node
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
d := deskGive{
|
||||
askedBy: link.Caller(),
|
||||
open: func(node, module, name, desk string) (int64, error) {
|
||||
return open.inventory.OpenSecretAsk(ctx, node, module, name, askedByName(link.Caller()), desk)
|
||||
},
|
||||
end: func(id int64, outcome string) error { return open.inventory.EndSecretAsk(ctx, id, outcome) },
|
||||
declares: func(module, name string) error { return open.inventory.DeclaresOwnSecret(ctx, module, name) },
|
||||
known: func(machine string) error {
|
||||
_, err := open.inventory.NodeByName(ctx, machine)
|
||||
return err
|
||||
},
|
||||
trusted: func(module string) (bool, error) { return open.inventory.RunsAsItsOwnAccount(ctx, module) },
|
||||
ask: func(machine string, args map[string]any) (json.RawMessage, error) {
|
||||
var result json.RawMessage
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
answer, err := link.AskSeatTool(ctx, conn, "node-launcher", "secret", machine, args,
|
||||
time.Duration(deskPromptWithin+5)*time.Second)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return errors.New(answer.Error)
|
||||
}
|
||||
result = answer.Result
|
||||
return nil
|
||||
})
|
||||
return result, err
|
||||
},
|
||||
accept: func(value string) (bool, error) {
|
||||
return open.inventory.AcceptGivenSecret(ctx, node, module, name, value)
|
||||
},
|
||||
record: func(act link.HandAct) error {
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
_, err := link.RecordHandAct(ctx, conn, act)
|
||||
return err
|
||||
})
|
||||
},
|
||||
announce: func(node, module, name, how string) error { return announceSecretGiven(ctx, node, module, name, how) },
|
||||
}
|
||||
words, err := d.give(node, module, name, desk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(words)
|
||||
return nil
|
||||
}
|
||||
|
||||
// kindSecretGiven is the condition every value given for a module's own secret raises (the review of 2026-10-09,
|
||||
// M4): on every channel, so a bot token changed by somebody else — a channel that now answers for them — is
|
||||
// heard of. It stays until the operator silences or clears it.
|
||||
const kindSecretGiven = "secret-given"
|
||||
|
||||
// secretGivenObservation is that condition: which secret, of which module on which machine, how and when.
|
||||
// The summary, for whoever looks closer, names the secret and the time. The words the operator reads are
|
||||
// held to the plain rule (conditions.PlainWords): no clock time — the channel says when, in the operator's
|
||||
// time — and the secret's name said as words. Words that broke the rule were replaced by the keeper with
|
||||
// "needs a look … a problem it calls secret given" (hq issue 359), which told the operator nothing.
|
||||
func secretGivenObservation(node, module, name, how string, at time.Time) conditions.Observation {
|
||||
key := node + "." + module + "." + name
|
||||
where := module + " on " + node
|
||||
// Within the bounds whatever the names' length: the module and machine, else the module, else the machine.
|
||||
headline := "New secret given for " + where
|
||||
for _, h := range []string{"New secret given for " + module, "New secret given on " + node} {
|
||||
if len(headline) > conditions.HeadlineMax {
|
||||
headline = h
|
||||
}
|
||||
}
|
||||
resolved := "You saw that " + module + " was given a new secret"
|
||||
if len(resolved) > conditions.HeadlineMax+20 {
|
||||
resolved = "You saw that a new secret was given on " + node
|
||||
}
|
||||
return conditions.Observation{Scope: conditions.ScopeMachine, ID: key, Token: kindSecretGiven, Kind: kindSecretGiven,
|
||||
Machine: node, Severity: conditions.Urgent, Source: kindSecretGiven,
|
||||
Summary: fmt.Sprintf("%s of %s on %s was given %s at %s", name, module, node, how,
|
||||
at.Local().Format("2006-01-02 15:04")),
|
||||
Headline: headline,
|
||||
Explanation: fmt.Sprintf("The secret %s of %s was given %s. If you gave it, nothing else is needed. If you "+
|
||||
"did not, somebody else now holds what %s acts with.", secretNameWords(name), where, how, module),
|
||||
Needs: "silence this if you just gave it; if you did not, give it again yourself so that only you hold it.",
|
||||
Resolved: resolved,
|
||||
Actions: []conditions.Action{conditions.SilenceAction(conditions.Key(conditions.ScopeMachine, key, kindSecretGiven))}}
|
||||
}
|
||||
|
||||
// secretNameWords is a secret's name as the operator reads it: "telegram-token" is "telegram token".
|
||||
func secretNameWords(name string) string {
|
||||
return strings.Join(strings.FieldsFunc(name, func(r rune) bool { return r == '-' || r == '_' || r == '.' }), " ")
|
||||
}
|
||||
|
||||
// announceSecretGiven raises it on this controller's keeper.
|
||||
func announceSecretGiven(ctx context.Context, node, module, name, how string) error {
|
||||
return withKeeper(ctx, func(k *conditions.Keeper) error {
|
||||
_, err := k.Observe(ctx, secretGivenObservation(node, module, name, how, time.Now()))
|
||||
return err
|
||||
})
|
||||
}
|
||||
@@ -1,512 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/secrets"
|
||||
)
|
||||
|
||||
const typed = "123456789:AAEhBP0av28P4XFQnIuR-o-7Xnz1kkUzW3g"
|
||||
|
||||
// aDesk is the desk path with a prompt the test answers as the operator would, and what it was asked kept.
|
||||
func aDesk(t *testing.T, answer func(args map[string]any) (json.RawMessage, error)) (deskGive, *[]string, *[]link.HandAct, *[]map[string]any) {
|
||||
t.Helper()
|
||||
var accepted []string
|
||||
var acts []link.HandAct
|
||||
var asked []map[string]any
|
||||
return deskGive{
|
||||
declares: func(module, name string) error {
|
||||
if module != "telegram" || name != "telegram-token" {
|
||||
return errors.New(module + " does not declare " + name + " as an own secret")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
ask: func(machine string, args map[string]any) (json.RawMessage, error) {
|
||||
asked = append(asked, args)
|
||||
return answer(args)
|
||||
},
|
||||
accept: func(value string) (bool, error) { accepted = append(accepted, value); return false, nil },
|
||||
record: func(a link.HandAct) error { acts = append(acts, a); return nil },
|
||||
}, &accepted, &acts, &asked
|
||||
}
|
||||
|
||||
func sealedTo(t *testing.T, value string) func(args map[string]any) (json.RawMessage, error) {
|
||||
return func(args map[string]any) (json.RawMessage, error) {
|
||||
sealed, err := secrets.Seal(args["seal_to"].(string), []byte(value+"\n"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(map[string]any{"sealed": sealed})
|
||||
return raw, nil
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0259 §10: the value typed at the desk is sealed as `secret accept` seals it, and is in no
|
||||
// answer, no prompt argument and no act recorded.
|
||||
func TestASecretGivenAtTheDeskIsSealedAndSaidNowhere(t *testing.T) {
|
||||
d, accepted, acts, asked := aDesk(t, sealedTo(t, typed))
|
||||
words, err := d.give("anchor", "telegram", "telegram-token", "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(*accepted) != 1 || (*accepted)[0] != typed {
|
||||
t.Fatalf("the value sealed is not what was typed, its line ending taken off")
|
||||
}
|
||||
if len(*acts) != 1 || (*acts)[0].Verb != "secret accept" || (*acts)[0].Cause != "given-at-the-desk" ||
|
||||
!strings.Contains((*acts)[0].Why, "at the desk on laptop") {
|
||||
t.Errorf("the act: %+v", *acts)
|
||||
}
|
||||
raw, _ := json.Marshal(struct {
|
||||
Words string
|
||||
Acts []link.HandAct
|
||||
Asked []map[string]any
|
||||
}{words, *acts, *asked})
|
||||
if strings.Contains(string(raw), typed) || strings.Contains(string(raw), "AAEhBP0") {
|
||||
t.Fatal("the value appears in what was said, asked or recorded")
|
||||
}
|
||||
if !strings.Contains(words, "push anchor") || !strings.Contains(words, "given at the desk on laptop") {
|
||||
t.Errorf("%q", words)
|
||||
}
|
||||
if p := (*asked)[0]; p["seal_to"] == "" || p["timeout_seconds"] != deskPromptWithin {
|
||||
t.Errorf("the prompt was asked %v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNothingIsAskedForASecretTheMeshWouldRefuse(t *testing.T) {
|
||||
for _, c := range [][2]string{{"telegram", "chat-id"}, {"nobody", "telegram-token"}} {
|
||||
d, accepted, _, asked := aDesk(t, sealedTo(t, typed))
|
||||
if _, err := d.give("anchor", c[0], c[1], "laptop"); err == nil || !strings.Contains(err.Error(), "nobody was asked") {
|
||||
t.Errorf("%v: %v", c, err)
|
||||
}
|
||||
if len(*asked) != 0 || len(*accepted) != 0 {
|
||||
t.Errorf("%v: the operator was asked anyway", c)
|
||||
}
|
||||
}
|
||||
d, _, _, _ := aDesk(t, sealedTo(t, typed))
|
||||
if _, err := d.give("anchor", "telegram", "telegram-token", ""); err == nil {
|
||||
t.Error("no desk was refused nowhere")
|
||||
}
|
||||
}
|
||||
|
||||
func TestADismissedEmptyLateOrForeignAnswerTakesNothing(t *testing.T) {
|
||||
for want, answer := range map[string]func(map[string]any) (json.RawMessage, error){
|
||||
"not answered within 25 seconds": func(map[string]any) (json.RawMessage, error) {
|
||||
return json.RawMessage(`{"cancelled":true,"timed_out":true}`), nil
|
||||
},
|
||||
"was dismissed": func(map[string]any) (json.RawMessage, error) { return json.RawMessage(`{"cancelled":true}`), nil },
|
||||
"answered empty": sealedTo(t, " "),
|
||||
"not sealed to this call": func(map[string]any) (json.RawMessage, error) {
|
||||
other, _, _ := secrets.Keypair()
|
||||
sealed, _ := secrets.Seal(other, []byte(typed))
|
||||
raw, _ := json.Marshal(map[string]any{"sealed": sealed})
|
||||
return raw, nil
|
||||
},
|
||||
"could not be asked": func(map[string]any) (json.RawMessage, error) { return nil, errors.New("no session answers") },
|
||||
} {
|
||||
d, accepted, acts, _ := aDesk(t, answer)
|
||||
_, err := d.give("anchor", "telegram", "telegram-token", "laptop")
|
||||
if err == nil || !strings.Contains(err.Error(), want) || strings.Contains(err.Error(), typed) {
|
||||
t.Errorf("want %q, got %v", want, err)
|
||||
}
|
||||
if len(*accepted) != 0 || len(*acts) != 0 {
|
||||
t.Errorf("%s: something was taken or recorded", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheGiveVerbRunsTheDeskPathAndTheControllerMayAskTheDesk(t *testing.T) {
|
||||
argv, err := argvFor("give", map[string]any{"node": "anchor", "module": "telegram", "secret": "telegram-token", "at": "laptop"})
|
||||
if err != nil || strings.Join(argv, " ") != "secret ask anchor telegram telegram-token --at laptop" {
|
||||
t.Fatalf("%v %v", argv, err)
|
||||
}
|
||||
if _, err := argvFor("give", map[string]any{"node": "anchor", "module": "telegram", "secret": "telegram-token"}); err == nil {
|
||||
t.Error("give without a desk was taken")
|
||||
}
|
||||
// secret-ask is the same line, with the desk the module's machine unless named (novox/hq ADR 0277).
|
||||
argv, err = argvFor("secret-ask", map[string]any{"node": "anchor", "module": "telegram", "secret": "telegram-token"})
|
||||
if err != nil || strings.Join(argv, " ") != "secret ask anchor telegram telegram-token" {
|
||||
t.Fatalf("%v %v", argv, err)
|
||||
}
|
||||
argv, err = argvFor("secret-ask", map[string]any{"node": "anchor", "module": "telegram", "secret": "telegram-token", "at": "laptop"})
|
||||
if err != nil || strings.Join(argv, " ") != "secret ask anchor telegram telegram-token --at laptop" {
|
||||
t.Fatalf("%v %v", argv, err)
|
||||
}
|
||||
perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found := false
|
||||
for _, p := range perms.Publish {
|
||||
found = found || p == "mesh.seat.node-launcher.tool.secret.*"
|
||||
}
|
||||
if !found {
|
||||
t.Error("the controller may not ask the desk's prompt")
|
||||
}
|
||||
}
|
||||
|
||||
// The review of 2026-10-09 (M4): the desk's prompt says who asks in words the caller does not choose — the
|
||||
// controller, which the bus alone lets ask it — and what for, from names the controller checked; the prompt
|
||||
// carries no free text of the caller's.
|
||||
func TestThePromptIsAskedByNameNeverByWordsTheCallerChose(t *testing.T) {
|
||||
d, _, _, asked := aDesk(t, sealedTo(t, typed))
|
||||
if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := (*asked)[0]
|
||||
if p["module"] != "telegram" || p["secret"] != "telegram-token" || p["node"] != "anchor" {
|
||||
t.Errorf("the prompt was not asked by name: %v", p)
|
||||
}
|
||||
for _, free := range []string{"prompt", "message"} {
|
||||
if _, there := p[free]; there {
|
||||
t.Errorf("the prompt carries the caller's %s: %v", free, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Every value given for a module's own secret is announced as a condition, on every channel (the review of
|
||||
// 2026-10-09, M4): a bot token changed by somebody else is a channel that now answers for them.
|
||||
func TestAValueGivenAtTheDeskIsAnnounced(t *testing.T) {
|
||||
d, _, _, _ := aDesk(t, sealedTo(t, typed))
|
||||
var said []string
|
||||
d.announce = func(node, module, name, how string) error {
|
||||
said = append(said, node+" "+module+" "+name+" "+how)
|
||||
return nil
|
||||
}
|
||||
if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(said) != 1 || !strings.Contains(said[0], "anchor telegram telegram-token") || !strings.Contains(said[0], "laptop") {
|
||||
t.Fatalf("announced %v", said)
|
||||
}
|
||||
o := secretGivenObservation("anchor", "telegram", "telegram-token", "at the desk on laptop", time.Date(2026, 10, 9, 12, 3, 0, 0, time.UTC))
|
||||
if o.Severity != conditions.Urgent || !strings.Contains(o.Explanation, "telegram token") ||
|
||||
len(o.Actions) == 0 || o.Key() == "" {
|
||||
t.Errorf("the announcement %+v", o)
|
||||
}
|
||||
if strings.Contains(o.Summary+o.Explanation+o.Said, typed) {
|
||||
t.Error("the announcement carries the value")
|
||||
}
|
||||
}
|
||||
|
||||
// **The secret-given condition says itself in its own plain words** (hq issue 359): the words it carries pass
|
||||
// the plain rule, from the controller's terminal and from a desk, so the keeper keeps them — they once held a
|
||||
// clock time, and the operator read "Novox needs a look … a problem it calls secret given" instead. Its
|
||||
// severity and its answer stay: it is heard on every channel, and silenced by the operator.
|
||||
func TestTheSecretGivenConditionSaysItselfInPlainWords(t *testing.T) {
|
||||
at := time.Date(2026, 10, 9, 23, 32, 0, 0, time.Local)
|
||||
for _, how := range []string{"at the controller's terminal", "at the desk on laptop"} {
|
||||
o := secretGivenObservation("anchor", "telegram", "telegram-token", how, at)
|
||||
w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs,
|
||||
Actions: o.Actions}
|
||||
if why, ok := conditions.PlainWords(w, o.Machine); !ok {
|
||||
t.Fatalf("given %s, the words are not plain: %s", how, why)
|
||||
}
|
||||
k, _ := withConditionsInMemory(t)
|
||||
c, err := k.Observe(t.Context(), o)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.Headline != "New secret given for telegram on anchor" || c.Severity != conditions.Urgent ||
|
||||
!strings.HasPrefix(c.Explanation, conditions.NeedsYou+" silence this") ||
|
||||
!strings.Contains(c.Explanation, "telegram token of telegram on anchor was given "+how) ||
|
||||
len(c.Actions) != 1 || c.Actions[0].Label != "Silence for a week" {
|
||||
t.Errorf("given %s, the keeper said %q / %q (%s, %v)", how, c.Headline, c.Explanation, c.Severity, c.Actions)
|
||||
}
|
||||
if strings.Contains(c.Headline+c.Explanation+c.Resolved+c.Needs, typed) {
|
||||
t.Error("the words carry the value")
|
||||
}
|
||||
}
|
||||
// A long module name keeps the headline and the resolved line within their bounds.
|
||||
for _, module := range []string{"a-module-with-a-rather-long-name-indeed",
|
||||
"a-module-with-a-name-so-long-that-no-headline-could-ever-hold-it"} {
|
||||
o := secretGivenObservation("anchor", module, "api-key", "at the controller's terminal", at)
|
||||
if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Explanation: o.Explanation,
|
||||
Resolved: o.Resolved, Needs: o.Needs, Actions: o.Actions}, o.Machine); !ok {
|
||||
t.Errorf("the module %s: %s", module, why)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The bus lets the controller alone ask the desk's prompt (the review of 2026-10-09, M4): the runtime, which
|
||||
// carries every agent's calls, and a person granted every tool are denied it, however wide their grant.
|
||||
func TestOnlyTheControllerMayAskTheDesksPrompt(t *testing.T) {
|
||||
for _, p := range []broker.Principal{
|
||||
{Kind: broker.KindNodeTools, Node: "laptop"},
|
||||
{Kind: broker.KindPerson, Module: "operator", Invokes: []string{"*"}},
|
||||
{Kind: broker.KindModule, Node: "laptop", Module: "lab", Invokes: []string{"seat:node-launcher.secret"}},
|
||||
} {
|
||||
perms, err := broker.PermissionsFor(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, subject := range []string{"mesh.seat.node-launcher.tool.secret.laptop", "mesh.seat.node-launcher.tool.secret",
|
||||
"mesh.mod.rofi.tool.node-launcher.secret", "mesh.mod.rofi.tool.node-launcher.secret.laptop"} {
|
||||
if broker.MayPublish(perms, subject) {
|
||||
t.Errorf("%s may publish %s", p.Username(), subject)
|
||||
}
|
||||
}
|
||||
}
|
||||
perms, _ := broker.PermissionsFor(broker.Principal{Kind: broker.KindController})
|
||||
if !broker.MayPublish(perms, "mesh.seat.node-launcher.tool.secret.laptop") {
|
||||
t.Error("the controller may not ask the desk's prompt")
|
||||
}
|
||||
}
|
||||
|
||||
// A value for a secret comes from the terminal or the desk, never through a verb (the review of 2026-10-09,
|
||||
// M4): `secret accept` with a value, run for a verb, is refused before anything is read.
|
||||
func TestASecretValueIsNeverAcceptedThroughAVerb(t *testing.T) {
|
||||
t.Setenv(verbVar, "mesh-controller.command")
|
||||
for _, args := range [][]string{
|
||||
{"accept", "anchor", "telegram", "telegram-token", "--from", "/dev/null"},
|
||||
{"accept", "anchor", "app", "db", "--from", "/dev/null", "--provider", "store"},
|
||||
} {
|
||||
err := secretCommand(context.Background(), args)
|
||||
if err == nil || !strings.Contains(err.Error(), "never through a verb") {
|
||||
t.Errorf("%v: %v", args, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The `give` and `secret-ask` verbs' own line passes the terminal-only rule of ADR 0266, and no `secret accept`
|
||||
// does: a value, a file, a provider or an extra word is still the terminal's alone (novox/hq ADR 0277).
|
||||
func TestOnlyTheGiveLinePassesTheTerminalRuleForSecrets(t *testing.T) {
|
||||
for _, argv := range [][]string{
|
||||
{"secret", "ask", "anchor", "telegram", "telegram-token", "--at", "laptop"},
|
||||
{"secret", "ask", "anchor", "telegram", "telegram-token"},
|
||||
} {
|
||||
if err := terminalOnly(argv); err != nil {
|
||||
t.Errorf("%v refused: %v", argv, err)
|
||||
}
|
||||
}
|
||||
for _, argv := range [][]string{
|
||||
{"secret", "accept", "anchor", "telegram", "telegram-token", "--at-desk", "laptop"},
|
||||
{"secret", "ask", "anchor", "telegram", "telegram-token", "--from", "/tmp/x"},
|
||||
{"secret", "ask", "anchor", "telegram", "telegram-token", "--at", "laptop", "--local"},
|
||||
{"secret", "ask", "anchor", "telegram", "--at", "laptop"},
|
||||
{"secret", "accept", "anchor", "telegram", "telegram-token"},
|
||||
{"secret", "accept", "anchor", "telegram", "telegram-token", "--from", "/tmp/x"},
|
||||
{"secret", "accept", "anchor", "telegram", "telegram-token", "--at-desk", "laptop", "--local"},
|
||||
{"secret", "accept", "anchor", "telegram", "--provider", "--at-desk", "laptop"},
|
||||
{"secret", "export", "anchor", "telegram", "telegram-token", "--at-desk", "laptop"},
|
||||
} {
|
||||
if err := terminalOnly(argv); err == nil {
|
||||
t.Errorf("%v passed the terminal rule", argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The confirmation review of 2026-10-09, N1-give: a desk's prompt is answered over the desk machine's bus, and
|
||||
// on a desk machine agents run as the operator, who holds its runtime's credential — so a trusted party's
|
||||
// secret (a module running as an account of its own: the Telegram bot's token) is never taken at a desk.
|
||||
// Refused before anybody is asked to type, whoever called, naming the terminal's line.
|
||||
func TestATrustedPartysSecretIsNeverTakenAtADesk(t *testing.T) {
|
||||
d, accepted, acts, asked := aDesk(t, sealedTo(t, typed))
|
||||
d.trusted = func(module string) (bool, error) { return module == "telegram", nil }
|
||||
_, err := d.give("anchor", "telegram", "telegram-token", "laptop")
|
||||
if err == nil || !strings.Contains(err.Error(), "controller's terminal alone") ||
|
||||
!strings.Contains(err.Error(), "secret accept anchor telegram telegram-token") {
|
||||
t.Fatalf("a trusted party's secret was taken at the desk, or refused without the line: %v", err)
|
||||
}
|
||||
if len(*asked)+len(*accepted)+len(*acts) != 0 {
|
||||
t.Errorf("asked %v, accepted %d, recorded %v", *asked, len(*accepted), *acts)
|
||||
}
|
||||
d.trusted = func(string) (bool, error) { return false, errors.New("the store did not answer") }
|
||||
if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); err == nil || len(*asked) != 0 {
|
||||
t.Errorf("a module not known to be untrusted was asked at the desk: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// And who may answer the desk's prompt at all: only the runtime of the machine it is asked on, carrying the
|
||||
// launcher that holds the seat there — never the controller, another machine's runtime, or a module's own
|
||||
// account (the confirmation review of 2026-10-09, N1-give).
|
||||
func TestOnlyTheDeskMachinesLauncherMayAnswerItsPrompt(t *testing.T) {
|
||||
launcher := broker.Declared{Module: "rofi", Holds: []broker.Seat{{Name: "node-launcher", Scope: "node",
|
||||
Serves: []string{"run", "secret"}}}}
|
||||
subject := "mesh.seat.node-launcher.tool.secret.laptop"
|
||||
for _, c := range []struct {
|
||||
p broker.Principal
|
||||
answers bool
|
||||
}{
|
||||
{broker.Principal{Kind: broker.KindNodeTools, Node: "laptop", Module: broker.RuntimeModule, Carries: []broker.Declared{launcher}}, true},
|
||||
{broker.Principal{Kind: broker.KindNodeTools, Node: "anchor", Module: broker.RuntimeModule, Carries: []broker.Declared{launcher}}, false},
|
||||
{broker.Principal{Kind: broker.KindController}, false},
|
||||
{broker.Principal{Kind: broker.KindModule, Node: "laptop", Module: "lab"}, false},
|
||||
{broker.Principal{Kind: broker.KindNode, Node: "laptop"}, false},
|
||||
} {
|
||||
perms, err := broker.PermissionsFor(c.p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := broker.MaySubscribe(perms, subject); got != c.answers {
|
||||
t.Errorf("%s may answer %s: %v, want %v", c.p.Username(), subject, got, c.answers)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// N1-give at the controller's terminal (the confirmation review of 2026-10-09): a trusted party's secret is
|
||||
// announced before it is kept, and not kept when the announcement fails; another module's is kept first and
|
||||
// a failed announcement is said, not undone.
|
||||
func TestATrustedPartysSecretGivenAtTheTerminalIsAnnouncedBeforeItIsKept(t *testing.T) {
|
||||
var order []string
|
||||
announce := func(fail bool) func() error {
|
||||
return func() error {
|
||||
order = append(order, "announce")
|
||||
if fail {
|
||||
return errors.New("no channel")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
keep := func() (bool, error) { order = append(order, "keep"); return false, nil }
|
||||
|
||||
order = nil
|
||||
if _, unannounced, err := keepGiven(true, announce(false), keep); err != nil || unannounced != nil ||
|
||||
strings.Join(order, ",") != "announce,keep" {
|
||||
t.Errorf("trusted: %v %v, order %v; want announced, then kept", unannounced, err, order)
|
||||
}
|
||||
order = nil
|
||||
if _, _, err := keepGiven(true, announce(true), keep); err == nil || strings.Join(order, ",") != "announce" {
|
||||
t.Errorf("trusted, announcement failed: %v, order %v; want refused and nothing kept", err, order)
|
||||
}
|
||||
order = nil
|
||||
if _, unannounced, err := keepGiven(false, announce(true), keep); err != nil || unannounced == nil ||
|
||||
strings.Join(order, ",") != "keep,announce" {
|
||||
t.Errorf("not trusted: %v %v, order %v; want kept, then the failed announcement said", unannounced, err, order)
|
||||
}
|
||||
order = nil
|
||||
failing := func() (bool, error) { order = append(order, "keep"); return false, errors.New("store away") }
|
||||
if _, _, err := keepGiven(false, announce(false), failing); err == nil || strings.Join(order, ",") != "keep" {
|
||||
t.Errorf("not trusted, keep failed: %v, order %v; want refused and nothing announced", err, order)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine the mesh does not know, as the secret's or as the desk, is refused before anybody is asked to type.
|
||||
func TestAGiveNamingAMachineTheMeshDoesNotKnowAsksNobody(t *testing.T) {
|
||||
for _, unknown := range []string{"elsewhere", "nodesk"} {
|
||||
d, accepted, acts, asked := aDesk(t, func(map[string]any) (json.RawMessage, error) {
|
||||
t.Fatal("the desk was asked")
|
||||
return nil, nil
|
||||
})
|
||||
d.known = func(machine string) error {
|
||||
if machine == unknown {
|
||||
return errors.New("no node " + machine)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
node, desk := "anchor", "laptop"
|
||||
if unknown == "elsewhere" {
|
||||
node = unknown
|
||||
} else {
|
||||
desk = unknown
|
||||
}
|
||||
_, err := d.give(node, "telegram", "telegram-token", desk)
|
||||
if err == nil || !strings.Contains(err.Error(), "nobody was asked") || !strings.Contains(err.Error(), unknown) {
|
||||
t.Errorf("%s: %v", unknown, err)
|
||||
}
|
||||
if len(*accepted)+len(*acts)+len(*asked) != 0 {
|
||||
t.Errorf("%s: something happened: %v %v %v", unknown, *accepted, *acts, *asked)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0277: the prompt names who asked — the controller's word on the bus's caller, cut to a name's
|
||||
// characters — and never a word the caller chose: there is no argument for it.
|
||||
func TestThePromptNamesWhoAskedFromTheBussWordAlone(t *testing.T) {
|
||||
d, _, acts, asked := aDesk(t, sealedTo(t, typed))
|
||||
d.askedBy = "g14/claude-code, through the mesh-controller seat"
|
||||
if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := (*asked)[0]["asked_by"]; got != "g14/claude-code" {
|
||||
t.Errorf("asked_by %q", got)
|
||||
}
|
||||
if len(*acts) != 1 || !strings.Contains((*acts)[0].Why, "asked by g14/claude-code") {
|
||||
t.Errorf("the record: %+v", *acts)
|
||||
}
|
||||
for in, want := range map[string]string{
|
||||
"jochen at a shell on novox": "jochen at a shell on novox",
|
||||
"laptop/agent": "laptop/agent",
|
||||
"Your bank asks\nType your PIN, now": "Your bank asks-Type your PIN",
|
||||
"": "an unnamed caller",
|
||||
"<b>x</b>": "an unnamed caller",
|
||||
strings.Repeat("a", 100): strings.Repeat("a", 80),
|
||||
"--prompt, something else": "an unnamed caller",
|
||||
} {
|
||||
if got := askedByName(in); got != want {
|
||||
t.Errorf("askedByName(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
// The verb's schema has no argument that reaches the prompt's words.
|
||||
for _, verb := range []string{"give", "secret-ask"} {
|
||||
for _, free := range []string{"asked_by", "prompt", "message", "value", "from"} {
|
||||
if _, err := argvFor(verb, map[string]any{"node": "anchor", "module": "telegram", "secret": "telegram-token",
|
||||
"at": "laptop", free: "x"}); err == nil {
|
||||
t.Errorf("%s takes %s", verb, free)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An ask for a secret is bounded before anybody is asked to type (ADR 0277): the record refuses it, nothing is
|
||||
// asked; and how every ask ends is recorded.
|
||||
func TestASecretAskIsBoundedAndItsEndRecorded(t *testing.T) {
|
||||
d, accepted, _, asked := aDesk(t, sealedTo(t, typed))
|
||||
var ended []string
|
||||
d.open = func(node, module, name, desk string) (int64, error) { return 7, nil }
|
||||
d.end = func(id int64, outcome string) error {
|
||||
ended = append(ended, fmt.Sprintf("%d %s", id, outcome))
|
||||
return nil
|
||||
}
|
||||
if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.open = func(node, module, name, desk string) (int64, error) {
|
||||
return 0, errors.New("an ask for telegram-token of telegram on anchor is still open")
|
||||
}
|
||||
_, err := d.give("anchor", "telegram", "telegram-token", "laptop")
|
||||
if err == nil || !strings.Contains(err.Error(), "nobody was asked to type anything") || !strings.Contains(err.Error(), "still open") {
|
||||
t.Errorf("a second ask: %v", err)
|
||||
}
|
||||
if len(*asked) != 1 || len(*accepted) != 1 {
|
||||
t.Errorf("asked %d, accepted %d", len(*asked), len(*accepted))
|
||||
}
|
||||
d.open = func(node, module, name, desk string) (int64, error) { return 8, nil }
|
||||
d.ask = func(string, map[string]any) (json.RawMessage, error) {
|
||||
return json.RawMessage(`{"cancelled":true}`), nil
|
||||
}
|
||||
if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); !errors.Is(err, errNothingGiven) {
|
||||
t.Errorf("a dismissed prompt: %v", err)
|
||||
}
|
||||
if strings.Join(ended, "; ") != "7 given; 8 dismissed" {
|
||||
t.Errorf("ended: %v", ended)
|
||||
}
|
||||
}
|
||||
|
||||
// A secret ask cannot be turned into a secret read: the line carries no value, the answer carries none, and every
|
||||
// other `secret` line is the terminal's.
|
||||
func TestASecretAskIsNeverASecretRead(t *testing.T) {
|
||||
t.Setenv(verbVar, "mesh-controller.secret-ask")
|
||||
for _, args := range [][]string{
|
||||
{"ask", "anchor", "telegram", "telegram-token", "the-value"},
|
||||
{"ask", "anchor", "telegram"},
|
||||
{"ask", "anchor", "telegram", "telegram-token", "--from", "/dev/null"},
|
||||
} {
|
||||
if err := secretCommand(context.Background(), args); err == nil {
|
||||
t.Errorf("secret %v was taken", args)
|
||||
}
|
||||
}
|
||||
for _, args := range [][]string{{"recover", "anchor", "telegram", "telegram-token"}, {"export"}} {
|
||||
if err := terminalOnly(append([]string{"secret"}, args...)); err == nil {
|
||||
t.Errorf("secret %v passed the terminal rule", args)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -74,12 +74,8 @@ type probe struct {
|
||||
// probeRegistry is the registry, in to-be 45's order. **The registry is the design's live form**: a
|
||||
// probe added to a design is a row added here.
|
||||
var probeRegistry = []probe{
|
||||
// D1 also says every module assigned and left out of a machine's declaration (novox/hq issue 380), from the
|
||||
// resolution it already makes: needs-operator for a setting nobody gave, left-out for any other cause.
|
||||
{ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation; no module " +
|
||||
"assigned to a machine is left out of its declaration unsaid",
|
||||
From: "issues 236, 263, 275, 380", Kind: "declaration-refused",
|
||||
Raises: []string{kindAwaitingPush, kindLeftOut, kindNeedsOperator}, Phase: 1,
|
||||
{ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation",
|
||||
From: "issues 236, 263, 275", Kind: "declaration-refused", Raises: []string{kindAwaitingPush}, Phase: 1,
|
||||
run: probeDeclarations},
|
||||
{ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6",
|
||||
From: "issue 262", Kind: "resolver-wrong", Phase: 1, run: probeResolvers},
|
||||
@@ -120,11 +116,6 @@ var probeRegistry = []probe{
|
||||
{ID: probeDeliveriesID, Asserts: "no delivery is held past its state's bound unsaid: mesh-delivery's " +
|
||||
"`stalled`, each with the transition its table lets healer H2 take", From: "ADR 0239",
|
||||
Kind: kindDeliveryStalled, Phase: 3, run: probeDeliveries},
|
||||
// The delivery budgets (novox/hq ADR 0282 decision 7): the newest delivery of each class within its budget,
|
||||
// read from mesh-delivery's `times`; a measurement said, never a delivery held.
|
||||
{ID: probeBudgetsID, Asserts: "the newest delivery of a leaf module ran on every machine within three minutes of " +
|
||||
"its merge, and of a core module within ten: mesh-delivery's `times`", From: "ADR 0282, ADR 0304",
|
||||
Kind: kindOverBudget, Phase: 3, run: probeBudgets},
|
||||
// A client of the bus reconnecting in a loop (novox/hq issue 327), from the server's record of closed
|
||||
// connections, which the bus's own module reads.
|
||||
{ID: probeReconnectsID, Asserts: "no user of the bus had its connection dropped more than twelve times in the " +
|
||||
@@ -135,17 +126,6 @@ var probeRegistry = []probe{
|
||||
{ID: agentAccountProbe, Asserts: "every machine that names an agent account has it judged, on its node-engine's " +
|
||||
"newest statement, unable to become root without a person", From: "ADR 0266, ADR 0259 §8",
|
||||
Kind: kindAgentCanBecomeRoot, Phase: 1, run: probeAgentAccounts},
|
||||
// Root where the trusted parties run (novox/hq ADR 0259 §8): while an agent can become root there without a
|
||||
// person, an answer proven there proves nothing.
|
||||
{ID: "D-root", Asserts: "no agent can become root without a person on a machine where the router or a channel " +
|
||||
"proving its sender runs: not by its own account, and not through a tool that runs its command as an account " +
|
||||
"that can", From: "ADR 0259 §8", Kind: kindRootNotFree, Phase: 2, run: probeAgentRoot},
|
||||
// A module's bucket or log filling toward its cap (novox/hq ADR 0297 §6): said at three quarters, cleared
|
||||
// below seven tenths, so one at its threshold is not raised and cleared on every run. One that cannot be
|
||||
// read is said on its own, and every other is still judged.
|
||||
{ID: probeFillID, Asserts: "every module's bucket and log holds less than three quarters of its cap; one " +
|
||||
"said filling is cleared once it holds less than seven tenths", From: "ADR 0297, issue 501",
|
||||
Kind: kindBucketOrLogFilling, Raises: []string{kindBucketOrLogUnread}, Phase: 1, run: probeFill},
|
||||
{ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals",
|
||||
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs},
|
||||
// The core's health definitions (novox/hq to-be 45 §8, ADR 0236): what a core component's new build is
|
||||
@@ -651,7 +631,7 @@ func localDoctor(ctx context.Context) (*doctor, func(), error) {
|
||||
open.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
k, err := keeperOn(ctx, js.Conn(), storeLooks{open.inventory})
|
||||
k, err := keeperOn(ctx, js.Conn())
|
||||
if err != nil {
|
||||
js.Close()
|
||||
open.Close()
|
||||
|
||||
@@ -205,7 +205,7 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
read, err := readForPlanning(ctx, inv)
|
||||
read, err := inv.ReadRepositories(ctx)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
@@ -411,18 +411,7 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot
|
||||
Commit: e.Source.BuiltFrom, Provided: e.Provided, RollOut: current[e.Manifest.Module].RollOut,
|
||||
Manifest: raw}
|
||||
for _, r := range read[e.Manifest.Module] {
|
||||
// The module's own build source is said apart: a gate that predates it would read an own
|
||||
// entry among Reads as a context of its own repository.
|
||||
if r.Own {
|
||||
if len(r.Paths) > 0 {
|
||||
mod.Sources = append(mod.Sources, snapshot.BuildSource{Own: true, Paths: r.Paths})
|
||||
}
|
||||
continue
|
||||
}
|
||||
mod.Reads = append(mod.Reads, snapshot.RepositoryName(r.Repository))
|
||||
if len(r.Paths) > 0 {
|
||||
mod.Sources = append(mod.Sources, snapshot.BuildSource{Repository: snapshot.RepositoryName(r.Repository), Paths: r.Paths})
|
||||
}
|
||||
}
|
||||
f.Modules = append(f.Modules, mod)
|
||||
if e.Provided || e.Source.Repository == "" {
|
||||
@@ -514,10 +503,6 @@ func composedAndValidated(ctx context.Context, open *stores, node string, gens m
|
||||
if declared.Epoch, err = open.inventory.SentEpoch(ctx, record.ID); err != nil {
|
||||
return sendable{}, nil, err
|
||||
}
|
||||
// And the generation it was last sent, as the would-send is (novox/hq issue 234).
|
||||
if declared.Generation, err = open.inventory.SentGeneration(ctx, record.ID); err != nil {
|
||||
return sendable{}, nil, err
|
||||
}
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
return sendable{}, nil, err
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A provider whose wait fails the controller's check lists who waits on it (novox/hq issue 450).
|
||||
//
|
||||
// A wait that does not check out is judged unhealthy (ADR 0283 decision 3), so the provider raises its unhealthy
|
||||
// condition and its consumers are held under it (ADR 0240 rule 5). What is stored is what the machine said, the
|
||||
// wait unchecked: read as said, the provider seems to wait for the operator, and the held consumers were never
|
||||
// listed on the condition that is open.
|
||||
|
||||
// refusedWait is a wait for a secret the database's manifest does not declare: it fails the check.
|
||||
var refusedWait = inventory.Wait{Part: "postgres-database", Secret: "certificate", What: "the database's certificate"}
|
||||
|
||||
// judgedRefused is the provider's resource as the controller judges it: unhealthy, saying why.
|
||||
func judgedRefused() inventory.ResourceHealth {
|
||||
r := waitingDatabaseFor(refusedWait)
|
||||
r.State, r.Waits = link.StateUnhealthy, nil
|
||||
r.Reason = "says it waits for the secret certificate, which db does not declare"
|
||||
return r
|
||||
}
|
||||
|
||||
// The consumer's statement arrives after the provider's, so it is the consumer's judging (sayWaiters) that must list
|
||||
// it at the provider's unhealthy condition, made urgent by who waits on it.
|
||||
func TestAProviderWhoseWaitFailedItsCheckListsWhoWaitsOnIt(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
stored := waitingDatabaseFor(refusedWait)
|
||||
open := judgeBoth(t, k, []inventory.ResourceHealth{judgedRefused()},
|
||||
map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Resources: []inventory.ResourceHealth{stored}}},
|
||||
shopFailingBeside(stored))
|
||||
provider := conditionOf(open, moduleUnhealthyKey("db", "anchor"))
|
||||
if len(open) != 1 || provider == nil {
|
||||
t.Fatalf("a provider whose wait failed its check and a consumer of it raised %v; want the provider's "+
|
||||
"unhealthy alone", openKeysOf(open))
|
||||
}
|
||||
if said := provider.Evidence[0].Said; !strings.Contains(said, "shop on laptop") {
|
||||
t.Fatalf("the provider's unhealthy condition does not list shop on laptop as waiting on it: %s", said)
|
||||
}
|
||||
if provider.Severity != conditions.Urgent {
|
||||
t.Fatalf("the provider's unhealthy condition is %s with a consumer waiting on it; want urgent", provider.Severity)
|
||||
}
|
||||
}
|
||||
|
||||
// A provider that waits for a secret already given is unhealthy to its consumers too, before its own condition opens:
|
||||
// they are held under it as under any unhealthy provider, never as waiting for the operator, and its condition, when
|
||||
// open, is said as unhealthy with who waits on it.
|
||||
func TestAProviderWaitingForASecretAlreadyGivenIsUnhealthyToItsConsumers(t *testing.T) {
|
||||
given := holdingOf(nil, shopFailingBeside(waitingDatabase()), shopOnTheDatabase)
|
||||
given.waits.given["db@anchor"] = map[string]time.Time{"licence": time.Now().Add(-time.Hour)}
|
||||
by, held := given.heldWith("laptop", "shop", []inventory.ResourceHealth{failingConsumer("shop")})
|
||||
if !held || by.provider != theDatabase || len(by.waits) > 0 {
|
||||
t.Fatalf("shop under a database waiting for a licence already given: held %v under %v with waits %v; want "+
|
||||
"held under db on anchor as unhealthy, no waits", held, by.provider, by.waits)
|
||||
}
|
||||
if _, uncovered := given.waitingUncovered("laptop", "shop", []inventory.ResourceHealth{failingConsumer("shop")}); uncovered {
|
||||
t.Fatalf("a provider whose wait failed its check is said as waiting for another part")
|
||||
}
|
||||
}
|
||||
@@ -1,148 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A wait on a clock of its own: each pause moves it on, and nothing waits in real time.
|
||||
func aFakeWait(bound time.Duration) (waitOut, *[]time.Duration) {
|
||||
clock := time.Date(2026, 10, 11, 11, 52, 2, 0, time.UTC)
|
||||
var paused []time.Duration
|
||||
return waitOut{bound: bound, first: time.Second, most: 10 * time.Second,
|
||||
now: func() time.Time { return clock },
|
||||
sleep: func(_ context.Context, d time.Duration) error {
|
||||
paused = append(paused, d)
|
||||
clock = clock.Add(d)
|
||||
return nil
|
||||
}}, &paused
|
||||
}
|
||||
|
||||
// notServed is what the bus says while node-tools re-serves, in the words seen on 2026-10-11.
|
||||
func notServed() error {
|
||||
return fmt.Errorf("cannot ask gitea.gitea_api: %w", link.ErrNothingServes)
|
||||
}
|
||||
|
||||
// A registration's forge question that lands while the tool runner re-serves is asked again, and the forge's
|
||||
// answer, seconds later, is the registration's (novox/hq issue 486).
|
||||
func TestTheForgeNotServedForAMomentIsWaitedOut(t *testing.T) {
|
||||
w, paused := aFakeWait(60 * time.Second)
|
||||
asked := 0
|
||||
facts, err := forgeWaitedOut(t.Context(), w, "novox/mesh-catalog", func(context.Context) (forgeFacts, error) {
|
||||
asked++
|
||||
if asked <= 2 {
|
||||
return forgeFacts{}, notServed()
|
||||
}
|
||||
return forgeFacts{ID: 42, Guarded: true}, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("a forge not served for two asks failed the registration: %v", err)
|
||||
}
|
||||
if facts.ID != 42 || !facts.Guarded || asked != 3 {
|
||||
t.Fatalf("the forge's answer was %+v after %d asks, want its answer on the third", facts, asked)
|
||||
}
|
||||
if want := []time.Duration{time.Second, 2 * time.Second}; fmt.Sprint(*paused) != fmt.Sprint(want) {
|
||||
t.Fatalf("paused %v, want %v", *paused, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A tool that did not answer within its wait has held the receive loop that long already: not asked again.
|
||||
func TestTheForgeNotAnsweringInTimeIsNotAskedAgain(t *testing.T) {
|
||||
w, paused := aFakeWait(60 * time.Second)
|
||||
asked := 0
|
||||
_, err := forgeWaitedOut(t.Context(), w, "novox/x", func(context.Context) (forgeFacts, error) {
|
||||
asked++
|
||||
return forgeFacts{}, fmt.Errorf("gitea did not answer: %w", link.ErrNoAnswer)
|
||||
})
|
||||
if !errors.Is(err, link.ErrNoAnswer) || asked != 1 || len(*paused) != 0 {
|
||||
t.Fatalf("a forge that did not answer in time was asked %d times, paused %v: %v", asked, *paused, err)
|
||||
}
|
||||
}
|
||||
|
||||
// An ask that hangs past the bound is cut off at the bound, not at its own wait, and the failure still says it
|
||||
// waited: the receive loop is held no longer than the bound (novox/hq issue 486).
|
||||
func TestAnAskHangingPastTheBoundIsCutOffAtIt(t *testing.T) {
|
||||
w := waitOut{bound: 200 * time.Millisecond, first: 10 * time.Millisecond, most: 50 * time.Millisecond,
|
||||
now: time.Now, sleep: sleepOrDone}
|
||||
asked := 0
|
||||
began := time.Now()
|
||||
_, err := forgeWaitedOut(t.Context(), w, "novox/mesh-catalog", func(ctx context.Context) (forgeFacts, error) {
|
||||
asked++
|
||||
if asked == 1 {
|
||||
return forgeFacts{}, notServed()
|
||||
}
|
||||
select { // link.Ask's own wait is 30 s; the bound must end it first
|
||||
case <-ctx.Done():
|
||||
return forgeFacts{}, fmt.Errorf("gitea did not answer: %w", link.ErrNoAnswer)
|
||||
case <-time.After(30 * time.Second):
|
||||
return forgeFacts{ID: 1, Guarded: true}, nil
|
||||
}
|
||||
})
|
||||
if took := time.Since(began); took > 2*time.Second {
|
||||
t.Fatalf("the wait took %s, past its bound of %s", took, w.bound)
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), "waited") || !strings.Contains(err.Error(), "asking 2 times") ||
|
||||
!strings.Contains(err.Error(), "issue 486") {
|
||||
t.Fatalf("the bound's end does not say it waited: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Past the bound the registration fails, saying it waited, for how long, and what the forge last said.
|
||||
func TestTheForgeNotServedPastTheBoundFailsSayingItWaited(t *testing.T) {
|
||||
w, paused := aFakeWait(60 * time.Second)
|
||||
_, err := forgeWaitedOut(t.Context(), w, "novox/mesh-catalog", func(context.Context) (forgeFacts, error) {
|
||||
return forgeFacts{}, notServed()
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("a forge never served was read as an answer")
|
||||
}
|
||||
var waited time.Duration
|
||||
for _, p := range *paused {
|
||||
waited += p
|
||||
}
|
||||
if waited > 60*time.Second || waited < 40*time.Second {
|
||||
t.Fatalf("waited %s in all, want close to and within the bound of 60s", waited)
|
||||
}
|
||||
for _, want := range []string{"waited " + waited.String(), "novox/mesh-catalog", "nothing serves", "issue 486"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the failure does not say %q: %v", want, err)
|
||||
}
|
||||
}
|
||||
if !errors.Is(err, link.ErrNothingServes) {
|
||||
t.Errorf("the failure lost what the forge last said: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A refusal is the forge's word: never asked again.
|
||||
func TestARefusalFromTheForgeIsNeverAskedAgain(t *testing.T) {
|
||||
w, paused := aFakeWait(60 * time.Second)
|
||||
asked := 0
|
||||
_, err := forgeWaitedOut(t.Context(), w, "novox/x", func(context.Context) (forgeFacts, error) {
|
||||
asked++
|
||||
return forgeFacts{}, errors.New("gitea.gitea_api: 404 Not Found")
|
||||
})
|
||||
if err == nil || asked != 1 || len(*paused) != 0 {
|
||||
t.Fatalf("a refusal was asked %d times, paused %v: %v", asked, *paused, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A registration whose own context ends stops waiting at once.
|
||||
func TestAWaitForTheForgeEndsWithItsContext(t *testing.T) {
|
||||
w, _ := aFakeWait(60 * time.Second)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
w.sleep = func(ctx context.Context, d time.Duration) error { cancel(); return sleepOrDone(ctx, d) }
|
||||
asked := 0
|
||||
_, err := forgeWaitedOut(ctx, w, "novox/x", func(context.Context) (forgeFacts, error) {
|
||||
asked++
|
||||
return forgeFacts{}, notServed()
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) || asked != 1 {
|
||||
t.Fatalf("asked %d times after the context ended: %v", asked, err)
|
||||
}
|
||||
}
|
||||
@@ -18,7 +18,7 @@ func foundDirectory(module string, since time.Time) inventory.ResourceHealth {
|
||||
return inventory.ResourceHealth{Module: module, Resource: module + ".data", Kind: link.KindDirectory,
|
||||
Target: "/srv/" + module, State: link.StateUnhealthy, Since: since,
|
||||
Reason: link.ReasonUsedAsFound + " owned by 1000:1000, mode 700, as found; root, mode 755 was declared and " +
|
||||
"not given it — `nox node hand-over laptop <directory>` on the control-node, with its path, hands it to the mesh"}
|
||||
"not given it — `mesh-host hand-over` at the machine hands it to the mesh"}
|
||||
}
|
||||
|
||||
func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) {
|
||||
|
||||
+23
-302
@@ -27,10 +27,8 @@ import (
|
||||
// **"Reported applied" is not enough.** ADR 0218 sent a module to one machine first and the rest once
|
||||
// that machine reported it applied. A build that applies and then does nothing, crashes, serves no
|
||||
// tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by
|
||||
// the component's health — the core's health definitions, or a module's own — within ten minutes of the
|
||||
// apply: a core module passing three times over at least two minutes, a leaf module once, as soon as the
|
||||
// node-engine states it healthy (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it). Only then
|
||||
// are the rest sent.
|
||||
// the component's health — the core's health definitions, or a module's own — passing three times over
|
||||
// at least two minutes, within ten minutes of the apply. Only then are the rest sent.
|
||||
//
|
||||
// **A failing gate stops the plan and puts the previous build back there.** The build is marked failed
|
||||
// at its gate — registration refuses it and nothing sends it again on its own — the module's registered
|
||||
@@ -45,38 +43,11 @@ var (
|
||||
gateSettle = 2 * time.Minute
|
||||
// gateBound is how long after the apply the build may take to become healthy.
|
||||
gateBound = 10 * time.Minute
|
||||
// gatePasses is how many consecutive judgings must find a core module healthy, gateEvery apart at the
|
||||
// least, after gateSettle. gateLeafPasses is a leaf module's: one, taken once the node-engine states every
|
||||
// resource of it healthy by its own check, with no settle (novox/hq ADR 0282 decision 4 as ADR 0304
|
||||
// decision 3 changed it, issue 524).
|
||||
gatePasses = 3
|
||||
gateLeafPasses = 1
|
||||
gateLeafSettles = false
|
||||
gateEvery = 40 * time.Second
|
||||
// gatePasses is how many consecutive judgings must find it healthy, gateEvery apart at the least.
|
||||
gatePasses = 3
|
||||
gateEvery = 40 * time.Second
|
||||
)
|
||||
|
||||
// coreModule is whether a module is judged as the core (novox/hq ADR 0282 decision 1): one on the controller's
|
||||
// own path, or one holding the mesh's resolver. Every other module is a leaf.
|
||||
func coreModule(module string, m catalogue.Manifest) bool {
|
||||
if _, own := onTheControllersPath[module]; own {
|
||||
return true
|
||||
}
|
||||
for _, c := range m.Claims {
|
||||
if c.Name == resolverSeat {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return coreComponent(module) != ""
|
||||
}
|
||||
|
||||
// gateWants is how many passes a module needs, and whether it must stay healthy gateSettle first.
|
||||
func gateWants(module string, m catalogue.Manifest) (passes int, settles bool) {
|
||||
if coreModule(module, m) {
|
||||
return gatePasses, true
|
||||
}
|
||||
return gateLeafPasses, gateLeafSettles
|
||||
}
|
||||
|
||||
// gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions
|
||||
// are raised by a plan as it judges, and kept or cleared by the probe on every run.
|
||||
const gateProbe = "DG"
|
||||
@@ -116,9 +87,6 @@ const (
|
||||
healthWaiting
|
||||
healthNotYet
|
||||
healthBroken
|
||||
// healthSuperseded is a judging that cannot go on: the machine was sent another build of the module
|
||||
// after the gate's send (novox/hq issue 352). No verdict on the build judged, and nothing put back.
|
||||
healthSuperseded
|
||||
)
|
||||
|
||||
// served is what one machine's node tools answered the bus's discovery with.
|
||||
@@ -150,69 +118,10 @@ type gateFacts struct {
|
||||
health map[string]inventory.NodeHealth
|
||||
healthErr error
|
||||
// heldOn is, per "<module>@<machine>", the provider its findings are held under (ADR 0240 rule 5).
|
||||
heldOn map[string]heldReading
|
||||
heldOn map[string]string
|
||||
// groupsAdded is, per module, whether the move judged puts an account in a group its previous build did
|
||||
// not (issue 318 review): the only move whose wait for a new login is excused.
|
||||
groupsAdded map[string]bool
|
||||
// waits is what the controller holds to check a module's wait for the operator (novox/hq ADR 0283): the
|
||||
// manifest of each module's build judged, and the secrets given on each machine.
|
||||
waits operatorWaitFacts
|
||||
// sent is, per machine, the declaration the gate's own send carried there (novox/hq issue 352): a
|
||||
// report is held against it, never against the send made last. sentBuilds is what each machine was
|
||||
// last sent of every module, and judged the commit of each module this gate judges: a machine last
|
||||
// sent another build of the module is not running the build judged.
|
||||
sent map[string]inventory.SentDeclaration
|
||||
sentBuilds map[string]map[string]string
|
||||
commits map[string]string
|
||||
}
|
||||
|
||||
// heldReading is the provider a module's findings are held under, as "<module> on <machine>", and, when that
|
||||
// provider only waits for the operator for the part the module needs, its wait in one sentence (novox/hq issue
|
||||
// 405): the module's gate then reads as a wait for a person (ADR 0254), a pass carrying the wait, as ADR 0283
|
||||
// decision 4 reads the waiting provider itself. A walk never waits on the operator's secret, there or here.
|
||||
type heldReading struct {
|
||||
on, waits string
|
||||
}
|
||||
|
||||
// heldReadings is, per "<module>@<machine>" in every machine's newest statement, what its findings are held under.
|
||||
func heldReadings(hold *holding) map[string]heldReading {
|
||||
out := map[string]heldReading{}
|
||||
for machine := range hold.healths {
|
||||
for module, by := range hold.heldModules(machine) {
|
||||
reading := heldReading{on: by.provider.Module + " on " + by.provider.Node}
|
||||
if len(by.waits) > 0 {
|
||||
reading.waits = operatorWaitSaid(by.provider.Module, by.provider.Node, by.waits)
|
||||
}
|
||||
out[module+"@"+machine] = reading
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// reportedOn says a machine's last report is on what the gate sent it (novox/hq issue 352): on that
|
||||
// declaration, or one it was sent after it — or, for a gate kept before sends were kept on it, on the
|
||||
// declaration last sent. On 2026-10-09 a release's gate read the control node's report against a newer
|
||||
// send another plan had just made there, and failed three builds the machine had reported healthy as
|
||||
// "has not reported on what it was sent".
|
||||
func (f gateFacts) reportedOn(machine string, r inventory.Reported) bool {
|
||||
if sent, kept := f.sent[machine]; kept {
|
||||
return sent.ReportsOn(r)
|
||||
}
|
||||
return r.Current
|
||||
}
|
||||
|
||||
// supersededOn says the machine was last sent another build of the module than the one this gate judges
|
||||
// (novox/hq issue 352): the judging cannot go on, whatever the machine reports. On 2026-10-09 a controller
|
||||
// put back by one gate judged another gate's newer controller build passed on the same machine, reading
|
||||
// the put-back build's health as the newer one's.
|
||||
func (f gateFacts) supersededOn(module, machine string) (string, bool) {
|
||||
judged, known := f.commits[module]
|
||||
sent, has := f.sentBuilds[machine][module]
|
||||
if !known || !has || judged == "" || sent == "" || sameCommit(sent, judged) {
|
||||
return "", false
|
||||
}
|
||||
return fmt.Sprintf("%s was sent %s %s after this gate's %s: the build judged no longer runs there, and "+
|
||||
"this judging is superseded by that send's", machine, module, short(sent), short(judged)), true
|
||||
}
|
||||
|
||||
// gatherGateFacts reads what a judging needs, from the store, the bus and this controller's memory. A
|
||||
@@ -254,7 +163,12 @@ var gatherGateFacts = func(ctx context.Context, open *stores, component string)
|
||||
// Whose findings wait on an unhealthy provider (ADR 0240 rule 5): their gates wait, not fail.
|
||||
if f.healthErr == nil && f.openErr == nil {
|
||||
if hold, err := readHolding(ctx, inv, f.open); err == nil {
|
||||
f.heldOn = heldReadings(hold)
|
||||
f.heldOn = map[string]string{}
|
||||
for machine := range f.health {
|
||||
for module, p := range hold.heldModules(machine) {
|
||||
f.heldOn[module+"@"+machine] = p.Module + " on " + p.Node
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if theLease != nil {
|
||||
@@ -285,12 +199,9 @@ func judgeHealth(module, component string, m catalogue.Manifest, machine string,
|
||||
return healthBroken, fmt.Sprintf("the witness on %s judged the %s %s and %s: %s", machine, r.Component,
|
||||
short(r.From), r.Outcome, r.Why)
|
||||
}
|
||||
if why, superseded := f.supersededOn(module, machine); superseded {
|
||||
return healthSuperseded, why
|
||||
}
|
||||
r, said := f.reports[machine]
|
||||
switch {
|
||||
case !said || r.At == nil || !f.reportedOn(machine, r):
|
||||
case !said || r.At == nil || !r.Current:
|
||||
return healthNotYet, fmt.Sprintf("%s has not reported on what it was sent", machine)
|
||||
case r.Outcome == inventory.OutcomeFailed || r.Outcome == inventory.OutcomeRefused:
|
||||
return healthBroken, fmt.Sprintf("%s %s what it was sent", machine, r.Outcome)
|
||||
@@ -298,20 +209,17 @@ func judgeHealth(module, component string, m catalogue.Manifest, machine string,
|
||||
return healthNotYet, fmt.Sprintf("%s reported %q", machine, r.Outcome)
|
||||
}
|
||||
// **No new condition about it**: about the machine itself, or naming the module on that machine,
|
||||
// raised since the judging began. The gate's own are not evidence about the build. A fault that was
|
||||
// there at the send and reopened since is not new; one that had cleared before the send and came back
|
||||
// after it is (OpenAt, novox/hq issue 348).
|
||||
// raised since the judging began. The gate's own are not evidence about the build.
|
||||
if f.judged {
|
||||
if f.openErr != nil {
|
||||
return healthNotYet, "what is wrong cannot be read, so whether the build made anything wrong is not known: " +
|
||||
firstLine(f.openErr.Error())
|
||||
}
|
||||
for _, c := range f.open {
|
||||
// A wait for a person's new login, for a directory used as found to be handed over, or for the
|
||||
// operator's secret or setting, is the module's reading, not a fault raised since the send: the gate
|
||||
// reads it from the statement below (ADR 0254, novox/hq issue 339, ADR 0283).
|
||||
if c.Source == gateProbe || c.OpenAt(since) || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound ||
|
||||
c.Kind == kindNeedsOperator {
|
||||
// A wait for a person's new login, or for a directory used as found to be handed over, is the module's
|
||||
// reading, not a fault raised since the send: the gate reads it from the statement below (ADR 0254,
|
||||
// novox/hq issue 339).
|
||||
if c.Source == gateProbe || c.Raised.Before(since) || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound {
|
||||
continue
|
||||
}
|
||||
onIt := c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) ||
|
||||
@@ -423,7 +331,7 @@ func aboutTheMachine(machine string, moved []string, since time.Time, f gateFact
|
||||
aboutIt := c.Subject.Scope == conditions.ScopeMachine && (c.Subject.ID == machine || c.Subject.Machine == machine ||
|
||||
slices.Contains(c.Subject.Also, machine))
|
||||
// A directory used as found waits for a person, whatever the send did (novox/hq issue 339).
|
||||
if !aboutIt || c.Source == gateProbe || c.OpenAt(since) || c.Kind == kindUsedAsFound {
|
||||
if !aboutIt || c.Source == gateProbe || c.Raised.Before(since) || c.Kind == kindUsedAsFound {
|
||||
kept = append(kept, c)
|
||||
continue
|
||||
}
|
||||
@@ -528,22 +436,6 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
return "", err
|
||||
}
|
||||
facts.groupsAdded = movesAddingGroups(ctx, open.inventory, g, pairs, shelf)
|
||||
facts.waits = gateWaitFacts(ctx, open.inventory, g, pairs, shelf, facts.health)
|
||||
facts.sent = g.Sent
|
||||
facts.commits, facts.sentBuilds = judgedCommits(g, pairs), map[string]map[string]string{}
|
||||
// A module this gate put back at once (putBackBroken) was sent its earlier build by the gate itself:
|
||||
// not another send, and not a judging superseded.
|
||||
for _, m := range g.Returned {
|
||||
delete(facts.commits, m)
|
||||
}
|
||||
for _, j := range pairs {
|
||||
if _, read := facts.sentBuilds[j.node]; read {
|
||||
continue
|
||||
}
|
||||
if builds, known, err := open.inventory.SentBuilds(ctx, j.node); err == nil && known {
|
||||
facts.sentBuilds[j.node] = builds
|
||||
}
|
||||
}
|
||||
// **What is wrong with a machine itself is the machine's** (novox/hq issue 281): read once for each
|
||||
// machine judged, apart from what is wrong with a module there, and never pinned on the module the
|
||||
// gate happens to be kept on.
|
||||
@@ -580,13 +472,6 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
if _, seen := reading[j.module]; !seen {
|
||||
modules = append(modules, j.module)
|
||||
}
|
||||
if h == healthSuperseded {
|
||||
// Decided at once (novox/hq issue 352): nothing of this gate's can be judged on a machine that
|
||||
// was sent another build of it, and nothing is put back — the later send is what runs there.
|
||||
g.Failing, g.Last = nil, ""
|
||||
decide(g, inventory.GateSuperseded, said, now)
|
||||
return g.Verdict, nil
|
||||
}
|
||||
if h == healthBroken && !slices.Contains(g.Broken, j.module) {
|
||||
g.Broken = append(g.Broken, j.module)
|
||||
if g.BrokenWhy == "" {
|
||||
@@ -637,24 +522,8 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
}
|
||||
}
|
||||
settled := now.Sub(*g.Since) >= gateSettle
|
||||
// What the send needs is what its most demanding module needs: a send carrying a core module is judged
|
||||
// as the core.
|
||||
wants, settles := 0, false
|
||||
for _, m := range modules {
|
||||
p, s := gateWants(m, shelf[m])
|
||||
wants, settles = max(wants, p), settles || s
|
||||
}
|
||||
if len(modules) == 0 {
|
||||
// A send judged on no machine of its modules (its first machine not running them) has no class to
|
||||
// be judged by: it is judged as the core, never passed at once without a reading.
|
||||
wants, settles = gatePasses, true
|
||||
}
|
||||
g.Wants = wants
|
||||
// A module's own pass is its own class's: a leaf beside a core module that fails keeps its build after
|
||||
// its one reading, and only what did not pass is put back (issue 318's rule, by class since issue 524).
|
||||
passedAlone := func(m string) bool {
|
||||
p, s := gateWants(m, shelf[m])
|
||||
return (settled || !s) && reading[m] <= healthPerson && g.Healthy[m] >= p
|
||||
return settled && reading[m] <= healthPerson && g.Healthy[m] >= gatePasses
|
||||
}
|
||||
// fail decides the send failed: what passed on its own keeps its pass, everything else is put back.
|
||||
fail := func(why string) {
|
||||
@@ -671,7 +540,6 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
pastBound := now.Sub(*g.Since) > gateBound
|
||||
switch {
|
||||
case worst == healthBroken:
|
||||
g.Read(now, false, g.BrokenWhy)
|
||||
var judging []string
|
||||
for _, m := range modules {
|
||||
if reading[m] != healthBroken && !passedAlone(m) {
|
||||
@@ -690,10 +558,8 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
// Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either —
|
||||
// the provider's own condition says what is wrong (ADR 0240 rule 5).
|
||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
||||
g.Read(now, false, why)
|
||||
case worst == healthNotYet:
|
||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
||||
g.Read(now, false, why)
|
||||
if pastBound {
|
||||
fail(fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why))
|
||||
}
|
||||
@@ -701,8 +567,7 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
// Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict.
|
||||
g.Passes++
|
||||
g.LastPass, g.Last, g.Failing = &now, "", nil
|
||||
g.Read(now, true, "")
|
||||
if g.Passes >= wants && (settled || !settles) {
|
||||
if g.Passes >= gatePasses && settled {
|
||||
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
|
||||
now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now)
|
||||
}
|
||||
@@ -710,40 +575,6 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
return g.Verdict, nil
|
||||
}
|
||||
|
||||
// judgedCommits is the commit of each module a gate judges: the gate's own To for its module, and each
|
||||
// carried move's. Pure.
|
||||
func judgedCommits(g *inventory.PlanGate, pairs []judged) map[string]string {
|
||||
out := map[string]string{}
|
||||
for _, c := range g.Carried {
|
||||
if c.To != "" {
|
||||
out[c.Module] = c.To
|
||||
}
|
||||
}
|
||||
if g.To != "" {
|
||||
for _, j := range pairs {
|
||||
if _, has := out[j.module]; !has && !slices.ContainsFunc(g.Carried, func(c inventory.CarriedMove) bool { return c.Module == j.module }) {
|
||||
out[j.module] = g.To
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sentNow is what each machine was just sent, read after a send for the gate to keep (novox/hq issue
|
||||
// 352): a machine whose send is not on record is left out, and its report is read as before.
|
||||
func sentNow(ctx context.Context, inv *inventory.Inventory, machines []string) map[string]inventory.SentDeclaration {
|
||||
out := map[string]inventory.SentDeclaration{}
|
||||
for _, n := range machines {
|
||||
if s, found, err := inv.SentTo(ctx, n); err == nil && found {
|
||||
out[n] = s
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// whyFor is a passing gate's why as one module's verdict says it: the send's, and that module's own wait
|
||||
// for a person, never another's (issue 318 review).
|
||||
func whyFor(g *inventory.PlanGate, module string) string {
|
||||
@@ -819,47 +650,6 @@ func movesAddingGroups(ctx context.Context, inv *inventory.Inventory, g *invento
|
||||
return out
|
||||
}
|
||||
|
||||
// gateWaitFacts reads what checks the waits for the operator of the modules a gate judges (novox/hq ADR 0283): the
|
||||
// manifest of the build judged — the one it moves to, else the catalogue's — and the secrets given on each machine
|
||||
// that says a module of them waits.
|
||||
func gateWaitFacts(ctx context.Context, inv *inventory.Inventory, g *inventory.PlanGate, pairs []judged,
|
||||
shelf map[string]catalogue.Manifest, health map[string]inventory.NodeHealth) operatorWaitFacts {
|
||||
var f operatorWaitFacts
|
||||
judgedManifests := map[string]catalogue.Manifest{}
|
||||
byMachine := map[string][]string{}
|
||||
for _, j := range pairs {
|
||||
waiting := false
|
||||
for _, r := range health[j.node].Resources {
|
||||
if r.Module == j.module && r.State == link.StateWaiting {
|
||||
waiting = true
|
||||
}
|
||||
}
|
||||
if !waiting {
|
||||
continue
|
||||
}
|
||||
byMachine[j.node] = append(byMachine[j.node], j.module)
|
||||
if _, done := judgedManifests[j.module]; done {
|
||||
continue
|
||||
}
|
||||
to := g.To
|
||||
for _, c := range g.Carried {
|
||||
if c.Module == j.module {
|
||||
to = c.To
|
||||
break
|
||||
}
|
||||
}
|
||||
if m, found, err := inv.ManifestAt(ctx, j.module, to); err == nil && found {
|
||||
judgedManifests[j.module] = m
|
||||
} else if m, known := shelf[j.module]; known {
|
||||
judgedManifests[j.module] = m
|
||||
}
|
||||
}
|
||||
for machine, modules := range byMachine {
|
||||
readWaitFacts(ctx, inv, machine, modules, judgedManifests, &f)
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// decide sets a gate's verdict.
|
||||
func decide(g *inventory.PlanGate, verdict, why string, now time.Time) {
|
||||
g.Verdict, g.Why, g.JudgedAt = verdict, why, &now
|
||||
@@ -1010,16 +800,6 @@ func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module str
|
||||
"back", module, short(state.Previous), inventory.KeptBuilds))
|
||||
return
|
||||
}
|
||||
if g.Component == lease.ComponentController {
|
||||
// **The controller is never put back to a build older than the store's schema** (novox/hq issue
|
||||
// 352): the build before it carries fewer migrations than the failed one applied, starts behind
|
||||
// its own records, and judges the next gate with what it can read. The current build is kept and
|
||||
// the condition says so; a person decides.
|
||||
if why, ok := controllerSchemaAllows(ctx, inv, previous); !ok {
|
||||
notBack(why)
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := inv.RestoreModule(ctx, previous); err != nil {
|
||||
notBack(err.Error())
|
||||
return
|
||||
@@ -1053,53 +833,6 @@ func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module str
|
||||
sayRollback(ctx, open, module, g, "")
|
||||
}
|
||||
|
||||
// controllerSchemaAllows says the store's schema lets this build of the controller be put back: the
|
||||
// build recorded, when it served, a reach at or past the highest migration the store has applied. One
|
||||
// that never recorded a reach is not proved safe, and is refused as such (novox/hq issue 352). Why
|
||||
// says what is kept and why when it is not.
|
||||
func controllerSchemaAllows(ctx context.Context, inv *inventory.Inventory, previous inventory.Build) (string, bool) {
|
||||
applied, err := inv.SchemaApplied(ctx)
|
||||
if err != nil {
|
||||
return "what the store's schema reaches cannot be read, so whether the build before it can read it is not " +
|
||||
"known; the current build is kept: " + err.Error(), false
|
||||
}
|
||||
build := versionOfBuild(previous)
|
||||
if build == "" {
|
||||
return fmt.Sprintf("the build before it (%s) names no bundle to know it by, so whether it can read the store's "+
|
||||
"schema (migration %04d) is not known; the current build is kept, and a person decides", short(previous.Commit), applied), false
|
||||
}
|
||||
reach, known, err := inv.SchemaReachOf(ctx, build)
|
||||
if err != nil {
|
||||
return "what the build before it knows of the store's schema cannot be read; the current build is kept: " + err.Error(), false
|
||||
}
|
||||
if !known {
|
||||
return fmt.Sprintf("the build before it (%s, %s) never recorded how far it reads the store's schema — a "+
|
||||
"controller records that when it serves — so it is not proved to read migration %04d, which the store "+
|
||||
"has applied; a controller older than its store starts behind its own records and judges with what it "+
|
||||
"can read, so the current build is kept, and a person decides", short(previous.Commit), build, applied), false
|
||||
}
|
||||
if reach < applied {
|
||||
return fmt.Sprintf("the build before it (%s, %s) reads the store's schema up to migration %04d, and the store "+
|
||||
"is at %04d: a controller older than its store starts behind its own records and judges with what it "+
|
||||
"can read, so the current build is kept, and a person decides", short(previous.Commit), build, reach, applied), false
|
||||
}
|
||||
return "", true
|
||||
}
|
||||
|
||||
// versionOfBuild is the version a build's bundle is delivered as — its archive's digest, short, as the
|
||||
// catalogue names it (`${version}`) — read from the build's artifacts; empty when none is a bundle.
|
||||
func versionOfBuild(b inventory.Build) string {
|
||||
for _, a := range b.Made {
|
||||
if a.Kind != catalogue.ArtifactBundle && a.Kind != catalogue.ArtifactArchive {
|
||||
continue
|
||||
}
|
||||
if _, hex, found := strings.Cut(a.Reference, "sha256:"); found && len(hex) >= 12 {
|
||||
return hex[:12]
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// rollbacks is what a failed send puts back, sent together (novox/hq issue 281): a gate that judged one
|
||||
// send judges what it moved as one, and what it found wanting goes back in one send per machine — not
|
||||
// in a send for each module, which is the churn that failed the gate in the first place.
|
||||
@@ -1385,9 +1118,9 @@ func gateLine(g *inventory.PlanGate) string {
|
||||
if g.Why != "" {
|
||||
line += ": " + g.Why
|
||||
} else if g.Last != "" {
|
||||
line += fmt.Sprintf(" (%d of %d passes; wanting: %s)", g.Passes, gateWanted(g), g.Last)
|
||||
line += fmt.Sprintf(" (%d of %d passes; wanting: %s)", g.Passes, gatePasses, g.Last)
|
||||
} else if g.Verdict == "" {
|
||||
line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gateWanted(g))
|
||||
line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gatePasses)
|
||||
}
|
||||
if g.Verdict == "" && len(g.Waits) > 0 {
|
||||
line += waitsSaid(g.Waits)
|
||||
@@ -1398,10 +1131,6 @@ func gateLine(g *inventory.PlanGate) string {
|
||||
if g.Rollback != "" {
|
||||
line += "; " + g.Rollback
|
||||
}
|
||||
// Before the send it judges (novox/hq issue 490).
|
||||
if said := grantsSaid(g.Grants); said != "" {
|
||||
line += "; " + said
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
@@ -1430,11 +1159,3 @@ func witnessObservation(node string, r lease.Rollback) conditions.Observation {
|
||||
Summary: fmt.Sprintf("the witness on %s judged the %s %s not healthy %s at %s: %s", node, r.Component,
|
||||
short(r.From), what, r.At.UTC().Format(time.RFC3339), r.Why)}
|
||||
}
|
||||
|
||||
// gateWanted is the passes a gate's send needs, three where the gate was judged before it was said.
|
||||
func gateWanted(g *inventory.PlanGate) int {
|
||||
if g.Wants > 0 {
|
||||
return g.Wants
|
||||
}
|
||||
return gatePasses
|
||||
}
|
||||
|
||||
@@ -53,7 +53,6 @@ func healthyContainer(module string) inventory.ResourceHealth {
|
||||
// tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass,
|
||||
// counted over the gate's spacing — before it says its own. The broken one alone is put back.
|
||||
func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
withGateBounds(t, 150*time.Millisecond, 20*time.Millisecond, 10*time.Second)
|
||||
@@ -285,7 +284,6 @@ func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) {
|
||||
// tools are marked and put back in the very judging that found them broken — their registered build is the
|
||||
// one before, and the laptop is sent it — while `app` goes on being judged, recovers, and keeps its own pass.
|
||||
func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
|
||||
@@ -113,10 +113,9 @@ func aGateMesh(t *testing.T) *gateMesh {
|
||||
}
|
||||
for node, h := range g.health {
|
||||
if h == healthNotYet {
|
||||
// Not reported on the send: neither the send made last, nor the gate's own (issue 352).
|
||||
f.rolledBack[node] = nil
|
||||
r := f.reports[node]
|
||||
r.Current, r.Declared, r.ReportedSequence = false, "", 0
|
||||
r.Current = false
|
||||
f.reports[node] = r
|
||||
}
|
||||
}
|
||||
@@ -153,7 +152,6 @@ func (g *gateMesh) plan(t *testing.T) inventory.Plan {
|
||||
// registered and sent to it again — and never reaches the second machine; it is marked, said as a
|
||||
// condition and an event, never registered or rolled back again.
|
||||
func TestABuildThatFailsItsGateIsRolledBackOnItsFirstMachineAndGoesNoFurther(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := g.open.inventory
|
||||
@@ -256,7 +254,7 @@ func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) {
|
||||
t.Fatalf("sent %v", g.sent)
|
||||
}
|
||||
gate := p.Modules["app"].Gate
|
||||
if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gateWanted(gate) {
|
||||
if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gatePasses {
|
||||
t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate)
|
||||
}
|
||||
if v, found, err := g.open.inventory.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
@@ -267,52 +265,8 @@ func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A leaf module passes its gate on one reading, as soon as its first machine is healthy, without waiting out the
|
||||
// core's settle time (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it, issue 524).
|
||||
func TestALeafModulePassesOnOneHealthyReading(t *testing.T) {
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
gateSettle = time.Hour // a core module could not pass for an hour
|
||||
advancePlans(ctx, g.open) // the first machine is sent the new build
|
||||
advancePlans(ctx, g.open) // one healthy judging
|
||||
p := g.plan(t)
|
||||
gate := p.Modules["app"].Gate
|
||||
if gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes != 1 || gate.Wants != 1 {
|
||||
t.Fatalf("a leaf module's gate after one healthy judging: %+v (plan %s: %s)", gate, p.State, p.Note)
|
||||
}
|
||||
if !strings.Contains(gate.Why, "healthy 1 times") {
|
||||
t.Errorf("the verdict says %q", gate.Why)
|
||||
}
|
||||
}
|
||||
|
||||
// Which modules are judged as the core: those on the controller's own path and the resolver's holder, three
|
||||
// readings after the settle time; every other module one (novox/hq ADR 0282 decision 1, ADR 0304 decision 3).
|
||||
func TestAModuleIsJudgedByItsClass(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
module string
|
||||
claims []catalogue.Claim
|
||||
passes int
|
||||
settles bool
|
||||
}{
|
||||
{"app", nil, 1, false},
|
||||
{"nzbget", nil, 1, false},
|
||||
{"mesh-controller", nil, 3, true},
|
||||
{"mesh-host", nil, 3, true},
|
||||
{"node-tools", nil, 3, true},
|
||||
{"nats", nil, 3, true},
|
||||
{catalogue.DeliverySeat, nil, 3, true},
|
||||
{"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, true},
|
||||
} {
|
||||
p, s := gateWants(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims})
|
||||
if p != c.passes || s != c.settles {
|
||||
t.Errorf("%s: %d passes, settles %v; want %d, %v", c.module, p, s, c.passes, c.settles)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A first machine that never becomes healthy fails its gate at the bound, and is put back.
|
||||
func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
advancePlans(ctx, g.open)
|
||||
@@ -638,13 +592,3 @@ func TestAWitnessesVerdictIsAConditionWhileItsReportsSayIt(t *testing.T) {
|
||||
t.Fatalf("a verdict the reports no longer carry is still said: %+v %v", obs, err)
|
||||
}
|
||||
}
|
||||
|
||||
// judgeLeavesAsTheCore makes a test's leaf modules judged as a core module is, three readings after the settle
|
||||
// time, for a test of the gate's other rules (a fault, the bound, a supersession) that needs a judging still
|
||||
// open after a healthy reading.
|
||||
func judgeLeavesAsTheCore(t *testing.T) {
|
||||
t.Helper()
|
||||
wasPasses, wasSettles := gateLeafPasses, gateLeafSettles
|
||||
gateLeafPasses, gateLeafSettles = gatePasses, true
|
||||
t.Cleanup(func() { gateLeafPasses, gateLeafSettles = wasPasses, wasSettles })
|
||||
}
|
||||
|
||||
@@ -1,300 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The assignment generation a declaration was composed from, and the record of every send (novox/hq
|
||||
// issue 234).
|
||||
//
|
||||
// On 2026-10-04 a declaration newer in sequence than every other named four fewer modules than the
|
||||
// assignments held, and a machine applied it and undeclared all four. The sequence orders arrival and
|
||||
// cannot tell a later send that carries an older view of the assignments. So a declaration now carries
|
||||
// the generation of the assignments it was composed from — a counter the store raises in the same
|
||||
// transaction as every assignment change — and a machine refuses one older than it applied, unless it is
|
||||
// a gate's put-back. And every send is written down with who sent it, from which generation and naming
|
||||
// which modules: the controller logged no send then, and which process sent the stale declaration could
|
||||
// not be read back from anything the mesh kept.
|
||||
|
||||
// kindOlderGeneration is S20's kind: a machine refused a send for the generation it was composed from.
|
||||
const kindOlderGeneration = "older-generation"
|
||||
|
||||
// generationRefusalsSaid is how long a refused send is said after its refusal: an hour, as an advisory is.
|
||||
const generationRefusalsSaid = advisoryQuiet
|
||||
|
||||
// stamp gives a composed declaration the order allotted to it before it was composed: its sequence, and
|
||||
// the epoch and generation when the machine reads them — and keeps the generation and acting epoch for the
|
||||
// record of the send whether or not the machine is sent them.
|
||||
//
|
||||
// **No put-back mark is sent.** A gate puts a machine back by composing it again (sendRollout), so its
|
||||
// declaration is allotted here like any other and carries the generation as it stands — never older than
|
||||
// what the machine applied. The node-engine reads a `put_back` key (mesh-host#82); this controller never
|
||||
// needs to send one.
|
||||
func (o order) stamp(d *sendable) {
|
||||
d.Sequence, d.Epoch = o.sequence, o.epoch
|
||||
d.composedFrom, d.actingEpoch = o.generation, o.acting
|
||||
d.Generation = 0
|
||||
if o.readsGeneration && o.generation > 0 {
|
||||
d.Generation = o.generation
|
||||
}
|
||||
}
|
||||
|
||||
// sentRecord is what the record of a send keeps beside its digest.
|
||||
type sentRecord struct {
|
||||
sequence int64
|
||||
epoch uint64
|
||||
generation int64
|
||||
// toldGeneration is the generation on the wire, which the would-send is stamped with: zero for a machine
|
||||
// whose node-engine has not said it reads one.
|
||||
toldGeneration int64
|
||||
sender string
|
||||
modules []string
|
||||
}
|
||||
|
||||
// sentRecordOf is a composed send's record: its sender is the caller this process acts for.
|
||||
func sentRecordOf(ctx context.Context, d sendable) sentRecord {
|
||||
return sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom,
|
||||
toldGeneration: d.Generation, sender: senderOf(callerOf(ctx)), modules: d.modules}
|
||||
}
|
||||
|
||||
// callerOf is who asked for what this process does: the seat call's caller when ctx belongs to one, the
|
||||
// caller the controller ran this command for, or the account at the shell.
|
||||
func callerOf(ctx context.Context) string {
|
||||
if c := link.CallerIn(ctx); c != "" {
|
||||
return c
|
||||
}
|
||||
return link.Caller()
|
||||
}
|
||||
|
||||
// senderOf is a send's sender in words: who asked, and the process and build that composed it — the
|
||||
// answer issue 234 could not find, because two controllers and a one-shot push were all sending then.
|
||||
func senderOf(caller string) string {
|
||||
host, _ := os.Hostname()
|
||||
return fmt.Sprintf("%s (pid %d on %s, build %s)", caller, os.Getpid(), host, version)
|
||||
}
|
||||
|
||||
// namedModules are the modules a declaration names: its machine's set, less what was left out of it.
|
||||
func namedModules(plan catalogue.Resolution, leftOut map[string]string) []string {
|
||||
out := make([]string, 0, len(plan.Modules))
|
||||
for _, m := range plan.Modules {
|
||||
if _, left := leftOut[m.Module]; !left {
|
||||
out = append(out, m.Module)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// heardGenerationRefusal keeps a machine's refusal of a send for its generation, so S20 names its sender
|
||||
// and says it (novox/hq issue 234): on the send it refused or — when the mesh has no record of sending that
|
||||
// sequence — as a refusal of its own, naming the sender as unknown. Either way S20 is raised: a refusal the
|
||||
// mesh cannot attribute is the louder fact, not a quieter one.
|
||||
//
|
||||
// **And raises the mesh's counter past what the machine applied, when the refused send was composed from
|
||||
// the counter as it stands** (counterBehind): then the sender's view was not stale — the counter is behind
|
||||
// the machine, which is a store put back from a backup — and every send after would be refused for ever. A
|
||||
// stale sender's generation is below the counter, and the counter is left alone for it.
|
||||
//
|
||||
// Nothing is sent from here. This runs in the controller's receive loop, and a push from it would hold that
|
||||
// loop, and the machine's hold, for as long as the push takes — the deaf controller of issues 184 and 185.
|
||||
// S20 names `push <node>` for that case instead.
|
||||
func heardGenerationRefusal(ctx context.Context, inv *inventory.Inventory, report link.Report) {
|
||||
r := report.OlderGeneration
|
||||
if r == nil || inv == nil || report.Node == "" {
|
||||
return
|
||||
}
|
||||
node, err := inv.NodeByName(ctx, report.Node)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the machine cannot be "+
|
||||
"read, so it is not raised: %v\n", report.Node, err)
|
||||
return
|
||||
}
|
||||
var raised int64
|
||||
if now, err := inv.AssignmentGeneration(ctx); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the mesh's own cannot be "+
|
||||
"read: %v\n", report.Node, err)
|
||||
} else if counterBehind(*r, now) {
|
||||
if raised, err = inv.RaiseAssignmentGeneration(ctx, r.Applied); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation (%d) is behind what %s applied (%d), "+
|
||||
"and could not be raised: %v\n", now, report.Node, r.Applied, err)
|
||||
raised = 0
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation was %d, behind what %s applied (%d) — "+
|
||||
"a store put back from a backup — and is raised to %d; `push %s` sends it what the mesh holds now\n",
|
||||
now, report.Node, r.Applied, raised, report.Node)
|
||||
}
|
||||
}
|
||||
send, found, err := inv.RefusedSend(ctx, node.ID, report.Sequence, r.Applied, raised)
|
||||
if err == nil && !found {
|
||||
send, err = inv.RecordUnrecordedRefusal(ctx, inventory.Send{Node: node.ID, Sequence: report.Sequence,
|
||||
Epoch: report.Epoch, Generation: r.Generation, Digest: report.Declared, RefusedApplied: r.Applied,
|
||||
CounterRaisedTo: raised})
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d for its generation, and the refusal could not be "+
|
||||
"kept, so it is not raised: %v\n", report.Node, report.Sequence, err)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d from %s: composed from assignment generation %d, "+
|
||||
"and it applied %d\n", report.Node, report.Sequence, send.Sender, r.Generation, r.Applied)
|
||||
}
|
||||
|
||||
// counterBehind says a refusal shows the mesh's counter behind the machine rather than a stale sender: the
|
||||
// refused send carried the counter as it stands now, and the machine applied more than that.
|
||||
func counterBehind(r link.GenerationRefusal, now int64) bool {
|
||||
return r.Generation >= now && r.Applied > r.Generation
|
||||
}
|
||||
|
||||
// watchGenerationRefusals is S20: every send a machine refused within the hour for the generation it was
|
||||
// composed from, naming who sent it (novox/hq issue 234). One per machine, the newest refusal.
|
||||
func watchGenerationRefusals(f *signalFacts) []conditions.Observation {
|
||||
seen := map[string]bool{}
|
||||
var out []conditions.Observation
|
||||
for _, s := range f.refusedSends {
|
||||
if s.RefusedAt == nil || f.now.Sub(*s.RefusedAt) > generationRefusalsSaid || seen[s.NodeName] {
|
||||
continue
|
||||
}
|
||||
seen[s.NodeName] = true
|
||||
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: s.NodeName, Kind: kindOlderGeneration,
|
||||
Machine: s.NodeName, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, and "+
|
||||
"%s applied generation %d — a sender composing from a view of the assignments the mesh has moved "+
|
||||
"past; it named %s", s.NodeName, s.Sequence, s.Sender, s.Generation, s.NodeName, s.RefusedApplied,
|
||||
modulesWords(s.Modules)),
|
||||
Said: fmt.Sprintf("refused at %s", s.RefusedAt.UTC().Format(time.RFC3339)),
|
||||
Headline: fmt.Sprintf("%s refused an out-of-date update", s.NodeName),
|
||||
Explanation: fmt.Sprintf("Something sent %s an update made from an older list of what runs there. %s "+
|
||||
"refused it, so nothing was removed. Nothing for you to do unless it repeats.", s.NodeName, s.NodeName),
|
||||
Resolved: fmt.Sprintf("%s has had no out-of-date update for an hour", s.NodeName)}
|
||||
if s.CounterRaisedTo > 0 {
|
||||
// Not a stale sender: the mesh's counter was behind the machine (a store put back from a backup) and
|
||||
// was raised; nothing has sent the machine its declaration since, so a person is asked to.
|
||||
o.Summary = fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, "+
|
||||
"the mesh's own, and %s applied generation %d — the mesh's counter was behind the machine (a store "+
|
||||
"put back from a backup?) and is raised to %d; `push %s` sends it what the mesh holds now",
|
||||
s.NodeName, s.Sequence, s.Sender, s.Generation, s.NodeName, s.RefusedApplied, s.CounterRaisedTo,
|
||||
s.NodeName)
|
||||
o.Explanation = fmt.Sprintf("Needs you: push %s. The mesh's record was older than %s, so %s refused its "+
|
||||
"update and kept what it had. The record is repaired; a push sends the update again.",
|
||||
s.NodeName, s.NodeName, s.NodeName)
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// modulesWords is a send's modules as a sentence says them.
|
||||
func modulesWords(modules []string) string {
|
||||
if len(modules) == 0 {
|
||||
return "no module the mesh recorded"
|
||||
}
|
||||
return strings.Join(modules, ", ")
|
||||
}
|
||||
|
||||
// writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234), and
|
||||
// what it answered (issue 485): nothing when the mesh has not recorded a send to it.
|
||||
func writeLastSend(ctx context.Context, w io.Writer, inv *inventory.Inventory, node string) error {
|
||||
s, found, err := inv.LastSend(ctx, node)
|
||||
if err != nil || !found {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(w, "%s:\n", node)
|
||||
for _, line := range lastSendLines(s) {
|
||||
fmt.Fprintln(w, line)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// lastSendLines is a machine's last send in full, as `node show` and `plan` say it (novox/hq issue 485): when,
|
||||
// by whom, under which lease epoch, from which assignment generation, naming which modules, and what the
|
||||
// machine answered of it — the refusal in its own words.
|
||||
func lastSendLines(s inventory.Send) []string {
|
||||
const stamp = "2006-01-02 15:04:05"
|
||||
epoch := "under no lease epoch"
|
||||
if s.Epoch > 0 {
|
||||
epoch = fmt.Sprintf("under lease epoch %d", s.Epoch)
|
||||
}
|
||||
generation := "no assignment generation recorded"
|
||||
if s.Generation > 0 {
|
||||
generation = fmt.Sprintf("assignment generation %d", s.Generation)
|
||||
}
|
||||
lines := []string{
|
||||
fmt.Sprintf(" last sent sequence %d at %s", s.Sequence, s.SentAt.Local().Format(stamp)),
|
||||
fmt.Sprintf(" by %s", s.Sender),
|
||||
fmt.Sprintf(" %s, composed from %s", epoch, generation),
|
||||
fmt.Sprintf(" naming %s", modulesWords(s.Modules)),
|
||||
}
|
||||
answer := func(words string) { lines = append(lines, " its answer "+words) }
|
||||
switch a := s.Answer; {
|
||||
case s.RefusedAt != nil:
|
||||
// Before the report: a refusal for the generation is kept on the send itself and is the machine's
|
||||
// answer to this send in particular.
|
||||
answer(fmt.Sprintf("refused it at %s: it had applied assignment generation %d, newer than this one",
|
||||
s.RefusedAt.Local().Format(stamp), s.RefusedApplied))
|
||||
case a.Outcome == "" || a.At == nil:
|
||||
answer("the machine has not reported on it yet")
|
||||
case a.Outcome == inventory.OutcomeApplied:
|
||||
answer("applied it at " + a.At.Local().Format(stamp))
|
||||
case a.Outcome == inventory.OutcomeRefused:
|
||||
answer("refused it at " + a.At.Local().Format(stamp) + ", in its words:")
|
||||
for _, line := range strings.Split(strings.TrimRight(a.Refused, "\n"), "\n") {
|
||||
lines = append(lines, " "+strings.TrimSpace(line))
|
||||
}
|
||||
default:
|
||||
answer(fmt.Sprintf("%s at %s: %d resource(s) failed — `status` says which", a.Outcome,
|
||||
a.At.Local().Format(stamp), a.Failed))
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
// sendAnswerWord is what a machine answered of a send, in a word or few, as status says it in one line.
|
||||
func sendAnswerWord(s inventory.Send) string {
|
||||
switch a := s.Answer; {
|
||||
case s.RefusedAt != nil:
|
||||
return fmt.Sprintf("refused: it had applied generation %d", s.RefusedApplied)
|
||||
case a.Outcome == "" || a.At == nil:
|
||||
return "not reported on yet"
|
||||
case a.Outcome == inventory.OutcomeFailed:
|
||||
return fmt.Sprintf("failed (%d resource(s))", a.Failed)
|
||||
default:
|
||||
return a.Outcome
|
||||
}
|
||||
}
|
||||
|
||||
// printLastSends says, one line per machine, when it was last sent a declaration, by whom, from which assignment
|
||||
// generation and what it answered (novox/hq issue 485). `node show <node>` says the send in full.
|
||||
func printLastSends(nodes []inventory.Node, sends map[string]inventory.Send, unread string) {
|
||||
if unread != "" {
|
||||
fmt.Printf("what each machine was last sent could NOT be read: %s\n\n", unread)
|
||||
return
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
return
|
||||
}
|
||||
fmt.Println("last sent, per machine:")
|
||||
for _, n := range nodes {
|
||||
s, found := sends[n.Name]
|
||||
if !found {
|
||||
fmt.Printf(" %-12s no send recorded\n", n.Name)
|
||||
continue
|
||||
}
|
||||
generation := "no generation"
|
||||
if s.Generation > 0 {
|
||||
generation = fmt.Sprintf("generation %d", s.Generation)
|
||||
}
|
||||
fmt.Printf(" %-12s %s by %s, %s — %s\n", n.Name, s.SentAt.Local().Format("2006-01-02 15:04"), s.Sender,
|
||||
generation, sendAnswerWord(s))
|
||||
}
|
||||
fmt.Printf("\n `node show <node>` says a machine's last send in full\n\n")
|
||||
}
|
||||
@@ -1,148 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The assignment generation a declaration was composed from (novox/hq issue 234).
|
||||
|
||||
func bodyKeys(t *testing.T, s sendable) map[string]any {
|
||||
t.Helper()
|
||||
raw, err := s.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keys map[string]any
|
||||
if err := json.Unmarshal(raw, &keys); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
// **The generation goes on the wire only to a machine whose node-engine said it reads one**: an older
|
||||
// node-engine decodes strictly and refuses an unknown key, whole. No put-back mark is ever sent: a gate
|
||||
// composes its put-back afresh, with the generation as it stands.
|
||||
func TestTheGenerationIsSentOnlyToAMachineThatReadsOne(t *testing.T) {
|
||||
resources := []map[string]any{{"id": "a", "type": "file", "path": "/etc/a", "content": "x\n"}}
|
||||
reads := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: true, acting: 57}
|
||||
var told sendable
|
||||
told.Resources = resources
|
||||
reads.stamp(&told)
|
||||
keys := bodyKeys(t, told)
|
||||
if keys["generation"] != float64(40) || keys["sequence"] != float64(12) {
|
||||
t.Fatalf("a machine that reads a generation was sent %v", keys)
|
||||
}
|
||||
if _, there := keys["put_back"]; there {
|
||||
t.Fatalf("a put-back mark was sent: %v", keys)
|
||||
}
|
||||
if told.composedFrom != 40 || told.actingEpoch != 57 {
|
||||
t.Errorf("the send does not keep what it was composed from for its record: %+v", told)
|
||||
}
|
||||
|
||||
older := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: false, acting: 57}
|
||||
var untold sendable
|
||||
untold.Resources = resources
|
||||
older.stamp(&untold)
|
||||
if _, there := bodyKeys(t, untold)["generation"]; there {
|
||||
t.Fatal("a machine whose node-engine never said it reads a generation was sent one")
|
||||
}
|
||||
if untold.composedFrom != 40 {
|
||||
t.Errorf("the generation it was composed from is recorded whether or not it was sent: %+v", untold)
|
||||
}
|
||||
}
|
||||
|
||||
// **The sender is named**: the caller of the seat call when there is one, else the shell's account, and the
|
||||
// process and build that composed it.
|
||||
func TestASendNamesItsSender(t *testing.T) {
|
||||
said := senderOf("g14.node-tools, through the mesh-controller seat")
|
||||
if !strings.Contains(said, "g14.node-tools") || !strings.Contains(said, "pid ") || !strings.Contains(said, "build ") {
|
||||
t.Fatalf("the sender reads %q", said)
|
||||
}
|
||||
}
|
||||
|
||||
// refusedSend is a send a machine refused for its generation at a moment.
|
||||
func refusedSend(at time.Time) inventory.Send {
|
||||
return inventory.Send{NodeName: "anchor", Sequence: 12, Epoch: 57, Generation: 38, RefusedApplied: 40,
|
||||
Sender: "a one-shot push by jochen at a shell on anchor (pid 4242 on anchor, build 2026.10.11)",
|
||||
Modules: []string{"docker"}, SentAt: at.Add(-time.Second), RefusedAt: &at, Recorded: true}
|
||||
}
|
||||
|
||||
// **A refused send is raised naming its sender** (novox/hq issue 234): who sent it, from which generation,
|
||||
// against which the machine applied, and its sequence — the facts that took a morning to look for.
|
||||
func TestARefusedSendIsRaisedNamingItsSender(t *testing.T) {
|
||||
now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC)
|
||||
f := calm(now)
|
||||
f.refusedSends = []inventory.Send{refusedSend(now.Add(-time.Minute))}
|
||||
got := watchGenerationRefusals(f)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
o := got[0]
|
||||
if o.Key() != "machine.anchor.older-generation" || o.Machine != "anchor" {
|
||||
t.Errorf("raised as %s about %q", o.Key(), o.Machine)
|
||||
}
|
||||
for _, want := range []string{"a one-shot push by jochen", "generation 38", "generation 40", "sequence 12"} {
|
||||
if !strings.Contains(o.Summary, want) {
|
||||
t.Errorf("the summary does not say %q: %s", want, o.Summary)
|
||||
}
|
||||
}
|
||||
if o.Headline == "" || o.Explanation == "" || o.Resolved == "" {
|
||||
t.Errorf("the condition is not worded for the operator: %+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
// **A refusal of a send the mesh has no record of is raised too, its sender said to be unknown** — never
|
||||
// only a line on stderr.
|
||||
func TestARefusalOfAnUnrecordedSendIsRaisedSayingTheSenderIsUnknown(t *testing.T) {
|
||||
now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC)
|
||||
f := calm(now)
|
||||
s := refusedSend(now.Add(-time.Minute))
|
||||
s.Recorded, s.Sender, s.Modules = false, "a sender the mesh has no record of (no send of this sequence was recorded)", nil
|
||||
f.refusedSends = []inventory.Send{s}
|
||||
got := watchGenerationRefusals(f)
|
||||
if len(got) != 1 || !strings.Contains(got[0].Summary, "no record of") {
|
||||
t.Fatalf("an unattributed refusal was not raised as one: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **When the refusal showed the mesh's counter behind the machine, the condition asks for a push** — it was
|
||||
// raised, and nothing has sent the machine its declaration since — and does not say there is nothing to do.
|
||||
func TestACounterBehindTheMachineAsksForAPush(t *testing.T) {
|
||||
now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC)
|
||||
f := calm(now)
|
||||
s := refusedSend(now.Add(-time.Minute))
|
||||
s.Generation, s.RefusedApplied, s.CounterRaisedTo = 12, 40, 41
|
||||
f.refusedSends = []inventory.Send{s}
|
||||
got := watchGenerationRefusals(f)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
if !strings.Contains(got[0].Summary, "`push anchor`") || !strings.Contains(got[0].Explanation, "push anchor") ||
|
||||
strings.Contains(got[0].Explanation, "Nothing for you to do") {
|
||||
t.Errorf("a counter behind the machine does not ask for a push: %s / %s", got[0].Summary, got[0].Explanation)
|
||||
}
|
||||
}
|
||||
|
||||
// **Only a refusal of the counter as it stands is the counter behind**: a stale sender's generation is below
|
||||
// it, and the counter is left alone for that.
|
||||
func TestOnlyARefusalOfTheCounterAsItStandsRaisesIt(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
refused, applied, now int64
|
||||
behind bool
|
||||
}{
|
||||
{refused: 12, applied: 40, now: 12, behind: true}, // the store was put back: the mesh says 12, the machine had 40
|
||||
{refused: 38, applied: 40, now: 41, behind: false}, // a stale sender: the counter is already past
|
||||
{refused: 38, applied: 40, now: 40, behind: false}, // a stale sender: the counter is where the machine is
|
||||
} {
|
||||
r := link.GenerationRefusal{Generation: c.refused, Applied: c.applied}
|
||||
if got := counterBehind(r, c.now); got != c.behind {
|
||||
t.Errorf("refused %d, applied %d, counter %d: behind %v, want %v", c.refused, c.applied, c.now, got, c.behind)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,198 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/identity"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The grants step on a store (novox/hq issue 490): the machine holding the bus is sent its new user list,
|
||||
// and nothing else — not the new build of a module it runs, not a module newly assigned there.
|
||||
//
|
||||
// The restic shape: anchor holds the bus and runs restic at c1, as does laptop; restic's c2 gives it a
|
||||
// health tool. The grants step's send to anchor composes restic at c1, carries the user list that grants
|
||||
// laptop's node-engine c2's tool, and records that anchor still runs c1. A module newly assigned to anchor
|
||||
// makes the step fail, said, rather than install it unjudged.
|
||||
func TestTheGrantsStepSendsTheUserListAndNothingElse(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
start := time.Now().Add(-time.Hour)
|
||||
build := func(module, commit string, at time.Time, manifest map[string]any) link.BuildResult {
|
||||
manifest["module"], manifest["version"] = module, "1"
|
||||
raw, _ := json.Marshal(manifest)
|
||||
return link.BuildResult{ID: link.NewBuildID(at), Repository: "novox/mesh-catalog", Path: "modules/" + module,
|
||||
On: "anchor", Module: module, Commit: commit, Manifest: raw,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
||||
}
|
||||
for _, b := range []link.BuildResult{
|
||||
build("nats", "n1", start, natsFixture()),
|
||||
build("restic", "c1", start.Add(time.Second), resticFixture(false)),
|
||||
build("wallpaper", "w1", start.Add(2*time.Second), wallpaperFixture()),
|
||||
} {
|
||||
if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, a := range [][2]string{{"anchor", "nats"}, {"anchor", "restic"}, {"laptop", "restic"}} {
|
||||
if _, err := inv.Assign(ctx, a[0], a[1]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: "node." + n, Kind: inventory.BusNode, Node: n}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
wasEpoch := epochForActs
|
||||
epochForActs = func(context.Context) (uint64, error) { return 7, nil }
|
||||
t.Cleanup(func() { epochForActs = wasEpoch })
|
||||
|
||||
// A send to anchor composed as sendToEach composes it: numbered, planned, declared, stamped.
|
||||
compose := func(under context.Context) (catalogue.Resolution, sendable) {
|
||||
t.Helper()
|
||||
numbered, err := allot(under, inv, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, settings, err := planFor(under, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(under, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationWith(under, open, "anchor", plan, settings, gens, Allocating)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
numbered.stamp(&declared)
|
||||
return plan, declared
|
||||
}
|
||||
record := func(declared sendable) {
|
||||
t.Helper()
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := recordSent(ctx, inv, "anchor", body, declared.Builds, declared.Epoch,
|
||||
sentRecordOf(ctx, declared)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// Kept for anchor alone. laptop is never recorded as sent, on purpose: composing anchor resolves laptop
|
||||
// too (who is on the private network), and a step that kept every machine's builds refused anchor's
|
||||
// composition for want of laptop's last send (repo-check of #230 at 0f853e93).
|
||||
kept := keepingEveryBuild(keepingRecorded(ctx), "anchor")
|
||||
|
||||
// What anchor was last sent: everything at the builds of before the merge, as an ordinary send sends it.
|
||||
_, before := compose(keepingRecorded(ctx))
|
||||
record(before)
|
||||
plan, declared := compose(kept)
|
||||
if only, err := grantsOnlyIn(ctx, open, "anchor", plan, declared); err != nil || only != "" {
|
||||
t.Fatalf("with nothing changed, the step reads %q, %v", only, err)
|
||||
}
|
||||
|
||||
// The merge: restic's c2 gives it a health tool.
|
||||
if _, _, err := takeIn(ctx, inv, build("restic", "c2", start.Add(time.Minute),
|
||||
resticFixture(true))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The step's send to anchor: restic as anchor runs it, c1, and the user list granting c2's tool.
|
||||
generation, err := inv.AssignmentGeneration(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, declared = compose(kept)
|
||||
for _, m := range plan.Modules {
|
||||
if m.Module == "restic" && len(m.Tools) > 0 {
|
||||
t.Fatalf("the grants step composed restic's new build on anchor: tools %v", m.Tools)
|
||||
}
|
||||
}
|
||||
if declared.Builds["restic"] != "c1" {
|
||||
t.Fatalf("the step's send records it carries restic %q; want c1, which anchor runs", declared.Builds["restic"])
|
||||
}
|
||||
if !strings.Contains(declared.BusUsers, "mesh.mod.restic.tool.backup_health.laptop") {
|
||||
t.Errorf("the step's declaration does not grant laptop's node-engine restic's new tool:\n%s", declared.BusUsers)
|
||||
}
|
||||
// The generation it carries is the current one (novox/hq issue 234), and the step does not raise it.
|
||||
if declared.composedFrom != generation {
|
||||
t.Errorf("the step's send was composed from generation %d; the mesh is at %d", declared.composedFrom, generation)
|
||||
}
|
||||
// The guard, on the very declaration the send sends: only the user list differs from the last send.
|
||||
if only, err := grantsOnlyIn(ctx, open, "anchor", plan, declared); err != nil || only != "" {
|
||||
t.Fatalf("the step reads as changing more than the user list: %q, %v", only, err)
|
||||
}
|
||||
// And the gate's refusal still reads the step's send: it moves nothing there.
|
||||
if names, err := ungatedIn(kept, open, []string{"anchor"}, ""); err != nil || strings.Join(names, ",") != "anchor" {
|
||||
t.Fatalf("the step's send is refused as a move: %v, %v", names, err)
|
||||
}
|
||||
// Recorded as the send records it: anchor still runs restic c1, its gate still to come there.
|
||||
record(declared)
|
||||
if sent, _, err := inv.SentBuilds(ctx, "anchor"); err != nil || sent["restic"] != "c1" {
|
||||
t.Fatalf("after the step anchor reads as sent restic %q (%v); want c1", sent["restic"], err)
|
||||
}
|
||||
if after, err := inv.AssignmentGeneration(ctx); err != nil || after != generation {
|
||||
t.Fatalf("the step moved the assignment generation from %d to %d (%v)", generation, after, err)
|
||||
}
|
||||
|
||||
// A module newly assigned to anchor: the step would install it, unjudged, so it is refused unsent.
|
||||
if _, err := inv.Assign(ctx, "anchor", "wallpaper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, declared = compose(kept)
|
||||
only, err := grantsOnlyIn(ctx, open, "anchor", plan, declared)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(only, "wallpaper newly assigned") || !strings.Contains(only, "+wallpaper") {
|
||||
t.Fatalf("a new assignment on the bus's machine reads %q; want the step refused, naming it", only)
|
||||
}
|
||||
// And the send itself refuses it, unsent: judged where it is composed, before the bus is dialled.
|
||||
identity.ForTest(t)
|
||||
if _, err := sendToEach(kept, open, []string{"anchor"}); !errors.Is(err, errNotGrantsOnly) ||
|
||||
!strings.Contains(err.Error(), "wallpaper") {
|
||||
t.Fatalf("the grants step's send of a new assignment was not refused by the send: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// fixtureImage is the image every fixture container of the grants step's tests runs.
|
||||
var fixtureImage = "registry.invalid:5000/restic/backup@sha256:" + strings.Repeat("b", 64)
|
||||
|
||||
// resticFixture is restic's manifest in the shape of mesh-catalog #210: at c2 (withTool) its backup judged by
|
||||
// its new tool `backup_health`, beside a check of another kind it does not run itself — a tool check alone
|
||||
// is refused (ADR 0227 rule 8, ADR 0240 rule 2) — and at c1 neither.
|
||||
func resticFixture(withTool bool) map[string]any {
|
||||
backup := map[string]any{"id": "backup", "type": "container", "name": "restic-backup", "image": fixtureImage}
|
||||
measure := map[string]any{"id": "measure", "type": "container", "name": "restic-measure", "image": fixtureImage}
|
||||
m := map[string]any{"resources": []any{backup, measure}}
|
||||
if withTool {
|
||||
backup["health"] = map[string]any{"kind": catalogue.HealthTool, "tool": "backup_health"}
|
||||
measure["health"] = map[string]any{"kind": "runtime"}
|
||||
m["tools"] = []string{"backup_health"}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// natsFixture is the bus's module: it holds mesh-broker and is sent the user list.
|
||||
func natsFixture() map[string]any {
|
||||
return map[string]any{"bus-users": "/var/lib/nats-module/conf/accounts.conf",
|
||||
"claims": []any{map[string]any{"name": catalogue.BrokerSeat, "scope": catalogue.ScopeMesh}}}
|
||||
}
|
||||
|
||||
// wallpaperFixture is a module the bus's machine is newly assigned.
|
||||
func wallpaperFixture() map[string]any {
|
||||
return map[string]any{"resources": []any{
|
||||
map[string]any{"id": "paper", "type": "container", "name": "wallpaper", "image": fixtureImage}}}
|
||||
}
|
||||
@@ -1,226 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The grants reach the bus before the gate (novox/hq issue 490).
|
||||
//
|
||||
// On 2026-10-11 a merge gave restic, assigned to every machine, a new health tool. Its walk sent restic to
|
||||
// its first machine and judged it there; that machine's node-engine asked the new tool and the bus refused
|
||||
// it, because the grant was in the bus's user list, which only the machine holding the bus carries — and
|
||||
// that machine runs restic too, so it was left out of the send while restic's new build waited there for
|
||||
// the very gate that could not pass. A person pushed it by hand, carrying restic's new build there unjudged.
|
||||
|
||||
// aSend is one send the walk made: to which machines, and whether every build there was kept.
|
||||
type aSend struct {
|
||||
names []string
|
||||
keepsAll bool
|
||||
judged []string
|
||||
}
|
||||
|
||||
// sendsRecorded replaces the walk's send and the reading of the bus's user list for one test: the machine
|
||||
// holding the bus is novox, and its list is behind as behind says.
|
||||
func sendsRecorded(t *testing.T, holder string, behind bool, refuse error) *[]aSend {
|
||||
t.Helper()
|
||||
var sends []aSend
|
||||
wasSend, wasBehind := sendRollout, brokerBehindOf
|
||||
t.Cleanup(func() { sendRollout, brokerBehindOf = wasSend, wasBehind })
|
||||
brokerBehindOf = func(_ context.Context, _ *stores, names []string) (string, bool, error) {
|
||||
if slices.Contains(names, holder) {
|
||||
return holder, false, nil
|
||||
}
|
||||
return holder, behind, nil
|
||||
}
|
||||
sendRollout = func(ctx context.Context, _ *stores, names []string) ([]string, error) {
|
||||
s := aSend{names: append([]string(nil), names...), keepsAll: len(names) == 1 && everyBuildKept(ctx, names[0])}
|
||||
for n := range scopeOf(ctx).judged {
|
||||
s.judged = append(s.judged, n)
|
||||
}
|
||||
sends = append(sends, s)
|
||||
if refuse != nil && s.keepsAll {
|
||||
return nil, refuse
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
return &sends
|
||||
}
|
||||
|
||||
// The restic shape: a new tool on a module on every machine, its first machine ace, the bus on novox.
|
||||
func TestAWalkSendsTheGrantsToTheBusBeforeTheFirstMachineIsJudged(t *testing.T) {
|
||||
sends := sendsRecorded(t, "novox", true, nil)
|
||||
sent, grants, err := sendJudged(t.Context(), nil, "ace")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(*sends) != 2 {
|
||||
t.Fatalf("the walk made %d send(s), want the grants step and then the first machine: %+v", len(*sends), *sends)
|
||||
}
|
||||
first, then := (*sends)[0], (*sends)[1]
|
||||
if strings.Join(first.names, ",") != "novox" || !first.keepsAll || len(first.judged) != 0 {
|
||||
t.Errorf("the first send is %+v, want novox alone, every build there kept, judging nothing", first)
|
||||
}
|
||||
if strings.Join(then.names, ",") != "ace" || then.keepsAll || strings.Join(then.judged, ",") != "ace" {
|
||||
t.Errorf("the second send is %+v, want ace, judged", then)
|
||||
}
|
||||
if strings.Join(sent, ",") != "ace" {
|
||||
t.Errorf("the gate's machines are %v, want ace alone: the bus's machine is not judged", sent)
|
||||
}
|
||||
if grants == nil || grants.Node != "novox" || grants.At == nil || grants.Failed != "" {
|
||||
t.Fatalf("the gate keeps %+v as its grants step", grants)
|
||||
}
|
||||
line := gateLine(&inventory.PlanGate{Machines: sent, Grants: grants})
|
||||
if !strings.Contains(line, "grants step: novox sent the bus's user list first, its builds kept") {
|
||||
t.Errorf("plans says the gate as %q", line)
|
||||
}
|
||||
}
|
||||
|
||||
// Nothing more when there is nothing to carry: the list unchanged, or the first machine holds the bus.
|
||||
func TestAWalkTakesNoGrantsStepWhenTheListIsAlreadyThere(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
node string
|
||||
behind bool
|
||||
}{{"the list unchanged", "ace", false}, {"the first machine holds the bus", "novox", true}} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
sends := sendsRecorded(t, "novox", c.behind, nil)
|
||||
_, grants, err := sendJudged(t.Context(), nil, c.node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(*sends) != 1 || (*sends)[0].keepsAll || grants != nil {
|
||||
t.Errorf("the walk made %+v with grants %+v, want the judged send alone", *sends, grants)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A grants step that cannot be sent is said on the gate, and the walk goes on as it did before.
|
||||
func TestAGrantsStepThatFailsIsSaidAndTheSendGoesOn(t *testing.T) {
|
||||
sends := sendsRecorded(t, "novox", true, errors.New("a bus upgrade waits for a person"))
|
||||
sent, grants, err := sendJudged(t.Context(), nil, "ace")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(sent, ",") != "ace" || len(*sends) != 2 {
|
||||
t.Errorf("the walk made %+v", *sends)
|
||||
}
|
||||
if grants == nil || grants.Failed == "" || grants.At != nil {
|
||||
t.Fatalf("the gate keeps %+v", grants)
|
||||
}
|
||||
if line := gateLine(&inventory.PlanGate{Machines: sent, Grants: grants}); !strings.Contains(line,
|
||||
"grants step to novox FAILED, sent without it: a bus upgrade waits for a person") {
|
||||
t.Errorf("plans says the gate as %q", line)
|
||||
}
|
||||
}
|
||||
|
||||
// A step whose send would change more than the user list is refused unsent by the send (sendToEach): the
|
||||
// gate says it FAILED with what differed, and the walk goes on.
|
||||
func TestAGrantsStepThatWouldCarryMoreIsNotSent(t *testing.T) {
|
||||
refusal := fmt.Errorf("%w: its send would change more than the bus's user list: -postgres.login-n8n", errNotGrantsOnly)
|
||||
sends := sendsRecorded(t, "novox", true, refusal)
|
||||
sent, grants, err := sendJudged(t.Context(), nil, "ace")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(sent, ",") != "ace" || len(*sends) != 2 {
|
||||
t.Errorf("the walk made %+v, want the refused step and then ace's judged send", *sends)
|
||||
}
|
||||
if grants == nil || grants.At != nil || !strings.Contains(grants.Failed, "-postgres.login-n8n") {
|
||||
t.Fatalf("the gate keeps %+v", grants)
|
||||
}
|
||||
}
|
||||
|
||||
// The step's exemption is the list's content alone: its path or mode moving is a change like any other.
|
||||
func TestOnlyTheUserListsContentIsExempt(t *testing.T) {
|
||||
list := "nats.bus-users"
|
||||
file := func(fields map[string]any) sentResource {
|
||||
r := map[string]any{"id": list, "type": "file", "path": "/conf/accounts.conf", "mode": "0600", "content": "a"}
|
||||
for k, v := range fields {
|
||||
r[k] = v
|
||||
}
|
||||
body, _ := json.Marshal(map[string]any{"resources": []any{r}})
|
||||
s, err := summarize(body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return s[0]
|
||||
}
|
||||
was := file(nil)
|
||||
if other := otherThanTheList(diffSent([]sentResource{was}, []sentResource{file(map[string]any{"content": "b"})}), list); len(other) != 0 {
|
||||
t.Errorf("the list's new content reads as more: %v", other)
|
||||
}
|
||||
for _, f := range []map[string]any{{"mode": "0644"}, {"path": "/elsewhere"}, {"content": "b", "owner": "nats"}} {
|
||||
other := otherThanTheList(diffSent([]sentResource{was}, []sentResource{file(f)}), list)
|
||||
if len(other) != 1 || !strings.HasPrefix(other[0], "~"+list) {
|
||||
t.Errorf("the list's resource changed by %v reads %v; want it refused", f, other)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The step keeps the builds of the machine it sends alone: the others its composition resolves are composed
|
||||
// as any send composes them.
|
||||
func TestTheGrantsStepKeepsOneMachinesBuilds(t *testing.T) {
|
||||
ctx := keepingEveryBuild(t.Context(), "novox")
|
||||
if !everyBuildKept(ctx, "novox") || everyBuildKept(ctx, "ace") || everyBuildKept(t.Context(), "novox") {
|
||||
t.Error("the grants step keeps the builds of a machine it does not send")
|
||||
}
|
||||
}
|
||||
|
||||
// The store test's manifests are manifests the controller takes in: checked here, where no store is needed,
|
||||
// so a fixture the module rules refuse fails before the store test is run (repo-check of #230 at 325575b2).
|
||||
func TestTheGrantsStepFixturesAreManifests(t *testing.T) {
|
||||
for name, m := range map[string]map[string]any{"nats": natsFixture(), "restic-c1": resticFixture(false),
|
||||
"restic-c2": resticFixture(true), "wallpaper": wallpaperFixture()} {
|
||||
m["module"], m["version"] = strings.SplitN(name, "-", 2)[0], "1"
|
||||
raw, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := catalogue.ParseManifest(raw); err != nil {
|
||||
t.Errorf("%s: %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The delivery plan says the step before the merge.
|
||||
func TestAChangePlanSaysTheGrantsStep(t *testing.T) {
|
||||
const catalogue_ = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||
entry := func(name string, on ...string) inventory.Entry {
|
||||
e := fromRepo(name, catalogue_, "modules/"+name)
|
||||
e.Source.BuiltFrom = "old"
|
||||
e.On = on
|
||||
return e
|
||||
}
|
||||
nats := entry("nats", "novox")
|
||||
nats.Manifest.BusUsers = "/etc/nats/users.conf"
|
||||
nats.Manifest.Claims = []catalogue.Claim{{Name: catalogue.BrokerSeat, Scope: catalogue.ScopeMesh}}
|
||||
restic := entry("restic", "ace", "novox", "shanks")
|
||||
only := entry("bus-tools", "novox")
|
||||
entries := []inventory.Entry{nats, restic, only}
|
||||
rollsOut := func(string) (inventory.Upgrade, bool) { return inventory.Upgrade{RollOut: true}, true }
|
||||
plan := func(paths ...string) link.ChangePlan {
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: paths}
|
||||
return changePlanOf("novox/mesh-catalog", "main", "head", reachOfMerge(m, entries, nil, nil), entries, rollsOut)
|
||||
}
|
||||
|
||||
text := planText(plan("modules/restic/module.json"))
|
||||
if !strings.Contains(text, "grants step: if this changes what the bus's user list says (a new tool, subject or "+
|
||||
"user), novox is sent that list alone, every build there kept, before restic is judged on its first machine") {
|
||||
t.Errorf("the change plan does not say the grants step:\n%s", text)
|
||||
}
|
||||
// A module only on the bus's machine goes there with the list in its own send: no step of its own.
|
||||
if text := planText(plan("modules/bus-tools/module.json")); strings.Contains(text, "grants step") {
|
||||
t.Errorf("a module on the bus's machine alone reads:\n%s", text)
|
||||
}
|
||||
}
|
||||
@@ -1,158 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A delivery group still assembles** (novox/hq ADR 0306 decision 2): a merge whose delivery group has a member
|
||||
// not yet merged waits for the merge window, whether or not a batch is assembling, so the group's members walk in
|
||||
// one batch, in their order (ADR 0276 decision 5). Which groups exist and where each member stands is
|
||||
// mesh-delivery's record (ADR 0239): a group is the pull requests of two repositories or more sharing a head
|
||||
// branch name, which is its id. The controller asks it, through its `groups` verb, when a merge from a head is
|
||||
// heard; a member whose pull request is open (proposed, checked, ready or rejected) is still to merge. When
|
||||
// mesh-delivery does not answer within groupAskWithin, or answers something unreadable, the merge waits: the safe
|
||||
// error is to wait, and its batch says why.
|
||||
|
||||
// groupAskWithin is how long mesh-delivery is given to say a merge's group, while the merge is being heard.
|
||||
var groupAskWithin = 3 * time.Second
|
||||
|
||||
// openPullStates are the states of a delivery whose pull request is open and not merged (mesh-delivery's table):
|
||||
// a member in one of them is still to merge. Closed unmerged is stopped, final, and waits for nothing.
|
||||
var openPullStates = map[string]bool{"proposed": true, "checked": true, "ready": true, "rejected": true}
|
||||
|
||||
// groupMember is one member of a delivery group as mesh-delivery's `groups` says it: `<owner/repo>@<commit> (state)`.
|
||||
type groupMember struct {
|
||||
Repository, State string
|
||||
}
|
||||
|
||||
// deliveryGroupOf is what mesh-delivery says of the group a head names: its members, and whether the head names a
|
||||
// group at all; an error when it cannot be asked or did not answer. A seam a test replaces.
|
||||
var deliveryGroupOf = func(ctx context.Context, held bool, head string) ([]groupMember, bool, error) {
|
||||
if !held {
|
||||
return nil, false, fmt.Errorf("no holder of %s is on record to say it", catalogue.DeliverySeat)
|
||||
}
|
||||
serving := servingBus.Load()
|
||||
if serving == nil {
|
||||
return nil, false, errors.New("this controller is not on the bus")
|
||||
}
|
||||
asking, cancel := context.WithTimeout(ctx, groupAskWithin)
|
||||
defer cancel()
|
||||
raw, err := askDeliveryOwner(asking, serving.Conn(), "groups", map[string]any{})
|
||||
if err != nil {
|
||||
if errors.Is(err, link.ErrNothingServes) {
|
||||
return nil, false, fmt.Errorf("nothing serves %s.groups", catalogue.DeliverySeat)
|
||||
}
|
||||
return nil, false, err
|
||||
}
|
||||
var lines []struct {
|
||||
ID string `json:"id"`
|
||||
Members []string `json:"members"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &lines); err != nil {
|
||||
return nil, false, fmt.Errorf("%s.groups answered something unreadable: %w", catalogue.DeliverySeat, err)
|
||||
}
|
||||
for _, l := range lines {
|
||||
if l.ID != head {
|
||||
continue
|
||||
}
|
||||
var members []groupMember
|
||||
for _, s := range l.Members {
|
||||
members = append(members, memberOf(s))
|
||||
}
|
||||
return members, true, nil
|
||||
}
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
// memberOf reads one member as `groups` says it: `novox/mesh-sdk@d41d255f28ad (held)`.
|
||||
func memberOf(s string) groupMember {
|
||||
id, state := strings.TrimSpace(s), ""
|
||||
if i := strings.LastIndex(id, " ("); i >= 0 && strings.HasSuffix(id, ")") {
|
||||
id, state = id[:i], id[i+2:len(id)-1]
|
||||
}
|
||||
repository, _, _ := strings.Cut(id, "@")
|
||||
return groupMember{Repository: repository, State: state}
|
||||
}
|
||||
|
||||
// groupWait says whether a merge heard now waits for the window for its delivery group, and why in words; nothing
|
||||
// when it names no head, its head names no group, or every other member is merged or closed. own is the merge's
|
||||
// kind (ownPath).
|
||||
//
|
||||
// A member of another repository whose merge this controller already keeps in a batch not yet cut of the same
|
||||
// kind is merged, whatever mesh-delivery says yet (it marks a member merged once its walk opened): the merge joins
|
||||
// that batch by their head. One kept in a batch of the other kind keeps the merge waiting, whatever mesh-delivery
|
||||
// says (review of #250): kinds never share a batch, so the merge's batch is a new one, and cut at once it would
|
||||
// walk before that member's batch is cut and break the group's order (ADR 0276 decision 5). Waiting, the two
|
||||
// batches are cut in the order their windows close, an own-path one first at one look, as cutBatchesHeld cuts
|
||||
// every pair of kinds.
|
||||
//
|
||||
// Asked before the plans are held: a slow answer holds up no walk.
|
||||
func groupWait(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, held, own bool) (string, error) {
|
||||
if m.Head == "" {
|
||||
return "", nil
|
||||
}
|
||||
same, other, err := keptFromHead(ctx, inv, m.Head, own)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(other) > 0 {
|
||||
return "group " + m.Head + " has a member waiting in a batch of the other kind: " + strings.Join(other, ", "), nil
|
||||
}
|
||||
members, isGroup, err := deliveryGroupOf(ctx, held, m.Head)
|
||||
if err != nil {
|
||||
return fmt.Sprintf("%s did not say whether group %s has a member still to merge (%v): it waits for the "+
|
||||
"window, the safe side", catalogue.DeliverySeat, m.Head, err), nil
|
||||
}
|
||||
if !isGroup {
|
||||
return "", nil
|
||||
}
|
||||
var open []string
|
||||
for _, g := range members {
|
||||
if strings.EqualFold(g.Repository, m.Owner+"/"+m.Repo) || same[strings.ToLower(g.Repository)] ||
|
||||
!openPullStates[g.State] {
|
||||
continue
|
||||
}
|
||||
open = append(open, g.Repository+" ("+g.State+")")
|
||||
}
|
||||
if len(open) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
return "group " + m.Head + " has a member still to merge: " + strings.Join(open, ", "), nil
|
||||
}
|
||||
|
||||
// keptFromHead are the merges from a head that batches not yet cut hold: the repositories (lower case) of those in
|
||||
// a batch of the kind own says, and, in words, each of the other kind with its batch.
|
||||
func keptFromHead(ctx context.Context, inv *inventory.Inventory, head string, own bool) (map[string]bool, []string, error) {
|
||||
batches, err := inv.Batches(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
same := map[string]bool{}
|
||||
var other []string
|
||||
for _, b := range batches {
|
||||
merges, err := inv.MergesOf(ctx, b.ID)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
for _, k := range merges {
|
||||
if announcedOf(k).Head != head {
|
||||
continue
|
||||
}
|
||||
if b.OwnPath() == own {
|
||||
same[strings.ToLower(k.Repository)] = true
|
||||
} else {
|
||||
other = append(other, fmt.Sprintf("%s@%s in %s", k.Repository, short(k.Commit), b.ID))
|
||||
}
|
||||
}
|
||||
}
|
||||
return same, other, nil
|
||||
}
|
||||
@@ -60,18 +60,11 @@ var handActVerbs = []handActVerb{
|
||||
// a person's word (ADR 0242), which the push itself reads from what it carried (recorded_push.go).
|
||||
{Verb: "push", Decision: "a recorded build moves only by a person's push: that push is the word its " +
|
||||
"upgrade policy asks for (ADR 0242)", DecidedWhen: pushedRecorded},
|
||||
// Stopping or starting a walk the operator chose on a warrant (novox/hq ADR 0259) is their decision.
|
||||
{Verb: "plans stop", Decision: "the operator's answer to an ask is their decision, not a repair (ADR 0259)",
|
||||
DecidedFor: []string{conditions.CauseOperatorAnswer}},
|
||||
{Verb: "plans stop"},
|
||||
{Verb: "plans close"},
|
||||
// A walk started by a person instead of its delivery's owner (novox/hq ADR 0239): the owner down, or
|
||||
// not trusted with it — either is a repair the owner should have made. Unless the operator chose it on
|
||||
// a warrant (ADR 0259).
|
||||
{Verb: "plans go", Decision: "the operator's answer to an ask is their decision, not a repair (ADR 0259)",
|
||||
DecidedFor: []string{conditions.CauseOperatorAnswer}},
|
||||
// An act the operator chose on a warrant (novox/hq ADR 0259): asked by the controller, answered on a
|
||||
// channel that proved who answered, performed by the controller as itself.
|
||||
{Verb: handActWarrant, Decision: "the operator chose it, answering what the controller asked (ADR 0259)"},
|
||||
// not trusted with it — either is a repair the owner should have made.
|
||||
{Verb: "plans go"},
|
||||
{Verb: "broker consumer-reset"},
|
||||
// Silencing the same condition twice says the condition, or what it watches, wants mending — unless
|
||||
// it is the operator's answer on a notification: a decision to live with it (novox/hq ADR 0258).
|
||||
@@ -84,9 +77,6 @@ var handActVerbs = []handActVerb{
|
||||
"deliberate test, never a repair", DecidedFor: []string{causeDrill}},
|
||||
// A drill: something broken on purpose to see the mesh raise and clear it. A person's test, never a
|
||||
// repair, however often it is run.
|
||||
// A credential a provider's software issued, taken from the provider and kept for its consumers (novox/hq ADR
|
||||
// 0305): the mesh working as decided, however often the software changes it.
|
||||
{Verb: handActIssued, Decision: "a credential the provider's software issued, kept for its consumers (ADR 0305)"},
|
||||
{Verb: "hand-act drill", Decision: "a drill is a person's deliberate test of the mesh, never a repair"},
|
||||
// A person's decisions by design.
|
||||
{Verb: "retire approve", Decision: "nothing is retired past its bound without a person (ADR 0230)"},
|
||||
@@ -113,9 +103,6 @@ var handActVerbs = []handActVerb{
|
||||
// to that judgement. Several values rotate for one leak, and a leak that recurs is a defect of the
|
||||
// module that prints them, an issue against it, not a healer that rotates. A rotation for any other
|
||||
// cause — a credential that stopped working — counts: a schedule or a healer could take it over.
|
||||
// A value given at the desk (novox/hq ADR 0259 §10): an outside party's key, such as a bot token, which
|
||||
// only a person can give. Their word, never a repair.
|
||||
{Verb: "secret accept", Decision: "a value an outside party issued is given by a person, at their desk"},
|
||||
{Verb: "secret rotate", Decision: "a value a person judged disclosed is replaced on their word",
|
||||
DecidedFor: []string{causeLeakedInLogs}},
|
||||
}
|
||||
@@ -254,9 +241,6 @@ func handActCommand(ctx context.Context, args []string) error {
|
||||
if len(args) > 0 && args[0] == "drill" {
|
||||
return handActDrill(ctx, args[1:])
|
||||
}
|
||||
if len(args) > 0 && args[0] == "warrant" {
|
||||
return handActWarrantCommand(ctx, args[1:])
|
||||
}
|
||||
if len(args) > 0 && args[0] != "list" && !strings.HasPrefix(args[0], "-") {
|
||||
return errors.New("hand-act record <what> --why <text> --cause <word> | hand-act drill <what> --why <text> " +
|
||||
"| hand-acts [--days N] [--json]")
|
||||
|
||||
@@ -1,220 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A hand-over's line is judged before anything is asked (novox/hq issue 356): a node's name and the directory's
|
||||
// absolute path exactly as the engine states it — no `..`, no doubled or trailing separator, nothing relative.
|
||||
func TestAHandOverLineIsJudgedBeforeItIsAsked(t *testing.T) {
|
||||
for _, args := range [][]string{
|
||||
{},
|
||||
{"laptop"},
|
||||
{"laptop", "/srv/notes", "extra"},
|
||||
{"", "/srv/notes"},
|
||||
{"--node", "/srv/notes"},
|
||||
{"laptop", "srv/notes"},
|
||||
{"laptop", "/srv/../etc"},
|
||||
{"laptop", "/srv//notes"},
|
||||
{"laptop", "/srv/notes/"},
|
||||
{"laptop", "/srv/notes/."},
|
||||
} {
|
||||
if _, _, err := handOverLine(args); err == nil {
|
||||
t.Errorf("%q was taken", args)
|
||||
}
|
||||
}
|
||||
node, path, err := handOverLine([]string{"laptop", "/srv/notes"})
|
||||
if err != nil || node != "laptop" || path != "/srv/notes" {
|
||||
t.Fatalf("read as %q %q %v", node, path, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Who hands over is the line's caller in the words every verb's caller is recorded in — the operator through
|
||||
// mesh-cli — or the controller's terminal when nobody is named.
|
||||
func TestAHandOverNamesWhoAsked(t *testing.T) {
|
||||
t.Setenv(link.CallerVar, " jo through mesh-cli on anchor ")
|
||||
if by := handOverBy(); by != "jo through mesh-cli on anchor" {
|
||||
t.Fatalf("by %q", by)
|
||||
}
|
||||
t.Setenv(link.CallerVar, "")
|
||||
if by := handOverBy(); by != "the controller's terminal" {
|
||||
t.Fatalf("by %q", by)
|
||||
}
|
||||
}
|
||||
|
||||
// **A hand-over is the controller's terminal's alone** (novox/hq issue 356, ADR 0266): through any verb, and
|
||||
// through mesh-cli outside the terminal, `node hand-over` is refused and nothing runs — at the next apply root
|
||||
// gives the directory to the account the module declares, and whoever may call a verb includes agents.
|
||||
func TestAHandOverIsRefusedThroughEveryVerb(t *testing.T) {
|
||||
line := []string{"node", "hand-over", "laptop", "/srv/notes"}
|
||||
if err := terminalOnly(line); err == nil {
|
||||
t.Fatal("node hand-over passed as a verb's line")
|
||||
}
|
||||
if _, err := argvFor("command", map[string]any{"command": "node hand-over laptop /srv/notes"}); err == nil {
|
||||
t.Fatal("the command verb composed node hand-over")
|
||||
}
|
||||
if _, err := ordinaryLine(line); err == nil {
|
||||
t.Fatal("node hand-over composed as an ordinary mesh-cli line")
|
||||
}
|
||||
if _, err := argvFor("node", map[string]any{"node": "laptop", "hand-over": "/srv/notes"}); err == nil {
|
||||
t.Fatal("the node verb composed a hand-over")
|
||||
}
|
||||
}
|
||||
|
||||
// The condition's words are plain and name no machine's binary; the operator's line, with the node and the path
|
||||
// (novox/hq ADR 0272), is in the summary the module's health gives (moduleHealthWord) and in the evidence.
|
||||
func TestTheUsedAsFoundConditionNamesTheOperatorsLine(t *testing.T) {
|
||||
rs := []inventory.ResourceHealth{foundDirectory("notes", time.Now().Add(-24*time.Hour))}
|
||||
o := usedAsFoundObservation("notes", "laptop", "notes on laptop uses notes.data as found", rs)
|
||||
if strings.Contains(o.Needs, "mesh-host") || strings.Contains(o.Explanation, "mesh-host") ||
|
||||
!strings.Contains(o.Needs, "control-node") {
|
||||
t.Fatalf("the condition's words: %q %q", o.Needs, o.Explanation)
|
||||
}
|
||||
if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Needs: o.Needs,
|
||||
Resolved: o.Resolved}, "laptop"); !ok {
|
||||
t.Fatalf("not plain: %s", why)
|
||||
}
|
||||
f := gateFacts{now: time.Now(), health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: time.Now(),
|
||||
Resources: rs}}}
|
||||
h, why := moduleHealthWord("notes", "laptop", time.Now().Add(-time.Hour), f)
|
||||
if h != healthPerson || !strings.Contains(why, "`nox node hand-over laptop <directory>` on the control-node") ||
|
||||
strings.Contains(why, "mesh-host") || strings.Contains(why, "/srv/") {
|
||||
t.Fatalf("the module's health reads %v %q; want the operator's line", h, why)
|
||||
}
|
||||
}
|
||||
|
||||
// **Nothing is asked of a node the mesh does not know, and the engine's refusal is the command's failure**
|
||||
// (review of issue 356): a refused hand-over never exits as a success.
|
||||
func TestAHandOverAsksOnlyAKnownNodeAndFailsOnARefusal(t *testing.T) {
|
||||
t.Setenv(link.CallerVar, "jo through mesh-cli on anchor")
|
||||
asked := 0
|
||||
ask := func(answer link.HandOverAnswer) func(node, path, by string) (link.HandOverAnswer, error) {
|
||||
return func(node, path, by string) (link.HandOverAnswer, error) {
|
||||
asked++
|
||||
if node != "laptop" || path != "/srv/notes" || by != "jo through mesh-cli on anchor" {
|
||||
t.Fatalf("asked %q %q %q", node, path, by)
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
}
|
||||
unknown := func(string) error { return errors.New("no node called laptop") }
|
||||
known := func(string) error { return nil }
|
||||
var out bytes.Buffer
|
||||
err := handOverAsked([]string{"laptop", "/srv/notes"}, unknown, ask(link.HandOverAnswer{Said: "x"}), &out)
|
||||
if err == nil || asked != 0 || !strings.Contains(err.Error(), "nothing was asked") {
|
||||
t.Fatalf("an unknown node: %v, asked %d", err, asked)
|
||||
}
|
||||
err = handOverAsked([]string{"laptop", "/srv/../etc"}, known, ask(link.HandOverAnswer{Said: "x"}), &out)
|
||||
if err == nil || asked != 0 {
|
||||
t.Fatalf("a refused line was asked: %v, asked %d", err, asked)
|
||||
}
|
||||
err = handOverAsked([]string{"laptop", "/srv/notes"}, known,
|
||||
ask(link.HandOverAnswer{Refused: "/srv/notes is not used as found; nothing was handed over"}), &out)
|
||||
if err == nil || !strings.Contains(err.Error(), "laptop refused: /srv/notes is not used as found") || out.Len() != 0 {
|
||||
t.Fatalf("a refusal: %v, printed %q", err, out.String())
|
||||
}
|
||||
err = handOverAsked([]string{"laptop", "/srv/notes"}, known, ask(link.HandOverAnswer{Said: "handed over"}), &out)
|
||||
if err != nil || !strings.HasPrefix(out.String(), "handed over\n") || !strings.Contains(out.String(), "`nox push laptop`") {
|
||||
t.Fatalf("a record: %v, printed %q", err, out.String())
|
||||
}
|
||||
failing := func(string, string, string) (link.HandOverAnswer, error) {
|
||||
return link.HandOverAnswer{}, errors.New("no engine")
|
||||
}
|
||||
if err := handOverAsked([]string{"laptop", "/srv/notes"}, known, failing, &out); err == nil {
|
||||
t.Fatal("an ask that failed was a success")
|
||||
}
|
||||
}
|
||||
|
||||
// The setuid search's line asks only a known node, one name and nothing else, and fails on a refusal
|
||||
// (novox/hq issue 361).
|
||||
func TestASetuidSearchAsksOnlyAKnownNodeAndFailsOnARefusal(t *testing.T) {
|
||||
t.Setenv(link.CallerVar, "jo through mesh-cli on anchor")
|
||||
asked := 0
|
||||
ask := func(answer link.HandOverAnswer) func(node, by string) (link.HandOverAnswer, error) {
|
||||
return func(node, by string) (link.HandOverAnswer, error) {
|
||||
asked++
|
||||
if node != "novox" || by != "jo through mesh-cli on anchor" {
|
||||
t.Fatalf("asked %q %q", node, by)
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
}
|
||||
known := func(string) error { return nil }
|
||||
var out bytes.Buffer
|
||||
for _, args := range [][]string{nil, {"novox", "extra"}, {"-x"}} {
|
||||
if err := setuidSearchAsked(args, known, ask(link.HandOverAnswer{Said: "x"}), &out); err == nil || asked != 0 {
|
||||
t.Fatalf("%v was asked: %v", args, err)
|
||||
}
|
||||
}
|
||||
if err := setuidSearchAsked([]string{"novox"}, func(string) error { return errors.New("no node called novox") },
|
||||
ask(link.HandOverAnswer{Said: "x"}), &out); err == nil || asked != 0 {
|
||||
t.Fatalf("an unknown node: %v", err)
|
||||
}
|
||||
if err := setuidSearchAsked([]string{"novox"}, known, ask(link.HandOverAnswer{Refused: "no; no search was started"}),
|
||||
&out); err == nil || !strings.Contains(err.Error(), "novox refused") || out.Len() != 0 {
|
||||
t.Fatalf("a refusal: %v, printed %q", err, out.String())
|
||||
}
|
||||
if err := setuidSearchAsked([]string{"novox"}, known, ask(link.HandOverAnswer{Said: "a new search starts"}),
|
||||
&out); err != nil || !strings.HasPrefix(out.String(), "a new search starts\n") {
|
||||
t.Fatalf("a start: %v, printed %q", err, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
// The directory judgement's line asks only a known node, one name and nothing else, fails on a refusal, and says it
|
||||
// changed nothing (novox/hq issue 358).
|
||||
func TestADirectoryJudgementAsksOnlyAKnownNodeAndSaysItIsReadOnly(t *testing.T) {
|
||||
t.Setenv(link.CallerVar, "jo through mesh-cli on anchor")
|
||||
asked := 0
|
||||
ask := func(answer link.HandOverAnswer) func(node, by string) (link.HandOverAnswer, error) {
|
||||
return func(node, by string) (link.HandOverAnswer, error) {
|
||||
asked++
|
||||
if node != "novox" || by != "jo through mesh-cli on anchor" {
|
||||
t.Fatalf("asked %q %q", node, by)
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
}
|
||||
known := func(string) error { return nil }
|
||||
var out bytes.Buffer
|
||||
for _, args := range [][]string{nil, {"novox", "extra"}, {"-x"}} {
|
||||
if err := judgeDirectoriesAsked(args, known, ask(link.HandOverAnswer{Said: "x"}), &out); err == nil || asked != 0 {
|
||||
t.Fatalf("%v was asked: %v", args, err)
|
||||
}
|
||||
}
|
||||
if err := judgeDirectoriesAsked([]string{"novox"}, func(string) error { return errors.New("no node called novox") },
|
||||
ask(link.HandOverAnswer{Said: "x"}), &out); err == nil || asked != 0 {
|
||||
t.Fatalf("an unknown node: %v", err)
|
||||
}
|
||||
if err := judgeDirectoriesAsked([]string{"novox"}, known, ask(link.HandOverAnswer{Refused: "no; nothing was judged"}),
|
||||
&out); err == nil || !strings.Contains(err.Error(), "novox refused") || out.Len() != 0 {
|
||||
t.Fatalf("a refusal: %v, printed %q", err, out.String())
|
||||
}
|
||||
said := "2 directories declared: 1 placed, 1 would be refused\n/var/lib/a: placed\n/var/lib/b: would be refused: …"
|
||||
if err := judgeDirectoriesAsked([]string{"novox"}, known, ask(link.HandOverAnswer{Said: said}), &out); err != nil ||
|
||||
out.String() != "novox (read-only: nothing on novox was changed)\n"+said+"\n" {
|
||||
t.Fatalf("a judgement: %v, printed %q", err, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
// The judgement is a seat verb an agent may call — it reads, and changes nothing on the node — and runs exactly
|
||||
// `node judge-directories <node>` (novox/hq issue 358).
|
||||
func TestTheDirectoryJudgementIsAVerbThatOnlyReads(t *testing.T) {
|
||||
argv, err := argvFor("judge-directories", map[string]any{"node": "novox"})
|
||||
if err != nil || strings.Join(argv, " ") != "node judge-directories novox" {
|
||||
t.Fatalf("the verb runs %v, %v", argv, err)
|
||||
}
|
||||
if _, err := argvFor("judge-directories", map[string]any{}); err == nil {
|
||||
t.Fatal("the verb ran with no node")
|
||||
}
|
||||
if !readingVerbs["judge-directories"] {
|
||||
t.Fatal("the judgement is not counted a reading verb")
|
||||
}
|
||||
}
|
||||
@@ -625,7 +625,7 @@ func planStale(ctx context.Context, inv *inventory.Inventory, p inventory.Plan)
|
||||
}
|
||||
for _, newer := range recent {
|
||||
if newer.ID == p.ID || !newer.Created.After(p.Created) || !repositoryMatches(newer.Repository, p.Repository) ||
|
||||
newer.Branch != p.Branch || newer.State == inventory.PlanSuperseded || newer.Batch() {
|
||||
newer.Branch != p.Branch || newer.State == inventory.PlanSuperseded {
|
||||
continue
|
||||
}
|
||||
return inventory.PlanSuperseded, fmt.Sprintf("superseded by %s (%s %s), a newer merge of the same repository "+
|
||||
|
||||
@@ -128,7 +128,7 @@ func (r *recordedDelivery) grant(context.Context, []readyNode) error { return ni
|
||||
|
||||
func (r *recordedDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
|
||||
r.declared = append(r.declared, s.node)
|
||||
return recordSent(ctx, r.inv, s.node, body, s.declared.Builds, s.declared.Epoch, sentRecordOf(ctx, s.declared))
|
||||
return recordSent(ctx, r.inv, s.node, body, s.declared.Builds, s.declared.Epoch)
|
||||
}
|
||||
|
||||
// aResolver is a module built from a repository, at a commit, with something on the machine that
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// hideTyping turns a terminal's echo off while a secret is typed at it, and gives back what restores it. On
|
||||
// anything that is not a terminal (a pipe, a file) it does nothing.
|
||||
func hideTyping(f *os.File) func() {
|
||||
fd := int(f.Fd())
|
||||
before, err := unix.IoctlGetTermios(fd, unix.TCGETS)
|
||||
if err != nil {
|
||||
return func() {}
|
||||
}
|
||||
hidden := *before
|
||||
hidden.Lflag &^= unix.ECHO
|
||||
if err := unix.IoctlSetTermios(fd, unix.TCSETS, &hidden); err != nil {
|
||||
return func() {}
|
||||
}
|
||||
return func() {
|
||||
_ = unix.IoctlSetTermios(fd, unix.TCSETS, before)
|
||||
_, _ = os.Stderr.WriteString("\n")
|
||||
}
|
||||
}
|
||||
@@ -1,221 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// novox/hq issue 348: on 2026-10-09 the control node's resolver stopped answering on its private address
|
||||
// at 10:57:57 UTC; the machine's network condition was raised at 10:58:45. The node-engine and the
|
||||
// controller were sent at 10:59:34. The new node-engine's first statement judged the names once — unknown,
|
||||
// "one look failed; a second decides" — and that statement cleared the condition, though its last evidence
|
||||
// still said "connection refused". The next look raised it again at 11:00:23, after the send, and both
|
||||
// builds failed their gate at 11:10 with "raised since it was sent" and were put back, for a fault that
|
||||
// began before they were sent.
|
||||
|
||||
// namesRefused is the control node's names part as its node-engine said it in the outage: its own
|
||||
// resolver, at its own address, refusing.
|
||||
func namesRefused(state string, streak int) link.NetworkPart {
|
||||
p := link.NetworkPart{Part: link.PartNames, State: state, Since: h0, Streak: streak}
|
||||
if state == link.StateUnhealthy {
|
||||
p.Reason = "1 of its 2 resolvers do not answer as the mesh's do"
|
||||
p.Said = "10.77.0.1 — anchor.internal (IPv4): read udp 10.77.0.1:35244->10.77.0.1:53: read: connection refused"
|
||||
p.Toward = []string{"10.77.0.1"}
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func networkSaying(parts ...link.NetworkPart) *link.NetworkHealth {
|
||||
state := link.StateHealthy
|
||||
for _, p := range parts {
|
||||
switch {
|
||||
case p.State == link.StateUnhealthy:
|
||||
state = link.StateUnhealthy
|
||||
case p.State == link.StateUnknown && state == link.StateHealthy:
|
||||
state = link.StateUnknown
|
||||
}
|
||||
}
|
||||
return &link.NetworkHealth{State: state, Since: h0, Parts: parts}
|
||||
}
|
||||
|
||||
// TestReplay348 replays the statements of the outage: the condition raised before the send is not
|
||||
// cleared by the restarted engine's first, undecided statement, and the gate does not count it against
|
||||
// the builds sent after it began.
|
||||
func TestReplay348(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv, k := open.inventory, conditionsFrom
|
||||
say := func(at time.Time, n *link.NetworkHealth) {
|
||||
t.Helper()
|
||||
if err := stateHealth(ctx, inv, k, "anchor", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
network := func() (conditions.Condition, bool) {
|
||||
t.Helper()
|
||||
list, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range list {
|
||||
if c.Key == "machine.anchor.network" {
|
||||
return c, true
|
||||
}
|
||||
}
|
||||
return conditions.Condition{}, false
|
||||
}
|
||||
|
||||
// 10:58:45 — the second failing look: raised.
|
||||
say(h0, networkSaying(namesRefused(link.StateUnhealthy, 2)))
|
||||
raised, ok := network()
|
||||
if !ok {
|
||||
t.Fatal("the resolver refusing on the control node raised nothing")
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
sent := time.Now().UTC()
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
|
||||
// 10:59:42 — the restarted engine's first statement: one look failed, a second decides.
|
||||
say(h0.Add(time.Minute), networkSaying(namesRefused(link.StateUnknown, 1)))
|
||||
if _, ok := network(); !ok {
|
||||
t.Fatal("a statement that judged nothing yet cleared the condition: the restarted engine's first look " +
|
||||
"said the fault was gone while it still refused")
|
||||
}
|
||||
// 11:00:23 — its second look: unhealthy again, the same raising.
|
||||
say(h0.Add(2*time.Minute), networkSaying(namesRefused(link.StateUnhealthy, 2)))
|
||||
again, ok := network()
|
||||
if !ok || !again.Raised.Equal(raised.Raised) || again.Count != 1 {
|
||||
t.Fatalf("the same raising was not kept: raised %s (first %s), count %d", again.Raised, raised.Raised, again.Count)
|
||||
}
|
||||
|
||||
// The gate on the control node, for a build sent after the fault began.
|
||||
open2, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := gateFacts{judged: true, open: open2}
|
||||
if w := aboutTheMachine("anchor", []string{"mesh-host"}, sent, f); w.whole != "" || len(w.on) != 0 {
|
||||
t.Fatalf("a fault from before the send held the build: %+v", w)
|
||||
}
|
||||
|
||||
// Decided healthy: cleared.
|
||||
say(h0.Add(3*time.Minute), networkSaying(link.NetworkPart{Part: link.PartNames, State: link.StateHealthy, Since: h0}))
|
||||
if c, ok := network(); ok {
|
||||
t.Fatalf("a statement that decides the names healthy left %s open", c.Key)
|
||||
}
|
||||
}
|
||||
|
||||
// A fault there at the send, cleared and reopened after it, is not raised since the send; one that cleared
|
||||
// before the send and came back after it is — a send that breaks a recovered machine fails its gate (review
|
||||
// of mesh-controller PR 179, A2). Read through OpenAt, by both of the gate's readings.
|
||||
func TestAFaultThatFlappedAfterTheSendIsNotTheSendsAndOneThatRecoveredBeforeItIs(t *testing.T) {
|
||||
since := h0
|
||||
network := func(first time.Time, gaps ...conditions.Gap) conditions.Condition {
|
||||
raised := since.Add(time.Minute)
|
||||
if len(gaps) > 0 {
|
||||
raised = gaps[len(gaps)-1].Reopened
|
||||
}
|
||||
return conditions.Condition{Key: "machine.anchor.network", Kind: kindMachineNetwork,
|
||||
Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor"},
|
||||
Summary: "anchor's network is not healthy", Source: sourceNetwork, First: first, Gaps: gaps, Raised: raised}
|
||||
}
|
||||
held := func(c conditions.Condition) bool {
|
||||
return aboutTheMachine("anchor", []string{"mesh-controller"}, since, gateFacts{judged: true,
|
||||
open: []conditions.Condition{c}}).whole != ""
|
||||
}
|
||||
// The day's case: raised before the send, cleared 8 s after it, reopened 49 s after it.
|
||||
flapped := network(since.Add(-49*time.Second),
|
||||
conditions.Gap{Cleared: since.Add(8 * time.Second), Reopened: since.Add(49 * time.Second)})
|
||||
if held(flapped) {
|
||||
t.Fatal("a fault there at the send, flapping after it, held the machine")
|
||||
}
|
||||
// Recovered before the send, broken again after it: the send's.
|
||||
recovered := network(since.Add(-time.Hour),
|
||||
conditions.Gap{Cleared: since.Add(-30 * time.Second), Reopened: since.Add(20 * time.Second)})
|
||||
if !held(recovered) {
|
||||
t.Fatal("a machine recovered at the send and broken after it passed the gate")
|
||||
}
|
||||
// An older gap, before the send, and the fault there at the send: not the send's.
|
||||
twice := network(since.Add(-time.Hour),
|
||||
conditions.Gap{Cleared: since.Add(-50 * time.Minute), Reopened: since.Add(-45 * time.Minute)},
|
||||
conditions.Gap{Cleared: since.Add(10 * time.Second), Reopened: since.Add(30 * time.Second)})
|
||||
if held(twice) {
|
||||
t.Fatal("a fault there at the send, with an older gap, held the machine")
|
||||
}
|
||||
// Raised after the send, never cleared: the send's.
|
||||
if !held(network(time.Time{})) {
|
||||
t.Fatal("a fault raised after the send held nothing")
|
||||
}
|
||||
// And a module's own, through judgeHealth.
|
||||
at := since.Add(2 * time.Minute)
|
||||
g := gateFacts{judged: true, now: at, reports: map[string]inventory.Reported{"anchor": {Node: "anchor",
|
||||
Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, engines: map[string]string{},
|
||||
served: map[string]served{}, rolledBack: map[string][]lease.Rollback{},
|
||||
open: []conditions.Condition{{Key: "provider.app.anchor.x.failing", Subject: conditions.Subject{
|
||||
Scope: conditions.ScopeProvider, ID: "app.anchor.x", Machine: "anchor"}, Summary: "failing",
|
||||
First: since.Add(-time.Hour), Raised: since.Add(time.Minute),
|
||||
Gaps: []conditions.Gap{{Cleared: since.Add(5 * time.Second), Reopened: since.Add(time.Minute)}}}}}
|
||||
if _, why := judgeHealth("app", "", catalogue.Manifest{Module: "app"}, "anchor", since, g); strings.HasPrefix(why, "raised since it was sent") {
|
||||
t.Fatalf("a module's own fault there at the send: %s", why)
|
||||
}
|
||||
g.open[0].Gaps[0].Cleared = since.Add(-5 * time.Second)
|
||||
if _, why := judgeHealth("app", "", catalogue.Manifest{Module: "app"}, "anchor", since, g); !strings.HasPrefix(why, "raised since it was sent") {
|
||||
t.Fatalf("a module's own fault, recovered at the send and back after it, was not counted: %s", why)
|
||||
}
|
||||
}
|
||||
|
||||
// Only an undecided part holds a condition that names it; a condition about another part clears, and a
|
||||
// statement unknown as a whole holds every part (review of PR 179, A4). Pure.
|
||||
func TestAnUndecidedPartHoldsOnlyWhatNamesIt(t *testing.T) {
|
||||
f := netFacts(map[string]*inventory.NetworkHealth{
|
||||
"anchor": aNetwork(link.StateUnknown, inventory.NetworkPart{Part: link.PartNames, State: link.StateUnknown, Streak: 1},
|
||||
inventory.NetworkPart{Part: link.PartRoute, State: link.StateHealthy}),
|
||||
"laptop": aNetwork(link.StateHealthy, inventory.NetworkPart{Part: link.PartNames, State: link.StateHealthy}),
|
||||
"spare": aNetwork(link.StateStarting, inventory.NetworkPart{Part: link.PartTunnel, State: link.StateHealthy}),
|
||||
"other": aNetwork(link.StateStarting, inventory.NetworkPart{Part: link.PartTunnel, State: link.StateStarting}),
|
||||
})
|
||||
u := undecidedParts(f)
|
||||
if !u["anchor"][link.PartNames] || u["anchor"][link.PartRoute] || u["laptop"] != nil || !u["spare"]["*"] ||
|
||||
!u["other"][link.PartTunnel] || u["other"]["*"] {
|
||||
t.Fatalf("undecided: %v", u)
|
||||
}
|
||||
about := func(machine, said string, also ...string) conditions.Condition {
|
||||
return conditions.Condition{Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: machine,
|
||||
Machine: machine, Also: also}, Evidence: []conditions.Evidence{{Said: said}}}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
c conditions.Condition
|
||||
held bool
|
||||
}{
|
||||
{about("anchor", "names since 2026-10-09 10:58:45 UTC: 10.77.0.1 — refused"), true},
|
||||
{about("anchor", "route since 2026-10-09 10:58:45 UTC: no default route"), false},
|
||||
{about("laptop", "names since 2026-10-09 10:58:45 UTC: refused"), false},
|
||||
{about("spare", "route since …: no default route"), true},
|
||||
{about("hub", "anchor: names: refused", "anchor"), true},
|
||||
{about("hub", "anchor: tunnel: no handshake", "anchor"), false},
|
||||
} {
|
||||
if got := heldUndecided(c.c, u); got != c.held {
|
||||
t.Errorf("%s %q held %v, want %v", c.c.Subject.Machine, c.c.Evidence[0].Said, got, c.held)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A release walks its modules without a record per module: D10 counts what its tier names as rolling,
|
||||
// so the node-engine a release walks is not "behind, and no plan is rolling it out" on its first machine.
|
||||
func TestAReleaseRollsOutWhatItsTierNames(t *testing.T) {
|
||||
plans := []inventory.Plan{
|
||||
{ID: "release-1", State: inventory.PlanRolling, Tiers: [][]string{{"mesh-host"}}, Modules: map[string]*inventory.PlanModule{}},
|
||||
{ID: "plan-2", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{"letta": {}}},
|
||||
}
|
||||
got := rollingModules(plans)
|
||||
if !got["mesh-host"] || !got["letta"] || len(got) != 2 {
|
||||
t.Fatalf("rolling: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -1,359 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
)
|
||||
|
||||
// novox/hq issue 352: on 2026-10-09 a release's gate on the control node read the machine's report against a
|
||||
// newer send another plan had just made there — not against its own send — and failed three builds the
|
||||
// machine had reported healthy ("has not reported on what it was sent"), put them back on every machine,
|
||||
// to a controller older than the store's schema, and that controller then judged the newer plan's
|
||||
// controller passed from the put-back build's health.
|
||||
|
||||
// TestReplay352 replays the walk on the backlog fixture: the release sends anchor and anchor reports;
|
||||
// another send reaches anchor, unreported; the gate still passes. And a send that moves a judged module
|
||||
// to another build supersedes the judging: no verdict, nothing put back.
|
||||
func TestReplay352(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
t.Run("a newer send to the judged machine does not unreport the gate's", testANewerSendDoesNotUnreportTheGatesOwn)
|
||||
t.Run("a send that moves the module supersedes the judging", testASendThatMovesTheModuleSupersedesTheJudging)
|
||||
}
|
||||
|
||||
func testANewerSendDoesNotUnreportTheGatesOwn(t *testing.T) {
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
advancePlans(ctx, b.open) // anchor is sent, and the fixture reports it applied
|
||||
// 16:31:37 — another plan sends anchor a newer declaration, which it has not reported on.
|
||||
carried, _, _ := inv.SentBuilds(ctx, "anchor")
|
||||
if err := inv.RecordSent(ctx, nodeID(t, b.open, "anchor"), "d-anchor-newer", carried); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reports, _ := inv.LastReports(ctx)
|
||||
for _, r := range reports {
|
||||
if r.Node == "anchor" && r.Current {
|
||||
t.Fatal("the fixture's newer send reads as reported")
|
||||
}
|
||||
}
|
||||
gateEvery, gateBound = 0, 0 // past the bound at once: before the fix, "has not reported" fails it here
|
||||
for i := 0; i < 4; i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
p := b.release(t)
|
||||
if p.State == inventory.PlanFailed || strings.Contains(p.Note, "has not reported") {
|
||||
t.Fatalf("the release failed on the newer send: %s %s", p.State, p.Note)
|
||||
}
|
||||
if g := p.Release.Gate; g != nil && (g.Sent == nil || g.Sent["anchor"].Digest == "") {
|
||||
t.Fatalf("the gate does not keep what it sent: %+v", g)
|
||||
}
|
||||
if v, found, err := inv.GateOf(ctx, "build-app-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("app's pass on anchor was not kept: %+v %v %v", v, found, err)
|
||||
}
|
||||
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" {
|
||||
t.Fatalf("app was put back to %s", current["app"].Commit)
|
||||
}
|
||||
}
|
||||
|
||||
// A plan's own first send waits while a release judges the same module on that machine with another build.
|
||||
func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
advancePlans(ctx, b.open) // the release judges app c2 on anchor
|
||||
_, _, _, err := gatedSend(ctx, b.open, "anchor", []inventory.CarriedMove{{Module: "app", Node: "anchor", From: "c2", To: "c3", Build: "build-app-c3"}})
|
||||
if !errors.Is(err, errWalkedElsewhere) || !strings.Contains(err.Error(), "release-") {
|
||||
t.Fatalf("a newer build of a judged module was sent under the release's gate: %v", err)
|
||||
}
|
||||
if len(b.sent) != 1 {
|
||||
t.Fatalf("sent %v", b.sent)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge plan's judging is superseded the same way: another send moved its module on the first machine.
|
||||
func TestAPlansJudgingIsSupersededByASendThatMovesItsModule(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := g.open.inventory
|
||||
advancePlans(ctx, g.open) // anchor is sent app c2 first
|
||||
if err := inv.RecordSent(ctx, nodeID(t, g.open, "anchor"), "d-anchor-c3", map[string]string{"app": "c3"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gateEvery = 0
|
||||
advancePlans(ctx, g.open)
|
||||
p := g.plan(t)
|
||||
if p.State != inventory.PlanSuperseded || !strings.Contains(p.Note, "superseded") || !strings.Contains(p.Note, "c3") {
|
||||
t.Fatalf("the plan is %s: %s", p.State, p.Note)
|
||||
}
|
||||
// Nothing put back: the registered build stands, the build is not marked, and the plan's gate made no
|
||||
// rollback (a release may walk what the other send left waiting on anchor; that is not a put-back).
|
||||
if r := p.Modules["app"].Gate.Rollback; r != "" {
|
||||
t.Fatalf("a superseded judging made a rollback: %q", r)
|
||||
}
|
||||
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" {
|
||||
t.Fatalf("app was put back to %s", current["app"].Commit)
|
||||
}
|
||||
if failed, _ := inv.GateFailed(ctx, "build-2"); failed {
|
||||
t.Fatal("a superseded build was marked failed")
|
||||
}
|
||||
}
|
||||
|
||||
func testASendThatMovesTheModuleSupersedesTheJudging(t *testing.T) {
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
advancePlans(ctx, b.open)
|
||||
// Another send moves app on anchor to a build this gate does not judge.
|
||||
carried, _, _ := inv.SentBuilds(ctx, "anchor")
|
||||
carried["app"] = "c3"
|
||||
if err := inv.RecordSent(ctx, nodeID(t, b.open, "anchor"), "d-anchor-c3", carried); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gateEvery = 0
|
||||
advancePlans(ctx, b.open)
|
||||
p := b.release(t)
|
||||
if p.State != inventory.PlanSuperseded || !strings.Contains(p.Note, "superseded") || !strings.Contains(p.Note, "c3") {
|
||||
t.Fatalf("the release is %s: %s", p.State, p.Note)
|
||||
}
|
||||
if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}}) {
|
||||
t.Fatalf("sent %v: a superseded judging puts nothing back", b.sent)
|
||||
}
|
||||
if _, found, _ := inv.GateOf(ctx, "build-app-c2"); found {
|
||||
t.Fatal("a superseded judging kept a verdict")
|
||||
}
|
||||
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" {
|
||||
t.Fatalf("app was put back to %s", current["app"].Commit)
|
||||
}
|
||||
}
|
||||
|
||||
// A report is on the gate's own send: the declaration itself, or one sequenced after it; a gate kept
|
||||
// without its send reads the report against the send made last, as before. Pure.
|
||||
func TestAReportIsHeldAgainstTheGatesOwnSend(t *testing.T) {
|
||||
sent := inventory.SentDeclaration{Digest: "d-490", Sequence: 490}
|
||||
for _, c := range []struct {
|
||||
r inventory.Reported
|
||||
want bool
|
||||
}{
|
||||
{inventory.Reported{Declared: "d-490", Current: false}, true},
|
||||
{inventory.Reported{Declared: "d-491", ReportedSequence: 491, Current: true}, true},
|
||||
{inventory.Reported{Declared: "d-489", ReportedSequence: 489, Current: false}, false},
|
||||
{inventory.Reported{Declared: "other", ReportedSequence: 490}, true}, // the same sequence, said by another digest
|
||||
{inventory.Reported{Declared: "d-495", ReportedSequence: 495, Current: true}, true}, // the last send: this one or a later one
|
||||
{inventory.Reported{Declared: "", Current: false}, false},
|
||||
} {
|
||||
if got := sent.ReportsOn(c.r); got != c.want {
|
||||
t.Errorf("%+v on %+v: %v", c.r, sent, got)
|
||||
}
|
||||
}
|
||||
byDigest := inventory.SentDeclaration{Digest: "d-1"}
|
||||
if !byDigest.ReportsOn(inventory.Reported{Declared: "d-1"}) || byDigest.ReportsOn(inventory.Reported{ReportedSequence: 5}) ||
|
||||
!byDigest.ReportsOn(inventory.Reported{Current: true}) {
|
||||
t.Fatal("a send kept without a sequence is matched by its digest and by the last send alone")
|
||||
}
|
||||
f := gateFacts{sent: map[string]inventory.SentDeclaration{"anchor": sent}}
|
||||
if !f.reportedOn("anchor", inventory.Reported{Declared: "d-490"}) || f.reportedOn("anchor", inventory.Reported{Declared: "d-1"}) {
|
||||
t.Fatal("a gate that kept its send read the report against something other than it")
|
||||
}
|
||||
if !f.reportedOn("laptop", inventory.Reported{Current: true}) || f.reportedOn("laptop", inventory.Reported{Current: false}) {
|
||||
t.Fatal("a gate that did not keep its send does not read the report against the send made last")
|
||||
}
|
||||
// Through the merge plan's first-machine wait too.
|
||||
at := time.Now()
|
||||
state := inventory.PlanModule{First: []string{"anchor"}, FirstAt: &at,
|
||||
Gate: &inventory.PlanGate{Machines: []string{"anchor"}, Sent: map[string]inventory.SentDeclaration{"anchor": sent}}}
|
||||
reports := []inventory.Reported{{Node: "anchor", At: &at, Outcome: inventory.OutcomeApplied, Current: false, Declared: "d-490"}}
|
||||
if step := nextRollout(state, []string{"anchor", "laptop"}, false, reports, at.Add(time.Minute), time.Hour); step.waiting != "" || step.failed != "" {
|
||||
t.Fatalf("the first machine's report on the plan's own send read as none: %+v", step)
|
||||
}
|
||||
reports[0].Declared = "d-480"
|
||||
if step := nextRollout(state, []string{"anchor", "laptop"}, false, reports, at.Add(time.Minute), time.Hour); step.waiting == "" {
|
||||
t.Fatalf("a report on an older send read as the plan's: %+v", step)
|
||||
}
|
||||
}
|
||||
|
||||
// A gate judges only the build the machine was last sent: last sent another build of the module, the
|
||||
// judging is superseded, whatever the machine reports. Pure.
|
||||
func TestAGateJudgesOnlyTheBuildTheMachineWasLastSent(t *testing.T) {
|
||||
at := time.Now()
|
||||
f := gateFacts{now: at, reports: map[string]inventory.Reported{"anchor": {Node: "anchor", Outcome: inventory.OutcomeApplied,
|
||||
At: &at, Current: true}}, engines: map[string]string{}, served: map[string]served{}, rolledBack: map[string][]lease.Rollback{},
|
||||
commits: map[string]string{"mesh-controller": "e6b00e2e"}, sentBuilds: map[string]map[string]string{"anchor": {"mesh-controller": "ef26d4cb"}}}
|
||||
taken := at.Add(-30 * time.Second)
|
||||
f.holder = &lease.Holder{Taken: taken, Health: &lease.Health{Ready: true}}
|
||||
h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "anchor", at.Add(-time.Minute), f)
|
||||
if h != healthSuperseded || !strings.Contains(why, "ef26d4cb") || !strings.Contains(why, "e6b00e2e") {
|
||||
t.Fatalf("a controller build the machine no longer runs: %v %q", h, why)
|
||||
}
|
||||
f.sentBuilds["anchor"]["mesh-controller"] = "e6b00e2e"
|
||||
if h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "anchor", at.Add(-time.Minute), f); h == healthSuperseded {
|
||||
t.Fatalf("the build sent read as another: %q", why)
|
||||
}
|
||||
delete(f.sentBuilds, "anchor")
|
||||
if h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "anchor", at.Add(-time.Minute), f); h == healthSuperseded {
|
||||
t.Fatalf("a machine whose send is not known read as superseded: %q", why)
|
||||
}
|
||||
g := &inventory.PlanGate{To: "c2", Carried: []inventory.CarriedMove{{Module: "late", Node: "anchor", To: "c5"}}}
|
||||
if got := judgedCommits(g, []judged{{"app", "anchor"}, {"late", "anchor"}}); got["app"] != "c2" || got["late"] != "c5" {
|
||||
t.Fatalf("judged commits %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A move of another build of a module to a machine where a release or a plan is judging that module
|
||||
// waits for that judging; the same build to that machine is already there. Pure.
|
||||
func TestAWalkWaitsForAJudgingOfTheSameModuleOnThatMachine(t *testing.T) {
|
||||
at := time.Now()
|
||||
release := inventory.Plan{ID: "release-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{
|
||||
Gate: &inventory.PlanGate{Machines: []string{"novox"}, Carried: []inventory.CarriedMove{
|
||||
{Module: "mesh-controller", Node: "novox", From: "ef26d4cb", To: "2913c54c"}}}}}
|
||||
merge := inventory.Plan{ID: "plan-1", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
|
||||
"app": {First: []string{"anchor"}, FirstAt: &at, Commit: "c2", Gate: &inventory.PlanGate{Machines: []string{"anchor"}}}}}
|
||||
f := moveFacts{plans: []inventory.Plan{release, merge}}
|
||||
for _, c := range []struct {
|
||||
module, node, to, want string
|
||||
}{
|
||||
{"mesh-controller", "novox", "e6b00e2e", "release-1"}, // the day's case: a newer controller to the judged machine
|
||||
{"mesh-controller", "novox", "2913c54c", ""}, // the same build: already there
|
||||
{"mesh-controller", "ace", "2913c54c", "release-1"}, // another machine while the first is judged
|
||||
{"mesh-host", "novox", "x", ""}, // a module the release does not carry
|
||||
{"app", "anchor", "c2", ""},
|
||||
{"app", "anchor", "c3", "plan-1"},
|
||||
{"app", "laptop", "c2", "plan-1"},
|
||||
} {
|
||||
if got := f.walkedBy(c.module, c.node, c.to); got != c.want {
|
||||
t.Errorf("%s %s to %s: walked by %q, want %q", c.module, c.to, c.node, got, c.want)
|
||||
}
|
||||
}
|
||||
release.Release.Gate.Verdict = inventory.GatePassed
|
||||
merge.Modules["app"].Gate.Verdict = inventory.GatePassed
|
||||
if f.walkedBy("mesh-controller", "novox", "e6b00e2e") != "" || f.walkedBy("app", "laptop", "c3") != "" {
|
||||
t.Fatal("a passed judging still holds a move")
|
||||
}
|
||||
release.Release.Gate.Verdict = ""
|
||||
f.plans[0].State = inventory.PlanSuperseded
|
||||
if f.walkedBy("mesh-controller", "novox", "e6b00e2e") != "" {
|
||||
t.Fatal("a closed release still holds a move")
|
||||
}
|
||||
}
|
||||
|
||||
// The controller is never put back to a build that reaches less of the store's schema than the store
|
||||
// has, or to one that never said what it reaches: the current build is kept, and the condition says so.
|
||||
func TestTheControllerIsNotPutBackToABuildOlderThanTheStoresSchema(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
keeper, _ := withConditionsInMemory(t)
|
||||
told := &conditions.Told{}
|
||||
was := doctorFrom
|
||||
doctorFrom = &doctor{open: open, keeper: keeper, teller: told}
|
||||
t.Cleanup(func() { doctorFrom = was })
|
||||
wasSend := sendRollout
|
||||
var sent [][]string
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
sent = append(sent, names)
|
||||
return names, nil
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = wasSend })
|
||||
|
||||
build := func(id, commit, digest string, asked time.Time) inventory.Build {
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: commit})
|
||||
b := inventory.Build{ID: id, Module: "mesh-controller", Commit: commit, Repository: "novox/mesh-controller", Path: ".",
|
||||
Manifest: manifest, Asked: asked, At: asked, Made: []inventory.Artifact{{Name: "controller", Kind: catalogue.ArtifactBundle,
|
||||
Reference: "mesh-artifact://mesh-controller/controller/blobs/sha256:" + digest}}}
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: commit},
|
||||
inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Path: ".", BuiltFrom: commit, Head: commit, Asked: asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
previous := build("build-old", "ef26d4cb", strings.Repeat("1", 64), time.Now().Add(-2*time.Hour))
|
||||
failed := build("build-new", "e6b00e2e", strings.Repeat("2", 64), time.Now().Add(-time.Minute))
|
||||
if versionOfBuild(previous) != strings.Repeat("1", 12) {
|
||||
t.Fatalf("the build's version is %q", versionOfBuild(previous))
|
||||
}
|
||||
applied, err := inv.SchemaApplied(ctx)
|
||||
if err != nil || applied < 87 {
|
||||
t.Fatalf("the store's schema reaches %d (%v)", applied, err)
|
||||
}
|
||||
// The build before never recorded what it reads: not proved, refused.
|
||||
if why, ok := controllerSchemaAllows(ctx, inv, previous); ok || !strings.Contains(why, "never recorded") {
|
||||
t.Fatalf("an unknown reach: %v %q", ok, why)
|
||||
}
|
||||
// It reads less than the store has: refused, naming both.
|
||||
if err := inv.RecordSchemaReach(ctx, versionOfBuild(previous), applied-1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if why, ok := controllerSchemaAllows(ctx, inv, previous); ok || !strings.Contains(why, "is at") {
|
||||
t.Fatalf("a reach behind the store: %v %q", ok, why)
|
||||
}
|
||||
// Through the gate: the failed build is marked, nothing is put back, nothing is sent, the condition is urgent.
|
||||
at := time.Now().Add(-5 * time.Minute)
|
||||
state := &inventory.PlanModule{Build: failed.ID, Commit: failed.Commit, Previous: previous.Commit, First: []string{"anchor"}, FirstAt: &at}
|
||||
p := inventory.Plan{ID: "plan-352", Repository: "novox/mesh-controller", Branch: "main", Commit: failed.Commit, Created: at,
|
||||
State: inventory.PlanRolling, Tiers: [][]string{{"mesh-controller"}}, Modules: map[string]*inventory.PlanModule{"mesh-controller": state}}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gateFailed(ctx, open, &p, "mesh-controller", state, []string{"anchor"}, "not healthy within 10m0s of its apply")
|
||||
if state.Gate.Rollback != inventory.NotRolledBack || !strings.Contains(p.Note, "NOT put back") || !strings.Contains(p.Note, "the current build is kept") {
|
||||
t.Fatalf("rollback %q: %s", state.Gate.Rollback, p.Note)
|
||||
}
|
||||
if len(sent) != 0 {
|
||||
t.Fatalf("sent %v: nothing is put back", sent)
|
||||
}
|
||||
if current, _ := inv.CurrentBuilds(ctx); current["mesh-controller"].Commit != failed.Commit {
|
||||
t.Fatalf("the module was put back to %s", current["mesh-controller"].Commit)
|
||||
}
|
||||
if marked, _ := inv.GateFailed(ctx, failed.ID); !marked {
|
||||
t.Fatal("the failed build is not marked failed at its gate")
|
||||
}
|
||||
open2, _ := keeper.Open(ctx)
|
||||
var found bool
|
||||
for _, c := range open2 {
|
||||
if c.Kind == kindRollbackFailed && c.Severity == conditions.Urgent && strings.Contains(c.Summary, "current build is kept") {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("no urgent rollback-failed condition saying the current build is kept: %+v", open2)
|
||||
}
|
||||
// Reaching the store: allowed.
|
||||
if err := inv.RecordSchemaReach(ctx, versionOfBuild(previous), applied); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if why, ok := controllerSchemaAllows(ctx, inv, previous); !ok {
|
||||
t.Fatalf("a build that reads the whole schema was refused: %q", why)
|
||||
}
|
||||
// A build with no bundle to know it by: refused.
|
||||
if why, ok := controllerSchemaAllows(ctx, inv, inventory.Build{Commit: "x"}); ok || !strings.Contains(why, "names no bundle") {
|
||||
t.Fatalf("a build without a bundle: %v %q", ok, why)
|
||||
}
|
||||
}
|
||||
|
||||
// The lease's holder names the build the declaration told it it is, and the version stamp only without one.
|
||||
func TestTheHolderNamesTheBuildTheDeclarationToldIt(t *testing.T) {
|
||||
t.Setenv(RunningBuildVar, " ad62528c47c7 ")
|
||||
if h := holderOf("x"); h.Build != "ad62528c47c7" {
|
||||
t.Fatalf("the holder's build is %q", h.Build)
|
||||
}
|
||||
t.Setenv(RunningBuildVar, "")
|
||||
if h := holderOf("x"); h.Build != version {
|
||||
t.Fatalf("without a declared version the holder's build is %q", h.Build)
|
||||
}
|
||||
if reach, err := schemaReach(); err != nil || reach < 87 {
|
||||
t.Fatalf("this build's reach is %d (%v)", reach, err)
|
||||
}
|
||||
}
|
||||
@@ -1,433 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq issue 459, ADR 0299: since the merge window one walk carries every merge of its batch, so a stop of one
|
||||
// group member through `delivery-stop` ended the walk of every other delivery in it. The stop now names the merges
|
||||
// to leave out: the walk is ended, and every other merge it answered is walked again in a fresh batch without them
|
||||
// — the stopped merge, and each later merge of its repository's branch, which contains it and ends with it.
|
||||
|
||||
// endsItsLine ends, as the test ends, a line the controller's output left open (a walk's ask prints " tier N: " and
|
||||
// the asked build the rest, which the tests' fake ask never prints): `go test -v` would put the test's "--- PASS"
|
||||
// after it, and the store-tests step of merge-check.sh, which reads that marker at a line's start, would say the
|
||||
// test never ran.
|
||||
func endsItsLine(t *testing.T) {
|
||||
t.Cleanup(func() { fmt.Println() })
|
||||
}
|
||||
|
||||
// leaved is a kept merge of a repository, made at a moment.
|
||||
func leaved(repository, commit string, made time.Time) inventory.BatchedMerge {
|
||||
return inventory.BatchedMerge{Repository: repository, Branch: "main", Commit: commit, Merged: made, Heard: made}
|
||||
}
|
||||
|
||||
// The stopped merge is left out, and so is each later merge of its repository's branch, naming the merge it
|
||||
// contains; an earlier merge of that repository and every other repository's merge are walked again.
|
||||
func TestLeftOutAreTheStoppedMergeAndTheMergesThatContainIt(t *testing.T) {
|
||||
c1 := leaved("novox/mesh-catalog", "c1aaaaaaaaaa", t0)
|
||||
c2 := leaved("novox/mesh-catalog", "c2bbbbbbbbbb", t0.Add(10*time.Second))
|
||||
c3 := leaved("novox/mesh-catalog", "c3cccccccccc", t0.Add(20*time.Second))
|
||||
o1 := leaved("novox/one", "o1dddddddddd", t0.Add(5*time.Second))
|
||||
again, out, err := leaveOut([]inventory.BatchedMerge{c1, o1, c2, c3}, []string{"novox/Mesh-Catalog@c2bbbbbbbbbb"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var walked []string
|
||||
for _, m := range again {
|
||||
walked = append(walked, m.Commit)
|
||||
}
|
||||
if !reflect.DeepEqual(walked, []string{c1.Commit, o1.Commit}) {
|
||||
t.Fatalf("walked again: %v; want the earlier catalogue merge and the other repository's", walked)
|
||||
}
|
||||
want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: c2.Commit},
|
||||
{Repository: "novox/mesh-catalog", Commit: c3.Commit, Contains: c2.Commit}}
|
||||
if !reflect.DeepEqual(out, want) {
|
||||
t.Fatalf("left out: %+v; want %+v", out, want)
|
||||
}
|
||||
// A merge the walk does not answer is refused, naming it: nothing ends.
|
||||
if _, _, err := leaveOut([]inventory.BatchedMerge{c1, o1}, []string{"novox/mesh-catalog@ffffffffffff"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "ffffffffffff") {
|
||||
t.Fatalf("a merge the walk does not answer was left out: %v", err)
|
||||
}
|
||||
if _, _, err := leaveOut([]inventory.BatchedMerge{c1}, []string{"no-commit"}); err == nil {
|
||||
t.Fatal("a merge without its commit was taken")
|
||||
}
|
||||
}
|
||||
|
||||
// moving is a kept merge that moved modules when it was heard.
|
||||
func moving(m inventory.BatchedMerge, modules ...string) inventory.BatchedMerge {
|
||||
m.Event, _ = json.Marshal(keptMerge{Moves: modules})
|
||||
return m
|
||||
}
|
||||
|
||||
// A merge of another repository that moves a module the stopped merge moves is left out too, naming the modules
|
||||
// it shares: the module is built at its branch, which holds the stopped merge (review of #232). And so on, until
|
||||
// none more is: a merge sharing a module with that one is left out as well.
|
||||
func TestAMergeSharingAStoppedMergesModuleIsLeftOut(t *testing.T) {
|
||||
stopped := moving(leaved("novox/mesh-catalog", "c1aaaaaaaaaa", t0), "app")
|
||||
reader := moving(leaved("novox/one", "o1bbbbbbbbbb", t0.Add(time.Second)), "app", "one")
|
||||
next := moving(leaved("novox/two", "t1cccccccccc", t0.Add(2*time.Second)), "one")
|
||||
free := moving(leaved("novox/three", "h1dddddddddd", t0.Add(3*time.Second)), "three")
|
||||
again, out, err := leaveOut([]inventory.BatchedMerge{stopped, reader, next, free}, []string{"novox/mesh-catalog@c1aaaaaaa"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(again) != 1 || again[0].Commit != free.Commit {
|
||||
t.Fatalf("walked again: %+v; want the merge that shares nothing alone", again)
|
||||
}
|
||||
want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: stopped.Commit},
|
||||
{Repository: "novox/one", Commit: reader.Commit, Contains: stopped.Commit, Shares: []string{"app"}},
|
||||
{Repository: "novox/two", Commit: next.Commit, Contains: stopped.Commit, Shares: []string{"one"}}}
|
||||
if !reflect.DeepEqual(out, want) {
|
||||
t.Fatalf("left out %+v; want %+v", out, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A walk withholds a module a stopped merge moved where it would build it only as a dependent; one a merge of its
|
||||
// own batch moves is built, and empty tiers go.
|
||||
func TestAWalkWithholdsAStoppedMergesDependents(t *testing.T) {
|
||||
plan := inventory.Plan{Tiers: [][]string{{"one"}, {"app"}, {"notes", "app2"}},
|
||||
Modules: map[string]*inventory.PlanModule{"one": {}, "app": {}, "notes": {}, "app2": {}}}
|
||||
got := withhold(&plan, []string{"app", "notes", "absent"}, []string{"notes"})
|
||||
if !reflect.DeepEqual(got, []string{"app"}) || !reflect.DeepEqual(plan.Tiers, [][]string{{"one"}, {"notes", "app2"}}) {
|
||||
t.Fatalf("withheld %v, tiers %v", got, plan.Tiers)
|
||||
}
|
||||
if _, in := plan.Modules["app"]; in {
|
||||
t.Fatal("a withheld module is still in the plan")
|
||||
}
|
||||
}
|
||||
|
||||
// The seat verb passes the merges to leave out to the command.
|
||||
func TestDeliveryStopTakesTheMergesToLeaveOut(t *testing.T) {
|
||||
got, err := argvFor("delivery-stop", map[string]any{"plan": "plan-1", "why": "w", "by": "jochen",
|
||||
"without": "novox/mesh-lab@b1,novox/mesh-lab@b2"})
|
||||
want := []string{"delivery", "stop", "plan-1", "--why", "w", "--by", "mesh-delivery for jochen",
|
||||
"--without", "novox/mesh-lab@b1,novox/mesh-lab@b2"}
|
||||
if err != nil || !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("delivery-stop with without → %v %v, wanted %v", got, err, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A walk of the catalogue's merge and another repository's is stopped without the catalogue's: it ends stopped,
|
||||
// names the batch that walks the other merge again and the merge it left out, and that batch is cut into a walk
|
||||
// that builds the other repository's module alone.
|
||||
func TestAStopWithoutAMergeWalksTheOthersAgain(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "pr/cat1aaaaaaaa")
|
||||
endsItsLine(t)
|
||||
asksRecorded(t)
|
||||
ctx := t.Context()
|
||||
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
|
||||
one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head)
|
||||
hear(t, open, cat, t0.Add(time.Second))
|
||||
hear(t, open, one, t0.Add(6*time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 || !ws[0].Open() {
|
||||
t.Fatalf("no open walk to stop: %+v", ws)
|
||||
}
|
||||
stopped := ws[0]
|
||||
p, said, err := stopWalkWithout(ctx, open.inventory, stopped.ID, "mesh-delivery for jochen", "its group stopped",
|
||||
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.State != inventory.PlanFailed || p.Delivery.Stopped == "" || p.Delivery.WalkedAgainBy == "" ||
|
||||
!reflect.DeepEqual(p.Delivery.LeftOut, []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: cat.Commit}}) {
|
||||
t.Fatalf("the stopped walk reads %s %+v", p.State, p.Delivery)
|
||||
}
|
||||
if said.WalkedAgainBy != p.Delivery.WalkedAgainBy || !reflect.DeepEqual(said.WalkedAgain, []string{"novox/one@" + one.Commit}) {
|
||||
t.Fatalf("the stop answered %+v", said)
|
||||
}
|
||||
again, err := open.inventory.MergesOf(ctx, p.Delivery.WalkedAgainBy)
|
||||
if err != nil || len(again) != 1 || again[0].Commit != one.Commit {
|
||||
t.Fatalf("the fresh batch answers %+v (%v); want the other repository's merge alone", again, err)
|
||||
}
|
||||
if left, _ := open.inventory.MergesOf(ctx, stopped.ID); len(left) != 1 || left[0].Commit != cat.Commit {
|
||||
t.Fatalf("the stopped walk still answers %+v; want the left-out merge alone", left)
|
||||
}
|
||||
cutAt(t, open, t0.Add(4*time.Minute))
|
||||
fresh, err := open.inventory.PlanByID(ctx, p.Delivery.WalkedAgainBy)
|
||||
if err != nil || fresh.Batch() || !fresh.Open() {
|
||||
t.Fatalf("the fresh batch was not cut into a walk: %s %v", fresh.State, err)
|
||||
}
|
||||
if _, in := fresh.Modules["one"]; !in {
|
||||
t.Fatalf("the fresh walk does not build one: %v", fresh.Modules)
|
||||
}
|
||||
if _, in := fresh.Modules["app"]; in {
|
||||
t.Fatalf("the fresh walk builds app, whose merge was left out: %v", fresh.Modules)
|
||||
}
|
||||
if len(fresh.Delivery.Merges) != 1 || fresh.Delivery.Merges[0].Commit != one.Commit {
|
||||
t.Fatalf("the fresh walk answers %+v", fresh.Delivery.Merges)
|
||||
}
|
||||
// A stopped walk starts no search of its own (ADR 0276 decision 3 is for a failed one).
|
||||
cutAt(t, open, t0.Add(5*time.Minute))
|
||||
if alone, _ := open.inventory.AloneMerges(ctx); len(alone) != 0 {
|
||||
t.Fatalf("a merge of the stopped walk waits to be walked alone: %+v", alone)
|
||||
}
|
||||
}
|
||||
|
||||
// A later merge of the stopped merge's repository contains it: it is left out too, named with the merge it
|
||||
// contains, and only the other repository's merge is walked again. With nothing left to walk, the walk is only
|
||||
// ended. A merge the walk does not answer is refused, and nothing ends.
|
||||
func TestAStopWithoutAMergeEndsTheMergesThatContainIt(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "pr/cat1aaaaaaaa")
|
||||
endsItsLine(t)
|
||||
asksRecorded(t)
|
||||
ctx := t.Context()
|
||||
first := catalogueMerge("cat1aaaaaaaa", "app", t0)
|
||||
later := catalogueMerge("cat2cccccccc", "notes", t0.Add(10*time.Second))
|
||||
one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), first.Head)
|
||||
hear(t, open, first, t0.Add(time.Second))
|
||||
hear(t, open, one, t0.Add(6*time.Second))
|
||||
hear(t, open, later, t0.Add(11*time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
walk := ws[0]
|
||||
|
||||
if _, _, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "x",
|
||||
[]string{"novox/mesh-catalog@ffffffffffff"}, t0.Add(3*time.Minute)); err == nil {
|
||||
t.Fatal("a stop leaving out a merge the walk does not answer was taken")
|
||||
}
|
||||
if p, _ := open.inventory.PlanByID(ctx, walk.ID); !p.Open() {
|
||||
t.Fatalf("a refused stop ended the walk: %s", p.State)
|
||||
}
|
||||
|
||||
p, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped",
|
||||
[]string{"novox/mesh-catalog@" + first.Commit}, t0.Add(3*time.Minute))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: first.Commit},
|
||||
{Repository: "novox/mesh-catalog", Commit: later.Commit, Contains: first.Commit}}
|
||||
if !reflect.DeepEqual(p.Delivery.LeftOut, want) || !reflect.DeepEqual(said.LeftOut, want) {
|
||||
t.Fatalf("left out %+v, said %+v; want %+v", p.Delivery.LeftOut, said.LeftOut, want)
|
||||
}
|
||||
if !strings.Contains(p.Note, later.Commit[:8]) || !strings.Contains(p.Note, p.Delivery.WalkedAgainBy) {
|
||||
t.Fatalf("the stopped walk's note does not name the merge ended with it and the fresh batch: %q", p.Note)
|
||||
}
|
||||
again, _ := open.inventory.MergesOf(ctx, p.Delivery.WalkedAgainBy)
|
||||
if len(again) != 1 || again[0].Commit != one.Commit {
|
||||
t.Fatalf("walked again: %+v; want the other repository's merge alone", again)
|
||||
}
|
||||
|
||||
// Nothing left once the other repository's merge is left out too: the fresh walk is stopped, and nothing more.
|
||||
cutAt(t, open, t0.Add(4*time.Minute))
|
||||
fresh, _ := open.inventory.PlanByID(ctx, p.Delivery.WalkedAgainBy)
|
||||
q, said, err := stopWalkWithout(ctx, open.inventory, fresh.ID, "mesh-delivery for jochen", "that one too",
|
||||
[]string{"novox/one@" + one.Commit}, t0.Add(5*time.Minute))
|
||||
if err != nil || q.State != inventory.PlanFailed || q.Delivery.WalkedAgainBy != "" || said.WalkedAgainBy != "" {
|
||||
t.Fatalf("a stop leaving nothing to walk: %s %+v %+v %v", q.State, q.Delivery, said, err)
|
||||
}
|
||||
if _, bs := walks(t, open); len(bs) != 0 {
|
||||
t.Fatalf("a batch of nothing was made: %+v", bs)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge heard while the walk ran waits in the open batch, at a later commit of the same repository. The merge
|
||||
// walked again joins that batch, never a batch of its own that would walk the older commit after the newer one
|
||||
// (review of #232): its window closes, and its walk is at the newer commit, answering both.
|
||||
func TestAStopWalksTheOthersAgainInTheOpenBatch(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "pr/cat1aaaaaaaa")
|
||||
endsItsLine(t)
|
||||
asksRecorded(t)
|
||||
ctx := t.Context()
|
||||
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
|
||||
one1 := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head)
|
||||
hear(t, open, cat, t0.Add(time.Second))
|
||||
hear(t, open, one1, t0.Add(6*time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
walk := ws[0]
|
||||
one2 := repoMerge("one", "one2cccccccc", t0.Add(3*time.Minute))
|
||||
hear(t, open, one2, t0.Add(3*time.Minute))
|
||||
_, bs := walks(t, open)
|
||||
if len(bs) != 1 {
|
||||
t.Fatalf("the merge heard while the walk ran is in %d batches", len(bs))
|
||||
}
|
||||
waiting := bs[0]
|
||||
p, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped",
|
||||
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.Delivery.WalkedAgainBy != waiting.ID || said.WalkedAgainBy != waiting.ID {
|
||||
t.Fatalf("walked again by %q (said %q); want the open batch %s", p.Delivery.WalkedAgainBy, said.WalkedAgainBy, waiting.ID)
|
||||
}
|
||||
if _, bs := walks(t, open); len(bs) != 1 {
|
||||
t.Fatalf("%d batches; want the open one alone", len(bs))
|
||||
}
|
||||
b, _ := open.inventory.PlanByID(ctx, waiting.ID)
|
||||
if b.Delivery.Batch == nil || !b.Delivery.Batch.Closed || !slices.Contains(b.Delivery.Batch.Withheld, "app") {
|
||||
t.Fatalf("the open batch reads %+v; want its window closed and app withheld", b.Delivery.Batch)
|
||||
}
|
||||
// Its window closed by the stop, not by the clock: cut at once.
|
||||
cutAt(t, open, t0.Add(3*time.Minute+11*time.Second))
|
||||
fresh, _ := open.inventory.PlanByID(ctx, waiting.ID)
|
||||
if fresh.Batch() || fresh.CommitOf("novox/one") != one2.Commit {
|
||||
t.Fatalf("the batch is %s at %q; want it cut, walking the newer commit", fresh.State, fresh.CommitOf("novox/one"))
|
||||
}
|
||||
var answered []string
|
||||
for _, m := range fresh.Delivery.Merges {
|
||||
answered = append(answered, m.Commit+">"+m.Carried)
|
||||
}
|
||||
if !slices.Contains(answered, one1.Commit+">"+one2.Commit) || !slices.Contains(answered, one2.Commit+">") {
|
||||
t.Fatalf("the walk answers %v; want the earlier merge carried by the later", answered)
|
||||
}
|
||||
}
|
||||
|
||||
// A stop names what the stopped walk already sent and nothing walks again, on which machines; and a later merge of
|
||||
// the stopped merge's repository that waits in a batch, which will deliver the stopped change. A stopped walk
|
||||
// that walked its merges again is refused a retry, saying why.
|
||||
func TestAStopNamesWhatStaysOnTheMachinesAndWhatStillCarriesIt(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "pr/cat1aaaaaaaa")
|
||||
endsItsLine(t)
|
||||
asksRecorded(t)
|
||||
ctx := t.Context()
|
||||
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
|
||||
one1 := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head)
|
||||
hear(t, open, cat, t0.Add(time.Second))
|
||||
hear(t, open, one1, t0.Add(6*time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
walk := ws[0]
|
||||
sent := t0.Add(150 * time.Second)
|
||||
for name, m := range walk.Modules {
|
||||
if name == "app" || name == "one" {
|
||||
m.First, m.FirstAt = []string{"anchor"}, &sent
|
||||
}
|
||||
}
|
||||
if err := open.inventory.SavePlan(ctx, &walk); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cat2 := catalogueMerge("cat2dddddddd", "notes", t0.Add(3*time.Minute))
|
||||
hear(t, open, cat2, t0.Add(3*time.Minute))
|
||||
_, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped",
|
||||
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(said.Kept, []inventory.PlanKept{{Module: "app", Machines: []string{"anchor"}}}) {
|
||||
t.Fatalf("kept %+v; want app on anchor alone (one is walked again)", said.Kept)
|
||||
}
|
||||
p, _ := open.inventory.PlanByID(ctx, walk.ID)
|
||||
if !reflect.DeepEqual(p.Delivery.Kept, said.Kept) || !strings.Contains(p.Note, "app on anchor") {
|
||||
t.Fatalf("the stopped walk's record does not name what stays: %+v %q", p.Delivery.Kept, p.Note)
|
||||
}
|
||||
if len(said.StillCarriedBy) != 1 || !strings.Contains(said.StillCarriedBy[0], cat2.Commit) {
|
||||
t.Fatalf("still carried by %v; want the later catalogue merge waiting in the batch", said.StillCarriedBy)
|
||||
}
|
||||
if _, err := retryPlan(ctx, open, walk.ID); err == nil || !strings.Contains(err.Error(), "was stopped without") {
|
||||
t.Fatalf("a walk stopped without some merges was retried, or refused for another reason: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A batch not yet cut is stopped without a merge: the merge goes to a stopped record of its own, the batch goes on
|
||||
// with the others and withholds the stopped merge's modules; with every merge left out, the batch is stopped.
|
||||
func TestAStopWithoutAMergeLeavesItOutOfABatch(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
endsItsLine(t)
|
||||
asksRecorded(t)
|
||||
ctx := t.Context()
|
||||
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
|
||||
one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head)
|
||||
hear(t, open, cat, t0.Add(time.Second))
|
||||
hear(t, open, one, t0.Add(6*time.Second))
|
||||
_, bs := walks(t, open)
|
||||
b := bs[0]
|
||||
r, said, err := stopWalkWithout(ctx, open.inventory, b.ID, "mesh-delivery for jochen", "its group stopped",
|
||||
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(10*time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if said.WalkedAgainBy != b.ID || said.Stopped == b.ID {
|
||||
t.Fatalf("said %+v; want a stopped record of its own, and the batch walking the others", said)
|
||||
}
|
||||
if left, _ := open.inventory.MergesOf(ctx, said.Stopped); len(left) != 1 || left[0].Commit != cat.Commit {
|
||||
t.Fatalf("the stopped record holds %+v", left)
|
||||
}
|
||||
if stopped, _ := open.inventory.PlanByID(ctx, said.Stopped); stopped.State != inventory.PlanFailed ||
|
||||
stopped.Delivery.Stopped == "" || stopped.Delivery.WalkedAgainBy != b.ID {
|
||||
t.Fatalf("the stopped record reads %s %+v", stopped.State, stopped.Delivery)
|
||||
}
|
||||
_ = r
|
||||
cutAt(t, open, t0.Add(3*time.Minute))
|
||||
w, _ := open.inventory.PlanByID(ctx, b.ID)
|
||||
if _, in := w.Modules["app"]; in || w.Batch() {
|
||||
t.Fatalf("the batch's walk is %s and builds %v; want it cut without app", w.State, w.Modules)
|
||||
}
|
||||
// A batch whose every merge is left out is stopped itself.
|
||||
hear(t, open, catalogueMerge("cat3eeeeeeee", "notes", t0.Add(4*time.Minute)), t0.Add(4*time.Minute))
|
||||
_, bs = walks(t, open)
|
||||
last := bs[0]
|
||||
if _, said, err := stopWalkWithout(ctx, open.inventory, last.ID, "mesh-delivery for jochen", "x",
|
||||
[]string{"novox/mesh-catalog@cat3eeeeeeee"}, t0.Add(4*time.Minute+time.Second)); err != nil || said.Stopped != last.ID {
|
||||
t.Fatalf("a batch left with nothing: %+v %v", said, err)
|
||||
}
|
||||
if p, _ := open.inventory.PlanByID(ctx, last.ID); p.State != inventory.PlanFailed || p.Batch() {
|
||||
t.Fatalf("the emptied batch is %s", p.State)
|
||||
}
|
||||
}
|
||||
|
||||
// The stopped walk, the batch and the merges moved are one act (ADR 0299 decision 7): a plan written by somebody
|
||||
// else since it was read refuses all of it, and no merge moves, no plan is written.
|
||||
func TestSavingPlansAndMovingMergesIsOneAct(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
endsItsLine(t)
|
||||
ctx := t.Context()
|
||||
hear(t, open, catalogueMerge("cat1aaaaaaaa", "app", t0), t0.Add(time.Second))
|
||||
_, bs := walks(t, open)
|
||||
stale := bs[0]
|
||||
moved := stale
|
||||
moved.Note = "written by somebody else"
|
||||
if err := open.inventory.SavePlan(ctx, &moved); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fresh := inventory.Plan{ID: "plan-fresh", Repository: "novox/mesh-catalog", Branch: "main", Commit: "x", Created: t0,
|
||||
State: inventory.PlanAssembling, Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{}}
|
||||
err := open.inventory.SavePlansMoving(ctx, []*inventory.Plan{&fresh, &stale},
|
||||
[]inventory.MergeMove{{Repository: "novox/mesh-catalog", Commit: "cat1aaaaaaaa", Plan: "plan-fresh"}})
|
||||
if !errors.Is(err, inventory.ErrPlanMoved) {
|
||||
t.Fatalf("a stale plan in the act: %v", err)
|
||||
}
|
||||
if _, err := open.inventory.PlanByID(ctx, "plan-fresh"); err == nil {
|
||||
t.Fatal("the new plan was written though the act was refused")
|
||||
}
|
||||
if m, _, _ := open.inventory.MergeOf(ctx, "novox/mesh-catalog", "cat1aaaaaaaa"); m.Plan != stale.ID {
|
||||
t.Fatalf("the merge moved to %q though the act was refused", m.Plan)
|
||||
}
|
||||
}
|
||||
|
||||
// What a stopped walk leaves on the machines names every machine its sends reached (review of #232): a module sent
|
||||
// with no first machine reached every machine running it; one sent first reached those, and the rest it was sent
|
||||
// to; a module a merge walked again moves is sent again, and is not named.
|
||||
func TestKeptNamesEveryMachineASendReached(t *testing.T) {
|
||||
at := t0
|
||||
p := inventory.Plan{Modules: map[string]*inventory.PlanModule{
|
||||
"together": {SentAt: &at},
|
||||
"first": {First: []string{"anchor"}, FirstAt: &at, Rest: map[string]inventory.SentDeclaration{"laptop": {}}},
|
||||
"again": {SentAt: &at},
|
||||
"unsent": {},
|
||||
}}
|
||||
got := keptOf(p, []string{"again"}, map[string][]string{"together": {"laptop", "anchor"}, "again": {"anchor"}})
|
||||
want := []inventory.PlanKept{{Module: "first", Machines: []string{"anchor", "laptop"}},
|
||||
{Module: "together", Machines: []string{"anchor", "laptop"}}}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("kept %+v; want %+v", got, want)
|
||||
}
|
||||
}
|
||||
@@ -1,102 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq issue 529: a walk waiting because its delivery is held for a person said only "its walk waits for
|
||||
// mesh-delivery's word — `plans go … --why …` starts it by hand", pointing a reader at a hand start instead of at
|
||||
// the person's decision. The line names what mesh-delivery says holds the walk: the held delivery, its reason and
|
||||
// `mesh-delivery.release`; or the delivery not at its turn.
|
||||
|
||||
func waitingWalk(now time.Time) inventory.Plan {
|
||||
return inventory.Plan{ID: "plan-1791793277221379309", Repository: "novox/mesh-sdk", Branch: "main",
|
||||
Commit: "9747f3e1b1ac", State: inventory.PlanBuilding, Tiers: [][]string{{"mesh-sdk"}},
|
||||
Created: now.Add(-9 * time.Minute), Modules: map[string]*inventory.PlanModule{},
|
||||
Delivery: &inventory.PlanDelivery{Awaits: "mesh-delivery"}}
|
||||
}
|
||||
|
||||
func TestAWalkWaitingOnAHeldDeliveryNamesTheHold(t *testing.T) {
|
||||
now := time.Date(2026, 10, 12, 8, 35, 0, 0, time.UTC)
|
||||
p := waitingWalk(now)
|
||||
held := "novox/mesh-sdk@d41d255f28ad held since 2026-10-12T08:23:00Z: its group adr-0307-state-history's composed " +
|
||||
"check did not pass for the heads that merged — a person decides: `mesh-delivery.release` (id " +
|
||||
"novox/mesh-sdk@d41d255f28ad, why) lets it go, `mesh-delivery.stop` ends it"
|
||||
waits := walkWaits{p.ID: {{ID: "novox/mesh-sdk@d41d255f28ad", State: "held", Walk: p.ID, Says: held}}}
|
||||
line := planLineSaid(p, now, pauseView{}, tierAtLeast, nil, waits)
|
||||
for _, want := range []string{"held for a person", "composed check did not pass", "mesh-delivery.release"} {
|
||||
if !strings.Contains(line, want) {
|
||||
t.Fatalf("the line does not say %q: %q", want, line)
|
||||
}
|
||||
}
|
||||
if strings.Contains(line, "starts it by hand") {
|
||||
t.Fatalf("a held walk's line points at a hand start: %q", line)
|
||||
}
|
||||
// Told nothing — an older mesh-delivery, or none answering — it reads as before.
|
||||
if line := planLineSaid(p, now, pauseView{}, tierAtLeast, nil, nil); !strings.Contains(line, "starts it by hand") {
|
||||
t.Fatalf("told nothing, the line reads %q", line)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWalkWaitingOnADeliveryNotAtItsTurnSaysSo(t *testing.T) {
|
||||
now := time.Date(2026, 10, 12, 8, 57, 0, 0, time.UTC)
|
||||
p := waitingWalk(now)
|
||||
waits := walkWaits{p.ID: {{ID: "novox/mesh-sdk@9747f3e1b1ac", State: "published", Walk: p.ID,
|
||||
Says: "not let go yet: novox/mesh-tools@3f0c1a6cec18 (published) carried by it not at its turn"}}}
|
||||
line := planLineSaid(p, now, pauseView{}, tierAtLeast, nil, waits)
|
||||
if !strings.Contains(line, "not at its turn") || !strings.Contains(line, "novox/mesh-tools@3f0c1a6cec18") {
|
||||
t.Fatalf("the line reads %q", line)
|
||||
}
|
||||
// No hand start offered: it would skip the group's order (review of issue 529).
|
||||
if strings.Contains(line, "plans go") {
|
||||
t.Fatalf("a walk not at its turn points at a hand start: %q", line)
|
||||
}
|
||||
}
|
||||
|
||||
// A walk nothing holds — the nzbgeek case, its merges' deliveries ended as nothing-to-move — is said so, and the hand
|
||||
// start is the way on.
|
||||
func TestAWalkHeldByNothingSaysWhyAndTheHandStart(t *testing.T) {
|
||||
now := time.Date(2026, 10, 11, 20, 30, 0, 0, time.UTC)
|
||||
p := waitingWalk(now)
|
||||
waits := walkWaits{p.ID: {{ID: "novox/mesh-media-catalog@cdcdcdcd0000", State: "unheld", Walk: p.ID,
|
||||
Says: "held by nothing: its merges' deliveries ended as nothing-to-move (novox/mesh-media-catalog@abababab0000), " +
|
||||
"yet the walk moves nzbgeek, nzbget — nothing gives it its word"}}}
|
||||
line := planLineSaid(p, now, pauseView{}, tierAtLeast, nil, waits)
|
||||
if !strings.Contains(line, "ended as nothing-to-move") || !strings.Contains(line, "plans go "+p.ID) {
|
||||
t.Fatalf("the line reads %q", line)
|
||||
}
|
||||
}
|
||||
|
||||
// What holds each waiting walk is asked of mesh-delivery's `stalled` with `walks`, a verb the controller's grant
|
||||
// already names; only lines naming a walk are kept, by walk.
|
||||
func TestWhatHoldsTheWaitingWalksIsAskedOfTheDeliveryOwner(t *testing.T) {
|
||||
was := askDeliveryOwner
|
||||
defer func() { askDeliveryOwner = was }()
|
||||
var asked map[string]any
|
||||
askDeliveryOwner = func(_ context.Context, _ *nats.Conn, verb string, args map[string]any) (json.RawMessage, error) {
|
||||
if verb != "stalled" {
|
||||
t.Fatalf("asked %s", verb)
|
||||
}
|
||||
asked = args
|
||||
return json.Marshal([]stalledLine{{ID: "novox/a@1", State: "held", Walk: "plan-1", Says: "held: why"},
|
||||
{ID: "novox/b@2", State: "held", Says: "a delivery past its bound, no walk"}})
|
||||
}
|
||||
got := walksWaiting(t.Context(), nil, true)
|
||||
if asked["walks"] != true {
|
||||
t.Fatalf("stalled was asked with %v", asked)
|
||||
}
|
||||
if len(got) != 1 || len(got["plan-1"]) != 1 || got["plan-1"][0].Says != "held: why" {
|
||||
t.Fatalf("the waiting walks are %+v", got)
|
||||
}
|
||||
if got := walksWaiting(t.Context(), nil, false); got != nil {
|
||||
t.Fatalf("with no holder on record it asked: %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -77,7 +77,7 @@ func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) {
|
||||
}
|
||||
cancel() // the sender is going away: its context is cancelled between the send and the record
|
||||
body := []byte(`{"declaration":1,"resources":[]}`)
|
||||
digest, err := recordSent(ctx, inv, "anchor", body, nil, 0, sentRecord{sender: "a test"})
|
||||
digest, err := recordSent(ctx, inv, "anchor", body, nil, 0)
|
||||
if err != nil {
|
||||
// NodeByName on the cancelled context may itself refuse; the record must still be possible
|
||||
// through the detached context, so look the node up again on a live one.
|
||||
|
||||
@@ -1,272 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/secrets"
|
||||
)
|
||||
|
||||
// `secret issued <provision>`: the credential a provider's software issued, taken from the provider and kept as
|
||||
// the provision's one credential for every consumer (novox/hq ADR 0305, issue 526).
|
||||
//
|
||||
// **The value comes from the provider, never from whoever asks.** A module served by its machine's runtime calls
|
||||
// under the runtime's one bus account, which carries that machine's agents' calls too, so a value handed to a
|
||||
// verb could be anybody's. This line carries none: it asks the provider's own tool, on the provider's machine, in
|
||||
// the controller's own name — the subject that names its caller, which the bus lets the controller alone publish
|
||||
// (issue 365) — with a sealing key made for this one call. The tool answers only the controller, the value sealed
|
||||
// to that key; the controller opens it, seals it to every holder and forgets it, as the desk's value is (ADR 0277
|
||||
// §23). Asking again fetches the same value again, and a value every holder already holds is not sealed anew.
|
||||
//
|
||||
// **What this trusts: the provider's machine.** The answer is taken from whoever answers the provider's tool
|
||||
// subject on the provider's machine first. On that machine its runtime serves the subject for every module it
|
||||
// carries, so anything that runtime runs, or anything else granted that subject there, could answer before the
|
||||
// provider does and hand the controller a value of its own. ADR 0305 accepts that by trusting the provider's
|
||||
// machine, as the provider itself is trusted; the bus keeps every other machine from answering it.
|
||||
|
||||
// handActIssued is the verb a change of an issued value is recorded under: the mesh working as decided.
|
||||
const handActIssued = "secret issued"
|
||||
|
||||
// issuedAsk is how long the provider's tool is given to answer.
|
||||
const issuedAsk = 20 * time.Second
|
||||
|
||||
// issuedProvider is one assignment of a module whose offer says its software issues the provision's credential.
|
||||
type issuedProvider struct {
|
||||
Node, Module, Tool string
|
||||
}
|
||||
|
||||
// issuedTaker is the line's work, with what it reaches handed in: the store, the bus and the sends.
|
||||
type issuedTaker struct {
|
||||
// providers are the assignments to ask, and whether any module offers the provision so at all.
|
||||
providers func(provision string) ([]issuedProvider, bool, error)
|
||||
ask func(p issuedProvider, args map[string]any) (link.Answer, error)
|
||||
keep func(provision, provider, value string) (inventory.IssuedKept, error)
|
||||
record func(act link.HandAct) error
|
||||
send func(machines []string) error
|
||||
}
|
||||
|
||||
// take asks every provider of the provision for its software's value and keeps each; it answers what it did, in
|
||||
// words that never carry the value or its fingerprint.
|
||||
func (t issuedTaker) take(provision string) (string, error) {
|
||||
providers, offered, err := t.providers(provision)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !offered {
|
||||
return "", fmt.Errorf("no module offers %q with a credential its software issues (\"credential\": "+
|
||||
"{\"issued-by\": \"software\", \"tool\": …}, ADR 0305), so there is nothing to take from a provider; "+
|
||||
"nothing was asked", provision)
|
||||
}
|
||||
if len(providers) == 0 {
|
||||
return fmt.Sprintf("no consumer holds %s from a provider whose software issues it yet, so there is nothing "+
|
||||
"to seal; a consumer that binds holds a value the mesh made until the provider next asks", provision), nil
|
||||
}
|
||||
var said []string
|
||||
var failed []string
|
||||
for _, p := range providers {
|
||||
words, err := t.takeFrom(provision, p)
|
||||
if err != nil {
|
||||
failed = append(failed, fmt.Sprintf("%s on %s: %v", p.Module, p.Node, err))
|
||||
continue
|
||||
}
|
||||
said = append(said, words)
|
||||
}
|
||||
out := strings.Join(said, "\n")
|
||||
if len(failed) > 0 {
|
||||
return out, fmt.Errorf("%s was not taken from every provider:\n %s", provision, strings.Join(failed, "\n "))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// takeFrom is one provider: asked, opened, kept, recorded and sent when it changed.
|
||||
func (t issuedTaker) takeFrom(provision string, p issuedProvider) (string, error) {
|
||||
public, private, err := secrets.Keypair()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("no key could be made to take the value: %w", err)
|
||||
}
|
||||
answer, err := t.ask(p, map[string]any{"provision": provision, "seal_to": public})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("its tool %s could not be asked: %w", p.Tool, err)
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return "", fmt.Errorf("its tool %s refused: %s", p.Tool, answer.Error)
|
||||
}
|
||||
var given struct {
|
||||
Sealed string `json:"sealed"`
|
||||
}
|
||||
if err := unmarshalAnswer(answer, &given); err != nil || given.Sealed == "" {
|
||||
return "", fmt.Errorf("its tool %s answered no sealed value", p.Tool)
|
||||
}
|
||||
opened, err := secrets.Open(private, given.Sealed)
|
||||
if err != nil {
|
||||
// Never the value, never what failed to open: only that it was not sealed to this call.
|
||||
return "", fmt.Errorf("its tool %s answered a value not sealed to this call; nothing was taken", p.Tool)
|
||||
}
|
||||
value := strings.TrimSpace(string(opened))
|
||||
for i := range opened {
|
||||
opened[i] = 0
|
||||
}
|
||||
if value == "" {
|
||||
return "", fmt.Errorf("its tool %s answered an empty value; nothing was taken", p.Tool)
|
||||
}
|
||||
kept, err := t.keep(provision, p.Node, value)
|
||||
value = ""
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
holders := make([]string, 0, len(kept.Holders))
|
||||
for _, h := range kept.Holders {
|
||||
holders = append(holders, fmt.Sprintf("%s on %s%s", h.ConsumerModule, h.Consumer, asLocal(h.Local)))
|
||||
}
|
||||
words := ""
|
||||
if len(kept.Unsealable) > 0 {
|
||||
var names []string
|
||||
for _, h := range kept.Unsealable {
|
||||
names = append(names, fmt.Sprintf("%s on %s", h.ConsumerModule, h.Consumer))
|
||||
}
|
||||
words = fmt.Sprintf("\n not sealed to %s: no sealing key on that machine", strings.Join(names, ", "))
|
||||
}
|
||||
if !kept.Changed {
|
||||
return fmt.Sprintf("%s from %s on %s: every holder already holds the value its software issued (%s); "+
|
||||
"nothing was sent%s", provision, p.Module, p.Node, strings.Join(holders, ", "), words), nil
|
||||
}
|
||||
sealedTo := make([]string, 0, len(kept.Sealed))
|
||||
for _, h := range kept.Sealed {
|
||||
sealedTo = append(sealedTo, fmt.Sprintf("%s on %s%s", h.ConsumerModule, h.Consumer, asLocal(h.Local)))
|
||||
}
|
||||
recorded := ""
|
||||
if err := t.record(link.HandAct{Verb: handActIssued, Args: []string{provision},
|
||||
Why: fmt.Sprintf("%s on %s gave the credential its software issued for %s; sealed to %s", p.Module, p.Node,
|
||||
provision, strings.Join(sealedTo, ", ")),
|
||||
Cause: "issued-by-software"}); err != nil {
|
||||
recorded = fmt.Sprintf("\n this was NOT recorded in the hand-act log, and is done anyway: %v", err)
|
||||
}
|
||||
if err := t.send(kept.Machines); err != nil {
|
||||
return "", fmt.Errorf("%s from %s on %s is sealed to %s and could not be sent: %w; `push --behind` sends it",
|
||||
provision, p.Module, p.Node, strings.Join(sealedTo, ", "), err)
|
||||
}
|
||||
return fmt.Sprintf("%s from %s on %s: the value its software issued is sealed to %s; sent %s, and each of "+
|
||||
"them starts again on it%s%s", provision, p.Module, p.Node, strings.Join(sealedTo, ", "),
|
||||
strings.Join(kept.Machines, ", "), words, recorded), nil
|
||||
}
|
||||
|
||||
// issuedProvidersOf are the assignments to ask for a provision: every machine providing it to a pair the mesh
|
||||
// holds, and on it the assigned module whose offer says its software issues the credential (ADR 0305 §1).
|
||||
func issuedProvidersOf(ctx context.Context, inv *inventory.Inventory, provision string) ([]issuedProvider, bool, error) {
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if !offersIssued(shelf, provision) {
|
||||
return nil, false, nil
|
||||
}
|
||||
holders, err := inv.HoldersOf(ctx, provision, "")
|
||||
if err != nil {
|
||||
return nil, true, err
|
||||
}
|
||||
nodes := map[string]bool{}
|
||||
for _, h := range holders {
|
||||
nodes[h.Provider] = true
|
||||
}
|
||||
names := make([]string, 0, len(nodes))
|
||||
for n := range nodes {
|
||||
names = append(names, n)
|
||||
}
|
||||
sort.Strings(names)
|
||||
var out []issuedProvider
|
||||
for _, node := range names {
|
||||
assigned, err := inv.Assigned(ctx, node)
|
||||
if err != nil {
|
||||
return nil, true, err
|
||||
}
|
||||
out = append(out, issuedOn(shelf, node, assigned, provision)...)
|
||||
}
|
||||
return out, true, nil
|
||||
}
|
||||
|
||||
// offersIssued is whether any module offers the provision with a credential its software issues.
|
||||
func offersIssued(shelf map[string]catalogue.Manifest, provision string) bool {
|
||||
for _, m := range shelf {
|
||||
if _, issued := m.IssuedCredentialOf(provision); issued {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// issuedOn are the modules assigned to one machine whose offer says their software issues the provision's
|
||||
// credential, each with the tool that answers it.
|
||||
func issuedOn(shelf map[string]catalogue.Manifest, node string, assigned []string, provision string) []issuedProvider {
|
||||
var out []issuedProvider
|
||||
for _, module := range assigned {
|
||||
if tool, issued := shelf[module].IssuedCredentialOf(provision); issued {
|
||||
out = append(out, issuedProvider{Node: node, Module: module, Tool: tool})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// issuedRefusesRotation refuses `rotate` of a provision its provider's software issues (ADR 0305 §9): a value the
|
||||
// mesh made would replace one the software holds, and every consumer would hold a dead credential until the
|
||||
// provider next asks.
|
||||
func issuedRefusesRotation(shelf map[string]catalogue.Manifest, provision string) error {
|
||||
if !offersIssued(shelf, provision) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%s's credential is issued by its provider's software, which the mesh cannot make or "+
|
||||
"rotate (ADR 0305): change it in that software, and `secret issued %s` takes the value it holds now. "+
|
||||
"Nothing was rotated", provision, provision)
|
||||
}
|
||||
|
||||
// secretIssued is `secret issued <provision>`, on this controller's stores and bus.
|
||||
func secretIssued(ctx context.Context, args []string) error {
|
||||
if len(args) != 1 || args[0] == "" || strings.HasPrefix(args[0], "-") {
|
||||
return errors.New("secret issued <provision>")
|
||||
}
|
||||
provision := args[0]
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
t := issuedTaker{
|
||||
providers: func(provision string) ([]issuedProvider, bool, error) {
|
||||
return issuedProvidersOf(ctx, open.inventory, provision)
|
||||
},
|
||||
ask: func(p issuedProvider, args map[string]any) (link.Answer, error) {
|
||||
var answer link.Answer
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
var err error
|
||||
answer, err = link.AskModuleToolOnAs(ctx, conn, broker.ControllerName, p.Module, p.Tool, p.Node,
|
||||
args, issuedAsk)
|
||||
return err
|
||||
})
|
||||
return answer, err
|
||||
},
|
||||
keep: func(provision, provider, value string) (inventory.IssuedKept, error) {
|
||||
return open.inventory.KeepIssuedSecret(ctx, provision, provider, value)
|
||||
},
|
||||
record: func(act link.HandAct) error {
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
_, err := link.RecordHandAct(ctx, conn, act)
|
||||
return err
|
||||
})
|
||||
},
|
||||
send: func(machines []string) error { return sendTo(ctx, open, machines) },
|
||||
}
|
||||
words, err := t.take(provision)
|
||||
if words != "" {
|
||||
fmt.Println(words)
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -1,218 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/secrets"
|
||||
)
|
||||
|
||||
// The `issued` verb runs `secret issued <provision>` and nothing else, and that line is the one `secret` line
|
||||
// beside rotate and the desk's that a verb may run (novox/hq ADR 0305 §3, §10, issue 526).
|
||||
func TestTheIssuedVerbRunsTheOneLineThatCarriesNoValue(t *testing.T) {
|
||||
argv, err := argvFor("issued", map[string]any{"provision": "plex-api"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(argv, " ") != "secret issued plex-api" {
|
||||
t.Fatalf("the verb composed %q", argv)
|
||||
}
|
||||
if _, err := argvFor("issued", map[string]any{}); err == nil {
|
||||
t.Fatal("the verb ran with no provision")
|
||||
}
|
||||
if _, err := argvFor("issued", map[string]any{"provision": "plex-api", "secret": "a value"}); err == nil {
|
||||
t.Fatal("the verb took a value beside the provision")
|
||||
}
|
||||
for _, line := range [][]string{
|
||||
{"secret", "issued", "plex-api", "--from", "/tmp/x"},
|
||||
{"secret", "issued", "--from"},
|
||||
{"secret", "issued"},
|
||||
{"secret", "accept", "ace", "kometa", "plex-api", "--provider", "ace"},
|
||||
{"secret", "recover", "ace", "kometa", "plex-api"},
|
||||
} {
|
||||
if err := terminalOnly(line); err == nil {
|
||||
t.Errorf("%q ran through a verb", line)
|
||||
}
|
||||
}
|
||||
if err := terminalOnly([]string{"secret", "issued", "plex-api"}); err != nil {
|
||||
t.Fatalf("the issued line was refused through its verb: %v", err)
|
||||
}
|
||||
found := false
|
||||
for _, v := range catalogue.ControllerVerbs {
|
||||
if v.Name == "issued" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("the controller's seat does not declare the issued verb")
|
||||
}
|
||||
}
|
||||
|
||||
// A provider's tool answering the value sealed to the call's key: what a provider does (ADR 0305 §2).
|
||||
func issuedSealed(t *testing.T, value string) func(issuedProvider, map[string]any) (link.Answer, error) {
|
||||
return func(_ issuedProvider, args map[string]any) (link.Answer, error) {
|
||||
key, _ := args["seal_to"].(string)
|
||||
if key == "" {
|
||||
t.Fatal("the provider was asked without a key to seal to")
|
||||
}
|
||||
if _, carried := args["value"]; carried {
|
||||
t.Fatal("the ask carried a value")
|
||||
}
|
||||
sealed, err := secrets.Seal(key, []byte(value))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(map[string]string{"sealed": sealed})
|
||||
return link.Answer{Result: raw}, nil
|
||||
}
|
||||
}
|
||||
|
||||
type issuedRecorder struct {
|
||||
kept []string
|
||||
acts []link.HandAct
|
||||
sent [][]string
|
||||
again bool // the value is held by every holder already
|
||||
}
|
||||
|
||||
func (r *issuedRecorder) taker(t *testing.T, ask func(issuedProvider, map[string]any) (link.Answer, error)) issuedTaker {
|
||||
return issuedTaker{
|
||||
providers: func(provision string) ([]issuedProvider, bool, error) {
|
||||
return []issuedProvider{{Node: "ace", Module: "plex", Tool: "plex_issued_credential"}}, true, nil
|
||||
},
|
||||
ask: ask,
|
||||
keep: func(provision, provider, value string) (inventory.IssuedKept, error) {
|
||||
r.kept = append(r.kept, value)
|
||||
holders := []inventory.Holder{{Provision: provision, Consumer: "ace", ConsumerModule: "kometa", Provider: provider},
|
||||
{Provision: provision, Consumer: "ace", ConsumerModule: "tautulli", Provider: provider}}
|
||||
if r.again {
|
||||
return inventory.IssuedKept{Holders: holders}, nil
|
||||
}
|
||||
return inventory.IssuedKept{Changed: true, Holders: holders, Sealed: holders, Machines: []string{"ace"}}, nil
|
||||
},
|
||||
record: func(act link.HandAct) error { r.acts = append(r.acts, act); return nil },
|
||||
send: func(machines []string) error { r.sent = append(r.sent, machines); return nil },
|
||||
}
|
||||
}
|
||||
|
||||
// The value the provider's tool sealed to the call is kept, recorded and sent, and the answer never says it.
|
||||
func TestAnIssuedValueIsTakenSealedFromTheProviderAndSent(t *testing.T) {
|
||||
var r issuedRecorder
|
||||
words, err := r.taker(t, issuedSealed(t, "the-servers-own-token")).take("plex-api")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(r.kept) != 1 || r.kept[0] != "the-servers-own-token" {
|
||||
t.Fatalf("the provider's value was not kept: %d kept", len(r.kept))
|
||||
}
|
||||
if len(r.sent) != 1 || strings.Join(r.sent[0], ",") != "ace" {
|
||||
t.Fatalf("the holders' machines were not sent: %v", r.sent)
|
||||
}
|
||||
if len(r.acts) != 1 || r.acts[0].Verb != handActIssued {
|
||||
t.Fatalf("the change was not recorded in the hand-act log: %+v", r.acts)
|
||||
}
|
||||
for _, said := range []string{words, r.acts[0].Why, strings.Join(r.acts[0].Args, " ")} {
|
||||
if strings.Contains(said, "the-servers-own-token") || strings.Contains(said, inventory.IssuedStamp("the-servers-own-token")) {
|
||||
t.Fatalf("the value or its fingerprint was said: %q", said)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(words, "kometa on ace") || !strings.Contains(words, "tautulli on ace") {
|
||||
t.Fatalf("the answer does not name the holders: %q", words)
|
||||
}
|
||||
}
|
||||
|
||||
// The same value again writes nothing, records nothing and sends nobody (ADR 0305 §5).
|
||||
func TestAnIssuedValueEveryHolderHoldsSendsNobody(t *testing.T) {
|
||||
r := issuedRecorder{again: true}
|
||||
words, err := r.taker(t, issuedSealed(t, "the-servers-own-token")).take("plex-api")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(r.sent) != 0 || len(r.acts) != 0 {
|
||||
t.Fatalf("an unchanged value was sent or recorded: %v %v", r.sent, r.acts)
|
||||
}
|
||||
if !strings.Contains(words, "nothing was sent") {
|
||||
t.Fatalf("the answer does not say nothing was sent: %q", words)
|
||||
}
|
||||
}
|
||||
|
||||
// A value not sealed to the call's key is not taken, nor is a refusal, nor an empty answer (ADR 0305 §2).
|
||||
func TestOnlyAValueSealedToTheCallIsTaken(t *testing.T) {
|
||||
other, _, _ := secrets.Keypair()
|
||||
elsewhere, _ := secrets.Seal(other, []byte("somebody's value"))
|
||||
raw, _ := json.Marshal(map[string]string{"sealed": elsewhere})
|
||||
plain, _ := json.Marshal(map[string]string{"value": "the-servers-own-token"})
|
||||
for name, ask := range map[string]func(issuedProvider, map[string]any) (link.Answer, error){
|
||||
"sealed to another key": func(issuedProvider, map[string]any) (link.Answer, error) { return link.Answer{Result: raw}, nil },
|
||||
"answered in the clear": func(issuedProvider, map[string]any) (link.Answer, error) { return link.Answer{Result: plain}, nil },
|
||||
"refused": func(issuedProvider, map[string]any) (link.Answer, error) {
|
||||
return link.Answer{Error: "the controller alone"}, nil
|
||||
},
|
||||
"not reached": func(issuedProvider, map[string]any) (link.Answer, error) {
|
||||
return link.Answer{}, errors.New("503 no responders")
|
||||
},
|
||||
"sealed empty": issuedSealed(t, " "),
|
||||
} {
|
||||
var r issuedRecorder
|
||||
if _, err := r.taker(t, ask).take("plex-api"); err == nil {
|
||||
t.Errorf("%s: taken", name)
|
||||
}
|
||||
if len(r.kept) != 0 || len(r.sent) != 0 {
|
||||
t.Errorf("%s: something was kept or sent", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A provision no offer says its software issues is refused before anything is asked.
|
||||
func TestAProvisionNoOfferSaysIsIssuedIsRefused(t *testing.T) {
|
||||
asked := false
|
||||
taker := issuedTaker{
|
||||
providers: func(string) ([]issuedProvider, bool, error) { return nil, false, nil },
|
||||
ask: func(issuedProvider, map[string]any) (link.Answer, error) {
|
||||
asked = true
|
||||
return link.Answer{}, nil
|
||||
},
|
||||
}
|
||||
if _, err := taker.take("postgres-database"); err == nil || !strings.Contains(err.Error(), "ADR 0305") {
|
||||
t.Fatalf("expected a refusal naming the record, got %v", err)
|
||||
}
|
||||
if asked {
|
||||
t.Fatal("a provider was asked for a provision no offer says its software issues")
|
||||
}
|
||||
}
|
||||
|
||||
// Which assignments are asked: the modules on the providing machine whose offer says so, with their tool.
|
||||
func TestTheProvidersAskedAreTheOffersThatSaySo(t *testing.T) {
|
||||
shelf := map[string]catalogue.Manifest{
|
||||
"plex": {Module: "plex", Provides: []catalogue.Offer{{Name: "plex-api", Credential: &catalogue.OfferCredential{
|
||||
IssuedBy: catalogue.IssuedBySoftware, Tool: "plex_issued_credential"}}}},
|
||||
"jellyfin": {Module: "jellyfin", Provides: []catalogue.Offer{{Name: "plex-api"}}},
|
||||
}
|
||||
got := issuedOn(shelf, "ace", []string{"jellyfin", "plex", "kometa"}, "plex-api")
|
||||
if len(got) != 1 || got[0] != (issuedProvider{Node: "ace", Module: "plex", Tool: "plex_issued_credential"}) {
|
||||
t.Fatalf("the providers asked: %+v", got)
|
||||
}
|
||||
if err := issuedRefusesRotation(shelf, "plex-api"); err == nil || !strings.Contains(err.Error(), "secret issued plex-api") {
|
||||
t.Fatalf("rotate of an issued provision was not refused with the way to take it: %v", err)
|
||||
}
|
||||
if err := issuedRefusesRotation(shelf, "postgres-database"); err != nil {
|
||||
t.Fatalf("rotate of an ordinary provision was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A change of an issued value is the mesh working as decided, never a repair (ADR 0305 §8).
|
||||
func TestAnIssuedValueIsADecisionInTheHandActLog(t *testing.T) {
|
||||
for _, v := range handActVerbs {
|
||||
if v.Verb == handActIssued {
|
||||
if v.Decision == "" {
|
||||
t.Fatal("secret issued counts as a repair")
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("secret issued is not in the hand-act table")
|
||||
}
|
||||
@@ -1,160 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq issue 485: the mesh records every send — sender, lease epoch, generation, modules — and only `plan`
|
||||
// said it. `status` says per machine when it was last sent a declaration and by whom, in one line; `node show`
|
||||
// says the last send in full and what the machine answered; and `status --json` carries it as fields, since
|
||||
// the store's reader may not read the controller's tables and these verbs are the only window onto them.
|
||||
|
||||
func aLastSend(answer inventory.SendAnswer) inventory.Send {
|
||||
sent := time.Date(2026, 10, 11, 18, 40, 5, 0, time.Local)
|
||||
return inventory.Send{NodeName: "laptop", Sequence: 12, Epoch: 57,
|
||||
Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12",
|
||||
Modules: []string{"docker", "sudo"}, SentAt: sent, Recorded: true, Answer: answer}
|
||||
}
|
||||
|
||||
func TestStatusSaysEachMachinesLastSendInOneLine(t *testing.T) {
|
||||
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.Local)
|
||||
asked := answers{
|
||||
nodes: []inventory.Node{{Name: "anchor", LastSeen: time.Now()}, {Name: "laptop", LastSeen: time.Now()}},
|
||||
lastSent: map[string]inventory.Send{"laptop": aLastSend(inventory.SendAnswer{
|
||||
Outcome: inventory.OutcomeApplied, At: &at})},
|
||||
}
|
||||
shown := printed(t, func() error { return printStatus(asked) })
|
||||
var laptop, anchor []string
|
||||
for _, line := range strings.Split(shown, "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) > 0 && fields[0] == "laptop" {
|
||||
laptop = append(laptop, line)
|
||||
}
|
||||
if len(fields) > 0 && fields[0] == "anchor" {
|
||||
anchor = append(anchor, line)
|
||||
}
|
||||
}
|
||||
if len(laptop) != 1 || !strings.Contains(laptop[0], "2026-10-11 18:40") ||
|
||||
!strings.Contains(laptop[0], "by a one-shot push by jochen at a shell on anchor") ||
|
||||
!strings.Contains(laptop[0], "generation 40") || !strings.Contains(laptop[0], "applied") {
|
||||
t.Fatalf("status does not say laptop's last send in one line:\n%s", shown)
|
||||
}
|
||||
if len(anchor) != 1 || !strings.Contains(anchor[0], "no send recorded") {
|
||||
t.Fatalf("status does not say anchor has no send recorded, in one line:\n%s", shown)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatusSaysTheLastSendsCouldNotBeRead(t *testing.T) {
|
||||
asked := answers{nodes: []inventory.Node{{Name: "laptop", LastSeen: time.Now()}},
|
||||
lastSentUnread: "the store went away"}
|
||||
if shown := printed(t, func() error { return printStatus(asked) }); !strings.Contains(shown, "the store went away") {
|
||||
t.Fatalf("a record of sends that could not be read is not said:\n%s", shown)
|
||||
}
|
||||
if asked.well() {
|
||||
t.Error("a mesh whose sends could not be read is called well")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeShowSaysTheLastSendInFull(t *testing.T) {
|
||||
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.Local)
|
||||
refused := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeRefused,
|
||||
Refused: "unknown field \"generation\"\nsecond line", At: &at})), "\n")
|
||||
for _, want := range []string{"sequence 12", "2026-10-11 18:40:05", "a one-shot push by jochen at a shell on anchor",
|
||||
"lease epoch 57", "generation 40", "docker, sudo", "refused", "unknown field \"generation\""} {
|
||||
if !strings.Contains(refused, want) {
|
||||
t.Errorf("node show's last send does not say %q:\n%s", want, refused)
|
||||
}
|
||||
}
|
||||
if applied := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeApplied,
|
||||
At: &at})), "\n"); !strings.Contains(applied, "applied it at 2026-10-11 18:41:00") {
|
||||
t.Errorf("an applied send is not said applied:\n%s", applied)
|
||||
}
|
||||
if waiting := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{})), "\n"); !strings.Contains(waiting,
|
||||
"not reported on it yet") {
|
||||
t.Errorf("a send not reported on is not said so:\n%s", waiting)
|
||||
}
|
||||
stale := aLastSend(inventory.SendAnswer{})
|
||||
when := at
|
||||
stale.RefusedAt, stale.RefusedApplied = &when, 44
|
||||
if s := strings.Join(lastSendLines(stale), "\n"); !strings.Contains(s, "refused it at 2026-10-11 18:41:00") ||
|
||||
!strings.Contains(s, "generation 44") {
|
||||
t.Errorf("a send refused for its generation is not said so:\n%s", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatusJSONCarriesEachMachinesLastSend(t *testing.T) {
|
||||
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.UTC)
|
||||
asked := answers{nodes: []inventory.Node{{Name: "anchor"}, {Name: "laptop"}},
|
||||
lastSent: map[string]inventory.Send{"laptop": aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeRefused,
|
||||
Refused: "unknown field", At: &at})}}
|
||||
body, err := statusAsJSON(asked)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got struct {
|
||||
LastSent []map[string]any `json:"lastSent"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got.LastSent) != 1 {
|
||||
t.Fatalf("status --json carries %d last sends, want laptop's only:\n%s", len(got.LastSent), body)
|
||||
}
|
||||
s := got.LastSent[0]
|
||||
if s["node"] != "laptop" || s["sequence"] != float64(12) || s["epoch"] != float64(57) ||
|
||||
s["generation"] != float64(40) || s["sender"] != "a one-shot push by jochen at a shell on anchor" ||
|
||||
s["digest"] != "d12" || s["sentAt"] == nil || s["answer"] != "refused" || s["refused"] != "unknown field" ||
|
||||
s["answeredAt"] == nil {
|
||||
t.Fatalf("laptop's last send reads %v", s)
|
||||
}
|
||||
if modules, _ := s["modules"].([]any); len(modules) != 2 || modules[0] != "docker" {
|
||||
t.Fatalf("the modules it named read %v", s["modules"])
|
||||
}
|
||||
}
|
||||
|
||||
// And through the store: a send recorded and refused by the machine is what `node show` and `status --json`
|
||||
// read back.
|
||||
func TestNodeShowAndStatusReadTheLastSendFromTheStore(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
record, err := open.inventory.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSentWith(ctx, record.ID, "d12", nil, 57, 40, inventory.Send{Sequence: 12,
|
||||
Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12",
|
||||
Modules: []string{"docker", "sudo"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeRefused,
|
||||
Declared: "d12", Refused: "unknown field \"generation\""}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
shown := printed(t, func() error { return showNode(ctx, open, "laptop") })
|
||||
for _, want := range []string{"a one-shot push by jochen at a shell on anchor", "lease epoch 57", "generation 40",
|
||||
"docker, sudo", "unknown field \"generation\""} {
|
||||
if !strings.Contains(shown, want) {
|
||||
t.Errorf("node show laptop does not say %q:\n%s", want, shown)
|
||||
}
|
||||
}
|
||||
body := printed(t, func() error { return statusFor(ctx, open, true) })
|
||||
var got struct {
|
||||
LastSent []struct {
|
||||
Node string `json:"node"`
|
||||
Sender string `json:"sender"`
|
||||
Answer string `json:"answer"`
|
||||
} `json:"lastSent"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(body), &got); err != nil {
|
||||
t.Fatalf("%v:\n%s", err, body)
|
||||
}
|
||||
if len(got.LastSent) != 1 || got.LastSent[0].Node != "laptop" || got.LastSent[0].Answer != "refused" ||
|
||||
got.LastSent[0].Sender != "a one-shot push by jochen at a shell on anchor" {
|
||||
t.Fatalf("status --json's last sends read %+v", got.LastSent)
|
||||
}
|
||||
}
|
||||
@@ -1,222 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// A module assigned to a machine and left out of its declaration is said (novox/hq issue 380).
|
||||
//
|
||||
// **An omission is a finding, never a refusal of the push** (ADR 0163, rule 6): the machine is sent everything
|
||||
// else, and the module's held things are kept. Until issue 380 the only place it showed was `plan`: nfs-server was
|
||||
// assigned to the home server for days, every send left it out for a setting nobody gave, and the operator believed
|
||||
// it ran. So the self-check (D1) judges every machine as the next send would (Resolution.LeftOutBecause, the send's
|
||||
// own judgement) and raises a condition on that machine for each module it leaves out:
|
||||
//
|
||||
// - a setting nobody gave (catalogue.UnsetSettingError) is the operator's to give: kind needs-operator, naming
|
||||
// the module, the setting and the command that sets it;
|
||||
// - any other cause is said as a module not working is: kind left-out, a warning with the reason as evidence.
|
||||
//
|
||||
// Never urgent and never escalated by age (as ADR 0283 decision 5): nothing the machine ran was undone. It clears
|
||||
// on the first run that no longer finds it — the module composed again, or no longer assigned. `status` and
|
||||
// `node show` list the same modules under "assigned, not applied" with the same reason.
|
||||
//
|
||||
// A module left out because its stored manifest has a key this controller does not know is not raised here: the
|
||||
// catalogue's own unknown-field condition says it once for the whole mesh (ADR 0262); it is still listed.
|
||||
|
||||
const (
|
||||
// probeLeftOutID is the self-check's probe that raises and clears these conditions: D1, which already
|
||||
// resolves every machine (probeDeclarations), so the judgement costs no resolution of its own.
|
||||
probeLeftOutID = "D1"
|
||||
// kindLeftOut is a module left out for any cause but a setting nobody gave; it is also every such condition's
|
||||
// token, whichever its kind, so a cause that changes is the same condition said anew.
|
||||
kindLeftOut = "left-out"
|
||||
)
|
||||
|
||||
// leftOutModule is one module of a machine's set that its declaration leaves out, and why.
|
||||
type leftOutModule struct {
|
||||
Module string
|
||||
// Setting is the setting nobody gave, when that is the cause.
|
||||
Setting string
|
||||
// Why is the declaration's own reason, whole.
|
||||
Why string
|
||||
// Unread is a stored manifest this controller cannot read whole (said by the catalogue's condition).
|
||||
Unread bool
|
||||
}
|
||||
|
||||
// leftOutOf is every module of a machine's resolution that a push would leave out, sorted. Pure: the judgement
|
||||
// the push makes (catalogue.Resolution.LeftOutBecause), nothing allocated.
|
||||
func leftOutOf(plan catalogue.Resolution, settings catalogue.SettingsBy, adopted bool) []leftOutModule {
|
||||
because := plan.LeftOutBecause(settings, adopted)
|
||||
out := make([]leftOutModule, 0, len(because))
|
||||
for module, why := range because {
|
||||
l := leftOutModule{Module: module, Why: oneLine(why.Error())}
|
||||
var unset *catalogue.UnsetSettingError
|
||||
var unread *catalogue.UnreadManifestError
|
||||
switch {
|
||||
case errors.As(why, &unset):
|
||||
l.Setting = unset.Setting
|
||||
case errors.As(why, &unread):
|
||||
l.Unread = true
|
||||
}
|
||||
out = append(out, l)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module })
|
||||
return out
|
||||
}
|
||||
|
||||
// settingCommand is the command that gives a module's setting on one machine.
|
||||
func settingCommand(module, setting, node string) string {
|
||||
return fmt.Sprintf("`settings set %s '{%q: …}' --node %s`", module, setting, node)
|
||||
}
|
||||
|
||||
// reason is why a module is left out, as `status`, `node show` and the condition's summary say it: for a setting
|
||||
// nobody gave, the setting and the command that gives it; otherwise the declaration's own words.
|
||||
func (l leftOutModule) reason(node string) string {
|
||||
if l.Setting != "" {
|
||||
return fmt.Sprintf("nothing sets its setting %q, so every send leaves it out of its declaration — %s sets it", l.Setting,
|
||||
settingCommand(l.Module, l.Setting, node))
|
||||
}
|
||||
return "every send leaves it out of its declaration: " + l.Why
|
||||
}
|
||||
|
||||
// leftOutObservations are the conditions a machine's left-out modules raise, one each.
|
||||
func leftOutObservations(node string, left []leftOutModule) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, l := range left {
|
||||
if l.Unread {
|
||||
continue
|
||||
}
|
||||
out = append(out, leftOutObservation(node, l))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// leftOutObservation is one module left out of one machine's declaration: needs-operator for a setting nobody
|
||||
// gave, left-out otherwise; a warning either way, the operator's to resolve.
|
||||
func leftOutObservation(node string, l leftOutModule) conditions.Observation {
|
||||
o := conditions.Observation{Scope: conditions.ScopeModule, ID: l.Module + "." + node, Token: kindLeftOut,
|
||||
Kind: kindLeftOut, Machine: node, Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s is assigned to %s and not applied: %s", l.Module, node, l.reason(node)),
|
||||
Said: l.Why}
|
||||
w := leftOutWords(l.Module, node, l.Setting)
|
||||
if l.Setting != "" {
|
||||
o.Kind = kindNeedsOperator
|
||||
} else {
|
||||
// The words of why may name a path or an address, which the operator's channel withholds: the
|
||||
// summary sends them to the evidence, and `plan` says them in full.
|
||||
o.Summary = fmt.Sprintf("%s is assigned to %s and not applied: every send leaves it out of its declaration, "+
|
||||
"because what is set for it does not fit its manifest — the evidence and `plan %s` say why", l.Module, node, node)
|
||||
}
|
||||
o.Headline, o.Explanation, o.Needs, o.Resolved = w.Headline, w.Explanation, w.Needs, w.Resolved
|
||||
return o
|
||||
}
|
||||
|
||||
// leftOutWords is what the operator reads of a module left out (ADR 0253): plain, the act named.
|
||||
func leftOutWords(module, node, setting string) words {
|
||||
w := words{
|
||||
Headline: fmt.Sprintf("%s is not applied on %s", module, node),
|
||||
Explanation: fmt.Sprintf("%s is assigned to %s, and every send to %s leaves it out of the declaration because what "+
|
||||
"is set for it does not fit its manifest. Nothing of it changes there; the rest of %s is sent as usual.",
|
||||
module, node, node, node),
|
||||
Needs: fmt.Sprintf("read why in the details, then change what is set for %s or unassign it.", module),
|
||||
Resolved: fmt.Sprintf("%s on %s is no longer left out", module, node),
|
||||
}
|
||||
if setting != "" {
|
||||
w.Explanation = fmt.Sprintf("%s is assigned to %s, and every send to %s leaves it out of the declaration because "+
|
||||
"nothing sets its setting %s. Nothing of it runs there until it is set; the rest of %s is sent as usual.",
|
||||
module, node, node, setting, node)
|
||||
w.Needs = fmt.Sprintf("set %s for %s on %s, or approve it when it is proposed to you.", setting, module, node)
|
||||
// A setting's name that is not plain (a dotted key) is in the summary instead.
|
||||
if _, ok := conditions.PlainWords(w, node); !ok {
|
||||
w.Explanation = fmt.Sprintf("%s is assigned to %s, and every send to %s leaves it out of the declaration because a "+
|
||||
"setting it needs is not set. Nothing of it runs there until it is set; the details name it.",
|
||||
module, node, node)
|
||||
w.Needs = fmt.Sprintf("set what %s needs on %s; the details name the setting.", module, node)
|
||||
}
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// notAppliedLines is a machine's "assigned, not applied" as `node show` prints it: one line a module, with the
|
||||
// reason its condition says.
|
||||
func notAppliedLines(node string, left []leftOutModule) []string {
|
||||
if len(left) == 0 {
|
||||
return nil
|
||||
}
|
||||
lines := []string{"", " assigned, not applied:"}
|
||||
for _, l := range left {
|
||||
lines = append(lines, fmt.Sprintf(" %-22s %s", l.Module, l.reason(node)))
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
// machineNotApplied is one module assigned to a machine and left out of its declaration, in `status --json`.
|
||||
type machineNotApplied struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Setting string `json:"setting,omitempty"`
|
||||
Reason string `json:"reason"`
|
||||
// UnreadManifest is a module left out because this controller cannot read its manifest whole: it raises no
|
||||
// condition of its own on the machine, since the catalogue's unknown-field condition says it once (ADR 0262).
|
||||
UnreadManifest bool `json:"unread-manifest,omitempty"`
|
||||
}
|
||||
|
||||
// notApplied is every machine's left-out modules, in a stated order, as `status --json` carries them.
|
||||
func notApplied(left map[string][]leftOutModule) []machineNotApplied {
|
||||
names := make([]string, 0, len(left))
|
||||
for name := range left {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
var out []machineNotApplied
|
||||
for _, name := range names {
|
||||
for _, l := range left[name] {
|
||||
out = append(out, machineNotApplied{Node: name, Module: l.Module, Setting: l.Setting, Reason: l.reason(name),
|
||||
UnreadManifest: l.Unread})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// printNotApplied is status's "assigned, not applied", every machine's.
|
||||
func printNotApplied(left map[string][]leftOutModule) {
|
||||
rows := notApplied(left)
|
||||
if len(rows) == 0 {
|
||||
return
|
||||
}
|
||||
unread := 0
|
||||
for _, r := range rows {
|
||||
if r.UnreadManifest {
|
||||
unread++
|
||||
}
|
||||
}
|
||||
// Which raise a condition is said, not implied (review of #223): one whose manifest this controller cannot
|
||||
// read is listed here and raises none of its own on the machine — the catalogue's condition says it.
|
||||
raises := "each raises a condition on its machine"
|
||||
switch {
|
||||
case unread == len(rows):
|
||||
raises = "none raises a condition on its machine: this controller cannot read their manifests, which the " +
|
||||
"catalogue's own condition says"
|
||||
case unread > 0:
|
||||
raises += fmt.Sprintf(", except the %d whose manifest this controller cannot read, which the catalogue's own "+
|
||||
"condition says", unread)
|
||||
}
|
||||
fmt.Printf("%d module(s) assigned, not applied — every send leaves them out of their declaration; %s:\n",
|
||||
len(rows), raises)
|
||||
for _, r := range rows {
|
||||
fmt.Printf(" %-12s %-22s %s\n", r.Node, r.Module, r.Reason)
|
||||
}
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
// isLeftOutCondition is whether a condition is a module left out of its declaration, which the machine's health
|
||||
// statements neither raise nor clear: by its token, which every one carries whatever its kind.
|
||||
func isLeftOutCondition(c conditions.Condition) bool {
|
||||
return c.Subject.Scope == conditions.ScopeModule && strings.HasSuffix(c.Key, "."+kindLeftOut)
|
||||
}
|
||||
@@ -1,259 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// novox/hq issue 380: nfs-server was assigned to the home server for days and every send left it out — "nfs-server
|
||||
// has a file that says ${setting:shares}, and nothing sets shares for it" — and nothing but `plan` said so. The
|
||||
// manifest is the catalogue's own at the commit that added it (mesh-catalog 48fba44), which still says
|
||||
// ${setting:shares}; the reason below is the one the live push printed.
|
||||
|
||||
// leftOutNFS is the resolution of a machine assigned that nfs-server, with the settings given.
|
||||
func leftOutNFS(t *testing.T) catalogue.Resolution {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile("testdata/left-out/nfs-server.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return catalogue.Resolution{Modules: []catalogue.Manifest{m}}
|
||||
}
|
||||
|
||||
// sharesGiven is the operator's setting for it on the machine.
|
||||
func sharesGiven(value string) catalogue.SettingsBy {
|
||||
return catalogue.SettingsBy{"nfs-server": {{From: "anchor", Values: map[string]any{"shares": value}}}}
|
||||
}
|
||||
|
||||
func TestAModuleLeftOutForASettingNobodyGaveNeedsTheOperatorNamingTheSettingAndTheCommand(t *testing.T) {
|
||||
left := leftOutOf(leftOutNFS(t), nil, false)
|
||||
if len(left) != 1 || left[0].Module != "nfs-server" || left[0].Setting != "shares" {
|
||||
t.Fatalf("left out: %+v", left)
|
||||
}
|
||||
if !strings.Contains(left[0].Why, `nothing sets "shares" for it`) {
|
||||
t.Fatalf("the reason is not the push's own: %s", left[0].Why)
|
||||
}
|
||||
obs := leftOutObservations("anchor", left)
|
||||
if len(obs) != 1 {
|
||||
t.Fatalf("raised %+v", obs)
|
||||
}
|
||||
o := obs[0]
|
||||
if o.Key() != "module.nfs-server.anchor.left-out" || o.Kind != kindNeedsOperator || o.Machine != "anchor" ||
|
||||
o.Severity != conditions.Warning || o.Resolver != conditions.ResolverOperator {
|
||||
t.Fatalf("raised %s as %s, %s, by %s, on %q", o.Key(), o.Kind, o.Severity, o.Resolver, o.Machine)
|
||||
}
|
||||
for _, want := range []string{"nfs-server", "anchor", `"shares"`, "`settings set nfs-server '{\"shares\": …}' --node anchor`"} {
|
||||
if !strings.Contains(o.Summary, want) {
|
||||
t.Errorf("the summary does not name %s: %s", want, o.Summary)
|
||||
}
|
||||
}
|
||||
if o.Said != left[0].Why {
|
||||
t.Errorf("the evidence is not the reason the send gives: %s", o.Said)
|
||||
}
|
||||
plainExample(t, o, "nfs-server is not applied on anchor",
|
||||
"Needs you: set shares for nfs-server on anchor, or approve it when it is proposed to you. nfs-server is assigned to "+
|
||||
"anchor, and every send to anchor leaves it out of the declaration because nothing sets its setting shares. "+
|
||||
"Nothing of it runs there until it is set; the rest of anchor is sent as usual.")
|
||||
}
|
||||
|
||||
func TestAModuleLeftOutForAnotherCauseIsAWarningWithTheReasonAsEvidence(t *testing.T) {
|
||||
// A setting stored that its manifest can no longer take: one with a line break (issue 339).
|
||||
left := leftOutOf(leftOutNFS(t), sharesGiven("library=/srv/library\nmedia=/srv/media"), false)
|
||||
if len(left) != 1 || left[0].Setting != "" {
|
||||
t.Fatalf("left out: %+v", left)
|
||||
}
|
||||
obs := leftOutObservations("anchor", left)
|
||||
if len(obs) != 1 {
|
||||
t.Fatalf("raised %+v", obs)
|
||||
}
|
||||
o := obs[0]
|
||||
if o.Key() != "module.nfs-server.anchor.left-out" || o.Kind != kindLeftOut || o.Severity != conditions.Warning {
|
||||
t.Fatalf("raised %s as %s, %s", o.Key(), o.Kind, o.Severity)
|
||||
}
|
||||
if !strings.Contains(o.Said, "holds a line break") || strings.Contains(o.Summary, "/srv/") {
|
||||
t.Fatalf("the reason is not the evidence, or the summary carries it to the channel:\n%s\n%s", o.Summary, o.Said)
|
||||
}
|
||||
plainExample(t, o, "nfs-server is not applied on anchor",
|
||||
"Needs you: read why in the details, then change what is set for nfs-server or unassign it. nfs-server is assigned to "+
|
||||
"anchor, and every send to anchor leaves it out of the declaration because what is set for it does not fit "+
|
||||
"its manifest. Nothing of it changes there; the rest of anchor is sent as usual.")
|
||||
}
|
||||
|
||||
// The self-check raises it, keeps it a warning however long it stands, and clears it when the module composes
|
||||
// again or is no longer assigned; a machine's health statement neither clears nor raises it.
|
||||
func TestALeftOutModulesConditionClearsWhenItComposesAgainOrIsUnassigned(t *testing.T) {
|
||||
plan, settings := leftOutNFS(t), catalogue.SettingsBy(nil)
|
||||
withProbes(t, probe{ID: probeLeftOutID, Asserts: "the test's", Kind: kindLeftOut, Phase: 1,
|
||||
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
|
||||
return leftOutObservations("anchor", leftOutOf(plan, settings, false)), nil
|
||||
}})
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||
defer k.Close(context.Background())
|
||||
d := &doctor{keeper: k, teller: &conditions.Told{}, host: "anchor"}
|
||||
key := "module.nfs-server.anchor.left-out"
|
||||
openOnes := func() map[string]conditions.Condition {
|
||||
t.Helper()
|
||||
open, err := k.Open(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := map[string]conditions.Condition{}
|
||||
for _, c := range open {
|
||||
out[c.Key] = c
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
d.runOnce(t.Context(), "a test")
|
||||
c, raised := openOnes()[key]
|
||||
if !raised || c.Kind != kindNeedsOperator || c.Severity != conditions.Warning {
|
||||
t.Fatalf("a module left out raised %+v", openOnes())
|
||||
}
|
||||
// A statement from the machine that says nothing of it — the module runs nothing there — leaves it open.
|
||||
if err := judgeModuleHealth(t.Context(), nil, k, "anchor", map[string][]inventory.ResourceHealth{}, nil,
|
||||
time.Now().Add(72*time.Hour)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, still := openOnes()[key]; !still {
|
||||
t.Fatal("a health statement that says nothing of the module cleared its left-out condition")
|
||||
}
|
||||
d.runOnce(t.Context(), "a test")
|
||||
if c := openOnes()[key]; c.Severity != conditions.Warning {
|
||||
t.Fatalf("after a health statement and days, it is %+v", openOnes())
|
||||
}
|
||||
|
||||
// The setting given: it composes, and the condition clears.
|
||||
settings = sharesGiven("library=/srv/library")
|
||||
d.runOnce(t.Context(), "a test")
|
||||
if _, still := openOnes()[key]; still {
|
||||
t.Fatalf("composed again, still open: %+v", openOnes())
|
||||
}
|
||||
|
||||
// Left out again, then unassigned: no longer in the machine's set, and it clears.
|
||||
settings = nil
|
||||
d.runOnce(t.Context(), "a test")
|
||||
if _, raised := openOnes()[key]; !raised {
|
||||
t.Fatal("left out again and not raised")
|
||||
}
|
||||
plan = catalogue.Resolution{}
|
||||
d.runOnce(t.Context(), "a test")
|
||||
if _, still := openOnes()[key]; still {
|
||||
t.Fatalf("unassigned, still open: %+v", openOnes())
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheLeftOutProbeIsInTheRegistry(t *testing.T) {
|
||||
for _, p := range probeRegistry {
|
||||
if p.ID == probeLeftOutID {
|
||||
if p.run == nil || !slices.Contains(p.Raises, kindLeftOut) || !slices.Contains(p.Raises, kindNeedsOperator) {
|
||||
t.Fatalf("%+v", p)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("no probe %s: a module left out is said nowhere", probeLeftOutID)
|
||||
}
|
||||
|
||||
// status and node show list it under "assigned, not applied" with the reason the condition says, and status is
|
||||
// not well while there is one.
|
||||
func TestStatusAndNodeListAModuleAssignedAndNotApplied(t *testing.T) {
|
||||
left := map[string][]leftOutModule{"anchor": leftOutOf(leftOutNFS(t), nil, false)}
|
||||
reason := leftOutObservations("anchor", left["anchor"])[0].Summary
|
||||
asked := answers{leftOut: left}
|
||||
if asked.well() {
|
||||
t.Fatal("a mesh with a module assigned and not applied is called well")
|
||||
}
|
||||
shown := printed(t, func() error { return printStatus(asked) })
|
||||
if !strings.Contains(shown, "1 module(s) assigned, not applied") || !strings.Contains(shown, "nfs-server") ||
|
||||
!strings.Contains(shown, "`settings set nfs-server '{\"shares\": …}' --node anchor` sets it") {
|
||||
t.Fatalf("status says:\n%s", shown)
|
||||
}
|
||||
if !strings.Contains(reason, left["anchor"][0].reason("anchor")) {
|
||||
t.Fatalf("status and the condition say different reasons:\n%s\n%s", shown, reason)
|
||||
}
|
||||
body, err := statusAsJSON(asked)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var doc struct {
|
||||
NotApplied []machineNotApplied `json:"not-applied"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &doc); err != nil || len(doc.NotApplied) != 1 ||
|
||||
doc.NotApplied[0].Setting != "shares" || doc.NotApplied[0].Node != "anchor" {
|
||||
t.Fatalf("status --json: %v %s", err, body)
|
||||
}
|
||||
lines := strings.Join(notAppliedLines("anchor", left["anchor"]), "\n")
|
||||
if !strings.Contains(lines, "assigned, not applied:") || !strings.Contains(lines, "nfs-server") ||
|
||||
!strings.Contains(lines, `nothing sets its setting "shares"`) {
|
||||
t.Fatalf("node show says:\n%s", lines)
|
||||
}
|
||||
}
|
||||
|
||||
// The skip is this probe's alone (review of #223): a part waiting for the operator (ADR 0283) keeps its own
|
||||
// needs-operator condition, `module.<m>.<node>.needs-operator`, which still clears on the first statement that no
|
||||
// longer names it — beside a left-out condition on the same machine, which stays.
|
||||
func TestAWaitsNeedsOperatorStillClearsWhenItsStatementStopsNamingItBesideALeftOutOne(t *testing.T) {
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||
defer k.Close(context.Background())
|
||||
left := leftOutObservations("anchor", leftOutOf(leftOutNFS(t), nil, false))
|
||||
if err := k.Reconcile(t.Context(), probeLeftOutID, left); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waiting := map[string][]inventory.ResourceHealth{"notes": {{Module: "notes", Resource: "server", State: link.StateWaiting,
|
||||
Waits: []inventory.Wait{{Setting: "domain", What: "the domain it serves"}}}}}
|
||||
if err := judgeModuleHealth(t.Context(), nil, k, "anchor", waiting, map[string]int{"notes": 2}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitKey, leftKey := needsOperatorKey("notes", "anchor"), "module.nfs-server.anchor.left-out"
|
||||
open := func() map[string]conditions.Condition {
|
||||
t.Helper()
|
||||
list, err := k.Open(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := map[string]conditions.Condition{}
|
||||
for _, c := range list {
|
||||
out[c.Key] = c
|
||||
}
|
||||
return out
|
||||
}
|
||||
if c, raised := open()[waitKey]; !raised || c.Kind != kindNeedsOperator {
|
||||
t.Fatalf("the wait raised %+v", open())
|
||||
}
|
||||
if c := open()[leftKey]; c.Kind != kindNeedsOperator {
|
||||
t.Fatalf("the left-out condition is not open as needs-operator: %+v", open())
|
||||
}
|
||||
// D1 runs again and still finds nfs-server left out: its reconcile clears only what D1 raised, never the wait,
|
||||
// which the machine's statement raised.
|
||||
if err := k.Reconcile(t.Context(), probeLeftOutID, left); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, still := open()[waitKey]; !still {
|
||||
t.Fatalf("D1's reconcile cleared the wait's needs-operator: %+v", open())
|
||||
}
|
||||
if err := judgeModuleHealth(t.Context(), nil, k, "anchor", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, still := open()[waitKey]; still {
|
||||
t.Fatalf("the statement no longer names the wait, and its needs-operator is still open: %+v", open())
|
||||
}
|
||||
if _, still := open()[leftKey]; !still {
|
||||
t.Fatalf("the left-out condition was cleared by a health statement: %+v", open())
|
||||
}
|
||||
}
|
||||
@@ -1,398 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The tests of novox/hq issue 525's window skip: a merge waits for the merge window only when a batch assembling
|
||||
// holds a merge related to it, or its delivery group may have a member still to merge (ADR 0304 decision 1, ADR
|
||||
// 0306 decisions 2 and 3). Walks stay one at a time.
|
||||
|
||||
// A merge from no head that shares nothing, with no walk open, is cut into its walk when it is heard: no window.
|
||||
func TestALoneMergeSharingNothingIsCutAtOnce(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
hear(t, open, lone(repoMerge("one", "c1", t0)), t0)
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 0 || !ws[0].Open() || ws[0].Commit != "c1" {
|
||||
t.Fatalf("a lone merge sharing nothing: %d walk(s) %+v, %d batch(es)", len(ws), ws, len(bs))
|
||||
}
|
||||
if ws[0].Times == nil || ws[0].Times.WindowClosed == nil || !ws[0].Times.WindowClosed.Equal(t0) {
|
||||
t.Fatalf("its window closed at %+v; want when it was heard, %s", ws[0].Times, t0)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge related to the batch assembling joins it and waits for the window, which its coming restarts: by its
|
||||
// repository, by a module in common (a dependent of what the batch moves), by an edge of the module graph between
|
||||
// their modules (one that widens nothing), or by its delivery group's head.
|
||||
func TestARelatedMergeJoinsTheBatchAndWaitsForTheWindow(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
for _, c := range []string{"same repository", "a module in common", "an edge of the graph", "the same group head"} {
|
||||
t.Run(c, func(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "pr/c1")
|
||||
asksWithPaths(t)
|
||||
inv := open.inventory
|
||||
first := repoMerge("one", "c1", t0)
|
||||
second := lone(repoMerge("two", "c2", t0.Add(10*time.Second)))
|
||||
why := map[string]string{"same repository": "both change novox/one", "a module in common": "both move two",
|
||||
"an edge of the graph": "two depends on builder (built-by)",
|
||||
"the same group head": "both are members of group " + first.Head}[c]
|
||||
switch c {
|
||||
case "same repository":
|
||||
second = lone(repoMerge("one", "c2", t0.Add(10*time.Second)))
|
||||
case "a module in common":
|
||||
// two is built on one: a merge of one moves two too.
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "two", Version: "1",
|
||||
Build: &catalogue.Build{On: []catalogue.BuildsOn{{Arg: "BASE", Module: "one"}}}},
|
||||
inventory.Source{Repository: "novox/two", Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "an edge of the graph":
|
||||
// two is built by the build machine: an edge that orders and widens nothing.
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "builder", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: "mesh-build-machine", Scope: catalogue.ScopeMesh}}},
|
||||
inventory.Source{Repository: "novox/builder", Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first = repoMerge("builder", "c1", t0)
|
||||
case "the same group head":
|
||||
second = fromHead(repoMerge("two", "c2", t0.Add(10*time.Second)), first.Head)
|
||||
}
|
||||
hear(t, open, first, t0)
|
||||
said := captured(t, func() error {
|
||||
return (following{open: open}).hearMerge(ctx, second, t0.Add(10*time.Second))
|
||||
})
|
||||
if !strings.Contains(said, "("+why+")") {
|
||||
t.Fatalf("the merge was not said related by %q: %s", why, said)
|
||||
}
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 {
|
||||
t.Fatalf("the related merge: %d walk(s), batches %+v", len(ws), bs)
|
||||
}
|
||||
if bs[0].Delivery.Batch.CutAtOnce {
|
||||
t.Fatal("a batch a related merge joined is cut at once")
|
||||
}
|
||||
cutAt(t, open, t0.Add(99*time.Second))
|
||||
if ws, _ := walks(t, open); len(ws) != 0 {
|
||||
t.Fatalf("cut before the window the related merge restarted closed: %+v", ws)
|
||||
}
|
||||
cutAt(t, open, t0.Add(100*time.Second))
|
||||
ws, bs = walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 0 || len(ws[0].Delivery.Merges) != 2 {
|
||||
t.Fatalf("after the window: %d walk(s), %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A merge from a head whose delivery group has a member still to merge waits for the window, even with no batch
|
||||
// assembling, and says why; its sibling joins the same batch by their head (ADR 0306 decision 2). A merge from a
|
||||
// head that names no group shares nothing and is cut at once, as one from no head is.
|
||||
func TestAMergeOfAGroupWaitsEvenAlone(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
groupsWaiting(t, "feat/x")
|
||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/x"), t0)
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 0 || len(bs) != 1 || bs[0].Delivery.Batch.CutAtOnce || windowClosed(bs[0], t0) {
|
||||
t.Fatalf("a group's merge alone: %d walk(s), batches %+v", len(ws), bs)
|
||||
}
|
||||
if words := batchWords(bs[0], t0); !strings.HasPrefix(words, "assembling: 90 s left") ||
|
||||
!strings.Contains(words, "group feat/x has a member still to merge: novox/elsewhere (ready)") {
|
||||
t.Fatalf("it reads %q", words)
|
||||
}
|
||||
hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(5*time.Second)), "feat/x"), t0.Add(5*time.Second))
|
||||
if _, bs := walks(t, open); len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 {
|
||||
t.Fatalf("the group's second member did not join the first: %+v", bs)
|
||||
}
|
||||
cutAt(t, open, t0.Add(94*time.Second))
|
||||
if ws, _ := walks(t, open); len(ws) != 0 {
|
||||
t.Fatalf("cut before its window closed: %+v", ws)
|
||||
}
|
||||
cutAt(t, open, t0.Add(95*time.Second))
|
||||
ws, bs = walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 0 || ws[0].CommitOf("novox/one") != "c1" || ws[0].CommitOf("novox/two") != "c2" {
|
||||
t.Fatalf("at its window: %d walk(s) %+v, %d batch(es)", len(ws), ws, len(bs))
|
||||
}
|
||||
}
|
||||
|
||||
// A merge from a head that names no group, sharing nothing, with no walk open, is cut at once.
|
||||
func TestAHeadedMergeInNoGroupIsCutAtOnce(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "fix/alone"), t0)
|
||||
if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 || !ws[0].Open() {
|
||||
t.Fatalf("a headed merge in no group: %d walk(s), %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
}
|
||||
|
||||
// When mesh-delivery cannot say a head's group, the merge waits for the window, the safe side, and its batch says
|
||||
// why.
|
||||
func TestAGroupNobodyCanSayWaits(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
was := deliveryGroupOf
|
||||
deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) {
|
||||
return nil, false, errors.New("nothing serves mesh-delivery.groups")
|
||||
}
|
||||
t.Cleanup(func() { deliveryGroupOf = was })
|
||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/x"), t0)
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 0 || len(bs) != 1 || bs[0].Delivery.Batch.CutAtOnce {
|
||||
t.Fatalf("a merge whose group nobody could say: %d walk(s), batches %+v", len(ws), bs)
|
||||
}
|
||||
if words := batchWords(bs[0], t0); !strings.Contains(words, "mesh-delivery did not say whether group feat/x has a "+
|
||||
"member still to merge (nothing serves mesh-delivery.groups): it waits for the window, the safe side") {
|
||||
t.Fatalf("it reads %q", words)
|
||||
}
|
||||
}
|
||||
|
||||
// A member counts as still to merge only while its pull request is open, and not once this controller keeps its
|
||||
// merge: a group whose other members are merged or closed makes nobody wait.
|
||||
func TestAGroupsMergedOrClosedMembersMakeNobodyWait(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
members := []groupMember{{Repository: "novox/one", State: "ready"}, {Repository: "novox/two", State: "published"},
|
||||
{Repository: "novox/three", State: "stopped"}}
|
||||
was := deliveryGroupOf
|
||||
deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) { return members, true, nil }
|
||||
t.Cleanup(func() { deliveryGroupOf = was })
|
||||
m := fromHead(repoMerge("one", "c1", t0), "feat/x")
|
||||
if why, err := groupWait(ctx, open.inventory, m, true, false); err != nil || why != "" {
|
||||
t.Fatalf("the group's other members are merged or closed, yet: %q %v", why, err)
|
||||
}
|
||||
members = append(members, groupMember{Repository: "novox/four", State: "checked"})
|
||||
if why, _ := groupWait(ctx, open.inventory, m, true, false); !strings.Contains(why, "novox/four (checked)") {
|
||||
t.Fatalf("an open member: %q", why)
|
||||
}
|
||||
// novox/four's merge is kept in a batch already: merged, whatever mesh-delivery says yet.
|
||||
hear(t, open, fromHead(repoMerge("two", "c4", t0), "feat/x"), t0)
|
||||
members[3].Repository = "novox/two"
|
||||
if why, _ := groupWait(ctx, open.inventory, m, true, false); why != "" {
|
||||
t.Fatalf("a member this controller keeps the merge of still waits: %q", why)
|
||||
}
|
||||
if got := memberOf("novox/mesh-sdk@d41d255f28ad (held)"); got != (groupMember{Repository: "novox/mesh-sdk", State: "held"}) {
|
||||
t.Fatalf("a member reads %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge related to two batches assembling folds the newer into the older, which answers every merge of both
|
||||
// and names nothing twice; the folded one names the batch that took it over.
|
||||
func TestAMergeRelatedToTwoBatchesFoldsThem(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "feat/a", "feat/b")
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/a"), t0)
|
||||
hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(5*time.Second)), "feat/b"), t0.Add(5*time.Second))
|
||||
_, bs := walks(t, open)
|
||||
if len(bs) != 2 {
|
||||
t.Fatalf("two merges sharing nothing: %d batch(es)", len(bs))
|
||||
}
|
||||
older, newer := bs[1], bs[0] // newest first
|
||||
// Of one's repository and from two's head: related to both.
|
||||
hear(t, open, fromHead(repoMerge("one", "c3", t0.Add(10*time.Second)), "feat/b"), t0.Add(10*time.Second))
|
||||
_, bs = walks(t, open)
|
||||
if len(bs) != 1 || bs[0].ID != older.ID || len(bs[0].Delivery.Merges) != 3 {
|
||||
t.Fatalf("after a merge related to both: batches %+v; want %s answering three merges", bs, older.ID)
|
||||
}
|
||||
folded, err := open.inventory.PlanByID(ctx, newer.ID)
|
||||
if err != nil || folded.State != inventory.PlanSuperseded || folded.Delivery == nil ||
|
||||
folded.Delivery.TakenOverBy != older.ID {
|
||||
t.Fatalf("the newer batch reads %s %+v (%v)", folded.State, folded.Delivery, err)
|
||||
}
|
||||
if left, _ := open.inventory.MergesOf(ctx, newer.ID); len(left) != 0 {
|
||||
t.Fatalf("the folded batch still answers %+v", left)
|
||||
}
|
||||
cutAt(t, open, t0.Add(100*time.Second))
|
||||
ws, bs := walks(t, open)
|
||||
var cut []inventory.Plan
|
||||
for _, w := range ws {
|
||||
if w.State != inventory.PlanSuperseded {
|
||||
cut = append(cut, w)
|
||||
}
|
||||
}
|
||||
if len(cut) != 1 || len(bs) != 0 || cut[0].ID != older.ID || cut[0].CommitOf("novox/one") != "c3" ||
|
||||
cut[0].CommitOf("novox/two") != "c2" {
|
||||
t.Fatalf("the folded batch's walk: %+v, %d batch(es)", cut, len(bs))
|
||||
}
|
||||
}
|
||||
|
||||
// With a walk open, a batch cut at once still waits for it, said queued behind it (walks stay one at a time, ADR
|
||||
// 0276 decision 4); it is cut once that walk ended, with no window to wait for.
|
||||
func TestACutAtOnceBatchQueuesBehindAnOpenWalk(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "pr/c1")
|
||||
asksWithPaths(t)
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(90*time.Second))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 || !ws[0].Open() {
|
||||
t.Fatalf("no walk open: %+v", ws)
|
||||
}
|
||||
first := ws[0]
|
||||
heard := t0.Add(100 * time.Second)
|
||||
hear(t, open, lone(repoMerge("two", "c2", heard)), heard)
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != first.ID ||
|
||||
!bs[0].Delivery.Batch.CutAtOnce {
|
||||
t.Fatalf("a lone merge during a walk: %d walk(s), batches %+v", len(ws), bs)
|
||||
}
|
||||
if !strings.HasPrefix(batchWords(bs[0], heard), "queued behind "+first.ID) {
|
||||
t.Fatalf("it reads %q", batchWords(bs[0], heard))
|
||||
}
|
||||
first.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(t.Context(), &first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, heard.Add(time.Second))
|
||||
ws, bs = walks(t, open)
|
||||
if len(ws) != 2 || len(bs) != 0 || ws[0].Commit != "c2" {
|
||||
t.Fatalf("the queued batch was not cut once the walk ended: %+v %+v", ws, bs)
|
||||
}
|
||||
if ws[0].Times == nil || ws[0].Times.WindowClosed == nil || !ws[0].Times.WindowClosed.Equal(heard) {
|
||||
t.Fatalf("its window closed at %+v; want when its merge was heard, %s", ws[0].Times, heard)
|
||||
}
|
||||
}
|
||||
|
||||
// A stopped walk's other merges are walked again in a batch they are related to, and a batch of their own when
|
||||
// they share nothing with the one assembling (ADR 0299, ADR 0304 decision 1): never in a batch of another
|
||||
// change, whose window the stop would close.
|
||||
func TestAStopWalksTheOthersAgainInABatchOfTheirOwnWhenTheyShareNothing(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "pr/cat1aaaaaaaa")
|
||||
endsItsLine(t)
|
||||
asksRecorded(t)
|
||||
ctx := t.Context()
|
||||
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
|
||||
one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head)
|
||||
hear(t, open, cat, t0.Add(time.Second))
|
||||
hear(t, open, one, t0.Add(6*time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
walk := ws[0]
|
||||
hear(t, open, repoMerge("two", "two1cccccccc", t0.Add(3*time.Minute)), t0.Add(3*time.Minute))
|
||||
_, bs := walks(t, open)
|
||||
if len(bs) != 1 {
|
||||
t.Fatalf("the merge heard while the walk ran is in %d batches", len(bs))
|
||||
}
|
||||
other := bs[0]
|
||||
_, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped",
|
||||
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if said.WalkedAgainBy == "" || said.WalkedAgainBy == other.ID {
|
||||
t.Fatalf("walked again by %q; want a batch of its own, not %s", said.WalkedAgainBy, other.ID)
|
||||
}
|
||||
b, _ := open.inventory.PlanByID(ctx, other.ID)
|
||||
if b.Delivery.Batch == nil || b.Delivery.Batch.Closed || len(b.Delivery.Merges) != 1 {
|
||||
t.Fatalf("the other change's batch reads %+v; want it untouched by the stop", b.Delivery)
|
||||
}
|
||||
}
|
||||
|
||||
// A catalogue batch's cut folds in every walk waiting for its delivery's word (cutBatch's carry), so what that
|
||||
// walk moves is the batch's too: a lone merge related to the waiting walk joins the batch assembling and waits for
|
||||
// its window, rather than being cut at once as sharing nothing with what the batch's own merges move.
|
||||
func TestALoneMergeRelatedToAWalkTheCutFoldsJoinsTheBatch(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "pr/c3")
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(90*time.Second))
|
||||
if ws, _ := walks(t, open); len(ws) != 1 || !ws[0].Waiting() {
|
||||
t.Fatalf("the walk does not wait for its word: %+v", ws)
|
||||
}
|
||||
hear(t, open, repoMerge("three", "c3", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
|
||||
hear(t, open, lone(repoMerge("one", "c2", t0.Add(2*time.Minute+5*time.Second))), t0.Add(2*time.Minute+5*time.Second))
|
||||
_, bs := walks(t, open)
|
||||
if len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 || bs[0].Delivery.Batch.CutAtOnce {
|
||||
t.Fatalf("a lone merge related to the waiting walk: batches %+v", bs)
|
||||
}
|
||||
}
|
||||
|
||||
// A delivery group with a member of each kind (review of #250): a merge on the controller's own path and a
|
||||
// catalogue merge never share a batch, so the later member cannot join the earlier one's. Kept in a batch of the
|
||||
// other kind, the earlier member keeps the later one waiting, whatever mesh-delivery says yet — never cut at once
|
||||
// to walk before it. The two batches are then cut in the order their windows close (cutBatchesHeld: the oldest
|
||||
// closed batch first, an own-path one first at one look, and the other queued behind a walk that started): the
|
||||
// earlier member's window closes first, so it walks first, in both orders.
|
||||
func TestAGroupAcrossBothKindsKeepsItsOrder(t *testing.T) {
|
||||
for _, ownFirst := range []bool{true, false} {
|
||||
name := "catalogue member first"
|
||||
if ownFirst {
|
||||
name = "controller member first"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
was := deliveryGroupOf
|
||||
// mesh-delivery still says both members ready: it marks one merged only once its walk opened.
|
||||
deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) {
|
||||
return []groupMember{{Repository: "novox/mesh-controller", State: "ready"},
|
||||
{Repository: "novox/mesh-catalog", State: "ready"}}, true, nil
|
||||
}
|
||||
t.Cleanup(func() { deliveryGroupOf = was })
|
||||
own := fromHead(repoMerge("mesh-controller", "k1", t0), "feat/both-kinds")
|
||||
cat := fromHead(catalogueMerge("cat1aaaaaaaa", "app", t0), "feat/both-kinds")
|
||||
first, second := cat, own
|
||||
if ownFirst {
|
||||
first, second = own, cat
|
||||
}
|
||||
second.MergedAt = t0.Add(10 * time.Second).Format(time.RFC3339Nano)
|
||||
hear(t, open, first, t0)
|
||||
hear(t, open, second, t0.Add(10*time.Second))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 0 || len(bs) != 2 {
|
||||
t.Fatalf("both members heard: %d walk(s), %d batch(es); want two batches, nothing cut", len(ws), len(bs))
|
||||
}
|
||||
later := bs[0] // newest first
|
||||
if later.Delivery.Batch.CutAtOnce || !strings.Contains(later.Delivery.Batch.Waits,
|
||||
"has a member waiting in a batch of the other kind: novox/"+first.Repo+"@") {
|
||||
t.Fatalf("the later member's batch reads %+v; want it waiting for the earlier member", later.Delivery.Batch)
|
||||
}
|
||||
// The earlier member's window closes first: its batch alone is cut.
|
||||
cutAt(t, open, t0.Add(90*time.Second))
|
||||
ws, bs = walks(t, open)
|
||||
if len(ws) != 1 || !ws[0].Open() || ws[0].CommitOf("novox/"+first.Repo) != first.Commit || len(bs) != 1 ||
|
||||
bs[0].ID != later.ID {
|
||||
t.Fatalf("at the earlier window: walks %+v, batches %+v; want the earlier member alone cut", ws, bs)
|
||||
}
|
||||
earlier := ws[0]
|
||||
// The later member's window closes while the earlier walk runs: it queues behind it.
|
||||
cutAt(t, open, t0.Add(100*time.Second))
|
||||
if _, bs := walks(t, open); len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != earlier.ID {
|
||||
t.Fatalf("the later member's batch does not queue behind the earlier walk: %+v", bs)
|
||||
}
|
||||
earlier.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &earlier); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(101*time.Second))
|
||||
got, err := open.inventory.PlanByID(ctx, later.ID)
|
||||
if err != nil || got.Batch() || got.CommitOf("novox/"+second.Repo) != second.Commit {
|
||||
t.Fatalf("the later member was not walked after the earlier: %s %v", got.State, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -98,9 +98,8 @@ func judgeNetworks(ctx context.Context, inv *inventory.Inventory, k *conditions.
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
}
|
||||
undecided := undecidedParts(f)
|
||||
for _, c := range open {
|
||||
if !slices.Contains(networkKinds, c.Kind) || said[c.Key] || heldUndecided(c, undecided) {
|
||||
if !slices.Contains(networkKinds, c.Kind) || said[c.Key] {
|
||||
continue
|
||||
}
|
||||
why := "no machine says it any more"
|
||||
@@ -117,59 +116,6 @@ func judgeNetworks(ctx context.Context, inv *inventory.Inventory, k *conditions.
|
||||
return nil
|
||||
}
|
||||
|
||||
// undecidedParts is, per machine, every part of its newest statement not yet judged: starting, or unknown
|
||||
// — one look failed and a second decides (novox/hq issue 348). Such a part does not say its fault is gone.
|
||||
// A statement whose parts are all decided but whose whole is unknown or starting holds every part. Pure.
|
||||
//
|
||||
// On 2026-10-09 the control node's resolver refused every question from 10:58 to 11:18 UTC. A build of
|
||||
// the node-engine sent at 10:59:34 restarted it; its first statement judged the names once (unknown, "one
|
||||
// look failed; a second decides"), and that statement cleared the control node's network condition while
|
||||
// its last evidence still said "connection refused". The second look raised it again forty seconds later —
|
||||
// after the send — and the gate failed the build for a fault from before it.
|
||||
func undecidedParts(f networkFacts) map[string]map[string]bool {
|
||||
out := map[string]map[string]bool{}
|
||||
for m, h := range f.healths {
|
||||
if h.Network == nil {
|
||||
continue
|
||||
}
|
||||
parts := map[string]bool{}
|
||||
for _, p := range h.Network.Parts {
|
||||
if p.State == link.StateUnknown || p.State == link.StateStarting {
|
||||
parts[p.Part] = true
|
||||
}
|
||||
}
|
||||
if len(parts) == 0 && (h.Network.State == link.StateUnknown || h.Network.State == link.StateStarting) {
|
||||
parts["*"] = true
|
||||
}
|
||||
if len(parts) > 0 {
|
||||
out[m] = parts
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// heldUndecided says an open network condition is kept rather than cleared: a part its newest evidence
|
||||
// names is undecided in the newest statement of a machine it is about. A condition about other parts
|
||||
// clears as before. Pure.
|
||||
func heldUndecided(c conditions.Condition, undecided map[string]map[string]bool) bool {
|
||||
said := ""
|
||||
if len(c.Evidence) > 0 {
|
||||
said = c.Evidence[0].Said
|
||||
}
|
||||
for _, m := range append([]string{c.Subject.Machine}, c.Subject.Also...) {
|
||||
parts := undecided[m]
|
||||
if parts["*"] {
|
||||
return true
|
||||
}
|
||||
for part := range parts {
|
||||
if strings.Contains(said, part+" since ") || strings.Contains(said, part+": ") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// pointed is one machine's failing part that points at another machine.
|
||||
type pointed struct {
|
||||
from string
|
||||
|
||||
@@ -63,15 +63,6 @@ func run() error {
|
||||
// 45 §6), so the next controller takes it at once rather than after its age.
|
||||
defer theLease.release()
|
||||
|
||||
// Every walk a verb keeps is said on the bus, whichever verb kept it (novox/hq ADR 0308 decision 1): the save
|
||||
// that starts or ends a walk keeps its mark, and no later save says it again. The serving controller says its
|
||||
// own saves.
|
||||
if args[0] != "serve" {
|
||||
said, done := sayingWhatAVerbKept(ctx)
|
||||
inventory.PlanSaved = said
|
||||
defer done()
|
||||
}
|
||||
|
||||
switch args[0] {
|
||||
case "build":
|
||||
return buildCommand(ctx, args[1:])
|
||||
@@ -87,8 +78,6 @@ func run() error {
|
||||
return rotateCommand(ctx, args[1:])
|
||||
case "ask":
|
||||
return askCommand(ctx, args[1:])
|
||||
case "rehearse":
|
||||
return rehearseCommand(ctx, args[1:])
|
||||
case "builds":
|
||||
return buildsCommand(ctx, args[1:])
|
||||
// The build queue, controlled by hand (novox/hq ADR 0219).
|
||||
@@ -320,7 +309,7 @@ func usage() {
|
||||
the self-check: the last verdict, a run now, the probes, the signals' ages
|
||||
healers [--days N] [--json] the healers, what they did lately, and their brake (to-be 45 §7)
|
||||
durations [--kind K] [--days N] [--json]
|
||||
apply, heartbeat, plan-tier, build and walk-phase durations
|
||||
apply, heartbeat, plan-tier and build durations, per machine or module
|
||||
collection [--json] kept archives held/unheld by a manifest, and what the sweep may let go
|
||||
builder issue <name> a broker account for a build machine, scoped to build work,
|
||||
delivered as the builder module's broker secret (module add it first)
|
||||
|
||||
@@ -1204,18 +1204,7 @@ func graphOfFacts(f snapshot.Facts) ([]inventory.Entry, map[string][]inventory.R
|
||||
entries = append(entries, inventory.Entry{Manifest: manifest, Provided: mod.Provided,
|
||||
Source: inventory.Source{Repository: mod.Repository, Path: mod.Path, BuiltFrom: mod.Commit}})
|
||||
for _, r := range mod.Reads {
|
||||
entry := inventory.ReadRepository{Repository: r}
|
||||
for _, s := range mod.Sources {
|
||||
if !s.Own && s.Repository == r && len(s.Paths) > 0 {
|
||||
entry.Paths = s.Paths
|
||||
}
|
||||
}
|
||||
read[mod.Name] = append(read[mod.Name], entry)
|
||||
}
|
||||
for _, s := range mod.Sources {
|
||||
if s.Own && len(s.Paths) > 0 {
|
||||
read[mod.Name] = append(read[mod.Name], inventory.ReadRepository{Own: true, Paths: s.Paths})
|
||||
}
|
||||
read[mod.Name] = append(read[mod.Name], inventory.ReadRepository{Repository: r})
|
||||
}
|
||||
}
|
||||
var edges []inventory.Edge
|
||||
|
||||
@@ -390,10 +390,9 @@ func TestAWithheldPathIsStoodInForByAPath(t *testing.T) {
|
||||
// controller's ask and by the facts — and finds the repository it checks from its origin.
|
||||
func TestACheckByHandClonesWhatTheSeatClones(t *testing.T) {
|
||||
for dir, refs := range map[string]map[string]string{
|
||||
"mesh-catalog": {"mesh-catalog": "main"},
|
||||
"mesh-host": {"mesh-host": "c0ffee"},
|
||||
"mesh-controller": {"mesh-controller": "c0ffee", "mesh-controller-main": "main", "mesh-lab": "main",
|
||||
"mesh-sdk": sdkPinnedByGoMod(t)},
|
||||
"mesh-catalog": {"mesh-catalog": "main"},
|
||||
"mesh-host": {"mesh-host": "c0ffee"},
|
||||
"mesh-controller": {"mesh-controller": "c0ffee", "mesh-controller-main": "main", "mesh-lab": "main"},
|
||||
} {
|
||||
got := besideRefs(dir, "c0ffee")
|
||||
for d, ref := range refs {
|
||||
@@ -413,24 +412,3 @@ func TestACheckByHandClonesWhatTheSeatClones(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sdkPinnedByGoMod is the SDK commit this tree's go.mod pins, read from the file rather than the build, so
|
||||
// sdkPinned is held to what the repository says (novox/hq issue 449).
|
||||
func sdkPinnedByGoMod(t *testing.T) string {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Join("..", "..", "go.mod"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, line := range strings.Split(string(raw), "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) >= 2 && fields[0] == sdkModule {
|
||||
if m := pseudoCommit.FindStringSubmatch(fields[1]); m != nil {
|
||||
return m[1]
|
||||
}
|
||||
return "go/" + fields[1]
|
||||
}
|
||||
}
|
||||
t.Fatalf("go.mod requires no %s", sdkModule)
|
||||
return ""
|
||||
}
|
||||
|
||||
@@ -141,12 +141,6 @@ func runForMeshCLI(ctx context.Context, node string, asked link.CLIAsked, v cliV
|
||||
if v.refused != "" {
|
||||
return link.CLIRefusal(v.refused)
|
||||
}
|
||||
// Standard input is the terminal's alone: a secret given at the terminal reaches `secret accept`, and no ordinary
|
||||
// call is handed what the asker's standard input held (novox/hq ADR 0259 §10, ADR 0272).
|
||||
if len(asked.Stdin) > 0 && !v.terminal {
|
||||
return link.CLIAnswer{Exit: 1, Why: v.why, Refused: "standard input is given to a line that runs as the " +
|
||||
"controller's terminal alone, and this one does not. Nothing ran"}
|
||||
}
|
||||
if cliServers[asked.Line[0]] {
|
||||
return link.CLIAnswer{Exit: 1, Why: v.why, Refused: fmt.Sprintf("%s serves until stopped, and is not a "+
|
||||
"command line mesh-cli runs. Nothing ran", asked.Line[0])}
|
||||
@@ -161,12 +155,8 @@ func runForMeshCLI(ctx context.Context, node string, asked link.CLIAsked, v cliV
|
||||
}
|
||||
cmd := selfCommand(ctx, line)
|
||||
cmd.Env = commandEnvironment(fmt.Sprintf("%s through mesh-cli on %s", asked.Account, node), verb, v.terminal)
|
||||
// No standard input unless mesh-cli carried one for a terminal line: a command that reads one gets nothing, and
|
||||
// fails saying so (ADR 0272 §5).
|
||||
// No standard input: a command that reads one gets nothing, and fails saying so (ADR 0272 §5).
|
||||
cmd.Stdin = nil
|
||||
if len(asked.Stdin) > 0 {
|
||||
cmd.Stdin = bytes.NewReader(asked.Stdin)
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout, cmd.Stderr = &stdout, &stderr
|
||||
err := cmd.Run()
|
||||
|
||||
@@ -1,99 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// secretAcceptWants makes the test binary, run as a command line, read a secret as `secret accept` reads it and
|
||||
// say whether it is the value whose SHA-256 the variable names (TestMain).
|
||||
const secretAcceptWants = "MESH_TEST_SECRET_ACCEPT_WANTS"
|
||||
|
||||
// readAsSecretAccept is that process: `secret accept <node> <module> <name> [--from -]`, the value read by
|
||||
// valueFor, compared by digest, and only the verdict printed.
|
||||
func readAsSecretAccept(want string, argv []string) int {
|
||||
if len(argv) < 5 || argv[0] != "secret" || argv[1] != "accept" {
|
||||
fmt.Printf("not a secret accept line: %q\n", argv)
|
||||
return 2
|
||||
}
|
||||
from := ""
|
||||
if len(argv) == 7 && argv[5] == "--from" {
|
||||
from = argv[6]
|
||||
}
|
||||
value, err := valueFor(argv[2], argv[3], argv[4], from)
|
||||
if err != nil {
|
||||
fmt.Printf("secret accept read nothing: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
sum := sha256.Sum256([]byte(asSupplied(value)))
|
||||
if hex.EncodeToString(sum[:]) != want {
|
||||
fmt.Printf("secret accept read something else (%d bytes)\n", len(value))
|
||||
return 1
|
||||
}
|
||||
fmt.Println("secret accept read the value it was given")
|
||||
return 0
|
||||
}
|
||||
|
||||
// novox/hq ADR 0259 §10, ADR 0272: what mesh-cli's standard input held reaches `secret accept` on a line that runs
|
||||
// as the controller's terminal, and appears nowhere else — not in the answer, not in the journal, not in the calls
|
||||
// record; an ordinary line carrying it is refused and nothing runs.
|
||||
func TestStandardInputReachesSecretAcceptAtTheTerminalAndNowhereElse(t *testing.T) {
|
||||
token := "123456789:AAEhBOweik6ad9r_QxGivenAtTheTerminal"
|
||||
sum := sha256.Sum256([]byte(token))
|
||||
t.Setenv(secretAcceptWants, hex.EncodeToString(sum[:]))
|
||||
var journal []string
|
||||
var mu sync.Mutex
|
||||
was := cliJournal
|
||||
cliJournal = func(line string) { mu.Lock(); journal = append(journal, line); mu.Unlock() }
|
||||
t.Cleanup(func() { cliJournal = was })
|
||||
ctx := context.Background()
|
||||
|
||||
for _, line := range [][]string{
|
||||
{"secret", "accept", "anchor", "telegram", "telegram-token", "--from", "-"},
|
||||
{"secret", "accept", "anchor", "telegram", "telegram-token"}, // the prompt's path, a line on standard input
|
||||
} {
|
||||
asked := cliAsked("operator", 1000, line...)
|
||||
asked.Stdin = []byte(token + "\n")
|
||||
a := runForMeshCLI(ctx, "control", asked, cliVerdict{terminal: true, why: "the terminal"})
|
||||
if a.Exit != 0 || !strings.Contains(string(a.Stdout), "read the value it was given") {
|
||||
t.Fatalf("%q: secret accept did not read what mesh-cli carried: %+v (%s)", line, a, a.Stdout)
|
||||
}
|
||||
if body, _ := json.Marshal(a); strings.Contains(string(body), "AAEh") || strings.Contains(string(body), base64.StdEncoding.EncodeToString([]byte(token))) {
|
||||
t.Fatalf("the answer carries the secret: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
// Without standard input, the line reads nothing, as before.
|
||||
a := runForMeshCLI(ctx, "control", cliAsked("operator", 1000, "secret", "accept", "anchor", "telegram",
|
||||
"telegram-token", "--from", "-"), cliVerdict{terminal: true, why: "the terminal"})
|
||||
if a.Exit == 0 {
|
||||
t.Fatalf("a line with no standard input read a value: %+v", a)
|
||||
}
|
||||
|
||||
// An ordinary call is never handed it: refused, and nothing ran.
|
||||
asked := cliAsked("operator", 1000, "status")
|
||||
asked.Stdin = []byte(token)
|
||||
a = runForMeshCLI(ctx, "laptop", asked, cliVerdict{why: "not the terminal"})
|
||||
if a.Exit == 0 || !strings.Contains(a.Refused, "terminal alone") || len(a.Stdout) != 0 {
|
||||
t.Fatalf("an ordinary line was given standard input: %+v", a)
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
for _, l := range journal {
|
||||
if strings.Contains(l, "AAEh") {
|
||||
t.Fatalf("the journal says the secret: %s", l)
|
||||
}
|
||||
}
|
||||
if len(journal) == 0 {
|
||||
t.Fatal("the lines were not said in the journal at all")
|
||||
}
|
||||
|
||||
}
|
||||
@@ -24,7 +24,7 @@ var cliNodes = []inventory.Node{
|
||||
{Name: "unnamed"},
|
||||
}
|
||||
|
||||
func cliAsked(account string, uid uint32, line ...string) link.CLIAsked {
|
||||
func asked(account string, uid uint32, line ...string) link.CLIAsked {
|
||||
return link.CLIAsked{Line: line, Account: account, UID: uid, Session: "session-1.scope"}
|
||||
}
|
||||
|
||||
@@ -39,13 +39,13 @@ func TestMeshCLIIsTheTerminalOnlyForTheControlNodesOperator(t *testing.T) {
|
||||
refused string
|
||||
why string
|
||||
}{
|
||||
{"the control-node's operator", "control", cliAsked("operator", 1000, "status"), control, true, "", "the controller's terminal"},
|
||||
{"another node's operator", "laptop", cliAsked("operator", 1000, "status"), control, false, "", "agents on laptop may run as operator"},
|
||||
{"another account", "control", cliAsked("agent", 1001, "status"), control, false, "operator account (operator) only", ""},
|
||||
{"root", "control", cliAsked("root", 0, "status"), control, false, "never root", ""},
|
||||
{"a node with no operator account", "unnamed", cliAsked("operator", 1000, "status"), control, false, "does not know unnamed's operator account", ""},
|
||||
{"a node the mesh does not know", "elsewhere", cliAsked("operator", 1000, "status"), control, false, "not a node this mesh knows", ""},
|
||||
{"two control-nodes", "control", cliAsked("operator", 1000, "status"), []string{"control", "laptop"}, false, "", "2 control-nodes"},
|
||||
{"the control-node's operator", "control", asked("operator", 1000, "status"), control, true, "", "the controller's terminal"},
|
||||
{"another node's operator", "laptop", asked("operator", 1000, "status"), control, false, "", "agents on laptop may run as operator"},
|
||||
{"another account", "control", asked("agent", 1001, "status"), control, false, "operator account (operator) only", ""},
|
||||
{"root", "control", asked("root", 0, "status"), control, false, "never root", ""},
|
||||
{"a node with no operator account", "unnamed", asked("operator", 1000, "status"), control, false, "does not know unnamed's operator account", ""},
|
||||
{"a node the mesh does not know", "elsewhere", asked("operator", 1000, "status"), control, false, "not a node this mesh knows", ""},
|
||||
{"two control-nodes", "control", asked("operator", 1000, "status"), []string{"control", "laptop"}, false, "", "2 control-nodes"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
v := judgeCLI(c.node, c.asked, cliNodes, c.control)
|
||||
@@ -70,7 +70,7 @@ func TestTheTerminalRunsWithoutAVerbAndAnOrdinaryCallNamesMeshCLI(t *testing.T)
|
||||
t.Setenv(servedVar, "1")
|
||||
ctx := context.Background()
|
||||
|
||||
a := runForMeshCLI(ctx, "control", cliAsked("operator", 1000, "status"), cliVerdict{terminal: true, why: "the terminal"})
|
||||
a := runForMeshCLI(ctx, "control", asked("operator", 1000, "status"), cliVerdict{terminal: true, why: "the terminal"})
|
||||
if a.Exit != 0 || a.Refused != "" || !a.Terminal {
|
||||
t.Fatalf("the terminal's line did not run: %+v", a)
|
||||
}
|
||||
@@ -79,7 +79,7 @@ func TestTheTerminalRunsWithoutAVerbAndAnOrdinaryCallNamesMeshCLI(t *testing.T)
|
||||
t.Fatalf("the terminal's line ran with %s", got)
|
||||
}
|
||||
|
||||
a = runForMeshCLI(ctx, "laptop", cliAsked("operator", 1000, "status"), cliVerdict{why: "not the terminal"})
|
||||
a = runForMeshCLI(ctx, "laptop", asked("operator", 1000, "status"), cliVerdict{why: "not the terminal"})
|
||||
if a.Exit != 0 || a.Terminal || a.Why != "not the terminal" {
|
||||
t.Fatalf("an ordinary line did not run as one: %+v", a)
|
||||
}
|
||||
@@ -93,21 +93,21 @@ func TestAnOrdinaryCallMeetsTheCommandVerbsRefusals(t *testing.T) {
|
||||
t.Setenv(echoEnvironment, "1")
|
||||
ctx := context.Background()
|
||||
ordinary := cliVerdict{why: "not the terminal"}
|
||||
a := runForMeshCLI(ctx, "laptop", cliAsked("operator", 1000, "cleanup", "delete", "x"), ordinary)
|
||||
a := runForMeshCLI(ctx, "laptop", asked("operator", 1000, "cleanup", "delete", "x"), ordinary)
|
||||
if a.Refused == "" || len(a.Stdout) != 0 || a.Exit != 1 || a.Why != "not the terminal" {
|
||||
t.Fatalf("a repair without --why ran as an ordinary call: %+v", a)
|
||||
}
|
||||
a = runForMeshCLI(ctx, "laptop", cliAsked("operator", 1000, "settings", "set", "claude-code", "{}"), ordinary)
|
||||
a = runForMeshCLI(ctx, "laptop", asked("operator", 1000, "settings", "set", "claude-code", "{}"), ordinary)
|
||||
if a.Refused != "" || !strings.Contains(string(a.Stdout), `verb="mesh-cli"`) {
|
||||
t.Fatalf("an ordinary settings set did not run through the settings verb's path with MESH_VERB set: %+v", a)
|
||||
}
|
||||
for _, server := range []string{"serve", "api", "board"} {
|
||||
a := runForMeshCLI(ctx, "control", cliAsked("operator", 1000, server), cliVerdict{terminal: true})
|
||||
a := runForMeshCLI(ctx, "control", asked("operator", 1000, server), cliVerdict{terminal: true})
|
||||
if a.Refused == "" || len(a.Stdout) != 0 {
|
||||
t.Fatalf("%s was run for mesh-cli: %+v", server, a)
|
||||
}
|
||||
}
|
||||
a = runForMeshCLI(ctx, "control", cliAsked("agent", 1001, "status"), cliVerdict{refused: "agent is not answered"})
|
||||
a = runForMeshCLI(ctx, "control", asked("agent", 1001, "status"), cliVerdict{refused: "agent is not answered"})
|
||||
if a.Refused != "agent is not answered" || len(a.Stdout) != 0 {
|
||||
t.Fatalf("a refused line ran: %+v", a)
|
||||
}
|
||||
@@ -184,9 +184,9 @@ func TestEveryMeshCLILineIsSaidInTheJournal(t *testing.T) {
|
||||
cliJournal = func(line string) { said = append(said, line) }
|
||||
t.Cleanup(func() { cliJournal = was })
|
||||
ctx := link.WithCallID(context.Background(), "call-1")
|
||||
runForMeshCLI(ctx, "control", cliAsked("operator", 1000, "settings", "set", "x", `{"password":"s3cret"}`),
|
||||
runForMeshCLI(ctx, "control", asked("operator", 1000, "settings", "set", "x", `{"password":"s3cret"}`),
|
||||
cliVerdict{terminal: true, why: "the terminal"})
|
||||
runForMeshCLI(ctx, "control", cliAsked("agent", 1001, "status"), cliVerdict{refused: "agent is not answered"})
|
||||
runForMeshCLI(ctx, "control", asked("agent", 1001, "status"), cliVerdict{refused: "agent is not answered"})
|
||||
all := strings.Join(said, "\n")
|
||||
if len(said) != 2 || !strings.Contains(all, "call-1") || !strings.Contains(all, "operator on control") ||
|
||||
!strings.Contains(all, "as the controller's terminal") || !strings.Contains(all, "refused") {
|
||||
@@ -213,7 +213,7 @@ func TestAnOrdinaryLineRunsNothingTheCommandVerbWouldRefuse(t *testing.T) {
|
||||
if _, err := ordinaryLine(line); err == nil {
|
||||
t.Errorf("%q composed as an ordinary line", line)
|
||||
}
|
||||
a := runForMeshCLI(context.Background(), "laptop", cliAsked("operator", 1000, line...), cliVerdict{why: "not the terminal"})
|
||||
a := runForMeshCLI(context.Background(), "laptop", asked("operator", 1000, line...), cliVerdict{why: "not the terminal"})
|
||||
if a.Refused == "" || len(a.Stdout) != 0 {
|
||||
t.Errorf("%q ran as an ordinary line: %+v", line, a)
|
||||
}
|
||||
@@ -279,11 +279,11 @@ func TestTheTerminalsMarkIsStrippedFromEveryOtherLine(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
a := runForMeshCLI(context.Background(), "laptop", cliAsked("operator", 1000, "status"), cliVerdict{why: "not the terminal"})
|
||||
a := runForMeshCLI(context.Background(), "laptop", asked("operator", 1000, "status"), cliVerdict{why: "not the terminal"})
|
||||
if got := string(a.Stdout); !strings.Contains(got, "terminal=false") || !strings.Contains(got, `verb="mesh-cli"`) {
|
||||
t.Fatalf("an ordinary line with the mark in the serving environment ran as %s", got)
|
||||
}
|
||||
a = runForMeshCLI(context.Background(), "control", cliAsked("operator", 1000, "status"), cliVerdict{terminal: true})
|
||||
a = runForMeshCLI(context.Background(), "control", asked("operator", 1000, "status"), cliVerdict{terminal: true})
|
||||
if got := string(a.Stdout); !strings.Contains(got, "terminal=true") {
|
||||
t.Fatalf("the terminal's line ran as %s", got)
|
||||
}
|
||||
|
||||
@@ -40,50 +40,6 @@ type merges interface {
|
||||
AnnouncedMerges(ctx context.Context, since time.Time) ([]link.AnnouncedMerge, error)
|
||||
}
|
||||
|
||||
// unheardMerges is the announcements of merges the controller has not heard (novox/hq ADR 0276): a merge kept
|
||||
// in a batch is not acted on until its batch is cut, which can be past mergeGrace behind a long walk, and is
|
||||
// not missed for that.
|
||||
type unheardMerges struct {
|
||||
merges
|
||||
inv *inventory.Inventory
|
||||
}
|
||||
|
||||
func (u unheardMerges) AnnouncedMerges(ctx context.Context, since time.Time) ([]link.AnnouncedMerge, error) {
|
||||
all, err := u.merges.AnnouncedMerges(ctx, since)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []link.AnnouncedMerge
|
||||
for _, a := range all {
|
||||
_, kept, err := u.inv.MergeOf(ctx, a.Owner+"/"+a.Repo, a.Commit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !kept {
|
||||
out = append(out, a)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// owedToTheRecord says a merge read as history is still owed to the record (novox/hq ADR 0276): the merges
|
||||
// reader knows, when it keeps them (unheardMerges).
|
||||
func owedToTheRecord(ctx context.Context, announced merges, m link.SourceMoved, entries []inventory.Entry,
|
||||
read map[string][]inventory.ReadRepository) bool {
|
||||
u, ok := announced.(unheardMerges)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
merged, err := time.Parse(time.RFC3339Nano, m.MergedAt)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
raw := m
|
||||
raw.MergedAt = ""
|
||||
owed, err := owedLate(ctx, u.inv, m, merged, wouldMove(raw, entries, read))
|
||||
return err == nil && owed
|
||||
}
|
||||
|
||||
// catchingUpOnMerges reads back the forge's announcements on a timer, until the context ends.
|
||||
func catchingUpOnMerges(ctx context.Context, open *stores, announced merges) {
|
||||
f := following{open}
|
||||
@@ -92,7 +48,7 @@ func catchingUpOnMerges(ctx context.Context, open *stores, announced merges) {
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
read, err := readForPlanning(ctx, open.inventory)
|
||||
read, err := open.inventory.ReadRepositories(ctx)
|
||||
return entries, read, err
|
||||
}
|
||||
failing := ""
|
||||
@@ -105,7 +61,7 @@ func catchingUpOnMerges(ctx context.Context, open *stores, announced merges) {
|
||||
case <-tick.C:
|
||||
}
|
||||
watchedMerges.begin()
|
||||
err := catchUpOnMerges(ctx, time.Now(), unheardMerges{announced, open.inventory}, catalogued, f.SourceMoved, func(format string, args ...any) {
|
||||
err := catchUpOnMerges(ctx, time.Now(), announced, catalogued, f.SourceMoved, func(format string, args ...any) {
|
||||
fmt.Printf(format+"\n", args...)
|
||||
})
|
||||
// What the pass found is what S5 says (novox/hq to-be 45 §3); a pass that could not read
|
||||
@@ -146,15 +102,10 @@ func catchUpOnMerges(ctx context.Context, now time.Time, announced merges,
|
||||
return err
|
||||
}
|
||||
for _, a := range all {
|
||||
// A merge the forge said no time of is dated by its announcement, so a packaging module's look can
|
||||
// make it history once acted on, and the catch-up does not act on it again every pass (ADR 0267).
|
||||
if a.SourceMoved.MergedAt == "" && !a.At.IsZero() {
|
||||
a.SourceMoved.MergedAt = a.At.UTC().Format(time.RFC3339)
|
||||
}
|
||||
if now.Sub(a.At) < mergeGrace {
|
||||
continue
|
||||
}
|
||||
if len(wouldMove(a.SourceMoved, entries, read)) == 0 && !owedToTheRecord(ctx, announced, a.SourceMoved, entries, read) {
|
||||
if len(wouldMove(a.SourceMoved, entries, read)) == 0 {
|
||||
continue
|
||||
}
|
||||
if entries, read, err = catalogued(ctx); err != nil {
|
||||
@@ -162,12 +113,7 @@ func catchUpOnMerges(ctx context.Context, now time.Time, announced merges,
|
||||
}
|
||||
moves := wouldMove(a.SourceMoved, entries, read)
|
||||
if len(moves) == 0 {
|
||||
if !owedToTheRecord(ctx, announced, a.SourceMoved, entries, read) {
|
||||
continue
|
||||
}
|
||||
raw := a.SourceMoved
|
||||
raw.MergedAt = ""
|
||||
moves = wouldMove(raw, entries, read)
|
||||
continue
|
||||
}
|
||||
var names []string
|
||||
for _, e := range moves {
|
||||
|
||||
@@ -74,21 +74,9 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke
|
||||
kept := inventory.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target,
|
||||
State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts,
|
||||
Check: r.Check, Needs: r.Needs, Account: r.Account, Root: r.Root}
|
||||
for _, w := range r.Waits {
|
||||
kept.Waits = append(kept.Waits, inventory.Wait{Part: w.Part, Secret: w.Secret, Setting: w.Setting, What: w.What})
|
||||
}
|
||||
resources = append(resources, kept)
|
||||
}
|
||||
// **A wait for the operator is checked before it is excused** (novox/hq ADR 0283): a waiting resource whose
|
||||
// wait does not check out is judged unhealthy, saying why; one that does is kept beside the unhealthy ones, so
|
||||
// judgeModuleHealth can say it as needs-operator. What is stored is what the machine said.
|
||||
var wf operatorWaitFacts
|
||||
if mods := waitingModules(resources); len(mods) > 0 {
|
||||
readWaitFacts(ctx, inv, node, mods, nil, &wf)
|
||||
}
|
||||
for _, r := range checkWaiting(node, resources, wf) {
|
||||
if (r.State == link.StateUnhealthy || r.State == link.StateWaiting) && r.Module != "" {
|
||||
unhealthy[r.Module] = append(unhealthy[r.Module], r)
|
||||
if r.State == link.StateUnhealthy && r.Module != "" {
|
||||
unhealthy[r.Module] = append(unhealthy[r.Module], kept)
|
||||
}
|
||||
}
|
||||
streaks := map[string]int{}
|
||||
@@ -147,17 +135,16 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
}
|
||||
standing := map[string]conditions.Condition{}
|
||||
for _, c := range open {
|
||||
// A module left out of the composition is the self-check's to raise and clear, never a statement's
|
||||
// (novox/hq issue 380): its needs-operator is not cleared for not being in what the machine runs.
|
||||
if (c.Kind == kindModuleUnhealthy || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound ||
|
||||
c.Kind == kindNeedsOperator) &&
|
||||
c.Subject.Machine == node && !isLeftOutCondition(c) {
|
||||
if (c.Kind == kindModuleUnhealthy || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound) &&
|
||||
c.Subject.Machine == node {
|
||||
standing[c.Key] = c
|
||||
}
|
||||
}
|
||||
hold, err := readHoldingFor(ctx, inv, open)
|
||||
if err != nil {
|
||||
return err
|
||||
var hold *holding
|
||||
if inv != nil {
|
||||
if hold, err = readHolding(ctx, inv, open); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
var problems []string
|
||||
modules := make([]string, 0, len(unhealthy))
|
||||
@@ -172,39 +159,6 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
heldOn := map[string]string{}
|
||||
providers := map[catalogue.Chosen]bool{}
|
||||
for _, m := range modules {
|
||||
// **A part that waits for the operator is said as that** (novox/hq ADR 0283): its waits already checked,
|
||||
// the operator's, never urgent, its words naming the act.
|
||||
if waits, waiting := operatorWait(m, unhealthy[m]); waiting {
|
||||
o := needsOperatorObservation(m, node, waits, unhealthy[m])
|
||||
// **A provider waiting for the operator says who waits on it** (novox/hq issue 405), as one waiting for a
|
||||
// login does: its consumers are held under it.
|
||||
if hold != nil {
|
||||
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
|
||||
}
|
||||
seen[o.Key()] = true
|
||||
became[m] = kindNeedsOperator
|
||||
if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen {
|
||||
continue
|
||||
}
|
||||
if _, err := k.Observe(ctx, o); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
continue
|
||||
}
|
||||
// **A wait for the operator beside anything else is said too** (novox/hq issue 405): a module with a checked
|
||||
// wait beside a new login owed, a directory used as found or a fault of its own is said as that, and the
|
||||
// operator's secret or setting it waits for was not mentioned until the other cleared. Its needs-operator
|
||||
// condition stands beside the other, on the same looks; what follows judges the rest without the wait.
|
||||
if waits, rest := besideAWait(m, unhealthy[m]); len(waits) > 0 {
|
||||
o := needsOperatorObservation(m, node, waits, waitingOf(m, unhealthy[m]))
|
||||
seen[o.Key()] = true
|
||||
if _, isOpen := standing[o.Key()]; streaks[m] >= moduleUnhealthyAfter || isOpen {
|
||||
if _, err := k.Observe(ctx, o); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
}
|
||||
unhealthy[m] = rest
|
||||
}
|
||||
// **A directory used as found is said as that** (novox/hq issue 339): the operator's to hand over at the
|
||||
// machine, never urgent — nothing is broken by the wait that a person was not told of — and its own kind,
|
||||
// so the gate never reads it as a fault of the build that happened to be sent beside it.
|
||||
@@ -241,24 +195,13 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
}
|
||||
o := moduleUnhealthyObservation(m, node, unhealthy[m])
|
||||
if hold != nil {
|
||||
if by, held := hold.heldWith(node, m, unhealthy[m]); held {
|
||||
// Held under the provider's condition: nothing of its own, and the provider's says it waits. The
|
||||
// clearing line says which the provider is: unhealthy, or waiting for the operator (issue 405).
|
||||
p := by.provider
|
||||
heldOn[o.Key()] = p.Module + " on " + p.Node + ", which is unhealthy"
|
||||
if len(by.waits) > 0 {
|
||||
heldOn[o.Key()] = p.Module + " on " + p.Node + ", which waits for you"
|
||||
}
|
||||
if p, held := hold.heldUnder(node, m, unhealthy[m]); held {
|
||||
// Held under the provider's condition: nothing of its own, and the provider's says it waits.
|
||||
heldOn[o.Key()] = p.Module + " on " + p.Node
|
||||
providers[p] = true
|
||||
continue
|
||||
}
|
||||
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
|
||||
// A provider that waits for the operator for a part this finding cannot be matched to does not hold it
|
||||
// (novox/hq issue 405): raised as its own, and saying the provider waits, so neither is hidden.
|
||||
if p, waiting := hold.waitingUncovered(node, m, unhealthy[m]); waiting {
|
||||
o.Said += fmt.Sprintf("; %s on %s waits for you, but not for anything %s is known to need, so %s's "+
|
||||
"fault is said on its own", p.Module, p.Node, m, m)
|
||||
}
|
||||
}
|
||||
seen[o.Key()] = true
|
||||
became[m] = kindModuleUnhealthy
|
||||
@@ -285,11 +228,8 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
if c.Kind == kindUsedAsFound {
|
||||
why = fmt.Sprintf("%s says no directory of %s is used as found any more", node, module)
|
||||
}
|
||||
if c.Kind == kindNeedsOperator {
|
||||
why = fmt.Sprintf("%s says %s no longer waits for the operator", node, module)
|
||||
}
|
||||
if on, held := heldOn[key]; held {
|
||||
why = fmt.Sprintf("what %s finds on %s waits on %s: held under its condition", module, node, on)
|
||||
why = fmt.Sprintf("what %s finds on %s waits on %s, which is unhealthy: held under its condition", module, node, on)
|
||||
}
|
||||
// **A condition that became the other kind** is not "working again" (issue 318 review): its clearing
|
||||
// line says what it became.
|
||||
@@ -298,12 +238,6 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
case c.Kind == kindModuleUnhealthy && became[module] == kindReloginNeeded:
|
||||
why = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
|
||||
resolved = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
|
||||
case c.Kind == kindModuleUnhealthy && became[module] == kindNeedsOperator:
|
||||
why = fmt.Sprintf("%s on %s now only waits for the operator", module, node)
|
||||
resolved = fmt.Sprintf("%s on %s now only waits for you", module, node)
|
||||
case c.Kind == kindNeedsOperator && became[module] == kindModuleUnhealthy:
|
||||
why = fmt.Sprintf("%s on %s no longer only waits for the operator, and is not healthy", module, node)
|
||||
resolved = fmt.Sprintf("What %s on %s waited for is given, and it still does not work", module, node)
|
||||
case c.Kind == kindReloginNeeded && became[module] == kindModuleUnhealthy:
|
||||
why = fmt.Sprintf("%s on %s no longer waits for a new login, and is not healthy", module, node)
|
||||
resolved = fmt.Sprintf("The new login on %s is done, and %s still does not work", node, module)
|
||||
@@ -328,74 +262,36 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
// sayWaiters observes a provider's open condition again, with who waits on it, from its machine's newest
|
||||
// statement. Nothing when its condition is not open: it is raised by its own statements, on its own looks.
|
||||
func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p catalogue.Chosen, now time.Time) error {
|
||||
// Its statement as it was judged, its waits checked (novox/hq issue 450): a wait that failed the check is
|
||||
// unhealthy, so the condition built here is the one its own statement raised, and who waits on it is listed.
|
||||
var raisedAt *conditions.Condition
|
||||
for i, c := range hold.open {
|
||||
if c.Key == moduleUnhealthyKey(p.Module, p.Node) || c.Key == reloginKey(p.Module, p.Node) {
|
||||
raisedAt = &hold.open[i]
|
||||
}
|
||||
}
|
||||
if raisedAt == nil {
|
||||
return nil
|
||||
}
|
||||
var rs []inventory.ResourceHealth
|
||||
for _, r := range hold.checked(p.Node) {
|
||||
if r.Module == p.Module && (r.State == link.StateUnhealthy || r.State == link.StateWaiting) {
|
||||
for _, r := range hold.healths[p.Node].Resources {
|
||||
if r.Module == p.Module && r.State == link.StateUnhealthy {
|
||||
rs = append(rs, r)
|
||||
}
|
||||
}
|
||||
if len(rs) == 0 {
|
||||
return nil
|
||||
}
|
||||
// The condition its own statement says it under: needs-operator while it only waits for the operator (novox/hq
|
||||
// issue 405), else that for the rest of it, a wait beside it said on its own.
|
||||
var o conditions.Observation
|
||||
if waits, waiting := operatorWait(p.Module, rs); waiting {
|
||||
o = needsOperatorObservation(p.Module, p.Node, waits, rs)
|
||||
} else {
|
||||
_, rest := besideAWait(p.Module, rs)
|
||||
o = moduleUnhealthyObservation(p.Module, p.Node, rest)
|
||||
if said, waits := personWait(p.Module, p.Node, rest); waits {
|
||||
o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rest)
|
||||
}
|
||||
o := moduleUnhealthyObservation(p.Module, p.Node, rs)
|
||||
if said, waits := personWait(p.Module, p.Node, rs); waits {
|
||||
o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rs)
|
||||
}
|
||||
raised := false
|
||||
for _, c := range hold.open {
|
||||
raised = raised || c.Key == o.Key()
|
||||
}
|
||||
if !raised {
|
||||
return nil // not open yet, or open as another kind: its own statement says it next
|
||||
if o.Key() != raisedAt.Key {
|
||||
return nil // its own statement says it next
|
||||
}
|
||||
sayWaitingOn(&o, hold.waitersOn(p))
|
||||
_, err := k.Observe(ctx, o)
|
||||
return err
|
||||
}
|
||||
|
||||
// besideAWait is, for a module with something not healthy beside a part waiting for the operator, the waits (checked
|
||||
// already) and the rest without the waiting parts (novox/hq issue 405); no waits when nothing waits, or when the
|
||||
// module only waits (operatorWait says that whole). Pure.
|
||||
func besideAWait(module string, rs []inventory.ResourceHealth) ([]inventory.Wait, []inventory.ResourceHealth) {
|
||||
if _, only := operatorWait(module, rs); only {
|
||||
return nil, rs
|
||||
}
|
||||
var waits []inventory.Wait
|
||||
var rest []inventory.ResourceHealth
|
||||
for _, r := range rs {
|
||||
if r.Module == module && r.State == link.StateWaiting {
|
||||
waits = append(waits, r.Waits...)
|
||||
continue
|
||||
}
|
||||
rest = append(rest, r)
|
||||
}
|
||||
if len(waits) == 0 {
|
||||
return nil, rs
|
||||
}
|
||||
return waits, rest
|
||||
}
|
||||
|
||||
// waitingOf is a module's waiting resources: the evidence of its wait.
|
||||
func waitingOf(module string, rs []inventory.ResourceHealth) []inventory.ResourceHealth {
|
||||
var out []inventory.ResourceHealth
|
||||
for _, r := range rs {
|
||||
if r.Module == module && r.State == link.StateWaiting {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// reloginKey is a module's relogin-needed condition on a machine.
|
||||
func reloginKey(module, node string) string {
|
||||
return conditions.Key(conditions.ScopeModule, module+"."+node, kindReloginNeeded)
|
||||
@@ -467,15 +363,12 @@ func waitingAccounts(module string, rs []inventory.ResourceHealth) []string {
|
||||
}
|
||||
|
||||
// sayWaitingOn adds to a module's condition the consumers held under it (to-be 48 §6): urgent while anyone
|
||||
// waits on it, whether it is not working or waits for a new login. **A wait for the operator's secret or setting
|
||||
// stays a warning** (ADR 0283 decision 5, novox/hq issue 405): who waits on it is listed, and nothing escalates it.
|
||||
// waits on it, whether it is not working or waits for a new login.
|
||||
func sayWaitingOn(o *conditions.Observation, waiters []string) {
|
||||
if len(waiters) == 0 {
|
||||
return
|
||||
}
|
||||
if o.Kind != kindNeedsOperator {
|
||||
o.Severity = conditions.Urgent
|
||||
}
|
||||
o.Severity = conditions.Urgent
|
||||
o.Said += "; " + waitingWords(waiters)
|
||||
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
|
||||
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
|
||||
@@ -497,11 +390,9 @@ func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHeal
|
||||
continue
|
||||
}
|
||||
words = append(words, fmt.Sprintf("its %s %s %s", r.Kind, r.Resource, reasonWords(r)))
|
||||
// The restart count runs from when the engine began watching it, not from the state's since (novox/hq
|
||||
// issue 458).
|
||||
said = append(said, fmt.Sprintf("%s (%s %s): %s, since %s, %d look(s) in a row, %s",
|
||||
r.Resource, r.Kind, r.Target, orNotSaid(r.Reason), r.Since.UTC().Format("2006-01-02 15:04:05 MST"),
|
||||
r.Streak, restartsWords(r.Restarts)))
|
||||
said = append(said, fmt.Sprintf("%s (%s %s): %s, %d look(s) in a row, %d restart(s) counted, since %s",
|
||||
r.Resource, r.Kind, r.Target, orNotSaid(r.Reason), r.Streak, r.Restarts,
|
||||
r.Since.UTC().Format("2006-01-02 15:04:05 MST")))
|
||||
}
|
||||
return conditions.Observation{Scope: conditions.ScopeModule, ID: module + "." + node, Token: "unhealthy",
|
||||
Kind: kindModuleUnhealthy, Machine: node, Severity: conditions.Warning, Source: sourceHealth,
|
||||
@@ -511,19 +402,9 @@ func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHeal
|
||||
Explanation: fmt.Sprintf("%s on %s is not healthy: %s. It clears as soon as it runs again.", module, node,
|
||||
namesWords(plain, 3)),
|
||||
Needs: needs,
|
||||
Actions: moduleActions(node, rs),
|
||||
Resolved: fmt.Sprintf("%s works again on %s", module, node)}
|
||||
}
|
||||
|
||||
// restartsWords is a restart count as what it is (novox/hq issue 458). The engine reports its own count
|
||||
// (mesh-host's liveness judge, kept.Counted): every rise of systemd's NRestarts or the runtime's RestartCount
|
||||
// after a start's grace, so only restarts by the Restart= setting or the restart policy. It keeps the count
|
||||
// across a recreate, a start by hand and its own restarts, and forgets it only when the resource is no longer
|
||||
// declared. So it counts automatic restarts since the engine began watching it, never since a state's "since".
|
||||
func restartsWords(n int) string {
|
||||
return fmt.Sprintf("%d automatic restart(s) since the node-engine began watching it", n)
|
||||
}
|
||||
|
||||
// reasonWords is why a resource is unhealthy, as a person reads it.
|
||||
func reasonWords(r inventory.ResourceHealth) string {
|
||||
// An account waiting for a new login (ADR 0252) is said in the mesh's words, not the engine's.
|
||||
@@ -532,17 +413,12 @@ func reasonWords(r inventory.ResourceHealth) string {
|
||||
}
|
||||
switch r.Reason {
|
||||
case "restarting":
|
||||
return fmt.Sprintf("keeps restarting (%s)", restartsWords(r.Restarts))
|
||||
return fmt.Sprintf("keeps restarting (%d restart(s) counted)", r.Restarts)
|
||||
case "down":
|
||||
return "is not running"
|
||||
case "":
|
||||
return "is unhealthy"
|
||||
}
|
||||
// A wait for the operator that did not check out is said as the controller found it (ADR 0283): names of
|
||||
// secrets and settings only, never what the check itself said.
|
||||
if strings.HasPrefix(r.Reason, waitRefusedPrefix) {
|
||||
return r.Reason
|
||||
}
|
||||
// What a declared check found says an endpoint, a path or an address: evidence, never the summary the
|
||||
// operator's channel carries (ADR 0234 §6). The summary names the check.
|
||||
if r.Check != "" {
|
||||
@@ -634,9 +510,7 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
|
||||
if h.HeardAt.Before(since) {
|
||||
return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module)
|
||||
}
|
||||
// **A wait for the operator is checked first** (novox/hq ADR 0283): one that does not check out is unhealthy.
|
||||
resources := checkWaiting(machine, h.Resources, f.waits)
|
||||
wait, waits := personWait(module, machine, resources)
|
||||
wait, waits := personWait(module, machine, h.Resources)
|
||||
// **Only a build whose own send put the account in a new group is excused** (issue 318 review): read from
|
||||
// what the controller sent, never from when the machine says the wait began — that time is the engine's
|
||||
// memory, reset by its restart and moved by a change of words. A build that adds no account group cannot
|
||||
@@ -644,18 +518,11 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
|
||||
if waits && !f.groupsAdded[module] {
|
||||
waits = false
|
||||
}
|
||||
var found, onWaiting []string
|
||||
var forOperator []inventory.Wait
|
||||
for _, r := range resources {
|
||||
var found []string
|
||||
for _, r := range h.Resources {
|
||||
if r.Module != module {
|
||||
continue
|
||||
}
|
||||
// **Any build is excused while its wait for the operator checks out** (ADR 0283 decision 4): a secret not
|
||||
// given is owed by every build alike, so it is no fault of this one, and the verdict carries it.
|
||||
if r.State == link.StateWaiting {
|
||||
forOperator = append(forOperator, r.Waits...)
|
||||
continue
|
||||
}
|
||||
if waits && r.State == link.StateUnhealthy {
|
||||
continue
|
||||
}
|
||||
@@ -673,16 +540,8 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
|
||||
return healthNotYet, fmt.Sprintf("its %s %s on %s is still starting", r.Kind, r.Resource, machine)
|
||||
case link.StateUnhealthy:
|
||||
if on, held := f.heldOn[module+"@"+machine]; held {
|
||||
// **Held under a provider that only waits for the operator** (novox/hq issue 405): a wait for a
|
||||
// person, a pass carrying the wait (ADR 0254, ADR 0283 decision 4) — the walk never waits for the
|
||||
// operator's secret, whichever module owes it.
|
||||
if on.waits != "" {
|
||||
onWaiting = append(onWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which waits for you: %s",
|
||||
r.Kind, r.Resource, machine, on.on, on.waits))
|
||||
continue
|
||||
}
|
||||
return healthWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which is unhealthy", r.Kind,
|
||||
r.Resource, machine, on.on)
|
||||
r.Resource, machine, on)
|
||||
}
|
||||
return healthNotYet, fmt.Sprintf("its %s %s on %s %s", r.Kind, r.Resource, machine, reasonWords(r))
|
||||
default:
|
||||
@@ -690,17 +549,14 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
|
||||
reasonAfter(r.Reason))
|
||||
}
|
||||
}
|
||||
if waits || len(found) > 0 || len(forOperator) > 0 || len(onWaiting) > 0 {
|
||||
said := onWaiting
|
||||
if waits || len(found) > 0 {
|
||||
var said []string
|
||||
if waits {
|
||||
said = append(said, wait)
|
||||
}
|
||||
if len(forOperator) > 0 {
|
||||
said = append(said, operatorWaitSaid(module, machine, forOperator))
|
||||
}
|
||||
if len(found) > 0 {
|
||||
said = append(said, fmt.Sprintf("on %s, %s uses %s as found and waits for the operator to hand it over "+
|
||||
"(`nox node hand-over %s <directory>` on the control-node)", machine, module, strings.Join(found, ", "), machine))
|
||||
said = append(said, fmt.Sprintf("on %s, %s uses %s as found and waits for a person to hand it over "+
|
||||
"(`mesh-host hand-over <directory>` at the machine)", machine, module, strings.Join(found, ", ")))
|
||||
}
|
||||
return healthPerson, strings.Join(said, "; ")
|
||||
}
|
||||
@@ -736,7 +592,7 @@ func healthLines(h inventory.NodeHealth, had bool, now time.Time) []string {
|
||||
line += " — " + r.Reason
|
||||
}
|
||||
if r.Restarts > 0 {
|
||||
line += ", " + restartsWords(r.Restarts)
|
||||
line += fmt.Sprintf(", %d restart(s) counted", r.Restarts)
|
||||
}
|
||||
if r.State == link.StateUnhealthy && h.Streaks[r.Module] < moduleUnhealthyAfter {
|
||||
line += " (unconfirmed: said once)"
|
||||
@@ -821,9 +677,7 @@ func usedAsFoundObservation(module, node, said string, rs []inventory.ResourceHe
|
||||
o.Explanation = fmt.Sprintf("A directory of %s was already on %s, with another owner or mode than %s declares. "+
|
||||
"The mesh left it as it was rather than hand it to an account, so %s may not be able to use it.",
|
||||
module, node, module, module)
|
||||
// Plain words (ADR 0253): the line itself — `nox node hand-over <node> <path>` on the control-node (ADR 0272,
|
||||
// issue 356) — is in the summary and the evidence, which name the directory; a path is never in these.
|
||||
o.Needs = "hand the directory over from the control-node, as the operator; the details name it and the line to type."
|
||||
o.Needs = fmt.Sprintf("on %s, run mesh-host hand-over with the directory's path as root.", node)
|
||||
o.Resolved = fmt.Sprintf("%s's directory on %s is the mesh's", module, node)
|
||||
o.Actions = nil
|
||||
return o
|
||||
|
||||
@@ -10,12 +10,10 @@ import (
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
@@ -399,9 +397,8 @@ func assignCommand(ctx context.Context, verb string, args []string) error {
|
||||
func settingsCommand(ctx context.Context, args []string) error {
|
||||
if len(args) == 0 {
|
||||
return errors.New("settings show <module> [--node <node>] [--history], settings set <module> <file> " +
|
||||
"[--node <node>] [--replace], settings clear <module> [--node <node>], settings preferences " +
|
||||
"[<module>] [--node <node>], settings propose <module> <file | --clear> [--node <node>] [--replace], " +
|
||||
"or settings proposals [<id>]")
|
||||
"[--node <node>] [--replace], settings clear <module> [--node <node>], or settings preferences " +
|
||||
"[<module>] [--node <node>]")
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
@@ -415,38 +412,12 @@ func settingsCommand(ctx context.Context, args []string) error {
|
||||
// **What a set removes is refused unless meant** (novox/hq ADR 0217). A layer is replaced whole,
|
||||
// and on 2026-10-05 setting one placement dropped a machine's whole layer for a module without a
|
||||
// word (novox/hq issue 304). Adding and changing keys needs nothing; removing one needs this.
|
||||
replace := set.Bool("replace", false, "for set and propose: remove the keys the new layer does not name")
|
||||
replace := set.Bool("replace", false, "for set: remove the keys the new layer does not name")
|
||||
history := set.Bool("history", false, "for show: the layers this one replaced, the latest first")
|
||||
clear := set.Bool("clear", false, "for propose: propose that the layer be removed")
|
||||
positionals, err := parseAround(set, args[1:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch args[0] {
|
||||
case "propose":
|
||||
// A trusted setting, proposed by anyone and set only on the operator's warrant (novox/hq ADR 0277):
|
||||
// the stores are opened there, so the proposal and the verb's refusals below never meet.
|
||||
if len(positionals) < 1 || len(positionals) > 2 {
|
||||
return errors.New("settings propose <module> <settings.json | {…}> [--node <node>] [--replace], or settings propose <module> --clear [--node <node>]")
|
||||
}
|
||||
values := ""
|
||||
if len(positionals) == 2 {
|
||||
values = positionals[1]
|
||||
}
|
||||
return proposeCommand(ctx, positionals[0], *node, values, *clear, *replace)
|
||||
case "proposals":
|
||||
if len(positionals) > 1 || *node != "" || *clear || *replace || *history {
|
||||
return errors.New("settings proposals [<id>]")
|
||||
}
|
||||
id := ""
|
||||
if len(positionals) == 1 {
|
||||
id = positionals[0]
|
||||
}
|
||||
return proposalsCommand(ctx, id)
|
||||
}
|
||||
if *clear {
|
||||
return errors.New("--clear is for settings propose; a layer is cleared with settings clear")
|
||||
}
|
||||
|
||||
where := "the whole mesh"
|
||||
if *node != "" {
|
||||
@@ -484,7 +455,7 @@ func settingsCommand(ctx context.Context, args []string) error {
|
||||
"removal is meant (novox/hq ADR 0217). Nothing was changed",
|
||||
positionals[0], where, strings.Join(removed, ", "), positionals[0], nodeFlag(*node))
|
||||
}
|
||||
if err := inv.SetSettingsBy(ctx, *node, positionals[0], values, setByWords()); err != nil {
|
||||
if err := inv.SetSettings(ctx, *node, positionals[0], values); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s on %s:\n", positionals[0], where)
|
||||
@@ -525,12 +496,8 @@ func settingsCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
for _, p := range past {
|
||||
shown, _ := json.MarshalIndent(p.Values, " ", " ")
|
||||
by := ""
|
||||
if p.SetBy != "" {
|
||||
by = "; " + p.SetBy
|
||||
}
|
||||
fmt.Printf("%s on %s, until %s (%s%s):\n %s\n", positionals[0], where,
|
||||
p.ReplacedAt.Local().Format("2006-01-02 15:04:05"), p.ReplacedBy, by, shown)
|
||||
fmt.Printf("%s on %s, until %s (%s):\n %s\n", positionals[0], where,
|
||||
p.ReplacedAt.Local().Format("2006-01-02 15:04:05"), p.ReplacedBy, shown)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -546,14 +513,6 @@ func settingsCommand(ctx context.Context, args []string) error {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(shown))
|
||||
// And who set it (novox/hq ADR 0277): a layer the operator approved on their phone says so.
|
||||
if setBy, setAt, has, err := inv.LayerOrigin(ctx, *node, positionals[0]); err != nil {
|
||||
return err
|
||||
} else if has && setBy != "" {
|
||||
fmt.Printf(" %s\n", setBy)
|
||||
} else if has {
|
||||
fmt.Printf(" set at %s; who set it was not kept\n", setAt.Local().Format("2006-01-02 15:04"))
|
||||
}
|
||||
}
|
||||
// Every value the module gives a default or a layer sets, and where it came from (novox/hq
|
||||
// ADR 0262): the default, the mesh's layer, or this node's. Said after the layer, which stays
|
||||
@@ -647,26 +606,17 @@ func settingsCommand(ctx context.Context, args []string) error {
|
||||
if err := refuseTerminalSettingsThroughAVerb(ctx, inv, before, nil, positionals[0], where); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inv.ClearSettingsBy(ctx, *node, positionals[0], setByWords()); err != nil {
|
||||
if err := inv.ClearSettings(ctx, *node, positionals[0]); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s on %s is back to what the module says\n", positionals[0], where)
|
||||
return nil
|
||||
|
||||
default:
|
||||
return fmt.Errorf("settings has no %q; it has show, set, clear, preferences, propose and proposals", args[0])
|
||||
return fmt.Errorf("settings has no %q; it has show, set, clear and preferences", args[0])
|
||||
}
|
||||
}
|
||||
|
||||
// setByWords is who sets a layer from this process, as the layer keeps it (novox/hq ADR 0277): the caller at the
|
||||
// controller's terminal, or through which verb.
|
||||
func setByWords() string {
|
||||
if verb, through := throughAVerb(); through {
|
||||
return "set by " + link.Caller() + " through " + verb + " at " + time.Now().Local().Format("2006-01-02 15:04")
|
||||
}
|
||||
return "set at the controller's terminal by " + link.Caller() + " at " + time.Now().Local().Format("2006-01-02 15:04")
|
||||
}
|
||||
|
||||
// describeEffective says each setting's value on a machine or the whole mesh, where it came from, and
|
||||
// the module's default when a layer overrides it.
|
||||
func describeEffective(module, where string, values []catalogue.SettingSource) string {
|
||||
@@ -1157,13 +1107,10 @@ func refuseTerminalSettingsThroughAVerb(ctx context.Context, inv *inventory.Inve
|
||||
}
|
||||
// A trusted mergeable file takes any key, so its module's whole layer is the terminal's (novox/hq issue 340).
|
||||
if files := catalogue.TrustedMergeable(shelf[module]); len(files) > 0 && !sameLayer(before, after) {
|
||||
return fmt.Errorf("the settings of %s on %s are set at the controller's terminal (`mesh-cli` on the control-node) or on "+
|
||||
"the operator's warrant, never through a verb alone (this line came through %q): %s merges whatever key a "+
|
||||
"layer sets into a file root or a consumer trusts, so any key could point the module at a listener of the "+
|
||||
"caller's, and whoever may call a verb includes agents (novox/hq issue 340; a file nothing trusts says "+
|
||||
"\"trusted\": false). Propose it instead: the settings verb with propose puts the exact values to the "+
|
||||
"operator on a channel that proves who answers, and the layer is set on their Approve (novox/hq ADR 0277). "+
|
||||
"Nothing was changed",
|
||||
return fmt.Errorf("the settings of %s on %s are set at the controller's terminal only (`mesh-cli` on the control-node), never through a verb (this "+
|
||||
"line came through %q): %s merges whatever key a layer sets into a file root or a consumer trusts, so "+
|
||||
"any key could point the module at a listener of the caller's, and whoever may call a verb includes "+
|
||||
"agents (novox/hq issue 340; a file nothing trusts says \"trusted\": false). Nothing was changed",
|
||||
module, where, verb, strings.Join(files, ", "))
|
||||
}
|
||||
for _, key := range catalogue.TerminalKeys(shelf[module]) {
|
||||
@@ -1172,13 +1119,11 @@ func refuseTerminalSettingsThroughAVerb(ctx context.Context, inv *inventory.Inve
|
||||
if string(was) == string(now) {
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf("%s of %s on %s is set at the controller's terminal (`mesh-cli` on the control-node) or on the "+
|
||||
"operator's warrant, never through a verb alone (this line came through %q): it says where root creates and "+
|
||||
"owns a module's directories, which of the machine's paths are mounted into its container, what the mesh's "+
|
||||
"consumers trust, or what a file root or a person's session obeys takes, and whoever may call a verb "+
|
||||
"includes agents (novox/hq issue 339; issue 340 for a mergeable file's own keys). Propose it instead: the "+
|
||||
"settings verb with propose puts the exact values to the operator on a channel that proves who answers, and "+
|
||||
"the layer is set on their Approve (novox/hq ADR 0277). Nothing was changed", key, module, where, verb)
|
||||
return fmt.Errorf("%s of %s on %s is set at the controller's terminal only (`mesh-cli` on the control-node), never through a verb (this "+
|
||||
"line came through %q): it says where root creates and owns a module's directories, which of "+
|
||||
"the machine's paths are mounted into its container, what the mesh's consumers trust, or what a file "+
|
||||
"root or a person's session obeys takes, and whoever may call a verb includes agents (novox/hq issue 339; "+
|
||||
"issue 340 for a mergeable file's own keys). Nothing was changed", key, module, where, verb)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -426,10 +426,10 @@ func overlayShow(ctx context.Context, open *stores) error {
|
||||
tunnel.Interface, tunnel.Range, tunnel.Port)
|
||||
case n.Hub && hubName == n.Name && tunnel.Interface != "":
|
||||
fmt.Printf(" hub — found a tunnel on %s and did NOT take it over: its key is not the tunnel's; "+
|
||||
"`nox-mesh-host overlay take --tunnel %s` on the machine takes it", tunnel.Interface, tunnel.Interface)
|
||||
"`mesh-host overlay take --tunnel %s` on the machine takes it", tunnel.Interface, tunnel.Interface)
|
||||
case n.Hub:
|
||||
fmt.Print(" hub — found no tunnel; if the machine runs the predecessor's, " +
|
||||
"`nox-mesh-host overlay take --tunnel <iface>` there adopts it (novox/hq ADR 0105)")
|
||||
"`mesh-host overlay take --tunnel <iface>` there adopts it (novox/hq ADR 0105)")
|
||||
case !n.Reachable():
|
||||
fmt.Print(" not dialable")
|
||||
}
|
||||
|
||||
@@ -3,12 +3,10 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/beside"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
@@ -241,12 +239,13 @@ func TestATakeoverIsNotComposedForAHubPlacedOffItsTunnel(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// theResolver is the catalogue's dnsmasq module as it is (internal/beside).
|
||||
// theResolver is the catalogue's dnsmasq module as it is, or the test is skipped where the
|
||||
// catalogue is not beside this checkout.
|
||||
func theResolver(t *testing.T) catalogue.Manifest {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Join(beside.Catalogue(t), "dnsmasq", "module.json"))
|
||||
raw, err := os.ReadFile("../../../mesh-catalog/modules/dnsmasq/module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
t.Skipf("the catalogue is not beside this checkout: %v", err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -18,7 +17,6 @@ import (
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/token"
|
||||
)
|
||||
|
||||
@@ -30,7 +28,7 @@ import (
|
||||
|
||||
func nodeCommand(ctx context.Context, args []string) error {
|
||||
if len(args) == 0 {
|
||||
return errors.New("node add <name>, node list, node show <name>, " + publicDomainUsage + ", or " + handOverUsage)
|
||||
return errors.New("node add <name>, node list, node show <name>, or " + publicDomainUsage)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
@@ -44,7 +42,7 @@ func nodeCommand(ctx context.Context, args []string) error {
|
||||
if len(args) != 2 {
|
||||
return errors.New("node show <name>")
|
||||
}
|
||||
return showNode(ctx, open, args[1])
|
||||
return showNode(ctx, inv, args[1])
|
||||
case "add":
|
||||
return addNode(ctx, inv, args[1:])
|
||||
|
||||
@@ -114,168 +112,11 @@ func nodeCommand(ctx context.Context, args []string) error {
|
||||
// an optional second argument is the home when it is not /home/<account>.
|
||||
return nodeAccount(ctx, inv, args[1:])
|
||||
|
||||
case "hand-over":
|
||||
// A directory the node-engine uses as found, handed to the mesh (novox/hq issue 356, issue 339). Here, at
|
||||
// the controller's terminal, and nowhere else: at the next apply root gives the directory to the account
|
||||
// the module declares, and whoever may call a verb includes agents.
|
||||
return nodeHandOver(ctx, open, args[1:])
|
||||
|
||||
case "setuid-search":
|
||||
// A fresh search for setuid programs on a node (novox/hq issue 361), after the operator changed by hand
|
||||
// what the last one found. Here, at the controller's terminal, and nowhere else: a search never makes a
|
||||
// machine free wrongly, but asked again and again it would keep the machine unjudged and its disks busy,
|
||||
// and whoever may call a verb includes agents.
|
||||
return nodeSetuidSearch(ctx, open, args[1:])
|
||||
|
||||
case "judge-directories":
|
||||
// Which of a node's declared directories the walk that follows no link another account could make would
|
||||
// refuse (novox/hq issue 358). Read-only on the node and here, so a verb may run it too.
|
||||
return nodeJudgeDirectories(ctx, open, args[1:])
|
||||
|
||||
default:
|
||||
return fmt.Errorf("node has no %q; it has add, list, show, public-domain, account, agent-account, hand-over, "+
|
||||
"setuid-search and judge-directories", args[0])
|
||||
return fmt.Errorf("node has no %q; it has add, list, show, public-domain, account and agent-account", args[0])
|
||||
}
|
||||
}
|
||||
|
||||
const handOverUsage = "node hand-over <node> <directory> — hand a directory the node-engine on <node> uses as found " +
|
||||
"to the mesh: its next apply gives it the declared owner and mode. The directory's absolute path, as the module's " +
|
||||
"condition names it"
|
||||
|
||||
// handOverLine reads a hand-over's line: the node and the directory's absolute path, exactly as the engine states
|
||||
// it. Judged before anything is asked, and judged again by the engine, which is the one that acts.
|
||||
func handOverLine(args []string) (node, path string, err error) {
|
||||
if len(args) != 2 {
|
||||
return "", "", errors.New(handOverUsage)
|
||||
}
|
||||
node, path = args[0], args[1]
|
||||
if node == "" || strings.HasPrefix(node, "-") {
|
||||
return "", "", fmt.Errorf("%q is not a node's name; %s", node, handOverUsage)
|
||||
}
|
||||
if !filepath.IsAbs(path) {
|
||||
return "", "", fmt.Errorf("%q is not an absolute path; %s", path, handOverUsage)
|
||||
}
|
||||
if filepath.Clean(path) != path {
|
||||
return "", "", fmt.Errorf("%q is not the directory's path as the engine states it (no `..`, no doubled or "+
|
||||
"trailing separator); %s", path, handOverUsage)
|
||||
}
|
||||
return node, path, nil
|
||||
}
|
||||
|
||||
// handOverBy is who hands the directory over, in the words a verb's caller is recorded in: the operator through
|
||||
// mesh-cli on the control-node, or whoever runs this controller's binary at its terminal.
|
||||
func handOverBy() string {
|
||||
if by := strings.TrimSpace(os.Getenv(link.CallerVar)); by != "" {
|
||||
return by
|
||||
}
|
||||
return "the controller's terminal"
|
||||
}
|
||||
|
||||
// nodeHandOver asks the node's engine to take a directory it uses as found as the mesh's, and says what came of
|
||||
// it. The engine records the hand-over or refuses; nothing is recorded here, because the directory is the
|
||||
// machine's and the engine is the one that reads it. The ask is signed with the mesh's key (issue 356's review).
|
||||
func nodeHandOver(ctx context.Context, open *stores, args []string) error {
|
||||
known := func(node string) error {
|
||||
_, err := open.inventory.NodeByName(ctx, node)
|
||||
return err
|
||||
}
|
||||
ask := func(node, path, by string) (link.HandOverAnswer, error) {
|
||||
ident, err := open.Identity(ctx)
|
||||
if err != nil {
|
||||
return link.HandOverAnswer{}, fmt.Errorf("the mesh's signing key cannot be read, so nothing was asked of %s: %w",
|
||||
node, err)
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return link.HandOverAnswer{}, err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return link.HandOverAnswer{}, fmt.Errorf("cannot reach the bus, so nothing was asked of %s: %w", node, err)
|
||||
}
|
||||
defer js.Close()
|
||||
return link.AskHandOver(ctx, js.Conn(), ident, node, path, by, link.HandOverWithin)
|
||||
}
|
||||
return handOverAsked(args, known, ask, os.Stdout)
|
||||
}
|
||||
|
||||
// handOverAsked is the hand-over's line with its two acts given: whether the mesh knows the node, and the ask.
|
||||
// Nothing is asked of a line or a node that is refused, and the engine's refusal is this command's failure —
|
||||
// never a success with the refusal printed.
|
||||
func handOverAsked(args []string, known func(node string) error,
|
||||
ask func(node, path, by string) (link.HandOverAnswer, error), out io.Writer) error {
|
||||
node, path, err := handOverLine(args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := known(node); err != nil {
|
||||
return fmt.Errorf("nothing was asked: %w", err)
|
||||
}
|
||||
answer, err := ask(node, path, handOverBy())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Refused != "" {
|
||||
return fmt.Errorf("%s refused: %s", node, answer.Refused)
|
||||
}
|
||||
fmt.Fprintln(out, answer.Said)
|
||||
fmt.Fprintf(out, " the module's condition clears once %s applies; `nox push %s` applies it now\n", node, node)
|
||||
return nil
|
||||
}
|
||||
|
||||
const setuidSearchUsage = "node setuid-search <node> — throw away the node-engine's last search for setuid " +
|
||||
"programs on <node> and start a full one: after a setuid-root program it found was removed by hand. Until it " +
|
||||
"completes, root-free says the node is not judged yet"
|
||||
|
||||
// nodeSetuidSearch asks the node's engine for a fresh search, signed with the mesh's key as a hand-over is.
|
||||
func nodeSetuidSearch(ctx context.Context, open *stores, args []string) error {
|
||||
known := func(node string) error {
|
||||
_, err := open.inventory.NodeByName(ctx, node)
|
||||
return err
|
||||
}
|
||||
ask := func(node, by string) (link.HandOverAnswer, error) {
|
||||
ident, err := open.Identity(ctx)
|
||||
if err != nil {
|
||||
return link.HandOverAnswer{}, fmt.Errorf("the mesh's signing key cannot be read, so nothing was asked of %s: %w",
|
||||
node, err)
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return link.HandOverAnswer{}, err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return link.HandOverAnswer{}, fmt.Errorf("cannot reach the bus, so nothing was asked of %s: %w", node, err)
|
||||
}
|
||||
defer js.Close()
|
||||
return link.AskSetuidSearch(ctx, js.Conn(), ident, node, by, link.HandOverWithin)
|
||||
}
|
||||
return setuidSearchAsked(args, known, ask, os.Stdout)
|
||||
}
|
||||
|
||||
// setuidSearchAsked is the line with its two acts given: whether the mesh knows the node, and the ask. The
|
||||
// engine's refusal is this command's failure.
|
||||
func setuidSearchAsked(args []string, known func(node string) error,
|
||||
ask func(node, by string) (link.HandOverAnswer, error), out io.Writer) error {
|
||||
if len(args) != 1 || args[0] == "" || strings.HasPrefix(args[0], "-") {
|
||||
return errors.New(setuidSearchUsage)
|
||||
}
|
||||
node := args[0]
|
||||
if err := known(node); err != nil {
|
||||
return fmt.Errorf("nothing was asked: %w", err)
|
||||
}
|
||||
answer, err := ask(node, handOverBy())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Refused != "" {
|
||||
return fmt.Errorf("%s refused: %s", node, answer.Refused)
|
||||
}
|
||||
fmt.Fprintln(out, answer.Said)
|
||||
fmt.Fprintf(out, " the controller's root-free verb shows the search's progress until it completes\n")
|
||||
return nil
|
||||
}
|
||||
|
||||
// addNode creates a node record, adopted when the operator says so (novox/hq ADR 0100).
|
||||
func addNode(ctx context.Context, inv *inventory.Inventory, args []string) error {
|
||||
set := flag.NewFlagSet("node add", flag.ContinueOnError)
|
||||
@@ -792,8 +633,7 @@ func roughly(d time.Duration) string {
|
||||
//
|
||||
// It is also where "what should it be configured as" is read. The same line that gates an
|
||||
// assignment carries `card1-DP-1`, and a person composing settings for that machine needs it.
|
||||
func showNode(ctx context.Context, stored *stores, name string) error {
|
||||
inv := stored.inventory
|
||||
func showNode(ctx context.Context, inv *inventory.Inventory, name string) error {
|
||||
node, err := inv.NodeByName(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -809,19 +649,6 @@ func showNode(ctx context.Context, stored *stores, name string) error {
|
||||
if err := showMode(ctx, inv, node); err != nil {
|
||||
return err
|
||||
}
|
||||
// What it was last sent, by whom, under which epoch, from which generation, naming which modules, and what
|
||||
// it answered (novox/hq issue 485): the record of sends is read through this verb, never the store's reader.
|
||||
// Unreadable is said, not taken for none.
|
||||
switch last, found, err := inv.LastSend(ctx, name); {
|
||||
case err != nil:
|
||||
fmt.Printf(" what it was last sent could NOT be read: %v\n", err)
|
||||
case !found:
|
||||
fmt.Printf(" last sent no send recorded since the mesh began keeping them\n")
|
||||
default:
|
||||
for _, line := range lastSendLines(last) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
}
|
||||
// Whom agents run as here, and whether that account can become root without a person (ADR 0266).
|
||||
for _, line := range agentAccountLines(ctx, inv, node) {
|
||||
fmt.Println(line)
|
||||
@@ -908,17 +735,6 @@ func showNode(ctx context.Context, stored *stores, name string) error {
|
||||
if len(assigned) > 0 {
|
||||
fmt.Printf("\n assigned: %s\n", strings.Join(assigned, ", "))
|
||||
}
|
||||
// And which of them a push leaves out, and why (novox/hq issue 380): judged as the push judges it, the same
|
||||
// reason its condition says. Not computable is said, never read as "all applied".
|
||||
plan, settings, err := planFor(ctx, stored, name)
|
||||
switch {
|
||||
case err != nil:
|
||||
fmt.Printf("\n whether a send leaves any assigned module out is NOT known: %s\n", oneLine(err.Error()))
|
||||
default:
|
||||
for _, line := range notAppliedLines(name, leftOutOf(plan, settings, node.Adopted)) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -942,57 +758,3 @@ func printJSONTo(w io.Writer, v any) error {
|
||||
fmt.Fprintln(w, string(body))
|
||||
return nil
|
||||
}
|
||||
|
||||
const judgeDirectoriesUsage = "node judge-directories <node> — read-only: which of the directories <node> is " +
|
||||
"declared the node-engine's walk that follows no link another account could make would refuse, one line each " +
|
||||
"(novox/hq issue 358). Nothing is made, chmodded or chowned"
|
||||
|
||||
// nodeJudgeDirectories asks the node's engine to judge its declared directories, signed with the mesh's key as a
|
||||
// hand-over is. Read-only: the engine changes nothing and keeps nothing.
|
||||
func nodeJudgeDirectories(ctx context.Context, open *stores, args []string) error {
|
||||
known := func(node string) error {
|
||||
_, err := open.inventory.NodeByName(ctx, node)
|
||||
return err
|
||||
}
|
||||
ask := func(node, by string) (link.HandOverAnswer, error) {
|
||||
ident, err := open.Identity(ctx)
|
||||
if err != nil {
|
||||
return link.HandOverAnswer{}, fmt.Errorf("the mesh's signing key cannot be read, so nothing was asked of %s: %w",
|
||||
node, err)
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return link.HandOverAnswer{}, err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return link.HandOverAnswer{}, fmt.Errorf("cannot reach the bus, so nothing was asked of %s: %w", node, err)
|
||||
}
|
||||
defer js.Close()
|
||||
return link.AskJudgeDirectories(ctx, js.Conn(), ident, node, by, link.HandOverWithin)
|
||||
}
|
||||
return judgeDirectoriesAsked(args, known, ask, os.Stdout)
|
||||
}
|
||||
|
||||
// judgeDirectoriesAsked is the line with its two acts given: whether the mesh knows the node, and the ask. The
|
||||
// engine's refusal is this command's failure.
|
||||
func judgeDirectoriesAsked(args []string, known func(node string) error,
|
||||
ask func(node, by string) (link.HandOverAnswer, error), out io.Writer) error {
|
||||
if len(args) != 1 || args[0] == "" || strings.HasPrefix(args[0], "-") {
|
||||
return errors.New(judgeDirectoriesUsage)
|
||||
}
|
||||
node := args[0]
|
||||
if err := known(node); err != nil {
|
||||
return fmt.Errorf("nothing was asked: %w", err)
|
||||
}
|
||||
answer, err := ask(node, handOverBy())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Refused != "" {
|
||||
return fmt.Errorf("%s refused: %s", node, answer.Refused)
|
||||
}
|
||||
fmt.Fprintf(out, "%s (read-only: nothing on %s was changed)\n", node, node)
|
||||
fmt.Fprintln(out, answer.Said)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,112 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// novox/hq issue 237: every line of an act's answer comes from one resolution of the node taken after
|
||||
// its write, so the answer agrees with `plan` for the node right after.
|
||||
|
||||
func aProviderAndItsConsumer(t *testing.T, open *stores) {
|
||||
t.Helper()
|
||||
register(t, open, catalogue.Manifest{Module: "db", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "rows", Scope: catalogue.ScopeNode}}})
|
||||
register(t, open, catalogue.Manifest{Module: "app", Version: "1", Requires: []string{"rows"}})
|
||||
}
|
||||
|
||||
// twoRivalsAndANote is a node that does not resolve — two modules claim one node seat there — and a
|
||||
// module beside them that changes nothing about it.
|
||||
func twoRivalsAndANote(t *testing.T, open *stores) {
|
||||
t.Helper()
|
||||
one, two := rivals()
|
||||
register(t, open, one)
|
||||
register(t, open, two)
|
||||
register(t, open, catalogue.Manifest{Module: "notes", Version: "1"})
|
||||
}
|
||||
|
||||
func TestAnUnassignThatLeavesItsNodeUnresolvedSaysSoOnceInItsAnswer(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
twoRivalsAndANote(t, open)
|
||||
// Straight into the store, as a node that came to this before the act.
|
||||
for _, m := range []string{"rival-one", "rival-two", "notes"} {
|
||||
if _, err := open.inventory.Assign(ctx, "laptop", m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
said := answered(t, func() error { return assignCommand(ctx, "unassign", []string{"laptop", "notes"}) })
|
||||
|
||||
if _, _, err := planFor(ctx, open, "laptop"); err == nil || !unresolvable(err) {
|
||||
t.Fatalf("laptop resolves with two claimants of one seat: %v", err)
|
||||
}
|
||||
if n := strings.Count(said, "laptop does not resolve now"); n != 1 {
|
||||
t.Errorf("the answer says laptop does not resolve %d times, want once:\n%s", n, said)
|
||||
}
|
||||
if strings.Contains(said, "is left out of the rest of the mesh") {
|
||||
t.Errorf("the answer carries a line from a read that is not its resolution:\n%s", said)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnassignThatLeavesItsNodeResolvedSaysNothingOfIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
aProviderAndItsConsumer(t, open)
|
||||
if _, err := assign(ctx, open, "laptop", "db", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said, err := unassign(ctx, open, "laptop", "app")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := planFor(ctx, open, "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(said, "does not resolve") || strings.Contains(said, "could not be read") {
|
||||
t.Errorf("a node that resolves is said not to:\n%s", said)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAssignThatLeavesItsNodeUnresolvedSaysOnlyThePlansRefusal(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
twoRivalsAndANote(t, open)
|
||||
if _, err := assign(ctx, open, "laptop", "rival-one"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Kept and not resolvable: a second claimant of the seat, which assignment keeps.
|
||||
said, err := assign(ctx, open, "laptop", "rival-two")
|
||||
if err == nil {
|
||||
t.Fatalf("two claimants of one node seat resolved:\n%s", said)
|
||||
}
|
||||
_, _, planErr := planFor(ctx, open, "laptop")
|
||||
if planErr == nil || planErr.Error() != err.Error() {
|
||||
t.Errorf("the act's refusal is not the plan's right after:\nact: %v\nplan: %v", err, planErr)
|
||||
}
|
||||
if strings.Contains(said, "now has") {
|
||||
t.Errorf("the answer says a seat is now held on a node that does not resolve:\n%s", said)
|
||||
}
|
||||
}
|
||||
|
||||
// The other readers of the whole mesh still say a machine they leave out, on standard error (novox/hq
|
||||
// issue 188); an act sends it elsewhere.
|
||||
func TestAMachineLeftOutIsSaidWhereTheContextSays(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
sayLeftOut(leftOutSaidBy(context.Background(), &buf), "laptop", errors.New("why"))
|
||||
if buf.String() != "laptop is left out of the rest of the mesh: it does not resolve: why\n" {
|
||||
t.Errorf("said %q", buf.String())
|
||||
}
|
||||
got := answered(t, func() error {
|
||||
sayLeftOut(context.Background(), "laptop", errors.New("why"))
|
||||
return nil
|
||||
})
|
||||
if !strings.Contains(got, "laptop is left out") {
|
||||
t.Errorf("without an act's writer the line did not reach standard error: %q", got)
|
||||
}
|
||||
}
|
||||
@@ -1,261 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A part that waits for the operator's secret or setting (novox/hq ADR 0283, issue 386).
|
||||
//
|
||||
// **A module's tool check may say it waits**: nothing of it is wrong but a part that cannot work until the operator
|
||||
// gives one of its own secrets or one of its settings. The node-engine states such a resource `waiting`, with what
|
||||
// it waits for. A module saying so is an assertion, so **the controller checks each wait before it excuses it**:
|
||||
//
|
||||
// - a wait for a secret names an own secret the module's manifest declares — by its name, or as a member of a
|
||||
// secret family — said `"issued-by": "outside"`, and the store holds no value a person gave for it on that
|
||||
// machine;
|
||||
// - a wait for a setting names a setting the manifest declares (checked by name only: the controller cannot tell
|
||||
// whether a free-form value covers a part, and the needs-operator condition is where a false one shows).
|
||||
//
|
||||
// An excused wait is read by the first-node gate as *waits for a person* (ADR 0254), a pass carried in the verdict,
|
||||
// for any build of the module — a secret not given is owed by every build alike. It is said to the operator as
|
||||
// `module.<module>.<machine>.needs-operator`, naming the act. A wait that fails the check is judged unhealthy, saying
|
||||
// why, and raises the module's `unhealthy` condition.
|
||||
|
||||
// kindNeedsOperator is a module's condition while a part of it waits for the operator's secret or setting.
|
||||
const kindNeedsOperator = "needs-operator"
|
||||
|
||||
// needsOperatorKey is a module's needs-operator condition on a machine.
|
||||
func needsOperatorKey(module, node string) string {
|
||||
return conditions.Key(conditions.ScopeModule, module+"."+node, kindNeedsOperator)
|
||||
}
|
||||
|
||||
// operatorWaitFacts is what the controller holds to check a module's waits: the manifest judged per module, and per
|
||||
// "<module>@<machine>" the own secrets a person gave there, with when. A module or a machine absent is not known,
|
||||
// and no wait of it is excused.
|
||||
type operatorWaitFacts struct {
|
||||
manifests map[string]catalogue.Manifest
|
||||
given map[string]map[string]time.Time
|
||||
}
|
||||
|
||||
// checkWait is nil when a wait is excused, and otherwise why not, in words. Pure.
|
||||
func checkWait(module, machine string, w inventory.Wait, f operatorWaitFacts) error {
|
||||
m, known := f.manifests[module]
|
||||
if !known {
|
||||
return fmt.Errorf("says it waits for %s, and the mesh holds no manifest of %s to check it against", waitNames(w), module)
|
||||
}
|
||||
switch {
|
||||
case w.Secret != "" && w.Setting != "", w.Secret == "" && w.Setting == "":
|
||||
return fmt.Errorf("says it waits, naming %s, where a wait names one secret or one setting", waitNames(w))
|
||||
case w.Setting != "":
|
||||
if _, declared := m.Settings[w.Setting]; !declared {
|
||||
return fmt.Errorf("says it waits for the setting %s, which %s does not declare", w.Setting, module)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
own, _, declared := m.OwnSecrets.Lookup(w.Secret)
|
||||
if !declared {
|
||||
return fmt.Errorf("says it waits for the secret %s, which %s does not declare", w.Secret, module)
|
||||
}
|
||||
if own.IssuedBy != catalogue.IssuedOutside {
|
||||
return fmt.Errorf("says it waits for the secret %s, which the mesh makes itself: only a secret issued outside "+
|
||||
"the mesh waits for the operator", w.Secret)
|
||||
}
|
||||
given, readable := f.given[module+"@"+machine]
|
||||
if !readable {
|
||||
return fmt.Errorf("says it waits for the secret %s, and what was given on %s could not be read", w.Secret, machine)
|
||||
}
|
||||
if at, was := given[w.Secret]; was {
|
||||
return fmt.Errorf("says it waits for the secret %s, which was given at %s", w.Secret,
|
||||
at.UTC().Format("2006-01-02 15:04 MST"))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// waitRefusedPrefix opens every reason checkWait gives, so the words of a refused wait are told from a check's own.
|
||||
const waitRefusedPrefix = "says it waits"
|
||||
|
||||
// waitNames is what a wait names, as "the secret x" or "the setting y".
|
||||
func waitNames(w inventory.Wait) string {
|
||||
switch {
|
||||
case w.Secret != "" && w.Setting != "":
|
||||
return "the secret " + w.Secret + " and the setting " + w.Setting
|
||||
case w.Secret != "":
|
||||
return "the secret " + w.Secret
|
||||
case w.Setting != "":
|
||||
return "the setting " + w.Setting
|
||||
}
|
||||
return "nothing"
|
||||
}
|
||||
|
||||
// checkWaiting reads one machine's resources against the facts: every waiting resource whose waits all check out is
|
||||
// kept as said; one with a wait that does not, or with no wait at all, is answered as unhealthy with why. Pure; the
|
||||
// statement as kept is not changed.
|
||||
func checkWaiting(machine string, rs []inventory.ResourceHealth, f operatorWaitFacts) []inventory.ResourceHealth {
|
||||
out := make([]inventory.ResourceHealth, 0, len(rs))
|
||||
for _, r := range rs {
|
||||
if r.State == link.StateWaiting {
|
||||
var why error
|
||||
if len(r.Waits) == 0 {
|
||||
why = fmt.Errorf("says it waits, and names nothing it waits for")
|
||||
}
|
||||
for _, w := range r.Waits {
|
||||
if why == nil {
|
||||
why = checkWait(r.Module, machine, w, f)
|
||||
}
|
||||
}
|
||||
if why != nil {
|
||||
r.State, r.Reason = link.StateUnhealthy, why.Error()
|
||||
}
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// operatorWait is whether everything not healthy of a module on a machine is waiting with its waits checked
|
||||
// (checkWaiting already applied), and those waits. A module with anything unhealthy, starting or unknown beside it
|
||||
// does not wait: it is judged as before.
|
||||
func operatorWait(module string, rs []inventory.ResourceHealth) ([]inventory.Wait, bool) {
|
||||
var waits []inventory.Wait
|
||||
for _, r := range rs {
|
||||
if r.Module != module {
|
||||
continue
|
||||
}
|
||||
switch r.State {
|
||||
case link.StateHealthy:
|
||||
case link.StateWaiting:
|
||||
waits = append(waits, r.Waits...)
|
||||
default:
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
return waits, len(waits) > 0
|
||||
}
|
||||
|
||||
// operatorWaitSaid is a module's wait for the operator in one sentence, for the gate's verdict and the condition's
|
||||
// summary: what the operator gives and what it names, and for a secret the line that opens the desk prompt.
|
||||
func operatorWaitSaid(module, machine string, waits []inventory.Wait) string {
|
||||
var parts []string
|
||||
for _, w := range waits {
|
||||
part := fmt.Sprintf("%s (%s", w.What, waitNames(w))
|
||||
if w.Secret != "" {
|
||||
part += fmt.Sprintf(", given with `nox secret ask %s %s %s`", machine, module, w.Secret)
|
||||
}
|
||||
parts = append(parts, part+")")
|
||||
}
|
||||
return fmt.Sprintf("%s on %s waits for the operator: %s", module, machine, strings.Join(parts, "; "))
|
||||
}
|
||||
|
||||
// needsOperatorObservation is a module whose only parts not healthy wait for the operator (ADR 0283): the operator's,
|
||||
// a warning however long it stands, its plain words naming the act and never saying there is nothing to do.
|
||||
func needsOperatorObservation(module, node string, waits []inventory.Wait, rs []inventory.ResourceHealth) conditions.Observation {
|
||||
o := moduleUnhealthyObservation(module, node, rs)
|
||||
o.Token, o.Kind, o.Resolver, o.Severity = kindNeedsOperator, kindNeedsOperator, conditions.ResolverOperator, conditions.Warning
|
||||
o.Summary = operatorWaitSaid(module, node, waits)
|
||||
w := needsOperatorWords(module, node, waits)
|
||||
o.Headline, o.Explanation, o.Needs, o.Resolved, o.Actions = w.Headline, w.Explanation, w.Needs, w.Resolved, nil
|
||||
return o
|
||||
}
|
||||
|
||||
// needsOperatorWords is what the operator reads of a module waiting for them (ADR 0253, ADR 0283): the act, for a
|
||||
// secret typed at the machine's desk prompt and for a setting approved when an agent proposes it. The secret's and
|
||||
// the setting's names, and the line, are in the summary for whoever looks closer.
|
||||
func needsOperatorWords(module, node string, waits []inventory.Wait) words {
|
||||
var acts []string
|
||||
seen := map[string]bool{}
|
||||
secret := false
|
||||
for _, w := range waits {
|
||||
var act string
|
||||
switch {
|
||||
case w.Secret != "":
|
||||
act, secret = fmt.Sprintf("type %s at %s's desk prompt", w.What, node), true
|
||||
case w.Setting != "":
|
||||
act = fmt.Sprintf("approve %s of %s on %s when it is proposed to you", w.Setting, module, node)
|
||||
}
|
||||
if act != "" && !seen[act] {
|
||||
seen[act] = true
|
||||
acts = append(acts, act)
|
||||
}
|
||||
}
|
||||
needs := strings.Join(acts, "; and ") + "."
|
||||
// Plain words hold one sentence of at most conditions.NeedsMax characters: several acts are named in the
|
||||
// summary instead.
|
||||
if len(acts) == 0 || len(needs) > conditions.NeedsMax {
|
||||
needs = fmt.Sprintf("give what %s waits for on %s; the details name each secret and setting.", module, node)
|
||||
}
|
||||
explanation := fmt.Sprintf("Part of %s on %s cannot work until you give what it waits for.", module, node)
|
||||
if secret {
|
||||
explanation += " A hidden prompt opens at the desk when the secret is asked for, and what you type there " +
|
||||
"is sealed to the machine."
|
||||
}
|
||||
explanation += " Its update is in place and nothing was undone; it carries on by itself once it is given."
|
||||
return words{
|
||||
Headline: fmt.Sprintf("%s waits for you on %s", module, node),
|
||||
Needs: needs,
|
||||
Explanation: explanation,
|
||||
Resolved: fmt.Sprintf("%s on %s no longer waits for you", module, node),
|
||||
}
|
||||
}
|
||||
|
||||
// readWaitFacts reads what the controller holds to check the waits of the modules named on one machine: the
|
||||
// manifests (the catalogue's, or those given) and the secrets given there. A read that fails leaves that module
|
||||
// unknown, so none of its waits is excused.
|
||||
func readWaitFacts(ctx context.Context, inv *inventory.Inventory, machine string, modules []string,
|
||||
manifests map[string]catalogue.Manifest, f *operatorWaitFacts) {
|
||||
if f.manifests == nil {
|
||||
f.manifests = map[string]catalogue.Manifest{}
|
||||
}
|
||||
if f.given == nil {
|
||||
f.given = map[string]map[string]time.Time{}
|
||||
}
|
||||
if inv == nil {
|
||||
return
|
||||
}
|
||||
var shelf map[string]catalogue.Manifest
|
||||
sort.Strings(modules)
|
||||
for _, module := range modules {
|
||||
if _, has := f.manifests[module]; !has {
|
||||
if m, given := manifests[module]; given {
|
||||
f.manifests[module] = m
|
||||
} else {
|
||||
if shelf == nil {
|
||||
var err error
|
||||
if shelf, err = inv.Catalogue(ctx); err != nil {
|
||||
shelf = map[string]catalogue.Manifest{}
|
||||
}
|
||||
}
|
||||
if m, known := shelf[module]; known {
|
||||
f.manifests[module] = m
|
||||
}
|
||||
}
|
||||
}
|
||||
if _, read := f.given[module+"@"+machine]; read {
|
||||
continue
|
||||
}
|
||||
if given, err := inv.GivenOwnSecrets(ctx, machine, module); err == nil {
|
||||
f.given[module+"@"+machine] = given
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// waitingModules is every module with a waiting resource in a statement.
|
||||
func waitingModules(rs []inventory.ResourceHealth) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, r := range rs {
|
||||
if r.State == link.StateWaiting && r.Module != "" && !seen[r.Module] {
|
||||
seen[r.Module] = true
|
||||
out = append(out, r.Module)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1,264 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A part that waits for the operator's secret or setting (novox/hq ADR 0283, issue 386).
|
||||
|
||||
var mountsManifest = catalogue.Manifest{Module: "mounts", Version: "1",
|
||||
Settings: map[string]catalogue.SettingDeclaration{"smb-users": {}, "sources": {}},
|
||||
OwnSecrets: catalogue.OwnSecrets{
|
||||
"smb-password-*": {Path: "/s/smb-password-*.secret", IssuedBy: catalogue.IssuedOutside},
|
||||
"broker": {Path: "/s/broker"},
|
||||
"made": {Path: "/s/made", Taken: catalogue.TakenAtStart},
|
||||
"licence": {Path: "/s/licence", IssuedBy: catalogue.IssuedOutside},
|
||||
}}
|
||||
|
||||
var passwordWait = inventory.Wait{Part: "the source games", Secret: "smb-password-games",
|
||||
What: "the password of the source games"}
|
||||
|
||||
var usernameWait = inventory.Wait{Part: "the source games", Setting: "smb-users", What: "the username of the source games"}
|
||||
|
||||
func waitingResource(waits ...inventory.Wait) inventory.ResourceHealth {
|
||||
return inventory.ResourceHealth{Module: "mounts", Resource: "mounts.watch", Kind: "process",
|
||||
Target: "mesh-mounts-watch.service", State: link.StateWaiting, Check: "tool",
|
||||
Reason: "the source games waits for its password", Waits: waits}
|
||||
}
|
||||
|
||||
func factsGiven(given map[string]time.Time) operatorWaitFacts {
|
||||
return operatorWaitFacts{manifests: map[string]catalogue.Manifest{"mounts": mountsManifest},
|
||||
given: map[string]map[string]time.Time{"mounts@workstation": given}}
|
||||
}
|
||||
|
||||
// Rule 3: a wait is excused only when what it names is the module's, issued outside the mesh, and not given there.
|
||||
func TestAWaitIsExcusedOnlyWhenItChecksOut(t *testing.T) {
|
||||
at := time.Date(2026, 10, 10, 15, 8, 0, 0, time.UTC)
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
w inventory.Wait
|
||||
f operatorWaitFacts
|
||||
says string // "" when excused
|
||||
}{
|
||||
{"a member of an outside family, not given", passwordWait, factsGiven(nil), ""},
|
||||
{"an outside secret by name, not given", inventory.Wait{Part: "p", Secret: "licence", What: "w"}, factsGiven(nil), ""},
|
||||
{"a declared setting", usernameWait, factsGiven(nil), ""},
|
||||
{"a member given", passwordWait, factsGiven(map[string]time.Time{"smb-password-games": at}), "was given at 2026-10-10 15:08"},
|
||||
{"a secret not declared", inventory.Wait{Part: "p", Secret: "smb-credentials", What: "w"}, factsGiven(nil), "does not declare"},
|
||||
{"a secret the mesh makes", inventory.Wait{Part: "p", Secret: "made", What: "w"}, factsGiven(nil), "mesh makes itself"},
|
||||
{"the bus account", inventory.Wait{Part: "p", Secret: "broker", What: "w"}, factsGiven(nil), "mesh makes itself"},
|
||||
{"a setting not declared", inventory.Wait{Part: "p", Setting: "logins", What: "w"}, factsGiven(nil), "does not declare"},
|
||||
{"both", inventory.Wait{Part: "p", Secret: "licence", Setting: "smb-users", What: "w"}, factsGiven(nil), "one secret or one setting"},
|
||||
{"neither", inventory.Wait{Part: "p", What: "w"}, factsGiven(nil), "one secret or one setting"},
|
||||
{"no manifest known", passwordWait, operatorWaitFacts{}, "no manifest"},
|
||||
{"what was given cannot be read", passwordWait,
|
||||
operatorWaitFacts{manifests: map[string]catalogue.Manifest{"mounts": mountsManifest}}, "could not be read"},
|
||||
} {
|
||||
err := checkWait("mounts", "workstation", c.w, c.f)
|
||||
switch {
|
||||
case c.says == "" && err != nil:
|
||||
t.Errorf("%s: refused: %v", c.name, err)
|
||||
case c.says != "" && (err == nil || !strings.Contains(err.Error(), c.says)):
|
||||
t.Errorf("%s: %v; want a refusal saying %q", c.name, err, c.says)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A waiting resource whose wait does not check out, or that names nothing, is judged unhealthy, saying why.
|
||||
func TestAWaitThatFailsItsCheckIsUnhealthy(t *testing.T) {
|
||||
given := factsGiven(map[string]time.Time{"smb-password-games": time.Now()})
|
||||
got := checkWaiting("workstation", []inventory.ResourceHealth{waitingResource(passwordWait)}, given)
|
||||
if got[0].State != link.StateUnhealthy || !strings.Contains(got[0].Reason, "was given") {
|
||||
t.Fatalf("a wait for a secret given: %+v", got[0])
|
||||
}
|
||||
got = checkWaiting("workstation", []inventory.ResourceHealth{waitingResource()}, factsGiven(nil))
|
||||
if got[0].State != link.StateUnhealthy || !strings.Contains(got[0].Reason, "names nothing") {
|
||||
t.Fatalf("a wait naming nothing: %+v", got[0])
|
||||
}
|
||||
got = checkWaiting("workstation", []inventory.ResourceHealth{waitingResource(passwordWait, usernameWait)}, factsGiven(nil))
|
||||
if got[0].State != link.StateWaiting {
|
||||
t.Fatalf("two waits that check out: %+v", got[0])
|
||||
}
|
||||
}
|
||||
|
||||
// Rule 4: the gate passes a module whose only parts not healthy wait for the operator, carrying the wait; anything
|
||||
// else beside it is judged as before; and any build is excused, not only one that added something.
|
||||
func TestTheGatePassesAWaitForTheOperatorCarriedAlong(t *testing.T) {
|
||||
now := time.Now()
|
||||
since := now.Add(-time.Minute)
|
||||
healthy := inventory.ResourceHealth{Module: "mounts", Resource: "mounts.apply", Kind: "process",
|
||||
Target: "mesh-mounts-apply.service", State: link.StateHealthy}
|
||||
f := gateFacts{now: now, waits: factsGiven(nil), groupsAdded: map[string]bool{"mounts": false},
|
||||
health: map[string]inventory.NodeHealth{"workstation": {Node: "workstation", HeardAt: now,
|
||||
Resources: []inventory.ResourceHealth{healthy, waitingResource(passwordWait)}}}}
|
||||
h, why := moduleHealthWord("mounts", "workstation", since, f)
|
||||
if h != healthPerson || !strings.Contains(why, "waits for the operator: the password of the source games") ||
|
||||
!strings.Contains(why, "nox secret ask workstation mounts smb-password-games") {
|
||||
t.Fatalf("an excused wait reads %v %q; want a wait for a person naming the act", h, why)
|
||||
}
|
||||
// A second resource unhealthy beside it: not yet, as before.
|
||||
down := healthy
|
||||
down.State, down.Reason = link.StateUnhealthy, "down"
|
||||
f.health["workstation"] = inventory.NodeHealth{Node: "workstation", HeardAt: now,
|
||||
Resources: []inventory.ResourceHealth{down, waitingResource(passwordWait)}}
|
||||
if h, why := moduleHealthWord("mounts", "workstation", since, f); h != healthNotYet {
|
||||
t.Fatalf("a resource down beside the wait reads %v %q", h, why)
|
||||
}
|
||||
// The password given and the module still saying it waits: not excused.
|
||||
f.waits = factsGiven(map[string]time.Time{"smb-password-games": now})
|
||||
f.health["workstation"] = inventory.NodeHealth{Node: "workstation", HeardAt: now,
|
||||
Resources: []inventory.ResourceHealth{healthy, waitingResource(passwordWait)}}
|
||||
if h, why := moduleHealthWord("mounts", "workstation", since, f); h != healthNotYet || !strings.Contains(why, "was given") {
|
||||
t.Fatalf("a wait for a secret given reads %v %q", h, why)
|
||||
}
|
||||
// Facts never read (no manifest): never excused.
|
||||
f.waits = operatorWaitFacts{}
|
||||
if h, _ := moduleHealthWord("mounts", "workstation", since, f); h != healthNotYet {
|
||||
t.Fatalf("a wait nothing could check reads %v", h)
|
||||
}
|
||||
}
|
||||
|
||||
func needsOperatorOpen(t *testing.T, k *conditions.Keeper) (*conditions.Condition, []conditions.Condition) {
|
||||
t.Helper()
|
||||
open, err := k.Open(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i, c := range open {
|
||||
if c.Key == needsOperatorKey("mounts", "workstation") {
|
||||
return &open[i], open
|
||||
}
|
||||
}
|
||||
return nil, open
|
||||
}
|
||||
|
||||
// Rule 5: two statements of an excused wait raise needs-operator, the operator's, a warning however long, naming the
|
||||
// act; a statement without it clears it.
|
||||
func TestTheNeedsOperatorConditionNamesTheAct(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
ctx := t.Context()
|
||||
rs := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}}
|
||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 1}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, _ := needsOperatorOpen(t, k); got != nil {
|
||||
t.Fatal("raised on one statement")
|
||||
}
|
||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 2}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, open := needsOperatorOpen(t, k)
|
||||
if got == nil {
|
||||
t.Fatalf("not raised on two statements: %+v", open)
|
||||
}
|
||||
for _, c := range open {
|
||||
if c.Kind == kindModuleUnhealthy {
|
||||
t.Fatalf("raised as a fault too: %+v", c)
|
||||
}
|
||||
}
|
||||
if got.Kind != kindNeedsOperator || got.Resolver != conditions.ResolverOperator || got.Severity != conditions.Warning {
|
||||
t.Fatalf("the condition: %+v", got)
|
||||
}
|
||||
if !strings.Contains(got.Needs, "type the password of the source games at workstation's desk prompt") ||
|
||||
!strings.Contains(got.Explanation, "hidden prompt opens at the desk") {
|
||||
t.Fatalf("its needs do not name the act: %q", got.Needs)
|
||||
}
|
||||
if strings.Contains(strings.ToLower(got.Explanation), "nothing for you") || !strings.Contains(got.Explanation, "nothing was undone") {
|
||||
t.Fatalf("its explanation: %q", got.Explanation)
|
||||
}
|
||||
if !strings.Contains(got.Summary, "smb-password-games") || !strings.Contains(got.Summary, "nox secret ask workstation mounts smb-password-games") {
|
||||
t.Fatalf("its summary does not name the secret and the line: %q", got.Summary)
|
||||
}
|
||||
// Long open is still a warning: only the operator can end it.
|
||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 3}, time.Now().Add(48*time.Hour)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, _ := needsOperatorOpen(t, k); got == nil || got.Severity == conditions.Urgent {
|
||||
t.Fatalf("after two days: %+v", got)
|
||||
}
|
||||
// Given: the next statement does not say it, and it clears.
|
||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, _ := needsOperatorOpen(t, k); got != nil {
|
||||
t.Fatal("not cleared once given")
|
||||
}
|
||||
}
|
||||
|
||||
func TestASettingsWaitAsksForTheApproval(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
ctx := t.Context()
|
||||
rs := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(usernameWait)}}
|
||||
for i := 1; i <= 2; i++ {
|
||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": i}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, _ := needsOperatorOpen(t, k)
|
||||
if got == nil || !strings.Contains(got.Needs, "approve smb-users of mounts on workstation when it is proposed to you") {
|
||||
t.Fatalf("the condition: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A wait that fails its check is the module's own fault: unhealthy, with why, and no needs-operator.
|
||||
func TestAWaitThatFailsItsCheckRaisesUnhealthy(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
ctx := t.Context()
|
||||
checked := checkWaiting("workstation", []inventory.ResourceHealth{waitingResource(passwordWait)},
|
||||
factsGiven(map[string]time.Time{"smb-password-games": time.Now()}))
|
||||
rs := map[string][]inventory.ResourceHealth{"mounts": checked}
|
||||
for i := 1; i <= 2; i++ {
|
||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": i}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, open := needsOperatorOpen(t, k)
|
||||
if got != nil {
|
||||
t.Fatalf("a wait for a secret given raised needs-operator: %+v", got)
|
||||
}
|
||||
unhealthy := false
|
||||
for _, c := range open {
|
||||
unhealthy = unhealthy || c.Key == moduleUnhealthyKey("mounts", "workstation")
|
||||
}
|
||||
if !unhealthy {
|
||||
t.Fatalf("not raised as unhealthy: %+v", open)
|
||||
}
|
||||
}
|
||||
|
||||
// A module that waited and is then broken says so when the wait clears, and the other way round.
|
||||
func TestANeedsOperatorThatBecameUnhealthySaysSo(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
ctx := t.Context()
|
||||
waiting := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}}
|
||||
for i := 1; i <= 2; i++ {
|
||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", waiting, map[string]int{"mounts": i}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
broken := waitingResource()
|
||||
broken.State, broken.Reason, broken.Waits = link.StateUnhealthy, "the source games refused its login", nil
|
||||
for i := 3; i <= 4; i++ {
|
||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": {broken}},
|
||||
map[string]int{"mounts": i}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, open := needsOperatorOpen(t, k)
|
||||
if got != nil {
|
||||
t.Fatal("needs-operator still open after it became a fault")
|
||||
}
|
||||
found := false
|
||||
for _, c := range open {
|
||||
found = found || c.Key == moduleUnhealthyKey("mounts", "workstation")
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("the fault is not raised: %+v", open)
|
||||
}
|
||||
}
|
||||
@@ -157,7 +157,7 @@ func TestAMergeRebuildsTheModulesItChanged(t *testing.T) {
|
||||
{"a file directly among the modules", merge([]string{"modules/README.md"}, false), ""},
|
||||
{"a root file beside a module's", merge([]string{"merge-check.sh", "modules/keycloak/x.ts"}, false), "keycloak"},
|
||||
} {
|
||||
if got := named(whatTheMergeTouched(candidates, known, c.m, nil)); got != c.want {
|
||||
if got := named(whatTheMergeTouched(candidates, known, c.m)); got != c.want {
|
||||
t.Errorf("%s: rebuilt %q, wanted %q", c.what, got, c.want)
|
||||
}
|
||||
}
|
||||
@@ -285,7 +285,7 @@ func TestAChangeInsideAModuleIsThatModulesHeldOrNot(t *testing.T) {
|
||||
{"an old announcer saying nothing", merge(showcase, nil, false), ""},
|
||||
{"a manifest the merge removed, said or not", merge([]string{"modules/gone/module.json", "modules/gone/x.ts"}, nil, true), ""},
|
||||
} {
|
||||
if got := named(whatTheMergeTouched(candidates, known, c.m, nil)); got != c.want {
|
||||
if got := named(whatTheMergeTouched(candidates, known, c.m)); got != c.want {
|
||||
t.Errorf("%s: rebuilt %q, wanted %q", c.what, got, c.want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,125 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A delivery over its budget is loud (novox/hq ADR 0282 decision 7, issue 382): the self-check reads
|
||||
// mesh-delivery's `times` and raises the warning `delivery.<class>.over-budget` when the newest delivery of a
|
||||
// class whose delivery time is known took longer than its class's budget — three minutes for a leaf module (ADR
|
||||
// 0304 decision 4), ten for a core module — naming the delivery and its longest phase. It clears when the next delivery of that class
|
||||
// lands within its budget. Measurement only: the condition says, it never holds or acts on a delivery.
|
||||
|
||||
// probeBudgetsID is the probe that reads the delivery times.
|
||||
const probeBudgetsID = "D16"
|
||||
|
||||
// kindOverBudget is what a delivery over its class's budget raises.
|
||||
const kindOverBudget = "over-budget"
|
||||
|
||||
// deliveryBudgets are the budgets by class (ADR 0282 decision 1, the leaf's as ADR 0304 decision 4 changed it);
|
||||
// the probe raises nothing for another class.
|
||||
var deliveryBudgets = map[string]time.Duration{inventory.ClassLeaf: 3 * time.Minute, inventory.ClassCore: 10 * time.Minute}
|
||||
|
||||
// timesAnswer is what mesh-delivery's `times` answers, as far as the probe reads it.
|
||||
type timesAnswer struct {
|
||||
Classes []timesClass `json:"classes"`
|
||||
}
|
||||
|
||||
// timesClass is one class's line of `times`.
|
||||
type timesClass struct {
|
||||
Class string `json:"class"`
|
||||
Latest *timesLatest `json:"latest,omitempty"`
|
||||
}
|
||||
|
||||
// timesLatest is the newest delivery of a class whose delivery time is known.
|
||||
type timesLatest struct {
|
||||
ID string `json:"id"`
|
||||
TookMS int64 `json:"took_ms"`
|
||||
Longest string `json:"longest,omitempty"`
|
||||
Landed string `json:"landed,omitempty"`
|
||||
}
|
||||
|
||||
// deliveryTimes is what the delivery's owner says of its delivery times; nothing when no holder is on record or
|
||||
// none answers (D3 says that one).
|
||||
func deliveryTimes(ctx context.Context, conn *nats.Conn, held bool) (*timesAnswer, error) {
|
||||
if !held {
|
||||
return nil, nil
|
||||
}
|
||||
raw, err := askDeliveryOwner(ctx, conn, "times", map[string]any{})
|
||||
if errors.Is(err, link.ErrNothingServes) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var a timesAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("%s.times answered something unreadable: %w", catalogue.DeliverySeat, err)
|
||||
}
|
||||
return &a, nil
|
||||
}
|
||||
|
||||
// overBudgetObservations are the conditions of the classes whose newest delivery took longer than its budget:
|
||||
// strictly longer, so a delivery of exactly its budget is within it.
|
||||
func overBudgetObservations(a *timesAnswer) []conditions.Observation {
|
||||
if a == nil {
|
||||
return nil
|
||||
}
|
||||
var out []conditions.Observation
|
||||
for _, c := range a.Classes {
|
||||
budget, ok := deliveryBudgets[c.Class]
|
||||
if !ok || c.Latest == nil {
|
||||
continue
|
||||
}
|
||||
took := time.Duration(c.Latest.TookMS) * time.Millisecond
|
||||
if took <= budget {
|
||||
continue
|
||||
}
|
||||
longest := c.Latest.Longest
|
||||
if longest == "" {
|
||||
longest = "not known"
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeDelivery, ID: c.Class, Kind: kindOverBudget,
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the %s delivery %s took %s from its merge to running everywhere, over its budget of %s; "+
|
||||
"its longest phase: %s — `mesh-delivery.times`", c.Class, c.Latest.ID, humanDuration(took),
|
||||
humanDuration(budget), longest),
|
||||
Said: fmt.Sprintf("%s took %s (budget %s), longest phase %s", c.Latest.ID, took.Round(time.Second), budget, longest),
|
||||
Headline: fmt.Sprintf("A %s delivery took %s, over its %s budget", c.Class, humanDuration(took),
|
||||
humanDuration(budget)),
|
||||
Explanation: fmt.Sprintf("The delivery %s took %s from its merge until every machine ran it; a %s module is "+
|
||||
"held to %s (ADR 0282). Most of the time went to %s. Nothing was held or changed because of this: it is "+
|
||||
"a measurement.", c.Latest.ID, humanDuration(took), c.Class, humanDuration(budget), longest),
|
||||
Resolved: fmt.Sprintf("the next %s delivery lands within %s", c.Class, humanDuration(budget)),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// probeBudgets is D16: the newest delivery of each class lands within its class's budget.
|
||||
func probeBudgets(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
entries, err := d.open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var conn *nats.Conn
|
||||
if d.js != nil {
|
||||
conn = d.js.Conn()
|
||||
}
|
||||
a, err := deliveryTimes(ctx, conn, deliverySeatHeld(entries))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return overBudgetObservations(a), nil
|
||||
}
|
||||
@@ -1,112 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A leaf delivery of 3 minutes 10 seconds raises delivery.leaf.over-budget naming its longest phase; one of
|
||||
// exactly three minutes, a core one of nine and a class without a budget raise nothing (ADR 0282 decision 7,
|
||||
// the leaf's budget as ADR 0304 decision 4 changed it).
|
||||
func TestADeliveryOverItsBudgetIsLoud(t *testing.T) {
|
||||
a := ×Answer{Classes: []timesClass{
|
||||
{Class: inventory.ClassLeaf, Latest: ×Latest{ID: "novox/mesh-catalog@abc", TookMS: (3*time.Minute + 10*time.Second).Milliseconds(),
|
||||
Longest: "judgement 2m40s"}},
|
||||
{Class: inventory.ClassCore, Latest: ×Latest{ID: "novox/mesh-controller@def", TookMS: (9 * time.Minute).Milliseconds(),
|
||||
Longest: "build 1m"}},
|
||||
{Class: "unclassed", Latest: ×Latest{ID: "x@y", TookMS: time.Hour.Milliseconds()}},
|
||||
}}
|
||||
obs := overBudgetObservations(a)
|
||||
if len(obs) != 1 {
|
||||
t.Fatalf("one class over its budget, got %d: %+v", len(obs), obs)
|
||||
}
|
||||
o := obs[0]
|
||||
if o.Key() != "delivery.leaf.over-budget" || !strings.Contains(o.Summary, "judgement 2m40s") ||
|
||||
!strings.Contains(o.Summary, "novox/mesh-catalog@abc") {
|
||||
t.Fatalf("the condition names its delivery and longest phase: %s — %s", o.Key(), o.Summary)
|
||||
}
|
||||
// The next leaf delivery within its budget clears it: the probe raises nothing for the class.
|
||||
a.Classes[0].Latest = ×Latest{ID: "novox/mesh-catalog@ghi", TookMS: (3 * time.Minute).Milliseconds()}
|
||||
if obs := overBudgetObservations(a); len(obs) != 0 {
|
||||
t.Fatalf("a delivery of exactly its budget is within it: %+v", obs)
|
||||
}
|
||||
a.Classes[1].Latest.TookMS = (10*time.Minute + time.Second).Milliseconds()
|
||||
if obs := overBudgetObservations(a); len(obs) != 1 || obs[0].Key() != "delivery.core.over-budget" {
|
||||
t.Fatalf("a core delivery over ten minutes: %+v", obs)
|
||||
}
|
||||
if obs := overBudgetObservations(nil); obs != nil {
|
||||
t.Fatal("no answer raises nothing")
|
||||
}
|
||||
// No delivery of a class with a known time yet: nothing said of it.
|
||||
if obs := overBudgetObservations(×Answer{Classes: []timesClass{{Class: inventory.ClassLeaf}}}); len(obs) != 0 {
|
||||
t.Fatalf("a class with no delivery: %+v", obs)
|
||||
}
|
||||
}
|
||||
|
||||
// The probe's question is one the controller's grant names: a question the bus refuses checks nothing.
|
||||
func TestTheControllerMayAskForTheDeliveryTimes(t *testing.T) {
|
||||
found := false
|
||||
for _, v := range broker.VerbsTheControllerAsksTheDeliveryOwner {
|
||||
found = found || (v.Seat == catalogue.DeliverySeat && v.Verb == "times")
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("the grant does not name mesh-delivery.times")
|
||||
}
|
||||
}
|
||||
|
||||
// A walk's class is core when it walks a module of the controller's own path or one holding the mesh's resolver,
|
||||
// leaf otherwise; its window closed at its batch's window, or its maximum, never after its cut.
|
||||
func TestAWalksClassAndWindowAreReadAtItsCut(t *testing.T) {
|
||||
entries := []inventory.Entry{
|
||||
{Manifest: catalogue.Manifest{Module: "dnsmasq", Claims: []catalogue.Claim{{Name: resolverSeat}}}},
|
||||
{Manifest: catalogue.Manifest{Module: "gitea"}},
|
||||
}
|
||||
walk := func(modules ...string) inventory.Plan {
|
||||
p := inventory.Plan{Modules: map[string]*inventory.PlanModule{}}
|
||||
for _, m := range modules {
|
||||
p.Modules[m] = &inventory.PlanModule{}
|
||||
}
|
||||
return p
|
||||
}
|
||||
for want, w := range map[string]inventory.Plan{
|
||||
inventory.ClassLeaf: walk("gitea"), inventory.ClassCore: walk("gitea", "mesh-controller"),
|
||||
} {
|
||||
if got := classOf(w, entries); got != want {
|
||||
t.Errorf("%v: %s, want %s", w.Modules, got, want)
|
||||
}
|
||||
}
|
||||
if got := classOf(walk("dnsmasq"), entries); got != inventory.ClassCore {
|
||||
t.Errorf("the resolver's holder is core: %s", got)
|
||||
}
|
||||
now := time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC)
|
||||
batch := inventory.Plan{Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{
|
||||
ClosesAt: now.Add(-20 * time.Second), AtMost: now.Add(5 * time.Minute)}}}
|
||||
times := walkTimesAtCut(batch, walk("gitea"), entries, now)
|
||||
if times.Cut == nil || !times.Cut.Equal(now) || times.WindowClosed == nil || !times.WindowClosed.Equal(now.Add(-20*time.Second)) ||
|
||||
times.Class != inventory.ClassLeaf {
|
||||
t.Fatalf("times at the cut: %+v", times)
|
||||
}
|
||||
batch.Delivery.Batch.AtMost = now.Add(-time.Minute)
|
||||
if times := walkTimesAtCut(batch, walk("gitea"), entries, now); !times.WindowClosed.Equal(now.Add(-time.Minute)) {
|
||||
t.Fatalf("a window closed at its maximum: %v", times.WindowClosed)
|
||||
}
|
||||
if times := walkTimesAtCut(inventory.Plan{Delivery: &inventory.PlanDelivery{Alone: true}}, walk("gitea"), entries, now); times.WindowClosed != nil {
|
||||
t.Fatalf("a merge walked alone had no window: %v", times.WindowClosed)
|
||||
}
|
||||
}
|
||||
|
||||
// What each machine of the rest was sent is kept only for the machines the send reached.
|
||||
func TestTheRestIsKeptForTheMachinesTheSendReached(t *testing.T) {
|
||||
got := restOf([]string{"ace", "g14"}, []string{"ace", "shanks"}, map[string]inventory.SentDeclaration{"ace": {Digest: "d1"}})
|
||||
if len(got) != 1 || got["ace"].Digest != "d1" {
|
||||
t.Fatalf("rest: %+v", got)
|
||||
}
|
||||
if restOf([]string{"g14"}, []string{"ace"}, nil) != nil {
|
||||
t.Fatal("no machine reached: nothing kept")
|
||||
}
|
||||
}
|
||||
@@ -221,24 +221,6 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
||||
w := usedAsFoundObservation(orModule(module), machineOr(o, "a machine"), o.Summary, nil)
|
||||
return words{Headline: w.Headline, Explanation: w.Explanation, Needs: w.Needs, Resolved: w.Resolved}
|
||||
}),
|
||||
kindNeedsOperator: worded(func(o conditions.Observation) words {
|
||||
// The observation carries the act itself (ADR 0283); these are its words when only the kind is known.
|
||||
module := ""
|
||||
if o.Scope == conditions.ScopeModule && o.Machine != "" {
|
||||
module = strings.TrimSuffix(o.ID, "."+o.Machine)
|
||||
}
|
||||
node := machineOr(o, "a machine")
|
||||
w := needsOperatorWords(orModule(module), node, nil)
|
||||
return w
|
||||
}),
|
||||
kindLeftOut: worded(func(o conditions.Observation) words {
|
||||
// The observation carries its own words (novox/hq issue 380); these are its kind's alone.
|
||||
module := ""
|
||||
if o.Scope == conditions.ScopeModule && o.Machine != "" {
|
||||
module = strings.TrimSuffix(o.ID, "."+o.Machine)
|
||||
}
|
||||
return leftOutWords(orModule(module), machineOr(o, "a machine"), "")
|
||||
}),
|
||||
kindProviderFailing: worded(func(o conditions.Observation) words {
|
||||
thing, consumer := conditions.ThingWords(o), idPart(o, 2)
|
||||
if consumer == "" {
|
||||
@@ -380,17 +362,6 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
||||
Explanation: "Its walk across the machines has not moved for longer than usual. Nothing is lost.",
|
||||
Resolved: "Resolved: the delivery moves again"}
|
||||
}),
|
||||
kindBatchNotCut: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Merged changes are not being delivered",
|
||||
Explanation: "Merges collected for one delivery should have been planned and were not. Nothing is lost.",
|
||||
Resolved: "Resolved: the merged changes are being delivered"}
|
||||
}),
|
||||
kindBatchBehindWalk: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Merged changes wait behind a slow delivery",
|
||||
Explanation: "Merges collected for the next delivery wait for the delivery before them, which is taking " +
|
||||
"longer than it should. Nothing is lost.",
|
||||
Resolved: "Resolved: the merged changes no longer wait"}
|
||||
}),
|
||||
kindWalkWaiting: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A delivery is waiting to start",
|
||||
Explanation: "A merged change is built, and mesh-delivery (the module that decides when a delivery goes " +
|
||||
@@ -614,19 +585,6 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
||||
Explanation: "The bus refused messages from a part of the mesh, so what they carried did not happen.",
|
||||
Resolved: "Resolved: the bus takes the messages again"}
|
||||
}),
|
||||
kindBucketOrLogFilling: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A module's bucket or log on the bus is filling up",
|
||||
Needs: "decide whether to raise its cap or have the module keep less.",
|
||||
Explanation: "A bucket or log a module keeps on the bus holds three quarters of its cap or more. When it " +
|
||||
"is full, the bus refuses what the module writes there.",
|
||||
Resolved: "It has room again"}
|
||||
}),
|
||||
kindBucketOrLogUnread: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A module's bucket or log could not be read",
|
||||
Explanation: "The controller could not read how full a bucket or log a module keeps on the bus is, so it " +
|
||||
"cannot say whether it is filling up. It asks again at its next check.",
|
||||
Resolved: "It can be read again"}
|
||||
}),
|
||||
"stream-wrong": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Part of the bus's storage is wrong",
|
||||
Needs: "check the machine the bus runs on; the details say what is missing.",
|
||||
@@ -722,8 +680,6 @@ func walkWaitingWords(w waitFacts, in time.Duration, severity conditions.Severit
|
||||
}
|
||||
|
||||
// waitingNeeds is what the operator does about a walk waiting past its urgent bound: nothing before it.
|
||||
// Start and Stop are also asked of the operator (novox/hq ADR 0259); the condition's own words keep saying
|
||||
// where they are given without a channel, and the ask's text drops that (askText).
|
||||
func waitingNeeds(severity conditions.Severity) string {
|
||||
if severity == conditions.Urgent {
|
||||
return "start it, or stop it, " + FromMeshMCPServer
|
||||
@@ -731,20 +687,6 @@ func waitingNeeds(severity conditions.Severity) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// waitingActions are the answers to a walk waiting past its urgent bound: start it, or stop it — the plan's
|
||||
// own verbs, approved by the operator (novox/hq ADR 0259). None before the bound.
|
||||
func waitingActions(plan string, severity conditions.Severity) []conditions.Action {
|
||||
if severity != conditions.Urgent || plan == "" {
|
||||
return nil
|
||||
}
|
||||
return []conditions.Action{
|
||||
{Label: "Start", Verb: "mesh-controller.plans", Level: conditions.LevelApprove,
|
||||
Arguments: map[string]string{"go": plan, "why": "", "cause": conditions.CauseOperatorAnswer}},
|
||||
{Label: "Stop", Verb: "mesh-controller.plans", Level: conditions.LevelApprove,
|
||||
Arguments: map[string]string{"stop": plan, "why": "", "cause": conditions.CauseOperatorAnswer}},
|
||||
}
|
||||
}
|
||||
|
||||
// moduleNeeds is what the operator can do about a module unhealthy on a machine: log in again where its
|
||||
// account's groups wait for it (ADR 0252), restart a failed service, or nothing where the mesh restarts it.
|
||||
// No answer is offered for a restart: a desk click performs only an acknowledgement (ADR 0258).
|
||||
@@ -759,35 +701,11 @@ func moduleNeeds(node string, rs []inventory.ResourceHealth) string {
|
||||
}
|
||||
}
|
||||
if unit != "" {
|
||||
// Also asked of the operator (moduleActions); the ask's text drops where (askText).
|
||||
return fmt.Sprintf("restart its service %s on %s %s", unit, node, FromMeshMCPServer)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// moduleActions are the answers to a module unhealthy on a machine: restart its failed service there,
|
||||
// approved by the operator (novox/hq ADR 0259) — none when the mesh restarts it, or a new login is what it
|
||||
// waits for.
|
||||
func moduleActions(node string, rs []inventory.ResourceHealth) []conditions.Action {
|
||||
for _, r := range rs {
|
||||
if strings.Contains(r.Reason, "relogin needed") {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
for _, r := range rs {
|
||||
if r.Kind != link.KindUnit || r.Target == "" {
|
||||
continue
|
||||
}
|
||||
scope := "system"
|
||||
if r.Account != "" {
|
||||
scope = "user"
|
||||
}
|
||||
return []conditions.Action{{Label: "Restart", Verb: "node-service-manager.restart", Machine: node,
|
||||
Level: conditions.LevelApprove, Arguments: map[string]string{"unit": r.Target, "scope": scope}}}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// FromMeshMCPServer ends what the operator needs when no notification can do it (ADR 0258), naming the mesh MCP
|
||||
// server (the glossary's word; "console" is retired): the answer is not an
|
||||
// acknowledgement, so it is given where the operator is known to be the one asking, until answers are
|
||||
@@ -849,9 +767,6 @@ func stalledWords(l stalledLine, o conditions.Observation) (headline, explanatio
|
||||
"to check it: the forge never announced it. It cannot merge until it is checked.", pull, long),
|
||||
fmt.Sprintf("%s has a merge check now, or is closed", pull), needs, nil
|
||||
}
|
||||
if l.State == "unanswered" {
|
||||
return unansweredWords(l, o)
|
||||
}
|
||||
held := l.State
|
||||
if held == "" {
|
||||
held = "held"
|
||||
@@ -861,19 +776,15 @@ func stalledWords(l stalledLine, o conditions.Observation) (headline, explanatio
|
||||
long = "for " + humanDuration(d)
|
||||
}
|
||||
if o.Resolver == conditions.ResolverOperator {
|
||||
// Asked of the operator, approved on a channel that proves who answered (novox/hq ADR 0259); the
|
||||
// router says where each can be answered, so the words do not.
|
||||
release := conditions.Action{Label: "Release", Verb: "mesh-delivery.release", Level: conditions.LevelApprove,
|
||||
Arguments: map[string]string{"id": l.ID, "why": ""}}
|
||||
stop := conditions.Action{Label: "Stop", Verb: "mesh-delivery.stop", Level: conditions.LevelApprove,
|
||||
Arguments: map[string]string{"id": l.ID, "why": ""}}
|
||||
// Words only: releasing or stopping a delivery is not an acknowledgement, so no desk click
|
||||
// performs it (ADR 0258).
|
||||
switch held {
|
||||
case "held":
|
||||
needs, actions = "release it, or stop it, "+FromMeshMCPServer, []conditions.Action{release, stop}
|
||||
needs = "release it, or stop it, " + FromMeshMCPServer
|
||||
case "ready", "checked":
|
||||
needs = "merge its pull request, or close it."
|
||||
default:
|
||||
needs, actions = "stop it "+FromMeshMCPServer, []conditions.Action{stop}
|
||||
needs = "stop it " + FromMeshMCPServer
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("Delivery of %s %s %s", name, held, long),
|
||||
@@ -881,89 +792,6 @@ func stalledWords(l stalledLine, o conditions.Observation) (headline, explanatio
|
||||
fmt.Sprintf("Delivery of %s is no longer %s", name, held), needs, actions
|
||||
}
|
||||
|
||||
// unansweredWords are the plain words of a pull request's head whose merge check was asked and has not answered
|
||||
// within its bound (novox/hq issue 438): mesh-delivery holds no delivery of it, so there is nothing to stop, release
|
||||
// or close, and no action is offered. The operator's acts are a new commit, which asks the check again, or a look
|
||||
// at the build queue, where a check that never ran may still wait.
|
||||
//
|
||||
// A head whose delivery is held in ready or rejected is said the same way (novox/hq issue 441), and mesh-delivery
|
||||
// names that state as data (Held). Its check answered once, so the build queue is the wrong place to look: the step
|
||||
// is a recheck of the delivery, named by its id, which asks the check again (novox/hq issue 451). It is advised, not
|
||||
// offered as an action, since the controller is not granted the recheck.
|
||||
func unansweredWords(l stalledLine, o conditions.Observation) (headline, explanation, resolved, needs string,
|
||||
actions []conditions.Action) {
|
||||
repository, _, _ := strings.Cut(l.ID, "@")
|
||||
pull := "A pull request of " + repoName(repository)
|
||||
if l.Number > 0 {
|
||||
pull = fmt.Sprintf("Pull request %s #%d", repoName(repository), l.Number)
|
||||
}
|
||||
long, limit := "for too long", ""
|
||||
if d, err := time.ParseDuration(l.For); err == nil {
|
||||
long = "for " + humanDuration(d)
|
||||
}
|
||||
if d, err := time.ParseDuration(l.Bound); err == nil {
|
||||
limit = ", past its limit of " + humanDuration(d)
|
||||
}
|
||||
check := "merge check"
|
||||
if said := uncheckedWaitWords(l); len(said) > 0 {
|
||||
check = "merge check (" + strings.Join(said, ", ") + ")"
|
||||
}
|
||||
which := "Its " + check
|
||||
switch l.Held {
|
||||
case "ready", "rejected":
|
||||
which = "Its delivery is " + l.Held + ", but on the forge its " + check
|
||||
if o.Resolver == conditions.ResolverOperator {
|
||||
needs = "recheck its delivery, which asks its check again — mesh-delivery.recheck with id " + l.ID +
|
||||
" and why — or push a new commit to its branch, " + FromMeshMCPServer
|
||||
}
|
||||
default:
|
||||
if o.Resolver == conditions.ResolverOperator {
|
||||
needs = "push a new commit to its branch, which asks the check again, or see whether its check still " +
|
||||
"waits in the build queue: mesh-controller.queue."
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("%s's merge check has not answered %s", pull, long),
|
||||
fmt.Sprintf("%s is open on a branch that requires the merge check. %s was asked and has not answered %s%s. "+
|
||||
"It cannot merge until its check answers.", pull, which, long, limit),
|
||||
fmt.Sprintf("%s has an answer from its merge check, or is closed", pull), needs, nil
|
||||
}
|
||||
|
||||
// uncheckedWaitWords are the merge checks an unanswered line waits on, as the operator reads them: from the checks
|
||||
// given as data, each time in the controller's local zone, as every other time in a message — "mesh/merge-gate
|
||||
// pending since 02:11" — never the UTC time inside mesh-delivery's finished words, which cannot be said again in
|
||||
// another zone (novox/hq issue 443). A line from a mesh-delivery that gives no such data is said by its words.
|
||||
func uncheckedWaitWords(l stalledLine) []string {
|
||||
if len(l.Checks) == 0 {
|
||||
return l.Waiting
|
||||
}
|
||||
out := make([]string, 0, len(l.Checks))
|
||||
for _, c := range l.Checks {
|
||||
switch c.State {
|
||||
case "never-set":
|
||||
out = append(out, c.Context+" never set")
|
||||
case "pending":
|
||||
out = append(out, c.Context+" pending"+sinceWords(c.Since))
|
||||
default:
|
||||
out = append(out, c.Context+" not answered")
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sinceWords is " since 02:11" in the controller's local zone, with its day when that is not today ("since 23:05 on
|
||||
// 9 Oct"), so the time stays absolute; a time not given, or not readable, is said so and never made up.
|
||||
func sinceWords(since string) string {
|
||||
at, err := time.Parse(time.RFC3339, since)
|
||||
if err != nil {
|
||||
return ", since a time the forge did not say"
|
||||
}
|
||||
local, today := at.In(wordsZone()), wordsNow().In(wordsZone())
|
||||
if local.YearDay() == today.YearDay() && local.Year() == today.Year() {
|
||||
return " since " + local.Format("15:04")
|
||||
}
|
||||
return " since " + local.Format("15:04 on 2 Jan")
|
||||
}
|
||||
|
||||
// causeWords is a hand-act's cause as a person says it.
|
||||
func causeWords(cause string) string {
|
||||
return strings.NewReplacer(".", " ", "_", " ").Replace(cause)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -66,21 +65,13 @@ func TestADeliveryWaitingNeedsNothingUntilItsBoundThenOffersStartAndStop(t *test
|
||||
t.Errorf("the summary lost the way on for whoever looks closer: %q", got[0].Summary)
|
||||
}
|
||||
|
||||
// Past four hours it is urgent, and asks the operator to start or stop it (novox/hq ADR 0259): the plan's
|
||||
// own verbs, approved, which the controller performs on the warrant. The router says where to answer.
|
||||
// Past four hours it is urgent, and offers the controller's own answers.
|
||||
f.waits[0].since = now.Add(-5 * time.Hour)
|
||||
got = watchWaits(f)
|
||||
plainExample(t, got[0], "openrazer delivery waiting to start",
|
||||
"Needs you: start it, or stop it, from the mesh MCP server; this notification cannot do it. The change to openrazer is merged and built, and mesh-delivery (the "+
|
||||
"module that decides when a delivery goes out) has not let it start for 5 hours, so mesh-delivery may "+
|
||||
"be stuck.", "Start", "Stop")
|
||||
for i, want := range []string{"go", "stop"} {
|
||||
a := got[0].Actions[i]
|
||||
if a.Verb != "mesh-controller.plans" || a.Arguments[want] != "plan-1791454185265004861" ||
|
||||
a.Level != conditions.LevelApprove || a.Arguments["cause"] != conditions.CauseOperatorAnswer {
|
||||
t.Errorf("%s: %+v", a.Label, a)
|
||||
}
|
||||
}
|
||||
"be stuck.")
|
||||
|
||||
// Many modules are counted, not listed in the headline.
|
||||
f.waits[0].modules = []string{"a", "b", "c", "d"}
|
||||
@@ -91,20 +82,16 @@ func TestADeliveryWaitingNeedsNothingUntilItsBoundThenOffersStartAndStop(t *test
|
||||
}
|
||||
|
||||
// **A module unhealthy**: "openrazer on g14 is not healthy: its unit openrazer-daemon.service failed in the
|
||||
// account's own service manager (exit-code)". Restarting is not an acknowledgement: it is asked of the
|
||||
// operator at the approve level (novox/hq ADR 0259), so a desk click never performs it (ADR 0258).
|
||||
// account's own service manager (exit-code)". Restarting is not an acknowledgement, so it is said in words
|
||||
// and offered as no answer (ADR 0258).
|
||||
func TestAModuleUnhealthyAsksForARestartInWords(t *testing.T) {
|
||||
o := moduleUnhealthyObservation("openrazer", "g14", []inventory.ResourceHealth{{Kind: link.KindUnit,
|
||||
Resource: "openrazer-daemon", Target: "openrazer-daemon.service", Account: "jochen",
|
||||
Resource: "openrazer-daemon", Target: "openrazer-daemon.service",
|
||||
Reason: "failed in the account's own service manager (exit-code)", Since: time.Now()}})
|
||||
plainExample(t, o, "openrazer not working on g14",
|
||||
"Needs you: restart its service openrazer-daemon on g14 from the mesh MCP server; this notification cannot do it. "+
|
||||
"openrazer on g14 is not healthy: its service openrazer-daemon stopped with an error. It clears as soon "+
|
||||
"as it runs again.", "Restart")
|
||||
if a := o.Actions[0]; a.Verb != "node-service-manager.restart" || a.Machine != "g14" || a.Level != conditions.LevelApprove ||
|
||||
a.Arguments["unit"] != "openrazer-daemon.service" || a.Arguments["scope"] != "user" {
|
||||
t.Errorf("restart: %+v", a)
|
||||
}
|
||||
"as it runs again.")
|
||||
// An account waiting for a new login (ADR 0252) asks for the login, held to the plain rule.
|
||||
o = moduleUnhealthyObservation("openrazer", "g14", []inventory.ResourceHealth{{Kind: "account",
|
||||
Resource: "operator-in-group", Target: "jochen", Reason: "relogin needed: the account is in the group"}})
|
||||
@@ -167,12 +154,7 @@ func TestADeliveryHeldAsksForReleaseOrStopInWords(t *testing.T) {
|
||||
Bound: "24h0m0s", H2: "none: the state is the operator's", Says: "it waits for the operator"}})
|
||||
plainExample(t, got[0], "Delivery of hq held for 36 hours",
|
||||
"Needs you: release it, or stop it, from the mesh MCP server; this notification cannot do it. A delivery of hq has been held for 36 hours, past its limit.",
|
||||
"Release", "Stop")
|
||||
for i, verb := range []string{"mesh-delivery.release", "mesh-delivery.stop"} {
|
||||
if a := got[0].Actions[i]; a.Verb != verb || a.Arguments["id"] != "novox/hq@055550802096" || a.Level != conditions.LevelApprove {
|
||||
t.Errorf("%+v", a)
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
// **Every kind the controller raises has plain words**, and its words are plain for a subject of every
|
||||
@@ -348,153 +330,3 @@ func TestAnUnannouncedPullRequestSaysToPushANewCommit(t *testing.T) {
|
||||
t.Fatalf("it reads %q / %q", o.Headline, o.Explanation)
|
||||
}
|
||||
}
|
||||
|
||||
// **A pull request whose merge check never answered says which check, since when, and what to do** (novox/hq issue
|
||||
// 438): mesh-delivery says one as a stalled line in state `unanswered`, the line below exactly as its test makes it.
|
||||
// No delivery of the head is held, so there is nothing to stop, release or close: no action is offered, and the
|
||||
// operator's acts are a new commit, which asks the check again, or a look at the build queue.
|
||||
func TestAnUnansweredMergeCheckSaysWhichCheckAndWhatToDo(t *testing.T) {
|
||||
var l stalledLine
|
||||
if err := json.Unmarshal([]byte(`{"id":"novox/mesh-controller@c11222026a3b","number":212,"state":"unanswered",`+
|
||||
`"waiting":["mesh/merge-gate pending since 2026-10-11T00:11:39Z","mesh/repo-check never set"],"for":"1h1m0s",`+
|
||||
`"bound":"1h0m0s","h2":"none: no delivery of it is held here, so there is none to close — the operator's",`+
|
||||
`"says":"novox/mesh-controller#212 is open on main, which requires the merge check, and its head's check `+
|
||||
`started and never answered: mesh/merge-gate pending since 2026-10-11T00:11:39Z, mesh/repo-check never set. `+
|
||||
`No delivery of it is held here, so nothing asks it again. A new commit on its branch announces it and asks `+
|
||||
`its check"}`), &l); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
obs := stalledObservations([]stalledLine{l})
|
||||
if len(obs) != 1 || obs[0].Resolver != conditions.ResolverOperator {
|
||||
t.Fatalf("an unanswered merge check is not the operator's: %+v", obs)
|
||||
}
|
||||
o := obs[0]
|
||||
t.Logf("headline: %s\nexplanation: %s\nneeds: %s\nresolved: %s", o.Headline, o.Explanation, o.Needs, o.Resolved)
|
||||
if len(o.Actions) != 0 {
|
||||
t.Fatalf("it offers an action a head with no delivery cannot take: %+v", o.Actions)
|
||||
}
|
||||
if strings.Contains(o.Needs, "stop") || strings.Contains(o.Needs, "release") || !strings.Contains(o.Needs, "commit") ||
|
||||
!strings.Contains(o.Needs, "in the build queue: mesh-controller.queue") || strings.Contains(o.Needs, "build seat") {
|
||||
t.Fatalf("it says to %q", o.Needs)
|
||||
}
|
||||
if o.Headline != "Pull request mesh-controller #212's merge check has not answered for 61 minutes" {
|
||||
t.Fatalf("its headline reads %q", o.Headline)
|
||||
}
|
||||
for _, want := range []string{"mesh/merge-gate pending since 2026-10-11T00:11:39Z", "mesh/repo-check never set",
|
||||
"past its limit of 60 minutes"} {
|
||||
if !strings.Contains(o.Explanation, want) {
|
||||
t.Fatalf("its explanation does not say %q: %q", want, o.Explanation)
|
||||
}
|
||||
}
|
||||
if strings.Contains(o.Explanation, "never asked") || strings.Contains(o.Explanation, "never announced") {
|
||||
t.Fatalf("it says the mesh was never asked, of a head whose check started: %q", o.Explanation)
|
||||
}
|
||||
|
||||
// A line from a mesh-delivery that names no checks still says the check did not answer, and offers nothing.
|
||||
var bare stalledLine
|
||||
if err := json.Unmarshal([]byte(`{"id":"novox/mesh-controller@c11222026a3b","number":212,"state":"unanswered",`+
|
||||
`"for":"1h1m0s","bound":"1h0m0s","h2":"none: no delivery of it is held here, so there is none to close — the operator's"}`),
|
||||
&bare); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
o = stalledObservations([]stalledLine{bare})[0]
|
||||
if len(o.Actions) != 0 || !strings.Contains(o.Explanation, "has not answered") || strings.Contains(o.Headline, "Delivery of") {
|
||||
t.Fatalf("a line naming no checks reads %q / %q, actions %+v", o.Headline, o.Explanation, o.Actions)
|
||||
}
|
||||
}
|
||||
|
||||
// **An unanswered merge check's time reaches the operator in their own time, never as raw UTC** (novox/hq issue
|
||||
// 443): mesh-delivery says each check waited on as data — its context, its state and since when, in RFC 3339 — beside
|
||||
// the finished words it gave before, and the controller words it in its local zone, as every other time in a message.
|
||||
// The line is the one mesh-delivery says, as in the report of issue 443, with the checks it now carries.
|
||||
func TestAnUnansweredMergeChecksTimeIsSaidInLocalTime(t *testing.T) {
|
||||
// The operator's zone given through the seam, never by writing time.Local: other tests' goroutines read it, and
|
||||
// the build seat runs the suite under the race detector.
|
||||
wasZone, wasNow := wordsZone, wordsNow
|
||||
wordsZone = func() *time.Location { return time.FixedZone("CEST", 2*60*60) }
|
||||
wordsNow = func() time.Time { return time.Date(2026, 10, 11, 1, 12, 39, 0, time.UTC) }
|
||||
t.Cleanup(func() { wordsZone, wordsNow = wasZone, wasNow })
|
||||
|
||||
var l stalledLine
|
||||
if err := json.Unmarshal([]byte(`{"id":"novox/mesh-controller@c11222026a3b","number":212,"state":"unanswered",`+
|
||||
`"waiting":["mesh/merge-gate pending since 2026-10-11T00:11:39Z","mesh/repo-check never set"],`+
|
||||
`"checks":[{"context":"mesh/merge-gate","state":"pending","since":"2026-10-11T00:11:39Z"},`+
|
||||
`{"context":"mesh/repo-check","state":"never-set"}],"for":"1h1m0s",`+
|
||||
`"bound":"1h0m0s","h2":"none: no delivery of it is held here, so there is none to close — the operator's",`+
|
||||
`"says":"novox/mesh-controller#212 is open on main, which requires the merge check, and its head's check `+
|
||||
`started and never answered: mesh/merge-gate pending since 2026-10-11T00:11:39Z, mesh/repo-check never set. `+
|
||||
`No delivery of it is held here, so nothing asks it again. A new commit on its branch announces it and asks `+
|
||||
`its check"}`), &l); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
o := stalledObservations([]stalledLine{l})[0]
|
||||
t.Logf("explanation: %s", o.Explanation)
|
||||
for _, raw := range []string{"2026-10-11T00:11:39Z", "00:11", "UTC"} {
|
||||
if strings.Contains(o.Explanation, raw) || strings.Contains(o.Headline, raw) {
|
||||
t.Fatalf("the operator reads %q: %q / %q", raw, o.Headline, o.Explanation)
|
||||
}
|
||||
}
|
||||
for _, want := range []string{"mesh/merge-gate pending since 02:11", "mesh/repo-check never set"} {
|
||||
if !strings.Contains(o.Explanation, want) {
|
||||
t.Fatalf("its explanation does not say %q: %q", want, o.Explanation)
|
||||
}
|
||||
}
|
||||
|
||||
// A check pending since another day than today says which day, so the time stays absolute.
|
||||
l.Checks[0].Since = "2026-10-09T21:05:00Z"
|
||||
if o = stalledObservations([]stalledLine{l})[0]; !strings.Contains(o.Explanation, "mesh/merge-gate pending since 23:05 on 9 Oct") {
|
||||
t.Fatalf("a check pending since an earlier day reads %q", o.Explanation)
|
||||
}
|
||||
|
||||
// A pending check whose time the forge did not say is said so, with no time made up.
|
||||
l.Checks[0].Since = ""
|
||||
if o = stalledObservations([]stalledLine{l})[0]; !strings.Contains(o.Explanation, "mesh/merge-gate pending, since a time the forge did not say") {
|
||||
t.Fatalf("a pending check without its time reads %q", o.Explanation)
|
||||
}
|
||||
}
|
||||
|
||||
// **An unanswered merge check on a ready or rejected delivery is told to recheck it** (novox/hq issue 451):
|
||||
// mesh-delivery says such a line too (issue 441) and names the delivery's state as data, `held`. That check already
|
||||
// answered once, so the build queue is the wrong place to look; a recheck of the delivery, named by its id, asks the
|
||||
// check again. A line with no delivery held keeps the advice of issue 438, and so does a line from a mesh-delivery
|
||||
// that does not say `held` yet: the controller never guesses the state from the words.
|
||||
func TestAnUnansweredMergeCheckOnAReadyOrRejectedDeliveryOffersARecheck(t *testing.T) {
|
||||
line := func(held string) stalledLine {
|
||||
var l stalledLine
|
||||
raw := `{"id":"novox/mesh-controller@c11222026a3b","number":212,"state":"unanswered",` +
|
||||
`"waiting":["mesh/merge-gate pending since 2026-10-11T00:11:39Z"],"for":"1h1m0s","bound":"1h0m0s",` +
|
||||
`"h2":"none: its delivery is ` + held + `, a state the operator moves — the operator's",` +
|
||||
`"says":"novox/mesh-controller#212 is open on main, which requires the merge check, and its delivery is ` +
|
||||
held + ` here, but on the forge its check has not answered: mesh/merge-gate pending since ` +
|
||||
`2026-10-11T00:11:39Z. The pull request waits on that status and nothing asks it again. A recheck of the ` +
|
||||
`delivery asks its check again, as does a new commit on its branch"`
|
||||
if held != "" {
|
||||
raw += `,"held":"` + held + `"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(raw+`}`), &l); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return l
|
||||
}
|
||||
for _, held := range []string{"ready", "rejected"} {
|
||||
o := stalledObservations([]stalledLine{line(held)})[0]
|
||||
t.Logf("%s: %s\nneeds: %s", held, o.Explanation, o.Needs)
|
||||
if !strings.Contains(o.Needs, "mesh-delivery.recheck") || !strings.Contains(o.Needs, "novox/mesh-controller@c11222026a3b") {
|
||||
t.Fatalf("a %s delivery's unanswered check is not offered its recheck: %q", held, o.Needs)
|
||||
}
|
||||
if strings.Contains(o.Needs, "build queue") || strings.Contains(o.Needs, "mesh-controller.queue") {
|
||||
t.Fatalf("a %s delivery's unanswered check, which answered once, is sent to the build queue: %q", held, o.Needs)
|
||||
}
|
||||
if !strings.Contains(o.Explanation, "Its delivery is "+held) {
|
||||
t.Fatalf("a %s delivery's explanation does not say its state: %q", held, o.Explanation)
|
||||
}
|
||||
if len(o.Actions) != 0 {
|
||||
t.Fatalf("it offers an action the controller is not granted: %+v", o.Actions)
|
||||
}
|
||||
}
|
||||
// No delivery held, or a mesh-delivery that does not say: the advice of issue 438, and no recheck.
|
||||
o := stalledObservations([]stalledLine{line("")})[0]
|
||||
if !strings.Contains(o.Needs, "in the build queue: mesh-controller.queue") || strings.Contains(o.Needs, "recheck") {
|
||||
t.Fatalf("a line with no delivery held says to %q", o.Needs)
|
||||
}
|
||||
}
|
||||
|
||||
+41
-125
@@ -7,7 +7,6 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
@@ -292,7 +291,7 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
|
||||
if err != nil {
|
||||
// Said, not skipped: a machine dropped here offers nothing and holds nothing as far
|
||||
// as every other machine's plan can tell (novox/hq issue 188).
|
||||
sayLeftOut(ctx, o.node.Name, err)
|
||||
fmt.Fprintf(os.Stderr, "%s is left out of the rest of the mesh: it does not resolve: %v\n", o.node.Name, err)
|
||||
continue
|
||||
}
|
||||
firstHeld = append(firstHeld, got.Claims...)
|
||||
@@ -337,7 +336,7 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
|
||||
// resolved without its offers, and one that consumes them cannot resolve here by
|
||||
// design — that is not the machine being dropped, it is the view being partial.
|
||||
if exclude == "" {
|
||||
sayLeftOut(ctx, o.node.Name, err)
|
||||
fmt.Fprintf(os.Stderr, "%s is left out of the rest of the mesh: it does not resolve: %v\n", o.node.Name, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
@@ -347,35 +346,6 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
|
||||
return world, nil
|
||||
}
|
||||
|
||||
// leftOutKey carries where theRestOfTheMesh says a machine it leaves out, for an act that answers with
|
||||
// its own resolution instead (novox/hq issue 237).
|
||||
type leftOutKey struct{}
|
||||
|
||||
// leftOutSaidBy is ctx with theRestOfTheMesh's "left out" lines written to w rather than to standard
|
||||
// error.
|
||||
//
|
||||
// **An act's answer is one resolution taken after its write** (novox/hq issue 237, decided 2026-10-11).
|
||||
// `assign` read the whole mesh once before its write — to record the seats held by derivation — and that
|
||||
// read said, on standard error, that the node did not resolve for lack of the seat the act was about to
|
||||
// give it. The controller seat's verb answers with both streams, so the stale line came last, after the
|
||||
// act's own "now has it held", and named as its remedy the assignment just made. An act gives io.Discard
|
||||
// here: what the node resolves to after the write is said by its own planFor, and every other machine
|
||||
// that does not resolve by blockedElsewhere, both read after the write.
|
||||
func leftOutSaidBy(ctx context.Context, w io.Writer) context.Context {
|
||||
return context.WithValue(ctx, leftOutKey{}, w)
|
||||
}
|
||||
|
||||
// sayLeftOut says that a machine is left out of the rest of the mesh, where ctx says to: standard
|
||||
// error unless an act said otherwise. Said, not skipped: a machine dropped there offers nothing and
|
||||
// holds nothing as far as every other machine's plan can tell (novox/hq issue 188).
|
||||
func sayLeftOut(ctx context.Context, node string, err error) {
|
||||
w, ok := ctx.Value(leftOutKey{}).(io.Writer)
|
||||
if !ok {
|
||||
w = os.Stderr
|
||||
}
|
||||
fmt.Fprintf(w, "%s is left out of the rest of the mesh: it does not resolve: %v\n", node, err)
|
||||
}
|
||||
|
||||
// declarationFor is everything a node would be sent.
|
||||
//
|
||||
// One place, because there were three and one of them was written before credentials existed and
|
||||
@@ -450,8 +420,7 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
||||
out := sendable{Resources: composed.Resources, Adoption: adoption,
|
||||
Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
|
||||
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld,
|
||||
unbound: with.Unbound, foreseen: composed.Foreseen, unplaced: composed.Unplaced,
|
||||
modules: namedModules(plan, composed.LeftOut)}
|
||||
unbound: with.Unbound, foreseen: composed.Foreseen, unplaced: composed.Unplaced}
|
||||
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
|
||||
// 0221). Read only on the send path: a question about what would be sent records nothing.
|
||||
if choosing == Allocating {
|
||||
@@ -543,6 +512,15 @@ func sortedKeysOf(m map[string]string) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// sayPlanDiff is `plan --diff`: what the declaration leaves out, then the diff. A module left out is not in
|
||||
// the body, so the diff alone would say "nothing would change" for a module just assigned whose settings
|
||||
// cannot compose — success-shaped silence. Said first, with why, as push and the plain plan say it
|
||||
// (novox/hq ADR 0163, rule 6).
|
||||
func sayPlanDiff(node string, declared sendable, diff func() error) error {
|
||||
reportLeftOut(node, declared)
|
||||
return diff()
|
||||
}
|
||||
|
||||
// reportLeftOut says which of a machine's modules its declaration leaves out and why (novox/hq ADR
|
||||
// 0163, rule 6), one line each: the machine is told everything else, and is told it was left out.
|
||||
func reportLeftOut(node string, declared sendable) {
|
||||
@@ -695,33 +673,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
needed := map[string]map[string]string{}
|
||||
foreseen := map[string]map[string]bool{}
|
||||
for _, m := range plan.Modules {
|
||||
// **A secret family is never made** (novox/hq ADR 0283): each member a person gave on this machine is
|
||||
// placed, and one not given is nothing — the module says it waits for it.
|
||||
for _, family := range m.OwnSecrets.Families() {
|
||||
members, err := inv.GivenMembers(ctx, node, m.Module, family)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
for _, g := range members {
|
||||
if _, fam, ok := m.OwnSecrets.Lookup(g.Name); !ok || fam != family {
|
||||
continue // a longer family's member, or a name no longer of this family
|
||||
}
|
||||
if !g.Current {
|
||||
if choosing == Allocating {
|
||||
return catalogue.Rendering{}, inventory.Node{}, fmt.Errorf(
|
||||
"%s on %s holds %q, which was given to the mesh rather than made by it, and %s has "+
|
||||
"since generated a new sealing key. The mesh cannot make another; give it again",
|
||||
m.Module, node, g.Name, node)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if needed[m.Module] == nil {
|
||||
needed[m.Module] = map[string]string{}
|
||||
}
|
||||
needed[m.Module][g.Name] = g.Sealed
|
||||
}
|
||||
}
|
||||
for name := range m.OwnSecrets.Plain() {
|
||||
for name := range m.OwnSecrets {
|
||||
// Minted on the send path and only read on every other. Making one is an insert, and
|
||||
// a question that writes is a question that can block against the machine it is about.
|
||||
var sealed string
|
||||
@@ -1296,11 +1248,9 @@ func planCommand(ctx context.Context, args []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// A module left out is not in the body, so the diff alone would say "nothing would change" for
|
||||
// a module just assigned whose settings cannot compose — success-shaped silence. Said first, with
|
||||
// why, as push and the plain plan say it (novox/hq ADR 0163, rule 6).
|
||||
reportLeftOut(args[0], declared)
|
||||
return writePlanDiff(ctx, open.inventory, args[0], body)
|
||||
return sayPlanDiff(args[0], declared, func() error {
|
||||
return writePlanDiff(ctx, open.inventory, args[0], body)
|
||||
})
|
||||
}
|
||||
if *asJSON {
|
||||
declared, err := declarationFor(ctx, open, args[0], plan, settings)
|
||||
@@ -1321,11 +1271,6 @@ func planCommand(ctx context.Context, args []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// What the machine was last told, by whom and from which assignment generation (novox/hq issue 234),
|
||||
// beside what it would be told now. A record that cannot be read is said, not taken for none.
|
||||
if err := writeLastSend(ctx, os.Stdout, open.inventory, args[0]); err != nil {
|
||||
fmt.Printf("what %s was last sent could not be read: %v\n", args[0], err)
|
||||
}
|
||||
// Which modules a push would leave out, and why — said before the plan, since the plan is of
|
||||
// what the machine would be told (novox/hq ADR 0163, rule 6). Judged, never composed: `plan`
|
||||
// without --json allocates nothing.
|
||||
@@ -1721,27 +1666,17 @@ func providerModuleOf(resolved catalogue.Resolution, open *stores, ctx context.C
|
||||
return ""
|
||||
}
|
||||
|
||||
// unheldLogger remembers what was last logged about each node's unmet seat dependencies, so the
|
||||
// log says each change once (novox/hq ADR 0207), on stderr so `status --json` stays a document.
|
||||
// A test gives its own writer and memory, and no longer shares the process's (novox/hq issue 486).
|
||||
// unheldLogged is what was last logged about each node's unmet seat dependencies, so the log says
|
||||
// each change once (novox/hq ADR 0207), on stderr so `status --json` stays a document.
|
||||
//
|
||||
// **The serving controller's log only.** planFor runs for every node on every push, assignment and
|
||||
// status — `blockedElsewhere` alone resolves the whole mesh — and a one-shot command starts with an
|
||||
// empty memory, so every node's report was "a change" and a push printed the whole mesh's list,
|
||||
// burying the line about the node it acted on. A command says what concerns its own act instead
|
||||
// (unheldChange, reportUnheldPushed); the full list is `status`'s.
|
||||
type unheldLogger struct {
|
||||
mu sync.Mutex
|
||||
logged map[string]string
|
||||
out io.Writer
|
||||
}
|
||||
|
||||
func newUnheldLogger(out io.Writer) *unheldLogger {
|
||||
return &unheldLogger{logged: map[string]string{}, out: out}
|
||||
}
|
||||
|
||||
var (
|
||||
unheldLog = newUnheldLogger(os.Stderr)
|
||||
unheldLogged = map[string]string{}
|
||||
unheldLoggedMu sync.Mutex
|
||||
logUnheldChanges bool
|
||||
)
|
||||
|
||||
@@ -1751,27 +1686,23 @@ func logUnheld(node string, unheld []catalogue.Unheld) {
|
||||
if !logUnheldChanges {
|
||||
return
|
||||
}
|
||||
unheldLog.log(node, unheld)
|
||||
}
|
||||
|
||||
func (l *unheldLogger) log(node string, unheld []catalogue.Unheld) {
|
||||
lines := make([]string, 0, len(unheld))
|
||||
for _, u := range unheld {
|
||||
lines = append(lines, u.String())
|
||||
}
|
||||
now := strings.Join(lines, "\n")
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
before, seen := l.logged[node]
|
||||
l.logged[node] = now
|
||||
unheldLoggedMu.Lock()
|
||||
before, seen := unheldLogged[node]
|
||||
unheldLogged[node] = now
|
||||
unheldLoggedMu.Unlock()
|
||||
if (seen && before == now) || (!seen && now == "") {
|
||||
return
|
||||
}
|
||||
if now == "" {
|
||||
fmt.Fprintf(l.out, "%s: every seat its modules depend on is held (novox/hq ADR 0207)\n", node)
|
||||
fmt.Fprintf(os.Stderr, "%s: every seat its modules depend on is held (novox/hq ADR 0207)\n", node)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(l.out, "%s: %d unmet seat dependenc(ies), reported and not refused (novox/hq ADR 0207):\n %s\n",
|
||||
fmt.Fprintf(os.Stderr, "%s: %d unmet seat dependenc(ies), reported and not refused (novox/hq ADR 0207):\n %s\n",
|
||||
node, len(lines), strings.Join(lines, "\n "))
|
||||
}
|
||||
|
||||
@@ -1780,43 +1711,28 @@ func (l *unheldLogger) log(node string, unheld []catalogue.Unheld) {
|
||||
// which includes every one of a module just assigned — and each now met that was not. Nothing about
|
||||
// any other node, and nothing that was already true before the act.
|
||||
func unheldChange(shelf map[string]catalogue.Manifest, node string, before, after []string) []string {
|
||||
return unheldChangeTo(shelf, node, before, after, unheldOver(shelf, node, after))
|
||||
}
|
||||
|
||||
// unheldOver is the node's unmet seat dependencies judged over a set of module names alone.
|
||||
func unheldOver(shelf map[string]catalogue.Manifest, node string, names []string) []catalogue.Unheld {
|
||||
var set []catalogue.Manifest
|
||||
for _, n := range names {
|
||||
if m, known := shelf[n]; known {
|
||||
set = append(set, m)
|
||||
judge := func(names []string) map[string]catalogue.Unheld {
|
||||
var set []catalogue.Manifest
|
||||
for _, n := range names {
|
||||
if m, known := shelf[n]; known {
|
||||
set = append(set, m)
|
||||
}
|
||||
}
|
||||
out := map[string]catalogue.Unheld{}
|
||||
for _, u := range catalogue.UnheldDependencies(shelf, node, set, nil) {
|
||||
out[u.Module+" "+u.Seat] = u
|
||||
}
|
||||
return out
|
||||
}
|
||||
return catalogue.UnheldDependencies(shelf, node, set, nil)
|
||||
}
|
||||
|
||||
// unheldChangeTo is unheldChange with the node's dependencies as they now stand given: `now` is what
|
||||
// the act's one resolution after its write says is unmet, and `after` the modules that resolution has
|
||||
// on the node (novox/hq issue 237). A line about how things stand after the act is never judged from
|
||||
// anything else, so it cannot disagree with `plan` for the node.
|
||||
func unheldChangeTo(shelf map[string]catalogue.Manifest, node string, before, after []string,
|
||||
now []catalogue.Unheld) []string {
|
||||
key := func(u catalogue.Unheld) string { return u.Module + " " + u.Seat }
|
||||
was := map[string]catalogue.Unheld{}
|
||||
for _, u := range unheldOver(shelf, node, before) {
|
||||
was[key(u)] = u
|
||||
}
|
||||
is := map[string]catalogue.Unheld{}
|
||||
for _, u := range now {
|
||||
is[key(u)] = u
|
||||
}
|
||||
was, now := judge(before), judge(after)
|
||||
var lines []string
|
||||
for _, k := range sortedNames(is) {
|
||||
for _, k := range sortedNames(now) {
|
||||
if _, already := was[k]; !already {
|
||||
lines = append(lines, "but "+is[k].String())
|
||||
lines = append(lines, "but "+now[k].String())
|
||||
}
|
||||
}
|
||||
for _, k := range sortedNames(was) {
|
||||
if _, still := is[k]; still {
|
||||
if _, still := now[k]; still {
|
||||
continue
|
||||
}
|
||||
u := was[k]
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// 2026-10-09: nfs-server was assigned to the home server, its file asked for a setting nothing set, and
|
||||
// `plan --diff` said "nothing would change". The diff now says what is left out, and why, before the diff.
|
||||
func TestPlanDiffSaysAModuleLeftOutBeforeTheDiff(t *testing.T) {
|
||||
declared := sendable{LeftOut: []string{"nfs-server"},
|
||||
leftOutWhy: map[string]string{"nfs-server": `nothing sets "shares" for it`}}
|
||||
said := printed(t, func() error {
|
||||
return sayPlanDiff("home", declared, func() error {
|
||||
writeDiff(os.Stdout, "home", sentDiff{}, nil)
|
||||
return nil
|
||||
})
|
||||
})
|
||||
left := strings.Index(said, "home: nfs-server left out")
|
||||
nothing := strings.Index(said, "home: nothing would change")
|
||||
if left < 0 || !strings.Contains(said, `nothing sets "shares" for it`) || nothing < left {
|
||||
t.Errorf("the left-out module and why, then the diff:\n%s", said)
|
||||
}
|
||||
}
|
||||
@@ -164,13 +164,12 @@ func retryRefusal(p inventory.Plan, plans []inventory.Plan) error {
|
||||
case p.Open():
|
||||
return fmt.Errorf("%s is still %s; nothing in it failed to retry — `rebuild <module>` asks one module again", p.ID, p.State)
|
||||
}
|
||||
|
||||
if len(failedIn(p)) > 0 {
|
||||
if q, found := newerOpenPlan(p, plans); found {
|
||||
return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and retrying %s would build "+
|
||||
"what that one replaced", q.ID, q.Repository, q.ID, short(q.Commit), p.ID)
|
||||
}
|
||||
return oneWalkAtATime(p, plans)
|
||||
return nil
|
||||
}
|
||||
stopped := stoppedRollouts(p)
|
||||
if len(stopped) == 0 {
|
||||
@@ -194,17 +193,6 @@ func retryRefusal(p inventory.Plan, plans []inventory.Plan) error {
|
||||
"the older build back", m, q.ID, q.State, q.Repository, short(q.Commit), p.ID)
|
||||
}
|
||||
}
|
||||
return oneWalkAtATime(p, plans)
|
||||
}
|
||||
|
||||
// oneWalkAtATime refuses a retry while another walk is open, started or waiting for its word (novox/hq ADR
|
||||
// 0276): a walk retried beside it would be two walks at once.
|
||||
func oneWalkAtATime(p inventory.Plan, plans []inventory.Plan) error {
|
||||
for _, q := range plans {
|
||||
if q.ID != p.ID && q.Open() && q.Release == nil {
|
||||
return fmt.Errorf("%s is open (%s): one walk at a time — retry %s once it ended", q.ID, q.Named(), p.ID)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -251,16 +239,6 @@ func resumed(p *inventory.Plan, why string) {
|
||||
if p.State == inventory.PlanFailed && len(failedIn(*p)) == 0 {
|
||||
p.State = inventory.PlanBuilding
|
||||
p.Note = why
|
||||
goesOnAgain(p)
|
||||
}
|
||||
}
|
||||
|
||||
// goesOnAgain clears a stop from a walk that walks again: a stop said why the walk ended, and a walk resumed has
|
||||
// not ended. Kept, it would end a retried walk that fails again as stopped, not failed (novox/hq ADR 0308
|
||||
// decision 1), and keep the search for the merge that brought the failure from running (searchAfterFailures).
|
||||
func goesOnAgain(p *inventory.Plan) {
|
||||
if p.Delivery != nil {
|
||||
p.Delivery.Stopped, p.Delivery.StoppedWhy = "", ""
|
||||
}
|
||||
}
|
||||
|
||||
@@ -286,43 +264,13 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
||||
return "", err
|
||||
}
|
||||
plans = append(plans, recent...)
|
||||
// **A failed walk whose earlier merges are walked alone is not retried** (novox/hq ADR 0276): the search for
|
||||
// the merge that brought the failure answers them now, and a retried walk would name them twice.
|
||||
// **A walk stopped without some merges is not retried** (novox/hq ADR 0299): the merges it left out were stopped,
|
||||
// and the others are another walk's now.
|
||||
if p.Delivery != nil && (p.Delivery.WalkedAgainBy != "" || len(p.Delivery.LeftOut) > 0) {
|
||||
return "", fmt.Errorf("%s was stopped without %s: those merges were stopped, and %s walks the others; a "+
|
||||
"newer merge, or `rebuild <module>`, builds again", p.ID, mergesLeftOutWords(p.Delivery.LeftOut),
|
||||
orWords(p.Delivery.WalkedAgainBy, "nothing"))
|
||||
}
|
||||
if p.Delivery != nil && len(p.Delivery.Merges) > 0 {
|
||||
kept, err := inv.MergesOf(ctx, p.ID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(kept) < len(p.Delivery.Merges) {
|
||||
return "", fmt.Errorf("%s's earlier merges are walked alone, to find which one brought its failure: "+
|
||||
"those walks answer them; a newer merge, or `rebuild <module>`, builds again", p.ID)
|
||||
}
|
||||
}
|
||||
// Settled from the build records before anything is judged (novox/hq issue 457): a build of the tier
|
||||
// that failed after the plan did is as failed as the one that failed it. What toRetry says is the
|
||||
// failed set every step below works from.
|
||||
var failed []string
|
||||
if p.Tier < len(p.Tiers) {
|
||||
recorded, byID, err := recordsOfAsked(ctx, inv, &p, p.Tiers[p.Tier])
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
failed = toRetry(&p, recorded, byID)
|
||||
}
|
||||
if err := retryRefusal(p, plans); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if again := unjudgedAtGate(p); len(failed) == 0 && len(again) > 0 {
|
||||
if again := unjudgedAtGate(p); len(failedIn(p)) == 0 && len(again) > 0 {
|
||||
return retryTierWhole(ctx, open, &p, again)
|
||||
}
|
||||
if len(failed) == 0 {
|
||||
if len(failedIn(p)) == 0 {
|
||||
return retryRollouts(ctx, open, &p)
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
@@ -333,6 +281,7 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
||||
for _, e := range entries {
|
||||
byName[e.Manifest.Module] = e
|
||||
}
|
||||
failed := failedIn(p)
|
||||
var asked []string
|
||||
for _, m := range failed {
|
||||
askModule(ctx, &p, m, byName)
|
||||
@@ -470,7 +419,6 @@ func retryRollouts(ctx context.Context, open *stores, p *inventory.Plan) (string
|
||||
said = append(said, m+" to "+strings.Join(again, ", "))
|
||||
}
|
||||
p.State = inventory.PlanRolling
|
||||
goesOnAgain(p)
|
||||
p.Note = fmt.Sprintf("tier %d retried by hand; sent %s first again", p.Tier, strings.Join(said, "; "))
|
||||
if err := open.inventory.SavePlan(ctx, p); err != nil {
|
||||
return "", err
|
||||
@@ -536,7 +484,6 @@ func retryTierWhole(ctx context.Context, open *stores, p *inventory.Plan, again
|
||||
}
|
||||
if p.State == inventory.PlanFailed && len(failedIn(*p)) == 0 {
|
||||
p.State = inventory.PlanBuilding
|
||||
goesOnAgain(p)
|
||||
p.Note = fmt.Sprintf("tier %d retried by hand: %s asked again; %s sent again with the tier", p.Tier,
|
||||
strings.Join(again, ", "), orNone(strings.Join(resent, ", ")))
|
||||
}
|
||||
@@ -554,16 +501,3 @@ func retryTierWhole(ctx context.Context, open *stores, p *inventory.Plan, again
|
||||
func sendAgain(s *inventory.PlanModule) {
|
||||
s.First, s.FirstAt, s.Gate, s.GatedBy, s.Previous, s.Why = nil, nil, nil, "", "", ""
|
||||
}
|
||||
|
||||
// toRetry is the modules a retry asks again: every one of the tier that failed, the plan's state
|
||||
// settled from the build records first (novox/hq issue 457). A plan fails on the first failure in its
|
||||
// tier, and an outcome arriving after that finds no open plan to answer — it is kept only in the build
|
||||
// records. Read from the plan alone, a retry asked only the build that failed first, and the records
|
||||
// then failed the plan again on the next: each failed build of a tier took a retry of its own. What
|
||||
// still runs is left asked, and its outcome is the plan's once the retry sets it building.
|
||||
func toRetry(p *inventory.Plan, recorded map[string][]inventory.Build, byID map[string]inventory.Build) []string {
|
||||
if p.Tier < len(p.Tiers) {
|
||||
settleFromRecords(p, p.Tiers[p.Tier], recorded, byID)
|
||||
}
|
||||
return failedIn(*p)
|
||||
}
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A retry asks every failed build of the tier, not one (novox/hq issue 457). Seen 2026-10-11: gitea
|
||||
// and plex both failed in tier 0 while the registry was held still; gitea's failure failed the plan,
|
||||
// and plex's, arriving after, found no open plan and was kept only in the build records. The first
|
||||
// retry asked gitea alone, the records then failed the plan again on plex, and a second retry asked
|
||||
// plex.
|
||||
func TestARetryAsksEveryFailedBuildOfTheTier(t *testing.T) {
|
||||
asked := time.Date(2026, 10, 11, 1, 29, 0, 0, time.UTC)
|
||||
failedAt := asked.Add(2 * time.Minute)
|
||||
p := inventory.Plan{ID: "plan-457", State: inventory.PlanFailed, Tier: 0,
|
||||
Tiers: [][]string{{"gitea", "plex"}}, Note: "gitea failed to build in tier 0",
|
||||
Modules: map[string]*inventory.PlanModule{
|
||||
"gitea": {State: "failed", AskedAt: &asked, Build: "build-gitea", Why: "cannot reach the registry"},
|
||||
"plex": {State: "asked", AskedAt: &asked, Build: "build-plex"},
|
||||
}}
|
||||
byID := map[string]inventory.Build{
|
||||
"build-plex": {ID: "build-plex", Module: "plex", At: failedAt, Failed: "cannot reach the registry"},
|
||||
}
|
||||
|
||||
if got := toRetry(&p, nil, byID); !reflect.DeepEqual(got, []string{"gitea", "plex"}) {
|
||||
t.Fatalf("a retry of a tier where gitea and plex failed asks %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A build of the tier still running when the plan failed is not asked again: its outcome is the plan's
|
||||
// once the retry sets it building, and one that is recorded built is taken as built.
|
||||
func TestARetryLeavesABuildThatRunsOrWorked(t *testing.T) {
|
||||
asked := time.Date(2026, 10, 11, 1, 29, 0, 0, time.UTC)
|
||||
builtAt := asked.Add(3 * time.Minute)
|
||||
p := inventory.Plan{ID: "plan-457", State: inventory.PlanFailed, Tier: 0,
|
||||
Tiers: [][]string{{"a", "b", "c"}},
|
||||
Modules: map[string]*inventory.PlanModule{
|
||||
"a": {State: "failed", AskedAt: &asked, Build: "build-a", Why: "broken"},
|
||||
"b": {State: "asked", AskedAt: &asked, Build: "build-b"},
|
||||
"c": {State: "asked", AskedAt: &asked, Build: "build-c"},
|
||||
}}
|
||||
byID := map[string]inventory.Build{"build-c": {ID: "build-c", Module: "c", At: builtAt, Commit: "c0ffee"}}
|
||||
|
||||
if got := toRetry(&p, nil, byID); !reflect.DeepEqual(got, []string{"a"}) {
|
||||
t.Fatalf("asks %v, want a alone", got)
|
||||
}
|
||||
if s := p.Modules["c"]; s.State != "built" || s.Commit != "c0ffee" {
|
||||
t.Errorf("c, recorded built, is %+v", s)
|
||||
}
|
||||
if s := p.Modules["b"]; s.State != "asked" {
|
||||
t.Errorf("b, still building, is %+v", s)
|
||||
}
|
||||
}
|
||||
@@ -15,16 +15,16 @@ import (
|
||||
// inventory.Dependencies (dependenciesOf over the records), and the merge planned by reachOfMerge — the
|
||||
// path a real merge takes, short of the bus.
|
||||
//
|
||||
// **A shared repository moves only what a change's files are in the build source of** (novox/hq ADR 0267,
|
||||
// issues 338 and 363): each build records the build source it said, and a merge is mapped onto those of the
|
||||
// newest builds. A build that said none (P below, as every build before ADR 0267) is read as before: P moves
|
||||
// on any merge to the repository it packages, though through no edge. The rows tagged 338 held the opposite
|
||||
// until ADR 0267 was built.
|
||||
func TestASharedRepositoryIsPlannedFromTheRecordedBuildSources(t *testing.T) {
|
||||
// **The repository rows are CURRENT BEHAVIOUR, documented — not the rule the operator states**
|
||||
// (novox/hq issue 338, and the decision pending on it): a build that read a repository gives its module a
|
||||
// packages edge to every module built from that repository, and mergeCandidates moves it on any merge to
|
||||
// that repository, whatever the files. So a change to C alone, or to a README, moves the module that
|
||||
// packages C's repository. ADR 0238 §3 records exactly that today ("a repository a recipe names"); the
|
||||
// expectations marked 338 change with that decision.
|
||||
func TestASharedRepositoryIsPlannedFromTheRecordsAsItIsToday(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
ctx := t.Context()
|
||||
asked := time.Now().Add(-time.Hour)
|
||||
sourcesOf := map[string][]inventory.BuildSource{}
|
||||
register := func(m catalogue.Manifest, repository, path string, against []string, read []inventory.ReadRepository) {
|
||||
t.Helper()
|
||||
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: repository, Seat: "git", Path: path,
|
||||
@@ -32,8 +32,7 @@ func TestASharedRepositoryIsPlannedFromTheRecordedBuildSources(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + m.Module, Repository: repository, Ref: "main",
|
||||
Module: m.Module, Commit: "old", On: "builder", Path: path, Against: against, Read: read, Asked: asked,
|
||||
Sources: sourcesOf[m.Module]}); err != nil {
|
||||
Module: m.Module, Commit: "old", On: "builder", Path: path, Against: against, Read: read, Asked: asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -41,16 +40,7 @@ func TestASharedRepositoryIsPlannedFromTheRecordedBuildSources(t *testing.T) {
|
||||
agent := catalogue.Manifest{Module: "build-agent", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}}
|
||||
|
||||
// The shape of issue 338, each build saying its build source (ADR 0267).
|
||||
gomod := []string{"go.mod", "go.sum"}
|
||||
sourcesOf["mesh-controller"] = []inventory.BuildSource{{Paths: append([]string{"module.json", "cmd/mesh-controller/",
|
||||
"internal/conditions/", "internal/broker/"}, gomod...)}}
|
||||
sourcesOf["build-agent"] = []inventory.BuildSource{
|
||||
{Paths: []string{"modules/build-agent/Dockerfile", "modules/build-agent/module.json"}},
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: append([]string{"cmd/mesh-builder/", "internal/broker/"}, gomod...)}}
|
||||
sourcesOf["route-proxy"] = []inventory.BuildSource{
|
||||
{Paths: []string{"modules/route-proxy/Dockerfile", "modules/route-proxy/module.json"}},
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: append([]string{"examples/route-proxy/", "internal/broker/"}, gomod...)}}
|
||||
// The shape of issue 338.
|
||||
register(catalogue.Manifest{Module: "mesh-controller", Version: "1"}, "novox/mesh-controller", "", nil, nil)
|
||||
register(agent, "novox/mesh-catalog", "modules/build-agent", nil, controllerRead)
|
||||
register(catalogue.Manifest{Module: "route-proxy", Version: "1"}, "novox/mesh-catalog", "modules/route-proxy", nil, controllerRead)
|
||||
@@ -91,15 +81,13 @@ func TestASharedRepositoryIsPlannedFromTheRecordedBuildSources(t *testing.T) {
|
||||
if !reflect.DeepEqual(shared, sharedRepositoryEdges) {
|
||||
t.Errorf("derived %v\nthe hand-written rows use %v", shared, sharedRepositoryEdges)
|
||||
}
|
||||
// Each kind derived from its record: built against (stands-on), build.on (declared); a read draws none.
|
||||
for _, e := range edges {
|
||||
if e.Kind == inventory.EdgePackages {
|
||||
t.Errorf("a packages edge was drawn (ADR 0267 rule 4): %v", e)
|
||||
}
|
||||
}
|
||||
// Each kind derived from its record: built against (stands-on), build.on (declared), read (packages).
|
||||
for _, want := range []inventory.Edge{
|
||||
dep("d", inventory.EdgeStandsOn, "a"),
|
||||
dep("e", inventory.EdgeDeclared, "b"),
|
||||
dep("p", inventory.EdgePackages, "a"),
|
||||
dep("p", inventory.EdgePackages, "b"),
|
||||
dep("p", inventory.EdgePackages, "c"),
|
||||
dep("d", inventory.EdgeBuiltBy, "build-agent"),
|
||||
} {
|
||||
found := false
|
||||
@@ -117,23 +105,15 @@ func TestASharedRepositoryIsPlannedFromTheRecordedBuildSources(t *testing.T) {
|
||||
want string
|
||||
issue338 bool
|
||||
}{
|
||||
{"A and B changed, C untouched: D after A, E after B; P, which said no build source, reads all", "one",
|
||||
{"A and B changed, C untouched: D after A, E after B; P packages their repository", "one",
|
||||
[]string{"modules/a/x.go", "modules/b/x.go"}, "a,b,p | d,e", false},
|
||||
{"C alone: C, and P, read whole as before; P after nothing", "one",
|
||||
[]string{"modules/c/x.go"}, "c,p", false},
|
||||
{"a README of the repository P packages: P, read whole as before", "one",
|
||||
[]string{"README.md"}, "p", false},
|
||||
{"C alone: C, and P, which packages C's repository", "one",
|
||||
[]string{"modules/c/x.go"}, "c,p", true},
|
||||
{"a README of the repository P packages: P moves, nothing built from it does", "one",
|
||||
[]string{"README.md"}, "p", true},
|
||||
{"the dependent's repository: D alone", "two", []string{"d/main.go"}, "d", false},
|
||||
{"a README of the controller's repository: no module", "mesh-controller",
|
||||
[]string{"README.md"}, "", true},
|
||||
{"the controller's own command: the controller alone", "mesh-controller",
|
||||
[]string{"cmd/mesh-controller/main.go"}, "mesh-controller", true},
|
||||
{"a package only the controller builds from: the controller alone", "mesh-controller",
|
||||
[]string{"internal/conditions/condition.go"}, "mesh-controller", true},
|
||||
{"the route proxy's program: the route proxy alone", "mesh-controller",
|
||||
[]string{"examples/route-proxy/main.go"}, "route-proxy", true},
|
||||
{"a package all three build from: all three", "mesh-controller",
|
||||
[]string{"internal/broker/broker.go"}, "mesh-controller | build-agent | route-proxy", false},
|
||||
{"a README of the controller's repository: all three, three tiers", "mesh-controller",
|
||||
[]string{"README.md"}, "mesh-controller | build-agent | route-proxy", true},
|
||||
{"the route proxy's directory in the catalogue: it alone", "mesh-catalog",
|
||||
[]string{"modules/route-proxy/module.json"}, "route-proxy", false},
|
||||
{"the build agent's directory: it alone, nothing it builds", "mesh-catalog",
|
||||
@@ -143,7 +123,7 @@ func TestASharedRepositoryIsPlannedFromTheRecordedBuildSources(t *testing.T) {
|
||||
if got != c.want {
|
||||
tag := ""
|
||||
if c.issue338 {
|
||||
tag = " (issue 338, flipped by ADR 0267)"
|
||||
tag = " (current behaviour, issue 338)"
|
||||
}
|
||||
t.Errorf("%s: planned %q, wanted %q%s", c.what, got, c.want, tag)
|
||||
}
|
||||
|
||||
@@ -1,15 +1,12 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
snapshot "github.com/novox/mesh-controller/internal/facts"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
@@ -24,7 +21,7 @@ import (
|
||||
// kind widens the plan orders the tiers
|
||||
// stands-on yes yes, after its base is built
|
||||
// declared yes yes, after its base is built
|
||||
// packages no no — retired by novox/hq ADR 0267; one recorded before is read and ignored
|
||||
// packages yes no, the same tier (a code dependency)
|
||||
// built-by no yes, after the build machine — except for what the build machine stands
|
||||
// on, and for the controller whose worker it binds
|
||||
// worker-of no yes, the build seat's holder after the controller (hq issue 206)
|
||||
@@ -126,9 +123,9 @@ func TestAPlanIsWhatTheChangeTouchedAndWhatIsBuiltOnIt(t *testing.T) {
|
||||
{what: "transitive: F on D on A, A changed",
|
||||
edges: []inventory.Edge{dep("f", standsOn, "d"), dep("d", standsOn, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x"}, want: "a | d | f"},
|
||||
{what: "transitive across kinds stops at a packages edge: F declared on D, D packages A (ADR 0267)",
|
||||
{what: "transitive across kinds: F declared on D, D packages A",
|
||||
edges: []inventory.Edge{dep("f", declared, "d"), dep("d", packages, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x"}, want: "a"},
|
||||
repo: "one", paths: []string{"modules/a/x"}, want: "a,d | f"},
|
||||
|
||||
// Each kind alone: X depends on A, A changed (widening), then both changed (ordering).
|
||||
{what: "stands-on (built against A's artifact) widens", edges: []inventory.Edge{dep("x", standsOn, "a")},
|
||||
@@ -139,8 +136,8 @@ func TestAPlanIsWhatTheChangeTouchedAndWhatIsBuiltOnIt(t *testing.T) {
|
||||
repo: "one", paths: []string{"modules/a/x"}, want: "a | x"},
|
||||
{what: "declared orders", edges: []inventory.Edge{dep("x", declared, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a | x"},
|
||||
{what: "packages, recorded before ADR 0267, widens nothing", edges: []inventory.Edge{dep("x", packages, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x"}, want: "a"},
|
||||
{what: "packages widens, into the same tier", edges: []inventory.Edge{dep("x", packages, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x"}, want: "a,x"},
|
||||
{what: "packages does not order", edges: []inventory.Edge{dep("x", packages, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a,x"},
|
||||
{what: "built-by never widens", edges: []inventory.Edge{dep("x", builtBy, "a")},
|
||||
@@ -191,7 +188,7 @@ func TestAPlanIsWhatTheChangeTouchedAndWhatIsBuiltOnIt(t *testing.T) {
|
||||
repo: "one", paths: []string{"modules/z/x"}, want: "z | a,b | d"},
|
||||
{what: "a diamond of mixed kinds orders on the ordering side only",
|
||||
edges: []inventory.Edge{dep("d", standsOn, "a"), dep("d", packages, "b")},
|
||||
repo: "one", paths: []string{"modules/b/x"}, want: "b"},
|
||||
repo: "one", paths: []string{"modules/b/x"}, want: "b,d"},
|
||||
|
||||
// A cycle the catalogue should never produce: what remains is one last tier, and said.
|
||||
{what: "a cycle is one last tier, not lost", edges: []inventory.Edge{dep("a", standsOn, "b"), dep("b", standsOn, "a"),
|
||||
@@ -228,16 +225,22 @@ func TestAPlanIsWhatTheChangeTouchedAndWhatIsBuiltOnIt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// **A shared repository moves only what a change's files are in the build source of** (novox/hq ADR 0267,
|
||||
// issue 338, issue 363). The controller is built from its repository's root as a Go bundle; the route proxy
|
||||
// and the build seat's holder build images whose context is that repository and which name the package they
|
||||
// compile. Each newest trunk build said its build source — the import closure of its program — and a merge
|
||||
// is mapped onto those. The rows tagged 338 held the opposite until ADR 0267 was built: every merge to the
|
||||
// controller's repository planned all three, in three tiers.
|
||||
// **CURRENT BEHAVIOUR, documented — not the rule the operator states.** novox/hq issue 338 (a module
|
||||
// built from a shared repository moves on every merge to it) and the decision pending on it would change
|
||||
// every row here. Today:
|
||||
//
|
||||
// The build sources are this repository's own programs as GoBuildSource reads them (held to that by
|
||||
// TestThisRepositorysProgramsHaveBuildSourcesOfTheirOwn in internal/builder), cut to what the rows need.
|
||||
func TestASharedRepositoryMovesOnlyWhatItsBuildSourceHolds(t *testing.T) {
|
||||
// - mesh-controller is built from its repository's root, so every file of that repository touches it;
|
||||
// - route-proxy and build-agent package the whole of that repository (a build context), so the build
|
||||
// record's `read` makes them move on any merge to it, whatever the files, and dependenciesOf gives
|
||||
// each a packages edge to every module built from it;
|
||||
// - built-by (route-proxy on build-agent) and worker-of (build-agent on the controller) make it three
|
||||
// tiers.
|
||||
//
|
||||
// These follow ADR 0238 §3 as written ("the whole repository for a module built from its root, and a
|
||||
// repository a recipe names"), so they are not failures; when the decision on issue 338 lands, these
|
||||
// expectations change with it. The edges are the ones dependenciesOf derives from this catalogue — held
|
||||
// to that by TestASharedRepositoryIsPlannedFromTheRecordsAsItIsToday, which derives them from the store.
|
||||
func TestASharedRepositoryMovesWhatPackagesItAsItDoesToday(t *testing.T) {
|
||||
const catalogueRepo = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||
const controllerRepo = "http://forge.internal:20000/novox/mesh-controller.git"
|
||||
entries := []inventory.Entry{
|
||||
@@ -246,26 +249,7 @@ func TestASharedRepositoryMovesOnlyWhatItsBuildSourceHolds(t *testing.T) {
|
||||
fromRepo("route-proxy", catalogueRepo, "modules/route-proxy"),
|
||||
fromRepo("gitea", catalogueRepo, "modules/gitea"),
|
||||
}
|
||||
gomod := []string{"go.mod", "go.sum", "vendor/modules.txt"}
|
||||
with := func(paths ...string) []string { return append(append([]string{}, gomod...), paths...) }
|
||||
read := map[string][]inventory.ReadRepository{
|
||||
"mesh-controller": {{Own: true, Paths: with("module.json", "cmd/mesh-controller/", "internal/conditions/",
|
||||
"internal/broker/", "internal/builder/", "internal/inventory/", "internal/inventory/migrations/**",
|
||||
"vendor/github.com/nats-io/nats.go/")}},
|
||||
"build-agent": {
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: with("cmd/mesh-builder/", "internal/broker/",
|
||||
"internal/builder/", "internal/inventory/", "internal/inventory/migrations/**", "vendor/github.com/nats-io/nats.go/")},
|
||||
{Own: true, Paths: []string{"modules/build-agent/Dockerfile", "modules/build-agent/module.json"}},
|
||||
},
|
||||
"route-proxy": {
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: with("examples/route-proxy/", "internal/broker/",
|
||||
"vendor/github.com/nats-io/nats.go/")},
|
||||
{Own: true, Paths: []string{"modules/route-proxy/Dockerfile", "modules/route-proxy/module.json"}},
|
||||
},
|
||||
}
|
||||
// With no build source said — before each module's first trunk build under ADR 0267, or while an
|
||||
// earlier merge's build of it is pending — a module is read as before.
|
||||
unsaid := map[string][]inventory.ReadRepository{
|
||||
"build-agent": {{Repository: "novox/mesh-controller", Ref: "main"}},
|
||||
"route-proxy": {{Repository: "novox/mesh-controller", Ref: "main"}},
|
||||
}
|
||||
@@ -273,176 +257,51 @@ func TestASharedRepositoryMovesOnlyWhatItsBuildSourceHolds(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
what, repo string
|
||||
paths []string
|
||||
read map[string][]inventory.ReadRepository
|
||||
want string
|
||||
unread string
|
||||
}{
|
||||
// The operator's acceptance: a merge of the controller's own code plans the controller alone.
|
||||
{"the controller's own command: the controller alone (338)", "mesh-controller",
|
||||
[]string{"cmd/mesh-controller/main.go"}, read, "mesh-controller", ""},
|
||||
{"a package only the controller builds from: the controller alone (338)", "mesh-controller",
|
||||
[]string{"internal/conditions/condition.go", "internal/conditions/bus.go"}, read, "mesh-controller", ""},
|
||||
{"a test beside the controller's command: nothing is built from it", "mesh-controller",
|
||||
[]string{"cmd/mesh-controller/main_test.go"}, read, "", "cmd/mesh-controller/main_test.go"},
|
||||
{"a README of the controller's repository: no module (338)", "mesh-controller",
|
||||
[]string{"README.md"}, read, "", "README.md"},
|
||||
{"the route proxy's program alone: the route proxy alone (338)", "mesh-controller",
|
||||
[]string{"examples/route-proxy/main.go"}, read, "route-proxy", ""},
|
||||
{"the build seat's program alone: its holder alone", "mesh-controller",
|
||||
[]string{"cmd/mesh-builder/main.go"}, read, "build-agent", ""},
|
||||
{"a package the build seat's program and the controller build from: both", "mesh-controller",
|
||||
[]string{"internal/builder/builder.go"}, read, "mesh-controller | build-agent", ""},
|
||||
{"a migration the controller and the build seat's program embed: both", "mesh-controller",
|
||||
[]string{"internal/inventory/migrations/0088-a-build-says-its-build-source.sql"}, read,
|
||||
"mesh-controller | build-agent", ""},
|
||||
// A package all three build from: all three; the build seat's holder after the controller whose worker
|
||||
// it binds (worker-of, issue 206), the proxy after the holder that builds it (built-by).
|
||||
{"a package all three build from: all three", "mesh-controller",
|
||||
[]string{"internal/broker/broker.go"}, read, "mesh-controller | build-agent | route-proxy", ""},
|
||||
{"a vendored package all three build from: all three", "mesh-controller",
|
||||
[]string{"vendor/github.com/nats-io/nats.go/nats.go"}, read, "mesh-controller | build-agent | route-proxy", ""},
|
||||
{"go.sum: all three", "mesh-controller",
|
||||
[]string{"go.sum"}, read, "mesh-controller | build-agent | route-proxy", ""},
|
||||
{"a file added to the proxy's package: the proxy", "mesh-controller",
|
||||
[]string{"examples/route-proxy/new.go"}, read, "route-proxy", ""},
|
||||
// Before any build source is said: as before.
|
||||
{"no build source said: a README moves all three, as before", "mesh-controller",
|
||||
[]string{"README.md"}, unsaid, "mesh-controller | build-agent | route-proxy", ""},
|
||||
// In the catalogue, where they live, each by its own build source.
|
||||
{"the route proxy's recipe: it alone", "mesh-catalog",
|
||||
[]string{"modules/route-proxy/Dockerfile"}, read, "route-proxy", ""},
|
||||
{"the route proxy's manifest: it alone", "mesh-catalog",
|
||||
[]string{"modules/route-proxy/module.json"}, read, "route-proxy", ""},
|
||||
{"the route proxy's README: nothing", "mesh-catalog",
|
||||
[]string{"modules/route-proxy/README.md"}, read, "", "modules/route-proxy/README.md"},
|
||||
{"the build agent's manifest: it alone, nothing it builds", "mesh-catalog",
|
||||
[]string{"modules/build-agent/module.json"}, read, "build-agent", ""},
|
||||
// The live three-tier plan of 2026-10-08 (issue 338), in the worker-of order (issue 206) that
|
||||
// TestAMergeIsPlannedInTiersAlongTheThreeKindsOfDependency's controller case holds too.
|
||||
{"a README of the controller's repository moves all three, in three tiers", "mesh-controller",
|
||||
[]string{"README.md"}, "mesh-controller | build-agent | route-proxy"},
|
||||
{"the controller's own code: the same", "mesh-controller",
|
||||
[]string{"cmd/mesh-controller/main.go"}, "mesh-controller | build-agent | route-proxy"},
|
||||
{"the route proxy's program alone: the same, the controller with it", "mesh-controller",
|
||||
[]string{"examples/route-proxy/main.go"}, "mesh-controller | build-agent | route-proxy"},
|
||||
// In the catalogue, where they live, the rule is path-precise.
|
||||
{"the route proxy's directory in the catalogue: it alone", "mesh-catalog",
|
||||
[]string{"modules/route-proxy/module.json"}, "route-proxy"},
|
||||
{"the build agent's directory: it alone, nothing it builds", "mesh-catalog",
|
||||
[]string{"modules/build-agent/module.json"}, "build-agent"},
|
||||
{"another module of the catalogue: neither", "mesh-catalog",
|
||||
[]string{"modules/gitea/index.ts"}, read, "gitea", ""},
|
||||
[]string{"modules/gitea/index.ts"}, "gitea"},
|
||||
} {
|
||||
r, got := planMerge(t, c.repo, c.paths, entries, c.read, edges)
|
||||
r, got := planMerge(t, c.repo, c.paths, entries, read, edges)
|
||||
if got != c.want {
|
||||
t.Errorf("%s: planned %q, wanted %q", c.what, got, c.want)
|
||||
t.Errorf("%s: planned %q, wanted %q (as today; issue 338)", c.what, got, c.want)
|
||||
}
|
||||
if u := strings.Join(r.Unread, ","); u != c.unread {
|
||||
t.Errorf("%s: unread %q, wanted %q", c.what, u, c.unread)
|
||||
if c.repo == "mesh-controller" && strings.Join(r.Unread, ",") != "" {
|
||||
t.Errorf("%s: a root-built module reads every file, and %v were said unread", c.what, r.Unread)
|
||||
}
|
||||
}
|
||||
// Files not all said: everything the repository builds, as before.
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main", Commit: "head",
|
||||
Paths: []string{"README.md"}, PathsTruncated: true}
|
||||
if got := tiered(reachOfMerge(m, entries, read, edges).Plan.Tiers); got != "mesh-controller | build-agent | route-proxy" {
|
||||
t.Errorf("a merge whose files were not all said: planned %q, wanted all three", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **A module whose recorded build source a plan has overtaken is read whole** (novox/hq ADR 0267): a merge
|
||||
// that added an import to the route proxy is planned; before a build of it works — still building, failed,
|
||||
// or its plan closed before reaching it — a merge changing only the newly imported package must still move
|
||||
// the proxy, since the build source its last build said does not hold that package.
|
||||
func TestAModuleAPlanOvertookIsReadWhole(t *testing.T) {
|
||||
built := time.Date(2026, 10, 10, 1, 0, 0, 0, time.UTC)
|
||||
read := map[string][]inventory.ReadRepository{
|
||||
"route-proxy": {
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"examples/route-proxy/", "go.mod"}, Built: built},
|
||||
{Own: true, Paths: []string{"modules/route-proxy/module.json"}, Built: built},
|
||||
},
|
||||
"mesh-controller": {{Own: true, Paths: []string{"module.json", "cmd/mesh-controller/"}, Built: built}},
|
||||
}
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main", Paths: []string{"internal/newly/imported.go"}}
|
||||
if readsFrom(read["route-proxy"], m) {
|
||||
t.Fatal("the said build source holds the new package: the fixture is wrong")
|
||||
}
|
||||
proxy := func(state string) map[string]*inventory.PlanModule {
|
||||
return map[string]*inventory.PlanModule{"route-proxy": {State: state}, "gitea": {State: "built"}}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
plans []inventory.Plan
|
||||
whole bool
|
||||
}{
|
||||
{"no plan", nil, false},
|
||||
{"a plan still building it", []inventory.Plan{{State: inventory.PlanBuilding, Created: built.Add(-time.Hour),
|
||||
Modules: proxy("building")}}, true},
|
||||
{"a plan made after its build that failed before building it", []inventory.Plan{{State: "failed",
|
||||
Created: built.Add(time.Minute), Modules: proxy("waiting")}}, true},
|
||||
{"a plan made after its build that built it", []inventory.Plan{{State: inventory.PlanDone,
|
||||
Created: built.Add(time.Minute), Modules: proxy("built")}}, false},
|
||||
{"a plan closed before its build", []inventory.Plan{{State: "failed", Created: built.Add(-time.Hour),
|
||||
Modules: proxy("waiting")}}, false},
|
||||
} {
|
||||
view := planningView(read, c.plans)
|
||||
if readsFrom(view["route-proxy"], m) != c.whole {
|
||||
t.Errorf("%s: read whole %v, wanted %v", c.what, !c.whole, c.whole)
|
||||
}
|
||||
if (ownSource(view["route-proxy"]) == nil) != c.whole {
|
||||
t.Errorf("%s: its own build source kept %v", c.what, ownSource(view["route-proxy"]) != nil)
|
||||
}
|
||||
if ownSource(view["mesh-controller"]) == nil {
|
||||
t.Errorf("%s: a module no plan holds lost its build source", c.what)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **A missed merge that moves only a module packaging the repository is acted on** (novox/hq ADR 0267,
|
||||
// issue 266): the catch-up asks wouldMove, which counts it; once a plan or build of it is made after the
|
||||
// merge, the merge is history for it, and the catch-up leaves it.
|
||||
func TestAMissedMergeMovingOnlyAPackagingModuleIsActedOnOnce(t *testing.T) {
|
||||
merged := time.Date(2026, 10, 10, 1, 0, 0, 0, time.UTC)
|
||||
entries := []inventory.Entry{
|
||||
fromRepo("mesh-controller", "http://forge.internal:20000/novox/mesh-controller.git", ""),
|
||||
fromRepo("route-proxy", "http://forge.internal:20000/novox/mesh-catalog.git", "modules/route-proxy"),
|
||||
}
|
||||
read := map[string][]inventory.ReadRepository{
|
||||
"mesh-controller": {{Own: true, Paths: []string{"module.json", "cmd/mesh-controller/"}, Built: merged.Add(-time.Hour)}},
|
||||
"route-proxy": {{Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"examples/route-proxy/"},
|
||||
Built: merged.Add(-time.Hour), Looked: merged.Add(-time.Hour)}},
|
||||
}
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main", Commit: "c1",
|
||||
MergedAt: merged.Format(time.RFC3339), Paths: []string{"examples/route-proxy/main.go"}}
|
||||
if got := wouldMove(m, entries, planningView(read, nil)); len(got) != 1 || got[0].Manifest.Module != "route-proxy" {
|
||||
t.Fatalf("a missed merge of the proxy's program would move %v", got)
|
||||
}
|
||||
// Acted on at once: the plan answering this very merge is a look, however close the clocks.
|
||||
atOnce := []inventory.Plan{{State: inventory.PlanBuilding, Commit: "c1", Created: merged.Add(2 * time.Second),
|
||||
Modules: map[string]*inventory.PlanModule{"route-proxy": {State: "building"}}}}
|
||||
if got := wouldMove(m, entries, planningView(read, atOnce)); len(got) != 0 {
|
||||
t.Fatalf("a merge whose own plan holds the proxy would move %v again", got)
|
||||
}
|
||||
acted := []inventory.Plan{{State: inventory.PlanBuilding, Created: merged.Add(2 * time.Minute),
|
||||
Modules: map[string]*inventory.PlanModule{"route-proxy": {State: "building"}}}}
|
||||
if got := wouldMove(m, entries, planningView(read, acted)); len(got) != 0 {
|
||||
t.Fatalf("a merge acted on for the proxy would move %v again", got)
|
||||
}
|
||||
// A plan that closed without building it looked at nothing: the merge is still news for it.
|
||||
closed := []inventory.Plan{{State: "failed", Created: merged.Add(2 * time.Minute),
|
||||
Modules: map[string]*inventory.PlanModule{"route-proxy": {State: "waiting"}}}}
|
||||
if got := wouldMove(m, entries, planningView(read, closed)); len(got) != 1 {
|
||||
t.Fatalf("a plan that never built the proxy hid the merge from it: %v", got)
|
||||
}
|
||||
// A look just before the merge, on clocks a little apart, is no look after it.
|
||||
skewed := []inventory.Plan{{State: inventory.PlanBuilding, Created: merged.Add(30 * time.Second),
|
||||
Modules: map[string]*inventory.PlanModule{"route-proxy": {State: "building"}}}}
|
||||
if got := wouldMove(m, entries, planningView(read, skewed)); len(got) != 1 {
|
||||
t.Fatalf("a look within the clocks' margin made the merge history: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// sharedRepositoryEdges is what dependenciesOf derives for the catalogue of the test above, sorted as it
|
||||
// sorts them: no packages edge (novox/hq ADR 0267 rule 4).
|
||||
// sorts them.
|
||||
var sharedRepositoryEdges = []inventory.Edge{
|
||||
dep("build-agent", inventory.EdgePackages, "mesh-controller"),
|
||||
dep("build-agent", inventory.EdgeWorkerOf, "mesh-controller"),
|
||||
dep("gitea", inventory.EdgeBuiltBy, "build-agent"),
|
||||
dep("mesh-controller", inventory.EdgeBuiltBy, "build-agent"),
|
||||
dep("route-proxy", inventory.EdgeBuiltBy, "build-agent"),
|
||||
dep("route-proxy", inventory.EdgePackages, "mesh-controller"),
|
||||
}
|
||||
|
||||
// **The planner's invariant, over random catalogues.** For any catalogue whose dependencies form no cycle
|
||||
// and any set of changed files in one repository:
|
||||
//
|
||||
// - the plan is exactly the modules of that repository whose directory holds a changed file (every file,
|
||||
// for a module built from the root), and everything reachable from them along stands-on and declared —
|
||||
// never along packages (novox/hq ADR 0267), built-by or worker-of;
|
||||
// for a module built from the root), and everything reachable from them along stands-on, declared and
|
||||
// packages — never along built-by or worker-of;
|
||||
// - every stands-on, declared, built-by and worker-of edge with both ends in the plan has the module
|
||||
// depended on in an earlier tier;
|
||||
// - no cycle is said.
|
||||
@@ -451,7 +310,7 @@ var sharedRepositoryEdges = []inventory.Edge{
|
||||
func TestAPlanIsTheTouchedModulesAndWhatIsReachableAlongTheWideningEdges(t *testing.T) {
|
||||
kinds := []string{inventory.EdgeStandsOn, inventory.EdgeDeclared, inventory.EdgePackages,
|
||||
inventory.EdgeBuiltBy, inventory.EdgeWorkerOf}
|
||||
widens := map[string]bool{inventory.EdgeStandsOn: true, inventory.EdgeDeclared: true}
|
||||
widens := map[string]bool{inventory.EdgeStandsOn: true, inventory.EdgeDeclared: true, inventory.EdgePackages: true}
|
||||
orders := map[string]bool{inventory.EdgeStandsOn: true, inventory.EdgeDeclared: true,
|
||||
inventory.EdgeBuiltBy: true, inventory.EdgeWorkerOf: true}
|
||||
// Directory names drawn from one pool, so two repositories hold directories of the same name, and one
|
||||
@@ -586,95 +445,3 @@ func describe(entries []inventory.Entry) []string {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// **The merge gate reads the build sources the snapshot carries** (novox/hq ADR 0267): the gate's plan of a
|
||||
// change is the merge handler's, so a snapshot taken by a controller that records build sources narrows the
|
||||
// gate's plan as it narrows the merge's; one without them reads every module as before.
|
||||
func TestTheGatePlansFromTheBuildSourcesTheSnapshotCarries(t *testing.T) {
|
||||
manifest := func(name string) json.RawMessage { return json.RawMessage(`{"module":"` + name + `","version":"1"}`) }
|
||||
facts := snapshot.Facts{Modules: []snapshot.Module{
|
||||
{Name: "mesh-controller", Repository: "novox/mesh-controller", Manifest: manifest("mesh-controller"),
|
||||
Sources: []snapshot.BuildSource{{Own: true, Paths: []string{"module.json", "cmd/mesh-controller/", "internal/broker/"}}}},
|
||||
{Name: "route-proxy", Repository: "novox/mesh-catalog", Path: "modules/route-proxy", Manifest: manifest("route-proxy"),
|
||||
Reads: []string{"novox/mesh-controller"},
|
||||
Sources: []snapshot.BuildSource{{Repository: "novox/mesh-controller", Paths: []string{"examples/route-proxy/", "internal/broker/"}},
|
||||
{Own: true, Paths: []string{"modules/route-proxy/module.json"}}}},
|
||||
}}
|
||||
for paths, want := range map[string]string{
|
||||
"cmd/mesh-controller/main.go": "mesh-controller",
|
||||
"examples/route-proxy/main.go": "route-proxy",
|
||||
"internal/broker/broker.go": "mesh-controller,route-proxy",
|
||||
"README.md": "",
|
||||
} {
|
||||
r, err := reachOfChange(facts, "novox/mesh-controller", []string{paths}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := tiered(r.Plan.Tiers); got != want {
|
||||
t.Errorf("%s: the gate planned %q, wanted %q", paths, got, want)
|
||||
}
|
||||
}
|
||||
// A snapshot without build sources: as before.
|
||||
for i := range facts.Modules {
|
||||
facts.Modules[i].Sources = nil
|
||||
}
|
||||
r, err := reachOfChange(facts, "novox/mesh-controller", []string{"README.md"}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := tiered(r.Plan.Tiers); got != "mesh-controller,route-proxy" {
|
||||
t.Errorf("a snapshot without build sources: the gate planned %q for a README", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **A plan says why each module is in it** (novox/hq ADR 0267, issue 363): the files of its build source the
|
||||
// merge changed, or why it is read whole — never that it packages a repository as though that moved it.
|
||||
func TestAPlanSaysWhyEachModuleIsInIt(t *testing.T) {
|
||||
const controllerRepo = "http://forge.internal:20000/novox/mesh-controller.git"
|
||||
controller := fromRepo("mesh-controller", controllerRepo, "")
|
||||
agent := fromRepo("build-agent", "http://forge.internal:20000/novox/mesh-catalog.git", "modules/build-agent")
|
||||
gitea := fromRepo("gitea", "http://forge.internal:20000/novox/mesh-catalog.git", "modules/gitea")
|
||||
built := time.Date(2026, 10, 10, 12, 26, 0, 0, time.UTC)
|
||||
read := map[string][]inventory.ReadRepository{
|
||||
"mesh-controller": {{Own: true, Paths: []string{"module.json", "cmd/mesh-controller/", "internal/link/"}, Built: built}},
|
||||
"build-agent": {
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"cmd/mesh-builder/", "internal/link/"}, Built: built},
|
||||
{Own: true, Paths: []string{"modules/build-agent/module.json"}, Built: built},
|
||||
},
|
||||
}
|
||||
merge := func(repo string, paths ...string) link.SourceMoved {
|
||||
return link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Paths: paths}
|
||||
}
|
||||
open := []inventory.Plan{{ID: "plan-1", State: inventory.PlanBuilding, Created: built.Add(time.Minute),
|
||||
Modules: map[string]*inventory.PlanModule{"build-agent": {State: "asked"}}}}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
e inventory.Entry
|
||||
read map[string][]inventory.ReadRepository
|
||||
m link.SourceMoved
|
||||
says string
|
||||
}{
|
||||
{"the controller's own closure", controller, read, merge("mesh-controller", "README.md", "internal/link/handacts.go"),
|
||||
"its build source changed: 1 changed file(s) in it, e.g. internal/link/handacts.go"},
|
||||
{"the build seat's closure, through its context", agent, read, merge("mesh-controller", "internal/link/handacts.go"),
|
||||
"its build source in novox/mesh-controller changed: 1 changed file(s) in it, e.g. internal/link/handacts.go"},
|
||||
{"an open plan has yet to build it", agent, planningView(read, open), merge("mesh-controller", "cmd/mesh-controller/main.go"),
|
||||
"read whole: plan plan-1 has not built it yet, so every file of novox/mesh-controller, which its build context is, is its build source"},
|
||||
{"nothing recorded, built from its root", controller, nil, merge("mesh-controller", "README.md"),
|
||||
"read whole: no build source recorded, so every file of its repository is its build source"},
|
||||
{"nothing recorded, in its directory", gitea, nil, merge("mesh-catalog", "modules/gitea/index.ts"),
|
||||
"read whole: no build source recorded, so its directory is its build source; e.g. modules/gitea/index.ts"},
|
||||
{"a root manifest is not in a module's directory", gitea, nil, merge("mesh-catalog", "module.json", "modules/gitea/x.ts"),
|
||||
"read whole: no build source recorded, so its directory is its build source; e.g. modules/gitea/x.ts"},
|
||||
{"a context on another branch says nothing of this one", agent, map[string][]inventory.ReadRepository{"build-agent": {
|
||||
{Repository: "novox/mesh-controller", Ref: "release", Paths: []string{"internal/link/"}},
|
||||
{Repository: "novox/mesh-controller", Ref: "main"}}}, merge("mesh-controller", "internal/link/handacts.go"),
|
||||
"read whole: no build source recorded, so every file of novox/mesh-controller, which its build context is, is its build source"},
|
||||
{"files not all said", controller, read, link.SourceMoved{Owner: "novox", Repo: "mesh-controller", PathsTruncated: true,
|
||||
Paths: []string{"x"}}, "read whole: the merge's changed files were not all said"},
|
||||
} {
|
||||
if got := whyMoved(c.e, c.read[c.e.Manifest.Module], c.m); got != c.says {
|
||||
t.Errorf("%s:\n said %q\n wanted %q", c.what, got, c.says)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,158 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// Following a plan (novox/hq issue 229): `plans wait <id>` holds its answer until the plan moves, reaches its
|
||||
// next tier or ends, or until its bound passes, and says which. Before it, an agent following a rollout had
|
||||
// `plans` answer once, in prose, and wrote a loop of its own round the mesh's tools to be told the plan moved.
|
||||
//
|
||||
// **No wait is longer than a call is answered within** (link.LongestAnswer, inside the console's thirty
|
||||
// seconds): a longer one would answer "still running" and leave its caller to ask `calls` — the loop again.
|
||||
// So a caller following a plan to its end asks again; each call answers the moment the plan moves.
|
||||
|
||||
// maxPlanWait is the longest bound a wait takes, and the bound when none is given: what a call is answered
|
||||
// within, less the time a verb's command takes to start and read the store.
|
||||
var maxPlanWait = link.LongestAnswer - planWaitStart
|
||||
|
||||
// planWaitStart is what the verb's window allows beyond the bound: starting the command and opening the stores.
|
||||
// A store slow enough to push the read and the answer past it degrades as any long call does: the caller is
|
||||
// answered that the call is still running, and `calls` with its id says what came of it.
|
||||
const planWaitStart = 5 * time.Second
|
||||
|
||||
// planWaitEvery is how often a wait reads the plan again.
|
||||
const planWaitEvery = time.Second
|
||||
|
||||
// The things a wait waits until.
|
||||
var planWaitUntil = []string{"moved", "tier", "done"}
|
||||
|
||||
// planWaitAsked checks what a wait is asked for: until (moved when empty) and a bound (a duration, or a number
|
||||
// of seconds; maxPlanWait when empty).
|
||||
func planWaitAsked(until, bound string) (string, time.Duration, error) {
|
||||
if until == "" {
|
||||
until = "moved"
|
||||
}
|
||||
known := false
|
||||
for _, u := range planWaitUntil {
|
||||
known = known || u == until
|
||||
}
|
||||
if !known {
|
||||
return "", 0, fmt.Errorf("a wait is until %s, not %q", strings.Join(planWaitUntil, ", "), until)
|
||||
}
|
||||
d := maxPlanWait
|
||||
if bound = strings.TrimSpace(bound); bound != "" {
|
||||
var err error
|
||||
if secs, isNumber := strconv.Atoi(bound); isNumber == nil {
|
||||
d = time.Duration(secs) * time.Second
|
||||
} else if d, err = time.ParseDuration(bound); err != nil {
|
||||
return "", 0, fmt.Errorf("the bound %q is not a duration (such as 20s) or a number of seconds", bound)
|
||||
}
|
||||
if d <= 0 || d > maxPlanWait {
|
||||
return "", 0, fmt.Errorf("the bound is at most %s, the longest a call is answered within; %s was asked "+
|
||||
"— to follow a plan longer, ask again: each call answers the moment the plan moves", maxPlanWait, d)
|
||||
}
|
||||
}
|
||||
return until, d, nil
|
||||
}
|
||||
|
||||
// planEnded is a plan that will not move again: done, failed or superseded. A batch being assembled is not one.
|
||||
func planEnded(p inventory.Plan) bool { return !p.Open() && !p.Batch() }
|
||||
|
||||
// waitOnPlan reads the plan until what it waits for happens or the bound passes, and answers the plan as last
|
||||
// read and what ended the wait: moved (any save of it), tier (its tier changed), done (it ended — which ends
|
||||
// every wait, since nothing comes after it), or bound. Moved and tier are judged against the plan as first read.
|
||||
func waitOnPlan(ctx context.Context, read func(context.Context) (inventory.Plan, error), until string,
|
||||
bound, every time.Duration) (inventory.Plan, string, error) {
|
||||
first, err := read(ctx)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, "", err
|
||||
}
|
||||
if planEnded(first) {
|
||||
return first, "done", nil
|
||||
}
|
||||
deadline := time.Now().Add(bound)
|
||||
tick := time.NewTicker(every)
|
||||
defer tick.Stop()
|
||||
last := first
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return last, "", ctx.Err()
|
||||
case <-tick.C:
|
||||
}
|
||||
if !time.Now().Before(deadline) {
|
||||
return last, "bound", nil
|
||||
}
|
||||
p, err := read(ctx)
|
||||
if err != nil {
|
||||
return last, "", err
|
||||
}
|
||||
last = p
|
||||
switch {
|
||||
case planEnded(p):
|
||||
return p, "done", nil
|
||||
case until == "moved" && p.Revision != first.Revision:
|
||||
return p, "moved", nil
|
||||
case until == "tier" && p.Tier != first.Tier:
|
||||
return p, "tier", nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// planWaited is what `plans wait` answers: the plan, what ended the wait, and the line a person reads (`text`,
|
||||
// which the verb answers as its output and not again in the data).
|
||||
type planWaited struct {
|
||||
// Came is what ended the wait: moved, tier, done, or bound when nothing did.
|
||||
Came string `json:"came"`
|
||||
Until string `json:"until"`
|
||||
Waited string `json:"waited"`
|
||||
Plan planStatus `json:"plan"`
|
||||
// Revision is the plan's as read: a caller asking again with the same plan sees whether it moved since.
|
||||
Revision int64 `json:"revision"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
func planWaitedDoc(p inventory.Plan, st planStatus, until, came string, waited time.Duration, line string) planWaited {
|
||||
said := map[string]string{
|
||||
"moved": "moved", "tier": "reached another tier", "done": "ended",
|
||||
"bound": "did not " + map[string]string{"moved": "move", "tier": "reach another tier", "done": "end"}[until],
|
||||
}[came]
|
||||
text := fmt.Sprintf("%s %s in %s (waited until %s)\n%s — %s\n", p.ID, said, waited.Round(time.Second), until,
|
||||
p.ID, line)
|
||||
return planWaited{Came: came, Until: until, Waited: waited.Round(time.Millisecond).String(), Plan: st,
|
||||
Revision: p.Revision, Text: text}
|
||||
}
|
||||
|
||||
// planWaitCommand is `plans wait <id>`: the plan followed until it moves, or the bound passes.
|
||||
func planWaitCommand(ctx context.Context, inv *inventory.Inventory, id, untilAsked, boundAsked string) error {
|
||||
until, bound, err := planWaitAsked(untilAsked, boundAsked)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
began := time.Now()
|
||||
p, came, err := waitOnPlan(ctx, func(ctx context.Context) (inventory.Plan, error) {
|
||||
return inv.PlanByID(ctx, id)
|
||||
}, until, bound, planWaitEvery)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Said as `plans <id>` says its first line, and as `status` says the plan.
|
||||
now, one := time.Now(), []inventory.Plan{p}
|
||||
bounds, pause := readTierBounds(ctx, inv, now), buildSeatPause(ctx, inv, one)
|
||||
line := planLineSaid(p, now, pause, bounds.of(p.Repository), func(module string) []string {
|
||||
on, _ := inv.Running(ctx, module)
|
||||
return on
|
||||
}, waitsOf(ctx, inv, one))
|
||||
doc := planWaitedDoc(p, planStatuses(one, now, pause, bounds)[0], until, came, now.Sub(began), line)
|
||||
return json.NewEncoder(os.Stdout).Encode(doc)
|
||||
}
|
||||
@@ -1,196 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A plan as the store would answer it at each read: the reads are counted, and each answers the next.
|
||||
type planReads struct {
|
||||
plans []inventory.Plan
|
||||
n int
|
||||
}
|
||||
|
||||
func (r *planReads) read(context.Context) (inventory.Plan, error) {
|
||||
p := r.plans[min(r.n, len(r.plans)-1)]
|
||||
r.n++
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func followedPlan(rev int64, state string, tier int) inventory.Plan {
|
||||
return inventory.Plan{ID: "plan-1", Repository: "novox/x", Commit: "abc", State: state, Tier: tier,
|
||||
Tiers: [][]string{{"a"}, {"b"}}, Revision: rev}
|
||||
}
|
||||
|
||||
// **`plans` waits for a plan, and says what ended the wait** (novox/hq issue 229): an agent following a
|
||||
// rollout is told when the plan moved, reached its next tier or ended, or that the bound passed — in one
|
||||
// call, without a loop of its own.
|
||||
func TestAWaitForAPlanEndsWhenWhatItWaitsForHappens(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
until string
|
||||
reads []inventory.Plan
|
||||
came string
|
||||
times int
|
||||
}{
|
||||
{"moved: any save", "moved", []inventory.Plan{
|
||||
followedPlan(1, inventory.PlanBuilding, 0), followedPlan(1, inventory.PlanBuilding, 0),
|
||||
followedPlan(2, inventory.PlanBuilding, 0)}, "moved", 3},
|
||||
{"tier: a save inside the tier is not enough", "tier", []inventory.Plan{
|
||||
followedPlan(1, inventory.PlanBuilding, 0), followedPlan(2, inventory.PlanBuilding, 0),
|
||||
followedPlan(3, inventory.PlanRolling, 1)}, "tier", 3},
|
||||
{"tier: a plan that ended has no next tier to wait for", "tier", []inventory.Plan{
|
||||
followedPlan(1, inventory.PlanBuilding, 0), followedPlan(2, inventory.PlanFailed, 0)}, "done", 2},
|
||||
{"done: a new tier is not the end", "done", []inventory.Plan{
|
||||
followedPlan(1, inventory.PlanBuilding, 0), followedPlan(2, inventory.PlanRolling, 1),
|
||||
followedPlan(3, inventory.PlanDone, 1)}, "done", 3},
|
||||
{"done: a plan already ended answers at once", "done", []inventory.Plan{
|
||||
followedPlan(4, inventory.PlanSuperseded, 1)}, "done", 1},
|
||||
{"the bound passes", "moved", []inventory.Plan{followedPlan(1, inventory.PlanBuilding, 0)}, "bound", -1},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
r := &planReads{plans: c.reads}
|
||||
p, came, err := waitOnPlan(context.Background(), r.read, c.until, 50*time.Millisecond, time.Millisecond)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if came != c.came {
|
||||
t.Fatalf("the wait ended as %q, want %q", came, c.came)
|
||||
}
|
||||
if c.times > 0 && r.n != c.times {
|
||||
t.Errorf("read the plan %d times, want %d: it answered before or after what it waited for", r.n, c.times)
|
||||
}
|
||||
if want := c.reads[min(r.n, len(c.reads))-1]; p.Revision != want.Revision {
|
||||
t.Errorf("answered revision %d, not the plan as it was last read (%d)", p.Revision, want.Revision)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// What a wait is asked for is checked before anything is read: what it waits until, and a bound no longer than
|
||||
// a call is answered within.
|
||||
func TestAWaitIsRefusedWhatItCannotDo(t *testing.T) {
|
||||
for _, bad := range []struct{ until, bound string }{{"landed", ""}, {"", "1h"}, {"", "-3s"}, {"", "soon"}} {
|
||||
if _, _, err := planWaitAsked(bad.until, bad.bound); err == nil {
|
||||
t.Errorf("until %q, bound %q was taken", bad.until, bad.bound)
|
||||
}
|
||||
}
|
||||
until, bound, err := planWaitAsked("", "")
|
||||
if err != nil || until != "moved" || bound != maxPlanWait {
|
||||
t.Fatalf("the defaults are %q %s %v, want moved and %s", until, bound, err, maxPlanWait)
|
||||
}
|
||||
if _, bound, _ := planWaitAsked("tier", "12"); bound != 12*time.Second {
|
||||
t.Errorf("a bare number is seconds; read %s", bound)
|
||||
}
|
||||
}
|
||||
|
||||
// **The answer is data, with its prose as a field beside it** (novox/hq issue 229): the verb's `answer` carries
|
||||
// the plan and what ended the wait, and its output is the line a person reads — never JSON inside a string.
|
||||
func TestAWaitForAPlanAnswersAsData(t *testing.T) {
|
||||
p := followedPlan(7, inventory.PlanRolling, 1)
|
||||
st := planStatuses([]inventory.Plan{p}, time.Now(), pauseView{}, tierBounds{})[0]
|
||||
doc := planWaitedDoc(p, st, "tier", "tier", 3*time.Second, "rolling, tier 2 of 2")
|
||||
stdout, _ := json.Marshal(doc)
|
||||
a := answerOf([]string{"plans", "wait", "plan-1"}, stdout, "", true)
|
||||
data, ok := a.Answer.(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("the answer is not data: %v", a.Answer)
|
||||
}
|
||||
if data["came"] != "tier" || data["revision"] != float64(7) {
|
||||
t.Errorf("the data does not say what ended the wait: %v", data)
|
||||
}
|
||||
if plan, _ := data["plan"].(map[string]any); plan["state"] != "rolling" || plan["tier"] != float64(1) {
|
||||
t.Errorf("the data does not carry the plan: %v", data["plan"])
|
||||
}
|
||||
if _, text := data["text"]; text {
|
||||
t.Errorf("the prose is in the data too: %v", data)
|
||||
}
|
||||
if strings.Contains(a.Output, "{") || !strings.Contains(a.Output, "plan-1") || !strings.Contains(a.Output, "tier") {
|
||||
t.Errorf("the output is not the line a person reads: %q", a.Output)
|
||||
}
|
||||
}
|
||||
|
||||
// The verb composes the wait, and asks for a window as long as its bound.
|
||||
func TestThePlansVerbComposesAWait(t *testing.T) {
|
||||
argv, err := argvFor("plans", map[string]any{"wait": "plan-1", "until": "done", "bound": "15s"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := strings.Join(argv, " "); got != "plans wait plan-1 --until done --bound 15s" {
|
||||
t.Fatalf("composed %q", got)
|
||||
}
|
||||
if _, err := argvFor("plans", map[string]any{"until": "done"}); err == nil {
|
||||
t.Error("until without wait was taken")
|
||||
}
|
||||
if _, err := argvFor("plans", map[string]any{"wait": "plan-1", "bound": "1h"}); err == nil {
|
||||
t.Error("a bound past what a call can wait was taken")
|
||||
}
|
||||
}
|
||||
|
||||
// Live, with the store: `plans wait` run as the verb runs it answers when the plan reaches its next tier, as
|
||||
// data, and says the bound passed when nothing moved (novox/hq issue 229).
|
||||
func TestAPlanFollowedFromTheStoreAnswersWhenItMoves(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
now := time.Now().UTC()
|
||||
walk := inventory.Plan{ID: "plan-229", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c229c229",
|
||||
Created: now, State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {}, "b": {}}}
|
||||
if err := open.inventory.SavePlan(ctx, &walk); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
moved := make(chan error, 1)
|
||||
go func() {
|
||||
time.Sleep(1500 * time.Millisecond)
|
||||
p, err := open.inventory.PlanByID(ctx, walk.ID)
|
||||
if err == nil {
|
||||
p.Tier, p.State = 1, inventory.PlanRolling
|
||||
err = open.inventory.SavePlan(ctx, &p)
|
||||
}
|
||||
moved <- err
|
||||
}()
|
||||
began := time.Now()
|
||||
var got planWaited
|
||||
out := printed(t, func() error {
|
||||
return plansCommand(ctx, []string{"wait", walk.ID, "--until", "tier", "--bound", "15s"})
|
||||
})
|
||||
if err := <-moved; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := json.Unmarshal([]byte(out), &got); err != nil {
|
||||
t.Fatalf("plans wait did not answer data: %v: %q", err, out)
|
||||
}
|
||||
if got.Came != "tier" || got.Plan.Tier != 1 || got.Plan.State != inventory.PlanRolling {
|
||||
t.Fatalf("the wait answered %+v", got)
|
||||
}
|
||||
if waited := time.Since(began); waited > 10*time.Second {
|
||||
t.Errorf("it answered after %s, not when the plan moved", waited)
|
||||
}
|
||||
out = printed(t, func() error {
|
||||
return plansCommand(ctx, []string{"wait", walk.ID, "--bound", "2"})
|
||||
})
|
||||
if err := json.Unmarshal([]byte(out), &got); err != nil || got.Came != "bound" || got.Until != "moved" {
|
||||
t.Fatalf("a plan that did not move answered %+v (%v)", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The verb's schema says the longest bound in words, and the words are the value (review 75 on #266): the
|
||||
// catalogue cannot read maxPlanWait, so this holds them together.
|
||||
func TestThePlansSchemaSaysTheLongestBoundThereIs(t *testing.T) {
|
||||
v, ok := controllerVerb("plans")
|
||||
if !ok {
|
||||
t.Fatal("no plans verb")
|
||||
}
|
||||
props, _ := v.Input["properties"].(map[string]any)
|
||||
bound, _ := props["bound"].(map[string]any)
|
||||
said, _ := bound["description"].(string)
|
||||
if !strings.Contains(said, maxPlanWait.String()+", the default, is the longest") {
|
||||
t.Fatalf("the schema says %q; the longest bound is %s", said, maxPlanWait)
|
||||
}
|
||||
}
|
||||
@@ -1,378 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// Who can become root where the trusted parties run (novox/hq ADR 0259 §8, as reviewed on 2026-10-09).
|
||||
//
|
||||
// The router and every channel proving its sender run as accounts of their own, so that no agent reads what
|
||||
// they hold or speaks as them. **Root on their machine undoes all of it**, and so does an agent running as the
|
||||
// operator's account there. A machine is **root-free** — an answer proven there may authorise — only when all
|
||||
// of these are measured, now, and hold:
|
||||
//
|
||||
// 1. the machine names an account agents run as (novox/hq ADR 0266), so no agent runs as the operator's
|
||||
// account, which may become root;
|
||||
// 2. its node-engine — running as root, which no agent controls — judged that account unable to become root
|
||||
// without a person, in a statement heard within the last 15 minutes (agentConfined, judgedConfined). The
|
||||
// engine gives that verdict only from a complete search for setuid programs younger than mesh-host's
|
||||
// rootsearch.FreshFor; before one, it says "not judged yet", which is no pass (novox/hq issue 361);
|
||||
// 3. the login shell's `execute` is not served there (novox/hq ADR 0268): its holder's setting withholds it
|
||||
// **and** the bus was asked and heard no `execute` answered there. `execute` runs commands as the machine's
|
||||
// runtime account, which the mesh's acting tools give passwordless sudo; that account is taken to become
|
||||
// root, always, so no measure of it is asked.
|
||||
//
|
||||
// **Nothing else is a pass.** A machine that names no agent account, an unknown machine, a store or bus that
|
||||
// could not be read, a verdict stale or absent — each is not root-free, and says why. The sudo module's own
|
||||
// measure is no longer part of this judgement: it ran in the machine's runtime, as the very account an agent
|
||||
// could become, so it could not be believed.
|
||||
//
|
||||
// The one judgement (judgeRoot) is read two ways: the self-check raises `root-not-free` on every machine where
|
||||
// the router or a module of its own account runs and the judgement fails; and the `root-free` verb answers it
|
||||
// live, to the router, which honours a verified sender only on its pass. A machine holding the operator's
|
||||
// graphical session, where a messaging client's desktop app may run, is not judged here: the operator accepted
|
||||
// that gap for now (hq issue 344).
|
||||
|
||||
// kindRootNotFree is the condition a trusted party's machine that is not root-free raises. Its own key, apart
|
||||
// from ADR 0266's agent-can-become-root (Token agent-root, from DA): the two judge different things — DA the
|
||||
// agent account alone, this the whole of root-free — and one key from two probes flapped between them (the
|
||||
// confirmation review of 2026-10-09).
|
||||
const kindRootNotFree = "root-not-free"
|
||||
|
||||
// routerSeat is the seat the router holds: where it runs counts as a trusted party's machine.
|
||||
const routerSeat = "operator-channel"
|
||||
|
||||
const (
|
||||
loginShellSeat = "node-login-shell"
|
||||
// loginShellVerb is the seat's verb that runs a command, and the name of the setting its holder withholds
|
||||
// it by (novox/hq ADR 0268).
|
||||
loginShellVerb = "execute"
|
||||
// executeServes is the one value of that setting that serves the verb; anything else withholds it.
|
||||
executeServes = "serve"
|
||||
)
|
||||
|
||||
// loginShellServed judges whether the login shell's execute is served on a machine, failing closed (novox/hq
|
||||
// ADR 0268): served while the holder's setting there is `serve` (or the holder has no such setting and claims
|
||||
// the verb), or while the bus heard the verb answered there, or while the bus could not be asked. why says
|
||||
// which, in words.
|
||||
func loginShellServed(holder string, claims bool, setting *any, heard, asked bool) (bool, string) {
|
||||
var why []string
|
||||
switch {
|
||||
case setting != nil:
|
||||
if v, _ := (*setting).(string); v == executeServes {
|
||||
why = append(why, holder+"'s execute setting there is "+executeServes)
|
||||
}
|
||||
case claims:
|
||||
why = append(why, holder+" claims execute and has no setting that withholds it")
|
||||
}
|
||||
if heard {
|
||||
why = append(why, "the bus hears execute answered there")
|
||||
} else if !asked {
|
||||
why = append(why, "the bus could not be asked whether execute is answered there")
|
||||
}
|
||||
return len(why) > 0, strings.Join(why, "; ")
|
||||
}
|
||||
|
||||
// rootFacts is what the judgement reads of one machine.
|
||||
type rootFacts struct {
|
||||
Machine string
|
||||
// Unread is every read that failed, in words: any one is a fail.
|
||||
Unread []string
|
||||
// AgentNamed is whether the machine names an agent account; Confined whether its node-engine judged it
|
||||
// unable to become root, freshly; ConfinedWhy the judgement's words either way.
|
||||
AgentNamed bool
|
||||
Confined bool
|
||||
ConfinedWhy string
|
||||
// Execute is whether the login shell's execute is served there; ExecuteWhy why, in words.
|
||||
Execute bool
|
||||
ExecuteWhy string
|
||||
// SearchPending is the agent account unjudged only because the node-engine's setuid search runs, within
|
||||
// the quiet the controller gives it (searchQuietFor; ADR 0266's quiet window, issue 361).
|
||||
SearchPending bool
|
||||
}
|
||||
|
||||
// rootVerdict is the judgement on one machine, as the root-free verb answers it.
|
||||
type rootVerdict struct {
|
||||
Machine string `json:"machine"`
|
||||
Free bool `json:"free"`
|
||||
Why string `json:"why"`
|
||||
Judged time.Time `json:"judged"`
|
||||
// Quiet is a machine not free only because its setuid search still runs, within searchQuietFor: the
|
||||
// self-check raises nothing for it then (ADR 0266's quiet window). It is never free for it.
|
||||
Quiet bool `json:"quiet,omitempty"`
|
||||
}
|
||||
|
||||
// judgeRoot is the one judgement: free only when nothing failed to read, an agent account is named and judged
|
||||
// confined, and execute is not served.
|
||||
func judgeRoot(f rootFacts, now time.Time) rootVerdict {
|
||||
v := rootVerdict{Machine: f.Machine, Judged: now.UTC()}
|
||||
var not []string
|
||||
if len(f.Unread) > 0 {
|
||||
not = append(not, "not measured: "+strings.Join(f.Unread, "; "))
|
||||
}
|
||||
switch {
|
||||
case f.ConfinedWhy == "":
|
||||
// Not read (said above), or nothing said of it: never a pass.
|
||||
if len(f.Unread) == 0 {
|
||||
not = append(not, "whether agents there can become root was not judged")
|
||||
}
|
||||
case !f.AgentNamed:
|
||||
not = append(not, "agents run as the operator's account there, which may become root ("+f.ConfinedWhy+")")
|
||||
case !f.Confined:
|
||||
not = append(not, f.ConfinedWhy)
|
||||
}
|
||||
if f.Execute {
|
||||
not = append(not, "the login shell runs any command an agent gives it as the machine's runtime account, "+
|
||||
"which can become root ("+orNoneKnown(f.ExecuteWhy)+")")
|
||||
}
|
||||
if len(not) > 0 {
|
||||
v.Why = strings.Join(not, "; ")
|
||||
// The one failure is the agent account not judged yet, because its search runs.
|
||||
v.Quiet = len(not) == 1 && f.SearchPending && f.AgentNamed && !f.Confined && len(f.Unread) == 0 && !f.Execute
|
||||
return v
|
||||
}
|
||||
v.Free = true
|
||||
v.Why = f.ConfinedWhy + "; the login shell's execute is not served there"
|
||||
return v
|
||||
}
|
||||
|
||||
// rootReader reads the facts of machines live: the catalogue's placements, the node-engine's verdicts and the
|
||||
// bus's discovery, each once per reader.
|
||||
type rootReader struct {
|
||||
entries []inventory.Entry
|
||||
read error
|
||||
heard map[string]map[string]map[string]bool
|
||||
asked error
|
||||
// confined is agentConfined; settings the login shell holder's settings on a machine. Replaceable in a test.
|
||||
confined func(ctx context.Context, node string, now time.Time) (named, confined bool, why string, err error)
|
||||
// quiet says the one thing keeping a machine's agent account unjudged is the node-engine's setuid
|
||||
// search, within searchQuietFor (ADR 0266, searchStillRunning). Read by the self-check alone, to raise nothing
|
||||
// then; nil reads no quiet. It never makes a machine root-free.
|
||||
quiet func(ctx context.Context, node string, now time.Time) bool
|
||||
settings func(ctx context.Context, node, module string) ([]catalogue.Layer, error)
|
||||
}
|
||||
|
||||
func newRootReader(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn) *rootReader {
|
||||
r := &rootReader{}
|
||||
r.entries, r.read = inv.Catalogued(ctx)
|
||||
if conn == nil {
|
||||
r.asked = fmt.Errorf("this process holds no connection to the bus")
|
||||
} else {
|
||||
r.heard, r.asked = discoverSeatVerbs(ctx, conn)
|
||||
}
|
||||
r.confined = func(ctx context.Context, node string, now time.Time) (bool, bool, string, error) {
|
||||
return agentConfined(ctx, inv, node, now)
|
||||
}
|
||||
r.settings = inv.SettingsFor
|
||||
return r
|
||||
}
|
||||
|
||||
// facts reads one machine, at now.
|
||||
func (r *rootReader) facts(ctx context.Context, machine string, now time.Time) rootFacts {
|
||||
f := rootFacts{Machine: machine}
|
||||
if r.read != nil {
|
||||
f.Unread = append(f.Unread, "the catalogue's placements could not be read: "+r.read.Error())
|
||||
}
|
||||
named, confined, why, err := r.confined(ctx, machine, now)
|
||||
if err != nil {
|
||||
f.Unread = append(f.Unread, "the account agents run as could not be read: "+err.Error())
|
||||
} else {
|
||||
f.AgentNamed, f.Confined, f.ConfinedWhy = named, confined, why
|
||||
}
|
||||
f.Execute, f.ExecuteWhy = r.executeServed(ctx, machine)
|
||||
if r.quiet != nil && f.AgentNamed && !f.Confined {
|
||||
f.SearchPending = r.quiet(ctx, machine, now)
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// executeServed is whether the login shell's execute is served on a machine, failing closed: the bus not
|
||||
// asked, the placements not read, or a holder's setting not read, is served.
|
||||
func (r *rootReader) executeServed(ctx context.Context, machine string) (bool, string) {
|
||||
heard := r.heard[loginShellSeat][loginShellVerb][machine]
|
||||
asked := r.asked == nil
|
||||
var whys []string
|
||||
served := false
|
||||
holders := 0
|
||||
for _, e := range r.entries {
|
||||
if !e.Manifest.ClaimsSeat(loginShellSeat) || !slices.Contains(e.On, machine) {
|
||||
continue
|
||||
}
|
||||
holders++
|
||||
var setting *any
|
||||
if _, declared := e.Manifest.Settings[loginShellVerb]; declared {
|
||||
layers, err := r.settings(ctx, machine, e.Manifest.Module)
|
||||
if err != nil {
|
||||
served = true
|
||||
whys = append(whys, e.Manifest.Module+"'s setting there could not be read: "+err.Error())
|
||||
continue
|
||||
}
|
||||
for _, s := range catalogue.Effective(e.Manifest, layers) {
|
||||
if s.Key == loginShellVerb {
|
||||
v := s.Value
|
||||
setting = &v
|
||||
}
|
||||
}
|
||||
}
|
||||
if s, why := loginShellServed(e.Manifest.Module, claimServes(e.Manifest, loginShellSeat, loginShellVerb),
|
||||
setting, heard, asked); s {
|
||||
served = true
|
||||
whys = append(whys, why)
|
||||
}
|
||||
}
|
||||
if holders == 0 {
|
||||
// Nobody is assigned to serve it; the bus must still hear nobody answering it.
|
||||
if s, why := loginShellServed("no holder", false, nil, heard, asked); s {
|
||||
served = true
|
||||
whys = append(whys, why)
|
||||
}
|
||||
}
|
||||
if r.read != nil {
|
||||
served = true
|
||||
whys = append(whys, "who holds the login shell there could not be read")
|
||||
}
|
||||
return served, strings.Join(whys, "; ")
|
||||
}
|
||||
|
||||
// claimServes says whether a manifest's claim of a seat names a verb among those it serves.
|
||||
func claimServes(m catalogue.Manifest, seat, verb string) bool {
|
||||
for _, c := range m.Claims {
|
||||
if c.Name == seat && slices.Contains(c.Serves, verb) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// judgeRootFree is the root-free verb's answer: each named machine judged now. It never fails: what could not
|
||||
// be read is a machine not free, saying so.
|
||||
func judgeRootFree(ctx context.Context, r *rootReader, machines []string, now time.Time) []rootVerdict {
|
||||
out := make([]rootVerdict, 0, len(machines))
|
||||
for _, m := range machines {
|
||||
out = append(out, judgeRoot(r.facts(ctx, m, now), now))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// trustedMachines are the machines where the router or a module of its own account runs, each with those
|
||||
// modules.
|
||||
func trustedMachines(entries []inventory.Entry) map[string][]string {
|
||||
trusted := map[string][]string{}
|
||||
for _, e := range entries {
|
||||
for _, node := range e.On {
|
||||
if e.Manifest.RunsAs != "" || e.Manifest.ClaimsSeat(routerSeat) {
|
||||
trusted[node] = append(trusted[node], e.Manifest.Module)
|
||||
}
|
||||
}
|
||||
}
|
||||
return trusted
|
||||
}
|
||||
|
||||
// agentRootObservation is the condition of a trusted party's machine that is not root-free.
|
||||
func agentRootObservation(v rootVerdict, trusted []string) conditions.Observation {
|
||||
trusted = append([]string(nil), trusted...)
|
||||
sort.Strings(trusted)
|
||||
return conditions.Observation{Scope: conditions.ScopeMachine, ID: v.Machine, Token: kindRootNotFree,
|
||||
Machine: v.Machine, Kind: kindRootNotFree, Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("%s is not root-free, where %s run: until it is, the router approves nothing proven "+
|
||||
"there (novox/hq ADR 0259 §8): %s", v.Machine, strings.Join(trusted, ", "), v.Why),
|
||||
Headline: "Phone answers held on " + v.Machine,
|
||||
Needs: "give the programs working for you on " + v.Machine + " an account that cannot become root.",
|
||||
Explanation: "The modules that prove your answers from your phone run on " + v.Machine + ", and the mesh " +
|
||||
"cannot show that a program working for you there is unable to become root or to act as you. Until " +
|
||||
"it can, answers from your phone can only acknowledge.",
|
||||
Resolved: "Answers from your phone can approve again on " + v.Machine}
|
||||
}
|
||||
|
||||
// probeAgentRoot is the probe: every trusted party's machine, judged by the one judgement.
|
||||
func probeAgentRoot(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
var conn *nats.Conn
|
||||
if d.js != nil {
|
||||
conn = d.js.Conn()
|
||||
}
|
||||
inv := d.open.inventory
|
||||
r := newRootReader(ctx, inv, conn)
|
||||
if r.read != nil {
|
||||
return nil, r.read
|
||||
}
|
||||
// The self-check alone reads ADR 0266's quiet window: nothing raised while a machine's setuid search
|
||||
// runs, within searchQuietFor. The root-free verb never reads it, so the machine still answers not free.
|
||||
r.quiet = func(ctx context.Context, node string, now time.Time) bool {
|
||||
n, err := inv.NodeByName(ctx, node)
|
||||
if err != nil || n.AgentAccount == "" {
|
||||
return false
|
||||
}
|
||||
h, had, err := inv.HealthOf(ctx, node)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
quiet, err := searchStillRunning(ctx, inv, node, n.AgentAccount, h, had, now)
|
||||
return err == nil && quiet
|
||||
}
|
||||
return rootObservations(ctx, r, trustedMachines(r.entries), time.Now()), nil
|
||||
}
|
||||
|
||||
// rootObservations judges the machines and says each that fails.
|
||||
func rootObservations(ctx context.Context, r *rootReader, trusted map[string][]string, now time.Time) []conditions.Observation {
|
||||
var machines []string
|
||||
for m := range trusted {
|
||||
machines = append(machines, m)
|
||||
}
|
||||
sort.Strings(machines)
|
||||
var out []conditions.Observation
|
||||
for _, v := range judgeRootFree(ctx, r, machines, now) {
|
||||
if !v.Free && !v.Quiet {
|
||||
out = append(out, agentRootObservation(v, trusted[v.Machine]))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// rootClock is the clock the root-free verb judges by.
|
||||
var rootClock = time.Now
|
||||
|
||||
// rootFreeAnswer is the root-free verb: the named machines, each judged now by the serving controller. Only it
|
||||
// answers: a process that is not serving says so, and a caller reads that as no machine free.
|
||||
func rootFreeAnswer(ctx context.Context, machines string, now time.Time) (any, error) {
|
||||
d := doctorFrom
|
||||
if d == nil || d.open == nil || d.open.inventory == nil {
|
||||
return nil, fmt.Errorf("this controller is not serving, so it judges no machine root-free: ask again, and " +
|
||||
"the serving controller answers")
|
||||
}
|
||||
var names []string
|
||||
for _, m := range strings.Split(machines, ",") {
|
||||
if m = strings.TrimSpace(m); m != "" && !slices.Contains(names, m) {
|
||||
names = append(names, m)
|
||||
}
|
||||
}
|
||||
if len(names) == 0 {
|
||||
return nil, fmt.Errorf("root-free judges the machines named, and none was")
|
||||
}
|
||||
var conn *nats.Conn
|
||||
if d.js != nil {
|
||||
conn = d.js.Conn()
|
||||
}
|
||||
return map[string]any{"machines": judgeRootFree(ctx, newRootReader(ctx, d.open.inventory, conn), names, now)}, nil
|
||||
}
|
||||
|
||||
// rootFreeNow is the machines judged root-free, for composing a push's memberships: only those that pass.
|
||||
func rootFreeNow(ctx context.Context, r *rootReader, machines []string, now time.Time) map[string]bool {
|
||||
free := map[string]bool{}
|
||||
for _, v := range judgeRootFree(ctx, r, machines, now) {
|
||||
if v.Free {
|
||||
free[v.Machine] = true
|
||||
}
|
||||
}
|
||||
return free
|
||||
}
|
||||
@@ -1,265 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
var rootNow = time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
// A machine is root-free only on a positive measure of each thing (the review of 2026-10-09, H2/H3): every
|
||||
// read succeeded, an agent account is named and judged confined by the node-engine, execute is not served.
|
||||
func TestRootFreeIsAPositiveMeasureAndNothingElse(t *testing.T) {
|
||||
pass := rootFacts{Machine: "anchor", AgentNamed: true, Confined: true,
|
||||
ConfinedWhy: "the agent account agents cannot become root without a person (judged 2026-10-09 12:00)"}
|
||||
if v := judgeRoot(pass, rootNow); !v.Free || v.Machine != "anchor" || !v.Judged.Equal(rootNow) {
|
||||
t.Fatalf("the one pass: %+v", v)
|
||||
}
|
||||
fails := map[string]func(*rootFacts){
|
||||
"a read failed": func(f *rootFacts) { f.Unread = []string{"the store did not answer"} },
|
||||
"no agent account named": func(f *rootFacts) { f.AgentNamed, f.Confined = false, false },
|
||||
"not confined": func(f *rootFacts) { f.Confined = false },
|
||||
"nothing said of it": func(f *rootFacts) { f.ConfinedWhy = "" },
|
||||
"execute served": func(f *rootFacts) { f.Execute, f.ExecuteWhy = true, "the bus hears execute answered there" },
|
||||
"confined, but agent read": func(f *rootFacts) { f.Unread, f.ConfinedWhy = []string{"x"}, "" },
|
||||
}
|
||||
for name, mutate := range fails {
|
||||
f := pass
|
||||
mutate(&f)
|
||||
if v := judgeRoot(f, rootNow); v.Free || v.Why == "" {
|
||||
t.Errorf("%s: judged %+v", name, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The reader fails closed on every case the review named: the agent account read only where the coding-agent
|
||||
// module runs (old :225), no account to measure taken for a pass (old :246), and a measure that did not answer
|
||||
// taken for "not root" (old :114, :123).
|
||||
func TestTheRootReaderFailsClosed(t *testing.T) {
|
||||
shell := catalogue.Manifest{Module: "zsh", Claims: []catalogue.Claim{{Name: loginShellSeat, Serves: []string{"execute"}}},
|
||||
Settings: map[string]catalogue.SettingDeclaration{"execute": {Default: "withhold"}}}
|
||||
reader := func() *rootReader {
|
||||
return &rootReader{
|
||||
entries: []inventory.Entry{{Manifest: shell, On: []string{"anchor"}}},
|
||||
heard: map[string]map[string]map[string]bool{},
|
||||
confined: func(context.Context, string, time.Time) (bool, bool, string, error) {
|
||||
return true, true, "the agent account agents cannot become root without a person", nil
|
||||
},
|
||||
settings: func(context.Context, string, string) ([]catalogue.Layer, error) { return nil, nil },
|
||||
}
|
||||
}
|
||||
ctx := context.Background()
|
||||
if v := judgeRootFree(ctx, reader(), []string{"anchor"}, rootNow)[0]; !v.Free {
|
||||
t.Fatalf("the control: agents confined, execute withheld and unheard, everything read: %+v", v)
|
||||
}
|
||||
cases := map[string]func(*rootReader){
|
||||
"no agent account named, no coding-agent module there": func(r *rootReader) {
|
||||
r.confined = func(context.Context, string, time.Time) (bool, bool, string, error) {
|
||||
return false, false, "anchor names no agent account: agents run as the operator account (ops)", nil
|
||||
}
|
||||
},
|
||||
"the node-engine's verdict not read": func(r *rootReader) {
|
||||
r.confined = func(context.Context, string, time.Time) (bool, bool, string, error) {
|
||||
return false, false, "", errors.New("the store did not answer")
|
||||
}
|
||||
},
|
||||
"a stale verdict": func(r *rootReader) {
|
||||
r.confined = func(context.Context, string, time.Time) (bool, bool, string, error) {
|
||||
return true, false, "the agent account agents is not judged: the machine's newest statement was heard at …", nil
|
||||
}
|
||||
},
|
||||
"the bus not asked": func(r *rootReader) { r.asked = errors.New("no bus") },
|
||||
"the placements not read": func(r *rootReader) { r.read = errors.New("no store") },
|
||||
"the holder's setting not read": func(r *rootReader) {
|
||||
r.settings = func(context.Context, string, string) ([]catalogue.Layer, error) { return nil, errors.New("no store") }
|
||||
},
|
||||
"execute heard on the bus": func(r *rootReader) {
|
||||
r.heard = map[string]map[string]map[string]bool{loginShellSeat: {"execute": {"anchor": true}}}
|
||||
},
|
||||
"a holder that serves execute": func(r *rootReader) {
|
||||
r.entries[0].Manifest.Settings = nil
|
||||
},
|
||||
}
|
||||
for name, mutate := range cases {
|
||||
r := reader()
|
||||
mutate(r)
|
||||
if v := judgeRootFree(ctx, r, []string{"anchor"}, rootNow)[0]; v.Free {
|
||||
t.Errorf("%s: judged free: %+v", name, v)
|
||||
}
|
||||
}
|
||||
// A machine with no login shell holder at all: still the bus must hear none.
|
||||
r := reader()
|
||||
r.entries = nil
|
||||
r.heard = map[string]map[string]map[string]bool{loginShellSeat: {"execute": {"anchor": true}}}
|
||||
if v := judgeRootFree(ctx, r, []string{"anchor"}, rootNow)[0]; v.Free {
|
||||
t.Errorf("execute answered by a module nobody assigned: %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
// The probe says agent-root, by the same judgement, on each machine where the router or a module of its own
|
||||
// account runs and that is not root-free; urgent and in plain words; nothing where every one is free.
|
||||
func TestTheProbeSaysEachMachineThatIsNotRootFree(t *testing.T) {
|
||||
entries := []inventory.Entry{
|
||||
{Manifest: catalogue.Manifest{Module: "telegram", RunsAs: "telegram"}, On: []string{"anchor"}},
|
||||
{Manifest: catalogue.Manifest{Module: "messenger", RunsAs: "messenger",
|
||||
Claims: []catalogue.Claim{{Name: routerSeat}}}, On: []string{"anchor"}},
|
||||
{Manifest: catalogue.Manifest{Module: "xorg", Claims: []catalogue.Claim{{Name: catalogue.DisplayServerSeat}}},
|
||||
On: []string{"laptop"}},
|
||||
}
|
||||
trusted := trustedMachines(entries)
|
||||
if len(trusted["anchor"]) != 2 || len(trusted["laptop"]) != 0 {
|
||||
t.Fatalf("trusted %v", trusted)
|
||||
}
|
||||
r := &rootReader{entries: entries, heard: map[string]map[string]map[string]bool{},
|
||||
confined: func(_ context.Context, node string, _ time.Time) (bool, bool, string, error) {
|
||||
return false, false, node + " names no agent account: agents run as the operator account (ops)", nil
|
||||
},
|
||||
settings: func(context.Context, string, string) ([]catalogue.Layer, error) { return nil, nil }}
|
||||
got := rootObservations(context.Background(), r, trusted, rootNow)
|
||||
if len(got) != 1 || got[0].Machine != "anchor" || got[0].Kind != kindRootNotFree || got[0].Severity != conditions.Urgent ||
|
||||
!strings.Contains(got[0].Summary, "messenger, telegram") || !strings.Contains(got[0].Summary, "names no agent account") {
|
||||
t.Fatalf("said %+v", got)
|
||||
}
|
||||
o := got[0]
|
||||
if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Explanation: o.Explanation,
|
||||
Needs: o.Needs, Resolved: o.Resolved}, o.Machine); !ok {
|
||||
t.Errorf("not plain: %s", why)
|
||||
}
|
||||
r.confined = func(context.Context, string, time.Time) (bool, bool, string, error) {
|
||||
return true, true, "confined", nil
|
||||
}
|
||||
if got := rootObservations(context.Background(), r, trusted, rootNow); len(got) != 0 {
|
||||
t.Errorf("said of a free machine: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0268: the login shell counts only where its execute is served, and fails closed.
|
||||
func TestTheLoginShellCountsOnlyWhereExecuteIsServed(t *testing.T) {
|
||||
val := func(v any) *any { return &v }
|
||||
cases := []struct {
|
||||
name string
|
||||
claims bool
|
||||
setting *any
|
||||
heard, asked bool
|
||||
served bool
|
||||
saysInTheWhys string
|
||||
}{
|
||||
{"withheld by the setting and silent on the bus", true, val("withhold"), false, true, false, ""},
|
||||
{"withheld by the setting, the machine not yet pushed", true, val("withhold"), true, true, true, "the bus hears"},
|
||||
{"the setting serves", true, val("serve"), false, true, true, "setting there is serve"},
|
||||
{"a wrong value withholds, as the holder does", true, val("Serve"), false, true, false, ""},
|
||||
{"the setting withholds, the bus could not be asked", true, val("withhold"), false, false, true, "could not be asked"},
|
||||
{"a holder with no such setting that claims execute", true, nil, false, true, true, "claims execute"},
|
||||
{"a holder with no such setting that does not claim it, heard all the same", false, nil, true, true, true, "the bus hears"},
|
||||
{"a holder with no such setting that does not claim it, silent", false, nil, false, true, false, ""},
|
||||
}
|
||||
for _, c := range cases {
|
||||
served, why := loginShellServed("zsh", c.claims, c.setting, c.heard, c.asked)
|
||||
if served != c.served || (c.saysInTheWhys != "" && !strings.Contains(why, c.saysInTheWhys)) {
|
||||
t.Errorf("%s: served %v (%q), want %v saying %q", c.name, served, why, c.served, c.saysInTheWhys)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The root-free verb is the serving controller's alone, answered in its process and never as a command; a
|
||||
// controller not serving answers an error, which the router reads as no machine free.
|
||||
func TestRootFreeIsAnsweredOnlyByTheServingController(t *testing.T) {
|
||||
was := doctorFrom
|
||||
doctorFrom = nil
|
||||
t.Cleanup(func() { doctorFrom = was })
|
||||
if _, err := rootFreeAnswer(context.Background(), "anchor", rootNow); err == nil {
|
||||
t.Error("a controller not serving judged a machine")
|
||||
}
|
||||
if !inProcess["root-free"] {
|
||||
t.Error("root-free is not answered in the serving process")
|
||||
}
|
||||
if _, err := argvFor("root-free", map[string]any{"machines": "anchor"}); err == nil {
|
||||
t.Error("root-free ran as a command")
|
||||
}
|
||||
// The router names its machines as a list (its contract with this verb); one text separated by commas is
|
||||
// the same; anything else in the list is refused.
|
||||
for _, given := range []any{[]any{"anchor", "relay"}, "anchor, relay"} {
|
||||
a, err := readArguments("root-free", map[string]any{"machines": given})
|
||||
if err != nil || a.given["machines"] != "anchor,relay" && a.given["machines"] != "anchor, relay" {
|
||||
t.Errorf("root-free given %v read %v (%v)", given, a, err)
|
||||
}
|
||||
}
|
||||
if _, err := readArguments("root-free", map[string]any{"machines": []any{"anchor", 7}}); err == nil {
|
||||
t.Error("root-free took a number for a machine")
|
||||
}
|
||||
if _, err := readArguments("status", map[string]any{"machines": []any{"anchor"}}); err == nil {
|
||||
t.Error("a verb that takes no list took one")
|
||||
}
|
||||
}
|
||||
|
||||
// The confirmation review of 2026-10-09: ADR 0266's quiet window (#175) keeps the self-check from raising
|
||||
// agent-can-become-root while the node-engine's setuid search runs and no complete one judges. It must not make root-free answer free:
|
||||
// root-free needs a complete, fresh verdict. A verdict still waiting for the search is "not judged" to
|
||||
// agentConfined, so the machine is not root-free, whatever the quiet says — and the same statement, complete
|
||||
// and healthy, is the control.
|
||||
func TestAMachineWaitingForItsFirstSetuidSearchIsNotRootFree(t *testing.T) {
|
||||
now := rootNow
|
||||
statement := func(state, reason string) inventory.NodeHealth {
|
||||
return inventory.NodeHealth{Node: "anchor", Contract: link.RootContract, SaidAt: now, HeardAt: now,
|
||||
Resources: []inventory.ResourceHealth{{Module: "claude-code", Resource: "agent", Kind: link.KindAccount,
|
||||
Target: "agents", State: state, Reason: reason, Root: link.RootNever}}}
|
||||
}
|
||||
judged := func(h inventory.NodeHealth) rootVerdict {
|
||||
confined, why := judgedConfined("agents", h, true, now)
|
||||
return judgeRoot(rootFacts{Machine: "anchor", AgentNamed: true, Confined: confined, ConfinedWhy: why}, now)
|
||||
}
|
||||
if v := judged(statement(link.StateHealthy, "")); !v.Free {
|
||||
t.Fatalf("the control: a complete healthy verdict, fresh: %+v", v)
|
||||
}
|
||||
pending := statement(link.StateUnknown, link.ReasonRootPending+": the search runs")
|
||||
if rootVerdictKind("agents", pending, true, now) != verdictPending {
|
||||
t.Fatal("the statement is not one the quiet window counts as waiting for the search")
|
||||
}
|
||||
if v := judged(pending); v.Free {
|
||||
t.Errorf("a machine whose setuid search is pending was judged root-free: %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
// The confirmation review of 2026-10-09, on #154 beside ADR 0266: D-root keeps ADR 0266's quiet window — nothing
|
||||
// raised while the one thing unjudged is the setuid search, within searchQuietFor — while the root-free verb
|
||||
// still answers the machine not free; and D-root's condition has a key of its own, apart from DA's.
|
||||
func TestRootNotFreeIsQuietWhileTheFirstSearchRunsAndKeyedApartFromDA(t *testing.T) {
|
||||
entries := []inventory.Entry{{Manifest: catalogue.Manifest{Module: "telegram", RunsAs: "telegram"}, On: []string{"anchor"}}}
|
||||
r := &rootReader{entries: entries, heard: map[string]map[string]map[string]bool{},
|
||||
confined: func(context.Context, string, time.Time) (bool, bool, string, error) {
|
||||
return true, false, "the agent account agents is not judged: the search for setuid programs runs", nil
|
||||
},
|
||||
settings: func(context.Context, string, string) ([]catalogue.Layer, error) { return nil, nil },
|
||||
quiet: func(context.Context, string, time.Time) bool { return true }}
|
||||
trusted := trustedMachines(entries)
|
||||
if got := rootObservations(context.Background(), r, trusted, rootNow); len(got) != 0 {
|
||||
t.Errorf("raised while the search runs: %+v", got)
|
||||
}
|
||||
if v := judgeRootFree(context.Background(), r, []string{"anchor"}, rootNow)[0]; v.Free || !v.Quiet {
|
||||
t.Errorf("root-free while the search runs: %+v", v)
|
||||
}
|
||||
// Quiet hides nothing else: the login shell served as well is said.
|
||||
r.heard = map[string]map[string]map[string]bool{loginShellSeat: {"execute": {"anchor": true}}}
|
||||
if got := rootObservations(context.Background(), r, trusted, rootNow); len(got) != 1 {
|
||||
t.Errorf("a second failure was kept quiet: %+v", got)
|
||||
}
|
||||
// Past searchQuietFor, said.
|
||||
r.heard, r.quiet = map[string]map[string]map[string]bool{}, func(context.Context, string, time.Time) bool { return false }
|
||||
got := rootObservations(context.Background(), r, trusted, rootNow)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("a search past searchQuietFor was not said: %+v", got)
|
||||
}
|
||||
// One key per judgement: DA's is machine.<m>.agent-root, this one its own.
|
||||
da := conditions.Observation{Scope: conditions.ScopeMachine, ID: "anchor", Token: "agent-root", Machine: "anchor"}
|
||||
if got[0].Key() == da.Key() {
|
||||
t.Errorf("D-root and DA share the key %s", da.Key())
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user