Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
dc5a8208a2 | ||
|
|
a5a355aeec |
@@ -3,8 +3,3 @@
|
|||||||
# A built binary. The lab writes one here when pointed at the repository root by mistake;
|
# A built binary. The lab writes one here when pointed at the repository root by mistake;
|
||||||
# built artifacts belong in build/, which is already ignored.
|
# built artifacts belong in build/, which is already ignored.
|
||||||
/mesh-builder
|
/mesh-builder
|
||||||
/mesh-controller
|
|
||||||
/postgres-provisioner
|
|
||||||
/redis-provisioner
|
|
||||||
# And where `go build` in cmd/mesh-controller leaves it (novox/hq issue 233).
|
|
||||||
/cmd/mesh-controller/mesh-controller
|
|
||||||
|
|||||||
@@ -99,22 +99,16 @@ proxy-image:
|
|||||||
@echo
|
@echo
|
||||||
@docker image inspect $(PROXY_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
@docker image inspect $(PROXY_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||||
|
|
||||||
# The whole check. Raises a database, runs the repository's own check against it -- merge-check.sh,
|
# The whole gate. Raises a database, runs everything against it, and takes it down again --
|
||||||
# the very script `mesh/repo-check` runs -- and takes the database down again, including when the
|
# including when the tests fail, which is why the teardown is not conditional.
|
||||||
# check fails, which is why the teardown is not conditional and the script's exit status is the
|
|
||||||
# target's.
|
|
||||||
#
|
#
|
||||||
# **It calls the script rather than restating it** (novox/hq issue 431): this target used to list its
|
# **Packages in parallel, under the race detector, each test on a bus of its own** (internal/testbus).
|
||||||
# own steps, its formatting step walked vendor/ where the script's does not, and it failed on
|
# It was one package at a time against one shared bus, because the live tests assert, read and remove
|
||||||
# third-party code no change could fix -- so a developer's check and the gate disagreed, and the steps
|
# the mesh's own objects by their fixed names, and two packages at once deleted what the other read; the
|
||||||
# after formatting never ran through it. The script is the one list of steps; this target only gives it
|
# suite was red run as Go runs it and read as noise. A bus per test, of the release the mesh runs, made
|
||||||
# a database (MESH_TEST_POSTGRES, exported above).
|
# it the same in any order. The timeout bounds a hang to a failure with a stack, never a stalled gate.
|
||||||
#
|
check: fmt vet postgres
|
||||||
# **Packages in parallel, under the race detector, each test on a bus of its own** (internal/testbus),
|
@go test -race -timeout 15m ./... ; status=$$? ; $(MAKE) postgres-stop ; exit $$status
|
||||||
# as the script runs them: a bus per test, of the release the mesh runs, makes the suite the same in any
|
|
||||||
# order, and the timeout bounds a hang to a failure with a stack, never a stalled gate.
|
|
||||||
check: postgres
|
|
||||||
@sh merge-check.sh ; status=$$? ; docker rm -f $(PG_CONTAINER) >/dev/null 2>&1 || true ; exit $$status
|
|
||||||
|
|
||||||
# Without a database the store's tests skip rather than fail, so this is the honest subset and not
|
# Without a database the store's tests skip rather than fail, so this is the honest subset and not
|
||||||
# the gate.
|
# the gate.
|
||||||
@@ -124,10 +118,8 @@ test:
|
|||||||
vet:
|
vet:
|
||||||
go vet ./...
|
go vet ./...
|
||||||
|
|
||||||
# Quick steps for a developer, not part of `check`. The directories gofmt reads must be the ones
|
|
||||||
# merge-check.sh lists, never `.`, which walks vendor/ (novox/hq issue 431).
|
|
||||||
fmt:
|
fmt:
|
||||||
@unformatted=$$(gofmt -l cmd internal examples) ; \
|
@unformatted=$$(gofmt -l . 2>/dev/null) ; \
|
||||||
if [ -n "$$unformatted" ] ; then echo "not gofmt'd:" ; echo "$$unformatted" ; exit 1 ; fi
|
if [ -n "$$unformatted" ] ; then echo "not gofmt'd:" ; echo "$$unformatted" ; exit 1 ; fi
|
||||||
|
|
||||||
postgres:
|
postgres:
|
||||||
|
|||||||
@@ -16,8 +16,7 @@ RUN CGO_ENABLED=0 go build -trimpath -ldflags '-s -w' -o /mesh-builder ./cmd/mes
|
|||||||
FROM ${ALPINE_BASE}
|
FROM ${ALPINE_BASE}
|
||||||
# git to clone what it is asked to build, and the docker client to build and push it. The daemon
|
# git to clone what it is asked to build, and the docker client to build and push it. The daemon
|
||||||
# is the machine's, reached through its socket — a build machine shares the runtime it was given
|
# is the machine's, reached through its socket — a build machine shares the runtime it was given
|
||||||
# rather than running one inside itself. xz to read an upstream release an archive is pinned to
|
# rather than running one inside itself.
|
||||||
# (novox/hq ADR 0316): Go's standard library reads gzip and bzip2 and not xz.
|
RUN apk add --no-cache git docker-cli
|
||||||
RUN apk add --no-cache git docker-cli xz
|
|
||||||
COPY --from=build /mesh-builder /usr/local/bin/mesh-builder
|
COPY --from=build /mesh-builder /usr/local/bin/mesh-builder
|
||||||
ENTRYPOINT ["/usr/local/bin/mesh-builder"]
|
ENTRYPOINT ["/usr/local/bin/mesh-builder"]
|
||||||
|
|||||||
+14
-66
@@ -4,7 +4,6 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
"io"
|
|
||||||
"slices"
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -55,9 +54,6 @@ func assignWith(ctx context.Context, open *stores, node string, opts assignOptio
|
|||||||
if err := refusedMovingTheController(modules); err != nil {
|
if err := refusedMovingTheController(modules); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
// Every line of the answer comes from the one resolution after the write (novox/hq issue 237):
|
|
||||||
// what the reads before it would say about a machine left out is not the answer's.
|
|
||||||
ctx = leftOutSaidBy(ctx, io.Discard)
|
|
||||||
// Held while it is recorded, so it cannot land between a converge's preview and its flip and
|
// Held while it is recorded, so it cannot land between a converge's preview and its flip and
|
||||||
// be taken without ever having been previewed (novox/hq ADR 0100).
|
// be taken without ever having been previewed (novox/hq ADR 0100).
|
||||||
ctx, release, err := holdNodes(ctx, open, []string{node})
|
ctx, release, err := holdNodes(ctx, open, []string{node})
|
||||||
@@ -129,28 +125,22 @@ func assignWith(ctx context.Context, open *stores, node string, opts assignOptio
|
|||||||
for _, line := range settled {
|
for _, line := range settled {
|
||||||
answer += "\n " + line
|
answer += "\n " + line
|
||||||
}
|
}
|
||||||
|
// What this act changed about this node's unmet seat dependencies, and nothing else (novox/hq
|
||||||
|
// ADR 0207): a dependency of a module just assigned, or one this assignment met. The rest of the
|
||||||
|
// node's list, and every other node's, is `status`'s.
|
||||||
|
for _, line := range unheldChange(shelf, node, before, append(append([]string(nil), before...), added...)) {
|
||||||
|
answer += "\n " + line
|
||||||
|
}
|
||||||
// Its bus credential, in the same act (novox/hq issue 203): an assignment pushed before its
|
// Its bus credential, in the same act (novox/hq issue 203): an assignment pushed before its
|
||||||
// credential exists delivers a process that cannot authenticate and crash-loops until somebody
|
// credential exists delivers a process that cannot authenticate and crash-loops until somebody
|
||||||
// runs a second verb and a second push. Issued here when the module speaks on the bus and has
|
// runs a second verb and a second push. Issued here when the module speaks on the bus and has
|
||||||
// no credential yet; kept when it has one, so re-assigning rotates nothing.
|
// no credential yet; kept when it has one, so re-assigning rotates nothing.
|
||||||
var issued []string
|
|
||||||
for _, module := range added {
|
for _, module := range added {
|
||||||
if line := issueOnAssign(ctx, open, node, module); line != "" {
|
if line := issueOnAssign(ctx, open, node, module); line != "" {
|
||||||
issued = append(issued, line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// **The one resolution the rest of the answer is said from** (novox/hq issue 237), taken after
|
|
||||||
// the write: what this act changed about this node's unmet seat dependencies, and nothing else
|
|
||||||
// (novox/hq ADR 0207) — a dependency of a module just assigned, or one this assignment met; the
|
|
||||||
// rest of the node's list, and every other node's, is `status`'s — and whether the node resolves.
|
|
||||||
after := resolvedAfter(ctx, open, shelf, node)
|
|
||||||
for _, line := range unheldChangeTo(shelf, node, before, after.modules, after.unheld) {
|
|
||||||
answer += "\n " + line
|
answer += "\n " + line
|
||||||
}
|
}
|
||||||
for _, line := range issued {
|
|
||||||
answer += "\n " + line
|
|
||||||
}
|
}
|
||||||
plan, err := after.plan, after.err
|
plan, _, err := planFor(ctx, open, node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Kept, and still refused. Both halves are the answer, and the rest of the mesh is still
|
// Kept, and still refused. Both halves are the answer, and the rest of the mesh is still
|
||||||
// worth reporting: this machine's refusal is rarely the only consequence.
|
// worth reporting: this machine's refusal is rarely the only consequence.
|
||||||
@@ -228,8 +218,6 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string)
|
|||||||
if len(modules) == 0 {
|
if len(modules) == 0 {
|
||||||
return "", fmt.Errorf("unassign %s names no module", node)
|
return "", fmt.Errorf("unassign %s names no module", node)
|
||||||
}
|
}
|
||||||
// One resolution after the write, as assign's answer (novox/hq issue 237).
|
|
||||||
ctx = leftOutSaidBy(ctx, io.Discard)
|
|
||||||
ctx, release, err := holdNodes(ctx, open, []string{node})
|
ctx, release, err := holdNodes(ctx, open, []string{node})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
@@ -274,22 +262,15 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string)
|
|||||||
}
|
}
|
||||||
answer := fmt.Sprintf("%s no longer runs %s — run `push %s` to make it so",
|
answer := fmt.Sprintf("%s no longer runs %s — run `push %s` to make it so",
|
||||||
node, strings.Join(modules, ", "), node)
|
node, strings.Join(modules, ", "), node)
|
||||||
after := resolvedAfter(ctx, open, shelf, node)
|
var left []string
|
||||||
for _, line := range unheldChangeTo(shelf, node, assigned, after.modules, after.unheld) {
|
for _, a := range assigned {
|
||||||
|
if !slices.Contains(modules, a) {
|
||||||
|
left = append(left, a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, line := range unheldChange(shelf, node, assigned, left) {
|
||||||
answer += "\n " + line
|
answer += "\n " + line
|
||||||
}
|
}
|
||||||
// Whether the node itself still resolves, from the same resolution: before, only the rest of the
|
|
||||||
// mesh's reads said it, on standard error and once per other machine.
|
|
||||||
if after.err != nil {
|
|
||||||
if unresolvable(after.err) {
|
|
||||||
answer += fmt.Sprintf("\n but %s does not resolve now, so nothing will be sent to it:", node)
|
|
||||||
} else {
|
|
||||||
answer += fmt.Sprintf("\n but whether %s still resolves could not be read:", node)
|
|
||||||
}
|
|
||||||
for _, line := range strings.Split(strings.TrimRight(after.err.Error(), "\n"), "\n") {
|
|
||||||
answer += "\n " + strings.TrimSpace(line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// What it leaves behind that is irreplaceable is kept and retired, never removed (novox/hq ADR 0233).
|
// What it leaves behind that is irreplaceable is kept and retired, never removed (novox/hq ADR 0233).
|
||||||
for _, line := range keptOnUnassign(ctx, open.inventory, node, modules) {
|
for _, line := range keptOnUnassign(ctx, open.inventory, node, modules) {
|
||||||
answer += "\n " + line
|
answer += "\n " + line
|
||||||
@@ -297,39 +278,6 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string)
|
|||||||
return answer + blockedElsewhere(ctx, open, node), nil
|
return answer + blockedElsewhere(ctx, open, node), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// afterAct is the one resolution of a node an act's answer is said from, taken after its write
|
|
||||||
// (novox/hq issue 237): the plan or why there is none, the modules it has on the node, and its unmet
|
|
||||||
// seat dependencies.
|
|
||||||
type afterAct struct {
|
|
||||||
plan catalogue.Resolution
|
|
||||||
err error
|
|
||||||
modules []string
|
|
||||||
unheld []catalogue.Unheld
|
|
||||||
}
|
|
||||||
|
|
||||||
// resolvedAfter resolves the node as `plan` does, once, after the act's write. When the node resolves,
|
|
||||||
// its modules and unmet seat dependencies are the resolution's — the closure, so a holder pulled in by
|
|
||||||
// a requirement counts as `plan` counts it. When it does not, they are judged over the assignments read
|
|
||||||
// after the write, which can only find more unmet than the closure would: a seat said held there is
|
|
||||||
// held in the closure too.
|
|
||||||
func resolvedAfter(ctx context.Context, open *stores, shelf map[string]catalogue.Manifest, node string) afterAct {
|
|
||||||
plan, _, err := planFor(ctx, open, node)
|
|
||||||
if err == nil {
|
|
||||||
out := afterAct{plan: plan, unheld: plan.Unheld}
|
|
||||||
for _, m := range plan.Modules {
|
|
||||||
out.modules = append(out.modules, m.Module)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
out := afterAct{err: err}
|
|
||||||
assigned, readErr := open.inventory.Assigned(ctx, node)
|
|
||||||
if readErr != nil {
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
out.modules, out.unheld = assigned, unheldOver(shelf, node, assigned)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// splitModules is a surface's one `module` field as the modules it names: several, comma-separated,
|
// splitModules is a surface's one `module` field as the modules it names: several, comma-separated,
|
||||||
// are one act (novox/hq ADR 0207), so the holders that depend on each other go on together from the
|
// are one act (novox/hq ADR 0207), so the holders that depend on each other go on together from the
|
||||||
// command API and the controller seat's verbs as they do from the command line.
|
// command API and the controller seat's verbs as they do from the command line.
|
||||||
|
|||||||
@@ -408,9 +408,6 @@ func comparisonOf(module string, c comparison, opts takeOptions) (preview string
|
|||||||
switch {
|
switch {
|
||||||
case sec.Origin == inventory.OriginAccepted:
|
case sec.Origin == inventory.OriginAccepted:
|
||||||
fmt.Fprintf(&b, " %s %s: accepted from a person, carried in as it is\n", what, name)
|
fmt.Fprintf(&b, " %s %s: accepted from a person, carried in as it is\n", what, name)
|
||||||
case sec.Origin == inventory.OriginIssued:
|
|
||||||
// Taken from the provider's software (novox/hq ADR 0305): the value the running service uses.
|
|
||||||
fmt.Fprintf(&b, " %s %s: issued by the provider's software, carried in as it is\n", what, name)
|
|
||||||
case opts.Mint[sec.Name]:
|
case opts.Mint[sec.Name]:
|
||||||
fmt.Fprintf(&b, " %s %s: minted by the mesh; the service takes the new value, as --mint said\n", what, name)
|
fmt.Fprintf(&b, " %s %s: minted by the mesh; the service takes the new value, as --mint said\n", what, name)
|
||||||
case foundData:
|
case foundData:
|
||||||
|
|||||||
@@ -1,180 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **A refusal the bus said once is counted once** (novox/hq issue 402). The bus refused the controller
|
|
||||||
// one publish at 18:52:59 UTC on 2026-10-10 and never again, yet S9 looked at the one remembered
|
|
||||||
// advisory every half minute for the hour it is kept, and the condition said "observed 15 time(s), last
|
|
||||||
// 13s ago" with a refusal in its evidence every 30 seconds: two of us read it as a refusal going on and
|
|
||||||
// went looking for a missing grant. The condition counts what the bus said, with when it last said it,
|
|
||||||
// never how often the controller looked.
|
|
||||||
func TestARefusalSaidOnceIsCountedOnceHoweverOftenItIsLookedAt(t *testing.T) {
|
|
||||||
refused := time.Date(2026, 10, 10, 18, 52, 59, 0, time.UTC)
|
|
||||||
now := refused.Add(10 * time.Second)
|
|
||||||
store := conditions.NewInMemory()
|
|
||||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
|
||||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
|
||||||
t.Cleanup(func() { k.Close(t.Context()) })
|
|
||||||
|
|
||||||
// The words the live condition bus.controller.refused carried on 2026-10-10.
|
|
||||||
said := "the bus refused the controller: nats: permissions violation: Permissions Violation for " +
|
|
||||||
`Publish to "mesh.seat.mesh-delivery.tool.times"`
|
|
||||||
advisory := link.Advisory{Kind: link.AdvisoryRefused, ID: "controller", Said: said,
|
|
||||||
First: refused, Last: refused, Count: 1}
|
|
||||||
look := func() conditions.Condition {
|
|
||||||
t.Helper()
|
|
||||||
f := &signalFacts{now: now, host: "novox", advisories: []link.Advisory{advisory}}
|
|
||||||
if err := k.Reconcile(t.Context(), "S9", watchAdvisories(f)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
c, found, err := conditions.ReadOne(t.Context(), store, "bus.controller.refused")
|
|
||||||
if err != nil || !found {
|
|
||||||
t.Fatalf("bus.controller.refused: found %v, %v", found, err)
|
|
||||||
}
|
|
||||||
return c
|
|
||||||
}
|
|
||||||
|
|
||||||
var c conditions.Condition
|
|
||||||
for range 15 {
|
|
||||||
c = look()
|
|
||||||
now = now.Add(30 * time.Second)
|
|
||||||
}
|
|
||||||
if c.Observations != 1 || len(c.Evidence) != 1 {
|
|
||||||
t.Fatalf("one refusal, looked at 15 times, reads as observed %d time(s) with %d evidence lines: %+v",
|
|
||||||
c.Observations, len(c.Evidence), c.Evidence)
|
|
||||||
}
|
|
||||||
if !c.LastObserved.Equal(refused) || !c.Evidence[0].At.Equal(refused) {
|
|
||||||
t.Fatalf("last observed %s, evidence at %s: the refusal was at %s", c.LastObserved, c.Evidence[0].At, refused)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The bus refuses it three times more between two looks: the condition counts four refusals, says
|
|
||||||
// the newest, and adds one line of evidence for the look that saw them.
|
|
||||||
again := now.Add(-5 * time.Second)
|
|
||||||
advisory.Last, advisory.Count = again, 4
|
|
||||||
c = look()
|
|
||||||
if c.Observations != 4 || len(c.Evidence) != 2 || !c.LastObserved.Equal(again) || !c.Evidence[0].At.Equal(again) {
|
|
||||||
t.Fatalf("four refusals read as observed %d time(s), last %s, evidence %+v", c.Observations, c.LastObserved, c.Evidence)
|
|
||||||
}
|
|
||||||
if !strings.Contains(c.Summary, "(4 times since 18:52 UTC)") {
|
|
||||||
t.Fatalf("summary %q", c.Summary)
|
|
||||||
}
|
|
||||||
now = now.Add(30 * time.Second)
|
|
||||||
if c = look(); c.Observations != 4 || len(c.Evidence) != 2 {
|
|
||||||
t.Fatalf("a look with nothing new counted: observed %d time(s), evidence %+v", c.Observations, c.Evidence)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **A condition that looks at the present still counts its looks** (novox/hq issue 402 changes only
|
|
||||||
// what reads a record): a machine silent for three looks was observed three times, and says so.
|
|
||||||
func TestAConditionOfThePresentCountsItsLooks(t *testing.T) {
|
|
||||||
now := time.Date(2026, 10, 10, 19, 0, 0, 0, time.UTC)
|
|
||||||
store := conditions.NewInMemory()
|
|
||||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
|
||||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
|
||||||
t.Cleanup(func() { k.Close(t.Context()) })
|
|
||||||
var c conditions.Condition
|
|
||||||
for range 3 {
|
|
||||||
var err error
|
|
||||||
if c, err = k.Observe(t.Context(), conditions.Observation{Scope: conditions.ScopeMachine, ID: "ace",
|
|
||||||
Kind: "silent", Machine: "ace", Severity: conditions.Warning, Summary: "ace has not been heard from",
|
|
||||||
Source: "S1"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
now = now.Add(30 * time.Second)
|
|
||||||
}
|
|
||||||
if c.Observations != 3 || len(c.Evidence) != 3 || !c.LastObserved.Equal(now.Add(-30*time.Second)) {
|
|
||||||
t.Fatalf("observed %d time(s), %d evidence lines, last %s", c.Observations, len(c.Evidence), c.LastObserved)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **One key, one watcher** (novox/hq issue 440). A consumer's max-deliveries condition is said from
|
|
||||||
// DEAD_LETTERS while it holds a message the consumer gave up on (issue 330). A max-deliveries advisory
|
|
||||||
// without a token names the same key; read beside it, each look added an observation and a line of
|
|
||||||
// evidence through the dead-letter half, and the two summaries overwrote each other on every look. The
|
|
||||||
// dead-letter row alone says that key, and counts the messages given up on, never the looks.
|
|
||||||
func TestAHeldDeadLetterIsCountedByWhatWasGivenUpNotByTheLooks(t *testing.T) {
|
|
||||||
gaveUp := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
|
|
||||||
now := gaveUp.Add(20 * time.Second)
|
|
||||||
store := conditions.NewInMemory()
|
|
||||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
|
||||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
|
||||||
t.Cleanup(func() { k.Close(t.Context()) })
|
|
||||||
|
|
||||||
const key = "bus.EVENTS.media_sonarr.max-deliveries"
|
|
||||||
held := map[string]int{"EVENTS.media_sonarr": 1}
|
|
||||||
newest := map[string]time.Time{"EVENTS.media_sonarr": gaveUp}
|
|
||||||
advisory := link.Advisory{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.media_sonarr", Stream: "EVENTS",
|
|
||||||
Consumer: "media_sonarr", Said: "sonarr on media handed message 7 over 5 times and gave up on it",
|
|
||||||
First: gaveUp, Last: gaveUp, Count: 1}
|
|
||||||
look := func() conditions.Condition {
|
|
||||||
t.Helper()
|
|
||||||
f := &signalFacts{now: now, host: "novox", advisories: []link.Advisory{advisory},
|
|
||||||
deadLetters: held, deadLettersNewest: newest}
|
|
||||||
if err := k.Reconcile(t.Context(), "S9", watchAdvisories(f)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
c, found, err := conditions.ReadOne(t.Context(), store, key)
|
|
||||||
if err != nil || !found {
|
|
||||||
t.Fatalf("%s: found %v, %v", key, found, err)
|
|
||||||
}
|
|
||||||
if !strings.Contains(c.Summary, "dead-letters verb") {
|
|
||||||
t.Fatalf("the summary is not the dead-letter row's: %q", c.Summary)
|
|
||||||
}
|
|
||||||
return c
|
|
||||||
}
|
|
||||||
|
|
||||||
var c conditions.Condition
|
|
||||||
for range 5 {
|
|
||||||
c = look()
|
|
||||||
now = now.Add(30 * time.Second)
|
|
||||||
}
|
|
||||||
if c.Observations != 1 || len(c.Evidence) != 1 || !c.LastObserved.Equal(gaveUp) {
|
|
||||||
t.Fatalf("one message given up on, looked at five times, reads as observed %d time(s), last %s, "+
|
|
||||||
"with %d evidence lines: %+v", c.Observations, c.LastObserved, len(c.Evidence), c.Evidence)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The consumer gives up on a second message between two looks: two given up on, one more line.
|
|
||||||
again := now.Add(-10 * time.Second)
|
|
||||||
held["EVENTS.media_sonarr"], newest["EVENTS.media_sonarr"] = 2, again
|
|
||||||
c = look()
|
|
||||||
if c.Observations != 2 || len(c.Evidence) != 2 || !c.LastObserved.Equal(again) {
|
|
||||||
t.Fatalf("two given up on read as observed %d time(s), last %s, evidence %+v", c.Observations,
|
|
||||||
c.LastObserved, c.Evidence)
|
|
||||||
}
|
|
||||||
now = now.Add(30 * time.Second)
|
|
||||||
if c = look(); c.Observations != 2 || len(c.Evidence) != 2 {
|
|
||||||
t.Fatalf("a look with nothing new counted: observed %d time(s), evidence %+v", c.Observations, c.Evidence)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The advisory still says the max-deliveries it alone knows: a message given up on that could not be
|
|
||||||
// kept, under a key of its own (issue 330), which issue 440's change leaves as it was.
|
|
||||||
func TestAMessageThatCouldNotBeKeptIsStillSaidFromTheAdvisory(t *testing.T) {
|
|
||||||
at := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
|
|
||||||
f := &signalFacts{now: at, host: "novox", advisories: []link.Advisory{{Kind: link.AdvisoryMaxDeliveries,
|
|
||||||
ID: "EVENTS.media_sonarr", Stream: "EVENTS", Consumer: "media_sonarr", Token: link.AdvisoryNotKept,
|
|
||||||
Said: "sonarr on media gave up on message 7, and it could not be kept", First: at, Last: at, Count: 1}}}
|
|
||||||
said := watchAdvisories(f)
|
|
||||||
if len(said) != 1 || said[0].Key() != "bus.EVENTS.media_sonarr.not-kept" {
|
|
||||||
t.Fatalf("%+v", said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A max-deliveries advisory without a token names a consumer's dead-letter key, which only DEAD_LETTERS
|
|
||||||
// says (novox/hq issues 330 and 440): with nothing held, the advisory alone raises nothing.
|
|
||||||
func TestAMaxDeliveriesAdvisoryAloneRaisesNothing(t *testing.T) {
|
|
||||||
at := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
|
|
||||||
f := &signalFacts{now: at, host: "novox", advisories: []link.Advisory{{Kind: link.AdvisoryMaxDeliveries,
|
|
||||||
ID: "EVENTS.media_sonarr", Stream: "EVENTS", Consumer: "media_sonarr",
|
|
||||||
Said: "sonarr on media handed message 7 over 5 times and gave up on it", First: at, Last: at, Count: 1}}}
|
|
||||||
if said := watchAdvisories(f); len(said) != 0 {
|
|
||||||
t.Fatalf("said %+v", said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,118 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// kindApplyFailing is S21's kind: a machine whose declarations have been refused, or applied only in part,
|
|
||||||
// for longer than one apply interval (novox/hq issue 233).
|
|
||||||
const kindApplyFailing = "apply-failing"
|
|
||||||
|
|
||||||
// applyEvery is how often a node-engine applies what it was last told when nothing new arrives: its
|
|
||||||
// ReconcileEvery. A failure still there after one more of them is not one the next apply clears.
|
|
||||||
const applyEvery = 5 * time.Minute
|
|
||||||
|
|
||||||
// packageManagerKey is how a node-engine names a package manager that does not answer among what failed
|
|
||||||
// (mesh-host's probe.go): "the package manager (pacman)".
|
|
||||||
const packageManagerKey = "the package manager ("
|
|
||||||
|
|
||||||
// watchApplyFailing is S21: **a machine that does not apply what it was sent is said by the mesh, not only
|
|
||||||
// to someone who asks for `status`** (novox/hq issue 233, issue 187's family). A workstation refused every
|
|
||||||
// declaration for an hour and a half because its package manager did not answer, and the mesh's only word
|
|
||||||
// on it was a line in `status`. Raised once the machine's first report of this failure is older than one apply
|
|
||||||
// interval **while the machine is heard** — judged by time, not by how many reports said it: a node-engine
|
|
||||||
// says a re-apply only when how it went changed, so the same failure is one report however long it lasts.
|
|
||||||
// Names the refusal's or the failure's words; cleared by the clean apply the node-engine then says. Not while
|
|
||||||
// the machine is silent, asleep or not known: S1 says silent, and asleep is not lost.
|
|
||||||
func watchApplyFailing(f *signalFacts) []conditions.Observation {
|
|
||||||
heard := map[string]bool{}
|
|
||||||
for _, m := range f.machines {
|
|
||||||
if !m.asleep() && !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) <= heartbeatBound(m.every) {
|
|
||||||
heard[m.name] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var out []conditions.Observation
|
|
||||||
for _, d := range f.doing {
|
|
||||||
if !d.Wrong() || d.Since == nil || f.now.Sub(*d.Since) <= applyEvery || !heard[d.Node] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
out = append(out, applyFailing(d, f.now))
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// applyFailing is one machine's S21 condition.
|
|
||||||
func applyFailing(d inventory.Doing, now time.Time) conditions.Observation {
|
|
||||||
m := d.Node
|
|
||||||
since := d.Since.UTC().Format("2006-01-02 15:04 MST")
|
|
||||||
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Kind: kindApplyFailing, Machine: m,
|
|
||||||
Severity: conditions.Warning, Said: doingWords(d),
|
|
||||||
Resolved: m + " applies what it is sent again"}
|
|
||||||
how := "refused its declaration whole"
|
|
||||||
if d.Outcome != inventory.OutcomeRefused {
|
|
||||||
how = "applied its declaration only in part"
|
|
||||||
}
|
|
||||||
o.Summary = fmt.Sprintf("%s has %s since %s (bound %s): %s", m, how, since, ago(applyEvery), failedNames(d))
|
|
||||||
o.Headline = m + " is not applying its update"
|
|
||||||
o.Explanation = fmt.Sprintf("%s has %s for %s, and trying again has not helped. The details say what fails.",
|
|
||||||
m, plainHow(d), humanDuration(now.Sub(*d.Since)))
|
|
||||||
if tool := unansweringTool(d); tool != "" {
|
|
||||||
o.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent "+
|
|
||||||
"is applied, since %s (bound %s)", m, tool, since, ago(applyEvery))
|
|
||||||
o.Headline = fmt.Sprintf("%s cannot install packages", m)
|
|
||||||
o.Explanation = fmt.Sprintf("The package manager on %s (%s) has not answered for %s, so no package reaches "+
|
|
||||||
"it. Everything else is still applied, so an update that repairs it can land; if none does, it needs a "+
|
|
||||||
"look on the machine.", m, tool, humanDuration(now.Sub(*d.Since)))
|
|
||||||
}
|
|
||||||
return o
|
|
||||||
}
|
|
||||||
|
|
||||||
// plainHow is how a machine failed, as the operator reads it.
|
|
||||||
func plainHow(d inventory.Doing) string {
|
|
||||||
if d.Outcome == inventory.OutcomeRefused {
|
|
||||||
return "refused every update"
|
|
||||||
}
|
|
||||||
return "applied its updates only in part"
|
|
||||||
}
|
|
||||||
|
|
||||||
// unansweringTool is the package manager a machine said does not answer, or "".
|
|
||||||
func unansweringTool(d inventory.Doing) string {
|
|
||||||
for _, r := range d.Failed {
|
|
||||||
if rest, ok := strings.CutPrefix(r.ID, packageManagerKey); ok {
|
|
||||||
return strings.TrimSuffix(rest, ")")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// failedNames is what failed, by name: the refusal, or the failed resources' ids.
|
|
||||||
func failedNames(d inventory.Doing) string {
|
|
||||||
if d.Outcome == inventory.OutcomeRefused {
|
|
||||||
return "refused"
|
|
||||||
}
|
|
||||||
ids := make([]string, 0, len(d.Failed))
|
|
||||||
for _, r := range d.Failed {
|
|
||||||
ids = append(ids, r.ID)
|
|
||||||
}
|
|
||||||
sort.Strings(ids)
|
|
||||||
return "failed " + strings.Join(ids, ", ")
|
|
||||||
}
|
|
||||||
|
|
||||||
// doingWords is the machine's own words on what failed: the evidence, kept inside the mesh.
|
|
||||||
func doingWords(d inventory.Doing) string {
|
|
||||||
if d.Outcome == inventory.OutcomeRefused {
|
|
||||||
return "refused: " + oneLine(d.Refused)
|
|
||||||
}
|
|
||||||
var parts []string
|
|
||||||
for _, r := range d.Failed {
|
|
||||||
parts = append(parts, r.ID+": "+oneLine(r.Error))
|
|
||||||
}
|
|
||||||
sort.Strings(parts)
|
|
||||||
return strings.Join(parts, "; ")
|
|
||||||
}
|
|
||||||
@@ -1,123 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// unansweringPackageManager is anchor's last report as the workstation's read on 2026-10-04, once its
|
|
||||||
// node-engine applies the rest (novox/hq issue 233): the package failed and the package manager did not answer,
|
|
||||||
// the same failure on times reports since for before now.
|
|
||||||
func unansweringPackageManager(now time.Time, forHowLong time.Duration, times int) inventory.Doing {
|
|
||||||
since := now.Add(-forHowLong)
|
|
||||||
return inventory.Doing{Node: "anchor", Outcome: inventory.OutcomeFailed, At: now.Add(-time.Second),
|
|
||||||
Since: &since, Times: times, Applied: 41, Failed: []inventory.FailedResource{
|
|
||||||
{ID: "apply", Error: `applying "pacman.base": nothing can be said about "base": this is the arch host and ` +
|
|
||||||
"pacman does not answer here. Either this machine is not Arch, or its package database is broken: " +
|
|
||||||
"pacman exited 1: error: config file /etc/pacman.conf could not be read: No such file or directory"},
|
|
||||||
{ID: "the package manager (pacman)", Error: "pacman does not answer here, so every package in this " +
|
|
||||||
"declaration fails and the rest of it is applied (novox/hq issue 233): pacman exited 1: error: config " +
|
|
||||||
"file /etc/pacman.conf could not be read: No such file or directory"}}}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **A machine whose package manager does not answer is raised by the mesh itself, naming the tool and the
|
|
||||||
// probe's words** (novox/hq issue 233): the words in the evidence, the path kept out of the summary.
|
|
||||||
func TestAPackageManagerThatDoesNotAnswerIsRaisedNamingItsWords(t *testing.T) {
|
|
||||||
now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
|
|
||||||
f := calm(now)
|
|
||||||
f.doing = []inventory.Doing{unansweringPackageManager(now, 90*time.Minute, 18)}
|
|
||||||
got := watchApplyFailing(f)
|
|
||||||
if len(got) != 1 {
|
|
||||||
t.Fatalf("raised %d conditions, want one: %+v", len(got), got)
|
|
||||||
}
|
|
||||||
o := got[0]
|
|
||||||
if o.Key() != "machine.anchor.apply-failing" || o.Machine != "anchor" {
|
|
||||||
t.Errorf("the condition is not the machine's: %s", o.Key())
|
|
||||||
}
|
|
||||||
if !strings.Contains(o.Summary, "pacman does not answer") || !strings.Contains(o.Summary, "since 2026-10-04 12:30") {
|
|
||||||
t.Errorf("the summary does not name the tool and since when: %s", o.Summary)
|
|
||||||
}
|
|
||||||
if strings.Contains(o.Summary, "/etc/") {
|
|
||||||
t.Errorf("the summary carries a path, which belongs in the evidence: %s", o.Summary)
|
|
||||||
}
|
|
||||||
if !strings.Contains(o.Said, "could not be read") {
|
|
||||||
t.Errorf("the evidence does not carry the probe's words: %s", o.Said)
|
|
||||||
}
|
|
||||||
if !strings.Contains(o.Headline, "cannot install packages") || o.Explanation == "" || o.Resolved == "" {
|
|
||||||
t.Errorf("the plain words do not say it: %+v", o)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **A whole refusal is raised too**, with the node-engine's words: what the workstation's report said before the
|
|
||||||
// node-engine applied the rest.
|
|
||||||
func TestARefusalThatGoesOnIsRaised(t *testing.T) {
|
|
||||||
now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
|
|
||||||
since := now.Add(-time.Hour)
|
|
||||||
f := calm(now)
|
|
||||||
f.doing = []inventory.Doing{{Node: "anchor", Outcome: inventory.OutcomeRefused, Since: &since, Times: 12,
|
|
||||||
Refused: "this is the arch host and pacman does not answer here", At: now.Add(-time.Minute)}}
|
|
||||||
got := watchApplyFailing(f)
|
|
||||||
if len(got) != 1 || !strings.Contains(got[0].Summary, "refused its declaration whole") ||
|
|
||||||
!strings.Contains(got[0].Said, "pacman does not answer") {
|
|
||||||
t.Fatalf("a refusal that went on for an hour was not raised with its words: %+v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **Judged by time while the machine is heard, not by how many reports said it** (review of novox/hq issue 233):
|
|
||||||
// a node-engine says the same failure once, so one report an apply interval old is a fault that goes on; one
|
|
||||||
// younger is not yet; and a silent or sleeping machine is S1's.
|
|
||||||
func TestOneReportOlderThanAnApplyIsRaisedAndAQuietMachineIsNot(t *testing.T) {
|
|
||||||
now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
|
|
||||||
f := calm(now)
|
|
||||||
f.doing = []inventory.Doing{unansweringPackageManager(now, 6*time.Minute, 1)}
|
|
||||||
if got := watchApplyFailing(f); len(got) != 1 {
|
|
||||||
t.Errorf("one report of a failure six minutes old was not raised: %+v", got)
|
|
||||||
}
|
|
||||||
f.doing = []inventory.Doing{unansweringPackageManager(now, 4*time.Minute, 1)}
|
|
||||||
if got := watchApplyFailing(f); len(got) != 0 {
|
|
||||||
t.Errorf("a failure four minutes old was raised before one apply could clear it: %+v", got)
|
|
||||||
}
|
|
||||||
f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 1)}
|
|
||||||
f.machines[0].lastHeard = now.Add(-time.Hour)
|
|
||||||
if got := watchApplyFailing(f); len(got) != 0 {
|
|
||||||
t.Errorf("a silent machine was raised for its last report: %+v", got)
|
|
||||||
}
|
|
||||||
f = calm(now)
|
|
||||||
f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 1)}
|
|
||||||
f.doing[0].Node = "a-machine-the-mesh-does-not-know"
|
|
||||||
if got := watchApplyFailing(f); len(got) != 0 {
|
|
||||||
t.Errorf("a machine not heard of was raised: %+v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **A failed re-apply followed by a clean one clears S21** (review of novox/hq issue 233): the node-engine says
|
|
||||||
// the clean re-apply, the store keeps it as applied with no failure since, and the condition the failure raised
|
|
||||||
// is cleared through the keeper on the next tick.
|
|
||||||
func TestACleanReapplyAfterAFailedOneClearsTheCondition(t *testing.T) {
|
|
||||||
now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
|
|
||||||
store := conditions.NewInMemory()
|
|
||||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
|
||||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
|
||||||
defer k.Close(context.Background())
|
|
||||||
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
|
||||||
|
|
||||||
failing := calm(now)
|
|
||||||
failing.doing = []inventory.Doing{unansweringPackageManager(now, 20*time.Minute, 1)}
|
|
||||||
w.see(t.Context(), failing)
|
|
||||||
open, err := k.Open(t.Context())
|
|
||||||
if err != nil || len(open) != 1 || open[0].Kind != kindApplyFailing {
|
|
||||||
t.Fatalf("the failed re-apply raised %+v (%v), want apply-failing", open, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
clean := calm(now.Add(5 * time.Minute))
|
|
||||||
clean.doing = nil // the clean re-apply recorded: NotDoingWhatTheyWereTold no longer lists the machine
|
|
||||||
w.see(t.Context(), clean)
|
|
||||||
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
|
||||||
t.Fatalf("the condition stayed open after the clean re-apply: %+v", open)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -213,8 +213,6 @@ type asker struct {
|
|||||||
|
|
||||||
saidNoRouter bool
|
saidNoRouter bool
|
||||||
saidNoGrant bool
|
saidNoGrant bool
|
||||||
// saidGrantCaveat is whether a grant held with a caveat was said, so it is said once.
|
|
||||||
saidGrantCaveat bool
|
|
||||||
|
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
nudged chan struct{}
|
nudged chan struct{}
|
||||||
@@ -319,13 +317,6 @@ func (a *asker) reconcile(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
return a.sayUnasked(ctx, unasked, "the bus does not hold the controller's grant to ask yet: "+why)
|
return a.sayUnasked(ctx, unasked, "the bus does not hold the controller's grant to ask yet: "+why)
|
||||||
}
|
}
|
||||||
if why != "" && !a.saidGrantCaveat {
|
|
||||||
// Held, with a caveat worth one line (an older bus module cannot confirm the reload).
|
|
||||||
a.logf("the controller asks: %s", why)
|
|
||||||
a.saidGrantCaveat = true
|
|
||||||
} else if why == "" {
|
|
||||||
a.saidGrantCaveat = false
|
|
||||||
}
|
|
||||||
a.saidNoGrant = false
|
a.saidNoGrant = false
|
||||||
}
|
}
|
||||||
channels := ""
|
channels := ""
|
||||||
|
|||||||
@@ -12,7 +12,6 @@ import (
|
|||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
)
|
)
|
||||||
@@ -809,207 +808,3 @@ func TestARefusalIsAskedAgainAndTheUndeliveredConditionClearsOnceTaken(t *testin
|
|||||||
t.Errorf("the explanation says its verdict twice: %q", e)
|
t.Errorf("the explanation says its verdict twice: %q", e)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// hq issue 353, the review: the bus holds the controller's grant to ask only when both facts hold — the mesh
|
|
||||||
// composes the controller's own grant, and the bus's module says it is in force on the running server — and
|
|
||||||
// each fact not known is not held, said with why.
|
|
||||||
func TestTheGrantIsHeldOnlyWhenComposedAndInForceOnTheBus(t *testing.T) {
|
|
||||||
yes := func(context.Context) (bool, string, error) { return true, "", nil }
|
|
||||||
no := func(why string) grantJudge {
|
|
||||||
return func(context.Context) (bool, string, error) { return false, why, nil }
|
|
||||||
}
|
|
||||||
broken := func(context.Context) (bool, string, error) { return false, "", errors.New("the store is away") }
|
|
||||||
for _, c := range []struct {
|
|
||||||
name string
|
|
||||||
composed, inForce grantJudge
|
|
||||||
held bool
|
|
||||||
says string
|
|
||||||
}{
|
|
||||||
{"composed and in force", yes, yes, true, ""},
|
|
||||||
{"not composed", no("no router is assigned"), yes, false, "no router is assigned"},
|
|
||||||
{"composed, written, not reloaded", yes, no("the server has not reloaded"), false, "has not reloaded"},
|
|
||||||
{"the composition unknown", broken, yes, false, "could not be worked out: the store is away"},
|
|
||||||
{"the bus unknown", yes, broken, false, "could not be worked out: the store is away"},
|
|
||||||
{"nothing judges the bus", yes, nil, false, "not judged"},
|
|
||||||
} {
|
|
||||||
held, why, err := grantHeldBy(c.composed, c.inForce, time.Now)(context.Background())
|
|
||||||
if err != nil || held != c.held || !strings.Contains(why, c.says) {
|
|
||||||
t.Errorf("%s: held %v (%q, %v)", c.name, held, why, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Remembered for grantLookEvery, then judged again.
|
|
||||||
now := time.Date(2026, 10, 9, 18, 0, 0, 0, time.UTC)
|
|
||||||
inForce := false
|
|
||||||
judge := grantHeldBy(yes, func(context.Context) (bool, string, error) { return inForce, "not yet", nil },
|
|
||||||
func() time.Time { return now })
|
|
||||||
if held, _, _ := judge(context.Background()); held {
|
|
||||||
t.Fatal("held before the bus enforces it")
|
|
||||||
}
|
|
||||||
inForce = true
|
|
||||||
if held, _, _ := judge(context.Background()); held {
|
|
||||||
t.Error("judged again inside grantLookEvery")
|
|
||||||
}
|
|
||||||
now = now.Add(grantLookEvery)
|
|
||||||
if held, _, _ := judge(context.Background()); !held {
|
|
||||||
t.Error("not judged again after grantLookEvery")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The controller's own grant, as composed: present while a router that takes asks is assigned, absent
|
|
||||||
// otherwise — whatever else in the user list changes.
|
|
||||||
func TestTheControllersOwnGrantToAskIsReadFromTheComposition(t *testing.T) {
|
|
||||||
router := broker.Declared{Module: "messenger", Holds: []broker.Seat{{Name: broker.AsksSeat, Scope: "mesh",
|
|
||||||
Accepts: []string{"ask", "cancel"}, Emits: []string{"decided"}, ByCaller: []string{"ask", "cancel", "decided"}}}}
|
|
||||||
other := broker.Declared{Module: "dunst"}
|
|
||||||
with := broker.Records{Nodes: []string{"anchor"}, Assigned: map[string][]broker.Declared{"anchor": {router, other}}}
|
|
||||||
without := broker.Records{Nodes: []string{"anchor"}, Assigned: map[string][]broker.Declared{"anchor": {other}}}
|
|
||||||
if may, err := controllerMayAsk(with); err != nil || !may {
|
|
||||||
t.Errorf("with a router assigned: %v %v", may, err)
|
|
||||||
}
|
|
||||||
if may, err := controllerMayAsk(without); err != nil || may {
|
|
||||||
t.Errorf("with no router assigned: %v %v", may, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The bus module's answer, with the record's word on the list sent as the check only where the answer does
|
|
||||||
// not settle it (reviews 22 and 31 of PR 188); each way it is not held names its own remedy.
|
|
||||||
func TestTheBusModulesAnswerNamesTheRemedyForEachCase(t *testing.T) {
|
|
||||||
const (
|
|
||||||
reloaded = `{"allowed":true,"in_force":true,"file_written":"2026-10-09T15:55:56Z","server_loaded":"2026-10-09T15:56:05Z"}`
|
|
||||||
notLoaded = `{"allowed":true,"in_force":false,"why":"the user list on the machine allows it, and the server has not reloaded since it was written","file_written":"2026-10-09T15:55:56Z","server_loaded":"2026-10-09T15:50:00Z"}`
|
|
||||||
noVarz = `{"allowed":true,"in_force":false,"why":"the server's last configuration load could not be read: refused"}`
|
|
||||||
noStat = `{"allowed":true,"in_force":false,"why":"when the bus's user list was written could not be read inside mesh-broker-nats: exit 1"}`
|
|
||||||
notAllowed = `{"allowed":false,"in_force":false,"why":"the user list on the machine does not allow it"}`
|
|
||||||
)
|
|
||||||
current := busListState{Machine: "anchor"}
|
|
||||||
behind := busListState{Machine: "anchor", Behind: true}
|
|
||||||
stepped := busListState{Machine: "anchor", Behind: true, BusStep: "sending anchor would replace the bus; `bus upgrade --why …`"}
|
|
||||||
uncomposed := busListState{Machine: "anchor", Behind: true, Uncomposed: errors.New("module foo requires a provision nobody provides")}
|
|
||||||
for _, c := range []struct {
|
|
||||||
name, answer string
|
|
||||||
askErr error
|
|
||||||
list busListState
|
|
||||||
listErr error
|
|
||||||
held bool
|
|
||||||
says, not string
|
|
||||||
}{
|
|
||||||
{"in force", reloaded, nil, behind, nil, true, "", "push"},
|
|
||||||
{"written, not reloaded", notLoaded, nil, current, nil, false, "within about 5 s", "push"},
|
|
||||||
{"in force not known: varz", noVarz, nil, current, nil, false, "is not known: the server's last configuration load", "wait"},
|
|
||||||
{"in force not known: stat", noStat, nil, current, nil, false, "is not known: when the bus's user list was written", "wait"},
|
|
||||||
{"not allowed, list behind", notAllowed, nil, behind, nil, false, "`push anchor` carries it", "5 s"},
|
|
||||||
{"not allowed, bus step pending", notAllowed, nil, stepped, nil, false, "`bus upgrade", "`push anchor`"},
|
|
||||||
{"not allowed, list uncomposed", notAllowed, nil, uncomposed, nil, false, "`plan anchor`", "`push anchor`"},
|
|
||||||
{"older module, allowed, list current", `{"allowed":true}`, nil, current, nil, true, "not confirmed", "push"},
|
|
||||||
{"older module, allowed, list behind", `{"allowed":true}`, nil, behind, nil, true, "not confirmed", "push"},
|
|
||||||
{"older module, allowed, list unknown", `{"allowed":true}`, nil, current, errors.New("the store is away"), true, "`bus upgrade`", "push"},
|
|
||||||
{"older module, not allowed, list behind", `{"allowed":false}`, nil, behind, nil, false, "`push anchor` carries it", "5 s"},
|
|
||||||
{"module cannot answer, list current", "", errors.New("no responders"), current, nil, true, "could not answer", "push"},
|
|
||||||
{"module cannot answer, list behind", "", errors.New("no responders"), behind, nil, false, "`push anchor` carries it", ""},
|
|
||||||
{"module cannot answer, list unknown", "", errors.New("no responders"), current, errors.New("the store is away"), false, "could not be worked out", ""},
|
|
||||||
{"answer unreadable, list behind", `not json`, nil, behind, nil, false, "could not be read", ""},
|
|
||||||
} {
|
|
||||||
judge := busGrantBy(
|
|
||||||
func(context.Context) (string, json.RawMessage, error) {
|
|
||||||
return "anchor", json.RawMessage(c.answer), c.askErr
|
|
||||||
},
|
|
||||||
func(context.Context) (busListState, error) { return c.list, c.listErr })
|
|
||||||
held, why, _ := judgeGrantOnce(judge)
|
|
||||||
if held != c.held || !strings.Contains(why, c.says) || (c.not != "" && strings.Contains(why, c.not)) {
|
|
||||||
t.Errorf("%s: held %v (%q)", c.name, held, why)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
none := busGrantBy(func(context.Context) (string, json.RawMessage, error) { return "", nil, nil }, nil)
|
|
||||||
if held, why, _ := judgeGrantOnce(none); held || !strings.Contains(why, "no module holds the bus") {
|
|
||||||
t.Errorf("no bus: held %v (%q)", held, why)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// judgeGrantOnce is the whole judgement with the composed grant held, as the asker makes it.
|
|
||||||
func judgeGrantOnce(bus grantJudge) (bool, string, error) {
|
|
||||||
yes := func(context.Context) (bool, string, error) { return true, "", nil }
|
|
||||||
return grantHeldBy(yes, bus, time.Now)(context.Background())
|
|
||||||
}
|
|
||||||
|
|
||||||
// The orders reviews 22 and 31 of PR 188 found: this controller live before the bus module that answers
|
|
||||||
// in_force, with the bus's machine behind on its list — and, after mesh-catalog #166 merged, unpushable until a
|
|
||||||
// person's bus upgrade. An older module that allows the ask asks (once, with the caveat said); a new module that
|
|
||||||
// has not reloaded names the wait, not a push; one that cannot say whether it reloaded names what is not known.
|
|
||||||
func TestAnOlderBusModuleNeitherSilencesTheAsksNorNamesTheWrongRemedy(t *testing.T) {
|
|
||||||
for _, c := range []struct {
|
|
||||||
name, answer string
|
|
||||||
list busListState
|
|
||||||
asks bool
|
|
||||||
says, not string
|
|
||||||
}{
|
|
||||||
{"older module, list current", `{"allowed":true}`, busListState{Machine: "anchor"}, true, "not confirmed", ""},
|
|
||||||
{"older module, list behind, bus step pending", `{"allowed":true}`,
|
|
||||||
busListState{Machine: "anchor", Behind: true, BusStep: "a planned step"}, true, "not confirmed", ""},
|
|
||||||
{"new module, not reloaded",
|
|
||||||
`{"allowed":true,"in_force":false,"why":"not reloaded","file_written":"2026-10-09T15:55:56Z","server_loaded":"2026-10-09T15:50:00Z"}`,
|
|
||||||
busListState{Machine: "anchor"}, false, "within about 5 s", "push"},
|
|
||||||
{"new module, reload not known", `{"allowed":true,"in_force":false,"why":"the server's last configuration load could not be read: refused"}`,
|
|
||||||
busListState{Machine: "anchor"}, false, "is not known", "wait"},
|
|
||||||
} {
|
|
||||||
t.Run(c.name, func(t *testing.T) {
|
|
||||||
r := newAskerRig(t)
|
|
||||||
var said []string
|
|
||||||
r.a.logf = func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }
|
|
||||||
var raised [][]conditions.Observation
|
|
||||||
r.a.raise = func(_ context.Context, obs []conditions.Observation) error {
|
|
||||||
raised = append(raised, obs)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
yes := func(context.Context) (bool, string, error) { return true, "", nil }
|
|
||||||
r.a.grantHeld = grantHeldBy(yes, busGrantBy(
|
|
||||||
func(context.Context) (string, json.RawMessage, error) {
|
|
||||||
return "anchor", json.RawMessage(c.answer), nil
|
|
||||||
},
|
|
||||||
func(context.Context) (busListState, error) { return c.list, nil }),
|
|
||||||
func() time.Time { return r.now })
|
|
||||||
r.open = []conditions.Condition{heldCondition()}
|
|
||||||
_ = r.a.reconcile(context.Background())
|
|
||||||
_ = r.a.reconcile(context.Background())
|
|
||||||
sent := r.asksSent(t)
|
|
||||||
if c.asks {
|
|
||||||
if len(sent) != 1 || sent[0].About != heldCondition().Key {
|
|
||||||
t.Fatalf("not asked: %+v", sent)
|
|
||||||
}
|
|
||||||
n := 0
|
|
||||||
for _, s := range said {
|
|
||||||
if strings.Contains(s, c.says) {
|
|
||||||
n++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if n != 1 {
|
|
||||||
t.Errorf("the caveat said %d times: %q", n, said)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if len(sent) != 0 {
|
|
||||||
t.Fatalf("asked while the bus does not hold the grant: %+v", sent)
|
|
||||||
}
|
|
||||||
if len(raised) == 0 || len(raised[len(raised)-1]) != 1 ||
|
|
||||||
raised[len(raised)-1][0].Kind != "asks-undelivered" {
|
|
||||||
t.Fatalf("not said as undelivered: %+v", raised)
|
|
||||||
}
|
|
||||||
summary := raised[len(raised)-1][0].Summary
|
|
||||||
if !strings.Contains(summary, c.says) || (c.not != "" && strings.Contains(summary, c.not)) {
|
|
||||||
t.Errorf("the condition names the wrong remedy: %q", summary)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The serving asker is wired to the real judgement: with records and a bus it cannot read, the grant is not
|
|
||||||
// held, and so nothing is asked (a stub saying yes, or no judgement at all, would ask).
|
|
||||||
func TestTheServingAskerJudgesTheGrantForReal(t *testing.T) {
|
|
||||||
a := servingAsker(&stores{}, nil, nil, nil)
|
|
||||||
if a.grantHeld == nil {
|
|
||||||
t.Fatal("the serving asker does not judge the bus's grant")
|
|
||||||
}
|
|
||||||
held, why, err := a.grantHeld(context.Background())
|
|
||||||
if err != nil || held || !strings.Contains(why, "could not be worked out") {
|
|
||||||
t.Errorf("held %v (%q, %v) with nothing to read", held, why, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -235,27 +235,12 @@ func routerHereIn(inv *inventory.Inventory) func(ctx context.Context) (bool, err
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// grantHeldIn says whether the bus holds the controller's grant to ask (novox/hq issue 353), judged on the two
|
// grantHeldIn says whether the bus holds the controller's grant to ask (novox/hq issue 353): the user list the
|
||||||
// facts that make it so, and failing closed on each:
|
// machine holding the bus was last sent is the one the mesh composes now (brokerBehind, the same judgement a
|
||||||
//
|
// push makes to send that machine first). While it is behind, the controller's ask is refused by the bus,
|
||||||
// 1. **composed**: the user list the mesh composes now grants the controller itself to publish its ask
|
// whatever the record says of the router, so nothing is asked and the operator is told to push that machine.
|
||||||
// (controllerMayAsk) — the controller's own grant, not whether any user of the list changed;
|
// Judged at most every grantLookEvery: composing the list resolves the bus's machine whole.
|
||||||
// 2. **in force**: the bus's own module, on the machine holding the bus, says the user list there allows it
|
func grantHeldIn(open *stores) func(ctx context.Context) (bool, string, error) {
|
||||||
// and the server reloaded after that list was written (the nats module's `nats_user_can`, `in_force`) — a
|
|
||||||
// list sent is not a list the bus enforces until it reloads.
|
|
||||||
//
|
|
||||||
// Anything that cannot be worked out is not held, with why, so the conditions that need the operator are said
|
|
||||||
// as undelivered rather than asked into a refusal.
|
|
||||||
func grantHeldIn(open *stores, conn *nats.Conn) func(ctx context.Context) (bool, string, error) {
|
|
||||||
return grantHeldBy(composedGrantIn(open), busGrantIn(open, conn), time.Now)
|
|
||||||
}
|
|
||||||
|
|
||||||
// grantJudge is one of the two facts: whether it holds, and why not.
|
|
||||||
type grantJudge func(ctx context.Context) (bool, string, error)
|
|
||||||
|
|
||||||
// grantHeldBy is the judgement over the two facts, remembered for grantLookEvery: composing the list resolves
|
|
||||||
// the bus's machine whole, and asking the bus's module is a call.
|
|
||||||
func grantHeldBy(composed, inForce grantJudge, now func() time.Time) func(ctx context.Context) (bool, string, error) {
|
|
||||||
var mu sync.Mutex
|
var mu sync.Mutex
|
||||||
var at time.Time
|
var at time.Time
|
||||||
var held bool
|
var held bool
|
||||||
@@ -263,251 +248,22 @@ func grantHeldBy(composed, inForce grantJudge, now func() time.Time) func(ctx co
|
|||||||
return func(ctx context.Context) (bool, string, error) {
|
return func(ctx context.Context) (bool, string, error) {
|
||||||
mu.Lock()
|
mu.Lock()
|
||||||
defer mu.Unlock()
|
defer mu.Unlock()
|
||||||
if !at.IsZero() && now().Sub(at) < grantLookEvery {
|
if !at.IsZero() && time.Since(at) < grantLookEvery {
|
||||||
return held, why, nil
|
return held, why, nil
|
||||||
}
|
}
|
||||||
held, why = judgeGrant(ctx, composed, inForce)
|
machine, behind, err := brokerBehind(ctx, open, nil)
|
||||||
at = now()
|
|
||||||
return held, why, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// judgeGrant is held only when both facts hold; an error is a fact not known, never a yes. A fact that holds
|
|
||||||
// with a caveat (a reload not confirmed) gives that caveat as the why of a yes.
|
|
||||||
func judgeGrant(ctx context.Context, composed, inForce grantJudge) (bool, string) {
|
|
||||||
var caveat []string
|
|
||||||
for _, fact := range []struct {
|
|
||||||
judge grantJudge
|
|
||||||
what string
|
|
||||||
}{{composed, "whether the mesh composes the controller a grant to ask"}, {inForce, "whether the bus enforces it"}} {
|
|
||||||
if fact.judge == nil {
|
|
||||||
return false, fact.what + " is not judged here"
|
|
||||||
}
|
|
||||||
ok, why, err := fact.judge(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return false, fact.what + " could not be worked out: " + err.Error()
|
|
||||||
}
|
|
||||||
if !ok {
|
|
||||||
return false, why
|
|
||||||
}
|
|
||||||
if why != "" {
|
|
||||||
caveat = append(caveat, why)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true, strings.Join(caveat, "; ")
|
|
||||||
}
|
|
||||||
|
|
||||||
// controllerMayAsk says whether the controller's own grant, as composed from these records, lets it publish its
|
|
||||||
// ask (ADR 0259 §3: composed while a router that takes asks under its asker's name is assigned).
|
|
||||||
func controllerMayAsk(records broker.Records) (bool, error) {
|
|
||||||
users, err := broker.Users(records)
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
for _, u := range users {
|
|
||||||
if u.Kind != broker.KindController {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
perms, err := broker.PermissionsFor(u)
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
return broker.MayPublish(perms, asks.AskSubject(askerName)), nil
|
|
||||||
}
|
|
||||||
return false, errors.New("the composition holds no controller")
|
|
||||||
}
|
|
||||||
|
|
||||||
// composedGrantIn is the first fact, from the mesh's records.
|
|
||||||
func composedGrantIn(open *stores) grantJudge {
|
|
||||||
return func(ctx context.Context) (bool, string, error) {
|
|
||||||
if open == nil || open.inventory == nil {
|
|
||||||
return false, "", errors.New("the mesh's records are not open")
|
|
||||||
}
|
|
||||||
records, err := open.inventory.BusRecords(ctx)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, "", err
|
return false, "", err
|
||||||
}
|
}
|
||||||
may, err := controllerMayAsk(records)
|
at, held, why = time.Now(), !behind, ""
|
||||||
if err != nil || may {
|
if behind {
|
||||||
return may, "", err
|
why = fmt.Sprintf("the bus's user list on %s is behind what the mesh composes, so the bus has not been "+
|
||||||
|
"given the controller's grant to ask; `push %s` carries it", machine, machine)
|
||||||
}
|
}
|
||||||
return false, "the mesh composes the controller no grant to ask: no router that takes asks under its " +
|
return held, why, nil
|
||||||
"asker's name is assigned", nil
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// busGrantIn is the second fact, asked of the bus's own module on the machine holding the bus, with the check
|
|
||||||
// main made before it (novox/hq issue 353, PR 186: is the user list sent to the bus's machine the one composed
|
|
||||||
// now) kept for the cases the module's word does not settle.
|
|
||||||
func busGrantIn(open *stores, conn *nats.Conn) grantJudge {
|
|
||||||
ask := func(ctx context.Context) (string, json.RawMessage, error) {
|
|
||||||
if open == nil || open.inventory == nil || conn == nil {
|
|
||||||
return "", nil, errors.New("the mesh's records or the bus are not open")
|
|
||||||
}
|
|
||||||
holders, err := seatHolders(ctx, open.inventory)
|
|
||||||
if err != nil {
|
|
||||||
return "", nil, err
|
|
||||||
}
|
|
||||||
h, held := holders[theBrokerSeat]
|
|
||||||
if !held || h.Node == "" || h.Module == "" {
|
|
||||||
return "", nil, nil
|
|
||||||
}
|
|
||||||
a, err := link.AskModuleToolOn(ctx, conn, h.Module, busUserCanTool, h.Node, map[string]any{
|
|
||||||
"user": broker.ControllerName, "action": "publish", "subject": asks.AskSubject(askerName)}, 15*time.Second)
|
|
||||||
if err != nil {
|
|
||||||
return h.Node, nil, err
|
|
||||||
}
|
|
||||||
if a.Error != "" {
|
|
||||||
return h.Node, nil, errors.New(a.Error)
|
|
||||||
}
|
|
||||||
return h.Node, a.Result, nil
|
|
||||||
}
|
|
||||||
list := func(ctx context.Context) (busListState, error) {
|
|
||||||
if open == nil || open.inventory == nil {
|
|
||||||
return busListState{}, errors.New("the mesh's records are not open")
|
|
||||||
}
|
|
||||||
s, err := brokerListState(ctx, open, nil)
|
|
||||||
if err != nil || s.Machine == "" || !s.Behind {
|
|
||||||
return s, err
|
|
||||||
}
|
|
||||||
step, err := busHeld(ctx, open.inventory, []string{s.Machine})
|
|
||||||
if err != nil {
|
|
||||||
return busListState{}, err
|
|
||||||
}
|
|
||||||
s.BusStep = step[s.Machine]
|
|
||||||
return s, nil
|
|
||||||
}
|
|
||||||
return busGrantBy(ask, list)
|
|
||||||
}
|
|
||||||
|
|
||||||
// busListState is the user list on the bus's machine as the controller's record knows it: whether the list
|
|
||||||
// sent there is behind the one composed now, why it could not be composed, and the bus step a send there waits
|
|
||||||
// on (that machine cannot be pushed until a person's `bus upgrade`).
|
|
||||||
type busListState struct {
|
|
||||||
Machine string
|
|
||||||
Behind bool
|
|
||||||
Uncomposed error
|
|
||||||
BusStep string
|
|
||||||
}
|
|
||||||
|
|
||||||
// remedy is the one thing that sends the bus's machine the list composed now: the plan's own reason when it
|
|
||||||
// cannot be composed, the bus upgrade while sends there wait on it, and otherwise a push of that machine.
|
|
||||||
func (s busListState) remedy(node string) string {
|
|
||||||
if s.Machine != "" {
|
|
||||||
node = s.Machine
|
|
||||||
}
|
|
||||||
switch {
|
|
||||||
case s.Uncomposed != nil:
|
|
||||||
return fmt.Sprintf("the user list for %s cannot be composed (%v); `plan %s` says what to fix first",
|
|
||||||
node, s.Uncomposed, node)
|
|
||||||
case s.BusStep != "":
|
|
||||||
return fmt.Sprintf("sends to %s wait on the bus's planned step, so a push will not carry it: %s", node, s.BusStep)
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("`push %s` carries it", node)
|
|
||||||
}
|
|
||||||
|
|
||||||
// busGrantBy judges the bus's word, and names for each way it is not held the one thing that makes it held:
|
|
||||||
//
|
|
||||||
// - **in force**: held;
|
|
||||||
// - **allowed, from a module older than `in_force`**: held, with the caveat that the reload is not confirmed.
|
|
||||||
// The file allows the publish, and the bus reloads within about 5 s of a change to it, so the worst case is
|
|
||||||
// one ask refused in those seconds, which is asked again (issue 369). The list check is not consulted:
|
|
||||||
// while a bus step is pending the bus's machine cannot be pushed, so any change to the composed list would
|
|
||||||
// read as behind for good and silence every ask until a person's bus upgrade (the review of PR 188);
|
|
||||||
// - **allowed, written, not reloaded** (the module gives both times): wait about 5 s;
|
|
||||||
// - **allowed, in force not known** (the module could not read a time): what is not known, said;
|
|
||||||
// - **not allowed, or the module cannot answer**: the list check decides — the remedy that sends the list
|
|
||||||
// when it is behind (busListState.remedy), and, for a module that cannot answer, held while the list sent
|
|
||||||
// is current, as main judged it.
|
|
||||||
//
|
|
||||||
// ask answers the machine holding the bus and its module's answer ("" when nothing holds the bus); list is the
|
|
||||||
// record's word on the list sent to that machine.
|
|
||||||
func busGrantBy(ask func(ctx context.Context) (string, json.RawMessage, error),
|
|
||||||
list func(ctx context.Context) (busListState, error)) grantJudge {
|
|
||||||
return func(ctx context.Context) (bool, string, error) {
|
|
||||||
node, raw, askErr := ask(ctx)
|
|
||||||
if askErr == nil && node == "" {
|
|
||||||
return false, "no module holds the bus, so nothing says what it enforces", nil
|
|
||||||
}
|
|
||||||
var answer busAnswer
|
|
||||||
if askErr == nil {
|
|
||||||
if answer, askErr = readBusAnswer(raw); askErr != nil {
|
|
||||||
askErr = fmt.Errorf("its answer could not be read: %w", askErr)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if askErr != nil {
|
|
||||||
if node == "" {
|
|
||||||
node = "the bus's machine"
|
|
||||||
}
|
|
||||||
s, err := list(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return false, "", fmt.Errorf("the bus module on %s could not answer (%v), and whether the list sent "+
|
|
||||||
"there is the one composed now could not be worked out: %w", node, askErr, err)
|
|
||||||
}
|
|
||||||
if s.Behind {
|
|
||||||
return false, fmt.Sprintf("the bus module on %s could not answer (%v), and the user list sent there is "+
|
|
||||||
"behind what the mesh composes; %s", node, askErr, s.remedy(node)), nil
|
|
||||||
}
|
|
||||||
return true, fmt.Sprintf("the bus module on %s could not answer (%v); judged by the user list sent there, "+
|
|
||||||
"which is the one composed now", node, askErr), nil
|
|
||||||
}
|
|
||||||
notAllowed := func(older string) (bool, string, error) {
|
|
||||||
s, err := list(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return false, "", fmt.Errorf("the user list on %s does not allow the controller to ask%s, and whether "+
|
|
||||||
"the list sent there is the one composed now could not be worked out: %w", node, older, err)
|
|
||||||
}
|
|
||||||
if s.Behind {
|
|
||||||
return false, fmt.Sprintf("the user list on %s does not allow the controller to ask%s: it is behind "+
|
|
||||||
"what the mesh composes; %s", node, older, s.remedy(node)), nil
|
|
||||||
}
|
|
||||||
return false, fmt.Sprintf("the user list on %s does not allow the controller to ask%s, though the list "+
|
|
||||||
"sent there is the one composed now; %s again", node, older, s.remedy(node)), nil
|
|
||||||
}
|
|
||||||
switch {
|
|
||||||
case answer.InForce == nil && answer.Allowed:
|
|
||||||
return true, fmt.Sprintf("the user list on %s allows the controller to ask, but its reload is not "+
|
|
||||||
"confirmed: the bus module there is older than the answer whether a grant is in force, and the bus "+
|
|
||||||
"upgrade (`bus upgrade`) carries one that confirms it", node), nil
|
|
||||||
case answer.InForce == nil:
|
|
||||||
return notAllowed(" (the bus module there is older than the answer whether a grant is in force)")
|
|
||||||
case !answer.Allowed:
|
|
||||||
return notAllowed("")
|
|
||||||
case *answer.InForce:
|
|
||||||
return true, "", nil
|
|
||||||
case answer.FileWritten != "" && answer.ServerLoaded != "":
|
|
||||||
return false, fmt.Sprintf("the bus on %s has the controller's grant to ask written (%s) but has not "+
|
|
||||||
"reloaded since (last load %s); it reloads in place within about 5 s, so nothing is to be done but wait",
|
|
||||||
node, answer.FileWritten, answer.ServerLoaded), nil
|
|
||||||
}
|
|
||||||
why := answer.Why
|
|
||||||
if why == "" {
|
|
||||||
why = "it gives no reason"
|
|
||||||
}
|
|
||||||
return false, fmt.Sprintf("the user list on %s allows the controller to ask, but whether the bus enforces it "+
|
|
||||||
"is not known: %s", node, why), nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// busUserCanTool is the bus module's answer to whether a user may do something, and whether it is in force.
|
|
||||||
const busUserCanTool = "nats_user_can"
|
|
||||||
|
|
||||||
// busAnswer is the bus module's answer to busUserCanTool; InForce is nil from a module older than that field,
|
|
||||||
// and FileWritten and ServerLoaded are given only when it could read both times (mesh-catalog's nats InForce).
|
|
||||||
type busAnswer struct {
|
|
||||||
Allowed bool `json:"allowed"`
|
|
||||||
InForce *bool `json:"in_force"`
|
|
||||||
Why string `json:"why"`
|
|
||||||
FileWritten string `json:"file_written"`
|
|
||||||
ServerLoaded string `json:"server_loaded"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func readBusAnswer(raw json.RawMessage) (busAnswer, error) {
|
|
||||||
var answer busAnswer
|
|
||||||
err := json.Unmarshal(raw, &answer)
|
|
||||||
return answer, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// grantLookEvery is how often the bus's user list is judged against the one its machine was last sent.
|
// grantLookEvery is how often the bus's user list is judged against the one its machine was last sent.
|
||||||
const grantLookEvery = 30 * time.Second
|
const grantLookEvery = 30 * time.Second
|
||||||
|
|
||||||
@@ -546,19 +302,7 @@ func startAsking(ctx context.Context, open *stores, server *link.Server, conn *n
|
|||||||
fmt.Printf("the operator cannot be asked: %v\n", err)
|
fmt.Printf("the operator cannot be asked: %v\n", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
a := servingAsker(open, conn, js, keeper)
|
a := &asker{
|
||||||
if err := server.Decides(a); err != nil {
|
|
||||||
fmt.Printf("the operator's answers cannot be heard, so nothing is asked: %v\n", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
askerFrom = a
|
|
||||||
go a.keep(ctx)
|
|
||||||
}
|
|
||||||
|
|
||||||
// servingAsker is the serving controller's asker, wired to the mesh: what it reads, publishes, performs and
|
|
||||||
// records, and the judgements it asks by — the bus's grant to ask among them (novox/hq issue 353).
|
|
||||||
func servingAsker(open *stores, conn *nats.Conn, js jetstream.JetStream, keeper *conditions.Keeper) *asker {
|
|
||||||
return &asker{
|
|
||||||
open: keeper.Open,
|
open: keeper.Open,
|
||||||
silence: func(ctx context.Context, key string, d time.Duration, by, why string) error {
|
silence: func(ctx context.Context, key string, d time.Duration, by, why string) error {
|
||||||
_, err := keeper.Silence(ctx, key, d, by, why)
|
_, err := keeper.Silence(ctx, key, d, by, why)
|
||||||
@@ -578,7 +322,7 @@ func servingAsker(open *stores, conn *nats.Conn, js jetstream.JetStream, keeper
|
|||||||
},
|
},
|
||||||
routerRecord: routerRecordOf(conn, open.inventory),
|
routerRecord: routerRecordOf(conn, open.inventory),
|
||||||
routerHere: routerHereIn(open.inventory),
|
routerHere: routerHereIn(open.inventory),
|
||||||
grantHeld: grantHeldIn(open, conn),
|
grantHeld: grantHeldIn(open),
|
||||||
channels: channelsIn(open.inventory),
|
channels: channelsIn(open.inventory),
|
||||||
raise: func(ctx context.Context, obs []conditions.Observation) error {
|
raise: func(ctx context.Context, obs []conditions.Observation) error {
|
||||||
return keeper.Reconcile(ctx, sourceAsker, obs)
|
return keeper.Reconcile(ctx, sourceAsker, obs)
|
||||||
@@ -586,4 +330,10 @@ func servingAsker(open *stores, conn *nats.Conn, js jetstream.JetStream, keeper
|
|||||||
now: time.Now,
|
now: time.Now,
|
||||||
logf: func(format string, args ...any) { fmt.Printf(format+"\n", args...) },
|
logf: func(format string, args ...any) { fmt.Printf(format+"\n", args...) },
|
||||||
}
|
}
|
||||||
|
if err := server.Decides(a); err != nil {
|
||||||
|
fmt.Printf("the operator's answers cannot be heard, so nothing is asked: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
askerFrom = a
|
||||||
|
go a.keep(ctx)
|
||||||
}
|
}
|
||||||
|
|||||||
+17
-330
@@ -37,11 +37,6 @@ import (
|
|||||||
// own commit, the newest first, one walk each, until one is delivered; the older ones are then answered
|
// own commit, the newest first, one walk each, until one is delivered; the older ones are then answered
|
||||||
// by that one.
|
// by that one.
|
||||||
//
|
//
|
||||||
// **A merge waits only for a batch it shares something with** (novox/hq ADR 0304 decision 1, ADR 0306 decisions
|
|
||||||
// 2 and 3, issue 525): it joins a batch it is related to, and one that is related to none and whose delivery
|
|
||||||
// group has no member still to merge is a batch whose window is closed at once (batchToJoin). Walks stay one at
|
|
||||||
// a time: such a batch is cut when no walk is open, and queued behind the one that is otherwise.
|
|
||||||
//
|
|
||||||
// A merge on the controller's own path (a module whose walk waits for nobody's word) never shares a batch with
|
// A merge on the controller's own path (a module whose walk waits for nobody's word) never shares a batch with
|
||||||
// one that waits for mesh-delivery's: each kind has a batch of its own, an own-path batch is cut first and folds
|
// one that waits for mesh-delivery's: each kind has a batch of its own, an own-path batch is cut first and folds
|
||||||
// no waiting catalogue walk, and a catalogue walk let go while another walk runs starts once it ended — so no
|
// no waiting catalogue walk, and a catalogue walk let go while another walk runs starts once it ended — so no
|
||||||
@@ -178,12 +173,6 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
|
|||||||
fmt.Printf("%s merged into %s (%.8s); nothing the mesh holds reads it\n", repository, m.Base, m.Commit)
|
fmt.Printf("%s merged into %s (%.8s); nothing the mesh holds reads it\n", repository, m.Base, m.Commit)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
// **Its delivery group's word, before the plans are held** (ADR 0306 decision 2): a slow answer holds up no walk.
|
|
||||||
own := ownPath(touches)
|
|
||||||
waits, err := groupWait(ctx, inv, m, deliverySeatHeld(entries), own)
|
|
||||||
if err != nil {
|
|
||||||
return notNow(err)
|
|
||||||
}
|
|
||||||
release, err := inv.HoldPlans(ctx, true)
|
release, err := inv.HoldPlans(ctx, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return notNow(err)
|
return notNow(err)
|
||||||
@@ -209,6 +198,7 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
|
|||||||
// **A merge on the controller's own path never shares a batch with one that waits for the delivery's
|
// **A merge on the controller's own path never shares a batch with one that waits for the delivery's
|
||||||
// word** (decided during the build, 2026-10-10): batched together, the catalogue's deliveries would start
|
// word** (decided during the build, 2026-10-10): batched together, the catalogue's deliveries would start
|
||||||
// with the controller's and skip their turn. Each kind has a batch of its own.
|
// with the controller's and skip their turn. Each kind has a batch of its own.
|
||||||
|
own := ownPath(touches)
|
||||||
if p, ok, err := answeredByALaterMerge(ctx, inv, heard, touches, own, deliverySeatHeld(entries)); err != nil {
|
if p, ok, err := answeredByALaterMerge(ctx, inv, heard, touches, own, deliverySeatHeld(entries)); err != nil {
|
||||||
return notNow(err)
|
return notNow(err)
|
||||||
} else if ok {
|
} else if ok {
|
||||||
@@ -228,7 +218,7 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
|
|||||||
return cutBatchesHeld(ctx, f.open, now)
|
return cutBatchesHeld(ctx, f.open, now)
|
||||||
}
|
}
|
||||||
|
|
||||||
batch, why, err := batchToJoin(ctx, inv, heard, entries, read, now, own, waits)
|
batch, err := theOpenBatch(ctx, inv, heard, now, own)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return notNow(err)
|
return notNow(err)
|
||||||
}
|
}
|
||||||
@@ -239,7 +229,7 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
|
|||||||
if err := keepBatch(ctx, inv, &batch, now, ""); err != nil {
|
if err := keepBatch(ctx, inv, &batch, now, ""); err != nil {
|
||||||
return notNow(err)
|
return notNow(err)
|
||||||
}
|
}
|
||||||
fmt.Printf("%s merged into %s (%.8s); %s, it joins %s — %s\n", repository, m.Base, m.Commit, why, batch.ID,
|
fmt.Printf("%s merged into %s (%.8s); it joins %s — %s\n", repository, m.Base, m.Commit, batch.ID,
|
||||||
batchWords(batch, now))
|
batchWords(batch, now))
|
||||||
return cutBatchesHeld(ctx, f.open, now)
|
return cutBatchesHeld(ctx, f.open, now)
|
||||||
}
|
}
|
||||||
@@ -330,211 +320,22 @@ func ownPath(touches []inventory.Entry) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// **A merge waits only for a batch it shares something with** (novox/hq ADR 0304 decision 1, as ADR 0306
|
// theOpenBatch is the batch a merge heard now joins: the one of its kind not yet cut, or a new one.
|
||||||
// decisions 2 and 3 read it). batchToJoin is the batch a merge heard now joins, and why in words:
|
func theOpenBatch(ctx context.Context, inv *inventory.Inventory, first inventory.BatchedMerge, now time.Time, own bool) (inventory.Plan, error) {
|
||||||
//
|
|
||||||
// - an assembling or queued batch of its kind it is related to (related.go): it joins it and waits for the
|
|
||||||
// window, which its coming restarts. Related to two or more, the newer are folded into the oldest, which
|
|
||||||
// answers all their merges: a merge never waits in two windows, and what it shares with each walks in one.
|
|
||||||
// - none, and its delivery group has a member still to merge, or mesh-delivery could not say (groupWait, waits
|
|
||||||
// names why): a new batch, which waits for the window, beside the ones it shares nothing with; its words say
|
|
||||||
// why.
|
|
||||||
// - none, and no member of its group is still to merge: a new batch whose window is closed at once. Cut at
|
|
||||||
// once means only that: walks stay one at a time (ADR 0276 decision 4), so with a walk open the batch is
|
|
||||||
// queued behind it as any closed batch is.
|
|
||||||
func batchToJoin(ctx context.Context, inv *inventory.Inventory, heard inventory.BatchedMerge, entries []inventory.Entry,
|
|
||||||
read map[string][]inventory.ReadRepository, now time.Time, own bool, waits string) (inventory.Plan, string, error) {
|
|
||||||
edges, err := inv.Dependencies(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return inventory.Plan{}, "", err
|
|
||||||
}
|
|
||||||
joins, whys, _, err := relatedBatches(ctx, inv, []inventory.BatchedMerge{heard}, own, "", entries, read, edges)
|
|
||||||
if err != nil {
|
|
||||||
return inventory.Plan{}, "", err
|
|
||||||
}
|
|
||||||
if len(joins) > 0 {
|
|
||||||
into := joins[0]
|
|
||||||
why := "related to " + into.ID + " (" + whys[0] + ")"
|
|
||||||
if len(joins) > 1 {
|
|
||||||
plans, moves, err := foldInto(ctx, inv, &into, joins[1:])
|
|
||||||
if err != nil {
|
|
||||||
return inventory.Plan{}, "", err
|
|
||||||
}
|
|
||||||
if err := inv.SavePlansMoving(ctx, plans, moves); err != nil {
|
|
||||||
return inventory.Plan{}, "", err
|
|
||||||
}
|
|
||||||
var folded []string
|
|
||||||
for i, b := range joins[1:] {
|
|
||||||
folded = append(folded, b.ID+" ("+whys[i+1]+")")
|
|
||||||
}
|
|
||||||
why += ", and to " + strings.Join(folded, ", ") + ", folded into it"
|
|
||||||
}
|
|
||||||
into.Delivery.Batch.CutAtOnce = false
|
|
||||||
return into, why, nil
|
|
||||||
}
|
|
||||||
if waits != "" {
|
|
||||||
// Its own window, beside the batches it shares nothing with: waiting is for its group's members, which
|
|
||||||
// are related to it by their head and join it (ADR 0306 decision 2).
|
|
||||||
b, err := newBatch(ctx, inv, heard, now, own, false, waits)
|
|
||||||
return b, waits + ", it waits for the window in a batch of its own", err
|
|
||||||
}
|
|
||||||
b, err := newBatch(ctx, inv, heard, now, own, true, "")
|
|
||||||
return b, "it shares nothing with a batch assembling and no member of a group is still to merge: it is cut at once", err
|
|
||||||
}
|
|
||||||
|
|
||||||
// relatedBatches are the batches not yet cut of a kind, oldest first, parted into those related to merges (with
|
|
||||||
// the first reason of each) and the others. Each batch is judged by what its cut would move (ADR 0306 decision
|
|
||||||
// 3): its merges' moves as they were kept and as they read now — the catalogue may have moved since a merge was
|
|
||||||
// heard — widened to their dependents, and, for a catalogue batch, every walk waiting for its word, which the
|
|
||||||
// cut folds in (cutBatch's carry). A merge kept with no moves leaves its batch unknown, related to everything.
|
|
||||||
// except is a walk not to count as folded: the one being stopped.
|
|
||||||
func relatedBatches(ctx context.Context, inv *inventory.Inventory, merges []inventory.BatchedMerge, own bool,
|
|
||||||
except string, entries []inventory.Entry, read map[string][]inventory.ReadRepository,
|
|
||||||
edges []inventory.Edge) (joins []inventory.Plan, whys []string, others []inventory.Plan, err error) {
|
|
||||||
batches, err := inv.Batches(ctx)
|
batches, err := inv.Batches(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, nil, err
|
return inventory.Plan{}, err
|
||||||
}
|
}
|
||||||
var kind []inventory.Plan
|
|
||||||
for _, b := range batches {
|
for _, b := range batches {
|
||||||
if b.OwnPath() == own {
|
if b.OwnPath() == own {
|
||||||
kind = append(kind, b)
|
return b, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(kind) == 0 {
|
|
||||||
return nil, nil, nil, nil
|
|
||||||
}
|
|
||||||
carry := newFootprint()
|
|
||||||
if !own {
|
|
||||||
// An own-path batch folds no waiting walk (cutBatchesHeld); a catalogue batch may fold every one.
|
|
||||||
plans, err := inv.OpenPlans(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, nil, err
|
|
||||||
}
|
|
||||||
for _, p := range plans {
|
|
||||||
if p.Release != nil || !p.Waiting() || p.ID == except {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
kept, err := inv.MergesOf(ctx, p.ID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, nil, err
|
|
||||||
}
|
|
||||||
carry = unionFootprint(carry, footprintOfWalk(p, kept))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
mine := footprintAsCut(merges, entries, read, edges)
|
|
||||||
for _, b := range kind {
|
|
||||||
held, err := inv.MergesOf(ctx, b.ID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, nil, err
|
|
||||||
}
|
|
||||||
if b.Delivery == nil {
|
|
||||||
b.Delivery = &inventory.PlanDelivery{}
|
|
||||||
}
|
|
||||||
if b.Delivery.Batch == nil {
|
|
||||||
b.Delivery.Batch = &inventory.PlanBatch{Own: own}
|
|
||||||
}
|
|
||||||
if ok, why := related(mine, unionFootprint(footprintAsCut(held, entries, read, edges), carry), edges); ok {
|
|
||||||
joins, whys = append(joins, b), append(whys, why)
|
|
||||||
} else {
|
|
||||||
others = append(others, b)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return joins, whys, others, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// footprintAsCut is a batch's footprint as a cut now would read it: footprintOfBatch, widened by the modules each
|
|
||||||
// merge moves as the catalogue reads now (planBatch judges each merge again at the cut, so what it moves may
|
|
||||||
// have changed since it was heard). Both are kept: when in doubt, related.
|
|
||||||
func footprintAsCut(merges []inventory.BatchedMerge, entries []inventory.Entry,
|
|
||||||
read map[string][]inventory.ReadRepository, edges []inventory.Edge) footprint {
|
|
||||||
f := footprintOfBatch(merges, edges)
|
|
||||||
var moved []string
|
|
||||||
for _, m := range merges {
|
|
||||||
s := announcedOf(m).SourceMoved
|
|
||||||
if s.Owner == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
s.MergedAt = "" // news, whatever a cut marked seen since (as planBatch reads it)
|
|
||||||
for _, e := range wouldMove(s, entries, read) {
|
|
||||||
moved = append(moved, e.Manifest.Module)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, name := range reachableFrom(moved, edges) {
|
|
||||||
f.modules[name] = true
|
|
||||||
}
|
|
||||||
return f
|
|
||||||
}
|
|
||||||
|
|
||||||
// unionFootprint is what two changes reach together.
|
|
||||||
func unionFootprint(a, b footprint) footprint {
|
|
||||||
f := newFootprint()
|
|
||||||
for _, from := range []footprint{a, b} {
|
|
||||||
for k := range from.repos {
|
|
||||||
f.repos[k] = true
|
|
||||||
}
|
|
||||||
for k := range from.modules {
|
|
||||||
f.modules[k] = true
|
|
||||||
}
|
|
||||||
for k := range from.heads {
|
|
||||||
f.heads[k] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
f.unknown = a.unknown
|
|
||||||
if f.unknown == "" {
|
|
||||||
f.unknown = b.unknown
|
|
||||||
}
|
|
||||||
return f
|
|
||||||
}
|
|
||||||
|
|
||||||
// foldInto folds batches into the oldest one related to a merge: the plans to write (into first, so no merge
|
|
||||||
// names a plan the store does not hold) and the merges to move, for one SavePlansMoving. The folded batches name
|
|
||||||
// the batch that took them over; a stop's closed window and its withheld modules (ADR 0299) go with their merges.
|
|
||||||
func foldInto(ctx context.Context, inv *inventory.Inventory, into *inventory.Plan, from []inventory.Plan) ([]*inventory.Plan,
|
|
||||||
[]inventory.MergeMove, error) {
|
|
||||||
if into.Delivery == nil {
|
|
||||||
into.Delivery = &inventory.PlanDelivery{}
|
|
||||||
}
|
|
||||||
if into.Delivery.Batch == nil {
|
|
||||||
into.Delivery.Batch = &inventory.PlanBatch{Own: into.OwnPath()}
|
|
||||||
}
|
|
||||||
plans := []*inventory.Plan{into}
|
|
||||||
var moves []inventory.MergeMove
|
|
||||||
for i := range from {
|
|
||||||
f := from[i]
|
|
||||||
merges, err := inv.MergesOf(ctx, f.ID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
for _, m := range merges {
|
|
||||||
moves = append(moves, inventory.MergeMove{Repository: m.Repository, Commit: m.Commit, Plan: into.ID, Alone: m.Alone})
|
|
||||||
}
|
|
||||||
if f.Delivery == nil {
|
|
||||||
f.Delivery = &inventory.PlanDelivery{}
|
|
||||||
}
|
|
||||||
if w := f.Delivery.Batch; w != nil {
|
|
||||||
into.Delivery.Batch.Closed = into.Delivery.Batch.Closed || w.Closed
|
|
||||||
into.Delivery.Batch.Withheld = union(into.Delivery.Batch.Withheld, w.Withheld)
|
|
||||||
sort.Strings(into.Delivery.Batch.Withheld)
|
|
||||||
}
|
|
||||||
f.State = inventory.PlanSuperseded
|
|
||||||
f.Delivery.Batch = nil
|
|
||||||
f.Delivery.TakenOverBy = into.ID
|
|
||||||
f.Note = fmt.Sprintf("folded into %s before it was cut: a merge related to both joined it, and %s answers "+
|
|
||||||
"its merges", into.ID, into.ID)
|
|
||||||
plans = append(plans, &f)
|
|
||||||
}
|
|
||||||
return plans, moves, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// newBatch is a new batch of a kind, kept before its first merge names it: a merge never names a plan the store
|
|
||||||
// does not hold. cutAtOnce closes its window at once; waits says why it waits for its group, when it does.
|
|
||||||
func newBatch(ctx context.Context, inv *inventory.Inventory, first inventory.BatchedMerge, now time.Time, own,
|
|
||||||
cutAtOnce bool, waits string) (inventory.Plan, error) {
|
|
||||||
b := inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch,
|
b := inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch,
|
||||||
Commit: first.Commit, Merged: first.Merged, Created: now, State: inventory.PlanAssembling,
|
Commit: first.Commit, Merged: first.Merged, Created: now, State: inventory.PlanAssembling,
|
||||||
Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{},
|
Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{},
|
||||||
Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: own, CutAtOnce: cutAtOnce, Waits: waits}}}
|
Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: own}}}
|
||||||
|
// Kept before its first merge names it: a merge never names a plan the store does not hold.
|
||||||
if err := inv.SavePlan(ctx, &b); err != nil {
|
if err := inv.SavePlan(ctx, &b); err != nil {
|
||||||
return inventory.Plan{}, err
|
return inventory.Plan{}, err
|
||||||
}
|
}
|
||||||
@@ -566,16 +367,7 @@ func keepBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan,
|
|||||||
}
|
}
|
||||||
closes, latest := windowOf(merges, b.Created, window, atMost)
|
closes, latest := windowOf(merges, b.Created, window, atMost)
|
||||||
b.Delivery.Merges = named
|
b.Delivery.Merges = named
|
||||||
var closed, cutAtOnce bool
|
b.Delivery.Batch = &inventory.PlanBatch{ClosesAt: closes, AtMost: latest, Behind: behind, Own: b.OwnPath()}
|
||||||
var withheld []string
|
|
||||||
var waits string
|
|
||||||
if was := b.Delivery.Batch; was != nil {
|
|
||||||
closed, withheld = was.Closed, was.Withheld // a stop's, which no look takes back (novox/hq ADR 0299)
|
|
||||||
cutAtOnce = was.CutAtOnce // the merge's, until another joins it (ADR 0304 decision 1)
|
|
||||||
waits = was.Waits // its group's, as said when it was made (ADR 0306 decision 2)
|
|
||||||
}
|
|
||||||
b.Delivery.Batch = &inventory.PlanBatch{ClosesAt: closes, AtMost: latest, Behind: behind, Own: b.OwnPath(),
|
|
||||||
Closed: closed, Withheld: withheld, CutAtOnce: cutAtOnce, Waits: waits}
|
|
||||||
b.State = inventory.PlanAssembling
|
b.State = inventory.PlanAssembling
|
||||||
if behind != "" && windowClosed(*b, now) {
|
if behind != "" && windowClosed(*b, now) {
|
||||||
b.State = inventory.PlanQueued
|
b.State = inventory.PlanQueued
|
||||||
@@ -604,14 +396,13 @@ func windowOf(merges []inventory.BatchedMerge, created time.Time, window, atMost
|
|||||||
return last.Add(window), created.Add(atMost)
|
return last.Add(window), created.Add(atMost)
|
||||||
}
|
}
|
||||||
|
|
||||||
// windowClosed says a batch's window is closed: closed by a stop, closed at once for a merge that shares
|
// windowClosed says a batch's window is closed: no merge for the window's length, or its maximum reached.
|
||||||
// nothing (ADR 0304 decision 1), no merge for the window's length, or its maximum reached.
|
|
||||||
func windowClosed(b inventory.Plan, now time.Time) bool {
|
func windowClosed(b inventory.Plan, now time.Time) bool {
|
||||||
if b.Delivery == nil || b.Delivery.Batch == nil {
|
if b.Delivery == nil || b.Delivery.Batch == nil {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
w := b.Delivery.Batch
|
w := b.Delivery.Batch
|
||||||
return w.Closed || w.CutAtOnce || !now.Before(w.ClosesAt) || !now.Before(w.AtMost)
|
return !now.Before(w.ClosesAt) || !now.Before(w.AtMost)
|
||||||
}
|
}
|
||||||
|
|
||||||
// carriedOf is a set of merges as a walk carries them: the latest merge of each repository's branch, and
|
// carriedOf is a set of merges as a walk carries them: the latest merge of each repository's branch, and
|
||||||
@@ -687,8 +478,7 @@ func spelledOf(m inventory.BatchedMerge) string {
|
|||||||
// namedOf is one merge as a walk or batch names it: its commit, and its pull request and moves as announced.
|
// namedOf is one merge as a walk or batch names it: its commit, and its pull request and moves as announced.
|
||||||
func namedOf(m inventory.BatchedMerge, repository string) inventory.PlanMerge {
|
func namedOf(m inventory.BatchedMerge, repository string) inventory.PlanMerge {
|
||||||
k := announcedOf(m)
|
k := announcedOf(m)
|
||||||
return inventory.PlanMerge{Repository: repository, Commit: m.Commit, Number: k.Number, Title: k.Title, Moves: k.Moves,
|
return inventory.PlanMerge{Repository: repository, Commit: m.Commit, Number: k.Number, Title: k.Title, Moves: k.Moves}
|
||||||
Merged: m.Merged, Heard: m.Heard}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// laterMerge says a was merged after b: by the forge's merge time, then by when each was heard.
|
// laterMerge says a was merged after b: by the forge's merge time, then by when each was heard.
|
||||||
@@ -820,9 +610,8 @@ func cutBatchesHeld(ctx context.Context, open *stores, now time.Time) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// The oldest batch whose window closed is cut, an own-path one before a catalogue one (it goes first, and
|
// The oldest batch whose window closed is cut, an own-path one before a catalogue one (it goes first, and
|
||||||
// the catalogue batch queues behind it rather than being cut and overtaken). Several of a kind may be open
|
// the catalogue batch queues behind it rather than being cut and overtaken); one of each kind at most is
|
||||||
// since a merge that shares nothing is cut at once (ADR 0304 decision 1): walks stay one at a time, so one
|
// open (theOpenBatch).
|
||||||
// is cut per look, and once its walk started the others are queued behind it.
|
|
||||||
sort.SliceStable(batches, func(i, j int) bool { return batches[i].OwnPath() && !batches[j].OwnPath() })
|
sort.SliceStable(batches, func(i, j int) bool { return batches[i].OwnPath() && !batches[j].OwnPath() })
|
||||||
for i := range batches {
|
for i := range batches {
|
||||||
b := &batches[i]
|
b := &batches[i]
|
||||||
@@ -972,18 +761,7 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
|
|||||||
}
|
}
|
||||||
sort.Strings(also)
|
sort.Strings(also)
|
||||||
ordered := groupOrderEdges(members, entries, read, edges)
|
ordered := groupOrderEdges(members, entries, read, edges)
|
||||||
var withheld []string
|
|
||||||
plan := planOfMoves(moved, append(append([]inventory.Edge{}, edges...), ordered...))
|
plan := planOfMoves(moved, append(append([]inventory.Edge{}, edges...), ordered...))
|
||||||
// **Withheld, a stopped merge's modules** (novox/hq ADR 0299): a module is built at its branch, which holds the
|
|
||||||
// merge a stop left out of the walk these merges were walked again from — built here only as a dependent, it
|
|
||||||
// would deliver that merge. One a merge of this batch moves is built: that merge carries the stopped change,
|
|
||||||
// and the stop named it.
|
|
||||||
if batch.Delivery != nil && batch.Delivery.Batch != nil {
|
|
||||||
if out := withhold(&plan, batch.Delivery.Batch.Withheld, moved); len(out) > 0 {
|
|
||||||
withheld = out
|
|
||||||
fmt.Printf(" withheld, a stopped merge's modules built only as dependents: %s\n", strings.Join(out, ", "))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
plan.ID, plan.Revision, plan.Created, plan.Epoch = batch.ID, batch.Revision, now, batch.Epoch
|
plan.ID, plan.Revision, plan.Created, plan.Epoch = batch.ID, batch.Revision, now, batch.Epoch
|
||||||
plan.Commits = commits
|
plan.Commits = commits
|
||||||
newest := newestCommit(commits)
|
newest := newestCommit(commits)
|
||||||
@@ -994,9 +772,6 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
|
|||||||
}
|
}
|
||||||
plan.Delivery.Merges = named
|
plan.Delivery.Merges = named
|
||||||
plan.Delivery.Alone = batch.Delivery != nil && batch.Delivery.Alone
|
plan.Delivery.Alone = batch.Delivery != nil && batch.Delivery.Alone
|
||||||
plan.Delivery.Withheld = withheld
|
|
||||||
// **Its moments and its class** (novox/hq ADR 0282 decision 6): measured, never acted on.
|
|
||||||
plan.Times = walkTimesAtCut(*batch, plan, entries, now)
|
|
||||||
if len(moved) == 0 {
|
if len(moved) == 0 {
|
||||||
plan.State = inventory.PlanDone
|
plan.State = inventory.PlanDone
|
||||||
plan.Tiers = [][]string{}
|
plan.Tiers = [][]string{}
|
||||||
@@ -1045,86 +820,6 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// withhold takes from a plan every module named that its merges do not move, and the tiers left empty: the
|
|
||||||
// modules taken.
|
|
||||||
func withhold(plan *inventory.Plan, names, moved []string) []string {
|
|
||||||
var out []string
|
|
||||||
for _, n := range names {
|
|
||||||
if _, in := plan.Modules[n]; in && !slices.Contains(moved, n) {
|
|
||||||
delete(plan.Modules, n)
|
|
||||||
out = append(out, n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(out) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
var tiers [][]string
|
|
||||||
for _, t := range plan.Tiers {
|
|
||||||
var kept []string
|
|
||||||
for _, n := range t {
|
|
||||||
if !slices.Contains(out, n) {
|
|
||||||
kept = append(kept, n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(kept) > 0 {
|
|
||||||
tiers = append(tiers, kept)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if tiers == nil {
|
|
||||||
tiers = [][]string{}
|
|
||||||
}
|
|
||||||
plan.Tiers = tiers
|
|
||||||
sort.Strings(out)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// walkTimesAtCut is a walk's own moments as it is cut (novox/hq ADR 0282 decision 6): when its batch's window
|
|
||||||
// closed — no merge for the window's length, or its maximum, whichever came first — when it was cut, and its
|
|
||||||
// class. A merge walked alone had no window.
|
|
||||||
func walkTimesAtCut(batch, walk inventory.Plan, entries []inventory.Entry, now time.Time) *inventory.PlanTimes {
|
|
||||||
cut := now
|
|
||||||
t := &inventory.PlanTimes{Cut: &cut, Class: classOf(walk, entries)}
|
|
||||||
if batch.Delivery != nil && batch.Delivery.Batch != nil {
|
|
||||||
w := batch.Delivery.Batch
|
|
||||||
closed := w.ClosesAt
|
|
||||||
if !w.AtMost.IsZero() && w.AtMost.Before(closed) {
|
|
||||||
closed = w.AtMost
|
|
||||||
}
|
|
||||||
if w.CutAtOnce && !batch.Created.IsZero() {
|
|
||||||
closed = batch.Created // closed when it was made (ADR 0304 decision 1)
|
|
||||||
}
|
|
||||||
if !closed.IsZero() {
|
|
||||||
if closed.After(now) {
|
|
||||||
closed = now
|
|
||||||
}
|
|
||||||
t.WindowClosed = &closed
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return t
|
|
||||||
}
|
|
||||||
|
|
||||||
// resolverSeat is the seat of the mesh's resolver: a module claiming it is a core module (ADR 0282 decision 1).
|
|
||||||
const resolverSeat = "mesh-dns-resolver"
|
|
||||||
|
|
||||||
// classOf is a walk's class (novox/hq ADR 0282 decision 1): core when it walks a module on the controller's own
|
|
||||||
// path or one holding the mesh's resolver, leaf otherwise.
|
|
||||||
func classOf(walk inventory.Plan, entries []inventory.Entry) string {
|
|
||||||
resolvers := map[string]bool{}
|
|
||||||
for _, e := range entries {
|
|
||||||
for _, c := range e.Manifest.Claims {
|
|
||||||
if c.Name == resolverSeat {
|
|
||||||
resolvers[e.Manifest.Module] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for m := range walk.Modules {
|
|
||||||
if _, own := onTheControllersPath[m]; own || resolvers[m] {
|
|
||||||
return inventory.ClassCore
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return inventory.ClassLeaf
|
|
||||||
}
|
|
||||||
|
|
||||||
// combinedMerges is a batch's merges as one merge per repository's branch: the latest, with every file the
|
// combinedMerges is a batch's merges as one merge per repository's branch: the latest, with every file the
|
||||||
// merges of it changed and said to be a module's — on a linear trunk the latest contains the others. A file is
|
// merges of it changed and said to be a module's — on a linear trunk the latest contains the others. A file is
|
||||||
// removed when the last merge of the batch that changed it removed it. merges are in the order they were made.
|
// removed when the last merge of the batch that changed it removed it. merges are in the order they were made.
|
||||||
@@ -1349,10 +1044,6 @@ func batchWords(b inventory.Plan, now time.Time) string {
|
|||||||
if b.State == inventory.PlanQueued {
|
if b.State == inventory.PlanQueued {
|
||||||
return fmt.Sprintf("queued behind %s; grouped: %s; plan not yet calculated", w.Behind, grouped)
|
return fmt.Sprintf("queued behind %s; grouped: %s; plan not yet calculated", w.Behind, grouped)
|
||||||
}
|
}
|
||||||
if w.CutAtOnce {
|
|
||||||
return fmt.Sprintf("cut at once, sharing nothing with a batch assembling; grouped: %s; plan not yet calculated",
|
|
||||||
grouped)
|
|
||||||
}
|
|
||||||
left := w.ClosesAt.Sub(now)
|
left := w.ClosesAt.Sub(now)
|
||||||
if w.AtMost.Before(w.ClosesAt) {
|
if w.AtMost.Before(w.ClosesAt) {
|
||||||
left = w.AtMost.Sub(now)
|
left = w.AtMost.Sub(now)
|
||||||
@@ -1360,12 +1051,8 @@ func batchWords(b inventory.Plan, now time.Time) string {
|
|||||||
if left < 0 {
|
if left < 0 {
|
||||||
left = 0
|
left = 0
|
||||||
}
|
}
|
||||||
waits := ""
|
return fmt.Sprintf("assembling: %s left (at the latest %s); grouped: %s; plan not yet calculated",
|
||||||
if w.Waits != "" {
|
secondsWords(left), w.AtMost.Local().Format("15:04:05"), grouped)
|
||||||
waits = " — " + w.Waits
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("assembling: %s left (at the latest %s)%s; grouped: %s; plan not yet calculated",
|
|
||||||
secondsWords(left), w.AtMost.Local().Format("15:04:05"), waits, grouped)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// groupedWords is a batch's commits, each with the earlier merges it answers.
|
// groupedWords is a batch's commits, each with the earlier merges it answers.
|
||||||
|
|||||||
@@ -34,7 +34,6 @@ func windowed(t *testing.T) *stores {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
t.Setenv(mergeWindowFileVar, settings)
|
t.Setenv(mergeWindowFileVar, settings)
|
||||||
noGroups(t)
|
|
||||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"},
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"},
|
||||||
inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Ref: "main", BuiltFrom: "c0",
|
inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Ref: "main", BuiltFrom: "c0",
|
||||||
Head: "c0"}); err != nil {
|
Head: "c0"}); err != nil {
|
||||||
@@ -59,70 +58,19 @@ func windowed(t *testing.T) *stores {
|
|||||||
// t0 is the moment a test's clock starts: merges are dated by it, and heard after it.
|
// t0 is the moment a test's clock starts: merges are dated by it, and heard after it.
|
||||||
var t0 = time.Now().UTC().Truncate(time.Second)
|
var t0 = time.Now().UTC().Truncate(time.Second)
|
||||||
|
|
||||||
// catalogueMerge is a merge of the catalogue changing one module's directory, made at a moment, from a pull
|
// catalogueMerge is a merge of the catalogue changing one module's directory, made at a moment.
|
||||||
// request's head branch of its own, as the forge announces every merge. Whether its head names a delivery group
|
|
||||||
// with a member still to merge is mesh-delivery's word, which windowed stubs as no group at all (noGroups); a test
|
|
||||||
// whose merges wait for the window as a group's says so (groupsWaiting).
|
|
||||||
func catalogueMerge(commit, module string, made time.Time) link.SourceMoved {
|
func catalogueMerge(commit, module string, made time.Time) link.SourceMoved {
|
||||||
return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Head: "pr/" + commit, Commit: commit,
|
return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit,
|
||||||
MergedAt: made.Format(time.RFC3339Nano), Paths: []string{"modules/" + module + "/module.json"},
|
MergedAt: made.Format(time.RFC3339Nano), Paths: []string{"modules/" + module + "/module.json"},
|
||||||
ModuleDirs: []string{"modules/" + module}, ModuleDirsSaid: true}
|
ModuleDirs: []string{"modules/" + module}, ModuleDirsSaid: true}
|
||||||
}
|
}
|
||||||
|
|
||||||
// repoMerge is a merge of a repository of one module's own, from a head branch of its own (catalogueMerge).
|
// repoMerge is a merge of a repository of one module's own.
|
||||||
func repoMerge(repo, commit string, made time.Time) link.SourceMoved {
|
func repoMerge(repo, commit string, made time.Time) link.SourceMoved {
|
||||||
return link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Head: "pr/" + commit, Commit: commit,
|
return link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Commit: commit,
|
||||||
MergedAt: made.Format(time.RFC3339Nano), Paths: []string{"main.go"}}
|
MergedAt: made.Format(time.RFC3339Nano), Paths: []string{"main.go"}}
|
||||||
}
|
}
|
||||||
|
|
||||||
// noGroups stubs mesh-delivery's word on groups for a test: no head names a group.
|
|
||||||
func noGroups(t *testing.T) {
|
|
||||||
t.Helper()
|
|
||||||
was := deliveryGroupOf
|
|
||||||
deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) { return nil, false, nil }
|
|
||||||
t.Cleanup(func() { deliveryGroupOf = was })
|
|
||||||
}
|
|
||||||
|
|
||||||
// groupsWaiting stubs mesh-delivery's word on groups for a test: each head named is a group with a member of another
|
|
||||||
// repository whose pull request is open (novox/elsewhere, ready), so a merge from it waits for the window (ADR 0306
|
|
||||||
// decision 2); every other head names no group.
|
|
||||||
func groupsWaiting(t *testing.T, heads ...string) {
|
|
||||||
t.Helper()
|
|
||||||
was := deliveryGroupOf
|
|
||||||
deliveryGroupOf = func(_ context.Context, _ bool, head string) ([]groupMember, bool, error) {
|
|
||||||
if slices.Contains(heads, head) {
|
|
||||||
return []groupMember{{Repository: "novox/elsewhere", State: "ready"}}, true, nil
|
|
||||||
}
|
|
||||||
return nil, false, nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { deliveryGroupOf = was })
|
|
||||||
}
|
|
||||||
|
|
||||||
// everyHeadWaits stubs mesh-delivery's word on groups for a test whose merges each wait for the window as they did
|
|
||||||
// before novox/hq ADR 0304: every head is a group with a member of another repository still to merge.
|
|
||||||
func everyHeadWaits(t *testing.T) {
|
|
||||||
t.Helper()
|
|
||||||
was := deliveryGroupOf
|
|
||||||
deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) {
|
|
||||||
return []groupMember{{Repository: "novox/elsewhere", State: "ready"}}, true, nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { deliveryGroupOf = was })
|
|
||||||
}
|
|
||||||
|
|
||||||
// fromHead is a merge from a head branch a test names: merges from one head are members of one delivery group,
|
|
||||||
// related by it (novox/hq ADR 0304 decision 1), so they share a batch whatever else they share.
|
|
||||||
func fromHead(m link.SourceMoved, head string) link.SourceMoved {
|
|
||||||
m.Head = head
|
|
||||||
return m
|
|
||||||
}
|
|
||||||
|
|
||||||
// lone is a merge that names no head: known to be no delivery group's, it waits for no window unless it is
|
|
||||||
// related to a batch assembling (ADR 0304 decision 1).
|
|
||||||
func lone(m link.SourceMoved) link.SourceMoved {
|
|
||||||
m.Head = ""
|
|
||||||
return m
|
|
||||||
}
|
|
||||||
|
|
||||||
// hear is a merge heard at a moment.
|
// hear is a merge heard at a moment.
|
||||||
func hear(t *testing.T, open *stores, m link.SourceMoved, now time.Time) {
|
func hear(t *testing.T, open *stores, m link.SourceMoved, now time.Time) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
@@ -160,7 +108,6 @@ func walks(t *testing.T, open *stores) (walks, batches []inventory.Plan) {
|
|||||||
// later commit, which names both merges — the earlier carried by the later.
|
// later commit, which names both merges — the earlier carried by the later.
|
||||||
func TestTwoMergesSecondsApartAreOneWalkAtTheLaterCommit(t *testing.T) {
|
func TestTwoMergesSecondsApartAreOneWalkAtTheLaterCommit(t *testing.T) {
|
||||||
open := windowed(t)
|
open := windowed(t)
|
||||||
everyHeadWaits(t)
|
|
||||||
asked := asksWithPaths(t)
|
asked := asksWithPaths(t)
|
||||||
claude := catalogueMerge("553b7191claude", "app", t0)
|
claude := catalogueMerge("553b7191claude", "app", t0)
|
||||||
dunst := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second))
|
dunst := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second))
|
||||||
@@ -205,10 +152,9 @@ func TestTwoMergesSecondsApartAreOneWalkAtTheLaterCommit(t *testing.T) {
|
|||||||
// Merges of three repositories in one window are one walk, one commit for each.
|
// Merges of three repositories in one window are one walk, one commit for each.
|
||||||
func TestThreeRepositoriesInOneWindowAreOneWalk(t *testing.T) {
|
func TestThreeRepositoriesInOneWindowAreOneWalk(t *testing.T) {
|
||||||
open := windowed(t)
|
open := windowed(t)
|
||||||
groupsWaiting(t, "feat/three")
|
|
||||||
asksRecorded(t)
|
asksRecorded(t)
|
||||||
for i, r := range []string{"one", "two", "three"} {
|
for i, r := range []string{"one", "two", "three"} {
|
||||||
hear(t, open, fromHead(repoMerge(r, "c-"+r, t0.Add(time.Duration(i)*20*time.Second)), "feat/three"),
|
hear(t, open, repoMerge(r, "c-"+r, t0.Add(time.Duration(i)*20*time.Second)),
|
||||||
t0.Add(time.Duration(i)*20*time.Second+time.Second))
|
t0.Add(time.Duration(i)*20*time.Second+time.Second))
|
||||||
}
|
}
|
||||||
cutAt(t, open, t0.Add(41*time.Second+90*time.Second))
|
cutAt(t, open, t0.Add(41*time.Second+90*time.Second))
|
||||||
@@ -241,10 +187,9 @@ func TestThreeRepositoriesInOneWindowAreOneWalk(t *testing.T) {
|
|||||||
// closes at most ten minutes after its first merge however busy.
|
// closes at most ten minutes after its first merge however busy.
|
||||||
func TestAMergeRestartsTheWindowAndTheMaximumClosesIt(t *testing.T) {
|
func TestAMergeRestartsTheWindowAndTheMaximumClosesIt(t *testing.T) {
|
||||||
open := windowed(t)
|
open := windowed(t)
|
||||||
groupsWaiting(t, "feat/both", "feat/busy")
|
|
||||||
asksWithPaths(t)
|
asksWithPaths(t)
|
||||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0)
|
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||||
hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(80*time.Second)), "feat/both"), t0.Add(80*time.Second))
|
hear(t, open, repoMerge("two", "c2", t0.Add(80*time.Second)), t0.Add(80*time.Second))
|
||||||
cutAt(t, open, t0.Add(100*time.Second))
|
cutAt(t, open, t0.Add(100*time.Second))
|
||||||
if ws, _ := walks(t, open); len(ws) != 0 {
|
if ws, _ := walks(t, open); len(ws) != 0 {
|
||||||
t.Fatal("the window closed 90 seconds after its first merge, not after its last")
|
t.Fatal("the window closed 90 seconds after its first merge, not after its last")
|
||||||
@@ -255,13 +200,12 @@ func TestAMergeRestartsTheWindowAndTheMaximumClosesIt(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
busy := windowed(t)
|
busy := windowed(t)
|
||||||
groupsWaiting(t, "feat/both", "feat/busy")
|
|
||||||
asksWithPaths(t)
|
asksWithPaths(t)
|
||||||
var last time.Time
|
var last time.Time
|
||||||
for i := 0; i < 12; i++ {
|
for i := 0; i < 12; i++ {
|
||||||
last = t0.Add(time.Duration(i) * time.Minute)
|
last = t0.Add(time.Duration(i) * time.Minute)
|
||||||
repo := []string{"one", "two", "three"}[i%3]
|
repo := []string{"one", "two", "three"}[i%3]
|
||||||
hear(t, busy, fromHead(repoMerge(repo, "c"+string(rune('a'+i)), last), "feat/busy"), last)
|
hear(t, busy, repoMerge(repo, "c"+string(rune('a'+i)), last), last)
|
||||||
if ws, _ := walks(t, busy); len(ws) > 0 {
|
if ws, _ := walks(t, busy); len(ws) > 0 {
|
||||||
if i != 10 {
|
if i != 10 {
|
||||||
t.Fatalf("a busy window closed at merge %d (minute %d), not at its maximum", i, i)
|
t.Fatalf("a busy window closed at merge %d (minute %d), not at its maximum", i, i)
|
||||||
@@ -279,11 +223,10 @@ func TestAMergeRestartsTheWindowAndTheMaximumClosesIt(t *testing.T) {
|
|||||||
// what is grouped, and that the plan is not yet calculated. It says so on the bus as plan-moved, too.
|
// what is grouped, and that the plan is not yet calculated. It says so on the bus as plan-moved, too.
|
||||||
func TestTheAssemblingBatchIsShown(t *testing.T) {
|
func TestTheAssemblingBatchIsShown(t *testing.T) {
|
||||||
open := windowed(t)
|
open := windowed(t)
|
||||||
groupsWaiting(t, "pr/553b7191claude")
|
|
||||||
asksWithPaths(t)
|
asksWithPaths(t)
|
||||||
var said []inventory.Plan
|
var said []inventory.Plan
|
||||||
was := inventory.PlanSaved
|
was := inventory.PlanSaved
|
||||||
inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { said = append(said, p) }
|
inventory.PlanSaved = func(p inventory.Plan) { said = append(said, p) }
|
||||||
t.Cleanup(func() { inventory.PlanSaved = was })
|
t.Cleanup(func() { inventory.PlanSaved = was })
|
||||||
now := time.Now().UTC()
|
now := time.Now().UTC()
|
||||||
claude := catalogueMerge("553b7191claude", "app", now.Add(-20*time.Second))
|
claude := catalogueMerge("553b7191claude", "app", now.Add(-20*time.Second))
|
||||||
@@ -292,7 +235,7 @@ func TestTheAssemblingBatchIsShown(t *testing.T) {
|
|||||||
dunst.Number, dunst.Title = 175, "dunst: the font the operator chose"
|
dunst.Number, dunst.Title = 175, "dunst: the font the operator chose"
|
||||||
hear(t, open, claude, now.Add(-19*time.Second))
|
hear(t, open, claude, now.Add(-19*time.Second))
|
||||||
hear(t, open, dunst, now.Add(-time.Second))
|
hear(t, open, dunst, now.Add(-time.Second))
|
||||||
hear(t, open, fromHead(repoMerge("one", "a6bc0931one", now), dunst.Head), now)
|
hear(t, open, repoMerge("one", "a6bc0931one", now), now)
|
||||||
out := captured(t, func() error { return plansCommand(t.Context(), nil) })
|
out := captured(t, func() error { return plansCommand(t.Context(), nil) })
|
||||||
lines := strings.Split(out, "\n")
|
lines := strings.Split(out, "\n")
|
||||||
first := lines[0]
|
first := lines[0]
|
||||||
@@ -417,7 +360,6 @@ func TestAWaitingWalkIsFoldedIntoTheNextBatch(t *testing.T) {
|
|||||||
// moves — named at once on a walk done — and joins the next batch when it does not.
|
// moves — named at once on a walk done — and joins the next batch when it does not.
|
||||||
func TestALateMergeIsAnsweredByTheWalkOfTheLaterOne(t *testing.T) {
|
func TestALateMergeIsAnsweredByTheWalkOfTheLaterOne(t *testing.T) {
|
||||||
open := windowed(t)
|
open := windowed(t)
|
||||||
everyHeadWaits(t)
|
|
||||||
asksWithPaths(t)
|
asksWithPaths(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
later := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second))
|
later := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second))
|
||||||
@@ -449,7 +391,6 @@ func TestALateMergeIsAnsweredByTheWalkOfTheLaterOne(t *testing.T) {
|
|||||||
// first; the first delivered answers the older ones, and the search stops.
|
// first; the first delivered answers the older ones, and the search stops.
|
||||||
func TestAFailedWalkWalksItsEarlierMergesAlone(t *testing.T) {
|
func TestAFailedWalkWalksItsEarlierMergesAlone(t *testing.T) {
|
||||||
open := windowed(t)
|
open := windowed(t)
|
||||||
everyHeadWaits(t)
|
|
||||||
asked := asksWithPaths(t)
|
asked := asksWithPaths(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
// Made before the cut marks their modules seen, as merges are: walked again, they are news all the same.
|
// Made before the cut marks their modules seen, as merges are: walked again, they are news all the same.
|
||||||
@@ -518,7 +459,6 @@ func TestAFailedWalkWalksItsEarlierMergesAlone(t *testing.T) {
|
|||||||
func TestAGroupsOrderBecomesTiersInsideTheWalk(t *testing.T) {
|
func TestAGroupsOrderBecomesTiersInsideTheWalk(t *testing.T) {
|
||||||
for _, grouped := range []bool{true, false} {
|
for _, grouped := range []bool{true, false} {
|
||||||
open := windowed(t)
|
open := windowed(t)
|
||||||
everyHeadWaits(t)
|
|
||||||
asksWithPaths(t)
|
asksWithPaths(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
for _, m := range []string{"mesh-host"} {
|
for _, m := range []string{"mesh-host"} {
|
||||||
@@ -556,10 +496,9 @@ func TestAGroupsOrderBecomesTiersInsideTheWalk(t *testing.T) {
|
|||||||
// store, a merge handed over again is not doubled, and the batch is cut when its window closes.
|
// store, a merge handed over again is not doubled, and the batch is cut when its window closes.
|
||||||
func TestABatchSurvivesARestart(t *testing.T) {
|
func TestABatchSurvivesARestart(t *testing.T) {
|
||||||
open := windowed(t)
|
open := windowed(t)
|
||||||
groupsWaiting(t, "feat/both")
|
|
||||||
asksWithPaths(t)
|
asksWithPaths(t)
|
||||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0)
|
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||||
hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(30*time.Second)), "feat/both"), t0.Add(30*time.Second))
|
hear(t, open, repoMerge("two", "c2", t0.Add(30*time.Second)), t0.Add(30*time.Second))
|
||||||
|
|
||||||
// A new controller: nothing of the first one's but the store.
|
// A new controller: nothing of the first one's but the store.
|
||||||
again, err := openStores(t.Context())
|
again, err := openStores(t.Context())
|
||||||
@@ -567,7 +506,7 @@ func TestABatchSurvivesARestart(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
t.Cleanup(again.Close)
|
t.Cleanup(again.Close)
|
||||||
hear(t, again, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0.Add(60*time.Second)) // the bus hands it over again
|
hear(t, again, repoMerge("one", "c1", t0), t0.Add(60*time.Second)) // the bus hands it over again
|
||||||
cutAt(t, again, t0.Add(119*time.Second))
|
cutAt(t, again, t0.Add(119*time.Second))
|
||||||
if ws, bs := walks(t, again); len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 {
|
if ws, bs := walks(t, again); len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 {
|
||||||
t.Fatalf("the restarted controller's batch: %d walk(s), %+v", len(ws), bs)
|
t.Fatalf("the restarted controller's batch: %d walk(s), %+v", len(ws), bs)
|
||||||
@@ -624,7 +563,6 @@ func TestAWaitingWalkNamesTheMergesItAnswers(t *testing.T) {
|
|||||||
// nothing, and the merge joins the next batch.
|
// nothing, and the merge joins the next batch.
|
||||||
func TestALateMergeOfAOneModuleRepositoryIsNamed(t *testing.T) {
|
func TestALateMergeOfAOneModuleRepositoryIsNamed(t *testing.T) {
|
||||||
open := windowed(t)
|
open := windowed(t)
|
||||||
everyHeadWaits(t)
|
|
||||||
asksWithPaths(t)
|
asksWithPaths(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
c2 := repoMerge("one", "c2", t0.Add(-50*time.Second))
|
c2 := repoMerge("one", "c2", t0.Add(-50*time.Second))
|
||||||
@@ -778,7 +716,6 @@ func TestTheCatchUpKeepsWhatACutMadeHistory(t *testing.T) {
|
|||||||
// still waits for the word, so no catalogue delivery skips its turn.
|
// still waits for the word, so no catalogue delivery skips its turn.
|
||||||
func TestAMergeOnTheControllersPathNeverSharesABatch(t *testing.T) {
|
func TestAMergeOnTheControllersPathNeverSharesABatch(t *testing.T) {
|
||||||
open := windowed(t)
|
open := windowed(t)
|
||||||
everyHeadWaits(t)
|
|
||||||
asked := asksWithPaths(t)
|
asked := asksWithPaths(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1",
|
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1",
|
||||||
@@ -923,7 +860,6 @@ func sameMerge(a, b inventory.PlanMerge) bool {
|
|||||||
// its delivery would skip its turn. It joins the next catalogue batch.
|
// its delivery would skip its turn. It joins the next catalogue batch.
|
||||||
func TestALateCatalogueMergeIsNotAnsweredByAnOwnPathWalk(t *testing.T) {
|
func TestALateCatalogueMergeIsNotAnsweredByAnOwnPathWalk(t *testing.T) {
|
||||||
open := windowed(t)
|
open := windowed(t)
|
||||||
everyHeadWaits(t)
|
|
||||||
asksWithPaths(t)
|
asksWithPaths(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
for _, m := range []struct {
|
for _, m := range []struct {
|
||||||
|
|||||||
@@ -48,10 +48,6 @@ func gatherBatches(ctx context.Context, inv *inventory.Inventory, now time.Time)
|
|||||||
if f.atMost.Before(f.closed) {
|
if f.atMost.Before(f.closed) {
|
||||||
f.closed = f.atMost
|
f.closed = f.atMost
|
||||||
}
|
}
|
||||||
if w.Batch.CutAtOnce && !b.Created.IsZero() {
|
|
||||||
// Its window closed when it was made, which is its latest too (ADR 0304 decision 1).
|
|
||||||
f.atMost, f.closed = b.Created, b.Created
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if f.behind != "" {
|
if f.behind != "" {
|
||||||
if walk, err := inv.PlanByID(ctx, f.behind); err == nil {
|
if walk, err := inv.PlanByID(ctx, f.behind); err == nil {
|
||||||
|
|||||||
@@ -1,207 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **A filling bucket or log is said before it is full** (novox/hq ADR 0297 §6, issue 501).
|
|
||||||
//
|
|
||||||
// A module's bucket and its log each have a cap on the bus, and when either is full the bus refuses
|
|
||||||
// the next write: the module stops doing what it writes for, and nothing said so before. On
|
|
||||||
// 2026-10-11 the issue tracker's bucket held a sixth of its cap after two days, growing toward a stop
|
|
||||||
// nobody would have heard coming. So the self-check reads every module's bucket and log, and one at
|
|
||||||
// fillRaiseAt of its cap or more raises a condition naming the module, the bucket or log, and how full
|
|
||||||
// it is.
|
|
||||||
//
|
|
||||||
// **Cleared by observation below fillClearBelow, not below fillRaiseAt**: a bucket or log hovering at
|
|
||||||
// its threshold would otherwise be raised and cleared on every run. Between the two, an open condition
|
|
||||||
// is kept and none is raised.
|
|
||||||
//
|
|
||||||
// **One that cannot be read is said, and the rest are still judged.** An unanswered question about one
|
|
||||||
// stream is no reason to know nothing of the others: it is a finding of its own, naming the bucket or
|
|
||||||
// log and why, and a fill condition already open for it is kept, because not knowing is not a pass.
|
|
||||||
|
|
||||||
// The fill thresholds, in percent of a bucket's or log's cap.
|
|
||||||
const (
|
|
||||||
fillRaiseAt = 75
|
|
||||||
fillClearBelow = 70
|
|
||||||
)
|
|
||||||
|
|
||||||
// The conditions the fill probe raises: one filling, and one that could not be read.
|
|
||||||
const (
|
|
||||||
kindBucketOrLogFilling = "bucket-or-log-filling"
|
|
||||||
kindBucketOrLogUnread = "bucket-or-log-unread"
|
|
||||||
)
|
|
||||||
|
|
||||||
// probeFillID is the probe's id in the registry.
|
|
||||||
const probeFillID = "D-fill"
|
|
||||||
|
|
||||||
// bucketOrLog is one module's bucket or log as the fill probe reads it.
|
|
||||||
type bucketOrLog struct {
|
|
||||||
Module string
|
|
||||||
// Kind is `bucket` or `log`; Name its local name; Stream the stream it is on the bus.
|
|
||||||
Kind, Name, Stream string
|
|
||||||
}
|
|
||||||
|
|
||||||
// filled is how full one bucket or log is, read from the bus.
|
|
||||||
type filled struct {
|
|
||||||
Bytes, Max uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
// percent is how full, in whole percent, rounded down.
|
|
||||||
func (f filled) percent() uint64 {
|
|
||||||
if f.Max == 0 {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
return f.Bytes * 100 / f.Max
|
|
||||||
}
|
|
||||||
|
|
||||||
// fillKey is where a bucket's or log's fill condition is kept.
|
|
||||||
func fillKey(s bucketOrLog) string { return conditions.Key(conditions.ScopeBus, s.Stream, "filling") }
|
|
||||||
|
|
||||||
// fillObservation is what one bucket's or log's fill says: a finding at fillRaiseAt or above, and,
|
|
||||||
// while its condition is open, at fillClearBelow or above too; nothing otherwise, which is what clears
|
|
||||||
// it. The operator's words are the kind's (plain_words.go); the summary and the evidence name the
|
|
||||||
// module, the bucket or log, and the fill.
|
|
||||||
func fillObservation(s bucketOrLog, f filled, open bool) (conditions.Observation, bool) {
|
|
||||||
if f.Max == 0 {
|
|
||||||
return conditions.Observation{}, false // one with no cap cannot fill
|
|
||||||
}
|
|
||||||
// Compared in bytes, not rounded percent: 74.9% is not 75%.
|
|
||||||
atRaise := f.Bytes*100 >= f.Max*fillRaiseAt
|
|
||||||
aboveClear := f.Bytes*100 >= f.Max*fillClearBelow
|
|
||||||
if !atRaise && !(open && aboveClear) {
|
|
||||||
return conditions.Observation{}, false
|
|
||||||
}
|
|
||||||
pct := f.percent()
|
|
||||||
return conditions.Observation{
|
|
||||||
Scope: conditions.ScopeBus, ID: s.Stream, Token: "filling", Kind: kindBucketOrLogFilling,
|
|
||||||
Severity: conditions.Warning,
|
|
||||||
Summary: fmt.Sprintf("%s's %s %s holds %s of %s, %d%%: at its cap the bus refuses the module's next write",
|
|
||||||
s.Module, s.Kind, s.Name, mibWords(f.Bytes), mibWords(f.Max), pct),
|
|
||||||
Said: fmt.Sprintf("module %s, %s %s (stream %s): %d of %d bytes, %d%%", s.Module, s.Kind, s.Name,
|
|
||||||
s.Stream, f.Bytes, f.Max, pct),
|
|
||||||
}, true
|
|
||||||
}
|
|
||||||
|
|
||||||
// unreadObservation says one bucket or log could not be read, and why. Asked over the network, so one
|
|
||||||
// look can be wrong: raised on the second look in a row (confirm.go).
|
|
||||||
func unreadObservation(s bucketOrLog, why error) conditions.Observation {
|
|
||||||
return conditions.Observation{
|
|
||||||
Scope: conditions.ScopeBus, ID: s.Stream, Token: "unread", Kind: kindBucketOrLogUnread,
|
|
||||||
Severity: conditions.Warning, Confirm: true,
|
|
||||||
Summary: fmt.Sprintf("%s's %s %s could not be read, so how full it is is not known", s.Module, s.Kind, s.Name),
|
|
||||||
Said: fmt.Sprintf("module %s, %s %s (stream %s): %v", s.Module, s.Kind, s.Name, s.Stream, why),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// keptFilling is an open fill condition said again for a bucket or log that could not be read this
|
|
||||||
// time: not knowing how full it is now is no reason to say it has room. Only ever made from a condition
|
|
||||||
// read back as open — its own summary and severity, never words made up for it.
|
|
||||||
func keptFilling(s bucketOrLog, open conditions.Condition, why error) conditions.Observation {
|
|
||||||
return conditions.Observation{
|
|
||||||
Scope: conditions.ScopeBus, ID: s.Stream, Token: "filling", Kind: kindBucketOrLogFilling,
|
|
||||||
Severity: open.Severity, Summary: open.Summary,
|
|
||||||
Said: fmt.Sprintf("module %s, %s %s (stream %s): not read this time (%v); kept open as it was",
|
|
||||||
s.Module, s.Kind, s.Name, s.Stream, why),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// mibWords is a size as a person reads it, in MiB with one decimal.
|
|
||||||
func mibWords(b uint64) string {
|
|
||||||
return fmt.Sprintf("%.1f MiB", float64(b)/(1024*1024))
|
|
||||||
}
|
|
||||||
|
|
||||||
// readFill is how full one bucket or log is on the bus; found false when it is not on the bus.
|
|
||||||
type readFill func(ctx context.Context, s bucketOrLog) (f filled, found bool, err error)
|
|
||||||
|
|
||||||
// openFill is the fill condition open under a key, if one is, or why it could not be read.
|
|
||||||
type openFill func(ctx context.Context, key string) (conditions.Condition, bool, error)
|
|
||||||
|
|
||||||
// judgeFills judges every bucket and log on its own: one that cannot be read is said as unread, with
|
|
||||||
// its fill condition kept only when that condition was read back and is open — when it cannot be read
|
|
||||||
// either, nothing is said of its fill, which the unread finding already covers. Every other is judged
|
|
||||||
// by its fill; for one of those, a condition that cannot be read is taken as open, so an unknown never
|
|
||||||
// clears a fill measured between fillClearBelow and fillRaiseAt.
|
|
||||||
func judgeFills(ctx context.Context, all []bucketOrLog, read readFill, open openFill) []conditions.Observation {
|
|
||||||
var out []conditions.Observation
|
|
||||||
for _, s := range all {
|
|
||||||
f, found, err := read(ctx, s)
|
|
||||||
if err != nil {
|
|
||||||
out = append(out, unreadObservation(s, err))
|
|
||||||
if c, isOpen, cerr := open(ctx, fillKey(s)); cerr == nil && isOpen {
|
|
||||||
out = append(out, keptFilling(s, c, err))
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if !found {
|
|
||||||
continue // not on the bus: created on the controller's next raise, and nothing there can fill
|
|
||||||
}
|
|
||||||
_, isOpen, cerr := open(ctx, fillKey(s))
|
|
||||||
isOpen = isOpen || cerr != nil
|
|
||||||
if o, said := fillObservation(s, f, isOpen); said {
|
|
||||||
out = append(out, o)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.SliceStable(out, func(i, j int) bool { return out[i].Key() < out[j].Key() })
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// declaredBucketsAndLogs is every module's bucket and log the catalogue declares, by its stream.
|
|
||||||
func declaredBucketsAndLogs(ctx context.Context, d *doctor) ([]bucketOrLog, error) {
|
|
||||||
buckets, err := d.open.inventory.DeclaredBuckets(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
logs, err := d.open.inventory.DeclaredLogs(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var out []bucketOrLog
|
|
||||||
for _, b := range buckets {
|
|
||||||
out = append(out, bucketOrLog{Module: b.Module, Kind: "bucket", Name: b.Name, Stream: "KV_" + b.Bucket()})
|
|
||||||
}
|
|
||||||
for _, l := range logs {
|
|
||||||
out = append(out, bucketOrLog{Module: l.Module, Kind: "log", Name: l.Name, Stream: l.Stream()})
|
|
||||||
}
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// probeFill reads every module's bucket and log on the bus and says each one filling toward its cap,
|
|
||||||
// and each one that could not be read.
|
|
||||||
func probeFill(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
|
||||||
all, err := declaredBucketsAndLogs(ctx, d)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
js := d.js.Context()
|
|
||||||
read := func(ctx context.Context, s bucketOrLog) (filled, bool, error) {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return filled{}, false, err
|
|
||||||
}
|
|
||||||
info, err := js.StreamInfo(s.Stream, nats.Context(ctx))
|
|
||||||
switch {
|
|
||||||
case errors.Is(err, nats.ErrStreamNotFound):
|
|
||||||
return filled{}, false, nil
|
|
||||||
case err != nil:
|
|
||||||
return filled{}, false, err
|
|
||||||
case info.Config.MaxBytes <= 0:
|
|
||||||
return filled{}, true, nil
|
|
||||||
}
|
|
||||||
return filled{Bytes: info.State.Bytes, Max: uint64(info.Config.MaxBytes)}, true, nil
|
|
||||||
}
|
|
||||||
open := func(ctx context.Context, key string) (conditions.Condition, bool, error) {
|
|
||||||
if d.keeper == nil {
|
|
||||||
return conditions.Condition{}, false, nil
|
|
||||||
}
|
|
||||||
return d.keeper.Get(ctx, key)
|
|
||||||
}
|
|
||||||
return judgeFills(ctx, all, read, open), nil
|
|
||||||
}
|
|
||||||
@@ -1,165 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **A filling log or bucket is said at three quarters of its cap and cleared below seven tenths**
|
|
||||||
// (novox/hq ADR 0297 §6): raised at 75% naming the module, the bucket or log and its fill; not at 74.9%;
|
|
||||||
// kept between 70% and 75% while it is open, and not raised there when it is not; and cleared — said no
|
|
||||||
// more — once it reads below 70%, open or not.
|
|
||||||
func TestAFillingBucketOrLogIsRaisedAtThreeQuartersAndClearedBelowSevenTenths(t *testing.T) {
|
|
||||||
const mib = 1024 * 1024
|
|
||||||
log := bucketOrLog{Module: "mesh-issues", Kind: "log", Name: "changes", Stream: "LOG_mesh-issues_changes"}
|
|
||||||
bucket := bucketOrLog{Module: "mesh-issues", Kind: "bucket", Name: "issues", Stream: "KV_mesh-issues_issues"}
|
|
||||||
for _, c := range []struct {
|
|
||||||
name string
|
|
||||||
of bucketOrLog
|
|
||||||
bytes uint64
|
|
||||||
max uint64
|
|
||||||
open bool
|
|
||||||
said bool
|
|
||||||
}{
|
|
||||||
{"a log at exactly 75%", log, 768 * mib, 1024 * mib, false, true},
|
|
||||||
{"a bucket at exactly 75%", bucket, 48 * mib, 64 * mib, false, true},
|
|
||||||
{"a bucket full", bucket, 64 * mib, 64 * mib, false, true},
|
|
||||||
{"a log just under 75%", log, 768*mib - 1, 1024 * mib, false, false},
|
|
||||||
{"a bucket at 72%, not open", bucket, 64 * mib * 72 / 100, 64 * mib, false, false},
|
|
||||||
{"a bucket at 72%, open: kept", bucket, 64 * mib * 72 / 100, 64 * mib, true, true},
|
|
||||||
{"a log at exactly 70%, open: kept", log, 700 * mib, 1000 * mib, true, true},
|
|
||||||
{"a log just under 70%, open: cleared", log, 700*mib - 1, 1000 * mib, true, false},
|
|
||||||
{"a bucket at 10%, open: cleared", bucket, 64 * mib / 10, 64 * mib, true, false},
|
|
||||||
{"a bucket with no cap", bucket, 100, 0, true, false},
|
|
||||||
} {
|
|
||||||
o, said := fillObservation(c.of, filled{Bytes: c.bytes, Max: c.max}, c.open)
|
|
||||||
if said != c.said {
|
|
||||||
t.Errorf("%s: said %v, want %v", c.name, said, c.said)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if !said {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if o.Key() != fillKey(c.of) || o.Kind != kindBucketOrLogFilling || o.Severity != conditions.Warning {
|
|
||||||
t.Errorf("%s: raised as %s (%s, %s)", c.name, o.Key(), o.Kind, o.Severity)
|
|
||||||
}
|
|
||||||
for _, part := range []string{c.of.Module, c.of.Kind + " " + c.of.Name, "%", " of "} {
|
|
||||||
if !strings.Contains(o.Summary, part) {
|
|
||||||
t.Errorf("%s: does not name %q: %q", c.name, part, o.Summary)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !strings.Contains(o.Said, "bytes") {
|
|
||||||
t.Errorf("%s: the evidence does not say the fill in bytes: %q", c.name, o.Said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
o, _ := fillObservation(log, filled{Bytes: 800 * mib, Max: 1024 * mib}, false)
|
|
||||||
if !strings.Contains(o.Summary, "800.0 MiB of 1024.0 MiB, 78%") {
|
|
||||||
t.Errorf("the fill is said as %q", o.Summary)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **One bucket or log that cannot be read does not stop the others being judged** (review of #231): it is
|
|
||||||
// said as unread with its reason, a fill condition open for it is kept, and every other bucket and log
|
|
||||||
// is judged by its own fill.
|
|
||||||
func TestAnUnreadableBucketOrLogIsSaidAndTheRestAreStillJudged(t *testing.T) {
|
|
||||||
const mib = 1024 * 1024
|
|
||||||
broken := bucketOrLog{Module: "a", Kind: "bucket", Name: "broken", Stream: "KV_a_broken"}
|
|
||||||
brokenOpen := bucketOrLog{Module: "a", Kind: "log", Name: "was-filling", Stream: "LOG_a_was-filling"}
|
|
||||||
full := bucketOrLog{Module: "b", Kind: "log", Name: "changes", Stream: "LOG_b_changes"}
|
|
||||||
empty := bucketOrLog{Module: "c", Kind: "bucket", Name: "quiet", Stream: "KV_c_quiet"}
|
|
||||||
absent := bucketOrLog{Module: "d", Kind: "bucket", Name: "not-yet", Stream: "KV_d_not-yet"}
|
|
||||||
refusal := errors.New("the bus did not answer")
|
|
||||||
read := func(_ context.Context, s bucketOrLog) (filled, bool, error) {
|
|
||||||
switch s {
|
|
||||||
case broken, brokenOpen:
|
|
||||||
return filled{}, false, refusal
|
|
||||||
case full:
|
|
||||||
return filled{Bytes: 900 * mib, Max: 1000 * mib}, true, nil
|
|
||||||
case empty:
|
|
||||||
return filled{Bytes: 1, Max: 64 * mib}, true, nil
|
|
||||||
}
|
|
||||||
return filled{}, false, nil
|
|
||||||
}
|
|
||||||
open := func(_ context.Context, key string) (conditions.Condition, bool, error) {
|
|
||||||
if key == fillKey(brokenOpen) {
|
|
||||||
return conditions.Condition{Key: key, Severity: conditions.Warning,
|
|
||||||
Summary: "a's log was-filling holds 800.0 MiB of 1024.0 MiB, 78%"}, true, nil
|
|
||||||
}
|
|
||||||
return conditions.Condition{}, false, nil
|
|
||||||
}
|
|
||||||
got := map[string]conditions.Observation{}
|
|
||||||
for _, o := range judgeFills(context.Background(), []bucketOrLog{broken, brokenOpen, full, empty, absent}, read, open) {
|
|
||||||
got[o.Key()] = o
|
|
||||||
}
|
|
||||||
for _, s := range []bucketOrLog{broken, brokenOpen} {
|
|
||||||
o, said := got[conditions.Key(conditions.ScopeBus, s.Stream, "unread")]
|
|
||||||
if !said || o.Kind != kindBucketOrLogUnread || !o.Confirm || !strings.Contains(o.Said, refusal.Error()) {
|
|
||||||
t.Errorf("%s could not be read and was said as %+v", s.Stream, o)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if o, kept := got[fillKey(brokenOpen)]; !kept || !strings.Contains(o.Said, "kept open") {
|
|
||||||
t.Errorf("an open fill condition of a log not read this time was not kept: %+v", o)
|
|
||||||
}
|
|
||||||
if _, said := got[fillKey(broken)]; said {
|
|
||||||
t.Error("a bucket not read and not filling was said filling")
|
|
||||||
}
|
|
||||||
if o, said := got[fillKey(full)]; !said || o.Kind != kindBucketOrLogFilling {
|
|
||||||
t.Errorf("a log at 90%% beside an unreadable one was not judged: %+v", got)
|
|
||||||
}
|
|
||||||
if len(got) != 4 {
|
|
||||||
t.Errorf("said %d findings, want 4 (two unread, one kept, one filling): %v", len(got), got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Both kinds have plain words of their own, which hold to the plain rule (novox/hq ADR 0253).
|
|
||||||
func TestAFillingOrUnreadBucketOrLogIsSaidInPlainWords(t *testing.T) {
|
|
||||||
for _, kind := range []string{kindBucketOrLogFilling, kindBucketOrLogUnread} {
|
|
||||||
wording, has := plainWordings[kind]
|
|
||||||
if !has {
|
|
||||||
t.Errorf("%s has no plain words", kind)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if why, ok := conditions.PlainWords(wording(conditions.Observation{Kind: kind})); !ok {
|
|
||||||
t.Errorf("%s: %s", kind, why)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **When neither the bucket or log nor its condition can be read, nothing is said of its fill** (second
|
|
||||||
// review of #231): the unread finding covers it, and no fill is invented for it. For one that is read and
|
|
||||||
// measured between seven tenths and three quarters, a condition that cannot be read is taken as open, so
|
|
||||||
// an unknown never clears it.
|
|
||||||
func TestNothingIsSaidOfAFillWhenNeitherItNorItsConditionCanBeRead(t *testing.T) {
|
|
||||||
const mib = 1024 * 1024
|
|
||||||
broken := bucketOrLog{Module: "a", Kind: "log", Name: "changes", Stream: "LOG_a_changes"}
|
|
||||||
between := bucketOrLog{Module: "b", Kind: "bucket", Name: "issues", Stream: "KV_b_issues"}
|
|
||||||
read := func(_ context.Context, s bucketOrLog) (filled, bool, error) {
|
|
||||||
if s == broken {
|
|
||||||
return filled{}, false, errors.New("the bus did not answer")
|
|
||||||
}
|
|
||||||
return filled{Bytes: 72 * mib, Max: 100 * mib}, true, nil
|
|
||||||
}
|
|
||||||
open := func(context.Context, string) (conditions.Condition, bool, error) {
|
|
||||||
return conditions.Condition{}, false, errors.New("the conditions bucket did not answer")
|
|
||||||
}
|
|
||||||
got := map[string]conditions.Observation{}
|
|
||||||
for _, o := range judgeFills(context.Background(), []bucketOrLog{broken, between}, read, open) {
|
|
||||||
got[o.Key()] = o
|
|
||||||
}
|
|
||||||
if o, said := got[fillKey(broken)]; said {
|
|
||||||
t.Fatalf("a fill was said for a log whose fill and condition could not be read: %+v", o)
|
|
||||||
}
|
|
||||||
if _, said := got[conditions.Key(conditions.ScopeBus, broken.Stream, "unread")]; !said {
|
|
||||||
t.Error("the log that could not be read was not said as unread")
|
|
||||||
}
|
|
||||||
if _, kept := got[fillKey(between)]; !kept {
|
|
||||||
t.Error("a bucket at 72% whose condition could not be read was cleared")
|
|
||||||
}
|
|
||||||
if len(got) != 2 {
|
|
||||||
t.Errorf("said %d findings, want 2: %v", len(got), got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -835,9 +835,6 @@ type answers struct {
|
|||||||
// public name on the machine went dark. The holds were correct; they were recorded only in the
|
// public name on the machine went dark. The holds were correct; they were recorded only in the
|
||||||
// machine's own state file, and the one visible symptom was a count that did not add up.
|
// machine's own state file, and the one visible symptom was a count that did not add up.
|
||||||
untaken map[string]map[string]int
|
untaken map[string]map[string]int
|
||||||
// leftOut is, per machine, every module of its set its composition leaves out, and why (novox/hq issue
|
|
||||||
// 380): assigned and not applied, which every push said only in passing. Not well while there is any.
|
|
||||||
leftOut map[string][]leftOutModule
|
|
||||||
// unheld is every module on a machine whose resources are applied through a seat nothing on
|
// unheld is every module on a machine whose resources are applied through a seat nothing on
|
||||||
// that machine holds (novox/hq ADR 0207), with the modules that could hold it. Reported, not
|
// that machine holds (novox/hq ADR 0207), with the modules that could hold it. Reported, not
|
||||||
// refused, until the switch — and while there is any, the mesh is not all well: the order the
|
// refused, until the switch — and while there is any, the mesh is not all well: the order the
|
||||||
@@ -866,10 +863,6 @@ type answers struct {
|
|||||||
// applied, is not well: an assignment somebody made is not made yet, or will not be.
|
// applied, is not well: an assignment somebody made is not made yet, or will not be.
|
||||||
pending []inventory.PendingAssignment
|
pending []inventory.PendingAssignment
|
||||||
pendingUnread string
|
pendingUnread string
|
||||||
// lastSent is every machine's last recorded send, by name, with what the machine answered of it (novox/hq
|
|
||||||
// issue 485); lastSentUnread why it could not be read.
|
|
||||||
lastSent map[string]inventory.Send
|
|
||||||
lastSentUnread string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
|
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
|
||||||
|
|||||||
@@ -153,97 +153,8 @@ type forgeFacts struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// askTheForge asks the forge, through its module's tools on the bus, for a repository's id and its branch's
|
// askTheForge asks the forge, through its module's tools on the bus, for a repository's id and its branch's
|
||||||
// protection, waiting out a forge not served for a moment (novox/hq issue 486). A variable so a test needs no
|
// protection. A variable so a test needs no forge.
|
||||||
// forge.
|
|
||||||
var askTheForge = func(ctx context.Context, owner, repo, branch string) (forgeFacts, error) {
|
var askTheForge = func(ctx context.Context, owner, repo, branch string) (forgeFacts, error) {
|
||||||
return forgeWaitedOut(ctx, forgeWait, owner+"/"+repo, func(ctx context.Context) (forgeFacts, error) {
|
|
||||||
return askTheForgeOnce(ctx, owner, repo, branch)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// waitOut bounds how long a question is asked again while its answer is "not now": a pause of first, doubling
|
|
||||||
// to most, for at most bound in all. now and sleep are the clock, so a test waits in no real time.
|
|
||||||
type waitOut struct {
|
|
||||||
bound, first, most time.Duration
|
|
||||||
now func() time.Time
|
|
||||||
sleep func(context.Context, time.Duration) error
|
|
||||||
}
|
|
||||||
|
|
||||||
// forgeWait is the registration's wait for the forge (novox/hq issue 486). On 2026-10-11 a build's registration
|
|
||||||
// asked the forge while node-tools on the control-node re-served after a rollout of the tool runner, heard
|
|
||||||
// "nothing serves gitea.gitea_api", and failed its plan; the forge answered again seconds later. A re-serve takes
|
|
||||||
// about 10 s, so 60 s waits one out with room to spare. It is a cap on the whole wait, an ask in flight included:
|
|
||||||
// the registration holds the controller's one receive loop while it waits, and a loop held two minutes reads as
|
|
||||||
// deaf. The pauses are those the build machine waits the registry with (novox/hq issue 457).
|
|
||||||
var forgeWait = waitOut{bound: 60 * time.Second, first: time.Second, most: 10 * time.Second,
|
|
||||||
now: time.Now, sleep: sleepOrDone}
|
|
||||||
|
|
||||||
// sleepOrDone pauses for d, or less when ctx ends first.
|
|
||||||
func sleepOrDone(ctx context.Context, d time.Duration) error {
|
|
||||||
t := time.NewTimer(d)
|
|
||||||
defer t.Stop()
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return ctx.Err()
|
|
||||||
case <-t.C:
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// forgeNotNow is an answer that says the forge's tool is not served now — the tool runner re-serving — not that
|
|
||||||
// the forge refused. Only that is asked again: a refusal, an unreadable answer or a repository with no id is the
|
|
||||||
// forge's word, and a tool that did not answer in time has already held the receive loop its whole wait.
|
|
||||||
func forgeNotNow(err error) bool { return errors.Is(err, link.ErrNothingServes) }
|
|
||||||
|
|
||||||
// forgeWaitedOut asks the forge about repository, asking again while it answers "not served now", all of it
|
|
||||||
// within w's bound, an ask in flight included (novox/hq issue 486). Past the bound it fails, saying it waited,
|
|
||||||
// for how long, and what the forge last said; when the caller's own context ends, it says that instead.
|
|
||||||
func forgeWaitedOut(ctx context.Context, w waitOut, repository string,
|
|
||||||
ask func(context.Context) (forgeFacts, error)) (forgeFacts, error) {
|
|
||||||
capped, cancel := context.WithTimeout(ctx, w.bound)
|
|
||||||
defer cancel()
|
|
||||||
began := w.now()
|
|
||||||
pause := w.first
|
|
||||||
for asked := 1; ; asked++ {
|
|
||||||
facts, err := ask(capped)
|
|
||||||
if err == nil {
|
|
||||||
if asked > 1 {
|
|
||||||
fmt.Printf("the forge answered about %s again, after %s (novox/hq issue 486)\n", repository,
|
|
||||||
w.now().Sub(began).Round(time.Second))
|
|
||||||
}
|
|
||||||
return facts, nil
|
|
||||||
}
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
return forgeFacts{}, err // the caller's own context ended: no wait of ours to speak of
|
|
||||||
}
|
|
||||||
gaveUp := func() (forgeFacts, error) {
|
|
||||||
return forgeFacts{}, fmt.Errorf("waited %s for the forge to answer about %s, asking %d times, and it "+
|
|
||||||
"did not (novox/hq issue 486): %w", w.now().Sub(began).Round(time.Second), repository, asked, err)
|
|
||||||
}
|
|
||||||
if capped.Err() != nil {
|
|
||||||
return gaveUp() // the bound ended an ask in flight
|
|
||||||
}
|
|
||||||
if !forgeNotNow(err) {
|
|
||||||
return forgeFacts{}, err
|
|
||||||
}
|
|
||||||
if w.now().Sub(began)+pause > w.bound {
|
|
||||||
return gaveUp()
|
|
||||||
}
|
|
||||||
fmt.Printf("the forge cannot be asked about %s now (%v); asking again in %s, for at most %s in all "+
|
|
||||||
"(novox/hq issue 486)\n", repository, err, pause, w.bound)
|
|
||||||
if serr := w.sleep(capped, pause); serr != nil {
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
return forgeFacts{}, fmt.Errorf("stopped waiting for the forge to answer about %s: %w", repository,
|
|
||||||
ctx.Err())
|
|
||||||
}
|
|
||||||
return gaveUp()
|
|
||||||
}
|
|
||||||
pause = min(2*pause, w.most)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// askTheForgeOnce asks the forge once, through its module's tools on the bus.
|
|
||||||
func askTheForgeOnce(ctx context.Context, owner, repo, branch string) (forgeFacts, error) {
|
|
||||||
js, err := aBus()
|
js, err := aBus()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return forgeFacts{}, err
|
return forgeFacts{}, err
|
||||||
|
|||||||
@@ -151,6 +151,13 @@ func busCommand(ctx context.Context, args []string) error {
|
|||||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||||
sub, args = args[0], args[1:]
|
sub, args = args[0], args[1:]
|
||||||
}
|
}
|
||||||
|
// The view's credential, a terminal line like a person's (bus_view.go).
|
||||||
|
switch sub {
|
||||||
|
case "view-credential":
|
||||||
|
return busViewCredential(ctx, args)
|
||||||
|
case "view-revoke":
|
||||||
|
return busViewRevoke(ctx, args)
|
||||||
|
}
|
||||||
set := flag.NewFlagSet("bus", flag.ContinueOnError)
|
set := flag.NewFlagSet("bus", flag.ContinueOnError)
|
||||||
snapshot := set.String("snapshot-taken", "", "where the streams' snapshot a person took is, while the mesh takes none itself")
|
snapshot := set.String("snapshot-taken", "", "where the streams' snapshot a person took is, while the mesh takes none itself")
|
||||||
reversible := set.Bool("reversible", false, "the new version can be undone by putting the old one back")
|
reversible := set.Bool("reversible", false, "the new version can be undone by putting the old one back")
|
||||||
@@ -160,14 +167,14 @@ func busCommand(ctx context.Context, args []string) error {
|
|||||||
if rest, err := parseAround(set, args); err != nil {
|
if rest, err := parseAround(set, args); err != nil {
|
||||||
return err
|
return err
|
||||||
} else if len(rest) > 0 {
|
} else if len(rest) > 0 {
|
||||||
return errors.New("bus [upgrade --why … --reversible|--irreversible [--snapshot-taken <where>]]")
|
return errors.New(busUsage)
|
||||||
}
|
}
|
||||||
switch sub {
|
switch sub {
|
||||||
case "":
|
case "":
|
||||||
return busStatus(ctx)
|
return busStatus(ctx)
|
||||||
case "upgrade":
|
case "upgrade":
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("bus says what a bus upgrade would do, or `bus upgrade` — not %q", sub)
|
return fmt.Errorf("bus says what a bus upgrade would do, or `bus upgrade`, `bus view-credential`, `bus view-revoke` — not %q", sub)
|
||||||
}
|
}
|
||||||
// Everything refused before anything is done.
|
// Everything refused before anything is done.
|
||||||
if err := why.require("bus upgrade"); err != nil {
|
if err := why.require("bus upgrade"); err != nil {
|
||||||
|
|||||||
@@ -0,0 +1,117 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The view's credential: the one read-only user a page in a browser connects to the bus as, over the
|
||||||
|
// bus module's WebSocket listener (novox/hq research 036, gap G1; broker.KindView).
|
||||||
|
//
|
||||||
|
// **A terminal line, like a person's credential** (operator.go): printed once, never stored — the mesh
|
||||||
|
// keeps a hash — and revoked by forgetting the row, which the next composition of the user list makes
|
||||||
|
// real. There is one view; issuing it again rotates its password.
|
||||||
|
const busUsage = "bus [upgrade --why … --reversible|--irreversible [--snapshot-taken <where>] | view-credential | view-revoke]"
|
||||||
|
|
||||||
|
// busWebSocketPort is the port the bus module's WebSocket listener is published on, mirrored from the
|
||||||
|
// nats module's manifest (its `bus-websocket` opening), because the credential names where to connect
|
||||||
|
// and the controller does not read the module's configuration. Reached across the overlay only: the
|
||||||
|
// opening is from the mesh, and the mesh's filter admits nothing else.
|
||||||
|
const busWebSocketPort = 4223
|
||||||
|
|
||||||
|
func busViewCredential(ctx context.Context, args []string) error {
|
||||||
|
if len(args) != 0 {
|
||||||
|
return errors.New("bus view-credential takes nothing: there is one view, and this prints its credential once")
|
||||||
|
}
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
|
||||||
|
// Refused here rather than at the next composition, where it would stop the whole file.
|
||||||
|
if _, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindView, PasswordHash: "x"}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: broker.ViewUser, Kind: inventory.BusView})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
where, err := broker.FromEnvironment()
|
||||||
|
if err != nil && !errors.Is(err, broker.ErrNotConfigured) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
host := where.Address
|
||||||
|
if h, _, err := net.SplitHostPort(where.Address); err == nil {
|
||||||
|
host = h
|
||||||
|
}
|
||||||
|
websocket := ""
|
||||||
|
if host != "" {
|
||||||
|
websocket = "ws://" + net.JoinHostPort(host, strconv.Itoa(busWebSocketPort))
|
||||||
|
}
|
||||||
|
held, err := json.Marshal(struct {
|
||||||
|
WebSocket string `json:"websocket,omitempty"`
|
||||||
|
URL string `json:"url,omitempty"`
|
||||||
|
Fingerprint string `json:"fingerprint,omitempty"`
|
||||||
|
User string `json:"user"`
|
||||||
|
Password string `json:"password"`
|
||||||
|
InboxPrefix string `json:"inbox_prefix"`
|
||||||
|
Hears []string `json:"hears"`
|
||||||
|
Reads string `json:"reads"`
|
||||||
|
HowToRead string `json:"how_to_read"`
|
||||||
|
}{
|
||||||
|
WebSocket: websocket, URL: "nats://" + where.Address, Fingerprint: where.Fingerprint,
|
||||||
|
User: broker.ViewUser, Password: password,
|
||||||
|
// The client must make its inboxes under the view's own prefix: its subscribe grant is
|
||||||
|
// `_INBOX.view.>` and no wider (design 25 §4), and a client's default inbox is not under it.
|
||||||
|
InboxPrefix: "_INBOX." + broker.ViewUser,
|
||||||
|
Hears: broker.ViewHears, Reads: broker.ViewBucket,
|
||||||
|
HowToRead: "direct reads only, no watch (a consumer is refused): list with a request to $JS.API.DIRECT.GET.KV_" +
|
||||||
|
broker.ViewBucket + ` carrying {"multi_last":["$KV.` + broker.ViewBucket + `.>"]}, answered until a 204 status; ` +
|
||||||
|
"read one key with $JS.API.DIRECT.GET.KV_" + broker.ViewBucket + ".$KV." + broker.ViewBucket + ".<number> " +
|
||||||
|
"(nats.js: kvm.open(bucket, {allow_direct: true}), never create); re-read the key an event's number names",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("issued the view, which hears %s and reads the bucket %s, and nothing else\n",
|
||||||
|
strings.Join(broker.ViewHears, ", "), broker.ViewBucket)
|
||||||
|
fmt.Println(" this is the only time the credential is printed; the mesh keeps a hash")
|
||||||
|
fmt.Println(" it works once the bus has been told, which is the next push to the machine holding mesh-broker —")
|
||||||
|
fmt.Println(" and while a new build of the bus module waits for that machine, the next `bus upgrade` a person starts,")
|
||||||
|
fmt.Println(" which is also what brings the WebSocket listener it connects through")
|
||||||
|
fmt.Println()
|
||||||
|
fmt.Println(string(held))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func busViewRevoke(ctx context.Context, args []string) error {
|
||||||
|
if len(args) != 0 {
|
||||||
|
return errors.New("bus view-revoke takes nothing: there is one view")
|
||||||
|
}
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
|
||||||
|
if err := open.inventory.ForgetBusUser(ctx, broker.ViewUser); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// **Revoked at the next composition, not now** — as a person is (operator revoke): the bus's users
|
||||||
|
// are a file, and the credential stops working when the file no longer names it.
|
||||||
|
fmt.Println("the view is forgotten, and its credential stops working at the next composition — " +
|
||||||
|
"push the machine holding mesh-broker to make it so")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -2,7 +2,6 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"slices"
|
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
@@ -29,14 +28,6 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
|
|||||||
for _, e := range entries {
|
for _, e := range entries {
|
||||||
byName[e.Manifest.Module] = e
|
byName[e.Manifest.Module] = e
|
||||||
}
|
}
|
||||||
// The machine holding the bus, whose declaration carries every module's grants (novox/hq issue 490).
|
|
||||||
holder := ""
|
|
||||||
for _, e := range entries {
|
|
||||||
if e.Manifest.BusUsers != "" && e.Manifest.ClaimsSeat(catalogue.BrokerSeat) && len(e.On) > 0 {
|
|
||||||
holder = e.On[0]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var granting []string
|
|
||||||
machines := map[string]*link.MachinePlan{}
|
machines := map[string]*link.MachinePlan{}
|
||||||
machine := func(name string) *link.MachinePlan {
|
machine := func(name string) *link.MachinePlan {
|
||||||
if machines[name] == nil {
|
if machines[name] == nil {
|
||||||
@@ -59,10 +50,6 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
|
|||||||
machine(on).Receives = append(machine(on).Receives, name)
|
machine(on).Receives = append(machine(on).Receives, name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !waits && holder != "" && !(len(e.On) == 1 && e.On[0] == holder) && len(e.On) > 0 &&
|
|
||||||
!slices.Contains(granting, name) {
|
|
||||||
granting = append(granting, name)
|
|
||||||
}
|
|
||||||
switch {
|
switch {
|
||||||
case (name == "nats" || catalogue.ProvidesBus(e.Manifest)) && len(e.On) > 0:
|
case (name == "nats" || catalogue.ProvidesBus(e.Manifest)) && len(e.On) > 0:
|
||||||
// Said before the merge (novox/hq issue 336): a new bus build holds every send to the bus's machine
|
// Said before the merge (novox/hq issue 336): a new bus build holds every send to the bus's machine
|
||||||
@@ -89,13 +76,6 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(granting) > 0 {
|
|
||||||
// Said before the merge (novox/hq issue 490): what the walk does when the change alters the bus's
|
|
||||||
// user list. Whether it does is known only once the builds are registered, so it is said as a rule.
|
|
||||||
p.Steps = append(p.Steps, fmt.Sprintf("%s: if this changes what the bus's user list says (a new tool, "+
|
|
||||||
"subject or user), %s is sent that list alone, every build there kept, before %s is judged on its first "+
|
|
||||||
"machine", grantsStepWord, holder, strings.Join(granting, ", ")))
|
|
||||||
}
|
|
||||||
var names []string
|
var names []string
|
||||||
for name := range machines {
|
for name := range machines {
|
||||||
names = append(names, name)
|
names = append(names, name)
|
||||||
@@ -108,10 +88,6 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
|
|||||||
return p
|
return p
|
||||||
}
|
}
|
||||||
|
|
||||||
// grantsStepWord names the grants step (novox/hq issue 490): the bus's user list sent alone to the machine
|
|
||||||
// holding the bus ahead of a walk's gate. Not the bus step, which replaces the bus itself.
|
|
||||||
const grantsStepWord = "grants step"
|
|
||||||
|
|
||||||
// busUpgradeNeeded is how a change plan says that merging it holds the bus's machine for a person's step.
|
// busUpgradeNeeded is how a change plan says that merging it holds the bus's machine for a person's step.
|
||||||
const busUpgradeNeeded = "merging this needs a person's bus upgrade"
|
const busUpgradeNeeded = "merging this needs a person's bus upgrade"
|
||||||
|
|
||||||
|
|||||||
@@ -179,14 +179,6 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
|||||||
}
|
}
|
||||||
fmt.Fprintf(out, ", keeps state %s", strings.Join(kept, ", "))
|
fmt.Fprintf(out, ", keeps state %s", strings.Join(kept, ", "))
|
||||||
}
|
}
|
||||||
// And the logs it keeps, with their caps (novox/hq ADR 0297).
|
|
||||||
if len(m.Logs) > 0 {
|
|
||||||
kept := make([]string, 0, len(m.Logs))
|
|
||||||
for _, l := range m.Logs {
|
|
||||||
kept = append(kept, fmt.Sprintf("%s (%d MiB)", l.Name, l.Cap()))
|
|
||||||
}
|
|
||||||
fmt.Fprintf(out, ", keeps log %s", strings.Join(kept, ", "))
|
|
||||||
}
|
|
||||||
if len(m.Reads) > 0 {
|
if len(m.Reads) > 0 {
|
||||||
fmt.Fprintf(out, ", reads %s", strings.Join(m.Reads, ", "))
|
fmt.Fprintf(out, ", reads %s", strings.Join(m.Reads, ", "))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,10 +6,8 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/beside"
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The check anybody can run is the check registration runs (novox/hq issue 148, ADR 0037): a manifest
|
// The check anybody can run is the check registration runs (novox/hq issue 148, ADR 0037): a manifest
|
||||||
@@ -58,7 +56,10 @@ func TestModuleCheckJudgesBetweenTheManifestsGiven(t *testing.T) {
|
|||||||
|
|
||||||
// The real catalogue passes the command, the way it passes the test that used to be the only check.
|
// The real catalogue passes the command, the way it passes the test that used to be the only check.
|
||||||
func TestModuleCheckPassesTheCatalogue(t *testing.T) {
|
func TestModuleCheckPassesTheCatalogue(t *testing.T) {
|
||||||
root := beside.Catalogue(t)
|
root := filepath.Join("..", "..", "..", "mesh-catalog", "modules")
|
||||||
|
if _, err := os.Stat(root); err != nil {
|
||||||
|
t.Skipf("catalogue sibling not present: %v", err)
|
||||||
|
}
|
||||||
paths, err := manifestsUnder(root)
|
paths, err := manifestsUnder(root)
|
||||||
if err != nil || len(paths) == 0 {
|
if err != nil || len(paths) == 0 {
|
||||||
t.Fatalf("no manifests under %s: %v", root, err)
|
t.Fatalf("no manifests under %s: %v", root, err)
|
||||||
|
|||||||
@@ -5,8 +5,6 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"path"
|
"path"
|
||||||
"regexp"
|
|
||||||
"runtime/debug"
|
|
||||||
"slices"
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -248,11 +246,9 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop
|
|||||||
if dir == "mesh-controller" {
|
if dir == "mesh-controller" {
|
||||||
beside["mesh-controller-main"] = link.CheckedOut{Repository: url, Ref: refs["mesh-controller-main"]}
|
beside["mesh-controller-main"] = link.CheckedOut{Repository: url, Ref: refs["mesh-controller-main"]}
|
||||||
if e.Source.Seat != "" {
|
if e.Source.Seat != "" {
|
||||||
for _, sibling := range []string{"mesh-lab", "mesh-sdk"} {
|
if lab, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository,
|
||||||
if url, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository,
|
"mesh-lab")}); err == nil {
|
||||||
sibling)}); err == nil {
|
beside["mesh-lab"] = link.CheckedOut{Repository: lab, Ref: refs["mesh-lab"]}
|
||||||
beside[sibling] = link.CheckedOut{Repository: url, Ref: refs[sibling]}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -275,54 +271,17 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop
|
|||||||
// catalogue, whose checkout beside is what tests read its files from, at its main, what the next merge
|
// catalogue, whose checkout beside is what tests read its files from, at its main, what the next merge
|
||||||
// builds from; and beside the controller its main, for a judge the running controller predates, and the
|
// builds from; and beside the controller its main, for a judge the running controller predates, and the
|
||||||
// lab's main, whose replays every check runs. **One rule, read by the check the controller asks for and by
|
// lab's main, whose replays every check runs. **One rule, read by the check the controller asks for and by
|
||||||
// the facts snapshot** (Facts.Beside), so a check run by hand clones what the build seat clones. Beside the
|
// the facts snapshot** (Facts.Beside), so a check run by hand clones what the build seat clones.
|
||||||
// controller also the SDK, checked out at the commit the running controller's go.mod pins (sdkPinned), not
|
|
||||||
// one a desktop holds (novox/hq issue 449). The checkout only places the clone: the conformance test reads the
|
|
||||||
// fixtures at the pin of the tree under check, from the clone's history, so a pull request moving the SDK is
|
|
||||||
// judged against the SDK it moves to (internal/link/conformance_test.go).
|
|
||||||
func besideRefs(dir, running string) map[string]string {
|
func besideRefs(dir, running string) map[string]string {
|
||||||
switch dir {
|
switch dir {
|
||||||
case "mesh-catalog":
|
case "mesh-catalog":
|
||||||
return map[string]string{dir: "main"}
|
return map[string]string{dir: "main"}
|
||||||
case "mesh-controller":
|
case "mesh-controller":
|
||||||
return map[string]string{dir: running, "mesh-controller-main": "main", "mesh-lab": "main",
|
return map[string]string{dir: running, "mesh-controller-main": "main", "mesh-lab": "main"}
|
||||||
"mesh-sdk": sdkPinned()}
|
|
||||||
}
|
}
|
||||||
return map[string]string{dir: running}
|
return map[string]string{dir: running}
|
||||||
}
|
}
|
||||||
|
|
||||||
// sdkModule is the Go module of the SDK the controller is built against.
|
|
||||||
const sdkModule = "git.novox.be/novox/mesh-sdk/go"
|
|
||||||
|
|
||||||
// pseudoCommit is the commit a Go pseudo-version names: v0.1.11-0.20261009143344-f047d0a4a970 → f047d0a4a970.
|
|
||||||
var pseudoCommit = regexp.MustCompile(`-([0-9a-f]{12})$`)
|
|
||||||
|
|
||||||
// sdkPinned is the ref of the SDK repository this controller was built from, as its go.mod pins it and its
|
|
||||||
// build records it: a pseudo-version's commit, or a release's tag (the SDK tags its Go module under go/).
|
|
||||||
// "main" only for a binary that records no SDK version — a local replace — which a running controller is
|
|
||||||
// not (novox/hq issue 449).
|
|
||||||
func sdkPinned() string {
|
|
||||||
info, ok := debug.ReadBuildInfo()
|
|
||||||
if !ok {
|
|
||||||
return "main"
|
|
||||||
}
|
|
||||||
for _, dep := range info.Deps {
|
|
||||||
if dep.Path != sdkModule {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if dep.Replace != nil {
|
|
||||||
dep = dep.Replace
|
|
||||||
}
|
|
||||||
if m := pseudoCommit.FindStringSubmatch(dep.Version); m != nil {
|
|
||||||
return m[1]
|
|
||||||
}
|
|
||||||
if strings.HasPrefix(dep.Version, "v") {
|
|
||||||
return "go/" + dep.Version
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return "main"
|
|
||||||
}
|
|
||||||
|
|
||||||
// siblingOf is another repository of the same owner: novox/mesh-controller → novox/mesh-lab.
|
// siblingOf is another repository of the same owner: novox/mesh-controller → novox/mesh-lab.
|
||||||
func siblingOf(repository, name string) string {
|
func siblingOf(repository, name string) string {
|
||||||
if cut := strings.LastIndex(repository, "/"); cut >= 0 {
|
if cut := strings.LastIndex(repository, "/"); cut >= 0 {
|
||||||
|
|||||||
@@ -1,76 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The looks of the open conditions — how often each was observed, when last, its newest evidence — are
|
|
||||||
// kept in the controller's store, not the conditions bucket (novox/hq issue 420, ADR 0289 §1): the bucket
|
|
||||||
// is written when what a condition says changes, never for a look that saw it as it was.
|
|
||||||
|
|
||||||
// storeLooks keeps the conditions' looks in the inventory.
|
|
||||||
type storeLooks struct{ inv *inventory.Inventory }
|
|
||||||
|
|
||||||
func (s storeLooks) KeepLooks(ctx context.Context, key string, l conditions.Looks) error {
|
|
||||||
evidence, err := json.Marshal(l.Evidence)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return s.inv.KeepConditionLooks(ctx, inventory.ConditionLooks{Key: key, Observations: l.Observations,
|
|
||||||
LastObserved: l.LastObserved, Evidence: evidence})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s storeLooks) Looked(ctx context.Context, key string) (conditions.Looks, bool, error) {
|
|
||||||
row, kept, err := s.inv.ConditionLooksOf(ctx, key)
|
|
||||||
if err != nil || !kept {
|
|
||||||
return conditions.Looks{}, kept, err
|
|
||||||
}
|
|
||||||
l, err := looksOf(row)
|
|
||||||
return l, err == nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s storeLooks) AllLooks(ctx context.Context) (map[string]conditions.Looks, error) {
|
|
||||||
rows, err := s.inv.AllConditionLooks(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
out := make(map[string]conditions.Looks, len(rows))
|
|
||||||
for key, row := range rows {
|
|
||||||
l, err := looksOf(row)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
out[key] = l
|
|
||||||
}
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s storeLooks) ForgetLooks(ctx context.Context, key string) error {
|
|
||||||
return s.inv.ForgetConditionLooks(ctx, key)
|
|
||||||
}
|
|
||||||
|
|
||||||
func looksOf(row inventory.ConditionLooks) (conditions.Looks, error) {
|
|
||||||
l := conditions.Looks{Observations: row.Observations, LastObserved: row.LastObserved.UTC()}
|
|
||||||
if len(row.Evidence) > 0 {
|
|
||||||
if err := json.Unmarshal(row.Evidence, &l.Evidence); err != nil {
|
|
||||||
return conditions.Looks{}, fmt.Errorf("the evidence kept for the condition %s cannot be read: %w", row.Key, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return l, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// looksHere is where a process that is not the serving controller reads the conditions' looks: the store,
|
|
||||||
// opened for it, closed by the function returned. A store that cannot be opened is said on every condition
|
|
||||||
// read through it, never read as none.
|
|
||||||
func looksHere(ctx context.Context) (conditions.LookStore, func()) {
|
|
||||||
inv, err := openInventory(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return conditions.NoLooks{Err: err}, func() {}
|
|
||||||
}
|
|
||||||
return storeLooks{inv}, inv.Close
|
|
||||||
}
|
|
||||||
@@ -1,87 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/testbus"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **Looks at an unchanged condition write the bucket once, and `conditions` still says how often and when
|
|
||||||
// it was seen** (novox/hq issue 420): the looks are in the controller's store, and the verb, run as a
|
|
||||||
// process of its own, reads them from there. On 2026-10-10 one open condition had rewritten the bucket 7 003
|
|
||||||
// times in two days.
|
|
||||||
func TestTheConditionsVerbReadsTheLooksTheBucketNoLongerHolds(t *testing.T) {
|
|
||||||
inv := inventory.ForTest(t)
|
|
||||||
bus := testbus.Start(t)
|
|
||||||
serving, err := broker.Dial(bus.ClientURL())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer serving.Close()
|
|
||||||
if err := serving.EnsureControllerBuckets(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
ctx := t.Context()
|
|
||||||
store, history, err := conditions.OnTheBus(ctx, serving.Conn())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
k := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history, Looks: storeLooks{inv}})
|
|
||||||
defer k.Close(context.Background())
|
|
||||||
look := conditions.Observation{Scope: conditions.ScopeMachine, ID: "anchor", Kind: "silent", Machine: "anchor",
|
|
||||||
Severity: conditions.Warning, Summary: "anchor has not been heard from", Source: "S1"}
|
|
||||||
for range 4 {
|
|
||||||
if _, err := k.Observe(ctx, look); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
api, err := jetstream.New(serving.Conn())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
stream, err := api.Stream(ctx, "KV_"+broker.ConditionsBucket)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
info, err := stream.Info(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if info.State.LastSeq != 1 {
|
|
||||||
t.Fatalf("four looks at one unchanged condition wrote the bucket %d times, want 1", info.State.LastSeq)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The verb as a process other than the serving controller: it reads the bucket and the store.
|
|
||||||
keptBefore := conditionsFrom
|
|
||||||
conditionsFrom = nil
|
|
||||||
defer func() { conditionsFrom = keptBefore }()
|
|
||||||
t.Setenv(broker.NATSVar, bus.ClientURL())
|
|
||||||
t.Setenv(broker.CertificateVar, "")
|
|
||||||
var shown bytes.Buffer
|
|
||||||
if err := showCondition(ctx, []string{"machine.anchor.silent", "--json"}, &shown); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var c conditions.Condition
|
|
||||||
if err := json.Unmarshal(shown.Bytes(), &c); err != nil {
|
|
||||||
t.Fatalf("%v: %s", err, shown.String())
|
|
||||||
}
|
|
||||||
if c.Observations != 4 || len(c.Evidence) != 4 || c.LastObserved.IsZero() || c.LooksUnread != "" ||
|
|
||||||
time.Since(c.LastObserved) > time.Minute {
|
|
||||||
t.Fatalf("conditions show reads %d observation(s), %d evidence line(s), last %s (%q)", c.Observations,
|
|
||||||
len(c.Evidence), c.LastObserved, c.LooksUnread)
|
|
||||||
}
|
|
||||||
listed, err := openConditions(ctx)
|
|
||||||
if err != nil || len(listed) != 1 || listed[0].Observations != 4 {
|
|
||||||
t.Fatalf("conditions lists %+v: %v", listed, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -24,18 +24,16 @@ import (
|
|||||||
// **`status` used to be the only place the mesh said it was wrong, and only to whoever asked.** Every
|
// **`status` used to be the only place the mesh said it was wrong, and only to whoever asked.** Every
|
||||||
// core failure of research 031 was found by a person looking. A condition is the mesh saying it: raised
|
// core failure of research 031 was found by a person looking. A condition is the mesh saying it: raised
|
||||||
// by a watchdog when a signal is late (signals.go), by a probe when an invariant does not hold
|
// by a watchdog when a signal is late (signals.go), by a probe when an invariant does not hold
|
||||||
// (doctor.go), or by an event a provider sends (standing.go); kept on the bus with since-when, and its
|
// (doctor.go), or by an event a provider sends (standing.go); kept on the bus with since-when and
|
||||||
// looks — how often, when last, the newest evidence — in the controller's store (condition_looks.go,
|
// evidence; said on the bus as it changes, for the operator's channel to carry; and cleared when an
|
||||||
// novox/hq issue 420); said on the bus as it changes, for the operator's channel to carry; and cleared when an
|
|
||||||
// observation says it is resolved. Nobody resolves one by hand. A person who knows silences it, for a
|
// observation says it is resolved. Nobody resolves one by hand. A person who knows silences it, for a
|
||||||
// while, with a reason, and that is recorded as a hand act.
|
// while, with a reason, and that is recorded as a hand act.
|
||||||
|
|
||||||
// conditionsFrom is the serving controller's keeper; nil in any other process, which opens its own.
|
// conditionsFrom is the serving controller's keeper; nil in any other process, which opens its own.
|
||||||
var conditionsFrom *conditions.Keeper
|
var conditionsFrom *conditions.Keeper
|
||||||
|
|
||||||
// keeperOn is a keeper over the store on a connection, saying its transitions on that connection, with
|
// keeperOn is a keeper over the store on a connection, saying its transitions on that connection.
|
||||||
// the conditions' looks kept in looks (condition_looks.go).
|
func keeperOn(ctx context.Context, conn *nats.Conn) (*conditions.Keeper, error) {
|
||||||
func keeperOn(ctx context.Context, conn *nats.Conn, looks conditions.LookStore) (*conditions.Keeper, error) {
|
|
||||||
store, history, err := conditions.OnTheBus(ctx, conn)
|
store, history, err := conditions.OnTheBus(ctx, conn)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -44,7 +42,7 @@ func keeperOn(ctx context.Context, conn *nats.Conn, looks conditions.LookStore)
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history, Looks: looks,
|
return conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history,
|
||||||
Teller: link.OverNATS{Conn: conn, JS: js},
|
Teller: link.OverNATS{Conn: conn, JS: js},
|
||||||
Say: func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) },
|
Say: func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) },
|
||||||
// What status leads with changed: composed again soon (a nudge outside the serving controller
|
// What status leads with changed: composed again soon (a nudge outside the serving controller
|
||||||
@@ -60,10 +58,8 @@ func withKeeper(ctx context.Context, f func(*conditions.Keeper) error) error {
|
|||||||
if conditionsFrom != nil {
|
if conditionsFrom != nil {
|
||||||
return f(conditionsFrom)
|
return f(conditionsFrom)
|
||||||
}
|
}
|
||||||
looks, closeLooks := looksHere(ctx)
|
|
||||||
defer closeLooks()
|
|
||||||
return onTheBus(func(conn *nats.Conn) error {
|
return onTheBus(func(conn *nats.Conn) error {
|
||||||
k, err := keeperOn(ctx, conn, looks)
|
k, err := keeperOn(ctx, conn)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -86,8 +82,6 @@ func openConditions(ctx context.Context) ([]conditions.Condition, error) {
|
|||||||
return nil, fmt.Errorf("this process has no bus to read the conditions from: %w", err)
|
return nil, fmt.Errorf("this process has no bus to read the conditions from: %w", err)
|
||||||
}
|
}
|
||||||
var out []conditions.Condition
|
var out []conditions.Condition
|
||||||
looks, closeLooks := looksHere(ctx)
|
|
||||||
defer closeLooks()
|
|
||||||
err := onTheBus(func(conn *nats.Conn) error {
|
err := onTheBus(func(conn *nats.Conn) error {
|
||||||
store, _, err := conditions.OnTheBus(ctx, conn)
|
store, _, err := conditions.OnTheBus(ctx, conn)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -95,7 +89,7 @@ func openConditions(ctx context.Context) ([]conditions.Condition, error) {
|
|||||||
}
|
}
|
||||||
reading, cancel := context.WithTimeout(ctx, 5*time.Second)
|
reading, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
out, err = conditions.ReadWith(reading, store, looks)
|
out, err = conditions.Read(reading, store)
|
||||||
return err
|
return err
|
||||||
})
|
})
|
||||||
return out, err
|
return out, err
|
||||||
@@ -258,14 +252,12 @@ func showCondition(ctx context.Context, args []string, w io.Writer) error {
|
|||||||
if conditionsFrom != nil {
|
if conditionsFrom != nil {
|
||||||
c, found, err = conditionsFrom.Get(ctx, key)
|
c, found, err = conditionsFrom.Get(ctx, key)
|
||||||
} else {
|
} else {
|
||||||
looks, closeLooks := looksHere(ctx)
|
|
||||||
defer closeLooks()
|
|
||||||
err = onTheBus(func(conn *nats.Conn) error {
|
err = onTheBus(func(conn *nats.Conn) error {
|
||||||
store, _, err := conditions.OnTheBus(ctx, conn)
|
store, _, err := conditions.OnTheBus(ctx, conn)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
c, found, err = conditions.ReadOneWith(ctx, store, looks, key)
|
c, found, err = conditions.ReadOne(ctx, store, key)
|
||||||
return err
|
return err
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -169,44 +169,6 @@ func TestAShrinkOfMoreThanHalfIsUrgent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// THE FALSE ALARM (issue 368), replayed through the store D13 reads: an agent's home moved from the
|
|
||||||
// operator's own home (94.7 MB) to the agent account's fresh one (490 B), and `data-shrank` was raised
|
|
||||||
// for data that was never lost. A moved item is read against its new path only, so nothing is raised —
|
|
||||||
// and a genuine shrink at the new path, a week of history later, still is.
|
|
||||||
func TestAMovedPathIsNoShrinkAndAShrinkThereStillIs(t *testing.T) {
|
|
||||||
inv := inventory.ForTest(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
shelf := shelfFor(t, houseManifest)
|
|
||||||
declared := []inventory.DeclaredData{{Module: "house", Item: "config", Class: "irreplaceable", Owned: true}}
|
|
||||||
start := time.Now().Add(-6 * time.Hour)
|
|
||||||
measure := func(at time.Time, path string, size int64) []conditions.Observation {
|
|
||||||
t.Helper()
|
|
||||||
if _, err := inv.RecordData(ctx, "home", declared, map[string]map[string]inventory.Measurement{"house": {
|
|
||||||
"config": {Path: path, Size: bytesOf(size), MeasuredAt: when(at), LastWrite: when(at),
|
|
||||||
LastBackup: when(at)}}}, "", at); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
records, err := inv.Data(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
peaks, err := inv.DataPeaks(ctx, at.Add(-shrinkWindow))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return dataFindings(records, peaks, shelf, nil, nil, at)
|
|
||||||
}
|
|
||||||
measure(start, "/home/operator/.claude", 94_700_000)
|
|
||||||
if got := findingsByKind(measure(start.Add(10*time.Minute), "/home/agent/.claude", 490)); got[kindDataShrank].Kind != "" {
|
|
||||||
t.Fatalf("a moved path raised a shrink: %+v", got[kindDataShrank])
|
|
||||||
}
|
|
||||||
measure(start.Add(2*time.Hour), "/home/agent/.claude", 300<<20)
|
|
||||||
got := findingsByKind(measure(start.Add(4*time.Hour), "/home/agent/.claude", 1<<20))[kindDataShrank]
|
|
||||||
if got.Severity != conditions.Urgent || !strings.Contains(got.Summary, "shrank") {
|
|
||||||
t.Fatalf("a genuine shrink at the new path was not raised: %+v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Data said to be written all the time and not written; data with no backup or an old one — urgent when
|
// Data said to be written all the time and not written; data with no backup or an old one — urgent when
|
||||||
// irreplaceable, a warning when valuable; and a new item given its bound before it is said.
|
// irreplaceable, a warning when valuable; and a new item given its bound before it is said.
|
||||||
func TestQuietDataAndMissingBackupsAreSaidByClass(t *testing.T) {
|
func TestQuietDataAndMissingBackupsAreSaidByClass(t *testing.T) {
|
||||||
|
|||||||
@@ -144,15 +144,11 @@ func actOnDeadLetter(ctx context.Context, on *busHandles, act string, id uint64,
|
|||||||
}
|
}
|
||||||
switch act {
|
switch act {
|
||||||
case "deliver":
|
case "deliver":
|
||||||
delivered, to, err := link.DeliverAgain(on.js, id)
|
_, to, err := link.DeliverAgain(on.js, id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
answer["delivered_on"] = to
|
answer["delivered_on"] = to
|
||||||
if delivered.Original != "" {
|
|
||||||
// What became of the ask in its seat's queue (novox/hq issue 334): removed, or left, and why.
|
|
||||||
answer["original"] = delivered.Original
|
|
||||||
}
|
|
||||||
answer["done"] = fmt.Sprintf("dead letter %d was delivered again to %s, and nobody else; it is no longer kept",
|
answer["done"] = fmt.Sprintf("dead letter %d was delivered again to %s, and nobody else; it is no longer kept",
|
||||||
id, consumerWho(d.Stream, d.Consumer))
|
id, consumerWho(d.Stream, d.Consumer))
|
||||||
case "drop":
|
case "drop":
|
||||||
|
|||||||
+24
-595
@@ -10,7 +10,6 @@ import (
|
|||||||
"slices"
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
@@ -83,38 +82,6 @@ func waitingNote(p inventory.Plan) string {
|
|||||||
p.Delivery.Awaits, p.ID)
|
p.Delivery.Awaits, p.ID)
|
||||||
}
|
}
|
||||||
|
|
||||||
// waitingSaid is what a walk waiting for its word says when the delivery's owner said what holds it (novox/hq
|
|
||||||
// issue 529): a delivery held for a person, its reason and the verb that releases it — the person's decision, not
|
|
||||||
// a hand start — or the deliveries not at their turn. Told nothing, it is waitingNote.
|
|
||||||
func waitingSaid(p inventory.Plan, lines []stalledLine) string {
|
|
||||||
if len(lines) == 0 {
|
|
||||||
return waitingNote(p)
|
|
||||||
}
|
|
||||||
var held, unheld, other []string
|
|
||||||
for _, l := range lines {
|
|
||||||
switch l.State {
|
|
||||||
case "held":
|
|
||||||
held = append(held, l.Says)
|
|
||||||
case "unheld":
|
|
||||||
unheld = append(unheld, l.Says)
|
|
||||||
default:
|
|
||||||
other = append(other, l.Says)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
switch {
|
|
||||||
case len(held) > 0:
|
|
||||||
return fmt.Sprintf("published; its walk waits for %s's word, and a delivery it carries is held for a person: %s",
|
|
||||||
p.Delivery.Awaits, strings.Join(held, "; "))
|
|
||||||
case len(other) > 0:
|
|
||||||
// Not at its turn: it goes when its turn comes. A hand start here would skip its group's order (review of
|
|
||||||
// novox/hq issue 529), so none is offered.
|
|
||||||
return fmt.Sprintf("published; its walk waits for %s's word: %s", p.Delivery.Awaits, strings.Join(other, "; "))
|
|
||||||
}
|
|
||||||
// Held by nothing: nothing will give it its word, so the hand start is the way on.
|
|
||||||
return fmt.Sprintf("published; its walk waits for %s's word: %s — `plans go %s --why …` starts it by hand",
|
|
||||||
p.Delivery.Awaits, strings.Join(unheld, "; "), p.ID)
|
|
||||||
}
|
|
||||||
|
|
||||||
// letGo gives a waiting walk its word: by the delivery's owner, or a person. The next advance asks its
|
// letGo gives a waiting walk its word: by the delivery's owner, or a person. The next advance asks its
|
||||||
// first tier. Refused for a walk that does not wait.
|
// first tier. Refused for a walk that does not wait.
|
||||||
func letGo(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, error) {
|
func letGo(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, error) {
|
||||||
@@ -150,419 +117,28 @@ func letGo(ctx context.Context, inv *inventory.Inventory, id, by, why string) (i
|
|||||||
// stopWalk ends a walk on its delivery's word: failed, said as stopped by whom, why. What it asked still
|
// stopWalk ends a walk on its delivery's word: failed, said as stopped by whom, why. What it asked still
|
||||||
// builds and registers; nothing further is asked or sent.
|
// builds and registers; nothing further is asked or sent.
|
||||||
func stopWalk(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, error) {
|
func stopWalk(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, error) {
|
||||||
p, _, err := stopWalkWithout(ctx, inv, id, by, why, nil, time.Now().UTC())
|
|
||||||
return p, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// stopAnswer is what a stop that leaves merges out says, as data (novox/hq ADR 0299): the record stopped, the
|
|
||||||
// batch that walks its other merges again with those merges, every merge left out, the modules the stopped walk
|
|
||||||
// already sent that stay on the machines named, the modules the batch's walk withholds, and every merge elsewhere
|
|
||||||
// that will deliver a stopped change when it is walked — a later merge of its repository, or one moving its modules.
|
|
||||||
type stopAnswer struct {
|
|
||||||
Stopped string `json:"stopped"`
|
|
||||||
WalkedAgainBy string `json:"walked_again_by,omitempty"`
|
|
||||||
WalkedAgain []string `json:"walked_again,omitempty"`
|
|
||||||
LeftOut []inventory.PlanLeftOut `json:"left_out,omitempty"`
|
|
||||||
Kept []inventory.PlanKept `json:"kept,omitempty"`
|
|
||||||
Withheld []string `json:"withheld,omitempty"`
|
|
||||||
StillCarriedBy []string `json:"still_carried_by,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// stopWalkWithout ends a walk on its delivery's word and, given merges to leave out, walks every other merge it
|
|
||||||
// answered again without them (novox/hq issue 459, ADR 0299): one walk carries every merge of its batch, so ending
|
|
||||||
// it for one group member ended every other delivery on it.
|
|
||||||
//
|
|
||||||
// - Left out are the merges named, each later merge of their repository's branch, which contains one, and each
|
|
||||||
// merge moving a module a left-out merge moves: a module is built at its branch, which holds the stopped merge.
|
|
||||||
// - The others join the open batch of their kind — merges heard while the walk ran wait there, and an older
|
|
||||||
// commit never walks after a newer one — or a new batch when none is open; its window is closed, and its walk
|
|
||||||
// withholds the left-out merges' modules where it would build them only as dependents.
|
|
||||||
// - Given a batch not yet cut, the merges left out leave it for a stopped record of their own, and the batch
|
|
||||||
// goes on without them; with nothing left in it, the batch itself is stopped.
|
|
||||||
// - The stopped record, the batch and every merge moved are written in one transaction (decision 7).
|
|
||||||
//
|
|
||||||
// With nothing left to walk, or no merge named, the walk is only ended. A merge named that the record does not
|
|
||||||
// answer refuses the stop whole.
|
|
||||||
func stopWalkWithout(ctx context.Context, inv *inventory.Inventory, id, by, why string, without []string,
|
|
||||||
now time.Time) (inventory.Plan, stopAnswer, error) {
|
|
||||||
said := stopAnswer{Stopped: id}
|
|
||||||
release, err := inv.HoldPlans(ctx, true)
|
release, err := inv.HoldPlans(ctx, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return inventory.Plan{}, said, err
|
return inventory.Plan{}, err
|
||||||
}
|
}
|
||||||
defer release()
|
defer release()
|
||||||
p, err := inv.PlanByID(ctx, id)
|
p, err := inv.PlanByID(ctx, id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return inventory.Plan{}, said, err
|
return inventory.Plan{}, err
|
||||||
}
|
}
|
||||||
batch := p.Batch()
|
if !p.Open() {
|
||||||
if !p.Open() && !(batch && len(without) > 0) {
|
return p, fmt.Errorf("%s is already %s", p.ID, p.State)
|
||||||
return p, said, fmt.Errorf("%s is already %s", p.ID, p.State)
|
|
||||||
}
|
}
|
||||||
var merges, again []inventory.BatchedMerge
|
if p.Delivery == nil {
|
||||||
if len(without) > 0 {
|
p.Delivery = &inventory.PlanDelivery{}
|
||||||
if merges, err = inv.MergesOf(ctx, p.ID); err != nil {
|
|
||||||
return p, said, err
|
|
||||||
}
|
}
|
||||||
if again, said.LeftOut, err = leaveOut(merges, without); err != nil {
|
p.Delivery.Stopped, p.Delivery.StoppedWhy = by, why
|
||||||
return p, said, fmt.Errorf("%s is not stopped: %w", p.ID, err)
|
p.State = inventory.PlanFailed
|
||||||
|
p.Note = fmt.Sprintf("stopped by %s at tier %d: %s", by, p.Tier, why)
|
||||||
|
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||||
|
return p, err
|
||||||
}
|
}
|
||||||
}
|
return p, nil
|
||||||
stopped := func(r *inventory.Plan) {
|
|
||||||
if r.Delivery == nil {
|
|
||||||
r.Delivery = &inventory.PlanDelivery{}
|
|
||||||
}
|
|
||||||
r.Delivery.Stopped, r.Delivery.StoppedWhy, r.Delivery.LeftOut = by, why, said.LeftOut
|
|
||||||
r.State = inventory.PlanFailed
|
|
||||||
r.Note = fmt.Sprintf("stopped by %s at tier %d: %s", by, r.Tier, why)
|
|
||||||
}
|
|
||||||
withheld := movesOf(leftOutMerges(merges, said.LeftOut))
|
|
||||||
said.Withheld = withheld
|
|
||||||
if said.StillCarriedBy, err = stillCarried(ctx, inv, p.ID, merges, said.LeftOut, withheld); err != nil {
|
|
||||||
return p, said, err
|
|
||||||
}
|
|
||||||
if batch {
|
|
||||||
return p, said, stopInBatch(ctx, inv, &p, merges, again, &said, stopped, now)
|
|
||||||
}
|
|
||||||
entries, err := inv.Catalogued(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return p, said, err
|
|
||||||
}
|
|
||||||
running := map[string][]string{}
|
|
||||||
for _, e := range entries {
|
|
||||||
running[e.Manifest.Module] = e.On
|
|
||||||
}
|
|
||||||
said.Kept = keptOf(p, movesOf(again), running)
|
|
||||||
stopped(&p)
|
|
||||||
p.Delivery.Kept = said.Kept
|
|
||||||
if len(again) == 0 {
|
|
||||||
if len(said.LeftOut) > 0 {
|
|
||||||
p.Note += fmt.Sprintf("; left out %s, nothing else to walk", mergesLeftOutWords(said.LeftOut))
|
|
||||||
}
|
|
||||||
return p, said, inv.SavePlan(ctx, &p)
|
|
||||||
}
|
|
||||||
target, folded, foldMoves, err := batchToWalkAgain(ctx, inv, p.ID, again, now)
|
|
||||||
if err != nil {
|
|
||||||
return p, said, fmt.Errorf("%s is not stopped: %w", p.ID, err)
|
|
||||||
}
|
|
||||||
target.Delivery.Batch.Closed = true
|
|
||||||
target.Delivery.Batch.Withheld = union(target.Delivery.Batch.Withheld, withheld)
|
|
||||||
sort.Strings(target.Delivery.Batch.Withheld)
|
|
||||||
said.WalkedAgainBy = target.ID
|
|
||||||
var moves []inventory.MergeMove
|
|
||||||
for _, m := range again {
|
|
||||||
said.WalkedAgain = append(said.WalkedAgain, m.Repository+"@"+m.Commit)
|
|
||||||
moves = append(moves, inventory.MergeMove{Repository: m.Repository, Commit: m.Commit, Plan: target.ID, Alone: m.Alone})
|
|
||||||
}
|
|
||||||
p.Delivery.WalkedAgainBy = target.ID
|
|
||||||
p.Note += fmt.Sprintf("; its other merges are walked again by %s: %s", target.ID, strings.Join(said.WalkedAgain, ", "))
|
|
||||||
if len(said.LeftOut) > 0 {
|
|
||||||
p.Note += "; left out " + mergesLeftOutWords(said.LeftOut)
|
|
||||||
}
|
|
||||||
if len(said.Kept) > 0 {
|
|
||||||
p.Note += "; " + keptWords(said.Kept)
|
|
||||||
}
|
|
||||||
// The batch first, so no merge names a plan the store does not hold; all of it, or nothing (decision 7) — the
|
|
||||||
// batches folded into it too.
|
|
||||||
if err := inv.SavePlansMoving(ctx, append([]*inventory.Plan{&target, &p}, folded...),
|
|
||||||
append(moves, foldMoves...)); err != nil {
|
|
||||||
return p, said, fmt.Errorf("%s is not stopped, and none of its merges moved: %w", p.ID, err)
|
|
||||||
}
|
|
||||||
// What the batch holds and names now: written again by the cutter's next look should this fail.
|
|
||||||
if err := keepBatch(ctx, inv, &target, now, ""); err != nil {
|
|
||||||
fmt.Printf(" %s holds its merges; its record is written again at the next look: %v\n", target.ID, err)
|
|
||||||
}
|
|
||||||
fmt.Printf("%s stopped by %s; %s walks its other merges again: %s\n", p.ID, by, target.ID,
|
|
||||||
strings.Join(said.WalkedAgain, ", "))
|
|
||||||
return p, said, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// stopInBatch leaves merges out of a batch not yet cut (ADR 0299): they go to a stopped record of their own, which
|
|
||||||
// names the batch as walking the others, and the batch withholds their modules; a batch left with none is stopped
|
|
||||||
// itself.
|
|
||||||
func stopInBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan, merges, again []inventory.BatchedMerge,
|
|
||||||
said *stopAnswer, stopped func(*inventory.Plan), now time.Time) error {
|
|
||||||
if len(again) == 0 {
|
|
||||||
stopped(b)
|
|
||||||
b.Delivery.Batch = nil
|
|
||||||
b.Note = "a batch whose every merge was left out by a stop: " + b.Note
|
|
||||||
return inv.SavePlan(ctx, b)
|
|
||||||
}
|
|
||||||
left := leftOutMerges(merges, said.LeftOut)
|
|
||||||
first := left[0]
|
|
||||||
r := inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch,
|
|
||||||
Commit: first.Commit, Merged: first.Merged, Created: now, Tiers: [][]string{},
|
|
||||||
Modules: map[string]*inventory.PlanModule{}}
|
|
||||||
if r.ID == b.ID {
|
|
||||||
r.ID += "-left"
|
|
||||||
}
|
|
||||||
stopped(&r)
|
|
||||||
r.Delivery.WalkedAgainBy = b.ID
|
|
||||||
r.Note += fmt.Sprintf("; left out of %s before it was walked: %s; %s walks the others", b.ID,
|
|
||||||
mergesLeftOutWords(said.LeftOut), b.ID)
|
|
||||||
b.Delivery.Batch.Withheld = union(b.Delivery.Batch.Withheld, said.Withheld)
|
|
||||||
sort.Strings(b.Delivery.Batch.Withheld)
|
|
||||||
var moves []inventory.MergeMove
|
|
||||||
for _, m := range left {
|
|
||||||
moves = append(moves, inventory.MergeMove{Repository: m.Repository, Commit: m.Commit, Plan: r.ID, Alone: m.Alone})
|
|
||||||
}
|
|
||||||
for _, m := range again {
|
|
||||||
said.WalkedAgain = append(said.WalkedAgain, m.Repository+"@"+m.Commit)
|
|
||||||
}
|
|
||||||
said.Stopped, said.WalkedAgainBy = r.ID, b.ID
|
|
||||||
if err := inv.SavePlansMoving(ctx, []*inventory.Plan{&r, b}, moves); err != nil {
|
|
||||||
return fmt.Errorf("%s is not changed, and none of its merges moved: %w", b.ID, err)
|
|
||||||
}
|
|
||||||
if err := keepBatch(ctx, inv, b, now, ""); err != nil {
|
|
||||||
fmt.Printf(" %s holds its merges; its record is written again at the next look: %v\n", b.ID, err)
|
|
||||||
}
|
|
||||||
fmt.Printf("%s: %s left out by a stop, kept by %s\n", b.ID, mergesLeftOutWords(said.LeftOut), r.ID)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// batchToWalkAgain is the batch a stopped walk's merges join (novox/hq ADR 0299): a batch of their kind not yet cut
|
|
||||||
// that they are related to (ADR 0304 decision 1, ADR 0306 decision 3), or a new one, not yet written. Related to
|
|
||||||
// two or more, the newer are folded into the oldest: the plans and merge moves that fold them, to be written in
|
|
||||||
// the stop's one act. The kind is read from what the merges moved; the stopped walk itself is no walk a cut folds.
|
|
||||||
func batchToWalkAgain(ctx context.Context, inv *inventory.Inventory, stopping string, again []inventory.BatchedMerge,
|
|
||||||
now time.Time) (inventory.Plan, []*inventory.Plan, []inventory.MergeMove, error) {
|
|
||||||
own := false
|
|
||||||
for _, n := range movesOf(again) {
|
|
||||||
if _, on := onTheControllersPath[n]; on {
|
|
||||||
own = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
entries, read, edges, err := theGraph(ctx, inv)
|
|
||||||
if err != nil {
|
|
||||||
return inventory.Plan{}, nil, nil, err
|
|
||||||
}
|
|
||||||
joins, _, _, err := relatedBatches(ctx, inv, again, own, stopping, entries, read, edges)
|
|
||||||
if err != nil {
|
|
||||||
return inventory.Plan{}, nil, nil, err
|
|
||||||
}
|
|
||||||
if len(joins) > 0 {
|
|
||||||
into := joins[0]
|
|
||||||
if len(joins) == 1 {
|
|
||||||
return into, nil, nil, nil
|
|
||||||
}
|
|
||||||
plans, moves, err := foldInto(ctx, inv, &into, joins[1:])
|
|
||||||
if err != nil {
|
|
||||||
return inventory.Plan{}, nil, nil, err
|
|
||||||
}
|
|
||||||
return into, plans[1:], moves, nil
|
|
||||||
}
|
|
||||||
first := again[0]
|
|
||||||
return inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch,
|
|
||||||
Commit: first.Commit, Merged: first.Merged, Created: now, State: inventory.PlanAssembling,
|
|
||||||
Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{},
|
|
||||||
Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: own}}}, nil, nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// leaveOut splits a walk's merges by the merges named to leave out, each `<repository>@<commit>` (a commit's
|
|
||||||
// first characters will do): the merges walked again, and those left out — each named one; each later merge of
|
|
||||||
// its repository's branch, which contains it; and each merge moving a module a left-out merge moves, which is
|
|
||||||
// built at its branch and so holds the stopped merge — until none more is. A name the walk does not answer is
|
|
||||||
// refused.
|
|
||||||
func leaveOut(merges []inventory.BatchedMerge, without []string) ([]inventory.BatchedMerge, []inventory.PlanLeftOut, error) {
|
|
||||||
var stopped []inventory.BatchedMerge
|
|
||||||
for _, w := range without {
|
|
||||||
repository, commit, ok := strings.Cut(strings.TrimSpace(w), "@")
|
|
||||||
if !ok || repository == "" || len(commit) < 7 {
|
|
||||||
return nil, nil, fmt.Errorf("%q names no merge: <repository>@<commit>", w)
|
|
||||||
}
|
|
||||||
i := slices.IndexFunc(merges, func(m inventory.BatchedMerge) bool {
|
|
||||||
return strings.EqualFold(m.Repository, repository) && strings.HasPrefix(m.Commit, commit)
|
|
||||||
})
|
|
||||||
if i < 0 {
|
|
||||||
return nil, nil, fmt.Errorf("it answers no merge %s@%s", repository, commit)
|
|
||||||
}
|
|
||||||
stopped = append(stopped, merges[i])
|
|
||||||
}
|
|
||||||
same := func(a, b inventory.BatchedMerge) bool {
|
|
||||||
return strings.EqualFold(a.Repository, b.Repository) && a.Commit == b.Commit
|
|
||||||
}
|
|
||||||
out := map[int]inventory.PlanLeftOut{}
|
|
||||||
for i, m := range merges {
|
|
||||||
if slices.ContainsFunc(stopped, func(s inventory.BatchedMerge) bool { return same(s, m) }) {
|
|
||||||
out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for grew := true; grew; {
|
|
||||||
grew = false
|
|
||||||
for i, m := range merges {
|
|
||||||
if _, left := out[i]; left {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
for j, o := range out {
|
|
||||||
l := merges[j]
|
|
||||||
contains := o.Contains
|
|
||||||
if contains == "" {
|
|
||||||
contains = l.Commit
|
|
||||||
}
|
|
||||||
if strings.EqualFold(l.Repository, m.Repository) && l.Branch == m.Branch && laterMerge(m, l) {
|
|
||||||
out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit, Contains: contains}
|
|
||||||
} else if shared := sharedMoves(m, l); len(shared) > 0 {
|
|
||||||
out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit, Contains: contains,
|
|
||||||
Shares: shared}
|
|
||||||
} else {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
grew = true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var again []inventory.BatchedMerge
|
|
||||||
var left []inventory.PlanLeftOut
|
|
||||||
for i, m := range merges {
|
|
||||||
if o, is := out[i]; is {
|
|
||||||
left = append(left, o)
|
|
||||||
} else {
|
|
||||||
again = append(again, m)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.SliceStable(left, func(i, j int) bool { return left[i].Contains == "" && left[j].Contains != "" })
|
|
||||||
return again, left, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// sharedMoves are the modules two kept merges both moved when they were heard.
|
|
||||||
func sharedMoves(a, b inventory.BatchedMerge) []string {
|
|
||||||
var out []string
|
|
||||||
theirs := announcedOf(b).Moves
|
|
||||||
for _, n := range announcedOf(a).Moves {
|
|
||||||
if slices.Contains(theirs, n) && !slices.Contains(out, n) {
|
|
||||||
out = append(out, n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.Strings(out)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// leftOutMerges are the kept merges a stop left out.
|
|
||||||
func leftOutMerges(merges []inventory.BatchedMerge, out []inventory.PlanLeftOut) []inventory.BatchedMerge {
|
|
||||||
var left []inventory.BatchedMerge
|
|
||||||
for _, m := range merges {
|
|
||||||
if slices.ContainsFunc(out, func(o inventory.PlanLeftOut) bool {
|
|
||||||
return strings.EqualFold(o.Repository, m.Repository) && o.Commit == m.Commit
|
|
||||||
}) {
|
|
||||||
left = append(left, m)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return left
|
|
||||||
}
|
|
||||||
|
|
||||||
// movesOf is every module kept merges moved when they were heard, sorted.
|
|
||||||
func movesOf(merges []inventory.BatchedMerge) []string {
|
|
||||||
var out []string
|
|
||||||
for _, m := range merges {
|
|
||||||
out = union(out, announcedOf(m).Moves)
|
|
||||||
}
|
|
||||||
sort.Strings(out)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// keptOf are the modules a stopped walk sent that no merge walked again moves, and the machines each reached:
|
|
||||||
// those keep running the stopped walk's build. A module sent to its first machines reached those, and the rest it
|
|
||||||
// was sent to after them; one sent with no first machine (its policy sends it to all together, or a later tier is
|
|
||||||
// built by it) reached every machine running it.
|
|
||||||
func keptOf(p inventory.Plan, walkedAgain []string, running map[string][]string) []inventory.PlanKept {
|
|
||||||
var out []inventory.PlanKept
|
|
||||||
for name, m := range p.Modules {
|
|
||||||
if m == nil || slices.Contains(walkedAgain, name) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
var machines []string
|
|
||||||
if m.FirstAt != nil || m.SentAt != nil {
|
|
||||||
machines = union(machines, m.First)
|
|
||||||
}
|
|
||||||
if m.SentAt != nil && len(m.First) == 0 {
|
|
||||||
machines = union(machines, running[name])
|
|
||||||
}
|
|
||||||
if m.Gate != nil && m.Gate.Since != nil {
|
|
||||||
machines = union(machines, m.Gate.Machines)
|
|
||||||
}
|
|
||||||
for machine := range m.Rest {
|
|
||||||
machines = union(machines, []string{machine})
|
|
||||||
}
|
|
||||||
if len(machines) == 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
sort.Strings(machines)
|
|
||||||
out = append(out, inventory.PlanKept{Module: name, Machines: machines})
|
|
||||||
}
|
|
||||||
sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module })
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// keptWords is what a stopped walk leaves on the machines, as a person reads it.
|
|
||||||
func keptWords(kept []inventory.PlanKept) string {
|
|
||||||
var words []string
|
|
||||||
for _, k := range kept {
|
|
||||||
words = append(words, k.Module+" on "+strings.Join(k.Machines, ", "))
|
|
||||||
}
|
|
||||||
return "left running the stopped walk's build, which nothing walks again: " + strings.Join(words, "; ")
|
|
||||||
}
|
|
||||||
|
|
||||||
// stillCarried are the merges outside the stopped record, in a batch or an open walk, that will deliver a stopped
|
|
||||||
// change when they are walked: a later merge of its repository's branch, which contains it, or one that moves a
|
|
||||||
// module it moves. Holding a merged change back for good is a revert's (ADR 0299).
|
|
||||||
func stillCarried(ctx context.Context, inv *inventory.Inventory, id string, merges []inventory.BatchedMerge,
|
|
||||||
out []inventory.PlanLeftOut, withheld []string) ([]string, error) {
|
|
||||||
var named []inventory.BatchedMerge
|
|
||||||
for _, m := range leftOutMerges(merges, out) {
|
|
||||||
if slices.ContainsFunc(out, func(o inventory.PlanLeftOut) bool { return o.Commit == m.Commit && o.Contains == "" }) {
|
|
||||||
named = append(named, m)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(named) == 0 {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
open, err := inv.OpenPlans(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
batches, err := inv.Batches(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var carried []string
|
|
||||||
for _, p := range append(open, batches...) {
|
|
||||||
if p.ID == id {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
ms, err := inv.MergesOf(ctx, p.ID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
for _, m := range ms {
|
|
||||||
later := slices.ContainsFunc(named, func(s inventory.BatchedMerge) bool {
|
|
||||||
return strings.EqualFold(s.Repository, m.Repository) && s.Branch == m.Branch && laterMerge(m, s)
|
|
||||||
})
|
|
||||||
shares := slices.ContainsFunc(announcedOf(m).Moves, func(n string) bool { return slices.Contains(withheld, n) })
|
|
||||||
if later || shares {
|
|
||||||
carried = append(carried, fmt.Sprintf("%s@%s (in %s)", m.Repository, m.Commit, p.ID))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.Strings(carried)
|
|
||||||
return carried, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// mergesLeftOutWords is the merges a stop left out, as a person reads them: each that contains a stopped one, or
|
|
||||||
// shares its modules, says so.
|
|
||||||
func mergesLeftOutWords(out []inventory.PlanLeftOut) string {
|
|
||||||
var words []string
|
|
||||||
for _, o := range out {
|
|
||||||
s := o.Repository + "@" + short(o.Commit)
|
|
||||||
switch {
|
|
||||||
case len(o.Shares) > 0:
|
|
||||||
s += " (which moves " + strings.Join(o.Shares, ", ") + " as " + short(o.Contains) + " does)"
|
|
||||||
case o.Contains != "":
|
|
||||||
s += " (which contains " + short(o.Contains) + ")"
|
|
||||||
}
|
|
||||||
words = append(words, s)
|
|
||||||
}
|
|
||||||
return strings.Join(words, ", ")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// orderMember is one member of a group, as mesh-delivery says it.
|
// orderMember is one member of a group, as mesh-delivery says it.
|
||||||
@@ -890,8 +466,6 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
|||||||
group := set.String("group", "", "a delivery group's id")
|
group := set.String("group", "", "a delivery group's id")
|
||||||
by := set.String("by", catalogue.DeliverySeat, "who says it")
|
by := set.String("by", catalogue.DeliverySeat, "who says it")
|
||||||
why := set.String("why", "", "why")
|
why := set.String("why", "", "why")
|
||||||
without := set.String("without", "", "merges a stop leaves out, <repository>@<commit>, comma-separated: the walk's "+
|
|
||||||
"other merges are walked again without them (novox/hq ADR 0299)")
|
|
||||||
limit := set.Int("n", 50, "how many ended walks to answer beside the open ones")
|
limit := set.Int("n", 50, "how many ended walks to answer beside the open ones")
|
||||||
planID := set.String("plan", "", "one walk")
|
planID := set.String("plan", "", "one walk")
|
||||||
positionals, err := parseAround(set, rest)
|
positionals, err := parseAround(set, rest)
|
||||||
@@ -949,7 +523,7 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
|||||||
if *group == "" && len(members) == 1 {
|
if *group == "" && len(members) == 1 {
|
||||||
// One head, checked again as the forge's announcement would have it (a recheck): the controller's
|
// One head, checked again as the forge's announcement would have it (a recheck): the controller's
|
||||||
// own path for a pull request, run because the delivery's owner asked.
|
// own path for a pull request, run because the delivery's owner asked.
|
||||||
if err := sayingOnTheBus(ctx, inv, func() error { return (following{open}).PullUpdated(ctx, members[0].pull()) }); err != nil {
|
if err := sayingOnTheBus(ctx, func() error { return (following{open}).PullUpdated(ctx, members[0].pull()) }); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return answer(map[string]any{"rechecked": members[0].ID})
|
return answer(map[string]any{"rechecked": members[0].ID})
|
||||||
@@ -967,7 +541,7 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
|||||||
return errors.New("delivery go <plan> [--by <who>] [--why <text>]")
|
return errors.New("delivery go <plan> [--by <who>] [--why <text>]")
|
||||||
}
|
}
|
||||||
var p inventory.Plan
|
var p inventory.Plan
|
||||||
if err := sayingOnTheBus(ctx, inv, func() (err error) {
|
if err := sayingOnTheBus(ctx, func() (err error) {
|
||||||
p, err = letGo(ctx, inv, positionals[0], *by, strings.TrimSpace(*why))
|
p, err = letGo(ctx, inv, positionals[0], *by, strings.TrimSpace(*why))
|
||||||
return err
|
return err
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
@@ -978,21 +552,15 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
|||||||
return nil
|
return nil
|
||||||
case "stop":
|
case "stop":
|
||||||
if len(positionals) != 1 || strings.TrimSpace(*why) == "" {
|
if len(positionals) != 1 || strings.TrimSpace(*why) == "" {
|
||||||
return errors.New("delivery stop <plan> --why <text> [--by <who>] [--without <repository>@<commit>,…]")
|
return errors.New("delivery stop <plan> --why <text> [--by <who>]")
|
||||||
}
|
}
|
||||||
var p inventory.Plan
|
var p inventory.Plan
|
||||||
var said stopAnswer
|
if err := sayingOnTheBus(ctx, func() (err error) {
|
||||||
if err := sayingOnTheBus(ctx, inv, func() (err error) {
|
p, err = stopWalk(ctx, inv, positionals[0], *by, strings.TrimSpace(*why))
|
||||||
p, said, err = stopWalkWithout(ctx, inv, positionals[0], *by, strings.TrimSpace(*why), splitList(*without),
|
|
||||||
time.Now().UTC())
|
|
||||||
return err
|
return err
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if *without != "" {
|
|
||||||
// What it left out and what it walks again, as data: the delivery's owner reads it (ADR 0299).
|
|
||||||
return answer(said)
|
|
||||||
}
|
|
||||||
fmt.Printf("%s stopped by %s at tier %d of %d; what was asked still builds and registers, nothing further "+
|
fmt.Printf("%s stopped by %s at tier %d of %d; what was asked still builds and registers, nothing further "+
|
||||||
"is asked or sent\n", p.ID, *by, p.Tier, len(p.Tiers))
|
"is asked or sent\n", p.ID, *by, p.Tier, len(p.Tiers))
|
||||||
return nil
|
return nil
|
||||||
@@ -1028,81 +596,12 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// Each walk's phases, with the rest's reports (novox/hq ADR 0282 decision 6).
|
|
||||||
now := time.Now()
|
|
||||||
for i := range walks {
|
|
||||||
walks[i].Phases = walks[i].WalkPhases(now, appliedFrom(ctx, inv))
|
|
||||||
}
|
|
||||||
return answer(map[string]any{"held": deliverySeatHeld(entries), "walks": walks,
|
return answer(map[string]any{"held": deliverySeatHeld(entries), "walks": walks,
|
||||||
"own-path": sortedKeysOf(ownPathWords())})
|
"own-path": sortedKeysOf(ownPathWords())})
|
||||||
}
|
}
|
||||||
return fmt.Errorf("delivery %s: plan, order, check, go, stop or walks", sub)
|
return fmt.Errorf("delivery %s: plan, order, check, go, stop or walks", sub)
|
||||||
}
|
}
|
||||||
|
|
||||||
// appliedFrom answers a machine's first report after a send from the controller's `apply` durations; a lookup
|
|
||||||
// that fails is a report not read, which leaves the walk's end unknown rather than wrong.
|
|
||||||
func appliedFrom(ctx context.Context, inv *inventory.Inventory) inventory.AppliedLookup {
|
|
||||||
return func(node string, sent time.Time) (inventory.AppliedReport, bool) {
|
|
||||||
r, ok, err := inv.FirstAppliedAfter(ctx, node, sent)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "the report of %s after %s could not be read: %v\n", node, sent.Format(time.RFC3339), err)
|
|
||||||
return inventory.AppliedReport{}, false
|
|
||||||
}
|
|
||||||
return r, ok
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// recordWalkPhases keeps, once, each phase of every walk ended lately whose end is known, as a duration of kind
|
|
||||||
// walk-phase per class (novox/hq ADR 0282 decision 6): what `durations` summarises. A walk whose end is unknown
|
|
||||||
// past ApplySilentAfter keeps its measured phases without its total.
|
|
||||||
func recordWalkPhases(ctx context.Context, inv *inventory.Inventory, now time.Time) error {
|
|
||||||
recent, err := inv.RecentPlans(ctx, 30)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
for _, p := range recent {
|
|
||||||
if p.State != inventory.PlanDone || p.Release != nil || now.Sub(p.Updated) > 2*time.Hour {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
anyKept, totalKept, err := inv.WalkPhasesKept(ctx, p.ID)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if totalKept {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
ph := p.WalkPhases(now, appliedFrom(ctx, inv))
|
|
||||||
if ph != nil && ph.End == nil && anyKept {
|
|
||||||
continue // kept without its end; kept again only once its end is known
|
|
||||||
}
|
|
||||||
if ph == nil || (ph.End == nil && now.Sub(p.Updated) < inventory.ApplySilentAfter+time.Minute) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
class := ph.Class
|
|
||||||
if class == "" {
|
|
||||||
class = "unclassed"
|
|
||||||
}
|
|
||||||
for _, x := range ph.Phases {
|
|
||||||
if x.State != inventory.PhaseMeasured || x.Start == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if err := inv.RecordDuration(ctx, inventory.Duration{Kind: inventory.DurationWalkPhase,
|
|
||||||
Subject: class + "/" + x.Name, Ref: fmt.Sprintf("%s/%s/%d", p.ID, x.Name, x.Tier), Started: *x.Start,
|
|
||||||
Took: time.Duration(x.TookMS) * time.Millisecond, Detail: p.Named()}); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if ph.End != nil && ph.From != nil {
|
|
||||||
if err := inv.RecordDuration(ctx, inventory.Duration{Kind: inventory.DurationWalkPhase,
|
|
||||||
Subject: class + "/total", Ref: p.ID + "/total", Started: *ph.From,
|
|
||||||
Took: time.Duration(ph.TotalMS) * time.Millisecond, Detail: ph.Said}); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ownPathWords is the controller's own path as words, for an answer.
|
// ownPathWords is the controller's own path as words, for an answer.
|
||||||
func ownPathWords() map[string]string { return onTheControllersPath }
|
func ownPathWords() map[string]string { return onTheControllersPath }
|
||||||
|
|
||||||
@@ -1224,19 +723,12 @@ func shelfOf(entries []inventory.Entry) map[string]catalogue.Manifest {
|
|||||||
// delivery it walks reads its steps from. Never in the way of the walk: said beside it, and a save that could
|
// delivery it walks reads its steps from. Never in the way of the walk: said beside it, and a save that could
|
||||||
// not be said is logged — the delivery's owner, which also asks for the walks it follows, finds it by
|
// not be said is logged — the delivery's owner, which also asks for the walks it follows, finds it by
|
||||||
// comparison. Records arriving out of order are told apart by the plan's revision.
|
// comparison. Records arriving out of order are told apart by the plan's revision.
|
||||||
//
|
func sayPlanMoved(ctx context.Context, bus link.Bus, p inventory.Plan) {
|
||||||
// On the save that starts the walk and the one that ends it, the walk also says what it touches, its bound and,
|
go publishPlanMoved(ctx, bus, p)
|
||||||
// at the end, its outcome (novox/hq ADR 0308 decision 1, walk_said.go), read from facts; nil facts say only what
|
|
||||||
// the walk recorded.
|
|
||||||
func sayPlanMoved(ctx context.Context, bus link.Bus, facts walkFacts, p inventory.Plan) {
|
|
||||||
go publishPlanMoved(ctx, bus, facts, p)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func publishPlanMoved(ctx context.Context, bus link.Bus, facts walkFacts, p inventory.Plan) {
|
func publishPlanMoved(ctx context.Context, bus link.Bus, p inventory.Plan) {
|
||||||
// Its phases so far (novox/hq ADR 0282 decision 6): the rest's reports come after the walk ends, and are
|
body, err := json.Marshal(p)
|
||||||
// read by whoever asks for the walk (`delivery walks`).
|
|
||||||
p.Phases = p.WalkPhases(time.Now(), nil)
|
|
||||||
body, err := json.Marshal(planMovedBody{Plan: p, walkSaid: sayWalk(ctx, facts, p)})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -1251,7 +743,7 @@ func publishPlanMoved(ctx context.Context, bus link.Bus, facts walkFacts, p inve
|
|||||||
// controller's, or a connection of the command's own — a verb runs as a command of its own, and what it kept
|
// controller's, or a connection of the command's own — a verb runs as a command of its own, and what it kept
|
||||||
// would otherwise be said by nobody until the delivery's owner read the walks back. Without a bus the command
|
// would otherwise be said by nobody until the delivery's owner read the walks back. Without a bus the command
|
||||||
// still runs; what it did is found by comparison.
|
// still runs; what it did is found by comparison.
|
||||||
func sayingOnTheBus(ctx context.Context, facts walkFacts, f func() error) error {
|
func sayingOnTheBus(ctx context.Context, f func() error) error {
|
||||||
if checkEvents != nil {
|
if checkEvents != nil {
|
||||||
return f()
|
return f()
|
||||||
}
|
}
|
||||||
@@ -1262,10 +754,9 @@ func sayingOnTheBus(ctx context.Context, facts walkFacts, f func() error) error
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
bus := link.OverNATS{Conn: conn, JS: js}
|
bus := link.OverNATS{Conn: conn, JS: js}
|
||||||
was := inventory.PlanSaved
|
|
||||||
checkEvents = bus
|
checkEvents = bus
|
||||||
inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { publishPlanMoved(ctx, bus, facts, p) }
|
inventory.PlanSaved = func(p inventory.Plan) { publishPlanMoved(ctx, bus, p) }
|
||||||
defer func() { checkEvents, inventory.PlanSaved = nil, was }()
|
defer func() { checkEvents, inventory.PlanSaved = nil, nil }()
|
||||||
ran = true
|
ran = true
|
||||||
return f()
|
return f()
|
||||||
})
|
})
|
||||||
@@ -1275,65 +766,3 @@ func sayingOnTheBus(ctx context.Context, facts walkFacts, f func() error) error
|
|||||||
}
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// sayingWhatAVerbKept is PlanSaved for a process that is not the serving controller: each walk the verb keeps is
|
|
||||||
// said as `plan-moved` before the save returns, with the facts of the inventory that kept it, on the bus the verb
|
|
||||||
// reaches. It reaches the bus on the first save and never for a verb that keeps no walk. When the bus cannot be
|
|
||||||
// reached, a save that started or ended a walk says so on the verb's error output: its mark is kept and no later
|
|
||||||
// save says it, so the router holds what the walk touched until its bound, or not at all for a start. done lets
|
|
||||||
// the bus go and takes the hook back.
|
|
||||||
func sayingWhatAVerbKept(ctx context.Context) (said func(*inventory.Inventory, inventory.Plan), done func()) {
|
|
||||||
var (
|
|
||||||
mu sync.Mutex
|
|
||||||
reached bool
|
|
||||||
bus link.Bus
|
|
||||||
closing func()
|
|
||||||
failed error
|
|
||||||
)
|
|
||||||
said = func(inv *inventory.Inventory, p inventory.Plan) {
|
|
||||||
mu.Lock()
|
|
||||||
defer mu.Unlock()
|
|
||||||
if !reached {
|
|
||||||
reached = true
|
|
||||||
if handActConn != nil {
|
|
||||||
bus, failed = overNATS(handActConn)
|
|
||||||
} else if js, err := aBus(); err != nil {
|
|
||||||
failed = err
|
|
||||||
} else if bus, failed = overNATS(js.Conn()); failed == nil {
|
|
||||||
closing = js.Close
|
|
||||||
} else {
|
|
||||||
js.Close()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if bus == nil {
|
|
||||||
switch p.WalkSaid {
|
|
||||||
case inventory.WalkStarted:
|
|
||||||
fmt.Fprintf(os.Stderr, "%s: its start is kept and could NOT be said as plan-moved (%v): the router "+
|
|
||||||
"does not hold what it touches\n", p.ID, failed)
|
|
||||||
case inventory.WalkEnded:
|
|
||||||
fmt.Fprintf(os.Stderr, "%s: its end is kept and could NOT be said as plan-moved (%v): the router holds "+
|
|
||||||
"what it touched until its bound\n", p.ID, failed)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
publishPlanMoved(ctx, bus, inv, p)
|
|
||||||
}
|
|
||||||
done = func() {
|
|
||||||
mu.Lock()
|
|
||||||
defer mu.Unlock()
|
|
||||||
inventory.PlanSaved = nil
|
|
||||||
if closing != nil {
|
|
||||||
closing()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return said, done
|
|
||||||
}
|
|
||||||
|
|
||||||
// overNATS is the bus a connection carries, with its JetStream.
|
|
||||||
func overNATS(conn *nats.Conn) (link.Bus, error) {
|
|
||||||
js, err := conn.JetStream()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return link.OverNATS{Conn: conn, JS: js}, nil
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -37,41 +37,12 @@ type stalledLine struct {
|
|||||||
// Number is the pull request's, for a line of a head the forge never announced (novox/hq issue 347).
|
// Number is the pull request's, for a line of a head the forge never announced (novox/hq issue 347).
|
||||||
Number int `json:"number,omitempty"`
|
Number int `json:"number,omitempty"`
|
||||||
State string `json:"state"`
|
State string `json:"state"`
|
||||||
// Walk is the walk a line is about: a waiting walk no live delivery holds, or, asked with `walks`, each walk
|
|
||||||
// waiting for the owner's word and what holds it (novox/hq issues 362, 529).
|
|
||||||
Walk string `json:"walk,omitempty"`
|
|
||||||
// Waiting are the merge checks a line of the state `unanswered` waits on, as mesh-delivery says each:
|
|
||||||
// "mesh/merge-gate pending since <UTC time>", "mesh/repo-check never set" (novox/hq issue 438).
|
|
||||||
Waiting []string `json:"waiting,omitempty"`
|
|
||||||
// Checks are the same merge checks as data, which the controller words in the operator's own time (novox/hq
|
|
||||||
// issue 443): a time inside a finished sentence cannot be said again in another zone. A mesh-delivery from before
|
|
||||||
// says none, and Waiting is said as it reads.
|
|
||||||
Checks []waitingCheck `json:"checks,omitempty"`
|
|
||||||
// Held is the state of the delivery of a line of the state `unanswered`, ready or rejected, when one is held
|
|
||||||
// (novox/hq issues 441, 451): its check answered once, so a recheck is the step. Empty for a head with no
|
|
||||||
// delivery, and from a mesh-delivery that does not say it yet; the words are never read for it.
|
|
||||||
Held string `json:"held,omitempty"`
|
|
||||||
For string `json:"for"`
|
For string `json:"for"`
|
||||||
Bound string `json:"bound"`
|
Bound string `json:"bound"`
|
||||||
H2 string `json:"h2"`
|
H2 string `json:"h2"`
|
||||||
Says string `json:"says"`
|
Says string `json:"says"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// waitingCheck is one merge check a stalled line waits on, as mesh-delivery gives it: its context, its state —
|
|
||||||
// "pending", or "never-set" — and, for a pending one, since when in RFC 3339, empty when the forge did not say.
|
|
||||||
type waitingCheck struct {
|
|
||||||
Context string `json:"context"`
|
|
||||||
State string `json:"state"`
|
|
||||||
Since string `json:"since,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// wordsNow is the time the words are said at, which says whether a time needs its day; a seam a test replaces.
|
|
||||||
var wordsNow = time.Now
|
|
||||||
|
|
||||||
// wordsZone is the zone a stalled line's times are said in: the controller's local zone, as every other time in its
|
|
||||||
// messages. A seam a test replaces, so that no test writes time.Local, which every goroutine of the package reads.
|
|
||||||
var wordsZone = func() *time.Location { return time.Local }
|
|
||||||
|
|
||||||
// operatorsOnly is whether the table leaves H2 nothing to do for the line: the state is the operator's.
|
// operatorsOnly is whether the table leaves H2 nothing to do for the line: the state is the operator's.
|
||||||
func (l stalledLine) operatorsOnly() bool { return l.H2 == "" || strings.HasPrefix(l.H2, "none") }
|
func (l stalledLine) operatorsOnly() bool { return l.H2 == "" || strings.HasPrefix(l.H2, "none") }
|
||||||
|
|
||||||
@@ -132,34 +103,6 @@ func deliveriesStalled(ctx context.Context, conn *nats.Conn, held bool) ([]stall
|
|||||||
return lines, nil
|
return lines, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// walkWaits are, by walk, the lines mesh-delivery says of a walk waiting for its word: what holds it.
|
|
||||||
type walkWaits map[string][]stalledLine
|
|
||||||
|
|
||||||
// walksWaiting asks the delivery's owner what holds each walk waiting for its word (novox/hq issue 529): its
|
|
||||||
// `stalled` asked with `walks`, so a line of `plans` names a held delivery, its reason and the verb that releases
|
|
||||||
// it, instead of pointing at a hand start. Nil when no holder is on record or none answers: the line then reads as
|
|
||||||
// before. An owner from before this answers its stalled lines, of which only those naming a walk are kept.
|
|
||||||
func walksWaiting(ctx context.Context, conn *nats.Conn, held bool) walkWaits {
|
|
||||||
if !held {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
raw, err := askDeliveryOwner(ctx, conn, "stalled", map[string]any{"walks": true})
|
|
||||||
if err != nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
var lines []stalledLine
|
|
||||||
if json.Unmarshal(raw, &lines) != nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
out := walkWaits{}
|
|
||||||
for _, l := range lines {
|
|
||||||
if l.Walk != "" {
|
|
||||||
out[l.Walk] = append(out[l.Walk], l)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// stalledObservations are the conditions of what is stalled.
|
// stalledObservations are the conditions of what is stalled.
|
||||||
func stalledObservations(lines []stalledLine) []conditions.Observation {
|
func stalledObservations(lines []stalledLine) []conditions.Observation {
|
||||||
out := make([]conditions.Observation, 0, len(lines))
|
out := make([]conditions.Observation, 0, len(lines))
|
||||||
|
|||||||
@@ -119,13 +119,6 @@ func TestTheBusMachineIsBehindByItsUserListAlone(t *testing.T) {
|
|||||||
if userListBehind("", "") {
|
if userListBehind("", "") {
|
||||||
t.Fatal("a machine sent no list reads as behind")
|
t.Fatal("a machine sent no list reads as behind")
|
||||||
}
|
}
|
||||||
// A list that could not be composed is behind: not known is never "not behind" (the review of hq issue 353).
|
|
||||||
if !listBehind("", errors.New("the plan cannot be worked out"), digestOf([]byte(list))) {
|
|
||||||
t.Fatal("a list nobody could compose reads as current")
|
|
||||||
}
|
|
||||||
if listBehind(list, nil, digestOf([]byte(list))) || !listBehind(list, nil, "") {
|
|
||||||
t.Fatal("listBehind does not read the digest as userListBehind does")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// The machine holding the bus goes first: its declaration carries the user list the new grants are
|
// The machine holding the bus goes first: its declaration carries the user list the new grants are
|
||||||
|
|||||||
@@ -74,12 +74,8 @@ type probe struct {
|
|||||||
// probeRegistry is the registry, in to-be 45's order. **The registry is the design's live form**: a
|
// probeRegistry is the registry, in to-be 45's order. **The registry is the design's live form**: a
|
||||||
// probe added to a design is a row added here.
|
// probe added to a design is a row added here.
|
||||||
var probeRegistry = []probe{
|
var probeRegistry = []probe{
|
||||||
// D1 also says every module assigned and left out of a machine's declaration (novox/hq issue 380), from the
|
{ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation",
|
||||||
// resolution it already makes: needs-operator for a setting nobody gave, left-out for any other cause.
|
From: "issues 236, 263, 275", Kind: "declaration-refused", Raises: []string{kindAwaitingPush}, Phase: 1,
|
||||||
{ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation; no module " +
|
|
||||||
"assigned to a machine is left out of its declaration unsaid",
|
|
||||||
From: "issues 236, 263, 275, 380", Kind: "declaration-refused",
|
|
||||||
Raises: []string{kindAwaitingPush, kindLeftOut, kindNeedsOperator}, Phase: 1,
|
|
||||||
run: probeDeclarations},
|
run: probeDeclarations},
|
||||||
{ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6",
|
{ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6",
|
||||||
From: "issue 262", Kind: "resolver-wrong", Phase: 1, run: probeResolvers},
|
From: "issue 262", Kind: "resolver-wrong", Phase: 1, run: probeResolvers},
|
||||||
@@ -120,11 +116,6 @@ var probeRegistry = []probe{
|
|||||||
{ID: probeDeliveriesID, Asserts: "no delivery is held past its state's bound unsaid: mesh-delivery's " +
|
{ID: probeDeliveriesID, Asserts: "no delivery is held past its state's bound unsaid: mesh-delivery's " +
|
||||||
"`stalled`, each with the transition its table lets healer H2 take", From: "ADR 0239",
|
"`stalled`, each with the transition its table lets healer H2 take", From: "ADR 0239",
|
||||||
Kind: kindDeliveryStalled, Phase: 3, run: probeDeliveries},
|
Kind: kindDeliveryStalled, Phase: 3, run: probeDeliveries},
|
||||||
// The delivery budgets (novox/hq ADR 0282 decision 7): the newest delivery of each class within its budget,
|
|
||||||
// read from mesh-delivery's `times`; a measurement said, never a delivery held.
|
|
||||||
{ID: probeBudgetsID, Asserts: "the newest delivery of a leaf module ran on every machine within three minutes of " +
|
|
||||||
"its merge, and of a core module within ten: mesh-delivery's `times`", From: "ADR 0282, ADR 0304",
|
|
||||||
Kind: kindOverBudget, Phase: 3, run: probeBudgets},
|
|
||||||
// A client of the bus reconnecting in a loop (novox/hq issue 327), from the server's record of closed
|
// A client of the bus reconnecting in a loop (novox/hq issue 327), from the server's record of closed
|
||||||
// connections, which the bus's own module reads.
|
// connections, which the bus's own module reads.
|
||||||
{ID: probeReconnectsID, Asserts: "no user of the bus had its connection dropped more than twelve times in the " +
|
{ID: probeReconnectsID, Asserts: "no user of the bus had its connection dropped more than twelve times in the " +
|
||||||
@@ -140,12 +131,6 @@ var probeRegistry = []probe{
|
|||||||
{ID: "D-root", Asserts: "no agent can become root without a person on a machine where the router or a channel " +
|
{ID: "D-root", Asserts: "no agent can become root without a person on a machine where the router or a channel " +
|
||||||
"proving its sender runs: not by its own account, and not through a tool that runs its command as an account " +
|
"proving its sender runs: not by its own account, and not through a tool that runs its command as an account " +
|
||||||
"that can", From: "ADR 0259 §8", Kind: kindRootNotFree, Phase: 2, run: probeAgentRoot},
|
"that can", From: "ADR 0259 §8", Kind: kindRootNotFree, Phase: 2, run: probeAgentRoot},
|
||||||
// A module's bucket or log filling toward its cap (novox/hq ADR 0297 §6): said at three quarters, cleared
|
|
||||||
// below seven tenths, so one at its threshold is not raised and cleared on every run. One that cannot be
|
|
||||||
// read is said on its own, and every other is still judged.
|
|
||||||
{ID: probeFillID, Asserts: "every module's bucket and log holds less than three quarters of its cap; one " +
|
|
||||||
"said filling is cleared once it holds less than seven tenths", From: "ADR 0297, issue 501",
|
|
||||||
Kind: kindBucketOrLogFilling, Raises: []string{kindBucketOrLogUnread}, Phase: 1, run: probeFill},
|
|
||||||
{ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals",
|
{ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals",
|
||||||
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs},
|
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs},
|
||||||
// The core's health definitions (novox/hq to-be 45 §8, ADR 0236): what a core component's new build is
|
// The core's health definitions (novox/hq to-be 45 §8, ADR 0236): what a core component's new build is
|
||||||
@@ -651,7 +636,7 @@ func localDoctor(ctx context.Context) (*doctor, func(), error) {
|
|||||||
open.Close()
|
open.Close()
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
k, err := keeperOn(ctx, js.Conn(), storeLooks{open.inventory})
|
k, err := keeperOn(ctx, js.Conn())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
js.Close()
|
js.Close()
|
||||||
open.Close()
|
open.Close()
|
||||||
|
|||||||
@@ -514,10 +514,6 @@ func composedAndValidated(ctx context.Context, open *stores, node string, gens m
|
|||||||
if declared.Epoch, err = open.inventory.SentEpoch(ctx, record.ID); err != nil {
|
if declared.Epoch, err = open.inventory.SentEpoch(ctx, record.ID); err != nil {
|
||||||
return sendable{}, nil, err
|
return sendable{}, nil, err
|
||||||
}
|
}
|
||||||
// And the generation it was last sent, as the would-send is (novox/hq issue 234).
|
|
||||||
if declared.Generation, err = open.inventory.SentGeneration(ctx, record.ID); err != nil {
|
|
||||||
return sendable{}, nil, err
|
|
||||||
}
|
|
||||||
body, err := declared.Body()
|
body, err := declared.Body()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return sendable{}, nil, err
|
return sendable{}, nil, err
|
||||||
|
|||||||
@@ -1,66 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A provider whose wait fails the controller's check lists who waits on it (novox/hq issue 450).
|
|
||||||
//
|
|
||||||
// A wait that does not check out is judged unhealthy (ADR 0283 decision 3), so the provider raises its unhealthy
|
|
||||||
// condition and its consumers are held under it (ADR 0240 rule 5). What is stored is what the machine said, the
|
|
||||||
// wait unchecked: read as said, the provider seems to wait for the operator, and the held consumers were never
|
|
||||||
// listed on the condition that is open.
|
|
||||||
|
|
||||||
// refusedWait is a wait for a secret the database's manifest does not declare: it fails the check.
|
|
||||||
var refusedWait = inventory.Wait{Part: "postgres-database", Secret: "certificate", What: "the database's certificate"}
|
|
||||||
|
|
||||||
// judgedRefused is the provider's resource as the controller judges it: unhealthy, saying why.
|
|
||||||
func judgedRefused() inventory.ResourceHealth {
|
|
||||||
r := waitingDatabaseFor(refusedWait)
|
|
||||||
r.State, r.Waits = link.StateUnhealthy, nil
|
|
||||||
r.Reason = "says it waits for the secret certificate, which db does not declare"
|
|
||||||
return r
|
|
||||||
}
|
|
||||||
|
|
||||||
// The consumer's statement arrives after the provider's, so it is the consumer's judging (sayWaiters) that must list
|
|
||||||
// it at the provider's unhealthy condition, made urgent by who waits on it.
|
|
||||||
func TestAProviderWhoseWaitFailedItsCheckListsWhoWaitsOnIt(t *testing.T) {
|
|
||||||
k, _ := withConditionsInMemory(t)
|
|
||||||
stored := waitingDatabaseFor(refusedWait)
|
|
||||||
open := judgeBoth(t, k, []inventory.ResourceHealth{judgedRefused()},
|
|
||||||
map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Resources: []inventory.ResourceHealth{stored}}},
|
|
||||||
shopFailingBeside(stored))
|
|
||||||
provider := conditionOf(open, moduleUnhealthyKey("db", "anchor"))
|
|
||||||
if len(open) != 1 || provider == nil {
|
|
||||||
t.Fatalf("a provider whose wait failed its check and a consumer of it raised %v; want the provider's "+
|
|
||||||
"unhealthy alone", openKeysOf(open))
|
|
||||||
}
|
|
||||||
if said := provider.Evidence[0].Said; !strings.Contains(said, "shop on laptop") {
|
|
||||||
t.Fatalf("the provider's unhealthy condition does not list shop on laptop as waiting on it: %s", said)
|
|
||||||
}
|
|
||||||
if provider.Severity != conditions.Urgent {
|
|
||||||
t.Fatalf("the provider's unhealthy condition is %s with a consumer waiting on it; want urgent", provider.Severity)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A provider that waits for a secret already given is unhealthy to its consumers too, before its own condition opens:
|
|
||||||
// they are held under it as under any unhealthy provider, never as waiting for the operator, and its condition, when
|
|
||||||
// open, is said as unhealthy with who waits on it.
|
|
||||||
func TestAProviderWaitingForASecretAlreadyGivenIsUnhealthyToItsConsumers(t *testing.T) {
|
|
||||||
given := holdingOf(nil, shopFailingBeside(waitingDatabase()), shopOnTheDatabase)
|
|
||||||
given.waits.given["db@anchor"] = map[string]time.Time{"licence": time.Now().Add(-time.Hour)}
|
|
||||||
by, held := given.heldWith("laptop", "shop", []inventory.ResourceHealth{failingConsumer("shop")})
|
|
||||||
if !held || by.provider != theDatabase || len(by.waits) > 0 {
|
|
||||||
t.Fatalf("shop under a database waiting for a licence already given: held %v under %v with waits %v; want "+
|
|
||||||
"held under db on anchor as unhealthy, no waits", held, by.provider, by.waits)
|
|
||||||
}
|
|
||||||
if _, uncovered := given.waitingUncovered("laptop", "shop", []inventory.ResourceHealth{failingConsumer("shop")}); uncovered {
|
|
||||||
t.Fatalf("a provider whose wait failed its check is said as waiting for another part")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,148 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A wait on a clock of its own: each pause moves it on, and nothing waits in real time.
|
|
||||||
func aFakeWait(bound time.Duration) (waitOut, *[]time.Duration) {
|
|
||||||
clock := time.Date(2026, 10, 11, 11, 52, 2, 0, time.UTC)
|
|
||||||
var paused []time.Duration
|
|
||||||
return waitOut{bound: bound, first: time.Second, most: 10 * time.Second,
|
|
||||||
now: func() time.Time { return clock },
|
|
||||||
sleep: func(_ context.Context, d time.Duration) error {
|
|
||||||
paused = append(paused, d)
|
|
||||||
clock = clock.Add(d)
|
|
||||||
return nil
|
|
||||||
}}, &paused
|
|
||||||
}
|
|
||||||
|
|
||||||
// notServed is what the bus says while node-tools re-serves, in the words seen on 2026-10-11.
|
|
||||||
func notServed() error {
|
|
||||||
return fmt.Errorf("cannot ask gitea.gitea_api: %w", link.ErrNothingServes)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A registration's forge question that lands while the tool runner re-serves is asked again, and the forge's
|
|
||||||
// answer, seconds later, is the registration's (novox/hq issue 486).
|
|
||||||
func TestTheForgeNotServedForAMomentIsWaitedOut(t *testing.T) {
|
|
||||||
w, paused := aFakeWait(60 * time.Second)
|
|
||||||
asked := 0
|
|
||||||
facts, err := forgeWaitedOut(t.Context(), w, "novox/mesh-catalog", func(context.Context) (forgeFacts, error) {
|
|
||||||
asked++
|
|
||||||
if asked <= 2 {
|
|
||||||
return forgeFacts{}, notServed()
|
|
||||||
}
|
|
||||||
return forgeFacts{ID: 42, Guarded: true}, nil
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("a forge not served for two asks failed the registration: %v", err)
|
|
||||||
}
|
|
||||||
if facts.ID != 42 || !facts.Guarded || asked != 3 {
|
|
||||||
t.Fatalf("the forge's answer was %+v after %d asks, want its answer on the third", facts, asked)
|
|
||||||
}
|
|
||||||
if want := []time.Duration{time.Second, 2 * time.Second}; fmt.Sprint(*paused) != fmt.Sprint(want) {
|
|
||||||
t.Fatalf("paused %v, want %v", *paused, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A tool that did not answer within its wait has held the receive loop that long already: not asked again.
|
|
||||||
func TestTheForgeNotAnsweringInTimeIsNotAskedAgain(t *testing.T) {
|
|
||||||
w, paused := aFakeWait(60 * time.Second)
|
|
||||||
asked := 0
|
|
||||||
_, err := forgeWaitedOut(t.Context(), w, "novox/x", func(context.Context) (forgeFacts, error) {
|
|
||||||
asked++
|
|
||||||
return forgeFacts{}, fmt.Errorf("gitea did not answer: %w", link.ErrNoAnswer)
|
|
||||||
})
|
|
||||||
if !errors.Is(err, link.ErrNoAnswer) || asked != 1 || len(*paused) != 0 {
|
|
||||||
t.Fatalf("a forge that did not answer in time was asked %d times, paused %v: %v", asked, *paused, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// An ask that hangs past the bound is cut off at the bound, not at its own wait, and the failure still says it
|
|
||||||
// waited: the receive loop is held no longer than the bound (novox/hq issue 486).
|
|
||||||
func TestAnAskHangingPastTheBoundIsCutOffAtIt(t *testing.T) {
|
|
||||||
w := waitOut{bound: 200 * time.Millisecond, first: 10 * time.Millisecond, most: 50 * time.Millisecond,
|
|
||||||
now: time.Now, sleep: sleepOrDone}
|
|
||||||
asked := 0
|
|
||||||
began := time.Now()
|
|
||||||
_, err := forgeWaitedOut(t.Context(), w, "novox/mesh-catalog", func(ctx context.Context) (forgeFacts, error) {
|
|
||||||
asked++
|
|
||||||
if asked == 1 {
|
|
||||||
return forgeFacts{}, notServed()
|
|
||||||
}
|
|
||||||
select { // link.Ask's own wait is 30 s; the bound must end it first
|
|
||||||
case <-ctx.Done():
|
|
||||||
return forgeFacts{}, fmt.Errorf("gitea did not answer: %w", link.ErrNoAnswer)
|
|
||||||
case <-time.After(30 * time.Second):
|
|
||||||
return forgeFacts{ID: 1, Guarded: true}, nil
|
|
||||||
}
|
|
||||||
})
|
|
||||||
if took := time.Since(began); took > 2*time.Second {
|
|
||||||
t.Fatalf("the wait took %s, past its bound of %s", took, w.bound)
|
|
||||||
}
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "waited") || !strings.Contains(err.Error(), "asking 2 times") ||
|
|
||||||
!strings.Contains(err.Error(), "issue 486") {
|
|
||||||
t.Fatalf("the bound's end does not say it waited: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Past the bound the registration fails, saying it waited, for how long, and what the forge last said.
|
|
||||||
func TestTheForgeNotServedPastTheBoundFailsSayingItWaited(t *testing.T) {
|
|
||||||
w, paused := aFakeWait(60 * time.Second)
|
|
||||||
_, err := forgeWaitedOut(t.Context(), w, "novox/mesh-catalog", func(context.Context) (forgeFacts, error) {
|
|
||||||
return forgeFacts{}, notServed()
|
|
||||||
})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("a forge never served was read as an answer")
|
|
||||||
}
|
|
||||||
var waited time.Duration
|
|
||||||
for _, p := range *paused {
|
|
||||||
waited += p
|
|
||||||
}
|
|
||||||
if waited > 60*time.Second || waited < 40*time.Second {
|
|
||||||
t.Fatalf("waited %s in all, want close to and within the bound of 60s", waited)
|
|
||||||
}
|
|
||||||
for _, want := range []string{"waited " + waited.String(), "novox/mesh-catalog", "nothing serves", "issue 486"} {
|
|
||||||
if !strings.Contains(err.Error(), want) {
|
|
||||||
t.Errorf("the failure does not say %q: %v", want, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !errors.Is(err, link.ErrNothingServes) {
|
|
||||||
t.Errorf("the failure lost what the forge last said: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A refusal is the forge's word: never asked again.
|
|
||||||
func TestARefusalFromTheForgeIsNeverAskedAgain(t *testing.T) {
|
|
||||||
w, paused := aFakeWait(60 * time.Second)
|
|
||||||
asked := 0
|
|
||||||
_, err := forgeWaitedOut(t.Context(), w, "novox/x", func(context.Context) (forgeFacts, error) {
|
|
||||||
asked++
|
|
||||||
return forgeFacts{}, errors.New("gitea.gitea_api: 404 Not Found")
|
|
||||||
})
|
|
||||||
if err == nil || asked != 1 || len(*paused) != 0 {
|
|
||||||
t.Fatalf("a refusal was asked %d times, paused %v: %v", asked, *paused, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A registration whose own context ends stops waiting at once.
|
|
||||||
func TestAWaitForTheForgeEndsWithItsContext(t *testing.T) {
|
|
||||||
w, _ := aFakeWait(60 * time.Second)
|
|
||||||
ctx, cancel := context.WithCancel(t.Context())
|
|
||||||
w.sleep = func(ctx context.Context, d time.Duration) error { cancel(); return sleepOrDone(ctx, d) }
|
|
||||||
asked := 0
|
|
||||||
_, err := forgeWaitedOut(ctx, w, "novox/x", func(context.Context) (forgeFacts, error) {
|
|
||||||
asked++
|
|
||||||
return forgeFacts{}, notServed()
|
|
||||||
})
|
|
||||||
if !errors.Is(err, context.Canceled) || asked != 1 {
|
|
||||||
t.Fatalf("asked %d times after the context ended: %v", asked, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+18
-143
@@ -27,10 +27,8 @@ import (
|
|||||||
// **"Reported applied" is not enough.** ADR 0218 sent a module to one machine first and the rest once
|
// **"Reported applied" is not enough.** ADR 0218 sent a module to one machine first and the rest once
|
||||||
// that machine reported it applied. A build that applies and then does nothing, crashes, serves no
|
// that machine reported it applied. A build that applies and then does nothing, crashes, serves no
|
||||||
// tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by
|
// tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by
|
||||||
// the component's health — the core's health definitions, or a module's own — within ten minutes of the
|
// the component's health — the core's health definitions, or a module's own — passing three times over
|
||||||
// apply: a core module passing three times over at least two minutes, a leaf module once, as soon as the
|
// at least two minutes, within ten minutes of the apply. Only then are the rest sent.
|
||||||
// node-engine states it healthy (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it). Only then
|
|
||||||
// are the rest sent.
|
|
||||||
//
|
//
|
||||||
// **A failing gate stops the plan and puts the previous build back there.** The build is marked failed
|
// **A failing gate stops the plan and puts the previous build back there.** The build is marked failed
|
||||||
// at its gate — registration refuses it and nothing sends it again on its own — the module's registered
|
// at its gate — registration refuses it and nothing sends it again on its own — the module's registered
|
||||||
@@ -45,38 +43,11 @@ var (
|
|||||||
gateSettle = 2 * time.Minute
|
gateSettle = 2 * time.Minute
|
||||||
// gateBound is how long after the apply the build may take to become healthy.
|
// gateBound is how long after the apply the build may take to become healthy.
|
||||||
gateBound = 10 * time.Minute
|
gateBound = 10 * time.Minute
|
||||||
// gatePasses is how many consecutive judgings must find a core module healthy, gateEvery apart at the
|
// gatePasses is how many consecutive judgings must find it healthy, gateEvery apart at the least.
|
||||||
// least, after gateSettle. gateLeafPasses is a leaf module's: one, taken once the node-engine states every
|
|
||||||
// resource of it healthy by its own check, with no settle (novox/hq ADR 0282 decision 4 as ADR 0304
|
|
||||||
// decision 3 changed it, issue 524).
|
|
||||||
gatePasses = 3
|
gatePasses = 3
|
||||||
gateLeafPasses = 1
|
|
||||||
gateLeafSettles = false
|
|
||||||
gateEvery = 40 * time.Second
|
gateEvery = 40 * time.Second
|
||||||
)
|
)
|
||||||
|
|
||||||
// coreModule is whether a module is judged as the core (novox/hq ADR 0282 decision 1): one on the controller's
|
|
||||||
// own path, or one holding the mesh's resolver. Every other module is a leaf.
|
|
||||||
func coreModule(module string, m catalogue.Manifest) bool {
|
|
||||||
if _, own := onTheControllersPath[module]; own {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
for _, c := range m.Claims {
|
|
||||||
if c.Name == resolverSeat {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return coreComponent(module) != ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// gateWants is how many passes a module needs, and whether it must stay healthy gateSettle first.
|
|
||||||
func gateWants(module string, m catalogue.Manifest) (passes int, settles bool) {
|
|
||||||
if coreModule(module, m) {
|
|
||||||
return gatePasses, true
|
|
||||||
}
|
|
||||||
return gateLeafPasses, gateLeafSettles
|
|
||||||
}
|
|
||||||
|
|
||||||
// gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions
|
// gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions
|
||||||
// are raised by a plan as it judges, and kept or cleared by the probe on every run.
|
// are raised by a plan as it judges, and kept or cleared by the probe on every run.
|
||||||
const gateProbe = "DG"
|
const gateProbe = "DG"
|
||||||
@@ -150,13 +121,10 @@ type gateFacts struct {
|
|||||||
health map[string]inventory.NodeHealth
|
health map[string]inventory.NodeHealth
|
||||||
healthErr error
|
healthErr error
|
||||||
// heldOn is, per "<module>@<machine>", the provider its findings are held under (ADR 0240 rule 5).
|
// heldOn is, per "<module>@<machine>", the provider its findings are held under (ADR 0240 rule 5).
|
||||||
heldOn map[string]heldReading
|
heldOn map[string]string
|
||||||
// groupsAdded is, per module, whether the move judged puts an account in a group its previous build did
|
// groupsAdded is, per module, whether the move judged puts an account in a group its previous build did
|
||||||
// not (issue 318 review): the only move whose wait for a new login is excused.
|
// not (issue 318 review): the only move whose wait for a new login is excused.
|
||||||
groupsAdded map[string]bool
|
groupsAdded map[string]bool
|
||||||
// waits is what the controller holds to check a module's wait for the operator (novox/hq ADR 0283): the
|
|
||||||
// manifest of each module's build judged, and the secrets given on each machine.
|
|
||||||
waits operatorWaitFacts
|
|
||||||
// sent is, per machine, the declaration the gate's own send carried there (novox/hq issue 352): a
|
// sent is, per machine, the declaration the gate's own send carried there (novox/hq issue 352): a
|
||||||
// report is held against it, never against the send made last. sentBuilds is what each machine was
|
// report is held against it, never against the send made last. sentBuilds is what each machine was
|
||||||
// last sent of every module, and judged the commit of each module this gate judges: a machine last
|
// last sent of every module, and judged the commit of each module this gate judges: a machine last
|
||||||
@@ -166,29 +134,6 @@ type gateFacts struct {
|
|||||||
commits map[string]string
|
commits map[string]string
|
||||||
}
|
}
|
||||||
|
|
||||||
// heldReading is the provider a module's findings are held under, as "<module> on <machine>", and, when that
|
|
||||||
// provider only waits for the operator for the part the module needs, its wait in one sentence (novox/hq issue
|
|
||||||
// 405): the module's gate then reads as a wait for a person (ADR 0254), a pass carrying the wait, as ADR 0283
|
|
||||||
// decision 4 reads the waiting provider itself. A walk never waits on the operator's secret, there or here.
|
|
||||||
type heldReading struct {
|
|
||||||
on, waits string
|
|
||||||
}
|
|
||||||
|
|
||||||
// heldReadings is, per "<module>@<machine>" in every machine's newest statement, what its findings are held under.
|
|
||||||
func heldReadings(hold *holding) map[string]heldReading {
|
|
||||||
out := map[string]heldReading{}
|
|
||||||
for machine := range hold.healths {
|
|
||||||
for module, by := range hold.heldModules(machine) {
|
|
||||||
reading := heldReading{on: by.provider.Module + " on " + by.provider.Node}
|
|
||||||
if len(by.waits) > 0 {
|
|
||||||
reading.waits = operatorWaitSaid(by.provider.Module, by.provider.Node, by.waits)
|
|
||||||
}
|
|
||||||
out[module+"@"+machine] = reading
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// reportedOn says a machine's last report is on what the gate sent it (novox/hq issue 352): on that
|
// reportedOn says a machine's last report is on what the gate sent it (novox/hq issue 352): on that
|
||||||
// declaration, or one it was sent after it — or, for a gate kept before sends were kept on it, on the
|
// declaration, or one it was sent after it — or, for a gate kept before sends were kept on it, on the
|
||||||
// declaration last sent. On 2026-10-09 a release's gate read the control node's report against a newer
|
// declaration last sent. On 2026-10-09 a release's gate read the control node's report against a newer
|
||||||
@@ -254,7 +199,12 @@ var gatherGateFacts = func(ctx context.Context, open *stores, component string)
|
|||||||
// Whose findings wait on an unhealthy provider (ADR 0240 rule 5): their gates wait, not fail.
|
// Whose findings wait on an unhealthy provider (ADR 0240 rule 5): their gates wait, not fail.
|
||||||
if f.healthErr == nil && f.openErr == nil {
|
if f.healthErr == nil && f.openErr == nil {
|
||||||
if hold, err := readHolding(ctx, inv, f.open); err == nil {
|
if hold, err := readHolding(ctx, inv, f.open); err == nil {
|
||||||
f.heldOn = heldReadings(hold)
|
f.heldOn = map[string]string{}
|
||||||
|
for machine := range f.health {
|
||||||
|
for module, p := range hold.heldModules(machine) {
|
||||||
|
f.heldOn[module+"@"+machine] = p.Module + " on " + p.Node
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if theLease != nil {
|
if theLease != nil {
|
||||||
@@ -307,11 +257,10 @@ func judgeHealth(module, component string, m catalogue.Manifest, machine string,
|
|||||||
firstLine(f.openErr.Error())
|
firstLine(f.openErr.Error())
|
||||||
}
|
}
|
||||||
for _, c := range f.open {
|
for _, c := range f.open {
|
||||||
// A wait for a person's new login, for a directory used as found to be handed over, or for the
|
// A wait for a person's new login, or for a directory used as found to be handed over, is the module's
|
||||||
// operator's secret or setting, is the module's reading, not a fault raised since the send: the gate
|
// reading, not a fault raised since the send: the gate reads it from the statement below (ADR 0254,
|
||||||
// reads it from the statement below (ADR 0254, novox/hq issue 339, ADR 0283).
|
// novox/hq issue 339).
|
||||||
if c.Source == gateProbe || c.OpenAt(since) || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound ||
|
if c.Source == gateProbe || c.OpenAt(since) || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound {
|
||||||
c.Kind == kindNeedsOperator {
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
onIt := c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) ||
|
onIt := c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) ||
|
||||||
@@ -528,7 +477,6 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
|||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
facts.groupsAdded = movesAddingGroups(ctx, open.inventory, g, pairs, shelf)
|
facts.groupsAdded = movesAddingGroups(ctx, open.inventory, g, pairs, shelf)
|
||||||
facts.waits = gateWaitFacts(ctx, open.inventory, g, pairs, shelf, facts.health)
|
|
||||||
facts.sent = g.Sent
|
facts.sent = g.Sent
|
||||||
facts.commits, facts.sentBuilds = judgedCommits(g, pairs), map[string]map[string]string{}
|
facts.commits, facts.sentBuilds = judgedCommits(g, pairs), map[string]map[string]string{}
|
||||||
// A module this gate put back at once (putBackBroken) was sent its earlier build by the gate itself:
|
// A module this gate put back at once (putBackBroken) was sent its earlier build by the gate itself:
|
||||||
@@ -637,24 +585,8 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
settled := now.Sub(*g.Since) >= gateSettle
|
settled := now.Sub(*g.Since) >= gateSettle
|
||||||
// What the send needs is what its most demanding module needs: a send carrying a core module is judged
|
|
||||||
// as the core.
|
|
||||||
wants, settles := 0, false
|
|
||||||
for _, m := range modules {
|
|
||||||
p, s := gateWants(m, shelf[m])
|
|
||||||
wants, settles = max(wants, p), settles || s
|
|
||||||
}
|
|
||||||
if len(modules) == 0 {
|
|
||||||
// A send judged on no machine of its modules (its first machine not running them) has no class to
|
|
||||||
// be judged by: it is judged as the core, never passed at once without a reading.
|
|
||||||
wants, settles = gatePasses, true
|
|
||||||
}
|
|
||||||
g.Wants = wants
|
|
||||||
// A module's own pass is its own class's: a leaf beside a core module that fails keeps its build after
|
|
||||||
// its one reading, and only what did not pass is put back (issue 318's rule, by class since issue 524).
|
|
||||||
passedAlone := func(m string) bool {
|
passedAlone := func(m string) bool {
|
||||||
p, s := gateWants(m, shelf[m])
|
return settled && reading[m] <= healthPerson && g.Healthy[m] >= gatePasses
|
||||||
return (settled || !s) && reading[m] <= healthPerson && g.Healthy[m] >= p
|
|
||||||
}
|
}
|
||||||
// fail decides the send failed: what passed on its own keeps its pass, everything else is put back.
|
// fail decides the send failed: what passed on its own keeps its pass, everything else is put back.
|
||||||
fail := func(why string) {
|
fail := func(why string) {
|
||||||
@@ -671,7 +603,6 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
|||||||
pastBound := now.Sub(*g.Since) > gateBound
|
pastBound := now.Sub(*g.Since) > gateBound
|
||||||
switch {
|
switch {
|
||||||
case worst == healthBroken:
|
case worst == healthBroken:
|
||||||
g.Read(now, false, g.BrokenWhy)
|
|
||||||
var judging []string
|
var judging []string
|
||||||
for _, m := range modules {
|
for _, m := range modules {
|
||||||
if reading[m] != healthBroken && !passedAlone(m) {
|
if reading[m] != healthBroken && !passedAlone(m) {
|
||||||
@@ -690,10 +621,8 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
|||||||
// Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either —
|
// Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either —
|
||||||
// the provider's own condition says what is wrong (ADR 0240 rule 5).
|
// the provider's own condition says what is wrong (ADR 0240 rule 5).
|
||||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
||||||
g.Read(now, false, why)
|
|
||||||
case worst == healthNotYet:
|
case worst == healthNotYet:
|
||||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
||||||
g.Read(now, false, why)
|
|
||||||
if pastBound {
|
if pastBound {
|
||||||
fail(fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why))
|
fail(fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why))
|
||||||
}
|
}
|
||||||
@@ -701,8 +630,7 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
|||||||
// Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict.
|
// Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict.
|
||||||
g.Passes++
|
g.Passes++
|
||||||
g.LastPass, g.Last, g.Failing = &now, "", nil
|
g.LastPass, g.Last, g.Failing = &now, "", nil
|
||||||
g.Read(now, true, "")
|
if g.Passes >= gatePasses && settled {
|
||||||
if g.Passes >= wants && (settled || !settles) {
|
|
||||||
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
|
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
|
||||||
now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now)
|
now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now)
|
||||||
}
|
}
|
||||||
@@ -819,47 +747,6 @@ func movesAddingGroups(ctx context.Context, inv *inventory.Inventory, g *invento
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// gateWaitFacts reads what checks the waits for the operator of the modules a gate judges (novox/hq ADR 0283): the
|
|
||||||
// manifest of the build judged — the one it moves to, else the catalogue's — and the secrets given on each machine
|
|
||||||
// that says a module of them waits.
|
|
||||||
func gateWaitFacts(ctx context.Context, inv *inventory.Inventory, g *inventory.PlanGate, pairs []judged,
|
|
||||||
shelf map[string]catalogue.Manifest, health map[string]inventory.NodeHealth) operatorWaitFacts {
|
|
||||||
var f operatorWaitFacts
|
|
||||||
judgedManifests := map[string]catalogue.Manifest{}
|
|
||||||
byMachine := map[string][]string{}
|
|
||||||
for _, j := range pairs {
|
|
||||||
waiting := false
|
|
||||||
for _, r := range health[j.node].Resources {
|
|
||||||
if r.Module == j.module && r.State == link.StateWaiting {
|
|
||||||
waiting = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !waiting {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
byMachine[j.node] = append(byMachine[j.node], j.module)
|
|
||||||
if _, done := judgedManifests[j.module]; done {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
to := g.To
|
|
||||||
for _, c := range g.Carried {
|
|
||||||
if c.Module == j.module {
|
|
||||||
to = c.To
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if m, found, err := inv.ManifestAt(ctx, j.module, to); err == nil && found {
|
|
||||||
judgedManifests[j.module] = m
|
|
||||||
} else if m, known := shelf[j.module]; known {
|
|
||||||
judgedManifests[j.module] = m
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for machine, modules := range byMachine {
|
|
||||||
readWaitFacts(ctx, inv, machine, modules, judgedManifests, &f)
|
|
||||||
}
|
|
||||||
return f
|
|
||||||
}
|
|
||||||
|
|
||||||
// decide sets a gate's verdict.
|
// decide sets a gate's verdict.
|
||||||
func decide(g *inventory.PlanGate, verdict, why string, now time.Time) {
|
func decide(g *inventory.PlanGate, verdict, why string, now time.Time) {
|
||||||
g.Verdict, g.Why, g.JudgedAt = verdict, why, &now
|
g.Verdict, g.Why, g.JudgedAt = verdict, why, &now
|
||||||
@@ -1385,9 +1272,9 @@ func gateLine(g *inventory.PlanGate) string {
|
|||||||
if g.Why != "" {
|
if g.Why != "" {
|
||||||
line += ": " + g.Why
|
line += ": " + g.Why
|
||||||
} else if g.Last != "" {
|
} else if g.Last != "" {
|
||||||
line += fmt.Sprintf(" (%d of %d passes; wanting: %s)", g.Passes, gateWanted(g), g.Last)
|
line += fmt.Sprintf(" (%d of %d passes; wanting: %s)", g.Passes, gatePasses, g.Last)
|
||||||
} else if g.Verdict == "" {
|
} else if g.Verdict == "" {
|
||||||
line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gateWanted(g))
|
line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gatePasses)
|
||||||
}
|
}
|
||||||
if g.Verdict == "" && len(g.Waits) > 0 {
|
if g.Verdict == "" && len(g.Waits) > 0 {
|
||||||
line += waitsSaid(g.Waits)
|
line += waitsSaid(g.Waits)
|
||||||
@@ -1398,10 +1285,6 @@ func gateLine(g *inventory.PlanGate) string {
|
|||||||
if g.Rollback != "" {
|
if g.Rollback != "" {
|
||||||
line += "; " + g.Rollback
|
line += "; " + g.Rollback
|
||||||
}
|
}
|
||||||
// Before the send it judges (novox/hq issue 490).
|
|
||||||
if said := grantsSaid(g.Grants); said != "" {
|
|
||||||
line += "; " + said
|
|
||||||
}
|
|
||||||
return line
|
return line
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1430,11 +1313,3 @@ func witnessObservation(node string, r lease.Rollback) conditions.Observation {
|
|||||||
Summary: fmt.Sprintf("the witness on %s judged the %s %s not healthy %s at %s: %s", node, r.Component,
|
Summary: fmt.Sprintf("the witness on %s judged the %s %s not healthy %s at %s: %s", node, r.Component,
|
||||||
short(r.From), what, r.At.UTC().Format(time.RFC3339), r.Why)}
|
short(r.From), what, r.At.UTC().Format(time.RFC3339), r.Why)}
|
||||||
}
|
}
|
||||||
|
|
||||||
// gateWanted is the passes a gate's send needs, three where the gate was judged before it was said.
|
|
||||||
func gateWanted(g *inventory.PlanGate) int {
|
|
||||||
if g.Wants > 0 {
|
|
||||||
return g.Wants
|
|
||||||
}
|
|
||||||
return gatePasses
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -53,7 +53,6 @@ func healthyContainer(module string) inventory.ResourceHealth {
|
|||||||
// tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass,
|
// tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass,
|
||||||
// counted over the gate's spacing — before it says its own. The broken one alone is put back.
|
// counted over the gate's spacing — before it says its own. The broken one alone is put back.
|
||||||
func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) {
|
func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) {
|
||||||
judgeLeavesAsTheCore(t)
|
|
||||||
open := aMesh(t)
|
open := aMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
withGateBounds(t, 150*time.Millisecond, 20*time.Millisecond, 10*time.Second)
|
withGateBounds(t, 150*time.Millisecond, 20*time.Millisecond, 10*time.Second)
|
||||||
@@ -285,7 +284,6 @@ func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) {
|
|||||||
// tools are marked and put back in the very judging that found them broken — their registered build is the
|
// tools are marked and put back in the very judging that found them broken — their registered build is the
|
||||||
// one before, and the laptop is sent it — while `app` goes on being judged, recovers, and keeps its own pass.
|
// one before, and the laptop is sent it — while `app` goes on being judged, recovers, and keeps its own pass.
|
||||||
func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testing.T) {
|
func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testing.T) {
|
||||||
judgeLeavesAsTheCore(t)
|
|
||||||
open := aMesh(t)
|
open := aMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
|
|||||||
@@ -153,7 +153,6 @@ func (g *gateMesh) plan(t *testing.T) inventory.Plan {
|
|||||||
// registered and sent to it again — and never reaches the second machine; it is marked, said as a
|
// registered and sent to it again — and never reaches the second machine; it is marked, said as a
|
||||||
// condition and an event, never registered or rolled back again.
|
// condition and an event, never registered or rolled back again.
|
||||||
func TestABuildThatFailsItsGateIsRolledBackOnItsFirstMachineAndGoesNoFurther(t *testing.T) {
|
func TestABuildThatFailsItsGateIsRolledBackOnItsFirstMachineAndGoesNoFurther(t *testing.T) {
|
||||||
judgeLeavesAsTheCore(t)
|
|
||||||
g := aGateMesh(t)
|
g := aGateMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
inv := g.open.inventory
|
inv := g.open.inventory
|
||||||
@@ -256,7 +255,7 @@ func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) {
|
|||||||
t.Fatalf("sent %v", g.sent)
|
t.Fatalf("sent %v", g.sent)
|
||||||
}
|
}
|
||||||
gate := p.Modules["app"].Gate
|
gate := p.Modules["app"].Gate
|
||||||
if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gateWanted(gate) {
|
if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gatePasses {
|
||||||
t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate)
|
t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate)
|
||||||
}
|
}
|
||||||
if v, found, err := g.open.inventory.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
if v, found, err := g.open.inventory.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||||
@@ -267,52 +266,8 @@ func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A leaf module passes its gate on one reading, as soon as its first machine is healthy, without waiting out the
|
|
||||||
// core's settle time (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it, issue 524).
|
|
||||||
func TestALeafModulePassesOnOneHealthyReading(t *testing.T) {
|
|
||||||
g := aGateMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
gateSettle = time.Hour // a core module could not pass for an hour
|
|
||||||
advancePlans(ctx, g.open) // the first machine is sent the new build
|
|
||||||
advancePlans(ctx, g.open) // one healthy judging
|
|
||||||
p := g.plan(t)
|
|
||||||
gate := p.Modules["app"].Gate
|
|
||||||
if gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes != 1 || gate.Wants != 1 {
|
|
||||||
t.Fatalf("a leaf module's gate after one healthy judging: %+v (plan %s: %s)", gate, p.State, p.Note)
|
|
||||||
}
|
|
||||||
if !strings.Contains(gate.Why, "healthy 1 times") {
|
|
||||||
t.Errorf("the verdict says %q", gate.Why)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Which modules are judged as the core: those on the controller's own path and the resolver's holder, three
|
|
||||||
// readings after the settle time; every other module one (novox/hq ADR 0282 decision 1, ADR 0304 decision 3).
|
|
||||||
func TestAModuleIsJudgedByItsClass(t *testing.T) {
|
|
||||||
for _, c := range []struct {
|
|
||||||
module string
|
|
||||||
claims []catalogue.Claim
|
|
||||||
passes int
|
|
||||||
settles bool
|
|
||||||
}{
|
|
||||||
{"app", nil, 1, false},
|
|
||||||
{"nzbget", nil, 1, false},
|
|
||||||
{"mesh-controller", nil, 3, true},
|
|
||||||
{"mesh-host", nil, 3, true},
|
|
||||||
{"node-tools", nil, 3, true},
|
|
||||||
{"nats", nil, 3, true},
|
|
||||||
{catalogue.DeliverySeat, nil, 3, true},
|
|
||||||
{"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, true},
|
|
||||||
} {
|
|
||||||
p, s := gateWants(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims})
|
|
||||||
if p != c.passes || s != c.settles {
|
|
||||||
t.Errorf("%s: %d passes, settles %v; want %d, %v", c.module, p, s, c.passes, c.settles)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A first machine that never becomes healthy fails its gate at the bound, and is put back.
|
// A first machine that never becomes healthy fails its gate at the bound, and is put back.
|
||||||
func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) {
|
func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) {
|
||||||
judgeLeavesAsTheCore(t)
|
|
||||||
g := aGateMesh(t)
|
g := aGateMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
advancePlans(ctx, g.open)
|
advancePlans(ctx, g.open)
|
||||||
@@ -638,13 +593,3 @@ func TestAWitnessesVerdictIsAConditionWhileItsReportsSayIt(t *testing.T) {
|
|||||||
t.Fatalf("a verdict the reports no longer carry is still said: %+v %v", obs, err)
|
t.Fatalf("a verdict the reports no longer carry is still said: %+v %v", obs, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// judgeLeavesAsTheCore makes a test's leaf modules judged as a core module is, three readings after the settle
|
|
||||||
// time, for a test of the gate's other rules (a fault, the bound, a supersession) that needs a judging still
|
|
||||||
// open after a healthy reading.
|
|
||||||
func judgeLeavesAsTheCore(t *testing.T) {
|
|
||||||
t.Helper()
|
|
||||||
wasPasses, wasSettles := gateLeafPasses, gateLeafSettles
|
|
||||||
gateLeafPasses, gateLeafSettles = gatePasses, true
|
|
||||||
t.Cleanup(func() { gateLeafPasses, gateLeafSettles = wasPasses, wasSettles })
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,300 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The assignment generation a declaration was composed from, and the record of every send (novox/hq
|
|
||||||
// issue 234).
|
|
||||||
//
|
|
||||||
// On 2026-10-04 a declaration newer in sequence than every other named four fewer modules than the
|
|
||||||
// assignments held, and a machine applied it and undeclared all four. The sequence orders arrival and
|
|
||||||
// cannot tell a later send that carries an older view of the assignments. So a declaration now carries
|
|
||||||
// the generation of the assignments it was composed from — a counter the store raises in the same
|
|
||||||
// transaction as every assignment change — and a machine refuses one older than it applied, unless it is
|
|
||||||
// a gate's put-back. And every send is written down with who sent it, from which generation and naming
|
|
||||||
// which modules: the controller logged no send then, and which process sent the stale declaration could
|
|
||||||
// not be read back from anything the mesh kept.
|
|
||||||
|
|
||||||
// kindOlderGeneration is S20's kind: a machine refused a send for the generation it was composed from.
|
|
||||||
const kindOlderGeneration = "older-generation"
|
|
||||||
|
|
||||||
// generationRefusalsSaid is how long a refused send is said after its refusal: an hour, as an advisory is.
|
|
||||||
const generationRefusalsSaid = advisoryQuiet
|
|
||||||
|
|
||||||
// stamp gives a composed declaration the order allotted to it before it was composed: its sequence, and
|
|
||||||
// the epoch and generation when the machine reads them — and keeps the generation and acting epoch for the
|
|
||||||
// record of the send whether or not the machine is sent them.
|
|
||||||
//
|
|
||||||
// **No put-back mark is sent.** A gate puts a machine back by composing it again (sendRollout), so its
|
|
||||||
// declaration is allotted here like any other and carries the generation as it stands — never older than
|
|
||||||
// what the machine applied. The node-engine reads a `put_back` key (mesh-host#82); this controller never
|
|
||||||
// needs to send one.
|
|
||||||
func (o order) stamp(d *sendable) {
|
|
||||||
d.Sequence, d.Epoch = o.sequence, o.epoch
|
|
||||||
d.composedFrom, d.actingEpoch = o.generation, o.acting
|
|
||||||
d.Generation = 0
|
|
||||||
if o.readsGeneration && o.generation > 0 {
|
|
||||||
d.Generation = o.generation
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sentRecord is what the record of a send keeps beside its digest.
|
|
||||||
type sentRecord struct {
|
|
||||||
sequence int64
|
|
||||||
epoch uint64
|
|
||||||
generation int64
|
|
||||||
// toldGeneration is the generation on the wire, which the would-send is stamped with: zero for a machine
|
|
||||||
// whose node-engine has not said it reads one.
|
|
||||||
toldGeneration int64
|
|
||||||
sender string
|
|
||||||
modules []string
|
|
||||||
}
|
|
||||||
|
|
||||||
// sentRecordOf is a composed send's record: its sender is the caller this process acts for.
|
|
||||||
func sentRecordOf(ctx context.Context, d sendable) sentRecord {
|
|
||||||
return sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom,
|
|
||||||
toldGeneration: d.Generation, sender: senderOf(callerOf(ctx)), modules: d.modules}
|
|
||||||
}
|
|
||||||
|
|
||||||
// callerOf is who asked for what this process does: the seat call's caller when ctx belongs to one, the
|
|
||||||
// caller the controller ran this command for, or the account at the shell.
|
|
||||||
func callerOf(ctx context.Context) string {
|
|
||||||
if c := link.CallerIn(ctx); c != "" {
|
|
||||||
return c
|
|
||||||
}
|
|
||||||
return link.Caller()
|
|
||||||
}
|
|
||||||
|
|
||||||
// senderOf is a send's sender in words: who asked, and the process and build that composed it — the
|
|
||||||
// answer issue 234 could not find, because two controllers and a one-shot push were all sending then.
|
|
||||||
func senderOf(caller string) string {
|
|
||||||
host, _ := os.Hostname()
|
|
||||||
return fmt.Sprintf("%s (pid %d on %s, build %s)", caller, os.Getpid(), host, version)
|
|
||||||
}
|
|
||||||
|
|
||||||
// namedModules are the modules a declaration names: its machine's set, less what was left out of it.
|
|
||||||
func namedModules(plan catalogue.Resolution, leftOut map[string]string) []string {
|
|
||||||
out := make([]string, 0, len(plan.Modules))
|
|
||||||
for _, m := range plan.Modules {
|
|
||||||
if _, left := leftOut[m.Module]; !left {
|
|
||||||
out = append(out, m.Module)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.Strings(out)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// heardGenerationRefusal keeps a machine's refusal of a send for its generation, so S20 names its sender
|
|
||||||
// and says it (novox/hq issue 234): on the send it refused or — when the mesh has no record of sending that
|
|
||||||
// sequence — as a refusal of its own, naming the sender as unknown. Either way S20 is raised: a refusal the
|
|
||||||
// mesh cannot attribute is the louder fact, not a quieter one.
|
|
||||||
//
|
|
||||||
// **And raises the mesh's counter past what the machine applied, when the refused send was composed from
|
|
||||||
// the counter as it stands** (counterBehind): then the sender's view was not stale — the counter is behind
|
|
||||||
// the machine, which is a store put back from a backup — and every send after would be refused for ever. A
|
|
||||||
// stale sender's generation is below the counter, and the counter is left alone for it.
|
|
||||||
//
|
|
||||||
// Nothing is sent from here. This runs in the controller's receive loop, and a push from it would hold that
|
|
||||||
// loop, and the machine's hold, for as long as the push takes — the deaf controller of issues 184 and 185.
|
|
||||||
// S20 names `push <node>` for that case instead.
|
|
||||||
func heardGenerationRefusal(ctx context.Context, inv *inventory.Inventory, report link.Report) {
|
|
||||||
r := report.OlderGeneration
|
|
||||||
if r == nil || inv == nil || report.Node == "" {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
node, err := inv.NodeByName(ctx, report.Node)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the machine cannot be "+
|
|
||||||
"read, so it is not raised: %v\n", report.Node, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var raised int64
|
|
||||||
if now, err := inv.AssignmentGeneration(ctx); err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the mesh's own cannot be "+
|
|
||||||
"read: %v\n", report.Node, err)
|
|
||||||
} else if counterBehind(*r, now) {
|
|
||||||
if raised, err = inv.RaiseAssignmentGeneration(ctx, r.Applied); err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation (%d) is behind what %s applied (%d), "+
|
|
||||||
"and could not be raised: %v\n", now, report.Node, r.Applied, err)
|
|
||||||
raised = 0
|
|
||||||
} else {
|
|
||||||
fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation was %d, behind what %s applied (%d) — "+
|
|
||||||
"a store put back from a backup — and is raised to %d; `push %s` sends it what the mesh holds now\n",
|
|
||||||
now, report.Node, r.Applied, raised, report.Node)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
send, found, err := inv.RefusedSend(ctx, node.ID, report.Sequence, r.Applied, raised)
|
|
||||||
if err == nil && !found {
|
|
||||||
send, err = inv.RecordUnrecordedRefusal(ctx, inventory.Send{Node: node.ID, Sequence: report.Sequence,
|
|
||||||
Epoch: report.Epoch, Generation: r.Generation, Digest: report.Declared, RefusedApplied: r.Applied,
|
|
||||||
CounterRaisedTo: raised})
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d for its generation, and the refusal could not be "+
|
|
||||||
"kept, so it is not raised: %v\n", report.Node, report.Sequence, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d from %s: composed from assignment generation %d, "+
|
|
||||||
"and it applied %d\n", report.Node, report.Sequence, send.Sender, r.Generation, r.Applied)
|
|
||||||
}
|
|
||||||
|
|
||||||
// counterBehind says a refusal shows the mesh's counter behind the machine rather than a stale sender: the
|
|
||||||
// refused send carried the counter as it stands now, and the machine applied more than that.
|
|
||||||
func counterBehind(r link.GenerationRefusal, now int64) bool {
|
|
||||||
return r.Generation >= now && r.Applied > r.Generation
|
|
||||||
}
|
|
||||||
|
|
||||||
// watchGenerationRefusals is S20: every send a machine refused within the hour for the generation it was
|
|
||||||
// composed from, naming who sent it (novox/hq issue 234). One per machine, the newest refusal.
|
|
||||||
func watchGenerationRefusals(f *signalFacts) []conditions.Observation {
|
|
||||||
seen := map[string]bool{}
|
|
||||||
var out []conditions.Observation
|
|
||||||
for _, s := range f.refusedSends {
|
|
||||||
if s.RefusedAt == nil || f.now.Sub(*s.RefusedAt) > generationRefusalsSaid || seen[s.NodeName] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
seen[s.NodeName] = true
|
|
||||||
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: s.NodeName, Kind: kindOlderGeneration,
|
|
||||||
Machine: s.NodeName, Severity: conditions.Warning,
|
|
||||||
Summary: fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, and "+
|
|
||||||
"%s applied generation %d — a sender composing from a view of the assignments the mesh has moved "+
|
|
||||||
"past; it named %s", s.NodeName, s.Sequence, s.Sender, s.Generation, s.NodeName, s.RefusedApplied,
|
|
||||||
modulesWords(s.Modules)),
|
|
||||||
Said: fmt.Sprintf("refused at %s", s.RefusedAt.UTC().Format(time.RFC3339)),
|
|
||||||
Headline: fmt.Sprintf("%s refused an out-of-date update", s.NodeName),
|
|
||||||
Explanation: fmt.Sprintf("Something sent %s an update made from an older list of what runs there. %s "+
|
|
||||||
"refused it, so nothing was removed. Nothing for you to do unless it repeats.", s.NodeName, s.NodeName),
|
|
||||||
Resolved: fmt.Sprintf("%s has had no out-of-date update for an hour", s.NodeName)}
|
|
||||||
if s.CounterRaisedTo > 0 {
|
|
||||||
// Not a stale sender: the mesh's counter was behind the machine (a store put back from a backup) and
|
|
||||||
// was raised; nothing has sent the machine its declaration since, so a person is asked to.
|
|
||||||
o.Summary = fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, "+
|
|
||||||
"the mesh's own, and %s applied generation %d — the mesh's counter was behind the machine (a store "+
|
|
||||||
"put back from a backup?) and is raised to %d; `push %s` sends it what the mesh holds now",
|
|
||||||
s.NodeName, s.Sequence, s.Sender, s.Generation, s.NodeName, s.RefusedApplied, s.CounterRaisedTo,
|
|
||||||
s.NodeName)
|
|
||||||
o.Explanation = fmt.Sprintf("Needs you: push %s. The mesh's record was older than %s, so %s refused its "+
|
|
||||||
"update and kept what it had. The record is repaired; a push sends the update again.",
|
|
||||||
s.NodeName, s.NodeName, s.NodeName)
|
|
||||||
}
|
|
||||||
out = append(out, o)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// modulesWords is a send's modules as a sentence says them.
|
|
||||||
func modulesWords(modules []string) string {
|
|
||||||
if len(modules) == 0 {
|
|
||||||
return "no module the mesh recorded"
|
|
||||||
}
|
|
||||||
return strings.Join(modules, ", ")
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234), and
|
|
||||||
// what it answered (issue 485): nothing when the mesh has not recorded a send to it.
|
|
||||||
func writeLastSend(ctx context.Context, w io.Writer, inv *inventory.Inventory, node string) error {
|
|
||||||
s, found, err := inv.LastSend(ctx, node)
|
|
||||||
if err != nil || !found {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
fmt.Fprintf(w, "%s:\n", node)
|
|
||||||
for _, line := range lastSendLines(s) {
|
|
||||||
fmt.Fprintln(w, line)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// lastSendLines is a machine's last send in full, as `node show` and `plan` say it (novox/hq issue 485): when,
|
|
||||||
// by whom, under which lease epoch, from which assignment generation, naming which modules, and what the
|
|
||||||
// machine answered of it — the refusal in its own words.
|
|
||||||
func lastSendLines(s inventory.Send) []string {
|
|
||||||
const stamp = "2006-01-02 15:04:05"
|
|
||||||
epoch := "under no lease epoch"
|
|
||||||
if s.Epoch > 0 {
|
|
||||||
epoch = fmt.Sprintf("under lease epoch %d", s.Epoch)
|
|
||||||
}
|
|
||||||
generation := "no assignment generation recorded"
|
|
||||||
if s.Generation > 0 {
|
|
||||||
generation = fmt.Sprintf("assignment generation %d", s.Generation)
|
|
||||||
}
|
|
||||||
lines := []string{
|
|
||||||
fmt.Sprintf(" last sent sequence %d at %s", s.Sequence, s.SentAt.Local().Format(stamp)),
|
|
||||||
fmt.Sprintf(" by %s", s.Sender),
|
|
||||||
fmt.Sprintf(" %s, composed from %s", epoch, generation),
|
|
||||||
fmt.Sprintf(" naming %s", modulesWords(s.Modules)),
|
|
||||||
}
|
|
||||||
answer := func(words string) { lines = append(lines, " its answer "+words) }
|
|
||||||
switch a := s.Answer; {
|
|
||||||
case s.RefusedAt != nil:
|
|
||||||
// Before the report: a refusal for the generation is kept on the send itself and is the machine's
|
|
||||||
// answer to this send in particular.
|
|
||||||
answer(fmt.Sprintf("refused it at %s: it had applied assignment generation %d, newer than this one",
|
|
||||||
s.RefusedAt.Local().Format(stamp), s.RefusedApplied))
|
|
||||||
case a.Outcome == "" || a.At == nil:
|
|
||||||
answer("the machine has not reported on it yet")
|
|
||||||
case a.Outcome == inventory.OutcomeApplied:
|
|
||||||
answer("applied it at " + a.At.Local().Format(stamp))
|
|
||||||
case a.Outcome == inventory.OutcomeRefused:
|
|
||||||
answer("refused it at " + a.At.Local().Format(stamp) + ", in its words:")
|
|
||||||
for _, line := range strings.Split(strings.TrimRight(a.Refused, "\n"), "\n") {
|
|
||||||
lines = append(lines, " "+strings.TrimSpace(line))
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
answer(fmt.Sprintf("%s at %s: %d resource(s) failed — `status` says which", a.Outcome,
|
|
||||||
a.At.Local().Format(stamp), a.Failed))
|
|
||||||
}
|
|
||||||
return lines
|
|
||||||
}
|
|
||||||
|
|
||||||
// sendAnswerWord is what a machine answered of a send, in a word or few, as status says it in one line.
|
|
||||||
func sendAnswerWord(s inventory.Send) string {
|
|
||||||
switch a := s.Answer; {
|
|
||||||
case s.RefusedAt != nil:
|
|
||||||
return fmt.Sprintf("refused: it had applied generation %d", s.RefusedApplied)
|
|
||||||
case a.Outcome == "" || a.At == nil:
|
|
||||||
return "not reported on yet"
|
|
||||||
case a.Outcome == inventory.OutcomeFailed:
|
|
||||||
return fmt.Sprintf("failed (%d resource(s))", a.Failed)
|
|
||||||
default:
|
|
||||||
return a.Outcome
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// printLastSends says, one line per machine, when it was last sent a declaration, by whom, from which assignment
|
|
||||||
// generation and what it answered (novox/hq issue 485). `node show <node>` says the send in full.
|
|
||||||
func printLastSends(nodes []inventory.Node, sends map[string]inventory.Send, unread string) {
|
|
||||||
if unread != "" {
|
|
||||||
fmt.Printf("what each machine was last sent could NOT be read: %s\n\n", unread)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if len(nodes) == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
fmt.Println("last sent, per machine:")
|
|
||||||
for _, n := range nodes {
|
|
||||||
s, found := sends[n.Name]
|
|
||||||
if !found {
|
|
||||||
fmt.Printf(" %-12s no send recorded\n", n.Name)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
generation := "no generation"
|
|
||||||
if s.Generation > 0 {
|
|
||||||
generation = fmt.Sprintf("generation %d", s.Generation)
|
|
||||||
}
|
|
||||||
fmt.Printf(" %-12s %s by %s, %s — %s\n", n.Name, s.SentAt.Local().Format("2006-01-02 15:04"), s.Sender,
|
|
||||||
generation, sendAnswerWord(s))
|
|
||||||
}
|
|
||||||
fmt.Printf("\n `node show <node>` says a machine's last send in full\n\n")
|
|
||||||
}
|
|
||||||
@@ -1,148 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The assignment generation a declaration was composed from (novox/hq issue 234).
|
|
||||||
|
|
||||||
func bodyKeys(t *testing.T, s sendable) map[string]any {
|
|
||||||
t.Helper()
|
|
||||||
raw, err := s.Body()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var keys map[string]any
|
|
||||||
if err := json.Unmarshal(raw, &keys); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return keys
|
|
||||||
}
|
|
||||||
|
|
||||||
// **The generation goes on the wire only to a machine whose node-engine said it reads one**: an older
|
|
||||||
// node-engine decodes strictly and refuses an unknown key, whole. No put-back mark is ever sent: a gate
|
|
||||||
// composes its put-back afresh, with the generation as it stands.
|
|
||||||
func TestTheGenerationIsSentOnlyToAMachineThatReadsOne(t *testing.T) {
|
|
||||||
resources := []map[string]any{{"id": "a", "type": "file", "path": "/etc/a", "content": "x\n"}}
|
|
||||||
reads := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: true, acting: 57}
|
|
||||||
var told sendable
|
|
||||||
told.Resources = resources
|
|
||||||
reads.stamp(&told)
|
|
||||||
keys := bodyKeys(t, told)
|
|
||||||
if keys["generation"] != float64(40) || keys["sequence"] != float64(12) {
|
|
||||||
t.Fatalf("a machine that reads a generation was sent %v", keys)
|
|
||||||
}
|
|
||||||
if _, there := keys["put_back"]; there {
|
|
||||||
t.Fatalf("a put-back mark was sent: %v", keys)
|
|
||||||
}
|
|
||||||
if told.composedFrom != 40 || told.actingEpoch != 57 {
|
|
||||||
t.Errorf("the send does not keep what it was composed from for its record: %+v", told)
|
|
||||||
}
|
|
||||||
|
|
||||||
older := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: false, acting: 57}
|
|
||||||
var untold sendable
|
|
||||||
untold.Resources = resources
|
|
||||||
older.stamp(&untold)
|
|
||||||
if _, there := bodyKeys(t, untold)["generation"]; there {
|
|
||||||
t.Fatal("a machine whose node-engine never said it reads a generation was sent one")
|
|
||||||
}
|
|
||||||
if untold.composedFrom != 40 {
|
|
||||||
t.Errorf("the generation it was composed from is recorded whether or not it was sent: %+v", untold)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **The sender is named**: the caller of the seat call when there is one, else the shell's account, and the
|
|
||||||
// process and build that composed it.
|
|
||||||
func TestASendNamesItsSender(t *testing.T) {
|
|
||||||
said := senderOf("g14.node-tools, through the mesh-controller seat")
|
|
||||||
if !strings.Contains(said, "g14.node-tools") || !strings.Contains(said, "pid ") || !strings.Contains(said, "build ") {
|
|
||||||
t.Fatalf("the sender reads %q", said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// refusedSend is a send a machine refused for its generation at a moment.
|
|
||||||
func refusedSend(at time.Time) inventory.Send {
|
|
||||||
return inventory.Send{NodeName: "anchor", Sequence: 12, Epoch: 57, Generation: 38, RefusedApplied: 40,
|
|
||||||
Sender: "a one-shot push by jochen at a shell on anchor (pid 4242 on anchor, build 2026.10.11)",
|
|
||||||
Modules: []string{"docker"}, SentAt: at.Add(-time.Second), RefusedAt: &at, Recorded: true}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **A refused send is raised naming its sender** (novox/hq issue 234): who sent it, from which generation,
|
|
||||||
// against which the machine applied, and its sequence — the facts that took a morning to look for.
|
|
||||||
func TestARefusedSendIsRaisedNamingItsSender(t *testing.T) {
|
|
||||||
now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC)
|
|
||||||
f := calm(now)
|
|
||||||
f.refusedSends = []inventory.Send{refusedSend(now.Add(-time.Minute))}
|
|
||||||
got := watchGenerationRefusals(f)
|
|
||||||
if len(got) != 1 {
|
|
||||||
t.Fatalf("%+v", got)
|
|
||||||
}
|
|
||||||
o := got[0]
|
|
||||||
if o.Key() != "machine.anchor.older-generation" || o.Machine != "anchor" {
|
|
||||||
t.Errorf("raised as %s about %q", o.Key(), o.Machine)
|
|
||||||
}
|
|
||||||
for _, want := range []string{"a one-shot push by jochen", "generation 38", "generation 40", "sequence 12"} {
|
|
||||||
if !strings.Contains(o.Summary, want) {
|
|
||||||
t.Errorf("the summary does not say %q: %s", want, o.Summary)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if o.Headline == "" || o.Explanation == "" || o.Resolved == "" {
|
|
||||||
t.Errorf("the condition is not worded for the operator: %+v", o)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **A refusal of a send the mesh has no record of is raised too, its sender said to be unknown** — never
|
|
||||||
// only a line on stderr.
|
|
||||||
func TestARefusalOfAnUnrecordedSendIsRaisedSayingTheSenderIsUnknown(t *testing.T) {
|
|
||||||
now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC)
|
|
||||||
f := calm(now)
|
|
||||||
s := refusedSend(now.Add(-time.Minute))
|
|
||||||
s.Recorded, s.Sender, s.Modules = false, "a sender the mesh has no record of (no send of this sequence was recorded)", nil
|
|
||||||
f.refusedSends = []inventory.Send{s}
|
|
||||||
got := watchGenerationRefusals(f)
|
|
||||||
if len(got) != 1 || !strings.Contains(got[0].Summary, "no record of") {
|
|
||||||
t.Fatalf("an unattributed refusal was not raised as one: %+v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **When the refusal showed the mesh's counter behind the machine, the condition asks for a push** — it was
|
|
||||||
// raised, and nothing has sent the machine its declaration since — and does not say there is nothing to do.
|
|
||||||
func TestACounterBehindTheMachineAsksForAPush(t *testing.T) {
|
|
||||||
now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC)
|
|
||||||
f := calm(now)
|
|
||||||
s := refusedSend(now.Add(-time.Minute))
|
|
||||||
s.Generation, s.RefusedApplied, s.CounterRaisedTo = 12, 40, 41
|
|
||||||
f.refusedSends = []inventory.Send{s}
|
|
||||||
got := watchGenerationRefusals(f)
|
|
||||||
if len(got) != 1 {
|
|
||||||
t.Fatalf("%+v", got)
|
|
||||||
}
|
|
||||||
if !strings.Contains(got[0].Summary, "`push anchor`") || !strings.Contains(got[0].Explanation, "push anchor") ||
|
|
||||||
strings.Contains(got[0].Explanation, "Nothing for you to do") {
|
|
||||||
t.Errorf("a counter behind the machine does not ask for a push: %s / %s", got[0].Summary, got[0].Explanation)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **Only a refusal of the counter as it stands is the counter behind**: a stale sender's generation is below
|
|
||||||
// it, and the counter is left alone for that.
|
|
||||||
func TestOnlyARefusalOfTheCounterAsItStandsRaisesIt(t *testing.T) {
|
|
||||||
for _, c := range []struct {
|
|
||||||
refused, applied, now int64
|
|
||||||
behind bool
|
|
||||||
}{
|
|
||||||
{refused: 12, applied: 40, now: 12, behind: true}, // the store was put back: the mesh says 12, the machine had 40
|
|
||||||
{refused: 38, applied: 40, now: 41, behind: false}, // a stale sender: the counter is already past
|
|
||||||
{refused: 38, applied: 40, now: 40, behind: false}, // a stale sender: the counter is where the machine is
|
|
||||||
} {
|
|
||||||
r := link.GenerationRefusal{Generation: c.refused, Applied: c.applied}
|
|
||||||
if got := counterBehind(r, c.now); got != c.behind {
|
|
||||||
t.Errorf("refused %d, applied %d, counter %d: behind %v, want %v", c.refused, c.applied, c.now, got, c.behind)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,198 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/identity"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The grants step on a store (novox/hq issue 490): the machine holding the bus is sent its new user list,
|
|
||||||
// and nothing else — not the new build of a module it runs, not a module newly assigned there.
|
|
||||||
//
|
|
||||||
// The restic shape: anchor holds the bus and runs restic at c1, as does laptop; restic's c2 gives it a
|
|
||||||
// health tool. The grants step's send to anchor composes restic at c1, carries the user list that grants
|
|
||||||
// laptop's node-engine c2's tool, and records that anchor still runs c1. A module newly assigned to anchor
|
|
||||||
// makes the step fail, said, rather than install it unjudged.
|
|
||||||
func TestTheGrantsStepSendsTheUserListAndNothingElse(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
start := time.Now().Add(-time.Hour)
|
|
||||||
build := func(module, commit string, at time.Time, manifest map[string]any) link.BuildResult {
|
|
||||||
manifest["module"], manifest["version"] = module, "1"
|
|
||||||
raw, _ := json.Marshal(manifest)
|
|
||||||
return link.BuildResult{ID: link.NewBuildID(at), Repository: "novox/mesh-catalog", Path: "modules/" + module,
|
|
||||||
On: "anchor", Module: module, Commit: commit, Manifest: raw,
|
|
||||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
|
||||||
}
|
|
||||||
for _, b := range []link.BuildResult{
|
|
||||||
build("nats", "n1", start, natsFixture()),
|
|
||||||
build("restic", "c1", start.Add(time.Second), resticFixture(false)),
|
|
||||||
build("wallpaper", "w1", start.Add(2*time.Second), wallpaperFixture()),
|
|
||||||
} {
|
|
||||||
if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, a := range [][2]string{{"anchor", "nats"}, {"anchor", "restic"}, {"laptop", "restic"}} {
|
|
||||||
if _, err := inv.Assign(ctx, a[0], a[1]); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, n := range []string{"anchor", "laptop"} {
|
|
||||||
if _, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: "node." + n, Kind: inventory.BusNode, Node: n}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
wasEpoch := epochForActs
|
|
||||||
epochForActs = func(context.Context) (uint64, error) { return 7, nil }
|
|
||||||
t.Cleanup(func() { epochForActs = wasEpoch })
|
|
||||||
|
|
||||||
// A send to anchor composed as sendToEach composes it: numbered, planned, declared, stamped.
|
|
||||||
compose := func(under context.Context) (catalogue.Resolution, sendable) {
|
|
||||||
t.Helper()
|
|
||||||
numbered, err := allot(under, inv, "anchor")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
plan, settings, err := planFor(under, open, "anchor")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
gens, err := generators(under, open)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
declared, err := declarationWith(under, open, "anchor", plan, settings, gens, Allocating)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
numbered.stamp(&declared)
|
|
||||||
return plan, declared
|
|
||||||
}
|
|
||||||
record := func(declared sendable) {
|
|
||||||
t.Helper()
|
|
||||||
body, err := declared.Body()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := recordSent(ctx, inv, "anchor", body, declared.Builds, declared.Epoch,
|
|
||||||
sentRecordOf(ctx, declared)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Kept for anchor alone. laptop is never recorded as sent, on purpose: composing anchor resolves laptop
|
|
||||||
// too (who is on the private network), and a step that kept every machine's builds refused anchor's
|
|
||||||
// composition for want of laptop's last send (repo-check of #230 at 0f853e93).
|
|
||||||
kept := keepingEveryBuild(keepingRecorded(ctx), "anchor")
|
|
||||||
|
|
||||||
// What anchor was last sent: everything at the builds of before the merge, as an ordinary send sends it.
|
|
||||||
_, before := compose(keepingRecorded(ctx))
|
|
||||||
record(before)
|
|
||||||
plan, declared := compose(kept)
|
|
||||||
if only, err := grantsOnlyIn(ctx, open, "anchor", plan, declared); err != nil || only != "" {
|
|
||||||
t.Fatalf("with nothing changed, the step reads %q, %v", only, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The merge: restic's c2 gives it a health tool.
|
|
||||||
if _, _, err := takeIn(ctx, inv, build("restic", "c2", start.Add(time.Minute),
|
|
||||||
resticFixture(true))); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The step's send to anchor: restic as anchor runs it, c1, and the user list granting c2's tool.
|
|
||||||
generation, err := inv.AssignmentGeneration(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
plan, declared = compose(kept)
|
|
||||||
for _, m := range plan.Modules {
|
|
||||||
if m.Module == "restic" && len(m.Tools) > 0 {
|
|
||||||
t.Fatalf("the grants step composed restic's new build on anchor: tools %v", m.Tools)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if declared.Builds["restic"] != "c1" {
|
|
||||||
t.Fatalf("the step's send records it carries restic %q; want c1, which anchor runs", declared.Builds["restic"])
|
|
||||||
}
|
|
||||||
if !strings.Contains(declared.BusUsers, "mesh.mod.restic.tool.backup_health.laptop") {
|
|
||||||
t.Errorf("the step's declaration does not grant laptop's node-engine restic's new tool:\n%s", declared.BusUsers)
|
|
||||||
}
|
|
||||||
// The generation it carries is the current one (novox/hq issue 234), and the step does not raise it.
|
|
||||||
if declared.composedFrom != generation {
|
|
||||||
t.Errorf("the step's send was composed from generation %d; the mesh is at %d", declared.composedFrom, generation)
|
|
||||||
}
|
|
||||||
// The guard, on the very declaration the send sends: only the user list differs from the last send.
|
|
||||||
if only, err := grantsOnlyIn(ctx, open, "anchor", plan, declared); err != nil || only != "" {
|
|
||||||
t.Fatalf("the step reads as changing more than the user list: %q, %v", only, err)
|
|
||||||
}
|
|
||||||
// And the gate's refusal still reads the step's send: it moves nothing there.
|
|
||||||
if names, err := ungatedIn(kept, open, []string{"anchor"}, ""); err != nil || strings.Join(names, ",") != "anchor" {
|
|
||||||
t.Fatalf("the step's send is refused as a move: %v, %v", names, err)
|
|
||||||
}
|
|
||||||
// Recorded as the send records it: anchor still runs restic c1, its gate still to come there.
|
|
||||||
record(declared)
|
|
||||||
if sent, _, err := inv.SentBuilds(ctx, "anchor"); err != nil || sent["restic"] != "c1" {
|
|
||||||
t.Fatalf("after the step anchor reads as sent restic %q (%v); want c1", sent["restic"], err)
|
|
||||||
}
|
|
||||||
if after, err := inv.AssignmentGeneration(ctx); err != nil || after != generation {
|
|
||||||
t.Fatalf("the step moved the assignment generation from %d to %d (%v)", generation, after, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A module newly assigned to anchor: the step would install it, unjudged, so it is refused unsent.
|
|
||||||
if _, err := inv.Assign(ctx, "anchor", "wallpaper"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
plan, declared = compose(kept)
|
|
||||||
only, err := grantsOnlyIn(ctx, open, "anchor", plan, declared)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !strings.Contains(only, "wallpaper newly assigned") || !strings.Contains(only, "+wallpaper") {
|
|
||||||
t.Fatalf("a new assignment on the bus's machine reads %q; want the step refused, naming it", only)
|
|
||||||
}
|
|
||||||
// And the send itself refuses it, unsent: judged where it is composed, before the bus is dialled.
|
|
||||||
identity.ForTest(t)
|
|
||||||
if _, err := sendToEach(kept, open, []string{"anchor"}); !errors.Is(err, errNotGrantsOnly) ||
|
|
||||||
!strings.Contains(err.Error(), "wallpaper") {
|
|
||||||
t.Fatalf("the grants step's send of a new assignment was not refused by the send: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// fixtureImage is the image every fixture container of the grants step's tests runs.
|
|
||||||
var fixtureImage = "registry.invalid:5000/restic/backup@sha256:" + strings.Repeat("b", 64)
|
|
||||||
|
|
||||||
// resticFixture is restic's manifest in the shape of mesh-catalog #210: at c2 (withTool) its backup judged by
|
|
||||||
// its new tool `backup_health`, beside a check of another kind it does not run itself — a tool check alone
|
|
||||||
// is refused (ADR 0227 rule 8, ADR 0240 rule 2) — and at c1 neither.
|
|
||||||
func resticFixture(withTool bool) map[string]any {
|
|
||||||
backup := map[string]any{"id": "backup", "type": "container", "name": "restic-backup", "image": fixtureImage}
|
|
||||||
measure := map[string]any{"id": "measure", "type": "container", "name": "restic-measure", "image": fixtureImage}
|
|
||||||
m := map[string]any{"resources": []any{backup, measure}}
|
|
||||||
if withTool {
|
|
||||||
backup["health"] = map[string]any{"kind": catalogue.HealthTool, "tool": "backup_health"}
|
|
||||||
measure["health"] = map[string]any{"kind": "runtime"}
|
|
||||||
m["tools"] = []string{"backup_health"}
|
|
||||||
}
|
|
||||||
return m
|
|
||||||
}
|
|
||||||
|
|
||||||
// natsFixture is the bus's module: it holds mesh-broker and is sent the user list.
|
|
||||||
func natsFixture() map[string]any {
|
|
||||||
return map[string]any{"bus-users": "/var/lib/nats-module/conf/accounts.conf",
|
|
||||||
"claims": []any{map[string]any{"name": catalogue.BrokerSeat, "scope": catalogue.ScopeMesh}}}
|
|
||||||
}
|
|
||||||
|
|
||||||
// wallpaperFixture is a module the bus's machine is newly assigned.
|
|
||||||
func wallpaperFixture() map[string]any {
|
|
||||||
return map[string]any{"resources": []any{
|
|
||||||
map[string]any{"id": "paper", "type": "container", "name": "wallpaper", "image": fixtureImage}}}
|
|
||||||
}
|
|
||||||
@@ -1,226 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The grants reach the bus before the gate (novox/hq issue 490).
|
|
||||||
//
|
|
||||||
// On 2026-10-11 a merge gave restic, assigned to every machine, a new health tool. Its walk sent restic to
|
|
||||||
// its first machine and judged it there; that machine's node-engine asked the new tool and the bus refused
|
|
||||||
// it, because the grant was in the bus's user list, which only the machine holding the bus carries — and
|
|
||||||
// that machine runs restic too, so it was left out of the send while restic's new build waited there for
|
|
||||||
// the very gate that could not pass. A person pushed it by hand, carrying restic's new build there unjudged.
|
|
||||||
|
|
||||||
// aSend is one send the walk made: to which machines, and whether every build there was kept.
|
|
||||||
type aSend struct {
|
|
||||||
names []string
|
|
||||||
keepsAll bool
|
|
||||||
judged []string
|
|
||||||
}
|
|
||||||
|
|
||||||
// sendsRecorded replaces the walk's send and the reading of the bus's user list for one test: the machine
|
|
||||||
// holding the bus is novox, and its list is behind as behind says.
|
|
||||||
func sendsRecorded(t *testing.T, holder string, behind bool, refuse error) *[]aSend {
|
|
||||||
t.Helper()
|
|
||||||
var sends []aSend
|
|
||||||
wasSend, wasBehind := sendRollout, brokerBehindOf
|
|
||||||
t.Cleanup(func() { sendRollout, brokerBehindOf = wasSend, wasBehind })
|
|
||||||
brokerBehindOf = func(_ context.Context, _ *stores, names []string) (string, bool, error) {
|
|
||||||
if slices.Contains(names, holder) {
|
|
||||||
return holder, false, nil
|
|
||||||
}
|
|
||||||
return holder, behind, nil
|
|
||||||
}
|
|
||||||
sendRollout = func(ctx context.Context, _ *stores, names []string) ([]string, error) {
|
|
||||||
s := aSend{names: append([]string(nil), names...), keepsAll: len(names) == 1 && everyBuildKept(ctx, names[0])}
|
|
||||||
for n := range scopeOf(ctx).judged {
|
|
||||||
s.judged = append(s.judged, n)
|
|
||||||
}
|
|
||||||
sends = append(sends, s)
|
|
||||||
if refuse != nil && s.keepsAll {
|
|
||||||
return nil, refuse
|
|
||||||
}
|
|
||||||
return names, nil
|
|
||||||
}
|
|
||||||
return &sends
|
|
||||||
}
|
|
||||||
|
|
||||||
// The restic shape: a new tool on a module on every machine, its first machine ace, the bus on novox.
|
|
||||||
func TestAWalkSendsTheGrantsToTheBusBeforeTheFirstMachineIsJudged(t *testing.T) {
|
|
||||||
sends := sendsRecorded(t, "novox", true, nil)
|
|
||||||
sent, grants, err := sendJudged(t.Context(), nil, "ace")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(*sends) != 2 {
|
|
||||||
t.Fatalf("the walk made %d send(s), want the grants step and then the first machine: %+v", len(*sends), *sends)
|
|
||||||
}
|
|
||||||
first, then := (*sends)[0], (*sends)[1]
|
|
||||||
if strings.Join(first.names, ",") != "novox" || !first.keepsAll || len(first.judged) != 0 {
|
|
||||||
t.Errorf("the first send is %+v, want novox alone, every build there kept, judging nothing", first)
|
|
||||||
}
|
|
||||||
if strings.Join(then.names, ",") != "ace" || then.keepsAll || strings.Join(then.judged, ",") != "ace" {
|
|
||||||
t.Errorf("the second send is %+v, want ace, judged", then)
|
|
||||||
}
|
|
||||||
if strings.Join(sent, ",") != "ace" {
|
|
||||||
t.Errorf("the gate's machines are %v, want ace alone: the bus's machine is not judged", sent)
|
|
||||||
}
|
|
||||||
if grants == nil || grants.Node != "novox" || grants.At == nil || grants.Failed != "" {
|
|
||||||
t.Fatalf("the gate keeps %+v as its grants step", grants)
|
|
||||||
}
|
|
||||||
line := gateLine(&inventory.PlanGate{Machines: sent, Grants: grants})
|
|
||||||
if !strings.Contains(line, "grants step: novox sent the bus's user list first, its builds kept") {
|
|
||||||
t.Errorf("plans says the gate as %q", line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Nothing more when there is nothing to carry: the list unchanged, or the first machine holds the bus.
|
|
||||||
func TestAWalkTakesNoGrantsStepWhenTheListIsAlreadyThere(t *testing.T) {
|
|
||||||
for _, c := range []struct {
|
|
||||||
name string
|
|
||||||
node string
|
|
||||||
behind bool
|
|
||||||
}{{"the list unchanged", "ace", false}, {"the first machine holds the bus", "novox", true}} {
|
|
||||||
t.Run(c.name, func(t *testing.T) {
|
|
||||||
sends := sendsRecorded(t, "novox", c.behind, nil)
|
|
||||||
_, grants, err := sendJudged(t.Context(), nil, c.node)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(*sends) != 1 || (*sends)[0].keepsAll || grants != nil {
|
|
||||||
t.Errorf("the walk made %+v with grants %+v, want the judged send alone", *sends, grants)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A grants step that cannot be sent is said on the gate, and the walk goes on as it did before.
|
|
||||||
func TestAGrantsStepThatFailsIsSaidAndTheSendGoesOn(t *testing.T) {
|
|
||||||
sends := sendsRecorded(t, "novox", true, errors.New("a bus upgrade waits for a person"))
|
|
||||||
sent, grants, err := sendJudged(t.Context(), nil, "ace")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if strings.Join(sent, ",") != "ace" || len(*sends) != 2 {
|
|
||||||
t.Errorf("the walk made %+v", *sends)
|
|
||||||
}
|
|
||||||
if grants == nil || grants.Failed == "" || grants.At != nil {
|
|
||||||
t.Fatalf("the gate keeps %+v", grants)
|
|
||||||
}
|
|
||||||
if line := gateLine(&inventory.PlanGate{Machines: sent, Grants: grants}); !strings.Contains(line,
|
|
||||||
"grants step to novox FAILED, sent without it: a bus upgrade waits for a person") {
|
|
||||||
t.Errorf("plans says the gate as %q", line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A step whose send would change more than the user list is refused unsent by the send (sendToEach): the
|
|
||||||
// gate says it FAILED with what differed, and the walk goes on.
|
|
||||||
func TestAGrantsStepThatWouldCarryMoreIsNotSent(t *testing.T) {
|
|
||||||
refusal := fmt.Errorf("%w: its send would change more than the bus's user list: -postgres.login-n8n", errNotGrantsOnly)
|
|
||||||
sends := sendsRecorded(t, "novox", true, refusal)
|
|
||||||
sent, grants, err := sendJudged(t.Context(), nil, "ace")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if strings.Join(sent, ",") != "ace" || len(*sends) != 2 {
|
|
||||||
t.Errorf("the walk made %+v, want the refused step and then ace's judged send", *sends)
|
|
||||||
}
|
|
||||||
if grants == nil || grants.At != nil || !strings.Contains(grants.Failed, "-postgres.login-n8n") {
|
|
||||||
t.Fatalf("the gate keeps %+v", grants)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The step's exemption is the list's content alone: its path or mode moving is a change like any other.
|
|
||||||
func TestOnlyTheUserListsContentIsExempt(t *testing.T) {
|
|
||||||
list := "nats.bus-users"
|
|
||||||
file := func(fields map[string]any) sentResource {
|
|
||||||
r := map[string]any{"id": list, "type": "file", "path": "/conf/accounts.conf", "mode": "0600", "content": "a"}
|
|
||||||
for k, v := range fields {
|
|
||||||
r[k] = v
|
|
||||||
}
|
|
||||||
body, _ := json.Marshal(map[string]any{"resources": []any{r}})
|
|
||||||
s, err := summarize(body)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return s[0]
|
|
||||||
}
|
|
||||||
was := file(nil)
|
|
||||||
if other := otherThanTheList(diffSent([]sentResource{was}, []sentResource{file(map[string]any{"content": "b"})}), list); len(other) != 0 {
|
|
||||||
t.Errorf("the list's new content reads as more: %v", other)
|
|
||||||
}
|
|
||||||
for _, f := range []map[string]any{{"mode": "0644"}, {"path": "/elsewhere"}, {"content": "b", "owner": "nats"}} {
|
|
||||||
other := otherThanTheList(diffSent([]sentResource{was}, []sentResource{file(f)}), list)
|
|
||||||
if len(other) != 1 || !strings.HasPrefix(other[0], "~"+list) {
|
|
||||||
t.Errorf("the list's resource changed by %v reads %v; want it refused", f, other)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The step keeps the builds of the machine it sends alone: the others its composition resolves are composed
|
|
||||||
// as any send composes them.
|
|
||||||
func TestTheGrantsStepKeepsOneMachinesBuilds(t *testing.T) {
|
|
||||||
ctx := keepingEveryBuild(t.Context(), "novox")
|
|
||||||
if !everyBuildKept(ctx, "novox") || everyBuildKept(ctx, "ace") || everyBuildKept(t.Context(), "novox") {
|
|
||||||
t.Error("the grants step keeps the builds of a machine it does not send")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The store test's manifests are manifests the controller takes in: checked here, where no store is needed,
|
|
||||||
// so a fixture the module rules refuse fails before the store test is run (repo-check of #230 at 325575b2).
|
|
||||||
func TestTheGrantsStepFixturesAreManifests(t *testing.T) {
|
|
||||||
for name, m := range map[string]map[string]any{"nats": natsFixture(), "restic-c1": resticFixture(false),
|
|
||||||
"restic-c2": resticFixture(true), "wallpaper": wallpaperFixture()} {
|
|
||||||
m["module"], m["version"] = strings.SplitN(name, "-", 2)[0], "1"
|
|
||||||
raw, err := json.Marshal(m)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := catalogue.ParseManifest(raw); err != nil {
|
|
||||||
t.Errorf("%s: %v", name, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The delivery plan says the step before the merge.
|
|
||||||
func TestAChangePlanSaysTheGrantsStep(t *testing.T) {
|
|
||||||
const catalogue_ = "http://forge.internal:20000/novox/mesh-catalog.git"
|
|
||||||
entry := func(name string, on ...string) inventory.Entry {
|
|
||||||
e := fromRepo(name, catalogue_, "modules/"+name)
|
|
||||||
e.Source.BuiltFrom = "old"
|
|
||||||
e.On = on
|
|
||||||
return e
|
|
||||||
}
|
|
||||||
nats := entry("nats", "novox")
|
|
||||||
nats.Manifest.BusUsers = "/etc/nats/users.conf"
|
|
||||||
nats.Manifest.Claims = []catalogue.Claim{{Name: catalogue.BrokerSeat, Scope: catalogue.ScopeMesh}}
|
|
||||||
restic := entry("restic", "ace", "novox", "shanks")
|
|
||||||
only := entry("bus-tools", "novox")
|
|
||||||
entries := []inventory.Entry{nats, restic, only}
|
|
||||||
rollsOut := func(string) (inventory.Upgrade, bool) { return inventory.Upgrade{RollOut: true}, true }
|
|
||||||
plan := func(paths ...string) link.ChangePlan {
|
|
||||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: paths}
|
|
||||||
return changePlanOf("novox/mesh-catalog", "main", "head", reachOfMerge(m, entries, nil, nil), entries, rollsOut)
|
|
||||||
}
|
|
||||||
|
|
||||||
text := planText(plan("modules/restic/module.json"))
|
|
||||||
if !strings.Contains(text, "grants step: if this changes what the bus's user list says (a new tool, subject or "+
|
|
||||||
"user), novox is sent that list alone, every build there kept, before restic is judged on its first machine") {
|
|
||||||
t.Errorf("the change plan does not say the grants step:\n%s", text)
|
|
||||||
}
|
|
||||||
// A module only on the bus's machine goes there with the list in its own send: no step of its own.
|
|
||||||
if text := planText(plan("modules/bus-tools/module.json")); strings.Contains(text, "grants step") {
|
|
||||||
t.Errorf("a module on the bus's machine alone reads:\n%s", text)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,158 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **A delivery group still assembles** (novox/hq ADR 0306 decision 2): a merge whose delivery group has a member
|
|
||||||
// not yet merged waits for the merge window, whether or not a batch is assembling, so the group's members walk in
|
|
||||||
// one batch, in their order (ADR 0276 decision 5). Which groups exist and where each member stands is
|
|
||||||
// mesh-delivery's record (ADR 0239): a group is the pull requests of two repositories or more sharing a head
|
|
||||||
// branch name, which is its id. The controller asks it, through its `groups` verb, when a merge from a head is
|
|
||||||
// heard; a member whose pull request is open (proposed, checked, ready or rejected) is still to merge. When
|
|
||||||
// mesh-delivery does not answer within groupAskWithin, or answers something unreadable, the merge waits: the safe
|
|
||||||
// error is to wait, and its batch says why.
|
|
||||||
|
|
||||||
// groupAskWithin is how long mesh-delivery is given to say a merge's group, while the merge is being heard.
|
|
||||||
var groupAskWithin = 3 * time.Second
|
|
||||||
|
|
||||||
// openPullStates are the states of a delivery whose pull request is open and not merged (mesh-delivery's table):
|
|
||||||
// a member in one of them is still to merge. Closed unmerged is stopped, final, and waits for nothing.
|
|
||||||
var openPullStates = map[string]bool{"proposed": true, "checked": true, "ready": true, "rejected": true}
|
|
||||||
|
|
||||||
// groupMember is one member of a delivery group as mesh-delivery's `groups` says it: `<owner/repo>@<commit> (state)`.
|
|
||||||
type groupMember struct {
|
|
||||||
Repository, State string
|
|
||||||
}
|
|
||||||
|
|
||||||
// deliveryGroupOf is what mesh-delivery says of the group a head names: its members, and whether the head names a
|
|
||||||
// group at all; an error when it cannot be asked or did not answer. A seam a test replaces.
|
|
||||||
var deliveryGroupOf = func(ctx context.Context, held bool, head string) ([]groupMember, bool, error) {
|
|
||||||
if !held {
|
|
||||||
return nil, false, fmt.Errorf("no holder of %s is on record to say it", catalogue.DeliverySeat)
|
|
||||||
}
|
|
||||||
serving := servingBus.Load()
|
|
||||||
if serving == nil {
|
|
||||||
return nil, false, errors.New("this controller is not on the bus")
|
|
||||||
}
|
|
||||||
asking, cancel := context.WithTimeout(ctx, groupAskWithin)
|
|
||||||
defer cancel()
|
|
||||||
raw, err := askDeliveryOwner(asking, serving.Conn(), "groups", map[string]any{})
|
|
||||||
if err != nil {
|
|
||||||
if errors.Is(err, link.ErrNothingServes) {
|
|
||||||
return nil, false, fmt.Errorf("nothing serves %s.groups", catalogue.DeliverySeat)
|
|
||||||
}
|
|
||||||
return nil, false, err
|
|
||||||
}
|
|
||||||
var lines []struct {
|
|
||||||
ID string `json:"id"`
|
|
||||||
Members []string `json:"members"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(raw, &lines); err != nil {
|
|
||||||
return nil, false, fmt.Errorf("%s.groups answered something unreadable: %w", catalogue.DeliverySeat, err)
|
|
||||||
}
|
|
||||||
for _, l := range lines {
|
|
||||||
if l.ID != head {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
var members []groupMember
|
|
||||||
for _, s := range l.Members {
|
|
||||||
members = append(members, memberOf(s))
|
|
||||||
}
|
|
||||||
return members, true, nil
|
|
||||||
}
|
|
||||||
return nil, false, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// memberOf reads one member as `groups` says it: `novox/mesh-sdk@d41d255f28ad (held)`.
|
|
||||||
func memberOf(s string) groupMember {
|
|
||||||
id, state := strings.TrimSpace(s), ""
|
|
||||||
if i := strings.LastIndex(id, " ("); i >= 0 && strings.HasSuffix(id, ")") {
|
|
||||||
id, state = id[:i], id[i+2:len(id)-1]
|
|
||||||
}
|
|
||||||
repository, _, _ := strings.Cut(id, "@")
|
|
||||||
return groupMember{Repository: repository, State: state}
|
|
||||||
}
|
|
||||||
|
|
||||||
// groupWait says whether a merge heard now waits for the window for its delivery group, and why in words; nothing
|
|
||||||
// when it names no head, its head names no group, or every other member is merged or closed. own is the merge's
|
|
||||||
// kind (ownPath).
|
|
||||||
//
|
|
||||||
// A member of another repository whose merge this controller already keeps in a batch not yet cut of the same
|
|
||||||
// kind is merged, whatever mesh-delivery says yet (it marks a member merged once its walk opened): the merge joins
|
|
||||||
// that batch by their head. One kept in a batch of the other kind keeps the merge waiting, whatever mesh-delivery
|
|
||||||
// says (review of #250): kinds never share a batch, so the merge's batch is a new one, and cut at once it would
|
|
||||||
// walk before that member's batch is cut and break the group's order (ADR 0276 decision 5). Waiting, the two
|
|
||||||
// batches are cut in the order their windows close, an own-path one first at one look, as cutBatchesHeld cuts
|
|
||||||
// every pair of kinds.
|
|
||||||
//
|
|
||||||
// Asked before the plans are held: a slow answer holds up no walk.
|
|
||||||
func groupWait(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, held, own bool) (string, error) {
|
|
||||||
if m.Head == "" {
|
|
||||||
return "", nil
|
|
||||||
}
|
|
||||||
same, other, err := keptFromHead(ctx, inv, m.Head, own)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if len(other) > 0 {
|
|
||||||
return "group " + m.Head + " has a member waiting in a batch of the other kind: " + strings.Join(other, ", "), nil
|
|
||||||
}
|
|
||||||
members, isGroup, err := deliveryGroupOf(ctx, held, m.Head)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Sprintf("%s did not say whether group %s has a member still to merge (%v): it waits for the "+
|
|
||||||
"window, the safe side", catalogue.DeliverySeat, m.Head, err), nil
|
|
||||||
}
|
|
||||||
if !isGroup {
|
|
||||||
return "", nil
|
|
||||||
}
|
|
||||||
var open []string
|
|
||||||
for _, g := range members {
|
|
||||||
if strings.EqualFold(g.Repository, m.Owner+"/"+m.Repo) || same[strings.ToLower(g.Repository)] ||
|
|
||||||
!openPullStates[g.State] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
open = append(open, g.Repository+" ("+g.State+")")
|
|
||||||
}
|
|
||||||
if len(open) == 0 {
|
|
||||||
return "", nil
|
|
||||||
}
|
|
||||||
return "group " + m.Head + " has a member still to merge: " + strings.Join(open, ", "), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// keptFromHead are the merges from a head that batches not yet cut hold: the repositories (lower case) of those in
|
|
||||||
// a batch of the kind own says, and, in words, each of the other kind with its batch.
|
|
||||||
func keptFromHead(ctx context.Context, inv *inventory.Inventory, head string, own bool) (map[string]bool, []string, error) {
|
|
||||||
batches, err := inv.Batches(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
same := map[string]bool{}
|
|
||||||
var other []string
|
|
||||||
for _, b := range batches {
|
|
||||||
merges, err := inv.MergesOf(ctx, b.ID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
for _, k := range merges {
|
|
||||||
if announcedOf(k).Head != head {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if b.OwnPath() == own {
|
|
||||||
same[strings.ToLower(k.Repository)] = true
|
|
||||||
} else {
|
|
||||||
other = append(other, fmt.Sprintf("%s@%s in %s", k.Repository, short(k.Commit), b.ID))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return same, other, nil
|
|
||||||
}
|
|
||||||
@@ -84,9 +84,6 @@ var handActVerbs = []handActVerb{
|
|||||||
"deliberate test, never a repair", DecidedFor: []string{causeDrill}},
|
"deliberate test, never a repair", DecidedFor: []string{causeDrill}},
|
||||||
// A drill: something broken on purpose to see the mesh raise and clear it. A person's test, never a
|
// A drill: something broken on purpose to see the mesh raise and clear it. A person's test, never a
|
||||||
// repair, however often it is run.
|
// repair, however often it is run.
|
||||||
// A credential a provider's software issued, taken from the provider and kept for its consumers (novox/hq ADR
|
|
||||||
// 0305): the mesh working as decided, however often the software changes it.
|
|
||||||
{Verb: handActIssued, Decision: "a credential the provider's software issued, kept for its consumers (ADR 0305)"},
|
|
||||||
{Verb: "hand-act drill", Decision: "a drill is a person's deliberate test of the mesh, never a repair"},
|
{Verb: "hand-act drill", Decision: "a drill is a person's deliberate test of the mesh, never a repair"},
|
||||||
// A person's decisions by design.
|
// A person's decisions by design.
|
||||||
{Verb: "retire approve", Decision: "nothing is retired past its bound without a person (ADR 0230)"},
|
{Verb: "retire approve", Decision: "nothing is retired past its bound without a person (ADR 0230)"},
|
||||||
|
|||||||
@@ -167,54 +167,3 @@ func TestASetuidSearchAsksOnlyAKnownNodeAndFailsOnARefusal(t *testing.T) {
|
|||||||
t.Fatalf("a start: %v, printed %q", err, out.String())
|
t.Fatalf("a start: %v, printed %q", err, out.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// The directory judgement's line asks only a known node, one name and nothing else, fails on a refusal, and says it
|
|
||||||
// changed nothing (novox/hq issue 358).
|
|
||||||
func TestADirectoryJudgementAsksOnlyAKnownNodeAndSaysItIsReadOnly(t *testing.T) {
|
|
||||||
t.Setenv(link.CallerVar, "jo through mesh-cli on anchor")
|
|
||||||
asked := 0
|
|
||||||
ask := func(answer link.HandOverAnswer) func(node, by string) (link.HandOverAnswer, error) {
|
|
||||||
return func(node, by string) (link.HandOverAnswer, error) {
|
|
||||||
asked++
|
|
||||||
if node != "novox" || by != "jo through mesh-cli on anchor" {
|
|
||||||
t.Fatalf("asked %q %q", node, by)
|
|
||||||
}
|
|
||||||
return answer, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
known := func(string) error { return nil }
|
|
||||||
var out bytes.Buffer
|
|
||||||
for _, args := range [][]string{nil, {"novox", "extra"}, {"-x"}} {
|
|
||||||
if err := judgeDirectoriesAsked(args, known, ask(link.HandOverAnswer{Said: "x"}), &out); err == nil || asked != 0 {
|
|
||||||
t.Fatalf("%v was asked: %v", args, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := judgeDirectoriesAsked([]string{"novox"}, func(string) error { return errors.New("no node called novox") },
|
|
||||||
ask(link.HandOverAnswer{Said: "x"}), &out); err == nil || asked != 0 {
|
|
||||||
t.Fatalf("an unknown node: %v", err)
|
|
||||||
}
|
|
||||||
if err := judgeDirectoriesAsked([]string{"novox"}, known, ask(link.HandOverAnswer{Refused: "no; nothing was judged"}),
|
|
||||||
&out); err == nil || !strings.Contains(err.Error(), "novox refused") || out.Len() != 0 {
|
|
||||||
t.Fatalf("a refusal: %v, printed %q", err, out.String())
|
|
||||||
}
|
|
||||||
said := "2 directories declared: 1 placed, 1 would be refused\n/var/lib/a: placed\n/var/lib/b: would be refused: …"
|
|
||||||
if err := judgeDirectoriesAsked([]string{"novox"}, known, ask(link.HandOverAnswer{Said: said}), &out); err != nil ||
|
|
||||||
out.String() != "novox (read-only: nothing on novox was changed)\n"+said+"\n" {
|
|
||||||
t.Fatalf("a judgement: %v, printed %q", err, out.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The judgement is a seat verb an agent may call — it reads, and changes nothing on the node — and runs exactly
|
|
||||||
// `node judge-directories <node>` (novox/hq issue 358).
|
|
||||||
func TestTheDirectoryJudgementIsAVerbThatOnlyReads(t *testing.T) {
|
|
||||||
argv, err := argvFor("judge-directories", map[string]any{"node": "novox"})
|
|
||||||
if err != nil || strings.Join(argv, " ") != "node judge-directories novox" {
|
|
||||||
t.Fatalf("the verb runs %v, %v", argv, err)
|
|
||||||
}
|
|
||||||
if _, err := argvFor("judge-directories", map[string]any{}); err == nil {
|
|
||||||
t.Fatal("the verb ran with no node")
|
|
||||||
}
|
|
||||||
if !readingVerbs["judge-directories"] {
|
|
||||||
t.Fatal("the judgement is not counted a reading verb")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -128,7 +128,7 @@ func (r *recordedDelivery) grant(context.Context, []readyNode) error { return ni
|
|||||||
|
|
||||||
func (r *recordedDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
|
func (r *recordedDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
|
||||||
r.declared = append(r.declared, s.node)
|
r.declared = append(r.declared, s.node)
|
||||||
return recordSent(ctx, r.inv, s.node, body, s.declared.Builds, s.declared.Epoch, sentRecordOf(ctx, s.declared))
|
return recordSent(ctx, r.inv, s.node, body, s.declared.Builds, s.declared.Epoch)
|
||||||
}
|
}
|
||||||
|
|
||||||
// aResolver is a module built from a repository, at a commit, with something on the machine that
|
// aResolver is a module built from a repository, at a commit, with something on the machine that
|
||||||
|
|||||||
@@ -25,7 +25,6 @@ import (
|
|||||||
// another send reaches anchor, unreported; the gate still passes. And a send that moves a judged module
|
// another send reaches anchor, unreported; the gate still passes. And a send that moves a judged module
|
||||||
// to another build supersedes the judging: no verdict, nothing put back.
|
// to another build supersedes the judging: no verdict, nothing put back.
|
||||||
func TestReplay352(t *testing.T) {
|
func TestReplay352(t *testing.T) {
|
||||||
judgeLeavesAsTheCore(t)
|
|
||||||
t.Run("a newer send to the judged machine does not unreport the gate's", testANewerSendDoesNotUnreportTheGatesOwn)
|
t.Run("a newer send to the judged machine does not unreport the gate's", testANewerSendDoesNotUnreportTheGatesOwn)
|
||||||
t.Run("a send that moves the module supersedes the judging", testASendThatMovesTheModuleSupersedesTheJudging)
|
t.Run("a send that moves the module supersedes the judging", testASendThatMovesTheModuleSupersedesTheJudging)
|
||||||
}
|
}
|
||||||
@@ -67,11 +66,10 @@ func testANewerSendDoesNotUnreportTheGatesOwn(t *testing.T) {
|
|||||||
|
|
||||||
// A plan's own first send waits while a release judges the same module on that machine with another build.
|
// A plan's own first send waits while a release judges the same module on that machine with another build.
|
||||||
func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) {
|
func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) {
|
||||||
judgeLeavesAsTheCore(t)
|
|
||||||
b := aBacklog(t)
|
b := aBacklog(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
advancePlans(ctx, b.open) // the release judges app c2 on anchor
|
advancePlans(ctx, b.open) // the release judges app c2 on anchor
|
||||||
_, _, _, err := gatedSend(ctx, b.open, "anchor", []inventory.CarriedMove{{Module: "app", Node: "anchor", From: "c2", To: "c3", Build: "build-app-c3"}})
|
_, _, err := gatedSend(ctx, b.open, "anchor", []inventory.CarriedMove{{Module: "app", Node: "anchor", From: "c2", To: "c3", Build: "build-app-c3"}})
|
||||||
if !errors.Is(err, errWalkedElsewhere) || !strings.Contains(err.Error(), "release-") {
|
if !errors.Is(err, errWalkedElsewhere) || !strings.Contains(err.Error(), "release-") {
|
||||||
t.Fatalf("a newer build of a judged module was sent under the release's gate: %v", err)
|
t.Fatalf("a newer build of a judged module was sent under the release's gate: %v", err)
|
||||||
}
|
}
|
||||||
@@ -82,7 +80,6 @@ func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) {
|
|||||||
|
|
||||||
// A merge plan's judging is superseded the same way: another send moved its module on the first machine.
|
// A merge plan's judging is superseded the same way: another send moved its module on the first machine.
|
||||||
func TestAPlansJudgingIsSupersededByASendThatMovesItsModule(t *testing.T) {
|
func TestAPlansJudgingIsSupersededByASendThatMovesItsModule(t *testing.T) {
|
||||||
judgeLeavesAsTheCore(t)
|
|
||||||
g := aGateMesh(t)
|
g := aGateMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
inv := g.open.inventory
|
inv := g.open.inventory
|
||||||
|
|||||||
@@ -1,433 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"reflect"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// novox/hq issue 459, ADR 0299: since the merge window one walk carries every merge of its batch, so a stop of one
|
|
||||||
// group member through `delivery-stop` ended the walk of every other delivery in it. The stop now names the merges
|
|
||||||
// to leave out: the walk is ended, and every other merge it answered is walked again in a fresh batch without them
|
|
||||||
// — the stopped merge, and each later merge of its repository's branch, which contains it and ends with it.
|
|
||||||
|
|
||||||
// endsItsLine ends, as the test ends, a line the controller's output left open (a walk's ask prints " tier N: " and
|
|
||||||
// the asked build the rest, which the tests' fake ask never prints): `go test -v` would put the test's "--- PASS"
|
|
||||||
// after it, and the store-tests step of merge-check.sh, which reads that marker at a line's start, would say the
|
|
||||||
// test never ran.
|
|
||||||
func endsItsLine(t *testing.T) {
|
|
||||||
t.Cleanup(func() { fmt.Println() })
|
|
||||||
}
|
|
||||||
|
|
||||||
// leaved is a kept merge of a repository, made at a moment.
|
|
||||||
func leaved(repository, commit string, made time.Time) inventory.BatchedMerge {
|
|
||||||
return inventory.BatchedMerge{Repository: repository, Branch: "main", Commit: commit, Merged: made, Heard: made}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The stopped merge is left out, and so is each later merge of its repository's branch, naming the merge it
|
|
||||||
// contains; an earlier merge of that repository and every other repository's merge are walked again.
|
|
||||||
func TestLeftOutAreTheStoppedMergeAndTheMergesThatContainIt(t *testing.T) {
|
|
||||||
c1 := leaved("novox/mesh-catalog", "c1aaaaaaaaaa", t0)
|
|
||||||
c2 := leaved("novox/mesh-catalog", "c2bbbbbbbbbb", t0.Add(10*time.Second))
|
|
||||||
c3 := leaved("novox/mesh-catalog", "c3cccccccccc", t0.Add(20*time.Second))
|
|
||||||
o1 := leaved("novox/one", "o1dddddddddd", t0.Add(5*time.Second))
|
|
||||||
again, out, err := leaveOut([]inventory.BatchedMerge{c1, o1, c2, c3}, []string{"novox/Mesh-Catalog@c2bbbbbbbbbb"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var walked []string
|
|
||||||
for _, m := range again {
|
|
||||||
walked = append(walked, m.Commit)
|
|
||||||
}
|
|
||||||
if !reflect.DeepEqual(walked, []string{c1.Commit, o1.Commit}) {
|
|
||||||
t.Fatalf("walked again: %v; want the earlier catalogue merge and the other repository's", walked)
|
|
||||||
}
|
|
||||||
want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: c2.Commit},
|
|
||||||
{Repository: "novox/mesh-catalog", Commit: c3.Commit, Contains: c2.Commit}}
|
|
||||||
if !reflect.DeepEqual(out, want) {
|
|
||||||
t.Fatalf("left out: %+v; want %+v", out, want)
|
|
||||||
}
|
|
||||||
// A merge the walk does not answer is refused, naming it: nothing ends.
|
|
||||||
if _, _, err := leaveOut([]inventory.BatchedMerge{c1, o1}, []string{"novox/mesh-catalog@ffffffffffff"}); err == nil ||
|
|
||||||
!strings.Contains(err.Error(), "ffffffffffff") {
|
|
||||||
t.Fatalf("a merge the walk does not answer was left out: %v", err)
|
|
||||||
}
|
|
||||||
if _, _, err := leaveOut([]inventory.BatchedMerge{c1}, []string{"no-commit"}); err == nil {
|
|
||||||
t.Fatal("a merge without its commit was taken")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// moving is a kept merge that moved modules when it was heard.
|
|
||||||
func moving(m inventory.BatchedMerge, modules ...string) inventory.BatchedMerge {
|
|
||||||
m.Event, _ = json.Marshal(keptMerge{Moves: modules})
|
|
||||||
return m
|
|
||||||
}
|
|
||||||
|
|
||||||
// A merge of another repository that moves a module the stopped merge moves is left out too, naming the modules
|
|
||||||
// it shares: the module is built at its branch, which holds the stopped merge (review of #232). And so on, until
|
|
||||||
// none more is: a merge sharing a module with that one is left out as well.
|
|
||||||
func TestAMergeSharingAStoppedMergesModuleIsLeftOut(t *testing.T) {
|
|
||||||
stopped := moving(leaved("novox/mesh-catalog", "c1aaaaaaaaaa", t0), "app")
|
|
||||||
reader := moving(leaved("novox/one", "o1bbbbbbbbbb", t0.Add(time.Second)), "app", "one")
|
|
||||||
next := moving(leaved("novox/two", "t1cccccccccc", t0.Add(2*time.Second)), "one")
|
|
||||||
free := moving(leaved("novox/three", "h1dddddddddd", t0.Add(3*time.Second)), "three")
|
|
||||||
again, out, err := leaveOut([]inventory.BatchedMerge{stopped, reader, next, free}, []string{"novox/mesh-catalog@c1aaaaaaa"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(again) != 1 || again[0].Commit != free.Commit {
|
|
||||||
t.Fatalf("walked again: %+v; want the merge that shares nothing alone", again)
|
|
||||||
}
|
|
||||||
want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: stopped.Commit},
|
|
||||||
{Repository: "novox/one", Commit: reader.Commit, Contains: stopped.Commit, Shares: []string{"app"}},
|
|
||||||
{Repository: "novox/two", Commit: next.Commit, Contains: stopped.Commit, Shares: []string{"one"}}}
|
|
||||||
if !reflect.DeepEqual(out, want) {
|
|
||||||
t.Fatalf("left out %+v; want %+v", out, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A walk withholds a module a stopped merge moved where it would build it only as a dependent; one a merge of its
|
|
||||||
// own batch moves is built, and empty tiers go.
|
|
||||||
func TestAWalkWithholdsAStoppedMergesDependents(t *testing.T) {
|
|
||||||
plan := inventory.Plan{Tiers: [][]string{{"one"}, {"app"}, {"notes", "app2"}},
|
|
||||||
Modules: map[string]*inventory.PlanModule{"one": {}, "app": {}, "notes": {}, "app2": {}}}
|
|
||||||
got := withhold(&plan, []string{"app", "notes", "absent"}, []string{"notes"})
|
|
||||||
if !reflect.DeepEqual(got, []string{"app"}) || !reflect.DeepEqual(plan.Tiers, [][]string{{"one"}, {"notes", "app2"}}) {
|
|
||||||
t.Fatalf("withheld %v, tiers %v", got, plan.Tiers)
|
|
||||||
}
|
|
||||||
if _, in := plan.Modules["app"]; in {
|
|
||||||
t.Fatal("a withheld module is still in the plan")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The seat verb passes the merges to leave out to the command.
|
|
||||||
func TestDeliveryStopTakesTheMergesToLeaveOut(t *testing.T) {
|
|
||||||
got, err := argvFor("delivery-stop", map[string]any{"plan": "plan-1", "why": "w", "by": "jochen",
|
|
||||||
"without": "novox/mesh-lab@b1,novox/mesh-lab@b2"})
|
|
||||||
want := []string{"delivery", "stop", "plan-1", "--why", "w", "--by", "mesh-delivery for jochen",
|
|
||||||
"--without", "novox/mesh-lab@b1,novox/mesh-lab@b2"}
|
|
||||||
if err != nil || !reflect.DeepEqual(got, want) {
|
|
||||||
t.Fatalf("delivery-stop with without → %v %v, wanted %v", got, err, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A walk of the catalogue's merge and another repository's is stopped without the catalogue's: it ends stopped,
|
|
||||||
// names the batch that walks the other merge again and the merge it left out, and that batch is cut into a walk
|
|
||||||
// that builds the other repository's module alone.
|
|
||||||
func TestAStopWithoutAMergeWalksTheOthersAgain(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
groupsWaiting(t, "pr/cat1aaaaaaaa")
|
|
||||||
endsItsLine(t)
|
|
||||||
asksRecorded(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
|
|
||||||
one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head)
|
|
||||||
hear(t, open, cat, t0.Add(time.Second))
|
|
||||||
hear(t, open, one, t0.Add(6*time.Second))
|
|
||||||
cutAt(t, open, t0.Add(2*time.Minute))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
if len(ws) != 1 || !ws[0].Open() {
|
|
||||||
t.Fatalf("no open walk to stop: %+v", ws)
|
|
||||||
}
|
|
||||||
stopped := ws[0]
|
|
||||||
p, said, err := stopWalkWithout(ctx, open.inventory, stopped.ID, "mesh-delivery for jochen", "its group stopped",
|
|
||||||
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if p.State != inventory.PlanFailed || p.Delivery.Stopped == "" || p.Delivery.WalkedAgainBy == "" ||
|
|
||||||
!reflect.DeepEqual(p.Delivery.LeftOut, []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: cat.Commit}}) {
|
|
||||||
t.Fatalf("the stopped walk reads %s %+v", p.State, p.Delivery)
|
|
||||||
}
|
|
||||||
if said.WalkedAgainBy != p.Delivery.WalkedAgainBy || !reflect.DeepEqual(said.WalkedAgain, []string{"novox/one@" + one.Commit}) {
|
|
||||||
t.Fatalf("the stop answered %+v", said)
|
|
||||||
}
|
|
||||||
again, err := open.inventory.MergesOf(ctx, p.Delivery.WalkedAgainBy)
|
|
||||||
if err != nil || len(again) != 1 || again[0].Commit != one.Commit {
|
|
||||||
t.Fatalf("the fresh batch answers %+v (%v); want the other repository's merge alone", again, err)
|
|
||||||
}
|
|
||||||
if left, _ := open.inventory.MergesOf(ctx, stopped.ID); len(left) != 1 || left[0].Commit != cat.Commit {
|
|
||||||
t.Fatalf("the stopped walk still answers %+v; want the left-out merge alone", left)
|
|
||||||
}
|
|
||||||
cutAt(t, open, t0.Add(4*time.Minute))
|
|
||||||
fresh, err := open.inventory.PlanByID(ctx, p.Delivery.WalkedAgainBy)
|
|
||||||
if err != nil || fresh.Batch() || !fresh.Open() {
|
|
||||||
t.Fatalf("the fresh batch was not cut into a walk: %s %v", fresh.State, err)
|
|
||||||
}
|
|
||||||
if _, in := fresh.Modules["one"]; !in {
|
|
||||||
t.Fatalf("the fresh walk does not build one: %v", fresh.Modules)
|
|
||||||
}
|
|
||||||
if _, in := fresh.Modules["app"]; in {
|
|
||||||
t.Fatalf("the fresh walk builds app, whose merge was left out: %v", fresh.Modules)
|
|
||||||
}
|
|
||||||
if len(fresh.Delivery.Merges) != 1 || fresh.Delivery.Merges[0].Commit != one.Commit {
|
|
||||||
t.Fatalf("the fresh walk answers %+v", fresh.Delivery.Merges)
|
|
||||||
}
|
|
||||||
// A stopped walk starts no search of its own (ADR 0276 decision 3 is for a failed one).
|
|
||||||
cutAt(t, open, t0.Add(5*time.Minute))
|
|
||||||
if alone, _ := open.inventory.AloneMerges(ctx); len(alone) != 0 {
|
|
||||||
t.Fatalf("a merge of the stopped walk waits to be walked alone: %+v", alone)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A later merge of the stopped merge's repository contains it: it is left out too, named with the merge it
|
|
||||||
// contains, and only the other repository's merge is walked again. With nothing left to walk, the walk is only
|
|
||||||
// ended. A merge the walk does not answer is refused, and nothing ends.
|
|
||||||
func TestAStopWithoutAMergeEndsTheMergesThatContainIt(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
groupsWaiting(t, "pr/cat1aaaaaaaa")
|
|
||||||
endsItsLine(t)
|
|
||||||
asksRecorded(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
first := catalogueMerge("cat1aaaaaaaa", "app", t0)
|
|
||||||
later := catalogueMerge("cat2cccccccc", "notes", t0.Add(10*time.Second))
|
|
||||||
one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), first.Head)
|
|
||||||
hear(t, open, first, t0.Add(time.Second))
|
|
||||||
hear(t, open, one, t0.Add(6*time.Second))
|
|
||||||
hear(t, open, later, t0.Add(11*time.Second))
|
|
||||||
cutAt(t, open, t0.Add(2*time.Minute))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
walk := ws[0]
|
|
||||||
|
|
||||||
if _, _, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "x",
|
|
||||||
[]string{"novox/mesh-catalog@ffffffffffff"}, t0.Add(3*time.Minute)); err == nil {
|
|
||||||
t.Fatal("a stop leaving out a merge the walk does not answer was taken")
|
|
||||||
}
|
|
||||||
if p, _ := open.inventory.PlanByID(ctx, walk.ID); !p.Open() {
|
|
||||||
t.Fatalf("a refused stop ended the walk: %s", p.State)
|
|
||||||
}
|
|
||||||
|
|
||||||
p, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped",
|
|
||||||
[]string{"novox/mesh-catalog@" + first.Commit}, t0.Add(3*time.Minute))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: first.Commit},
|
|
||||||
{Repository: "novox/mesh-catalog", Commit: later.Commit, Contains: first.Commit}}
|
|
||||||
if !reflect.DeepEqual(p.Delivery.LeftOut, want) || !reflect.DeepEqual(said.LeftOut, want) {
|
|
||||||
t.Fatalf("left out %+v, said %+v; want %+v", p.Delivery.LeftOut, said.LeftOut, want)
|
|
||||||
}
|
|
||||||
if !strings.Contains(p.Note, later.Commit[:8]) || !strings.Contains(p.Note, p.Delivery.WalkedAgainBy) {
|
|
||||||
t.Fatalf("the stopped walk's note does not name the merge ended with it and the fresh batch: %q", p.Note)
|
|
||||||
}
|
|
||||||
again, _ := open.inventory.MergesOf(ctx, p.Delivery.WalkedAgainBy)
|
|
||||||
if len(again) != 1 || again[0].Commit != one.Commit {
|
|
||||||
t.Fatalf("walked again: %+v; want the other repository's merge alone", again)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Nothing left once the other repository's merge is left out too: the fresh walk is stopped, and nothing more.
|
|
||||||
cutAt(t, open, t0.Add(4*time.Minute))
|
|
||||||
fresh, _ := open.inventory.PlanByID(ctx, p.Delivery.WalkedAgainBy)
|
|
||||||
q, said, err := stopWalkWithout(ctx, open.inventory, fresh.ID, "mesh-delivery for jochen", "that one too",
|
|
||||||
[]string{"novox/one@" + one.Commit}, t0.Add(5*time.Minute))
|
|
||||||
if err != nil || q.State != inventory.PlanFailed || q.Delivery.WalkedAgainBy != "" || said.WalkedAgainBy != "" {
|
|
||||||
t.Fatalf("a stop leaving nothing to walk: %s %+v %+v %v", q.State, q.Delivery, said, err)
|
|
||||||
}
|
|
||||||
if _, bs := walks(t, open); len(bs) != 0 {
|
|
||||||
t.Fatalf("a batch of nothing was made: %+v", bs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A merge heard while the walk ran waits in the open batch, at a later commit of the same repository. The merge
|
|
||||||
// walked again joins that batch, never a batch of its own that would walk the older commit after the newer one
|
|
||||||
// (review of #232): its window closes, and its walk is at the newer commit, answering both.
|
|
||||||
func TestAStopWalksTheOthersAgainInTheOpenBatch(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
groupsWaiting(t, "pr/cat1aaaaaaaa")
|
|
||||||
endsItsLine(t)
|
|
||||||
asksRecorded(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
|
|
||||||
one1 := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head)
|
|
||||||
hear(t, open, cat, t0.Add(time.Second))
|
|
||||||
hear(t, open, one1, t0.Add(6*time.Second))
|
|
||||||
cutAt(t, open, t0.Add(2*time.Minute))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
walk := ws[0]
|
|
||||||
one2 := repoMerge("one", "one2cccccccc", t0.Add(3*time.Minute))
|
|
||||||
hear(t, open, one2, t0.Add(3*time.Minute))
|
|
||||||
_, bs := walks(t, open)
|
|
||||||
if len(bs) != 1 {
|
|
||||||
t.Fatalf("the merge heard while the walk ran is in %d batches", len(bs))
|
|
||||||
}
|
|
||||||
waiting := bs[0]
|
|
||||||
p, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped",
|
|
||||||
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if p.Delivery.WalkedAgainBy != waiting.ID || said.WalkedAgainBy != waiting.ID {
|
|
||||||
t.Fatalf("walked again by %q (said %q); want the open batch %s", p.Delivery.WalkedAgainBy, said.WalkedAgainBy, waiting.ID)
|
|
||||||
}
|
|
||||||
if _, bs := walks(t, open); len(bs) != 1 {
|
|
||||||
t.Fatalf("%d batches; want the open one alone", len(bs))
|
|
||||||
}
|
|
||||||
b, _ := open.inventory.PlanByID(ctx, waiting.ID)
|
|
||||||
if b.Delivery.Batch == nil || !b.Delivery.Batch.Closed || !slices.Contains(b.Delivery.Batch.Withheld, "app") {
|
|
||||||
t.Fatalf("the open batch reads %+v; want its window closed and app withheld", b.Delivery.Batch)
|
|
||||||
}
|
|
||||||
// Its window closed by the stop, not by the clock: cut at once.
|
|
||||||
cutAt(t, open, t0.Add(3*time.Minute+11*time.Second))
|
|
||||||
fresh, _ := open.inventory.PlanByID(ctx, waiting.ID)
|
|
||||||
if fresh.Batch() || fresh.CommitOf("novox/one") != one2.Commit {
|
|
||||||
t.Fatalf("the batch is %s at %q; want it cut, walking the newer commit", fresh.State, fresh.CommitOf("novox/one"))
|
|
||||||
}
|
|
||||||
var answered []string
|
|
||||||
for _, m := range fresh.Delivery.Merges {
|
|
||||||
answered = append(answered, m.Commit+">"+m.Carried)
|
|
||||||
}
|
|
||||||
if !slices.Contains(answered, one1.Commit+">"+one2.Commit) || !slices.Contains(answered, one2.Commit+">") {
|
|
||||||
t.Fatalf("the walk answers %v; want the earlier merge carried by the later", answered)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A stop names what the stopped walk already sent and nothing walks again, on which machines; and a later merge of
|
|
||||||
// the stopped merge's repository that waits in a batch, which will deliver the stopped change. A stopped walk
|
|
||||||
// that walked its merges again is refused a retry, saying why.
|
|
||||||
func TestAStopNamesWhatStaysOnTheMachinesAndWhatStillCarriesIt(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
groupsWaiting(t, "pr/cat1aaaaaaaa")
|
|
||||||
endsItsLine(t)
|
|
||||||
asksRecorded(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
|
|
||||||
one1 := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head)
|
|
||||||
hear(t, open, cat, t0.Add(time.Second))
|
|
||||||
hear(t, open, one1, t0.Add(6*time.Second))
|
|
||||||
cutAt(t, open, t0.Add(2*time.Minute))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
walk := ws[0]
|
|
||||||
sent := t0.Add(150 * time.Second)
|
|
||||||
for name, m := range walk.Modules {
|
|
||||||
if name == "app" || name == "one" {
|
|
||||||
m.First, m.FirstAt = []string{"anchor"}, &sent
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := open.inventory.SavePlan(ctx, &walk); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
cat2 := catalogueMerge("cat2dddddddd", "notes", t0.Add(3*time.Minute))
|
|
||||||
hear(t, open, cat2, t0.Add(3*time.Minute))
|
|
||||||
_, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped",
|
|
||||||
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !reflect.DeepEqual(said.Kept, []inventory.PlanKept{{Module: "app", Machines: []string{"anchor"}}}) {
|
|
||||||
t.Fatalf("kept %+v; want app on anchor alone (one is walked again)", said.Kept)
|
|
||||||
}
|
|
||||||
p, _ := open.inventory.PlanByID(ctx, walk.ID)
|
|
||||||
if !reflect.DeepEqual(p.Delivery.Kept, said.Kept) || !strings.Contains(p.Note, "app on anchor") {
|
|
||||||
t.Fatalf("the stopped walk's record does not name what stays: %+v %q", p.Delivery.Kept, p.Note)
|
|
||||||
}
|
|
||||||
if len(said.StillCarriedBy) != 1 || !strings.Contains(said.StillCarriedBy[0], cat2.Commit) {
|
|
||||||
t.Fatalf("still carried by %v; want the later catalogue merge waiting in the batch", said.StillCarriedBy)
|
|
||||||
}
|
|
||||||
if _, err := retryPlan(ctx, open, walk.ID); err == nil || !strings.Contains(err.Error(), "was stopped without") {
|
|
||||||
t.Fatalf("a walk stopped without some merges was retried, or refused for another reason: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A batch not yet cut is stopped without a merge: the merge goes to a stopped record of its own, the batch goes on
|
|
||||||
// with the others and withholds the stopped merge's modules; with every merge left out, the batch is stopped.
|
|
||||||
func TestAStopWithoutAMergeLeavesItOutOfABatch(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
endsItsLine(t)
|
|
||||||
asksRecorded(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
|
|
||||||
one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head)
|
|
||||||
hear(t, open, cat, t0.Add(time.Second))
|
|
||||||
hear(t, open, one, t0.Add(6*time.Second))
|
|
||||||
_, bs := walks(t, open)
|
|
||||||
b := bs[0]
|
|
||||||
r, said, err := stopWalkWithout(ctx, open.inventory, b.ID, "mesh-delivery for jochen", "its group stopped",
|
|
||||||
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(10*time.Second))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if said.WalkedAgainBy != b.ID || said.Stopped == b.ID {
|
|
||||||
t.Fatalf("said %+v; want a stopped record of its own, and the batch walking the others", said)
|
|
||||||
}
|
|
||||||
if left, _ := open.inventory.MergesOf(ctx, said.Stopped); len(left) != 1 || left[0].Commit != cat.Commit {
|
|
||||||
t.Fatalf("the stopped record holds %+v", left)
|
|
||||||
}
|
|
||||||
if stopped, _ := open.inventory.PlanByID(ctx, said.Stopped); stopped.State != inventory.PlanFailed ||
|
|
||||||
stopped.Delivery.Stopped == "" || stopped.Delivery.WalkedAgainBy != b.ID {
|
|
||||||
t.Fatalf("the stopped record reads %s %+v", stopped.State, stopped.Delivery)
|
|
||||||
}
|
|
||||||
_ = r
|
|
||||||
cutAt(t, open, t0.Add(3*time.Minute))
|
|
||||||
w, _ := open.inventory.PlanByID(ctx, b.ID)
|
|
||||||
if _, in := w.Modules["app"]; in || w.Batch() {
|
|
||||||
t.Fatalf("the batch's walk is %s and builds %v; want it cut without app", w.State, w.Modules)
|
|
||||||
}
|
|
||||||
// A batch whose every merge is left out is stopped itself.
|
|
||||||
hear(t, open, catalogueMerge("cat3eeeeeeee", "notes", t0.Add(4*time.Minute)), t0.Add(4*time.Minute))
|
|
||||||
_, bs = walks(t, open)
|
|
||||||
last := bs[0]
|
|
||||||
if _, said, err := stopWalkWithout(ctx, open.inventory, last.ID, "mesh-delivery for jochen", "x",
|
|
||||||
[]string{"novox/mesh-catalog@cat3eeeeeeee"}, t0.Add(4*time.Minute+time.Second)); err != nil || said.Stopped != last.ID {
|
|
||||||
t.Fatalf("a batch left with nothing: %+v %v", said, err)
|
|
||||||
}
|
|
||||||
if p, _ := open.inventory.PlanByID(ctx, last.ID); p.State != inventory.PlanFailed || p.Batch() {
|
|
||||||
t.Fatalf("the emptied batch is %s", p.State)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The stopped walk, the batch and the merges moved are one act (ADR 0299 decision 7): a plan written by somebody
|
|
||||||
// else since it was read refuses all of it, and no merge moves, no plan is written.
|
|
||||||
func TestSavingPlansAndMovingMergesIsOneAct(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
endsItsLine(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
hear(t, open, catalogueMerge("cat1aaaaaaaa", "app", t0), t0.Add(time.Second))
|
|
||||||
_, bs := walks(t, open)
|
|
||||||
stale := bs[0]
|
|
||||||
moved := stale
|
|
||||||
moved.Note = "written by somebody else"
|
|
||||||
if err := open.inventory.SavePlan(ctx, &moved); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
fresh := inventory.Plan{ID: "plan-fresh", Repository: "novox/mesh-catalog", Branch: "main", Commit: "x", Created: t0,
|
|
||||||
State: inventory.PlanAssembling, Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{}}
|
|
||||||
err := open.inventory.SavePlansMoving(ctx, []*inventory.Plan{&fresh, &stale},
|
|
||||||
[]inventory.MergeMove{{Repository: "novox/mesh-catalog", Commit: "cat1aaaaaaaa", Plan: "plan-fresh"}})
|
|
||||||
if !errors.Is(err, inventory.ErrPlanMoved) {
|
|
||||||
t.Fatalf("a stale plan in the act: %v", err)
|
|
||||||
}
|
|
||||||
if _, err := open.inventory.PlanByID(ctx, "plan-fresh"); err == nil {
|
|
||||||
t.Fatal("the new plan was written though the act was refused")
|
|
||||||
}
|
|
||||||
if m, _, _ := open.inventory.MergeOf(ctx, "novox/mesh-catalog", "cat1aaaaaaaa"); m.Plan != stale.ID {
|
|
||||||
t.Fatalf("the merge moved to %q though the act was refused", m.Plan)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// What a stopped walk leaves on the machines names every machine its sends reached (review of #232): a module sent
|
|
||||||
// with no first machine reached every machine running it; one sent first reached those, and the rest it was sent
|
|
||||||
// to; a module a merge walked again moves is sent again, and is not named.
|
|
||||||
func TestKeptNamesEveryMachineASendReached(t *testing.T) {
|
|
||||||
at := t0
|
|
||||||
p := inventory.Plan{Modules: map[string]*inventory.PlanModule{
|
|
||||||
"together": {SentAt: &at},
|
|
||||||
"first": {First: []string{"anchor"}, FirstAt: &at, Rest: map[string]inventory.SentDeclaration{"laptop": {}}},
|
|
||||||
"again": {SentAt: &at},
|
|
||||||
"unsent": {},
|
|
||||||
}}
|
|
||||||
got := keptOf(p, []string{"again"}, map[string][]string{"together": {"laptop", "anchor"}, "again": {"anchor"}})
|
|
||||||
want := []inventory.PlanKept{{Module: "first", Machines: []string{"anchor", "laptop"}},
|
|
||||||
{Module: "together", Machines: []string{"anchor", "laptop"}}}
|
|
||||||
if !reflect.DeepEqual(got, want) {
|
|
||||||
t.Fatalf("kept %+v; want %+v", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,102 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// novox/hq issue 529: a walk waiting because its delivery is held for a person said only "its walk waits for
|
|
||||||
// mesh-delivery's word — `plans go … --why …` starts it by hand", pointing a reader at a hand start instead of at
|
|
||||||
// the person's decision. The line names what mesh-delivery says holds the walk: the held delivery, its reason and
|
|
||||||
// `mesh-delivery.release`; or the delivery not at its turn.
|
|
||||||
|
|
||||||
func waitingWalk(now time.Time) inventory.Plan {
|
|
||||||
return inventory.Plan{ID: "plan-1791793277221379309", Repository: "novox/mesh-sdk", Branch: "main",
|
|
||||||
Commit: "9747f3e1b1ac", State: inventory.PlanBuilding, Tiers: [][]string{{"mesh-sdk"}},
|
|
||||||
Created: now.Add(-9 * time.Minute), Modules: map[string]*inventory.PlanModule{},
|
|
||||||
Delivery: &inventory.PlanDelivery{Awaits: "mesh-delivery"}}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAWalkWaitingOnAHeldDeliveryNamesTheHold(t *testing.T) {
|
|
||||||
now := time.Date(2026, 10, 12, 8, 35, 0, 0, time.UTC)
|
|
||||||
p := waitingWalk(now)
|
|
||||||
held := "novox/mesh-sdk@d41d255f28ad held since 2026-10-12T08:23:00Z: its group adr-0307-state-history's composed " +
|
|
||||||
"check did not pass for the heads that merged — a person decides: `mesh-delivery.release` (id " +
|
|
||||||
"novox/mesh-sdk@d41d255f28ad, why) lets it go, `mesh-delivery.stop` ends it"
|
|
||||||
waits := walkWaits{p.ID: {{ID: "novox/mesh-sdk@d41d255f28ad", State: "held", Walk: p.ID, Says: held}}}
|
|
||||||
line := planLineSaid(p, now, pauseView{}, tierAtLeast, nil, waits)
|
|
||||||
for _, want := range []string{"held for a person", "composed check did not pass", "mesh-delivery.release"} {
|
|
||||||
if !strings.Contains(line, want) {
|
|
||||||
t.Fatalf("the line does not say %q: %q", want, line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if strings.Contains(line, "starts it by hand") {
|
|
||||||
t.Fatalf("a held walk's line points at a hand start: %q", line)
|
|
||||||
}
|
|
||||||
// Told nothing — an older mesh-delivery, or none answering — it reads as before.
|
|
||||||
if line := planLineSaid(p, now, pauseView{}, tierAtLeast, nil, nil); !strings.Contains(line, "starts it by hand") {
|
|
||||||
t.Fatalf("told nothing, the line reads %q", line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAWalkWaitingOnADeliveryNotAtItsTurnSaysSo(t *testing.T) {
|
|
||||||
now := time.Date(2026, 10, 12, 8, 57, 0, 0, time.UTC)
|
|
||||||
p := waitingWalk(now)
|
|
||||||
waits := walkWaits{p.ID: {{ID: "novox/mesh-sdk@9747f3e1b1ac", State: "published", Walk: p.ID,
|
|
||||||
Says: "not let go yet: novox/mesh-tools@3f0c1a6cec18 (published) carried by it not at its turn"}}}
|
|
||||||
line := planLineSaid(p, now, pauseView{}, tierAtLeast, nil, waits)
|
|
||||||
if !strings.Contains(line, "not at its turn") || !strings.Contains(line, "novox/mesh-tools@3f0c1a6cec18") {
|
|
||||||
t.Fatalf("the line reads %q", line)
|
|
||||||
}
|
|
||||||
// No hand start offered: it would skip the group's order (review of issue 529).
|
|
||||||
if strings.Contains(line, "plans go") {
|
|
||||||
t.Fatalf("a walk not at its turn points at a hand start: %q", line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A walk nothing holds — the nzbgeek case, its merges' deliveries ended as nothing-to-move — is said so, and the hand
|
|
||||||
// start is the way on.
|
|
||||||
func TestAWalkHeldByNothingSaysWhyAndTheHandStart(t *testing.T) {
|
|
||||||
now := time.Date(2026, 10, 11, 20, 30, 0, 0, time.UTC)
|
|
||||||
p := waitingWalk(now)
|
|
||||||
waits := walkWaits{p.ID: {{ID: "novox/mesh-media-catalog@cdcdcdcd0000", State: "unheld", Walk: p.ID,
|
|
||||||
Says: "held by nothing: its merges' deliveries ended as nothing-to-move (novox/mesh-media-catalog@abababab0000), " +
|
|
||||||
"yet the walk moves nzbgeek, nzbget — nothing gives it its word"}}}
|
|
||||||
line := planLineSaid(p, now, pauseView{}, tierAtLeast, nil, waits)
|
|
||||||
if !strings.Contains(line, "ended as nothing-to-move") || !strings.Contains(line, "plans go "+p.ID) {
|
|
||||||
t.Fatalf("the line reads %q", line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// What holds each waiting walk is asked of mesh-delivery's `stalled` with `walks`, a verb the controller's grant
|
|
||||||
// already names; only lines naming a walk are kept, by walk.
|
|
||||||
func TestWhatHoldsTheWaitingWalksIsAskedOfTheDeliveryOwner(t *testing.T) {
|
|
||||||
was := askDeliveryOwner
|
|
||||||
defer func() { askDeliveryOwner = was }()
|
|
||||||
var asked map[string]any
|
|
||||||
askDeliveryOwner = func(_ context.Context, _ *nats.Conn, verb string, args map[string]any) (json.RawMessage, error) {
|
|
||||||
if verb != "stalled" {
|
|
||||||
t.Fatalf("asked %s", verb)
|
|
||||||
}
|
|
||||||
asked = args
|
|
||||||
return json.Marshal([]stalledLine{{ID: "novox/a@1", State: "held", Walk: "plan-1", Says: "held: why"},
|
|
||||||
{ID: "novox/b@2", State: "held", Says: "a delivery past its bound, no walk"}})
|
|
||||||
}
|
|
||||||
got := walksWaiting(t.Context(), nil, true)
|
|
||||||
if asked["walks"] != true {
|
|
||||||
t.Fatalf("stalled was asked with %v", asked)
|
|
||||||
}
|
|
||||||
if len(got) != 1 || len(got["plan-1"]) != 1 || got["plan-1"][0].Says != "held: why" {
|
|
||||||
t.Fatalf("the waiting walks are %+v", got)
|
|
||||||
}
|
|
||||||
if got := walksWaiting(t.Context(), nil, false); got != nil {
|
|
||||||
t.Fatalf("with no holder on record it asked: %+v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -77,7 +77,7 @@ func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) {
|
|||||||
}
|
}
|
||||||
cancel() // the sender is going away: its context is cancelled between the send and the record
|
cancel() // the sender is going away: its context is cancelled between the send and the record
|
||||||
body := []byte(`{"declaration":1,"resources":[]}`)
|
body := []byte(`{"declaration":1,"resources":[]}`)
|
||||||
digest, err := recordSent(ctx, inv, "anchor", body, nil, 0, sentRecord{sender: "a test"})
|
digest, err := recordSent(ctx, inv, "anchor", body, nil, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// NodeByName on the cancelled context may itself refuse; the record must still be possible
|
// NodeByName on the cancelled context may itself refuse; the record must still be possible
|
||||||
// through the detached context, so look the node up again on a live one.
|
// through the detached context, so look the node up again on a live one.
|
||||||
|
|||||||
@@ -1,272 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
"github.com/novox/mesh-controller/internal/secrets"
|
|
||||||
)
|
|
||||||
|
|
||||||
// `secret issued <provision>`: the credential a provider's software issued, taken from the provider and kept as
|
|
||||||
// the provision's one credential for every consumer (novox/hq ADR 0305, issue 526).
|
|
||||||
//
|
|
||||||
// **The value comes from the provider, never from whoever asks.** A module served by its machine's runtime calls
|
|
||||||
// under the runtime's one bus account, which carries that machine's agents' calls too, so a value handed to a
|
|
||||||
// verb could be anybody's. This line carries none: it asks the provider's own tool, on the provider's machine, in
|
|
||||||
// the controller's own name — the subject that names its caller, which the bus lets the controller alone publish
|
|
||||||
// (issue 365) — with a sealing key made for this one call. The tool answers only the controller, the value sealed
|
|
||||||
// to that key; the controller opens it, seals it to every holder and forgets it, as the desk's value is (ADR 0277
|
|
||||||
// §23). Asking again fetches the same value again, and a value every holder already holds is not sealed anew.
|
|
||||||
//
|
|
||||||
// **What this trusts: the provider's machine.** The answer is taken from whoever answers the provider's tool
|
|
||||||
// subject on the provider's machine first. On that machine its runtime serves the subject for every module it
|
|
||||||
// carries, so anything that runtime runs, or anything else granted that subject there, could answer before the
|
|
||||||
// provider does and hand the controller a value of its own. ADR 0305 accepts that by trusting the provider's
|
|
||||||
// machine, as the provider itself is trusted; the bus keeps every other machine from answering it.
|
|
||||||
|
|
||||||
// handActIssued is the verb a change of an issued value is recorded under: the mesh working as decided.
|
|
||||||
const handActIssued = "secret issued"
|
|
||||||
|
|
||||||
// issuedAsk is how long the provider's tool is given to answer.
|
|
||||||
const issuedAsk = 20 * time.Second
|
|
||||||
|
|
||||||
// issuedProvider is one assignment of a module whose offer says its software issues the provision's credential.
|
|
||||||
type issuedProvider struct {
|
|
||||||
Node, Module, Tool string
|
|
||||||
}
|
|
||||||
|
|
||||||
// issuedTaker is the line's work, with what it reaches handed in: the store, the bus and the sends.
|
|
||||||
type issuedTaker struct {
|
|
||||||
// providers are the assignments to ask, and whether any module offers the provision so at all.
|
|
||||||
providers func(provision string) ([]issuedProvider, bool, error)
|
|
||||||
ask func(p issuedProvider, args map[string]any) (link.Answer, error)
|
|
||||||
keep func(provision, provider, value string) (inventory.IssuedKept, error)
|
|
||||||
record func(act link.HandAct) error
|
|
||||||
send func(machines []string) error
|
|
||||||
}
|
|
||||||
|
|
||||||
// take asks every provider of the provision for its software's value and keeps each; it answers what it did, in
|
|
||||||
// words that never carry the value or its fingerprint.
|
|
||||||
func (t issuedTaker) take(provision string) (string, error) {
|
|
||||||
providers, offered, err := t.providers(provision)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if !offered {
|
|
||||||
return "", fmt.Errorf("no module offers %q with a credential its software issues (\"credential\": "+
|
|
||||||
"{\"issued-by\": \"software\", \"tool\": …}, ADR 0305), so there is nothing to take from a provider; "+
|
|
||||||
"nothing was asked", provision)
|
|
||||||
}
|
|
||||||
if len(providers) == 0 {
|
|
||||||
return fmt.Sprintf("no consumer holds %s from a provider whose software issues it yet, so there is nothing "+
|
|
||||||
"to seal; a consumer that binds holds a value the mesh made until the provider next asks", provision), nil
|
|
||||||
}
|
|
||||||
var said []string
|
|
||||||
var failed []string
|
|
||||||
for _, p := range providers {
|
|
||||||
words, err := t.takeFrom(provision, p)
|
|
||||||
if err != nil {
|
|
||||||
failed = append(failed, fmt.Sprintf("%s on %s: %v", p.Module, p.Node, err))
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
said = append(said, words)
|
|
||||||
}
|
|
||||||
out := strings.Join(said, "\n")
|
|
||||||
if len(failed) > 0 {
|
|
||||||
return out, fmt.Errorf("%s was not taken from every provider:\n %s", provision, strings.Join(failed, "\n "))
|
|
||||||
}
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// takeFrom is one provider: asked, opened, kept, recorded and sent when it changed.
|
|
||||||
func (t issuedTaker) takeFrom(provision string, p issuedProvider) (string, error) {
|
|
||||||
public, private, err := secrets.Keypair()
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("no key could be made to take the value: %w", err)
|
|
||||||
}
|
|
||||||
answer, err := t.ask(p, map[string]any{"provision": provision, "seal_to": public})
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("its tool %s could not be asked: %w", p.Tool, err)
|
|
||||||
}
|
|
||||||
if answer.Error != "" {
|
|
||||||
return "", fmt.Errorf("its tool %s refused: %s", p.Tool, answer.Error)
|
|
||||||
}
|
|
||||||
var given struct {
|
|
||||||
Sealed string `json:"sealed"`
|
|
||||||
}
|
|
||||||
if err := unmarshalAnswer(answer, &given); err != nil || given.Sealed == "" {
|
|
||||||
return "", fmt.Errorf("its tool %s answered no sealed value", p.Tool)
|
|
||||||
}
|
|
||||||
opened, err := secrets.Open(private, given.Sealed)
|
|
||||||
if err != nil {
|
|
||||||
// Never the value, never what failed to open: only that it was not sealed to this call.
|
|
||||||
return "", fmt.Errorf("its tool %s answered a value not sealed to this call; nothing was taken", p.Tool)
|
|
||||||
}
|
|
||||||
value := strings.TrimSpace(string(opened))
|
|
||||||
for i := range opened {
|
|
||||||
opened[i] = 0
|
|
||||||
}
|
|
||||||
if value == "" {
|
|
||||||
return "", fmt.Errorf("its tool %s answered an empty value; nothing was taken", p.Tool)
|
|
||||||
}
|
|
||||||
kept, err := t.keep(provision, p.Node, value)
|
|
||||||
value = ""
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
holders := make([]string, 0, len(kept.Holders))
|
|
||||||
for _, h := range kept.Holders {
|
|
||||||
holders = append(holders, fmt.Sprintf("%s on %s%s", h.ConsumerModule, h.Consumer, asLocal(h.Local)))
|
|
||||||
}
|
|
||||||
words := ""
|
|
||||||
if len(kept.Unsealable) > 0 {
|
|
||||||
var names []string
|
|
||||||
for _, h := range kept.Unsealable {
|
|
||||||
names = append(names, fmt.Sprintf("%s on %s", h.ConsumerModule, h.Consumer))
|
|
||||||
}
|
|
||||||
words = fmt.Sprintf("\n not sealed to %s: no sealing key on that machine", strings.Join(names, ", "))
|
|
||||||
}
|
|
||||||
if !kept.Changed {
|
|
||||||
return fmt.Sprintf("%s from %s on %s: every holder already holds the value its software issued (%s); "+
|
|
||||||
"nothing was sent%s", provision, p.Module, p.Node, strings.Join(holders, ", "), words), nil
|
|
||||||
}
|
|
||||||
sealedTo := make([]string, 0, len(kept.Sealed))
|
|
||||||
for _, h := range kept.Sealed {
|
|
||||||
sealedTo = append(sealedTo, fmt.Sprintf("%s on %s%s", h.ConsumerModule, h.Consumer, asLocal(h.Local)))
|
|
||||||
}
|
|
||||||
recorded := ""
|
|
||||||
if err := t.record(link.HandAct{Verb: handActIssued, Args: []string{provision},
|
|
||||||
Why: fmt.Sprintf("%s on %s gave the credential its software issued for %s; sealed to %s", p.Module, p.Node,
|
|
||||||
provision, strings.Join(sealedTo, ", ")),
|
|
||||||
Cause: "issued-by-software"}); err != nil {
|
|
||||||
recorded = fmt.Sprintf("\n this was NOT recorded in the hand-act log, and is done anyway: %v", err)
|
|
||||||
}
|
|
||||||
if err := t.send(kept.Machines); err != nil {
|
|
||||||
return "", fmt.Errorf("%s from %s on %s is sealed to %s and could not be sent: %w; `push --behind` sends it",
|
|
||||||
provision, p.Module, p.Node, strings.Join(sealedTo, ", "), err)
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("%s from %s on %s: the value its software issued is sealed to %s; sent %s, and each of "+
|
|
||||||
"them starts again on it%s%s", provision, p.Module, p.Node, strings.Join(sealedTo, ", "),
|
|
||||||
strings.Join(kept.Machines, ", "), words, recorded), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// issuedProvidersOf are the assignments to ask for a provision: every machine providing it to a pair the mesh
|
|
||||||
// holds, and on it the assigned module whose offer says its software issues the credential (ADR 0305 §1).
|
|
||||||
func issuedProvidersOf(ctx context.Context, inv *inventory.Inventory, provision string) ([]issuedProvider, bool, error) {
|
|
||||||
shelf, err := inv.Catalogue(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, false, err
|
|
||||||
}
|
|
||||||
if !offersIssued(shelf, provision) {
|
|
||||||
return nil, false, nil
|
|
||||||
}
|
|
||||||
holders, err := inv.HoldersOf(ctx, provision, "")
|
|
||||||
if err != nil {
|
|
||||||
return nil, true, err
|
|
||||||
}
|
|
||||||
nodes := map[string]bool{}
|
|
||||||
for _, h := range holders {
|
|
||||||
nodes[h.Provider] = true
|
|
||||||
}
|
|
||||||
names := make([]string, 0, len(nodes))
|
|
||||||
for n := range nodes {
|
|
||||||
names = append(names, n)
|
|
||||||
}
|
|
||||||
sort.Strings(names)
|
|
||||||
var out []issuedProvider
|
|
||||||
for _, node := range names {
|
|
||||||
assigned, err := inv.Assigned(ctx, node)
|
|
||||||
if err != nil {
|
|
||||||
return nil, true, err
|
|
||||||
}
|
|
||||||
out = append(out, issuedOn(shelf, node, assigned, provision)...)
|
|
||||||
}
|
|
||||||
return out, true, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// offersIssued is whether any module offers the provision with a credential its software issues.
|
|
||||||
func offersIssued(shelf map[string]catalogue.Manifest, provision string) bool {
|
|
||||||
for _, m := range shelf {
|
|
||||||
if _, issued := m.IssuedCredentialOf(provision); issued {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// issuedOn are the modules assigned to one machine whose offer says their software issues the provision's
|
|
||||||
// credential, each with the tool that answers it.
|
|
||||||
func issuedOn(shelf map[string]catalogue.Manifest, node string, assigned []string, provision string) []issuedProvider {
|
|
||||||
var out []issuedProvider
|
|
||||||
for _, module := range assigned {
|
|
||||||
if tool, issued := shelf[module].IssuedCredentialOf(provision); issued {
|
|
||||||
out = append(out, issuedProvider{Node: node, Module: module, Tool: tool})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// issuedRefusesRotation refuses `rotate` of a provision its provider's software issues (ADR 0305 §9): a value the
|
|
||||||
// mesh made would replace one the software holds, and every consumer would hold a dead credential until the
|
|
||||||
// provider next asks.
|
|
||||||
func issuedRefusesRotation(shelf map[string]catalogue.Manifest, provision string) error {
|
|
||||||
if !offersIssued(shelf, provision) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return fmt.Errorf("%s's credential is issued by its provider's software, which the mesh cannot make or "+
|
|
||||||
"rotate (ADR 0305): change it in that software, and `secret issued %s` takes the value it holds now. "+
|
|
||||||
"Nothing was rotated", provision, provision)
|
|
||||||
}
|
|
||||||
|
|
||||||
// secretIssued is `secret issued <provision>`, on this controller's stores and bus.
|
|
||||||
func secretIssued(ctx context.Context, args []string) error {
|
|
||||||
if len(args) != 1 || args[0] == "" || strings.HasPrefix(args[0], "-") {
|
|
||||||
return errors.New("secret issued <provision>")
|
|
||||||
}
|
|
||||||
provision := args[0]
|
|
||||||
open, err := openStores(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer open.Close()
|
|
||||||
t := issuedTaker{
|
|
||||||
providers: func(provision string) ([]issuedProvider, bool, error) {
|
|
||||||
return issuedProvidersOf(ctx, open.inventory, provision)
|
|
||||||
},
|
|
||||||
ask: func(p issuedProvider, args map[string]any) (link.Answer, error) {
|
|
||||||
var answer link.Answer
|
|
||||||
err := onTheBus(func(conn *nats.Conn) error {
|
|
||||||
var err error
|
|
||||||
answer, err = link.AskModuleToolOnAs(ctx, conn, broker.ControllerName, p.Module, p.Tool, p.Node,
|
|
||||||
args, issuedAsk)
|
|
||||||
return err
|
|
||||||
})
|
|
||||||
return answer, err
|
|
||||||
},
|
|
||||||
keep: func(provision, provider, value string) (inventory.IssuedKept, error) {
|
|
||||||
return open.inventory.KeepIssuedSecret(ctx, provision, provider, value)
|
|
||||||
},
|
|
||||||
record: func(act link.HandAct) error {
|
|
||||||
return onTheBus(func(conn *nats.Conn) error {
|
|
||||||
_, err := link.RecordHandAct(ctx, conn, act)
|
|
||||||
return err
|
|
||||||
})
|
|
||||||
},
|
|
||||||
send: func(machines []string) error { return sendTo(ctx, open, machines) },
|
|
||||||
}
|
|
||||||
words, err := t.take(provision)
|
|
||||||
if words != "" {
|
|
||||||
fmt.Println(words)
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
@@ -1,218 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
"github.com/novox/mesh-controller/internal/secrets"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The `issued` verb runs `secret issued <provision>` and nothing else, and that line is the one `secret` line
|
|
||||||
// beside rotate and the desk's that a verb may run (novox/hq ADR 0305 §3, §10, issue 526).
|
|
||||||
func TestTheIssuedVerbRunsTheOneLineThatCarriesNoValue(t *testing.T) {
|
|
||||||
argv, err := argvFor("issued", map[string]any{"provision": "plex-api"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if strings.Join(argv, " ") != "secret issued plex-api" {
|
|
||||||
t.Fatalf("the verb composed %q", argv)
|
|
||||||
}
|
|
||||||
if _, err := argvFor("issued", map[string]any{}); err == nil {
|
|
||||||
t.Fatal("the verb ran with no provision")
|
|
||||||
}
|
|
||||||
if _, err := argvFor("issued", map[string]any{"provision": "plex-api", "secret": "a value"}); err == nil {
|
|
||||||
t.Fatal("the verb took a value beside the provision")
|
|
||||||
}
|
|
||||||
for _, line := range [][]string{
|
|
||||||
{"secret", "issued", "plex-api", "--from", "/tmp/x"},
|
|
||||||
{"secret", "issued", "--from"},
|
|
||||||
{"secret", "issued"},
|
|
||||||
{"secret", "accept", "ace", "kometa", "plex-api", "--provider", "ace"},
|
|
||||||
{"secret", "recover", "ace", "kometa", "plex-api"},
|
|
||||||
} {
|
|
||||||
if err := terminalOnly(line); err == nil {
|
|
||||||
t.Errorf("%q ran through a verb", line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := terminalOnly([]string{"secret", "issued", "plex-api"}); err != nil {
|
|
||||||
t.Fatalf("the issued line was refused through its verb: %v", err)
|
|
||||||
}
|
|
||||||
found := false
|
|
||||||
for _, v := range catalogue.ControllerVerbs {
|
|
||||||
if v.Name == "issued" {
|
|
||||||
found = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !found {
|
|
||||||
t.Fatal("the controller's seat does not declare the issued verb")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A provider's tool answering the value sealed to the call's key: what a provider does (ADR 0305 §2).
|
|
||||||
func issuedSealed(t *testing.T, value string) func(issuedProvider, map[string]any) (link.Answer, error) {
|
|
||||||
return func(_ issuedProvider, args map[string]any) (link.Answer, error) {
|
|
||||||
key, _ := args["seal_to"].(string)
|
|
||||||
if key == "" {
|
|
||||||
t.Fatal("the provider was asked without a key to seal to")
|
|
||||||
}
|
|
||||||
if _, carried := args["value"]; carried {
|
|
||||||
t.Fatal("the ask carried a value")
|
|
||||||
}
|
|
||||||
sealed, err := secrets.Seal(key, []byte(value))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
raw, _ := json.Marshal(map[string]string{"sealed": sealed})
|
|
||||||
return link.Answer{Result: raw}, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type issuedRecorder struct {
|
|
||||||
kept []string
|
|
||||||
acts []link.HandAct
|
|
||||||
sent [][]string
|
|
||||||
again bool // the value is held by every holder already
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *issuedRecorder) taker(t *testing.T, ask func(issuedProvider, map[string]any) (link.Answer, error)) issuedTaker {
|
|
||||||
return issuedTaker{
|
|
||||||
providers: func(provision string) ([]issuedProvider, bool, error) {
|
|
||||||
return []issuedProvider{{Node: "ace", Module: "plex", Tool: "plex_issued_credential"}}, true, nil
|
|
||||||
},
|
|
||||||
ask: ask,
|
|
||||||
keep: func(provision, provider, value string) (inventory.IssuedKept, error) {
|
|
||||||
r.kept = append(r.kept, value)
|
|
||||||
holders := []inventory.Holder{{Provision: provision, Consumer: "ace", ConsumerModule: "kometa", Provider: provider},
|
|
||||||
{Provision: provision, Consumer: "ace", ConsumerModule: "tautulli", Provider: provider}}
|
|
||||||
if r.again {
|
|
||||||
return inventory.IssuedKept{Holders: holders}, nil
|
|
||||||
}
|
|
||||||
return inventory.IssuedKept{Changed: true, Holders: holders, Sealed: holders, Machines: []string{"ace"}}, nil
|
|
||||||
},
|
|
||||||
record: func(act link.HandAct) error { r.acts = append(r.acts, act); return nil },
|
|
||||||
send: func(machines []string) error { r.sent = append(r.sent, machines); return nil },
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The value the provider's tool sealed to the call is kept, recorded and sent, and the answer never says it.
|
|
||||||
func TestAnIssuedValueIsTakenSealedFromTheProviderAndSent(t *testing.T) {
|
|
||||||
var r issuedRecorder
|
|
||||||
words, err := r.taker(t, issuedSealed(t, "the-servers-own-token")).take("plex-api")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(r.kept) != 1 || r.kept[0] != "the-servers-own-token" {
|
|
||||||
t.Fatalf("the provider's value was not kept: %d kept", len(r.kept))
|
|
||||||
}
|
|
||||||
if len(r.sent) != 1 || strings.Join(r.sent[0], ",") != "ace" {
|
|
||||||
t.Fatalf("the holders' machines were not sent: %v", r.sent)
|
|
||||||
}
|
|
||||||
if len(r.acts) != 1 || r.acts[0].Verb != handActIssued {
|
|
||||||
t.Fatalf("the change was not recorded in the hand-act log: %+v", r.acts)
|
|
||||||
}
|
|
||||||
for _, said := range []string{words, r.acts[0].Why, strings.Join(r.acts[0].Args, " ")} {
|
|
||||||
if strings.Contains(said, "the-servers-own-token") || strings.Contains(said, inventory.IssuedStamp("the-servers-own-token")) {
|
|
||||||
t.Fatalf("the value or its fingerprint was said: %q", said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !strings.Contains(words, "kometa on ace") || !strings.Contains(words, "tautulli on ace") {
|
|
||||||
t.Fatalf("the answer does not name the holders: %q", words)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The same value again writes nothing, records nothing and sends nobody (ADR 0305 §5).
|
|
||||||
func TestAnIssuedValueEveryHolderHoldsSendsNobody(t *testing.T) {
|
|
||||||
r := issuedRecorder{again: true}
|
|
||||||
words, err := r.taker(t, issuedSealed(t, "the-servers-own-token")).take("plex-api")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(r.sent) != 0 || len(r.acts) != 0 {
|
|
||||||
t.Fatalf("an unchanged value was sent or recorded: %v %v", r.sent, r.acts)
|
|
||||||
}
|
|
||||||
if !strings.Contains(words, "nothing was sent") {
|
|
||||||
t.Fatalf("the answer does not say nothing was sent: %q", words)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A value not sealed to the call's key is not taken, nor is a refusal, nor an empty answer (ADR 0305 §2).
|
|
||||||
func TestOnlyAValueSealedToTheCallIsTaken(t *testing.T) {
|
|
||||||
other, _, _ := secrets.Keypair()
|
|
||||||
elsewhere, _ := secrets.Seal(other, []byte("somebody's value"))
|
|
||||||
raw, _ := json.Marshal(map[string]string{"sealed": elsewhere})
|
|
||||||
plain, _ := json.Marshal(map[string]string{"value": "the-servers-own-token"})
|
|
||||||
for name, ask := range map[string]func(issuedProvider, map[string]any) (link.Answer, error){
|
|
||||||
"sealed to another key": func(issuedProvider, map[string]any) (link.Answer, error) { return link.Answer{Result: raw}, nil },
|
|
||||||
"answered in the clear": func(issuedProvider, map[string]any) (link.Answer, error) { return link.Answer{Result: plain}, nil },
|
|
||||||
"refused": func(issuedProvider, map[string]any) (link.Answer, error) {
|
|
||||||
return link.Answer{Error: "the controller alone"}, nil
|
|
||||||
},
|
|
||||||
"not reached": func(issuedProvider, map[string]any) (link.Answer, error) {
|
|
||||||
return link.Answer{}, errors.New("503 no responders")
|
|
||||||
},
|
|
||||||
"sealed empty": issuedSealed(t, " "),
|
|
||||||
} {
|
|
||||||
var r issuedRecorder
|
|
||||||
if _, err := r.taker(t, ask).take("plex-api"); err == nil {
|
|
||||||
t.Errorf("%s: taken", name)
|
|
||||||
}
|
|
||||||
if len(r.kept) != 0 || len(r.sent) != 0 {
|
|
||||||
t.Errorf("%s: something was kept or sent", name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A provision no offer says its software issues is refused before anything is asked.
|
|
||||||
func TestAProvisionNoOfferSaysIsIssuedIsRefused(t *testing.T) {
|
|
||||||
asked := false
|
|
||||||
taker := issuedTaker{
|
|
||||||
providers: func(string) ([]issuedProvider, bool, error) { return nil, false, nil },
|
|
||||||
ask: func(issuedProvider, map[string]any) (link.Answer, error) {
|
|
||||||
asked = true
|
|
||||||
return link.Answer{}, nil
|
|
||||||
},
|
|
||||||
}
|
|
||||||
if _, err := taker.take("postgres-database"); err == nil || !strings.Contains(err.Error(), "ADR 0305") {
|
|
||||||
t.Fatalf("expected a refusal naming the record, got %v", err)
|
|
||||||
}
|
|
||||||
if asked {
|
|
||||||
t.Fatal("a provider was asked for a provision no offer says its software issues")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Which assignments are asked: the modules on the providing machine whose offer says so, with their tool.
|
|
||||||
func TestTheProvidersAskedAreTheOffersThatSaySo(t *testing.T) {
|
|
||||||
shelf := map[string]catalogue.Manifest{
|
|
||||||
"plex": {Module: "plex", Provides: []catalogue.Offer{{Name: "plex-api", Credential: &catalogue.OfferCredential{
|
|
||||||
IssuedBy: catalogue.IssuedBySoftware, Tool: "plex_issued_credential"}}}},
|
|
||||||
"jellyfin": {Module: "jellyfin", Provides: []catalogue.Offer{{Name: "plex-api"}}},
|
|
||||||
}
|
|
||||||
got := issuedOn(shelf, "ace", []string{"jellyfin", "plex", "kometa"}, "plex-api")
|
|
||||||
if len(got) != 1 || got[0] != (issuedProvider{Node: "ace", Module: "plex", Tool: "plex_issued_credential"}) {
|
|
||||||
t.Fatalf("the providers asked: %+v", got)
|
|
||||||
}
|
|
||||||
if err := issuedRefusesRotation(shelf, "plex-api"); err == nil || !strings.Contains(err.Error(), "secret issued plex-api") {
|
|
||||||
t.Fatalf("rotate of an issued provision was not refused with the way to take it: %v", err)
|
|
||||||
}
|
|
||||||
if err := issuedRefusesRotation(shelf, "postgres-database"); err != nil {
|
|
||||||
t.Fatalf("rotate of an ordinary provision was refused: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A change of an issued value is the mesh working as decided, never a repair (ADR 0305 §8).
|
|
||||||
func TestAnIssuedValueIsADecisionInTheHandActLog(t *testing.T) {
|
|
||||||
for _, v := range handActVerbs {
|
|
||||||
if v.Verb == handActIssued {
|
|
||||||
if v.Decision == "" {
|
|
||||||
t.Fatal("secret issued counts as a repair")
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
t.Fatal("secret issued is not in the hand-act table")
|
|
||||||
}
|
|
||||||
@@ -1,160 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// novox/hq issue 485: the mesh records every send — sender, lease epoch, generation, modules — and only `plan`
|
|
||||||
// said it. `status` says per machine when it was last sent a declaration and by whom, in one line; `node show`
|
|
||||||
// says the last send in full and what the machine answered; and `status --json` carries it as fields, since
|
|
||||||
// the store's reader may not read the controller's tables and these verbs are the only window onto them.
|
|
||||||
|
|
||||||
func aLastSend(answer inventory.SendAnswer) inventory.Send {
|
|
||||||
sent := time.Date(2026, 10, 11, 18, 40, 5, 0, time.Local)
|
|
||||||
return inventory.Send{NodeName: "laptop", Sequence: 12, Epoch: 57,
|
|
||||||
Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12",
|
|
||||||
Modules: []string{"docker", "sudo"}, SentAt: sent, Recorded: true, Answer: answer}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStatusSaysEachMachinesLastSendInOneLine(t *testing.T) {
|
|
||||||
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.Local)
|
|
||||||
asked := answers{
|
|
||||||
nodes: []inventory.Node{{Name: "anchor", LastSeen: time.Now()}, {Name: "laptop", LastSeen: time.Now()}},
|
|
||||||
lastSent: map[string]inventory.Send{"laptop": aLastSend(inventory.SendAnswer{
|
|
||||||
Outcome: inventory.OutcomeApplied, At: &at})},
|
|
||||||
}
|
|
||||||
shown := printed(t, func() error { return printStatus(asked) })
|
|
||||||
var laptop, anchor []string
|
|
||||||
for _, line := range strings.Split(shown, "\n") {
|
|
||||||
fields := strings.Fields(line)
|
|
||||||
if len(fields) > 0 && fields[0] == "laptop" {
|
|
||||||
laptop = append(laptop, line)
|
|
||||||
}
|
|
||||||
if len(fields) > 0 && fields[0] == "anchor" {
|
|
||||||
anchor = append(anchor, line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(laptop) != 1 || !strings.Contains(laptop[0], "2026-10-11 18:40") ||
|
|
||||||
!strings.Contains(laptop[0], "by a one-shot push by jochen at a shell on anchor") ||
|
|
||||||
!strings.Contains(laptop[0], "generation 40") || !strings.Contains(laptop[0], "applied") {
|
|
||||||
t.Fatalf("status does not say laptop's last send in one line:\n%s", shown)
|
|
||||||
}
|
|
||||||
if len(anchor) != 1 || !strings.Contains(anchor[0], "no send recorded") {
|
|
||||||
t.Fatalf("status does not say anchor has no send recorded, in one line:\n%s", shown)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStatusSaysTheLastSendsCouldNotBeRead(t *testing.T) {
|
|
||||||
asked := answers{nodes: []inventory.Node{{Name: "laptop", LastSeen: time.Now()}},
|
|
||||||
lastSentUnread: "the store went away"}
|
|
||||||
if shown := printed(t, func() error { return printStatus(asked) }); !strings.Contains(shown, "the store went away") {
|
|
||||||
t.Fatalf("a record of sends that could not be read is not said:\n%s", shown)
|
|
||||||
}
|
|
||||||
if asked.well() {
|
|
||||||
t.Error("a mesh whose sends could not be read is called well")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNodeShowSaysTheLastSendInFull(t *testing.T) {
|
|
||||||
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.Local)
|
|
||||||
refused := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeRefused,
|
|
||||||
Refused: "unknown field \"generation\"\nsecond line", At: &at})), "\n")
|
|
||||||
for _, want := range []string{"sequence 12", "2026-10-11 18:40:05", "a one-shot push by jochen at a shell on anchor",
|
|
||||||
"lease epoch 57", "generation 40", "docker, sudo", "refused", "unknown field \"generation\""} {
|
|
||||||
if !strings.Contains(refused, want) {
|
|
||||||
t.Errorf("node show's last send does not say %q:\n%s", want, refused)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if applied := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeApplied,
|
|
||||||
At: &at})), "\n"); !strings.Contains(applied, "applied it at 2026-10-11 18:41:00") {
|
|
||||||
t.Errorf("an applied send is not said applied:\n%s", applied)
|
|
||||||
}
|
|
||||||
if waiting := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{})), "\n"); !strings.Contains(waiting,
|
|
||||||
"not reported on it yet") {
|
|
||||||
t.Errorf("a send not reported on is not said so:\n%s", waiting)
|
|
||||||
}
|
|
||||||
stale := aLastSend(inventory.SendAnswer{})
|
|
||||||
when := at
|
|
||||||
stale.RefusedAt, stale.RefusedApplied = &when, 44
|
|
||||||
if s := strings.Join(lastSendLines(stale), "\n"); !strings.Contains(s, "refused it at 2026-10-11 18:41:00") ||
|
|
||||||
!strings.Contains(s, "generation 44") {
|
|
||||||
t.Errorf("a send refused for its generation is not said so:\n%s", s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStatusJSONCarriesEachMachinesLastSend(t *testing.T) {
|
|
||||||
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.UTC)
|
|
||||||
asked := answers{nodes: []inventory.Node{{Name: "anchor"}, {Name: "laptop"}},
|
|
||||||
lastSent: map[string]inventory.Send{"laptop": aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeRefused,
|
|
||||||
Refused: "unknown field", At: &at})}}
|
|
||||||
body, err := statusAsJSON(asked)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var got struct {
|
|
||||||
LastSent []map[string]any `json:"lastSent"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(body, &got); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(got.LastSent) != 1 {
|
|
||||||
t.Fatalf("status --json carries %d last sends, want laptop's only:\n%s", len(got.LastSent), body)
|
|
||||||
}
|
|
||||||
s := got.LastSent[0]
|
|
||||||
if s["node"] != "laptop" || s["sequence"] != float64(12) || s["epoch"] != float64(57) ||
|
|
||||||
s["generation"] != float64(40) || s["sender"] != "a one-shot push by jochen at a shell on anchor" ||
|
|
||||||
s["digest"] != "d12" || s["sentAt"] == nil || s["answer"] != "refused" || s["refused"] != "unknown field" ||
|
|
||||||
s["answeredAt"] == nil {
|
|
||||||
t.Fatalf("laptop's last send reads %v", s)
|
|
||||||
}
|
|
||||||
if modules, _ := s["modules"].([]any); len(modules) != 2 || modules[0] != "docker" {
|
|
||||||
t.Fatalf("the modules it named read %v", s["modules"])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// And through the store: a send recorded and refused by the machine is what `node show` and `status --json`
|
|
||||||
// read back.
|
|
||||||
func TestNodeShowAndStatusReadTheLastSendFromTheStore(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
record, err := open.inventory.NodeByName(ctx, "laptop")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := open.inventory.RecordSentWith(ctx, record.ID, "d12", nil, 57, 40, inventory.Send{Sequence: 12,
|
|
||||||
Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12",
|
|
||||||
Modules: []string{"docker", "sudo"}}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeRefused,
|
|
||||||
Declared: "d12", Refused: "unknown field \"generation\""}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
shown := printed(t, func() error { return showNode(ctx, open, "laptop") })
|
|
||||||
for _, want := range []string{"a one-shot push by jochen at a shell on anchor", "lease epoch 57", "generation 40",
|
|
||||||
"docker, sudo", "unknown field \"generation\""} {
|
|
||||||
if !strings.Contains(shown, want) {
|
|
||||||
t.Errorf("node show laptop does not say %q:\n%s", want, shown)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
body := printed(t, func() error { return statusFor(ctx, open, true) })
|
|
||||||
var got struct {
|
|
||||||
LastSent []struct {
|
|
||||||
Node string `json:"node"`
|
|
||||||
Sender string `json:"sender"`
|
|
||||||
Answer string `json:"answer"`
|
|
||||||
} `json:"lastSent"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal([]byte(body), &got); err != nil {
|
|
||||||
t.Fatalf("%v:\n%s", err, body)
|
|
||||||
}
|
|
||||||
if len(got.LastSent) != 1 || got.LastSent[0].Node != "laptop" || got.LastSent[0].Answer != "refused" ||
|
|
||||||
got.LastSent[0].Sender != "a one-shot push by jochen at a shell on anchor" {
|
|
||||||
t.Fatalf("status --json's last sends read %+v", got.LastSent)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,222 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A module assigned to a machine and left out of its declaration is said (novox/hq issue 380).
|
|
||||||
//
|
|
||||||
// **An omission is a finding, never a refusal of the push** (ADR 0163, rule 6): the machine is sent everything
|
|
||||||
// else, and the module's held things are kept. Until issue 380 the only place it showed was `plan`: nfs-server was
|
|
||||||
// assigned to the home server for days, every send left it out for a setting nobody gave, and the operator believed
|
|
||||||
// it ran. So the self-check (D1) judges every machine as the next send would (Resolution.LeftOutBecause, the send's
|
|
||||||
// own judgement) and raises a condition on that machine for each module it leaves out:
|
|
||||||
//
|
|
||||||
// - a setting nobody gave (catalogue.UnsetSettingError) is the operator's to give: kind needs-operator, naming
|
|
||||||
// the module, the setting and the command that sets it;
|
|
||||||
// - any other cause is said as a module not working is: kind left-out, a warning with the reason as evidence.
|
|
||||||
//
|
|
||||||
// Never urgent and never escalated by age (as ADR 0283 decision 5): nothing the machine ran was undone. It clears
|
|
||||||
// on the first run that no longer finds it — the module composed again, or no longer assigned. `status` and
|
|
||||||
// `node show` list the same modules under "assigned, not applied" with the same reason.
|
|
||||||
//
|
|
||||||
// A module left out because its stored manifest has a key this controller does not know is not raised here: the
|
|
||||||
// catalogue's own unknown-field condition says it once for the whole mesh (ADR 0262); it is still listed.
|
|
||||||
|
|
||||||
const (
|
|
||||||
// probeLeftOutID is the self-check's probe that raises and clears these conditions: D1, which already
|
|
||||||
// resolves every machine (probeDeclarations), so the judgement costs no resolution of its own.
|
|
||||||
probeLeftOutID = "D1"
|
|
||||||
// kindLeftOut is a module left out for any cause but a setting nobody gave; it is also every such condition's
|
|
||||||
// token, whichever its kind, so a cause that changes is the same condition said anew.
|
|
||||||
kindLeftOut = "left-out"
|
|
||||||
)
|
|
||||||
|
|
||||||
// leftOutModule is one module of a machine's set that its declaration leaves out, and why.
|
|
||||||
type leftOutModule struct {
|
|
||||||
Module string
|
|
||||||
// Setting is the setting nobody gave, when that is the cause.
|
|
||||||
Setting string
|
|
||||||
// Why is the declaration's own reason, whole.
|
|
||||||
Why string
|
|
||||||
// Unread is a stored manifest this controller cannot read whole (said by the catalogue's condition).
|
|
||||||
Unread bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// leftOutOf is every module of a machine's resolution that a push would leave out, sorted. Pure: the judgement
|
|
||||||
// the push makes (catalogue.Resolution.LeftOutBecause), nothing allocated.
|
|
||||||
func leftOutOf(plan catalogue.Resolution, settings catalogue.SettingsBy, adopted bool) []leftOutModule {
|
|
||||||
because := plan.LeftOutBecause(settings, adopted)
|
|
||||||
out := make([]leftOutModule, 0, len(because))
|
|
||||||
for module, why := range because {
|
|
||||||
l := leftOutModule{Module: module, Why: oneLine(why.Error())}
|
|
||||||
var unset *catalogue.UnsetSettingError
|
|
||||||
var unread *catalogue.UnreadManifestError
|
|
||||||
switch {
|
|
||||||
case errors.As(why, &unset):
|
|
||||||
l.Setting = unset.Setting
|
|
||||||
case errors.As(why, &unread):
|
|
||||||
l.Unread = true
|
|
||||||
}
|
|
||||||
out = append(out, l)
|
|
||||||
}
|
|
||||||
sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module })
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// settingCommand is the command that gives a module's setting on one machine.
|
|
||||||
func settingCommand(module, setting, node string) string {
|
|
||||||
return fmt.Sprintf("`settings set %s '{%q: …}' --node %s`", module, setting, node)
|
|
||||||
}
|
|
||||||
|
|
||||||
// reason is why a module is left out, as `status`, `node show` and the condition's summary say it: for a setting
|
|
||||||
// nobody gave, the setting and the command that gives it; otherwise the declaration's own words.
|
|
||||||
func (l leftOutModule) reason(node string) string {
|
|
||||||
if l.Setting != "" {
|
|
||||||
return fmt.Sprintf("nothing sets its setting %q, so every send leaves it out of its declaration — %s sets it", l.Setting,
|
|
||||||
settingCommand(l.Module, l.Setting, node))
|
|
||||||
}
|
|
||||||
return "every send leaves it out of its declaration: " + l.Why
|
|
||||||
}
|
|
||||||
|
|
||||||
// leftOutObservations are the conditions a machine's left-out modules raise, one each.
|
|
||||||
func leftOutObservations(node string, left []leftOutModule) []conditions.Observation {
|
|
||||||
var out []conditions.Observation
|
|
||||||
for _, l := range left {
|
|
||||||
if l.Unread {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
out = append(out, leftOutObservation(node, l))
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// leftOutObservation is one module left out of one machine's declaration: needs-operator for a setting nobody
|
|
||||||
// gave, left-out otherwise; a warning either way, the operator's to resolve.
|
|
||||||
func leftOutObservation(node string, l leftOutModule) conditions.Observation {
|
|
||||||
o := conditions.Observation{Scope: conditions.ScopeModule, ID: l.Module + "." + node, Token: kindLeftOut,
|
|
||||||
Kind: kindLeftOut, Machine: node, Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
|
|
||||||
Summary: fmt.Sprintf("%s is assigned to %s and not applied: %s", l.Module, node, l.reason(node)),
|
|
||||||
Said: l.Why}
|
|
||||||
w := leftOutWords(l.Module, node, l.Setting)
|
|
||||||
if l.Setting != "" {
|
|
||||||
o.Kind = kindNeedsOperator
|
|
||||||
} else {
|
|
||||||
// The words of why may name a path or an address, which the operator's channel withholds: the
|
|
||||||
// summary sends them to the evidence, and `plan` says them in full.
|
|
||||||
o.Summary = fmt.Sprintf("%s is assigned to %s and not applied: every send leaves it out of its declaration, "+
|
|
||||||
"because what is set for it does not fit its manifest — the evidence and `plan %s` say why", l.Module, node, node)
|
|
||||||
}
|
|
||||||
o.Headline, o.Explanation, o.Needs, o.Resolved = w.Headline, w.Explanation, w.Needs, w.Resolved
|
|
||||||
return o
|
|
||||||
}
|
|
||||||
|
|
||||||
// leftOutWords is what the operator reads of a module left out (ADR 0253): plain, the act named.
|
|
||||||
func leftOutWords(module, node, setting string) words {
|
|
||||||
w := words{
|
|
||||||
Headline: fmt.Sprintf("%s is not applied on %s", module, node),
|
|
||||||
Explanation: fmt.Sprintf("%s is assigned to %s, and every send to %s leaves it out of the declaration because what "+
|
|
||||||
"is set for it does not fit its manifest. Nothing of it changes there; the rest of %s is sent as usual.",
|
|
||||||
module, node, node, node),
|
|
||||||
Needs: fmt.Sprintf("read why in the details, then change what is set for %s or unassign it.", module),
|
|
||||||
Resolved: fmt.Sprintf("%s on %s is no longer left out", module, node),
|
|
||||||
}
|
|
||||||
if setting != "" {
|
|
||||||
w.Explanation = fmt.Sprintf("%s is assigned to %s, and every send to %s leaves it out of the declaration because "+
|
|
||||||
"nothing sets its setting %s. Nothing of it runs there until it is set; the rest of %s is sent as usual.",
|
|
||||||
module, node, node, setting, node)
|
|
||||||
w.Needs = fmt.Sprintf("set %s for %s on %s, or approve it when it is proposed to you.", setting, module, node)
|
|
||||||
// A setting's name that is not plain (a dotted key) is in the summary instead.
|
|
||||||
if _, ok := conditions.PlainWords(w, node); !ok {
|
|
||||||
w.Explanation = fmt.Sprintf("%s is assigned to %s, and every send to %s leaves it out of the declaration because a "+
|
|
||||||
"setting it needs is not set. Nothing of it runs there until it is set; the details name it.",
|
|
||||||
module, node, node)
|
|
||||||
w.Needs = fmt.Sprintf("set what %s needs on %s; the details name the setting.", module, node)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return w
|
|
||||||
}
|
|
||||||
|
|
||||||
// notAppliedLines is a machine's "assigned, not applied" as `node show` prints it: one line a module, with the
|
|
||||||
// reason its condition says.
|
|
||||||
func notAppliedLines(node string, left []leftOutModule) []string {
|
|
||||||
if len(left) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
lines := []string{"", " assigned, not applied:"}
|
|
||||||
for _, l := range left {
|
|
||||||
lines = append(lines, fmt.Sprintf(" %-22s %s", l.Module, l.reason(node)))
|
|
||||||
}
|
|
||||||
return lines
|
|
||||||
}
|
|
||||||
|
|
||||||
// machineNotApplied is one module assigned to a machine and left out of its declaration, in `status --json`.
|
|
||||||
type machineNotApplied struct {
|
|
||||||
Node string `json:"node"`
|
|
||||||
Module string `json:"module"`
|
|
||||||
Setting string `json:"setting,omitempty"`
|
|
||||||
Reason string `json:"reason"`
|
|
||||||
// UnreadManifest is a module left out because this controller cannot read its manifest whole: it raises no
|
|
||||||
// condition of its own on the machine, since the catalogue's unknown-field condition says it once (ADR 0262).
|
|
||||||
UnreadManifest bool `json:"unread-manifest,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// notApplied is every machine's left-out modules, in a stated order, as `status --json` carries them.
|
|
||||||
func notApplied(left map[string][]leftOutModule) []machineNotApplied {
|
|
||||||
names := make([]string, 0, len(left))
|
|
||||||
for name := range left {
|
|
||||||
names = append(names, name)
|
|
||||||
}
|
|
||||||
sort.Strings(names)
|
|
||||||
var out []machineNotApplied
|
|
||||||
for _, name := range names {
|
|
||||||
for _, l := range left[name] {
|
|
||||||
out = append(out, machineNotApplied{Node: name, Module: l.Module, Setting: l.Setting, Reason: l.reason(name),
|
|
||||||
UnreadManifest: l.Unread})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// printNotApplied is status's "assigned, not applied", every machine's.
|
|
||||||
func printNotApplied(left map[string][]leftOutModule) {
|
|
||||||
rows := notApplied(left)
|
|
||||||
if len(rows) == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
unread := 0
|
|
||||||
for _, r := range rows {
|
|
||||||
if r.UnreadManifest {
|
|
||||||
unread++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Which raise a condition is said, not implied (review of #223): one whose manifest this controller cannot
|
|
||||||
// read is listed here and raises none of its own on the machine — the catalogue's condition says it.
|
|
||||||
raises := "each raises a condition on its machine"
|
|
||||||
switch {
|
|
||||||
case unread == len(rows):
|
|
||||||
raises = "none raises a condition on its machine: this controller cannot read their manifests, which the " +
|
|
||||||
"catalogue's own condition says"
|
|
||||||
case unread > 0:
|
|
||||||
raises += fmt.Sprintf(", except the %d whose manifest this controller cannot read, which the catalogue's own "+
|
|
||||||
"condition says", unread)
|
|
||||||
}
|
|
||||||
fmt.Printf("%d module(s) assigned, not applied — every send leaves them out of their declaration; %s:\n",
|
|
||||||
len(rows), raises)
|
|
||||||
for _, r := range rows {
|
|
||||||
fmt.Printf(" %-12s %-22s %s\n", r.Node, r.Module, r.Reason)
|
|
||||||
}
|
|
||||||
fmt.Println()
|
|
||||||
}
|
|
||||||
|
|
||||||
// isLeftOutCondition is whether a condition is a module left out of its declaration, which the machine's health
|
|
||||||
// statements neither raise nor clear: by its token, which every one carries whatever its kind.
|
|
||||||
func isLeftOutCondition(c conditions.Condition) bool {
|
|
||||||
return c.Subject.Scope == conditions.ScopeModule && strings.HasSuffix(c.Key, "."+kindLeftOut)
|
|
||||||
}
|
|
||||||
@@ -1,259 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"os"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// novox/hq issue 380: nfs-server was assigned to the home server for days and every send left it out — "nfs-server
|
|
||||||
// has a file that says ${setting:shares}, and nothing sets shares for it" — and nothing but `plan` said so. The
|
|
||||||
// manifest is the catalogue's own at the commit that added it (mesh-catalog 48fba44), which still says
|
|
||||||
// ${setting:shares}; the reason below is the one the live push printed.
|
|
||||||
|
|
||||||
// leftOutNFS is the resolution of a machine assigned that nfs-server, with the settings given.
|
|
||||||
func leftOutNFS(t *testing.T) catalogue.Resolution {
|
|
||||||
t.Helper()
|
|
||||||
raw, err := os.ReadFile("testdata/left-out/nfs-server.json")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
m, err := catalogue.ParseManifest(raw)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return catalogue.Resolution{Modules: []catalogue.Manifest{m}}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sharesGiven is the operator's setting for it on the machine.
|
|
||||||
func sharesGiven(value string) catalogue.SettingsBy {
|
|
||||||
return catalogue.SettingsBy{"nfs-server": {{From: "anchor", Values: map[string]any{"shares": value}}}}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAModuleLeftOutForASettingNobodyGaveNeedsTheOperatorNamingTheSettingAndTheCommand(t *testing.T) {
|
|
||||||
left := leftOutOf(leftOutNFS(t), nil, false)
|
|
||||||
if len(left) != 1 || left[0].Module != "nfs-server" || left[0].Setting != "shares" {
|
|
||||||
t.Fatalf("left out: %+v", left)
|
|
||||||
}
|
|
||||||
if !strings.Contains(left[0].Why, `nothing sets "shares" for it`) {
|
|
||||||
t.Fatalf("the reason is not the push's own: %s", left[0].Why)
|
|
||||||
}
|
|
||||||
obs := leftOutObservations("anchor", left)
|
|
||||||
if len(obs) != 1 {
|
|
||||||
t.Fatalf("raised %+v", obs)
|
|
||||||
}
|
|
||||||
o := obs[0]
|
|
||||||
if o.Key() != "module.nfs-server.anchor.left-out" || o.Kind != kindNeedsOperator || o.Machine != "anchor" ||
|
|
||||||
o.Severity != conditions.Warning || o.Resolver != conditions.ResolverOperator {
|
|
||||||
t.Fatalf("raised %s as %s, %s, by %s, on %q", o.Key(), o.Kind, o.Severity, o.Resolver, o.Machine)
|
|
||||||
}
|
|
||||||
for _, want := range []string{"nfs-server", "anchor", `"shares"`, "`settings set nfs-server '{\"shares\": …}' --node anchor`"} {
|
|
||||||
if !strings.Contains(o.Summary, want) {
|
|
||||||
t.Errorf("the summary does not name %s: %s", want, o.Summary)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if o.Said != left[0].Why {
|
|
||||||
t.Errorf("the evidence is not the reason the send gives: %s", o.Said)
|
|
||||||
}
|
|
||||||
plainExample(t, o, "nfs-server is not applied on anchor",
|
|
||||||
"Needs you: set shares for nfs-server on anchor, or approve it when it is proposed to you. nfs-server is assigned to "+
|
|
||||||
"anchor, and every send to anchor leaves it out of the declaration because nothing sets its setting shares. "+
|
|
||||||
"Nothing of it runs there until it is set; the rest of anchor is sent as usual.")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAModuleLeftOutForAnotherCauseIsAWarningWithTheReasonAsEvidence(t *testing.T) {
|
|
||||||
// A setting stored that its manifest can no longer take: one with a line break (issue 339).
|
|
||||||
left := leftOutOf(leftOutNFS(t), sharesGiven("library=/srv/library\nmedia=/srv/media"), false)
|
|
||||||
if len(left) != 1 || left[0].Setting != "" {
|
|
||||||
t.Fatalf("left out: %+v", left)
|
|
||||||
}
|
|
||||||
obs := leftOutObservations("anchor", left)
|
|
||||||
if len(obs) != 1 {
|
|
||||||
t.Fatalf("raised %+v", obs)
|
|
||||||
}
|
|
||||||
o := obs[0]
|
|
||||||
if o.Key() != "module.nfs-server.anchor.left-out" || o.Kind != kindLeftOut || o.Severity != conditions.Warning {
|
|
||||||
t.Fatalf("raised %s as %s, %s", o.Key(), o.Kind, o.Severity)
|
|
||||||
}
|
|
||||||
if !strings.Contains(o.Said, "holds a line break") || strings.Contains(o.Summary, "/srv/") {
|
|
||||||
t.Fatalf("the reason is not the evidence, or the summary carries it to the channel:\n%s\n%s", o.Summary, o.Said)
|
|
||||||
}
|
|
||||||
plainExample(t, o, "nfs-server is not applied on anchor",
|
|
||||||
"Needs you: read why in the details, then change what is set for nfs-server or unassign it. nfs-server is assigned to "+
|
|
||||||
"anchor, and every send to anchor leaves it out of the declaration because what is set for it does not fit "+
|
|
||||||
"its manifest. Nothing of it changes there; the rest of anchor is sent as usual.")
|
|
||||||
}
|
|
||||||
|
|
||||||
// The self-check raises it, keeps it a warning however long it stands, and clears it when the module composes
|
|
||||||
// again or is no longer assigned; a machine's health statement neither clears nor raises it.
|
|
||||||
func TestALeftOutModulesConditionClearsWhenItComposesAgainOrIsUnassigned(t *testing.T) {
|
|
||||||
plan, settings := leftOutNFS(t), catalogue.SettingsBy(nil)
|
|
||||||
withProbes(t, probe{ID: probeLeftOutID, Asserts: "the test's", Kind: kindLeftOut, Phase: 1,
|
|
||||||
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
|
|
||||||
return leftOutObservations("anchor", leftOutOf(plan, settings, false)), nil
|
|
||||||
}})
|
|
||||||
store := conditions.NewInMemory()
|
|
||||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
|
||||||
defer k.Close(context.Background())
|
|
||||||
d := &doctor{keeper: k, teller: &conditions.Told{}, host: "anchor"}
|
|
||||||
key := "module.nfs-server.anchor.left-out"
|
|
||||||
openOnes := func() map[string]conditions.Condition {
|
|
||||||
t.Helper()
|
|
||||||
open, err := k.Open(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
out := map[string]conditions.Condition{}
|
|
||||||
for _, c := range open {
|
|
||||||
out[c.Key] = c
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
d.runOnce(t.Context(), "a test")
|
|
||||||
c, raised := openOnes()[key]
|
|
||||||
if !raised || c.Kind != kindNeedsOperator || c.Severity != conditions.Warning {
|
|
||||||
t.Fatalf("a module left out raised %+v", openOnes())
|
|
||||||
}
|
|
||||||
// A statement from the machine that says nothing of it — the module runs nothing there — leaves it open.
|
|
||||||
if err := judgeModuleHealth(t.Context(), nil, k, "anchor", map[string][]inventory.ResourceHealth{}, nil,
|
|
||||||
time.Now().Add(72*time.Hour)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, still := openOnes()[key]; !still {
|
|
||||||
t.Fatal("a health statement that says nothing of the module cleared its left-out condition")
|
|
||||||
}
|
|
||||||
d.runOnce(t.Context(), "a test")
|
|
||||||
if c := openOnes()[key]; c.Severity != conditions.Warning {
|
|
||||||
t.Fatalf("after a health statement and days, it is %+v", openOnes())
|
|
||||||
}
|
|
||||||
|
|
||||||
// The setting given: it composes, and the condition clears.
|
|
||||||
settings = sharesGiven("library=/srv/library")
|
|
||||||
d.runOnce(t.Context(), "a test")
|
|
||||||
if _, still := openOnes()[key]; still {
|
|
||||||
t.Fatalf("composed again, still open: %+v", openOnes())
|
|
||||||
}
|
|
||||||
|
|
||||||
// Left out again, then unassigned: no longer in the machine's set, and it clears.
|
|
||||||
settings = nil
|
|
||||||
d.runOnce(t.Context(), "a test")
|
|
||||||
if _, raised := openOnes()[key]; !raised {
|
|
||||||
t.Fatal("left out again and not raised")
|
|
||||||
}
|
|
||||||
plan = catalogue.Resolution{}
|
|
||||||
d.runOnce(t.Context(), "a test")
|
|
||||||
if _, still := openOnes()[key]; still {
|
|
||||||
t.Fatalf("unassigned, still open: %+v", openOnes())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTheLeftOutProbeIsInTheRegistry(t *testing.T) {
|
|
||||||
for _, p := range probeRegistry {
|
|
||||||
if p.ID == probeLeftOutID {
|
|
||||||
if p.run == nil || !slices.Contains(p.Raises, kindLeftOut) || !slices.Contains(p.Raises, kindNeedsOperator) {
|
|
||||||
t.Fatalf("%+v", p)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
t.Fatalf("no probe %s: a module left out is said nowhere", probeLeftOutID)
|
|
||||||
}
|
|
||||||
|
|
||||||
// status and node show list it under "assigned, not applied" with the reason the condition says, and status is
|
|
||||||
// not well while there is one.
|
|
||||||
func TestStatusAndNodeListAModuleAssignedAndNotApplied(t *testing.T) {
|
|
||||||
left := map[string][]leftOutModule{"anchor": leftOutOf(leftOutNFS(t), nil, false)}
|
|
||||||
reason := leftOutObservations("anchor", left["anchor"])[0].Summary
|
|
||||||
asked := answers{leftOut: left}
|
|
||||||
if asked.well() {
|
|
||||||
t.Fatal("a mesh with a module assigned and not applied is called well")
|
|
||||||
}
|
|
||||||
shown := printed(t, func() error { return printStatus(asked) })
|
|
||||||
if !strings.Contains(shown, "1 module(s) assigned, not applied") || !strings.Contains(shown, "nfs-server") ||
|
|
||||||
!strings.Contains(shown, "`settings set nfs-server '{\"shares\": …}' --node anchor` sets it") {
|
|
||||||
t.Fatalf("status says:\n%s", shown)
|
|
||||||
}
|
|
||||||
if !strings.Contains(reason, left["anchor"][0].reason("anchor")) {
|
|
||||||
t.Fatalf("status and the condition say different reasons:\n%s\n%s", shown, reason)
|
|
||||||
}
|
|
||||||
body, err := statusAsJSON(asked)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var doc struct {
|
|
||||||
NotApplied []machineNotApplied `json:"not-applied"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(body, &doc); err != nil || len(doc.NotApplied) != 1 ||
|
|
||||||
doc.NotApplied[0].Setting != "shares" || doc.NotApplied[0].Node != "anchor" {
|
|
||||||
t.Fatalf("status --json: %v %s", err, body)
|
|
||||||
}
|
|
||||||
lines := strings.Join(notAppliedLines("anchor", left["anchor"]), "\n")
|
|
||||||
if !strings.Contains(lines, "assigned, not applied:") || !strings.Contains(lines, "nfs-server") ||
|
|
||||||
!strings.Contains(lines, `nothing sets its setting "shares"`) {
|
|
||||||
t.Fatalf("node show says:\n%s", lines)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The skip is this probe's alone (review of #223): a part waiting for the operator (ADR 0283) keeps its own
|
|
||||||
// needs-operator condition, `module.<m>.<node>.needs-operator`, which still clears on the first statement that no
|
|
||||||
// longer names it — beside a left-out condition on the same machine, which stays.
|
|
||||||
func TestAWaitsNeedsOperatorStillClearsWhenItsStatementStopsNamingItBesideALeftOutOne(t *testing.T) {
|
|
||||||
store := conditions.NewInMemory()
|
|
||||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
|
||||||
defer k.Close(context.Background())
|
|
||||||
left := leftOutObservations("anchor", leftOutOf(leftOutNFS(t), nil, false))
|
|
||||||
if err := k.Reconcile(t.Context(), probeLeftOutID, left); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
waiting := map[string][]inventory.ResourceHealth{"notes": {{Module: "notes", Resource: "server", State: link.StateWaiting,
|
|
||||||
Waits: []inventory.Wait{{Setting: "domain", What: "the domain it serves"}}}}}
|
|
||||||
if err := judgeModuleHealth(t.Context(), nil, k, "anchor", waiting, map[string]int{"notes": 2}, time.Now()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
waitKey, leftKey := needsOperatorKey("notes", "anchor"), "module.nfs-server.anchor.left-out"
|
|
||||||
open := func() map[string]conditions.Condition {
|
|
||||||
t.Helper()
|
|
||||||
list, err := k.Open(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
out := map[string]conditions.Condition{}
|
|
||||||
for _, c := range list {
|
|
||||||
out[c.Key] = c
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
if c, raised := open()[waitKey]; !raised || c.Kind != kindNeedsOperator {
|
|
||||||
t.Fatalf("the wait raised %+v", open())
|
|
||||||
}
|
|
||||||
if c := open()[leftKey]; c.Kind != kindNeedsOperator {
|
|
||||||
t.Fatalf("the left-out condition is not open as needs-operator: %+v", open())
|
|
||||||
}
|
|
||||||
// D1 runs again and still finds nfs-server left out: its reconcile clears only what D1 raised, never the wait,
|
|
||||||
// which the machine's statement raised.
|
|
||||||
if err := k.Reconcile(t.Context(), probeLeftOutID, left); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, still := open()[waitKey]; !still {
|
|
||||||
t.Fatalf("D1's reconcile cleared the wait's needs-operator: %+v", open())
|
|
||||||
}
|
|
||||||
if err := judgeModuleHealth(t.Context(), nil, k, "anchor", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, still := open()[waitKey]; still {
|
|
||||||
t.Fatalf("the statement no longer names the wait, and its needs-operator is still open: %+v", open())
|
|
||||||
}
|
|
||||||
if _, still := open()[leftKey]; !still {
|
|
||||||
t.Fatalf("the left-out condition was cleared by a health statement: %+v", open())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,398 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The tests of novox/hq issue 525's window skip: a merge waits for the merge window only when a batch assembling
|
|
||||||
// holds a merge related to it, or its delivery group may have a member still to merge (ADR 0304 decision 1, ADR
|
|
||||||
// 0306 decisions 2 and 3). Walks stay one at a time.
|
|
||||||
|
|
||||||
// A merge from no head that shares nothing, with no walk open, is cut into its walk when it is heard: no window.
|
|
||||||
func TestALoneMergeSharingNothingIsCutAtOnce(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
asksWithPaths(t)
|
|
||||||
hear(t, open, lone(repoMerge("one", "c1", t0)), t0)
|
|
||||||
ws, bs := walks(t, open)
|
|
||||||
if len(ws) != 1 || len(bs) != 0 || !ws[0].Open() || ws[0].Commit != "c1" {
|
|
||||||
t.Fatalf("a lone merge sharing nothing: %d walk(s) %+v, %d batch(es)", len(ws), ws, len(bs))
|
|
||||||
}
|
|
||||||
if ws[0].Times == nil || ws[0].Times.WindowClosed == nil || !ws[0].Times.WindowClosed.Equal(t0) {
|
|
||||||
t.Fatalf("its window closed at %+v; want when it was heard, %s", ws[0].Times, t0)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A merge related to the batch assembling joins it and waits for the window, which its coming restarts: by its
|
|
||||||
// repository, by a module in common (a dependent of what the batch moves), by an edge of the module graph between
|
|
||||||
// their modules (one that widens nothing), or by its delivery group's head.
|
|
||||||
func TestARelatedMergeJoinsTheBatchAndWaitsForTheWindow(t *testing.T) {
|
|
||||||
ctx := t.Context()
|
|
||||||
for _, c := range []string{"same repository", "a module in common", "an edge of the graph", "the same group head"} {
|
|
||||||
t.Run(c, func(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
groupsWaiting(t, "pr/c1")
|
|
||||||
asksWithPaths(t)
|
|
||||||
inv := open.inventory
|
|
||||||
first := repoMerge("one", "c1", t0)
|
|
||||||
second := lone(repoMerge("two", "c2", t0.Add(10*time.Second)))
|
|
||||||
why := map[string]string{"same repository": "both change novox/one", "a module in common": "both move two",
|
|
||||||
"an edge of the graph": "two depends on builder (built-by)",
|
|
||||||
"the same group head": "both are members of group " + first.Head}[c]
|
|
||||||
switch c {
|
|
||||||
case "same repository":
|
|
||||||
second = lone(repoMerge("one", "c2", t0.Add(10*time.Second)))
|
|
||||||
case "a module in common":
|
|
||||||
// two is built on one: a merge of one moves two too.
|
|
||||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "two", Version: "1",
|
|
||||||
Build: &catalogue.Build{On: []catalogue.BuildsOn{{Arg: "BASE", Module: "one"}}}},
|
|
||||||
inventory.Source{Repository: "novox/two", Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
case "an edge of the graph":
|
|
||||||
// two is built by the build machine: an edge that orders and widens nothing.
|
|
||||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "builder", Version: "1",
|
|
||||||
Claims: []catalogue.Claim{{Name: "mesh-build-machine", Scope: catalogue.ScopeMesh}}},
|
|
||||||
inventory.Source{Repository: "novox/builder", Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
first = repoMerge("builder", "c1", t0)
|
|
||||||
case "the same group head":
|
|
||||||
second = fromHead(repoMerge("two", "c2", t0.Add(10*time.Second)), first.Head)
|
|
||||||
}
|
|
||||||
hear(t, open, first, t0)
|
|
||||||
said := captured(t, func() error {
|
|
||||||
return (following{open: open}).hearMerge(ctx, second, t0.Add(10*time.Second))
|
|
||||||
})
|
|
||||||
if !strings.Contains(said, "("+why+")") {
|
|
||||||
t.Fatalf("the merge was not said related by %q: %s", why, said)
|
|
||||||
}
|
|
||||||
ws, bs := walks(t, open)
|
|
||||||
if len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 {
|
|
||||||
t.Fatalf("the related merge: %d walk(s), batches %+v", len(ws), bs)
|
|
||||||
}
|
|
||||||
if bs[0].Delivery.Batch.CutAtOnce {
|
|
||||||
t.Fatal("a batch a related merge joined is cut at once")
|
|
||||||
}
|
|
||||||
cutAt(t, open, t0.Add(99*time.Second))
|
|
||||||
if ws, _ := walks(t, open); len(ws) != 0 {
|
|
||||||
t.Fatalf("cut before the window the related merge restarted closed: %+v", ws)
|
|
||||||
}
|
|
||||||
cutAt(t, open, t0.Add(100*time.Second))
|
|
||||||
ws, bs = walks(t, open)
|
|
||||||
if len(ws) != 1 || len(bs) != 0 || len(ws[0].Delivery.Merges) != 2 {
|
|
||||||
t.Fatalf("after the window: %d walk(s), %d batch(es)", len(ws), len(bs))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A merge from a head whose delivery group has a member still to merge waits for the window, even with no batch
|
|
||||||
// assembling, and says why; its sibling joins the same batch by their head (ADR 0306 decision 2). A merge from a
|
|
||||||
// head that names no group shares nothing and is cut at once, as one from no head is.
|
|
||||||
func TestAMergeOfAGroupWaitsEvenAlone(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
asksWithPaths(t)
|
|
||||||
groupsWaiting(t, "feat/x")
|
|
||||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/x"), t0)
|
|
||||||
ws, bs := walks(t, open)
|
|
||||||
if len(ws) != 0 || len(bs) != 1 || bs[0].Delivery.Batch.CutAtOnce || windowClosed(bs[0], t0) {
|
|
||||||
t.Fatalf("a group's merge alone: %d walk(s), batches %+v", len(ws), bs)
|
|
||||||
}
|
|
||||||
if words := batchWords(bs[0], t0); !strings.HasPrefix(words, "assembling: 90 s left") ||
|
|
||||||
!strings.Contains(words, "group feat/x has a member still to merge: novox/elsewhere (ready)") {
|
|
||||||
t.Fatalf("it reads %q", words)
|
|
||||||
}
|
|
||||||
hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(5*time.Second)), "feat/x"), t0.Add(5*time.Second))
|
|
||||||
if _, bs := walks(t, open); len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 {
|
|
||||||
t.Fatalf("the group's second member did not join the first: %+v", bs)
|
|
||||||
}
|
|
||||||
cutAt(t, open, t0.Add(94*time.Second))
|
|
||||||
if ws, _ := walks(t, open); len(ws) != 0 {
|
|
||||||
t.Fatalf("cut before its window closed: %+v", ws)
|
|
||||||
}
|
|
||||||
cutAt(t, open, t0.Add(95*time.Second))
|
|
||||||
ws, bs = walks(t, open)
|
|
||||||
if len(ws) != 1 || len(bs) != 0 || ws[0].CommitOf("novox/one") != "c1" || ws[0].CommitOf("novox/two") != "c2" {
|
|
||||||
t.Fatalf("at its window: %d walk(s) %+v, %d batch(es)", len(ws), ws, len(bs))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A merge from a head that names no group, sharing nothing, with no walk open, is cut at once.
|
|
||||||
func TestAHeadedMergeInNoGroupIsCutAtOnce(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
asksWithPaths(t)
|
|
||||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "fix/alone"), t0)
|
|
||||||
if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 || !ws[0].Open() {
|
|
||||||
t.Fatalf("a headed merge in no group: %d walk(s), %d batch(es)", len(ws), len(bs))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// When mesh-delivery cannot say a head's group, the merge waits for the window, the safe side, and its batch says
|
|
||||||
// why.
|
|
||||||
func TestAGroupNobodyCanSayWaits(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
asksWithPaths(t)
|
|
||||||
was := deliveryGroupOf
|
|
||||||
deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) {
|
|
||||||
return nil, false, errors.New("nothing serves mesh-delivery.groups")
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { deliveryGroupOf = was })
|
|
||||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/x"), t0)
|
|
||||||
ws, bs := walks(t, open)
|
|
||||||
if len(ws) != 0 || len(bs) != 1 || bs[0].Delivery.Batch.CutAtOnce {
|
|
||||||
t.Fatalf("a merge whose group nobody could say: %d walk(s), batches %+v", len(ws), bs)
|
|
||||||
}
|
|
||||||
if words := batchWords(bs[0], t0); !strings.Contains(words, "mesh-delivery did not say whether group feat/x has a "+
|
|
||||||
"member still to merge (nothing serves mesh-delivery.groups): it waits for the window, the safe side") {
|
|
||||||
t.Fatalf("it reads %q", words)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A member counts as still to merge only while its pull request is open, and not once this controller keeps its
|
|
||||||
// merge: a group whose other members are merged or closed makes nobody wait.
|
|
||||||
func TestAGroupsMergedOrClosedMembersMakeNobodyWait(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
asksWithPaths(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
members := []groupMember{{Repository: "novox/one", State: "ready"}, {Repository: "novox/two", State: "published"},
|
|
||||||
{Repository: "novox/three", State: "stopped"}}
|
|
||||||
was := deliveryGroupOf
|
|
||||||
deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) { return members, true, nil }
|
|
||||||
t.Cleanup(func() { deliveryGroupOf = was })
|
|
||||||
m := fromHead(repoMerge("one", "c1", t0), "feat/x")
|
|
||||||
if why, err := groupWait(ctx, open.inventory, m, true, false); err != nil || why != "" {
|
|
||||||
t.Fatalf("the group's other members are merged or closed, yet: %q %v", why, err)
|
|
||||||
}
|
|
||||||
members = append(members, groupMember{Repository: "novox/four", State: "checked"})
|
|
||||||
if why, _ := groupWait(ctx, open.inventory, m, true, false); !strings.Contains(why, "novox/four (checked)") {
|
|
||||||
t.Fatalf("an open member: %q", why)
|
|
||||||
}
|
|
||||||
// novox/four's merge is kept in a batch already: merged, whatever mesh-delivery says yet.
|
|
||||||
hear(t, open, fromHead(repoMerge("two", "c4", t0), "feat/x"), t0)
|
|
||||||
members[3].Repository = "novox/two"
|
|
||||||
if why, _ := groupWait(ctx, open.inventory, m, true, false); why != "" {
|
|
||||||
t.Fatalf("a member this controller keeps the merge of still waits: %q", why)
|
|
||||||
}
|
|
||||||
if got := memberOf("novox/mesh-sdk@d41d255f28ad (held)"); got != (groupMember{Repository: "novox/mesh-sdk", State: "held"}) {
|
|
||||||
t.Fatalf("a member reads %+v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A merge related to two batches assembling folds the newer into the older, which answers every merge of both
|
|
||||||
// and names nothing twice; the folded one names the batch that took it over.
|
|
||||||
func TestAMergeRelatedToTwoBatchesFoldsThem(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
groupsWaiting(t, "feat/a", "feat/b")
|
|
||||||
asksWithPaths(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/a"), t0)
|
|
||||||
hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(5*time.Second)), "feat/b"), t0.Add(5*time.Second))
|
|
||||||
_, bs := walks(t, open)
|
|
||||||
if len(bs) != 2 {
|
|
||||||
t.Fatalf("two merges sharing nothing: %d batch(es)", len(bs))
|
|
||||||
}
|
|
||||||
older, newer := bs[1], bs[0] // newest first
|
|
||||||
// Of one's repository and from two's head: related to both.
|
|
||||||
hear(t, open, fromHead(repoMerge("one", "c3", t0.Add(10*time.Second)), "feat/b"), t0.Add(10*time.Second))
|
|
||||||
_, bs = walks(t, open)
|
|
||||||
if len(bs) != 1 || bs[0].ID != older.ID || len(bs[0].Delivery.Merges) != 3 {
|
|
||||||
t.Fatalf("after a merge related to both: batches %+v; want %s answering three merges", bs, older.ID)
|
|
||||||
}
|
|
||||||
folded, err := open.inventory.PlanByID(ctx, newer.ID)
|
|
||||||
if err != nil || folded.State != inventory.PlanSuperseded || folded.Delivery == nil ||
|
|
||||||
folded.Delivery.TakenOverBy != older.ID {
|
|
||||||
t.Fatalf("the newer batch reads %s %+v (%v)", folded.State, folded.Delivery, err)
|
|
||||||
}
|
|
||||||
if left, _ := open.inventory.MergesOf(ctx, newer.ID); len(left) != 0 {
|
|
||||||
t.Fatalf("the folded batch still answers %+v", left)
|
|
||||||
}
|
|
||||||
cutAt(t, open, t0.Add(100*time.Second))
|
|
||||||
ws, bs := walks(t, open)
|
|
||||||
var cut []inventory.Plan
|
|
||||||
for _, w := range ws {
|
|
||||||
if w.State != inventory.PlanSuperseded {
|
|
||||||
cut = append(cut, w)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(cut) != 1 || len(bs) != 0 || cut[0].ID != older.ID || cut[0].CommitOf("novox/one") != "c3" ||
|
|
||||||
cut[0].CommitOf("novox/two") != "c2" {
|
|
||||||
t.Fatalf("the folded batch's walk: %+v, %d batch(es)", cut, len(bs))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// With a walk open, a batch cut at once still waits for it, said queued behind it (walks stay one at a time, ADR
|
|
||||||
// 0276 decision 4); it is cut once that walk ended, with no window to wait for.
|
|
||||||
func TestACutAtOnceBatchQueuesBehindAnOpenWalk(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
groupsWaiting(t, "pr/c1")
|
|
||||||
asksWithPaths(t)
|
|
||||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
|
||||||
cutAt(t, open, t0.Add(90*time.Second))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
if len(ws) != 1 || !ws[0].Open() {
|
|
||||||
t.Fatalf("no walk open: %+v", ws)
|
|
||||||
}
|
|
||||||
first := ws[0]
|
|
||||||
heard := t0.Add(100 * time.Second)
|
|
||||||
hear(t, open, lone(repoMerge("two", "c2", heard)), heard)
|
|
||||||
ws, bs := walks(t, open)
|
|
||||||
if len(ws) != 1 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != first.ID ||
|
|
||||||
!bs[0].Delivery.Batch.CutAtOnce {
|
|
||||||
t.Fatalf("a lone merge during a walk: %d walk(s), batches %+v", len(ws), bs)
|
|
||||||
}
|
|
||||||
if !strings.HasPrefix(batchWords(bs[0], heard), "queued behind "+first.ID) {
|
|
||||||
t.Fatalf("it reads %q", batchWords(bs[0], heard))
|
|
||||||
}
|
|
||||||
first.State = inventory.PlanDone
|
|
||||||
if err := open.inventory.SavePlan(t.Context(), &first); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
cutAt(t, open, heard.Add(time.Second))
|
|
||||||
ws, bs = walks(t, open)
|
|
||||||
if len(ws) != 2 || len(bs) != 0 || ws[0].Commit != "c2" {
|
|
||||||
t.Fatalf("the queued batch was not cut once the walk ended: %+v %+v", ws, bs)
|
|
||||||
}
|
|
||||||
if ws[0].Times == nil || ws[0].Times.WindowClosed == nil || !ws[0].Times.WindowClosed.Equal(heard) {
|
|
||||||
t.Fatalf("its window closed at %+v; want when its merge was heard, %s", ws[0].Times, heard)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A stopped walk's other merges are walked again in a batch they are related to, and a batch of their own when
|
|
||||||
// they share nothing with the one assembling (ADR 0299, ADR 0304 decision 1): never in a batch of another
|
|
||||||
// change, whose window the stop would close.
|
|
||||||
func TestAStopWalksTheOthersAgainInABatchOfTheirOwnWhenTheyShareNothing(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
groupsWaiting(t, "pr/cat1aaaaaaaa")
|
|
||||||
endsItsLine(t)
|
|
||||||
asksRecorded(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
|
|
||||||
one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head)
|
|
||||||
hear(t, open, cat, t0.Add(time.Second))
|
|
||||||
hear(t, open, one, t0.Add(6*time.Second))
|
|
||||||
cutAt(t, open, t0.Add(2*time.Minute))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
walk := ws[0]
|
|
||||||
hear(t, open, repoMerge("two", "two1cccccccc", t0.Add(3*time.Minute)), t0.Add(3*time.Minute))
|
|
||||||
_, bs := walks(t, open)
|
|
||||||
if len(bs) != 1 {
|
|
||||||
t.Fatalf("the merge heard while the walk ran is in %d batches", len(bs))
|
|
||||||
}
|
|
||||||
other := bs[0]
|
|
||||||
_, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped",
|
|
||||||
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if said.WalkedAgainBy == "" || said.WalkedAgainBy == other.ID {
|
|
||||||
t.Fatalf("walked again by %q; want a batch of its own, not %s", said.WalkedAgainBy, other.ID)
|
|
||||||
}
|
|
||||||
b, _ := open.inventory.PlanByID(ctx, other.ID)
|
|
||||||
if b.Delivery.Batch == nil || b.Delivery.Batch.Closed || len(b.Delivery.Merges) != 1 {
|
|
||||||
t.Fatalf("the other change's batch reads %+v; want it untouched by the stop", b.Delivery)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A catalogue batch's cut folds in every walk waiting for its delivery's word (cutBatch's carry), so what that
|
|
||||||
// walk moves is the batch's too: a lone merge related to the waiting walk joins the batch assembling and waits for
|
|
||||||
// its window, rather than being cut at once as sharing nothing with what the batch's own merges move.
|
|
||||||
func TestALoneMergeRelatedToAWalkTheCutFoldsJoinsTheBatch(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
groupsWaiting(t, "pr/c3")
|
|
||||||
asksWithPaths(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1",
|
|
||||||
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
|
|
||||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main",
|
|
||||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
|
||||||
cutAt(t, open, t0.Add(90*time.Second))
|
|
||||||
if ws, _ := walks(t, open); len(ws) != 1 || !ws[0].Waiting() {
|
|
||||||
t.Fatalf("the walk does not wait for its word: %+v", ws)
|
|
||||||
}
|
|
||||||
hear(t, open, repoMerge("three", "c3", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
|
|
||||||
hear(t, open, lone(repoMerge("one", "c2", t0.Add(2*time.Minute+5*time.Second))), t0.Add(2*time.Minute+5*time.Second))
|
|
||||||
_, bs := walks(t, open)
|
|
||||||
if len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 || bs[0].Delivery.Batch.CutAtOnce {
|
|
||||||
t.Fatalf("a lone merge related to the waiting walk: batches %+v", bs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A delivery group with a member of each kind (review of #250): a merge on the controller's own path and a
|
|
||||||
// catalogue merge never share a batch, so the later member cannot join the earlier one's. Kept in a batch of the
|
|
||||||
// other kind, the earlier member keeps the later one waiting, whatever mesh-delivery says yet — never cut at once
|
|
||||||
// to walk before it. The two batches are then cut in the order their windows close (cutBatchesHeld: the oldest
|
|
||||||
// closed batch first, an own-path one first at one look, and the other queued behind a walk that started): the
|
|
||||||
// earlier member's window closes first, so it walks first, in both orders.
|
|
||||||
func TestAGroupAcrossBothKindsKeepsItsOrder(t *testing.T) {
|
|
||||||
for _, ownFirst := range []bool{true, false} {
|
|
||||||
name := "catalogue member first"
|
|
||||||
if ownFirst {
|
|
||||||
name = "controller member first"
|
|
||||||
}
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
asksWithPaths(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
was := deliveryGroupOf
|
|
||||||
// mesh-delivery still says both members ready: it marks one merged only once its walk opened.
|
|
||||||
deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) {
|
|
||||||
return []groupMember{{Repository: "novox/mesh-controller", State: "ready"},
|
|
||||||
{Repository: "novox/mesh-catalog", State: "ready"}}, true, nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { deliveryGroupOf = was })
|
|
||||||
own := fromHead(repoMerge("mesh-controller", "k1", t0), "feat/both-kinds")
|
|
||||||
cat := fromHead(catalogueMerge("cat1aaaaaaaa", "app", t0), "feat/both-kinds")
|
|
||||||
first, second := cat, own
|
|
||||||
if ownFirst {
|
|
||||||
first, second = own, cat
|
|
||||||
}
|
|
||||||
second.MergedAt = t0.Add(10 * time.Second).Format(time.RFC3339Nano)
|
|
||||||
hear(t, open, first, t0)
|
|
||||||
hear(t, open, second, t0.Add(10*time.Second))
|
|
||||||
ws, bs := walks(t, open)
|
|
||||||
if len(ws) != 0 || len(bs) != 2 {
|
|
||||||
t.Fatalf("both members heard: %d walk(s), %d batch(es); want two batches, nothing cut", len(ws), len(bs))
|
|
||||||
}
|
|
||||||
later := bs[0] // newest first
|
|
||||||
if later.Delivery.Batch.CutAtOnce || !strings.Contains(later.Delivery.Batch.Waits,
|
|
||||||
"has a member waiting in a batch of the other kind: novox/"+first.Repo+"@") {
|
|
||||||
t.Fatalf("the later member's batch reads %+v; want it waiting for the earlier member", later.Delivery.Batch)
|
|
||||||
}
|
|
||||||
// The earlier member's window closes first: its batch alone is cut.
|
|
||||||
cutAt(t, open, t0.Add(90*time.Second))
|
|
||||||
ws, bs = walks(t, open)
|
|
||||||
if len(ws) != 1 || !ws[0].Open() || ws[0].CommitOf("novox/"+first.Repo) != first.Commit || len(bs) != 1 ||
|
|
||||||
bs[0].ID != later.ID {
|
|
||||||
t.Fatalf("at the earlier window: walks %+v, batches %+v; want the earlier member alone cut", ws, bs)
|
|
||||||
}
|
|
||||||
earlier := ws[0]
|
|
||||||
// The later member's window closes while the earlier walk runs: it queues behind it.
|
|
||||||
cutAt(t, open, t0.Add(100*time.Second))
|
|
||||||
if _, bs := walks(t, open); len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != earlier.ID {
|
|
||||||
t.Fatalf("the later member's batch does not queue behind the earlier walk: %+v", bs)
|
|
||||||
}
|
|
||||||
earlier.State = inventory.PlanDone
|
|
||||||
if err := open.inventory.SavePlan(ctx, &earlier); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
cutAt(t, open, t0.Add(101*time.Second))
|
|
||||||
got, err := open.inventory.PlanByID(ctx, later.ID)
|
|
||||||
if err != nil || got.Batch() || got.CommitOf("novox/"+second.Repo) != second.Commit {
|
|
||||||
t.Fatalf("the later member was not walked after the earlier: %s %v", got.State, err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -63,15 +63,6 @@ func run() error {
|
|||||||
// 45 §6), so the next controller takes it at once rather than after its age.
|
// 45 §6), so the next controller takes it at once rather than after its age.
|
||||||
defer theLease.release()
|
defer theLease.release()
|
||||||
|
|
||||||
// Every walk a verb keeps is said on the bus, whichever verb kept it (novox/hq ADR 0308 decision 1): the save
|
|
||||||
// that starts or ends a walk keeps its mark, and no later save says it again. The serving controller says its
|
|
||||||
// own saves.
|
|
||||||
if args[0] != "serve" {
|
|
||||||
said, done := sayingWhatAVerbKept(ctx)
|
|
||||||
inventory.PlanSaved = said
|
|
||||||
defer done()
|
|
||||||
}
|
|
||||||
|
|
||||||
switch args[0] {
|
switch args[0] {
|
||||||
case "build":
|
case "build":
|
||||||
return buildCommand(ctx, args[1:])
|
return buildCommand(ctx, args[1:])
|
||||||
@@ -320,7 +311,7 @@ func usage() {
|
|||||||
the self-check: the last verdict, a run now, the probes, the signals' ages
|
the self-check: the last verdict, a run now, the probes, the signals' ages
|
||||||
healers [--days N] [--json] the healers, what they did lately, and their brake (to-be 45 §7)
|
healers [--days N] [--json] the healers, what they did lately, and their brake (to-be 45 §7)
|
||||||
durations [--kind K] [--days N] [--json]
|
durations [--kind K] [--days N] [--json]
|
||||||
apply, heartbeat, plan-tier, build and walk-phase durations
|
apply, heartbeat, plan-tier and build durations, per machine or module
|
||||||
collection [--json] kept archives held/unheld by a manifest, and what the sweep may let go
|
collection [--json] kept archives held/unheld by a manifest, and what the sweep may let go
|
||||||
builder issue <name> a broker account for a build machine, scoped to build work,
|
builder issue <name> a broker account for a build machine, scoped to build work,
|
||||||
delivered as the builder module's broker secret (module add it first)
|
delivered as the builder module's broker secret (module add it first)
|
||||||
|
|||||||
@@ -392,8 +392,7 @@ func TestACheckByHandClonesWhatTheSeatClones(t *testing.T) {
|
|||||||
for dir, refs := range map[string]map[string]string{
|
for dir, refs := range map[string]map[string]string{
|
||||||
"mesh-catalog": {"mesh-catalog": "main"},
|
"mesh-catalog": {"mesh-catalog": "main"},
|
||||||
"mesh-host": {"mesh-host": "c0ffee"},
|
"mesh-host": {"mesh-host": "c0ffee"},
|
||||||
"mesh-controller": {"mesh-controller": "c0ffee", "mesh-controller-main": "main", "mesh-lab": "main",
|
"mesh-controller": {"mesh-controller": "c0ffee", "mesh-controller-main": "main", "mesh-lab": "main"},
|
||||||
"mesh-sdk": sdkPinnedByGoMod(t)},
|
|
||||||
} {
|
} {
|
||||||
got := besideRefs(dir, "c0ffee")
|
got := besideRefs(dir, "c0ffee")
|
||||||
for d, ref := range refs {
|
for d, ref := range refs {
|
||||||
@@ -413,24 +412,3 @@ func TestACheckByHandClonesWhatTheSeatClones(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// sdkPinnedByGoMod is the SDK commit this tree's go.mod pins, read from the file rather than the build, so
|
|
||||||
// sdkPinned is held to what the repository says (novox/hq issue 449).
|
|
||||||
func sdkPinnedByGoMod(t *testing.T) string {
|
|
||||||
t.Helper()
|
|
||||||
raw, err := os.ReadFile(filepath.Join("..", "..", "go.mod"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, line := range strings.Split(string(raw), "\n") {
|
|
||||||
fields := strings.Fields(line)
|
|
||||||
if len(fields) >= 2 && fields[0] == sdkModule {
|
|
||||||
if m := pseudoCommit.FindStringSubmatch(fields[1]); m != nil {
|
|
||||||
return m[1]
|
|
||||||
}
|
|
||||||
return "go/" + fields[1]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
t.Fatalf("go.mod requires no %s", sdkModule)
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -74,21 +74,9 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke
|
|||||||
kept := inventory.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target,
|
kept := inventory.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target,
|
||||||
State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts,
|
State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts,
|
||||||
Check: r.Check, Needs: r.Needs, Account: r.Account, Root: r.Root}
|
Check: r.Check, Needs: r.Needs, Account: r.Account, Root: r.Root}
|
||||||
for _, w := range r.Waits {
|
|
||||||
kept.Waits = append(kept.Waits, inventory.Wait{Part: w.Part, Secret: w.Secret, Setting: w.Setting, What: w.What})
|
|
||||||
}
|
|
||||||
resources = append(resources, kept)
|
resources = append(resources, kept)
|
||||||
}
|
if r.State == link.StateUnhealthy && r.Module != "" {
|
||||||
// **A wait for the operator is checked before it is excused** (novox/hq ADR 0283): a waiting resource whose
|
unhealthy[r.Module] = append(unhealthy[r.Module], kept)
|
||||||
// wait does not check out is judged unhealthy, saying why; one that does is kept beside the unhealthy ones, so
|
|
||||||
// judgeModuleHealth can say it as needs-operator. What is stored is what the machine said.
|
|
||||||
var wf operatorWaitFacts
|
|
||||||
if mods := waitingModules(resources); len(mods) > 0 {
|
|
||||||
readWaitFacts(ctx, inv, node, mods, nil, &wf)
|
|
||||||
}
|
|
||||||
for _, r := range checkWaiting(node, resources, wf) {
|
|
||||||
if (r.State == link.StateUnhealthy || r.State == link.StateWaiting) && r.Module != "" {
|
|
||||||
unhealthy[r.Module] = append(unhealthy[r.Module], r)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
streaks := map[string]int{}
|
streaks := map[string]int{}
|
||||||
@@ -147,18 +135,17 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
|||||||
}
|
}
|
||||||
standing := map[string]conditions.Condition{}
|
standing := map[string]conditions.Condition{}
|
||||||
for _, c := range open {
|
for _, c := range open {
|
||||||
// A module left out of the composition is the self-check's to raise and clear, never a statement's
|
if (c.Kind == kindModuleUnhealthy || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound) &&
|
||||||
// (novox/hq issue 380): its needs-operator is not cleared for not being in what the machine runs.
|
c.Subject.Machine == node {
|
||||||
if (c.Kind == kindModuleUnhealthy || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound ||
|
|
||||||
c.Kind == kindNeedsOperator) &&
|
|
||||||
c.Subject.Machine == node && !isLeftOutCondition(c) {
|
|
||||||
standing[c.Key] = c
|
standing[c.Key] = c
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
hold, err := readHoldingFor(ctx, inv, open)
|
var hold *holding
|
||||||
if err != nil {
|
if inv != nil {
|
||||||
|
if hold, err = readHolding(ctx, inv, open); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
}
|
||||||
var problems []string
|
var problems []string
|
||||||
modules := make([]string, 0, len(unhealthy))
|
modules := make([]string, 0, len(unhealthy))
|
||||||
for m := range unhealthy {
|
for m := range unhealthy {
|
||||||
@@ -172,39 +159,6 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
|||||||
heldOn := map[string]string{}
|
heldOn := map[string]string{}
|
||||||
providers := map[catalogue.Chosen]bool{}
|
providers := map[catalogue.Chosen]bool{}
|
||||||
for _, m := range modules {
|
for _, m := range modules {
|
||||||
// **A part that waits for the operator is said as that** (novox/hq ADR 0283): its waits already checked,
|
|
||||||
// the operator's, never urgent, its words naming the act.
|
|
||||||
if waits, waiting := operatorWait(m, unhealthy[m]); waiting {
|
|
||||||
o := needsOperatorObservation(m, node, waits, unhealthy[m])
|
|
||||||
// **A provider waiting for the operator says who waits on it** (novox/hq issue 405), as one waiting for a
|
|
||||||
// login does: its consumers are held under it.
|
|
||||||
if hold != nil {
|
|
||||||
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
|
|
||||||
}
|
|
||||||
seen[o.Key()] = true
|
|
||||||
became[m] = kindNeedsOperator
|
|
||||||
if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if _, err := k.Observe(ctx, o); err != nil {
|
|
||||||
problems = append(problems, err.Error())
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
// **A wait for the operator beside anything else is said too** (novox/hq issue 405): a module with a checked
|
|
||||||
// wait beside a new login owed, a directory used as found or a fault of its own is said as that, and the
|
|
||||||
// operator's secret or setting it waits for was not mentioned until the other cleared. Its needs-operator
|
|
||||||
// condition stands beside the other, on the same looks; what follows judges the rest without the wait.
|
|
||||||
if waits, rest := besideAWait(m, unhealthy[m]); len(waits) > 0 {
|
|
||||||
o := needsOperatorObservation(m, node, waits, waitingOf(m, unhealthy[m]))
|
|
||||||
seen[o.Key()] = true
|
|
||||||
if _, isOpen := standing[o.Key()]; streaks[m] >= moduleUnhealthyAfter || isOpen {
|
|
||||||
if _, err := k.Observe(ctx, o); err != nil {
|
|
||||||
problems = append(problems, err.Error())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
unhealthy[m] = rest
|
|
||||||
}
|
|
||||||
// **A directory used as found is said as that** (novox/hq issue 339): the operator's to hand over at the
|
// **A directory used as found is said as that** (novox/hq issue 339): the operator's to hand over at the
|
||||||
// machine, never urgent — nothing is broken by the wait that a person was not told of — and its own kind,
|
// machine, never urgent — nothing is broken by the wait that a person was not told of — and its own kind,
|
||||||
// so the gate never reads it as a fault of the build that happened to be sent beside it.
|
// so the gate never reads it as a fault of the build that happened to be sent beside it.
|
||||||
@@ -241,24 +195,13 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
|||||||
}
|
}
|
||||||
o := moduleUnhealthyObservation(m, node, unhealthy[m])
|
o := moduleUnhealthyObservation(m, node, unhealthy[m])
|
||||||
if hold != nil {
|
if hold != nil {
|
||||||
if by, held := hold.heldWith(node, m, unhealthy[m]); held {
|
if p, held := hold.heldUnder(node, m, unhealthy[m]); held {
|
||||||
// Held under the provider's condition: nothing of its own, and the provider's says it waits. The
|
// Held under the provider's condition: nothing of its own, and the provider's says it waits.
|
||||||
// clearing line says which the provider is: unhealthy, or waiting for the operator (issue 405).
|
heldOn[o.Key()] = p.Module + " on " + p.Node
|
||||||
p := by.provider
|
|
||||||
heldOn[o.Key()] = p.Module + " on " + p.Node + ", which is unhealthy"
|
|
||||||
if len(by.waits) > 0 {
|
|
||||||
heldOn[o.Key()] = p.Module + " on " + p.Node + ", which waits for you"
|
|
||||||
}
|
|
||||||
providers[p] = true
|
providers[p] = true
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
|
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
|
||||||
// A provider that waits for the operator for a part this finding cannot be matched to does not hold it
|
|
||||||
// (novox/hq issue 405): raised as its own, and saying the provider waits, so neither is hidden.
|
|
||||||
if p, waiting := hold.waitingUncovered(node, m, unhealthy[m]); waiting {
|
|
||||||
o.Said += fmt.Sprintf("; %s on %s waits for you, but not for anything %s is known to need, so %s's "+
|
|
||||||
"fault is said on its own", p.Module, p.Node, m, m)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
seen[o.Key()] = true
|
seen[o.Key()] = true
|
||||||
became[m] = kindModuleUnhealthy
|
became[m] = kindModuleUnhealthy
|
||||||
@@ -285,11 +228,8 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
|||||||
if c.Kind == kindUsedAsFound {
|
if c.Kind == kindUsedAsFound {
|
||||||
why = fmt.Sprintf("%s says no directory of %s is used as found any more", node, module)
|
why = fmt.Sprintf("%s says no directory of %s is used as found any more", node, module)
|
||||||
}
|
}
|
||||||
if c.Kind == kindNeedsOperator {
|
|
||||||
why = fmt.Sprintf("%s says %s no longer waits for the operator", node, module)
|
|
||||||
}
|
|
||||||
if on, held := heldOn[key]; held {
|
if on, held := heldOn[key]; held {
|
||||||
why = fmt.Sprintf("what %s finds on %s waits on %s: held under its condition", module, node, on)
|
why = fmt.Sprintf("what %s finds on %s waits on %s, which is unhealthy: held under its condition", module, node, on)
|
||||||
}
|
}
|
||||||
// **A condition that became the other kind** is not "working again" (issue 318 review): its clearing
|
// **A condition that became the other kind** is not "working again" (issue 318 review): its clearing
|
||||||
// line says what it became.
|
// line says what it became.
|
||||||
@@ -298,12 +238,6 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
|||||||
case c.Kind == kindModuleUnhealthy && became[module] == kindReloginNeeded:
|
case c.Kind == kindModuleUnhealthy && became[module] == kindReloginNeeded:
|
||||||
why = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
|
why = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
|
||||||
resolved = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
|
resolved = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
|
||||||
case c.Kind == kindModuleUnhealthy && became[module] == kindNeedsOperator:
|
|
||||||
why = fmt.Sprintf("%s on %s now only waits for the operator", module, node)
|
|
||||||
resolved = fmt.Sprintf("%s on %s now only waits for you", module, node)
|
|
||||||
case c.Kind == kindNeedsOperator && became[module] == kindModuleUnhealthy:
|
|
||||||
why = fmt.Sprintf("%s on %s no longer only waits for the operator, and is not healthy", module, node)
|
|
||||||
resolved = fmt.Sprintf("What %s on %s waited for is given, and it still does not work", module, node)
|
|
||||||
case c.Kind == kindReloginNeeded && became[module] == kindModuleUnhealthy:
|
case c.Kind == kindReloginNeeded && became[module] == kindModuleUnhealthy:
|
||||||
why = fmt.Sprintf("%s on %s no longer waits for a new login, and is not healthy", module, node)
|
why = fmt.Sprintf("%s on %s no longer waits for a new login, and is not healthy", module, node)
|
||||||
resolved = fmt.Sprintf("The new login on %s is done, and %s still does not work", node, module)
|
resolved = fmt.Sprintf("The new login on %s is done, and %s still does not work", node, module)
|
||||||
@@ -328,74 +262,36 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
|||||||
// sayWaiters observes a provider's open condition again, with who waits on it, from its machine's newest
|
// sayWaiters observes a provider's open condition again, with who waits on it, from its machine's newest
|
||||||
// statement. Nothing when its condition is not open: it is raised by its own statements, on its own looks.
|
// statement. Nothing when its condition is not open: it is raised by its own statements, on its own looks.
|
||||||
func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p catalogue.Chosen, now time.Time) error {
|
func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p catalogue.Chosen, now time.Time) error {
|
||||||
// Its statement as it was judged, its waits checked (novox/hq issue 450): a wait that failed the check is
|
var raisedAt *conditions.Condition
|
||||||
// unhealthy, so the condition built here is the one its own statement raised, and who waits on it is listed.
|
for i, c := range hold.open {
|
||||||
|
if c.Key == moduleUnhealthyKey(p.Module, p.Node) || c.Key == reloginKey(p.Module, p.Node) {
|
||||||
|
raisedAt = &hold.open[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if raisedAt == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
var rs []inventory.ResourceHealth
|
var rs []inventory.ResourceHealth
|
||||||
for _, r := range hold.checked(p.Node) {
|
for _, r := range hold.healths[p.Node].Resources {
|
||||||
if r.Module == p.Module && (r.State == link.StateUnhealthy || r.State == link.StateWaiting) {
|
if r.Module == p.Module && r.State == link.StateUnhealthy {
|
||||||
rs = append(rs, r)
|
rs = append(rs, r)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(rs) == 0 {
|
if len(rs) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
// The condition its own statement says it under: needs-operator while it only waits for the operator (novox/hq
|
o := moduleUnhealthyObservation(p.Module, p.Node, rs)
|
||||||
// issue 405), else that for the rest of it, a wait beside it said on its own.
|
if said, waits := personWait(p.Module, p.Node, rs); waits {
|
||||||
var o conditions.Observation
|
o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rs)
|
||||||
if waits, waiting := operatorWait(p.Module, rs); waiting {
|
|
||||||
o = needsOperatorObservation(p.Module, p.Node, waits, rs)
|
|
||||||
} else {
|
|
||||||
_, rest := besideAWait(p.Module, rs)
|
|
||||||
o = moduleUnhealthyObservation(p.Module, p.Node, rest)
|
|
||||||
if said, waits := personWait(p.Module, p.Node, rest); waits {
|
|
||||||
o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rest)
|
|
||||||
}
|
}
|
||||||
}
|
if o.Key() != raisedAt.Key {
|
||||||
raised := false
|
return nil // its own statement says it next
|
||||||
for _, c := range hold.open {
|
|
||||||
raised = raised || c.Key == o.Key()
|
|
||||||
}
|
|
||||||
if !raised {
|
|
||||||
return nil // not open yet, or open as another kind: its own statement says it next
|
|
||||||
}
|
}
|
||||||
sayWaitingOn(&o, hold.waitersOn(p))
|
sayWaitingOn(&o, hold.waitersOn(p))
|
||||||
_, err := k.Observe(ctx, o)
|
_, err := k.Observe(ctx, o)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// besideAWait is, for a module with something not healthy beside a part waiting for the operator, the waits (checked
|
|
||||||
// already) and the rest without the waiting parts (novox/hq issue 405); no waits when nothing waits, or when the
|
|
||||||
// module only waits (operatorWait says that whole). Pure.
|
|
||||||
func besideAWait(module string, rs []inventory.ResourceHealth) ([]inventory.Wait, []inventory.ResourceHealth) {
|
|
||||||
if _, only := operatorWait(module, rs); only {
|
|
||||||
return nil, rs
|
|
||||||
}
|
|
||||||
var waits []inventory.Wait
|
|
||||||
var rest []inventory.ResourceHealth
|
|
||||||
for _, r := range rs {
|
|
||||||
if r.Module == module && r.State == link.StateWaiting {
|
|
||||||
waits = append(waits, r.Waits...)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
rest = append(rest, r)
|
|
||||||
}
|
|
||||||
if len(waits) == 0 {
|
|
||||||
return nil, rs
|
|
||||||
}
|
|
||||||
return waits, rest
|
|
||||||
}
|
|
||||||
|
|
||||||
// waitingOf is a module's waiting resources: the evidence of its wait.
|
|
||||||
func waitingOf(module string, rs []inventory.ResourceHealth) []inventory.ResourceHealth {
|
|
||||||
var out []inventory.ResourceHealth
|
|
||||||
for _, r := range rs {
|
|
||||||
if r.Module == module && r.State == link.StateWaiting {
|
|
||||||
out = append(out, r)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// reloginKey is a module's relogin-needed condition on a machine.
|
// reloginKey is a module's relogin-needed condition on a machine.
|
||||||
func reloginKey(module, node string) string {
|
func reloginKey(module, node string) string {
|
||||||
return conditions.Key(conditions.ScopeModule, module+"."+node, kindReloginNeeded)
|
return conditions.Key(conditions.ScopeModule, module+"."+node, kindReloginNeeded)
|
||||||
@@ -467,15 +363,12 @@ func waitingAccounts(module string, rs []inventory.ResourceHealth) []string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// sayWaitingOn adds to a module's condition the consumers held under it (to-be 48 §6): urgent while anyone
|
// sayWaitingOn adds to a module's condition the consumers held under it (to-be 48 §6): urgent while anyone
|
||||||
// waits on it, whether it is not working or waits for a new login. **A wait for the operator's secret or setting
|
// waits on it, whether it is not working or waits for a new login.
|
||||||
// stays a warning** (ADR 0283 decision 5, novox/hq issue 405): who waits on it is listed, and nothing escalates it.
|
|
||||||
func sayWaitingOn(o *conditions.Observation, waiters []string) {
|
func sayWaitingOn(o *conditions.Observation, waiters []string) {
|
||||||
if len(waiters) == 0 {
|
if len(waiters) == 0 {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if o.Kind != kindNeedsOperator {
|
|
||||||
o.Severity = conditions.Urgent
|
o.Severity = conditions.Urgent
|
||||||
}
|
|
||||||
o.Said += "; " + waitingWords(waiters)
|
o.Said += "; " + waitingWords(waiters)
|
||||||
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
|
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
|
||||||
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
|
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
|
||||||
@@ -497,11 +390,9 @@ func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHeal
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
words = append(words, fmt.Sprintf("its %s %s %s", r.Kind, r.Resource, reasonWords(r)))
|
words = append(words, fmt.Sprintf("its %s %s %s", r.Kind, r.Resource, reasonWords(r)))
|
||||||
// The restart count runs from when the engine began watching it, not from the state's since (novox/hq
|
said = append(said, fmt.Sprintf("%s (%s %s): %s, %d look(s) in a row, %d restart(s) counted, since %s",
|
||||||
// issue 458).
|
r.Resource, r.Kind, r.Target, orNotSaid(r.Reason), r.Streak, r.Restarts,
|
||||||
said = append(said, fmt.Sprintf("%s (%s %s): %s, since %s, %d look(s) in a row, %s",
|
r.Since.UTC().Format("2006-01-02 15:04:05 MST")))
|
||||||
r.Resource, r.Kind, r.Target, orNotSaid(r.Reason), r.Since.UTC().Format("2006-01-02 15:04:05 MST"),
|
|
||||||
r.Streak, restartsWords(r.Restarts)))
|
|
||||||
}
|
}
|
||||||
return conditions.Observation{Scope: conditions.ScopeModule, ID: module + "." + node, Token: "unhealthy",
|
return conditions.Observation{Scope: conditions.ScopeModule, ID: module + "." + node, Token: "unhealthy",
|
||||||
Kind: kindModuleUnhealthy, Machine: node, Severity: conditions.Warning, Source: sourceHealth,
|
Kind: kindModuleUnhealthy, Machine: node, Severity: conditions.Warning, Source: sourceHealth,
|
||||||
@@ -515,15 +406,6 @@ func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHeal
|
|||||||
Resolved: fmt.Sprintf("%s works again on %s", module, node)}
|
Resolved: fmt.Sprintf("%s works again on %s", module, node)}
|
||||||
}
|
}
|
||||||
|
|
||||||
// restartsWords is a restart count as what it is (novox/hq issue 458). The engine reports its own count
|
|
||||||
// (mesh-host's liveness judge, kept.Counted): every rise of systemd's NRestarts or the runtime's RestartCount
|
|
||||||
// after a start's grace, so only restarts by the Restart= setting or the restart policy. It keeps the count
|
|
||||||
// across a recreate, a start by hand and its own restarts, and forgets it only when the resource is no longer
|
|
||||||
// declared. So it counts automatic restarts since the engine began watching it, never since a state's "since".
|
|
||||||
func restartsWords(n int) string {
|
|
||||||
return fmt.Sprintf("%d automatic restart(s) since the node-engine began watching it", n)
|
|
||||||
}
|
|
||||||
|
|
||||||
// reasonWords is why a resource is unhealthy, as a person reads it.
|
// reasonWords is why a resource is unhealthy, as a person reads it.
|
||||||
func reasonWords(r inventory.ResourceHealth) string {
|
func reasonWords(r inventory.ResourceHealth) string {
|
||||||
// An account waiting for a new login (ADR 0252) is said in the mesh's words, not the engine's.
|
// An account waiting for a new login (ADR 0252) is said in the mesh's words, not the engine's.
|
||||||
@@ -532,17 +414,12 @@ func reasonWords(r inventory.ResourceHealth) string {
|
|||||||
}
|
}
|
||||||
switch r.Reason {
|
switch r.Reason {
|
||||||
case "restarting":
|
case "restarting":
|
||||||
return fmt.Sprintf("keeps restarting (%s)", restartsWords(r.Restarts))
|
return fmt.Sprintf("keeps restarting (%d restart(s) counted)", r.Restarts)
|
||||||
case "down":
|
case "down":
|
||||||
return "is not running"
|
return "is not running"
|
||||||
case "":
|
case "":
|
||||||
return "is unhealthy"
|
return "is unhealthy"
|
||||||
}
|
}
|
||||||
// A wait for the operator that did not check out is said as the controller found it (ADR 0283): names of
|
|
||||||
// secrets and settings only, never what the check itself said.
|
|
||||||
if strings.HasPrefix(r.Reason, waitRefusedPrefix) {
|
|
||||||
return r.Reason
|
|
||||||
}
|
|
||||||
// What a declared check found says an endpoint, a path or an address: evidence, never the summary the
|
// What a declared check found says an endpoint, a path or an address: evidence, never the summary the
|
||||||
// operator's channel carries (ADR 0234 §6). The summary names the check.
|
// operator's channel carries (ADR 0234 §6). The summary names the check.
|
||||||
if r.Check != "" {
|
if r.Check != "" {
|
||||||
@@ -634,9 +511,7 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
|
|||||||
if h.HeardAt.Before(since) {
|
if h.HeardAt.Before(since) {
|
||||||
return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module)
|
return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module)
|
||||||
}
|
}
|
||||||
// **A wait for the operator is checked first** (novox/hq ADR 0283): one that does not check out is unhealthy.
|
wait, waits := personWait(module, machine, h.Resources)
|
||||||
resources := checkWaiting(machine, h.Resources, f.waits)
|
|
||||||
wait, waits := personWait(module, machine, resources)
|
|
||||||
// **Only a build whose own send put the account in a new group is excused** (issue 318 review): read from
|
// **Only a build whose own send put the account in a new group is excused** (issue 318 review): read from
|
||||||
// what the controller sent, never from when the machine says the wait began — that time is the engine's
|
// what the controller sent, never from when the machine says the wait began — that time is the engine's
|
||||||
// memory, reset by its restart and moved by a change of words. A build that adds no account group cannot
|
// memory, reset by its restart and moved by a change of words. A build that adds no account group cannot
|
||||||
@@ -644,18 +519,11 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
|
|||||||
if waits && !f.groupsAdded[module] {
|
if waits && !f.groupsAdded[module] {
|
||||||
waits = false
|
waits = false
|
||||||
}
|
}
|
||||||
var found, onWaiting []string
|
var found []string
|
||||||
var forOperator []inventory.Wait
|
for _, r := range h.Resources {
|
||||||
for _, r := range resources {
|
|
||||||
if r.Module != module {
|
if r.Module != module {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
// **Any build is excused while its wait for the operator checks out** (ADR 0283 decision 4): a secret not
|
|
||||||
// given is owed by every build alike, so it is no fault of this one, and the verdict carries it.
|
|
||||||
if r.State == link.StateWaiting {
|
|
||||||
forOperator = append(forOperator, r.Waits...)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if waits && r.State == link.StateUnhealthy {
|
if waits && r.State == link.StateUnhealthy {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -673,16 +541,8 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
|
|||||||
return healthNotYet, fmt.Sprintf("its %s %s on %s is still starting", r.Kind, r.Resource, machine)
|
return healthNotYet, fmt.Sprintf("its %s %s on %s is still starting", r.Kind, r.Resource, machine)
|
||||||
case link.StateUnhealthy:
|
case link.StateUnhealthy:
|
||||||
if on, held := f.heldOn[module+"@"+machine]; held {
|
if on, held := f.heldOn[module+"@"+machine]; held {
|
||||||
// **Held under a provider that only waits for the operator** (novox/hq issue 405): a wait for a
|
|
||||||
// person, a pass carrying the wait (ADR 0254, ADR 0283 decision 4) — the walk never waits for the
|
|
||||||
// operator's secret, whichever module owes it.
|
|
||||||
if on.waits != "" {
|
|
||||||
onWaiting = append(onWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which waits for you: %s",
|
|
||||||
r.Kind, r.Resource, machine, on.on, on.waits))
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
return healthWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which is unhealthy", r.Kind,
|
return healthWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which is unhealthy", r.Kind,
|
||||||
r.Resource, machine, on.on)
|
r.Resource, machine, on)
|
||||||
}
|
}
|
||||||
return healthNotYet, fmt.Sprintf("its %s %s on %s %s", r.Kind, r.Resource, machine, reasonWords(r))
|
return healthNotYet, fmt.Sprintf("its %s %s on %s %s", r.Kind, r.Resource, machine, reasonWords(r))
|
||||||
default:
|
default:
|
||||||
@@ -690,14 +550,11 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
|
|||||||
reasonAfter(r.Reason))
|
reasonAfter(r.Reason))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if waits || len(found) > 0 || len(forOperator) > 0 || len(onWaiting) > 0 {
|
if waits || len(found) > 0 {
|
||||||
said := onWaiting
|
var said []string
|
||||||
if waits {
|
if waits {
|
||||||
said = append(said, wait)
|
said = append(said, wait)
|
||||||
}
|
}
|
||||||
if len(forOperator) > 0 {
|
|
||||||
said = append(said, operatorWaitSaid(module, machine, forOperator))
|
|
||||||
}
|
|
||||||
if len(found) > 0 {
|
if len(found) > 0 {
|
||||||
said = append(said, fmt.Sprintf("on %s, %s uses %s as found and waits for the operator to hand it over "+
|
said = append(said, fmt.Sprintf("on %s, %s uses %s as found and waits for the operator to hand it over "+
|
||||||
"(`nox node hand-over %s <directory>` on the control-node)", machine, module, strings.Join(found, ", "), machine))
|
"(`nox node hand-over %s <directory>` on the control-node)", machine, module, strings.Join(found, ", "), machine))
|
||||||
@@ -736,7 +593,7 @@ func healthLines(h inventory.NodeHealth, had bool, now time.Time) []string {
|
|||||||
line += " — " + r.Reason
|
line += " — " + r.Reason
|
||||||
}
|
}
|
||||||
if r.Restarts > 0 {
|
if r.Restarts > 0 {
|
||||||
line += ", " + restartsWords(r.Restarts)
|
line += fmt.Sprintf(", %d restart(s) counted", r.Restarts)
|
||||||
}
|
}
|
||||||
if r.State == link.StateUnhealthy && h.Streaks[r.Module] < moduleUnhealthyAfter {
|
if r.State == link.StateUnhealthy && h.Streaks[r.Module] < moduleUnhealthyAfter {
|
||||||
line += " (unconfirmed: said once)"
|
line += " (unconfirmed: said once)"
|
||||||
|
|||||||
@@ -3,12 +3,10 @@ package main
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
|
||||||
"reflect"
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/beside"
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/overlay"
|
"github.com/novox/mesh-controller/internal/overlay"
|
||||||
@@ -241,12 +239,13 @@ func TestATakeoverIsNotComposedForAHubPlacedOffItsTunnel(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// theResolver is the catalogue's dnsmasq module as it is (internal/beside).
|
// theResolver is the catalogue's dnsmasq module as it is, or the test is skipped where the
|
||||||
|
// catalogue is not beside this checkout.
|
||||||
func theResolver(t *testing.T) catalogue.Manifest {
|
func theResolver(t *testing.T) catalogue.Manifest {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
raw, err := os.ReadFile(filepath.Join(beside.Catalogue(t), "dnsmasq", "module.json"))
|
raw, err := os.ReadFile("../../../mesh-catalog/modules/dnsmasq/module.json")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Skipf("the catalogue is not beside this checkout: %v", err)
|
||||||
}
|
}
|
||||||
m, err := catalogue.ParseManifest(raw)
|
m, err := catalogue.ParseManifest(raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -44,7 +44,7 @@ func nodeCommand(ctx context.Context, args []string) error {
|
|||||||
if len(args) != 2 {
|
if len(args) != 2 {
|
||||||
return errors.New("node show <name>")
|
return errors.New("node show <name>")
|
||||||
}
|
}
|
||||||
return showNode(ctx, open, args[1])
|
return showNode(ctx, inv, args[1])
|
||||||
case "add":
|
case "add":
|
||||||
return addNode(ctx, inv, args[1:])
|
return addNode(ctx, inv, args[1:])
|
||||||
|
|
||||||
@@ -127,14 +127,9 @@ func nodeCommand(ctx context.Context, args []string) error {
|
|||||||
// and whoever may call a verb includes agents.
|
// and whoever may call a verb includes agents.
|
||||||
return nodeSetuidSearch(ctx, open, args[1:])
|
return nodeSetuidSearch(ctx, open, args[1:])
|
||||||
|
|
||||||
case "judge-directories":
|
|
||||||
// Which of a node's declared directories the walk that follows no link another account could make would
|
|
||||||
// refuse (novox/hq issue 358). Read-only on the node and here, so a verb may run it too.
|
|
||||||
return nodeJudgeDirectories(ctx, open, args[1:])
|
|
||||||
|
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("node has no %q; it has add, list, show, public-domain, account, agent-account, hand-over, "+
|
return fmt.Errorf("node has no %q; it has add, list, show, public-domain, account, agent-account, hand-over "+
|
||||||
"setuid-search and judge-directories", args[0])
|
"and setuid-search", args[0])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -792,8 +787,7 @@ func roughly(d time.Duration) string {
|
|||||||
//
|
//
|
||||||
// It is also where "what should it be configured as" is read. The same line that gates an
|
// It is also where "what should it be configured as" is read. The same line that gates an
|
||||||
// assignment carries `card1-DP-1`, and a person composing settings for that machine needs it.
|
// assignment carries `card1-DP-1`, and a person composing settings for that machine needs it.
|
||||||
func showNode(ctx context.Context, stored *stores, name string) error {
|
func showNode(ctx context.Context, inv *inventory.Inventory, name string) error {
|
||||||
inv := stored.inventory
|
|
||||||
node, err := inv.NodeByName(ctx, name)
|
node, err := inv.NodeByName(ctx, name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -809,19 +803,6 @@ func showNode(ctx context.Context, stored *stores, name string) error {
|
|||||||
if err := showMode(ctx, inv, node); err != nil {
|
if err := showMode(ctx, inv, node); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// What it was last sent, by whom, under which epoch, from which generation, naming which modules, and what
|
|
||||||
// it answered (novox/hq issue 485): the record of sends is read through this verb, never the store's reader.
|
|
||||||
// Unreadable is said, not taken for none.
|
|
||||||
switch last, found, err := inv.LastSend(ctx, name); {
|
|
||||||
case err != nil:
|
|
||||||
fmt.Printf(" what it was last sent could NOT be read: %v\n", err)
|
|
||||||
case !found:
|
|
||||||
fmt.Printf(" last sent no send recorded since the mesh began keeping them\n")
|
|
||||||
default:
|
|
||||||
for _, line := range lastSendLines(last) {
|
|
||||||
fmt.Println(line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Whom agents run as here, and whether that account can become root without a person (ADR 0266).
|
// Whom agents run as here, and whether that account can become root without a person (ADR 0266).
|
||||||
for _, line := range agentAccountLines(ctx, inv, node) {
|
for _, line := range agentAccountLines(ctx, inv, node) {
|
||||||
fmt.Println(line)
|
fmt.Println(line)
|
||||||
@@ -908,17 +889,6 @@ func showNode(ctx context.Context, stored *stores, name string) error {
|
|||||||
if len(assigned) > 0 {
|
if len(assigned) > 0 {
|
||||||
fmt.Printf("\n assigned: %s\n", strings.Join(assigned, ", "))
|
fmt.Printf("\n assigned: %s\n", strings.Join(assigned, ", "))
|
||||||
}
|
}
|
||||||
// And which of them a push leaves out, and why (novox/hq issue 380): judged as the push judges it, the same
|
|
||||||
// reason its condition says. Not computable is said, never read as "all applied".
|
|
||||||
plan, settings, err := planFor(ctx, stored, name)
|
|
||||||
switch {
|
|
||||||
case err != nil:
|
|
||||||
fmt.Printf("\n whether a send leaves any assigned module out is NOT known: %s\n", oneLine(err.Error()))
|
|
||||||
default:
|
|
||||||
for _, line := range notAppliedLines(name, leftOutOf(plan, settings, node.Adopted)) {
|
|
||||||
fmt.Println(line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -942,57 +912,3 @@ func printJSONTo(w io.Writer, v any) error {
|
|||||||
fmt.Fprintln(w, string(body))
|
fmt.Fprintln(w, string(body))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
const judgeDirectoriesUsage = "node judge-directories <node> — read-only: which of the directories <node> is " +
|
|
||||||
"declared the node-engine's walk that follows no link another account could make would refuse, one line each " +
|
|
||||||
"(novox/hq issue 358). Nothing is made, chmodded or chowned"
|
|
||||||
|
|
||||||
// nodeJudgeDirectories asks the node's engine to judge its declared directories, signed with the mesh's key as a
|
|
||||||
// hand-over is. Read-only: the engine changes nothing and keeps nothing.
|
|
||||||
func nodeJudgeDirectories(ctx context.Context, open *stores, args []string) error {
|
|
||||||
known := func(node string) error {
|
|
||||||
_, err := open.inventory.NodeByName(ctx, node)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
ask := func(node, by string) (link.HandOverAnswer, error) {
|
|
||||||
ident, err := open.Identity(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return link.HandOverAnswer{}, fmt.Errorf("the mesh's signing key cannot be read, so nothing was asked of %s: %w",
|
|
||||||
node, err)
|
|
||||||
}
|
|
||||||
address, err := broker.BusAddress()
|
|
||||||
if err != nil {
|
|
||||||
return link.HandOverAnswer{}, err
|
|
||||||
}
|
|
||||||
js, err := broker.Dial(address)
|
|
||||||
if err != nil {
|
|
||||||
return link.HandOverAnswer{}, fmt.Errorf("cannot reach the bus, so nothing was asked of %s: %w", node, err)
|
|
||||||
}
|
|
||||||
defer js.Close()
|
|
||||||
return link.AskJudgeDirectories(ctx, js.Conn(), ident, node, by, link.HandOverWithin)
|
|
||||||
}
|
|
||||||
return judgeDirectoriesAsked(args, known, ask, os.Stdout)
|
|
||||||
}
|
|
||||||
|
|
||||||
// judgeDirectoriesAsked is the line with its two acts given: whether the mesh knows the node, and the ask. The
|
|
||||||
// engine's refusal is this command's failure.
|
|
||||||
func judgeDirectoriesAsked(args []string, known func(node string) error,
|
|
||||||
ask func(node, by string) (link.HandOverAnswer, error), out io.Writer) error {
|
|
||||||
if len(args) != 1 || args[0] == "" || strings.HasPrefix(args[0], "-") {
|
|
||||||
return errors.New(judgeDirectoriesUsage)
|
|
||||||
}
|
|
||||||
node := args[0]
|
|
||||||
if err := known(node); err != nil {
|
|
||||||
return fmt.Errorf("nothing was asked: %w", err)
|
|
||||||
}
|
|
||||||
answer, err := ask(node, handOverBy())
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if answer.Refused != "" {
|
|
||||||
return fmt.Errorf("%s refused: %s", node, answer.Refused)
|
|
||||||
}
|
|
||||||
fmt.Fprintf(out, "%s (read-only: nothing on %s was changed)\n", node, node)
|
|
||||||
fmt.Fprintln(out, answer.Said)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,112 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
|
||||||
)
|
|
||||||
|
|
||||||
// novox/hq issue 237: every line of an act's answer comes from one resolution of the node taken after
|
|
||||||
// its write, so the answer agrees with `plan` for the node right after.
|
|
||||||
|
|
||||||
func aProviderAndItsConsumer(t *testing.T, open *stores) {
|
|
||||||
t.Helper()
|
|
||||||
register(t, open, catalogue.Manifest{Module: "db", Version: "1",
|
|
||||||
Provides: []catalogue.Offer{{Name: "rows", Scope: catalogue.ScopeNode}}})
|
|
||||||
register(t, open, catalogue.Manifest{Module: "app", Version: "1", Requires: []string{"rows"}})
|
|
||||||
}
|
|
||||||
|
|
||||||
// twoRivalsAndANote is a node that does not resolve — two modules claim one node seat there — and a
|
|
||||||
// module beside them that changes nothing about it.
|
|
||||||
func twoRivalsAndANote(t *testing.T, open *stores) {
|
|
||||||
t.Helper()
|
|
||||||
one, two := rivals()
|
|
||||||
register(t, open, one)
|
|
||||||
register(t, open, two)
|
|
||||||
register(t, open, catalogue.Manifest{Module: "notes", Version: "1"})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAnUnassignThatLeavesItsNodeUnresolvedSaysSoOnceInItsAnswer(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
twoRivalsAndANote(t, open)
|
|
||||||
// Straight into the store, as a node that came to this before the act.
|
|
||||||
for _, m := range []string{"rival-one", "rival-two", "notes"} {
|
|
||||||
if _, err := open.inventory.Assign(ctx, "laptop", m); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
said := answered(t, func() error { return assignCommand(ctx, "unassign", []string{"laptop", "notes"}) })
|
|
||||||
|
|
||||||
if _, _, err := planFor(ctx, open, "laptop"); err == nil || !unresolvable(err) {
|
|
||||||
t.Fatalf("laptop resolves with two claimants of one seat: %v", err)
|
|
||||||
}
|
|
||||||
if n := strings.Count(said, "laptop does not resolve now"); n != 1 {
|
|
||||||
t.Errorf("the answer says laptop does not resolve %d times, want once:\n%s", n, said)
|
|
||||||
}
|
|
||||||
if strings.Contains(said, "is left out of the rest of the mesh") {
|
|
||||||
t.Errorf("the answer carries a line from a read that is not its resolution:\n%s", said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAnUnassignThatLeavesItsNodeResolvedSaysNothingOfIt(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
aProviderAndItsConsumer(t, open)
|
|
||||||
if _, err := assign(ctx, open, "laptop", "db", "app"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
said, err := unassign(ctx, open, "laptop", "app")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, _, err := planFor(ctx, open, "laptop"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if strings.Contains(said, "does not resolve") || strings.Contains(said, "could not be read") {
|
|
||||||
t.Errorf("a node that resolves is said not to:\n%s", said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAnAssignThatLeavesItsNodeUnresolvedSaysOnlyThePlansRefusal(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
twoRivalsAndANote(t, open)
|
|
||||||
if _, err := assign(ctx, open, "laptop", "rival-one"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
// Kept and not resolvable: a second claimant of the seat, which assignment keeps.
|
|
||||||
said, err := assign(ctx, open, "laptop", "rival-two")
|
|
||||||
if err == nil {
|
|
||||||
t.Fatalf("two claimants of one node seat resolved:\n%s", said)
|
|
||||||
}
|
|
||||||
_, _, planErr := planFor(ctx, open, "laptop")
|
|
||||||
if planErr == nil || planErr.Error() != err.Error() {
|
|
||||||
t.Errorf("the act's refusal is not the plan's right after:\nact: %v\nplan: %v", err, planErr)
|
|
||||||
}
|
|
||||||
if strings.Contains(said, "now has") {
|
|
||||||
t.Errorf("the answer says a seat is now held on a node that does not resolve:\n%s", said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The other readers of the whole mesh still say a machine they leave out, on standard error (novox/hq
|
|
||||||
// issue 188); an act sends it elsewhere.
|
|
||||||
func TestAMachineLeftOutIsSaidWhereTheContextSays(t *testing.T) {
|
|
||||||
var buf bytes.Buffer
|
|
||||||
sayLeftOut(leftOutSaidBy(context.Background(), &buf), "laptop", errors.New("why"))
|
|
||||||
if buf.String() != "laptop is left out of the rest of the mesh: it does not resolve: why\n" {
|
|
||||||
t.Errorf("said %q", buf.String())
|
|
||||||
}
|
|
||||||
got := answered(t, func() error {
|
|
||||||
sayLeftOut(context.Background(), "laptop", errors.New("why"))
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
if !strings.Contains(got, "laptop is left out") {
|
|
||||||
t.Errorf("without an act's writer the line did not reach standard error: %q", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,261 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A part that waits for the operator's secret or setting (novox/hq ADR 0283, issue 386).
|
|
||||||
//
|
|
||||||
// **A module's tool check may say it waits**: nothing of it is wrong but a part that cannot work until the operator
|
|
||||||
// gives one of its own secrets or one of its settings. The node-engine states such a resource `waiting`, with what
|
|
||||||
// it waits for. A module saying so is an assertion, so **the controller checks each wait before it excuses it**:
|
|
||||||
//
|
|
||||||
// - a wait for a secret names an own secret the module's manifest declares — by its name, or as a member of a
|
|
||||||
// secret family — said `"issued-by": "outside"`, and the store holds no value a person gave for it on that
|
|
||||||
// machine;
|
|
||||||
// - a wait for a setting names a setting the manifest declares (checked by name only: the controller cannot tell
|
|
||||||
// whether a free-form value covers a part, and the needs-operator condition is where a false one shows).
|
|
||||||
//
|
|
||||||
// An excused wait is read by the first-node gate as *waits for a person* (ADR 0254), a pass carried in the verdict,
|
|
||||||
// for any build of the module — a secret not given is owed by every build alike. It is said to the operator as
|
|
||||||
// `module.<module>.<machine>.needs-operator`, naming the act. A wait that fails the check is judged unhealthy, saying
|
|
||||||
// why, and raises the module's `unhealthy` condition.
|
|
||||||
|
|
||||||
// kindNeedsOperator is a module's condition while a part of it waits for the operator's secret or setting.
|
|
||||||
const kindNeedsOperator = "needs-operator"
|
|
||||||
|
|
||||||
// needsOperatorKey is a module's needs-operator condition on a machine.
|
|
||||||
func needsOperatorKey(module, node string) string {
|
|
||||||
return conditions.Key(conditions.ScopeModule, module+"."+node, kindNeedsOperator)
|
|
||||||
}
|
|
||||||
|
|
||||||
// operatorWaitFacts is what the controller holds to check a module's waits: the manifest judged per module, and per
|
|
||||||
// "<module>@<machine>" the own secrets a person gave there, with when. A module or a machine absent is not known,
|
|
||||||
// and no wait of it is excused.
|
|
||||||
type operatorWaitFacts struct {
|
|
||||||
manifests map[string]catalogue.Manifest
|
|
||||||
given map[string]map[string]time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkWait is nil when a wait is excused, and otherwise why not, in words. Pure.
|
|
||||||
func checkWait(module, machine string, w inventory.Wait, f operatorWaitFacts) error {
|
|
||||||
m, known := f.manifests[module]
|
|
||||||
if !known {
|
|
||||||
return fmt.Errorf("says it waits for %s, and the mesh holds no manifest of %s to check it against", waitNames(w), module)
|
|
||||||
}
|
|
||||||
switch {
|
|
||||||
case w.Secret != "" && w.Setting != "", w.Secret == "" && w.Setting == "":
|
|
||||||
return fmt.Errorf("says it waits, naming %s, where a wait names one secret or one setting", waitNames(w))
|
|
||||||
case w.Setting != "":
|
|
||||||
if _, declared := m.Settings[w.Setting]; !declared {
|
|
||||||
return fmt.Errorf("says it waits for the setting %s, which %s does not declare", w.Setting, module)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
own, _, declared := m.OwnSecrets.Lookup(w.Secret)
|
|
||||||
if !declared {
|
|
||||||
return fmt.Errorf("says it waits for the secret %s, which %s does not declare", w.Secret, module)
|
|
||||||
}
|
|
||||||
if own.IssuedBy != catalogue.IssuedOutside {
|
|
||||||
return fmt.Errorf("says it waits for the secret %s, which the mesh makes itself: only a secret issued outside "+
|
|
||||||
"the mesh waits for the operator", w.Secret)
|
|
||||||
}
|
|
||||||
given, readable := f.given[module+"@"+machine]
|
|
||||||
if !readable {
|
|
||||||
return fmt.Errorf("says it waits for the secret %s, and what was given on %s could not be read", w.Secret, machine)
|
|
||||||
}
|
|
||||||
if at, was := given[w.Secret]; was {
|
|
||||||
return fmt.Errorf("says it waits for the secret %s, which was given at %s", w.Secret,
|
|
||||||
at.UTC().Format("2006-01-02 15:04 MST"))
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// waitRefusedPrefix opens every reason checkWait gives, so the words of a refused wait are told from a check's own.
|
|
||||||
const waitRefusedPrefix = "says it waits"
|
|
||||||
|
|
||||||
// waitNames is what a wait names, as "the secret x" or "the setting y".
|
|
||||||
func waitNames(w inventory.Wait) string {
|
|
||||||
switch {
|
|
||||||
case w.Secret != "" && w.Setting != "":
|
|
||||||
return "the secret " + w.Secret + " and the setting " + w.Setting
|
|
||||||
case w.Secret != "":
|
|
||||||
return "the secret " + w.Secret
|
|
||||||
case w.Setting != "":
|
|
||||||
return "the setting " + w.Setting
|
|
||||||
}
|
|
||||||
return "nothing"
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkWaiting reads one machine's resources against the facts: every waiting resource whose waits all check out is
|
|
||||||
// kept as said; one with a wait that does not, or with no wait at all, is answered as unhealthy with why. Pure; the
|
|
||||||
// statement as kept is not changed.
|
|
||||||
func checkWaiting(machine string, rs []inventory.ResourceHealth, f operatorWaitFacts) []inventory.ResourceHealth {
|
|
||||||
out := make([]inventory.ResourceHealth, 0, len(rs))
|
|
||||||
for _, r := range rs {
|
|
||||||
if r.State == link.StateWaiting {
|
|
||||||
var why error
|
|
||||||
if len(r.Waits) == 0 {
|
|
||||||
why = fmt.Errorf("says it waits, and names nothing it waits for")
|
|
||||||
}
|
|
||||||
for _, w := range r.Waits {
|
|
||||||
if why == nil {
|
|
||||||
why = checkWait(r.Module, machine, w, f)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if why != nil {
|
|
||||||
r.State, r.Reason = link.StateUnhealthy, why.Error()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
out = append(out, r)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// operatorWait is whether everything not healthy of a module on a machine is waiting with its waits checked
|
|
||||||
// (checkWaiting already applied), and those waits. A module with anything unhealthy, starting or unknown beside it
|
|
||||||
// does not wait: it is judged as before.
|
|
||||||
func operatorWait(module string, rs []inventory.ResourceHealth) ([]inventory.Wait, bool) {
|
|
||||||
var waits []inventory.Wait
|
|
||||||
for _, r := range rs {
|
|
||||||
if r.Module != module {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
switch r.State {
|
|
||||||
case link.StateHealthy:
|
|
||||||
case link.StateWaiting:
|
|
||||||
waits = append(waits, r.Waits...)
|
|
||||||
default:
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return waits, len(waits) > 0
|
|
||||||
}
|
|
||||||
|
|
||||||
// operatorWaitSaid is a module's wait for the operator in one sentence, for the gate's verdict and the condition's
|
|
||||||
// summary: what the operator gives and what it names, and for a secret the line that opens the desk prompt.
|
|
||||||
func operatorWaitSaid(module, machine string, waits []inventory.Wait) string {
|
|
||||||
var parts []string
|
|
||||||
for _, w := range waits {
|
|
||||||
part := fmt.Sprintf("%s (%s", w.What, waitNames(w))
|
|
||||||
if w.Secret != "" {
|
|
||||||
part += fmt.Sprintf(", given with `nox secret ask %s %s %s`", machine, module, w.Secret)
|
|
||||||
}
|
|
||||||
parts = append(parts, part+")")
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("%s on %s waits for the operator: %s", module, machine, strings.Join(parts, "; "))
|
|
||||||
}
|
|
||||||
|
|
||||||
// needsOperatorObservation is a module whose only parts not healthy wait for the operator (ADR 0283): the operator's,
|
|
||||||
// a warning however long it stands, its plain words naming the act and never saying there is nothing to do.
|
|
||||||
func needsOperatorObservation(module, node string, waits []inventory.Wait, rs []inventory.ResourceHealth) conditions.Observation {
|
|
||||||
o := moduleUnhealthyObservation(module, node, rs)
|
|
||||||
o.Token, o.Kind, o.Resolver, o.Severity = kindNeedsOperator, kindNeedsOperator, conditions.ResolverOperator, conditions.Warning
|
|
||||||
o.Summary = operatorWaitSaid(module, node, waits)
|
|
||||||
w := needsOperatorWords(module, node, waits)
|
|
||||||
o.Headline, o.Explanation, o.Needs, o.Resolved, o.Actions = w.Headline, w.Explanation, w.Needs, w.Resolved, nil
|
|
||||||
return o
|
|
||||||
}
|
|
||||||
|
|
||||||
// needsOperatorWords is what the operator reads of a module waiting for them (ADR 0253, ADR 0283): the act, for a
|
|
||||||
// secret typed at the machine's desk prompt and for a setting approved when an agent proposes it. The secret's and
|
|
||||||
// the setting's names, and the line, are in the summary for whoever looks closer.
|
|
||||||
func needsOperatorWords(module, node string, waits []inventory.Wait) words {
|
|
||||||
var acts []string
|
|
||||||
seen := map[string]bool{}
|
|
||||||
secret := false
|
|
||||||
for _, w := range waits {
|
|
||||||
var act string
|
|
||||||
switch {
|
|
||||||
case w.Secret != "":
|
|
||||||
act, secret = fmt.Sprintf("type %s at %s's desk prompt", w.What, node), true
|
|
||||||
case w.Setting != "":
|
|
||||||
act = fmt.Sprintf("approve %s of %s on %s when it is proposed to you", w.Setting, module, node)
|
|
||||||
}
|
|
||||||
if act != "" && !seen[act] {
|
|
||||||
seen[act] = true
|
|
||||||
acts = append(acts, act)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
needs := strings.Join(acts, "; and ") + "."
|
|
||||||
// Plain words hold one sentence of at most conditions.NeedsMax characters: several acts are named in the
|
|
||||||
// summary instead.
|
|
||||||
if len(acts) == 0 || len(needs) > conditions.NeedsMax {
|
|
||||||
needs = fmt.Sprintf("give what %s waits for on %s; the details name each secret and setting.", module, node)
|
|
||||||
}
|
|
||||||
explanation := fmt.Sprintf("Part of %s on %s cannot work until you give what it waits for.", module, node)
|
|
||||||
if secret {
|
|
||||||
explanation += " A hidden prompt opens at the desk when the secret is asked for, and what you type there " +
|
|
||||||
"is sealed to the machine."
|
|
||||||
}
|
|
||||||
explanation += " Its update is in place and nothing was undone; it carries on by itself once it is given."
|
|
||||||
return words{
|
|
||||||
Headline: fmt.Sprintf("%s waits for you on %s", module, node),
|
|
||||||
Needs: needs,
|
|
||||||
Explanation: explanation,
|
|
||||||
Resolved: fmt.Sprintf("%s on %s no longer waits for you", module, node),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// readWaitFacts reads what the controller holds to check the waits of the modules named on one machine: the
|
|
||||||
// manifests (the catalogue's, or those given) and the secrets given there. A read that fails leaves that module
|
|
||||||
// unknown, so none of its waits is excused.
|
|
||||||
func readWaitFacts(ctx context.Context, inv *inventory.Inventory, machine string, modules []string,
|
|
||||||
manifests map[string]catalogue.Manifest, f *operatorWaitFacts) {
|
|
||||||
if f.manifests == nil {
|
|
||||||
f.manifests = map[string]catalogue.Manifest{}
|
|
||||||
}
|
|
||||||
if f.given == nil {
|
|
||||||
f.given = map[string]map[string]time.Time{}
|
|
||||||
}
|
|
||||||
if inv == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var shelf map[string]catalogue.Manifest
|
|
||||||
sort.Strings(modules)
|
|
||||||
for _, module := range modules {
|
|
||||||
if _, has := f.manifests[module]; !has {
|
|
||||||
if m, given := manifests[module]; given {
|
|
||||||
f.manifests[module] = m
|
|
||||||
} else {
|
|
||||||
if shelf == nil {
|
|
||||||
var err error
|
|
||||||
if shelf, err = inv.Catalogue(ctx); err != nil {
|
|
||||||
shelf = map[string]catalogue.Manifest{}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if m, known := shelf[module]; known {
|
|
||||||
f.manifests[module] = m
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if _, read := f.given[module+"@"+machine]; read {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if given, err := inv.GivenOwnSecrets(ctx, machine, module); err == nil {
|
|
||||||
f.given[module+"@"+machine] = given
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// waitingModules is every module with a waiting resource in a statement.
|
|
||||||
func waitingModules(rs []inventory.ResourceHealth) []string {
|
|
||||||
seen := map[string]bool{}
|
|
||||||
var out []string
|
|
||||||
for _, r := range rs {
|
|
||||||
if r.State == link.StateWaiting && r.Module != "" && !seen[r.Module] {
|
|
||||||
seen[r.Module] = true
|
|
||||||
out = append(out, r.Module)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
@@ -1,264 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A part that waits for the operator's secret or setting (novox/hq ADR 0283, issue 386).
|
|
||||||
|
|
||||||
var mountsManifest = catalogue.Manifest{Module: "mounts", Version: "1",
|
|
||||||
Settings: map[string]catalogue.SettingDeclaration{"smb-users": {}, "sources": {}},
|
|
||||||
OwnSecrets: catalogue.OwnSecrets{
|
|
||||||
"smb-password-*": {Path: "/s/smb-password-*.secret", IssuedBy: catalogue.IssuedOutside},
|
|
||||||
"broker": {Path: "/s/broker"},
|
|
||||||
"made": {Path: "/s/made", Taken: catalogue.TakenAtStart},
|
|
||||||
"licence": {Path: "/s/licence", IssuedBy: catalogue.IssuedOutside},
|
|
||||||
}}
|
|
||||||
|
|
||||||
var passwordWait = inventory.Wait{Part: "the source games", Secret: "smb-password-games",
|
|
||||||
What: "the password of the source games"}
|
|
||||||
|
|
||||||
var usernameWait = inventory.Wait{Part: "the source games", Setting: "smb-users", What: "the username of the source games"}
|
|
||||||
|
|
||||||
func waitingResource(waits ...inventory.Wait) inventory.ResourceHealth {
|
|
||||||
return inventory.ResourceHealth{Module: "mounts", Resource: "mounts.watch", Kind: "process",
|
|
||||||
Target: "mesh-mounts-watch.service", State: link.StateWaiting, Check: "tool",
|
|
||||||
Reason: "the source games waits for its password", Waits: waits}
|
|
||||||
}
|
|
||||||
|
|
||||||
func factsGiven(given map[string]time.Time) operatorWaitFacts {
|
|
||||||
return operatorWaitFacts{manifests: map[string]catalogue.Manifest{"mounts": mountsManifest},
|
|
||||||
given: map[string]map[string]time.Time{"mounts@workstation": given}}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Rule 3: a wait is excused only when what it names is the module's, issued outside the mesh, and not given there.
|
|
||||||
func TestAWaitIsExcusedOnlyWhenItChecksOut(t *testing.T) {
|
|
||||||
at := time.Date(2026, 10, 10, 15, 8, 0, 0, time.UTC)
|
|
||||||
for _, c := range []struct {
|
|
||||||
name string
|
|
||||||
w inventory.Wait
|
|
||||||
f operatorWaitFacts
|
|
||||||
says string // "" when excused
|
|
||||||
}{
|
|
||||||
{"a member of an outside family, not given", passwordWait, factsGiven(nil), ""},
|
|
||||||
{"an outside secret by name, not given", inventory.Wait{Part: "p", Secret: "licence", What: "w"}, factsGiven(nil), ""},
|
|
||||||
{"a declared setting", usernameWait, factsGiven(nil), ""},
|
|
||||||
{"a member given", passwordWait, factsGiven(map[string]time.Time{"smb-password-games": at}), "was given at 2026-10-10 15:08"},
|
|
||||||
{"a secret not declared", inventory.Wait{Part: "p", Secret: "smb-credentials", What: "w"}, factsGiven(nil), "does not declare"},
|
|
||||||
{"a secret the mesh makes", inventory.Wait{Part: "p", Secret: "made", What: "w"}, factsGiven(nil), "mesh makes itself"},
|
|
||||||
{"the bus account", inventory.Wait{Part: "p", Secret: "broker", What: "w"}, factsGiven(nil), "mesh makes itself"},
|
|
||||||
{"a setting not declared", inventory.Wait{Part: "p", Setting: "logins", What: "w"}, factsGiven(nil), "does not declare"},
|
|
||||||
{"both", inventory.Wait{Part: "p", Secret: "licence", Setting: "smb-users", What: "w"}, factsGiven(nil), "one secret or one setting"},
|
|
||||||
{"neither", inventory.Wait{Part: "p", What: "w"}, factsGiven(nil), "one secret or one setting"},
|
|
||||||
{"no manifest known", passwordWait, operatorWaitFacts{}, "no manifest"},
|
|
||||||
{"what was given cannot be read", passwordWait,
|
|
||||||
operatorWaitFacts{manifests: map[string]catalogue.Manifest{"mounts": mountsManifest}}, "could not be read"},
|
|
||||||
} {
|
|
||||||
err := checkWait("mounts", "workstation", c.w, c.f)
|
|
||||||
switch {
|
|
||||||
case c.says == "" && err != nil:
|
|
||||||
t.Errorf("%s: refused: %v", c.name, err)
|
|
||||||
case c.says != "" && (err == nil || !strings.Contains(err.Error(), c.says)):
|
|
||||||
t.Errorf("%s: %v; want a refusal saying %q", c.name, err, c.says)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A waiting resource whose wait does not check out, or that names nothing, is judged unhealthy, saying why.
|
|
||||||
func TestAWaitThatFailsItsCheckIsUnhealthy(t *testing.T) {
|
|
||||||
given := factsGiven(map[string]time.Time{"smb-password-games": time.Now()})
|
|
||||||
got := checkWaiting("workstation", []inventory.ResourceHealth{waitingResource(passwordWait)}, given)
|
|
||||||
if got[0].State != link.StateUnhealthy || !strings.Contains(got[0].Reason, "was given") {
|
|
||||||
t.Fatalf("a wait for a secret given: %+v", got[0])
|
|
||||||
}
|
|
||||||
got = checkWaiting("workstation", []inventory.ResourceHealth{waitingResource()}, factsGiven(nil))
|
|
||||||
if got[0].State != link.StateUnhealthy || !strings.Contains(got[0].Reason, "names nothing") {
|
|
||||||
t.Fatalf("a wait naming nothing: %+v", got[0])
|
|
||||||
}
|
|
||||||
got = checkWaiting("workstation", []inventory.ResourceHealth{waitingResource(passwordWait, usernameWait)}, factsGiven(nil))
|
|
||||||
if got[0].State != link.StateWaiting {
|
|
||||||
t.Fatalf("two waits that check out: %+v", got[0])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Rule 4: the gate passes a module whose only parts not healthy wait for the operator, carrying the wait; anything
|
|
||||||
// else beside it is judged as before; and any build is excused, not only one that added something.
|
|
||||||
func TestTheGatePassesAWaitForTheOperatorCarriedAlong(t *testing.T) {
|
|
||||||
now := time.Now()
|
|
||||||
since := now.Add(-time.Minute)
|
|
||||||
healthy := inventory.ResourceHealth{Module: "mounts", Resource: "mounts.apply", Kind: "process",
|
|
||||||
Target: "mesh-mounts-apply.service", State: link.StateHealthy}
|
|
||||||
f := gateFacts{now: now, waits: factsGiven(nil), groupsAdded: map[string]bool{"mounts": false},
|
|
||||||
health: map[string]inventory.NodeHealth{"workstation": {Node: "workstation", HeardAt: now,
|
|
||||||
Resources: []inventory.ResourceHealth{healthy, waitingResource(passwordWait)}}}}
|
|
||||||
h, why := moduleHealthWord("mounts", "workstation", since, f)
|
|
||||||
if h != healthPerson || !strings.Contains(why, "waits for the operator: the password of the source games") ||
|
|
||||||
!strings.Contains(why, "nox secret ask workstation mounts smb-password-games") {
|
|
||||||
t.Fatalf("an excused wait reads %v %q; want a wait for a person naming the act", h, why)
|
|
||||||
}
|
|
||||||
// A second resource unhealthy beside it: not yet, as before.
|
|
||||||
down := healthy
|
|
||||||
down.State, down.Reason = link.StateUnhealthy, "down"
|
|
||||||
f.health["workstation"] = inventory.NodeHealth{Node: "workstation", HeardAt: now,
|
|
||||||
Resources: []inventory.ResourceHealth{down, waitingResource(passwordWait)}}
|
|
||||||
if h, why := moduleHealthWord("mounts", "workstation", since, f); h != healthNotYet {
|
|
||||||
t.Fatalf("a resource down beside the wait reads %v %q", h, why)
|
|
||||||
}
|
|
||||||
// The password given and the module still saying it waits: not excused.
|
|
||||||
f.waits = factsGiven(map[string]time.Time{"smb-password-games": now})
|
|
||||||
f.health["workstation"] = inventory.NodeHealth{Node: "workstation", HeardAt: now,
|
|
||||||
Resources: []inventory.ResourceHealth{healthy, waitingResource(passwordWait)}}
|
|
||||||
if h, why := moduleHealthWord("mounts", "workstation", since, f); h != healthNotYet || !strings.Contains(why, "was given") {
|
|
||||||
t.Fatalf("a wait for a secret given reads %v %q", h, why)
|
|
||||||
}
|
|
||||||
// Facts never read (no manifest): never excused.
|
|
||||||
f.waits = operatorWaitFacts{}
|
|
||||||
if h, _ := moduleHealthWord("mounts", "workstation", since, f); h != healthNotYet {
|
|
||||||
t.Fatalf("a wait nothing could check reads %v", h)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func needsOperatorOpen(t *testing.T, k *conditions.Keeper) (*conditions.Condition, []conditions.Condition) {
|
|
||||||
t.Helper()
|
|
||||||
open, err := k.Open(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for i, c := range open {
|
|
||||||
if c.Key == needsOperatorKey("mounts", "workstation") {
|
|
||||||
return &open[i], open
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil, open
|
|
||||||
}
|
|
||||||
|
|
||||||
// Rule 5: two statements of an excused wait raise needs-operator, the operator's, a warning however long, naming the
|
|
||||||
// act; a statement without it clears it.
|
|
||||||
func TestTheNeedsOperatorConditionNamesTheAct(t *testing.T) {
|
|
||||||
k, _ := withConditionsInMemory(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
rs := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}}
|
|
||||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 1}, time.Now()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got, _ := needsOperatorOpen(t, k); got != nil {
|
|
||||||
t.Fatal("raised on one statement")
|
|
||||||
}
|
|
||||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 2}, time.Now()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
got, open := needsOperatorOpen(t, k)
|
|
||||||
if got == nil {
|
|
||||||
t.Fatalf("not raised on two statements: %+v", open)
|
|
||||||
}
|
|
||||||
for _, c := range open {
|
|
||||||
if c.Kind == kindModuleUnhealthy {
|
|
||||||
t.Fatalf("raised as a fault too: %+v", c)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if got.Kind != kindNeedsOperator || got.Resolver != conditions.ResolverOperator || got.Severity != conditions.Warning {
|
|
||||||
t.Fatalf("the condition: %+v", got)
|
|
||||||
}
|
|
||||||
if !strings.Contains(got.Needs, "type the password of the source games at workstation's desk prompt") ||
|
|
||||||
!strings.Contains(got.Explanation, "hidden prompt opens at the desk") {
|
|
||||||
t.Fatalf("its needs do not name the act: %q", got.Needs)
|
|
||||||
}
|
|
||||||
if strings.Contains(strings.ToLower(got.Explanation), "nothing for you") || !strings.Contains(got.Explanation, "nothing was undone") {
|
|
||||||
t.Fatalf("its explanation: %q", got.Explanation)
|
|
||||||
}
|
|
||||||
if !strings.Contains(got.Summary, "smb-password-games") || !strings.Contains(got.Summary, "nox secret ask workstation mounts smb-password-games") {
|
|
||||||
t.Fatalf("its summary does not name the secret and the line: %q", got.Summary)
|
|
||||||
}
|
|
||||||
// Long open is still a warning: only the operator can end it.
|
|
||||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 3}, time.Now().Add(48*time.Hour)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got, _ := needsOperatorOpen(t, k); got == nil || got.Severity == conditions.Urgent {
|
|
||||||
t.Fatalf("after two days: %+v", got)
|
|
||||||
}
|
|
||||||
// Given: the next statement does not say it, and it clears.
|
|
||||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got, _ := needsOperatorOpen(t, k); got != nil {
|
|
||||||
t.Fatal("not cleared once given")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestASettingsWaitAsksForTheApproval(t *testing.T) {
|
|
||||||
k, _ := withConditionsInMemory(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
rs := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(usernameWait)}}
|
|
||||||
for i := 1; i <= 2; i++ {
|
|
||||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": i}, time.Now()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
got, _ := needsOperatorOpen(t, k)
|
|
||||||
if got == nil || !strings.Contains(got.Needs, "approve smb-users of mounts on workstation when it is proposed to you") {
|
|
||||||
t.Fatalf("the condition: %+v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A wait that fails its check is the module's own fault: unhealthy, with why, and no needs-operator.
|
|
||||||
func TestAWaitThatFailsItsCheckRaisesUnhealthy(t *testing.T) {
|
|
||||||
k, _ := withConditionsInMemory(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
checked := checkWaiting("workstation", []inventory.ResourceHealth{waitingResource(passwordWait)},
|
|
||||||
factsGiven(map[string]time.Time{"smb-password-games": time.Now()}))
|
|
||||||
rs := map[string][]inventory.ResourceHealth{"mounts": checked}
|
|
||||||
for i := 1; i <= 2; i++ {
|
|
||||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": i}, time.Now()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
got, open := needsOperatorOpen(t, k)
|
|
||||||
if got != nil {
|
|
||||||
t.Fatalf("a wait for a secret given raised needs-operator: %+v", got)
|
|
||||||
}
|
|
||||||
unhealthy := false
|
|
||||||
for _, c := range open {
|
|
||||||
unhealthy = unhealthy || c.Key == moduleUnhealthyKey("mounts", "workstation")
|
|
||||||
}
|
|
||||||
if !unhealthy {
|
|
||||||
t.Fatalf("not raised as unhealthy: %+v", open)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A module that waited and is then broken says so when the wait clears, and the other way round.
|
|
||||||
func TestANeedsOperatorThatBecameUnhealthySaysSo(t *testing.T) {
|
|
||||||
k, _ := withConditionsInMemory(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
waiting := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}}
|
|
||||||
for i := 1; i <= 2; i++ {
|
|
||||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", waiting, map[string]int{"mounts": i}, time.Now()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
broken := waitingResource()
|
|
||||||
broken.State, broken.Reason, broken.Waits = link.StateUnhealthy, "the source games refused its login", nil
|
|
||||||
for i := 3; i <= 4; i++ {
|
|
||||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": {broken}},
|
|
||||||
map[string]int{"mounts": i}, time.Now()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
got, open := needsOperatorOpen(t, k)
|
|
||||||
if got != nil {
|
|
||||||
t.Fatal("needs-operator still open after it became a fault")
|
|
||||||
}
|
|
||||||
found := false
|
|
||||||
for _, c := range open {
|
|
||||||
found = found || c.Key == moduleUnhealthyKey("mounts", "workstation")
|
|
||||||
}
|
|
||||||
if !found {
|
|
||||||
t.Fatalf("the fault is not raised: %+v", open)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,125 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A delivery over its budget is loud (novox/hq ADR 0282 decision 7, issue 382): the self-check reads
|
|
||||||
// mesh-delivery's `times` and raises the warning `delivery.<class>.over-budget` when the newest delivery of a
|
|
||||||
// class whose delivery time is known took longer than its class's budget — three minutes for a leaf module (ADR
|
|
||||||
// 0304 decision 4), ten for a core module — naming the delivery and its longest phase. It clears when the next delivery of that class
|
|
||||||
// lands within its budget. Measurement only: the condition says, it never holds or acts on a delivery.
|
|
||||||
|
|
||||||
// probeBudgetsID is the probe that reads the delivery times.
|
|
||||||
const probeBudgetsID = "D16"
|
|
||||||
|
|
||||||
// kindOverBudget is what a delivery over its class's budget raises.
|
|
||||||
const kindOverBudget = "over-budget"
|
|
||||||
|
|
||||||
// deliveryBudgets are the budgets by class (ADR 0282 decision 1, the leaf's as ADR 0304 decision 4 changed it);
|
|
||||||
// the probe raises nothing for another class.
|
|
||||||
var deliveryBudgets = map[string]time.Duration{inventory.ClassLeaf: 3 * time.Minute, inventory.ClassCore: 10 * time.Minute}
|
|
||||||
|
|
||||||
// timesAnswer is what mesh-delivery's `times` answers, as far as the probe reads it.
|
|
||||||
type timesAnswer struct {
|
|
||||||
Classes []timesClass `json:"classes"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// timesClass is one class's line of `times`.
|
|
||||||
type timesClass struct {
|
|
||||||
Class string `json:"class"`
|
|
||||||
Latest *timesLatest `json:"latest,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// timesLatest is the newest delivery of a class whose delivery time is known.
|
|
||||||
type timesLatest struct {
|
|
||||||
ID string `json:"id"`
|
|
||||||
TookMS int64 `json:"took_ms"`
|
|
||||||
Longest string `json:"longest,omitempty"`
|
|
||||||
Landed string `json:"landed,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// deliveryTimes is what the delivery's owner says of its delivery times; nothing when no holder is on record or
|
|
||||||
// none answers (D3 says that one).
|
|
||||||
func deliveryTimes(ctx context.Context, conn *nats.Conn, held bool) (*timesAnswer, error) {
|
|
||||||
if !held {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
raw, err := askDeliveryOwner(ctx, conn, "times", map[string]any{})
|
|
||||||
if errors.Is(err, link.ErrNothingServes) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var a timesAnswer
|
|
||||||
if err := json.Unmarshal(raw, &a); err != nil {
|
|
||||||
return nil, fmt.Errorf("%s.times answered something unreadable: %w", catalogue.DeliverySeat, err)
|
|
||||||
}
|
|
||||||
return &a, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// overBudgetObservations are the conditions of the classes whose newest delivery took longer than its budget:
|
|
||||||
// strictly longer, so a delivery of exactly its budget is within it.
|
|
||||||
func overBudgetObservations(a *timesAnswer) []conditions.Observation {
|
|
||||||
if a == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
var out []conditions.Observation
|
|
||||||
for _, c := range a.Classes {
|
|
||||||
budget, ok := deliveryBudgets[c.Class]
|
|
||||||
if !ok || c.Latest == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
took := time.Duration(c.Latest.TookMS) * time.Millisecond
|
|
||||||
if took <= budget {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
longest := c.Latest.Longest
|
|
||||||
if longest == "" {
|
|
||||||
longest = "not known"
|
|
||||||
}
|
|
||||||
out = append(out, conditions.Observation{Scope: conditions.ScopeDelivery, ID: c.Class, Kind: kindOverBudget,
|
|
||||||
Severity: conditions.Warning,
|
|
||||||
Summary: fmt.Sprintf("the %s delivery %s took %s from its merge to running everywhere, over its budget of %s; "+
|
|
||||||
"its longest phase: %s — `mesh-delivery.times`", c.Class, c.Latest.ID, humanDuration(took),
|
|
||||||
humanDuration(budget), longest),
|
|
||||||
Said: fmt.Sprintf("%s took %s (budget %s), longest phase %s", c.Latest.ID, took.Round(time.Second), budget, longest),
|
|
||||||
Headline: fmt.Sprintf("A %s delivery took %s, over its %s budget", c.Class, humanDuration(took),
|
|
||||||
humanDuration(budget)),
|
|
||||||
Explanation: fmt.Sprintf("The delivery %s took %s from its merge until every machine ran it; a %s module is "+
|
|
||||||
"held to %s (ADR 0282). Most of the time went to %s. Nothing was held or changed because of this: it is "+
|
|
||||||
"a measurement.", c.Latest.ID, humanDuration(took), c.Class, humanDuration(budget), longest),
|
|
||||||
Resolved: fmt.Sprintf("the next %s delivery lands within %s", c.Class, humanDuration(budget)),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// probeBudgets is D16: the newest delivery of each class lands within its class's budget.
|
|
||||||
func probeBudgets(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
|
||||||
entries, err := d.open.inventory.Catalogued(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var conn *nats.Conn
|
|
||||||
if d.js != nil {
|
|
||||||
conn = d.js.Conn()
|
|
||||||
}
|
|
||||||
a, err := deliveryTimes(ctx, conn, deliverySeatHeld(entries))
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return overBudgetObservations(a), nil
|
|
||||||
}
|
|
||||||
@@ -1,112 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A leaf delivery of 3 minutes 10 seconds raises delivery.leaf.over-budget naming its longest phase; one of
|
|
||||||
// exactly three minutes, a core one of nine and a class without a budget raise nothing (ADR 0282 decision 7,
|
|
||||||
// the leaf's budget as ADR 0304 decision 4 changed it).
|
|
||||||
func TestADeliveryOverItsBudgetIsLoud(t *testing.T) {
|
|
||||||
a := ×Answer{Classes: []timesClass{
|
|
||||||
{Class: inventory.ClassLeaf, Latest: ×Latest{ID: "novox/mesh-catalog@abc", TookMS: (3*time.Minute + 10*time.Second).Milliseconds(),
|
|
||||||
Longest: "judgement 2m40s"}},
|
|
||||||
{Class: inventory.ClassCore, Latest: ×Latest{ID: "novox/mesh-controller@def", TookMS: (9 * time.Minute).Milliseconds(),
|
|
||||||
Longest: "build 1m"}},
|
|
||||||
{Class: "unclassed", Latest: ×Latest{ID: "x@y", TookMS: time.Hour.Milliseconds()}},
|
|
||||||
}}
|
|
||||||
obs := overBudgetObservations(a)
|
|
||||||
if len(obs) != 1 {
|
|
||||||
t.Fatalf("one class over its budget, got %d: %+v", len(obs), obs)
|
|
||||||
}
|
|
||||||
o := obs[0]
|
|
||||||
if o.Key() != "delivery.leaf.over-budget" || !strings.Contains(o.Summary, "judgement 2m40s") ||
|
|
||||||
!strings.Contains(o.Summary, "novox/mesh-catalog@abc") {
|
|
||||||
t.Fatalf("the condition names its delivery and longest phase: %s — %s", o.Key(), o.Summary)
|
|
||||||
}
|
|
||||||
// The next leaf delivery within its budget clears it: the probe raises nothing for the class.
|
|
||||||
a.Classes[0].Latest = ×Latest{ID: "novox/mesh-catalog@ghi", TookMS: (3 * time.Minute).Milliseconds()}
|
|
||||||
if obs := overBudgetObservations(a); len(obs) != 0 {
|
|
||||||
t.Fatalf("a delivery of exactly its budget is within it: %+v", obs)
|
|
||||||
}
|
|
||||||
a.Classes[1].Latest.TookMS = (10*time.Minute + time.Second).Milliseconds()
|
|
||||||
if obs := overBudgetObservations(a); len(obs) != 1 || obs[0].Key() != "delivery.core.over-budget" {
|
|
||||||
t.Fatalf("a core delivery over ten minutes: %+v", obs)
|
|
||||||
}
|
|
||||||
if obs := overBudgetObservations(nil); obs != nil {
|
|
||||||
t.Fatal("no answer raises nothing")
|
|
||||||
}
|
|
||||||
// No delivery of a class with a known time yet: nothing said of it.
|
|
||||||
if obs := overBudgetObservations(×Answer{Classes: []timesClass{{Class: inventory.ClassLeaf}}}); len(obs) != 0 {
|
|
||||||
t.Fatalf("a class with no delivery: %+v", obs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The probe's question is one the controller's grant names: a question the bus refuses checks nothing.
|
|
||||||
func TestTheControllerMayAskForTheDeliveryTimes(t *testing.T) {
|
|
||||||
found := false
|
|
||||||
for _, v := range broker.VerbsTheControllerAsksTheDeliveryOwner {
|
|
||||||
found = found || (v.Seat == catalogue.DeliverySeat && v.Verb == "times")
|
|
||||||
}
|
|
||||||
if !found {
|
|
||||||
t.Fatal("the grant does not name mesh-delivery.times")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A walk's class is core when it walks a module of the controller's own path or one holding the mesh's resolver,
|
|
||||||
// leaf otherwise; its window closed at its batch's window, or its maximum, never after its cut.
|
|
||||||
func TestAWalksClassAndWindowAreReadAtItsCut(t *testing.T) {
|
|
||||||
entries := []inventory.Entry{
|
|
||||||
{Manifest: catalogue.Manifest{Module: "dnsmasq", Claims: []catalogue.Claim{{Name: resolverSeat}}}},
|
|
||||||
{Manifest: catalogue.Manifest{Module: "gitea"}},
|
|
||||||
}
|
|
||||||
walk := func(modules ...string) inventory.Plan {
|
|
||||||
p := inventory.Plan{Modules: map[string]*inventory.PlanModule{}}
|
|
||||||
for _, m := range modules {
|
|
||||||
p.Modules[m] = &inventory.PlanModule{}
|
|
||||||
}
|
|
||||||
return p
|
|
||||||
}
|
|
||||||
for want, w := range map[string]inventory.Plan{
|
|
||||||
inventory.ClassLeaf: walk("gitea"), inventory.ClassCore: walk("gitea", "mesh-controller"),
|
|
||||||
} {
|
|
||||||
if got := classOf(w, entries); got != want {
|
|
||||||
t.Errorf("%v: %s, want %s", w.Modules, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if got := classOf(walk("dnsmasq"), entries); got != inventory.ClassCore {
|
|
||||||
t.Errorf("the resolver's holder is core: %s", got)
|
|
||||||
}
|
|
||||||
now := time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC)
|
|
||||||
batch := inventory.Plan{Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{
|
|
||||||
ClosesAt: now.Add(-20 * time.Second), AtMost: now.Add(5 * time.Minute)}}}
|
|
||||||
times := walkTimesAtCut(batch, walk("gitea"), entries, now)
|
|
||||||
if times.Cut == nil || !times.Cut.Equal(now) || times.WindowClosed == nil || !times.WindowClosed.Equal(now.Add(-20*time.Second)) ||
|
|
||||||
times.Class != inventory.ClassLeaf {
|
|
||||||
t.Fatalf("times at the cut: %+v", times)
|
|
||||||
}
|
|
||||||
batch.Delivery.Batch.AtMost = now.Add(-time.Minute)
|
|
||||||
if times := walkTimesAtCut(batch, walk("gitea"), entries, now); !times.WindowClosed.Equal(now.Add(-time.Minute)) {
|
|
||||||
t.Fatalf("a window closed at its maximum: %v", times.WindowClosed)
|
|
||||||
}
|
|
||||||
if times := walkTimesAtCut(inventory.Plan{Delivery: &inventory.PlanDelivery{Alone: true}}, walk("gitea"), entries, now); times.WindowClosed != nil {
|
|
||||||
t.Fatalf("a merge walked alone had no window: %v", times.WindowClosed)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// What each machine of the rest was sent is kept only for the machines the send reached.
|
|
||||||
func TestTheRestIsKeptForTheMachinesTheSendReached(t *testing.T) {
|
|
||||||
got := restOf([]string{"ace", "g14"}, []string{"ace", "shanks"}, map[string]inventory.SentDeclaration{"ace": {Digest: "d1"}})
|
|
||||||
if len(got) != 1 || got["ace"].Digest != "d1" {
|
|
||||||
t.Fatalf("rest: %+v", got)
|
|
||||||
}
|
|
||||||
if restOf([]string{"g14"}, []string{"ace"}, nil) != nil {
|
|
||||||
t.Fatal("no machine reached: nothing kept")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -81,13 +81,6 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
|||||||
"It usually catches up by itself.", m, m),
|
"It usually catches up by itself.", m, m),
|
||||||
Resolved: m + " applied the change"}
|
Resolved: m + " applied the change"}
|
||||||
}),
|
}),
|
||||||
kindApplyFailing: worded(func(o conditions.Observation) words {
|
|
||||||
m := machineOr(o, "a machine")
|
|
||||||
return words{Headline: m + " is not applying its update",
|
|
||||||
Explanation: fmt.Sprintf("%s has refused or only partly applied what the mesh sent it, again and again. "+
|
|
||||||
"The details say what fails.", conditions.Capital(m)),
|
|
||||||
Resolved: m + " applies what it is sent again"}
|
|
||||||
}),
|
|
||||||
"tools-silent": worded(func(o conditions.Observation) words {
|
"tools-silent": worded(func(o conditions.Observation) words {
|
||||||
m := machineOr(o, "a machine")
|
m := machineOr(o, "a machine")
|
||||||
return words{Headline: "Tools on " + m + " are not answering",
|
return words{Headline: "Tools on " + m + " are not answering",
|
||||||
@@ -228,24 +221,6 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
|||||||
w := usedAsFoundObservation(orModule(module), machineOr(o, "a machine"), o.Summary, nil)
|
w := usedAsFoundObservation(orModule(module), machineOr(o, "a machine"), o.Summary, nil)
|
||||||
return words{Headline: w.Headline, Explanation: w.Explanation, Needs: w.Needs, Resolved: w.Resolved}
|
return words{Headline: w.Headline, Explanation: w.Explanation, Needs: w.Needs, Resolved: w.Resolved}
|
||||||
}),
|
}),
|
||||||
kindNeedsOperator: worded(func(o conditions.Observation) words {
|
|
||||||
// The observation carries the act itself (ADR 0283); these are its words when only the kind is known.
|
|
||||||
module := ""
|
|
||||||
if o.Scope == conditions.ScopeModule && o.Machine != "" {
|
|
||||||
module = strings.TrimSuffix(o.ID, "."+o.Machine)
|
|
||||||
}
|
|
||||||
node := machineOr(o, "a machine")
|
|
||||||
w := needsOperatorWords(orModule(module), node, nil)
|
|
||||||
return w
|
|
||||||
}),
|
|
||||||
kindLeftOut: worded(func(o conditions.Observation) words {
|
|
||||||
// The observation carries its own words (novox/hq issue 380); these are its kind's alone.
|
|
||||||
module := ""
|
|
||||||
if o.Scope == conditions.ScopeModule && o.Machine != "" {
|
|
||||||
module = strings.TrimSuffix(o.ID, "."+o.Machine)
|
|
||||||
}
|
|
||||||
return leftOutWords(orModule(module), machineOr(o, "a machine"), "")
|
|
||||||
}),
|
|
||||||
kindProviderFailing: worded(func(o conditions.Observation) words {
|
kindProviderFailing: worded(func(o conditions.Observation) words {
|
||||||
thing, consumer := conditions.ThingWords(o), idPart(o, 2)
|
thing, consumer := conditions.ThingWords(o), idPart(o, 2)
|
||||||
if consumer == "" {
|
if consumer == "" {
|
||||||
@@ -621,19 +596,6 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
|||||||
Explanation: "The bus refused messages from a part of the mesh, so what they carried did not happen.",
|
Explanation: "The bus refused messages from a part of the mesh, so what they carried did not happen.",
|
||||||
Resolved: "Resolved: the bus takes the messages again"}
|
Resolved: "Resolved: the bus takes the messages again"}
|
||||||
}),
|
}),
|
||||||
kindBucketOrLogFilling: worded(func(o conditions.Observation) words {
|
|
||||||
return words{Headline: "A module's bucket or log on the bus is filling up",
|
|
||||||
Needs: "decide whether to raise its cap or have the module keep less.",
|
|
||||||
Explanation: "A bucket or log a module keeps on the bus holds three quarters of its cap or more. When it " +
|
|
||||||
"is full, the bus refuses what the module writes there.",
|
|
||||||
Resolved: "It has room again"}
|
|
||||||
}),
|
|
||||||
kindBucketOrLogUnread: worded(func(o conditions.Observation) words {
|
|
||||||
return words{Headline: "A module's bucket or log could not be read",
|
|
||||||
Explanation: "The controller could not read how full a bucket or log a module keeps on the bus is, so it " +
|
|
||||||
"cannot say whether it is filling up. It asks again at its next check.",
|
|
||||||
Resolved: "It can be read again"}
|
|
||||||
}),
|
|
||||||
"stream-wrong": worded(func(o conditions.Observation) words {
|
"stream-wrong": worded(func(o conditions.Observation) words {
|
||||||
return words{Headline: "Part of the bus's storage is wrong",
|
return words{Headline: "Part of the bus's storage is wrong",
|
||||||
Needs: "check the machine the bus runs on; the details say what is missing.",
|
Needs: "check the machine the bus runs on; the details say what is missing.",
|
||||||
@@ -856,9 +818,6 @@ func stalledWords(l stalledLine, o conditions.Observation) (headline, explanatio
|
|||||||
"to check it: the forge never announced it. It cannot merge until it is checked.", pull, long),
|
"to check it: the forge never announced it. It cannot merge until it is checked.", pull, long),
|
||||||
fmt.Sprintf("%s has a merge check now, or is closed", pull), needs, nil
|
fmt.Sprintf("%s has a merge check now, or is closed", pull), needs, nil
|
||||||
}
|
}
|
||||||
if l.State == "unanswered" {
|
|
||||||
return unansweredWords(l, o)
|
|
||||||
}
|
|
||||||
held := l.State
|
held := l.State
|
||||||
if held == "" {
|
if held == "" {
|
||||||
held = "held"
|
held = "held"
|
||||||
@@ -888,89 +847,6 @@ func stalledWords(l stalledLine, o conditions.Observation) (headline, explanatio
|
|||||||
fmt.Sprintf("Delivery of %s is no longer %s", name, held), needs, actions
|
fmt.Sprintf("Delivery of %s is no longer %s", name, held), needs, actions
|
||||||
}
|
}
|
||||||
|
|
||||||
// unansweredWords are the plain words of a pull request's head whose merge check was asked and has not answered
|
|
||||||
// within its bound (novox/hq issue 438): mesh-delivery holds no delivery of it, so there is nothing to stop, release
|
|
||||||
// or close, and no action is offered. The operator's acts are a new commit, which asks the check again, or a look
|
|
||||||
// at the build queue, where a check that never ran may still wait.
|
|
||||||
//
|
|
||||||
// A head whose delivery is held in ready or rejected is said the same way (novox/hq issue 441), and mesh-delivery
|
|
||||||
// names that state as data (Held). Its check answered once, so the build queue is the wrong place to look: the step
|
|
||||||
// is a recheck of the delivery, named by its id, which asks the check again (novox/hq issue 451). It is advised, not
|
|
||||||
// offered as an action, since the controller is not granted the recheck.
|
|
||||||
func unansweredWords(l stalledLine, o conditions.Observation) (headline, explanation, resolved, needs string,
|
|
||||||
actions []conditions.Action) {
|
|
||||||
repository, _, _ := strings.Cut(l.ID, "@")
|
|
||||||
pull := "A pull request of " + repoName(repository)
|
|
||||||
if l.Number > 0 {
|
|
||||||
pull = fmt.Sprintf("Pull request %s #%d", repoName(repository), l.Number)
|
|
||||||
}
|
|
||||||
long, limit := "for too long", ""
|
|
||||||
if d, err := time.ParseDuration(l.For); err == nil {
|
|
||||||
long = "for " + humanDuration(d)
|
|
||||||
}
|
|
||||||
if d, err := time.ParseDuration(l.Bound); err == nil {
|
|
||||||
limit = ", past its limit of " + humanDuration(d)
|
|
||||||
}
|
|
||||||
check := "merge check"
|
|
||||||
if said := uncheckedWaitWords(l); len(said) > 0 {
|
|
||||||
check = "merge check (" + strings.Join(said, ", ") + ")"
|
|
||||||
}
|
|
||||||
which := "Its " + check
|
|
||||||
switch l.Held {
|
|
||||||
case "ready", "rejected":
|
|
||||||
which = "Its delivery is " + l.Held + ", but on the forge its " + check
|
|
||||||
if o.Resolver == conditions.ResolverOperator {
|
|
||||||
needs = "recheck its delivery, which asks its check again — mesh-delivery.recheck with id " + l.ID +
|
|
||||||
" and why — or push a new commit to its branch, " + FromMeshMCPServer
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
if o.Resolver == conditions.ResolverOperator {
|
|
||||||
needs = "push a new commit to its branch, which asks the check again, or see whether its check still " +
|
|
||||||
"waits in the build queue: mesh-controller.queue."
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("%s's merge check has not answered %s", pull, long),
|
|
||||||
fmt.Sprintf("%s is open on a branch that requires the merge check. %s was asked and has not answered %s%s. "+
|
|
||||||
"It cannot merge until its check answers.", pull, which, long, limit),
|
|
||||||
fmt.Sprintf("%s has an answer from its merge check, or is closed", pull), needs, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// uncheckedWaitWords are the merge checks an unanswered line waits on, as the operator reads them: from the checks
|
|
||||||
// given as data, each time in the controller's local zone, as every other time in a message — "mesh/merge-gate
|
|
||||||
// pending since 02:11" — never the UTC time inside mesh-delivery's finished words, which cannot be said again in
|
|
||||||
// another zone (novox/hq issue 443). A line from a mesh-delivery that gives no such data is said by its words.
|
|
||||||
func uncheckedWaitWords(l stalledLine) []string {
|
|
||||||
if len(l.Checks) == 0 {
|
|
||||||
return l.Waiting
|
|
||||||
}
|
|
||||||
out := make([]string, 0, len(l.Checks))
|
|
||||||
for _, c := range l.Checks {
|
|
||||||
switch c.State {
|
|
||||||
case "never-set":
|
|
||||||
out = append(out, c.Context+" never set")
|
|
||||||
case "pending":
|
|
||||||
out = append(out, c.Context+" pending"+sinceWords(c.Since))
|
|
||||||
default:
|
|
||||||
out = append(out, c.Context+" not answered")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// sinceWords is " since 02:11" in the controller's local zone, with its day when that is not today ("since 23:05 on
|
|
||||||
// 9 Oct"), so the time stays absolute; a time not given, or not readable, is said so and never made up.
|
|
||||||
func sinceWords(since string) string {
|
|
||||||
at, err := time.Parse(time.RFC3339, since)
|
|
||||||
if err != nil {
|
|
||||||
return ", since a time the forge did not say"
|
|
||||||
}
|
|
||||||
local, today := at.In(wordsZone()), wordsNow().In(wordsZone())
|
|
||||||
if local.YearDay() == today.YearDay() && local.Year() == today.Year() {
|
|
||||||
return " since " + local.Format("15:04")
|
|
||||||
}
|
|
||||||
return " since " + local.Format("15:04 on 2 Jan")
|
|
||||||
}
|
|
||||||
|
|
||||||
// causeWords is a hand-act's cause as a person says it.
|
// causeWords is a hand-act's cause as a person says it.
|
||||||
func causeWords(cause string) string {
|
func causeWords(cause string) string {
|
||||||
return strings.NewReplacer(".", " ", "_", " ").Replace(cause)
|
return strings.NewReplacer(".", " ", "_", " ").Replace(cause)
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -348,153 +347,3 @@ func TestAnUnannouncedPullRequestSaysToPushANewCommit(t *testing.T) {
|
|||||||
t.Fatalf("it reads %q / %q", o.Headline, o.Explanation)
|
t.Fatalf("it reads %q / %q", o.Headline, o.Explanation)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// **A pull request whose merge check never answered says which check, since when, and what to do** (novox/hq issue
|
|
||||||
// 438): mesh-delivery says one as a stalled line in state `unanswered`, the line below exactly as its test makes it.
|
|
||||||
// No delivery of the head is held, so there is nothing to stop, release or close: no action is offered, and the
|
|
||||||
// operator's acts are a new commit, which asks the check again, or a look at the build queue.
|
|
||||||
func TestAnUnansweredMergeCheckSaysWhichCheckAndWhatToDo(t *testing.T) {
|
|
||||||
var l stalledLine
|
|
||||||
if err := json.Unmarshal([]byte(`{"id":"novox/mesh-controller@c11222026a3b","number":212,"state":"unanswered",`+
|
|
||||||
`"waiting":["mesh/merge-gate pending since 2026-10-11T00:11:39Z","mesh/repo-check never set"],"for":"1h1m0s",`+
|
|
||||||
`"bound":"1h0m0s","h2":"none: no delivery of it is held here, so there is none to close — the operator's",`+
|
|
||||||
`"says":"novox/mesh-controller#212 is open on main, which requires the merge check, and its head's check `+
|
|
||||||
`started and never answered: mesh/merge-gate pending since 2026-10-11T00:11:39Z, mesh/repo-check never set. `+
|
|
||||||
`No delivery of it is held here, so nothing asks it again. A new commit on its branch announces it and asks `+
|
|
||||||
`its check"}`), &l); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
obs := stalledObservations([]stalledLine{l})
|
|
||||||
if len(obs) != 1 || obs[0].Resolver != conditions.ResolverOperator {
|
|
||||||
t.Fatalf("an unanswered merge check is not the operator's: %+v", obs)
|
|
||||||
}
|
|
||||||
o := obs[0]
|
|
||||||
t.Logf("headline: %s\nexplanation: %s\nneeds: %s\nresolved: %s", o.Headline, o.Explanation, o.Needs, o.Resolved)
|
|
||||||
if len(o.Actions) != 0 {
|
|
||||||
t.Fatalf("it offers an action a head with no delivery cannot take: %+v", o.Actions)
|
|
||||||
}
|
|
||||||
if strings.Contains(o.Needs, "stop") || strings.Contains(o.Needs, "release") || !strings.Contains(o.Needs, "commit") ||
|
|
||||||
!strings.Contains(o.Needs, "in the build queue: mesh-controller.queue") || strings.Contains(o.Needs, "build seat") {
|
|
||||||
t.Fatalf("it says to %q", o.Needs)
|
|
||||||
}
|
|
||||||
if o.Headline != "Pull request mesh-controller #212's merge check has not answered for 61 minutes" {
|
|
||||||
t.Fatalf("its headline reads %q", o.Headline)
|
|
||||||
}
|
|
||||||
for _, want := range []string{"mesh/merge-gate pending since 2026-10-11T00:11:39Z", "mesh/repo-check never set",
|
|
||||||
"past its limit of 60 minutes"} {
|
|
||||||
if !strings.Contains(o.Explanation, want) {
|
|
||||||
t.Fatalf("its explanation does not say %q: %q", want, o.Explanation)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if strings.Contains(o.Explanation, "never asked") || strings.Contains(o.Explanation, "never announced") {
|
|
||||||
t.Fatalf("it says the mesh was never asked, of a head whose check started: %q", o.Explanation)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A line from a mesh-delivery that names no checks still says the check did not answer, and offers nothing.
|
|
||||||
var bare stalledLine
|
|
||||||
if err := json.Unmarshal([]byte(`{"id":"novox/mesh-controller@c11222026a3b","number":212,"state":"unanswered",`+
|
|
||||||
`"for":"1h1m0s","bound":"1h0m0s","h2":"none: no delivery of it is held here, so there is none to close — the operator's"}`),
|
|
||||||
&bare); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
o = stalledObservations([]stalledLine{bare})[0]
|
|
||||||
if len(o.Actions) != 0 || !strings.Contains(o.Explanation, "has not answered") || strings.Contains(o.Headline, "Delivery of") {
|
|
||||||
t.Fatalf("a line naming no checks reads %q / %q, actions %+v", o.Headline, o.Explanation, o.Actions)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **An unanswered merge check's time reaches the operator in their own time, never as raw UTC** (novox/hq issue
|
|
||||||
// 443): mesh-delivery says each check waited on as data — its context, its state and since when, in RFC 3339 — beside
|
|
||||||
// the finished words it gave before, and the controller words it in its local zone, as every other time in a message.
|
|
||||||
// The line is the one mesh-delivery says, as in the report of issue 443, with the checks it now carries.
|
|
||||||
func TestAnUnansweredMergeChecksTimeIsSaidInLocalTime(t *testing.T) {
|
|
||||||
// The operator's zone given through the seam, never by writing time.Local: other tests' goroutines read it, and
|
|
||||||
// the build seat runs the suite under the race detector.
|
|
||||||
wasZone, wasNow := wordsZone, wordsNow
|
|
||||||
wordsZone = func() *time.Location { return time.FixedZone("CEST", 2*60*60) }
|
|
||||||
wordsNow = func() time.Time { return time.Date(2026, 10, 11, 1, 12, 39, 0, time.UTC) }
|
|
||||||
t.Cleanup(func() { wordsZone, wordsNow = wasZone, wasNow })
|
|
||||||
|
|
||||||
var l stalledLine
|
|
||||||
if err := json.Unmarshal([]byte(`{"id":"novox/mesh-controller@c11222026a3b","number":212,"state":"unanswered",`+
|
|
||||||
`"waiting":["mesh/merge-gate pending since 2026-10-11T00:11:39Z","mesh/repo-check never set"],`+
|
|
||||||
`"checks":[{"context":"mesh/merge-gate","state":"pending","since":"2026-10-11T00:11:39Z"},`+
|
|
||||||
`{"context":"mesh/repo-check","state":"never-set"}],"for":"1h1m0s",`+
|
|
||||||
`"bound":"1h0m0s","h2":"none: no delivery of it is held here, so there is none to close — the operator's",`+
|
|
||||||
`"says":"novox/mesh-controller#212 is open on main, which requires the merge check, and its head's check `+
|
|
||||||
`started and never answered: mesh/merge-gate pending since 2026-10-11T00:11:39Z, mesh/repo-check never set. `+
|
|
||||||
`No delivery of it is held here, so nothing asks it again. A new commit on its branch announces it and asks `+
|
|
||||||
`its check"}`), &l); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
o := stalledObservations([]stalledLine{l})[0]
|
|
||||||
t.Logf("explanation: %s", o.Explanation)
|
|
||||||
for _, raw := range []string{"2026-10-11T00:11:39Z", "00:11", "UTC"} {
|
|
||||||
if strings.Contains(o.Explanation, raw) || strings.Contains(o.Headline, raw) {
|
|
||||||
t.Fatalf("the operator reads %q: %q / %q", raw, o.Headline, o.Explanation)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, want := range []string{"mesh/merge-gate pending since 02:11", "mesh/repo-check never set"} {
|
|
||||||
if !strings.Contains(o.Explanation, want) {
|
|
||||||
t.Fatalf("its explanation does not say %q: %q", want, o.Explanation)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A check pending since another day than today says which day, so the time stays absolute.
|
|
||||||
l.Checks[0].Since = "2026-10-09T21:05:00Z"
|
|
||||||
if o = stalledObservations([]stalledLine{l})[0]; !strings.Contains(o.Explanation, "mesh/merge-gate pending since 23:05 on 9 Oct") {
|
|
||||||
t.Fatalf("a check pending since an earlier day reads %q", o.Explanation)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A pending check whose time the forge did not say is said so, with no time made up.
|
|
||||||
l.Checks[0].Since = ""
|
|
||||||
if o = stalledObservations([]stalledLine{l})[0]; !strings.Contains(o.Explanation, "mesh/merge-gate pending, since a time the forge did not say") {
|
|
||||||
t.Fatalf("a pending check without its time reads %q", o.Explanation)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **An unanswered merge check on a ready or rejected delivery is told to recheck it** (novox/hq issue 451):
|
|
||||||
// mesh-delivery says such a line too (issue 441) and names the delivery's state as data, `held`. That check already
|
|
||||||
// answered once, so the build queue is the wrong place to look; a recheck of the delivery, named by its id, asks the
|
|
||||||
// check again. A line with no delivery held keeps the advice of issue 438, and so does a line from a mesh-delivery
|
|
||||||
// that does not say `held` yet: the controller never guesses the state from the words.
|
|
||||||
func TestAnUnansweredMergeCheckOnAReadyOrRejectedDeliveryOffersARecheck(t *testing.T) {
|
|
||||||
line := func(held string) stalledLine {
|
|
||||||
var l stalledLine
|
|
||||||
raw := `{"id":"novox/mesh-controller@c11222026a3b","number":212,"state":"unanswered",` +
|
|
||||||
`"waiting":["mesh/merge-gate pending since 2026-10-11T00:11:39Z"],"for":"1h1m0s","bound":"1h0m0s",` +
|
|
||||||
`"h2":"none: its delivery is ` + held + `, a state the operator moves — the operator's",` +
|
|
||||||
`"says":"novox/mesh-controller#212 is open on main, which requires the merge check, and its delivery is ` +
|
|
||||||
held + ` here, but on the forge its check has not answered: mesh/merge-gate pending since ` +
|
|
||||||
`2026-10-11T00:11:39Z. The pull request waits on that status and nothing asks it again. A recheck of the ` +
|
|
||||||
`delivery asks its check again, as does a new commit on its branch"`
|
|
||||||
if held != "" {
|
|
||||||
raw += `,"held":"` + held + `"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal([]byte(raw+`}`), &l); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return l
|
|
||||||
}
|
|
||||||
for _, held := range []string{"ready", "rejected"} {
|
|
||||||
o := stalledObservations([]stalledLine{line(held)})[0]
|
|
||||||
t.Logf("%s: %s\nneeds: %s", held, o.Explanation, o.Needs)
|
|
||||||
if !strings.Contains(o.Needs, "mesh-delivery.recheck") || !strings.Contains(o.Needs, "novox/mesh-controller@c11222026a3b") {
|
|
||||||
t.Fatalf("a %s delivery's unanswered check is not offered its recheck: %q", held, o.Needs)
|
|
||||||
}
|
|
||||||
if strings.Contains(o.Needs, "build queue") || strings.Contains(o.Needs, "mesh-controller.queue") {
|
|
||||||
t.Fatalf("a %s delivery's unanswered check, which answered once, is sent to the build queue: %q", held, o.Needs)
|
|
||||||
}
|
|
||||||
if !strings.Contains(o.Explanation, "Its delivery is "+held) {
|
|
||||||
t.Fatalf("a %s delivery's explanation does not say its state: %q", held, o.Explanation)
|
|
||||||
}
|
|
||||||
if len(o.Actions) != 0 {
|
|
||||||
t.Fatalf("it offers an action the controller is not granted: %+v", o.Actions)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// No delivery held, or a mesh-delivery that does not say: the advice of issue 438, and no recheck.
|
|
||||||
o := stalledObservations([]stalledLine{line("")})[0]
|
|
||||||
if !strings.Contains(o.Needs, "in the build queue: mesh-controller.queue") || strings.Contains(o.Needs, "recheck") {
|
|
||||||
t.Fatalf("a line with no delivery held says to %q", o.Needs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
+23
-129
@@ -7,7 +7,6 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
|
||||||
"os"
|
"os"
|
||||||
"slices"
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
@@ -292,7 +291,7 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
// Said, not skipped: a machine dropped here offers nothing and holds nothing as far
|
// Said, not skipped: a machine dropped here offers nothing and holds nothing as far
|
||||||
// as every other machine's plan can tell (novox/hq issue 188).
|
// as every other machine's plan can tell (novox/hq issue 188).
|
||||||
sayLeftOut(ctx, o.node.Name, err)
|
fmt.Fprintf(os.Stderr, "%s is left out of the rest of the mesh: it does not resolve: %v\n", o.node.Name, err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
firstHeld = append(firstHeld, got.Claims...)
|
firstHeld = append(firstHeld, got.Claims...)
|
||||||
@@ -337,7 +336,7 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
|
|||||||
// resolved without its offers, and one that consumes them cannot resolve here by
|
// resolved without its offers, and one that consumes them cannot resolve here by
|
||||||
// design — that is not the machine being dropped, it is the view being partial.
|
// design — that is not the machine being dropped, it is the view being partial.
|
||||||
if exclude == "" {
|
if exclude == "" {
|
||||||
sayLeftOut(ctx, o.node.Name, err)
|
fmt.Fprintf(os.Stderr, "%s is left out of the rest of the mesh: it does not resolve: %v\n", o.node.Name, err)
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -347,35 +346,6 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
|
|||||||
return world, nil
|
return world, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// leftOutKey carries where theRestOfTheMesh says a machine it leaves out, for an act that answers with
|
|
||||||
// its own resolution instead (novox/hq issue 237).
|
|
||||||
type leftOutKey struct{}
|
|
||||||
|
|
||||||
// leftOutSaidBy is ctx with theRestOfTheMesh's "left out" lines written to w rather than to standard
|
|
||||||
// error.
|
|
||||||
//
|
|
||||||
// **An act's answer is one resolution taken after its write** (novox/hq issue 237, decided 2026-10-11).
|
|
||||||
// `assign` read the whole mesh once before its write — to record the seats held by derivation — and that
|
|
||||||
// read said, on standard error, that the node did not resolve for lack of the seat the act was about to
|
|
||||||
// give it. The controller seat's verb answers with both streams, so the stale line came last, after the
|
|
||||||
// act's own "now has it held", and named as its remedy the assignment just made. An act gives io.Discard
|
|
||||||
// here: what the node resolves to after the write is said by its own planFor, and every other machine
|
|
||||||
// that does not resolve by blockedElsewhere, both read after the write.
|
|
||||||
func leftOutSaidBy(ctx context.Context, w io.Writer) context.Context {
|
|
||||||
return context.WithValue(ctx, leftOutKey{}, w)
|
|
||||||
}
|
|
||||||
|
|
||||||
// sayLeftOut says that a machine is left out of the rest of the mesh, where ctx says to: standard
|
|
||||||
// error unless an act said otherwise. Said, not skipped: a machine dropped there offers nothing and
|
|
||||||
// holds nothing as far as every other machine's plan can tell (novox/hq issue 188).
|
|
||||||
func sayLeftOut(ctx context.Context, node string, err error) {
|
|
||||||
w, ok := ctx.Value(leftOutKey{}).(io.Writer)
|
|
||||||
if !ok {
|
|
||||||
w = os.Stderr
|
|
||||||
}
|
|
||||||
fmt.Fprintf(w, "%s is left out of the rest of the mesh: it does not resolve: %v\n", node, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// declarationFor is everything a node would be sent.
|
// declarationFor is everything a node would be sent.
|
||||||
//
|
//
|
||||||
// One place, because there were three and one of them was written before credentials existed and
|
// One place, because there were three and one of them was written before credentials existed and
|
||||||
@@ -450,8 +420,7 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
|||||||
out := sendable{Resources: composed.Resources, Adoption: adoption,
|
out := sendable{Resources: composed.Resources, Adoption: adoption,
|
||||||
Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
|
Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
|
||||||
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld,
|
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld,
|
||||||
unbound: with.Unbound, foreseen: composed.Foreseen, unplaced: composed.Unplaced,
|
unbound: with.Unbound, foreseen: composed.Foreseen, unplaced: composed.Unplaced}
|
||||||
modules: namedModules(plan, composed.LeftOut)}
|
|
||||||
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
|
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
|
||||||
// 0221). Read only on the send path: a question about what would be sent records nothing.
|
// 0221). Read only on the send path: a question about what would be sent records nothing.
|
||||||
if choosing == Allocating {
|
if choosing == Allocating {
|
||||||
@@ -695,33 +664,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
|||||||
needed := map[string]map[string]string{}
|
needed := map[string]map[string]string{}
|
||||||
foreseen := map[string]map[string]bool{}
|
foreseen := map[string]map[string]bool{}
|
||||||
for _, m := range plan.Modules {
|
for _, m := range plan.Modules {
|
||||||
// **A secret family is never made** (novox/hq ADR 0283): each member a person gave on this machine is
|
for name := range m.OwnSecrets {
|
||||||
// placed, and one not given is nothing — the module says it waits for it.
|
|
||||||
for _, family := range m.OwnSecrets.Families() {
|
|
||||||
members, err := inv.GivenMembers(ctx, node, m.Module, family)
|
|
||||||
if err != nil {
|
|
||||||
return catalogue.Rendering{}, inventory.Node{}, err
|
|
||||||
}
|
|
||||||
for _, g := range members {
|
|
||||||
if _, fam, ok := m.OwnSecrets.Lookup(g.Name); !ok || fam != family {
|
|
||||||
continue // a longer family's member, or a name no longer of this family
|
|
||||||
}
|
|
||||||
if !g.Current {
|
|
||||||
if choosing == Allocating {
|
|
||||||
return catalogue.Rendering{}, inventory.Node{}, fmt.Errorf(
|
|
||||||
"%s on %s holds %q, which was given to the mesh rather than made by it, and %s has "+
|
|
||||||
"since generated a new sealing key. The mesh cannot make another; give it again",
|
|
||||||
m.Module, node, g.Name, node)
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if needed[m.Module] == nil {
|
|
||||||
needed[m.Module] = map[string]string{}
|
|
||||||
}
|
|
||||||
needed[m.Module][g.Name] = g.Sealed
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for name := range m.OwnSecrets.Plain() {
|
|
||||||
// Minted on the send path and only read on every other. Making one is an insert, and
|
// Minted on the send path and only read on every other. Making one is an insert, and
|
||||||
// a question that writes is a question that can block against the machine it is about.
|
// a question that writes is a question that can block against the machine it is about.
|
||||||
var sealed string
|
var sealed string
|
||||||
@@ -947,14 +890,9 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return catalogue.Rendering{}, inventory.Node{}, err
|
return catalogue.Rendering{}, inventory.Node{}, err
|
||||||
}
|
}
|
||||||
grantsGroups, err := engineGrantsGroups(ctx, inv, node)
|
|
||||||
if err != nil {
|
|
||||||
return catalogue.Rendering{}, inventory.Node{}, err
|
|
||||||
}
|
|
||||||
return catalogue.Rendering{
|
return catalogue.Rendering{
|
||||||
ReadsHealth: readsHealth,
|
ReadsHealth: readsHealth,
|
||||||
JudgesRoot: judgesRoot,
|
JudgesRoot: judgesRoot,
|
||||||
GrantsGroups: grantsGroups,
|
|
||||||
BusMembership: memberships[node],
|
BusMembership: memberships[node],
|
||||||
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Foreseen: foreseen, Ports: ports,
|
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Foreseen: foreseen, Ports: ports,
|
||||||
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
||||||
@@ -987,16 +925,6 @@ func engineJudgesRoot(ctx context.Context, inv *inventory.Inventory, node string
|
|||||||
return had && stated.Contract >= link.RootContract, nil
|
return had && stated.Contract >= link.RootContract, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// engineGrantsGroups says whether a machine's node-engine reads an access's `grant` (novox/hq ADR 0312), by its own
|
|
||||||
// newest statement, for the same reason as engineReadsHealth: an older engine parses strictly.
|
|
||||||
func engineGrantsGroups(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) {
|
|
||||||
stated, had, err := inv.HealthOf(ctx, node)
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
return had && stated.Contract >= link.GrantContract, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// zonesInTheMesh is every zone a module in the mesh declares, where the mesh placed it (novox/hq ADR
|
// zonesInTheMesh is every zone a module in the mesh declares, where the mesh placed it (novox/hq ADR
|
||||||
// 0199): the zone settled from that node's settings, the node's private address, the port the
|
// 0199): the zone settled from that node's settings, the node's private address, the port the
|
||||||
// answering listen is published on there.
|
// answering listen is published on there.
|
||||||
@@ -1336,11 +1264,6 @@ func planCommand(ctx context.Context, args []string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// What the machine was last told, by whom and from which assignment generation (novox/hq issue 234),
|
|
||||||
// beside what it would be told now. A record that cannot be read is said, not taken for none.
|
|
||||||
if err := writeLastSend(ctx, os.Stdout, open.inventory, args[0]); err != nil {
|
|
||||||
fmt.Printf("what %s was last sent could not be read: %v\n", args[0], err)
|
|
||||||
}
|
|
||||||
// Which modules a push would leave out, and why — said before the plan, since the plan is of
|
// Which modules a push would leave out, and why — said before the plan, since the plan is of
|
||||||
// what the machine would be told (novox/hq ADR 0163, rule 6). Judged, never composed: `plan`
|
// what the machine would be told (novox/hq ADR 0163, rule 6). Judged, never composed: `plan`
|
||||||
// without --json allocates nothing.
|
// without --json allocates nothing.
|
||||||
@@ -1736,27 +1659,17 @@ func providerModuleOf(resolved catalogue.Resolution, open *stores, ctx context.C
|
|||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// unheldLogger remembers what was last logged about each node's unmet seat dependencies, so the
|
// unheldLogged is what was last logged about each node's unmet seat dependencies, so the log says
|
||||||
// log says each change once (novox/hq ADR 0207), on stderr so `status --json` stays a document.
|
// each change once (novox/hq ADR 0207), on stderr so `status --json` stays a document.
|
||||||
// A test gives its own writer and memory, and no longer shares the process's (novox/hq issue 486).
|
|
||||||
//
|
//
|
||||||
// **The serving controller's log only.** planFor runs for every node on every push, assignment and
|
// **The serving controller's log only.** planFor runs for every node on every push, assignment and
|
||||||
// status — `blockedElsewhere` alone resolves the whole mesh — and a one-shot command starts with an
|
// status — `blockedElsewhere` alone resolves the whole mesh — and a one-shot command starts with an
|
||||||
// empty memory, so every node's report was "a change" and a push printed the whole mesh's list,
|
// empty memory, so every node's report was "a change" and a push printed the whole mesh's list,
|
||||||
// burying the line about the node it acted on. A command says what concerns its own act instead
|
// burying the line about the node it acted on. A command says what concerns its own act instead
|
||||||
// (unheldChange, reportUnheldPushed); the full list is `status`'s.
|
// (unheldChange, reportUnheldPushed); the full list is `status`'s.
|
||||||
type unheldLogger struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
logged map[string]string
|
|
||||||
out io.Writer
|
|
||||||
}
|
|
||||||
|
|
||||||
func newUnheldLogger(out io.Writer) *unheldLogger {
|
|
||||||
return &unheldLogger{logged: map[string]string{}, out: out}
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
var (
|
||||||
unheldLog = newUnheldLogger(os.Stderr)
|
unheldLogged = map[string]string{}
|
||||||
|
unheldLoggedMu sync.Mutex
|
||||||
logUnheldChanges bool
|
logUnheldChanges bool
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -1766,27 +1679,23 @@ func logUnheld(node string, unheld []catalogue.Unheld) {
|
|||||||
if !logUnheldChanges {
|
if !logUnheldChanges {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
unheldLog.log(node, unheld)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l *unheldLogger) log(node string, unheld []catalogue.Unheld) {
|
|
||||||
lines := make([]string, 0, len(unheld))
|
lines := make([]string, 0, len(unheld))
|
||||||
for _, u := range unheld {
|
for _, u := range unheld {
|
||||||
lines = append(lines, u.String())
|
lines = append(lines, u.String())
|
||||||
}
|
}
|
||||||
now := strings.Join(lines, "\n")
|
now := strings.Join(lines, "\n")
|
||||||
l.mu.Lock()
|
unheldLoggedMu.Lock()
|
||||||
defer l.mu.Unlock()
|
before, seen := unheldLogged[node]
|
||||||
before, seen := l.logged[node]
|
unheldLogged[node] = now
|
||||||
l.logged[node] = now
|
unheldLoggedMu.Unlock()
|
||||||
if (seen && before == now) || (!seen && now == "") {
|
if (seen && before == now) || (!seen && now == "") {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if now == "" {
|
if now == "" {
|
||||||
fmt.Fprintf(l.out, "%s: every seat its modules depend on is held (novox/hq ADR 0207)\n", node)
|
fmt.Fprintf(os.Stderr, "%s: every seat its modules depend on is held (novox/hq ADR 0207)\n", node)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
fmt.Fprintf(l.out, "%s: %d unmet seat dependenc(ies), reported and not refused (novox/hq ADR 0207):\n %s\n",
|
fmt.Fprintf(os.Stderr, "%s: %d unmet seat dependenc(ies), reported and not refused (novox/hq ADR 0207):\n %s\n",
|
||||||
node, len(lines), strings.Join(lines, "\n "))
|
node, len(lines), strings.Join(lines, "\n "))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1795,43 +1704,28 @@ func (l *unheldLogger) log(node string, unheld []catalogue.Unheld) {
|
|||||||
// which includes every one of a module just assigned — and each now met that was not. Nothing about
|
// which includes every one of a module just assigned — and each now met that was not. Nothing about
|
||||||
// any other node, and nothing that was already true before the act.
|
// any other node, and nothing that was already true before the act.
|
||||||
func unheldChange(shelf map[string]catalogue.Manifest, node string, before, after []string) []string {
|
func unheldChange(shelf map[string]catalogue.Manifest, node string, before, after []string) []string {
|
||||||
return unheldChangeTo(shelf, node, before, after, unheldOver(shelf, node, after))
|
judge := func(names []string) map[string]catalogue.Unheld {
|
||||||
}
|
|
||||||
|
|
||||||
// unheldOver is the node's unmet seat dependencies judged over a set of module names alone.
|
|
||||||
func unheldOver(shelf map[string]catalogue.Manifest, node string, names []string) []catalogue.Unheld {
|
|
||||||
var set []catalogue.Manifest
|
var set []catalogue.Manifest
|
||||||
for _, n := range names {
|
for _, n := range names {
|
||||||
if m, known := shelf[n]; known {
|
if m, known := shelf[n]; known {
|
||||||
set = append(set, m)
|
set = append(set, m)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return catalogue.UnheldDependencies(shelf, node, set, nil)
|
out := map[string]catalogue.Unheld{}
|
||||||
|
for _, u := range catalogue.UnheldDependencies(shelf, node, set, nil) {
|
||||||
|
out[u.Module+" "+u.Seat] = u
|
||||||
}
|
}
|
||||||
|
return out
|
||||||
// unheldChangeTo is unheldChange with the node's dependencies as they now stand given: `now` is what
|
|
||||||
// the act's one resolution after its write says is unmet, and `after` the modules that resolution has
|
|
||||||
// on the node (novox/hq issue 237). A line about how things stand after the act is never judged from
|
|
||||||
// anything else, so it cannot disagree with `plan` for the node.
|
|
||||||
func unheldChangeTo(shelf map[string]catalogue.Manifest, node string, before, after []string,
|
|
||||||
now []catalogue.Unheld) []string {
|
|
||||||
key := func(u catalogue.Unheld) string { return u.Module + " " + u.Seat }
|
|
||||||
was := map[string]catalogue.Unheld{}
|
|
||||||
for _, u := range unheldOver(shelf, node, before) {
|
|
||||||
was[key(u)] = u
|
|
||||||
}
|
|
||||||
is := map[string]catalogue.Unheld{}
|
|
||||||
for _, u := range now {
|
|
||||||
is[key(u)] = u
|
|
||||||
}
|
}
|
||||||
|
was, now := judge(before), judge(after)
|
||||||
var lines []string
|
var lines []string
|
||||||
for _, k := range sortedNames(is) {
|
for _, k := range sortedNames(now) {
|
||||||
if _, already := was[k]; !already {
|
if _, already := was[k]; !already {
|
||||||
lines = append(lines, "but "+is[k].String())
|
lines = append(lines, "but "+now[k].String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, k := range sortedNames(was) {
|
for _, k := range sortedNames(was) {
|
||||||
if _, still := is[k]; still {
|
if _, still := now[k]; still {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
u := was[k]
|
u := was[k]
|
||||||
|
|||||||
@@ -251,16 +251,6 @@ func resumed(p *inventory.Plan, why string) {
|
|||||||
if p.State == inventory.PlanFailed && len(failedIn(*p)) == 0 {
|
if p.State == inventory.PlanFailed && len(failedIn(*p)) == 0 {
|
||||||
p.State = inventory.PlanBuilding
|
p.State = inventory.PlanBuilding
|
||||||
p.Note = why
|
p.Note = why
|
||||||
goesOnAgain(p)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// goesOnAgain clears a stop from a walk that walks again: a stop said why the walk ended, and a walk resumed has
|
|
||||||
// not ended. Kept, it would end a retried walk that fails again as stopped, not failed (novox/hq ADR 0308
|
|
||||||
// decision 1), and keep the search for the merge that brought the failure from running (searchAfterFailures).
|
|
||||||
func goesOnAgain(p *inventory.Plan) {
|
|
||||||
if p.Delivery != nil {
|
|
||||||
p.Delivery.Stopped, p.Delivery.StoppedWhy = "", ""
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -288,13 +278,6 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
|||||||
plans = append(plans, recent...)
|
plans = append(plans, recent...)
|
||||||
// **A failed walk whose earlier merges are walked alone is not retried** (novox/hq ADR 0276): the search for
|
// **A failed walk whose earlier merges are walked alone is not retried** (novox/hq ADR 0276): the search for
|
||||||
// the merge that brought the failure answers them now, and a retried walk would name them twice.
|
// the merge that brought the failure answers them now, and a retried walk would name them twice.
|
||||||
// **A walk stopped without some merges is not retried** (novox/hq ADR 0299): the merges it left out were stopped,
|
|
||||||
// and the others are another walk's now.
|
|
||||||
if p.Delivery != nil && (p.Delivery.WalkedAgainBy != "" || len(p.Delivery.LeftOut) > 0) {
|
|
||||||
return "", fmt.Errorf("%s was stopped without %s: those merges were stopped, and %s walks the others; a "+
|
|
||||||
"newer merge, or `rebuild <module>`, builds again", p.ID, mergesLeftOutWords(p.Delivery.LeftOut),
|
|
||||||
orWords(p.Delivery.WalkedAgainBy, "nothing"))
|
|
||||||
}
|
|
||||||
if p.Delivery != nil && len(p.Delivery.Merges) > 0 {
|
if p.Delivery != nil && len(p.Delivery.Merges) > 0 {
|
||||||
kept, err := inv.MergesOf(ctx, p.ID)
|
kept, err := inv.MergesOf(ctx, p.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -305,24 +288,13 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
|||||||
"those walks answer them; a newer merge, or `rebuild <module>`, builds again", p.ID)
|
"those walks answer them; a newer merge, or `rebuild <module>`, builds again", p.ID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Settled from the build records before anything is judged (novox/hq issue 457): a build of the tier
|
|
||||||
// that failed after the plan did is as failed as the one that failed it. What toRetry says is the
|
|
||||||
// failed set every step below works from.
|
|
||||||
var failed []string
|
|
||||||
if p.Tier < len(p.Tiers) {
|
|
||||||
recorded, byID, err := recordsOfAsked(ctx, inv, &p, p.Tiers[p.Tier])
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
failed = toRetry(&p, recorded, byID)
|
|
||||||
}
|
|
||||||
if err := retryRefusal(p, plans); err != nil {
|
if err := retryRefusal(p, plans); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
if again := unjudgedAtGate(p); len(failed) == 0 && len(again) > 0 {
|
if again := unjudgedAtGate(p); len(failedIn(p)) == 0 && len(again) > 0 {
|
||||||
return retryTierWhole(ctx, open, &p, again)
|
return retryTierWhole(ctx, open, &p, again)
|
||||||
}
|
}
|
||||||
if len(failed) == 0 {
|
if len(failedIn(p)) == 0 {
|
||||||
return retryRollouts(ctx, open, &p)
|
return retryRollouts(ctx, open, &p)
|
||||||
}
|
}
|
||||||
entries, err := inv.Catalogued(ctx)
|
entries, err := inv.Catalogued(ctx)
|
||||||
@@ -333,6 +305,7 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
|||||||
for _, e := range entries {
|
for _, e := range entries {
|
||||||
byName[e.Manifest.Module] = e
|
byName[e.Manifest.Module] = e
|
||||||
}
|
}
|
||||||
|
failed := failedIn(p)
|
||||||
var asked []string
|
var asked []string
|
||||||
for _, m := range failed {
|
for _, m := range failed {
|
||||||
askModule(ctx, &p, m, byName)
|
askModule(ctx, &p, m, byName)
|
||||||
@@ -470,7 +443,6 @@ func retryRollouts(ctx context.Context, open *stores, p *inventory.Plan) (string
|
|||||||
said = append(said, m+" to "+strings.Join(again, ", "))
|
said = append(said, m+" to "+strings.Join(again, ", "))
|
||||||
}
|
}
|
||||||
p.State = inventory.PlanRolling
|
p.State = inventory.PlanRolling
|
||||||
goesOnAgain(p)
|
|
||||||
p.Note = fmt.Sprintf("tier %d retried by hand; sent %s first again", p.Tier, strings.Join(said, "; "))
|
p.Note = fmt.Sprintf("tier %d retried by hand; sent %s first again", p.Tier, strings.Join(said, "; "))
|
||||||
if err := open.inventory.SavePlan(ctx, p); err != nil {
|
if err := open.inventory.SavePlan(ctx, p); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
@@ -536,7 +508,6 @@ func retryTierWhole(ctx context.Context, open *stores, p *inventory.Plan, again
|
|||||||
}
|
}
|
||||||
if p.State == inventory.PlanFailed && len(failedIn(*p)) == 0 {
|
if p.State == inventory.PlanFailed && len(failedIn(*p)) == 0 {
|
||||||
p.State = inventory.PlanBuilding
|
p.State = inventory.PlanBuilding
|
||||||
goesOnAgain(p)
|
|
||||||
p.Note = fmt.Sprintf("tier %d retried by hand: %s asked again; %s sent again with the tier", p.Tier,
|
p.Note = fmt.Sprintf("tier %d retried by hand: %s asked again; %s sent again with the tier", p.Tier,
|
||||||
strings.Join(again, ", "), orNone(strings.Join(resent, ", ")))
|
strings.Join(again, ", "), orNone(strings.Join(resent, ", ")))
|
||||||
}
|
}
|
||||||
@@ -554,16 +525,3 @@ func retryTierWhole(ctx context.Context, open *stores, p *inventory.Plan, again
|
|||||||
func sendAgain(s *inventory.PlanModule) {
|
func sendAgain(s *inventory.PlanModule) {
|
||||||
s.First, s.FirstAt, s.Gate, s.GatedBy, s.Previous, s.Why = nil, nil, nil, "", "", ""
|
s.First, s.FirstAt, s.Gate, s.GatedBy, s.Previous, s.Why = nil, nil, nil, "", "", ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// toRetry is the modules a retry asks again: every one of the tier that failed, the plan's state
|
|
||||||
// settled from the build records first (novox/hq issue 457). A plan fails on the first failure in its
|
|
||||||
// tier, and an outcome arriving after that finds no open plan to answer — it is kept only in the build
|
|
||||||
// records. Read from the plan alone, a retry asked only the build that failed first, and the records
|
|
||||||
// then failed the plan again on the next: each failed build of a tier took a retry of its own. What
|
|
||||||
// still runs is left asked, and its outcome is the plan's once the retry sets it building.
|
|
||||||
func toRetry(p *inventory.Plan, recorded map[string][]inventory.Build, byID map[string]inventory.Build) []string {
|
|
||||||
if p.Tier < len(p.Tiers) {
|
|
||||||
settleFromRecords(p, p.Tiers[p.Tier], recorded, byID)
|
|
||||||
}
|
|
||||||
return failedIn(*p)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,57 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"reflect"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A retry asks every failed build of the tier, not one (novox/hq issue 457). Seen 2026-10-11: gitea
|
|
||||||
// and plex both failed in tier 0 while the registry was held still; gitea's failure failed the plan,
|
|
||||||
// and plex's, arriving after, found no open plan and was kept only in the build records. The first
|
|
||||||
// retry asked gitea alone, the records then failed the plan again on plex, and a second retry asked
|
|
||||||
// plex.
|
|
||||||
func TestARetryAsksEveryFailedBuildOfTheTier(t *testing.T) {
|
|
||||||
asked := time.Date(2026, 10, 11, 1, 29, 0, 0, time.UTC)
|
|
||||||
failedAt := asked.Add(2 * time.Minute)
|
|
||||||
p := inventory.Plan{ID: "plan-457", State: inventory.PlanFailed, Tier: 0,
|
|
||||||
Tiers: [][]string{{"gitea", "plex"}}, Note: "gitea failed to build in tier 0",
|
|
||||||
Modules: map[string]*inventory.PlanModule{
|
|
||||||
"gitea": {State: "failed", AskedAt: &asked, Build: "build-gitea", Why: "cannot reach the registry"},
|
|
||||||
"plex": {State: "asked", AskedAt: &asked, Build: "build-plex"},
|
|
||||||
}}
|
|
||||||
byID := map[string]inventory.Build{
|
|
||||||
"build-plex": {ID: "build-plex", Module: "plex", At: failedAt, Failed: "cannot reach the registry"},
|
|
||||||
}
|
|
||||||
|
|
||||||
if got := toRetry(&p, nil, byID); !reflect.DeepEqual(got, []string{"gitea", "plex"}) {
|
|
||||||
t.Fatalf("a retry of a tier where gitea and plex failed asks %v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A build of the tier still running when the plan failed is not asked again: its outcome is the plan's
|
|
||||||
// once the retry sets it building, and one that is recorded built is taken as built.
|
|
||||||
func TestARetryLeavesABuildThatRunsOrWorked(t *testing.T) {
|
|
||||||
asked := time.Date(2026, 10, 11, 1, 29, 0, 0, time.UTC)
|
|
||||||
builtAt := asked.Add(3 * time.Minute)
|
|
||||||
p := inventory.Plan{ID: "plan-457", State: inventory.PlanFailed, Tier: 0,
|
|
||||||
Tiers: [][]string{{"a", "b", "c"}},
|
|
||||||
Modules: map[string]*inventory.PlanModule{
|
|
||||||
"a": {State: "failed", AskedAt: &asked, Build: "build-a", Why: "broken"},
|
|
||||||
"b": {State: "asked", AskedAt: &asked, Build: "build-b"},
|
|
||||||
"c": {State: "asked", AskedAt: &asked, Build: "build-c"},
|
|
||||||
}}
|
|
||||||
byID := map[string]inventory.Build{"build-c": {ID: "build-c", Module: "c", At: builtAt, Commit: "c0ffee"}}
|
|
||||||
|
|
||||||
if got := toRetry(&p, nil, byID); !reflect.DeepEqual(got, []string{"a"}) {
|
|
||||||
t.Fatalf("asks %v, want a alone", got)
|
|
||||||
}
|
|
||||||
if s := p.Modules["c"]; s.State != "built" || s.Commit != "c0ffee" {
|
|
||||||
t.Errorf("c, recorded built, is %+v", s)
|
|
||||||
}
|
|
||||||
if s := p.Modules["b"]; s.State != "asked" {
|
|
||||||
t.Errorf("b, still building, is %+v", s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -78,11 +78,6 @@ func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation
|
|||||||
return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err)
|
return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err)
|
||||||
}
|
}
|
||||||
var problems, foreseen []string
|
var problems, foreseen []string
|
||||||
// Each module the next send leaves out of this machine's declaration (novox/hq issue 380), judged from this
|
|
||||||
// resolution as the send judges it: a finding, never a refusal.
|
|
||||||
if err == nil {
|
|
||||||
out = append(out, leftOutObservations(n.Name, leftOutOf(plan, settings, n.Adopted))...)
|
|
||||||
}
|
|
||||||
if err == nil && gensErr == nil {
|
if err == nil && gensErr == nil {
|
||||||
var declared sendable
|
var declared sendable
|
||||||
if declared, err = declarationWith(ctx, open, n.Name, plan, settings, gens, Foreseeing); err == nil {
|
if declared, err = declarationWith(ctx, open, n.Name, plan, settings, gens, Foreseeing); err == nil {
|
||||||
|
|||||||
@@ -591,7 +591,7 @@ func proposeCommand(ctx context.Context, module, node, valuesArg string, clear,
|
|||||||
return err
|
return err
|
||||||
},
|
},
|
||||||
routerHere: routerHereIn(open.inventory),
|
routerHere: routerHereIn(open.inventory),
|
||||||
grantHeld: grantHeldIn(open, conn),
|
grantHeld: grantHeldIn(open),
|
||||||
routerRecord: func(ctx context.Context, id string) (string, error) {
|
routerRecord: func(ctx context.Context, id string) (string, error) {
|
||||||
bucket, err := asksRecords(ctx, open.inventory)
|
bucket, err := asksRecords(ctx, open.inventory)
|
||||||
if err != nil || bucket == "" {
|
if err != nil || bucket == "" {
|
||||||
|
|||||||
@@ -37,32 +37,6 @@ type holding struct {
|
|||||||
shelf map[string]catalogue.Manifest
|
shelf map[string]catalogue.Manifest
|
||||||
// providers memoises providerFor by machine, consumer and provision.
|
// providers memoises providerFor by machine, consumer and provision.
|
||||||
providers map[string]providerLookup
|
providers map[string]providerLookup
|
||||||
// waits is what the waits of a statement are checked against, read as they are asked for (novox/hq issue 450).
|
|
||||||
waits operatorWaitFacts
|
|
||||||
}
|
|
||||||
|
|
||||||
// checked is a machine's newest statement as the controller judges it: each wait checked (ADR 0283 decision 3), so
|
|
||||||
// a wait that does not check out reads as unhealthy, as it did when the statement was judged (novox/hq issue 450).
|
|
||||||
// What is stored is what the machine said, the waits unchecked; read as stored, a provider whose wait failed its
|
|
||||||
// check seems to wait for the operator while its unhealthy condition is open.
|
|
||||||
func (h *holding) checked(machine string) []inventory.ResourceHealth {
|
|
||||||
rs := h.healths[machine].Resources
|
|
||||||
mods := waitingModules(rs)
|
|
||||||
if len(mods) == 0 {
|
|
||||||
return rs
|
|
||||||
}
|
|
||||||
if h.waits.manifests == nil {
|
|
||||||
h.waits.manifests = map[string]catalogue.Manifest{}
|
|
||||||
}
|
|
||||||
for _, module := range mods {
|
|
||||||
if _, has := h.waits.manifests[module]; !has {
|
|
||||||
if m, ok := h.manifestOf(module); ok {
|
|
||||||
h.waits.manifests[module] = m
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
readWaitFacts(h.ctx, h.inv, machine, mods, nil, &h.waits)
|
|
||||||
return checkWaiting(machine, rs, h.waits)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type providerLookup struct {
|
type providerLookup struct {
|
||||||
@@ -79,15 +53,6 @@ func readHolding(ctx context.Context, inv *inventory.Inventory, open []condition
|
|||||||
return &holding{ctx: ctx, inv: inv, healths: healths, open: open, providers: map[string]providerLookup{}}, nil
|
return &holding{ctx: ctx, inv: inv, healths: healths, open: open, providers: map[string]providerLookup{}}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// readHoldingFor is how a judging reads its holding: nothing without a store. A variable so a test can hand a
|
|
||||||
// judging the record it holds under (novox/hq issue 405).
|
|
||||||
var readHoldingFor = func(ctx context.Context, inv *inventory.Inventory, open []conditions.Condition) (*holding, error) {
|
|
||||||
if inv == nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
return readHolding(ctx, inv, open)
|
|
||||||
}
|
|
||||||
|
|
||||||
// heldFinding says a resource's state is a finding of its declared check that names a provision: what
|
// heldFinding says a resource's state is a finding of its declared check that names a provision: what
|
||||||
// may be held. Down and restarting are liveness, the resource's own.
|
// may be held. Down and restarting are liveness, the resource's own.
|
||||||
func heldFinding(r inventory.ResourceHealth) bool {
|
func heldFinding(r inventory.ResourceHealth) bool {
|
||||||
@@ -141,138 +106,43 @@ func (h *holding) lookUpProvider(machine, consumer, provision string) (catalogue
|
|||||||
return catalogue.Chosen{}, false
|
return catalogue.Chosen{}, false
|
||||||
}
|
}
|
||||||
|
|
||||||
// providerState is how a provider stands on the record: unhealthy when its unhealthy condition is open or its
|
// unhealthy says a provider is unhealthy on the record: its condition is open, or its machine's newest
|
||||||
// machine's newest statement says a resource of it is unhealthy; else waiting for the operator when that statement
|
// statement says a resource of it is unhealthy.
|
||||||
// says a resource of it waits, with the waits it names, or its needs-operator condition is open (waits then
|
func (h *holding) unhealthy(p catalogue.Chosen) bool {
|
||||||
// unknown); else healthy.
|
key := moduleUnhealthyKey(p.Module, p.Node)
|
||||||
func (h *holding) providerState(p catalogue.Chosen) (unhealthy, waiting bool, waits []inventory.Wait) {
|
|
||||||
for _, c := range h.open {
|
for _, c := range h.open {
|
||||||
if c.Key == moduleUnhealthyKey(p.Module, p.Node) {
|
if c.Key == key {
|
||||||
return true, false, nil
|
return true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, r := range h.checked(p.Node) {
|
for _, r := range h.healths[p.Node].Resources {
|
||||||
if r.Module != p.Module {
|
if r.Module == p.Module && r.State == link.StateUnhealthy {
|
||||||
continue
|
return true
|
||||||
}
|
|
||||||
switch r.State {
|
|
||||||
case link.StateUnhealthy:
|
|
||||||
return true, false, nil
|
|
||||||
case link.StateWaiting:
|
|
||||||
waiting = true
|
|
||||||
waits = append(waits, r.Waits...)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !waiting {
|
return false
|
||||||
for _, c := range h.open {
|
|
||||||
waiting = waiting || c.Key == needsOperatorKey(p.Module, p.Node)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false, waiting, waits
|
|
||||||
}
|
|
||||||
|
|
||||||
// manifestOf is a module's manifest from the catalogue, read once; false when it cannot be read.
|
|
||||||
func (h *holding) manifestOf(module string) (catalogue.Manifest, bool) {
|
|
||||||
if h.shelf == nil && h.inv != nil {
|
|
||||||
shelf, err := h.inv.Catalogue(h.ctx)
|
|
||||||
if err != nil {
|
|
||||||
return catalogue.Manifest{}, false
|
|
||||||
}
|
|
||||||
h.shelf = shelf
|
|
||||||
}
|
|
||||||
m, ok := h.shelf[module]
|
|
||||||
return m, ok
|
|
||||||
}
|
|
||||||
|
|
||||||
// covering is the waits of a provider that cover a provision it gives (novox/hq issue 405): a module that waits
|
|
||||||
// says nothing else of it is wrong and names each part that waits (ADR 0283 decision 1), so only a consumer of
|
|
||||||
// the waiting part waits on it. A wait covers a provision when its part is that provision, or the secret it waits
|
|
||||||
// for is the provision's shared credential (ADR 0158). Nothing when no wait can be matched: a consumer failing
|
|
||||||
// then is not held, since holding it would hide a fault that may be its own.
|
|
||||||
func (h *holding) covering(p catalogue.Chosen, provision string, waits []inventory.Wait) []inventory.Wait {
|
|
||||||
credential := ""
|
|
||||||
if m, ok := h.manifestOf(p.Module); ok {
|
|
||||||
credential, _ = m.SharedCredentialOf(provision)
|
|
||||||
}
|
|
||||||
var out []inventory.Wait
|
|
||||||
for _, w := range waits {
|
|
||||||
if w.Part == provision || (w.Secret != "" && w.Secret == credential) {
|
|
||||||
out = append(out, w)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// heldUnderProvider is the provider a consumer's findings are held under, and — when that provider only waits for the
|
|
||||||
// operator, for the part the consumer needs — the waits that hold it.
|
|
||||||
type heldUnderProvider struct {
|
|
||||||
provider catalogue.Chosen
|
|
||||||
// waits is set when every finding held waits on a provider that only waits for the operator: the consumer's
|
|
||||||
// gate then reads as ADR 0254's waits for a person, a pass with the wait carried (ADR 0283 decision 4).
|
|
||||||
waits []inventory.Wait
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// heldUnder is the provider a consumer's unhealthy resources wait on: when every one of them is a finding
|
// heldUnder is the provider a consumer's unhealthy resources wait on: when every one of them is a finding
|
||||||
// of a check naming a provision whose provider for this consumer is unhealthy on the record, or waits for the
|
// of a check naming a provision whose provider for this consumer is unhealthy on the record. False when any
|
||||||
// operator for the part that gives it (novox/hq issue 405). False when any is the consumer's own.
|
// is the consumer's own.
|
||||||
func (h *holding) heldUnder(machine, module string, rs []inventory.ResourceHealth) (catalogue.Chosen, bool) {
|
func (h *holding) heldUnder(machine, module string, rs []inventory.ResourceHealth) (catalogue.Chosen, bool) {
|
||||||
by, held := h.heldWith(machine, module, rs)
|
var on catalogue.Chosen
|
||||||
return by.provider, held
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *holding) heldWith(machine, module string, rs []inventory.ResourceHealth) (heldUnderProvider, bool) {
|
|
||||||
var on heldUnderProvider
|
|
||||||
onlyWaits := true
|
|
||||||
for _, r := range rs {
|
for _, r := range rs {
|
||||||
if r.State != link.StateUnhealthy {
|
if r.State != link.StateUnhealthy {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if !heldFinding(r) {
|
if !heldFinding(r) {
|
||||||
return heldUnderProvider{}, false
|
|
||||||
}
|
|
||||||
p, ok := h.providerFor(machine, module, r.Needs)
|
|
||||||
if !ok || (p.Node == machine && p.Module == module) {
|
|
||||||
return heldUnderProvider{}, false
|
|
||||||
}
|
|
||||||
unhealthy, waiting, waits := h.providerState(p)
|
|
||||||
switch {
|
|
||||||
case unhealthy:
|
|
||||||
onlyWaits = false
|
|
||||||
case waiting:
|
|
||||||
covered := h.covering(p, r.Needs, waits)
|
|
||||||
if len(covered) == 0 {
|
|
||||||
return heldUnderProvider{}, false
|
|
||||||
}
|
|
||||||
on.waits = append(on.waits, covered...)
|
|
||||||
default:
|
|
||||||
return heldUnderProvider{}, false
|
|
||||||
}
|
|
||||||
on.provider = p
|
|
||||||
}
|
|
||||||
if !onlyWaits {
|
|
||||||
on.waits = nil
|
|
||||||
}
|
|
||||||
return on, on.provider.Module != ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// waitingUncovered is a provider of a consumer's failing finding that waits for the operator for a part the
|
|
||||||
// finding cannot be matched to (novox/hq issue 405): the consumer is raised on its own, and its condition says
|
|
||||||
// the provider waits, so neither is hidden.
|
|
||||||
func (h *holding) waitingUncovered(machine, module string, rs []inventory.ResourceHealth) (catalogue.Chosen, bool) {
|
|
||||||
for _, r := range rs {
|
|
||||||
if r.State != link.StateUnhealthy || !heldFinding(r) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
p, ok := h.providerFor(machine, module, r.Needs)
|
|
||||||
if !ok || (p.Node == machine && p.Module == module) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if unhealthy, waiting, waits := h.providerState(p); !unhealthy && waiting && len(h.covering(p, r.Needs, waits)) == 0 {
|
|
||||||
return p, true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return catalogue.Chosen{}, false
|
return catalogue.Chosen{}, false
|
||||||
}
|
}
|
||||||
|
p, ok := h.providerFor(machine, module, r.Needs)
|
||||||
|
if !ok || (p.Node == machine && p.Module == module) || !h.unhealthy(p) {
|
||||||
|
return catalogue.Chosen{}, false
|
||||||
|
}
|
||||||
|
on = p
|
||||||
|
}
|
||||||
|
return on, on.Module != ""
|
||||||
|
}
|
||||||
|
|
||||||
// waitersOn is every consumer held under a provider, as "<module> on <machine>", sorted.
|
// waitersOn is every consumer held under a provider, as "<module> on <machine>", sorted.
|
||||||
func (h *holding) waitersOn(p catalogue.Chosen) []string {
|
func (h *holding) waitersOn(p catalogue.Chosen) []string {
|
||||||
@@ -295,8 +165,8 @@ func (h *holding) waitersOn(p catalogue.Chosen) []string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// heldModules is, for one machine's statement, each module whose finding is held, with the provider.
|
// heldModules is, for one machine's statement, each module whose finding is held, with the provider.
|
||||||
func (h *holding) heldModules(machine string) map[string]heldUnderProvider {
|
func (h *holding) heldModules(machine string) map[string]catalogue.Chosen {
|
||||||
out := map[string]heldUnderProvider{}
|
out := map[string]catalogue.Chosen{}
|
||||||
byModule := map[string][]inventory.ResourceHealth{}
|
byModule := map[string][]inventory.ResourceHealth{}
|
||||||
for _, r := range h.healths[machine].Resources {
|
for _, r := range h.healths[machine].Resources {
|
||||||
if r.Module != "" && r.State == link.StateUnhealthy {
|
if r.Module != "" && r.State == link.StateUnhealthy {
|
||||||
@@ -304,7 +174,7 @@ func (h *holding) heldModules(machine string) map[string]heldUnderProvider {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
for module, rs := range byModule {
|
for module, rs := range byModule {
|
||||||
if on, held := h.heldWith(machine, module, rs); held {
|
if on, held := h.heldUnder(machine, module, rs); held {
|
||||||
out[module] = on
|
out[module] = on
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+22
-119
@@ -249,7 +249,7 @@ func serve(ctx context.Context) (err error) {
|
|||||||
// (novox/hq to-be 45 §9).
|
// (novox/hq to-be 45 §9).
|
||||||
checkEvents = bus
|
checkEvents = bus
|
||||||
// And every walk this controller keeps in a new state, for the delivery it walks (novox/hq ADR 0239).
|
// And every walk this controller keeps in a new state, for the delivery it walks (novox/hq ADR 0239).
|
||||||
inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { sayPlanMoved(ctx, bus, open.inventory, p) }
|
inventory.PlanSaved = func(p inventory.Plan) { sayPlanMoved(ctx, bus, p) }
|
||||||
if err := server.Checks(following{open}); err != nil {
|
if err := server.Checks(following{open}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -357,15 +357,9 @@ func declare(ctx context.Context, args []string) error {
|
|||||||
// The epoch it carried, if a person wrote one in, is what the machine heard.
|
// The epoch it carried, if a person wrote one in, is what the machine heard.
|
||||||
var carried struct {
|
var carried struct {
|
||||||
Epoch uint64 `json:"epoch"`
|
Epoch uint64 `json:"epoch"`
|
||||||
Sequence int64 `json:"sequence"`
|
|
||||||
Generation int64 `json:"generation"`
|
|
||||||
}
|
}
|
||||||
_ = json.Unmarshal(raw, &carried)
|
_ = json.Unmarshal(raw, &carried)
|
||||||
// And recorded as every send is (novox/hq issue 234): by hand, from what it carried; the modules it
|
if _, err := recordSent(ctx, inv, node, raw, nil, carried.Epoch); err != nil {
|
||||||
// named are not known, since the mesh did not compose it.
|
|
||||||
if _, err := recordSent(ctx, inv, node, raw, nil, carried.Epoch, sentRecord{sequence: carried.Sequence,
|
|
||||||
epoch: carried.Epoch, generation: carried.Generation, toldGeneration: carried.Generation,
|
|
||||||
sender: senderOf(callerOf(ctx)) + ", a declaration sent by hand"}); err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw))
|
fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw))
|
||||||
@@ -804,7 +798,7 @@ func composeEach(names []string, allot func(node string) (order, error),
|
|||||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
numbered.stamp(&declared)
|
declared.Sequence, declared.Epoch = numbered.sequence, numbered.epoch
|
||||||
if len(declared.Resources) == 0 {
|
if len(declared.Resources) == 0 {
|
||||||
// Sent, not skipped (novox/hq issue 127). A node whose declaration composes to
|
// Sent, not skipped (novox/hq issue 127). A node whose declaration composes to
|
||||||
// nothing may have HELD something before — the broker opening a placement gave it,
|
// nothing may have HELD something before — the broker opening a placement gave it,
|
||||||
@@ -1022,8 +1016,7 @@ func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (stri
|
|||||||
}
|
}
|
||||||
// After it is away, not before. A digest recorded for something that failed to send would make
|
// After it is away, not before. A digest recorded for something that failed to send would make
|
||||||
// the machine look current for a declaration it never received.
|
// the machine look current for a declaration it never received.
|
||||||
digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds, s.declared.Epoch,
|
digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds, s.declared.Epoch)
|
||||||
sentRecordOf(ctx, s.declared))
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
@@ -1084,58 +1077,42 @@ func brokerFirst(names []string, holder string, behind bool) []string {
|
|||||||
// send anywhere, and a module running there always put it in its first wave. A digest of the list
|
// send anywhere, and a module running there always put it in its first wave. A digest of the list
|
||||||
// last sent is kept for this (ADR 0043: the list is composed on each push, never kept itself).
|
// last sent is kept for this (ADR 0043: the list is composed on each push, never kept itself).
|
||||||
func brokerBehind(ctx context.Context, open *stores, names []string) (string, bool, error) {
|
func brokerBehind(ctx context.Context, open *stores, names []string) (string, bool, error) {
|
||||||
s, err := brokerListState(ctx, open, names)
|
|
||||||
return s.Machine, s.Behind, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// brokerListState is brokerBehind with why the list could not be composed, when it could not: the plan's own
|
|
||||||
// reason, which a remedy names rather than a push that would fail on it (the review of hq issue 353).
|
|
||||||
func brokerListState(ctx context.Context, open *stores, names []string) (busListState, error) {
|
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
holders, err := seatHolders(ctx, inv)
|
holders, err := seatHolders(ctx, inv)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return busListState{}, err
|
return "", false, err
|
||||||
}
|
}
|
||||||
h, held := holders[theBrokerSeat]
|
h, held := holders[theBrokerSeat]
|
||||||
if !held || h.Node == "" {
|
if !held || h.Node == "" {
|
||||||
return busListState{}, nil
|
return "", false, nil
|
||||||
}
|
}
|
||||||
shelf, err := inv.Catalogue(ctx)
|
shelf, err := inv.Catalogue(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return busListState{}, err
|
return "", false, err
|
||||||
}
|
}
|
||||||
if m, known := shelf[h.Module]; !known || m.BusUsers == "" {
|
if m, known := shelf[h.Module]; !known || m.BusUsers == "" {
|
||||||
// A holder that is sent no user list carries no grant: nothing to send first.
|
// A holder that is sent no user list carries no grant: nothing to send first.
|
||||||
return busListState{}, nil
|
return "", false, nil
|
||||||
}
|
}
|
||||||
for _, n := range names {
|
for _, n := range names {
|
||||||
if n == h.Node {
|
if n == h.Node {
|
||||||
return busListState{Machine: h.Node}, nil
|
return h.Node, false, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// **What cannot be worked out is behind** (the review of hq issue 353). The list it would be sent cannot be
|
|
||||||
// composed, so whether it carries the grants a send relies on is not known; it is taken as behind, so a push
|
|
||||||
// sends that machine first and says why it cannot (`plan` says it too), and a rollout that needs the list
|
|
||||||
// refuses rather than sending code whose grants the bus may not hold. It said "not behind" until then: a
|
|
||||||
// send went ahead as if the bus held a list nobody could compose.
|
|
||||||
list, composeErr := "", error(nil)
|
|
||||||
plan, _, err := planFor(ctx, open, h.Node)
|
plan, _, err := planFor(ctx, open, h.Node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
composeErr = err
|
// It cannot be worked out: sending it would refuse the whole send, and `plan` says why.
|
||||||
} else if list, _, err = busUserList(ctx, inv, plan.Modules); err != nil {
|
return h.Node, false, nil
|
||||||
composeErr = err
|
}
|
||||||
|
list, _, err := busUserList(ctx, inv, plan.Modules)
|
||||||
|
if err != nil {
|
||||||
|
return h.Node, false, nil
|
||||||
}
|
}
|
||||||
sent, err := inv.SentBusUsers(ctx, h.Node)
|
sent, err := inv.SentBusUsers(ctx, h.Node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return busListState{}, err
|
return "", false, err
|
||||||
}
|
}
|
||||||
return busListState{Machine: h.Node, Behind: listBehind(list, composeErr, sent), Uncomposed: composeErr}, nil
|
return h.Node, userListBehind(list, sent), nil
|
||||||
}
|
|
||||||
|
|
||||||
// listBehind is whether the bus's machine is behind: the list composed now is not the one last sent, or it
|
|
||||||
// could not be composed at all — not known is behind, never "not behind".
|
|
||||||
func listBehind(list string, composeErr error, sentDigest string) bool {
|
|
||||||
return composeErr != nil || userListBehind(list, sentDigest)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// userListBehind is whether the user list composed now is not the one last sent, by its digest. An
|
// userListBehind is whether the user list composed now is not the one last sent, by its digest. An
|
||||||
@@ -1251,18 +1228,7 @@ func sendToEach(ctx context.Context, open *stores, names []string) ([]string, er
|
|||||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
numbered.stamp(&declared)
|
declared.Sequence, declared.Epoch = numbered.sequence, numbered.epoch
|
||||||
// **The grants step sends the user list and nothing else** (novox/hq issue 490): judged on this very
|
|
||||||
// declaration, the one sent, so nothing composed between a check and the send can slip through.
|
|
||||||
if everyBuildKept(ctx, name) {
|
|
||||||
other, err := grantsOnlyIn(ctx, open, name, plan, declared)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if other != "" {
|
|
||||||
return nil, fmt.Errorf("%w: %s", errNotGrantsOnly, other)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
reportLeftOut(name, declared)
|
reportLeftOut(name, declared)
|
||||||
sending = append(sending, readyNode{name, declared})
|
sending = append(sending, readyNode{name, declared})
|
||||||
}
|
}
|
||||||
@@ -1327,15 +1293,6 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, se
|
|||||||
if _, err := broker.RaiseBuckets(broker.OnConn(bus.Conn), buckets); err != nil {
|
if _, err := broker.RaiseBuckets(broker.OnConn(bus.Conn), buckets); err != nil {
|
||||||
return fmt.Errorf("the modules' state could not be asserted on the bus: %w", err)
|
return fmt.Errorf("the modules' state could not be asserted on the bus: %w", err)
|
||||||
}
|
}
|
||||||
// **And every declared log, for the same reason** (novox/hq ADR 0297 §2): a membership names its
|
|
||||||
// logs, and a module whose log does not exist fails its first append.
|
|
||||||
logs, err := open.inventory.DeclaredLogs(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("the modules' logs could not be read, so no log was asserted: %w", err)
|
|
||||||
}
|
|
||||||
if _, err := broker.RaiseLogs(broker.OnConn(bus.Conn), logs); err != nil {
|
|
||||||
return fmt.Errorf("the modules' logs could not be asserted on the bus: %w", err)
|
|
||||||
}
|
|
||||||
// Every membership is tried, and the first failure named once.
|
// Every membership is tried, and the first failure named once.
|
||||||
issued := 0
|
issued := 0
|
||||||
refused := map[string]error{}
|
refused := map[string]error{}
|
||||||
@@ -1436,11 +1393,6 @@ func wouldSendFrom(ctx context.Context, open *stores,
|
|||||||
if declared.Epoch, err = open.inventory.SentEpoch(ctx, n.ID); err != nil {
|
if declared.Epoch, err = open.inventory.SentEpoch(ctx, n.ID); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
// And the generation it was last sent, for the same reason (novox/hq issue 234): an assignment
|
|
||||||
// elsewhere in the mesh is not a change of this machine.
|
|
||||||
if declared.Generation, err = open.inventory.SentGeneration(ctx, n.ID); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
body, err := declared.Body()
|
body, err := declared.Body()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -1519,28 +1471,13 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string)
|
|||||||
fmt.Printf("the bus holds state nothing declares any more, kept because it is data: %s — "+
|
fmt.Printf("the bus holds state nothing declares any more, kept because it is data: %s — "+
|
||||||
"removing it is a person's act\n", strings.Join(undeclared, ", "))
|
"removing it is a person's act\n", strings.Join(undeclared, ", "))
|
||||||
}
|
}
|
||||||
// Every module's log (novox/hq ADR 0297), from the catalogue, as its buckets: one that nothing
|
|
||||||
// declares any more is said and kept — a log is a module's record, and no path of the mesh removes it.
|
|
||||||
logs, err := inv.DeclaredLogs(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
unlogged, err := broker.RaiseLogs(js, logs)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if len(unlogged) > 0 {
|
|
||||||
fmt.Printf("the bus holds logs nothing declares any more, kept because they are data: %s — "+
|
|
||||||
"removing one is a person's act\n", strings.Join(unlogged, ", "))
|
|
||||||
}
|
|
||||||
// And how every module hears what it consumes: asserted with the rest above, counted here.
|
// And how every module hears what it consumes: asserted with the rest above, counted here.
|
||||||
hearing, err := moduleConsumerCount(ctx, inv)
|
hearing, err := moduleConsumerCount(ctx, inv)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
fmt.Printf("the bus at %s has its streams, %d machine(s) can hear a declaration, %d module(s) "+
|
fmt.Printf("the bus at %s has its streams, %d machine(s) can hear a declaration, %d module(s) "+
|
||||||
"can hear what they consume, %d bucket(s) of state and %d log(s)\n", broker.BareAddress(address), len(names), hearing,
|
"can hear what they consume, and %d bucket(s) of state\n", broker.BareAddress(address), len(names), hearing, len(buckets))
|
||||||
len(buckets), len(logs))
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1594,12 +1531,6 @@ var epochForActs = func(ctx context.Context) (uint64, error) { return theLease.e
|
|||||||
type order struct {
|
type order struct {
|
||||||
sequence int64
|
sequence int64
|
||||||
epoch uint64
|
epoch uint64
|
||||||
// generation is the assignment generation read before the composition, and readsGeneration whether the
|
|
||||||
// machine's node-engine said it reads one (novox/hq issue 234); acting is the lease epoch the sender acts
|
|
||||||
// under, whether or not the machine is sent it — both kept for the record of the send.
|
|
||||||
generation int64
|
|
||||||
readsGeneration bool
|
|
||||||
acting uint64
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// allot takes the next sequence for a machine — the number its next declaration carries — under the
|
// allot takes the next sequence for a machine — the number its next declaration carries — under the
|
||||||
@@ -1613,7 +1544,6 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, e
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return order{}, err
|
return order{}, err
|
||||||
}
|
}
|
||||||
acting := epoch
|
|
||||||
if epoch > 0 {
|
if epoch > 0 {
|
||||||
reads, err := inv.ReadsEpoch(ctx, record.ID)
|
reads, err := inv.ReadsEpoch(ctx, record.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1623,24 +1553,11 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, e
|
|||||||
epoch = 0
|
epoch = 0
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// **The generation is read here, before the composition reads a single assignment** (novox/hq issue
|
|
||||||
// 234). A generation only grows, so one read before is never newer than the view composed after it:
|
|
||||||
// the declaration may claim a generation older than its content, never newer — and a claim newer than
|
|
||||||
// the content is exactly the stale send the machine must be able to refuse.
|
|
||||||
generation, err := inv.AssignmentGeneration(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return order{}, err
|
|
||||||
}
|
|
||||||
readsGeneration, err := inv.ReadsGeneration(ctx, record.ID)
|
|
||||||
if err != nil {
|
|
||||||
return order{}, err
|
|
||||||
}
|
|
||||||
seq, err := inv.NextSequence(ctx, record.ID)
|
seq, err := inv.NextSequence(ctx, record.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return order{}, err
|
return order{}, err
|
||||||
}
|
}
|
||||||
return order{sequence: seq, epoch: epoch, generation: generation, readsGeneration: readsGeneration,
|
return order{sequence: seq, epoch: epoch}, nil
|
||||||
acting: acting}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// recordSent writes down what a machine was just sent, and returns the digest.
|
// recordSent writes down what a machine was just sent, and returns the digest.
|
||||||
@@ -1655,7 +1572,7 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, e
|
|||||||
// And the build of each module it carried (novox/hq issue 259, ADR 0221), nil when that is not known:
|
// And the build of each module it carried (novox/hq issue 259, ADR 0221), nil when that is not known:
|
||||||
// what tells a machine held back by a policy or a plan from one a push left behind.
|
// what tells a machine held back by a policy or a plan from one a push left behind.
|
||||||
func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte,
|
func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte,
|
||||||
builds map[string]string, epoch uint64, sent sentRecord) (string, error) {
|
builds map[string]string, epoch uint64) (string, error) {
|
||||||
kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
record, err := inv.NodeByName(kept, node)
|
record, err := inv.NodeByName(kept, node)
|
||||||
@@ -1663,21 +1580,7 @@ func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body
|
|||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
digest := digestOf(body)
|
digest := digestOf(body)
|
||||||
// The digest and the generation it carried are written in one transaction (novox/hq issue 234), so the
|
if err := inv.RecordSentUnder(kept, record.ID, digest, builds, epoch); err != nil {
|
||||||
// would-send is never stamped with a generation the machine was not sent; and the send itself, who sent
|
|
||||||
// it and from which generation, beside them. A record of the send that cannot be written does not make a
|
|
||||||
// send that is away look failed: it is said, loudly, and the machine's own record stands.
|
|
||||||
err = inv.RecordSentWith(kept, record.ID, digest, builds, epoch, sent.toldGeneration, inventory.Send{
|
|
||||||
Sequence: sent.sequence, Epoch: int64(sent.epoch), Sender: sent.sender, Generation: sent.generation,
|
|
||||||
Digest: digest, Modules: sent.modules})
|
|
||||||
var unrecorded *inventory.SendNotRecordedError
|
|
||||||
switch {
|
|
||||||
case errors.As(err, &unrecorded):
|
|
||||||
fmt.Fprintf(os.Stderr, "mesh-controller: SEND NOT RECORDED: %s was sent declaration %s (sequence %d), and who "+
|
|
||||||
"sent it and from which generation could not be written down, so a refusal of it will name no sender "+
|
|
||||||
"(novox/hq issue 234): %v\n", node, short(digest), sent.sequence, unrecorded.Err)
|
|
||||||
fmt.Printf("%s: sent, and the record of who sent it could NOT be written: %v\n", node, unrecorded.Err)
|
|
||||||
case err != nil:
|
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
// And what it was, summarised, so the next push can be compared with it (novox/hq ADR 0217).
|
// And what it was, summarised, so the next push can be compared with it (novox/hq ADR 0217).
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"regexp"
|
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
@@ -68,95 +67,18 @@ func listQueue(ctx context.Context, args []string, w io.Writer) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
now := time.Now()
|
|
||||||
if *asJSON {
|
if *asJSON {
|
||||||
body, err := json.MarshalIndent(queueAnswerOf(q, now), "", " ")
|
body, err := json.MarshalIndent(q, "", " ")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
fmt.Fprintln(w, string(body))
|
fmt.Fprintln(w, string(body))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
fmt.Fprint(w, queueText(q, now))
|
fmt.Fprint(w, queueText(q, time.Now()))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// queueAnswer is the queue as data (novox/hq issue 455): what a caller reads to say how far back an ask
|
|
||||||
// waits — "waiting in the build queue behind 2" — which the table alone could not tell it. Text is the
|
|
||||||
// table, written from the same read: the verb answers it as its output and the rest as its answer, so the
|
|
||||||
// two never disagree.
|
|
||||||
type queueAnswer struct {
|
|
||||||
Seat string `json:"seat"`
|
|
||||||
Waiting int `json:"waiting"`
|
|
||||||
InFlight int `json:"in-flight"`
|
|
||||||
Dead int `json:"dead"`
|
|
||||||
MaxDeliver int `json:"max-deliver"`
|
|
||||||
Asks []queuedAnswer `json:"asks"`
|
|
||||||
Text string `json:"text"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// queuedAnswer is one ask as data. Never what the request carries beyond its source — not `held`, every
|
|
||||||
// artifact the mesh has built, nor a check's checkouts beside it.
|
|
||||||
type queuedAnswer struct {
|
|
||||||
ID string `json:"id"`
|
|
||||||
State string `json:"state"`
|
|
||||||
// Kind is "check" for a pull request's merge check, "build" for anything else.
|
|
||||||
Kind string `json:"kind"`
|
|
||||||
Repository string `json:"repository"`
|
|
||||||
Path string `json:"path,omitempty"`
|
|
||||||
Ref string `json:"ref,omitempty"`
|
|
||||||
// Commit is the ref when it names a commit; empty when the ask follows a branch or the default.
|
|
||||||
Commit string `json:"commit,omitempty"`
|
|
||||||
// Machine is the one building an ask in flight; WasOn the one that started it and handed it back.
|
|
||||||
Machine string `json:"machine,omitempty"`
|
|
||||||
WasOn string `json:"was-on,omitempty"`
|
|
||||||
AskedAt time.Time `json:"asked-at,omitzero"`
|
|
||||||
// Since is when the ask came to its state: its start on the machine for one in flight that a
|
|
||||||
// machine said it started, otherwise when it was asked.
|
|
||||||
Since time.Time `json:"since,omitzero"`
|
|
||||||
// Position is its place among the waiting, 1 being handed out next; 0 for an ask not waiting.
|
|
||||||
Position int `json:"position,omitempty"`
|
|
||||||
Seq uint64 `json:"seq"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// aCommitRef is a ref that names a commit by its full hash (SHA-1 or SHA-256).
|
|
||||||
var aCommitRef = regexp.MustCompile(`^(?:[0-9a-f]{40}|[0-9a-f]{64})$`)
|
|
||||||
|
|
||||||
// queueAnswerOf is the queue as data, in the table's order: in flight, the waiting from the next one to
|
|
||||||
// be handed out, then the dead. A pure function of one read (novox/hq issue 455).
|
|
||||||
func queueAnswerOf(q link.Queue, now time.Time) queueAnswer {
|
|
||||||
waiting, running, dead := q.Of(link.AskWaiting), q.Of(link.AskInFlight), q.Of(link.AskDead)
|
|
||||||
out := queueAnswer{Seat: q.Seat, Waiting: len(waiting), InFlight: len(running), Dead: len(dead),
|
|
||||||
MaxDeliver: q.MaxDeliver, Asks: []queuedAnswer{}, Text: queueText(q, now)}
|
|
||||||
one := func(a link.QueuedAsk, position int) queuedAnswer {
|
|
||||||
kind := "build"
|
|
||||||
if a.Request.Check != nil {
|
|
||||||
kind = "check"
|
|
||||||
}
|
|
||||||
since := a.AskedAt
|
|
||||||
if a.State == link.AskInFlight && a.On != "" && !a.Started.IsZero() {
|
|
||||||
since = a.Started
|
|
||||||
}
|
|
||||||
commit := ""
|
|
||||||
if aCommitRef.MatchString(a.Ref) {
|
|
||||||
commit = a.Ref
|
|
||||||
}
|
|
||||||
return queuedAnswer{ID: a.ID, State: a.State, Kind: kind, Repository: link.WithoutPassword(a.Repository), Path: a.Path,
|
|
||||||
Ref: a.Ref, Commit: commit, Machine: a.On, WasOn: a.Was, AskedAt: a.AskedAt, Since: since,
|
|
||||||
Position: position, Seq: a.Seq}
|
|
||||||
}
|
|
||||||
for _, a := range running {
|
|
||||||
out.Asks = append(out.Asks, one(a, 0))
|
|
||||||
}
|
|
||||||
for i, a := range waiting {
|
|
||||||
out.Asks = append(out.Asks, one(a, i+1))
|
|
||||||
}
|
|
||||||
for _, a := range dead {
|
|
||||||
out.Asks = append(out.Asks, one(a, 0))
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// queueText is the queue as a person reads it: a summary line, then each kind in queue order.
|
// queueText is the queue as a person reads it: a summary line, then each kind in queue order.
|
||||||
// Never what an ask carries as `held` — that is every artifact the mesh has built.
|
// Never what an ask carries as `held` — that is every artifact the mesh has built.
|
||||||
func queueText(q link.Queue, now time.Time) string {
|
func queueText(q link.Queue, now time.Time) string {
|
||||||
@@ -170,7 +92,7 @@ func queueText(q link.Queue, now time.Time) string {
|
|||||||
return "asked " + now.Sub(t).Round(time.Second).String() + " ago"
|
return "asked " + now.Sub(t).Round(time.Second).String() + " ago"
|
||||||
}
|
}
|
||||||
what := func(a link.QueuedAsk) string {
|
what := func(a link.QueuedAsk) string {
|
||||||
s := link.WithoutPassword(a.Repository)
|
s := a.Repository
|
||||||
if a.Path != "" {
|
if a.Path != "" {
|
||||||
s += " at " + a.Path
|
s += " at " + a.Path
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,137 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"reflect"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The queue answers as data as well as text (novox/hq issue 455): each ask's repository, commit, kind,
|
|
||||||
// machine, since and position among the waiting, read from the same queue the table is written from.
|
|
||||||
|
|
||||||
const aCommit = "0123456789abcdef0123456789abcdef01234567"
|
|
||||||
|
|
||||||
func aQueueOfEachKind(now time.Time) link.Queue {
|
|
||||||
asked := now.Add(-5 * time.Minute)
|
|
||||||
started := now.Add(-2 * time.Minute)
|
|
||||||
return link.Queue{Seat: link.TheBuildMachine, Delivered: 2, AckPending: 1, MaxDeliver: 5, Asks: []link.QueuedAsk{
|
|
||||||
{Seq: 1, ID: "build-dead", Repository: "ssh://forge/novox/x", AskedAt: asked, State: link.AskDead},
|
|
||||||
{Seq: 2, ID: "build-running", Repository: "ssh://forge/novox/mesh-host", Ref: aCommit, AskedAt: asked,
|
|
||||||
State: link.AskInFlight, On: "ace", Started: started,
|
|
||||||
Request: link.BuildRequest{ID: "build-running", Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-host"}}},
|
|
||||||
{Seq: 3, ID: "build-first", Repository: "ssh://forge/novox/mesh-catalog", Ref: "main", AskedAt: asked,
|
|
||||||
State: link.AskWaiting},
|
|
||||||
{Seq: 4, ID: "build-second", Repository: "ssh://forge/novox/mesh-catalog", Path: "modules/gitea", Ref: aCommit,
|
|
||||||
AskedAt: now.Add(-time.Minute), State: link.AskWaiting,
|
|
||||||
Request: link.BuildRequest{ID: "build-second", Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog"}}},
|
|
||||||
}}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTheQueueVerbAnswersEachAskAsData(t *testing.T) {
|
|
||||||
now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC)
|
|
||||||
q := aQueueOfEachKind(now)
|
|
||||||
got := queueAnswerOf(q, now)
|
|
||||||
if got.Waiting != 2 || got.InFlight != 1 || got.Dead != 1 || got.Seat != link.TheBuildMachine {
|
|
||||||
t.Fatalf("counts: %+v", got)
|
|
||||||
}
|
|
||||||
if got.Text != queueText(q, now) {
|
|
||||||
t.Fatalf("the data's text is not the table read from the same queue:\n%s", got.Text)
|
|
||||||
}
|
|
||||||
byID := map[string]queuedAnswer{}
|
|
||||||
for _, a := range got.Asks {
|
|
||||||
byID[a.ID] = a
|
|
||||||
}
|
|
||||||
running := byID["build-running"]
|
|
||||||
if running.Kind != "check" || running.Machine != "ace" || running.Commit != aCommit || running.Position != 0 ||
|
|
||||||
!running.Since.Equal(now.Add(-2*time.Minute)) || running.State != link.AskInFlight {
|
|
||||||
t.Errorf("in flight: %+v", running)
|
|
||||||
}
|
|
||||||
first, second := byID["build-first"], byID["build-second"]
|
|
||||||
if first.Position != 1 || first.Kind != "build" || first.Commit != "" || first.Ref != "main" || first.Machine != "" ||
|
|
||||||
!first.Since.Equal(now.Add(-5*time.Minute)) {
|
|
||||||
t.Errorf("first waiting: %+v", first)
|
|
||||||
}
|
|
||||||
if second.Position != 2 || second.Kind != "check" || second.Commit != aCommit || second.Path != "modules/gitea" ||
|
|
||||||
second.Repository != "ssh://forge/novox/mesh-catalog" {
|
|
||||||
t.Errorf("second waiting: %+v", second)
|
|
||||||
}
|
|
||||||
if dead := byID["build-dead"]; dead.Position != 0 || dead.State != link.AskDead {
|
|
||||||
t.Errorf("dead: %+v", dead)
|
|
||||||
}
|
|
||||||
// In the table's order: in flight, then the waiting by position, then the dead.
|
|
||||||
var order []string
|
|
||||||
for _, a := range got.Asks {
|
|
||||||
order = append(order, a.ID)
|
|
||||||
}
|
|
||||||
if want := []string{"build-running", "build-first", "build-second", "build-dead"}; !reflect.DeepEqual(order, want) {
|
|
||||||
t.Errorf("order %v, want %v", order, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The verb asks its command for JSON, and its answer carries the data while its output stays the table a
|
|
||||||
// person reads, so a text caller reads what it always did.
|
|
||||||
func TestTheQueueVerbKeepsTheTableAndCarriesTheData(t *testing.T) {
|
|
||||||
argv, err := argvFor("queue", nil)
|
|
||||||
if err != nil || !reflect.DeepEqual(argv, []string{"queue", "--json"}) {
|
|
||||||
t.Fatalf("queue's command line: %v %v", argv, err)
|
|
||||||
}
|
|
||||||
now := time.Now()
|
|
||||||
q := aQueueOfEachKind(now)
|
|
||||||
body, err := json.Marshal(queueAnswerOf(q, now))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
answer := answerOf(argv, body, "", true)
|
|
||||||
if answer.Output != queueText(q, now) {
|
|
||||||
t.Fatalf("the output is not the table:\n%s", answer.Output)
|
|
||||||
}
|
|
||||||
data, ok := answer.Answer.(map[string]any)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("no data: %#v", answer.Answer)
|
|
||||||
}
|
|
||||||
if _, doubled := data["text"]; doubled {
|
|
||||||
t.Error("the table is said twice: once as the output and again inside the data")
|
|
||||||
}
|
|
||||||
asks, _ := data["asks"].([]any)
|
|
||||||
if len(asks) != 4 || data["waiting"] != float64(2) {
|
|
||||||
t.Fatalf("data: %v", data)
|
|
||||||
}
|
|
||||||
second := asks[2].(map[string]any)
|
|
||||||
if second["position"] != float64(2) || second["kind"] != "check" || second["commit"] != aCommit {
|
|
||||||
t.Errorf("second waiting: %v", second)
|
|
||||||
}
|
|
||||||
for _, a := range asks {
|
|
||||||
if _, held := a.(map[string]any)["held"]; held {
|
|
||||||
t.Error("an ask carries what the mesh holds")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// A failure is said as text, as before.
|
|
||||||
failed := answerOf(argv, nil, "mesh-controller: the bus cannot be reached\n", false)
|
|
||||||
if failed.Answer != nil || !strings.Contains(failed.Output, "cannot be reached") {
|
|
||||||
t.Errorf("a failure: %+v", failed)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A repository named with a password never shows it, in the table or the data.
|
|
||||||
func TestTheQueueNeverShowsARepositorysPassword(t *testing.T) {
|
|
||||||
if got := link.WithoutPassword("https://agent:s3cret@forge.example/novox/x.git"); strings.Contains(got, "s3cret") ||
|
|
||||||
!strings.Contains(got, "agent@forge.example/novox/x.git") {
|
|
||||||
t.Errorf("redacted: %s", got)
|
|
||||||
}
|
|
||||||
now := time.Now()
|
|
||||||
q := link.Queue{Seat: link.TheBuildMachine, Asks: []link.QueuedAsk{{Seq: 1, ID: "build-1",
|
|
||||||
Repository: "https://agent:s3cret@forge.example/novox/x.git", State: link.AskWaiting}}}
|
|
||||||
body, _ := json.Marshal(queueAnswerOf(q, now))
|
|
||||||
if strings.Contains(string(body), "s3cret") {
|
|
||||||
t.Errorf("the queue said a password: %s", body)
|
|
||||||
}
|
|
||||||
for _, kept := range []string{"ssh://git@forge:222/novox/x.git", "novox/x", "git@forge:novox/x.git", ""} {
|
|
||||||
if got := link.WithoutPassword(kept); got != kept {
|
|
||||||
t.Errorf("%q became %q", kept, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -326,7 +326,7 @@ func TestTheQueueVerbsRunTheirCommands(t *testing.T) {
|
|||||||
args map[string]any
|
args map[string]any
|
||||||
want []string
|
want []string
|
||||||
}{
|
}{
|
||||||
{"queue", nil, []string{"queue", "--json"}},
|
{"queue", nil, []string{"queue"}},
|
||||||
{"cancel", map[string]any{"id": "build-1"}, []string{"cancel", "build-1"}},
|
{"cancel", map[string]any{"id": "build-1"}, []string{"cancel", "build-1"}},
|
||||||
{"clear", nil, []string{"clear"}},
|
{"clear", nil, []string{"clear"}},
|
||||||
{"clear", map[string]any{"dead": "true"}, []string{"clear", "--dead"}},
|
{"clear", map[string]any{"dead": "true"}, []string{"clear", "--dead"}},
|
||||||
|
|||||||
@@ -69,9 +69,6 @@ type meshStatus struct {
|
|||||||
// **A document without this said an outage was a well mesh.** Read from what each machine
|
// **A document without this said an outage was a well mesh.** Read from what each machine
|
||||||
// reported, so it is the machine's account and not the mesh's take-time listing.
|
// reported, so it is the machine's account and not the mesh's take-time listing.
|
||||||
Untaken []machineUntaken `json:"untaken,omitempty"`
|
Untaken []machineUntaken `json:"untaken,omitempty"`
|
||||||
// NotApplied is every module assigned to a machine and left out of its composition, with why (novox/hq
|
|
||||||
// issue 380). Absent when every module composes.
|
|
||||||
NotApplied []machineNotApplied `json:"not-applied,omitempty"`
|
|
||||||
// Filtered is every converged machine that is not filtered by the mesh alone (novox/hq ADR
|
// Filtered is every converged machine that is not filtered by the mesh alone (novox/hq ADR
|
||||||
// 0168), one entry per rule set the mesh did not write — the found firewall in force again,
|
// 0168), one entry per rule set the mesh did not write — the found firewall in force again,
|
||||||
// or a chain nobody speaks for. Absent when every converged machine is filtered by the mesh
|
// or a chain nobody speaks for. Absent when every converged machine is filtered by the mesh
|
||||||
@@ -109,33 +106,6 @@ type meshStatus struct {
|
|||||||
// why they could not be read.
|
// why they could not be read.
|
||||||
Pending []pendingStatus `json:"pending,omitempty"`
|
Pending []pendingStatus `json:"pending,omitempty"`
|
||||||
PendingUnread string `json:"pendingUnread,omitempty"`
|
PendingUnread string `json:"pendingUnread,omitempty"`
|
||||||
// LastSent is every machine's last recorded send — who sent it, under which lease epoch, from which
|
|
||||||
// assignment generation, naming which modules — and what the machine answered of it (novox/hq issue 485).
|
|
||||||
// A machine the mesh has recorded no send to is absent. LastSentUnread says why it could not be read.
|
|
||||||
LastSent []machineLastSent `json:"lastSent"`
|
|
||||||
LastSentUnread string `json:"lastSentUnread,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// machineLastSent is one machine's last send as status says it.
|
|
||||||
type machineLastSent struct {
|
|
||||||
Node string `json:"node"`
|
|
||||||
Sequence int64 `json:"sequence"`
|
|
||||||
SentAt time.Time `json:"sentAt"`
|
|
||||||
Sender string `json:"sender"`
|
|
||||||
Epoch int64 `json:"epoch"`
|
|
||||||
Generation int64 `json:"generation"`
|
|
||||||
Digest string `json:"digest"`
|
|
||||||
Modules []string `json:"modules"`
|
|
||||||
// Answer is what the machine said of it: applied, failed or refused, as its report has it; empty when it
|
|
||||||
// has not reported on it. Refused is its words when it refused it whole, Failed how many resources failed.
|
|
||||||
Answer string `json:"answer"`
|
|
||||||
Refused string `json:"refused,omitempty"`
|
|
||||||
Failed int `json:"failed,omitempty"`
|
|
||||||
AnsweredAt *time.Time `json:"answeredAt,omitempty"`
|
|
||||||
// RefusedForGeneration is when the machine refused it for coming from an older assignment generation than
|
|
||||||
// it applied, and AppliedGeneration that generation; absent when it did not.
|
|
||||||
RefusedForGeneration *time.Time `json:"refusedForGeneration,omitempty"`
|
|
||||||
AppliedGeneration int64 `json:"appliedGeneration,omitempty"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// pendingStatus is one pending assignment as status says it.
|
// pendingStatus is one pending assignment as status says it.
|
||||||
@@ -281,7 +251,6 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
out.Unheld = asked.unheld
|
out.Unheld = asked.unheld
|
||||||
out.NotApplied = notApplied(asked.leftOut)
|
|
||||||
out.HandActsThisWeek, out.HandActsUnread = asked.handActs, asked.handActsUnread
|
out.HandActsThisWeek, out.HandActsUnread = asked.handActs, asked.handActsUnread
|
||||||
out.HealsThisWeek, out.HealsUnread = asked.heals, asked.healsUnread
|
out.HealsThisWeek, out.HealsUnread = asked.heals, asked.healsUnread
|
||||||
// In brief, as `conditions` lists them: status leads with every open condition, and their whole
|
// In brief, as `conditions` lists them: status leads with every open condition, and their whole
|
||||||
@@ -294,7 +263,6 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
|||||||
out.Pending = append(out.Pending, pendingStatus{Node: p.Node, Module: p.Module, State: p.State,
|
out.Pending = append(out.Pending, pendingStatus{Node: p.Node, Module: p.Module, State: p.State,
|
||||||
Build: p.Build, Repository: p.Repository, Path: p.Path, Since: p.Since, Settled: p.Settled, Note: p.Note})
|
Build: p.Build, Repository: p.Repository, Path: p.Path, Since: p.Since, Settled: p.Settled, Note: p.Note})
|
||||||
}
|
}
|
||||||
out.LastSent, out.LastSentUnread = lastSentStatus(asked.lastSent), asked.lastSentUnread
|
|
||||||
for name := range asked.refused {
|
for name := range asked.refused {
|
||||||
out.Unresolved = append(out.Unresolved, machineUnresolved{
|
out.Unresolved = append(out.Unresolved, machineUnresolved{
|
||||||
Node: name, Problem: asked.refused[name]})
|
Node: name, Problem: asked.refused[name]})
|
||||||
@@ -351,20 +319,3 @@ func say(value any) error {
|
|||||||
fmt.Println(string(body))
|
fmt.Println(string(body))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// lastSentStatus is every machine's last send as status --json says it, by machine name (novox/hq issue 485).
|
|
||||||
func lastSentStatus(sends map[string]inventory.Send) []machineLastSent {
|
|
||||||
out := []machineLastSent{}
|
|
||||||
for _, s := range sends {
|
|
||||||
modules := s.Modules
|
|
||||||
if modules == nil {
|
|
||||||
modules = []string{}
|
|
||||||
}
|
|
||||||
out = append(out, machineLastSent{Node: s.NodeName, Sequence: s.Sequence, SentAt: s.SentAt, Sender: s.Sender,
|
|
||||||
Epoch: s.Epoch, Generation: s.Generation, Digest: s.Digest, Modules: modules, Answer: s.Answer.Outcome,
|
|
||||||
Refused: s.Answer.Refused, Failed: s.Answer.Failed, AnsweredAt: s.Answer.At,
|
|
||||||
RefusedForGeneration: s.RefusedAt, AppliedGeneration: s.RefusedApplied})
|
|
||||||
}
|
|
||||||
sort.Slice(out, func(i, j int) bool { return out[i].Node < out[j].Node })
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -80,7 +80,7 @@ func TestAVerbsOwnProcessNamesItsConnection(t *testing.T) {
|
|||||||
// A silence acts, so it is still its own process; it dials, and finds no such condition.
|
// A silence acts, so it is still its own process; it dials, and finds no such condition.
|
||||||
_, _ = handlers["conditions"](context.Background(),
|
_, _ = handlers["conditions"](context.Background(),
|
||||||
json.RawMessage(`{"silence":"bus.nothing.here","for":"1h","why":"a test"}`))
|
json.RawMessage(`{"silence":"bus.nothing.here","for":"1h","why":"a test"}`))
|
||||||
names := closedNames(t, bus, "mesh-controller verb conditions")
|
names := closedNames(t, bus)
|
||||||
found := false
|
found := false
|
||||||
for _, n := range names {
|
for _, n := range names {
|
||||||
found = found || n == "mesh-controller verb conditions"
|
found = found || n == "mesh-controller verb conditions"
|
||||||
|
|||||||
@@ -61,57 +61,20 @@ func keptExcept(ctx context.Context) (map[string]bool, bool) {
|
|||||||
return skip, on
|
return skip, on
|
||||||
}
|
}
|
||||||
|
|
||||||
type keepEveryBuildKey struct{}
|
|
||||||
|
|
||||||
// keepingEveryBuild is a context whose sends compose every module of one machine — recorded or rolling out —
|
|
||||||
// at the build that machine was last sent (novox/hq issue 490): the grants step, which sends the machine
|
|
||||||
// holding the bus its new user list and nothing of any module's new code. That code waits there for a gate
|
|
||||||
// like any other move; the list must not, or the first machine's gate is judged against a bus that refuses
|
|
||||||
// what the change newly grants.
|
|
||||||
//
|
|
||||||
// **That machine alone.** Composing one machine resolves the others too — who is on the private network,
|
|
||||||
// who answers a requirement — on the same context, and those are no part of the step's send: they are
|
|
||||||
// composed as any send composes them. Kept for every machine, a machine whose last send is not known
|
|
||||||
// refused the bus's machine's composition.
|
|
||||||
func keepingEveryBuild(ctx context.Context, node string) context.Context {
|
|
||||||
return context.WithValue(ctx, keepEveryBuildKey{}, node)
|
|
||||||
}
|
|
||||||
|
|
||||||
// everyBuildKept is whether this context keeps every module of this machine at the build it runs.
|
|
||||||
func everyBuildKept(ctx context.Context, node string) bool {
|
|
||||||
on, _ := ctx.Value(keepEveryBuildKey{}).(string)
|
|
||||||
return on != "" && on == node
|
|
||||||
}
|
|
||||||
|
|
||||||
// recordedKept is, for a send under keepingRecorded, every recorded module the machine was last sent a
|
// recordedKept is, for a send under keepingRecorded, every recorded module the machine was last sent a
|
||||||
// build of that the mesh's build is not identical to: module → the commit it keeps. Nil when the context
|
// build of that the mesh's build is not identical to: module → the commit it keeps. Nil when the context
|
||||||
// keeps nothing, or when what the machine was last sent is not known (it is then held whole elsewhere —
|
// keeps nothing, or when what the machine was last sent is not known (it is then held whole elsewhere —
|
||||||
// ADR 0221).
|
// ADR 0221).
|
||||||
//
|
|
||||||
// Under keepingEveryBuild, every module the machine was sent is kept, whatever its policy (novox/hq issue
|
|
||||||
// 490), and a machine whose last send is not known refuses the send: there is nothing to keep it at, and
|
|
||||||
// composing the mesh's builds would be the very move the grants step must not make.
|
|
||||||
func recordedKept(ctx context.Context, open *stores, node string) (map[string]string, error) {
|
func recordedKept(ctx context.Context, open *stores, node string) (map[string]string, error) {
|
||||||
every := everyBuildKept(ctx, node)
|
|
||||||
skip, on := keptExcept(ctx)
|
skip, on := keptExcept(ctx)
|
||||||
if !on && !every {
|
if !on {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
if every {
|
|
||||||
skip = nil
|
|
||||||
}
|
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
sent, known, err := inv.SentBuilds(ctx, node)
|
sent, known, err := inv.SentBuilds(ctx, node)
|
||||||
if err != nil {
|
if err != nil || !known {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if !known {
|
|
||||||
if every {
|
|
||||||
return nil, fmt.Errorf("what %s was last sent is not known, so the bus's user list cannot be sent "+
|
|
||||||
"there alone with every build kept (novox/hq issue 490)", node)
|
|
||||||
}
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
current, err := inv.CurrentBuilds(ctx)
|
current, err := inv.CurrentBuilds(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -120,7 +83,7 @@ func recordedKept(ctx context.Context, open *stores, node string) (map[string]st
|
|||||||
out := map[string]string{}
|
out := map[string]string{}
|
||||||
for m, was := range sent {
|
for m, was := range sent {
|
||||||
now, held := current[m]
|
now, held := current[m]
|
||||||
if !held || (now.RollOut && !every) || skip[m] || was == "" || sameCommit(was, now.Commit) {
|
if !held || now.RollOut || skip[m] || was == "" || sameCommit(was, now.Commit) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if f == nil {
|
if f == nil {
|
||||||
@@ -155,10 +118,6 @@ func keepRecorded(ctx context.Context, open *stores, node string, shelf map[stri
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if !found && everyBuildKept(ctx, node) {
|
|
||||||
return nil, fmt.Errorf("%s runs %s's build %s, which the build records no longer hold: the bus's user "+
|
|
||||||
"list cannot be sent there alone with it kept (novox/hq issue 490)", node, m, short(kept[m]))
|
|
||||||
}
|
|
||||||
if !found {
|
if !found {
|
||||||
return nil, fmt.Errorf("%s records rather than rolls out, and %s runs its build %s, which the build "+
|
return nil, fmt.Errorf("%s records rather than rolls out, and %s runs its build %s, which the build "+
|
||||||
"records no longer hold: this send cannot keep it and does not move it — `push %s` sends the new "+
|
"records no longer hold: this send cannot keep it and does not move it — `push %s` sends the new "+
|
||||||
|
|||||||
@@ -1,111 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// What relates two changes (novox/hq ADR 0304 decision 1, as ADR 0306 decision 3 reads it): two changes are
|
|
||||||
// related when they share a repository, a module, a delivery group's head, or an edge of the module graph
|
|
||||||
// between their modules. A module is every one the walk builds or sends: the modules its merges moved and the
|
|
||||||
// modules that depend on them, not only the ones the merges touched. Unrelated changes may walk beside each
|
|
||||||
// other (ADR 0304 decision 2, ADR 0282 decision 3); nothing walks so yet, and nothing but tests reads this.
|
|
||||||
|
|
||||||
// footprint is what one change reaches: its repositories (lower case), its modules, and its delivery group
|
|
||||||
// heads (the head branch names of its merges). unknown says a merge of it was kept without the modules it
|
|
||||||
// moves, so what it reaches is not known: it is related to every change, never wrongly to none.
|
|
||||||
type footprint struct {
|
|
||||||
repos, modules, heads map[string]bool
|
|
||||||
unknown string
|
|
||||||
}
|
|
||||||
|
|
||||||
func newFootprint() footprint {
|
|
||||||
return footprint{repos: map[string]bool{}, modules: map[string]bool{}, heads: map[string]bool{}}
|
|
||||||
}
|
|
||||||
|
|
||||||
// footprintOfWalk is a walk's footprint: the repositories it carries and the modules it plans. A walk keeps no
|
|
||||||
// head of its merges; kept are the merges the record holds for it, whose announcements say their heads, and
|
|
||||||
// may be nil.
|
|
||||||
func footprintOfWalk(p inventory.Plan, kept []inventory.BatchedMerge) footprint {
|
|
||||||
f := newFootprint()
|
|
||||||
for _, c := range p.Carried() {
|
|
||||||
f.repos[strings.ToLower(c.Repository)] = true
|
|
||||||
}
|
|
||||||
for name := range p.Modules {
|
|
||||||
f.modules[name] = true
|
|
||||||
}
|
|
||||||
for _, m := range kept {
|
|
||||||
f.repos[strings.ToLower(m.Repository)] = true
|
|
||||||
if h := announcedOf(m).Head; h != "" {
|
|
||||||
f.heads[h] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return f
|
|
||||||
}
|
|
||||||
|
|
||||||
// footprintOfBatch is an assembling batch's footprint: each merge's repository and head, and the modules its
|
|
||||||
// merges moved when heard, widened to their dependents (planOfMoves). A merge kept with no moves (one owed a
|
|
||||||
// later look, or kept for its packaging) is judged again at the cut and may move anything then: its batch is
|
|
||||||
// unknown, related to everything.
|
|
||||||
func footprintOfBatch(merges []inventory.BatchedMerge, edges []inventory.Edge) footprint {
|
|
||||||
f := newFootprint()
|
|
||||||
var moved []string
|
|
||||||
for _, m := range merges {
|
|
||||||
k := announcedOf(m)
|
|
||||||
f.repos[strings.ToLower(m.Repository)] = true
|
|
||||||
if k.Head != "" {
|
|
||||||
f.heads[k.Head] = true
|
|
||||||
}
|
|
||||||
if len(k.Moves) == 0 && f.unknown == "" {
|
|
||||||
f.unknown = "the modules a merge of " + m.Repository + " moves are not known until its batch is cut"
|
|
||||||
}
|
|
||||||
moved = append(moved, k.Moves...)
|
|
||||||
}
|
|
||||||
for _, name := range reachableFrom(moved, edges) {
|
|
||||||
f.modules[name] = true
|
|
||||||
}
|
|
||||||
return f
|
|
||||||
}
|
|
||||||
|
|
||||||
// related says whether two changes are related, and the first reason in plain words; false and empty when
|
|
||||||
// they are not.
|
|
||||||
func related(a, b footprint, edges []inventory.Edge) (bool, string) {
|
|
||||||
if a.unknown != "" {
|
|
||||||
return true, a.unknown
|
|
||||||
}
|
|
||||||
if b.unknown != "" {
|
|
||||||
return true, b.unknown
|
|
||||||
}
|
|
||||||
if r := firstShared(a.repos, b.repos); r != "" {
|
|
||||||
return true, "both change " + r
|
|
||||||
}
|
|
||||||
if m := firstShared(a.modules, b.modules); m != "" {
|
|
||||||
return true, "both move " + m
|
|
||||||
}
|
|
||||||
if h := firstShared(a.heads, b.heads); h != "" {
|
|
||||||
return true, "both are members of group " + h
|
|
||||||
}
|
|
||||||
for _, e := range edges {
|
|
||||||
if a.modules[e.From] && b.modules[e.To] || b.modules[e.From] && a.modules[e.To] {
|
|
||||||
return true, e.From + " depends on " + e.To + " (" + e.Kind + ")"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false, ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// firstShared is the first key, in order, both sets hold; empty when none.
|
|
||||||
func firstShared(a, b map[string]bool) string {
|
|
||||||
var both []string
|
|
||||||
for k := range a {
|
|
||||||
if b[k] {
|
|
||||||
both = append(both, k)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(both) == 0 {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
sort.Strings(both)
|
|
||||||
return both[0]
|
|
||||||
}
|
|
||||||
@@ -1,110 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// heardMerge is a kept merge of a repository from a head branch that moved modules.
|
|
||||||
func heardMerge(repository, head string, moves ...string) inventory.BatchedMerge {
|
|
||||||
m := inventory.BatchedMerge{Repository: repository, Branch: "main", Commit: "c-" + repository}
|
|
||||||
m.Event, _ = json.Marshal(keptMerge{SourceMoved: link.SourceMoved{Head: head}, Moves: moves})
|
|
||||||
return m
|
|
||||||
}
|
|
||||||
|
|
||||||
// walkOf is a walk carrying one repository's commit and planning modules.
|
|
||||||
func walkOf(repository string, modules ...string) inventory.Plan {
|
|
||||||
p := inventory.Plan{Repository: repository, Commit: "w-" + repository, Modules: map[string]*inventory.PlanModule{}}
|
|
||||||
for _, name := range modules {
|
|
||||||
p.Modules[name] = &inventory.PlanModule{}
|
|
||||||
}
|
|
||||||
return p
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRelatedChanges: two changes are related by a repository, a module, a group head or a graph edge between
|
|
||||||
// their modules, and by a module reached only through dependents (novox/hq ADR 0304 decision 1, ADR 0306
|
|
||||||
// decision 3); otherwise they are not.
|
|
||||||
func TestRelatedChanges(t *testing.T) {
|
|
||||||
edges := []inventory.Edge{
|
|
||||||
{From: "nzbget", To: "base-image", Kind: inventory.EdgeStandsOn},
|
|
||||||
{From: "app", To: "build-machine", Kind: inventory.EdgeBuiltBy},
|
|
||||||
}
|
|
||||||
cases := []struct {
|
|
||||||
name string
|
|
||||||
a, b footprint
|
|
||||||
why string
|
|
||||||
}{
|
|
||||||
{"repository, case aside",
|
|
||||||
footprintOfWalk(walkOf("novox/Mesh-Catalog", "notes"), nil),
|
|
||||||
footprintOfBatch([]inventory.BatchedMerge{heardMerge("NOVOX/mesh-catalog", "fix/a", "app")}, nil),
|
|
||||||
"both change novox/mesh-catalog"},
|
|
||||||
{"module",
|
|
||||||
footprintOfWalk(walkOf("novox/one", "nzbget"), nil),
|
|
||||||
footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/two", "fix/b", "nzbget")}, nil),
|
|
||||||
"both move nzbget"},
|
|
||||||
{"edge, walk depends on batch",
|
|
||||||
footprintOfWalk(walkOf("novox/one", "nzbget"), nil),
|
|
||||||
footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/two", "fix/b", "base-image")}, nil),
|
|
||||||
"nzbget depends on base-image (stands-on)"},
|
|
||||||
{"edge, batch depends on walk",
|
|
||||||
footprintOfWalk(walkOf("novox/one", "build-machine"), nil),
|
|
||||||
footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/two", "fix/b", "app")}, nil),
|
|
||||||
"app depends on build-machine (built-by)"},
|
|
||||||
{"group head",
|
|
||||||
footprintOfWalk(walkOf("novox/one", "notes"), []inventory.BatchedMerge{heardMerge("novox/one", "feat/x")}),
|
|
||||||
footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/two", "feat/x", "app")}, nil),
|
|
||||||
"both are members of group feat/x"},
|
|
||||||
{"module reached through dependents",
|
|
||||||
footprintOfWalk(walkOf("novox/one", "nzbget"), nil),
|
|
||||||
footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/two", "fix/b", "base-image")}, edges),
|
|
||||||
"both move nzbget"},
|
|
||||||
{"unrelated",
|
|
||||||
footprintOfWalk(walkOf("novox/one", "notes"), []inventory.BatchedMerge{heardMerge("novox/one", "feat/x")}),
|
|
||||||
footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/two", "feat/y", "app")}, edges),
|
|
||||||
""},
|
|
||||||
}
|
|
||||||
for _, c := range cases {
|
|
||||||
t.Run(c.name, func(t *testing.T) {
|
|
||||||
for _, pair := range [][2]footprint{{c.a, c.b}, {c.b, c.a}} {
|
|
||||||
is, why := related(pair[0], pair[1], edges)
|
|
||||||
if is != (c.why != "") || why != c.why {
|
|
||||||
t.Errorf("related = %v, %q; want %q", is, why, c.why)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestFootprintOfBatchWidensToDependents: a batch's modules are what its merges moved and every module that
|
|
||||||
// depends on them, along the edges that widen a walk.
|
|
||||||
func TestFootprintOfBatchWidensToDependents(t *testing.T) {
|
|
||||||
edges := []inventory.Edge{
|
|
||||||
{From: "nzbget", To: "base-image", Kind: inventory.EdgeStandsOn},
|
|
||||||
{From: "base-image", To: "build-machine", Kind: inventory.EdgeBuiltBy},
|
|
||||||
}
|
|
||||||
f := footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/two", "", "base-image")}, edges)
|
|
||||||
if !f.modules["base-image"] || !f.modules["nzbget"] || f.modules["build-machine"] || len(f.heads) != 0 {
|
|
||||||
t.Errorf("footprint = %+v; want base-image and nzbget, no head", f)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A batch holding a merge kept with no moves (one owed a later look, judged again at the cut) is related to
|
|
||||||
// every change, in both orders: what it will move is not known, and an unknown change is never unrelated
|
|
||||||
// (issue 525 review).
|
|
||||||
func TestABatchWhoseMovesAreNotKnownIsRelatedToEverything(t *testing.T) {
|
|
||||||
unknown := footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/one", "fix/a", "app"), heardMerge("novox/two", "fix/b")}, nil)
|
|
||||||
other := footprintOfWalk(walkOf("novox/three", "nzbget"), nil)
|
|
||||||
for _, pair := range [][2]footprint{{unknown, other}, {other, unknown}} {
|
|
||||||
if ok, why := related(pair[0], pair[1], nil); !ok || !strings.Contains(why, "not known") {
|
|
||||||
t.Fatalf("an unknown batch read as %v (%q)", ok, why)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
known := footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/one", "fix/a", "app")}, nil)
|
|
||||||
if ok, why := related(known, other, nil); ok {
|
|
||||||
t.Fatalf("a known, unrelated batch read as related: %q", why)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+14
-197
@@ -2,7 +2,6 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
"errors"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -254,19 +253,8 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
// The grants step composes every module at the build its machine was last sent (novox/hq issue 490):
|
|
||||||
// a move it keeps is not made. Read, not assumed, so a move it could not keep is still refused here.
|
|
||||||
var keeps map[string]string
|
|
||||||
if everyBuildKept(ctx, n) {
|
|
||||||
if keeps, err = recordedKept(ctx, open, n); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var waiting []string
|
var waiting []string
|
||||||
for _, mv := range moves {
|
for _, mv := range moves {
|
||||||
if was, kept := keeps[mv.Module]; kept && sameCommit(was, mv.From) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if !scope.judged[n] && !scope.modules[mv.Module] {
|
if !scope.judged[n] && !scope.modules[mv.Module] {
|
||||||
waiting = append(waiting, fmt.Sprintf("%s %s → %s", mv.Module, short(mv.From), short(mv.To)))
|
waiting = append(waiting, fmt.Sprintf("%s %s → %s", mv.Module, short(mv.From), short(mv.To)))
|
||||||
}
|
}
|
||||||
@@ -295,18 +283,15 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
|
|||||||
// owns are the moves the send exists for — every module of a plan's tier whose first machine this is, in
|
// owns are the moves the send exists for — every module of a plan's tier whose first machine this is, in
|
||||||
// one send (novox/hq issue 281): a send carries the machine's whole declaration (ADR 0221), so a send per
|
// one send (novox/hq issue 281): a send carries the machine's whole declaration (ADR 0221), so a send per
|
||||||
// module was the same declaration sent again and again, each one setting aside the one before.
|
// module was the same declaration sent again and again, each one setting aside the one before.
|
||||||
//
|
func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, []string, error) {
|
||||||
// And the grants step first, when the send changes what the bus's user list must say (novox/hq issue 490):
|
|
||||||
// answered for the gate to keep, nil when there was none.
|
|
||||||
func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, []string, *inventory.GrantsStep, error) {
|
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
f, err := readMoveFacts(ctx, inv)
|
f, err := readMoveFacts(ctx, inv)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
moves, err := machineMoves(ctx, open, f, node, true)
|
moves, err := machineMoves(ctx, open, f, node, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
own := func(module string) bool {
|
own := func(module string) bool {
|
||||||
return slices.ContainsFunc(owns, func(o inventory.CarriedMove) bool { return o.Module == module })
|
return slices.ContainsFunc(owns, func(o inventory.CarriedMove) bool { return o.Module == module })
|
||||||
@@ -316,7 +301,7 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if id := f.walkedBy(mv.Module, node, mv.To); id != "" {
|
if id := f.walkedBy(mv.Module, node, mv.To); id != "" {
|
||||||
return nil, nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
return nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
||||||
mv.Module, short(mv.To), node, id)
|
mv.Module, short(mv.To), node, id)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -324,7 +309,7 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
|
|||||||
// plan, or a release, on this machine is not crossed with a newer build; this send waits for its gate.
|
// plan, or a release, on this machine is not crossed with a newer build; this send waits for its gate.
|
||||||
for _, o := range owns {
|
for _, o := range owns {
|
||||||
if id := f.walkedBy(o.Module, node, o.To); id != "" {
|
if id := f.walkedBy(o.Module, node, o.To); id != "" {
|
||||||
return nil, nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
return nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
||||||
o.Module, short(o.To), node, id)
|
o.Module, short(o.To), node, id)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -338,22 +323,22 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(moves) == 0 && len(owns) == 0 {
|
if len(moves) == 0 && len(owns) == 0 {
|
||||||
return nil, nil, nil, nil
|
return nil, nil, nil
|
||||||
}
|
}
|
||||||
for i := range moves {
|
for i := range moves {
|
||||||
if moves[i].Build == "" {
|
if moves[i].Build == "" {
|
||||||
if moves[i].Build, err = inv.BuildOf(ctx, moves[i].Module, moves[i].To); err != nil {
|
if moves[i].Build, err = inv.BuildOf(ctx, moves[i].Module, moves[i].To); err != nil {
|
||||||
return nil, nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
sort.Slice(moves, func(i, j int) bool { return moves[i].Module < moves[j].Module })
|
sort.Slice(moves, func(i, j int) bool { return moves[i].Module < moves[j].Module })
|
||||||
sayRecreations(ctx, open, moves)
|
sayRecreations(ctx, open, moves)
|
||||||
sent, grants, err := sendJudged(ctx, open, node)
|
sent, err := sendRollout(withScope(ctx, sendScope{judged: map[string]bool{node: true}}), open, []string{node})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
return moves, sent, grants, nil
|
return moves, sent, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// passCarried keeps a pass as the verdict of every build the gate judged beside its own module.
|
// passCarried keeps a pass as the verdict of every build the gate judged beside its own module.
|
||||||
@@ -689,7 +674,7 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
|||||||
r.Next++
|
r.Next++
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
moves, sent, grants, err := gatedSend(ctx, open, node, nil)
|
moves, sent, err := gatedSend(ctx, open, node, nil)
|
||||||
if errors.Is(err, errWalkedElsewhere) {
|
if errors.Is(err, errWalkedElsewhere) {
|
||||||
note := fmt.Sprintf("waiting before %s: %v", node, err)
|
note := fmt.Sprintf("waiting before %s: %v", node, err)
|
||||||
changed := p.Note != note
|
changed := p.Note != note
|
||||||
@@ -704,12 +689,8 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
|||||||
r.Next++
|
r.Next++
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
r.Gate = &inventory.PlanGate{Machines: sent, Since: &now, Carried: moves, Sent: sentNow(ctx, open.inventory, sent),
|
r.Gate = &inventory.PlanGate{Machines: sent, Since: &now, Carried: moves, Sent: sentNow(ctx, open.inventory, sent)}
|
||||||
Grants: grants}
|
|
||||||
p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves))
|
p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves))
|
||||||
if said := grantsSaid(grants); said != "" {
|
|
||||||
p.Note += "; " + said
|
|
||||||
}
|
|
||||||
if said := recreationsSaid(moves); said != "" {
|
if said := recreationsSaid(moves); said != "" {
|
||||||
p.Note += "; " + said
|
p.Note += "; " + said
|
||||||
}
|
}
|
||||||
@@ -812,19 +793,14 @@ func backlogCommand(ctx context.Context, sub string, args []string) error {
|
|||||||
}
|
}
|
||||||
defer open.Close()
|
defer open.Close()
|
||||||
if sub == "release-backlog" {
|
if sub == "release-backlog" {
|
||||||
// Said on the bus (novox/hq ADR 0308 decision 1): the release plan starts walking here, and its start is
|
|
||||||
// said by the verb that opens it.
|
|
||||||
var p *inventory.Plan
|
|
||||||
if err := sayingOnTheBus(ctx, open.inventory, func() error {
|
|
||||||
release, err := open.inventory.HoldPlans(ctx, true)
|
release, err := open.inventory.HoldPlans(ctx, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer release()
|
defer release()
|
||||||
why.record(ctx, "upgrade release-backlog", nil)
|
why.record(ctx, "upgrade release-backlog", nil)
|
||||||
p, err = releaseBacklog(ctx, open, link.Caller())
|
p, err := releaseBacklog(ctx, open, link.Caller())
|
||||||
return err
|
if err != nil {
|
||||||
}); err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if p == nil {
|
if p == nil {
|
||||||
@@ -940,162 +916,3 @@ func recreationsSaid(moves []inventory.CarriedMove) string {
|
|||||||
}
|
}
|
||||||
return strings.Join(said, "; ")
|
return strings.Join(said, "; ")
|
||||||
}
|
}
|
||||||
|
|
||||||
// The grants step (novox/hq issue 490).
|
|
||||||
//
|
|
||||||
// The bus's user list — every module's grants, composed mesh-wide from the catalogue — travels only in the
|
|
||||||
// declaration of the machine holding the bus. A send to any other machine that changed it put that machine
|
|
||||||
// first (issue 249), unless a build waited there for a gate: then ungatedIn left it out, and said so. On
|
|
||||||
// 2026-10-11 a merge gave restic a new health tool; its walk sent restic to its first machine and judged it
|
|
||||||
// there, the machine's node-engine asked the tool and the bus refused it ("this host's grant does not name
|
|
||||||
// …"), because the machine holding the bus also runs restic, whose new build waited for that very gate. The
|
|
||||||
// gate could not pass; a person pushed the bus's machine by hand, which carried restic's new build there
|
|
||||||
// unjudged.
|
|
||||||
//
|
|
||||||
// So before a gated send, when the user list composed now is not the one the machine holding the bus was
|
|
||||||
// last sent, that machine is sent its declaration with **every build kept at the one it runs**: the new
|
|
||||||
// list, and nothing of any module's new code. Said on the gate (`plans`), and, when it cannot be sent, said
|
|
||||||
// with why and passed over: the send goes on as it did before, and its gate says what it finds.
|
|
||||||
|
|
||||||
// brokerBehindOf is brokerBehind: a variable so a test of the walk needs no store.
|
|
||||||
var brokerBehindOf = brokerBehind
|
|
||||||
|
|
||||||
// grantsStep sends the machine holding the bus its user list alone, ahead of a gated send to node, when the
|
|
||||||
// list changed; answers what it did, or nil when there was nothing to send.
|
|
||||||
func grantsStep(ctx context.Context, open *stores, node string) *inventory.GrantsStep {
|
|
||||||
holder, behind, err := brokerBehindOf(ctx, open, []string{node})
|
|
||||||
if err != nil {
|
|
||||||
fmt.Printf("grants step before %s: whether the bus's user list changed could not be read: %v\n", node, err)
|
|
||||||
return &inventory.GrantsStep{Node: "the machine holding the bus", Failed: err.Error()}
|
|
||||||
}
|
|
||||||
if holder == "" || holder == node || !behind {
|
|
||||||
// No bus with a user list, the gate's own machine holds it (its send carries the list first), or the
|
|
||||||
// list is the one already sent.
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if _, err := sendRollout(keepingEveryBuild(withScope(ctx, sendScope{}), holder), open, []string{holder}); err != nil {
|
|
||||||
fmt.Printf("grants step: the bus's user list could not be sent to %s ahead of %s, which is sent without "+
|
|
||||||
"it — the bus may refuse what the send newly grants: %v\n", holder, node, err)
|
|
||||||
return &inventory.GrantsStep{Node: holder, Failed: err.Error()}
|
|
||||||
}
|
|
||||||
now := time.Now().UTC()
|
|
||||||
fmt.Printf("grants step: %s sent the bus's user list alone, every build there kept, before %s is judged\n",
|
|
||||||
holder, node)
|
|
||||||
return &inventory.GrantsStep{Node: holder, At: &now}
|
|
||||||
}
|
|
||||||
|
|
||||||
// errNotGrantsOnly is a grants step refused unsent: its declaration would change more than the user list.
|
|
||||||
var errNotGrantsOnly = errors.New("the grants step would change more than the bus's user list, and is not sent")
|
|
||||||
|
|
||||||
// grantsOnlyIn is what a grants step's declaration, composed for sending, would change on the machine holding
|
|
||||||
// the bus besides its user list, against what it was last sent (novox/hq issue 490); empty when nothing else.
|
|
||||||
// Judged by sendToEach on the very declaration it then sends — one composition, judged, then sent or refused
|
|
||||||
// unsent. Kept builds are not the whole of a declaration: a new assignment, a settings change, an assignment
|
|
||||||
// taken away, a provision's logins are composed as the mesh holds them now, and a step that carried any of
|
|
||||||
// it would be the unjudged change this step exists to avoid. Compared resource by resource with the summary
|
|
||||||
// the last send kept (ADR 0217): exact, rather than a list of the cases that can differ. A module there that
|
|
||||||
// was never sent is a new assignment, refused whatever its resources.
|
|
||||||
func grantsOnlyIn(ctx context.Context, open *stores, holder string, plan catalogue.Resolution, declared sendable) (string, error) {
|
|
||||||
inv := open.inventory
|
|
||||||
sent, known, err := inv.SentBuilds(ctx, holder)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if !known {
|
|
||||||
return fmt.Sprintf("what %s was last sent is not known", holder), nil
|
|
||||||
}
|
|
||||||
var other []string
|
|
||||||
listID := ""
|
|
||||||
for _, m := range plan.Modules {
|
|
||||||
if _, was := sent[m.Module]; !was {
|
|
||||||
other = append(other, m.Module+" newly assigned")
|
|
||||||
}
|
|
||||||
if m.BusUsers != "" && m.ClaimsSeat(catalogue.BrokerSeat) {
|
|
||||||
listID = m.Module + "." + catalogue.BusUsersID()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var facts *moveFacts
|
|
||||||
for m, was := range sent {
|
|
||||||
now, carried := declared.Builds[m]
|
|
||||||
if !carried || sameCommit(now, was) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if facts == nil {
|
|
||||||
f, err := readMoveFacts(ctx, inv)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
facts = &f
|
|
||||||
}
|
|
||||||
if !facts.identical(m, was, now) {
|
|
||||||
other = append(other, fmt.Sprintf("%s %s → %s", m, short(was), short(now)))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, f := range declared.foreseen {
|
|
||||||
other = append(other, f+" would be minted")
|
|
||||||
}
|
|
||||||
body, err := declared.Body()
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
after, err := summarize(body)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
prev, err := inv.SentSummary(ctx, holder)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if len(prev) == 0 {
|
|
||||||
return fmt.Sprintf("what %s was last sent is not kept for comparison", holder), nil
|
|
||||||
}
|
|
||||||
var before []sentResource
|
|
||||||
if err := json.Unmarshal(prev, &before); err != nil {
|
|
||||||
return "", fmt.Errorf("what %s was last sent is kept in a form this version does not read: %w", holder, err)
|
|
||||||
}
|
|
||||||
other = append(other, otherThanTheList(diffSent(before, after), listID)...)
|
|
||||||
sort.Strings(other)
|
|
||||||
if len(other) == 0 {
|
|
||||||
return "", nil
|
|
||||||
}
|
|
||||||
return "its send would change more than the bus's user list: " + strings.Join(other, "; "), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// otherThanTheList is what a diff against the last send changes besides the user list's content: every
|
|
||||||
// resource added, removed or changed, but the list's own resource when its content is all that changed —
|
|
||||||
// its path, mode and every other field must be as last sent (novox/hq issue 490).
|
|
||||||
func otherThanTheList(d sentDiff, listID string) []string {
|
|
||||||
var other []string
|
|
||||||
for _, id := range d.Added {
|
|
||||||
other = append(other, "+"+id)
|
|
||||||
}
|
|
||||||
for _, id := range d.Removed {
|
|
||||||
other = append(other, "-"+id)
|
|
||||||
}
|
|
||||||
for _, c := range d.Changed {
|
|
||||||
if c.ID == listID && len(c.Fields) == 1 && c.Fields[0] == "content" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
other = append(other, fmt.Sprintf("~%s (%s)", c.ID, strings.Join(c.Fields, ", ")))
|
|
||||||
}
|
|
||||||
return other
|
|
||||||
}
|
|
||||||
|
|
||||||
// sendJudged is a gated send's sends: the grants step when the user list changed, then the machine judged.
|
|
||||||
func sendJudged(ctx context.Context, open *stores, node string) ([]string, *inventory.GrantsStep, error) {
|
|
||||||
grants := grantsStep(ctx, open, node)
|
|
||||||
sent, err := sendRollout(withScope(ctx, sendScope{judged: map[string]bool{node: true}}), open, []string{node})
|
|
||||||
return sent, grants, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// grantsSaid is a grants step as `plans` and a walk's note say it.
|
|
||||||
func grantsSaid(g *inventory.GrantsStep) string {
|
|
||||||
switch {
|
|
||||||
case g == nil:
|
|
||||||
return ""
|
|
||||||
case g.Failed != "":
|
|
||||||
return fmt.Sprintf(grantsStepWord+" to %s FAILED, sent without it: %s", g.Node, g.Failed)
|
|
||||||
default:
|
|
||||||
return fmt.Sprintf(grantsStepWord+": %s sent the bus's user list first, its builds kept", g.Node)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -11,8 +11,6 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
)
|
)
|
||||||
@@ -512,14 +510,6 @@ func advancePlans(ctx context.Context, open *stores) {
|
|||||||
advanceHeld(ctx, open)
|
advanceHeld(ctx, open)
|
||||||
}
|
}
|
||||||
|
|
||||||
// keepWalkPhases keeps where the walks that ended lately spent their time (novox/hq ADR 0282), outside the hold
|
|
||||||
// on the plans so it never lengthens it: measured, never acted on, and an error only said.
|
|
||||||
func keepWalkPhases(ctx context.Context, open *stores) {
|
|
||||||
if err := recordWalkPhases(ctx, open.inventory, time.Now()); err != nil {
|
|
||||||
fmt.Printf("plans: the phases of the walks ended lately could not be kept: %v\n", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// advanceHeld is advancePlans for a caller already holding the plans.
|
// advanceHeld is advancePlans for a caller already holding the plans.
|
||||||
func advanceHeld(ctx context.Context, open *stores) {
|
func advanceHeld(ctx context.Context, open *stores) {
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
@@ -651,10 +641,27 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
|||||||
// the controller in its first tier: the build that produced the new one is recorded, and the
|
// the controller in its first tier: the build that produced the new one is recorded, and the
|
||||||
// plan never hears it. The record is the fact; a build recorded after the ask is that tier's
|
// plan never hears it. The record is the fact; a build recorded after the ask is that tier's
|
||||||
// outcome, whoever was listening.
|
// outcome, whoever was listening.
|
||||||
recorded, byID, err := recordsOfAsked(ctx, inv, p, tier)
|
recorded := map[string][]inventory.Build{}
|
||||||
|
byID := map[string]inventory.Build{}
|
||||||
|
for _, m := range tier {
|
||||||
|
if s := p.Modules[m]; s != nil && s.State == "asked" {
|
||||||
|
builds, err := inv.Builds(ctx, m, 5)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
|
recorded[m] = builds
|
||||||
|
// Its own ask's record, by id — found even when the outcome named no module (ADR 0219).
|
||||||
|
if s.Build != "" {
|
||||||
|
b, found, err := inv.BuildByID(ctx, s.Build)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if found {
|
||||||
|
byID[s.Build] = b
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
if settleFromRecords(p, tier, recorded, byID) {
|
if settleFromRecords(p, tier, recorded, byID) {
|
||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
@@ -857,12 +864,8 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
|||||||
strings.Join(machines, ", "), p.Tier, err)
|
strings.Join(machines, ", "), p.Tier, err)
|
||||||
}
|
}
|
||||||
now := time.Now().UTC()
|
now := time.Now().UTC()
|
||||||
// What each machine of the rest was sent, kept for when it reports it applied (novox/hq ADR 0282 decision
|
|
||||||
// 6): measured, never acted on.
|
|
||||||
sentWhat := sentNow(ctx, open.inventory, sent)
|
|
||||||
for _, m := range rest {
|
for _, m := range rest {
|
||||||
p.Modules[m].SentAt = &now
|
p.Modules[m].SentAt = &now
|
||||||
p.Modules[m].Rest = restOf(restTo[m], sent, sentWhat)
|
|
||||||
}
|
}
|
||||||
fmt.Printf("%s: tier %d built; sent %s to %s, one send each\n", p.ID, p.Tier, strings.Join(rest, ", "),
|
fmt.Printf("%s: tier %d built; sent %s to %s, one send each\n", p.ID, p.Tier, strings.Join(rest, ", "),
|
||||||
strings.Join(sent, ", "))
|
strings.Join(sent, ", "))
|
||||||
@@ -943,20 +946,6 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
|||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// restOf is, for one module, every machine of its rest that the send reached and what it carried there.
|
|
||||||
func restOf(to, sent []string, what map[string]inventory.SentDeclaration) map[string]inventory.SentDeclaration {
|
|
||||||
out := map[string]inventory.SentDeclaration{}
|
|
||||||
for _, n := range to {
|
|
||||||
if slices.Contains(sent, n) {
|
|
||||||
out[n] = what[n]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(out) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// firstSend sends one machine, in one send, every module of the plan's tier whose first machine it is
|
// firstSend sends one machine, in one send, every module of the plan's tier whose first machine it is
|
||||||
// (novox/hq issue 281), and records the send on each: what that machine ran of it before — read once,
|
// (novox/hq issue 281), and records the send on each: what that machine ran of it before — read once,
|
||||||
// before the send, so a module the same send carries is never read as already moved — the machines it
|
// before the send, so a module the same send carries is never read as already moved — the machines it
|
||||||
@@ -973,7 +962,7 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
|||||||
}
|
}
|
||||||
// A gated send (ADR 0236): everything waiting on the machine goes with the tier, and the gate judges
|
// A gated send (ADR 0236): everything waiting on the machine goes with the tier, and the gate judges
|
||||||
// all of it there.
|
// all of it there.
|
||||||
carried, sent, grants, err := gatedSend(ctx, open, node, owns)
|
carried, sent, err := gatedSend(ctx, open, node, owns)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -994,7 +983,6 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
|||||||
s.GatedBy = ""
|
s.GatedBy = ""
|
||||||
if m == lead {
|
if m == lead {
|
||||||
s.Gate.Carried = carried
|
s.Gate.Carried = carried
|
||||||
s.Gate.Grants = grants
|
|
||||||
} else {
|
} else {
|
||||||
s.GatedBy = lead
|
s.GatedBy = lead
|
||||||
}
|
}
|
||||||
@@ -1004,10 +992,6 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
|||||||
strings.Join(sent, ", "))
|
strings.Join(sent, ", "))
|
||||||
fmt.Printf("%s: tier %d built; sent %d module(s) to %s first in one send (%s), the rest once its gate passes\n",
|
fmt.Printf("%s: tier %d built; sent %d module(s) to %s first in one send (%s), the rest once its gate passes\n",
|
||||||
p.ID, p.Tier, len(modules), strings.Join(sent, ", "), strings.Join(modules, ", "))
|
p.ID, p.Tier, len(modules), strings.Join(sent, ", "), strings.Join(modules, ", "))
|
||||||
// The grants step taken before it (novox/hq issue 490), said with it.
|
|
||||||
if said := grantsSaid(grants); said != "" {
|
|
||||||
p.Note += "; " + said
|
|
||||||
}
|
|
||||||
// What the send recreates, said with it (novox/hq ADR 0245).
|
// What the send recreates, said with it (novox/hq ADR 0245).
|
||||||
if said := recreationsSaid(carried); said != "" {
|
if said := recreationsSaid(carried); said != "" {
|
||||||
p.Note += "; " + said
|
p.Note += "; " + said
|
||||||
@@ -1209,7 +1193,6 @@ func sayUnsent(p *inventory.Plan, rollsOut func(string) bool) {
|
|||||||
// planTicker advances open plans on a timer, for the steps outcomes alone cannot take.
|
// planTicker advances open plans on a timer, for the steps outcomes alone cannot take.
|
||||||
func planTicker(ctx context.Context, open *stores) {
|
func planTicker(ctx context.Context, open *stores) {
|
||||||
advancePlans(ctx, open)
|
advancePlans(ctx, open)
|
||||||
keepWalkPhases(ctx, open)
|
|
||||||
tick := time.NewTicker(30 * time.Second)
|
tick := time.NewTicker(30 * time.Second)
|
||||||
defer tick.Stop()
|
defer tick.Stop()
|
||||||
for {
|
for {
|
||||||
@@ -1218,7 +1201,6 @@ func planTicker(ctx context.Context, open *stores) {
|
|||||||
return
|
return
|
||||||
case <-tick.C:
|
case <-tick.C:
|
||||||
advancePlans(ctx, open)
|
advancePlans(ctx, open)
|
||||||
keepWalkPhases(ctx, open)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1260,13 +1242,6 @@ func planLineWith(p inventory.Plan, now time.Time, pause pauseView, bound time.D
|
|||||||
// planLineOn is planLineWith naming the plan by what it moves and where (planHeadline), given what runs each
|
// planLineOn is planLineWith naming the plan by what it moves and where (planHeadline), given what runs each
|
||||||
// module; nil names no machine.
|
// module; nil names no machine.
|
||||||
func planLineOn(p inventory.Plan, now time.Time, pause pauseView, bound time.Duration, running func(string) []string) string {
|
func planLineOn(p inventory.Plan, now time.Time, pause pauseView, bound time.Duration, running func(string) []string) string {
|
||||||
return planLineSaid(p, now, pause, bound, running, nil)
|
|
||||||
}
|
|
||||||
|
|
||||||
// planLineSaid is planLineOn saying, of a walk waiting for its delivery's word, what the delivery's owner says
|
|
||||||
// holds it (novox/hq issue 529); nil waits says it as before.
|
|
||||||
func planLineSaid(p inventory.Plan, now time.Time, pause pauseView, bound time.Duration, running func(string) []string,
|
|
||||||
waits walkWaits) string {
|
|
||||||
name := planHeadline(p, running)
|
name := planHeadline(p, running)
|
||||||
where := fmt.Sprintf("tier %d of %d", min(p.Tier+1, len(p.Tiers)), len(p.Tiers))
|
where := fmt.Sprintf("tier %d of %d", min(p.Tier+1, len(p.Tiers)), len(p.Tiers))
|
||||||
switch p.State {
|
switch p.State {
|
||||||
@@ -1283,7 +1258,7 @@ func planLineSaid(p inventory.Plan, now time.Time, pause pauseView, bound time.D
|
|||||||
since := now.Sub(inTierSince(p)).Round(time.Second)
|
since := now.Sub(inTierSince(p)).Round(time.Second)
|
||||||
if p.Waiting() {
|
if p.Waiting() {
|
||||||
// Waiting for its delivery's word is no lateness of the walk's (novox/hq ADR 0239).
|
// Waiting for its delivery's word is no lateness of the walk's (novox/hq ADR 0239).
|
||||||
return fmt.Sprintf("%s · %s, %s, for %s", name, where, waitingSaid(p, waits[p.ID]), since)
|
return fmt.Sprintf("%s · %s, %s, for %s", name, where, waitingNote(p), since)
|
||||||
}
|
}
|
||||||
if deferred(p) {
|
if deferred(p) {
|
||||||
// Nor is waiting for the walk before it to end (one walk at a time, novox/hq ADR 0276).
|
// Nor is waiting for the walk before it to end (one walk at a time, novox/hq ADR 0276).
|
||||||
@@ -1488,11 +1463,11 @@ func plansCommand(ctx context.Context, args []string) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
bounds := readTierBounds(ctx, inv, now)
|
bounds := readTierBounds(ctx, inv, now)
|
||||||
fmt.Printf("%s — %s\n", p.ID, planLineSaid(p, now, buildSeatPause(ctx, inv, []inventory.Plan{p}),
|
fmt.Printf("%s — %s\n", p.ID, planLineOn(p, now, buildSeatPause(ctx, inv, []inventory.Plan{p}),
|
||||||
bounds.of(p.Repository), func(module string) []string {
|
bounds.of(p.Repository), func(module string) []string {
|
||||||
on, _ := inv.Running(ctx, module)
|
on, _ := inv.Running(ctx, module)
|
||||||
return on
|
return on
|
||||||
}, waitsOf(ctx, inv, []inventory.Plan{p})))
|
}))
|
||||||
if r := p.Release; r != nil {
|
if r := p.Release; r != nil {
|
||||||
// A release plan's walk (ADR 0236): machines done, the one judged, those to come.
|
// A release plan's walk (ADR 0236): machines done, the one judged, those to come.
|
||||||
fmt.Printf(" machines in order: %s; done: %s; skipped: %s\n", strings.Join(r.Order, ", "),
|
fmt.Printf(" machines in order: %s; done: %s; skipped: %s\n", strings.Join(r.Order, ", "),
|
||||||
@@ -1559,7 +1534,7 @@ func plansCommand(ctx context.Context, args []string) error {
|
|||||||
if len(positionals) == 2 && positionals[0] == "go" {
|
if len(positionals) == 2 && positionals[0] == "go" {
|
||||||
why.record(ctx, "plans go", positionals[1:])
|
why.record(ctx, "plans go", positionals[1:])
|
||||||
var p inventory.Plan
|
var p inventory.Plan
|
||||||
if err := sayingOnTheBus(ctx, inv, func() (err error) {
|
if err := sayingOnTheBus(ctx, func() (err error) {
|
||||||
p, err = letGo(ctx, inv, positionals[1], "a person ("+whoAsked()+")", strings.TrimSpace(*why.why))
|
p, err = letGo(ctx, inv, positionals[1], "a person ("+whoAsked()+")", strings.TrimSpace(*why.why))
|
||||||
return err
|
return err
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
@@ -1571,13 +1546,8 @@ func plansCommand(ctx context.Context, args []string) error {
|
|||||||
}
|
}
|
||||||
// `retry` (novox/hq ADR 0219): a failed plan's failed builds asked again, and the plan goes on.
|
// `retry` (novox/hq ADR 0219): a failed plan's failed builds asked again, and the plan goes on.
|
||||||
if len(positionals) == 2 && positionals[0] == "retry" {
|
if len(positionals) == 2 && positionals[0] == "retry" {
|
||||||
// Said on the bus (novox/hq ADR 0308 decision 1): a retried walk starts again, and its start is said
|
said, err := retryPlan(ctx, open, positionals[1])
|
||||||
// by the verb that starts it.
|
if err != nil {
|
||||||
var said string
|
|
||||||
if err := sayingOnTheBus(ctx, inv, func() (err error) {
|
|
||||||
said, err = retryPlan(ctx, open, positionals[1])
|
|
||||||
return err
|
|
||||||
}); err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
fmt.Println(said)
|
fmt.Println(said)
|
||||||
@@ -1591,29 +1561,26 @@ func plansCommand(ctx context.Context, args []string) error {
|
|||||||
if positionals[0] == "close" {
|
if positionals[0] == "close" {
|
||||||
how = "closed"
|
how = "closed"
|
||||||
}
|
}
|
||||||
// Said on the bus (novox/hq ADR 0308 decision 1): the walk's end, outcome stopped, is said by the verb
|
|
||||||
// that ends it, so what it touched is not held until its bound.
|
|
||||||
var p inventory.Plan
|
|
||||||
if err := sayingOnTheBus(ctx, inv, func() error {
|
|
||||||
release, err := inv.HoldPlans(ctx, true)
|
release, err := inv.HoldPlans(ctx, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer release()
|
defer release()
|
||||||
if p, err = inv.PlanByID(ctx, positionals[1]); err != nil {
|
p, err := inv.PlanByID(ctx, positionals[1])
|
||||||
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if !p.Open() {
|
if !p.Open() {
|
||||||
return fmt.Errorf("%s is already %s", p.ID, p.State)
|
return fmt.Errorf("%s is already %s", p.ID, p.State)
|
||||||
}
|
}
|
||||||
why.record(ctx, "plans "+positionals[0], positionals[1:])
|
why.record(ctx, "plans "+positionals[0], positionals[1:])
|
||||||
stopByHand(&p, how, "a person ("+whoAsked()+")", strings.TrimSpace(*why.why))
|
p.State = inventory.PlanFailed
|
||||||
|
p.Note = how + " by hand at tier " + fmt.Sprint(p.Tier) + ": " + strings.TrimSpace(*why.why)
|
||||||
sayUnsent(&p, func(m string) bool {
|
sayUnsent(&p, func(m string) bool {
|
||||||
u, err := inv.UpgradeOf(ctx, m)
|
u, err := inv.UpgradeOf(ctx, m)
|
||||||
return err == nil && u.RollOut
|
return err == nil && u.RollOut
|
||||||
})
|
})
|
||||||
return inv.SavePlan(ctx, &p)
|
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||||
}); err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
fmt.Printf("%s %s at tier %d of %d; what was asked still builds and registers, nothing further is asked\n",
|
fmt.Printf("%s %s at tier %d of %d; what was asked still builds and registers, nothing further is asked\n",
|
||||||
@@ -1647,41 +1614,15 @@ func plansCommand(ctx context.Context, args []string) error {
|
|||||||
on, _ := inv.Running(ctx, module)
|
on, _ := inv.Running(ctx, module)
|
||||||
return on
|
return on
|
||||||
}
|
}
|
||||||
waits := waitsOf(ctx, inv, plans)
|
|
||||||
for _, p := range plans {
|
for _, p := range plans {
|
||||||
if p.Batch() {
|
if p.Batch() {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
fmt.Printf("%-28s %s\n", p.ID, planLineSaid(p, now, pause, bounds.of(p.Repository), running, waits))
|
fmt.Printf("%-28s %s\n", p.ID, planLineOn(p, now, pause, bounds.of(p.Repository), running))
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// waitsOf asks the delivery's owner what holds the walks among these that wait for its word (novox/hq issue 529),
|
|
||||||
// once, and only when one waits; nil when none waits, no holder is on record, or the bus or the owner does not
|
|
||||||
// answer within a few seconds — the lines then read as before.
|
|
||||||
func waitsOf(ctx context.Context, inv *inventory.Inventory, plans []inventory.Plan) walkWaits {
|
|
||||||
waiting := false
|
|
||||||
for _, p := range plans {
|
|
||||||
waiting = waiting || p.Open() && p.Waiting()
|
|
||||||
}
|
|
||||||
if !waiting {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
entries, err := inv.Catalogued(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
var waits walkWaits
|
|
||||||
_ = onTheBus(func(conn *nats.Conn) error {
|
|
||||||
waits = walksWaiting(ctx, conn, deliverySeatHeld(entries))
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
return waits
|
|
||||||
}
|
|
||||||
|
|
||||||
// planWhatIf is the plan a merge would produce, computed the way the merge handler computes one
|
// planWhatIf is the plan a merge would produce, computed the way the merge handler computes one
|
||||||
// and saved nowhere: the modules the repository's changed files touch (or the modules named), what
|
// and saved nowhere: the modules the repository's changed files touch (or the modules named), what
|
||||||
// packages their source, everything reachable from them, in tiers. For reading before merging.
|
// packages their source, everything reachable from them, in tiers. For reading before merging.
|
||||||
@@ -1807,34 +1748,6 @@ func splitList(s string) []string {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// recordsOfAsked is what settleFromRecords reads: for every module of the tier still `asked`, its last
|
|
||||||
// builds and the record of its own ask, by id.
|
|
||||||
func recordsOfAsked(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan, tier []string) (
|
|
||||||
map[string][]inventory.Build, map[string]inventory.Build, error) {
|
|
||||||
recorded := map[string][]inventory.Build{}
|
|
||||||
byID := map[string]inventory.Build{}
|
|
||||||
for _, m := range tier {
|
|
||||||
if s := p.Modules[m]; s != nil && s.State == "asked" {
|
|
||||||
builds, err := inv.Builds(ctx, m, 5)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
recorded[m] = builds
|
|
||||||
// Its own ask's record, by id — found even when the outcome named no module (ADR 0219).
|
|
||||||
if s.Build != "" {
|
|
||||||
b, found, err := inv.BuildByID(ctx, s.Build)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
if found {
|
|
||||||
byID[s.Build] = b
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return recorded, byID, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// settleFromRecords marks every module of the tier still `asked` built — or failed — from a build
|
// settleFromRecords marks every module of the tier still `asked` built — or failed — from a build
|
||||||
// recorded after it was asked, and says whether it changed anything (novox/hq 04-ISSUES/214).
|
// recorded after it was asked, and says whether it changed anything (novox/hq 04-ISSUES/214).
|
||||||
// Newest first, as Builds answers: the first record after the ask is the outcome of that ask.
|
// Newest first, as Builds answers: the first record after the ask is the outcome of that ask.
|
||||||
@@ -1932,15 +1845,3 @@ func forgetDeleted(ctx context.Context, inv *inventory.Inventory, module string,
|
|||||||
"`module forget %s`: %v\n", p.ID, module, module, firstLine(err.Error()))
|
"`module forget %s`: %v\n", p.ID, module, module, firstLine(err.Error()))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// stopByHand ends a walk a person stopped or closed (`plans stop|close`): failed, with who stopped it and why
|
|
||||||
// kept as the delivery's owner keeps its own stop, so the walk ends with outcome stopped, not failed (novox/hq
|
|
||||||
// ADR 0308 decision 1).
|
|
||||||
func stopByHand(p *inventory.Plan, how, by, why string) {
|
|
||||||
p.State = inventory.PlanFailed
|
|
||||||
p.Note = how + " by hand at tier " + fmt.Sprint(p.Tier) + ": " + why
|
|
||||||
if p.Delivery == nil {
|
|
||||||
p.Delivery = &inventory.PlanDelivery{}
|
|
||||||
}
|
|
||||||
p.Delivery.Stopped, p.Delivery.StoppedWhy = by, why
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -160,7 +160,6 @@ func (b *backlogMesh) release(t *testing.T) inventory.Plan {
|
|||||||
// The backlog goes out one machine at a time: the first judged before the second is sent, each build's
|
// The backlog goes out one machine at a time: the first judged before the second is sent, each build's
|
||||||
// pass kept; a rebuild with the same bytes is no move at all; and a send outside a gate is refused.
|
// pass kept; a rebuild with the same bytes is no move at all; and a send outside a gate is refused.
|
||||||
func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) {
|
func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) {
|
||||||
judgeLeavesAsTheCore(t)
|
|
||||||
b := aBacklog(t)
|
b := aBacklog(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
inv := b.open.inventory
|
inv := b.open.inventory
|
||||||
|
|||||||
@@ -1,85 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestReplay237 replays novox/hq issue 237 (2026-10-04): a workstation already ran a module that
|
|
||||||
// depends on a node seat nothing there held, and the module holding that seat was assigned to it. The
|
|
||||||
// answer said the seat was now held, and then that the workstation was left out of the rest of the mesh
|
|
||||||
// for lack of it, naming as the remedy the assignment just made. `plan` for the node, right after,
|
|
||||||
// resolved. The outcome asserted: every line of the answer, as the controller seat's verb gives it
|
|
||||||
// (standard output, then standard error), agrees with `plan` for the node right after the act. Written
|
|
||||||
// with only what the controller had before its fix — register, assign's command, planFor — so it is laid
|
|
||||||
// over the commit before.
|
|
||||||
func TestReplay237(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
register(t, open, aContainer("web"))
|
|
||||||
register(t, open, catalogue.Manifest{Module: "docker", Version: "1",
|
|
||||||
Claims: []catalogue.Claim{{Name: catalogue.ContainerRuntimeSeat, Scope: catalogue.ScopeNode}}})
|
|
||||||
// Already on the node, from before the rule refused it: the node does not resolve until the
|
|
||||||
// holder comes.
|
|
||||||
if _, err := open.inventory.Assign(ctx, "laptop", "web"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
said := answered(t, func() error { return assignCommand(ctx, "assign", []string{"laptop", "docker"}) })
|
|
||||||
|
|
||||||
if _, _, err := planFor(ctx, open, "laptop"); err != nil {
|
|
||||||
t.Fatalf("laptop does not resolve right after the act: %v\nthe answer was:\n%s", err, said)
|
|
||||||
}
|
|
||||||
if !strings.Contains(said, "web on laptop now has "+catalogue.ContainerRuntimeSeat+" held") {
|
|
||||||
t.Errorf("the answer does not say the seat is now held:\n%s", said)
|
|
||||||
}
|
|
||||||
for _, not := range []string{"laptop is left out", "does not resolve", "which nothing on laptop holds"} {
|
|
||||||
if strings.Contains(said, not) {
|
|
||||||
t.Errorf("the answer says %q, which `plan` right after contradicts:\n%s", not, said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// answered is what a command says as the controller seat's verb answers it: standard output, then
|
|
||||||
// standard error (answerOf).
|
|
||||||
func answered(t *testing.T, f func() error) string {
|
|
||||||
t.Helper()
|
|
||||||
read := func(w **os.File) (func() string, error) {
|
|
||||||
old := *w
|
|
||||||
r, pw, err := os.Pipe()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
*w = pw
|
|
||||||
got := make(chan string)
|
|
||||||
go func() {
|
|
||||||
var buf bytes.Buffer
|
|
||||||
_, _ = io.Copy(&buf, r)
|
|
||||||
got <- buf.String()
|
|
||||||
}()
|
|
||||||
return func() string {
|
|
||||||
_ = pw.Close()
|
|
||||||
*w = old
|
|
||||||
return <-got
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
stdout, err := read(&os.Stdout)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
stderr, err := read(&os.Stderr)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
runErr := f()
|
|
||||||
said := stdout() + stderr()
|
|
||||||
if runErr != nil {
|
|
||||||
t.Fatalf("%v\n%s", runErr, said)
|
|
||||||
}
|
|
||||||
return said
|
|
||||||
}
|
|
||||||
@@ -11,7 +11,6 @@ import (
|
|||||||
"github.com/nats-io/nats-server/v2/server"
|
"github.com/nats-io/nats-server/v2/server"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/testbus"
|
"github.com/novox/mesh-controller/internal/testbus"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -31,7 +30,7 @@ func TestReplay327(t *testing.T) {
|
|||||||
if err := serving.EnsureControllerBuckets(); err != nil {
|
if err := serving.EnsureControllerBuckets(); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
keeper, err := keeperOn(context.Background(), serving.Conn(), conditions.NewMemoryLooks())
|
keeper, err := keeperOn(context.Background(), serving.Conn())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -84,12 +83,8 @@ func asAProcess(t *testing.T, url string) {
|
|||||||
t.Setenv(broker.CertificateVar, "")
|
t.Setenv(broker.CertificateVar, "")
|
||||||
}
|
}
|
||||||
|
|
||||||
// closedNames are the names of the connections the bus saw closed, waited for by want's own name: the
|
// closedNames are the names of the connections the bus saw closed.
|
||||||
// bus can register another connection's close first (the test's own setup connection, say), so returning
|
func closedNames(t *testing.T, bus *server.Server) []string {
|
||||||
// on the first non-empty list names whichever closed first, not necessarily the one being waited for
|
|
||||||
// (novox/hq issue 543). closedNames instead waits until want itself appears among the closed connections,
|
|
||||||
// or the deadline passes, and returns whatever names it last saw either way.
|
|
||||||
func closedNames(t *testing.T, bus *server.Server, want string) []string {
|
|
||||||
t.Helper()
|
t.Helper()
|
||||||
deadline := time.Now().Add(5 * time.Second)
|
deadline := time.Now().Add(5 * time.Second)
|
||||||
var names []string
|
var names []string
|
||||||
@@ -99,14 +94,10 @@ func closedNames(t *testing.T, bus *server.Server, want string) []string {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
names = names[:0]
|
names = names[:0]
|
||||||
found := false
|
|
||||||
for _, c := range connz.Conns {
|
for _, c := range connz.Conns {
|
||||||
names = append(names, c.Name)
|
names = append(names, c.Name)
|
||||||
if c.Name == want {
|
|
||||||
found = true
|
|
||||||
}
|
}
|
||||||
}
|
if len(names) > 0 {
|
||||||
if found {
|
|
||||||
return names
|
return names
|
||||||
}
|
}
|
||||||
time.Sleep(50 * time.Millisecond)
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
|||||||
@@ -77,7 +77,6 @@ func testAWalkStoppedAfterItsGatePassedKeepsThePass(t *testing.T) {
|
|||||||
// the first send is older than the wait for a first machine's report. Read before the pass is taken over, the
|
// the first send is older than the wait for a first machine's report. Read before the pass is taken over, the
|
||||||
// carried module's step said the first machine never applied it, and the passed build was put back.
|
// carried module's step said the first machine never applied it, and the passed build was put back.
|
||||||
func TestACarriedModuleTakesItsLeadsPassBeforeItsStepIsRead(t *testing.T) {
|
func TestACarriedModuleTakesItsLeadsPassBeforeItsStepIsRead(t *testing.T) {
|
||||||
judgeLeavesAsTheCore(t)
|
|
||||||
tm := aTierMesh(t, "m01", "m02")
|
tm := aTierMesh(t, "m01", "m02")
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
inv := tm.open.inventory
|
inv := tm.open.inventory
|
||||||
|
|||||||
@@ -1,28 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A restart count runs from when the node-engine began watching the resource, so it is never said beside
|
|
||||||
// "since" as if it counted from there (novox/hq issue 458): 3890 restarts beside "since 00:58" sent a reader after a crash loop that was not.
|
|
||||||
func TestARestartCountSaysItCountsFromWhenTheEngineBeganWatching(t *testing.T) {
|
|
||||||
since := time.Date(2026, 10, 11, 0, 58, 0, 0, time.UTC)
|
|
||||||
r := inventory.ResourceHealth{Module: "messenger", Resource: "messenger.own-runtime", Kind: "process",
|
|
||||||
Target: "messenger-runtime.service", State: link.StateUnhealthy, Reason: "restarting", Since: since,
|
|
||||||
Streak: 2, Restarts: 3890}
|
|
||||||
h := inventory.NodeHealth{HeardAt: since, Resources: []inventory.ResourceHealth{r},
|
|
||||||
Streaks: map[string]int{"messenger": 2}}
|
|
||||||
lines := strings.Join(healthLines(h, true, since.Add(time.Minute)), "\n")
|
|
||||||
o := moduleUnhealthyObservation("messenger", "anchor", []inventory.ResourceHealth{r})
|
|
||||||
for name, said := range map[string]string{"node show": lines, "evidence": o.Said, "summary": o.Summary} {
|
|
||||||
if !strings.Contains(said, "3890 automatic restart(s) since the node-engine began watching it") || strings.Contains(said, "counted") {
|
|
||||||
t.Errorf("%s says the restart count as if it were since then:\n%s", name, said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -55,15 +55,6 @@ func rotateCommand(ctx context.Context, args []string) error {
|
|||||||
defer open.Close()
|
defer open.Close()
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
|
|
||||||
// A credential the provider's software issued is not the mesh's to remake (novox/hq ADR 0305 §9).
|
|
||||||
shelf, err := inv.Catalogue(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := issuedRefusesRotation(shelf, provision); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
holders, err := inv.HoldersOf(ctx, provision, *only)
|
holders, err := inv.HoldersOf(ctx, provision, *only)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -162,7 +162,6 @@ func TestABusRebuiltFromAnUnchangedSourceDemandsNoBusStep(t *testing.T) {
|
|||||||
// A plan's build made from the source every machine running the module runs is not sent and not
|
// A plan's build made from the source every machine running the module runs is not sent and not
|
||||||
// judged: nothing moves. A build of a changed source is sent to its first machine, gated, as before.
|
// judged: nothing moves. A build of a changed source is sent to its first machine, gated, as before.
|
||||||
func TestAPlanSendsNothingForAnUnchangedSource(t *testing.T) {
|
func TestAPlanSendsNothingForAnUnchangedSource(t *testing.T) {
|
||||||
judgeLeavesAsTheCore(t)
|
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
name string
|
name string
|
||||||
source string
|
source string
|
||||||
|
|||||||
@@ -15,8 +15,7 @@ import (
|
|||||||
func serviceManagerHolder() catalogue.Manifest {
|
func serviceManagerHolder() catalogue.Manifest {
|
||||||
return catalogue.Manifest{Module: "systemd", Version: "1",
|
return catalogue.Manifest{Module: "systemd", Version: "1",
|
||||||
Claims: []catalogue.Claim{{Name: catalogue.ServiceManagerSeat, Scope: catalogue.ScopeNode,
|
Claims: []catalogue.Claim{{Name: catalogue.ServiceManagerSeat, Scope: catalogue.ScopeNode,
|
||||||
Serves: []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal",
|
Serves: []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal"}}},
|
||||||
"reset-failed", "wanted-by", "unlink-dangling"}}},
|
|
||||||
Resources: []map[string]any{{"id": "systemd", "type": "package", "package": "systemd"}}}
|
Resources: []map[string]any{{"id": "systemd", "type": "package", "package": "systemd"}}}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -314,11 +314,6 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return []string{"node", "show", str("node")}, nil
|
return []string{"node", "show", str("node")}, nil
|
||||||
case "judge-directories":
|
|
||||||
if err := need("node"); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return []string{"node", "judge-directories", str("node")}, nil
|
|
||||||
case "modules":
|
case "modules":
|
||||||
return []string{"module", "list", "--json"}, nil
|
return []string{"module", "list", "--json"}, nil
|
||||||
case "seats":
|
case "seats":
|
||||||
@@ -415,10 +410,6 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
|||||||
if b := str("by"); b != "" {
|
if b := str("by"); b != "" {
|
||||||
argv = append(argv, "--by", catalogue.DeliverySeat+" for "+b)
|
argv = append(argv, "--by", catalogue.DeliverySeat+" for "+b)
|
||||||
}
|
}
|
||||||
// The merges to leave out: the walk's other merges are walked again without them (novox/hq ADR 0299).
|
|
||||||
if w := str("without"); w != "" {
|
|
||||||
argv = append(argv, "--without", w)
|
|
||||||
}
|
|
||||||
return argv, nil
|
return argv, nil
|
||||||
case "delivery-walks":
|
case "delivery-walks":
|
||||||
argv := []string{"delivery", "walks"}
|
argv := []string{"delivery", "walks"}
|
||||||
@@ -431,8 +422,7 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
|||||||
return argv, nil
|
return argv, nil
|
||||||
// The build queue (novox/hq ADR 0219).
|
// The build queue (novox/hq ADR 0219).
|
||||||
case "queue":
|
case "queue":
|
||||||
// As data, the table its output (novox/hq issue 455).
|
return []string{"queue"}, nil
|
||||||
return []string{"queue", "--json"}, nil
|
|
||||||
case "cancel", "kill":
|
case "cancel", "kill":
|
||||||
if err := need("id"); err != nil {
|
if err := need("id"); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -783,13 +773,6 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return []string{"secret", "ask", str("node"), str("module"), str("secret"), "--at", str("at")}, nil
|
return []string{"secret", "ask", str("node"), str("module"), str("secret"), "--at", str("at")}, nil
|
||||||
case "issued":
|
|
||||||
// The credential a provider's software issued, taken from the provider (novox/hq ADR 0305): no value in
|
|
||||||
// the arguments, the provision alone.
|
|
||||||
if err := need("provision"); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return []string{"secret", "issued", str("provision")}, nil
|
|
||||||
case "secret-ask":
|
case "secret-ask":
|
||||||
// A module's own secret asked for, typed by the operator at the desk (novox/hq ADR 0277): never a value
|
// A module's own secret asked for, typed by the operator at the desk (novox/hq ADR 0277): never a value
|
||||||
// in the arguments. The desk is the module's machine unless at names another.
|
// in the arguments. The desk is the module's machine unless at names another.
|
||||||
@@ -983,14 +966,7 @@ var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "co
|
|||||||
// What the records say the registries may keep (novox/hq ADR 0251).
|
// What the records say the registries may keep (novox/hq ADR 0251).
|
||||||
"artifacts": true, "collect": true, "images": true, "mirrors": true,
|
"artifacts": true, "collect": true, "images": true, "mirrors": true,
|
||||||
// The delivery's owner's verbs answer JSON where they read (plan, order, check, walks) — novox/hq ADR 0239.
|
// The delivery's owner's verbs answer JSON where they read (plan, order, check, walks) — novox/hq ADR 0239.
|
||||||
"delivery": true,
|
"delivery": true}
|
||||||
// The build queue, each ask with its position among the waiting (novox/hq issue 455).
|
|
||||||
"queue": true}
|
|
||||||
|
|
||||||
// tableBesideVerbs are the JSON verbs whose document carries the table a person reads as its `text`: the
|
|
||||||
// verb answers that table as its output, as it did before it answered data, and the rest as `answer` —
|
|
||||||
// both from the one read, so they never disagree (novox/hq issue 455).
|
|
||||||
var tableBesideVerbs = map[string]bool{"queue": true}
|
|
||||||
|
|
||||||
// overviewVerbs are the JSON verbs whose answer is said once, as data, and not again as the text the
|
// overviewVerbs are the JSON verbs whose answer is said once, as data, and not again as the text the
|
||||||
// command printed: the overviews, which grow with the mesh (novox/hq issue 314). Every reader of theirs
|
// command printed: the overviews, which grow with the mesh (novox/hq issue 314). Every reader of theirs
|
||||||
@@ -1110,12 +1086,6 @@ func answerOf(argv []string, stdout []byte, stderr string, ok bool) verbAnswer {
|
|||||||
var parsed any
|
var parsed any
|
||||||
if json.Unmarshal(bytes.TrimSpace(stdout), &parsed) == nil {
|
if json.Unmarshal(bytes.TrimSpace(stdout), &parsed) == nil {
|
||||||
answer.Answer = parsed
|
answer.Answer = parsed
|
||||||
if doc, isDoc := parsed.(map[string]any); isDoc && tableBesideVerbs[argv[0]] {
|
|
||||||
if table, said := doc["text"].(string); said {
|
|
||||||
answer.Output = table + stderr
|
|
||||||
delete(doc, "text")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if overviewVerbs[argv[0]] {
|
if overviewVerbs[argv[0]] {
|
||||||
// Once, as data: the same document again as text doubled an answer that already
|
// Once, as data: the same document again as text doubled an answer that already
|
||||||
// outgrew one message of the bus (novox/hq issue 314).
|
// outgrew one message of the bus (novox/hq issue 314).
|
||||||
@@ -1396,7 +1366,7 @@ func splitCommandLine(line string) ([]string, error) {
|
|||||||
var words []string
|
var words []string
|
||||||
var cur strings.Builder
|
var cur strings.Builder
|
||||||
inWord := false
|
inWord := false
|
||||||
quote, opened := rune(0), 0
|
quote := rune(0)
|
||||||
runes := []rune(line)
|
runes := []rune(line)
|
||||||
for i := 0; i < len(runes); i++ {
|
for i := 0; i < len(runes); i++ {
|
||||||
r := runes[i]
|
r := runes[i]
|
||||||
@@ -1417,7 +1387,7 @@ func splitCommandLine(line string) ([]string, error) {
|
|||||||
cur.WriteRune(r)
|
cur.WriteRune(r)
|
||||||
}
|
}
|
||||||
case r == '\'' || r == '"':
|
case r == '\'' || r == '"':
|
||||||
quote, opened = r, i
|
quote = r
|
||||||
inWord = true
|
inWord = true
|
||||||
case r == '\\' && i+1 < len(runes):
|
case r == '\\' && i+1 < len(runes):
|
||||||
i++
|
i++
|
||||||
@@ -1435,7 +1405,7 @@ func splitCommandLine(line string) ([]string, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if quote != 0 {
|
if quote != 0 {
|
||||||
return nil, unclosedQuote(quote, opened)
|
return nil, fmt.Errorf("command has an unclosed %c quote", quote)
|
||||||
}
|
}
|
||||||
if inWord {
|
if inWord {
|
||||||
words = append(words, cur.String())
|
words = append(words, cur.String())
|
||||||
@@ -1443,17 +1413,6 @@ func splitCommandLine(line string) ([]string, error) {
|
|||||||
return words, nil
|
return words, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// unclosedQuote is the refusal of a line whose quote is never closed. Most often an apostrophe inside
|
|
||||||
// a single-quoted value ended that quote early and a later quote was left open, so the refusal says
|
|
||||||
// where the open quote is and how a quote is written inside a quoted value — the shell's own two
|
|
||||||
// ways, which this splitter already reads (novox/hq issue 294). It quotes none of the line: a refusal
|
|
||||||
// is kept on the bus as the call's answer, and the line may carry a setting's value.
|
|
||||||
func unclosedQuote(quote rune, opened int) error {
|
|
||||||
return fmt.Errorf("command has an unclosed %c quote, opened at character %d — an apostrophe "+
|
|
||||||
"inside a single-quoted value ends it; write a ' inside single quotes as '\\'' (it'\\''s), or use "+
|
|
||||||
"double quotes and write \\\" for a \" and \\\\ for a \\ inside them", quote, opened+1)
|
|
||||||
}
|
|
||||||
|
|
||||||
// seatAnnouncement is what the controller says it serves on the bus (novox/hq ADR 0197): the
|
// seatAnnouncement is what the controller says it serves on the bus (novox/hq ADR 0197): the
|
||||||
// mesh-controller seat, one endpoint per verb it answers, each with the seat's own description and
|
// mesh-controller seat, one endpoint per verb it answers, each with the seat's own description and
|
||||||
// argument schema — the same facts `tools` answers from the records, as NATS's services format.
|
// argument schema — the same facts `tools` answers from the records, as NATS's services format.
|
||||||
@@ -1498,9 +1457,7 @@ func seatAnnouncement(handlers map[string]link.ToolHandler) micro.Info {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// nodeReads are the `node` subcommands a verb may run: the ones that only read.
|
// nodeReads are the `node` subcommands a verb may run: the ones that only read.
|
||||||
var nodeReads = map[string]bool{"list": true, "show": true,
|
var nodeReads = map[string]bool{"list": true, "show": true}
|
||||||
// Read-only on the node too: the engine judges its directories and changes nothing (novox/hq issue 358).
|
|
||||||
"judge-directories": true}
|
|
||||||
|
|
||||||
// flagsOnly says a command line's rest names no subcommand: empty, or beginning with a flag. For a command
|
// flagsOnly says a command line's rest names no subcommand: empty, or beginning with a flag. For a command
|
||||||
// with no subcommands every word is a flag, its value or a name it reads.
|
// with no subcommands every word is a flag, its value or a name it reads.
|
||||||
@@ -1611,14 +1568,6 @@ func givenAtTheDesk(argv []string) bool {
|
|||||||
return argv[5] == "--at" && argv[6] != "" && !strings.HasPrefix(argv[6], "-")
|
return argv[5] == "--at" && argv[6] != "" && !strings.HasPrefix(argv[6], "-")
|
||||||
}
|
}
|
||||||
|
|
||||||
// takenFromTheProvider is exactly the line the `issued` verb composes, and nothing beside it: `secret issued
|
|
||||||
// <provision>` (novox/hq ADR 0305 §10). It carries no value — the value comes from the provider's own tool, sealed to
|
|
||||||
// the call — and answers none.
|
|
||||||
func takenFromTheProvider(argv []string) bool {
|
|
||||||
return len(argv) == 3 && argv[0] == "secret" && argv[1] == "issued" && argv[2] != "" &&
|
|
||||||
!strings.HasPrefix(argv[2], "-")
|
|
||||||
}
|
|
||||||
|
|
||||||
// terminalOnly refuses, through any verb, a command that is the operator's at the controller's terminal
|
// terminalOnly refuses, through any verb, a command that is the operator's at the controller's terminal
|
||||||
// alone (novox/hq ADR 0266). **Every `node` subcommand that is not a read**: `node account` and
|
// alone (novox/hq ADR 0266). **Every `node` subcommand that is not a read**: `node account` and
|
||||||
// `node agent-account` above all. Whoever may call a verb includes agents, and an agent that named itself
|
// `node agent-account` above all. Whoever may call a verb includes agents, and an agent that named itself
|
||||||
@@ -1635,7 +1584,7 @@ func terminalOnly(argv []string) error {
|
|||||||
// Of a secret's commands only rotation, which seals the new value to the machine that uses it, and the
|
// Of a secret's commands only rotation, which seals the new value to the machine that uses it, and the
|
||||||
// `give` verb's own line: an own secret typed by the operator into the desk's hidden prompt, sealed to this
|
// `give` verb's own line: an own secret typed by the operator into the desk's hidden prompt, sealed to this
|
||||||
// call and then to the module's machine, so no value travels in the verb or its answer (hq ADR 0259 §10).
|
// call and then to the module's machine, so no value travels in the verb or its answer (hq ADR 0259 §10).
|
||||||
if argv[0] == "secret" && (len(argv) < 2 || argv[1] != "rotate") && !givenAtTheDesk(argv) && !takenFromTheProvider(argv) {
|
if argv[0] == "secret" && (len(argv) < 2 || argv[1] != "rotate") && !givenAtTheDesk(argv) {
|
||||||
return terminalRefusal("secret %s is run at the controller's terminal only, never through a verb: accepting, "+
|
return terminalRefusal("secret %s is run at the controller's terminal only, never through a verb: accepting, "+
|
||||||
"recovering or exporting a secret hands it to whoever asks, and that includes agents (novox/hq ADR "+
|
"recovering or exporting a secret hands it to whoever asks, and that includes agents (novox/hq ADR "+
|
||||||
"0266). Nothing was done", strings.Join(argv[1:], " "))
|
"0266). Nothing was done", strings.Join(argv[1:], " "))
|
||||||
|
|||||||
@@ -256,7 +256,7 @@ func stringLit(e ast.Expr) (string, bool) {
|
|||||||
var accountedFlags = map[string]map[string]string{
|
var accountedFlags = map[string]map[string]string{
|
||||||
"status": {"json": "set by the verb: the answer is data"},
|
"status": {"json": "set by the verb: the answer is data"},
|
||||||
"seats": {"json": "set by the verb: the answer is data"},
|
"seats": {"json": "set by the verb: the answer is data"},
|
||||||
"queue": {"json": "set by the verb: the answer is data, its output the table (novox/hq issue 455)"},
|
"queue": {"json": "not set: the verb answers the table a person reads"},
|
||||||
"plan": {"json": "set by the verb unless files is asked"},
|
"plan": {"json": "set by the verb unless files is asked"},
|
||||||
"push": {
|
"push": {
|
||||||
"wait": "set by the verb to 0: a tool call cannot hold a connection for a whole apply",
|
"wait": "set by the verb to 0: a tool call cannot hold a connection for a whole apply",
|
||||||
|
|||||||
@@ -415,48 +415,3 @@ func TestNoVerbSetsTheOperatorsKeyOrRevealsASecret(t *testing.T) {
|
|||||||
t.Fatalf("behind %v: %v", behind, err)
|
t.Fatalf("behind %v: %v", behind, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A value with a quote in it can be said on a `command` line, as in a shell, and a line whose quote
|
|
||||||
// is left open is refused naming where it opened and how a quote is written, quoting none of it (novox/hq issue 294: an
|
|
||||||
// apostrophe inside a single-quoted JSON value cut the line, and the refusal named no cause).
|
|
||||||
func TestAQuotedValueCanHoldAQuote(t *testing.T) {
|
|
||||||
for _, c := range []struct{ line, want string }{
|
|
||||||
{`settings set claude-code '{"role":"the operator'\''s laptop"}'`, `{"role":"the operator's laptop"}`},
|
|
||||||
{`settings set claude-code "{\"role\":\"the operator's laptop\"}"`, `{"role":"the operator's laptop"}`},
|
|
||||||
{"x \"a \\\\ b\nc\"", "a \\ b\nc"},
|
|
||||||
{`x 'a\b'`, `a\b`},
|
|
||||||
} {
|
|
||||||
argv, err := splitCommandLine(c.line)
|
|
||||||
if err != nil || argv[len(argv)-1] != c.want {
|
|
||||||
t.Errorf("%s: read back %q %v, want %q", c.line, argv, err, c.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := splitCommandLine(`settings set claude-code '{"role":"the operator's laptop"}' --node g14`)
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("an apostrophe that leaves a quote open was accepted")
|
|
||||||
}
|
|
||||||
for _, want := range []string{"character 57", `'\''`, `\"`} {
|
|
||||||
if !strings.Contains(err.Error(), want) {
|
|
||||||
t.Errorf("the refusal does not say %q: %v", want, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if strings.Contains(err.Error(), "laptop") || strings.Contains(err.Error(), "g14") {
|
|
||||||
t.Errorf("the refusal quotes the line, which may carry a setting's value: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Every value reads back as it was when written the way the refusal says: single-quoted with '\”
|
|
||||||
// for each quote, or double-quoted with \ before each " and \ — newlines and backslashes included.
|
|
||||||
func TestAQuotedValueRoundTrips(t *testing.T) {
|
|
||||||
for _, v := range []string{`plain`, `it's`, `say "hi"`, `back\slash\`, "two\nlines", `'"\'\"`, `''`, ``} {
|
|
||||||
single := "x '" + strings.ReplaceAll(v, "'", `'\''`) + "'"
|
|
||||||
double := `x "` + strings.NewReplacer(`\`, `\\`, `"`, `\"`).Replace(v) + `"`
|
|
||||||
for _, line := range []string{single, double} {
|
|
||||||
argv, err := splitCommandLine(line)
|
|
||||||
if err != nil || len(argv) != 2 || argv[1] != v {
|
|
||||||
t.Errorf("%s: read back %q %v, want %q", line, argv, err, v)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -41,9 +41,6 @@ func secretCommand(ctx context.Context, args []string) error {
|
|||||||
case "accept":
|
case "accept":
|
||||||
case "ask":
|
case "ask":
|
||||||
return secretAsk(ctx, args[1:])
|
return secretAsk(ctx, args[1:])
|
||||||
case "issued":
|
|
||||||
// The credential a provider's software issued, taken from the provider (novox/hq ADR 0305).
|
|
||||||
return secretIssued(ctx, args[1:])
|
|
||||||
case "rotate":
|
case "rotate":
|
||||||
return secretRotate(ctx, args[1:])
|
return secretRotate(ctx, args[1:])
|
||||||
case "recover":
|
case "recover":
|
||||||
@@ -171,7 +168,6 @@ func secretAsk(ctx context.Context, args []string) error {
|
|||||||
|
|
||||||
const secretUsage = "secret rotate <node> <module> <name> [--why <text> [--cause <word>]]\n" +
|
const secretUsage = "secret rotate <node> <module> <name> [--why <text> [--cause <word>]]\n" +
|
||||||
"secret ask <node> <module> <name> [--at <machine>]\n" +
|
"secret ask <node> <module> <name> [--at <machine>]\n" +
|
||||||
"secret issued <provision>\n" +
|
|
||||||
"secret accept <node> <module> <name> [--from <file> | --at-desk <machine>] [--provider <node> [--local <name>]]\n" +
|
"secret accept <node> <module> <name> [--from <file> | --at-desk <machine>] [--provider <node> [--local <name>]]\n" +
|
||||||
"secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>] [--provider <node>]\n" +
|
"secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>] [--provider <node>]\n" +
|
||||||
"secret export [--out <file>]"
|
"secret export [--out <file>]"
|
||||||
|
|||||||
@@ -27,19 +27,6 @@ type sendable struct {
|
|||||||
// said it reads one is sent none, because an older node-engine refuses a key it does not know, whole
|
// said it reads one is sent none, because an older node-engine refuses a key it does not know, whole
|
||||||
// (link/order.go, the contract).
|
// (link/order.go, the contract).
|
||||||
Epoch uint64
|
Epoch uint64
|
||||||
// Generation is the assignment generation it was composed from (novox/hq issue 234): a machine that
|
|
||||||
// applied a later one refuses it, so a send composed from a view of the assignments the mesh has moved
|
|
||||||
// past cannot undeclare what is still assigned. Zero is not sent at all — every machine whose
|
|
||||||
// node-engine has not said it reads one is sent none, for the reason the epoch is not (an older
|
|
||||||
// node-engine refuses a key it does not know, whole).
|
|
||||||
Generation int64
|
|
||||||
// composedFrom is the generation read before this declaration was composed, and actingEpoch the
|
|
||||||
// lease epoch its sender acted under — whether or not the machine is sent either — for the record of
|
|
||||||
// the send; never on the wire.
|
|
||||||
composedFrom int64
|
|
||||||
actingEpoch uint64
|
|
||||||
// modules are the modules this declaration names, for the record of the send; never on the wire.
|
|
||||||
modules []string
|
|
||||||
// Adoption is nil for a converged node, and then the body is byte for byte what it was before
|
// Adoption is nil for a converged node, and then the body is byte for byte what it was before
|
||||||
// adoption existed: an older host parses the envelope strictly and would refuse the key.
|
// adoption existed: an older host parses the envelope strictly and would refuse the key.
|
||||||
Adoption *adoptionEnvelope
|
Adoption *adoptionEnvelope
|
||||||
@@ -107,9 +94,6 @@ func (s sendable) Body() ([]byte, error) {
|
|||||||
if len(s.LeftOut) > 0 {
|
if len(s.LeftOut) > 0 {
|
||||||
envelope["left_out"] = s.LeftOut
|
envelope["left_out"] = s.LeftOut
|
||||||
}
|
}
|
||||||
if s.Generation > 0 {
|
|
||||||
envelope["generation"] = s.Generation
|
|
||||||
}
|
|
||||||
// An empty declaration is deliberate here — the node owns nothing the mesh put there
|
// An empty declaration is deliberate here — the node owns nothing the mesh put there
|
||||||
// (novox/hq issue 127) — and the host refuses an empty body unless it is told the emptiness
|
// (novox/hq issue 127) — and the host refuses an empty body unless it is told the emptiness
|
||||||
// is meant, so a truncated or mis-composed body is never mistaken for "own nothing".
|
// is meant, so a truncated or mis-composed body is never mistaken for "own nothing".
|
||||||
|
|||||||
@@ -226,20 +226,6 @@ var signalsTable = []signalRow{
|
|||||||
newest: func(f *signalFacts) time.Time {
|
newest: func(f *signalFacts) time.Time {
|
||||||
return newestOf(f.batches, func(b batchFacts) time.Time { return b.closed })
|
return newestOf(f.batches, func(b batchFacts) time.Time { return b.closed })
|
||||||
}},
|
}},
|
||||||
{Row: "S20", Signal: "a send refused for the assignment generation it was composed from",
|
|
||||||
Emitter: "node-engine", Trigger: "each refusal (novox/hq issue 234)",
|
|
||||||
Bound: "none: raised at the first refusal, naming its sender, the generation it came from and the one the " +
|
|
||||||
"machine applied; cleared an hour after the last",
|
|
||||||
Kind: kindOlderGeneration, Severity: conditions.Warning, Phase: 2,
|
|
||||||
needs: func(f *signalFacts) error { return f.refusedSendsErr }, watch: watchGenerationRefusals,
|
|
||||||
newest: func(f *signalFacts) time.Time {
|
|
||||||
return newestOf(f.refusedSends, func(s inventory.Send) time.Time {
|
|
||||||
if s.RefusedAt == nil {
|
|
||||||
return time.Time{}
|
|
||||||
}
|
|
||||||
return *s.RefusedAt
|
|
||||||
})
|
|
||||||
}},
|
|
||||||
{Row: "S17", Signal: "a send held for the bus's planned step is told to a person", Emitter: "controller's plan",
|
{Row: "S17", Signal: "a send held for the bus's planned step is told to a person", Emitter: "controller's plan",
|
||||||
Trigger: "each send refused because it would replace the bus outside its step (novox/hq issue 336)",
|
Trigger: "each send refused because it would replace the bus outside its step (novox/hq issue 336)",
|
||||||
Bound: "none: raised at the first refusal, for the operator, naming what waits, the bus build from and to, " +
|
Bound: "none: raised at the first refusal, for the operator, naming what waits, the bus build from and to, " +
|
||||||
@@ -254,16 +240,6 @@ var signalsTable = []signalRow{
|
|||||||
newest: func(f *signalFacts) time.Time {
|
newest: func(f *signalFacts) time.Time {
|
||||||
return newestOf(f.bus.waits, func(w busWaitFacts) time.Time { return w.since })
|
return newestOf(f.bus.waits, func(w busWaitFacts) time.Time { return w.since })
|
||||||
}},
|
}},
|
||||||
{Row: "S21", Signal: "a clean apply", Emitter: "node-engine", Trigger: "each report of a declaration refused or applied only in part (novox/hq issue 233)",
|
|
||||||
Bound: "one apply interval (5 min) past the first report of the same failure, judged by time while the machine " +
|
|
||||||
"is heard — the node-engine says a re-apply only when how it went changed; not while it is silent or " +
|
|
||||||
"asleep; naming the refusal, or the package manager that does not answer with its words; cleared by the " +
|
|
||||||
"clean apply it then says",
|
|
||||||
Kind: kindApplyFailing, Severity: conditions.Warning, Phase: 2,
|
|
||||||
needs: func(f *signalFacts) error { return f.doingErr }, watch: watchApplyFailing,
|
|
||||||
newest: func(f *signalFacts) time.Time {
|
|
||||||
return newestOf(f.doing, func(d inventory.Doing) time.Time { return d.At })
|
|
||||||
}},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// watchFacts is S14: the snapshot a merge check is fed is older than its bound, or none was kept since
|
// watchFacts is S14: the snapshot a merge check is fed is older than its bound, or none was kept since
|
||||||
@@ -547,14 +523,6 @@ func watchAdvisories(f *signalFacts) []conditions.Observation {
|
|||||||
if a.Kind == link.AdvisoryConsumerLost && !f.lostConsumers[a.Stream+"."+a.Consumer] {
|
if a.Kind == link.AdvisoryConsumerLost && !f.lostConsumers[a.Stream+"."+a.Consumer] {
|
||||||
continue // it exists again, or the mesh no longer expects it: a removal, not a loss
|
continue // it exists again, or the mesh no longer expects it: a removal, not a loss
|
||||||
}
|
}
|
||||||
if a.Kind == link.AdvisoryMaxDeliveries && a.Token == "" {
|
|
||||||
// A consumer's max-deliveries key is the dead-letter row's (novox/hq issue 330): said while
|
|
||||||
// DEAD_LETTERS holds what it gave up on, cleared when that is delivered again or dropped. The
|
|
||||||
// listener records no such advisory today; one read here as well added a look to the count and
|
|
||||||
// overwrote the row's words on every look (novox/hq issue 440). Only one that could not be kept
|
|
||||||
// (AdvisoryNotKept), which DEAD_LETTERS cannot say, is said from here.
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
severity := conditions.Warning
|
severity := conditions.Warning
|
||||||
times := ""
|
times := ""
|
||||||
if a.Count > 1 {
|
if a.Count > 1 {
|
||||||
@@ -565,10 +533,7 @@ func watchAdvisories(f *signalFacts) []conditions.Observation {
|
|||||||
machine = f.host
|
machine = f.host
|
||||||
}
|
}
|
||||||
o := conditions.Observation{Scope: conditions.ScopeBus, ID: a.ID, Kind: a.Kind, Token: a.Token,
|
o := conditions.Observation{Scope: conditions.ScopeBus, ID: a.ID, Kind: a.Kind, Token: a.Token,
|
||||||
Machine: machine, Severity: severity, Summary: a.Said + times, Said: a.Said,
|
Machine: machine, Severity: severity, Summary: a.Said + times, Said: a.Said}
|
||||||
// The advisory is remembered and read again on every look for an hour: the condition counts
|
|
||||||
// what the bus said and when, not the looks (novox/hq issue 402).
|
|
||||||
Happened: a.Last, Times: a.Count}
|
|
||||||
if a.Kind == link.AdvisoryMaxDeliveries && a.Token == link.AdvisoryNotKept {
|
if a.Kind == link.AdvisoryMaxDeliveries && a.Token == link.AdvisoryNotKept {
|
||||||
o.Machine = consumerMachine(a.Stream, a.Consumer)
|
o.Machine = consumerMachine(a.Stream, a.Consumer)
|
||||||
o.Headline = clip(conditions.Capital(fmt.Sprintf("%s gave up on a message, not kept",
|
o.Headline = clip(conditions.Capital(fmt.Sprintf("%s gave up on a message, not kept",
|
||||||
@@ -600,17 +565,7 @@ func watchDeadLetters(f *signalFacts) []conditions.Observation {
|
|||||||
messages, them = fmt.Sprintf("%d messages", n), "them"
|
messages, them = fmt.Sprintf("%d messages", n), "them"
|
||||||
}
|
}
|
||||||
who := consumerWho(stream, consumer)
|
who := consumerWho(stream, consumer)
|
||||||
// DEAD_LETTERS is a record of what was given up on: the condition says when the newest held message
|
|
||||||
// was kept, and its count is a running total of the messages given up on while it is open, never
|
|
||||||
// how often the controller looked (novox/hq issues 402 and 440). It is at least the number held now,
|
|
||||||
// and does not go down when one is delivered again or dropped. Without that time it counts its
|
|
||||||
// looks, as a source of the present does.
|
|
||||||
happened, times := f.deadLettersNewest[key], 0
|
|
||||||
if !happened.IsZero() {
|
|
||||||
times = n
|
|
||||||
}
|
|
||||||
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: key, Kind: link.AdvisoryMaxDeliveries,
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: key, Kind: link.AdvisoryMaxDeliveries,
|
||||||
Happened: happened, Times: times,
|
|
||||||
Machine: consumerMachine(stream, consumer), Severity: conditions.Warning,
|
Machine: consumerMachine(stream, consumer), Severity: conditions.Warning,
|
||||||
Summary: fmt.Sprintf("%s gave up on %s; %s kept in %s until delivered again or dropped, with why, "+
|
Summary: fmt.Sprintf("%s gave up on %s; %s kept in %s until delivered again or dropped, with why, "+
|
||||||
"through the controller's dead-letters verb", link.ConsumerInWords(stream, consumer), messages,
|
"through the controller's dead-letters verb", link.ConsumerInWords(stream, consumer), messages,
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user