Compare commits

..
3 Commits
Author SHA1 Message Date
jschoubben fc573f951a Remove only a move's proven second copy, and never delete a handed-out original (hq issue 558)
mesh/merge-gate out of date: judged against trunk 13711fb5b91b, the trunk is now 13be75eb65c0; judged again on a recheck or when it is merged
mesh/repo-check out of date: judged against trunk 13711fb5b91b, the trunk is now 13be75eb65c0; judged again on a recheck or when it is merged
mesh/delivery checking
Second review of #276: a dead letter delivered again beside its original
shares its request id and was removed as a copy, losing the ask; and a
publish that outlived the pause could delete an original a holder had
taken. A copy now names the message it copies, removals are said, the
publish waits inside the pause's margin, and an original handed out
meanwhile keeps its place while the copy goes.
2026-10-12 21:46:49 +02:00
jschoubben 1ded7dff34 Make a queue move lossless and run its passes on the acting controller alone (hq issue 558)
mesh/merge-gate out of date: judged against trunk 4dbd4066ca54, the trunk is now c0c379af2b3c; judged again on a recheck or when it is merged
mesh/repo-check out of date: judged against trunk 4dbd4066ca54, the trunk is now c0c379af2b3c; judged again on a recheck or when it is merged
mesh/delivery superseded: a newer head of the same pull request
Review of #276: a move deleted before it published, so a failed publish
lost an ask; a verb's own process ran a pass beside the serving one, so
one could resume the queue while the other moved asks and a build ran
twice. The copy is now published first and second copies removed by id,
asks only signal the acting controller's loop, moves stop with 5 s of
the pause left, a cancel finds a moved ask by id, and an unreadable ask
keeps its age.
2026-10-12 21:38:11 +02:00
jschoubben c6e234455f Hand a delivery build out before pull-request checks (hq issue 558)
mesh/delivery superseded: a newer head of the same pull request
mesh/merge-gate out of date: judged against trunk 4dbd4066ca54, the trunk is now c0c379af2b3c; judged again on a recheck or when it is merged
mesh/repo-check out of date: judged against trunk 4dbd4066ca54, the trunk is now c0c379af2b3c; judged again on a recheck or when it is merged
A merged change's walk waited minutes on its build behind checks in a
first-come queue. The queue now hands deliveries out first, a check that
has waited ten minutes before deliveries asked after it, and says each
ask's kind; the order is kept in the work queue itself.
2026-10-12 21:25:46 +02:00
31 changed files with 1029 additions and 1453 deletions
+1 -13
View File
@@ -66,11 +66,6 @@ broker to take work from or a registry to publish into:
Without --registry the artifacts stay in this machine's container runtime, named by the
digest of their own configuration. The result is printed as JSON.
mesh-builder ready
exits 0 once the agent running in this container is bound to its build work and pulls it, and
1 before: its container's health check.
`
func run() error {
@@ -81,18 +76,12 @@ func run() error {
return nil
case "build":
return buildOnce(context.Background(), os.Args[2:])
case "ready":
// The container's health check (novox/hq issue 544): exits non-zero until the agent takes work.
return ready()
default:
fmt.Print(usage)
return nil
}
}
if err := notReady(); err != nil {
return err
}
credential, err := brokerFrom()
if err != nil {
return err
@@ -145,8 +134,7 @@ func run() error {
// **What an earlier holder here left goes before anything is taken** (novox/hq issue 306): a holder
// recreated mid-check left the check's store and bus running, and the ask went to another machine.
h.removeLeftBehind()
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused,
Taking: func() { taking(seat) }})
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused})
defer machine.Close()
// The seat's verbs, on this machine's subjects. Only the seat that declares them: the retired
-44
View File
@@ -1,44 +0,0 @@
package main
import (
"errors"
"fmt"
"os"
"path/filepath"
)
// **The build agent says it is ready** (novox/hq issue 544). It listens on nothing, so no http or tcp
// check can see it work; it writes a mark once it is bound to its worker and pulls build work, and its
// container's health check runs `mesh-builder ready`, which passes only while the mark is there. Without
// it the node-engine could judge the agent only alive, and every walk that moved it waited out the whole
// grace on each machine.
//
// The mark lives in the container's own temporary directory, never in the workspace the machine keeps:
// a new container starts without it, and a restarted one removes it before it binds again.
func readyMark() string { return filepath.Join(os.TempDir(), "mesh-builder.ready") }
// notReady removes the mark: a holder starting is not taking work until it says so again.
func notReady() error {
if err := os.Remove(readyMark()); err != nil && !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("cannot remove the mark that this agent takes build work (%s): %w", readyMark(), err)
}
return nil
}
// taking writes the mark. A mark that cannot be written is said, and the agent goes on building: its
// check then says it is not ready, which is the truth the gate should hear.
func taking(seat string) {
if err := os.WriteFile(readyMark(), []byte(seat+"\n"), 0o644); err != nil {
fmt.Fprintf(os.Stderr, "cannot mark that this agent takes build work (%s): %v\n", readyMark(), err)
return
}
fmt.Fprintf(os.Stderr, "taking build work from %s\n", seat)
}
// ready is the health check: nil when this container's agent is bound and pulling build work.
func ready() error {
if _, err := os.Stat(readyMark()); err != nil {
return fmt.Errorf("not taking build work yet: %s is not there", readyMark())
}
return nil
}
-32
View File
@@ -1,32 +0,0 @@
package main
import (
"os"
"testing"
)
// The health check passes only between the agent saying it takes work and its next start (novox/hq
// issue 544).
func TestTheAgentIsReadyOnlyOnceItTakesWork(t *testing.T) {
t.Setenv("TMPDIR", t.TempDir())
if err := ready(); err == nil {
t.Fatal("an agent that never took work is ready")
}
taking("mesh-build-machine")
if err := ready(); err != nil {
t.Fatalf("an agent taking work is not ready: %v", err)
}
// Started again, it is not ready until it binds again.
if err := notReady(); err != nil {
t.Fatal(err)
}
if err := ready(); err == nil {
t.Fatal("an agent starting again is ready before it takes work")
}
if err := notReady(); err != nil {
t.Fatalf("removing a mark that is not there failed: %v", err)
}
if _, err := os.Stat(readyMark()); err == nil {
t.Fatal("the mark is still there")
}
}
+1 -11
View File
@@ -1109,16 +1109,6 @@ const resolverSeat = "mesh-dns-resolver"
// classOf is a walk's class (novox/hq ADR 0282 decision 1): core when it walks a module on the controller's own
// path or one holding the mesh's resolver, leaf otherwise.
func classOf(walk inventory.Plan, entries []inventory.Entry) string {
modules := make([]string, 0, len(walk.Modules))
for m := range walk.Modules {
modules = append(modules, m)
}
return classOfModules(modules, entries)
}
// classOfModules is the class of modules moved together: core when one is on the controller's own path or holds
// the mesh's resolver, leaf otherwise (ADR 0282 decision 1).
func classOfModules(modules []string, entries []inventory.Entry) string {
resolvers := map[string]bool{}
for _, e := range entries {
for _, c := range e.Manifest.Claims {
@@ -1127,7 +1117,7 @@ func classOfModules(modules []string, entries []inventory.Entry) string {
}
}
}
for _, m := range modules {
for m := range walk.Modules {
if _, own := onTheControllersPath[m]; own || resolvers[m] {
return inventory.ClassCore
}
-24
View File
@@ -2,7 +2,6 @@ package main
import (
"fmt"
"path"
"slices"
"sort"
"strings"
@@ -106,32 +105,9 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
p.Machines = append(p.Machines, *machines[name])
}
p.Summary = summaryOf(p)
p.Class = changeClassOf(p, entries)
return p
}
// classNone is the class of a change that moves no module: nothing for a walk to move (novox/hq ADR 0313 §2).
const classNone = "none"
// changeClassOf is a change plan's class (novox/hq ADR 0313 §2): that of the slowest-budgeted module it moves,
// itself or as a dependent — core when one is on the controller's own path or holds the mesh's resolver, leaf
// otherwise — and none when it moves nothing. A module it adds a directory for counts by its own class: the
// catalogue names a module's directory by the module, so the directory's last element is its name. The graph holds
// no manifest of it yet, so a new module is core by its name alone (on the controller's own path), never by a
// resolver claim its manifest makes.
func changeClassOf(p link.ChangePlan, entries []inventory.Entry) string {
if len(p.Moved) == 0 && len(p.Dependents) == 0 && len(p.New) == 0 {
return classNone
}
modules := append(append([]string{}, p.Moved...), p.Dependents...)
for _, dir := range p.New {
if name := path.Base(strings.Trim(dir, "/")); name != "." && name != "/" && name != "" {
modules = append(modules, name)
}
}
return classOfModules(modules, entries)
}
// grantsStepWord names the grants step (novox/hq issue 490): the bus's user list sent alone to the machine
// holding the bus ahead of a walk's gate. Not the bus step, which replaces the bus itself.
const grantsStepWord = "grants step"
-54
View File
@@ -1,54 +0,0 @@
package main
import (
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// **A change plan says its class** (novox/hq ADR 0313 §2): the issue tracker holds a fix to the budget of the
// slowest-budgeted module the fix moves, and a fix that moves no module has no fix budget. The plan says core,
// leaf or none, so mesh-delivery can pass it on with the fix's delivery.
func TestAChangePlanSaysTheClassOfWhatItMoves(t *testing.T) {
const repo = "http://forge.internal:20000/novox/mesh-catalog.git"
entry := func(name, path string) inventory.Entry {
e := fromRepo(name, repo, path)
e.Source.BuiltFrom = "old"
e.On = []string{"anchor"}
return e
}
gitea := entry("gitea", "modules/gitea")
resolver := entry("dnsmasq", "modules/dnsmasq")
resolver.Manifest.Claims = []catalogue.Claim{{Name: resolverSeat}}
delivery := entry(catalogue.DeliverySeat, "modules/mesh-delivery")
webApp := entry("photos", "modules/photos")
entries := []inventory.Entry{gitea, resolver, delivery, webApp}
// photos stands on gitea: a change to gitea moves photos as its dependent.
edges := []inventory.Edge{{From: "photos", To: "gitea", Kind: inventory.EdgeStandsOn}}
policy := func(string) (inventory.Upgrade, bool) { return inventory.Upgrade{RollOut: true}, true }
plan := func(paths ...string) link.ChangePlan {
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: paths}
return changePlanOf("novox/mesh-catalog", "main", "head", reachOfMerge(m, entries, nil, edges), entries, policy)
}
for _, c := range []struct {
why string
paths []string
want string
}{
{"a leaf module and its dependent", []string{"modules/gitea/index.ts"}, inventory.ClassLeaf},
{"the delivery's owner, on the controller's own path", []string{"modules/mesh-delivery/cmd/x.go"}, inventory.ClassCore},
{"the mesh's resolver", []string{"modules/dnsmasq/module.json"}, inventory.ClassCore},
{"a leaf and a core module together: the slowest-budgeted", []string{"modules/gitea/index.ts", "modules/dnsmasq/module.json"}, inventory.ClassCore},
{"nothing any module's build reads", []string{"README.md"}, "none"},
{"a new leaf module alone", []string{"modules/weather/module.json"}, inventory.ClassLeaf},
{"a new module on the controller's own path, by its own class", []string{"next/mesh-delivery/module.json"}, inventory.ClassCore},
{"a new leaf module beside a core one moved", []string{"modules/weather/module.json", "modules/dnsmasq/module.json"}, inventory.ClassCore},
} {
if got := plan(c.paths...).Class; got != c.want {
t.Errorf("%s: class %q, want %q", c.why, got, c.want)
}
}
}
-135
View File
@@ -1,135 +0,0 @@
package main
import (
"fmt"
"sort"
"strings"
"time"
"github.com/novox/mesh-controller/internal/inventory"
)
// machineLines is what a machine said it is made of (novox/hq issue 160), as `node` shows it: every fact in
// one block with its age, the architecture and kernel the profile always carried among them, and each fact
// the machine could not determine said as unknown with its why — never as a zero, and never left out.
func machineLines(a inventory.MachineAccount, had bool, now time.Time) []string {
if !had {
return []string{" what it is made of: never said — this machine has not reported a profile"}
}
m := a.Machine
if m == nil {
return []string{
" what it is made of: not said — its node-engine is older than the one that says it",
" " + factLine("architecture", orUnsaid(a.Architecture)),
" " + factLine("kernel", orUnsaid(a.Kernel)),
}
}
age := roughly(now.Sub(m.ObservedAt)) + " ago"
if m.ObservedAt.IsZero() {
// Said with no time: its age is not known, and is never read as two thousand years.
age = "at a time it did not say (age unknown)"
}
out := []string{fmt.Sprintf(" what it is made of, as it said %s:", age)}
add := func(name, value string) { out = append(out, " "+factLine(name, value)) }
unknown := map[string]string{}
for k, v := range m.Unknown {
unknown[k] = v
}
// said is the value, or the fact's own reason it is unknown; a fact neither said nor unknown is a
// node-engine that broke its own rule, and is said so rather than shown blank.
said := func(fact, value string) string {
if value != "" {
delete(unknown, fact)
return value
}
if why, ok := unknown[fact]; ok {
delete(unknown, fact)
return "unknown: " + why
}
return "unknown: the machine said neither it nor why"
}
add("architecture", orUnsaid(a.Architecture))
release := said("kernel release", m.KernelRelease)
if m.KernelRelease == "" {
release = "(release " + release + ")"
}
add("kernel", orUnsaid(a.Kernel)+" "+release)
dist := said("distribution", m.Distribution)
if m.Distribution != "" {
dist += " " + said("version", m.Version)
}
add("distribution", dist)
add("processor", said("processor", m.Processor))
add("cores", said("cores", countWords(m.Cores)))
add("threads", said("threads", countWords(m.Threads)))
add("memory", said("memory", bytesWords(m.MemoryBytes)))
add("virtual", said("virtual or physical", m.Virtual))
up := ""
if m.BootedAt != nil {
up = fmt.Sprintf("%s, since %s", roughly(now.Sub(*m.BootedAt)), m.BootedAt.Local().Format("2006-01-02 15:04"))
}
add("up", said("uptime", up))
add("timezone", said("timezone", m.Timezone))
if len(m.Filesystems) == 0 {
add("filesystems", said("filesystems", ""))
} else {
for i, f := range m.Filesystems {
name := ""
if i == 0 {
name = "filesystems"
}
add(name, fmt.Sprintf("%s %s, %s free (%s %s)", f.Mount, bytesWords(f.SizeBytes),
bytesWords(f.FreeBytes), f.Type, f.Device))
}
if why, ok := unknown["filesystems"]; ok {
add("", "and some unknown: "+why)
delete(unknown, "filesystems")
}
}
add("public address", said("public address", ""))
// Any other fact a newer node-engine named unknown, so nothing it said is dropped.
rest := make([]string, 0, len(unknown))
for k := range unknown {
rest = append(rest, k)
}
sort.Strings(rest)
for _, k := range rest {
add(k, "unknown: "+unknown[k])
}
return out
}
func factLine(name, value string) string { return fmt.Sprintf("%-15s %s", name, value) }
func orUnsaid(s string) string {
if strings.TrimSpace(s) == "" {
return "not said"
}
return s
}
// countWords is a number a machine said, or nothing when it said none (its unknown says why).
func countWords(n int) string {
if n <= 0 {
return ""
}
return fmt.Sprint(n)
}
// bytesWords is an amount of bytes a person reads, in binary units; nothing when none was said.
func bytesWords(n int64) string {
if n <= 0 {
return ""
}
units := []string{"B", "KiB", "MiB", "GiB", "TiB", "PiB"}
v, u := float64(n), 0
for v >= 1024 && u < len(units)-1 {
v /= 1024
u++
}
if u == 0 {
return fmt.Sprintf("%d B", n)
}
return fmt.Sprintf("%.1f %s", v, units[u])
}
-88
View File
@@ -1,88 +0,0 @@
package main
import (
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
)
// Defends novox/hq issue 160: `node` shows every fact a machine said it is made of, in one block with its
// age, the architecture and kernel among them; and a fact it could not determine as unknown with why.
func TestNodeShowsWhatAMachineIsMadeOfWithItsAge(t *testing.T) {
now := time.Date(2026, 10, 12, 15, 0, 0, 0, time.UTC)
booted := now.Add(-75 * time.Hour)
a := inventory.MachineAccount{Architecture: "amd64", Kernel: "linux", Machine: &inventory.MadeOf{
ObservedAt: now.Add(-3 * time.Hour), KernelRelease: "7.2.8-arch1-2", Distribution: "Arch Linux",
Version: "rolling", Processor: "AMD Ryzen 9 8945HS", Cores: 8, Threads: 16, MemoryBytes: 32891494400,
Virtual: "physical", BootedAt: &booted, Timezone: "Europe/Brussels",
Filesystems: []inventory.MountedFilesystem{{Mount: "/", Device: "/dev/nvme0n1p3", Type: "ext4",
SizeBytes: 987978522624, FreeBytes: 31830110208}},
Unknown: map[string]string{"public address": "how it learns it is not decided yet"},
}}
got := strings.Join(machineLines(a, true, now), "\n")
t.Log("\n" + got)
for _, want := range []string{
"what it is made of, as it said 3h ago:",
"architecture amd64",
"kernel linux 7.2.8-arch1-2",
"distribution Arch Linux rolling",
"processor AMD Ryzen 9 8945HS",
"cores 8",
"threads 16",
"memory 30.6 GiB",
"virtual physical",
"up 3d, since",
"timezone Europe/Brussels",
"filesystems / 920.1 GiB, 29.6 GiB free (ext4 /dev/nvme0n1p3)",
"public address unknown: how it learns it is not decided yet",
} {
if !strings.Contains(got, want) {
t.Errorf("missing %q", want)
}
}
}
func TestAFactAMachineCouldNotReadIsShownUnknownNeverZero(t *testing.T) {
now := time.Date(2026, 10, 12, 15, 0, 0, 0, time.UTC)
a := inventory.MachineAccount{Architecture: "arm64", Kernel: "linux", Machine: &inventory.MadeOf{
ObservedAt: now, Unknown: map[string]string{"memory": "MemTotal not found in /proc/meminfo",
"cores": "/proc/cpuinfo says no core id", "kernel release": "permission denied"}}}
got := strings.Join(machineLines(a, true, now), "\n")
t.Log("\n" + got)
for _, want := range []string{
"memory unknown: MemTotal not found in /proc/meminfo",
"cores unknown: /proc/cpuinfo says no core id",
"kernel linux (release unknown: permission denied)",
// Neither said nor named unknown: the node-engine broke its rule, and that is shown.
"timezone unknown: the machine said neither it nor why",
} {
if !strings.Contains(got, want) {
t.Errorf("missing %q", want)
}
}
if strings.Contains(got, " 0 ") || strings.Contains(got, "0 B") {
t.Error("a fact the machine could not read was shown as a zero")
}
}
func TestAnOlderNodeEngineStillShowsArchitectureAndKernel(t *testing.T) {
got := strings.Join(machineLines(inventory.MachineAccount{Architecture: "amd64", Kernel: "linux"}, true,
time.Now()), "\n")
if !strings.Contains(got, "older than the one that says it") || !strings.Contains(got, "architecture amd64") ||
!strings.Contains(got, "kernel linux") {
t.Errorf("%s", got)
}
if got := strings.Join(machineLines(inventory.MachineAccount{}, false, time.Now()), "\n"); !strings.Contains(got, "never said") {
t.Errorf("%s", got)
}
}
func TestAnAccountWithoutItsTimeSaysAgeUnknown(t *testing.T) {
got := machineLines(inventory.MachineAccount{Architecture: "amd64", Kernel: "linux",
Machine: &inventory.MadeOf{KernelRelease: "7.2"}}, true, time.Now())[0]
if !strings.Contains(got, "age unknown") || strings.Contains(got, "d ago") {
t.Errorf("%s", got)
}
}
-11
View File
@@ -875,17 +875,6 @@ func showNode(ctx context.Context, stored *stores, name string) error {
}
}
// What it is made of, in one block with its age (novox/hq issue 160): collected and shown in the same
// breath, so no fact the machine says is kept and read by nothing.
if account, had, err := inv.MachineOf(ctx, name); err != nil {
fmt.Printf("\n what it is made of could NOT be read: %v\n", err)
} else {
fmt.Println()
for _, line := range machineLines(account, had, time.Now()) {
fmt.Println(line)
}
}
held, err := inv.Profile(ctx, name)
if err != nil {
return err
+56 -14
View File
@@ -14,7 +14,6 @@ import (
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/inventory"
@@ -81,6 +80,30 @@ func listQueue(ctx context.Context, args []string, w io.Writer) error {
return nil
}
// keepingTheQueueInOrder runs the build queue's passes while this controller acts (novox/hq issue 558):
// the one process that puts the queue in order, so no two passes pause and resume it over each other.
// The build seat is read from the catalogue at most every half minute.
func keepingTheQueueInOrder(ctx context.Context, open *stores, js *broker.JetStream) {
if js == nil {
return
}
seat, readAt := link.TheBuildMachine, time.Time{}
seatNow := func() string {
if time.Since(readAt) < watchEvery {
return seat
}
if entries, err := open.inventory.Catalogued(ctx); err == nil {
seat, readAt = buildSeatAmong(entries), time.Now()
} else {
fmt.Printf("the build seat is read as %s: the catalogue cannot be read: %v\n", seat, err)
}
return seat
}
link.KeepingInOrder(ctx, js, seatNow, link.Holding, func(format string, args ...any) {
fmt.Printf(format+"\n", args...)
})
}
// queueAnswer is the queue as data (novox/hq issue 455): what a caller reads to say how far back an ask
// waits — "waiting in the build queue behind 2" — which the table alone could not tell it. Text is the
// table, written from the same read: the verb answers it as its output and the rest as its answer, so the
@@ -100,7 +123,8 @@ type queueAnswer struct {
type queuedAnswer struct {
ID string `json:"id"`
State string `json:"state"`
// Kind is "check" for a pull request's merge check, "build" for anything else.
// Kind is "delivery" for a build a walk waits for, "check" for a pull request's check and any ask
// whose kind cannot be told (link.KindOf, novox/hq issue 558).
Kind string `json:"kind"`
Repository string `json:"repository"`
Path string `json:"path,omitempty"`
@@ -129,10 +153,7 @@ func queueAnswerOf(q link.Queue, now time.Time) queueAnswer {
out := queueAnswer{Seat: q.Seat, Waiting: len(waiting), InFlight: len(running), Dead: len(dead),
MaxDeliver: q.MaxDeliver, Asks: []queuedAnswer{}, Text: queueText(q, now)}
one := func(a link.QueuedAsk, position int) queuedAnswer {
kind := "build"
if a.Request.Check != nil {
kind = "check"
}
kind := link.KindOf(a.Request)
since := a.AskedAt
if a.State == link.AskInFlight && a.On != "" && !a.Started.IsZero() {
since = a.Started
@@ -189,19 +210,20 @@ func queueText(q link.Queue, now time.Time) string {
case a.Was != "":
where = fmt.Sprintf("handed back by %s, to be handed out again", a.Was)
}
fmt.Fprintf(&b, " %-26s %s — %s, %s (seq %d)\n", a.ID, what(a), where, ago(a.AskedAt), a.Seq)
fmt.Fprintf(&b, " %-26s %-8s %s — %s, %s (seq %d)\n", a.ID, link.KindOf(a.Request), what(a), where,
ago(a.AskedAt), a.Seq)
}
}
if len(waiting) > 0 {
fmt.Fprintln(&b, "\nwaiting:")
fmt.Fprintln(&b, "\nwaiting, in the order they are handed out:")
for _, a := range waiting {
fmt.Fprintf(&b, " %-26s %s — %s (seq %d)\n", a.ID, what(a), ago(a.AskedAt), a.Seq)
fmt.Fprintf(&b, " %-26s %-8s %s — %s (seq %d)\n", a.ID, link.KindOf(a.Request), what(a), ago(a.AskedAt), a.Seq)
}
}
if len(dead) > 0 {
fmt.Fprintf(&b, "\ndead — handed out as often as the worker allows (%d) and never settled, still held:\n", q.MaxDeliver)
for _, a := range dead {
fmt.Fprintf(&b, " %-26s %s — %s (seq %d)\n", a.ID, what(a), ago(a.AskedAt), a.Seq)
fmt.Fprintf(&b, " %-26s %-8s %s — %s (seq %d)\n", a.ID, link.KindOf(a.Request), what(a), ago(a.AskedAt), a.Seq)
}
}
switch {
@@ -275,6 +297,15 @@ func cancelAsk(ctx context.Context, js *broker.JetStream, open *stores, seat str
}
}
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
// **Where it waits now** (novox/hq issue 558): moved into order since it was read, a waiting ask has a
// later sequence, and the worker handing out what was behind its old one is not it being taken.
if ask.State == link.AskWaiting {
if q, err := link.ReadQueue(ctx, js, seat); err == nil {
if now, ok := q.Find(ask.ID); ok && now.State == link.AskWaiting {
ask.Seq = now.Seq
}
}
}
switch ask.State {
case link.AskWaiting:
if taken, err := takenSince(js, worker, ask.Seq); err != nil {
@@ -296,14 +327,25 @@ func cancelAsk(ctx context.Context, js *broker.JetStream, open *stores, seat str
"ends it there", ask.ID, started, ask.ID)
}
}
if err := js.Context().DeleteMsg(worker.Stream, ask.Seq); err != nil && !errors.Is(err, nats.ErrMsgNotFound) &&
!errors.Is(err, jetstream.ErrMsgNotFound) && !strings.Contains(err.Error(), "no message found") {
// Found by its id when it was moved into order since it was read (novox/hq issue 558).
removed, err := link.DeleteAsk(ctx, js, seat, ask)
if err != nil {
return "", fmt.Errorf("%s is marked cancelled and could not be deleted from the queue (seq %d): %w — a "+
"holder taking it ends it as cancelled", ask.ID, ask.Seq, err)
}
recordCancelled(ctx, open, ask)
return fmt.Sprintf("cancelled %s (%s, %s): deleted from the %s queue and recorded failed, %s — a plan "+
"that asked for it fails with that", ask.ID, ask.Repository, ask.State, seat, link.CancelledByHand), nil
return cancelSaid(ask, seat, removed), nil
}
// cancelSaid is what a cancel says: deleted from the queue only when it was.
func cancelSaid(ask link.QueuedAsk, seat string, removed uint64) string {
where := fmt.Sprintf("deleted from the %s queue", seat)
if removed == 0 {
where = fmt.Sprintf("no longer in the %s queue to delete, and marked cancelled, so a holder taking it "+
"ends it", seat)
}
return fmt.Sprintf("cancelled %s (%s, %s): %s and recorded failed, %s — a plan "+
"that asked for it fails with that", ask.ID, ask.Repository, ask.State, where, link.CancelledByHand)
}
// holderLooks is how long a cancel waits for a holder that took the ask before the mark to say it
+22 -2
View File
@@ -24,7 +24,7 @@ func aQueueOfEachKind(now time.Time) link.Queue {
State: link.AskInFlight, On: "ace", Started: started,
Request: link.BuildRequest{ID: "build-running", Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-host"}}},
{Seq: 3, ID: "build-first", Repository: "ssh://forge/novox/mesh-catalog", Ref: "main", AskedAt: asked,
State: link.AskWaiting},
State: link.AskWaiting, Request: link.BuildRequest{ID: "build-first", Repository: "ssh://forge/novox/mesh-catalog"}},
{Seq: 4, ID: "build-second", Repository: "ssh://forge/novox/mesh-catalog", Path: "modules/gitea", Ref: aCommit,
AskedAt: now.Add(-time.Minute), State: link.AskWaiting,
Request: link.BuildRequest{ID: "build-second", Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog"}}},
@@ -51,7 +51,7 @@ func TestTheQueueVerbAnswersEachAskAsData(t *testing.T) {
t.Errorf("in flight: %+v", running)
}
first, second := byID["build-first"], byID["build-second"]
if first.Position != 1 || first.Kind != "build" || first.Commit != "" || first.Ref != "main" || first.Machine != "" ||
if first.Position != 1 || first.Kind != "delivery" || first.Commit != "" || first.Ref != "main" || first.Machine != "" ||
!first.Since.Equal(now.Add(-5*time.Minute)) {
t.Errorf("first waiting: %+v", first)
}
@@ -62,6 +62,13 @@ func TestTheQueueVerbAnswersEachAskAsData(t *testing.T) {
if dead := byID["build-dead"]; dead.Position != 0 || dead.State != link.AskDead {
t.Errorf("dead: %+v", dead)
}
// And the table says each ask's kind too (novox/hq issue 558).
for _, line := range []string{"build-running check ", "build-first delivery ",
"build-second check ", "build-dead check "} {
if !strings.Contains(got.Text, line) {
t.Errorf("the table does not say %q:\n%s", line, got.Text)
}
}
// In the table's order: in flight, then the waiting by position, then the dead.
var order []string
for _, a := range got.Asks {
@@ -135,3 +142,16 @@ func TestTheQueueNeverShowsARepositorysPassword(t *testing.T) {
}
}
}
// A cancel says it deleted the ask only when it did: moved into order and gone since, it says so (novox/hq
// issue 558).
func TestACancelSaysDeletedOnlyWhenItDeleted(t *testing.T) {
ask := link.QueuedAsk{ID: "build-1", Repository: "novox/x", State: link.AskWaiting, Seq: 4}
if said := cancelSaid(ask, link.TheBuildMachine, 9); !strings.Contains(said, "deleted from the") {
t.Errorf("removed, it says: %s", said)
}
if said := cancelSaid(ask, link.TheBuildMachine, 0); strings.Contains(said, "deleted from") ||
!strings.Contains(said, "no longer in the") {
t.Errorf("gone already, it says: %s", said)
}
}
-90
View File
@@ -1,90 +0,0 @@
package main
import (
"encoding/json"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
)
// TestReplay201ThePlansSend replays novox/hq issue 201 (2026-10-02): two merges a minute apart made two plans; the
// controller's plan rolled the newer controller onto the control node, and the node-engine's plan then sent that
// machine the controller build before it, whose binary refused the seat row the newer one had widened. Here the
// control node, anchor, was last sent the newer controller c2 while the mesh's record of the controller is the
// build before it, c1, which passed its gate. The outcome asserted: a send that is no person's push — composed as a
// plan's send composes it (keepingRecorded, what sendKeeps gives every send through sendToEach) — carries the
// controller at c2, the build the machine was last sent, and not the older record; a person's push still carries
// the record. Written with only what the controller had before its fix (mesh-controller #273) — the inventory's
// build, gate, assignment and send records, keepingRecorded, planFor and declarationFor — so it is laid over the
// commit before; the fix's own TestReplay201 reads what the fix added. Needs the store (MESH_TEST_POSTGRES), and
// is named in testdata/store-tests.
func TestReplay201ThePlansSend(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
const older, newer = "c1aaaaaaaaaa", "c2bbbbbbbbbb"
old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute)
record := func(commit string, asked time.Time) inventory.Build {
t.Helper()
manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: "1"})
b := inventory.Build{ID: "build-mesh-controller-" + commit, Module: "mesh-controller", Commit: commit,
Repository: "novox/mesh-controller", Manifest: manifest, Asked: asked, At: asked,
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + commit + "0000000000000000"}}}
if err := inv.RecordBuild(ctx, b); err != nil {
t.Fatal(err)
}
return b
}
// The older controller rolled out and passed its gate; the newer one was asked after it and registered.
before := record(older, old)
if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: before.ID, Module: "mesh-controller", Commit: older,
Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil {
t.Fatal(err)
}
record(newer, now)
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, inventory.Source{
Repository: "novox/mesh-controller", BuiltFrom: newer, Head: newer, Asked: now}); err != nil {
t.Fatal(err)
}
if _, err := inv.Assign(ctx, "anchor", "mesh-controller"); err != nil {
t.Fatal(err)
}
// The controller's plan sent anchor the newer controller.
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"mesh-controller": newer}); err != nil {
t.Fatal(err)
}
// And the mesh's record of the controller is the build before it — whatever made it so.
if err := inv.RestoreModule(ctx, before); err != nil {
t.Fatal(err)
}
// The controller each send to anchor carries: a plan's send keeps recorded builds (keepingRecorded, what
// sendKeeps gives every send but a person's push); a person's push composes the mesh's record.
controllerSent := func(sends string, keeping bool) string {
t.Helper()
c := ctx
if keeping {
c = keepingRecorded(ctx)
}
plan, settings, err := planFor(c, open, "anchor")
if err != nil {
t.Fatalf("%s to anchor did not compose: %v", sends, err)
}
declared, err := declarationFor(c, open, "anchor", plan, settings)
if err != nil {
t.Fatalf("%s to anchor did not compose: %v", sends, err)
}
return declared.Builds["mesh-controller"]
}
if got := controllerSent("a plan's send", true); got != newer {
t.Fatalf("a plan's send carried anchor the controller %s, older than the %s anchor was last sent: the "+
"older binary would read the row its successor widened", short(got), short(newer))
}
if got := controllerSent("a person's push", false); got != older {
t.Fatalf("a person's push carried the controller %s, not the mesh's record %s", short(got), short(older))
}
}
+8 -41
View File
@@ -34,10 +34,8 @@ type seatRow struct {
Delivers string `json:"delivers,omitempty"`
Decision string `json:"decision"`
// Replicated says the seat may be held on several machines at once (novox/hq ADR 0223).
Replicated bool `json:"replicated,omitempty"`
// Paths are the paths on its machine the seat's holder owns (novox/hq ADR 0287).
Paths []catalogue.SeatPath `json:"paths,omitempty"`
Holders []seatHolder `json:"holders"`
Replicated bool `json:"replicated,omitempty"`
Holders []seatHolder `json:"holders"`
}
// seatsHeld is every seat the mesh defines with its holders, and every claim held that names no
@@ -51,7 +49,7 @@ func seatsHeld(seats []catalogue.Seat, held []catalogue.Held) ([]seatRow, []cata
rows := make([]seatRow, 0, len(seats))
for _, s := range seats {
row := seatRow{Seat: s.Name, Scope: s.Scope, Delivers: s.Delivers, Decision: s.Decision,
Replicated: s.Replicated, Paths: s.Paths, Holders: []seatHolder{}}
Replicated: s.Replicated, Holders: []seatHolder{}}
seen := map[seatHolder]bool{}
for _, h := range held {
// Resolve the held claim to a seat rather than comparing names, so a record naming a
@@ -257,16 +255,12 @@ func seatsCommand(ctx context.Context, args []string) error {
return err
}
rows, outside := seatsHeld(catalogue.Seats(), world.Held)
// Which module owns each seat's path on each machine (novox/hq ADR 0287): the seat naming it, and
// that seat's holder there, with the stepping back applied.
owners := catalogue.PathOwners(catalogue.Seats(), world.Held)
if *asJSON {
out := struct {
Seats []seatRow `json:"seats"`
Owners []catalogue.PathOwner `json:"path-owners,omitempty"`
Outside []catalogue.Held `json:"outside,omitempty"`
}{rows, owners, outside}
Seats []seatRow `json:"seats"`
Outside []catalogue.Held `json:"outside,omitempty"`
}{rows, outside}
body, err := json.MarshalIndent(out, "", " ")
if err != nil {
return err
@@ -276,7 +270,7 @@ func seatsCommand(ctx context.Context, args []string) error {
}
w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
fmt.Fprintln(w, "SEAT\tSCOPE\tDELIVERS\tPATHS\tHELD BY")
fmt.Fprintln(w, "SEAT\tSCOPE\tDELIVERS\tHELD BY")
for _, r := range rows {
delivers := r.Delivers
if delivers == "" {
@@ -290,22 +284,11 @@ func seatsCommand(ctx context.Context, args []string) error {
}
holders = strings.Join(parts, ", ")
}
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", r.Seat, r.Scope, delivers, seatPathsText(r.Paths), holders)
fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", r.Seat, r.Scope, delivers, holders)
}
if err := w.Flush(); err != nil {
return err
}
if len(owners) > 0 {
fmt.Println("\nwho owns each seat's path, by machine (novox/hq ADR 0287):")
w = tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
fmt.Fprintln(w, "PATH\tMACHINE\tSEAT\tMODULE")
for _, o := range owners {
fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", o.Path, o.Node, o.Seat, o.Module)
}
if err := w.Flush(); err != nil {
return err
}
}
if len(outside) > 0 {
fmt.Println("\nheld, and not a seat this mesh defines (registered before the set closed — novox/hq ADR 0110):")
for _, h := range outside {
@@ -314,19 +297,3 @@ func seatsCommand(ctx context.Context, args []string) error {
}
return nil
}
// seatPathsText is a seat's paths in one column: each path, and the seat it steps back for, if any.
func seatPathsText(paths []catalogue.SeatPath) string {
if len(paths) == 0 {
return "—"
}
parts := make([]string, 0, len(paths))
for _, p := range paths {
if p.StepsBackFor != "" {
parts = append(parts, p.Path+" (steps back for "+p.StepsBackFor+")")
continue
}
parts = append(parts, p.Path)
}
return strings.Join(parts, ", ")
}
-19
View File
@@ -74,22 +74,3 @@ func TestAKindedBenchIsNotHandedOver(t *testing.T) {
}
}
}
// The listing shows each seat's paths, and the seat a path steps back for (novox/hq ADR 0287).
func TestTheSeatsListingShowsEachSeatsPaths(t *testing.T) {
rows, _ := seatsHeld(catalogue.Seats(), nil)
got := map[string]string{}
for _, r := range rows {
got[r.Seat] = seatPathsText(r.Paths)
}
for seat, want := range map[string]string{
"node-hostname": "/etc/hosts, /etc/hostname",
"node-uplink": "/etc/resolv.conf (steps back for node-resolver)",
"node-resolver": "/etc/resolv.conf",
"node-packet-filter": "—",
} {
if got[seat] != want {
t.Errorf("%s lists its paths as %q, not %q", seat, got[seat], want)
}
}
}
+3
View File
@@ -739,6 +739,9 @@ func watchTheMesh(ctx context.Context, open *stores, server *link.Server, bus li
// and the answer chosen is performed on its warrant.
startAsking(watching, open, server, bus.Conn, keeper)
go forgettingOldHeals(watching, open.inventory)
// And the build queue's order (novox/hq issue 558): a check that has waited its while goes ahead of the
// deliveries asked after it even when nothing new is asked.
go keepingTheQueueInOrder(watching, open, server.JetStream())
fmt.Printf("watching the mesh: %d signal(s) every %s, %d probe(s) every %s; what is wrong is kept in %s "+
"and said as %s events\n", len(watchedRows()), watchEvery, len(runnableProbes()), doctorEvery,
broker.ConditionsBucket, conditions.Seat)
+34 -29
View File
@@ -1,7 +1,5 @@
package catalogue
import "path"
// The machine's own resolver (novox/hq ADR 0247).
//
// **Most machines have none.** Every machine lists the mesh's resolvers in /etc/resolv.conf and nothing
@@ -68,39 +66,43 @@ func holdsSeat(m Manifest, seat string) bool {
return false
}
// stepsBack is the module as it composes among the modules of one machine (novox/hq ADR 0287,
// ADR 0247):
// **where it holds a seat that names a path stepping back for another seat, and that other seat's holder
// is on the machine and declares the path, its own rendering of the path is left out**, and nothing else
// of it changes. Only a file the mesh renders for it steps back, as before: a resource on the path is
// composed as declared (the declaration copies a module's resources whole), so an uplink holder declaring
// the resolver file as a resource beside the resolver's holder is two owners of one path, and refused. The stepping back is the seat's definition — node-uplink stepping back from the resolver
// file for node-resolver — so any other module declaring the path beside the owner's is still two owners
// of one path, and is refused as before.
func stepsBack(m Manifest, modules []Manifest) Manifest {
drop := map[string]bool{}
for _, c := range m.Claims {
seat, known := SeatNamed(c.Name)
if !known {
continue
}
for _, p := range seat.Paths {
if p.StepsBackFor == "" {
continue
}
for _, o := range modules {
if o.Module != m.Module && holdsSeat(o, p.StepsBackFor) && declaresPath(o, p.Path) {
drop[p.Path] = true
}
}
// rendersResolverFile says whether a module asks the mesh to render the machine's resolver file.
func rendersResolverFile(m Manifest) bool {
for _, f := range m.Facts {
if !f.Home && f.Path == ResolverFile {
return true
}
}
if len(drop) == 0 {
return false
}
// ResolverFileOwner is the module that writes the machine's resolver file among the modules on one
// machine (novox/hq ADR 0247): the holder of node-resolver when one renders it, else nobody is named here
// and the file is whoever's it always was — the uplink holder's (ADR 0223).
func ResolverFileOwner(modules []Manifest) string {
for _, m := range modules {
if holdsSeat(m, ResolverSeat) && rendersResolverFile(m) {
return m.Module
}
}
return ""
}
// stepsBack is the module as it composes on a machine whose resolver file is the node-resolver holder's:
// **the uplink holder's rendering of that file is left out**, and nothing else of it changes. Only the
// uplink's holder steps back — any other module rendering the file beside the resolver's is still two
// owners of one path, and is refused as before.
func stepsBack(m Manifest, modules []Manifest) Manifest {
if !holdsSeat(m, "node-uplink") || !rendersResolverFile(m) {
return m
}
owner := ResolverFileOwner(modules)
if owner == "" || owner == m.Module {
return m
}
facts := make(map[string]RosterFile, len(m.Facts))
for name, f := range m.Facts {
if !f.Home && drop[path.Clean(f.Path)] {
if !f.Home && f.Path == ResolverFile {
continue
}
facts[name] = f
@@ -111,6 +113,9 @@ func stepsBack(m Manifest, modules []Manifest) Manifest {
// steppedBack is every module of one machine as it composes there (stepsBack, each).
func steppedBack(modules []Manifest) []Manifest {
if ResolverFileOwner(modules) == "" {
return modules
}
out := make([]Manifest, len(modules))
for i, m := range modules {
out[i] = stepsBack(m, modules)
+1 -3
View File
@@ -957,9 +957,7 @@ func checkResources(modules []Manifest) []string {
// Judged as the machine composes them: where the node-resolver holder writes the resolver file, the
// uplink holder's rendering of it is not composed, so it is not a second owner (novox/hq ADR 0247).
modules = steppedBack(modules)
// And only a claimant of a seat naming a path declares it — the controller's generated resources
// included, which are judged here at composition (novox/hq ADR 0287).
problems := seatPathsOnMachine(modules)
var problems []string
owner := map[string]string{}
ownedPath := map[string]string{} // path → owning module, for the access check below
packagePresent, packageAbsent := map[string][]string{}, map[string][]string{}
-315
View File
@@ -1,315 +0,0 @@
package catalogue
import (
"fmt"
"path"
"strings"
)
// The paths a seat's holder owns (novox/hq ADR 0287).
//
// **Who owns a machine-wide file is the role's question, answered once.** Until this, which module owned
// /etc/hosts or /etc/resolv.conf was decided record by record (ADRs 0199, 0222, 0223, 0241) and enforced
// only after the fact, by checkResources finding two modules on one machine declaring the same path: a
// module that did not hold node-hostname could declare /etc/hosts on a machine where nobody held it, and
// nothing refused it. Now a seat's definition names the paths the role owns on any machine, and three
// refusals follow from it, each where the mesh can still say no:
//
// - a claim whose module does not declare every path its seat names, by CanHold, at registration and at
// handover;
// - a module declaring a path a seat names while claiming none of the seats that name it, at
// registration (CatalogueProblems), and what the controller generates for a machine, at composition;
// - a module's own seat naming a path another seat already names, at registration — the second loses.
//
// Reading a seat's path is access, never a resource, so it stays allowed.
// SeatPath is one path a seat's holder owns: absolute, literal, on the machine where the holder runs.
type SeatPath struct {
Path string `json:"path"`
// StepsBackFor is a node seat whose holder owns the path instead, on a machine where that seat is held
// (novox/hq ADR 0287, ADR 0247): this seat's holder still declares the path, and its declaration is not
// composed there. Only the mesh's own seats step back, each by a record.
StepsBackFor string `json:"steps-back-for,omitempty"`
}
// seatPathProblem is what is wrong with one path a seat names, judged alone: absolute, written literally
// with no placeholder, and clean, so it compares with what a module declares byte for byte.
func seatPathProblem(seat, scope, p string) string {
switch {
case scope != ScopeNode:
return fmt.Sprintf("the seat %s names the path %q and is a %s seat; only a node seat names paths, "+
"because a path is a machine's (novox/hq ADR 0287)", seat, p, scope)
case strings.Contains(p, "${"):
return fmt.Sprintf("the seat %s names the path %q, which has a placeholder; a seat's path is written "+
"literally (novox/hq ADR 0287)", seat, p)
case !strings.HasPrefix(p, "/") || path.Clean(p) != p:
return fmt.Sprintf("the seat %s names the path %q, which is not a clean absolute path (novox/hq ADR 0287)",
seat, p)
case path.Dir(p) == "/":
// The root or a top-level directory is every role's, never one role's own (ADR 0287).
return fmt.Sprintf("the seat %s names the path %q, the root or a top-level directory, which no one "+
"role owns (novox/hq ADR 0287)", seat, p)
}
return ""
}
// SeatPathProblems is what is wrong with the paths a set of the mesh's own seats names (novox/hq ADR
// 0287): only a node seat names one, each absolute and literal; every path has exactly one seat naming it
// without stepping back; and a seat steps back only for another node seat naming the same path without
// stepping back, so the stepping back cannot cycle.
func SeatPathProblems(set []Seat) []string {
var problems []string
owners := map[string][]string{} // path → the seats naming it without stepping back
stepping := map[string][]SeatPath{} // seat → the paths it names stepping back
byName := map[string]Seat{}
for _, s := range set {
byName[s.Name] = s
for _, p := range s.Paths {
if why := seatPathProblem(s.Name, s.Scope, p.Path); why != "" {
problems = append(problems, why)
}
if p.StepsBackFor == "" {
owners[p.Path] = append(owners[p.Path], s.Name)
} else {
stepping[s.Name] = append(stepping[s.Name], p)
}
}
}
for _, s := range set {
for _, p := range stepping[s.Name] {
other, known := byName[p.StepsBackFor]
switch {
case !known || other.Scope != ScopeNode:
problems = append(problems, fmt.Sprintf("the seat %s steps back from %q for %q, which is not a "+
"node seat of the mesh's (novox/hq ADR 0287)", s.Name, p.Path, p.StepsBackFor))
case !namesWithoutSteppingBack(other, p.Path):
problems = append(problems, fmt.Sprintf("the seat %s steps back from %q for %s, which does not "+
"own that path itself; a seat steps back only for the one that owns it (novox/hq ADR 0287)",
s.Name, p.Path, other.Name))
}
}
}
for _, p := range sortedKeys(owners) {
if len(owners[p]) > 1 {
problems = append(problems, fmt.Sprintf("the path %q is named by %s, each without stepping back; "+
"no two seats name one path (novox/hq ADR 0287)", p, strings.Join(owners[p], " and ")))
}
}
for _, s := range set {
for _, p := range stepping[s.Name] {
if len(owners[p.Path]) == 0 {
problems = append(problems, fmt.Sprintf("the path %q has no seat naming it without stepping back "+
"(novox/hq ADR 0287)", p.Path))
}
}
}
return problems
}
func namesWithoutSteppingBack(s Seat, p string) bool {
for _, sp := range s.Paths {
if sp.Path == p && sp.StepsBackFor == "" {
return true
}
}
return false
}
// declaredPaths is every machine path a module declares it owns: a resource's literal path, and a file
// the mesh renders for it outside the operator's home. An access is not here — reading is not owning.
//
// **Cleaned, because the machine writes the cleaned path**: /etc//hosts, /etc/./hosts, /etc/../etc/hosts and
// /etc/hosts/ are all /etc/hosts to it, and a seat's path is clean by its own rule, so a declaration compared
// as written would pass for a path nobody named.
func declaredPaths(m Manifest) map[string]bool {
out := map[string]bool{}
for _, r := range m.Resources {
if p, ok := r["path"].(string); ok && strings.HasPrefix(p, "/") {
out[path.Clean(p)] = true
}
}
for _, f := range m.Facts {
if !f.Home && strings.HasPrefix(f.Path, "/") {
out[path.Clean(f.Path)] = true
}
}
return out
}
// declaresPath says whether a module declares one path, as a resource or as a file the mesh renders.
func declaresPath(m Manifest, p string) bool { return declaredPaths(m)[p] }
// missingPaths is each path a seat names that a module does not declare, in the seat's order.
func missingPaths(m Manifest, paths []SeatPath) []string {
if len(paths) == 0 {
return nil
}
has := declaredPaths(m)
var missing []string
for _, p := range paths {
if !has[p.Path] {
missing = append(missing, p.Path)
}
}
return missing
}
// pathSeats is which seats name each path: the mesh's own, and then those modules declare, in the order
// given. A module's seat naming a path a seat already names loses it, and the refusal says so (novox/hq
// ADR 0287 — as for a seat's name, the second declaration loses).
func pathSeats(order []string, shelf Shelf, declaredBy map[string]string) (map[string][]string, []string) {
named := map[string][]string{}
for _, s := range Seats() {
for _, p := range s.Paths {
named[p.Path] = append(named[p.Path], s.Name)
}
}
var problems []string
for _, module := range order {
for _, s := range shelf[module].DefinesSeats {
if declaredBy != nil && declaredBy[s.Name] != module {
continue // a second declaration of the seat's name, refused for that already
}
for _, p := range s.Paths {
if others := named[p]; len(others) > 0 {
problems = append(problems, fmt.Sprintf(
"%s declares the seat %s naming the path %q, which %s already names; no two seats "+
"name one path (novox/hq ADR 0287)", module, s.Name, p, strings.Join(others, " and ")))
continue
}
named[p] = append(named[p], s.Name)
}
}
}
return named, problems
}
// unclaimedSeatPaths is each path a module declares that a seat names while it claims none of the seats
// naming it (novox/hq ADR 0287): only a claimant declares a seat's path, and a module claiming any one of
// them may.
func unclaimedSeatPaths(m Manifest, named map[string][]string) []string {
claims := map[string]bool{}
for _, c := range m.Claims {
claims[canonicalSeat(c.Name)] = true
}
var problems []string
declared := declaredPaths(m)
for _, p := range sortedKeys(declared) {
seats := named[p]
if len(seats) == 0 {
continue
}
claimsOne := false
for _, s := range seats {
claimsOne = claimsOne || claims[s]
}
if !claimsOne {
problems = append(problems, fmt.Sprintf(
"%s declares the path %q, which the seat %s names, and claims none of them; only a holder of "+
"a seat naming a path declares it (novox/hq ADR 0287)", m.Module, p, strings.Join(seats, " and ")))
}
}
return problems
}
// declaredSeatPathProblems is what one module's own seat declaration can be judged on alone: a path
// only on a node seat, absolute, literal and clean.
func declaredSeatPathProblems(m Manifest, s SeatDeclaration) []string {
var problems []string
for _, p := range s.Paths {
if why := seatPathProblem(s.Name, s.At(), p); why != "" {
problems = append(problems, m.Module+": "+why)
}
}
return problems
}
// asSeatPaths is a module's own seat's paths in the shape the mesh's seats name theirs. A module's seat
// never steps back: only the mesh's own seats do, each by a record.
func asSeatPaths(paths []string) []SeatPath {
out := make([]SeatPath, 0, len(paths))
for _, p := range paths {
out = append(out, SeatPath{Path: p})
}
return out
}
// PathOwner is the module that owns one seat's path on one machine (novox/hq ADR 0287): the seat naming
// the path, and that seat's holder there, with the stepping back applied.
type PathOwner struct {
Path string `json:"path"`
Node string `json:"node"`
Seat string `json:"seat"`
Module string `json:"module"`
}
// PathOwners is, for every path a seat in the set names and every machine where a seat naming it is held,
// who owns the path there. A seat that steps back for another held on the same machine is passed over.
func PathOwners(set []Seat, held []Held) []PathOwner {
// node → seat → holding module
holders := map[string]map[string]string{}
for _, h := range held {
s, ok := SeatNamed(h.Claim)
if !ok || h.Scope != ScopeNode || s.Scope != ScopeNode {
continue
}
if holders[h.Node] == nil {
holders[h.Node] = map[string]string{}
}
if _, taken := holders[h.Node][s.Name]; !taken {
holders[h.Node][s.Name] = h.Module
}
}
naming := map[string][]Seat{} // path → the seats naming it
steps := map[string]map[string]string{} // path → seat → the seat it steps back for
for _, s := range set {
for _, p := range s.Paths {
naming[p.Path] = append(naming[p.Path], s)
if p.StepsBackFor != "" {
if steps[p.Path] == nil {
steps[p.Path] = map[string]string{}
}
steps[p.Path][s.Name] = p.StepsBackFor
}
}
}
var out []PathOwner
for _, p := range sortedKeys(naming) {
for _, node := range sortedKeys(holders) {
for _, s := range naming[p] {
module, holds := holders[node][s.Name]
if !holds {
continue
}
if forSeat := steps[p][s.Name]; forSeat != "" {
if _, there := holders[node][forSeat]; there {
continue
}
}
out = append(out, PathOwner{Path: p, Node: node, Seat: s.Name, Module: module})
break
}
}
}
return out
}
// seatPathsOnMachine is each path the modules of one machine declare that a seat names, for a module
// claiming none of the seats naming it (novox/hq ADR 0287). Registration refuses the same of a manifest;
// this holds what the controller generates for a machine to it too, because a computed module's resources
// exist only once composed (checkResources is asked them there). The seats are the mesh's own and those the
// machine's modules declare.
func seatPathsOnMachine(modules []Manifest) []string {
shelf := Shelf{}
var order []string
for _, m := range modules {
shelf[m.Module] = m
order = append(order, m.Module)
}
named, _ := pathSeats(order, shelf, nil)
var problems []string
for _, m := range modules {
problems = append(problems, unclaimedSeatPaths(m, named)...)
}
return problems
}
-311
View File
@@ -1,311 +0,0 @@
package catalogue
import (
"strings"
"testing"
)
// Defends novox/hq ADR 0287 (issue 412): a seat names the paths its holder owns, a claim without them is
// refused, and only a claimant declares a seat's path.
// The starting set, exactly: node-hostname names /etc/hosts and /etc/hostname, node-uplink the resolver
// file stepping back for node-resolver, node-resolver the resolver file, and every other seat names none.
func TestTheSeatsThatNamePathsAreTheStartingSet(t *testing.T) {
want := map[string]string{
"node-hostname": "/etc/hosts /etc/hostname",
"node-uplink": "/etc/resolv.conf>node-resolver",
ResolverSeat: "/etc/resolv.conf",
}
for _, s := range DefaultSeats() {
var got []string
for _, p := range s.Paths {
if p.StepsBackFor != "" {
got = append(got, p.Path+">"+p.StepsBackFor)
} else {
got = append(got, p.Path)
}
}
if strings.Join(got, " ") != want[s.Name] {
t.Errorf("%s names %v; ADR 0287 gives it %q", s.Name, got, want[s.Name])
}
}
}
// The compiled set keeps the rules: node seats only, absolute and literal, one seat per path without
// stepping back, and a stepping back that cannot cycle.
func TestTheCompiledSeatPathsKeepTheRules(t *testing.T) {
if p := SeatPathProblems(DefaultSeats()); len(p) != 0 {
t.Fatalf("the compiled set breaks ADR 0287: %v", p)
}
for _, c := range []struct {
why string
set []Seat
want string
}{
{"a mesh seat naming a path", []Seat{{Name: "mesh-x", Scope: ScopeMesh, Paths: []SeatPath{{Path: "/etc/x"}}}},
"only a node seat names paths"},
{"a placeholder", []Seat{{Name: "node-x", Scope: ScopeNode, Paths: []SeatPath{{Path: "/etc/${setting:x}"}}}},
"placeholder"},
{"a relative path", []Seat{{Name: "node-x", Scope: ScopeNode, Paths: []SeatPath{{Path: "etc/x"}}}},
"not a clean absolute path"},
{"two owners", []Seat{
{Name: "node-a", Scope: ScopeNode, Paths: []SeatPath{{Path: "/etc/x"}}},
{Name: "node-b", Scope: ScopeNode, Paths: []SeatPath{{Path: "/etc/x"}}}}, "no two seats name one path"},
{"a cycle", []Seat{
{Name: "node-a", Scope: ScopeNode, Paths: []SeatPath{{Path: "/etc/x", StepsBackFor: "node-b"}}},
{Name: "node-b", Scope: ScopeNode, Paths: []SeatPath{{Path: "/etc/x", StepsBackFor: "node-a"}}}},
"does not own that path itself"},
{"stepping back for nobody", []Seat{
{Name: "node-a", Scope: ScopeNode, Paths: []SeatPath{{Path: "/etc/x", StepsBackFor: "node-z"}}}},
"not a node seat"},
} {
if got := strings.Join(SeatPathProblems(c.set), "; "); !strings.Contains(got, c.want) {
t.Errorf("%s: %q does not say %q", c.why, got, c.want)
}
}
}
// What the store's rows carry is never the paths: a row read back keeps the compiled ones.
func TestTheSeatPathsSurviveTheStoresRows(t *testing.T) {
before := seats
t.Cleanup(func() { seats = before })
UseSeats([]Seat{{Name: "node-hostname", Scope: ScopeNode}})
s, _ := SeatNamed("node-hostname")
if len(s.Paths) != 2 {
t.Fatalf("node-hostname read back from its row names %v", s.Paths)
}
}
func hostnameSeat(t *testing.T) Seat {
t.Helper()
s, ok := SeatNamed("node-hostname")
if !ok {
t.Fatal("no node-hostname seat")
}
return s
}
func hostnameHolder(resources ...map[string]any) Manifest {
return Manifest{Module: "names", Version: "1", Claims: []Claim{{Name: "node-hostname", Scope: ScopeNode,
Serves: []string{"entries", "add", "remove"}}}, Resources: resources}
}
// A holder declares every path its seat names, as a resource or as a file the mesh renders for it, or
// CanHold refuses it naming the seat and each missing path — the judgement registration and handover share.
func TestAClaimWithoutTheSeatsPathsIsRefused(t *testing.T) {
seat := hostnameSeat(t)
hosts := map[string]any{"id": "hosts", "type": "file", "path": "/etc/hosts", "content": "x"}
name := map[string]any{"id": "name", "type": "file", "path": "/etc/hostname", "content": "x"}
if err := CanHold(hostnameHolder(hosts, name), seat); err != nil {
t.Errorf("a holder declaring both paths as resources was refused: %v", err)
}
rendered := hostnameHolder(hosts)
rendered.Facts = map[string]RosterFile{"name": {Path: "/etc/hostname", Template: "{{.Node}}\n"}}
if err := CanHold(rendered, seat); err != nil {
t.Errorf("a holder declaring a path as a file the mesh renders was refused: %v", err)
}
err := CanHold(hostnameHolder(), seat)
if err == nil {
t.Fatal("a holder declaring neither path was accepted")
}
for _, want := range []string{"node-hostname", "/etc/hosts", "/etc/hostname", "ADR 0287"} {
if !strings.Contains(err.Error(), want) {
t.Errorf("the refusal does not name %s: %v", want, err)
}
}
// A file under the operator's home is placed per account: it is not the machine's path.
home := hostnameHolder(hosts)
home.Facts = map[string]RosterFile{"name": {Path: "/etc/hostname", Home: true, Template: "x"}}
if err := CanHold(home, seat); err == nil {
t.Error("a file in the operator's home counted as the machine's /etc/hostname")
}
// At registration, the same refusal.
if got := strings.Join(CatalogueProblems(Shelf{"names": hostnameHolder(hosts)}), "; "); !strings.Contains(got,
"does not declare /etc/hostname") {
t.Errorf("registration let a holder without /etc/hostname through: %q", got)
}
}
// Only a claimant declares a seat's path: a module claiming no seat that names /etc/hosts and declaring
// it is refused at registration naming the seat; the uplink's holder and the resolver's both declaring
// the resolver file are each a claimant of a seat naming it. Reading a seat's path is access, and allowed.
func TestOnlyAClaimantDeclaresASeatsPath(t *testing.T) {
stranger := Manifest{Module: "stranger", Version: "1", Resources: []map[string]any{
{"id": "hosts", "type": "file", "path": "/etc/hosts", "into": "block", "content": "10.0.0.1 x\n"}}}
got := strings.Join(CatalogueProblems(Shelf{"stranger": stranger}), "; ")
if !strings.Contains(got, `stranger declares the path "/etc/hosts", which the seat node-hostname names`) {
t.Errorf("a module declaring /etc/hosts without claiming node-hostname was not refused: %q", got)
}
// However the path is spelled (novox/hq issue 412 review).
unclean := Manifest{Module: "unclean", Version: "1", Resources: []map[string]any{
{"id": "hosts", "type": "file", "path": "/etc/../etc/hosts", "content": "10.0.0.1 x\n"}}}
if got := strings.Join(CatalogueProblems(Shelf{"unclean": unclean}), "; "); !strings.Contains(got, "node-hostname") {
t.Errorf("/etc/../etc/hosts from a module not claiming node-hostname was not refused: %q", got)
}
reader := Manifest{Module: "reader", Version: "1", Accesses: []Access{{ID: "hosts", Path: "/etc/hosts"}}}
if got := CatalogueProblems(Shelf{"reader": reader}); len(got) != 0 {
t.Errorf("reading /etc/hosts was refused: %v", got)
}
uplink := Manifest{Module: "uplink", Version: "1", Claims: []Claim{{Name: "node-uplink", Scope: ScopeNode}},
Facts: map[string]RosterFile{"resolvers": {Path: ResolverFile, Template: "nameserver 10.42.0.1\n"}}}
resolver := localResolver()
resolver.Claims[0].Serves = []string{"routes", "route", "unroute"}
if got := CatalogueProblems(Shelf{"uplink": uplink, resolver.Module: resolver}); len(got) != 0 {
t.Errorf("the uplink's holder and the resolver's declaring the resolver file were refused: %v", got)
}
}
// A module's own seat names paths under the same rules: a node seat only, literal, and none another seat
// names — the second declaration loses. Its holder declares them, as a mesh seat's does.
func TestAModulesOwnSeatNamesPathsUnderTheSameRules(t *testing.T) {
meshScoped := Manifest{Module: "a", Version: "1", DefinesSeats: []SeatDeclaration{
{Name: "a-seat", Paths: []string{"/etc/a"}}}}
if got := strings.Join(declaredSeatProblems(meshScoped), "; "); !strings.Contains(got, "only a node seat names paths") {
t.Errorf("a module's mesh seat naming a path was accepted: %q", got)
}
placeholder := Manifest{Module: "a", Version: "1", DefinesSeats: []SeatDeclaration{
{Name: "a-seat", Scope: ScopeNode, Paths: []string{"${dir:state}/a"}}}}
if got := strings.Join(declaredSeatProblems(placeholder), "; "); !strings.Contains(got, "placeholder") {
t.Errorf("a module's seat naming a path with a placeholder was accepted: %q", got)
}
taken := Manifest{Module: "b", Version: "1", DefinesSeats: []SeatDeclaration{
{Name: "b-seat", Scope: ScopeNode, Paths: []string{"/etc/hosts"}}}}
if got := strings.Join(CatalogueProblems(Shelf{"b": taken}), "; "); !strings.Contains(got,
`b declares the seat b-seat naming the path "/etc/hosts", which node-hostname already names`) {
t.Errorf("a module's seat naming /etc/hosts was accepted: %q", got)
}
own := Manifest{Module: "c", Version: "1",
DefinesSeats: []SeatDeclaration{{Name: "c-seat", Scope: ScopeNode, Paths: []string{"/etc/c.conf"}}},
Claims: []Claim{{Name: "c-seat", Scope: ScopeNode}}}
if got := strings.Join(CatalogueProblems(Shelf{"c": own}), "; "); !strings.Contains(got,
"c claims c-seat but does not declare /etc/c.conf") {
t.Errorf("a holder of a module's seat without its path was accepted: %q", got)
}
own.Resources = []map[string]any{{"id": "conf", "type": "file", "path": "/etc/c.conf", "content": "x"}}
if got := CatalogueProblems(Shelf{"c": own}); len(got) != 0 {
t.Errorf("a holder declaring its own seat's path was refused: %v", got)
}
other := Manifest{Module: "d", Version: "1", Resources: []map[string]any{
{"id": "conf", "type": "file", "path": "/etc/c.conf", "content": "x"}}}
if got := strings.Join(CatalogueProblems(Shelf{"c": own, "d": other}), "; "); !strings.Contains(got,
`d declares the path "/etc/c.conf", which the seat c-seat names`) {
t.Errorf("a module not claiming c-seat declared its path: %q", got)
}
}
type writesHosts struct{}
func (writesHosts) Resources(string) ([]map[string]any, bool, error) {
return []map[string]any{{"id": "hosts", "type": "file", "path": "/etc/hosts", "into": "block",
"content": "10.42.0.1 anchor\n"}}, true, nil
}
// The controller is bound too: what it generates for a machine on a seat's path is refused at composition.
func TestAGeneratedResourceOnASeatsPathIsRefused(t *testing.T) {
r := Resolution{Node: "workstation", Modules: []Manifest{
{Module: "mesh-names", Computed: "mesh-names"},
}}
_, err := r.Declaration(Rendering{Generators: map[string]Generator{"mesh-names": writesHosts{}}})
if err == nil {
t.Fatal("a generated resource on /etc/hosts was composed")
}
for _, want := range []string{"mesh-names", "/etc/hosts", "node-hostname"} {
if !strings.Contains(err.Error(), want) {
t.Errorf("the refusal does not name %s: %v", want, err)
}
}
}
// Who owns a seat's path on a machine: the seat naming it and its holder there, with the stepping back
// applied — the resolver's holder where node-resolver is held, else the uplink's.
func TestPathOwnersApplyTheSteppingBack(t *testing.T) {
held := []Held{
{Claim: "node-uplink", Scope: ScopeNode, Node: "laptop", Module: "networkmanager"},
{Claim: ResolverSeat, Scope: ScopeNode, Node: "laptop", Module: "systemd-resolved"},
{Claim: "node-uplink", Scope: ScopeNode, Node: "server", Module: "systemd-networkd"},
{Claim: "node-hostname", Scope: ScopeNode, Node: "server", Module: "hostname"},
}
var got []string
for _, o := range PathOwners(Seats(), held) {
got = append(got, o.Path+" "+o.Node+" "+o.Seat+" "+o.Module)
}
want := []string{
"/etc/hostname server node-hostname hostname",
"/etc/hosts server node-hostname hostname",
"/etc/resolv.conf laptop node-resolver systemd-resolved",
"/etc/resolv.conf server node-uplink systemd-networkd",
}
if strings.Join(got, "\n") != strings.Join(want, "\n") {
t.Errorf("the owners are\n%s\nnot\n%s", strings.Join(got, "\n"), strings.Join(want, "\n"))
}
}
// The stepping back comes from the seat, and is of a rendered file only: an uplink holder declaring the
// resolver file as a resource beside a node-resolver holder is two writers of one path, refused when the
// machine is resolved — never composed twice — as before the seat named its paths.
func TestAnUplinkDeclaringTheResolverFileAsAResourceIsRefusedBesideTheResolver(t *testing.T) {
shelf := resolverShelf(t)
uplink := shelf["systemd-networkd"]
uplink.Facts = nil
uplink.Resources = append(append([]map[string]any(nil), uplink.Resources...),
map[string]any{"id": "resolvers", "type": "file", "path": ResolverFile, "content": "nameserver 10.42.0.1\n"})
shelf["systemd-networkd"] = uplink
shelf["local-resolver"] = localResolver()
_, err := Resolve(shelf, []string{"dnsmasq", "systemd-networkd", "local-resolver"}, splitDNSLaptop(), World{})
if err == nil || !strings.Contains(err.Error(), `both declare the path "/etc/resolv.conf"`) {
t.Fatalf("an uplink declaring the resolver file as a resource beside the resolver was not refused: %v", err)
}
// Its rendered file steps back, by the seat's definition, and nothing else of it does.
rendered := Manifest{Module: "uplink", Version: "1", Claims: []Claim{{Name: "node-uplink"}},
Facts: map[string]RosterFile{"resolvers": {Path: "/etc//resolv.conf", Template: "x"},
"other": {Path: "/etc/uplink.conf", Template: "x"}}}
got := stepsBack(rendered, []Manifest{rendered, localResolver()})
if len(got.Facts) != 1 || got.Facts["other"].Path == "" {
t.Errorf("the uplink's holder renders %v beside the resolver", got.Facts)
}
}
// A path is compared as the machine writes it: cleaned. Each spelling of /etc/hosts is /etc/hosts.
func TestAnUncleanSpellingOfASeatsPathIsTheSeatsPath(t *testing.T) {
for _, spelled := range []string{"/etc//hosts", "/etc/./hosts", "/etc/../etc/hosts", "/etc/hosts/"} {
stranger := Manifest{Module: "stranger", Version: "1", Resources: []map[string]any{
{"id": "hosts", "type": "file", "path": spelled, "content": "10.0.0.1 x\n"}}}
if got := strings.Join(CatalogueProblems(Shelf{"stranger": stranger}), "; "); !strings.Contains(got,
`stranger declares the path "/etc/hosts", which the seat node-hostname names`) {
t.Errorf("%s from a module not claiming node-hostname was not refused: %q", spelled, got)
}
holder := hostnameHolder(map[string]any{"id": "hosts", "type": "file", "path": spelled, "content": "x"},
map[string]any{"id": "name", "type": "file", "path": "/etc/hostname", "content": "x"})
if err := CanHold(holder, hostnameSeat(t)); err != nil {
t.Errorf("a holder declaring %s was refused: %v", spelled, err)
}
}
}
// A seat names a role's own path, never the root or a top-level directory.
func TestASeatNamesNoRootOrTopLevelDirectory(t *testing.T) {
for _, p := range []string{"/", "/etc"} {
m := Manifest{Module: "a", Version: "1", DefinesSeats: []SeatDeclaration{
{Name: "a-seat", Scope: ScopeNode, Paths: []string{p}}}}
if got := strings.Join(declaredSeatProblems(m), "; "); !strings.Contains(got, "top-level directory") {
t.Errorf("a module's seat naming %s was accepted: %q", p, got)
}
}
}
// The catalogue as it is passes: no holder of a seat naming a path is missing one, and nobody else
// declares one.
func TestTheCataloguePassesTheSeatPathRules(t *testing.T) {
shelf := Shelf{}
for _, m := range theCatalogue(t) {
shelf[m.Module] = m
}
for _, p := range CatalogueProblems(shelf) {
if strings.Contains(p, "ADR 0287") {
t.Errorf("the catalogue breaks ADR 0287: %s", p)
}
}
}
+4 -25
View File
@@ -53,11 +53,6 @@ type Seat struct {
// ${contribution:<seat>:<kind>}. Compiled, never stored: like the protocol, it is the mesh's
// definition of the role, and the store's rows carry no column for it.
Receives []Receivable
// Paths are the paths on its machine the seat's holder owns (novox/hq ADR 0287): a holder declares
// every one, and a module claiming no seat that names a path may not declare it. Only a node seat
// names paths. Compiled, never stored, like Receives: adding or taking away a path is a decision,
// recorded, and the store's rows carry no column for it.
Paths []SeatPath
// Decision is the record that made it a seat.
Decision string
}
@@ -183,8 +178,6 @@ var defaultSeats = append([]Seat{
// holds none of it. Named node-hosts-file until ADR 0223; the former name resolves to it as an
// alias on a mesh that knew it.
{Name: "node-hostname", Scope: ScopeNode, Decision: "novox/hq ADR 0199, ADR 0223",
// The machine's names are the role's files, whatever program fills it (novox/hq ADR 0287).
Paths: []SeatPath{{Path: "/etc/hosts"}, {Path: "/etc/hostname"}},
Serves: []Verb{
{Name: "entries", Description: "Every line of this machine's /etc/hosts, each marked whose it is: " +
"the operator's, or the block of the module or tool that writes it.",
@@ -306,17 +299,12 @@ var defaultSeats = append([]Seat{
// resolver file as it is and who wrote it, and every link with its addresses, routes and the resolvers
// the manager knows for it — the same for every holder, so a question about a machine's names is asked
// the same way whatever manages its network. Optional until both holders serve them.
//
// It owns the resolver file, stepping back for node-resolver where that is held (novox/hq ADR 0287,
// ADR 0247).
{Name: "node-uplink", Scope: ScopeNode, Decision: "novox/hq ADR 0117", Serves: uplinkVerbs(),
Paths: []SeatPath{{Path: ResolverFile, StepsBackFor: ResolverSeat}}},
{Name: "node-uplink", Scope: ScopeNode, Decision: "novox/hq ADR 0117", Serves: uplinkVerbs()},
// The machine's own resolver (novox/hq ADR 0247): held only where something requires `split-dns` — a
// VPN client whose domains must go to its own servers while every other name goes to the mesh's.
// Where it is held, its holder writes the resolver file and the uplink's holder steps back from it
// (node_resolver.go). It knows nothing of any VPN: its verbs route domains to servers over a link.
{Name: ResolverSeat, Scope: ScopeNode, Decision: "novox/hq ADR 0247", Serves: resolverVerbs(),
Paths: []SeatPath{{Path: ResolverFile}}},
{Name: ResolverSeat, Scope: ScopeNode, Decision: "novox/hq ADR 0247", Serves: resolverVerbs()},
// A machine's shares and the shares it mounts (novox/hq ADR 0263): the holder of node-nfs-server
// exports a machine's folders to the private network and provides each as `nfs-share`; the holder of
// node-mounts writes a mount and an automount unit per share on a machine that asks (shares.go).
@@ -376,11 +364,10 @@ func UseSeats(s []Seat) {
row.Accepts, row.Emits, row.Serves = d.Accepts, d.Emits, d.Serves
}
}
// What a seat receives, whether it is replicated and the paths it names are never stored
// (novox/hq ADR 0212, ADR 0223, ADR 0287), so they are always the compiled ones.
// What a seat receives and whether it is replicated are never stored (novox/hq ADR 0212, ADR
// 0223), so they are always the compiled ones.
if d, known := byName[row.Name]; known {
row.Receives = d.Receives
row.Paths = d.Paths
row.Replicated = d.Replicated
row.Serves = optionalAsCompiled(row.Serves, d.Serves)
}
@@ -554,14 +541,6 @@ func CanHold(m Manifest, seat Seat) error {
return fmt.Errorf("%s claims %s, whose holder answers for %q, and %s does not provide %q at %s scope",
m.Module, seat.Name, seat.Delivers, m.Module, seat.Delivers, seat.Scope)
}
// **A holder owns what the role owns** (novox/hq ADR 0287): every path the seat names, declared as
// a resource or as a file the mesh renders for it, so which module owns /etc/hosts on a machine is
// answered by the seat's definition and not by reading every manifest.
if missing := missingPaths(m, seat.Paths); len(missing) > 0 {
return fmt.Errorf("%s claims %s but does not declare %s, which that seat's holder owns (novox/hq "+
"ADR 0287) — a holder declares every path its seat names, as a resource or as a file the mesh "+
"renders for it", m.Module, seat.Name, strings.Join(missing, ", "))
}
// **Serving the seat's tools is a condition of holding it** (novox/hq ADR 0132). A holder that
// does not answer what the role promises is every caller's timeout, found at registration and
// at handover instead, naming the verbs rather than the fact that something is missing.
-17
View File
@@ -67,10 +67,6 @@ type SeatDeclaration struct {
// Records are state buckets of the declaring module that each user reads under its own name — the
// keys `<user>.…` and no other (ADR 0259 §3).
Records []string `json:"records,omitempty"`
// Paths are the paths on its machine the seat's holder owns (novox/hq ADR 0287), under the rules the
// mesh's own seats name theirs by: a node seat only, each absolute and literal, and none another seat
// names. A module's seat never steps back.
Paths []string `json:"paths,omitempty"`
}
// KindedBenches are the seats that may be kinded (novox/hq ADR 0234 §2): `channel` sends to the operator,
@@ -167,7 +163,6 @@ func declaredSeatProblems(m Manifest) []string {
}
}
problems = append(problems, trafficProblems(m, s)...)
problems = append(problems, declaredSeatPathProblems(m, s)...)
}
for _, u := range m.Uses {
@@ -276,10 +271,6 @@ func CatalogueProblems(shelf Shelf) []string {
return ok
}
// Which seats name each path (novox/hq ADR 0287): a module's seat naming one another seat names loses.
named, pathProblems := pathSeats(shelfOrder(shelf), shelf, declaredBy)
problems = append(problems, pathProblems...)
// Who claims each kind of a kinded bench, so a second claim of one kind is refused (ADR 0234 §2).
kindsTaken := map[string]string{}
@@ -336,15 +327,7 @@ func CatalogueProblems(shelf Shelf) []string {
"%s claims %s but does not serve %s, which that seat's protocol promises",
module, c.Name, strings.Join(missing, ", ")))
}
// And owns what the seat's holder owns, as a holder of the mesh's own seats does (ADR 0287).
if missing := missingPaths(m, asSeatPaths(s.Paths)); len(missing) > 0 {
problems = append(problems, fmt.Sprintf(
"%s claims %s but does not declare %s, which that seat's holder owns (novox/hq ADR 0287)",
module, c.Name, strings.Join(missing, ", ")))
}
}
// Only a claimant declares a seat's path (novox/hq ADR 0287).
problems = append(problems, unclaimedSeatPaths(m, named)...)
}
// **A trusted holder runs as its own account** (novox/hq ADR 0259 §8): a module saying warrants, or
// speaking for a kind that proves its sender, is never carried by a machine's runtime.
+2 -5
View File
@@ -33,9 +33,7 @@ func TestTheUplinkSeatPromisesItsVerbsAndRequiresNoneYet(t *testing.T) {
if !slices.Equal(names, []string{"resolvers", "links"}) {
t.Fatalf("node-uplink promises %v", names)
}
// The resolver file is the seat's path (novox/hq ADR 0287), which every holder of today renders.
today := Manifest{Module: "networkmanager", Version: "1", Claims: []Claim{{Name: "node-uplink", Scope: ScopeNode}},
Facts: map[string]RosterFile{"resolvers": {Path: ResolverFile, Template: "x"}}}
today := Manifest{Module: "networkmanager", Version: "1", Claims: []Claim{{Name: "node-uplink", Scope: ScopeNode}}}
if err := CanHold(today, seat); err != nil {
t.Fatalf("a holder that serves no verb yet is refused: %v", err)
}
@@ -57,8 +55,7 @@ func TestTheUplinkVerbsReadBackFromTheRowStayOptional(t *testing.T) {
// The row as seeding stores it: both verbs, the mark not a column.
UseSeats([]Seat{{Name: "node-uplink", Scope: ScopeNode, Serves: []Verb{{Name: "resolvers"}, {Name: "links"}}}})
seat := uplinkSeat(t)
holder := Manifest{Module: "systemd-networkd", Version: "1", Claims: []Claim{{Name: "node-uplink", Scope: ScopeNode}},
Facts: map[string]RosterFile{"resolvers": {Path: ResolverFile, Template: "x"}}}
holder := Manifest{Module: "systemd-networkd", Version: "1", Claims: []Claim{{Name: "node-uplink", Scope: ScopeNode}}}
if err := CanHold(holder, seat); err != nil {
t.Fatalf("read back from the row, the verbs are required: %v", err)
}
-74
View File
@@ -1,74 +0,0 @@
package inventory
import (
"context"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/jackc/pgx/v5"
)
// MachineAccount is what a machine said it is made of (novox/hq issue 160), from the profile it last
// reported: the architecture and kernel family the profile always carried, and the facts a node-engine that
// knows how says beside them. Read by `node`, so what a machine says of itself is visible in one place —
// collected and shown in the same breath, not collected and read by nothing (issue 160's pattern).
type MachineAccount struct {
Architecture string `json:"architecture"`
Kernel string `json:"kernel"`
// Machine is nil when the node-engine that reported is older than the one that says it.
Machine *MadeOf `json:"machine"`
}
// MadeOf is the node-engine's account of what it is made of, as it put it on the wire. A fact it could
// not determine is absent here and named in Unknown with why — never a zero.
type MadeOf struct {
ObservedAt time.Time `json:"observed_at"`
KernelRelease string `json:"kernel_release"`
Distribution string `json:"distribution"`
Version string `json:"version"`
Processor string `json:"processor"`
Cores int `json:"cores"`
Threads int `json:"threads"`
MemoryBytes int64 `json:"memory_bytes"`
Virtual string `json:"virtual"`
BootedAt *time.Time `json:"booted_at"`
Timezone string `json:"timezone"`
Filesystems []MountedFilesystem `json:"filesystems"`
Unknown map[string]string `json:"unknown"`
}
// MountedFilesystem is one local filesystem a machine has mounted.
type MountedFilesystem struct {
Mount string `json:"mount"`
Device string `json:"device"`
Type string `json:"type"`
SizeBytes int64 `json:"size_bytes"`
FreeBytes int64 `json:"free_bytes"`
}
// MachineOf is what a node last said it is made of. had is false for a node that has never reported a
// profile at all, which is a different answer from one whose node-engine does not say the facts yet.
func (i *Inventory) MachineOf(ctx context.Context, nodeName string) (account MachineAccount, had bool, err error) {
var raw []byte
err = i.store.Pool().QueryRow(ctx, `select profile from node where name = $1`, nodeName).Scan(&raw)
if errors.Is(err, pgx.ErrNoRows) {
return MachineAccount{}, false, fmt.Errorf("%w: %s", ErrNoSuchNode, nodeName)
}
if err != nil {
return MachineAccount{}, false, err
}
return machineFrom(raw)
}
func machineFrom(raw []byte) (MachineAccount, bool, error) {
if len(raw) == 0 {
return MachineAccount{}, false, nil
}
var account MachineAccount
if err := json.Unmarshal(raw, &account); err != nil {
return MachineAccount{}, false, err
}
return account, true, nil
}
-42
View File
@@ -1,42 +0,0 @@
package inventory
import (
"testing"
)
// The node-engine's words for a machine's facts (novox/hq issue 160), as mesh-host's profile.Machine puts
// them on the wire. A rename on either side fails here rather than as a blank on a real machine.
func TestAMachinesAccountIsReadInTheNodeEnginesWords(t *testing.T) {
raw := []byte(`{"architecture":"amd64","kernel":"linux","capabilities":[],
"machine":{"observed_at":"2026-10-12T15:00:00Z","kernel_release":"7.2.8-arch1-2",
"distribution":"Arch Linux","version":"rolling","processor":"AMD Ryzen 9","cores":8,"threads":16,
"memory_bytes":32891494400,"virtual":"physical","booted_at":"2026-10-09T08:50:34Z",
"timezone":"Europe/Brussels",
"filesystems":[{"mount":"/","device":"/dev/nvme0n1p3","type":"ext4","size_bytes":100,"free_bytes":40}],
"unknown":{"public address":"not decided"}}}`)
a, had, err := machineFrom(raw)
if err != nil || !had {
t.Fatalf("had %v, err %v", had, err)
}
m := a.Machine
if a.Architecture != "amd64" || a.Kernel != "linux" || m == nil {
t.Fatalf("%+v", a)
}
if m.KernelRelease != "7.2.8-arch1-2" || m.Distribution != "Arch Linux" || m.Version != "rolling" ||
m.Processor != "AMD Ryzen 9" || m.Cores != 8 || m.Threads != 16 || m.MemoryBytes != 32891494400 ||
m.Virtual != "physical" || m.BootedAt == nil || m.Timezone != "Europe/Brussels" ||
len(m.Filesystems) != 1 || m.Filesystems[0].FreeBytes != 40 || m.Unknown["public address"] == "" ||
m.ObservedAt.IsZero() {
t.Errorf("%+v", m)
}
}
func TestAProfileWithoutTheFactsIsAnOlderNodeEngineNotNothing(t *testing.T) {
a, had, err := machineFrom([]byte(`{"architecture":"amd64","kernel":"linux","capabilities":[]}`))
if err != nil || !had || a.Machine != nil || a.Architecture != "amd64" {
t.Errorf("%+v had %v err %v", a, had, err)
}
if _, had, _ := machineFrom(nil); had {
t.Error("a node that never reported read as having reported")
}
}
+4 -7
View File
@@ -79,6 +79,8 @@ func (b *natsBuilds) Ask(ctx context.Context, request BuildRequest) error {
if _, err := b.js.Context().Publish(BuildWorkOf(b.role()), body, nats.Context(publish)); err != nil {
return fmt.Errorf("cannot submit a build: %w", err)
}
// The acting controller puts it in order (novox/hq issue 558); this only says an ask joined.
signalAsked()
return nil
}
@@ -113,6 +115,8 @@ func (b *natsBuilds) Submit(ctx context.Context, request BuildRequest,
if _, err := b.js.Context().Publish(BuildWorkOf(b.role()), body, nats.Context(publish)); err != nil {
return BuildResult{}, fmt.Errorf("%w: cannot submit a build: %w", ErrNotHandedOver, err)
}
// The acting controller puts it in order (novox/hq issue 558); this only says an ask joined.
signalAsked()
waiting, cancelWait := context.WithTimeout(ctx, wait)
defer cancelWait()
@@ -167,10 +171,6 @@ type MachineOptions struct {
// Paused is asked before every fetch: while it says so, nothing new is taken, and a build
// already running finishes. Nil is never paused.
Paused func() bool
// Taking is called once, when this machine is bound to its worker and pulls work: from then on it
// can take a build, which is what its container's health check reads (novox/hq issue 544). Nil is
// told nothing.
Taking func()
}
// MachineOverNATS takes build work from the current build role.
@@ -236,9 +236,6 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
"the mesh's to answer, not this machine's", m.seat, err)
}
m.sub = sub
if m.opts.Taking != nil {
m.opts.Taking()
}
for {
if ctx.Err() != nil {
-33
View File
@@ -368,36 +368,3 @@ func TestNatsAMachineOnTheRetiredBuildRoleTakesThatRolesAsks(t *testing.T) {
t.Errorf("the retired role's holder did not answer: %+v", result)
}
}
// A holder says once that it takes work, and only once it is bound to its worker (novox/hq issue 544):
// its container's health check reads that, so a holder that could not bind is never said ready.
func TestNatsAHolderSaysItTakesWorkOnceItIsBound(t *testing.T) {
js := aBusWithTheBuildRole(t)
ctx, stop := context.WithCancel(context.Background())
defer stop()
said := make(chan struct{}, 4)
machine := MachineOverNATSWith(js, "anchor", TheBuildMachine, MachineOptions{Taking: func() { said <- struct{}{} }})
defer machine.Close()
go func() { _ = machine.Take(ctx, func(context.Context, Build) {}) }()
select {
case <-said:
case <-time.After(5 * time.Second):
t.Fatal("a holder bound to its worker never said it takes work")
}
time.Sleep(300 * time.Millisecond)
if len(said) != 0 {
t.Fatalf("a holder said it takes work %d more time(s)", len(said))
}
// A role the mesh raised no worker for cannot be bound, and its holder says nothing.
bounded, end := context.WithTimeout(ctx, 10*time.Second)
defer end()
other := MachineOverNATSWith(js, "anchor", "mesh-no-such-seat", MachineOptions{Taking: func() { said <- struct{}{} }})
defer other.Close()
if err := other.Take(bounded, func(context.Context, Build) {}); err == nil {
t.Fatal("a machine with no worker took work")
}
if len(said) != 0 {
t.Fatal("a machine that could not bind said it takes work")
}
}
-3
View File
@@ -325,9 +325,6 @@ type ChangePlan struct {
Steps []string `json:"steps,omitempty"`
// Summary is the plan in one line, for a commit status.
Summary string `json:"summary"`
// Class is the class of the slowest-budgeted module the change moves (novox/hq ADR 0282 §1, ADR 0313 §2):
// core, leaf, or none when it moves no module at all — a fix that moves nothing has no fix budget.
Class string `json:"class,omitempty"`
// Trunk is the trunk commit the head was merged into when it was judged (novox/hq ADR 0323 §3), twelve
// characters: set on the plan a verdict carries, which mesh-delivery keeps with its other inputs.
Trunk string `json:"trunk,omitempty"`
+26 -10
View File
@@ -62,7 +62,11 @@ type QueuedAsk struct {
Path string `json:"path,omitempty"`
Ref string `json:"ref,omitempty"`
AskedAt time.Time `json:"asked-at,omitempty"`
State string `json:"state"`
// Stored is when the stream first took it, a move into order kept (novox/hq issue 558).
Stored time.Time `json:"stored,omitempty"`
State string `json:"state"`
// movedFrom is the MovedFromHeader it carries: the sequence of the message a move copied it from.
movedFrom string
// On and Started are the holder building an in-flight ask and when it started, from its started
// event. Was is the holder that started an in-flight ask and is no longer building it — handed
// back, to be handed out again — with Started its start there.
@@ -236,15 +240,7 @@ func ReadQueue(ctx context.Context, js *broker.JetStream, seat string) (Queue, e
if err != nil {
return q, fmt.Errorf("reading the %s work queue: %w", seat, err)
}
ask := QueuedAsk{Seq: msg.Sequence}
if json.Unmarshal(msg.Data, &ask.Request) == nil {
r := ask.Request
ask.ID, ask.Repository, ask.Path, ask.Ref = r.ID, r.Repository, r.Path, r.Ref
if at, ok := BuildAskedAt(r.ID); ok {
ask.AskedAt = at
}
}
asks = append(asks, ask)
asks = append(asks, askOf(msg))
seq = msg.Sequence + 1
}
@@ -269,6 +265,26 @@ func ReadQueue(ctx context.Context, js *broker.JetStream, seat string) (Queue, e
return q, nil
}
// askOf is one ask as the stream holds it: what its request says, and when the stream took it.
func askOf(msg *jetstream.RawStreamMsg) QueuedAsk {
ask := QueuedAsk{Seq: msg.Sequence, Stored: msg.Time}
// Moved into order, it keeps when the stream first took it (novox/hq issue 558).
if at, err := time.Parse(time.RFC3339Nano, msg.Header.Get(QueuedAtHeader)); err == nil {
ask.Stored = at
}
if json.Unmarshal(msg.Data, &ask.Request) == nil {
r := ask.Request
ask.ID, ask.Repository, ask.Path, ask.Ref = r.ID, r.Repository, r.Path, r.Ref
if at, ok := BuildAskedAt(r.ID); ok {
ask.AskedAt = at
}
} else {
// Nothing read is nothing kept: a half-read request would pass for a delivery (novox/hq issue 558).
ask.Request = BuildRequest{}
}
return ask
}
// WithoutPassword is a repository as a listing may say it: a URL's password, should one ever be written
// into it, left out (novox/hq issue 455). Only for what is shown: the ask itself keeps what it was given,
// which a cancel's outcome and a rebuild's match are read from.
+426
View File
@@ -0,0 +1,426 @@
package link
import (
"context"
"errors"
"fmt"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
)
// The order the build queue hands asks out in (novox/hq issue 558).
//
// **A build a walk waits for goes before a pull request's check.** The queue was first come, first served,
// and a merged change's delivery build waited minutes behind checks while its walk, and every walk after
// it, waited on the build. So:
//
// - a delivery is handed out before any check;
// - a check that has waited CheckWaitsAtMost goes before every delivery asked after it, so no check
// waits for ever behind a stream of deliveries;
// - within a kind, first come, first served, by when it was asked;
// - an ask whose kind cannot be told counts as a check: a delivery taken late breaks a budget, a check
// taken late breaks nothing.
//
// **Kept in the stream, never beside it.** The queue is the seat's work queue, which the server hands out
// in stream order to whichever holder pulls; nothing else holds the order, so a restarted controller
// finds it as it was left. An ask that must go behind another is moved to the tail — deleted from where
// it was and published again, its body unchanged, under the same id — while the worker is paused, so no
// holder takes it in the moment it moves. Only waiting asks move: what a holder took is never touched.
// The kinds of ask, as the queue says them.
const (
KindDelivery = "delivery"
KindCheck = "check"
)
// CheckWaitsAtMost is how long a check waits before it goes ahead of the deliveries asked after it.
var CheckWaitsAtMost = 10 * time.Minute
// KindOf is an ask's kind: a check for a pull request's check — a recheck the merge tool asks included —
// a dry run, or an ask nothing could be read from; a delivery for any other build.
func KindOf(r BuildRequest) string {
if r.ID == "" || r.Check != nil || r.DryRun {
return KindCheck
}
return KindDelivery
}
// askedOrStored is when an ask was asked, as its id says, or else when the queue took it.
func (a QueuedAsk) askedOrStored() time.Time {
if !a.AskedAt.IsZero() {
return a.AskedAt
}
return a.Stored
}
// QueueOrder is the order the waiting asks are to be handed out in, now: the deliveries and every check
// that has waited CheckWaitsAtMost, by when each was asked, then the other checks, by when each was asked.
func QueueOrder(waiting []QueuedAsk, now time.Time) []QueuedAsk {
var first, then []QueuedAsk
for _, a := range waiting {
if KindOf(a.Request) == KindCheck && now.Sub(a.askedOrStored()) < CheckWaitsAtMost {
then = append(then, a)
continue
}
first = append(first, a)
}
byAsking := func(s []QueuedAsk) {
sort.SliceStable(s, func(i, j int) bool {
if ti, tj := s[i].askedOrStored(), s[j].askedOrStored(); !ti.Equal(tj) {
return ti.Before(tj)
}
return s[i].Seq < s[j].Seq
})
}
byAsking(first)
byAsking(then)
return append(first, then...)
}
// toMove is the asks to move to the tail, in this order, for the queue to stand as order says: everything
// after the longest start of order that the queue already holds in that order.
func toMove(queue, order []QueuedAsk) []QueuedAsk {
at := make(map[uint64]int, len(queue))
for i, a := range queue {
at[a.Seq] = i
}
kept, last := 0, -1
for kept < len(order) {
i, ok := at[order[kept].Seq]
if !ok || i < last {
break
}
last = i
kept++
}
return order[kept:]
}
// orderPasses keeps one pass at a time. Only the acting controller's loop runs one (KeepingInOrder), so
// there is one; the lock holds a test's two at once to that too.
var orderPasses sync.Mutex
// pausedFor bounds a pause the pass could not end itself: a controller that stops mid-pass leaves the
// worker paused this long at most. moveMargin is the part of it no move starts in: a pass stops moving
// with at least this left, and the next pass moves the rest, so no move runs past the pause.
const (
pausedFor = 20 * time.Second
moveMargin = 5 * time.Second
publishWaits = 3 * time.Second
)
// QueuedAtHeader carries, on a moved ask, when the stream first took it: a move keeps an ask's place in
// time, its id unread or not.
const QueuedAtHeader = "Mesh-Queued-At"
// MovedFromHeader names, on a move's copy, the sequence of the message it is a copy of: with QueuedAtHeader,
// what proves a second message of one id is a move's copy and not an ask delivered again (ADR 0293), whose
// headers are its original's — and so never name that original's own sequence.
const MovedFromHeader = "Mesh-Moved-From"
// Pass is what one pass did: the asks it moved, and each second copy it removed, in words.
type Pass struct {
Moved []string
Removed []string
}
// asked is the signal an ask sends the acting controller's pass loop, in this process. An ask asked in
// another process — a verb run as its own command — is seen by the loop as the queue's last sequence moving.
var asked = make(chan struct{}, 1)
// signalAsked tells the pass loop an ask joined the queue. Never blocks, never runs a pass.
func signalAsked() {
select {
case asked <- struct{}{}:
default:
}
}
// PutInOrder moves the seat's waiting asks into the order QueueOrder says, and says each it moved. A queue
// already in order is read and left alone; the worker is paused only while asks move. Run by the acting
// controller alone (KeepingInOrder): two passes in two processes would each resume what the other paused.
func PutInOrder(ctx context.Context, js *broker.JetStream, seat string) (Pass, error) {
var pass Pass
orderPasses.Lock()
defer orderPasses.Unlock()
api, err := jetstream.New(js.Conn())
if err != nil {
return pass, err
}
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
waiting, extra, delivered, err := readWaiting(ctx, api, worker, seat)
if err != nil || (len(extra) == 0 && len(toMove(waiting, QueueOrder(waiting, time.Now()))) == 0) {
return pass, err
}
// Paused, then read again: what the worker handed out up to the pause is not waiting any more.
pauseEnds := time.Now().Add(pausedFor)
if delivered >= 0 {
if _, err := api.PauseConsumer(ctx, worker.Stream, worker.Name, pauseEnds); err != nil {
return pass, fmt.Errorf("the %s worker could not be paused to put its queue in order, so it is left "+
"first come, first served: %w", seat, err)
}
defer func() {
resuming, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
defer cancel()
if _, err := api.ResumeConsumer(resuming, worker.Stream, worker.Name); err != nil {
fmt.Printf("the %s worker could not be resumed after its queue was put in order; it resumes by "+
"itself by %s: %v\n", seat, pauseEnds.Format(time.RFC3339), err)
}
}()
if waiting, extra, _, err = readWaiting(ctx, api, worker, seat); err != nil {
return pass, err
}
}
stream, err := api.Stream(ctx, worker.Stream)
if err != nil {
return pass, err
}
// A second copy of one ask — a move that published and stopped before it deleted — goes first.
for _, a := range extra {
if err := stream.DeleteMsg(ctx, a.Seq); err != nil && !errors.Is(err, jetstream.ErrMsgNotFound) {
return pass, fmt.Errorf("a second copy of %s (seq %d) could not be removed: %w", a.ID, a.Seq, err)
}
pass.Removed = append(pass.Removed, fmt.Sprintf("%s at seq %d, a move's second copy of the message at seq %s",
a.ID, a.Seq, a.movedFrom))
}
for _, a := range toMove(waiting, QueueOrder(waiting, time.Now())) {
if delivered >= 0 && time.Until(pauseEnds) < moveMargin {
// The rest waits for the next pass, under a pause of its own.
break
}
ok, err := moveToTheTail(ctx, js, stream, worker.Name, seat, a)
if err != nil {
return pass, err
}
if ok {
pass.Moved = append(pass.Moved, a.ID)
}
}
return pass, nil
}
// readWaiting is every ask the worker has not yet handed out, in queue order, but each second copy of one
// ask; the second copies, to remove; and the last sequence the worker handed out, -1 when there is no
// worker, so nothing is handed out to pause.
//
// **A second copy is proven, never guessed from an id**: a waiting message whose MovedFromHeader names an
// earlier message of the same request id still in the queue, and whose QueuedAtHeader is that message's
// first-queued time — a move that published its copy and stopped before it deleted the original. The
// original stays, waiting or handed out (a holder's then, and the copy would build it twice). Any other
// second message of one id — an ask a dead letter delivered again beside its original (ADR 0293) — is a
// real ask and stays.
func readWaiting(ctx context.Context, api jetstream.JetStream, worker broker.Consumer, seat string) (
waiting, extra []QueuedAsk, delivered int64, err error) {
stream, err := api.Stream(ctx, worker.Stream)
if err != nil {
return nil, nil, 0, fmt.Errorf("the %s work queue (%s) cannot be read: %w", seat, worker.Stream, err)
}
delivered = -1
if consumer, err := stream.Consumer(ctx, worker.Name); err == nil {
info, err := consumer.Info(ctx)
if err != nil {
return nil, nil, 0, fmt.Errorf("the worker of %s cannot be read: %w", seat, err)
}
delivered = int64(info.Delivered.Stream)
} else if !errors.Is(err, jetstream.ErrConsumerNotFound) {
return nil, nil, 0, fmt.Errorf("the worker of %s cannot be read: %w", seat, err)
}
// From the first: the stream holds only what is unsettled, so this is a handful of reads.
// Every message by sequence, with its id and first-queued time, to prove a copy against.
type held struct {
id string
stored time.Time
}
bySeq := map[string]held{}
var seq uint64 = 1
for n := 0; n < 10000; n++ {
msg, err := stream.GetMsg(ctx, seq, jetstream.WithGetMsgSubject(BuildWorkOf(seat)))
if errors.Is(err, jetstream.ErrMsgNotFound) {
break
}
if err != nil {
return nil, nil, 0, fmt.Errorf("reading the %s work queue: %w", seat, err)
}
seq = msg.Sequence + 1
a := askOf(msg)
a.movedFrom = msg.Header.Get(MovedFromHeader)
from, known := bySeq[a.movedFrom]
again := a.ID != "" && a.movedFrom != "" && known && from.id == a.ID && from.stored.Equal(a.Stored)
bySeq[strconv.FormatUint(a.Seq, 10)] = held{id: a.ID, stored: a.Stored}
switch {
case int64(a.Seq) <= delivered:
case again:
extra = append(extra, a)
default:
waiting = append(waiting, a)
}
}
return waiting, extra, delivered, nil
}
// moveToTheTail publishes the ask's body again at the queue's tail and then deletes it where it was. **The
// copy first**: a publish that fails deletes nothing, and a pass that stops between the two leaves two
// copies, which the next pass sees by their one id and makes one (readWaiting). A move never loses an ask.
// Looked at once more just before: one the worker has handed out since is a holder's now and stays. False
// when it was not moved.
func moveToTheTail(ctx context.Context, js *broker.JetStream, stream jetstream.Stream, workerName, seat string,
a QueuedAsk) (bool, error) {
if consumer, err := stream.Consumer(ctx, workerName); err == nil {
info, err := consumer.Info(ctx)
if err != nil {
return false, err
}
if info.Delivered.Stream >= a.Seq {
return false, nil
}
}
msg, err := stream.GetMsg(ctx, a.Seq)
if errors.Is(err, jetstream.ErrMsgNotFound) {
return false, nil
}
if err != nil {
return false, err
}
again := &nats.Msg{Subject: msg.Subject, Data: msg.Data, Header: nats.Header{}}
for k, v := range msg.Header {
// The server's own headers stay behind: a message id would be taken for a duplicate and dropped.
if !strings.HasPrefix(k, "Nats-") {
again.Header[k] = v
}
}
// Its place in time goes with it: when the stream first took it, as askOf read it; and the message it
// is a copy of, which proves it one.
again.Header.Set(QueuedAtHeader, askOf(msg).Stored.UTC().Format(time.RFC3339Nano))
again.Header.Set(MovedFromHeader, strconv.FormatUint(a.Seq, 10))
// Inside the margin, so the publish ends while the worker is still paused.
publishing, cancel := context.WithTimeout(ctx, publishWaits)
ack, err := js.Context().PublishMsg(again, nats.Context(publishing))
cancel()
if err != nil {
return false, fmt.Errorf("%s (seq %d) could not be put behind, and stays where it is: %w", a.ID, a.Seq, err)
}
// **Handed out meanwhile** — a bus slow enough that the pause ended: the original is a holder's, and the
// copy goes instead, so it is built once.
if consumer, err := stream.Consumer(ctx, workerName); err == nil {
info, err := consumer.Info(ctx)
if err != nil || info.Delivered.Stream >= a.Seq {
if derr := stream.DeleteMsg(ctx, ack.Sequence); derr != nil && !errors.Is(derr, jetstream.ErrMsgNotFound) {
return false, fmt.Errorf("%s was handed out while it moved and its copy (seq %d) could not be "+
"removed; the next pass removes it: %w", a.ID, ack.Sequence, derr)
}
if err != nil {
return false, fmt.Errorf("whether %s was handed out while it moved cannot be read, so its copy was "+
"removed and it stays where it was: %w", a.ID, err)
}
if info.Delivered.Stream >= ack.Sequence {
return false, fmt.Errorf("%s was handed out while it moved, and its copy too: it may be built twice", a.ID)
}
return false, nil
}
}
if err := stream.DeleteMsg(ctx, a.Seq); err != nil && !errors.Is(err, jetstream.ErrMsgNotFound) {
return false, fmt.Errorf("%s was copied behind and its original (seq %d) not removed; the next pass "+
"removes the second copy: %w", a.ID, a.Seq, err)
}
return true, nil
}
// KeepingInOrder runs the passes, on the acting controller alone, until ctx ends: when an ask in this
// process says so, when the queue's last sequence moved — an ask from another process — and every
// passEvery, for a check that comes of age with nothing asked. seat says the build seat to order; acting
// whether this controller acts (Holding); said what a pass did.
func KeepingInOrder(ctx context.Context, js *broker.JetStream, seat func() string, acting func() bool,
said func(format string, args ...any)) {
tick := time.NewTicker(orderLooks)
defer tick.Stop()
var lastSeq uint64
var lastPass time.Time
for {
signalled := false
select {
case <-ctx.Done():
return
case <-asked:
signalled = true
case <-tick.C:
}
if !acting() {
continue
}
name := seat()
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: name, Accepts: []string{"build"}})
info, err := js.Context().StreamInfo(worker.Stream)
if err != nil {
continue
}
if !signalled && info.State.LastSeq == lastSeq && time.Since(lastPass) < passEvery {
continue
}
lastSeq, lastPass = info.State.LastSeq, time.Now()
passing, cancel := context.WithTimeout(ctx, pausedFor+10*time.Second)
pass, err := PutInOrder(passing, js, name)
cancel()
for _, r := range pass.Removed {
said("the %s queue: removed %s", name, r)
}
if len(pass.Moved) > 0 {
said("the %s queue put in order: %s moved behind what goes first", name, strings.Join(pass.Moved, ", "))
}
if err != nil {
said("the %s queue could not be put in order: %v", name, err)
}
if info, err := js.Context().StreamInfo(worker.Stream); err == nil {
// Its own moves are not an ask.
lastSeq = info.State.LastSeq
}
}
}
// orderLooks is how often the loop looks whether an ask joined from another process; passEvery how often
// it passes with nothing asked.
var (
orderLooks = 2 * time.Second
passEvery = 30 * time.Second
)
// DeleteAsk removes a waiting or dead ask from the queue: at its sequence, or — moved into order since it
// was read — the copy carrying its id. Says the sequence it removed, 0 when neither was there any more.
func DeleteAsk(ctx context.Context, js *broker.JetStream, seat string, ask QueuedAsk) (uint64, error) {
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
gone := func(err error) bool {
return errors.Is(err, nats.ErrMsgNotFound) || errors.Is(err, jetstream.ErrMsgNotFound) ||
(err != nil && strings.Contains(err.Error(), "no message found"))
}
err := js.Context().DeleteMsg(worker.Stream, ask.Seq)
if err == nil {
return ask.Seq, nil
}
if !gone(err) {
return 0, err
}
if ask.ID == "" {
return 0, nil
}
q, err := ReadQueue(ctx, js, seat)
if err != nil {
return 0, err
}
for _, a := range q.Asks {
if a.ID == ask.ID && a.Seq != ask.Seq && a.State != AskInFlight {
if err := js.Context().DeleteMsg(worker.Stream, a.Seq); err != nil && !gone(err) {
return 0, err
} else if err == nil {
return a.Seq, nil
}
}
}
return 0, nil
}
+441
View File
@@ -0,0 +1,441 @@
package link
import (
"context"
"encoding/json"
"fmt"
"reflect"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
)
// The order the build queue hands asks out in (novox/hq issue 558): a build a walk waits for before any
// pull request's check, a check that has waited ten minutes before every delivery asked after it, and
// first come, first served within a kind.
func anAsk(seq uint64, id string, asked time.Time, check bool) QueuedAsk {
r := BuildRequest{ID: id, Repository: "/r"}
if check {
r.Check = &CheckRequest{Owner: "novox", Repo: "r"}
}
return QueuedAsk{Seq: seq, ID: id, AskedAt: asked, Request: r, State: AskWaiting}
}
func idsOf(asks []QueuedAsk) []string {
out := []string{}
for _, a := range asks {
out = append(out, a.ID)
}
return out
}
func TestADeliveryIsHandedOutBeforeTheChecksAskedBeforeIt(t *testing.T) {
now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC)
waiting := []QueuedAsk{
anAsk(1, "check-a", now.Add(-3*time.Minute), true),
anAsk(2, "check-b", now.Add(-2*time.Minute), true),
anAsk(3, "delivery-d", now.Add(-time.Minute), false),
}
want := []string{"delivery-d", "check-a", "check-b"}
if got := idsOf(QueueOrder(waiting, now)); !reflect.DeepEqual(got, want) {
t.Fatalf("handed out as %v, want %v", got, want)
}
// Only the checks ahead of it move behind it; the delivery stays where it is.
if got := idsOf(toMove(waiting, QueueOrder(waiting, now))); !reflect.DeepEqual(got, []string{"check-a", "check-b"}) {
t.Fatalf("moved %v", got)
}
}
func TestACheckThatWaitedTenMinutesGoesBeforeALaterDelivery(t *testing.T) {
now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC)
// As the queue stands after a delivery put the young checks behind it, ten minutes ago and since.
waiting := []QueuedAsk{
anAsk(5, "delivery-early", now.Add(-12*time.Minute), false),
anAsk(6, "delivery-later", now.Add(-time.Minute), false),
anAsk(7, "check-old", now.Add(-11*time.Minute), true),
anAsk(8, "check-young", now.Add(-9*time.Minute), true),
}
want := []string{"delivery-early", "check-old", "delivery-later", "check-young"}
if got := idsOf(QueueOrder(waiting, now)); !reflect.DeepEqual(got, want) {
t.Fatalf("handed out as %v, want %v", got, want)
}
if got := idsOf(toMove(waiting, QueueOrder(waiting, now))); !reflect.DeepEqual(got, []string{"delivery-later", "check-young"}) {
t.Fatalf("moved %v", got)
}
}
func TestAnAskWhoseKindCannotBeToldIsACheck(t *testing.T) {
for _, r := range []BuildRequest{
{}, // unreadable: nothing was read from it
{ID: "build-1", Check: &CheckRequest{}}, // a pull request's check, or a recheck the merge tool asked
{ID: "build-2", DryRun: true}, // looked at, never delivered
} {
if got := KindOf(r); got != KindCheck {
t.Errorf("%+v is a %s", r, got)
}
}
if got := KindOf(BuildRequest{ID: "build-3", Repository: "/r"}); got != KindDelivery {
t.Errorf("a build is a %s", got)
}
}
// Against a real server: two checks wait, a delivery is asked, and the next free holder takes the delivery.
func TestNatsADeliveryIsTakenBeforeTheChecksQueuedAheadOfIt(t *testing.T) {
js := aBusWithTheBuildRole(t)
ctx, stop := context.WithCancel(context.Background())
defer stop()
asks := BuildsOn(js, TheBuildMachine)
now := time.Now()
for i, r := range []BuildRequest{
{ID: NewBuildID(now.Add(-3 * time.Minute)), Repository: "/a", Check: &CheckRequest{Owner: "novox", Repo: "a"}},
{ID: NewBuildID(now.Add(-2 * time.Minute)), Repository: "/b", Check: &CheckRequest{Owner: "novox", Repo: "b"}},
{ID: NewBuildID(now), Repository: "/d"},
} {
if i == 2 {
// And one nobody can read, whose kind cannot be told: it waits with the checks.
if _, err := js.Context().Publish(BuildWork(), []byte("not an ask")); err != nil {
t.Fatal(err)
}
}
if err := asks.Ask(ctx, r); err != nil {
t.Fatal(err)
}
}
// The acting controller's loop puts it in order, told by the asks.
go KeepingInOrder(ctx, js, func() string { return TheBuildMachine }, func() bool { return true },
func(string, ...any) {})
time.Sleep(500 * time.Millisecond)
took := make(chan string, 4)
machine := MachineOverNATS(js, "anchor")
defer machine.Close()
go func() {
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
took <- work.Request().Repository
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
_ = work.Done()
})
}()
var order []string
for len(order) < 3 {
select {
case r := <-took:
order = append(order, r)
case <-time.After(10 * time.Second):
t.Fatalf("taken so far: %v", order)
}
}
if want := []string{"/d", "/a", "/b"}; !reflect.DeepEqual(order, want) {
t.Fatalf("taken as %v, want %v", order, want)
}
}
// --- the moves, against a real server ------------------------------------------------------------
func publishAsk(t *testing.T, js *broker.JetStream, r BuildRequest) {
t.Helper()
body, _ := json.Marshal(r)
if _, err := js.Context().Publish(BuildWork(), body); err != nil {
t.Fatal(err)
}
}
func aCheck(asked time.Time, repository string) BuildRequest {
return BuildRequest{ID: NewBuildID(asked), Repository: repository, Check: &CheckRequest{Owner: "novox", Repo: "r"}}
}
func aDelivery(asked time.Time, repository string) BuildRequest {
return BuildRequest{ID: NewBuildID(asked), Repository: repository}
}
// queued is the stream's asks, in queue order.
func queued(t *testing.T, js *broker.JetStream) []QueuedAsk {
t.Helper()
q, err := ReadQueue(context.Background(), js, TheBuildMachine)
if err != nil {
t.Fatal(err)
}
return q.Asks
}
// takeAll runs a holder that records how often it was handed each repository.
func takeAll(t *testing.T, ctx context.Context, js *broker.JetStream) (chan string, func() map[string]int) {
t.Helper()
took := make(chan string, 100)
var mu sync.Mutex
counts := map[string]int{}
machine := MachineOverNATS(js, "anchor")
t.Cleanup(machine.Close)
go func() {
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
mu.Lock()
counts[work.Request().Repository]++
mu.Unlock()
took <- work.Request().Repository
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
_ = work.Done()
})
}()
return took, func() map[string]int {
mu.Lock()
defer mu.Unlock()
out := map[string]int{}
for k, v := range counts {
out[k] = v
}
return out
}
}
// A move that published its copy and stopped before deleting the original leaves two copies of one ask;
// the next pass makes them one, and a copy of one a holder already took is removed, never built twice.
func TestNatsAMoveCutShortLeavesOneCopyAfterTheNextPass(t *testing.T) {
js := aBusWithTheBuildRole(t)
ctx := t.Context()
now := time.Now()
check := aCheck(now.Add(-2*time.Minute), "/c")
publishAsk(t, js, check)
publishAsk(t, js, aDelivery(now.Add(-time.Minute), "/d"))
moveCopyOf(t, js, check.ID) // the copy, published; the original never deleted
if pass, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(pass.Removed) != 1 {
t.Fatalf("removed %v: %v", pass.Removed, err)
}
var got []string
for _, a := range queued(t, js) {
got = append(got, a.Repository)
}
if want := []string{"/d", "/c"}; !reflect.DeepEqual(got, want) {
t.Fatalf("after the pass the queue holds %v, want %v", got, want)
}
// And one a holder took before the copy was published: the copy goes, the holder's stays.
taken := aCheck(now, "/taken")
publishAsk(t, js, taken)
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: TheBuildMachine, Accepts: []string{"build"}})
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
if err != nil {
t.Fatal(err)
}
defer func() { _ = sub.Unsubscribe() }()
// Take everything waiting up to and including /taken, and keep /taken unsettled.
for {
msgs, err := sub.Fetch(1, nats.MaxWait(2*time.Second))
if err != nil || len(msgs) == 0 {
t.Fatalf("nothing to take: %v", err)
}
var r BuildRequest
_ = json.Unmarshal(msgs[0].Data, &r)
if r.Repository == "/taken" {
break
}
_ = msgs[0].Ack()
}
moveCopyOf(t, js, taken.ID)
if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil {
t.Fatal(err)
}
n := 0
for _, a := range queued(t, js) {
if a.ID == taken.ID {
n++
}
}
if n != 1 {
t.Fatalf("%d copies of the ask a holder took are in the queue, want 1", n)
}
}
// moveCopyOf publishes what a move publishes for the ask with this id, as a pass stopped before its delete
// would leave it.
func moveCopyOf(t *testing.T, js *broker.JetStream, id string) {
t.Helper()
for _, a := range queued(t, js) {
if a.ID != id {
continue
}
body, _ := json.Marshal(a.Request)
msg := &nats.Msg{Subject: BuildWork(), Data: body, Header: nats.Header{}}
msg.Header.Set(QueuedAtHeader, a.Stored.UTC().Format(time.RFC3339Nano))
msg.Header.Set(MovedFromHeader, fmt.Sprint(a.Seq))
if _, err := js.Context().PublishMsg(msg); err != nil {
t.Fatal(err)
}
return
}
t.Fatalf("%s is not queued", id)
}
// A dead letter delivered again whose original stayed beside it (ADR 0293: its removal failed) is a real
// ask with the same id, and is never removed as a second copy — even when the original was itself a move's
// copy, whose headers the delivery carries.
func TestNatsAnAskDeliveredAgainBesideItsOriginalStays(t *testing.T) {
js := aBusWithTheBuildRole(t)
ctx := t.Context()
ask := aCheck(time.Now(), "/again")
publishAsk(t, js, aCheck(time.Now().Add(-time.Minute), "/before"))
publishAsk(t, js, ask)
publishAsk(t, js, aDelivery(time.Now(), "/d"))
if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil { // /again becomes a move's copy
t.Fatal(err)
}
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: TheBuildMachine, Accepts: []string{"build"}})
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
if err != nil {
t.Fatal(err)
}
defer func() { _ = sub.Unsubscribe() }()
var original *nats.Msg
for original == nil {
msgs, err := sub.Fetch(1, nats.MaxWait(2*time.Second))
if err != nil || len(msgs) == 0 {
t.Fatalf("nothing to take: %v", err)
}
var r BuildRequest
_ = json.Unmarshal(msgs[0].Data, &r)
if r.ID == ask.ID {
original = msgs[0] // given up on, never settled: dead, and kept
continue
}
_ = msgs[0].Ack()
}
// Delivered again as DeliverAgain does: its headers, and its own.
again := &nats.Msg{Subject: BuildWork(), Data: original.Data, Header: nats.Header{}}
for k, v := range original.Header {
again.Header[k] = v
}
again.Header.Set(AgainHeader, "7")
if _, err := js.Context().PublishMsg(again); err != nil {
t.Fatal(err)
}
pass, err := PutInOrder(ctx, js, TheBuildMachine)
if err != nil || len(pass.Removed) != 0 {
t.Fatalf("removed %v: %v", pass.Removed, err)
}
n := 0
for _, a := range queued(t, js) {
if a.ID == ask.ID {
n++
}
}
if n != 2 {
t.Fatalf("%d messages of the ask delivered again are queued, want its original and the delivery", n)
}
}
// Two passes at once: nothing is built twice, and nothing is lost.
func TestNatsTwoPassesAtOnceBuildNothingTwice(t *testing.T) {
js := aBusWithTheBuildRole(t)
ctx, stop := context.WithCancel(t.Context())
defer stop()
took, counts := takeAll(t, ctx, js)
now := time.Now()
const asks = 24
var passes sync.WaitGroup
for p := 0; p < 2; p++ {
passes.Add(1)
go func() {
defer passes.Done()
for i := 0; i < 15; i++ {
_, _ = PutInOrder(ctx, js, TheBuildMachine)
}
}()
}
for i := 0; i < asks; i++ {
repository := fmt.Sprintf("/%02d", i)
if i%3 == 2 {
publishAsk(t, js, aDelivery(now.Add(time.Duration(i)*time.Millisecond), repository))
} else {
publishAsk(t, js, aCheck(now.Add(time.Duration(i)*time.Millisecond), repository))
}
}
passes.Wait()
for got := 0; got < asks; got++ {
select {
case <-took:
case <-time.After(15 * time.Second):
t.Fatalf("taken %d of %d: %v", got, asks, counts())
}
}
time.Sleep(time.Second)
c := counts()
for i := 0; i < asks; i++ {
if n := c[fmt.Sprintf("/%02d", i)]; n != 1 {
t.Errorf("/%02d was built %d times", i, n)
}
}
}
// A check that has waited ten minutes goes before a delivery asked after it, even one queued ahead of it.
func TestNatsACheckThatWaitedTenMinutesIsTakenBeforeALaterDelivery(t *testing.T) {
js := aBusWithTheBuildRole(t)
ctx, stop := context.WithCancel(t.Context())
defer stop()
now := time.Now()
publishAsk(t, js, aDelivery(now.Add(-time.Minute), "/delivery"))
publishAsk(t, js, aCheck(now.Add(-11*time.Minute), "/old-check"))
if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil {
t.Fatal(err)
}
took, _ := takeAll(t, ctx, js)
var order []string
for len(order) < 2 {
select {
case r := <-took:
order = append(order, r)
case <-time.After(10 * time.Second):
t.Fatalf("taken so far: %v", order)
}
}
if want := []string{"/old-check", "/delivery"}; !reflect.DeepEqual(order, want) {
t.Fatalf("taken as %v, want %v", order, want)
}
}
// A cancel that read an ask before it moved deletes the moved copy, found by its id, and says when
// there was nothing left to delete.
func TestNatsACancelRacingAMoveDeletesTheMovedCopy(t *testing.T) {
js := aBusWithTheBuildRole(t)
ctx := t.Context()
now := time.Now()
publishAsk(t, js, aCheck(now.Add(-time.Minute), "/c"))
read := queued(t, js)[0]
publishAsk(t, js, aDelivery(now, "/d"))
if pass, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(pass.Moved) != 1 {
t.Fatalf("moved %v: %v", pass.Moved, err)
}
removed, err := DeleteAsk(ctx, js, TheBuildMachine, read)
if err != nil || removed == 0 || removed == read.Seq {
t.Fatalf("removed seq %d (read at %d): %v", removed, read.Seq, err)
}
for _, a := range queued(t, js) {
if a.ID == read.ID {
t.Fatalf("the cancelled ask is still queued at seq %d", a.Seq)
}
}
if removed, err := DeleteAsk(ctx, js, TheBuildMachine, read); err != nil || removed != 0 {
t.Fatalf("an ask gone already is said removed at %d: %v", removed, err)
}
}
// An ask nothing can be read from keeps its place in time when it moves.
func TestNatsAnUnreadableAskKeepsItsAgeWhenItMoves(t *testing.T) {
js := aBusWithTheBuildRole(t)
ctx := t.Context()
if _, err := js.Context().Publish(BuildWork(), []byte("not an ask")); err != nil {
t.Fatal(err)
}
first := queued(t, js)[0].Stored
time.Sleep(50 * time.Millisecond)
publishAsk(t, js, aDelivery(time.Now(), "/d"))
if pass, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(pass.Moved) != 1 {
t.Fatalf("moved %v: %v", pass.Moved, err)
}
asks := queued(t, js)
if len(asks) != 2 || asks[1].ID != "" || !asks[1].Stored.Equal(first) {
t.Fatalf("after the move the unreadable ask was taken at %v, first at %v: %+v", asks[1].Stored, first, asks)
}
}
-1
View File
@@ -35,4 +35,3 @@
./cmd/mesh-controller TestARollbackWhoseSendWasRefusedIsCompletedByTheNextSend
./cmd/mesh-controller TestAGateJudgesTheOtherMovesBesideAKeptCore
./cmd/mesh-controller TestReplay401
./cmd/mesh-controller TestReplay201ThePlansSend