Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fc573f951a | ||
|
|
1ded7dff34 | ||
|
|
c6e234455f |
@@ -66,11 +66,6 @@ broker to take work from or a registry to publish into:
|
||||
|
||||
Without --registry the artifacts stay in this machine's container runtime, named by the
|
||||
digest of their own configuration. The result is printed as JSON.
|
||||
|
||||
mesh-builder ready
|
||||
|
||||
exits 0 once the agent running in this container is bound to its build work and pulls it, and
|
||||
1 before: its container's health check.
|
||||
`
|
||||
|
||||
func run() error {
|
||||
@@ -81,18 +76,12 @@ func run() error {
|
||||
return nil
|
||||
case "build":
|
||||
return buildOnce(context.Background(), os.Args[2:])
|
||||
case "ready":
|
||||
// The container's health check (novox/hq issue 544): exits non-zero until the agent takes work.
|
||||
return ready()
|
||||
default:
|
||||
fmt.Print(usage)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if err := notReady(); err != nil {
|
||||
return err
|
||||
}
|
||||
credential, err := brokerFrom()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -145,8 +134,7 @@ func run() error {
|
||||
// **What an earlier holder here left goes before anything is taken** (novox/hq issue 306): a holder
|
||||
// recreated mid-check left the check's store and bus running, and the ask went to another machine.
|
||||
h.removeLeftBehind()
|
||||
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused,
|
||||
Taking: func() { taking(seat) }})
|
||||
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused})
|
||||
defer machine.Close()
|
||||
|
||||
// The seat's verbs, on this machine's subjects. Only the seat that declares them: the retired
|
||||
|
||||
@@ -1,44 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// **The build agent says it is ready** (novox/hq issue 544). It listens on nothing, so no http or tcp
|
||||
// check can see it work; it writes a mark once it is bound to its worker and pulls build work, and its
|
||||
// container's health check runs `mesh-builder ready`, which passes only while the mark is there. Without
|
||||
// it the node-engine could judge the agent only alive, and every walk that moved it waited out the whole
|
||||
// grace on each machine.
|
||||
//
|
||||
// The mark lives in the container's own temporary directory, never in the workspace the machine keeps:
|
||||
// a new container starts without it, and a restarted one removes it before it binds again.
|
||||
func readyMark() string { return filepath.Join(os.TempDir(), "mesh-builder.ready") }
|
||||
|
||||
// notReady removes the mark: a holder starting is not taking work until it says so again.
|
||||
func notReady() error {
|
||||
if err := os.Remove(readyMark()); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("cannot remove the mark that this agent takes build work (%s): %w", readyMark(), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// taking writes the mark. A mark that cannot be written is said, and the agent goes on building: its
|
||||
// check then says it is not ready, which is the truth the gate should hear.
|
||||
func taking(seat string) {
|
||||
if err := os.WriteFile(readyMark(), []byte(seat+"\n"), 0o644); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "cannot mark that this agent takes build work (%s): %v\n", readyMark(), err)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "taking build work from %s\n", seat)
|
||||
}
|
||||
|
||||
// ready is the health check: nil when this container's agent is bound and pulling build work.
|
||||
func ready() error {
|
||||
if _, err := os.Stat(readyMark()); err != nil {
|
||||
return fmt.Errorf("not taking build work yet: %s is not there", readyMark())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The health check passes only between the agent saying it takes work and its next start (novox/hq
|
||||
// issue 544).
|
||||
func TestTheAgentIsReadyOnlyOnceItTakesWork(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
if err := ready(); err == nil {
|
||||
t.Fatal("an agent that never took work is ready")
|
||||
}
|
||||
taking("mesh-build-machine")
|
||||
if err := ready(); err != nil {
|
||||
t.Fatalf("an agent taking work is not ready: %v", err)
|
||||
}
|
||||
// Started again, it is not ready until it binds again.
|
||||
if err := notReady(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ready(); err == nil {
|
||||
t.Fatal("an agent starting again is ready before it takes work")
|
||||
}
|
||||
if err := notReady(); err != nil {
|
||||
t.Fatalf("removing a mark that is not there failed: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(readyMark()); err == nil {
|
||||
t.Fatal("the mark is still there")
|
||||
}
|
||||
}
|
||||
@@ -1109,16 +1109,6 @@ const resolverSeat = "mesh-dns-resolver"
|
||||
// classOf is a walk's class (novox/hq ADR 0282 decision 1): core when it walks a module on the controller's own
|
||||
// path or one holding the mesh's resolver, leaf otherwise.
|
||||
func classOf(walk inventory.Plan, entries []inventory.Entry) string {
|
||||
modules := make([]string, 0, len(walk.Modules))
|
||||
for m := range walk.Modules {
|
||||
modules = append(modules, m)
|
||||
}
|
||||
return classOfModules(modules, entries)
|
||||
}
|
||||
|
||||
// classOfModules is the class of modules moved together: core when one is on the controller's own path or holds
|
||||
// the mesh's resolver, leaf otherwise (ADR 0282 decision 1).
|
||||
func classOfModules(modules []string, entries []inventory.Entry) string {
|
||||
resolvers := map[string]bool{}
|
||||
for _, e := range entries {
|
||||
for _, c := range e.Manifest.Claims {
|
||||
@@ -1127,7 +1117,7 @@ func classOfModules(modules []string, entries []inventory.Entry) string {
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, m := range modules {
|
||||
for m := range walk.Modules {
|
||||
if _, own := onTheControllersPath[m]; own || resolvers[m] {
|
||||
return inventory.ClassCore
|
||||
}
|
||||
|
||||
@@ -2,7 +2,6 @@ package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -106,32 +105,9 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
|
||||
p.Machines = append(p.Machines, *machines[name])
|
||||
}
|
||||
p.Summary = summaryOf(p)
|
||||
p.Class = changeClassOf(p, entries)
|
||||
return p
|
||||
}
|
||||
|
||||
// classNone is the class of a change that moves no module: nothing for a walk to move (novox/hq ADR 0313 §2).
|
||||
const classNone = "none"
|
||||
|
||||
// changeClassOf is a change plan's class (novox/hq ADR 0313 §2): that of the slowest-budgeted module it moves,
|
||||
// itself or as a dependent — core when one is on the controller's own path or holds the mesh's resolver, leaf
|
||||
// otherwise — and none when it moves nothing. A module it adds a directory for counts by its own class: the
|
||||
// catalogue names a module's directory by the module, so the directory's last element is its name. The graph holds
|
||||
// no manifest of it yet, so a new module is core by its name alone (on the controller's own path), never by a
|
||||
// resolver claim its manifest makes.
|
||||
func changeClassOf(p link.ChangePlan, entries []inventory.Entry) string {
|
||||
if len(p.Moved) == 0 && len(p.Dependents) == 0 && len(p.New) == 0 {
|
||||
return classNone
|
||||
}
|
||||
modules := append(append([]string{}, p.Moved...), p.Dependents...)
|
||||
for _, dir := range p.New {
|
||||
if name := path.Base(strings.Trim(dir, "/")); name != "." && name != "/" && name != "" {
|
||||
modules = append(modules, name)
|
||||
}
|
||||
}
|
||||
return classOfModules(modules, entries)
|
||||
}
|
||||
|
||||
// grantsStepWord names the grants step (novox/hq issue 490): the bus's user list sent alone to the machine
|
||||
// holding the bus ahead of a walk's gate. Not the bus step, which replaces the bus itself.
|
||||
const grantsStepWord = "grants step"
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A change plan says its class** (novox/hq ADR 0313 §2): the issue tracker holds a fix to the budget of the
|
||||
// slowest-budgeted module the fix moves, and a fix that moves no module has no fix budget. The plan says core,
|
||||
// leaf or none, so mesh-delivery can pass it on with the fix's delivery.
|
||||
func TestAChangePlanSaysTheClassOfWhatItMoves(t *testing.T) {
|
||||
const repo = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||
entry := func(name, path string) inventory.Entry {
|
||||
e := fromRepo(name, repo, path)
|
||||
e.Source.BuiltFrom = "old"
|
||||
e.On = []string{"anchor"}
|
||||
return e
|
||||
}
|
||||
gitea := entry("gitea", "modules/gitea")
|
||||
resolver := entry("dnsmasq", "modules/dnsmasq")
|
||||
resolver.Manifest.Claims = []catalogue.Claim{{Name: resolverSeat}}
|
||||
delivery := entry(catalogue.DeliverySeat, "modules/mesh-delivery")
|
||||
webApp := entry("photos", "modules/photos")
|
||||
entries := []inventory.Entry{gitea, resolver, delivery, webApp}
|
||||
// photos stands on gitea: a change to gitea moves photos as its dependent.
|
||||
edges := []inventory.Edge{{From: "photos", To: "gitea", Kind: inventory.EdgeStandsOn}}
|
||||
policy := func(string) (inventory.Upgrade, bool) { return inventory.Upgrade{RollOut: true}, true }
|
||||
plan := func(paths ...string) link.ChangePlan {
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: paths}
|
||||
return changePlanOf("novox/mesh-catalog", "main", "head", reachOfMerge(m, entries, nil, edges), entries, policy)
|
||||
}
|
||||
|
||||
for _, c := range []struct {
|
||||
why string
|
||||
paths []string
|
||||
want string
|
||||
}{
|
||||
{"a leaf module and its dependent", []string{"modules/gitea/index.ts"}, inventory.ClassLeaf},
|
||||
{"the delivery's owner, on the controller's own path", []string{"modules/mesh-delivery/cmd/x.go"}, inventory.ClassCore},
|
||||
{"the mesh's resolver", []string{"modules/dnsmasq/module.json"}, inventory.ClassCore},
|
||||
{"a leaf and a core module together: the slowest-budgeted", []string{"modules/gitea/index.ts", "modules/dnsmasq/module.json"}, inventory.ClassCore},
|
||||
{"nothing any module's build reads", []string{"README.md"}, "none"},
|
||||
{"a new leaf module alone", []string{"modules/weather/module.json"}, inventory.ClassLeaf},
|
||||
{"a new module on the controller's own path, by its own class", []string{"next/mesh-delivery/module.json"}, inventory.ClassCore},
|
||||
{"a new leaf module beside a core one moved", []string{"modules/weather/module.json", "modules/dnsmasq/module.json"}, inventory.ClassCore},
|
||||
} {
|
||||
if got := plan(c.paths...).Class; got != c.want {
|
||||
t.Errorf("%s: class %q, want %q", c.why, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,135 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// machineLines is what a machine said it is made of (novox/hq issue 160), as `node` shows it: every fact in
|
||||
// one block with its age, the architecture and kernel the profile always carried among them, and each fact
|
||||
// the machine could not determine said as unknown with its why — never as a zero, and never left out.
|
||||
func machineLines(a inventory.MachineAccount, had bool, now time.Time) []string {
|
||||
if !had {
|
||||
return []string{" what it is made of: never said — this machine has not reported a profile"}
|
||||
}
|
||||
m := a.Machine
|
||||
if m == nil {
|
||||
return []string{
|
||||
" what it is made of: not said — its node-engine is older than the one that says it",
|
||||
" " + factLine("architecture", orUnsaid(a.Architecture)),
|
||||
" " + factLine("kernel", orUnsaid(a.Kernel)),
|
||||
}
|
||||
}
|
||||
age := roughly(now.Sub(m.ObservedAt)) + " ago"
|
||||
if m.ObservedAt.IsZero() {
|
||||
// Said with no time: its age is not known, and is never read as two thousand years.
|
||||
age = "at a time it did not say (age unknown)"
|
||||
}
|
||||
out := []string{fmt.Sprintf(" what it is made of, as it said %s:", age)}
|
||||
add := func(name, value string) { out = append(out, " "+factLine(name, value)) }
|
||||
unknown := map[string]string{}
|
||||
for k, v := range m.Unknown {
|
||||
unknown[k] = v
|
||||
}
|
||||
// said is the value, or the fact's own reason it is unknown; a fact neither said nor unknown is a
|
||||
// node-engine that broke its own rule, and is said so rather than shown blank.
|
||||
said := func(fact, value string) string {
|
||||
if value != "" {
|
||||
delete(unknown, fact)
|
||||
return value
|
||||
}
|
||||
if why, ok := unknown[fact]; ok {
|
||||
delete(unknown, fact)
|
||||
return "unknown: " + why
|
||||
}
|
||||
return "unknown: the machine said neither it nor why"
|
||||
}
|
||||
|
||||
add("architecture", orUnsaid(a.Architecture))
|
||||
release := said("kernel release", m.KernelRelease)
|
||||
if m.KernelRelease == "" {
|
||||
release = "(release " + release + ")"
|
||||
}
|
||||
add("kernel", orUnsaid(a.Kernel)+" "+release)
|
||||
dist := said("distribution", m.Distribution)
|
||||
if m.Distribution != "" {
|
||||
dist += " " + said("version", m.Version)
|
||||
}
|
||||
add("distribution", dist)
|
||||
add("processor", said("processor", m.Processor))
|
||||
add("cores", said("cores", countWords(m.Cores)))
|
||||
add("threads", said("threads", countWords(m.Threads)))
|
||||
add("memory", said("memory", bytesWords(m.MemoryBytes)))
|
||||
add("virtual", said("virtual or physical", m.Virtual))
|
||||
up := ""
|
||||
if m.BootedAt != nil {
|
||||
up = fmt.Sprintf("%s, since %s", roughly(now.Sub(*m.BootedAt)), m.BootedAt.Local().Format("2006-01-02 15:04"))
|
||||
}
|
||||
add("up", said("uptime", up))
|
||||
add("timezone", said("timezone", m.Timezone))
|
||||
if len(m.Filesystems) == 0 {
|
||||
add("filesystems", said("filesystems", ""))
|
||||
} else {
|
||||
for i, f := range m.Filesystems {
|
||||
name := ""
|
||||
if i == 0 {
|
||||
name = "filesystems"
|
||||
}
|
||||
add(name, fmt.Sprintf("%s %s, %s free (%s %s)", f.Mount, bytesWords(f.SizeBytes),
|
||||
bytesWords(f.FreeBytes), f.Type, f.Device))
|
||||
}
|
||||
if why, ok := unknown["filesystems"]; ok {
|
||||
add("", "and some unknown: "+why)
|
||||
delete(unknown, "filesystems")
|
||||
}
|
||||
}
|
||||
add("public address", said("public address", ""))
|
||||
// Any other fact a newer node-engine named unknown, so nothing it said is dropped.
|
||||
rest := make([]string, 0, len(unknown))
|
||||
for k := range unknown {
|
||||
rest = append(rest, k)
|
||||
}
|
||||
sort.Strings(rest)
|
||||
for _, k := range rest {
|
||||
add(k, "unknown: "+unknown[k])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func factLine(name, value string) string { return fmt.Sprintf("%-15s %s", name, value) }
|
||||
|
||||
func orUnsaid(s string) string {
|
||||
if strings.TrimSpace(s) == "" {
|
||||
return "not said"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// countWords is a number a machine said, or nothing when it said none (its unknown says why).
|
||||
func countWords(n int) string {
|
||||
if n <= 0 {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprint(n)
|
||||
}
|
||||
|
||||
// bytesWords is an amount of bytes a person reads, in binary units; nothing when none was said.
|
||||
func bytesWords(n int64) string {
|
||||
if n <= 0 {
|
||||
return ""
|
||||
}
|
||||
units := []string{"B", "KiB", "MiB", "GiB", "TiB", "PiB"}
|
||||
v, u := float64(n), 0
|
||||
for v >= 1024 && u < len(units)-1 {
|
||||
v /= 1024
|
||||
u++
|
||||
}
|
||||
if u == 0 {
|
||||
return fmt.Sprintf("%d B", n)
|
||||
}
|
||||
return fmt.Sprintf("%.1f %s", v, units[u])
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// Defends novox/hq issue 160: `node` shows every fact a machine said it is made of, in one block with its
|
||||
// age, the architecture and kernel among them; and a fact it could not determine as unknown with why.
|
||||
func TestNodeShowsWhatAMachineIsMadeOfWithItsAge(t *testing.T) {
|
||||
now := time.Date(2026, 10, 12, 15, 0, 0, 0, time.UTC)
|
||||
booted := now.Add(-75 * time.Hour)
|
||||
a := inventory.MachineAccount{Architecture: "amd64", Kernel: "linux", Machine: &inventory.MadeOf{
|
||||
ObservedAt: now.Add(-3 * time.Hour), KernelRelease: "7.2.8-arch1-2", Distribution: "Arch Linux",
|
||||
Version: "rolling", Processor: "AMD Ryzen 9 8945HS", Cores: 8, Threads: 16, MemoryBytes: 32891494400,
|
||||
Virtual: "physical", BootedAt: &booted, Timezone: "Europe/Brussels",
|
||||
Filesystems: []inventory.MountedFilesystem{{Mount: "/", Device: "/dev/nvme0n1p3", Type: "ext4",
|
||||
SizeBytes: 987978522624, FreeBytes: 31830110208}},
|
||||
Unknown: map[string]string{"public address": "how it learns it is not decided yet"},
|
||||
}}
|
||||
got := strings.Join(machineLines(a, true, now), "\n")
|
||||
t.Log("\n" + got)
|
||||
for _, want := range []string{
|
||||
"what it is made of, as it said 3h ago:",
|
||||
"architecture amd64",
|
||||
"kernel linux 7.2.8-arch1-2",
|
||||
"distribution Arch Linux rolling",
|
||||
"processor AMD Ryzen 9 8945HS",
|
||||
"cores 8",
|
||||
"threads 16",
|
||||
"memory 30.6 GiB",
|
||||
"virtual physical",
|
||||
"up 3d, since",
|
||||
"timezone Europe/Brussels",
|
||||
"filesystems / 920.1 GiB, 29.6 GiB free (ext4 /dev/nvme0n1p3)",
|
||||
"public address unknown: how it learns it is not decided yet",
|
||||
} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("missing %q", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFactAMachineCouldNotReadIsShownUnknownNeverZero(t *testing.T) {
|
||||
now := time.Date(2026, 10, 12, 15, 0, 0, 0, time.UTC)
|
||||
a := inventory.MachineAccount{Architecture: "arm64", Kernel: "linux", Machine: &inventory.MadeOf{
|
||||
ObservedAt: now, Unknown: map[string]string{"memory": "MemTotal not found in /proc/meminfo",
|
||||
"cores": "/proc/cpuinfo says no core id", "kernel release": "permission denied"}}}
|
||||
got := strings.Join(machineLines(a, true, now), "\n")
|
||||
t.Log("\n" + got)
|
||||
for _, want := range []string{
|
||||
"memory unknown: MemTotal not found in /proc/meminfo",
|
||||
"cores unknown: /proc/cpuinfo says no core id",
|
||||
"kernel linux (release unknown: permission denied)",
|
||||
// Neither said nor named unknown: the node-engine broke its rule, and that is shown.
|
||||
"timezone unknown: the machine said neither it nor why",
|
||||
} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("missing %q", want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(got, " 0 ") || strings.Contains(got, "0 B") {
|
||||
t.Error("a fact the machine could not read was shown as a zero")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOlderNodeEngineStillShowsArchitectureAndKernel(t *testing.T) {
|
||||
got := strings.Join(machineLines(inventory.MachineAccount{Architecture: "amd64", Kernel: "linux"}, true,
|
||||
time.Now()), "\n")
|
||||
if !strings.Contains(got, "older than the one that says it") || !strings.Contains(got, "architecture amd64") ||
|
||||
!strings.Contains(got, "kernel linux") {
|
||||
t.Errorf("%s", got)
|
||||
}
|
||||
if got := strings.Join(machineLines(inventory.MachineAccount{}, false, time.Now()), "\n"); !strings.Contains(got, "never said") {
|
||||
t.Errorf("%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAccountWithoutItsTimeSaysAgeUnknown(t *testing.T) {
|
||||
got := machineLines(inventory.MachineAccount{Architecture: "amd64", Kernel: "linux",
|
||||
Machine: &inventory.MadeOf{KernelRelease: "7.2"}}, true, time.Now())[0]
|
||||
if !strings.Contains(got, "age unknown") || strings.Contains(got, "d ago") {
|
||||
t.Errorf("%s", got)
|
||||
}
|
||||
}
|
||||
@@ -875,17 +875,6 @@ func showNode(ctx context.Context, stored *stores, name string) error {
|
||||
}
|
||||
}
|
||||
|
||||
// What it is made of, in one block with its age (novox/hq issue 160): collected and shown in the same
|
||||
// breath, so no fact the machine says is kept and read by nothing.
|
||||
if account, had, err := inv.MachineOf(ctx, name); err != nil {
|
||||
fmt.Printf("\n what it is made of could NOT be read: %v\n", err)
|
||||
} else {
|
||||
fmt.Println()
|
||||
for _, line := range machineLines(account, had, time.Now()) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
}
|
||||
|
||||
held, err := inv.Profile(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
@@ -81,6 +80,30 @@ func listQueue(ctx context.Context, args []string, w io.Writer) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// keepingTheQueueInOrder runs the build queue's passes while this controller acts (novox/hq issue 558):
|
||||
// the one process that puts the queue in order, so no two passes pause and resume it over each other.
|
||||
// The build seat is read from the catalogue at most every half minute.
|
||||
func keepingTheQueueInOrder(ctx context.Context, open *stores, js *broker.JetStream) {
|
||||
if js == nil {
|
||||
return
|
||||
}
|
||||
seat, readAt := link.TheBuildMachine, time.Time{}
|
||||
seatNow := func() string {
|
||||
if time.Since(readAt) < watchEvery {
|
||||
return seat
|
||||
}
|
||||
if entries, err := open.inventory.Catalogued(ctx); err == nil {
|
||||
seat, readAt = buildSeatAmong(entries), time.Now()
|
||||
} else {
|
||||
fmt.Printf("the build seat is read as %s: the catalogue cannot be read: %v\n", seat, err)
|
||||
}
|
||||
return seat
|
||||
}
|
||||
link.KeepingInOrder(ctx, js, seatNow, link.Holding, func(format string, args ...any) {
|
||||
fmt.Printf(format+"\n", args...)
|
||||
})
|
||||
}
|
||||
|
||||
// queueAnswer is the queue as data (novox/hq issue 455): what a caller reads to say how far back an ask
|
||||
// waits — "waiting in the build queue behind 2" — which the table alone could not tell it. Text is the
|
||||
// table, written from the same read: the verb answers it as its output and the rest as its answer, so the
|
||||
@@ -100,7 +123,8 @@ type queueAnswer struct {
|
||||
type queuedAnswer struct {
|
||||
ID string `json:"id"`
|
||||
State string `json:"state"`
|
||||
// Kind is "check" for a pull request's merge check, "build" for anything else.
|
||||
// Kind is "delivery" for a build a walk waits for, "check" for a pull request's check and any ask
|
||||
// whose kind cannot be told (link.KindOf, novox/hq issue 558).
|
||||
Kind string `json:"kind"`
|
||||
Repository string `json:"repository"`
|
||||
Path string `json:"path,omitempty"`
|
||||
@@ -129,10 +153,7 @@ func queueAnswerOf(q link.Queue, now time.Time) queueAnswer {
|
||||
out := queueAnswer{Seat: q.Seat, Waiting: len(waiting), InFlight: len(running), Dead: len(dead),
|
||||
MaxDeliver: q.MaxDeliver, Asks: []queuedAnswer{}, Text: queueText(q, now)}
|
||||
one := func(a link.QueuedAsk, position int) queuedAnswer {
|
||||
kind := "build"
|
||||
if a.Request.Check != nil {
|
||||
kind = "check"
|
||||
}
|
||||
kind := link.KindOf(a.Request)
|
||||
since := a.AskedAt
|
||||
if a.State == link.AskInFlight && a.On != "" && !a.Started.IsZero() {
|
||||
since = a.Started
|
||||
@@ -189,19 +210,20 @@ func queueText(q link.Queue, now time.Time) string {
|
||||
case a.Was != "":
|
||||
where = fmt.Sprintf("handed back by %s, to be handed out again", a.Was)
|
||||
}
|
||||
fmt.Fprintf(&b, " %-26s %s — %s, %s (seq %d)\n", a.ID, what(a), where, ago(a.AskedAt), a.Seq)
|
||||
fmt.Fprintf(&b, " %-26s %-8s %s — %s, %s (seq %d)\n", a.ID, link.KindOf(a.Request), what(a), where,
|
||||
ago(a.AskedAt), a.Seq)
|
||||
}
|
||||
}
|
||||
if len(waiting) > 0 {
|
||||
fmt.Fprintln(&b, "\nwaiting:")
|
||||
fmt.Fprintln(&b, "\nwaiting, in the order they are handed out:")
|
||||
for _, a := range waiting {
|
||||
fmt.Fprintf(&b, " %-26s %s — %s (seq %d)\n", a.ID, what(a), ago(a.AskedAt), a.Seq)
|
||||
fmt.Fprintf(&b, " %-26s %-8s %s — %s (seq %d)\n", a.ID, link.KindOf(a.Request), what(a), ago(a.AskedAt), a.Seq)
|
||||
}
|
||||
}
|
||||
if len(dead) > 0 {
|
||||
fmt.Fprintf(&b, "\ndead — handed out as often as the worker allows (%d) and never settled, still held:\n", q.MaxDeliver)
|
||||
for _, a := range dead {
|
||||
fmt.Fprintf(&b, " %-26s %s — %s (seq %d)\n", a.ID, what(a), ago(a.AskedAt), a.Seq)
|
||||
fmt.Fprintf(&b, " %-26s %-8s %s — %s (seq %d)\n", a.ID, link.KindOf(a.Request), what(a), ago(a.AskedAt), a.Seq)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
@@ -275,6 +297,15 @@ func cancelAsk(ctx context.Context, js *broker.JetStream, open *stores, seat str
|
||||
}
|
||||
}
|
||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
|
||||
// **Where it waits now** (novox/hq issue 558): moved into order since it was read, a waiting ask has a
|
||||
// later sequence, and the worker handing out what was behind its old one is not it being taken.
|
||||
if ask.State == link.AskWaiting {
|
||||
if q, err := link.ReadQueue(ctx, js, seat); err == nil {
|
||||
if now, ok := q.Find(ask.ID); ok && now.State == link.AskWaiting {
|
||||
ask.Seq = now.Seq
|
||||
}
|
||||
}
|
||||
}
|
||||
switch ask.State {
|
||||
case link.AskWaiting:
|
||||
if taken, err := takenSince(js, worker, ask.Seq); err != nil {
|
||||
@@ -296,14 +327,25 @@ func cancelAsk(ctx context.Context, js *broker.JetStream, open *stores, seat str
|
||||
"ends it there", ask.ID, started, ask.ID)
|
||||
}
|
||||
}
|
||||
if err := js.Context().DeleteMsg(worker.Stream, ask.Seq); err != nil && !errors.Is(err, nats.ErrMsgNotFound) &&
|
||||
!errors.Is(err, jetstream.ErrMsgNotFound) && !strings.Contains(err.Error(), "no message found") {
|
||||
// Found by its id when it was moved into order since it was read (novox/hq issue 558).
|
||||
removed, err := link.DeleteAsk(ctx, js, seat, ask)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s is marked cancelled and could not be deleted from the queue (seq %d): %w — a "+
|
||||
"holder taking it ends it as cancelled", ask.ID, ask.Seq, err)
|
||||
}
|
||||
recordCancelled(ctx, open, ask)
|
||||
return fmt.Sprintf("cancelled %s (%s, %s): deleted from the %s queue and recorded failed, %s — a plan "+
|
||||
"that asked for it fails with that", ask.ID, ask.Repository, ask.State, seat, link.CancelledByHand), nil
|
||||
return cancelSaid(ask, seat, removed), nil
|
||||
}
|
||||
|
||||
// cancelSaid is what a cancel says: deleted from the queue only when it was.
|
||||
func cancelSaid(ask link.QueuedAsk, seat string, removed uint64) string {
|
||||
where := fmt.Sprintf("deleted from the %s queue", seat)
|
||||
if removed == 0 {
|
||||
where = fmt.Sprintf("no longer in the %s queue to delete, and marked cancelled, so a holder taking it "+
|
||||
"ends it", seat)
|
||||
}
|
||||
return fmt.Sprintf("cancelled %s (%s, %s): %s and recorded failed, %s — a plan "+
|
||||
"that asked for it fails with that", ask.ID, ask.Repository, ask.State, where, link.CancelledByHand)
|
||||
}
|
||||
|
||||
// holderLooks is how long a cancel waits for a holder that took the ask before the mark to say it
|
||||
|
||||
@@ -24,7 +24,7 @@ func aQueueOfEachKind(now time.Time) link.Queue {
|
||||
State: link.AskInFlight, On: "ace", Started: started,
|
||||
Request: link.BuildRequest{ID: "build-running", Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-host"}}},
|
||||
{Seq: 3, ID: "build-first", Repository: "ssh://forge/novox/mesh-catalog", Ref: "main", AskedAt: asked,
|
||||
State: link.AskWaiting},
|
||||
State: link.AskWaiting, Request: link.BuildRequest{ID: "build-first", Repository: "ssh://forge/novox/mesh-catalog"}},
|
||||
{Seq: 4, ID: "build-second", Repository: "ssh://forge/novox/mesh-catalog", Path: "modules/gitea", Ref: aCommit,
|
||||
AskedAt: now.Add(-time.Minute), State: link.AskWaiting,
|
||||
Request: link.BuildRequest{ID: "build-second", Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog"}}},
|
||||
@@ -51,7 +51,7 @@ func TestTheQueueVerbAnswersEachAskAsData(t *testing.T) {
|
||||
t.Errorf("in flight: %+v", running)
|
||||
}
|
||||
first, second := byID["build-first"], byID["build-second"]
|
||||
if first.Position != 1 || first.Kind != "build" || first.Commit != "" || first.Ref != "main" || first.Machine != "" ||
|
||||
if first.Position != 1 || first.Kind != "delivery" || first.Commit != "" || first.Ref != "main" || first.Machine != "" ||
|
||||
!first.Since.Equal(now.Add(-5*time.Minute)) {
|
||||
t.Errorf("first waiting: %+v", first)
|
||||
}
|
||||
@@ -62,6 +62,13 @@ func TestTheQueueVerbAnswersEachAskAsData(t *testing.T) {
|
||||
if dead := byID["build-dead"]; dead.Position != 0 || dead.State != link.AskDead {
|
||||
t.Errorf("dead: %+v", dead)
|
||||
}
|
||||
// And the table says each ask's kind too (novox/hq issue 558).
|
||||
for _, line := range []string{"build-running check ", "build-first delivery ",
|
||||
"build-second check ", "build-dead check "} {
|
||||
if !strings.Contains(got.Text, line) {
|
||||
t.Errorf("the table does not say %q:\n%s", line, got.Text)
|
||||
}
|
||||
}
|
||||
// In the table's order: in flight, then the waiting by position, then the dead.
|
||||
var order []string
|
||||
for _, a := range got.Asks {
|
||||
@@ -135,3 +142,16 @@ func TestTheQueueNeverShowsARepositorysPassword(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A cancel says it deleted the ask only when it did: moved into order and gone since, it says so (novox/hq
|
||||
// issue 558).
|
||||
func TestACancelSaysDeletedOnlyWhenItDeleted(t *testing.T) {
|
||||
ask := link.QueuedAsk{ID: "build-1", Repository: "novox/x", State: link.AskWaiting, Seq: 4}
|
||||
if said := cancelSaid(ask, link.TheBuildMachine, 9); !strings.Contains(said, "deleted from the") {
|
||||
t.Errorf("removed, it says: %s", said)
|
||||
}
|
||||
if said := cancelSaid(ask, link.TheBuildMachine, 0); strings.Contains(said, "deleted from") ||
|
||||
!strings.Contains(said, "no longer in the") {
|
||||
t.Errorf("gone already, it says: %s", said)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,90 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// TestReplay201ThePlansSend replays novox/hq issue 201 (2026-10-02): two merges a minute apart made two plans; the
|
||||
// controller's plan rolled the newer controller onto the control node, and the node-engine's plan then sent that
|
||||
// machine the controller build before it, whose binary refused the seat row the newer one had widened. Here the
|
||||
// control node, anchor, was last sent the newer controller c2 while the mesh's record of the controller is the
|
||||
// build before it, c1, which passed its gate. The outcome asserted: a send that is no person's push — composed as a
|
||||
// plan's send composes it (keepingRecorded, what sendKeeps gives every send through sendToEach) — carries the
|
||||
// controller at c2, the build the machine was last sent, and not the older record; a person's push still carries
|
||||
// the record. Written with only what the controller had before its fix (mesh-controller #273) — the inventory's
|
||||
// build, gate, assignment and send records, keepingRecorded, planFor and declarationFor — so it is laid over the
|
||||
// commit before; the fix's own TestReplay201 reads what the fix added. Needs the store (MESH_TEST_POSTGRES), and
|
||||
// is named in testdata/store-tests.
|
||||
func TestReplay201ThePlansSend(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
const older, newer = "c1aaaaaaaaaa", "c2bbbbbbbbbb"
|
||||
old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute)
|
||||
record := func(commit string, asked time.Time) inventory.Build {
|
||||
t.Helper()
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: "1"})
|
||||
b := inventory.Build{ID: "build-mesh-controller-" + commit, Module: "mesh-controller", Commit: commit,
|
||||
Repository: "novox/mesh-controller", Manifest: manifest, Asked: asked, At: asked,
|
||||
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + commit + "0000000000000000"}}}
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// The older controller rolled out and passed its gate; the newer one was asked after it and registered.
|
||||
before := record(older, old)
|
||||
if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: before.ID, Module: "mesh-controller", Commit: older,
|
||||
Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record(newer, now)
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, inventory.Source{
|
||||
Repository: "novox/mesh-controller", BuiltFrom: newer, Head: newer, Asked: now}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "anchor", "mesh-controller"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The controller's plan sent anchor the newer controller.
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"mesh-controller": newer}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// And the mesh's record of the controller is the build before it — whatever made it so.
|
||||
if err := inv.RestoreModule(ctx, before); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The controller each send to anchor carries: a plan's send keeps recorded builds (keepingRecorded, what
|
||||
// sendKeeps gives every send but a person's push); a person's push composes the mesh's record.
|
||||
controllerSent := func(sends string, keeping bool) string {
|
||||
t.Helper()
|
||||
c := ctx
|
||||
if keeping {
|
||||
c = keepingRecorded(ctx)
|
||||
}
|
||||
plan, settings, err := planFor(c, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatalf("%s to anchor did not compose: %v", sends, err)
|
||||
}
|
||||
declared, err := declarationFor(c, open, "anchor", plan, settings)
|
||||
if err != nil {
|
||||
t.Fatalf("%s to anchor did not compose: %v", sends, err)
|
||||
}
|
||||
return declared.Builds["mesh-controller"]
|
||||
}
|
||||
|
||||
if got := controllerSent("a plan's send", true); got != newer {
|
||||
t.Fatalf("a plan's send carried anchor the controller %s, older than the %s anchor was last sent: the "+
|
||||
"older binary would read the row its successor widened", short(got), short(newer))
|
||||
}
|
||||
if got := controllerSent("a person's push", false); got != older {
|
||||
t.Fatalf("a person's push carried the controller %s, not the mesh's record %s", short(got), short(older))
|
||||
}
|
||||
}
|
||||
@@ -34,10 +34,8 @@ type seatRow struct {
|
||||
Delivers string `json:"delivers,omitempty"`
|
||||
Decision string `json:"decision"`
|
||||
// Replicated says the seat may be held on several machines at once (novox/hq ADR 0223).
|
||||
Replicated bool `json:"replicated,omitempty"`
|
||||
// Paths are the paths on its machine the seat's holder owns (novox/hq ADR 0287).
|
||||
Paths []catalogue.SeatPath `json:"paths,omitempty"`
|
||||
Holders []seatHolder `json:"holders"`
|
||||
Replicated bool `json:"replicated,omitempty"`
|
||||
Holders []seatHolder `json:"holders"`
|
||||
}
|
||||
|
||||
// seatsHeld is every seat the mesh defines with its holders, and every claim held that names no
|
||||
@@ -51,7 +49,7 @@ func seatsHeld(seats []catalogue.Seat, held []catalogue.Held) ([]seatRow, []cata
|
||||
rows := make([]seatRow, 0, len(seats))
|
||||
for _, s := range seats {
|
||||
row := seatRow{Seat: s.Name, Scope: s.Scope, Delivers: s.Delivers, Decision: s.Decision,
|
||||
Replicated: s.Replicated, Paths: s.Paths, Holders: []seatHolder{}}
|
||||
Replicated: s.Replicated, Holders: []seatHolder{}}
|
||||
seen := map[seatHolder]bool{}
|
||||
for _, h := range held {
|
||||
// Resolve the held claim to a seat rather than comparing names, so a record naming a
|
||||
@@ -257,16 +255,12 @@ func seatsCommand(ctx context.Context, args []string) error {
|
||||
return err
|
||||
}
|
||||
rows, outside := seatsHeld(catalogue.Seats(), world.Held)
|
||||
// Which module owns each seat's path on each machine (novox/hq ADR 0287): the seat naming it, and
|
||||
// that seat's holder there, with the stepping back applied.
|
||||
owners := catalogue.PathOwners(catalogue.Seats(), world.Held)
|
||||
|
||||
if *asJSON {
|
||||
out := struct {
|
||||
Seats []seatRow `json:"seats"`
|
||||
Owners []catalogue.PathOwner `json:"path-owners,omitempty"`
|
||||
Outside []catalogue.Held `json:"outside,omitempty"`
|
||||
}{rows, owners, outside}
|
||||
Seats []seatRow `json:"seats"`
|
||||
Outside []catalogue.Held `json:"outside,omitempty"`
|
||||
}{rows, outside}
|
||||
body, err := json.MarshalIndent(out, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -276,7 +270,7 @@ func seatsCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
|
||||
w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
|
||||
fmt.Fprintln(w, "SEAT\tSCOPE\tDELIVERS\tPATHS\tHELD BY")
|
||||
fmt.Fprintln(w, "SEAT\tSCOPE\tDELIVERS\tHELD BY")
|
||||
for _, r := range rows {
|
||||
delivers := r.Delivers
|
||||
if delivers == "" {
|
||||
@@ -290,22 +284,11 @@ func seatsCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
holders = strings.Join(parts, ", ")
|
||||
}
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", r.Seat, r.Scope, delivers, seatPathsText(r.Paths), holders)
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", r.Seat, r.Scope, delivers, holders)
|
||||
}
|
||||
if err := w.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(owners) > 0 {
|
||||
fmt.Println("\nwho owns each seat's path, by machine (novox/hq ADR 0287):")
|
||||
w = tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
|
||||
fmt.Fprintln(w, "PATH\tMACHINE\tSEAT\tMODULE")
|
||||
for _, o := range owners {
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", o.Path, o.Node, o.Seat, o.Module)
|
||||
}
|
||||
if err := w.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(outside) > 0 {
|
||||
fmt.Println("\nheld, and not a seat this mesh defines (registered before the set closed — novox/hq ADR 0110):")
|
||||
for _, h := range outside {
|
||||
@@ -314,19 +297,3 @@ func seatsCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// seatPathsText is a seat's paths in one column: each path, and the seat it steps back for, if any.
|
||||
func seatPathsText(paths []catalogue.SeatPath) string {
|
||||
if len(paths) == 0 {
|
||||
return "—"
|
||||
}
|
||||
parts := make([]string, 0, len(paths))
|
||||
for _, p := range paths {
|
||||
if p.StepsBackFor != "" {
|
||||
parts = append(parts, p.Path+" (steps back for "+p.StepsBackFor+")")
|
||||
continue
|
||||
}
|
||||
parts = append(parts, p.Path)
|
||||
}
|
||||
return strings.Join(parts, ", ")
|
||||
}
|
||||
|
||||
@@ -74,22 +74,3 @@ func TestAKindedBenchIsNotHandedOver(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The listing shows each seat's paths, and the seat a path steps back for (novox/hq ADR 0287).
|
||||
func TestTheSeatsListingShowsEachSeatsPaths(t *testing.T) {
|
||||
rows, _ := seatsHeld(catalogue.Seats(), nil)
|
||||
got := map[string]string{}
|
||||
for _, r := range rows {
|
||||
got[r.Seat] = seatPathsText(r.Paths)
|
||||
}
|
||||
for seat, want := range map[string]string{
|
||||
"node-hostname": "/etc/hosts, /etc/hostname",
|
||||
"node-uplink": "/etc/resolv.conf (steps back for node-resolver)",
|
||||
"node-resolver": "/etc/resolv.conf",
|
||||
"node-packet-filter": "—",
|
||||
} {
|
||||
if got[seat] != want {
|
||||
t.Errorf("%s lists its paths as %q, not %q", seat, got[seat], want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -739,6 +739,9 @@ func watchTheMesh(ctx context.Context, open *stores, server *link.Server, bus li
|
||||
// and the answer chosen is performed on its warrant.
|
||||
startAsking(watching, open, server, bus.Conn, keeper)
|
||||
go forgettingOldHeals(watching, open.inventory)
|
||||
// And the build queue's order (novox/hq issue 558): a check that has waited its while goes ahead of the
|
||||
// deliveries asked after it even when nothing new is asked.
|
||||
go keepingTheQueueInOrder(watching, open, server.JetStream())
|
||||
fmt.Printf("watching the mesh: %d signal(s) every %s, %d probe(s) every %s; what is wrong is kept in %s "+
|
||||
"and said as %s events\n", len(watchedRows()), watchEvery, len(runnableProbes()), doctorEvery,
|
||||
broker.ConditionsBucket, conditions.Seat)
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
package catalogue
|
||||
|
||||
import "path"
|
||||
|
||||
// The machine's own resolver (novox/hq ADR 0247).
|
||||
//
|
||||
// **Most machines have none.** Every machine lists the mesh's resolvers in /etc/resolv.conf and nothing
|
||||
@@ -68,39 +66,43 @@ func holdsSeat(m Manifest, seat string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// stepsBack is the module as it composes among the modules of one machine (novox/hq ADR 0287,
|
||||
// ADR 0247):
|
||||
// **where it holds a seat that names a path stepping back for another seat, and that other seat's holder
|
||||
// is on the machine and declares the path, its own rendering of the path is left out**, and nothing else
|
||||
// of it changes. Only a file the mesh renders for it steps back, as before: a resource on the path is
|
||||
// composed as declared (the declaration copies a module's resources whole), so an uplink holder declaring
|
||||
// the resolver file as a resource beside the resolver's holder is two owners of one path, and refused. The stepping back is the seat's definition — node-uplink stepping back from the resolver
|
||||
// file for node-resolver — so any other module declaring the path beside the owner's is still two owners
|
||||
// of one path, and is refused as before.
|
||||
func stepsBack(m Manifest, modules []Manifest) Manifest {
|
||||
drop := map[string]bool{}
|
||||
for _, c := range m.Claims {
|
||||
seat, known := SeatNamed(c.Name)
|
||||
if !known {
|
||||
continue
|
||||
}
|
||||
for _, p := range seat.Paths {
|
||||
if p.StepsBackFor == "" {
|
||||
continue
|
||||
}
|
||||
for _, o := range modules {
|
||||
if o.Module != m.Module && holdsSeat(o, p.StepsBackFor) && declaresPath(o, p.Path) {
|
||||
drop[p.Path] = true
|
||||
}
|
||||
}
|
||||
// rendersResolverFile says whether a module asks the mesh to render the machine's resolver file.
|
||||
func rendersResolverFile(m Manifest) bool {
|
||||
for _, f := range m.Facts {
|
||||
if !f.Home && f.Path == ResolverFile {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if len(drop) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
// ResolverFileOwner is the module that writes the machine's resolver file among the modules on one
|
||||
// machine (novox/hq ADR 0247): the holder of node-resolver when one renders it, else nobody is named here
|
||||
// and the file is whoever's it always was — the uplink holder's (ADR 0223).
|
||||
func ResolverFileOwner(modules []Manifest) string {
|
||||
for _, m := range modules {
|
||||
if holdsSeat(m, ResolverSeat) && rendersResolverFile(m) {
|
||||
return m.Module
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// stepsBack is the module as it composes on a machine whose resolver file is the node-resolver holder's:
|
||||
// **the uplink holder's rendering of that file is left out**, and nothing else of it changes. Only the
|
||||
// uplink's holder steps back — any other module rendering the file beside the resolver's is still two
|
||||
// owners of one path, and is refused as before.
|
||||
func stepsBack(m Manifest, modules []Manifest) Manifest {
|
||||
if !holdsSeat(m, "node-uplink") || !rendersResolverFile(m) {
|
||||
return m
|
||||
}
|
||||
owner := ResolverFileOwner(modules)
|
||||
if owner == "" || owner == m.Module {
|
||||
return m
|
||||
}
|
||||
facts := make(map[string]RosterFile, len(m.Facts))
|
||||
for name, f := range m.Facts {
|
||||
if !f.Home && drop[path.Clean(f.Path)] {
|
||||
if !f.Home && f.Path == ResolverFile {
|
||||
continue
|
||||
}
|
||||
facts[name] = f
|
||||
@@ -111,6 +113,9 @@ func stepsBack(m Manifest, modules []Manifest) Manifest {
|
||||
|
||||
// steppedBack is every module of one machine as it composes there (stepsBack, each).
|
||||
func steppedBack(modules []Manifest) []Manifest {
|
||||
if ResolverFileOwner(modules) == "" {
|
||||
return modules
|
||||
}
|
||||
out := make([]Manifest, len(modules))
|
||||
for i, m := range modules {
|
||||
out[i] = stepsBack(m, modules)
|
||||
|
||||
@@ -957,9 +957,7 @@ func checkResources(modules []Manifest) []string {
|
||||
// Judged as the machine composes them: where the node-resolver holder writes the resolver file, the
|
||||
// uplink holder's rendering of it is not composed, so it is not a second owner (novox/hq ADR 0247).
|
||||
modules = steppedBack(modules)
|
||||
// And only a claimant of a seat naming a path declares it — the controller's generated resources
|
||||
// included, which are judged here at composition (novox/hq ADR 0287).
|
||||
problems := seatPathsOnMachine(modules)
|
||||
var problems []string
|
||||
owner := map[string]string{}
|
||||
ownedPath := map[string]string{} // path → owning module, for the access check below
|
||||
packagePresent, packageAbsent := map[string][]string{}, map[string][]string{}
|
||||
|
||||
@@ -1,315 +0,0 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The paths a seat's holder owns (novox/hq ADR 0287).
|
||||
//
|
||||
// **Who owns a machine-wide file is the role's question, answered once.** Until this, which module owned
|
||||
// /etc/hosts or /etc/resolv.conf was decided record by record (ADRs 0199, 0222, 0223, 0241) and enforced
|
||||
// only after the fact, by checkResources finding two modules on one machine declaring the same path: a
|
||||
// module that did not hold node-hostname could declare /etc/hosts on a machine where nobody held it, and
|
||||
// nothing refused it. Now a seat's definition names the paths the role owns on any machine, and three
|
||||
// refusals follow from it, each where the mesh can still say no:
|
||||
//
|
||||
// - a claim whose module does not declare every path its seat names, by CanHold, at registration and at
|
||||
// handover;
|
||||
// - a module declaring a path a seat names while claiming none of the seats that name it, at
|
||||
// registration (CatalogueProblems), and what the controller generates for a machine, at composition;
|
||||
// - a module's own seat naming a path another seat already names, at registration — the second loses.
|
||||
//
|
||||
// Reading a seat's path is access, never a resource, so it stays allowed.
|
||||
|
||||
// SeatPath is one path a seat's holder owns: absolute, literal, on the machine where the holder runs.
|
||||
type SeatPath struct {
|
||||
Path string `json:"path"`
|
||||
// StepsBackFor is a node seat whose holder owns the path instead, on a machine where that seat is held
|
||||
// (novox/hq ADR 0287, ADR 0247): this seat's holder still declares the path, and its declaration is not
|
||||
// composed there. Only the mesh's own seats step back, each by a record.
|
||||
StepsBackFor string `json:"steps-back-for,omitempty"`
|
||||
}
|
||||
|
||||
// seatPathProblem is what is wrong with one path a seat names, judged alone: absolute, written literally
|
||||
// with no placeholder, and clean, so it compares with what a module declares byte for byte.
|
||||
func seatPathProblem(seat, scope, p string) string {
|
||||
switch {
|
||||
case scope != ScopeNode:
|
||||
return fmt.Sprintf("the seat %s names the path %q and is a %s seat; only a node seat names paths, "+
|
||||
"because a path is a machine's (novox/hq ADR 0287)", seat, p, scope)
|
||||
case strings.Contains(p, "${"):
|
||||
return fmt.Sprintf("the seat %s names the path %q, which has a placeholder; a seat's path is written "+
|
||||
"literally (novox/hq ADR 0287)", seat, p)
|
||||
case !strings.HasPrefix(p, "/") || path.Clean(p) != p:
|
||||
return fmt.Sprintf("the seat %s names the path %q, which is not a clean absolute path (novox/hq ADR 0287)",
|
||||
seat, p)
|
||||
case path.Dir(p) == "/":
|
||||
// The root or a top-level directory is every role's, never one role's own (ADR 0287).
|
||||
return fmt.Sprintf("the seat %s names the path %q, the root or a top-level directory, which no one "+
|
||||
"role owns (novox/hq ADR 0287)", seat, p)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// SeatPathProblems is what is wrong with the paths a set of the mesh's own seats names (novox/hq ADR
|
||||
// 0287): only a node seat names one, each absolute and literal; every path has exactly one seat naming it
|
||||
// without stepping back; and a seat steps back only for another node seat naming the same path without
|
||||
// stepping back, so the stepping back cannot cycle.
|
||||
func SeatPathProblems(set []Seat) []string {
|
||||
var problems []string
|
||||
owners := map[string][]string{} // path → the seats naming it without stepping back
|
||||
stepping := map[string][]SeatPath{} // seat → the paths it names stepping back
|
||||
byName := map[string]Seat{}
|
||||
for _, s := range set {
|
||||
byName[s.Name] = s
|
||||
for _, p := range s.Paths {
|
||||
if why := seatPathProblem(s.Name, s.Scope, p.Path); why != "" {
|
||||
problems = append(problems, why)
|
||||
}
|
||||
if p.StepsBackFor == "" {
|
||||
owners[p.Path] = append(owners[p.Path], s.Name)
|
||||
} else {
|
||||
stepping[s.Name] = append(stepping[s.Name], p)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, s := range set {
|
||||
for _, p := range stepping[s.Name] {
|
||||
other, known := byName[p.StepsBackFor]
|
||||
switch {
|
||||
case !known || other.Scope != ScopeNode:
|
||||
problems = append(problems, fmt.Sprintf("the seat %s steps back from %q for %q, which is not a "+
|
||||
"node seat of the mesh's (novox/hq ADR 0287)", s.Name, p.Path, p.StepsBackFor))
|
||||
case !namesWithoutSteppingBack(other, p.Path):
|
||||
problems = append(problems, fmt.Sprintf("the seat %s steps back from %q for %s, which does not "+
|
||||
"own that path itself; a seat steps back only for the one that owns it (novox/hq ADR 0287)",
|
||||
s.Name, p.Path, other.Name))
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, p := range sortedKeys(owners) {
|
||||
if len(owners[p]) > 1 {
|
||||
problems = append(problems, fmt.Sprintf("the path %q is named by %s, each without stepping back; "+
|
||||
"no two seats name one path (novox/hq ADR 0287)", p, strings.Join(owners[p], " and ")))
|
||||
}
|
||||
}
|
||||
for _, s := range set {
|
||||
for _, p := range stepping[s.Name] {
|
||||
if len(owners[p.Path]) == 0 {
|
||||
problems = append(problems, fmt.Sprintf("the path %q has no seat naming it without stepping back "+
|
||||
"(novox/hq ADR 0287)", p.Path))
|
||||
}
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
func namesWithoutSteppingBack(s Seat, p string) bool {
|
||||
for _, sp := range s.Paths {
|
||||
if sp.Path == p && sp.StepsBackFor == "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// declaredPaths is every machine path a module declares it owns: a resource's literal path, and a file
|
||||
// the mesh renders for it outside the operator's home. An access is not here — reading is not owning.
|
||||
//
|
||||
// **Cleaned, because the machine writes the cleaned path**: /etc//hosts, /etc/./hosts, /etc/../etc/hosts and
|
||||
// /etc/hosts/ are all /etc/hosts to it, and a seat's path is clean by its own rule, so a declaration compared
|
||||
// as written would pass for a path nobody named.
|
||||
func declaredPaths(m Manifest) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, r := range m.Resources {
|
||||
if p, ok := r["path"].(string); ok && strings.HasPrefix(p, "/") {
|
||||
out[path.Clean(p)] = true
|
||||
}
|
||||
}
|
||||
for _, f := range m.Facts {
|
||||
if !f.Home && strings.HasPrefix(f.Path, "/") {
|
||||
out[path.Clean(f.Path)] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// declaresPath says whether a module declares one path, as a resource or as a file the mesh renders.
|
||||
func declaresPath(m Manifest, p string) bool { return declaredPaths(m)[p] }
|
||||
|
||||
// missingPaths is each path a seat names that a module does not declare, in the seat's order.
|
||||
func missingPaths(m Manifest, paths []SeatPath) []string {
|
||||
if len(paths) == 0 {
|
||||
return nil
|
||||
}
|
||||
has := declaredPaths(m)
|
||||
var missing []string
|
||||
for _, p := range paths {
|
||||
if !has[p.Path] {
|
||||
missing = append(missing, p.Path)
|
||||
}
|
||||
}
|
||||
return missing
|
||||
}
|
||||
|
||||
// pathSeats is which seats name each path: the mesh's own, and then those modules declare, in the order
|
||||
// given. A module's seat naming a path a seat already names loses it, and the refusal says so (novox/hq
|
||||
// ADR 0287 — as for a seat's name, the second declaration loses).
|
||||
func pathSeats(order []string, shelf Shelf, declaredBy map[string]string) (map[string][]string, []string) {
|
||||
named := map[string][]string{}
|
||||
for _, s := range Seats() {
|
||||
for _, p := range s.Paths {
|
||||
named[p.Path] = append(named[p.Path], s.Name)
|
||||
}
|
||||
}
|
||||
var problems []string
|
||||
for _, module := range order {
|
||||
for _, s := range shelf[module].DefinesSeats {
|
||||
if declaredBy != nil && declaredBy[s.Name] != module {
|
||||
continue // a second declaration of the seat's name, refused for that already
|
||||
}
|
||||
for _, p := range s.Paths {
|
||||
if others := named[p]; len(others) > 0 {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s declares the seat %s naming the path %q, which %s already names; no two seats "+
|
||||
"name one path (novox/hq ADR 0287)", module, s.Name, p, strings.Join(others, " and ")))
|
||||
continue
|
||||
}
|
||||
named[p] = append(named[p], s.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
return named, problems
|
||||
}
|
||||
|
||||
// unclaimedSeatPaths is each path a module declares that a seat names while it claims none of the seats
|
||||
// naming it (novox/hq ADR 0287): only a claimant declares a seat's path, and a module claiming any one of
|
||||
// them may.
|
||||
func unclaimedSeatPaths(m Manifest, named map[string][]string) []string {
|
||||
claims := map[string]bool{}
|
||||
for _, c := range m.Claims {
|
||||
claims[canonicalSeat(c.Name)] = true
|
||||
}
|
||||
var problems []string
|
||||
declared := declaredPaths(m)
|
||||
for _, p := range sortedKeys(declared) {
|
||||
seats := named[p]
|
||||
if len(seats) == 0 {
|
||||
continue
|
||||
}
|
||||
claimsOne := false
|
||||
for _, s := range seats {
|
||||
claimsOne = claimsOne || claims[s]
|
||||
}
|
||||
if !claimsOne {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s declares the path %q, which the seat %s names, and claims none of them; only a holder of "+
|
||||
"a seat naming a path declares it (novox/hq ADR 0287)", m.Module, p, strings.Join(seats, " and ")))
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// declaredSeatPathProblems is what one module's own seat declaration can be judged on alone: a path
|
||||
// only on a node seat, absolute, literal and clean.
|
||||
func declaredSeatPathProblems(m Manifest, s SeatDeclaration) []string {
|
||||
var problems []string
|
||||
for _, p := range s.Paths {
|
||||
if why := seatPathProblem(s.Name, s.At(), p); why != "" {
|
||||
problems = append(problems, m.Module+": "+why)
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// asSeatPaths is a module's own seat's paths in the shape the mesh's seats name theirs. A module's seat
|
||||
// never steps back: only the mesh's own seats do, each by a record.
|
||||
func asSeatPaths(paths []string) []SeatPath {
|
||||
out := make([]SeatPath, 0, len(paths))
|
||||
for _, p := range paths {
|
||||
out = append(out, SeatPath{Path: p})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// PathOwner is the module that owns one seat's path on one machine (novox/hq ADR 0287): the seat naming
|
||||
// the path, and that seat's holder there, with the stepping back applied.
|
||||
type PathOwner struct {
|
||||
Path string `json:"path"`
|
||||
Node string `json:"node"`
|
||||
Seat string `json:"seat"`
|
||||
Module string `json:"module"`
|
||||
}
|
||||
|
||||
// PathOwners is, for every path a seat in the set names and every machine where a seat naming it is held,
|
||||
// who owns the path there. A seat that steps back for another held on the same machine is passed over.
|
||||
func PathOwners(set []Seat, held []Held) []PathOwner {
|
||||
// node → seat → holding module
|
||||
holders := map[string]map[string]string{}
|
||||
for _, h := range held {
|
||||
s, ok := SeatNamed(h.Claim)
|
||||
if !ok || h.Scope != ScopeNode || s.Scope != ScopeNode {
|
||||
continue
|
||||
}
|
||||
if holders[h.Node] == nil {
|
||||
holders[h.Node] = map[string]string{}
|
||||
}
|
||||
if _, taken := holders[h.Node][s.Name]; !taken {
|
||||
holders[h.Node][s.Name] = h.Module
|
||||
}
|
||||
}
|
||||
naming := map[string][]Seat{} // path → the seats naming it
|
||||
steps := map[string]map[string]string{} // path → seat → the seat it steps back for
|
||||
for _, s := range set {
|
||||
for _, p := range s.Paths {
|
||||
naming[p.Path] = append(naming[p.Path], s)
|
||||
if p.StepsBackFor != "" {
|
||||
if steps[p.Path] == nil {
|
||||
steps[p.Path] = map[string]string{}
|
||||
}
|
||||
steps[p.Path][s.Name] = p.StepsBackFor
|
||||
}
|
||||
}
|
||||
}
|
||||
var out []PathOwner
|
||||
for _, p := range sortedKeys(naming) {
|
||||
for _, node := range sortedKeys(holders) {
|
||||
for _, s := range naming[p] {
|
||||
module, holds := holders[node][s.Name]
|
||||
if !holds {
|
||||
continue
|
||||
}
|
||||
if forSeat := steps[p][s.Name]; forSeat != "" {
|
||||
if _, there := holders[node][forSeat]; there {
|
||||
continue
|
||||
}
|
||||
}
|
||||
out = append(out, PathOwner{Path: p, Node: node, Seat: s.Name, Module: module})
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// seatPathsOnMachine is each path the modules of one machine declare that a seat names, for a module
|
||||
// claiming none of the seats naming it (novox/hq ADR 0287). Registration refuses the same of a manifest;
|
||||
// this holds what the controller generates for a machine to it too, because a computed module's resources
|
||||
// exist only once composed (checkResources is asked them there). The seats are the mesh's own and those the
|
||||
// machine's modules declare.
|
||||
func seatPathsOnMachine(modules []Manifest) []string {
|
||||
shelf := Shelf{}
|
||||
var order []string
|
||||
for _, m := range modules {
|
||||
shelf[m.Module] = m
|
||||
order = append(order, m.Module)
|
||||
}
|
||||
named, _ := pathSeats(order, shelf, nil)
|
||||
var problems []string
|
||||
for _, m := range modules {
|
||||
problems = append(problems, unclaimedSeatPaths(m, named)...)
|
||||
}
|
||||
return problems
|
||||
}
|
||||
@@ -1,311 +0,0 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Defends novox/hq ADR 0287 (issue 412): a seat names the paths its holder owns, a claim without them is
|
||||
// refused, and only a claimant declares a seat's path.
|
||||
|
||||
// The starting set, exactly: node-hostname names /etc/hosts and /etc/hostname, node-uplink the resolver
|
||||
// file stepping back for node-resolver, node-resolver the resolver file, and every other seat names none.
|
||||
func TestTheSeatsThatNamePathsAreTheStartingSet(t *testing.T) {
|
||||
want := map[string]string{
|
||||
"node-hostname": "/etc/hosts /etc/hostname",
|
||||
"node-uplink": "/etc/resolv.conf>node-resolver",
|
||||
ResolverSeat: "/etc/resolv.conf",
|
||||
}
|
||||
for _, s := range DefaultSeats() {
|
||||
var got []string
|
||||
for _, p := range s.Paths {
|
||||
if p.StepsBackFor != "" {
|
||||
got = append(got, p.Path+">"+p.StepsBackFor)
|
||||
} else {
|
||||
got = append(got, p.Path)
|
||||
}
|
||||
}
|
||||
if strings.Join(got, " ") != want[s.Name] {
|
||||
t.Errorf("%s names %v; ADR 0287 gives it %q", s.Name, got, want[s.Name])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The compiled set keeps the rules: node seats only, absolute and literal, one seat per path without
|
||||
// stepping back, and a stepping back that cannot cycle.
|
||||
func TestTheCompiledSeatPathsKeepTheRules(t *testing.T) {
|
||||
if p := SeatPathProblems(DefaultSeats()); len(p) != 0 {
|
||||
t.Fatalf("the compiled set breaks ADR 0287: %v", p)
|
||||
}
|
||||
for _, c := range []struct {
|
||||
why string
|
||||
set []Seat
|
||||
want string
|
||||
}{
|
||||
{"a mesh seat naming a path", []Seat{{Name: "mesh-x", Scope: ScopeMesh, Paths: []SeatPath{{Path: "/etc/x"}}}},
|
||||
"only a node seat names paths"},
|
||||
{"a placeholder", []Seat{{Name: "node-x", Scope: ScopeNode, Paths: []SeatPath{{Path: "/etc/${setting:x}"}}}},
|
||||
"placeholder"},
|
||||
{"a relative path", []Seat{{Name: "node-x", Scope: ScopeNode, Paths: []SeatPath{{Path: "etc/x"}}}},
|
||||
"not a clean absolute path"},
|
||||
{"two owners", []Seat{
|
||||
{Name: "node-a", Scope: ScopeNode, Paths: []SeatPath{{Path: "/etc/x"}}},
|
||||
{Name: "node-b", Scope: ScopeNode, Paths: []SeatPath{{Path: "/etc/x"}}}}, "no two seats name one path"},
|
||||
{"a cycle", []Seat{
|
||||
{Name: "node-a", Scope: ScopeNode, Paths: []SeatPath{{Path: "/etc/x", StepsBackFor: "node-b"}}},
|
||||
{Name: "node-b", Scope: ScopeNode, Paths: []SeatPath{{Path: "/etc/x", StepsBackFor: "node-a"}}}},
|
||||
"does not own that path itself"},
|
||||
{"stepping back for nobody", []Seat{
|
||||
{Name: "node-a", Scope: ScopeNode, Paths: []SeatPath{{Path: "/etc/x", StepsBackFor: "node-z"}}}},
|
||||
"not a node seat"},
|
||||
} {
|
||||
if got := strings.Join(SeatPathProblems(c.set), "; "); !strings.Contains(got, c.want) {
|
||||
t.Errorf("%s: %q does not say %q", c.why, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// What the store's rows carry is never the paths: a row read back keeps the compiled ones.
|
||||
func TestTheSeatPathsSurviveTheStoresRows(t *testing.T) {
|
||||
before := seats
|
||||
t.Cleanup(func() { seats = before })
|
||||
UseSeats([]Seat{{Name: "node-hostname", Scope: ScopeNode}})
|
||||
s, _ := SeatNamed("node-hostname")
|
||||
if len(s.Paths) != 2 {
|
||||
t.Fatalf("node-hostname read back from its row names %v", s.Paths)
|
||||
}
|
||||
}
|
||||
|
||||
func hostnameSeat(t *testing.T) Seat {
|
||||
t.Helper()
|
||||
s, ok := SeatNamed("node-hostname")
|
||||
if !ok {
|
||||
t.Fatal("no node-hostname seat")
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func hostnameHolder(resources ...map[string]any) Manifest {
|
||||
return Manifest{Module: "names", Version: "1", Claims: []Claim{{Name: "node-hostname", Scope: ScopeNode,
|
||||
Serves: []string{"entries", "add", "remove"}}}, Resources: resources}
|
||||
}
|
||||
|
||||
// A holder declares every path its seat names, as a resource or as a file the mesh renders for it, or
|
||||
// CanHold refuses it naming the seat and each missing path — the judgement registration and handover share.
|
||||
func TestAClaimWithoutTheSeatsPathsIsRefused(t *testing.T) {
|
||||
seat := hostnameSeat(t)
|
||||
hosts := map[string]any{"id": "hosts", "type": "file", "path": "/etc/hosts", "content": "x"}
|
||||
name := map[string]any{"id": "name", "type": "file", "path": "/etc/hostname", "content": "x"}
|
||||
|
||||
if err := CanHold(hostnameHolder(hosts, name), seat); err != nil {
|
||||
t.Errorf("a holder declaring both paths as resources was refused: %v", err)
|
||||
}
|
||||
rendered := hostnameHolder(hosts)
|
||||
rendered.Facts = map[string]RosterFile{"name": {Path: "/etc/hostname", Template: "{{.Node}}\n"}}
|
||||
if err := CanHold(rendered, seat); err != nil {
|
||||
t.Errorf("a holder declaring a path as a file the mesh renders was refused: %v", err)
|
||||
}
|
||||
err := CanHold(hostnameHolder(), seat)
|
||||
if err == nil {
|
||||
t.Fatal("a holder declaring neither path was accepted")
|
||||
}
|
||||
for _, want := range []string{"node-hostname", "/etc/hosts", "/etc/hostname", "ADR 0287"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not name %s: %v", want, err)
|
||||
}
|
||||
}
|
||||
// A file under the operator's home is placed per account: it is not the machine's path.
|
||||
home := hostnameHolder(hosts)
|
||||
home.Facts = map[string]RosterFile{"name": {Path: "/etc/hostname", Home: true, Template: "x"}}
|
||||
if err := CanHold(home, seat); err == nil {
|
||||
t.Error("a file in the operator's home counted as the machine's /etc/hostname")
|
||||
}
|
||||
// At registration, the same refusal.
|
||||
if got := strings.Join(CatalogueProblems(Shelf{"names": hostnameHolder(hosts)}), "; "); !strings.Contains(got,
|
||||
"does not declare /etc/hostname") {
|
||||
t.Errorf("registration let a holder without /etc/hostname through: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Only a claimant declares a seat's path: a module claiming no seat that names /etc/hosts and declaring
|
||||
// it is refused at registration naming the seat; the uplink's holder and the resolver's both declaring
|
||||
// the resolver file are each a claimant of a seat naming it. Reading a seat's path is access, and allowed.
|
||||
func TestOnlyAClaimantDeclaresASeatsPath(t *testing.T) {
|
||||
stranger := Manifest{Module: "stranger", Version: "1", Resources: []map[string]any{
|
||||
{"id": "hosts", "type": "file", "path": "/etc/hosts", "into": "block", "content": "10.0.0.1 x\n"}}}
|
||||
got := strings.Join(CatalogueProblems(Shelf{"stranger": stranger}), "; ")
|
||||
if !strings.Contains(got, `stranger declares the path "/etc/hosts", which the seat node-hostname names`) {
|
||||
t.Errorf("a module declaring /etc/hosts without claiming node-hostname was not refused: %q", got)
|
||||
}
|
||||
// However the path is spelled (novox/hq issue 412 review).
|
||||
unclean := Manifest{Module: "unclean", Version: "1", Resources: []map[string]any{
|
||||
{"id": "hosts", "type": "file", "path": "/etc/../etc/hosts", "content": "10.0.0.1 x\n"}}}
|
||||
if got := strings.Join(CatalogueProblems(Shelf{"unclean": unclean}), "; "); !strings.Contains(got, "node-hostname") {
|
||||
t.Errorf("/etc/../etc/hosts from a module not claiming node-hostname was not refused: %q", got)
|
||||
}
|
||||
reader := Manifest{Module: "reader", Version: "1", Accesses: []Access{{ID: "hosts", Path: "/etc/hosts"}}}
|
||||
if got := CatalogueProblems(Shelf{"reader": reader}); len(got) != 0 {
|
||||
t.Errorf("reading /etc/hosts was refused: %v", got)
|
||||
}
|
||||
uplink := Manifest{Module: "uplink", Version: "1", Claims: []Claim{{Name: "node-uplink", Scope: ScopeNode}},
|
||||
Facts: map[string]RosterFile{"resolvers": {Path: ResolverFile, Template: "nameserver 10.42.0.1\n"}}}
|
||||
resolver := localResolver()
|
||||
resolver.Claims[0].Serves = []string{"routes", "route", "unroute"}
|
||||
if got := CatalogueProblems(Shelf{"uplink": uplink, resolver.Module: resolver}); len(got) != 0 {
|
||||
t.Errorf("the uplink's holder and the resolver's declaring the resolver file were refused: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A module's own seat names paths under the same rules: a node seat only, literal, and none another seat
|
||||
// names — the second declaration loses. Its holder declares them, as a mesh seat's does.
|
||||
func TestAModulesOwnSeatNamesPathsUnderTheSameRules(t *testing.T) {
|
||||
meshScoped := Manifest{Module: "a", Version: "1", DefinesSeats: []SeatDeclaration{
|
||||
{Name: "a-seat", Paths: []string{"/etc/a"}}}}
|
||||
if got := strings.Join(declaredSeatProblems(meshScoped), "; "); !strings.Contains(got, "only a node seat names paths") {
|
||||
t.Errorf("a module's mesh seat naming a path was accepted: %q", got)
|
||||
}
|
||||
placeholder := Manifest{Module: "a", Version: "1", DefinesSeats: []SeatDeclaration{
|
||||
{Name: "a-seat", Scope: ScopeNode, Paths: []string{"${dir:state}/a"}}}}
|
||||
if got := strings.Join(declaredSeatProblems(placeholder), "; "); !strings.Contains(got, "placeholder") {
|
||||
t.Errorf("a module's seat naming a path with a placeholder was accepted: %q", got)
|
||||
}
|
||||
|
||||
taken := Manifest{Module: "b", Version: "1", DefinesSeats: []SeatDeclaration{
|
||||
{Name: "b-seat", Scope: ScopeNode, Paths: []string{"/etc/hosts"}}}}
|
||||
if got := strings.Join(CatalogueProblems(Shelf{"b": taken}), "; "); !strings.Contains(got,
|
||||
`b declares the seat b-seat naming the path "/etc/hosts", which node-hostname already names`) {
|
||||
t.Errorf("a module's seat naming /etc/hosts was accepted: %q", got)
|
||||
}
|
||||
|
||||
own := Manifest{Module: "c", Version: "1",
|
||||
DefinesSeats: []SeatDeclaration{{Name: "c-seat", Scope: ScopeNode, Paths: []string{"/etc/c.conf"}}},
|
||||
Claims: []Claim{{Name: "c-seat", Scope: ScopeNode}}}
|
||||
if got := strings.Join(CatalogueProblems(Shelf{"c": own}), "; "); !strings.Contains(got,
|
||||
"c claims c-seat but does not declare /etc/c.conf") {
|
||||
t.Errorf("a holder of a module's seat without its path was accepted: %q", got)
|
||||
}
|
||||
own.Resources = []map[string]any{{"id": "conf", "type": "file", "path": "/etc/c.conf", "content": "x"}}
|
||||
if got := CatalogueProblems(Shelf{"c": own}); len(got) != 0 {
|
||||
t.Errorf("a holder declaring its own seat's path was refused: %v", got)
|
||||
}
|
||||
other := Manifest{Module: "d", Version: "1", Resources: []map[string]any{
|
||||
{"id": "conf", "type": "file", "path": "/etc/c.conf", "content": "x"}}}
|
||||
if got := strings.Join(CatalogueProblems(Shelf{"c": own, "d": other}), "; "); !strings.Contains(got,
|
||||
`d declares the path "/etc/c.conf", which the seat c-seat names`) {
|
||||
t.Errorf("a module not claiming c-seat declared its path: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
type writesHosts struct{}
|
||||
|
||||
func (writesHosts) Resources(string) ([]map[string]any, bool, error) {
|
||||
return []map[string]any{{"id": "hosts", "type": "file", "path": "/etc/hosts", "into": "block",
|
||||
"content": "10.42.0.1 anchor\n"}}, true, nil
|
||||
}
|
||||
|
||||
// The controller is bound too: what it generates for a machine on a seat's path is refused at composition.
|
||||
func TestAGeneratedResourceOnASeatsPathIsRefused(t *testing.T) {
|
||||
r := Resolution{Node: "workstation", Modules: []Manifest{
|
||||
{Module: "mesh-names", Computed: "mesh-names"},
|
||||
}}
|
||||
_, err := r.Declaration(Rendering{Generators: map[string]Generator{"mesh-names": writesHosts{}}})
|
||||
if err == nil {
|
||||
t.Fatal("a generated resource on /etc/hosts was composed")
|
||||
}
|
||||
for _, want := range []string{"mesh-names", "/etc/hosts", "node-hostname"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not name %s: %v", want, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Who owns a seat's path on a machine: the seat naming it and its holder there, with the stepping back
|
||||
// applied — the resolver's holder where node-resolver is held, else the uplink's.
|
||||
func TestPathOwnersApplyTheSteppingBack(t *testing.T) {
|
||||
held := []Held{
|
||||
{Claim: "node-uplink", Scope: ScopeNode, Node: "laptop", Module: "networkmanager"},
|
||||
{Claim: ResolverSeat, Scope: ScopeNode, Node: "laptop", Module: "systemd-resolved"},
|
||||
{Claim: "node-uplink", Scope: ScopeNode, Node: "server", Module: "systemd-networkd"},
|
||||
{Claim: "node-hostname", Scope: ScopeNode, Node: "server", Module: "hostname"},
|
||||
}
|
||||
var got []string
|
||||
for _, o := range PathOwners(Seats(), held) {
|
||||
got = append(got, o.Path+" "+o.Node+" "+o.Seat+" "+o.Module)
|
||||
}
|
||||
want := []string{
|
||||
"/etc/hostname server node-hostname hostname",
|
||||
"/etc/hosts server node-hostname hostname",
|
||||
"/etc/resolv.conf laptop node-resolver systemd-resolved",
|
||||
"/etc/resolv.conf server node-uplink systemd-networkd",
|
||||
}
|
||||
if strings.Join(got, "\n") != strings.Join(want, "\n") {
|
||||
t.Errorf("the owners are\n%s\nnot\n%s", strings.Join(got, "\n"), strings.Join(want, "\n"))
|
||||
}
|
||||
}
|
||||
|
||||
// The stepping back comes from the seat, and is of a rendered file only: an uplink holder declaring the
|
||||
// resolver file as a resource beside a node-resolver holder is two writers of one path, refused when the
|
||||
// machine is resolved — never composed twice — as before the seat named its paths.
|
||||
func TestAnUplinkDeclaringTheResolverFileAsAResourceIsRefusedBesideTheResolver(t *testing.T) {
|
||||
shelf := resolverShelf(t)
|
||||
uplink := shelf["systemd-networkd"]
|
||||
uplink.Facts = nil
|
||||
uplink.Resources = append(append([]map[string]any(nil), uplink.Resources...),
|
||||
map[string]any{"id": "resolvers", "type": "file", "path": ResolverFile, "content": "nameserver 10.42.0.1\n"})
|
||||
shelf["systemd-networkd"] = uplink
|
||||
shelf["local-resolver"] = localResolver()
|
||||
_, err := Resolve(shelf, []string{"dnsmasq", "systemd-networkd", "local-resolver"}, splitDNSLaptop(), World{})
|
||||
if err == nil || !strings.Contains(err.Error(), `both declare the path "/etc/resolv.conf"`) {
|
||||
t.Fatalf("an uplink declaring the resolver file as a resource beside the resolver was not refused: %v", err)
|
||||
}
|
||||
// Its rendered file steps back, by the seat's definition, and nothing else of it does.
|
||||
rendered := Manifest{Module: "uplink", Version: "1", Claims: []Claim{{Name: "node-uplink"}},
|
||||
Facts: map[string]RosterFile{"resolvers": {Path: "/etc//resolv.conf", Template: "x"},
|
||||
"other": {Path: "/etc/uplink.conf", Template: "x"}}}
|
||||
got := stepsBack(rendered, []Manifest{rendered, localResolver()})
|
||||
if len(got.Facts) != 1 || got.Facts["other"].Path == "" {
|
||||
t.Errorf("the uplink's holder renders %v beside the resolver", got.Facts)
|
||||
}
|
||||
}
|
||||
|
||||
// A path is compared as the machine writes it: cleaned. Each spelling of /etc/hosts is /etc/hosts.
|
||||
func TestAnUncleanSpellingOfASeatsPathIsTheSeatsPath(t *testing.T) {
|
||||
for _, spelled := range []string{"/etc//hosts", "/etc/./hosts", "/etc/../etc/hosts", "/etc/hosts/"} {
|
||||
stranger := Manifest{Module: "stranger", Version: "1", Resources: []map[string]any{
|
||||
{"id": "hosts", "type": "file", "path": spelled, "content": "10.0.0.1 x\n"}}}
|
||||
if got := strings.Join(CatalogueProblems(Shelf{"stranger": stranger}), "; "); !strings.Contains(got,
|
||||
`stranger declares the path "/etc/hosts", which the seat node-hostname names`) {
|
||||
t.Errorf("%s from a module not claiming node-hostname was not refused: %q", spelled, got)
|
||||
}
|
||||
holder := hostnameHolder(map[string]any{"id": "hosts", "type": "file", "path": spelled, "content": "x"},
|
||||
map[string]any{"id": "name", "type": "file", "path": "/etc/hostname", "content": "x"})
|
||||
if err := CanHold(holder, hostnameSeat(t)); err != nil {
|
||||
t.Errorf("a holder declaring %s was refused: %v", spelled, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A seat names a role's own path, never the root or a top-level directory.
|
||||
func TestASeatNamesNoRootOrTopLevelDirectory(t *testing.T) {
|
||||
for _, p := range []string{"/", "/etc"} {
|
||||
m := Manifest{Module: "a", Version: "1", DefinesSeats: []SeatDeclaration{
|
||||
{Name: "a-seat", Scope: ScopeNode, Paths: []string{p}}}}
|
||||
if got := strings.Join(declaredSeatProblems(m), "; "); !strings.Contains(got, "top-level directory") {
|
||||
t.Errorf("a module's seat naming %s was accepted: %q", p, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue as it is passes: no holder of a seat naming a path is missing one, and nobody else
|
||||
// declares one.
|
||||
func TestTheCataloguePassesTheSeatPathRules(t *testing.T) {
|
||||
shelf := Shelf{}
|
||||
for _, m := range theCatalogue(t) {
|
||||
shelf[m.Module] = m
|
||||
}
|
||||
for _, p := range CatalogueProblems(shelf) {
|
||||
if strings.Contains(p, "ADR 0287") {
|
||||
t.Errorf("the catalogue breaks ADR 0287: %s", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -53,11 +53,6 @@ type Seat struct {
|
||||
// ${contribution:<seat>:<kind>}. Compiled, never stored: like the protocol, it is the mesh's
|
||||
// definition of the role, and the store's rows carry no column for it.
|
||||
Receives []Receivable
|
||||
// Paths are the paths on its machine the seat's holder owns (novox/hq ADR 0287): a holder declares
|
||||
// every one, and a module claiming no seat that names a path may not declare it. Only a node seat
|
||||
// names paths. Compiled, never stored, like Receives: adding or taking away a path is a decision,
|
||||
// recorded, and the store's rows carry no column for it.
|
||||
Paths []SeatPath
|
||||
// Decision is the record that made it a seat.
|
||||
Decision string
|
||||
}
|
||||
@@ -183,8 +178,6 @@ var defaultSeats = append([]Seat{
|
||||
// holds none of it. Named node-hosts-file until ADR 0223; the former name resolves to it as an
|
||||
// alias on a mesh that knew it.
|
||||
{Name: "node-hostname", Scope: ScopeNode, Decision: "novox/hq ADR 0199, ADR 0223",
|
||||
// The machine's names are the role's files, whatever program fills it (novox/hq ADR 0287).
|
||||
Paths: []SeatPath{{Path: "/etc/hosts"}, {Path: "/etc/hostname"}},
|
||||
Serves: []Verb{
|
||||
{Name: "entries", Description: "Every line of this machine's /etc/hosts, each marked whose it is: " +
|
||||
"the operator's, or the block of the module or tool that writes it.",
|
||||
@@ -306,17 +299,12 @@ var defaultSeats = append([]Seat{
|
||||
// resolver file as it is and who wrote it, and every link with its addresses, routes and the resolvers
|
||||
// the manager knows for it — the same for every holder, so a question about a machine's names is asked
|
||||
// the same way whatever manages its network. Optional until both holders serve them.
|
||||
//
|
||||
// It owns the resolver file, stepping back for node-resolver where that is held (novox/hq ADR 0287,
|
||||
// ADR 0247).
|
||||
{Name: "node-uplink", Scope: ScopeNode, Decision: "novox/hq ADR 0117", Serves: uplinkVerbs(),
|
||||
Paths: []SeatPath{{Path: ResolverFile, StepsBackFor: ResolverSeat}}},
|
||||
{Name: "node-uplink", Scope: ScopeNode, Decision: "novox/hq ADR 0117", Serves: uplinkVerbs()},
|
||||
// The machine's own resolver (novox/hq ADR 0247): held only where something requires `split-dns` — a
|
||||
// VPN client whose domains must go to its own servers while every other name goes to the mesh's.
|
||||
// Where it is held, its holder writes the resolver file and the uplink's holder steps back from it
|
||||
// (node_resolver.go). It knows nothing of any VPN: its verbs route domains to servers over a link.
|
||||
{Name: ResolverSeat, Scope: ScopeNode, Decision: "novox/hq ADR 0247", Serves: resolverVerbs(),
|
||||
Paths: []SeatPath{{Path: ResolverFile}}},
|
||||
{Name: ResolverSeat, Scope: ScopeNode, Decision: "novox/hq ADR 0247", Serves: resolverVerbs()},
|
||||
// A machine's shares and the shares it mounts (novox/hq ADR 0263): the holder of node-nfs-server
|
||||
// exports a machine's folders to the private network and provides each as `nfs-share`; the holder of
|
||||
// node-mounts writes a mount and an automount unit per share on a machine that asks (shares.go).
|
||||
@@ -376,11 +364,10 @@ func UseSeats(s []Seat) {
|
||||
row.Accepts, row.Emits, row.Serves = d.Accepts, d.Emits, d.Serves
|
||||
}
|
||||
}
|
||||
// What a seat receives, whether it is replicated and the paths it names are never stored
|
||||
// (novox/hq ADR 0212, ADR 0223, ADR 0287), so they are always the compiled ones.
|
||||
// What a seat receives and whether it is replicated are never stored (novox/hq ADR 0212, ADR
|
||||
// 0223), so they are always the compiled ones.
|
||||
if d, known := byName[row.Name]; known {
|
||||
row.Receives = d.Receives
|
||||
row.Paths = d.Paths
|
||||
row.Replicated = d.Replicated
|
||||
row.Serves = optionalAsCompiled(row.Serves, d.Serves)
|
||||
}
|
||||
@@ -554,14 +541,6 @@ func CanHold(m Manifest, seat Seat) error {
|
||||
return fmt.Errorf("%s claims %s, whose holder answers for %q, and %s does not provide %q at %s scope",
|
||||
m.Module, seat.Name, seat.Delivers, m.Module, seat.Delivers, seat.Scope)
|
||||
}
|
||||
// **A holder owns what the role owns** (novox/hq ADR 0287): every path the seat names, declared as
|
||||
// a resource or as a file the mesh renders for it, so which module owns /etc/hosts on a machine is
|
||||
// answered by the seat's definition and not by reading every manifest.
|
||||
if missing := missingPaths(m, seat.Paths); len(missing) > 0 {
|
||||
return fmt.Errorf("%s claims %s but does not declare %s, which that seat's holder owns (novox/hq "+
|
||||
"ADR 0287) — a holder declares every path its seat names, as a resource or as a file the mesh "+
|
||||
"renders for it", m.Module, seat.Name, strings.Join(missing, ", "))
|
||||
}
|
||||
// **Serving the seat's tools is a condition of holding it** (novox/hq ADR 0132). A holder that
|
||||
// does not answer what the role promises is every caller's timeout, found at registration and
|
||||
// at handover instead, naming the verbs rather than the fact that something is missing.
|
||||
|
||||
@@ -67,10 +67,6 @@ type SeatDeclaration struct {
|
||||
// Records are state buckets of the declaring module that each user reads under its own name — the
|
||||
// keys `<user>.…` and no other (ADR 0259 §3).
|
||||
Records []string `json:"records,omitempty"`
|
||||
// Paths are the paths on its machine the seat's holder owns (novox/hq ADR 0287), under the rules the
|
||||
// mesh's own seats name theirs by: a node seat only, each absolute and literal, and none another seat
|
||||
// names. A module's seat never steps back.
|
||||
Paths []string `json:"paths,omitempty"`
|
||||
}
|
||||
|
||||
// KindedBenches are the seats that may be kinded (novox/hq ADR 0234 §2): `channel` sends to the operator,
|
||||
@@ -167,7 +163,6 @@ func declaredSeatProblems(m Manifest) []string {
|
||||
}
|
||||
}
|
||||
problems = append(problems, trafficProblems(m, s)...)
|
||||
problems = append(problems, declaredSeatPathProblems(m, s)...)
|
||||
}
|
||||
|
||||
for _, u := range m.Uses {
|
||||
@@ -276,10 +271,6 @@ func CatalogueProblems(shelf Shelf) []string {
|
||||
return ok
|
||||
}
|
||||
|
||||
// Which seats name each path (novox/hq ADR 0287): a module's seat naming one another seat names loses.
|
||||
named, pathProblems := pathSeats(shelfOrder(shelf), shelf, declaredBy)
|
||||
problems = append(problems, pathProblems...)
|
||||
|
||||
// Who claims each kind of a kinded bench, so a second claim of one kind is refused (ADR 0234 §2).
|
||||
kindsTaken := map[string]string{}
|
||||
|
||||
@@ -336,15 +327,7 @@ func CatalogueProblems(shelf Shelf) []string {
|
||||
"%s claims %s but does not serve %s, which that seat's protocol promises",
|
||||
module, c.Name, strings.Join(missing, ", ")))
|
||||
}
|
||||
// And owns what the seat's holder owns, as a holder of the mesh's own seats does (ADR 0287).
|
||||
if missing := missingPaths(m, asSeatPaths(s.Paths)); len(missing) > 0 {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s claims %s but does not declare %s, which that seat's holder owns (novox/hq ADR 0287)",
|
||||
module, c.Name, strings.Join(missing, ", ")))
|
||||
}
|
||||
}
|
||||
// Only a claimant declares a seat's path (novox/hq ADR 0287).
|
||||
problems = append(problems, unclaimedSeatPaths(m, named)...)
|
||||
}
|
||||
// **A trusted holder runs as its own account** (novox/hq ADR 0259 §8): a module saying warrants, or
|
||||
// speaking for a kind that proves its sender, is never carried by a machine's runtime.
|
||||
|
||||
@@ -33,9 +33,7 @@ func TestTheUplinkSeatPromisesItsVerbsAndRequiresNoneYet(t *testing.T) {
|
||||
if !slices.Equal(names, []string{"resolvers", "links"}) {
|
||||
t.Fatalf("node-uplink promises %v", names)
|
||||
}
|
||||
// The resolver file is the seat's path (novox/hq ADR 0287), which every holder of today renders.
|
||||
today := Manifest{Module: "networkmanager", Version: "1", Claims: []Claim{{Name: "node-uplink", Scope: ScopeNode}},
|
||||
Facts: map[string]RosterFile{"resolvers": {Path: ResolverFile, Template: "x"}}}
|
||||
today := Manifest{Module: "networkmanager", Version: "1", Claims: []Claim{{Name: "node-uplink", Scope: ScopeNode}}}
|
||||
if err := CanHold(today, seat); err != nil {
|
||||
t.Fatalf("a holder that serves no verb yet is refused: %v", err)
|
||||
}
|
||||
@@ -57,8 +55,7 @@ func TestTheUplinkVerbsReadBackFromTheRowStayOptional(t *testing.T) {
|
||||
// The row as seeding stores it: both verbs, the mark not a column.
|
||||
UseSeats([]Seat{{Name: "node-uplink", Scope: ScopeNode, Serves: []Verb{{Name: "resolvers"}, {Name: "links"}}}})
|
||||
seat := uplinkSeat(t)
|
||||
holder := Manifest{Module: "systemd-networkd", Version: "1", Claims: []Claim{{Name: "node-uplink", Scope: ScopeNode}},
|
||||
Facts: map[string]RosterFile{"resolvers": {Path: ResolverFile, Template: "x"}}}
|
||||
holder := Manifest{Module: "systemd-networkd", Version: "1", Claims: []Claim{{Name: "node-uplink", Scope: ScopeNode}}}
|
||||
if err := CanHold(holder, seat); err != nil {
|
||||
t.Fatalf("read back from the row, the verbs are required: %v", err)
|
||||
}
|
||||
|
||||
@@ -1,74 +0,0 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// MachineAccount is what a machine said it is made of (novox/hq issue 160), from the profile it last
|
||||
// reported: the architecture and kernel family the profile always carried, and the facts a node-engine that
|
||||
// knows how says beside them. Read by `node`, so what a machine says of itself is visible in one place —
|
||||
// collected and shown in the same breath, not collected and read by nothing (issue 160's pattern).
|
||||
type MachineAccount struct {
|
||||
Architecture string `json:"architecture"`
|
||||
Kernel string `json:"kernel"`
|
||||
// Machine is nil when the node-engine that reported is older than the one that says it.
|
||||
Machine *MadeOf `json:"machine"`
|
||||
}
|
||||
|
||||
// MadeOf is the node-engine's account of what it is made of, as it put it on the wire. A fact it could
|
||||
// not determine is absent here and named in Unknown with why — never a zero.
|
||||
type MadeOf struct {
|
||||
ObservedAt time.Time `json:"observed_at"`
|
||||
KernelRelease string `json:"kernel_release"`
|
||||
Distribution string `json:"distribution"`
|
||||
Version string `json:"version"`
|
||||
Processor string `json:"processor"`
|
||||
Cores int `json:"cores"`
|
||||
Threads int `json:"threads"`
|
||||
MemoryBytes int64 `json:"memory_bytes"`
|
||||
Virtual string `json:"virtual"`
|
||||
BootedAt *time.Time `json:"booted_at"`
|
||||
Timezone string `json:"timezone"`
|
||||
Filesystems []MountedFilesystem `json:"filesystems"`
|
||||
Unknown map[string]string `json:"unknown"`
|
||||
}
|
||||
|
||||
// MountedFilesystem is one local filesystem a machine has mounted.
|
||||
type MountedFilesystem struct {
|
||||
Mount string `json:"mount"`
|
||||
Device string `json:"device"`
|
||||
Type string `json:"type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
FreeBytes int64 `json:"free_bytes"`
|
||||
}
|
||||
|
||||
// MachineOf is what a node last said it is made of. had is false for a node that has never reported a
|
||||
// profile at all, which is a different answer from one whose node-engine does not say the facts yet.
|
||||
func (i *Inventory) MachineOf(ctx context.Context, nodeName string) (account MachineAccount, had bool, err error) {
|
||||
var raw []byte
|
||||
err = i.store.Pool().QueryRow(ctx, `select profile from node where name = $1`, nodeName).Scan(&raw)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return MachineAccount{}, false, fmt.Errorf("%w: %s", ErrNoSuchNode, nodeName)
|
||||
}
|
||||
if err != nil {
|
||||
return MachineAccount{}, false, err
|
||||
}
|
||||
return machineFrom(raw)
|
||||
}
|
||||
|
||||
func machineFrom(raw []byte) (MachineAccount, bool, error) {
|
||||
if len(raw) == 0 {
|
||||
return MachineAccount{}, false, nil
|
||||
}
|
||||
var account MachineAccount
|
||||
if err := json.Unmarshal(raw, &account); err != nil {
|
||||
return MachineAccount{}, false, err
|
||||
}
|
||||
return account, true, nil
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The node-engine's words for a machine's facts (novox/hq issue 160), as mesh-host's profile.Machine puts
|
||||
// them on the wire. A rename on either side fails here rather than as a blank on a real machine.
|
||||
func TestAMachinesAccountIsReadInTheNodeEnginesWords(t *testing.T) {
|
||||
raw := []byte(`{"architecture":"amd64","kernel":"linux","capabilities":[],
|
||||
"machine":{"observed_at":"2026-10-12T15:00:00Z","kernel_release":"7.2.8-arch1-2",
|
||||
"distribution":"Arch Linux","version":"rolling","processor":"AMD Ryzen 9","cores":8,"threads":16,
|
||||
"memory_bytes":32891494400,"virtual":"physical","booted_at":"2026-10-09T08:50:34Z",
|
||||
"timezone":"Europe/Brussels",
|
||||
"filesystems":[{"mount":"/","device":"/dev/nvme0n1p3","type":"ext4","size_bytes":100,"free_bytes":40}],
|
||||
"unknown":{"public address":"not decided"}}}`)
|
||||
a, had, err := machineFrom(raw)
|
||||
if err != nil || !had {
|
||||
t.Fatalf("had %v, err %v", had, err)
|
||||
}
|
||||
m := a.Machine
|
||||
if a.Architecture != "amd64" || a.Kernel != "linux" || m == nil {
|
||||
t.Fatalf("%+v", a)
|
||||
}
|
||||
if m.KernelRelease != "7.2.8-arch1-2" || m.Distribution != "Arch Linux" || m.Version != "rolling" ||
|
||||
m.Processor != "AMD Ryzen 9" || m.Cores != 8 || m.Threads != 16 || m.MemoryBytes != 32891494400 ||
|
||||
m.Virtual != "physical" || m.BootedAt == nil || m.Timezone != "Europe/Brussels" ||
|
||||
len(m.Filesystems) != 1 || m.Filesystems[0].FreeBytes != 40 || m.Unknown["public address"] == "" ||
|
||||
m.ObservedAt.IsZero() {
|
||||
t.Errorf("%+v", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAProfileWithoutTheFactsIsAnOlderNodeEngineNotNothing(t *testing.T) {
|
||||
a, had, err := machineFrom([]byte(`{"architecture":"amd64","kernel":"linux","capabilities":[]}`))
|
||||
if err != nil || !had || a.Machine != nil || a.Architecture != "amd64" {
|
||||
t.Errorf("%+v had %v err %v", a, had, err)
|
||||
}
|
||||
if _, had, _ := machineFrom(nil); had {
|
||||
t.Error("a node that never reported read as having reported")
|
||||
}
|
||||
}
|
||||
@@ -79,6 +79,8 @@ func (b *natsBuilds) Ask(ctx context.Context, request BuildRequest) error {
|
||||
if _, err := b.js.Context().Publish(BuildWorkOf(b.role()), body, nats.Context(publish)); err != nil {
|
||||
return fmt.Errorf("cannot submit a build: %w", err)
|
||||
}
|
||||
// The acting controller puts it in order (novox/hq issue 558); this only says an ask joined.
|
||||
signalAsked()
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -113,6 +115,8 @@ func (b *natsBuilds) Submit(ctx context.Context, request BuildRequest,
|
||||
if _, err := b.js.Context().Publish(BuildWorkOf(b.role()), body, nats.Context(publish)); err != nil {
|
||||
return BuildResult{}, fmt.Errorf("%w: cannot submit a build: %w", ErrNotHandedOver, err)
|
||||
}
|
||||
// The acting controller puts it in order (novox/hq issue 558); this only says an ask joined.
|
||||
signalAsked()
|
||||
|
||||
waiting, cancelWait := context.WithTimeout(ctx, wait)
|
||||
defer cancelWait()
|
||||
@@ -167,10 +171,6 @@ type MachineOptions struct {
|
||||
// Paused is asked before every fetch: while it says so, nothing new is taken, and a build
|
||||
// already running finishes. Nil is never paused.
|
||||
Paused func() bool
|
||||
// Taking is called once, when this machine is bound to its worker and pulls work: from then on it
|
||||
// can take a build, which is what its container's health check reads (novox/hq issue 544). Nil is
|
||||
// told nothing.
|
||||
Taking func()
|
||||
}
|
||||
|
||||
// MachineOverNATS takes build work from the current build role.
|
||||
@@ -236,9 +236,6 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
|
||||
"the mesh's to answer, not this machine's", m.seat, err)
|
||||
}
|
||||
m.sub = sub
|
||||
if m.opts.Taking != nil {
|
||||
m.opts.Taking()
|
||||
}
|
||||
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
|
||||
@@ -368,36 +368,3 @@ func TestNatsAMachineOnTheRetiredBuildRoleTakesThatRolesAsks(t *testing.T) {
|
||||
t.Errorf("the retired role's holder did not answer: %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
// A holder says once that it takes work, and only once it is bound to its worker (novox/hq issue 544):
|
||||
// its container's health check reads that, so a holder that could not bind is never said ready.
|
||||
func TestNatsAHolderSaysItTakesWorkOnceItIsBound(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
said := make(chan struct{}, 4)
|
||||
machine := MachineOverNATSWith(js, "anchor", TheBuildMachine, MachineOptions{Taking: func() { said <- struct{}{} }})
|
||||
defer machine.Close()
|
||||
go func() { _ = machine.Take(ctx, func(context.Context, Build) {}) }()
|
||||
select {
|
||||
case <-said:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("a holder bound to its worker never said it takes work")
|
||||
}
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if len(said) != 0 {
|
||||
t.Fatalf("a holder said it takes work %d more time(s)", len(said))
|
||||
}
|
||||
|
||||
// A role the mesh raised no worker for cannot be bound, and its holder says nothing.
|
||||
bounded, end := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer end()
|
||||
other := MachineOverNATSWith(js, "anchor", "mesh-no-such-seat", MachineOptions{Taking: func() { said <- struct{}{} }})
|
||||
defer other.Close()
|
||||
if err := other.Take(bounded, func(context.Context, Build) {}); err == nil {
|
||||
t.Fatal("a machine with no worker took work")
|
||||
}
|
||||
if len(said) != 0 {
|
||||
t.Fatal("a machine that could not bind said it takes work")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -325,9 +325,6 @@ type ChangePlan struct {
|
||||
Steps []string `json:"steps,omitempty"`
|
||||
// Summary is the plan in one line, for a commit status.
|
||||
Summary string `json:"summary"`
|
||||
// Class is the class of the slowest-budgeted module the change moves (novox/hq ADR 0282 §1, ADR 0313 §2):
|
||||
// core, leaf, or none when it moves no module at all — a fix that moves nothing has no fix budget.
|
||||
Class string `json:"class,omitempty"`
|
||||
// Trunk is the trunk commit the head was merged into when it was judged (novox/hq ADR 0323 §3), twelve
|
||||
// characters: set on the plan a verdict carries, which mesh-delivery keeps with its other inputs.
|
||||
Trunk string `json:"trunk,omitempty"`
|
||||
|
||||
+26
-10
@@ -62,7 +62,11 @@ type QueuedAsk struct {
|
||||
Path string `json:"path,omitempty"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
AskedAt time.Time `json:"asked-at,omitempty"`
|
||||
State string `json:"state"`
|
||||
// Stored is when the stream first took it, a move into order kept (novox/hq issue 558).
|
||||
Stored time.Time `json:"stored,omitempty"`
|
||||
State string `json:"state"`
|
||||
// movedFrom is the MovedFromHeader it carries: the sequence of the message a move copied it from.
|
||||
movedFrom string
|
||||
// On and Started are the holder building an in-flight ask and when it started, from its started
|
||||
// event. Was is the holder that started an in-flight ask and is no longer building it — handed
|
||||
// back, to be handed out again — with Started its start there.
|
||||
@@ -236,15 +240,7 @@ func ReadQueue(ctx context.Context, js *broker.JetStream, seat string) (Queue, e
|
||||
if err != nil {
|
||||
return q, fmt.Errorf("reading the %s work queue: %w", seat, err)
|
||||
}
|
||||
ask := QueuedAsk{Seq: msg.Sequence}
|
||||
if json.Unmarshal(msg.Data, &ask.Request) == nil {
|
||||
r := ask.Request
|
||||
ask.ID, ask.Repository, ask.Path, ask.Ref = r.ID, r.Repository, r.Path, r.Ref
|
||||
if at, ok := BuildAskedAt(r.ID); ok {
|
||||
ask.AskedAt = at
|
||||
}
|
||||
}
|
||||
asks = append(asks, ask)
|
||||
asks = append(asks, askOf(msg))
|
||||
seq = msg.Sequence + 1
|
||||
}
|
||||
|
||||
@@ -269,6 +265,26 @@ func ReadQueue(ctx context.Context, js *broker.JetStream, seat string) (Queue, e
|
||||
return q, nil
|
||||
}
|
||||
|
||||
// askOf is one ask as the stream holds it: what its request says, and when the stream took it.
|
||||
func askOf(msg *jetstream.RawStreamMsg) QueuedAsk {
|
||||
ask := QueuedAsk{Seq: msg.Sequence, Stored: msg.Time}
|
||||
// Moved into order, it keeps when the stream first took it (novox/hq issue 558).
|
||||
if at, err := time.Parse(time.RFC3339Nano, msg.Header.Get(QueuedAtHeader)); err == nil {
|
||||
ask.Stored = at
|
||||
}
|
||||
if json.Unmarshal(msg.Data, &ask.Request) == nil {
|
||||
r := ask.Request
|
||||
ask.ID, ask.Repository, ask.Path, ask.Ref = r.ID, r.Repository, r.Path, r.Ref
|
||||
if at, ok := BuildAskedAt(r.ID); ok {
|
||||
ask.AskedAt = at
|
||||
}
|
||||
} else {
|
||||
// Nothing read is nothing kept: a half-read request would pass for a delivery (novox/hq issue 558).
|
||||
ask.Request = BuildRequest{}
|
||||
}
|
||||
return ask
|
||||
}
|
||||
|
||||
// WithoutPassword is a repository as a listing may say it: a URL's password, should one ever be written
|
||||
// into it, left out (novox/hq issue 455). Only for what is shown: the ask itself keeps what it was given,
|
||||
// which a cancel's outcome and a rebuild's match are read from.
|
||||
|
||||
@@ -0,0 +1,426 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// The order the build queue hands asks out in (novox/hq issue 558).
|
||||
//
|
||||
// **A build a walk waits for goes before a pull request's check.** The queue was first come, first served,
|
||||
// and a merged change's delivery build waited minutes behind checks while its walk, and every walk after
|
||||
// it, waited on the build. So:
|
||||
//
|
||||
// - a delivery is handed out before any check;
|
||||
// - a check that has waited CheckWaitsAtMost goes before every delivery asked after it, so no check
|
||||
// waits for ever behind a stream of deliveries;
|
||||
// - within a kind, first come, first served, by when it was asked;
|
||||
// - an ask whose kind cannot be told counts as a check: a delivery taken late breaks a budget, a check
|
||||
// taken late breaks nothing.
|
||||
//
|
||||
// **Kept in the stream, never beside it.** The queue is the seat's work queue, which the server hands out
|
||||
// in stream order to whichever holder pulls; nothing else holds the order, so a restarted controller
|
||||
// finds it as it was left. An ask that must go behind another is moved to the tail — deleted from where
|
||||
// it was and published again, its body unchanged, under the same id — while the worker is paused, so no
|
||||
// holder takes it in the moment it moves. Only waiting asks move: what a holder took is never touched.
|
||||
|
||||
// The kinds of ask, as the queue says them.
|
||||
const (
|
||||
KindDelivery = "delivery"
|
||||
KindCheck = "check"
|
||||
)
|
||||
|
||||
// CheckWaitsAtMost is how long a check waits before it goes ahead of the deliveries asked after it.
|
||||
var CheckWaitsAtMost = 10 * time.Minute
|
||||
|
||||
// KindOf is an ask's kind: a check for a pull request's check — a recheck the merge tool asks included —
|
||||
// a dry run, or an ask nothing could be read from; a delivery for any other build.
|
||||
func KindOf(r BuildRequest) string {
|
||||
if r.ID == "" || r.Check != nil || r.DryRun {
|
||||
return KindCheck
|
||||
}
|
||||
return KindDelivery
|
||||
}
|
||||
|
||||
// askedOrStored is when an ask was asked, as its id says, or else when the queue took it.
|
||||
func (a QueuedAsk) askedOrStored() time.Time {
|
||||
if !a.AskedAt.IsZero() {
|
||||
return a.AskedAt
|
||||
}
|
||||
return a.Stored
|
||||
}
|
||||
|
||||
// QueueOrder is the order the waiting asks are to be handed out in, now: the deliveries and every check
|
||||
// that has waited CheckWaitsAtMost, by when each was asked, then the other checks, by when each was asked.
|
||||
func QueueOrder(waiting []QueuedAsk, now time.Time) []QueuedAsk {
|
||||
var first, then []QueuedAsk
|
||||
for _, a := range waiting {
|
||||
if KindOf(a.Request) == KindCheck && now.Sub(a.askedOrStored()) < CheckWaitsAtMost {
|
||||
then = append(then, a)
|
||||
continue
|
||||
}
|
||||
first = append(first, a)
|
||||
}
|
||||
byAsking := func(s []QueuedAsk) {
|
||||
sort.SliceStable(s, func(i, j int) bool {
|
||||
if ti, tj := s[i].askedOrStored(), s[j].askedOrStored(); !ti.Equal(tj) {
|
||||
return ti.Before(tj)
|
||||
}
|
||||
return s[i].Seq < s[j].Seq
|
||||
})
|
||||
}
|
||||
byAsking(first)
|
||||
byAsking(then)
|
||||
return append(first, then...)
|
||||
}
|
||||
|
||||
// toMove is the asks to move to the tail, in this order, for the queue to stand as order says: everything
|
||||
// after the longest start of order that the queue already holds in that order.
|
||||
func toMove(queue, order []QueuedAsk) []QueuedAsk {
|
||||
at := make(map[uint64]int, len(queue))
|
||||
for i, a := range queue {
|
||||
at[a.Seq] = i
|
||||
}
|
||||
kept, last := 0, -1
|
||||
for kept < len(order) {
|
||||
i, ok := at[order[kept].Seq]
|
||||
if !ok || i < last {
|
||||
break
|
||||
}
|
||||
last = i
|
||||
kept++
|
||||
}
|
||||
return order[kept:]
|
||||
}
|
||||
|
||||
// orderPasses keeps one pass at a time. Only the acting controller's loop runs one (KeepingInOrder), so
|
||||
// there is one; the lock holds a test's two at once to that too.
|
||||
var orderPasses sync.Mutex
|
||||
|
||||
// pausedFor bounds a pause the pass could not end itself: a controller that stops mid-pass leaves the
|
||||
// worker paused this long at most. moveMargin is the part of it no move starts in: a pass stops moving
|
||||
// with at least this left, and the next pass moves the rest, so no move runs past the pause.
|
||||
const (
|
||||
pausedFor = 20 * time.Second
|
||||
moveMargin = 5 * time.Second
|
||||
publishWaits = 3 * time.Second
|
||||
)
|
||||
|
||||
// QueuedAtHeader carries, on a moved ask, when the stream first took it: a move keeps an ask's place in
|
||||
// time, its id unread or not.
|
||||
const QueuedAtHeader = "Mesh-Queued-At"
|
||||
|
||||
// MovedFromHeader names, on a move's copy, the sequence of the message it is a copy of: with QueuedAtHeader,
|
||||
// what proves a second message of one id is a move's copy and not an ask delivered again (ADR 0293), whose
|
||||
// headers are its original's — and so never name that original's own sequence.
|
||||
const MovedFromHeader = "Mesh-Moved-From"
|
||||
|
||||
// Pass is what one pass did: the asks it moved, and each second copy it removed, in words.
|
||||
type Pass struct {
|
||||
Moved []string
|
||||
Removed []string
|
||||
}
|
||||
|
||||
// asked is the signal an ask sends the acting controller's pass loop, in this process. An ask asked in
|
||||
// another process — a verb run as its own command — is seen by the loop as the queue's last sequence moving.
|
||||
var asked = make(chan struct{}, 1)
|
||||
|
||||
// signalAsked tells the pass loop an ask joined the queue. Never blocks, never runs a pass.
|
||||
func signalAsked() {
|
||||
select {
|
||||
case asked <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// PutInOrder moves the seat's waiting asks into the order QueueOrder says, and says each it moved. A queue
|
||||
// already in order is read and left alone; the worker is paused only while asks move. Run by the acting
|
||||
// controller alone (KeepingInOrder): two passes in two processes would each resume what the other paused.
|
||||
func PutInOrder(ctx context.Context, js *broker.JetStream, seat string) (Pass, error) {
|
||||
var pass Pass
|
||||
orderPasses.Lock()
|
||||
defer orderPasses.Unlock()
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
return pass, err
|
||||
}
|
||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
|
||||
waiting, extra, delivered, err := readWaiting(ctx, api, worker, seat)
|
||||
if err != nil || (len(extra) == 0 && len(toMove(waiting, QueueOrder(waiting, time.Now()))) == 0) {
|
||||
return pass, err
|
||||
}
|
||||
// Paused, then read again: what the worker handed out up to the pause is not waiting any more.
|
||||
pauseEnds := time.Now().Add(pausedFor)
|
||||
if delivered >= 0 {
|
||||
if _, err := api.PauseConsumer(ctx, worker.Stream, worker.Name, pauseEnds); err != nil {
|
||||
return pass, fmt.Errorf("the %s worker could not be paused to put its queue in order, so it is left "+
|
||||
"first come, first served: %w", seat, err)
|
||||
}
|
||||
defer func() {
|
||||
resuming, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
||||
defer cancel()
|
||||
if _, err := api.ResumeConsumer(resuming, worker.Stream, worker.Name); err != nil {
|
||||
fmt.Printf("the %s worker could not be resumed after its queue was put in order; it resumes by "+
|
||||
"itself by %s: %v\n", seat, pauseEnds.Format(time.RFC3339), err)
|
||||
}
|
||||
}()
|
||||
if waiting, extra, _, err = readWaiting(ctx, api, worker, seat); err != nil {
|
||||
return pass, err
|
||||
}
|
||||
}
|
||||
stream, err := api.Stream(ctx, worker.Stream)
|
||||
if err != nil {
|
||||
return pass, err
|
||||
}
|
||||
// A second copy of one ask — a move that published and stopped before it deleted — goes first.
|
||||
for _, a := range extra {
|
||||
if err := stream.DeleteMsg(ctx, a.Seq); err != nil && !errors.Is(err, jetstream.ErrMsgNotFound) {
|
||||
return pass, fmt.Errorf("a second copy of %s (seq %d) could not be removed: %w", a.ID, a.Seq, err)
|
||||
}
|
||||
pass.Removed = append(pass.Removed, fmt.Sprintf("%s at seq %d, a move's second copy of the message at seq %s",
|
||||
a.ID, a.Seq, a.movedFrom))
|
||||
}
|
||||
for _, a := range toMove(waiting, QueueOrder(waiting, time.Now())) {
|
||||
if delivered >= 0 && time.Until(pauseEnds) < moveMargin {
|
||||
// The rest waits for the next pass, under a pause of its own.
|
||||
break
|
||||
}
|
||||
ok, err := moveToTheTail(ctx, js, stream, worker.Name, seat, a)
|
||||
if err != nil {
|
||||
return pass, err
|
||||
}
|
||||
if ok {
|
||||
pass.Moved = append(pass.Moved, a.ID)
|
||||
}
|
||||
}
|
||||
return pass, nil
|
||||
}
|
||||
|
||||
// readWaiting is every ask the worker has not yet handed out, in queue order, but each second copy of one
|
||||
// ask; the second copies, to remove; and the last sequence the worker handed out, -1 when there is no
|
||||
// worker, so nothing is handed out to pause.
|
||||
//
|
||||
// **A second copy is proven, never guessed from an id**: a waiting message whose MovedFromHeader names an
|
||||
// earlier message of the same request id still in the queue, and whose QueuedAtHeader is that message's
|
||||
// first-queued time — a move that published its copy and stopped before it deleted the original. The
|
||||
// original stays, waiting or handed out (a holder's then, and the copy would build it twice). Any other
|
||||
// second message of one id — an ask a dead letter delivered again beside its original (ADR 0293) — is a
|
||||
// real ask and stays.
|
||||
func readWaiting(ctx context.Context, api jetstream.JetStream, worker broker.Consumer, seat string) (
|
||||
waiting, extra []QueuedAsk, delivered int64, err error) {
|
||||
stream, err := api.Stream(ctx, worker.Stream)
|
||||
if err != nil {
|
||||
return nil, nil, 0, fmt.Errorf("the %s work queue (%s) cannot be read: %w", seat, worker.Stream, err)
|
||||
}
|
||||
delivered = -1
|
||||
if consumer, err := stream.Consumer(ctx, worker.Name); err == nil {
|
||||
info, err := consumer.Info(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, 0, fmt.Errorf("the worker of %s cannot be read: %w", seat, err)
|
||||
}
|
||||
delivered = int64(info.Delivered.Stream)
|
||||
} else if !errors.Is(err, jetstream.ErrConsumerNotFound) {
|
||||
return nil, nil, 0, fmt.Errorf("the worker of %s cannot be read: %w", seat, err)
|
||||
}
|
||||
// From the first: the stream holds only what is unsettled, so this is a handful of reads.
|
||||
// Every message by sequence, with its id and first-queued time, to prove a copy against.
|
||||
type held struct {
|
||||
id string
|
||||
stored time.Time
|
||||
}
|
||||
bySeq := map[string]held{}
|
||||
var seq uint64 = 1
|
||||
for n := 0; n < 10000; n++ {
|
||||
msg, err := stream.GetMsg(ctx, seq, jetstream.WithGetMsgSubject(BuildWorkOf(seat)))
|
||||
if errors.Is(err, jetstream.ErrMsgNotFound) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil, 0, fmt.Errorf("reading the %s work queue: %w", seat, err)
|
||||
}
|
||||
seq = msg.Sequence + 1
|
||||
a := askOf(msg)
|
||||
a.movedFrom = msg.Header.Get(MovedFromHeader)
|
||||
from, known := bySeq[a.movedFrom]
|
||||
again := a.ID != "" && a.movedFrom != "" && known && from.id == a.ID && from.stored.Equal(a.Stored)
|
||||
bySeq[strconv.FormatUint(a.Seq, 10)] = held{id: a.ID, stored: a.Stored}
|
||||
switch {
|
||||
case int64(a.Seq) <= delivered:
|
||||
case again:
|
||||
extra = append(extra, a)
|
||||
default:
|
||||
waiting = append(waiting, a)
|
||||
}
|
||||
}
|
||||
return waiting, extra, delivered, nil
|
||||
}
|
||||
|
||||
// moveToTheTail publishes the ask's body again at the queue's tail and then deletes it where it was. **The
|
||||
// copy first**: a publish that fails deletes nothing, and a pass that stops between the two leaves two
|
||||
// copies, which the next pass sees by their one id and makes one (readWaiting). A move never loses an ask.
|
||||
// Looked at once more just before: one the worker has handed out since is a holder's now and stays. False
|
||||
// when it was not moved.
|
||||
func moveToTheTail(ctx context.Context, js *broker.JetStream, stream jetstream.Stream, workerName, seat string,
|
||||
a QueuedAsk) (bool, error) {
|
||||
if consumer, err := stream.Consumer(ctx, workerName); err == nil {
|
||||
info, err := consumer.Info(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if info.Delivered.Stream >= a.Seq {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
msg, err := stream.GetMsg(ctx, a.Seq)
|
||||
if errors.Is(err, jetstream.ErrMsgNotFound) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
again := &nats.Msg{Subject: msg.Subject, Data: msg.Data, Header: nats.Header{}}
|
||||
for k, v := range msg.Header {
|
||||
// The server's own headers stay behind: a message id would be taken for a duplicate and dropped.
|
||||
if !strings.HasPrefix(k, "Nats-") {
|
||||
again.Header[k] = v
|
||||
}
|
||||
}
|
||||
// Its place in time goes with it: when the stream first took it, as askOf read it; and the message it
|
||||
// is a copy of, which proves it one.
|
||||
again.Header.Set(QueuedAtHeader, askOf(msg).Stored.UTC().Format(time.RFC3339Nano))
|
||||
again.Header.Set(MovedFromHeader, strconv.FormatUint(a.Seq, 10))
|
||||
// Inside the margin, so the publish ends while the worker is still paused.
|
||||
publishing, cancel := context.WithTimeout(ctx, publishWaits)
|
||||
ack, err := js.Context().PublishMsg(again, nats.Context(publishing))
|
||||
cancel()
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("%s (seq %d) could not be put behind, and stays where it is: %w", a.ID, a.Seq, err)
|
||||
}
|
||||
// **Handed out meanwhile** — a bus slow enough that the pause ended: the original is a holder's, and the
|
||||
// copy goes instead, so it is built once.
|
||||
if consumer, err := stream.Consumer(ctx, workerName); err == nil {
|
||||
info, err := consumer.Info(ctx)
|
||||
if err != nil || info.Delivered.Stream >= a.Seq {
|
||||
if derr := stream.DeleteMsg(ctx, ack.Sequence); derr != nil && !errors.Is(derr, jetstream.ErrMsgNotFound) {
|
||||
return false, fmt.Errorf("%s was handed out while it moved and its copy (seq %d) could not be "+
|
||||
"removed; the next pass removes it: %w", a.ID, ack.Sequence, derr)
|
||||
}
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("whether %s was handed out while it moved cannot be read, so its copy was "+
|
||||
"removed and it stays where it was: %w", a.ID, err)
|
||||
}
|
||||
if info.Delivered.Stream >= ack.Sequence {
|
||||
return false, fmt.Errorf("%s was handed out while it moved, and its copy too: it may be built twice", a.ID)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
if err := stream.DeleteMsg(ctx, a.Seq); err != nil && !errors.Is(err, jetstream.ErrMsgNotFound) {
|
||||
return false, fmt.Errorf("%s was copied behind and its original (seq %d) not removed; the next pass "+
|
||||
"removes the second copy: %w", a.ID, a.Seq, err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// KeepingInOrder runs the passes, on the acting controller alone, until ctx ends: when an ask in this
|
||||
// process says so, when the queue's last sequence moved — an ask from another process — and every
|
||||
// passEvery, for a check that comes of age with nothing asked. seat says the build seat to order; acting
|
||||
// whether this controller acts (Holding); said what a pass did.
|
||||
func KeepingInOrder(ctx context.Context, js *broker.JetStream, seat func() string, acting func() bool,
|
||||
said func(format string, args ...any)) {
|
||||
tick := time.NewTicker(orderLooks)
|
||||
defer tick.Stop()
|
||||
var lastSeq uint64
|
||||
var lastPass time.Time
|
||||
for {
|
||||
signalled := false
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-asked:
|
||||
signalled = true
|
||||
case <-tick.C:
|
||||
}
|
||||
if !acting() {
|
||||
continue
|
||||
}
|
||||
name := seat()
|
||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: name, Accepts: []string{"build"}})
|
||||
info, err := js.Context().StreamInfo(worker.Stream)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if !signalled && info.State.LastSeq == lastSeq && time.Since(lastPass) < passEvery {
|
||||
continue
|
||||
}
|
||||
lastSeq, lastPass = info.State.LastSeq, time.Now()
|
||||
passing, cancel := context.WithTimeout(ctx, pausedFor+10*time.Second)
|
||||
pass, err := PutInOrder(passing, js, name)
|
||||
cancel()
|
||||
for _, r := range pass.Removed {
|
||||
said("the %s queue: removed %s", name, r)
|
||||
}
|
||||
if len(pass.Moved) > 0 {
|
||||
said("the %s queue put in order: %s moved behind what goes first", name, strings.Join(pass.Moved, ", "))
|
||||
}
|
||||
if err != nil {
|
||||
said("the %s queue could not be put in order: %v", name, err)
|
||||
}
|
||||
if info, err := js.Context().StreamInfo(worker.Stream); err == nil {
|
||||
// Its own moves are not an ask.
|
||||
lastSeq = info.State.LastSeq
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// orderLooks is how often the loop looks whether an ask joined from another process; passEvery how often
|
||||
// it passes with nothing asked.
|
||||
var (
|
||||
orderLooks = 2 * time.Second
|
||||
passEvery = 30 * time.Second
|
||||
)
|
||||
|
||||
// DeleteAsk removes a waiting or dead ask from the queue: at its sequence, or — moved into order since it
|
||||
// was read — the copy carrying its id. Says the sequence it removed, 0 when neither was there any more.
|
||||
func DeleteAsk(ctx context.Context, js *broker.JetStream, seat string, ask QueuedAsk) (uint64, error) {
|
||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
|
||||
gone := func(err error) bool {
|
||||
return errors.Is(err, nats.ErrMsgNotFound) || errors.Is(err, jetstream.ErrMsgNotFound) ||
|
||||
(err != nil && strings.Contains(err.Error(), "no message found"))
|
||||
}
|
||||
err := js.Context().DeleteMsg(worker.Stream, ask.Seq)
|
||||
if err == nil {
|
||||
return ask.Seq, nil
|
||||
}
|
||||
if !gone(err) {
|
||||
return 0, err
|
||||
}
|
||||
if ask.ID == "" {
|
||||
return 0, nil
|
||||
}
|
||||
q, err := ReadQueue(ctx, js, seat)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for _, a := range q.Asks {
|
||||
if a.ID == ask.ID && a.Seq != ask.Seq && a.State != AskInFlight {
|
||||
if err := js.Context().DeleteMsg(worker.Stream, a.Seq); err != nil && !gone(err) {
|
||||
return 0, err
|
||||
} else if err == nil {
|
||||
return a.Seq, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0, nil
|
||||
}
|
||||
@@ -0,0 +1,441 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// The order the build queue hands asks out in (novox/hq issue 558): a build a walk waits for before any
|
||||
// pull request's check, a check that has waited ten minutes before every delivery asked after it, and
|
||||
// first come, first served within a kind.
|
||||
|
||||
func anAsk(seq uint64, id string, asked time.Time, check bool) QueuedAsk {
|
||||
r := BuildRequest{ID: id, Repository: "/r"}
|
||||
if check {
|
||||
r.Check = &CheckRequest{Owner: "novox", Repo: "r"}
|
||||
}
|
||||
return QueuedAsk{Seq: seq, ID: id, AskedAt: asked, Request: r, State: AskWaiting}
|
||||
}
|
||||
|
||||
func idsOf(asks []QueuedAsk) []string {
|
||||
out := []string{}
|
||||
for _, a := range asks {
|
||||
out = append(out, a.ID)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestADeliveryIsHandedOutBeforeTheChecksAskedBeforeIt(t *testing.T) {
|
||||
now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC)
|
||||
waiting := []QueuedAsk{
|
||||
anAsk(1, "check-a", now.Add(-3*time.Minute), true),
|
||||
anAsk(2, "check-b", now.Add(-2*time.Minute), true),
|
||||
anAsk(3, "delivery-d", now.Add(-time.Minute), false),
|
||||
}
|
||||
want := []string{"delivery-d", "check-a", "check-b"}
|
||||
if got := idsOf(QueueOrder(waiting, now)); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("handed out as %v, want %v", got, want)
|
||||
}
|
||||
// Only the checks ahead of it move behind it; the delivery stays where it is.
|
||||
if got := idsOf(toMove(waiting, QueueOrder(waiting, now))); !reflect.DeepEqual(got, []string{"check-a", "check-b"}) {
|
||||
t.Fatalf("moved %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestACheckThatWaitedTenMinutesGoesBeforeALaterDelivery(t *testing.T) {
|
||||
now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC)
|
||||
// As the queue stands after a delivery put the young checks behind it, ten minutes ago and since.
|
||||
waiting := []QueuedAsk{
|
||||
anAsk(5, "delivery-early", now.Add(-12*time.Minute), false),
|
||||
anAsk(6, "delivery-later", now.Add(-time.Minute), false),
|
||||
anAsk(7, "check-old", now.Add(-11*time.Minute), true),
|
||||
anAsk(8, "check-young", now.Add(-9*time.Minute), true),
|
||||
}
|
||||
want := []string{"delivery-early", "check-old", "delivery-later", "check-young"}
|
||||
if got := idsOf(QueueOrder(waiting, now)); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("handed out as %v, want %v", got, want)
|
||||
}
|
||||
if got := idsOf(toMove(waiting, QueueOrder(waiting, now))); !reflect.DeepEqual(got, []string{"delivery-later", "check-young"}) {
|
||||
t.Fatalf("moved %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAskWhoseKindCannotBeToldIsACheck(t *testing.T) {
|
||||
for _, r := range []BuildRequest{
|
||||
{}, // unreadable: nothing was read from it
|
||||
{ID: "build-1", Check: &CheckRequest{}}, // a pull request's check, or a recheck the merge tool asked
|
||||
{ID: "build-2", DryRun: true}, // looked at, never delivered
|
||||
} {
|
||||
if got := KindOf(r); got != KindCheck {
|
||||
t.Errorf("%+v is a %s", r, got)
|
||||
}
|
||||
}
|
||||
if got := KindOf(BuildRequest{ID: "build-3", Repository: "/r"}); got != KindDelivery {
|
||||
t.Errorf("a build is a %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Against a real server: two checks wait, a delivery is asked, and the next free holder takes the delivery.
|
||||
func TestNatsADeliveryIsTakenBeforeTheChecksQueuedAheadOfIt(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
asks := BuildsOn(js, TheBuildMachine)
|
||||
now := time.Now()
|
||||
for i, r := range []BuildRequest{
|
||||
{ID: NewBuildID(now.Add(-3 * time.Minute)), Repository: "/a", Check: &CheckRequest{Owner: "novox", Repo: "a"}},
|
||||
{ID: NewBuildID(now.Add(-2 * time.Minute)), Repository: "/b", Check: &CheckRequest{Owner: "novox", Repo: "b"}},
|
||||
{ID: NewBuildID(now), Repository: "/d"},
|
||||
} {
|
||||
if i == 2 {
|
||||
// And one nobody can read, whose kind cannot be told: it waits with the checks.
|
||||
if _, err := js.Context().Publish(BuildWork(), []byte("not an ask")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := asks.Ask(ctx, r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// The acting controller's loop puts it in order, told by the asks.
|
||||
go KeepingInOrder(ctx, js, func() string { return TheBuildMachine }, func() bool { return true },
|
||||
func(string, ...any) {})
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
took := make(chan string, 4)
|
||||
machine := MachineOverNATS(js, "anchor")
|
||||
defer machine.Close()
|
||||
go func() {
|
||||
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
|
||||
took <- work.Request().Repository
|
||||
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
|
||||
_ = work.Done()
|
||||
})
|
||||
}()
|
||||
var order []string
|
||||
for len(order) < 3 {
|
||||
select {
|
||||
case r := <-took:
|
||||
order = append(order, r)
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatalf("taken so far: %v", order)
|
||||
}
|
||||
}
|
||||
if want := []string{"/d", "/a", "/b"}; !reflect.DeepEqual(order, want) {
|
||||
t.Fatalf("taken as %v, want %v", order, want)
|
||||
}
|
||||
}
|
||||
|
||||
// --- the moves, against a real server ------------------------------------------------------------
|
||||
|
||||
func publishAsk(t *testing.T, js *broker.JetStream, r BuildRequest) {
|
||||
t.Helper()
|
||||
body, _ := json.Marshal(r)
|
||||
if _, err := js.Context().Publish(BuildWork(), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func aCheck(asked time.Time, repository string) BuildRequest {
|
||||
return BuildRequest{ID: NewBuildID(asked), Repository: repository, Check: &CheckRequest{Owner: "novox", Repo: "r"}}
|
||||
}
|
||||
|
||||
func aDelivery(asked time.Time, repository string) BuildRequest {
|
||||
return BuildRequest{ID: NewBuildID(asked), Repository: repository}
|
||||
}
|
||||
|
||||
// queued is the stream's asks, in queue order.
|
||||
func queued(t *testing.T, js *broker.JetStream) []QueuedAsk {
|
||||
t.Helper()
|
||||
q, err := ReadQueue(context.Background(), js, TheBuildMachine)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return q.Asks
|
||||
}
|
||||
|
||||
// takeAll runs a holder that records how often it was handed each repository.
|
||||
func takeAll(t *testing.T, ctx context.Context, js *broker.JetStream) (chan string, func() map[string]int) {
|
||||
t.Helper()
|
||||
took := make(chan string, 100)
|
||||
var mu sync.Mutex
|
||||
counts := map[string]int{}
|
||||
machine := MachineOverNATS(js, "anchor")
|
||||
t.Cleanup(machine.Close)
|
||||
go func() {
|
||||
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
|
||||
mu.Lock()
|
||||
counts[work.Request().Repository]++
|
||||
mu.Unlock()
|
||||
took <- work.Request().Repository
|
||||
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
|
||||
_ = work.Done()
|
||||
})
|
||||
}()
|
||||
return took, func() map[string]int {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
out := map[string]int{}
|
||||
for k, v := range counts {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
}
|
||||
|
||||
// A move that published its copy and stopped before deleting the original leaves two copies of one ask;
|
||||
// the next pass makes them one, and a copy of one a holder already took is removed, never built twice.
|
||||
func TestNatsAMoveCutShortLeavesOneCopyAfterTheNextPass(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
ctx := t.Context()
|
||||
now := time.Now()
|
||||
check := aCheck(now.Add(-2*time.Minute), "/c")
|
||||
publishAsk(t, js, check)
|
||||
publishAsk(t, js, aDelivery(now.Add(-time.Minute), "/d"))
|
||||
moveCopyOf(t, js, check.ID) // the copy, published; the original never deleted
|
||||
if pass, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(pass.Removed) != 1 {
|
||||
t.Fatalf("removed %v: %v", pass.Removed, err)
|
||||
}
|
||||
var got []string
|
||||
for _, a := range queued(t, js) {
|
||||
got = append(got, a.Repository)
|
||||
}
|
||||
if want := []string{"/d", "/c"}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("after the pass the queue holds %v, want %v", got, want)
|
||||
}
|
||||
|
||||
// And one a holder took before the copy was published: the copy goes, the holder's stays.
|
||||
taken := aCheck(now, "/taken")
|
||||
publishAsk(t, js, taken)
|
||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: TheBuildMachine, Accepts: []string{"build"}})
|
||||
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
// Take everything waiting up to and including /taken, and keep /taken unsettled.
|
||||
for {
|
||||
msgs, err := sub.Fetch(1, nats.MaxWait(2*time.Second))
|
||||
if err != nil || len(msgs) == 0 {
|
||||
t.Fatalf("nothing to take: %v", err)
|
||||
}
|
||||
var r BuildRequest
|
||||
_ = json.Unmarshal(msgs[0].Data, &r)
|
||||
if r.Repository == "/taken" {
|
||||
break
|
||||
}
|
||||
_ = msgs[0].Ack()
|
||||
}
|
||||
moveCopyOf(t, js, taken.ID)
|
||||
if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n := 0
|
||||
for _, a := range queued(t, js) {
|
||||
if a.ID == taken.ID {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("%d copies of the ask a holder took are in the queue, want 1", n)
|
||||
}
|
||||
}
|
||||
|
||||
// moveCopyOf publishes what a move publishes for the ask with this id, as a pass stopped before its delete
|
||||
// would leave it.
|
||||
func moveCopyOf(t *testing.T, js *broker.JetStream, id string) {
|
||||
t.Helper()
|
||||
for _, a := range queued(t, js) {
|
||||
if a.ID != id {
|
||||
continue
|
||||
}
|
||||
body, _ := json.Marshal(a.Request)
|
||||
msg := &nats.Msg{Subject: BuildWork(), Data: body, Header: nats.Header{}}
|
||||
msg.Header.Set(QueuedAtHeader, a.Stored.UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set(MovedFromHeader, fmt.Sprint(a.Seq))
|
||||
if _, err := js.Context().PublishMsg(msg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
t.Fatalf("%s is not queued", id)
|
||||
}
|
||||
|
||||
// A dead letter delivered again whose original stayed beside it (ADR 0293: its removal failed) is a real
|
||||
// ask with the same id, and is never removed as a second copy — even when the original was itself a move's
|
||||
// copy, whose headers the delivery carries.
|
||||
func TestNatsAnAskDeliveredAgainBesideItsOriginalStays(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
ctx := t.Context()
|
||||
ask := aCheck(time.Now(), "/again")
|
||||
publishAsk(t, js, aCheck(time.Now().Add(-time.Minute), "/before"))
|
||||
publishAsk(t, js, ask)
|
||||
publishAsk(t, js, aDelivery(time.Now(), "/d"))
|
||||
if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil { // /again becomes a move's copy
|
||||
t.Fatal(err)
|
||||
}
|
||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: TheBuildMachine, Accepts: []string{"build"}})
|
||||
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
var original *nats.Msg
|
||||
for original == nil {
|
||||
msgs, err := sub.Fetch(1, nats.MaxWait(2*time.Second))
|
||||
if err != nil || len(msgs) == 0 {
|
||||
t.Fatalf("nothing to take: %v", err)
|
||||
}
|
||||
var r BuildRequest
|
||||
_ = json.Unmarshal(msgs[0].Data, &r)
|
||||
if r.ID == ask.ID {
|
||||
original = msgs[0] // given up on, never settled: dead, and kept
|
||||
continue
|
||||
}
|
||||
_ = msgs[0].Ack()
|
||||
}
|
||||
// Delivered again as DeliverAgain does: its headers, and its own.
|
||||
again := &nats.Msg{Subject: BuildWork(), Data: original.Data, Header: nats.Header{}}
|
||||
for k, v := range original.Header {
|
||||
again.Header[k] = v
|
||||
}
|
||||
again.Header.Set(AgainHeader, "7")
|
||||
if _, err := js.Context().PublishMsg(again); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pass, err := PutInOrder(ctx, js, TheBuildMachine)
|
||||
if err != nil || len(pass.Removed) != 0 {
|
||||
t.Fatalf("removed %v: %v", pass.Removed, err)
|
||||
}
|
||||
n := 0
|
||||
for _, a := range queued(t, js) {
|
||||
if a.ID == ask.ID {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 2 {
|
||||
t.Fatalf("%d messages of the ask delivered again are queued, want its original and the delivery", n)
|
||||
}
|
||||
}
|
||||
|
||||
// Two passes at once: nothing is built twice, and nothing is lost.
|
||||
func TestNatsTwoPassesAtOnceBuildNothingTwice(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
defer stop()
|
||||
took, counts := takeAll(t, ctx, js)
|
||||
now := time.Now()
|
||||
const asks = 24
|
||||
var passes sync.WaitGroup
|
||||
for p := 0; p < 2; p++ {
|
||||
passes.Add(1)
|
||||
go func() {
|
||||
defer passes.Done()
|
||||
for i := 0; i < 15; i++ {
|
||||
_, _ = PutInOrder(ctx, js, TheBuildMachine)
|
||||
}
|
||||
}()
|
||||
}
|
||||
for i := 0; i < asks; i++ {
|
||||
repository := fmt.Sprintf("/%02d", i)
|
||||
if i%3 == 2 {
|
||||
publishAsk(t, js, aDelivery(now.Add(time.Duration(i)*time.Millisecond), repository))
|
||||
} else {
|
||||
publishAsk(t, js, aCheck(now.Add(time.Duration(i)*time.Millisecond), repository))
|
||||
}
|
||||
}
|
||||
passes.Wait()
|
||||
for got := 0; got < asks; got++ {
|
||||
select {
|
||||
case <-took:
|
||||
case <-time.After(15 * time.Second):
|
||||
t.Fatalf("taken %d of %d: %v", got, asks, counts())
|
||||
}
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
c := counts()
|
||||
for i := 0; i < asks; i++ {
|
||||
if n := c[fmt.Sprintf("/%02d", i)]; n != 1 {
|
||||
t.Errorf("/%02d was built %d times", i, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A check that has waited ten minutes goes before a delivery asked after it, even one queued ahead of it.
|
||||
func TestNatsACheckThatWaitedTenMinutesIsTakenBeforeALaterDelivery(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
defer stop()
|
||||
now := time.Now()
|
||||
publishAsk(t, js, aDelivery(now.Add(-time.Minute), "/delivery"))
|
||||
publishAsk(t, js, aCheck(now.Add(-11*time.Minute), "/old-check"))
|
||||
if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
took, _ := takeAll(t, ctx, js)
|
||||
var order []string
|
||||
for len(order) < 2 {
|
||||
select {
|
||||
case r := <-took:
|
||||
order = append(order, r)
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatalf("taken so far: %v", order)
|
||||
}
|
||||
}
|
||||
if want := []string{"/old-check", "/delivery"}; !reflect.DeepEqual(order, want) {
|
||||
t.Fatalf("taken as %v, want %v", order, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A cancel that read an ask before it moved deletes the moved copy, found by its id, and says when
|
||||
// there was nothing left to delete.
|
||||
func TestNatsACancelRacingAMoveDeletesTheMovedCopy(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
ctx := t.Context()
|
||||
now := time.Now()
|
||||
publishAsk(t, js, aCheck(now.Add(-time.Minute), "/c"))
|
||||
read := queued(t, js)[0]
|
||||
publishAsk(t, js, aDelivery(now, "/d"))
|
||||
if pass, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(pass.Moved) != 1 {
|
||||
t.Fatalf("moved %v: %v", pass.Moved, err)
|
||||
}
|
||||
removed, err := DeleteAsk(ctx, js, TheBuildMachine, read)
|
||||
if err != nil || removed == 0 || removed == read.Seq {
|
||||
t.Fatalf("removed seq %d (read at %d): %v", removed, read.Seq, err)
|
||||
}
|
||||
for _, a := range queued(t, js) {
|
||||
if a.ID == read.ID {
|
||||
t.Fatalf("the cancelled ask is still queued at seq %d", a.Seq)
|
||||
}
|
||||
}
|
||||
if removed, err := DeleteAsk(ctx, js, TheBuildMachine, read); err != nil || removed != 0 {
|
||||
t.Fatalf("an ask gone already is said removed at %d: %v", removed, err)
|
||||
}
|
||||
}
|
||||
|
||||
// An ask nothing can be read from keeps its place in time when it moves.
|
||||
func TestNatsAnUnreadableAskKeepsItsAgeWhenItMoves(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
ctx := t.Context()
|
||||
if _, err := js.Context().Publish(BuildWork(), []byte("not an ask")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first := queued(t, js)[0].Stored
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
publishAsk(t, js, aDelivery(time.Now(), "/d"))
|
||||
if pass, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(pass.Moved) != 1 {
|
||||
t.Fatalf("moved %v: %v", pass.Moved, err)
|
||||
}
|
||||
asks := queued(t, js)
|
||||
if len(asks) != 2 || asks[1].ID != "" || !asks[1].Stored.Equal(first) {
|
||||
t.Fatalf("after the move the unreadable ask was taken at %v, first at %v: %+v", asks[1].Stored, first, asks)
|
||||
}
|
||||
}
|
||||
Vendored
-1
@@ -35,4 +35,3 @@
|
||||
./cmd/mesh-controller TestARollbackWhoseSendWasRefusedIsCompletedByTheNextSend
|
||||
./cmd/mesh-controller TestAGateJudgesTheOtherMovesBesideAKeptCore
|
||||
./cmd/mesh-controller TestReplay401
|
||||
./cmd/mesh-controller TestReplay201ThePlansSend
|
||||
|
||||
Reference in New Issue
Block a user