Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9e233e1d87 | ||
|
|
0d11b4fc93 | ||
|
|
d34b4f32e1 | ||
|
|
d941d798e9 | ||
|
|
b7f70bc4c0 | ||
|
|
8431ac9656 | ||
|
|
bc2a448512 |
@@ -1,66 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
)
|
||||
|
||||
// A container waiting because it said busy is news when it starts waiting and when it is replaced, never on
|
||||
// its output alone (novox/hq ADR 0335): the walk waiting on this machine starts its gate at the report that
|
||||
// says it replaced, and a reconcile is otherwise silent.
|
||||
func TestABusyWaitIsSaidWhenItStartsAndWhenItEnds(t *testing.T) {
|
||||
quiet := link.Report{Outward: []string{"eth0"}}
|
||||
waiting := link.Report{Outward: []string{"eth0"}, Busy: []link.Busy{{ID: "build-agent.server",
|
||||
Container: "mesh-build-agent", State: "waiting", Said: "building a"}}}
|
||||
still := link.Report{Outward: []string{"eth0"}, Busy: []link.Busy{{ID: "build-agent.server",
|
||||
Container: "mesh-build-agent", State: "waiting", Said: "building b, a minute on"}}}
|
||||
replaced := link.Report{Outward: []string{"eth0"}, Busy: []link.Busy{{ID: "build-agent.server",
|
||||
Container: "mesh-build-agent", State: "replaced-when-idle"}}}
|
||||
if adoptionFingerprint(waiting) == adoptionFingerprint(quiet) {
|
||||
t.Error("a container starting to wait is not news")
|
||||
}
|
||||
if adoptionFingerprint(still) != adoptionFingerprint(waiting) {
|
||||
t.Error("a wait whose output changed reads as news on every ask")
|
||||
}
|
||||
if adoptionFingerprint(replaced) == adoptionFingerprint(waiting) {
|
||||
t.Error("a container replaced after it waited is not news")
|
||||
}
|
||||
if !worthSaying(link.Report{Busy: replaced.Busy}) {
|
||||
t.Error("a replacement after a wait is not worth saying")
|
||||
}
|
||||
}
|
||||
|
||||
// While a container waits, a pass is asked for every so often, not at the reconcile's five minutes; with
|
||||
// nothing waiting, none (novox/hq ADR 0335 §2).
|
||||
func TestAWaitIsAskedAgainSoonerThanTheReconcile(t *testing.T) {
|
||||
if BusyAskEvery >= ReconcileEvery || BusyAskEvery > time.Minute {
|
||||
t.Fatalf("a waiting container is asked every %s", BusyAskEvery)
|
||||
}
|
||||
var reconciles atomic.Int32
|
||||
queue := &link.Queue{Reconcile: func(context.Context) (link.Report, bool) {
|
||||
reconciles.Add(1)
|
||||
return link.Report{}, false
|
||||
}}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go queue.Run(ctx)
|
||||
busyWaiting.Store(false)
|
||||
t.Cleanup(func() { busyWaiting.Store(false) })
|
||||
go busyPasses(ctx, queue, 10*time.Millisecond)
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if n := reconciles.Load(); n != 0 {
|
||||
t.Fatalf("%d passes were asked with nothing waiting", n)
|
||||
}
|
||||
busyWaiting.Store(true)
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for reconciles.Load() == 0 && time.Now().Before(deadline) {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
if reconciles.Load() == 0 {
|
||||
t.Fatal("no pass was asked while a container waits")
|
||||
}
|
||||
}
|
||||
+2
-44
@@ -24,7 +24,6 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
@@ -1260,7 +1259,6 @@ func runLink(ctx context.Context, opts options) error {
|
||||
queue.ReconcileDue()
|
||||
}
|
||||
go holdTheMachine(ctx, queue)
|
||||
go busyPasses(ctx, queue, BusyAskEvery)
|
||||
// And whether what it runs stays up, looked at on its own clock and said when it changes (novox/hq
|
||||
// ADR 0240) — between applies, which is when a container crash-loops.
|
||||
if judging != nil {
|
||||
@@ -1399,12 +1397,6 @@ func adoptionFingerprint(r link.Report) string {
|
||||
// it could only speak when a declaration arrived, and a declaration cannot be composed until it
|
||||
// has spoken: a machine waiting for a push that is waiting for the machine.
|
||||
parts := []string{"firewall=" + r.Firewall, "outward=" + strings.Join(r.Outward, ",")}
|
||||
// And a container waiting because it said busy, or replaced after it waited (novox/hq ADR 0335): the walk
|
||||
// waiting on this machine starts its gate at the report that says it replaced. Not its output, which
|
||||
// may change on every ask.
|
||||
for _, b := range r.Busy {
|
||||
parts = append(parts, "busy "+b.ID+"="+b.State)
|
||||
}
|
||||
for _, h := range r.Held {
|
||||
parts = append(parts, "held "+h.ID+"="+h.Changed)
|
||||
}
|
||||
@@ -1744,7 +1736,7 @@ func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health
|
||||
// LibraryContract: this engine reads a resource's declared `health` and judges it (ADR 0240 Phase B), judges a
|
||||
// user's declared `root` (novox/hq ADR 0266), and reads a declaration's identities, groups and libraries
|
||||
// (novox/hq ADR 0328), which is what tells the controller it may be sent any of those fields.
|
||||
h := &link.Health{Contract: link.BusyContract, At: st.At.UTC(), Resources: []link.ResourceHealth{}}
|
||||
h := &link.Health{Contract: link.OwnGroupContract, At: st.At.UTC(), Resources: []link.ResourceHealth{}}
|
||||
for _, r := range st.Resources {
|
||||
h.Resources = append(h.Resources, link.ResourceHealth{Module: r.Module, Resource: r.ID, Kind: r.Kind,
|
||||
Target: r.Target, State: r.State, Reason: r.Reason, Since: r.Since.UTC(), Streak: r.Streak,
|
||||
@@ -1876,32 +1868,6 @@ func holdTheMachine(ctx context.Context, queue *link.Queue) {
|
||||
}
|
||||
}
|
||||
|
||||
// busyWaiting is whether the last apply left a container running because it said busy (novox/hq ADR 0335).
|
||||
var busyWaiting atomic.Bool
|
||||
|
||||
// BusyAskEvery is how often a container waiting because it said busy is asked again (novox/hq ADR 0335 §2):
|
||||
// each ask is a pass of its own, so a build that ends is followed by its replacement within this, not
|
||||
// within the reconcile's five minutes — and the bound of 15 minutes is met within this of its end.
|
||||
const BusyAskEvery = 20 * time.Second
|
||||
|
||||
// busyPasses asks for a pass every so often while a container waits because it said busy. **It asks; it
|
||||
// does not apply**, as holdTheMachine: the queue's worker does, when its turn comes, so a pass never holds
|
||||
// the machine for the wait, and a delivery waiting is that pass's apply.
|
||||
func busyPasses(ctx context.Context, queue *link.Queue, every time.Duration) {
|
||||
ticker := time.NewTicker(every)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if busyWaiting.Load() {
|
||||
queue.ReconcileDue()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// reconcileKept is the reconcile's act, as the queue's worker runs it: hold the machine to what it
|
||||
// was last told, read once it is this act's turn, and say what did not go.
|
||||
//
|
||||
@@ -1953,7 +1919,7 @@ func worthSaying(report link.Report) bool {
|
||||
}
|
||||
return len(report.Held) > 0 || report.Firewall != "" || len(report.Outward) > 0 ||
|
||||
len(report.Filters) > 0 || report.FoundFirewall != nil || len(report.Windows) > 0 ||
|
||||
len(report.Rollbacks) > 0 || len(report.Busy) > 0
|
||||
len(report.Rollbacks) > 0
|
||||
}
|
||||
|
||||
// announceOr is what the apply writes its detail with, given what the caller has to say things with.
|
||||
@@ -2202,14 +2168,6 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
|
||||
for _, w := range outcome.Windows {
|
||||
report.Windows = append(report.Windows, link.Window{Step: w.Step, Holds: w.Holds, Since: w.Opened, Until: w.Until})
|
||||
}
|
||||
// And a container left running because it said busy, or replaced after it waited (novox/hq ADR 0335):
|
||||
// the walk waiting on this machine reads it. While one waits, this machine asks it again sooner than its
|
||||
// reconcile (busyPasses).
|
||||
for _, b := range outcome.Busy {
|
||||
report.Busy = append(report.Busy, link.Busy{ID: b.ID, Container: b.Container, State: b.State,
|
||||
Since: b.Since, Until: b.Until, Said: b.Said})
|
||||
}
|
||||
busyWaiting.Store(outcome.Waiting())
|
||||
// And what runs here that nobody asked for (novox/hq ADR 0163).
|
||||
if strays, err := apply.Strays(ctx, apply.ExecRunner, updated); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "mesh-host: applied, and could not list what else runs here: %v\n", err)
|
||||
|
||||
@@ -87,10 +87,10 @@ func TestTheStatementSaysAnAccountJudgedForRoot(t *testing.T) {
|
||||
Groups: []string{"openrazer"}}, State: accounts.Healthy, Since: at},
|
||||
}}
|
||||
h := withAccounts(healthAsReported(liveness.Statement{At: at}, nil), as)
|
||||
// At RootContract or later: an engine that also reads a container's busy says BusyContract (novox/hq ADR
|
||||
// 0335), one above ADR 0334's OwnGroupContract.
|
||||
if h.Contract < link.RootContract || link.RootContract != 3 || h.Contract != link.BusyContract ||
|
||||
link.BusyContract != link.OwnGroupContract+1 {
|
||||
// At RootContract or later: an engine that also reads an identity's runner says OwnGroupContract (novox/hq ADR
|
||||
// 0334), one above ADR 0328's LibraryContract.
|
||||
if h.Contract < link.RootContract || link.RootContract != 3 || h.Contract != link.OwnGroupContract ||
|
||||
link.OwnGroupContract != link.LibraryContract+1 {
|
||||
t.Fatalf("contract %d", h.Contract)
|
||||
}
|
||||
got := map[string]string{}
|
||||
|
||||
+8
-1
@@ -12,7 +12,8 @@ mesh runs on:
|
||||
2 a database per context `inventory`, `identity` and `licences`
|
||||
3 those contexts' schemas mesh-controller migrate
|
||||
4 the bus's certificate mesh-controller broker certificate
|
||||
5 the bus runs NATS, with the controller's account as its first user
|
||||
5 the bus runs NATS as mesh-broker-nats, where the nats module keeps it
|
||||
(ADR 0333), with the controller's account as its first user
|
||||
6 the control plane runs mesh-controller serve, given the bus (MESH_BUS_NATS)
|
||||
```
|
||||
|
||||
@@ -26,6 +27,12 @@ rootsecrets.go). A template whose bus connection it does not recognise is refuse
|
||||
naming the placeholder it looked for. The bundle that raised the previous broker was deleted with
|
||||
ADR 0331; there is one bundle, and every lab bed names it.
|
||||
|
||||
**The bus is raised where the `nats` module keeps it** (novox/hq ADR 0333): the container
|
||||
`mesh-broker-nats`, its configuration and user list in `/var/lib/nats-module/conf`, its streams in
|
||||
`/var/lib/mesh-broker-nats`, its certificate in `/var/lib/mesh-broker-tls`, published on the node's bus
|
||||
port. At the installer's broker step the module adopts it: its own image replaces the upstream one, once,
|
||||
on the same streams, users and certificate. `internal/bootstrap` holds the bundle to the module's manifest.
|
||||
|
||||
Build an installer carrying a builder, and point it at this:
|
||||
|
||||
```
|
||||
|
||||
@@ -141,6 +141,30 @@
|
||||
"migrate"],
|
||||
"verify": ["sh", "-c", "docker exec mesh-store psql -U postgres -d inventory -tAc \"select to_regclass('public.node')\" | grep -qx node && docker exec mesh-store psql -U postgres -d identity -tAc \"select to_regclass('public.signing_key')\" | grep -qx signing_key && docker exec mesh-store psql -U postgres -d licences -tAc \"select to_regclass('public.licence')\" | grep -qx licence"]
|
||||
},
|
||||
// **The bus is raised where the `nats` module keeps it** (novox/hq ADR 0333 §1): the container
|
||||
// `mesh-broker-nats`, its configuration and user list in /var/lib/nats-module/conf, its streams in
|
||||
// /var/lib/mesh-broker-nats and its certificate in /var/lib/mesh-broker-tls — the names and paths the
|
||||
// catalogue's module declares. So adopting it as that module at the broker step keeps it: the
|
||||
// module's built image takes the place of the upstream one below, once, on the same streams, users
|
||||
// and certificate, and nothing else moves. mesh-host's internal/bootstrap holds the two to each other
|
||||
// against the module's own manifest (busLayout), at the broker step and in its tests.
|
||||
//
|
||||
// Directories on the machine, not volumes the runtime names: the module mounts these paths, and the
|
||||
// control plane, once it runs as the mesh's process, reads the certificate at its path on the machine.
|
||||
// The certificate's directory is readable by everyone because the certificate is public and the control
|
||||
// plane runs as nobody; its key is written 0600 by the program that makes it.
|
||||
{
|
||||
"id": "bus-tls-dir",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/mesh-broker-tls",
|
||||
"mode": "0755"
|
||||
},
|
||||
{
|
||||
"id": "bus-data-dir",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/mesh-broker-nats",
|
||||
"mode": "0700"
|
||||
},
|
||||
{
|
||||
"id": "bus-certificate",
|
||||
"type": "action",
|
||||
@@ -151,42 +175,53 @@
|
||||
// it, and asking nothing of the image it writes into. Self-signed on purpose: a host pins
|
||||
// this server's exact certificate (novox/hq ADR 0004), and at this moment there is no mesh
|
||||
// to ask an authority of.
|
||||
// `--user 0:0` because the volume is root's and this image runs as nobody, which is right
|
||||
// for the long-running control plane and wrong for a one-shot writing into a fresh volume.
|
||||
"command": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls",
|
||||
// `--user 0:0` because the directory is root's and this image runs as nobody, which is right
|
||||
// for the long-running control plane and wrong for a one-shot writing into a fresh directory.
|
||||
"command": ["docker", "run", "--rm", "--user", "0:0", "-v", "/var/lib/mesh-broker-tls:/tls",
|
||||
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
||||
"broker", "certificate", "--into", "/tls"],
|
||||
"verify": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls",
|
||||
"verify": ["docker", "run", "--rm", "--user", "0:0", "-v", "/var/lib/mesh-broker-tls:/tls",
|
||||
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
||||
"broker", "certificate", "--check", "--into", "/tls"]
|
||||
},
|
||||
// **Seeds, written only where nothing is** (create-once). The bus's configuration and user list are
|
||||
// the bundle's until the bus is adopted, and the module's from then on (ADR 0333 §4): an installer
|
||||
// run again after the adoption applies this bundle again, and an ordinary file would put the
|
||||
// bundle's one user back over every user the mesh composed, and leave the node unable to reach its
|
||||
// bus. Until the adoption the installer places the composed list over the seed (enrol.go).
|
||||
{
|
||||
"id": "bus-conf-dir",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/mesh-bus-conf",
|
||||
"path": "/var/lib/nats-module/conf",
|
||||
"mode": "0700"
|
||||
},
|
||||
{
|
||||
"id": "bus-conf",
|
||||
"type": "file",
|
||||
"path": "/var/lib/mesh-bus-conf/nats.conf",
|
||||
"path": "/var/lib/nats-module/conf/nats.conf",
|
||||
"mode": "0644",
|
||||
"content": "port: 4222\nhttp: 127.0.0.1:8222\n\ntls {\n cert_file: \"/tls/tls.crt\"\n key_file: \"/tls/tls.key\"\n}\n\njetstream {\n store_dir: \"/data\"\n}\n\ninclude accounts.conf\n"
|
||||
"create-once": true,
|
||||
"content": "port: 4222\nhttp: 0.0.0.0:8222\n\ntls {\n cert_file: \"/tls/tls.crt\"\n key_file: \"/tls/tls.key\"\n}\n\njetstream {\n store_dir: \"/data\"\n}\n\ninclude accounts.conf\n"
|
||||
},
|
||||
{
|
||||
"id": "bus-accounts",
|
||||
"type": "file",
|
||||
"path": "/var/lib/mesh-bus-conf/accounts.conf",
|
||||
"path": "/var/lib/nats-module/conf/accounts.conf",
|
||||
"mode": "0600",
|
||||
"create-once": true,
|
||||
"content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n MESH {\n jetstream: enabled\n users = [\n { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.ACK.DEAD_LETTER_NOTICES.controller.>\", \"$JS.API.>\", \"$KV.mesh-controller_asked.>\", \"$KV.mesh-controller_calls.>\", \"$KV.mesh-controller_hand-acts.>\", \"$KV.mesh-controller_conditions.>\", \"$KV.mesh-controller_condition-history.>\", \"$KV.mesh-controller_lease.>\", \"$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>\", \"$KV.SEAT_NODE_BUILD_AGENT_cancelled.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.node-build-agent.accept.>\", \"mesh.seat.mesh-build-machine.tool.>\", \"mesh.seat.node-build-agent.tool.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\", \"mesh.seat.mesh-controller.event.rolled-back\", \"mesh.seat.mesh-controller.event.checked\", \"mesh.seat.mesh-controller.event.plan-moved\", \"mesh.seat.mesh-controller.event.condition-raised\", \"mesh.seat.mesh-controller.event.condition-changed\", \"mesh.seat.mesh-controller.event.condition-cleared\", \"mesh.seat.mesh-controller.event.doctor-heartbeat\", \"mesh.seat.mesh-controller.event.secret-replaced\", \"mesh.seat.mesh-controller.event.healer-acted\", \"$SRV.INFO\", \"mesh.seat.node-intrusion-prevention.tool.banned.*\", \"mesh.seat.node-backup.tool.backed-up.*\", \"mesh.seat.node-backup.tool.now.*\", \"mesh.seat.mesh-delivery.tool.stalled\", \"mesh.seat.mesh-delivery.tool.close\", \"mesh.seat.node-launcher.tool.secret.*\", \"mesh.seat.mesh-delivery.tool.release\", \"mesh.seat.mesh-delivery.tool.stop\", \"mesh.seat.mesh-delivery.tool.times\", \"mesh.seat.mesh-delivery.tool.groups\", \"mesh.seat.node-service-manager.tool.restart.*\", \"mesh.seat.mesh-controller.tool.plans\", \"mesh.events.dead.>\", \"mesh.again.>\"] }\n subscribe: { allow: [\"$JS.API.>\", \"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>\", \"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.*.event.provisioner.failing\", \"mesh.mod.*.event.provisioner.recovered\", \"mesh.mod.*.event.provisioner.retirement\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.gitea.event.pull.updated\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.node-build-agent.event.built\", \"mesh.seat.operator-channel.event.decided.mesh-controller\", \"mesh.seat.mesh-controller.tool.>\", \"$SRV.PING\", \"$SRV.INFO\", \"$SRV.PING.mesh-controller\", \"$SRV.PING.mesh-controller.>\", \"$SRV.INFO.mesh-controller\", \"$SRV.INFO.mesh-controller.>\", \"$SRV.STATS\", \"$SRV.STATS.mesh-controller\", \"$SRV.STATS.mesh-controller.>\"] }\n allow_responses: { max: 1, ttl: \"1m\" }\n } }\n ]\n }\n}\n"
|
||||
},
|
||||
// The upstream server, until the broker step adopts it: the `nats` module's own image, built by the
|
||||
// mesh, takes its place there once, on these same directories (ADR 0333 §3). The published port is the
|
||||
// node's bus port, 5671 unless given, which every enrolment token names; the installer gives the
|
||||
// module the setting that maps its declared 4222 to it (ADR 0333 §2).
|
||||
{
|
||||
"id": "broker",
|
||||
"type": "container",
|
||||
"name": "mesh-broker",
|
||||
"name": "mesh-broker-nats",
|
||||
"image": "192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927",
|
||||
"ports": ["5671:4222", "127.0.0.1:8222:8222"],
|
||||
"volumes": ["mesh-broker-data:/data", "mesh-broker-tls:/tls:ro", "/var/lib/mesh-bus-conf:/etc/nats:ro"],
|
||||
"volumes": ["/var/lib/mesh-broker-nats:/data", "/var/lib/nats-module/conf:/etc/nats:ro", "/var/lib/mesh-broker-tls:/tls:ro"],
|
||||
"args": ["-c", "/etc/nats/nats.conf", "-js"]
|
||||
},
|
||||
{
|
||||
@@ -202,7 +237,7 @@
|
||||
"image": "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
||||
"network": "host",
|
||||
"args": ["serve"],
|
||||
"volumes": ["mesh-broker-tls:/broker-tls:ro"],
|
||||
"volumes": ["/var/lib/mesh-broker-tls:/broker-tls:ro"],
|
||||
"env": {
|
||||
"MESH_STORE_INVENTORY": "postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable",
|
||||
"MESH_STORE_IDENTITY": "postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable",
|
||||
|
||||
+2
-40
@@ -92,11 +92,6 @@ type Outcome struct {
|
||||
// reads is, for a container, the digest of each file it was created reading, by path — so
|
||||
// the next apply can say which one changed (novox/hq 04-ISSUES/103).
|
||||
reads map[string]string
|
||||
// busySince is, for a container left running because it said busy, when the first apply that would
|
||||
// have replaced it ran (novox/hq ADR 0335): kept in its record, so the bound holds across passes and
|
||||
// across a restart of the node-engine. busy is what the report says of it.
|
||||
busySince *time.Time
|
||||
busy *BusyWait
|
||||
}
|
||||
|
||||
// Report is what an apply did, in the order it did it.
|
||||
@@ -113,20 +108,6 @@ type Report struct {
|
||||
// 224): a machine whose server is stopped at 03:31 because its collector is running is
|
||||
// working, not broken, and the report is where that difference is said.
|
||||
Windows []Window `json:"windows,omitempty"`
|
||||
// Busy is every container this apply left running because it said busy, or replaced after it had
|
||||
// waited (novox/hq ADR 0335): a machine whose walk waits for it says why, and how it ended.
|
||||
Busy []BusyWait `json:"busy,omitempty"`
|
||||
}
|
||||
|
||||
// Waiting says whether a container of this apply waits to be replaced because it is busy: what makes the
|
||||
// node-engine ask again sooner than its reconcile (novox/hq ADR 0335 §2).
|
||||
func (r Report) Waiting() bool {
|
||||
for _, b := range r.Busy {
|
||||
if b.State == BusyWaiting {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Changed reports whether anything about the machine actually moved. An apply that changed
|
||||
@@ -137,10 +118,7 @@ func (r Report) Changed() bool {
|
||||
// whose account's manager is not running, which nothing moved either (novox/hq ADR 0177);
|
||||
// held still is a container a maintenance window stopped, which the apply did not touch
|
||||
// (novox/hq issue 224).
|
||||
// waits busy is a container left running because it said busy (novox/hq ADR 0335), which nothing
|
||||
// moved either.
|
||||
if o.Action != "unchanged" && o.Action != "held" && o.Action != "waiting" && o.Action != heldStill &&
|
||||
o.Action != waitsBusy {
|
||||
if o.Action != "unchanged" && o.Action != "held" && o.Action != "waiting" && o.Action != heldStill {
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -1009,7 +987,6 @@ func ApplyMindingWindows(
|
||||
KeptMode: keptMode,
|
||||
KeptOwner: keptOwner,
|
||||
Reads: outcome.reads,
|
||||
BusySince: outcome.busySince,
|
||||
Stateless: outcome.stateless,
|
||||
Scope: outcome.scope,
|
||||
User: outcome.user,
|
||||
@@ -1069,10 +1046,7 @@ func ApplyMindingWindows(
|
||||
Action: "forgotten", Detail: detail})
|
||||
log(fmt.Sprintf(" forgotten %s (%s): %s", from.ID, from.Target, detail))
|
||||
}
|
||||
if outcome.busy != nil {
|
||||
report.Busy = append(report.Busy, *outcome.busy)
|
||||
}
|
||||
if outcome.Action == heldStill || outcome.Action == waitsBusy {
|
||||
if outcome.Action == heldStill {
|
||||
// Not changed: nothing about it moved, so nothing that restarts on it restarts.
|
||||
log(fmt.Sprintf(" %s %s (%s): %s", outcome.Action, outcome.ID, outcome.Target, outcome.Detail))
|
||||
} else if outcome.Action != "unchanged" {
|
||||
@@ -2733,15 +2707,6 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner,
|
||||
out.Action = "unchanged"
|
||||
return out, nil
|
||||
case existed:
|
||||
// **A container that says it is busy is asked before it is replaced** (novox/hq ADR 0335 §2): left
|
||||
// running while it says busy, within the bound, and replaced on the first pass it says idle. Only a
|
||||
// running one: a stopped container holds no work in hand.
|
||||
if r.Busy != nil && before.Running {
|
||||
if waits := askBeforeReplacing(ctx, r, run, cri, previous, &out); waits {
|
||||
out.reads = wasReading
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
if _, err := run(ctx, cri, "rm", "-f", r.Name); err != nil {
|
||||
return out, fmt.Errorf("replacing container %s: %w", r.Name, err)
|
||||
}
|
||||
@@ -2842,9 +2807,6 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner,
|
||||
default:
|
||||
out.Detail = "replaced; a container's configuration is fixed when it is created"
|
||||
}
|
||||
if out.busy != nil {
|
||||
out.Detail += "; " + out.busy.Words()
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -1,139 +0,0 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// A container that says it is busy is replaced once it is idle, within a bound (novox/hq ADR 0335).
|
||||
//
|
||||
// **Asked, never waited on.** The apply asks the container once, and while it says busy leaves it running
|
||||
// as it is and goes on with everything else of the declaration: the machine's lock is held for one ask,
|
||||
// never for the wait. The wait is carried between passes in the container's record (store.Applied's
|
||||
// BusySince), so the next pass — the node-engine asks again sooner than its reconcile while anything waits
|
||||
// — asks again, and so does a node-engine started again in between: the bound runs from the first apply
|
||||
// that would have replaced the container, whoever applied it.
|
||||
|
||||
// waitsBusy is the action of a container left running because it said busy (novox/hq ADR 0335 §2). Its own
|
||||
// word rather than "unchanged": its declaration moved, and the machine is not yet what was declared.
|
||||
const waitsBusy = "waits-busy"
|
||||
|
||||
// What a report says of a busy container (novox/hq ADR 0335 §2, §3).
|
||||
const (
|
||||
// BusyWaiting is a container left running because it said busy, waiting to be replaced.
|
||||
BusyWaiting = "waiting"
|
||||
// BusyReplacedIdle is a container replaced on the first pass it said idle, after it had waited.
|
||||
BusyReplacedIdle = "replaced-when-idle"
|
||||
// BusyReplacedAtBound is a container still busy at the bound, replaced anyway.
|
||||
BusyReplacedAtBound = "replaced-at-bound"
|
||||
)
|
||||
|
||||
// BusyWait is one container a busy command held, as a report says it.
|
||||
type BusyWait struct {
|
||||
// ID is the resource's id (`<module>.<resource>`), Container its runtime name.
|
||||
ID string `json:"id"`
|
||||
Container string `json:"container"`
|
||||
// State is BusyWaiting, BusyReplacedIdle or BusyReplacedAtBound.
|
||||
State string `json:"state"`
|
||||
// Since is when the first apply that would have replaced it ran; Until when the bound replaces it.
|
||||
Since time.Time `json:"since"`
|
||||
Until time.Time `json:"until"`
|
||||
// Said is the last output its busy command gave.
|
||||
Said string `json:"said,omitempty"`
|
||||
}
|
||||
|
||||
// Words is the wait in a sentence, for an outcome's detail and the console.
|
||||
func (b BusyWait) Words() string {
|
||||
since := b.Since.UTC().Format(time.RFC3339)
|
||||
said := ""
|
||||
if b.Said != "" {
|
||||
said = "; its busy command last said: " + b.Said
|
||||
}
|
||||
switch b.State {
|
||||
case BusyReplacedAtBound:
|
||||
return fmt.Sprintf("replaced while still busy: it said busy since %s and the bound of %s ran out "+
|
||||
"(novox/hq ADR 0335 §3)%s", since, declaration.BusyBound, said)
|
||||
case BusyReplacedIdle:
|
||||
return fmt.Sprintf("replaced once it said idle, after waiting since %s (novox/hq ADR 0335 §2)%s", since, said)
|
||||
}
|
||||
return fmt.Sprintf("busy: left running and replaced on the first pass it says idle, or at %s at the latest; "+
|
||||
"waiting since %s (novox/hq ADR 0335 §2)%s", b.Until.UTC().Format(time.RFC3339), since, said)
|
||||
}
|
||||
|
||||
// busyClock is the clock the bound is read on. A test moves it.
|
||||
var busyClock = time.Now
|
||||
|
||||
// busySaidMost is how much of a busy command's output a report carries: its last lines, enough to say what
|
||||
// it held.
|
||||
const busySaidMost = 400
|
||||
|
||||
// askBeforeReplacing asks a running container that declares busy whether it may be replaced now, and says
|
||||
// in out what came of it. True when it waits: out is then the whole outcome of this pass. False when the
|
||||
// caller replaces it — idle, or past the bound — with out carrying the wait it ended, if there was one.
|
||||
func askBeforeReplacing(ctx context.Context, r *declaration.Container, run Runner, cri string,
|
||||
previous store.Applied, out *Outcome) bool {
|
||||
now := busyClock().UTC()
|
||||
busy, said := askBusy(ctx, r, run, cri)
|
||||
var since time.Time
|
||||
waited := previous.BusySince != nil && previous.Target == r.Name
|
||||
if waited {
|
||||
since = previous.BusySince.UTC()
|
||||
}
|
||||
wait := BusyWait{ID: r.ID, Container: r.Name, Since: since, Said: said}
|
||||
switch {
|
||||
case !busy && !waited:
|
||||
// Idle at the first ask: replaced at once, as a container that declares nothing is.
|
||||
return false
|
||||
case !busy:
|
||||
wait.State, wait.Until = BusyReplacedIdle, since.Add(declaration.BusyBound)
|
||||
out.busy = &wait
|
||||
return false
|
||||
}
|
||||
if !waited {
|
||||
since = now
|
||||
wait.Since = now
|
||||
}
|
||||
wait.Until = since.Add(declaration.BusyBound)
|
||||
if !now.Before(wait.Until) {
|
||||
wait.State = BusyReplacedAtBound
|
||||
out.busy = &wait
|
||||
return false
|
||||
}
|
||||
wait.State = BusyWaiting
|
||||
out.Action, out.Detail = waitsBusy, wait.Words()
|
||||
out.busySince, out.busy = &since, &wait
|
||||
return true
|
||||
}
|
||||
|
||||
// askBusy runs a container's busy command inside it, by its shell, as an exec health check is run: busy is
|
||||
// an exit of 0 and nothing else. An ask that could not be made, or did not answer within its timeout, is
|
||||
// not busy: the bound is what keeps a container whose command never answers from being held.
|
||||
func askBusy(ctx context.Context, r *declaration.Container, run Runner, cri string) (bool, string) {
|
||||
ask, cancel := context.WithTimeout(ctx, r.Busy.Within())
|
||||
defer cancel()
|
||||
output, err := run(ask, cri, "exec", r.Name, "/bin/sh", "-c", r.Busy.Command)
|
||||
said := strings.TrimSpace(output)
|
||||
if err != nil && said == "" {
|
||||
said = err.Error()
|
||||
}
|
||||
return err == nil, lastOf(said, busySaidMost)
|
||||
}
|
||||
|
||||
// lastOf is the end of a text, at most n bytes, on one line.
|
||||
func lastOf(s string, n int) string {
|
||||
s = strings.Join(strings.Fields(s), " ")
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
from := len(s) - n
|
||||
for from < len(s) && !utf8.RuneStart(s[from]) {
|
||||
from++
|
||||
}
|
||||
return "…" + s[from:]
|
||||
}
|
||||
@@ -1,326 +0,0 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// A container that says it is busy is replaced once it is idle, within a bound (novox/hq ADR 0335). The
|
||||
// machine of these tests runs two containers, a build agent that declares `busy` and a web server that
|
||||
// does not; every call that replaces a container is counted.
|
||||
|
||||
type aBusyMachine struct {
|
||||
mu sync.Mutex
|
||||
// containers is each container's name and the spec it was created with; running whether it runs.
|
||||
containers map[string]string
|
||||
busy bool // what the agent's busy command answers
|
||||
said string // what it prints
|
||||
asked int
|
||||
calls []string
|
||||
}
|
||||
|
||||
func newBusyMachine() *aBusyMachine {
|
||||
return &aBusyMachine{containers: map[string]string{}}
|
||||
}
|
||||
|
||||
func (m *aBusyMachine) run(_ context.Context, _ string, args ...string) (string, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
switch args[0] {
|
||||
case "info", "image":
|
||||
return "27.0\n", nil
|
||||
case "container": // inspect
|
||||
name := args[len(args)-1]
|
||||
spec, there := m.containers[name]
|
||||
if !there {
|
||||
return "", errors.New("no such container")
|
||||
}
|
||||
return "true\t" + spec, nil
|
||||
case "exec":
|
||||
m.asked++
|
||||
m.calls = append(m.calls, "ask "+args[1])
|
||||
if strings.Join(args[2:], " ") != "/bin/sh -c mesh-builder busy" {
|
||||
return "", errors.New("asked something else: " + strings.Join(args[2:], " "))
|
||||
}
|
||||
if m.busy {
|
||||
return m.said + "\n", nil
|
||||
}
|
||||
return m.said + "\n", errors.New("exited 1")
|
||||
case "rm":
|
||||
name := args[len(args)-1]
|
||||
m.calls = append(m.calls, "rm "+name)
|
||||
delete(m.containers, name)
|
||||
case "run":
|
||||
name := args[slices.Index(args, "--name")+1]
|
||||
m.calls = append(m.calls, "create "+name)
|
||||
for i, a := range args {
|
||||
if a == "--label" && strings.HasPrefix(args[i+1], specLabel+"=") {
|
||||
m.containers[name] = strings.TrimPrefix(args[i+1], specLabel+"=")
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (m *aBusyMachine) mark() int {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return len(m.calls)
|
||||
}
|
||||
|
||||
// since is every call since mark that replaced a container, or asked one.
|
||||
func (m *aBusyMachine) since(mark int) []string {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return append([]string(nil), m.calls[mark:]...)
|
||||
}
|
||||
|
||||
func (m *aBusyMachine) answer(busy bool, said string) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.busy, m.said = busy, said
|
||||
}
|
||||
|
||||
// twoContainers is the declaration at a version: v moves the agent's image tag through its env, w the web's.
|
||||
func twoContainers(t *testing.T, agent, web string) *declaration.Declaration {
|
||||
t.Helper()
|
||||
return parse(t, `{"declaration":1,"resources":[
|
||||
{"id":"build-agent.server","type":"container","name":"mesh-build-agent","image":"`+pinned+`",
|
||||
"env":{"V":"`+agent+`"},"busy":{"command":"mesh-builder busy","timeout":"5s"}},
|
||||
{"id":"web.server","type":"container","name":"web","image":"`+pinned+`","env":{"V":"`+web+`"}}
|
||||
]}`)
|
||||
}
|
||||
|
||||
// atClock moves the bound's clock for one test.
|
||||
func atClock(t *testing.T, at *time.Time) {
|
||||
t.Helper()
|
||||
was := busyClock
|
||||
busyClock = func() time.Time { return *at }
|
||||
t.Cleanup(func() { busyClock = was })
|
||||
}
|
||||
|
||||
func busyOf(r Report, id string) (BusyWait, bool) {
|
||||
for _, b := range r.Busy {
|
||||
if b.ID == id {
|
||||
return b, true
|
||||
}
|
||||
}
|
||||
return BusyWait{}, false
|
||||
}
|
||||
|
||||
func applyBusy(t *testing.T, m *aBusyMachine, d *declaration.Declaration, known store.State) (Report, store.State) {
|
||||
t.Helper()
|
||||
report, state, err := Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, m.run, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("the apply failed: %v", err)
|
||||
}
|
||||
return report, state
|
||||
}
|
||||
|
||||
// The decision's first check: an apply that would replace a container whose busy command exits 0 leaves it
|
||||
// running, applies the rest, and replaces it on the first pass the command exits non-zero.
|
||||
func TestABusyContainerIsLeftRunningAndReplacedOnTheFirstPassItSaysIdle(t *testing.T) {
|
||||
now := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
atClock(t, &now)
|
||||
m := newBusyMachine()
|
||||
_, state := applyBusy(t, m, twoContainers(t, "1", "1"), store.State{})
|
||||
|
||||
// A merge moves both: the agent holds a build, the web server holds nothing.
|
||||
m.answer(true, "building novox/mesh-controller (check of #280) since 00:58")
|
||||
mark := m.mark()
|
||||
report, state := applyBusy(t, m, twoContainers(t, "2", "2"), state)
|
||||
if got := m.since(mark); !slices.Equal(got, []string{"ask mesh-build-agent", "rm web", "create web"}) {
|
||||
t.Fatalf("a busy agent was not left running while the rest applied: %v", got)
|
||||
}
|
||||
o := outcomeOf(report, "build-agent.server")
|
||||
if o.Action != waitsBusy {
|
||||
t.Fatalf("a busy container was reported %q, want %q: %+v", o.Action, waitsBusy, o)
|
||||
}
|
||||
for _, says := range []string{"busy", "first pass it says idle", "check of #280", "ADR 0335"} {
|
||||
if !strings.Contains(o.Detail, says) {
|
||||
t.Errorf("the outcome does not say %q: %s", says, o.Detail)
|
||||
}
|
||||
}
|
||||
if (Report{Outcomes: []Outcome{o}}).Changed() {
|
||||
t.Errorf("a container left running reads as a change: %+v", o)
|
||||
}
|
||||
b, said := busyOf(report, "build-agent.server")
|
||||
if !said || b.State != BusyWaiting || !b.Since.Equal(now) || !b.Until.Equal(now.Add(15*time.Minute)) ||
|
||||
b.Container != "mesh-build-agent" || !strings.Contains(b.Said, "check of #280") {
|
||||
t.Fatalf("the report does not say the agent waits, since when and until when: %+v", report.Busy)
|
||||
}
|
||||
if !report.Waiting() {
|
||||
t.Error("a report with a container waiting does not say so")
|
||||
}
|
||||
if rec, _ := state.Find("build-agent.server"); rec.BusySince == nil || !rec.BusySince.Equal(now) {
|
||||
t.Fatalf("the wait is not kept in the container's record: %+v", rec)
|
||||
}
|
||||
|
||||
// Asked again a minute later, still busy: still left running, the wait counted from the first apply.
|
||||
now = now.Add(time.Minute)
|
||||
mark = m.mark()
|
||||
report, state = applyBusy(t, m, twoContainers(t, "2", "2"), state)
|
||||
if got := m.since(mark); !slices.Equal(got, []string{"ask mesh-build-agent"}) {
|
||||
t.Fatalf("a second pass while busy did more than ask: %v", got)
|
||||
}
|
||||
if b, _ := busyOf(report, "build-agent.server"); b.State != BusyWaiting || !b.Since.Equal(now.Add(-time.Minute)) {
|
||||
t.Fatalf("the second pass does not keep the wait from the first apply: %+v", report.Busy)
|
||||
}
|
||||
|
||||
// The build ends: the next pass replaces it, and says it waited.
|
||||
now = now.Add(time.Minute)
|
||||
m.answer(false, "idle")
|
||||
mark = m.mark()
|
||||
report, state = applyBusy(t, m, twoContainers(t, "2", "2"), state)
|
||||
if got := m.since(mark); !slices.Equal(got, []string{"ask mesh-build-agent", "rm mesh-build-agent", "create mesh-build-agent"}) {
|
||||
t.Fatalf("the agent was not replaced on the first pass it said idle: %v", got)
|
||||
}
|
||||
if o := outcomeOf(report, "build-agent.server"); o.Action != "updated" || !strings.Contains(o.Detail, "once it said idle") {
|
||||
t.Errorf("the replacement does not say it waited: %+v", o)
|
||||
}
|
||||
if b, _ := busyOf(report, "build-agent.server"); b.State != BusyReplacedIdle {
|
||||
t.Errorf("the report does not say the agent was replaced when idle: %+v", report.Busy)
|
||||
}
|
||||
if report.Waiting() {
|
||||
t.Error("a report with nothing waiting says something waits")
|
||||
}
|
||||
if rec, _ := state.Find("build-agent.server"); rec.BusySince != nil {
|
||||
t.Errorf("the wait is still kept after the replacement: %+v", rec)
|
||||
}
|
||||
|
||||
// And the pass after that has nothing to say about it.
|
||||
report, _ = applyBusy(t, m, twoContainers(t, "2", "2"), state)
|
||||
if len(report.Busy) != 0 {
|
||||
t.Errorf("a container replaced is still said: %+v", report.Busy)
|
||||
}
|
||||
}
|
||||
|
||||
// An idle container is replaced at once, as one that declares nothing is: asked once, nothing waits.
|
||||
func TestAnIdleContainerIsReplacedAtOnce(t *testing.T) {
|
||||
m := newBusyMachine()
|
||||
_, state := applyBusy(t, m, twoContainers(t, "1", "1"), store.State{})
|
||||
m.answer(false, "")
|
||||
mark := m.mark()
|
||||
report, _ := applyBusy(t, m, twoContainers(t, "2", "1"), state)
|
||||
if got := m.since(mark); !slices.Equal(got, []string{"ask mesh-build-agent", "rm mesh-build-agent", "create mesh-build-agent"}) {
|
||||
t.Fatalf("an idle agent was not replaced at once: %v", got)
|
||||
}
|
||||
if len(report.Busy) != 0 {
|
||||
t.Errorf("an idle container replaced at once is said as a wait: %+v", report.Busy)
|
||||
}
|
||||
// And a container whose declaration did not move is never asked.
|
||||
mark = m.mark()
|
||||
applyBusy(t, m, twoContainers(t, "2", "1"), state)
|
||||
if got := m.since(mark); slices.Contains(got, "ask mesh-build-agent") {
|
||||
t.Errorf("a container nothing replaces was asked: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Past the bound it is replaced anyway, the report carrying the command's last output — and the bound holds
|
||||
// across a restart of the node-engine: the wait is read back from the state on disk.
|
||||
func TestABusyContainerIsReplacedAtTheBoundEvenAfterARestart(t *testing.T) {
|
||||
now := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
atClock(t, &now)
|
||||
m := newBusyMachine()
|
||||
_, state := applyBusy(t, m, twoContainers(t, "1", "1"), store.State{})
|
||||
m.answer(true, "building novox/mesh-host (a check that never ends)")
|
||||
_, state = applyBusy(t, m, twoContainers(t, "2", "1"), state)
|
||||
|
||||
// The node-engine stops and starts again: what it knows is what it saved.
|
||||
path := filepath.Join(t.TempDir(), "state.json")
|
||||
if err := store.Save(path, state); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reloaded, err := store.Load(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
now = now.Add(14 * time.Minute)
|
||||
mark := m.mark()
|
||||
report, reloaded := applyBusy(t, m, twoContainers(t, "2", "1"), reloaded)
|
||||
if got := m.since(mark); slices.Contains(got, "rm mesh-build-agent") {
|
||||
t.Fatalf("a container busy within the bound was replaced after a restart: %v", got)
|
||||
}
|
||||
if b, _ := busyOf(report, "build-agent.server"); b.State != BusyWaiting || !b.Since.Equal(now.Add(-14*time.Minute)) {
|
||||
t.Fatalf("a restart started the wait again: %+v", report.Busy)
|
||||
}
|
||||
|
||||
now = now.Add(time.Minute)
|
||||
mark = m.mark()
|
||||
report, _ = applyBusy(t, m, twoContainers(t, "2", "1"), reloaded)
|
||||
if got := m.since(mark); !slices.Equal(got, []string{"ask mesh-build-agent", "rm mesh-build-agent", "create mesh-build-agent"}) {
|
||||
t.Fatalf("a container still busy at the bound was not replaced: %v", got)
|
||||
}
|
||||
b, _ := busyOf(report, "build-agent.server")
|
||||
if b.State != BusyReplacedAtBound || !strings.Contains(b.Said, "never ends") {
|
||||
t.Fatalf("the report does not say it replaced a busy container, with its last output: %+v", report.Busy)
|
||||
}
|
||||
if o := outcomeOf(report, "build-agent.server"); !strings.Contains(o.Detail, "still busy") ||
|
||||
!strings.Contains(o.Detail, "never ends") {
|
||||
t.Errorf("the outcome does not say a busy container was replaced, with what it said: %s", o.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
// While one container waits, a send that changes another module on the same machine is applied at once: the
|
||||
// apply returns after one ask, and holds nothing for the wait.
|
||||
func TestASendToAnotherModuleIsAppliedWhileOneWaits(t *testing.T) {
|
||||
now := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
atClock(t, &now)
|
||||
m := newBusyMachine()
|
||||
_, state := applyBusy(t, m, twoContainers(t, "1", "1"), store.State{})
|
||||
m.answer(true, "building")
|
||||
_, state = applyBusy(t, m, twoContainers(t, "2", "1"), state)
|
||||
|
||||
// The next send moves only the web server.
|
||||
mark := m.mark()
|
||||
done := make(chan struct{})
|
||||
var report Report
|
||||
go func() {
|
||||
defer close(done)
|
||||
report, _ = applyBusy(t, m, twoContainers(t, "2", "2"), state)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("an apply while a container waits did not return")
|
||||
}
|
||||
got := m.since(mark)
|
||||
if !slices.Contains(got, "create web") || slices.Contains(got, "rm mesh-build-agent") {
|
||||
t.Fatalf("the other module's send was not applied at once beside the wait: %v", got)
|
||||
}
|
||||
if o := outcomeOf(report, "web.server"); o.Action != "updated" {
|
||||
t.Errorf("the web server was reported %q: %+v", o.Action, o)
|
||||
}
|
||||
}
|
||||
|
||||
// A container's busy is held to its shape where it arrives.
|
||||
func TestBusyIsReadAndHeldToItsShape(t *testing.T) {
|
||||
d := twoContainers(t, "1", "1")
|
||||
c := d.Resources[0].(*declaration.Container)
|
||||
if c.Busy == nil || c.Busy.Command != "mesh-builder busy" || c.Busy.Within() != 5*time.Second {
|
||||
t.Fatalf("busy was not read: %+v", c.Busy)
|
||||
}
|
||||
for name, raw := range map[string]string{
|
||||
"no command": `"busy":{"command":" "}`,
|
||||
"a long timeout": `"busy":{"command":"x","timeout":"5m"}`,
|
||||
"not a duration": `"busy":{"command":"x","timeout":"soon"}`,
|
||||
"an unknown key": `"busy":{"command":"x","every":"1m"}`,
|
||||
"on a step": `"run-once":true,"busy":{"command":"x"}`,
|
||||
} {
|
||||
_, err := declaration.Parse([]byte(`{"declaration":1,"resources":[{"id":"a.b","type":"container",` +
|
||||
`"name":"b","image":"` + pinned + `",` + raw + `}]}`))
|
||||
if err == nil {
|
||||
t.Errorf("%s: a busy of that shape was accepted", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -36,7 +36,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/firewall"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
"github.com/novox/mesh-host/internal/tunnel"
|
||||
)
|
||||
|
||||
@@ -293,14 +292,12 @@ type Result struct {
|
||||
Agent string `json:"host-agent,omitempty"`
|
||||
|
||||
// Registry is the mesh's own artifact store, once it answers.
|
||||
Registry string `json:"registry,omitempty"`
|
||||
RegistryReplied int `json:"registry-replied,omitempty"`
|
||||
RegistryKnown bool `json:"registry-already-registered,omitempty"`
|
||||
RegistryRunning string `json:"registry-container,omitempty"`
|
||||
// RegistryTrusted are the names the runtime was told to trust the registry by, before the push.
|
||||
RegistryTrusted []string `json:"registry-trusted,omitempty"`
|
||||
PublishedAs string `json:"control-plane-image,omitempty"`
|
||||
PublishedAlready bool `json:"control-plane-image-already-published,omitempty"`
|
||||
Registry string `json:"registry,omitempty"`
|
||||
RegistryReplied int `json:"registry-replied,omitempty"`
|
||||
RegistryKnown bool `json:"registry-already-registered,omitempty"`
|
||||
RegistryRunning string `json:"registry-container,omitempty"`
|
||||
PublishedAs string `json:"control-plane-image,omitempty"`
|
||||
PublishedAlready bool `json:"control-plane-image-already-published,omitempty"`
|
||||
|
||||
// Permanent is the control plane as an ordinary module.
|
||||
Permanent string `json:"permanent-container,omitempty"`
|
||||
@@ -558,9 +555,7 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
|
||||
// Named rather than attempted. Everything from step 6 on is a conversation with a
|
||||
// control plane that a dry run has not raised, so there is nothing to ask and nothing
|
||||
// honest to report about the answers.
|
||||
say(" would then enrol " + o.Node + ", have the runtime trust the registry as " +
|
||||
strings.Join(RegistryNames(o), ", ") + ",")
|
||||
say(" install the registry from " +
|
||||
say(" would then enrol " + o.Node + ", install the registry from " +
|
||||
o.Catalogue + ", push the control plane's image into it,")
|
||||
say(" reinstall the control plane as a module, and drop " +
|
||||
rewritten.TempName)
|
||||
@@ -659,10 +654,26 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
|
||||
return result, failed(StepOperator, err)
|
||||
}
|
||||
|
||||
// ---- 7. registry, 8. publish -----------------------------------------------------------
|
||||
published, err := RegistryAndPublish(ctx, o, d, sys, temporary, controlPlaneImage, &result, say)
|
||||
// ---- 7. registry ----------------------------------------------------------------------
|
||||
say("registry — somewhere for this mesh to keep its own images")
|
||||
registry, err := InstallRegistry(ctx, o, d, temporary, say)
|
||||
result.Registry, result.RegistryRunning = registry.Address, registry.Container
|
||||
result.RegistryKnown, result.RegistryReplied = registry.Known, registry.Answered
|
||||
if err != nil {
|
||||
return result, err
|
||||
return result, failed(StepRegistry, err)
|
||||
}
|
||||
|
||||
// ---- 8. publish -----------------------------------------------------------------------
|
||||
say("publish — the control plane's image gets its first manifest digest")
|
||||
// **The image this mesh built, not the one the installer carried.** The carried one is the
|
||||
// builder; publishing it here would put a builder in the registry under the control plane's
|
||||
// name and install it as the control plane — which is exactly what happened the first time
|
||||
// this ran, and presented as a control plane that started, printed a builder's usage, and
|
||||
// exited cleanly over and over.
|
||||
published, err := PublishControlPlane(ctx, o, d, controlPlaneImage, say)
|
||||
result.PublishedAs, result.PublishedAlready = published.Reference, published.Already
|
||||
if err != nil {
|
||||
return result, failed(StepPublish, err)
|
||||
}
|
||||
|
||||
// ---- 9. control plane -----------------------------------------------------------------
|
||||
@@ -734,7 +745,7 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
|
||||
|
||||
// ---- broker ---------------------------------------------------------------------------
|
||||
say("broker — the foundation's bus, adopted as the nats module: one bus, not two")
|
||||
if err := InstallBroker(ctx, o, permanentControl, rewritten.Declaration, say); err != nil {
|
||||
if err := InstallBroker(ctx, o, permanentControl, rewritten.Declaration, d.Fetch, say); err != nil {
|
||||
return result, failed(StepBroker, err)
|
||||
}
|
||||
|
||||
@@ -792,44 +803,3 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// RegistryAndPublish is steps 7 and 8: the runtime trusts the mesh's registry, the registry is
|
||||
// installed and answers, and the control plane's image is pushed into it.
|
||||
//
|
||||
// **One function so the order is held by a test** (novox/hq issue 92). The runtime refuses plain
|
||||
// HTTP to a registry it has not been told to trust, and the module that tells it is applied long
|
||||
// after the push — so the trust is written and confirmed here, before anything is tagged or pushed.
|
||||
// A refusal names the step it happened in.
|
||||
func RegistryAndPublish(ctx context.Context, o Options, d Deps, sys system.System, temporary controlPlane,
|
||||
controlPlaneImage string, result *Result, say func(string)) (Published, error) {
|
||||
|
||||
// ---- 7. registry ----------------------------------------------------------------------
|
||||
say("registry — somewhere for this mesh to keep its own images")
|
||||
// Trusted by the runtime before anything is pushed to it, by every name the mesh will use for
|
||||
// it here: the module that writes this trust is applied long after the push (novox/hq issue 92).
|
||||
trusted, err := TrustTheRegistry(ctx, o, sys, d.Run, say)
|
||||
result.RegistryTrusted = trusted
|
||||
if err != nil {
|
||||
return Published{}, failed(StepRegistry, err)
|
||||
}
|
||||
registry, err := InstallRegistry(ctx, o, d, temporary, say)
|
||||
result.Registry, result.RegistryRunning = registry.Address, registry.Container
|
||||
result.RegistryKnown, result.RegistryReplied = registry.Known, registry.Answered
|
||||
if err != nil {
|
||||
return Published{}, failed(StepRegistry, err)
|
||||
}
|
||||
|
||||
// ---- 8. publish -----------------------------------------------------------------------
|
||||
say("publish — the control plane's image gets its first manifest digest")
|
||||
// **The image this mesh built, not the one the installer carried.** The carried one is the
|
||||
// builder; publishing it here would put a builder in the registry under the control plane's
|
||||
// name and install it as the control plane — which is exactly what happened the first time
|
||||
// this ran, and presented as a control plane that started, printed a builder's usage, and
|
||||
// exited cleanly over and over.
|
||||
published, err := PublishControlPlane(ctx, o, d, controlPlaneImage, say)
|
||||
result.PublishedAs, result.PublishedAlready = published.Reference, published.Already
|
||||
if err != nil {
|
||||
return published, failed(StepPublish, err)
|
||||
}
|
||||
return published, nil
|
||||
}
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// BuilderModule is the module that lets a mesh produce anything at all.
|
||||
const BuilderModule = "builder"
|
||||
// BuilderModule is the module that lets a mesh produce anything at all: the build agent, which holds
|
||||
// the node seat `node-build-agent` (novox/hq ADR 0190). The one-holder `builder` module it replaced
|
||||
// left the catalogue on 2026-10-03, and genesis went on asking for it (issue 596).
|
||||
const BuilderModule = "build-agent"
|
||||
|
||||
// BuilderRepository is what its image is called in this mesh's own registry.
|
||||
const BuilderRepository = "mesh-builder"
|
||||
@@ -50,10 +53,10 @@ func InstallBuilder(ctx context.Context, o Options, d Deps, control controlPlane
|
||||
manifest, err := readManifest(o.Catalogue, BuilderModule)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("%w\n"+
|
||||
"This is the manifest that makes the builder an ordinary module. Without it the mesh "+
|
||||
"This is the manifest that makes the build agent an ordinary module. Without it the mesh "+
|
||||
"has the image and no way to run it, so nothing can be built here", err)
|
||||
}
|
||||
pinned, places, err := pinPlaceholder(manifest, published.Reference, BuilderModule)
|
||||
pinned, places, err := pinTheCarriedBuilder(manifest, published.Reference)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
@@ -66,21 +69,72 @@ func InstallBuilder(ctx context.Context, o Options, d Deps, control controlPlane
|
||||
}
|
||||
|
||||
// Before the push, so the account is in the declaration the machine receives rather than in
|
||||
// the one after it. A builder that arrives without its credential starts, finds nothing it may
|
||||
// read, and waits — which looks exactly like a builder with no work.
|
||||
account := o.Node + "-" + BuilderModule
|
||||
if _, err := control.tell(ctx, "builder", "issue", account, "--node", o.Node); err != nil {
|
||||
// Said and carried on. An account that already exists is the ordinary case on a re-run,
|
||||
// and the push below is what makes either state true on the machine.
|
||||
if !strings.Contains(err.Error(), "already") {
|
||||
return out, fmt.Errorf("the builder has no broker account, so it can take no work: %w", err)
|
||||
}
|
||||
say(" broker account " + account + " — already issued")
|
||||
} else {
|
||||
out.Account = true
|
||||
say(" broker account " + account + ", scoped to what it consumes and emits")
|
||||
// the one after it. A build agent that arrives without its credential starts, finds nothing it
|
||||
// may read, and waits — which looks exactly like an agent with no work. The same act as any
|
||||
// module's account (`module issue`), which a re-run repeats harmlessly.
|
||||
if _, err := control.tell(ctx, "module", "issue", BuilderModule, "--node", o.Node); err != nil {
|
||||
return out, fmt.Errorf("the build agent has no account on the bus, so it can take no work: %w", err)
|
||||
}
|
||||
out.Account = true
|
||||
say(" bus account " + BuilderModule + " on " + o.Node + ", scoped to what it consumes and emits")
|
||||
|
||||
out.Installed.Pushed, err = pushNode(ctx, o, control, say)
|
||||
return out, err
|
||||
}
|
||||
|
||||
// pinTheCarriedBuilder names the image this installer carried and published in the build agent's
|
||||
// manifest, in place of the artifact the mesh would build for it.
|
||||
//
|
||||
// **The one module genesis cannot have built** (novox/hq ADR 0073): the build agent is what builds,
|
||||
// and at this step the mesh has no build agent. The catalogue's manifest names its server container by
|
||||
// the artifact it builds (`"artifact": "server"`, an image built from mesh-controller's
|
||||
// cmd/mesh-builder); the image this installer carries is that build, made from the same program. So
|
||||
// every container built from that image artifact runs the published reference instead, and the rest
|
||||
// of the manifest — its build, its seat, its requirements — is the catalogue's as written. The mesh
|
||||
// rebuilds it from the catalogue like any module from then on.
|
||||
//
|
||||
// A manifest still carrying the old placeholder digest is pinned the old way. One naming neither an
|
||||
// image artifact nor a placeholder is refused: there would be nothing to put the carried image in.
|
||||
func pinTheCarriedBuilder(manifest []byte, reference string) ([]byte, int, error) {
|
||||
if bytes.Contains(manifest, []byte(placeholderDigest)) {
|
||||
return pinPlaceholder(manifest, reference, BuilderModule)
|
||||
}
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(manifest, &m); err != nil {
|
||||
return nil, 0, fmt.Errorf("the %s module's manifest is not readable: %w", BuilderModule, err)
|
||||
}
|
||||
images := map[string]bool{}
|
||||
if build, ok := m["build"].(map[string]any); ok {
|
||||
artifacts, _ := build["artifacts"].([]any)
|
||||
for _, a := range artifacts {
|
||||
if art, ok := a.(map[string]any); ok && art["kind"] == "image" {
|
||||
if name, ok := art["name"].(string); ok {
|
||||
images[name] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
places := 0
|
||||
resources, _ := m["resources"].([]any)
|
||||
for _, r := range resources {
|
||||
res, ok := r.(map[string]any)
|
||||
if !ok || res["type"] != "container" {
|
||||
continue
|
||||
}
|
||||
if name, ok := res["artifact"].(string); ok && images[name] {
|
||||
delete(res, "artifact")
|
||||
res["image"] = reference
|
||||
places++
|
||||
}
|
||||
}
|
||||
if places == 0 {
|
||||
return nil, 0, fmt.Errorf(
|
||||
"the %s module's manifest runs no container from an image it builds, so there is nowhere to "+
|
||||
"put the builder this installer carried and published (%s)", BuilderModule, reference)
|
||||
}
|
||||
pinned, err := json.MarshalIndent(m, "", " ")
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return append(pinned, '\n'), places, nil
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package bootstrap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -44,15 +45,15 @@ func TestTheBuildersPackageRegistryPortIsASettingAndNotTheManifest(t *testing.T)
|
||||
[]byte(theBuildersBinding), quietly); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := c.settings["builder-settings.json"]; got != `{"serves":{"port":3100}}` {
|
||||
if got := c.settings[BuilderModule+"-settings.json"]; got != `{"serves":{"port":3100}}` {
|
||||
t.Errorf("the builder was told %q about the package registry's port", got)
|
||||
}
|
||||
// The manifest reaches the mesh as the catalogue wrote it. A port rewritten into it here is a
|
||||
// port the next registration from the catalogue silently takes back.
|
||||
if got := c.settings["builder-module.json"]; got != theBuildersBinding {
|
||||
if got := c.settings[BuilderModule+"-module.json"]; got != theBuildersBinding {
|
||||
t.Errorf("the installer changed the builder's manifest:\n%s", got)
|
||||
}
|
||||
set, push := c.index("settings set builder"), c.index("push anchor")
|
||||
set, push := c.index("settings set "+BuilderModule), c.index("push anchor")
|
||||
if set < 0 || push < 0 || set > push {
|
||||
t.Fatalf("the port was not set before the push that raises the builder: %v", c.told)
|
||||
}
|
||||
@@ -167,3 +168,73 @@ func TestAForgeManifestTheCatalogueDoesNotHaveIsNamed(t *testing.T) {
|
||||
t.Errorf("a missing forge manifest was not explained: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// theCataloguesModule is the catalogue's own manifest of a module, never a copy: the clone beside a merge
|
||||
// check (MESH_CHECK_BESIDE), where it must be; else a checkout beside this one; else the test skips aloud.
|
||||
func theCataloguesModule(t *testing.T, module string) []byte {
|
||||
t.Helper()
|
||||
inside := filepath.Join("mesh-catalog", "modules", module, "module.json")
|
||||
if root := os.Getenv("MESH_CHECK_BESIDE"); root != "" {
|
||||
raw, err := os.ReadFile(filepath.Join(root, inside))
|
||||
if err != nil {
|
||||
t.Fatalf("the catalogue beside this check (MESH_CHECK_BESIDE=%s) has no %s: %v", root, inside, err)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join("..", "..", "..", inside))
|
||||
if err != nil {
|
||||
t.Skipf("NOT JUDGED: no catalogue beside this checkout and no MESH_CHECK_BESIDE; the merge check judges it")
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
// **Genesis installs the build agent the catalogue holds** (novox/hq issue 596, ADR 0190): the
|
||||
// one-holder `builder` left the catalogue on 2026-10-03, and genesis stopped at step 13 asking for it.
|
||||
// The catalogue's own manifest is read, and the image the installer carried is put where the manifest
|
||||
// names the image it builds.
|
||||
func TestGenesisInstallsTheCataloguesBuildAgent(t *testing.T) {
|
||||
manifest := theCataloguesModule(t, BuilderModule)
|
||||
const reference = "127.0.0.1:5000/mesh-builder@sha256:" + "4444444444444444444444444444444444444444444444444444444444444444"
|
||||
pinned, places, err := pinTheCarriedBuilder(manifest, reference)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Claims []struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"claims"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(pinned, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m.Module != BuilderModule || len(m.Claims) == 0 || m.Claims[0].Name != "node-build-agent" {
|
||||
t.Fatalf("genesis installs %q claiming %v; the build agent holds node-build-agent (ADR 0190)", m.Module, m.Claims)
|
||||
}
|
||||
running := 0
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] != "container" {
|
||||
continue
|
||||
}
|
||||
if _, still := r["artifact"]; still {
|
||||
t.Errorf("the container %v still names an artifact nothing has built yet", r["name"])
|
||||
}
|
||||
if r["image"] == reference {
|
||||
running++
|
||||
}
|
||||
}
|
||||
if places != 1 || running != 1 {
|
||||
t.Errorf("the carried builder runs in %d container(s) (%d place(s) said); the build agent runs one", running, places)
|
||||
}
|
||||
}
|
||||
|
||||
// A manifest that runs nothing from an image it builds has nowhere to put the carried builder, and is
|
||||
// refused rather than registered running nothing.
|
||||
func TestABuildAgentManifestWithNowhereForTheCarriedBuilderIsRefused(t *testing.T) {
|
||||
_, _, err := pinTheCarriedBuilder([]byte(`{"module":"build-agent","resources":[{"id":"server","type":"container","name":"x","artifact":"server"}],"build":{"artifacts":[{"name":"server","kind":"bundle"}]}}`),
|
||||
"127.0.0.1:5000/mesh-builder@sha256:"+strings.Repeat("4", 64))
|
||||
if err == nil || !strings.Contains(err.Error(), "nowhere to put") {
|
||||
t.Fatalf("a manifest with no image artifact was pinned: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -311,6 +311,17 @@ func secretsByVariableIn(manifest []byte) (map[string]string, error) {
|
||||
for name, path := range m.OwnSecrets {
|
||||
secretAt[path] = name
|
||||
}
|
||||
// **A file the module itself declares is its own, not a secret** (novox/hq issue 597). The
|
||||
// controller reads its merge window's settings from a file its manifest writes from its settings
|
||||
// (MESH_MERGE_WINDOW_FILE, ADR 0276): the mesh writes it with the module's declaration, and there
|
||||
// is nothing for genesis to carry in. Only a `…_FILE` naming neither an own-secret nor a file of
|
||||
// the module is refused.
|
||||
declared := map[string]bool{}
|
||||
for _, r := range m.Resources {
|
||||
if r.Type == "file" && r.Path != "" {
|
||||
declared[r.Path] = true
|
||||
}
|
||||
}
|
||||
|
||||
wanted := map[string]string{}
|
||||
for _, r := range m.Resources {
|
||||
@@ -333,10 +344,13 @@ func secretsByVariableIn(manifest []byte) (map[string]string, error) {
|
||||
on = from
|
||||
}
|
||||
secret, named := secretAt[on]
|
||||
if !named && declared[on] {
|
||||
continue
|
||||
}
|
||||
if !named {
|
||||
return nil, fmt.Errorf(
|
||||
"the %s module's container reads %s from %s, and no own-secret of that "+
|
||||
"module writes that file.\n"+
|
||||
"the %s module's container reads %s from %s, and neither an own-secret nor "+
|
||||
"a file of that module writes that file.\n"+
|
||||
"So the mesh would seal nothing there and the control plane would find "+
|
||||
"an empty file where a connection string has to be. The manifest has to "+
|
||||
"name the two ends the same, directly or through a mount",
|
||||
|
||||
@@ -94,10 +94,10 @@ func Enrol(ctx context.Context, o Options, sys system.System, control controlPla
|
||||
switch mine, err := identity.Load(where); {
|
||||
case err == nil && mine.Node == o.Node:
|
||||
say(" already enrolled " + o.Node + " — " + where)
|
||||
// **Enrolled is not adopted** (novox/hq ADR 0331 §5). This run's apply wrote the bundle's
|
||||
// user list again — the controller's entry and nobody else's — over the list placed when
|
||||
// the machine enrolled. Until the bus is the nats module, that list is still the
|
||||
// installer's to place, so it is placed again, before anything dials the bus as a node.
|
||||
// **Enrolled is not adopted** (novox/hq ADR 0331 §5). Until the bus is the nats module, its
|
||||
// user list is the installer's to place, so it is placed again, before anything dials the
|
||||
// bus as a node: the controller's composition is the one that is right, whatever an earlier
|
||||
// run left there. The bundle seeds the list only where none is (create-once, ADR 0333 §4).
|
||||
if err := placeUntilAdopted(ctx, control, say); err != nil {
|
||||
return out, err
|
||||
}
|
||||
@@ -168,9 +168,14 @@ func Enrol(ctx context.Context, o Options, sys system.System, control controlPla
|
||||
// configuration includes, and the container that reads it. The installer raised them, so it is the
|
||||
// one that knows them (examples/foundation-first-node.lock). A bridge until the bus is adopted as the
|
||||
// nats module, which receives the list in its declaration from then on (novox/hq ADR 0331 §5).
|
||||
//
|
||||
// **The module's path and the module's container** (novox/hq ADR 0333 §4): the bundle raises the bus
|
||||
// where the `nats` module keeps it, so the list placed here is the one the module goes on writing, in
|
||||
// the same file, read by the same container. The tests hold both to the bundle and to the module's
|
||||
// manifest (`bus-users` and its server container).
|
||||
const (
|
||||
busAccounts = "/var/lib/mesh-bus-conf/accounts.conf"
|
||||
busContainer = "mesh-broker"
|
||||
busAccounts = "/var/lib/nats-module/conf/accounts.conf"
|
||||
busContainer = "mesh-broker-nats"
|
||||
)
|
||||
|
||||
// placeTheBusUsers writes the mesh's composed user list beside the bus the installer raised, and makes
|
||||
|
||||
@@ -315,8 +315,8 @@ func TestAFirstNodeIsLetOntoTheBusItRaised(t *testing.T) {
|
||||
|
||||
placed, enrol := []int{}, -1
|
||||
for i, c := range runtime.commands {
|
||||
if strings.HasPrefix(c, "sh -c") && strings.Contains(c, "kill -s HUP mesh-broker") &&
|
||||
strings.Contains(c, "/var/lib/mesh-bus-conf/accounts.conf") {
|
||||
if strings.HasPrefix(c, "sh -c") && strings.Contains(c, "kill -s HUP mesh-broker-nats") &&
|
||||
strings.Contains(c, "/var/lib/nats-module/conf/accounts.conf") {
|
||||
placed = append(placed, i)
|
||||
}
|
||||
if strings.HasPrefix(c, "/usr/local/bin/mesh-host enrol") {
|
||||
@@ -365,7 +365,7 @@ func TestARerunOnAnEnrolledMachineWhoseBusIsNotAdoptedPlacesTheUsersAgain(t *tes
|
||||
placed := false
|
||||
for _, c := range runtime.commands {
|
||||
if strings.HasPrefix(c, "sh -c") && strings.Contains(c, "node.anchor") &&
|
||||
strings.Contains(c, "/var/lib/mesh-bus-conf/accounts.conf") && strings.Contains(c, "kill -s HUP mesh-broker") {
|
||||
strings.Contains(c, "/var/lib/nats-module/conf/accounts.conf") && strings.Contains(c, "kill -s HUP mesh-broker-nats") {
|
||||
placed = true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -180,3 +180,85 @@ func TestAProcessFormNamingNoOneReplacementIsRefused(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// theStepNineSecretsOf is what step 9 delivers for a process-form controller manifest: its genesis
|
||||
// container made, pinned, and its secrets read from that container's environment.
|
||||
func theStepNineSecretsOf(t *testing.T, manifest []byte) (map[string]string, error) {
|
||||
t.Helper()
|
||||
form, err := processFormOf(manifest)
|
||||
if err != nil || !form.Found {
|
||||
t.Fatalf("the manifest has no process form: %v", err)
|
||||
}
|
||||
genesis, err := genesisForm(manifest, form, builtImage)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pinned, _, err := pinImage(genesis, builtImage, "registry.internal:5000/mesh-controller@sha256:"+strings.Repeat("e", 64), ControlPlaneModule)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return secretsByVariableIn(pinned)
|
||||
}
|
||||
|
||||
// **A file the controller's manifest writes itself is not a secret** (novox/hq issue 597). The merge
|
||||
// window's settings reach the controller in a file its manifest declares from its settings
|
||||
// (MESH_MERGE_WINDOW_FILE, ADR 0276); genesis refused it as a secret nobody writes, and stopped at
|
||||
// step 11 of 24.
|
||||
func TestAFileTheControllersManifestWritesIsNotASecret(t *testing.T) {
|
||||
manifest := strings.Replace(processFormManifest,
|
||||
`{"id": "controller", "type": "process",`,
|
||||
`{"id": "merge-window", "type": "file", "path": "${dir:mesh-state}/merge-window.json", "mode": "0600",
|
||||
"owner": "mesh-controller", "content": "{\"merge-window\": \"${setting:merge-window}\"}\n"},
|
||||
{"id": "controller", "type": "process",`, 1)
|
||||
manifest = strings.Replace(manifest, `"MESH_BUS_NATS_FILE": "${dir:mesh-state}/bus"}`,
|
||||
`"MESH_BUS_NATS_FILE": "${dir:mesh-state}/bus",
|
||||
"MESH_MERGE_WINDOW_FILE": "${dir:mesh-state}/merge-window.json"}`, 1)
|
||||
if !strings.Contains(manifest, "MESH_MERGE_WINDOW_FILE") || !strings.Contains(manifest, `"id": "merge-window"`) {
|
||||
t.Fatal("the fixture was not changed")
|
||||
}
|
||||
wanted, err := theStepNineSecretsOf(t, []byte(manifest))
|
||||
if err != nil {
|
||||
t.Fatalf("step 9 refuses a file the manifest writes itself: %v", err)
|
||||
}
|
||||
if _, asked := wanted["MESH_MERGE_WINDOW"]; asked {
|
||||
t.Errorf("the merge window's settings are delivered as a secret: %v", wanted)
|
||||
}
|
||||
if wanted["MESH_STORE_INVENTORY"] != "inventory" || wanted["MESH_BUS_NATS"] != "bus" {
|
||||
t.Errorf("step 9 delivers %v; the store's and the bus's secrets are still wanted", wanted)
|
||||
}
|
||||
}
|
||||
|
||||
// And a `…_FILE` that neither an own-secret nor a file of the module writes is still refused.
|
||||
func TestAFileNothingOfTheControllerWritesIsStillRefused(t *testing.T) {
|
||||
manifest := strings.Replace(processFormManifest, `"MESH_BUS_NATS_FILE": "${dir:mesh-state}/bus"}`,
|
||||
`"MESH_BUS_NATS_FILE": "${dir:mesh-state}/bus",
|
||||
"MESH_MERGE_WINDOW_FILE": "${dir:mesh-state}/merge-window.json"}`, 1)
|
||||
if _, err := theStepNineSecretsOf(t, []byte(manifest)); err == nil || !strings.Contains(err.Error(), "MESH_MERGE_WINDOW_FILE") {
|
||||
t.Fatalf("a file nothing writes was accepted, or refused without its name: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The controller's own manifest, as the mesh runs it, passes step 9 (issue 597): in a merge check the
|
||||
// node-engine's check has the controller cloned beside it (MESH_CHECK_BESIDE), where it must be; else a
|
||||
// checkout beside this one; else it skips aloud.
|
||||
func TestTheControllersOwnManifestPassesStepNine(t *testing.T) {
|
||||
var manifest []byte
|
||||
if root := os.Getenv("MESH_CHECK_BESIDE"); root != "" {
|
||||
raw, err := os.ReadFile(filepath.Join(root, "mesh-controller", "module.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("the controller is not beside this check in MESH_CHECK_BESIDE=%s: %v", root, err)
|
||||
}
|
||||
manifest = raw
|
||||
} else if raw, err := os.ReadFile(filepath.Join("..", "..", "..", "mesh-controller", "module.json")); err == nil {
|
||||
manifest = raw
|
||||
} else {
|
||||
t.Skip("NOT JUDGED: no controller beside this checkout and no MESH_CHECK_BESIDE; the merge check judges it")
|
||||
}
|
||||
wanted, err := theStepNineSecretsOf(t, manifest)
|
||||
if err != nil {
|
||||
t.Fatalf("genesis's step 9 refuses the controller the mesh runs: %v", err)
|
||||
}
|
||||
if wanted["MESH_STORE_INVENTORY"] != "inventory" {
|
||||
t.Errorf("step 9 delivers %v", wanted)
|
||||
}
|
||||
}
|
||||
|
||||
+205
-27
@@ -4,7 +4,9 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
)
|
||||
@@ -165,13 +167,21 @@ const BusModule = "nats"
|
||||
// (ADR 0331 §1, §5). The installer's own placing of that list (enrol.go placeTheBusUsers) was the
|
||||
// bridge to here and is not run again.
|
||||
//
|
||||
// **What is compared is the container's name, not its image.** The store's module pins the upstream
|
||||
// postgres the foundation runs; the bus's module builds its server image (it carries the mesh's
|
||||
// snapshot tool), so no pinned reference can be equal to the foundation's. A module whose server is
|
||||
// not the foundation's container by name would raise a second bus beside the first, and is refused
|
||||
// by name before anything is registered.
|
||||
// **The bundle raises the bus where the module keeps it** (novox/hq ADR 0333 §1): the same container
|
||||
// name, the same streams directory, configuration directory, user list and certificate. That is checked
|
||||
// here against the module's own manifest before anything is registered, and a difference is refused
|
||||
// naming each one — under another name or on other paths the module would raise a second bus beside the
|
||||
// first, or an empty one in its place.
|
||||
//
|
||||
// **The image is the one thing that changes, once** (ADR 0333 §3). The store's module pins the upstream
|
||||
// postgres the foundation runs; the bus's module builds its server image (it carries the mesh's snapshot
|
||||
// tool), so no pinned reference can be equal to the foundation's. The applier recreates the container on
|
||||
// the module's image, on the same streams, users and certificate. This says so before the push and waits
|
||||
// after it until the container runs the module's image and the bus answers, so nothing after this step
|
||||
// dials a bus that is restarting.
|
||||
func InstallBroker(ctx context.Context, o Options, control controlPlane,
|
||||
foundation *declaration.Declaration, say func(string)) error {
|
||||
foundation *declaration.Declaration, fetch func(context.Context, string) (int, string, error),
|
||||
say func(string)) error {
|
||||
|
||||
const module = BusModule
|
||||
manifest, err := readManifest(o.Catalogue, module)
|
||||
@@ -182,12 +192,23 @@ func InstallBroker(ctx context.Context, o Options, control controlPlane,
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := busServerIsTheFoundations(manifest, bus, module); err != nil {
|
||||
if err := busLayoutIsTheModules(manifest, foundation, module); err != nil {
|
||||
return err
|
||||
}
|
||||
say(" adopting " + bus.Name + " — the bus the foundation raised, its streams kept")
|
||||
if err := installProvider(ctx, o, control, module, manifest, "(the server, adopting the foundation's bus)",
|
||||
nil, say); err != nil {
|
||||
func() error {
|
||||
if running := runningImage(ctx, control, bus.Name); running != "" && running != bus.Image {
|
||||
say(" already restarted " + bus.Name + " — it runs the module's image")
|
||||
return nil
|
||||
}
|
||||
say(" restarting " + bus.Name + " — once, onto the " + module + " module's image; its " +
|
||||
"streams, users and certificate stay where they are (novox/hq ADR 0333 §3)")
|
||||
return nil
|
||||
}, say); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := waitForTheAdoptedBus(ctx, o, control, bus, fetch, say); err != nil {
|
||||
return err
|
||||
}
|
||||
say(" adopted " + bus.Name + " — the foundation's bus is now the " + module +
|
||||
@@ -195,41 +216,190 @@ func InstallBroker(ctx context.Context, o Options, control controlPlane,
|
||||
return nil
|
||||
}
|
||||
|
||||
// busServerIsTheFoundations checks that the bus module's server container is the one the
|
||||
// foundation raised, by name, so the applier takes it over rather than raising a second bus.
|
||||
func busServerIsTheFoundations(manifest []byte, bus *declaration.Container, module string) error {
|
||||
// busMonitor is where the bus says it is healthy: its monitoring port, published on the machine's own
|
||||
// loopback by the bundle and by the module alike (both "127.0.0.1:8222:8222"). /healthz answers ok only
|
||||
// once the server is up and JetStream has its streams back.
|
||||
const busMonitor = "http://127.0.0.1:8222/healthz"
|
||||
|
||||
// runningImage is the image the named container was created from, or "" when there is none.
|
||||
func runningImage(ctx context.Context, control controlPlane, name string) string {
|
||||
asking, cancel := context.WithTimeout(ctx, control.timeout)
|
||||
defer cancel()
|
||||
out, err := control.run(asking, "docker", "inspect", "--format", "{{.Config.Image}}", name)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(out)
|
||||
}
|
||||
|
||||
// waitForTheAdoptedBus waits until the bus container runs an image other than the one the bundle raised
|
||||
// it from — the module's, after the applier's one recreate — and the bus answers on its monitor.
|
||||
//
|
||||
// Bounded by --wait, and asked at least once. A bus still on the bundle's image when it runs out is
|
||||
// refused, naming what it runs: the push was sent and the node has not applied it, and every step after
|
||||
// this one reaches the mesh through that bus.
|
||||
func waitForTheAdoptedBus(ctx context.Context, o Options, control controlPlane, bus *declaration.Container,
|
||||
fetch func(context.Context, string) (int, string, error), say func(string)) error {
|
||||
|
||||
deadline := time.Now().Add(o.Wait)
|
||||
var running, said string
|
||||
for {
|
||||
running = runningImage(ctx, control, bus.Name)
|
||||
if running != "" && running != bus.Image {
|
||||
code, body, err := fetch(ctx, busMonitor)
|
||||
if err == nil && code == http.StatusOK {
|
||||
say(" bus answers " + bus.Name + " on " + running)
|
||||
return nil
|
||||
}
|
||||
said = strings.TrimSpace(body)
|
||||
if err != nil {
|
||||
said = err.Error()
|
||||
}
|
||||
}
|
||||
if !time.Now().Before(deadline) {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(answerEvery):
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case running == "":
|
||||
return fmt.Errorf("the bus container %s is not there after the %s module was pushed, so nothing "+
|
||||
"after this step could reach the mesh. The node-engine's journal says why it did not raise it",
|
||||
bus.Name, BusModule)
|
||||
case running == bus.Image:
|
||||
return fmt.Errorf("%s still runs the bundle's image %s %s after the %s module was pushed: the node "+
|
||||
"has not applied it. Its streams are untouched; run this installer again once the node-engine "+
|
||||
"applies (its journal says why it did not)", bus.Name, bus.Image, o.Wait, BusModule)
|
||||
}
|
||||
return fmt.Errorf("%s runs the %s module's image %s and its monitor (%s) did not answer ok within %s: %s",
|
||||
bus.Name, BusModule, running, busMonitor, o.Wait, orUnknown(said))
|
||||
}
|
||||
|
||||
// BusUsersID is the bundle's resource for the bus's first user list.
|
||||
const BusUsersID = "bus-accounts"
|
||||
|
||||
// busLayout is where a bus lives: its container, the machine's directories it mounts as its streams,
|
||||
// its configuration and its certificate, and the file its user list is.
|
||||
type busLayout struct {
|
||||
Container, Streams, Conf, Certificate, Users string
|
||||
}
|
||||
|
||||
// Where the server reads each, inside its container: the same in the bundle and the module, since both
|
||||
// run the same server with the same configuration.
|
||||
const (
|
||||
busStreamsInside = "/data"
|
||||
busConfInside = "/etc/nats"
|
||||
busCertificateInside = "/tls"
|
||||
)
|
||||
|
||||
// foundationBusLayout reads the layout of the bus the bundle raises.
|
||||
func foundationBusLayout(foundation *declaration.Declaration) (busLayout, error) {
|
||||
bus, err := brokerIn(foundation)
|
||||
if err != nil {
|
||||
return busLayout{}, err
|
||||
}
|
||||
at := mountedFrom(bus.Volumes)
|
||||
l := busLayout{Container: bus.Name, Streams: at[busStreamsInside], Conf: at[busConfInside],
|
||||
Certificate: at[busCertificateInside]}
|
||||
for _, r := range foundation.Resources {
|
||||
if f, ok := r.(*declaration.File); ok && r.Identity() == BusUsersID {
|
||||
l.Users = f.Path
|
||||
}
|
||||
}
|
||||
return l, nil
|
||||
}
|
||||
|
||||
// moduleBusLayout reads the layout of the bus the module declares: its server container, the mounts of
|
||||
// that container with every `${access:<id>}` resolved to the access's path, and its `bus-users`.
|
||||
func moduleBusLayout(manifest []byte, module string) (busLayout, error) {
|
||||
var m struct {
|
||||
BusUsers string `json:"bus-users"`
|
||||
Resources []struct {
|
||||
Type string `json:"type"`
|
||||
Name string `json:"name"`
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Name string `json:"name"`
|
||||
Volumes []string `json:"volumes"`
|
||||
} `json:"resources"`
|
||||
Accesses []struct {
|
||||
ID string `json:"id"`
|
||||
Path string `json:"path"`
|
||||
} `json:"accesses"`
|
||||
}
|
||||
if err := json.Unmarshal(manifest, &m); err != nil {
|
||||
return fmt.Errorf("the %s module's manifest is not readable: %w", module, err)
|
||||
return busLayout{}, fmt.Errorf("the %s module's manifest is not readable: %w", module, err)
|
||||
}
|
||||
var names []string
|
||||
for _, r := range m.Resources {
|
||||
if r.Type != "container" {
|
||||
continue
|
||||
var containers []int
|
||||
for i, r := range m.Resources {
|
||||
if r.Type == "container" {
|
||||
containers = append(containers, i)
|
||||
}
|
||||
if r.Name == bus.Name {
|
||||
return nil
|
||||
}
|
||||
if len(containers) != 1 {
|
||||
var names []string
|
||||
for _, i := range containers {
|
||||
names = append(names, m.Resources[i].Name)
|
||||
}
|
||||
names = append(names, r.Name)
|
||||
return busLayout{}, fmt.Errorf("the %s module declares %s; the bus it adopts is one server, so it "+
|
||||
"declares one container", module, describeNames(names))
|
||||
}
|
||||
server := m.Resources[containers[0]]
|
||||
resolved := make([]string, 0, len(server.Volumes))
|
||||
for _, v := range server.Volumes {
|
||||
for _, a := range m.Accesses {
|
||||
v = strings.ReplaceAll(v, "${access:"+a.ID+"}", a.Path)
|
||||
}
|
||||
resolved = append(resolved, v)
|
||||
}
|
||||
at := mountedFrom(resolved)
|
||||
return busLayout{Container: server.Name, Streams: at[busStreamsInside], Conf: at[busConfInside],
|
||||
Certificate: at[busCertificateInside], Users: m.BusUsers}, nil
|
||||
}
|
||||
|
||||
// busLayoutIsTheModules checks that the bus the bundle raised is where the module keeps it, field by
|
||||
// field, so the applier takes it over rather than raising a second bus beside it or an empty one in its
|
||||
// place (novox/hq ADR 0333 §1). Every difference is named, with both sides.
|
||||
func busLayoutIsTheModules(manifest []byte, foundation *declaration.Declaration, module string) error {
|
||||
theirs, err := moduleBusLayout(manifest, module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ours, err := foundationBusLayout(foundation)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var differ []string
|
||||
for _, f := range []struct{ what, module, bundle string }{
|
||||
{"the server container", theirs.Container, ours.Container},
|
||||
{"its streams (" + busStreamsInside + ")", theirs.Streams, ours.Streams},
|
||||
{"its configuration (" + busConfInside + ")", theirs.Conf, ours.Conf},
|
||||
{"its certificate (" + busCertificateInside + ")", theirs.Certificate, ours.Certificate},
|
||||
{"its user list", theirs.Users, ours.Users},
|
||||
} {
|
||||
if f.module == "" || f.module != f.bundle {
|
||||
differ = append(differ, fmt.Sprintf("%s: the module's is %q, the bundle's %q", f.what,
|
||||
f.module, f.bundle))
|
||||
}
|
||||
}
|
||||
if len(differ) == 0 {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf(
|
||||
"the %s module's server is %s, and the foundation's bus is the container %q.\n"+
|
||||
"The module adopts the foundation's bus in place, so its server has to carry the name the "+
|
||||
"foundation raised; under another name it would raise a second bus beside the first, and "+
|
||||
"the mesh would run two (novox/hq ADR 0331 §1). Nothing has been registered",
|
||||
module, describeNames(names), bus.Name)
|
||||
"the bus the foundation raised is not where the %s module keeps it:\n - %s\n"+
|
||||
"The module adopts the foundation's bus in place, so the bundle raises it with the module's "+
|
||||
"name and paths; otherwise adopting it would raise a second bus beside the first, or an empty "+
|
||||
"one on other paths (novox/hq ADR 0333 §1). Nothing has been registered",
|
||||
module, strings.Join(differ, "\n - "))
|
||||
}
|
||||
|
||||
func describeNames(names []string) string {
|
||||
if len(names) == 0 {
|
||||
return "no container at all"
|
||||
}
|
||||
return "the container " + strings.Join(names, ", ")
|
||||
return "the containers " + strings.Join(names, ", ")
|
||||
}
|
||||
|
||||
// InstallVault installs the vault as a foundation module (novox/hq ADR 0085, amended). Nothing to
|
||||
@@ -308,3 +478,11 @@ func serverMatchesFoundation(manifest []byte, store *declaration.Container, modu
|
||||
"store with. Its server container has to carry the name the foundation raised",
|
||||
module, store.Name)
|
||||
}
|
||||
|
||||
// orUnknown is s, or a word for nothing: trust.go held it, and this branch takes trust.go out.
|
||||
func orUnknown(s string) string {
|
||||
if strings.TrimSpace(s) == "" {
|
||||
return "(nothing)"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
@@ -2,10 +2,16 @@ package bootstrap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
)
|
||||
|
||||
// aCatalogue is a catalogue checkout holding these modules' manifests.
|
||||
@@ -24,32 +30,73 @@ func aCatalogue(t *testing.T, manifests map[string]string) string {
|
||||
return dir
|
||||
}
|
||||
|
||||
// aBusModule is the nats module's manifest as far as adoption reads it: a server container by name.
|
||||
func aBusModule(server string) string {
|
||||
// aBusModule is the nats module's manifest as far as adoption reads it: its server container, the
|
||||
// directories it mounts, its certificate through an access, and its user list — the shape of the
|
||||
// catalogue's modules/nats/module.json.
|
||||
func aBusModule(l busLayout) string {
|
||||
return `{"module": "nats", "version": "1",
|
||||
"claims": [{"name": "mesh-broker", "scope": "mesh"}],
|
||||
"bus-users": "/var/lib/nats-module/conf/accounts.conf",
|
||||
"resources": [{"id": "server", "type": "container", "name": "` + server + `", "ports": ["4222:4222"], "artifact": "server"}],
|
||||
"bus-users": "` + l.Users + `",
|
||||
"resources": [
|
||||
{"id": "server", "type": "container", "name": "` + l.Container + `", "ports": ["4222:4222", "127.0.0.1:8222:8222"],
|
||||
"volumes": ["` + l.Streams + `:/data", "` + l.Conf + `:/etc/nats:ro", "${access:tls}:/tls:ro"], "artifact": "server"}],
|
||||
"accesses": [{"id": "tls", "path": "` + l.Certificate + `", "mode": "read"}],
|
||||
"build": {"artifacts": [{"name": "server", "kind": "image", "from": "Dockerfile"}]}}`
|
||||
}
|
||||
|
||||
func busStep(t *testing.T, catalogue string) (*asked, error) {
|
||||
// theModulesLayout is where the catalogue's nats module keeps its bus (mesh-catalog modules/nats).
|
||||
var theModulesLayout = busLayout{Container: "mesh-broker-nats", Streams: "/var/lib/mesh-broker-nats",
|
||||
Conf: "/var/lib/nats-module/conf", Certificate: "/var/lib/mesh-broker-tls",
|
||||
Users: "/var/lib/nats-module/conf/accounts.conf"}
|
||||
|
||||
// theModulesImage is what the bus runs once the applier recreated it on the module's built image.
|
||||
const theModulesImage = "127.0.0.1:5000/nats@sha256:3333333333333333333333333333333333333333333333333333333333333333"
|
||||
|
||||
// aBusThatAdopts answers the broker step as a node that applied the push: the bus container runs the
|
||||
// module's image, and its monitor says ok.
|
||||
func aBusThatAdopts() (*asked, func(context.Context, string) (int, string, error)) {
|
||||
runtime := &asked{answer: aMeshThatAgrees(map[string]string{
|
||||
"inspect --format {{.Config.Image}} mesh-broker-nats": theModulesImage + "\n",
|
||||
})}
|
||||
return runtime, func(_ context.Context, url string) (int, string, error) {
|
||||
if url != busMonitor {
|
||||
return 0, "", fmt.Errorf("asked %s", url)
|
||||
}
|
||||
return http.StatusOK, `{"status":"ok"}`, nil
|
||||
}
|
||||
}
|
||||
|
||||
func busStepWith(t *testing.T, o Options, runtime *asked,
|
||||
fetch func(context.Context, string) (int, string, error)) ([]string, error) {
|
||||
t.Helper()
|
||||
rewritten, err := Rewrite(theRealBundle(t), held)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runtime := &asked{answer: aMeshThatAgrees(nil)}
|
||||
control := controlPlane{container: "mesh-controller", run: runtime.run, timeout: 0}
|
||||
err = InstallBroker(context.Background(), installing(t, catalogue), control, rewritten.Declaration, func(string) {})
|
||||
control := controlPlane{container: "mesh-controller", run: runtime.run, timeout: time.Second}
|
||||
var said []string
|
||||
err = InstallBroker(context.Background(), o, control, rewritten.Declaration, fetch,
|
||||
func(line string) { said = append(said, line) })
|
||||
return said, err
|
||||
}
|
||||
|
||||
func busStep(t *testing.T, catalogue string) (*asked, error) {
|
||||
t.Helper()
|
||||
runtime, fetch := aBusThatAdopts()
|
||||
o := installing(t, catalogue)
|
||||
o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"}
|
||||
_, err := busStepWith(t, o, runtime, fetch)
|
||||
return runtime, err
|
||||
}
|
||||
|
||||
// The broker step adopts the foundation's bus as the nats module (novox/hq ADR 0331 §1): the module
|
||||
// the mesh goes on managing, registered from the catalogue — never lavinmq, which left it.
|
||||
func TestTheBrokerStepAdoptsTheBusAsTheNatsModule(t *testing.T) {
|
||||
catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule("mesh-broker")})
|
||||
runtime, _ := busStep(t, catalogue)
|
||||
catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)})
|
||||
runtime, err := busStep(t, catalogue)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !runtime.ran("module add /nats-module.json") {
|
||||
t.Fatalf("the nats module was not registered: %v", runtime.commands)
|
||||
}
|
||||
@@ -74,15 +121,23 @@ func TestTheBrokerStepFailsForAModuleTheCatalogueDoesNotHold(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A bus module whose server is not the foundation's container by name would raise a second bus
|
||||
// beside the first; it is refused naming both, before anything is registered.
|
||||
func TestABusModuleWhoseServerIsNotTheFoundationsIsRefused(t *testing.T) {
|
||||
catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule("mesh-broker-nats")})
|
||||
// A bus module that keeps its bus anywhere else than the bundle raised it would raise a second bus
|
||||
// beside the first, or an empty one on other paths (novox/hq ADR 0333 §1). It is refused naming every
|
||||
// difference with both sides, before anything is registered. The layout is the one the bundle had
|
||||
// before ADR 0333, kept as a module's here: the refusal genesis stopped at, at step 18 (issue 590).
|
||||
func TestABusModuleWhoseLayoutIsNotTheFoundationsIsRefused(t *testing.T) {
|
||||
before := busLayout{Container: "mesh-broker", Streams: "mesh-broker-data", Conf: "/var/lib/mesh-bus-conf",
|
||||
Certificate: "mesh-broker-tls", Users: "/var/lib/mesh-bus-conf/accounts.conf"}
|
||||
catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(before)})
|
||||
runtime, err := busStep(t, catalogue)
|
||||
if err == nil {
|
||||
t.Fatal("a bus module with another server was adopted")
|
||||
t.Fatal("a bus module kept elsewhere was adopted")
|
||||
}
|
||||
for _, named := range []string{"mesh-broker-nats", `"mesh-broker"`} {
|
||||
for _, named := range []string{
|
||||
`"mesh-broker"`, `"mesh-broker-nats"`, `"mesh-broker-data"`, `"/var/lib/mesh-broker-nats"`,
|
||||
`"/var/lib/mesh-bus-conf"`, `"/var/lib/nats-module/conf"`, `"mesh-broker-tls"`, `"/var/lib/mesh-broker-tls"`,
|
||||
`"/var/lib/mesh-bus-conf/accounts.conf"`, `"/var/lib/nats-module/conf/accounts.conf"`,
|
||||
} {
|
||||
if !strings.Contains(err.Error(), named) {
|
||||
t.Errorf("the refusal does not name %s: %v", named, err)
|
||||
}
|
||||
@@ -91,3 +146,274 @@ func TestABusModuleWhoseServerIsNotTheFoundationsIsRefused(t *testing.T) {
|
||||
t.Errorf("something was registered: %v", runtime.commands)
|
||||
}
|
||||
}
|
||||
|
||||
// One field moved is refused as surely as all of them: a module whose streams are elsewhere would be
|
||||
// raised empty on the bus's own name.
|
||||
func TestABusModuleWithItsStreamsElsewhereIsRefused(t *testing.T) {
|
||||
moved := theModulesLayout
|
||||
moved.Streams = "/srv/nats"
|
||||
catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(moved)})
|
||||
runtime, err := busStep(t, catalogue)
|
||||
if err == nil || !strings.Contains(err.Error(), `"/srv/nats"`) || !strings.Contains(err.Error(), "streams") {
|
||||
t.Fatalf("a module with its streams elsewhere was not refused naming them: %v", err)
|
||||
}
|
||||
if strings.Contains(err.Error(), "certificate") {
|
||||
t.Errorf("the refusal names what agrees: %v", err)
|
||||
}
|
||||
if runtime.ran("module add") {
|
||||
t.Errorf("something was registered: %v", runtime.commands)
|
||||
}
|
||||
}
|
||||
|
||||
// The node's bus port is kept (novox/hq ADR 0333 §2): the module declares 4222, and the broker step
|
||||
// gives it the setting that maps that to the port the bundle published, 5671 unless given — before the
|
||||
// push, so the first declaration already names it.
|
||||
func TestTheBrokerStepGivesTheModuleTheNodesBusPort(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
given int
|
||||
want string
|
||||
}{{0, `{"ports":{"4222":5671}}`}, {5771, `{"ports":{"4222":5771}}`}} {
|
||||
catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)})
|
||||
runtime, fetch := aBusThatAdopts()
|
||||
o := installing(t, catalogue)
|
||||
o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"}
|
||||
o.Ports = FoundationPorts{Bus: c.given}
|
||||
said, err := busStepWith(t, o, runtime, fetch)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
set, pushed := -1, -1
|
||||
for i, command := range runtime.commands {
|
||||
if strings.Contains(command, "settings set nats /nats-settings.json --node anchor") && set < 0 {
|
||||
set = i
|
||||
}
|
||||
if strings.Contains(command, "push anchor") && pushed < 0 {
|
||||
pushed = i
|
||||
}
|
||||
}
|
||||
if set < 0 || pushed < 0 || set > pushed {
|
||||
t.Fatalf("bus port %d: the module's port setting was not given before the push: %v", c.given, runtime.commands)
|
||||
}
|
||||
if !strings.Contains(strings.Join(said, "\n"), "settings nats on anchor: "+c.want) {
|
||||
t.Errorf("bus port %d: the setting given is not %s:\n%s", c.given, c.want, strings.Join(said, "\n"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The one restart is said before it happens, and the step goes on only once the bus runs the module's
|
||||
// image and answers (novox/hq ADR 0333 §3).
|
||||
func TestTheBrokerStepSaysTheRestartAndWaitsForTheBusToAnswer(t *testing.T) {
|
||||
previous := answerEvery
|
||||
answerEvery = time.Millisecond
|
||||
defer func() { answerEvery = previous }()
|
||||
|
||||
catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)})
|
||||
rewritten, err := Rewrite(theRealBundle(t), held)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bundleImage := containerNamed(rewritten.Declaration, "mesh-broker-nats").Image
|
||||
|
||||
// The push is sent; the node takes a few passes to recreate the bus, and the server a few more to
|
||||
// answer.
|
||||
pushed, inspected, monitored := false, 0, 0
|
||||
runtime := &asked{}
|
||||
agrees := aMeshThatAgrees(nil)
|
||||
runtime.answer = func(name string, args []string) (string, error) {
|
||||
joined := strings.Join(args, " ")
|
||||
switch {
|
||||
case strings.Contains(joined, "push anchor"):
|
||||
pushed = true
|
||||
case strings.Contains(joined, "inspect --format {{.Config.Image}} mesh-broker-nats"):
|
||||
if !pushed {
|
||||
return bundleImage + "\n", nil
|
||||
}
|
||||
inspected++
|
||||
if inspected < 3 {
|
||||
return bundleImage + "\n", nil
|
||||
}
|
||||
return theModulesImage + "\n", nil
|
||||
}
|
||||
return agrees(name, args)
|
||||
}
|
||||
fetch := func(context.Context, string) (int, string, error) {
|
||||
monitored++
|
||||
if monitored < 3 {
|
||||
return http.StatusServiceUnavailable, `{"status":"unavailable","error":"JetStream stream catching up"}`, nil
|
||||
}
|
||||
return http.StatusOK, `{"status":"ok"}`, nil
|
||||
}
|
||||
o := installing(t, catalogue)
|
||||
o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"}
|
||||
o.Wait = 10 * time.Second
|
||||
said, err := busStepWith(t, o, runtime, fetch)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := strings.Join(said, "\n")
|
||||
restart := strings.Index(text, "restarting mesh-broker-nats — once")
|
||||
answers := strings.Index(text, "bus answers mesh-broker-nats on "+theModulesImage)
|
||||
adopted := strings.Index(text, "adopted mesh-broker-nats")
|
||||
if restart < 0 || answers < restart || adopted < answers {
|
||||
t.Fatalf("the step did not say the restart, then the bus answering, then the adoption:\n%s", text)
|
||||
}
|
||||
if inspected < 3 || monitored < 3 {
|
||||
t.Errorf("the step went on before the bus ran the module's image and answered (asked its image %d "+
|
||||
"times, its monitor %d)", inspected, monitored)
|
||||
}
|
||||
}
|
||||
|
||||
// A bus still on the bundle's image when the wait runs out is refused, naming the image: every step after
|
||||
// this one reaches the mesh through that bus, and the push was not applied.
|
||||
func TestABusTheNodeDidNotRecreateIsRefused(t *testing.T) {
|
||||
previous := answerEvery
|
||||
answerEvery = time.Millisecond
|
||||
defer func() { answerEvery = previous }()
|
||||
|
||||
catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)})
|
||||
rewritten, err := Rewrite(theRealBundle(t), held)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bundleImage := containerNamed(rewritten.Declaration, "mesh-broker-nats").Image
|
||||
runtime := &asked{answer: aMeshThatAgrees(map[string]string{
|
||||
"inspect --format {{.Config.Image}} mesh-broker-nats": bundleImage + "\n",
|
||||
})}
|
||||
fetch := func(context.Context, string) (int, string, error) { return http.StatusOK, `{"status":"ok"}`, nil }
|
||||
o := installing(t, catalogue)
|
||||
o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"}
|
||||
o.Wait = 20 * time.Millisecond
|
||||
_, err = busStepWith(t, o, runtime, fetch)
|
||||
if err == nil {
|
||||
t.Fatal("a bus the node never recreated was taken as adopted")
|
||||
}
|
||||
if !strings.Contains(err.Error(), bundleImage) || !strings.Contains(err.Error(), "not applied") {
|
||||
t.Errorf("the refusal does not say the bus still runs the bundle's image: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A re-run after the adoption finds the bus on the module's image already, and says so rather than
|
||||
// announcing a restart that does not happen.
|
||||
func TestARerunAfterTheAdoptionSaysTheBusIsTheModulesAlready(t *testing.T) {
|
||||
catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)})
|
||||
runtime, fetch := aBusThatAdopts()
|
||||
o := installing(t, catalogue)
|
||||
o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"}
|
||||
said, err := busStepWith(t, o, runtime, fetch)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := strings.Join(said, "\n")
|
||||
if strings.Contains(text, "restarting") || !strings.Contains(text, "already restarted mesh-broker-nats") {
|
||||
t.Errorf("a re-run announced a restart, or did not say the bus is the module's already:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
// theCataloguesBusModule is the catalogue's own nats manifest, never a copy of it (novox/hq ADR 0333,
|
||||
// "How it is checked"): in a merge check the clone the build seat puts beside this repository
|
||||
// (MESH_CHECK_BESIDE, the catalogue at its main), where it must be; elsewhere a checkout of the catalogue
|
||||
// beside this one. With neither, the test says so and skips — a skip is not a pass, and the merge check
|
||||
// never skips it.
|
||||
func theCataloguesBusModule(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
const inside = "mesh-catalog/modules/nats/module.json"
|
||||
if root := os.Getenv("MESH_CHECK_BESIDE"); root != "" {
|
||||
raw, err := os.ReadFile(filepath.Join(root, inside))
|
||||
if err != nil {
|
||||
t.Fatalf("the catalogue is not beside this check in MESH_CHECK_BESIDE=%s, so the bundle's bus "+
|
||||
"cannot be held to the module's: %v", root, err)
|
||||
}
|
||||
t.Logf("judged against %s as cloned beside this check", filepath.Join(root, inside))
|
||||
return raw
|
||||
}
|
||||
beside := filepath.Join("..", "..", "..", inside)
|
||||
raw, err := os.ReadFile(beside)
|
||||
if err != nil {
|
||||
t.Skipf("NOT JUDGED: no catalogue beside this checkout (%s) and no MESH_CHECK_BESIDE, so the bundle's "+
|
||||
"bus is not held to the nats module's manifest here; the merge check holds it", beside)
|
||||
}
|
||||
t.Logf("judged against the catalogue checked out at %s", beside)
|
||||
return raw
|
||||
}
|
||||
|
||||
// **The genesis bundle takes the nats module's layout** (novox/hq ADR 0333 §1), read from the module's
|
||||
// manifest: the bus's container, its streams, its configuration, its certificate and its user list are
|
||||
// where the module keeps them, so adopting the bus keeps it. Before ADR 0333 the bundle raised
|
||||
// `mesh-broker` on volumes the module never mounts, and genesis stopped at step 18 (issue 590).
|
||||
func TestTheBundleRaisesTheBusWhereTheCataloguesModuleKeepsIt(t *testing.T) {
|
||||
manifest := theCataloguesBusModule(t)
|
||||
rewritten, err := Rewrite(theRealBundle(t), held)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := busLayoutIsTheModules(manifest, rewritten.Declaration, BusModule); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// And the port the installer's setting maps is the one the module declares (ADR 0333 §2).
|
||||
var m struct {
|
||||
Resources []struct {
|
||||
Type string `json:"type"`
|
||||
Ports []string `json:"ports"`
|
||||
} `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(manifest, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared := false
|
||||
for _, r := range m.Resources {
|
||||
for _, p := range r.Ports {
|
||||
if p == fmt.Sprintf("%d:4222", busModulePort) {
|
||||
declared = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !declared {
|
||||
t.Errorf("the nats module does not publish its server on %d, the port the installer's setting maps "+
|
||||
"to the node's bus port", busModulePort)
|
||||
}
|
||||
}
|
||||
|
||||
// The bundle as the installer reads it agrees with the module's layout as this repository states it,
|
||||
// and with where the installer's bridge writes (ADR 0333 §4): the bridge places the users in the file the
|
||||
// bus reads, and makes the container that reads it re-read it.
|
||||
func TestTheBridgeWritesWhereTheBundleRaisesTheBus(t *testing.T) {
|
||||
rewritten, err := Rewrite(theRealBundle(t), held)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ours, err := foundationBusLayout(rewritten.Declaration)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ours != theModulesLayout {
|
||||
t.Errorf("the bundle raises the bus at %+v; the nats module keeps it at %+v", ours, theModulesLayout)
|
||||
}
|
||||
if ours.Users != busAccounts || ours.Container != busContainer {
|
||||
t.Errorf("the bridge writes %s and signals %s; the bundle's bus reads %s and is %s",
|
||||
busAccounts, busContainer, ours.Users, ours.Container)
|
||||
}
|
||||
if filepath.Dir(ours.Users) != ours.Conf {
|
||||
t.Errorf("the user list %s is not in the configuration directory %s the bus reads", ours.Users, ours.Conf)
|
||||
}
|
||||
}
|
||||
|
||||
// The bundle's bus configuration and user list are seeds (create-once): an installer run again after
|
||||
// the adoption applies the bundle again, and must not put the bundle's one user back over the users the
|
||||
// module writes there (ADR 0333 §4).
|
||||
func TestTheBundlesBusFilesAreSeeds(t *testing.T) {
|
||||
rewritten, err := Rewrite(theRealBundle(t), held)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seeds := map[string]bool{}
|
||||
for _, r := range rewritten.Declaration.Resources {
|
||||
if f, ok := r.(*declaration.File); ok && f.CreateOnce {
|
||||
seeds[f.Path] = true
|
||||
}
|
||||
}
|
||||
for _, path := range []string{busAccounts, theModulesLayout.Conf + "/nats.conf"} {
|
||||
if !seeds[path] {
|
||||
t.Errorf("the bundle writes %s over whatever is there; it is a seed, written only where nothing is", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -72,7 +72,7 @@ func TestGivenPortsMoveOnlyTheMachinesSide(t *testing.T) {
|
||||
if len(storeC.Ports) != 1 || storeC.Ports[0] != "5433:5432" {
|
||||
t.Errorf("the store publishes %v", storeC.Ports)
|
||||
}
|
||||
bus := containerNamed(r.Declaration, "mesh-broker")
|
||||
bus := containerNamed(r.Declaration, "mesh-broker-nats")
|
||||
if strings.Join(bus.Ports, " ") != "5771:4222 127.0.0.1:8222:8222" {
|
||||
t.Errorf("the bus publishes %v", bus.Ports)
|
||||
}
|
||||
|
||||
@@ -127,13 +127,6 @@ func Preflight(ctx context.Context, o Options, d Deps, say func(string)) ([]byte
|
||||
if err := RefuseAMachineInUse(ctx, o, d.Run, say); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The registry's trust is written at step 7, before the push (novox/hq issue 92). What can be
|
||||
// known about whether it will take, without writing anything, is asked now.
|
||||
if o.pivots() {
|
||||
if err := RuntimeWillTakeTheTrust(ctx, o, d.Run, say); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Can this machine reach what the bundle's images come from?
|
||||
//
|
||||
|
||||
@@ -1,317 +0,0 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
)
|
||||
|
||||
// The registry's trust, written by genesis before anything is pushed to it (novox/hq issue 92).
|
||||
//
|
||||
// The mesh's registry speaks plain HTTP on purpose: every path to it is inside the private
|
||||
// network's encryption (novox/hq ADR 0082). The container runtime refuses plain HTTP to a registry
|
||||
// it has not been told to trust, and the module that tells it — `docker`, which writes the
|
||||
// artifact store's reach into the runtime's configuration — is applied long after genesis pushes
|
||||
// the control plane's image. So genesis writes that trust itself, first, with the same writer the
|
||||
// module uses: into the runtime's file beside what is there (ADR 0102), the runtime reloaded and
|
||||
// never restarted. It then asks the runtime whether it will speak to every name, and refuses with
|
||||
// the runtime's answer before anything is pushed.
|
||||
//
|
||||
// **The handover.** What genesis writes is kept in no record of its own: no record of the foundation
|
||||
// could hold it, because the retire step re-applies the bundle and takes away a carried record the
|
||||
// bundle does not name. The `docker` module, applied later, declares the artifact store's reach —
|
||||
// the mesh name on the node's port — finds it already in the file it has no record of, and takes it
|
||||
// as its own (ADR 0102: a declared member found in an unrecorded file is the mesh's). That member
|
||||
// does not change at the handover. The module's first apply usually still changes the file and
|
||||
// reloads the runtime once, for the keys it writes beside it (`live-restore`, `log-driver`,
|
||||
// `log-opts`); a reload stops nothing. The address the operator gave, when it is not the mesh name,
|
||||
// stays as part of the machine's own configuration, as the hand-written trust it replaces would have.
|
||||
//
|
||||
// **This rests on an assumption about order.** The module takes the mesh name's member over only
|
||||
// because its FIRST apply on this node already declares it: genesis assigns no `docker` before the
|
||||
// network step (16), and once the node is on the network the controller composes the reach as
|
||||
// `<node>.<suffix>:<port>`.
|
||||
// Were `docker` applied earlier — while the reach is empty or `127.0.0.1:<port>`, as it is before the
|
||||
// network — its record would exist without the mesh name, and the member would stay the machine's
|
||||
// for good: undeclaring the module would never take it out. Installing the node's seat holders
|
||||
// earlier in genesis must revisit this.
|
||||
//
|
||||
// **The suffix** is read from this process's environment, and the controller reads its own: a mesh
|
||||
// with a custom `MESH_INTERNAL_SUFFIX` sets it for both.
|
||||
|
||||
// runtimeConfig is the container runtime's configuration, which the trust is written into. A
|
||||
// variable so a test writes into a file of its own.
|
||||
var runtimeConfig = "/etc/docker/daemon.json"
|
||||
|
||||
// runtimeUnit is the runtime's service, reloaded when the file changed.
|
||||
const runtimeUnit = "docker.service"
|
||||
|
||||
// internalSuffixVar and defaultInternalSuffix are what a node's mesh name ends in: the same variable
|
||||
// and default the controller composes `<node>.<suffix>` from (mesh-controller internal/overlay),
|
||||
// which is the name the artifact store's reach — and so every push by the mesh's builder — uses.
|
||||
const (
|
||||
internalSuffixVar = "MESH_INTERNAL_SUFFIX"
|
||||
defaultInternalSuffix = "internal"
|
||||
)
|
||||
|
||||
// RegistryNames are the names the mesh will use for its registry on this node, each as host:port.
|
||||
//
|
||||
// The address the operator gave, and the node's mesh name: the controller's reach for the artifact
|
||||
// store, which the builder pushes to without being told. Both on the port the registry module
|
||||
// serves on this node — the node's setting, never the module's default (novox/hq issue 088). A
|
||||
// loopback name is left out: the runtime trusts loopback without being told, and the mesh never
|
||||
// names it to another node.
|
||||
func RegistryNames(o Options) []string {
|
||||
port := strconv.Itoa(o.Ports.orDefaults().Registry)
|
||||
var names []string
|
||||
add := func(hostport string) {
|
||||
host, _, err := net.SplitHostPort(hostport)
|
||||
if err != nil || host == "" || isLoopback(host) {
|
||||
return
|
||||
}
|
||||
for _, n := range names {
|
||||
if n == hostport {
|
||||
return
|
||||
}
|
||||
}
|
||||
names = append(names, hostport)
|
||||
}
|
||||
if host, _, err := net.SplitHostPort(o.Registry); err == nil {
|
||||
add(net.JoinHostPort(strings.Trim(host, "[]"), port))
|
||||
}
|
||||
if node := strings.TrimSpace(o.Node); node != "" {
|
||||
add(net.JoinHostPort(node+"."+internalSuffix(), port))
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func internalSuffix() string {
|
||||
if v := strings.TrimSpace(os.Getenv(internalSuffixVar)); v != "" {
|
||||
return strings.TrimPrefix(v, ".")
|
||||
}
|
||||
return defaultInternalSuffix
|
||||
}
|
||||
|
||||
func isLoopback(host string) bool {
|
||||
if strings.EqualFold(host, "localhost") {
|
||||
return true
|
||||
}
|
||||
ip := net.ParseIP(strings.Trim(host, "[]"))
|
||||
return ip != nil && ip.IsLoopback()
|
||||
}
|
||||
|
||||
// trustDeclaration is the declaration genesis applies to write the trust: the same two resources
|
||||
// the `docker` module declares for it — the runtime's file written into, the runtime reloaded on it.
|
||||
func trustDeclaration(names []string) (*declaration.Declaration, error) {
|
||||
keys, err := json.Marshal(map[string][]string{"insecure-registries": names})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := json.Marshal(map[string]any{
|
||||
"declaration": 1,
|
||||
"resources": []map[string]any{
|
||||
{"id": "genesis.registry-trust", "type": "file", "path": runtimeConfig, "mode": "0644",
|
||||
"into": "json", "content": string(keys) + "\n"},
|
||||
{"id": "genesis.registry-trust-reload", "type": "service", "unit": runtimeUnit,
|
||||
"state": "running", "reload-on": []string{"genesis.registry-trust"}},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return declaration.ParseTrusted(raw)
|
||||
}
|
||||
|
||||
// TrustTheRegistry writes the runtime's trust of every name the mesh serves its registry on here,
|
||||
// then refuses unless the runtime says it will speak plain HTTP to each.
|
||||
func TrustTheRegistry(ctx context.Context, o Options, sys system.System, run Runner,
|
||||
say func(string)) ([]string, error) {
|
||||
|
||||
names := RegistryNames(o)
|
||||
if len(names) == 0 {
|
||||
say(" trusted only loopback names this registry, which the runtime trusts already")
|
||||
return nil, nil
|
||||
}
|
||||
d, err := trustDeclaration(names)
|
||||
if err != nil {
|
||||
return names, err
|
||||
}
|
||||
|
||||
// One apply at a time on this machine, as ApplyBundle does: a host already running here may be
|
||||
// writing the same file.
|
||||
unlock, err := store.Lock(o.State, func() { say(" waiting another apply holds this node's state") })
|
||||
if err != nil {
|
||||
return names, err
|
||||
}
|
||||
report, _, err := apply.Apply(ctx, sys, d, store.State{}, store.OriginCarried, run,
|
||||
func(line string) { say(" " + strings.TrimPrefix(line, " ")) }, refuseSealed)
|
||||
unlock()
|
||||
if err != nil {
|
||||
return names, fmt.Errorf("the registry's trust could not be written into %s: %w\n"+
|
||||
"The registry is plain HTTP over the private network (novox/hq ADR 0082), and the container "+
|
||||
"runtime refuses to push to it until it trusts it. Nothing has been pushed", runtimeConfig, err)
|
||||
}
|
||||
if report.Changed() {
|
||||
say(" trusted " + strings.Join(names, ", ") + " — written into " + runtimeConfig +
|
||||
", the runtime reloaded")
|
||||
} else {
|
||||
say(" trusted " + strings.Join(names, ", ") + " — " + runtimeConfig + " already says so")
|
||||
}
|
||||
return names, runtimeWillSpeakTo(ctx, o, run, names)
|
||||
}
|
||||
|
||||
// runtimeWillSpeakTo asks the runtime, until it says it trusts every name or the wait runs out.
|
||||
//
|
||||
// The file being right is not the runtime trusting it: a reload the runtime refused, or a flag on
|
||||
// its command line that contradicts the file, leaves the push to fail at step 8 with an HTTPS error.
|
||||
func runtimeWillSpeakTo(ctx context.Context, o Options, run Runner, names []string) error {
|
||||
deadline := time.Now().Add(o.Wait)
|
||||
var missing []string
|
||||
var last error
|
||||
for {
|
||||
asking, cancel := context.WithTimeout(ctx, o.Timeout)
|
||||
out, err := run(asking, "docker", "info", "--format", "{{json .RegistryConfig}}")
|
||||
cancel()
|
||||
last = err
|
||||
if err == nil {
|
||||
missing, last = untrusted(out, names)
|
||||
if last == nil && len(missing) == 0 {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(answerEvery):
|
||||
}
|
||||
}
|
||||
if last != nil {
|
||||
return fmt.Errorf("the container runtime could not be asked whether it trusts the registry: %w", last)
|
||||
}
|
||||
return fmt.Errorf(
|
||||
"the container runtime does not trust %s after %s says it should, so it would refuse to push to "+
|
||||
"it.\nThe file was written and the runtime reloaded; a runtime that still refuses was started "+
|
||||
"with its own registry options, or did not take the reload — `journalctl -u %s` says which. "+
|
||||
"Nothing has been pushed", strings.Join(missing, ", "), runtimeConfig, runtimeUnit)
|
||||
}
|
||||
|
||||
// untrusted reads the runtime's registry configuration and gives back the names it would speak
|
||||
// HTTPS to: a name is trusted when the runtime lists it as not secure, or when its address falls in
|
||||
// a range the runtime trusts.
|
||||
func untrusted(registryConfig string, names []string) ([]string, error) {
|
||||
var c struct {
|
||||
InsecureRegistryCIDRs []string `json:"InsecureRegistryCIDRs"`
|
||||
IndexConfigs map[string]struct {
|
||||
Secure bool `json:"Secure"`
|
||||
} `json:"IndexConfigs"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(strings.TrimSpace(registryConfig)), &c); err != nil {
|
||||
return nil, fmt.Errorf("the runtime's registry configuration is not readable: %w", err)
|
||||
}
|
||||
var missing []string
|
||||
for _, name := range names {
|
||||
if index, ok := c.IndexConfigs[name]; ok && !index.Secure {
|
||||
continue
|
||||
}
|
||||
host, _, _ := net.SplitHostPort(name)
|
||||
if ip := net.ParseIP(strings.Trim(host, "[]")); ip != nil && inAny(ip, c.InsecureRegistryCIDRs) {
|
||||
continue
|
||||
}
|
||||
missing = append(missing, name)
|
||||
}
|
||||
return missing, nil
|
||||
}
|
||||
|
||||
func inAny(ip net.IP, cidrs []string) bool {
|
||||
for _, cidr := range cidrs {
|
||||
if _, n, err := net.ParseCIDR(cidr); err == nil && n.Contains(ip) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// procRoot is where a process's command line is read from. A variable so a test gives its own.
|
||||
var procRoot = "/proc"
|
||||
|
||||
// RuntimeWillTakeTheTrust is preflight's half of the registry's trust: everything that can be known
|
||||
// about it without writing anything, asked at step 1 so a machine that cannot take the trust is
|
||||
// refused before the foundation is raised, rather than at step 7 after it (novox/hq issue 92).
|
||||
//
|
||||
// - The runtime's file is absent or a JSON object: the mesh writes into an object, and refuses to
|
||||
// write over anything else (ADR 0102).
|
||||
// - The runtime's service is known to systemd and active: the trust is taken by reloading it.
|
||||
// - The runtime was not started with `--insecure-registry`: dockerd refuses a directive given both
|
||||
// as a flag and in its file, so it would refuse the reload and keep distrusting the registry.
|
||||
func RuntimeWillTakeTheTrust(ctx context.Context, o Options, run Runner, say func(string)) error {
|
||||
names := RegistryNames(o)
|
||||
if len(names) == 0 {
|
||||
return nil
|
||||
}
|
||||
if raw, err := os.ReadFile(runtimeConfig); err == nil {
|
||||
var object map[string]json.RawMessage
|
||||
if len(strings.TrimSpace(string(raw))) > 0 {
|
||||
if err := json.Unmarshal(raw, &object); err != nil || object == nil {
|
||||
return fmt.Errorf("%s is not a JSON object, and the mesh writes the trust of its registry "+
|
||||
"(%s) into it beside what is there, never over it (novox/hq ADR 0102). Make it an object "+
|
||||
"or remove it; nothing has been changed", runtimeConfig, strings.Join(names, ", "))
|
||||
}
|
||||
}
|
||||
} else if !os.IsNotExist(err) {
|
||||
return fmt.Errorf("the runtime's configuration %s cannot be read: %w", runtimeConfig, err)
|
||||
}
|
||||
|
||||
asking, cancel := context.WithTimeout(ctx, o.Timeout)
|
||||
out, err := run(asking, "systemctl", "show", runtimeUnit, "--property=LoadState,ActiveState,MainPID")
|
||||
cancel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("systemd could not be asked about %s: %w\nThe registry's trust is taken by "+
|
||||
"reloading it; nothing has been changed", runtimeUnit, err)
|
||||
}
|
||||
props := map[string]string{}
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
if k, v, ok := strings.Cut(strings.TrimSpace(line), "="); ok {
|
||||
props[k] = v
|
||||
}
|
||||
}
|
||||
if props["LoadState"] != "loaded" || props["ActiveState"] != "active" {
|
||||
return fmt.Errorf("%s is %s and %s, and the registry's trust (%s) is taken by reloading it. "+
|
||||
"Genesis needs the runtime run by systemd as %s; nothing has been changed",
|
||||
runtimeUnit, orUnknown(props["LoadState"]), orUnknown(props["ActiveState"]),
|
||||
strings.Join(names, ", "), runtimeUnit)
|
||||
}
|
||||
if pid := props["MainPID"]; pid != "" && pid != "0" {
|
||||
if cmdline, err := os.ReadFile(procRoot + "/" + pid + "/cmdline"); err == nil {
|
||||
for _, arg := range strings.Split(string(cmdline), "\x00") {
|
||||
if strings.HasPrefix(arg, "--insecure-registry") {
|
||||
return fmt.Errorf("the runtime was started with %s, and dockerd refuses insecure registries "+
|
||||
"given both as a flag and in %s — so it would refuse the reload that makes it trust "+
|
||||
"%s. Move the flag's registries into the file and restart the runtime once; nothing "+
|
||||
"has been changed", arg, runtimeConfig, strings.Join(names, ", "))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
say(" registry trust " + runtimeConfig + " can be written into and " + runtimeUnit + " reloaded")
|
||||
return nil
|
||||
}
|
||||
|
||||
func orUnknown(s string) string {
|
||||
if s == "" {
|
||||
return "unknown"
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -1,347 +0,0 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
)
|
||||
|
||||
// novox/hq issue 92: genesis pushed the control plane's image to its registry before anything had
|
||||
// told the container runtime to trust that registry, so the push failed with "server gave HTTP
|
||||
// response to HTTPS client" — first by the address given, then by the mesh name the builder used,
|
||||
// and the trust the mesh wrote later named the registry's default port rather than the node's.
|
||||
|
||||
// aRuntime is a container runtime on systemd whose registry configuration is what its file said at
|
||||
// its last reload: what a real dockerd does, and why a written file is not yet a trusted registry.
|
||||
type aRuntime struct {
|
||||
path string
|
||||
trusted []string
|
||||
commands []string
|
||||
// ignoresReload is a runtime started with options of its own, which a reload does not change.
|
||||
ignoresReload bool
|
||||
}
|
||||
|
||||
func (r *aRuntime) run(_ context.Context, name string, args ...string) (string, error) {
|
||||
line := name + " " + strings.Join(args, " ")
|
||||
r.commands = append(r.commands, line)
|
||||
switch {
|
||||
case name == "docker" && len(args) > 0 && args[0] == "info":
|
||||
index := map[string]map[string]any{"docker.io": {"Secure": true}}
|
||||
for _, n := range r.trusted {
|
||||
index[n] = map[string]any{"Name": n, "Secure": false}
|
||||
}
|
||||
out, _ := json.Marshal(map[string]any{"InsecureRegistryCIDRs": []string{"127.0.0.0/8"}, "IndexConfigs": index})
|
||||
return string(out), nil
|
||||
case name == "systemctl" && strings.Contains(line, "reload"):
|
||||
if !r.ignoresReload {
|
||||
r.trusted = r.trusted[:0]
|
||||
raw, _ := os.ReadFile(r.path)
|
||||
var c struct {
|
||||
Insecure []string `json:"insecure-registries"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &c)
|
||||
r.trusted = append(r.trusted, c.Insecure...)
|
||||
}
|
||||
case strings.Contains(line, "is-enabled"):
|
||||
return "enabled", nil
|
||||
case strings.Contains(line, "show") && strings.Contains(line, "ActiveState"):
|
||||
return "LoadState=loaded\nActiveState=active\nSubState=running", nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (r *aRuntime) did(word string) int {
|
||||
n := 0
|
||||
for _, c := range r.commands {
|
||||
if strings.Contains(c, "systemctl") && strings.Contains(c, " "+word+" ") || strings.HasSuffix(c, " "+word) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func trustSetup(t *testing.T, existing string) (Options, *aRuntime, system.System) {
|
||||
t.Helper()
|
||||
restore := apply.ForTests(t.TempDir(), t.TempDir())
|
||||
t.Cleanup(restore)
|
||||
dir := t.TempDir()
|
||||
was := runtimeConfig
|
||||
runtimeConfig = filepath.Join(dir, "daemon.json")
|
||||
t.Cleanup(func() { runtimeConfig = was })
|
||||
if existing != "" {
|
||||
if err := os.WriteFile(runtimeConfig, []byte(existing), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
t.Setenv(internalSuffixVar, "")
|
||||
sys, err := system.For("arch")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
o := Options{Node: "anchor", Registry: "192.0.2.10:5100", Ports: FoundationPorts{Registry: 5100},
|
||||
State: filepath.Join(dir, "state", "state.json"), Timeout: time.Second, Wait: 0}
|
||||
return o, &aRuntime{path: runtimeConfig}, sys
|
||||
}
|
||||
|
||||
func insecureIn(t *testing.T) (string, map[string]json.RawMessage) {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(runtimeConfig)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var o map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &o); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var list []string
|
||||
_ = json.Unmarshal(o["insecure-registries"], &list)
|
||||
return fmt.Sprint(list), o
|
||||
}
|
||||
|
||||
func TestGenesisTrustsTheRegistryByEveryNameBeforeItPushes(t *testing.T) {
|
||||
o, runtime, sys := trustSetup(t, `{"data-root":"/srv/docker","insecure-registries":["198.51.100.7:5000"]}`)
|
||||
|
||||
names, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The address given and the mesh name the builder pushes by, both on the node's port.
|
||||
if fmt.Sprint(names) != "[192.0.2.10:5100 anchor.internal:5100]" {
|
||||
t.Errorf("trusted by %v", names)
|
||||
}
|
||||
list, object := insecureIn(t)
|
||||
if list != "[198.51.100.7:5000 192.0.2.10:5100 anchor.internal:5100]" {
|
||||
t.Errorf("the runtime's list is %s", list)
|
||||
}
|
||||
if string(object["data-root"]) != `"/srv/docker"` {
|
||||
t.Errorf("the machine's own key was not kept: %s", object["data-root"])
|
||||
}
|
||||
if runtime.did("reload") != 1 || runtime.did("restart") != 0 || runtime.did("stop") != 0 {
|
||||
t.Errorf("the runtime was not reloaded exactly once and never restarted: %v", runtime.commands)
|
||||
}
|
||||
|
||||
// Re-run: nothing changes, nothing is reloaded.
|
||||
runtime.commands = nil
|
||||
if _, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if runtime.did("reload") != 0 {
|
||||
t.Errorf("a re-run reloaded the runtime: %v", runtime.commands)
|
||||
}
|
||||
}
|
||||
|
||||
// The port is the node's setting for the registry, not the module's default, whatever the operator
|
||||
// typed the address with — and loopback, which the runtime trusts already, is not written.
|
||||
func TestTheRegistrysTrustNamesThePortTheNodeServesItOn(t *testing.T) {
|
||||
o := Options{Node: "anchor", Registry: "127.0.0.1:5100", Ports: FoundationPorts{Registry: 5100}}
|
||||
t.Setenv(internalSuffixVar, "")
|
||||
if got := fmt.Sprint(RegistryNames(o)); got != "[anchor.internal:5100]" {
|
||||
t.Errorf("names %s", got)
|
||||
}
|
||||
o.Ports = FoundationPorts{}
|
||||
o.Registry = "192.0.2.10:5000"
|
||||
t.Setenv(internalSuffixVar, ".mesh.example")
|
||||
if got := fmt.Sprint(RegistryNames(o)); got != "[192.0.2.10:5000 anchor.mesh.example:5000]" {
|
||||
t.Errorf("names %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A runtime that still refuses after the write is refused here, with the names, before any push.
|
||||
func TestGenesisRefusesARuntimeThatStillDoesNotTrustTheRegistry(t *testing.T) {
|
||||
o, runtime, sys := trustSetup(t, "")
|
||||
runtime.ignoresReload = true
|
||||
_, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {})
|
||||
if err == nil {
|
||||
t.Fatal("a runtime that does not trust the registry was let through to the push")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "192.0.2.10:5100, anchor.internal:5100") ||
|
||||
!strings.Contains(err.Error(), "Nothing has been pushed") {
|
||||
t.Errorf("the refusal does not say what: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The handover: the `docker` module, applied later, writes the reach the controller composes for
|
||||
// the artifact store — the mesh name and the node's port. It finds it there: the file does not
|
||||
// change, the runtime is not reloaded, and the member it declares is recorded as the module's from
|
||||
// then on (a member found in a file with no record of it is the mesh's, ADR 0102), so the module
|
||||
// owns the trust it writes and taking the module away takes that member out.
|
||||
func TestTheDockerModuleTakesTheTrustOverWithoutAChange(t *testing.T) {
|
||||
o, runtime, sys := trustSetup(t, `{"live-restore":true}`)
|
||||
if _, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before, _ := os.ReadFile(runtimeConfig)
|
||||
|
||||
module, err := declaration.ParseTrusted([]byte(fmt.Sprintf(`{"declaration":1,"resources":[
|
||||
{"id":"docker.daemon","type":"file","path":%q,"mode":"0644","into":"json",
|
||||
"content":"{\"live-restore\": true, \"insecure-registries\": [\"anchor.internal:5100\"]}\n"},
|
||||
{"id":"docker.runtime","type":"service","unit":"docker.service","state":"running","boot":"enabled",
|
||||
"reload-on":["docker.daemon"]}]}`, runtimeConfig)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runtime.commands = nil
|
||||
_, state, err := apply.Apply(context.Background(), sys, module, store.State{}, store.OriginDeclared,
|
||||
runtime.run, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after, _ := os.ReadFile(runtimeConfig)
|
||||
if string(before) != string(after) {
|
||||
t.Errorf("the handover changed the file:\n%s\n%s", before, after)
|
||||
}
|
||||
if runtime.did("reload") != 0 || runtime.did("restart") != 0 {
|
||||
t.Errorf("the handover touched the runtime: %v", runtime.commands)
|
||||
}
|
||||
if rec, _ := state.Find("docker.daemon"); rec.Into == nil || len(rec.Into.Added["insecure-registries"]) != 1 {
|
||||
t.Errorf("the module did not take the mesh name's member over: %+v", rec.Into)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARangeTheRuntimeTrustsCoversAnAddressInIt(t *testing.T) {
|
||||
missing, err := untrusted(`{"InsecureRegistryCIDRs":["10.42.0.0/16"],"IndexConfigs":{}}`,
|
||||
[]string{"10.42.0.1:5000", "anchor.internal:5000"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fmt.Sprint(missing) != "[anchor.internal:5000]" {
|
||||
t.Errorf("missing %v", missing)
|
||||
}
|
||||
}
|
||||
|
||||
// **The order, which is the fault issue 92 is about.** Steps 7 and 8 against a runtime that behaves
|
||||
// as dockerd does: a push to a registry it has not been told to trust is refused with the error the
|
||||
// first genesis met. The trust must be written and confirmed before anything is tagged or pushed;
|
||||
// moving the trust after the push, or dropping it, fails this test.
|
||||
func TestNothingIsPushedToTheRegistryBeforeTheRuntimeTrustsIt(t *testing.T) {
|
||||
o, runtime, sys := trustSetup(t, `{"data-root":"/srv/docker"}`)
|
||||
o.Catalogue = catalogueWith(t, RegistryModule, upstreamRegistryManifest)
|
||||
o.Registry = "192.0.2.10:5100"
|
||||
|
||||
pushed := false
|
||||
mesh := aMeshThatAgrees(nil)
|
||||
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
||||
if name == "systemctl" || (name == "docker" && args[0] == "info") {
|
||||
return runtime.run(ctx, name, args...)
|
||||
}
|
||||
runtime.commands = append(runtime.commands, name+" "+strings.Join(args, " "))
|
||||
if name == "docker" {
|
||||
switch args[0] {
|
||||
case "tag":
|
||||
return "", nil
|
||||
case "push":
|
||||
trusted := false
|
||||
for _, n := range runtime.trusted {
|
||||
trusted = trusted || strings.HasPrefix(args[1], n+"/")
|
||||
}
|
||||
if !trusted {
|
||||
return "", fmt.Errorf(`Get "https://%s/v2/": http: server gave HTTP response to HTTPS client`, o.Registry)
|
||||
}
|
||||
pushed = true
|
||||
return "", nil
|
||||
case "image":
|
||||
return `["` + o.Registry + `/mesh-controller@sha256:` + strings.Repeat("a", 64) + `"]`, nil
|
||||
}
|
||||
}
|
||||
return mesh(name, args)
|
||||
}
|
||||
d := Deps{Run: run, Fetch: func(_ context.Context, url string) (int, string, error) {
|
||||
if strings.HasSuffix(url, "/v2/") || pushed {
|
||||
return http.StatusOK, `{"name":"mesh-controller","tags":["genesis"]}`, nil
|
||||
}
|
||||
return http.StatusNotFound, "", nil
|
||||
}}
|
||||
var result Result
|
||||
published, err := RegistryAndPublish(context.Background(), o, d, sys,
|
||||
controlPlane{container: "temp-mesh-controller", run: run, timeout: time.Second}, held, &result, func(string) {})
|
||||
if err != nil {
|
||||
t.Fatalf("genesis did not get past the push: %v\ncommands: %v", err, runtime.commands)
|
||||
}
|
||||
if !strings.HasPrefix(published.Reference, o.Registry+"/mesh-controller@sha256:") {
|
||||
t.Errorf("published as %q", published.Reference)
|
||||
}
|
||||
|
||||
firstPush, reloaded, confirmed := -1, -1, -1
|
||||
for i, c := range runtime.commands {
|
||||
switch {
|
||||
case firstPush < 0 && (strings.HasPrefix(c, "docker tag") || strings.HasPrefix(c, "docker push")):
|
||||
firstPush = i
|
||||
case strings.HasPrefix(c, "systemctl") && strings.Contains(c, "reload"):
|
||||
reloaded = i
|
||||
case strings.HasPrefix(c, "docker info"):
|
||||
confirmed = i
|
||||
}
|
||||
}
|
||||
if firstPush < 0 || reloaded < 0 || confirmed < 0 || !(reloaded < confirmed && confirmed < firstPush) {
|
||||
t.Errorf("the trust was not written (%d) and confirmed (%d) before the first tag or push (%d): %v",
|
||||
reloaded, confirmed, firstPush, runtime.commands)
|
||||
}
|
||||
if fmt.Sprint(result.RegistryTrusted) != "[192.0.2.10:5100 anchor.internal:5100]" {
|
||||
t.Errorf("the result says it trusted %v", result.RegistryTrusted)
|
||||
}
|
||||
}
|
||||
|
||||
// Preflight asks, without writing, what would stop the trust from taking at step 7.
|
||||
func TestPreflightRefusesARuntimeThatCouldNotTakeTheTrust(t *testing.T) {
|
||||
proc := t.TempDir()
|
||||
was := procRoot
|
||||
procRoot = proc
|
||||
t.Cleanup(func() { procRoot = was })
|
||||
if err := os.MkdirAll(filepath.Join(proc, "42"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
unit := func(load, active string) Runner {
|
||||
return func(_ context.Context, name string, args ...string) (string, error) {
|
||||
return "LoadState=" + load + "\nActiveState=" + active + "\nMainPID=42\n", nil
|
||||
}
|
||||
}
|
||||
cmdline := func(args ...string) {
|
||||
if err := os.WriteFile(filepath.Join(proc, "42", "cmdline"), []byte(strings.Join(args, "\x00")+"\x00"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
cmdline("/usr/bin/dockerd", "-H", "fd://")
|
||||
|
||||
for _, c := range []struct {
|
||||
name, file string
|
||||
run Runner
|
||||
flag bool
|
||||
refused string
|
||||
}{
|
||||
{"a runtime that can take it", `{"data-root":"/srv/docker"}`, unit("loaded", "active"), false, ""},
|
||||
{"no file yet", "", unit("loaded", "active"), false, ""},
|
||||
{"a file that is not an object", `["not", "an", "object"]`, unit("loaded", "active"), false, "not a JSON object"},
|
||||
{"a runtime systemd does not run", `{}`, unit("not-found", "inactive"), false, "not-found and inactive"},
|
||||
{"a runtime with the flag", `{}`, unit("loaded", "active"), true, "--insecure-registry=10.0.0.1:5000"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
o, _, _ := trustSetup(t, c.file)
|
||||
if c.flag {
|
||||
cmdline("/usr/bin/dockerd", "--insecure-registry=10.0.0.1:5000")
|
||||
defer cmdline("/usr/bin/dockerd", "-H", "fd://")
|
||||
}
|
||||
before, _ := os.ReadFile(runtimeConfig)
|
||||
err := RuntimeWillTakeTheTrust(context.Background(), o, c.run, func(string) {})
|
||||
switch {
|
||||
case c.refused == "" && err != nil:
|
||||
t.Errorf("refused: %v", err)
|
||||
case c.refused != "" && (err == nil || !strings.Contains(err.Error(), c.refused)):
|
||||
t.Errorf("wanted a refusal naming %q, got %v", c.refused, err)
|
||||
}
|
||||
if after, _ := os.ReadFile(runtimeConfig); string(after) != string(before) {
|
||||
t.Error("preflight wrote the runtime's file")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -89,7 +89,7 @@ func TestAFoundationThatIsUpAndAnsweringIsAccepted(t *testing.T) {
|
||||
// run-once and scheduled shapes are excluded on purpose — a step that has exited is not a
|
||||
// fault. The name is `temp-mesh-controller` because the permanent one is a module and takes the
|
||||
// plain name (novox/hq ADR 0067), which is what makes the two of them coexist at all.
|
||||
want := []string{"mesh-store", "mesh-broker", "temp-mesh-controller"}
|
||||
want := []string{"mesh-store", "mesh-broker-nats", "temp-mesh-controller"}
|
||||
if len(verified.Running) != len(want) {
|
||||
t.Fatalf("confirmed %v running, want %v", verified.Running, want)
|
||||
}
|
||||
@@ -132,7 +132,7 @@ func TestAControlPlaneThatIsStillStartingIsWaitedFor(t *testing.T) {
|
||||
// than being told only that something is not what it should be.
|
||||
func TestAContainerThatExitedIsNamedWithItsState(t *testing.T) {
|
||||
runtime := &asked{answer: func(_ string, args []string) (string, error) {
|
||||
if args[0] == "container" && args[len(args)-1] == "mesh-broker" {
|
||||
if args[0] == "container" && args[len(args)-1] == "mesh-broker-nats" {
|
||||
return "false exited\n", nil
|
||||
}
|
||||
if args[0] == "container" {
|
||||
@@ -146,7 +146,7 @@ func TestAContainerThatExitedIsNamedWithItsState(t *testing.T) {
|
||||
if err == nil {
|
||||
t.Fatal("a container that had exited was reported as part of a running foundation")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "mesh-broker") || !strings.Contains(err.Error(), "exited") {
|
||||
if !strings.Contains(err.Error(), "mesh-broker-nats") || !strings.Contains(err.Error(), "exited") {
|
||||
t.Errorf("the failure does not say which container is in what state: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,87 +0,0 @@
|
||||
package declaration
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Busy is how a long-running container says it holds work that would be lost if it were stopped now
|
||||
// (novox/hq ADR 0335 §1): a command run inside it, by its shell, as an exec health check is, that exits 0
|
||||
// while it is busy and non-zero when it is idle. Being asked is also the container's notice that it is
|
||||
// about to be replaced: from the first ask it takes no new work, so its answer ends once the work in hand
|
||||
// ends.
|
||||
//
|
||||
// **Not part of what the container is.** Busy is not in the container's spec: a changed command changes
|
||||
// when the container is replaced, never whether — a container is not recreated for its busy command alone.
|
||||
type Busy struct {
|
||||
Command string `json:"command"`
|
||||
// Timeout is how long one ask may take; empty is BusyTimeoutDefault.
|
||||
Timeout string `json:"timeout,omitempty"`
|
||||
}
|
||||
|
||||
// The bounds of an ask (novox/hq ADR 0335).
|
||||
const (
|
||||
BusyTimeoutDefault = 10 * time.Second
|
||||
// BusyTimeoutCeiling is the longest one ask may take: an ask runs inside an apply, and an apply waits
|
||||
// for nothing longer than a look.
|
||||
BusyTimeoutCeiling = time.Minute
|
||||
// BusyBound is how long a container that says busy is left running after the first apply that would
|
||||
// have replaced it (ADR 0335 §3). The node-engine's, not the module's: no module holds its own
|
||||
// replacement for ever.
|
||||
BusyBound = 15 * time.Minute
|
||||
)
|
||||
|
||||
// UnmarshalJSON reads a busy strictly, as everything a declaration carries is read.
|
||||
func (b *Busy) UnmarshalJSON(raw []byte) error {
|
||||
type plain Busy
|
||||
var p plain
|
||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&p); err != nil {
|
||||
return fmt.Errorf("busy: %w", err)
|
||||
}
|
||||
*b = Busy(p)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Within is how long one ask may take. Validated on arrival, so an unreadable timeout is the default.
|
||||
func (b *Busy) Within() time.Duration {
|
||||
if d, err := time.ParseDuration(b.Timeout); err == nil && d > 0 {
|
||||
return d
|
||||
}
|
||||
return BusyTimeoutDefault
|
||||
}
|
||||
|
||||
// problems holds a container's busy to its shape: a one-line command, a timeout within its bounds, and only
|
||||
// on a container that stays up.
|
||||
func (b *Busy) problems(where string, longRunning bool) []string {
|
||||
if b == nil {
|
||||
return nil
|
||||
}
|
||||
var problems []string
|
||||
say := func(format string, args ...any) {
|
||||
problems = append(problems, where+": "+fmt.Sprintf(format, args...))
|
||||
}
|
||||
if !longRunning {
|
||||
say("busy is asked of a container that stays up; a step or a scheduled run is never replaced mid-run")
|
||||
}
|
||||
if strings.TrimSpace(b.Command) == "" {
|
||||
say("busy needs a command")
|
||||
}
|
||||
if strings.ContainsAny(b.Command, "\n\r") {
|
||||
say("busy's command is one line")
|
||||
}
|
||||
if b.Timeout != "" {
|
||||
d, err := time.ParseDuration(b.Timeout)
|
||||
switch {
|
||||
case err != nil:
|
||||
say("busy's timeout is a duration, not %q", b.Timeout)
|
||||
case d <= 0 || d > BusyTimeoutCeiling:
|
||||
say("an ask of busy takes more than nothing and at most %s, not %s", BusyTimeoutCeiling, d)
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
@@ -1239,11 +1239,6 @@ type Container struct {
|
||||
// timing, judged by the node-engine beside liveness. Absent: judged alive or not, and nothing more.
|
||||
Health *Health `json:"health,omitempty"`
|
||||
|
||||
// Busy is how this container says it holds work a replacement would lose (novox/hq ADR 0335): asked
|
||||
// before it is replaced, and while it says busy it is left running, within the bound. Absent: never
|
||||
// busy, and replaced at once, as before.
|
||||
Busy *Busy `json:"busy,omitempty"`
|
||||
|
||||
// RunAs is who the container runs as, resolved by the controller from the module's identity and the
|
||||
// image's declared interface (novox/hq ADR 0325 §2, §3): told by environment (the image reads PUID and
|
||||
// PGID, say) or by the runtime's own user option. Part of the spec, so a change recreates it. Absent: the
|
||||
@@ -1323,7 +1318,6 @@ func (c *Container) Target() string { return c.Name }
|
||||
|
||||
func (c *Container) validate(where string, _ bool) []string {
|
||||
problems := c.Health.problems(where, true, !c.RunOnce && c.Schedule == "")
|
||||
problems = append(problems, c.Busy.problems(where, !c.RunOnce && c.Schedule == "")...)
|
||||
if c.Name == "" {
|
||||
problems = append(problems, where+": a container needs a name")
|
||||
}
|
||||
|
||||
@@ -174,14 +174,6 @@ type Report struct {
|
||||
// broken.
|
||||
Windows []Window `json:"windows,omitempty"`
|
||||
|
||||
// Busy is every container the apply that reports left running because it said busy, and every one it
|
||||
// replaced after it had waited, with how (novox/hq ADR 0335): `waiting`, `replaced-when-idle` or
|
||||
// `replaced-at-bound`, since when, until when at the latest, and the last output of its busy command.
|
||||
// A walk waiting on this machine for a module whose container is listed `waiting` waits for the report
|
||||
// that no longer lists it so: its first-machine gate starts then (ADR 0335 §4). Absent from an engine
|
||||
// older than BusyContract, which never waits.
|
||||
Busy []Busy `json:"busy,omitempty"`
|
||||
|
||||
// Strays is what runs on the machine that the mesh neither wrote nor holds (novox/hq ADR
|
||||
// 0163): containers nobody declared and nobody holds, the ones a cutover leaves behind.
|
||||
Strays []Stray `json:"strays,omitempty"`
|
||||
@@ -288,13 +280,6 @@ const LibraryContract = 6
|
||||
// before it, its presence is what tells the controller this host may be sent `runner` on an identity.
|
||||
const OwnGroupContract = 7
|
||||
|
||||
// BusyContract is the statement of a host that also reads a container's `busy` (novox/hq ADR 0335): it asks a
|
||||
// container that declares it before replacing it, leaves it running while it says busy, within a bound of
|
||||
// 15 minutes, and says the wait in its report's `busy`. Like the fields before it, its presence is what tells
|
||||
// the controller this host may be sent `busy` on a container; an older host refuses the whole declaration
|
||||
// for it.
|
||||
const BusyContract = 8
|
||||
|
||||
// Health is one statement of every long-running resource's state on this machine (to-be 48 §4). Said in
|
||||
// every report, as an event on each change, and again every minute while anything is not healthy — so a
|
||||
// lost statement is not a lost fault.
|
||||
@@ -746,17 +731,6 @@ type Window struct {
|
||||
Until time.Time `json:"until"`
|
||||
}
|
||||
|
||||
// Busy is one container a busy command held (novox/hq ADR 0335), as a report says it. ID is the resource's
|
||||
// id, `<module>.<resource>`.
|
||||
type Busy struct {
|
||||
ID string `json:"id"`
|
||||
Container string `json:"container"`
|
||||
State string `json:"state"`
|
||||
Since time.Time `json:"since"`
|
||||
Until time.Time `json:"until"`
|
||||
Said string `json:"said,omitempty"`
|
||||
}
|
||||
|
||||
// A Stray is a container the mesh neither wrote nor holds (ADR 0163).
|
||||
type Stray struct {
|
||||
Kind string `json:"kind"`
|
||||
|
||||
@@ -155,11 +155,6 @@ type Applied struct {
|
||||
// container's spec label; this is the same information kept per file, so that when the spec
|
||||
// no longer matches the host can say WHICH file changed rather than only that something did.
|
||||
Reads map[string]string `json:"reads,omitempty"`
|
||||
|
||||
// BusySince is, for a container left running because it said busy, when the first apply that would
|
||||
// have replaced it ran (novox/hq ADR 0335 §3): kept here, beside the machine's state, so the bound runs
|
||||
// from then across every pass and across a restart of the node-engine. Absent while nothing waits.
|
||||
BusySince *time.Time `json:"busy_since,omitempty"`
|
||||
}
|
||||
|
||||
// Into is what a file written into held before the mesh's keys, or before the mesh's block.
|
||||
|
||||
Reference in New Issue
Block a user