diff --git a/Makefile b/Makefile index c9921e8..ebe96e7 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,29 @@ +SYSTEM ?= arch # The gate. Green is the definition of done (novox/hq how-we-build §5). VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo development) -LDFLAGS := -s -w -X main.version=$(VERSION) +LDFLAGS := -s -w -X main.builtFor=$(SYSTEM) -X main.version=$(VERSION) -.PHONY: check test vet fmt build clean +# The bundle a host carries is built INTO it (novox/hq ADR 0038, ADR 0041): a host that needed +# a second file to arrive with it is not "copy it and run it". +BUNDLE ?= -check: fmt vet test build +.PHONY: check test vet fmt build clean host + +check: fmt vet test packaging-test build + +# One binary per operating system (novox/hq ADR 0060). The system is pinned at link time; a +# host built without one refuses to touch a machine rather than guessing. +hosts: + @for s in arch alpine android; do \ + CGO_ENABLED=0 go build -ldflags="-s -w -X main.builtFor=$$s -X main.version=$(VERSION)" \ + -o mesh-host-$$s ./cmd/mesh-host || exit 1; \ + echo "built mesh-host-$$s"; \ + done + +packaging-test: + @./packaging/rollback_test.sh + @./packaging/launch_test.sh + @./packaging/roused_test.sh fmt: @test -z "$$(gofmt -l . )" || { echo "unformatted:"; gofmt -l . ; exit 1; } @@ -16,9 +35,27 @@ vet: test: go test ./... -count=1 -# Static on purpose: copy it onto a machine and run it is the whole installation. +# A default build carries no bundle and refuses to reconcile, which is the honest state for a +# host nobody has told what a substrate is. build: CGO_ENABLED=0 go build -ldflags="$(LDFLAGS)" -o mesh-host ./cmd/mesh-host +# A host for a real machine, carrying a real bundle: +# make host SYSTEM=arch BUNDLE=path/to/substrate.lock +# +# The bundle replaces the one for SYSTEM, because its contents are per operating system — +# package names and unit names differ (novox/hq ADR 0005). +host: + @test -n "$(BUNDLE)" || { echo "BUNDLE= is required; a host with no bundle cannot raise a first node"; exit 1; } + @test -f "$(BUNDLE)" || { echo "no such bundle: $(BUNDLE)"; exit 1; } + @test -f internal/bundle/substrate-$(SYSTEM).lock || { echo "no bundle slot for SYSTEM=$(SYSTEM)"; exit 1; } + @cp internal/bundle/substrate-$(SYSTEM).lock internal/bundle/substrate-$(SYSTEM).lock.default + @cp "$(BUNDLE)" internal/bundle/substrate-$(SYSTEM).lock + @CGO_ENABLED=0 go build -ldflags="$(LDFLAGS)" -o mesh-host ./cmd/mesh-host; \ + status=$$?; \ + mv internal/bundle/substrate-$(SYSTEM).lock.default internal/bundle/substrate-$(SYSTEM).lock; \ + exit $$status + @echo "built for $(SYSTEM) carrying $(BUNDLE)" + clean: rm -f mesh-host diff --git a/README.md b/README.md index a2ac393..d440a3b 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ mesh-host profile ``` That is the whole installation. One statically linked binary, nothing else present, no runtime -to install first ([`novox/hq` ADR 0041](https://git.novox.be/novox/hq)). +to install first ([`novox/hq` ADR 0005](https://git.novox.be/novox/hq)). ## What it is for @@ -20,15 +20,47 @@ the one. **It does not decide.** Anything needing knowledge of another node is the control plane's, and the host never queries the mesh database. It receives declarations and applies them. -## What exists today - -**Stage 1 only: it reports.** It applies nothing, connects to nothing, and listens on nothing. +## Joining a mesh ``` -mesh-host profile what this machine can be asked to do -mesh-host inventory what this machine is, and what it holds - --json machine-readable - --timeout how long any single probe may take (default 10s) +mesh-host enrol --token --name +``` + +**The node generates its own identity** — an Ed25519 keypair whose private half never leaves the +machine. The mesh records the public half. Nothing is issued to this node; it arrives holding its +identity, and what it receives is being known. + +**The broker's certificate is checked before this machine sends anything.** The token pins a +fingerprint; the connection is refused if what answers presents anything else. That refusal has +its own error and says plainly that retrying will not help, because it does not mean the network +is down — it means the mesh was substituted, and since this host applies whatever the link +delivers, that would be the whole machine. + +There is no certificate authority involved and no hostname check. At bootstrap the broker is +self-signed and reached at an address rather than a name, so there is nothing to trace and nothing +to match. One exact certificate, or nothing, which is stricter than either. + +**An already-enrolled machine refuses to enrol again.** The mesh believes its first identity, so +replacing it is deliberate: remove the identity file first. + +**What is not built is the link itself.** Enrolment verifies the broker and generates the identity, +and then stops, having saved nothing — so it can be run again unchanged. + +## What exists today + +**Stages 1 and 2.** It reports what a machine is, and it applies a declaration to one. It +connects to nothing and listens on nothing — what it applies comes from a file. + +``` +mesh-host profile what this machine can be asked to do +mesh-host inventory what this machine is, and what it holds +mesh-host apply FILE make this machine match a declaration from a file +mesh-host reconcile make this machine match the declaration this host carries +mesh-host bundle show what this host carries +mesh-host owned what this host has applied and still owns + --json machine-readable + --state where this node keeps what it knows + --dry-run read and check the declaration, change nothing ``` ``` @@ -46,8 +78,69 @@ linux/amd64 cannot be asked to: [firewall privileged] ``` -Stages 2 to 4 — applying from a pinned bundle, the link and the local store, and enrolment — -are designed and not built. +## Applying + +A declaration is JSON, versioned, and an **ordered list** of resources — the order is stated +rather than derived, because deriving it would be the host deciding +([`novox/hq` ADR 0005](https://git.novox.be/novox/hq)). The vocabulary is `directory`, `file` +and `service`, and **anything outside it refuses the whole declaration**: a host that skipped +what it did not understand would apply most of a declaration and report success. + +```json +{"declaration":1,"resources":[ + {"id":"mesh-etc","type":"directory","path":"/etc/mesh","mode":"0755"}, + {"id":"node-conf","type":"file","path":"/etc/mesh/node.conf","content":"role = anchor\n","mode":"0640"}, + {"id":"journal","type":"service","unit":"systemd-journald.service","state":"running"} +]} +``` + +**It converges rather than executes.** Applying twice changes nothing the second time; applying +to a drifted machine returns it. A mode is *maintained*, not merely set — a permission applied +at creation is not a permission held. + +**It owns a footprint, and only that.** What it applied and is no longer declared is removed; +what it did not create is never touched. It knows which is which because it recorded what it +did, after each thing worked. + +**A failed step fails the apply.** No step runs after a failure, and the error carries what had +already been done — the machine is in whatever state that left it, and pretending otherwise is +the fault this exists to prevent. + +## The bundle a host carries + +A host built for a machine carries its declaration **inside the binary**: + +``` +make host BUNDLE=path/to/substrate.lock +``` + +`mesh-host reconcile` then applies it. That is the first node's path — no mesh present, nothing +fetched, nothing else copied onto the machine. `copy it and run it` stops being true the moment +a second file has to arrive with it, which is why the bundle is embedded rather than beside it. + +**A default build carries nothing and refuses to reconcile**, saying so. A host that applied +nothing and reported success would look exactly like one that raised a first node, and the +difference would surface later as a mesh that never came up with nothing to point at. + +Stages 3 and 4 — the link, and enrolment — are designed and not built. + +## What stage 2 does not yet prove + +The design defines stage 2 as *the host applies `substrate.lock` with no mesh present*, and +calls out the claim underneath it: **that one host can raise the substrate alone**. + +The mechanism is proved — a sealed machine, one binary, and it configures itself from what it +carries. The claim is not. The substrate is four container services, and: + +- the vocabulary has no container type, because a container needs an image and where images + come from is open ([`novox/hq` research 012](https://git.novox.be/novox/hq)); +- what belongs in a substrate is not known — the closure for a one-node mesh is what + research 011 and 012 exist to answer; +- and the machine used to test this has no container runtime, because a sealed network cannot + install one. + +So `substrate.lock` here is a real bundle with a placeholder's content. Saying that plainly +beats shipping a host that claims a substrate it has never raised. ## A capability is detected, never assumed @@ -63,6 +156,12 @@ firewall is asked to list a ruleset, which needs the privilege as well as the to nobody can act on. The reason is what a person reads when a node will not take work they expected it to take. +**A unit that does not exist is not a unit that is stopped.** `systemctl is-active` says +`inactive` for both, so declaring a unit stopped reported success for a unit the host cannot +manage at all. `LoadState` separates them. Found by applying inside a raised machine, not by +reasoning — and its sibling: removing an orphaned service whose unit has since been uninstalled +used to fail the whole apply, which left a node able to apply *nothing*, ever. + **Exit codes are not the whole answer.** Found by running against a real machine rather than by reasoning: `systemctl is-system-running` exits non-zero for every state except `running` — including `degraded`, which means some units failed and the init is emphatically there. Reading @@ -80,7 +179,7 @@ CGO_ENABLED=0 go build -ldflags="-s -w" -o mesh-host ./cmd/mesh-host Roughly 3 MB, static, no dynamic dependencies. Cross-compiles with `GOOS`/`GOARCH`; a host is built once per architecture and copied, never built on the machine it runs on. -**Mocking the boundary is forbidden** ([`novox/hq` ADR 0034](https://git.novox.be/novox/hq)). +**Mocking the boundary is forbidden** ([`novox/hq` ADR 0017](https://git.novox.be/novox/hq)). Every detector is exercised against a fake runner for its logic *and* against this machine for its behaviour. The tests do not assert which capabilities a machine has — that varies, and is the point of detecting — they assert that detection tells the truth about whatever is there. @@ -96,3 +195,29 @@ repository carries implementation and does not carry decisions. - `02-DECISIONS/0039-the-link-is-the-security-boundary.md` — a node owns no password - `02-DECISIONS/0041-the-host-depends-on-nothing.md` — why this is a static binary, and Go - `04-ISSUES/007-an-installed-package-is-not-a-capability` — why detection works this way + +## Checks that cross into the control plane's repository + +Two things are agreed between this repository and `novox/mesh-control`, and each is a separate +struct on each side. A field renamed on one of them fails **silently** — the crossing succeeds and +something is simply absent — so both are checked by handing one side's real output to the other's +real parser. Neither runs by default; each skips with a reason, because a repository that fails +without its neighbour checked out is a repository nobody can build. + +**What the mesh sends, read by this host:** + +``` +mesh-control: ./build/mesh-control plan --json > /tmp/d.json +mesh-host: MESH_EMITTED=/tmp/d.json go test ./internal/declaration/ -v +``` + +**What this node says when it joins, read by the mesh:** + +``` +mesh-host: MESH_ENROL_OUT=/tmp/enrol.json go test ./internal/link/ +mesh-control: MESH_ENROL=/tmp/enrol.json make check +``` + +The second writes the private half of the sealing key beside the request, so the mesh's suite can +prove that what it sealed is openable rather than merely present. A key that is correctly named +and simply *wrong* passes every check that only looks at the message. diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index fb1db70..77b0558 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -8,33 +8,58 @@ package main import ( "context" + "crypto/sha256" + "encoding/base64" + "encoding/hex" "encoding/json" + "errors" "flag" "fmt" "os" "os/signal" + "sort" + "strings" "syscall" "text/tabwriter" "time" + "github.com/novox/mesh-host/internal/apply" + "github.com/novox/mesh-host/internal/bundle" + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/identity" "github.com/novox/mesh-host/internal/inventory" + "github.com/novox/mesh-host/internal/link" "github.com/novox/mesh-host/internal/profile" + "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/system" + "github.com/novox/mesh-host/internal/upgrade" ) // version is stamped at build time. Unset in a development build, and said so rather than // defaulted to something that looks like a release. +// builtFor names the operating system this host was built for, set at link time +// (novox/hq ADR 0005). A host built without one refuses to do anything that touches the +// machine, rather than guessing and calling a package manager that is not there. +var builtFor = "" + var version = "development build" const usage = `mesh-host — the node host profile what this machine can be asked to do inventory what this machine is, and what it holds + apply FILE make this machine match a declaration from a file + reconcile make this machine match the declaration this host carries + bundle show what this host carries + owned what this host has applied and still owns version --json machine-readable output --timeout how long any single probe may take (default 10s) + --state where this node keeps what it knows (default /var/lib/mesh-host/state.json) + --dry-run read the declaration and refuse it if wrong, but change nothing -Stage 1: reports only. It applies nothing, connects to nothing, listens on nothing. +It connects to nothing and listens on nothing. What it applies comes from a file. ` func main() { @@ -45,7 +70,7 @@ func main() { command, opts, err := parseArgs(os.Args[1:]) if err == nil { - err = run(ctx, command, opts.json, opts.timeout) + err = run(ctx, command, opts) } if err != nil { fmt.Fprintf(os.Stderr, "mesh-host: %v\n", err) @@ -54,8 +79,13 @@ func main() { } type options struct { - json bool - timeout time.Duration + json bool + timeout time.Duration + state string + token string + nodeName string + dryRun bool + file string } // parseArgs takes the subcommand first, then its flags. @@ -65,7 +95,7 @@ type options struct { // passed, silently ignored, with a successful exit. That is the fault this whole project keeps // naming, so the parser takes the subcommand off the front and parses what follows. func parseArgs(args []string) (string, options, error) { - opts := options{timeout: 10 * time.Second} + opts := options{timeout: 10 * time.Second, state: store.DefaultPath} command := "" if len(args) > 0 { @@ -78,19 +108,47 @@ func parseArgs(args []string) (string, options, error) { set.Usage = func() { fmt.Fprint(os.Stderr, usage) } set.BoolVar(&opts.json, "json", false, "machine-readable output") set.DurationVar(&opts.timeout, "timeout", opts.timeout, "how long any single probe may take") + set.StringVar(&opts.state, "state", opts.state, "where this node keeps what it knows") + set.BoolVar(&opts.dryRun, "dry-run", false, "read and check the declaration, change nothing") + set.StringVar(&opts.token, "token", "", "enrol: the one-time token, carried here by a person") + set.StringVar(&opts.nodeName, "name", "", "enrol: override the name the token carries") - if err := set.Parse(args); err != nil { - return "", opts, err + // Parsed in a loop, because the standard library stops at the FIRST non-flag argument. + // `mesh-host inventory --json` hit that once, and taking the subcommand off the front + // fixed only half of it: `mesh-host apply decl.json --dry-run` left --dry-run unread in + // exactly the same way. A flag may sit before, after or between positionals, and one that + // is silently dropped is the fault this whole project keeps naming. + var positionals []string + rest := args + for { + if err := set.Parse(rest); err != nil { + return "", opts, err + } + rest = set.Args() + if len(rest) == 0 { + break + } + positionals = append(positionals, rest[0]) + rest = rest[1:] + } + + if command == "apply" { + if len(positionals) != 1 { + return "", opts, errors.New("apply needs exactly one declaration file") + } + opts.file = positionals[0] + return command, opts, nil } // Anything left over was neither the command nor a flag. Refused rather than ignored: a // mistyped argument that changes nothing and reports success is worse than an error. - if rest := set.Args(); len(rest) > 0 { - return "", opts, fmt.Errorf("unexpected argument %q — try `mesh-host help`", rest[0]) + if len(positionals) > 0 { + return "", opts, fmt.Errorf("unexpected argument %q — try `mesh-host help`", positionals[0]) } return command, opts, nil } -func run(ctx context.Context, command string, jsonOut bool, timeout time.Duration) error { +func run(ctx context.Context, command string, opts options) error { + jsonOut, timeout := opts.json, opts.timeout switch command { case "profile": p := profile.Detect(ctx, profile.Default(nil), timeout) @@ -108,6 +166,70 @@ func run(ctx context.Context, command string, jsonOut bool, timeout time.Duratio writeInventory(inv) return nil + case "apply": + raw, err := os.ReadFile(opts.file) + if err != nil { + return fmt.Errorf("reading the declaration: %w", err) + } + // ParseTrusted: a file handed to the host by someone already running it as root is + // not the link. novox/hq ADR 0005 bounds what a REMOTE party may push; someone who + // can write this file and run this binary can do anything the binary can, so refusing + // them an action would buy nothing and would make an action untestable except by + // rebuilding the bundle. + d, err := declaration.ParseFileTrusted(raw) + if err != nil { + return err + } + return runApply(ctx, opts, d, opts.file) + + case "reconcile": + // The first node's path. novox/hq ADR 0004: no mesh reachable means the declaration + // comes from the bundle the host carries. There is no link yet, so this is currently + // the only source — which is a stage, not a design, and saying so beats implying the + // other source exists. + d, err := bundle.Load(builtFor) + if err != nil { + return err + } + return runApply(ctx, opts, d, "the carried bundle") + + case "bundle": + if bundle.IsEmpty(builtFor) { + fmt.Println("this host carries no bundle") + return nil + } + _, err := bundle.Load(builtFor) + if err != nil { + // Asked before it matters, rather than discovered on a first node. + return fmt.Errorf("this host carries a bundle it cannot itself read: %w", err) + } + os.Stdout.Write(bundle.Raw(builtFor)) + return nil + + case "owned": + known, err := store.Load(opts.state) + if err != nil { + return err + } + if jsonOut { + return writeJSON(known) + } + if len(known.Resources) == 0 { + fmt.Println("this host has applied nothing on this machine") + return nil + } + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + for _, r := range known.Resources { + fmt.Fprintf(w, " %s\t%s\t%s\n", r.Type, r.ID, r.Target) + } + return w.Flush() + + case "enrol", "enroll": + return enrol(ctx, opts) + + case "run": + return runLink(ctx, opts) + case "version": fmt.Println(version) return nil @@ -170,7 +292,7 @@ func writeInventory(inv inventory.Inventory) { writeProfile(inv.Profile) // Printed last and never hidden. An inventory that quietly omits what it could not read - // is the same fault as a report assembled from intent (novox/hq ADR 0035). + // is the same fault as a report assembled from intent (novox/hq ADR 0018). if len(inv.Unreadable) > 0 { fmt.Println("\ncould not read:") for _, u := range inv.Unreadable { @@ -178,3 +300,539 @@ func writeInventory(inv inventory.Inventory) { } } } + +// runApply reads a declaration and makes the machine match it. +// +// The state is loaded before anything is touched and saved after, including when the apply +// fails part-way: what was applied before the failure is on the machine, and a host that did +// not record it would believe it owns less than it does and leave that behind forever. +func runApply(ctx context.Context, opts options, d *declaration.Declaration, source string) error { + known, err := store.Load(opts.state) + if err != nil { + return err + } + + if opts.dryRun { + fmt.Printf("%s: %d resource(s), version %d — accepted, nothing applied\n", + source, len(d.Resources), d.Version) + return nil + } + + sys, err := system.For(builtFor) + if err != nil { + return err + } + // Refuse a declaration naming a shape this host cannot apply, before anything is applied. + // An android host has no `package` applier, and finding that out half way through is the + // half-configured machine this host exists to prevent (novox/hq ADR 0005). + if err := system.Check(sys, d); err != nil { + return err + } + // And prove this is the machine the host was built for. Installing the arch host on Alpine + // must say so once, at the start, rather than failing later inside pacman. + if err := sys.Confirm(ctx, apply.ExecRunner); err != nil { + return err + } + + report, updated, applyErr := apply.Apply(ctx, sys, d, known, store.OriginCarried, + apply.ExecRunner, func(line string) { + if !opts.json { + fmt.Println(line) + } + }, sealOpener(opts.state)) + + // Saved whichever way it went. Recording only on success would lose the footprint of a + // failed apply, and that footprint is on the machine either way. + if saveErr := store.Save(opts.state, updated); saveErr != nil { + if applyErr != nil { + return fmt.Errorf("%w\n\nand the node's state could not be saved: %v", applyErr, saveErr) + } + return saveErr + } + if applyErr != nil { + return applyErr + } + + // Only now, and only after a clean apply: this version got as far as a completed + // reconcile, which is the whole of what "known good" claims (novox/hq ADR 0005). Not + // health — a disconnected node is ordinary, and a resource that fails is the machine's + // problem rather than the binary's. + // + // A failure to record is reported and does not fail the apply. The apply worked; what is + // lost is a rollback's ability to come back here, which is worse to hide than to say. + if version != "" { + if err := upgrade.RecordKnownGood(upgrade.KnownGoodPath(opts.state), version); err != nil { + fmt.Fprintf(os.Stderr, + "mesh-host: applied, but could not record %s as known-good: %v\n"+ + " a rollback would have nothing to return to.\n", version, err) + } + } + // And tell the launcher this start worked. Without it the counter only climbs, and a node + // that has been up for months rolls itself back on its third ordinary restart. + if err := upgrade.ClearAttempts(upgrade.AttemptsPath(opts.state)); err != nil { + fmt.Fprintf(os.Stderr, + "mesh-host: applied, but could not clear the start counter: %v\n"+ + " this node may roll itself back after a few more restarts.\n", err) + } + + if opts.json { + return writeJSON(report) + } + if !report.Changed() { + fmt.Printf("%s: already matches — %d resource(s) checked\n", source, len(report.Outcomes)) + return nil + } + fmt.Printf("%s: applied — %d resource(s)\n", source, len(report.Outcomes)) + return nil +} + +// enrol joins this machine to a mesh. +// +// novox/hq 09-the-node-lifecycle: the token carries four things, the node dials the broker over +// the underlay, checks the certificate against the pin *before sending anything*, and presents +// the one-time secret together with a public key it generated itself. +// +// The mesh issues no identity. This machine arrives holding one; what it receives is being known. +func enrol(ctx context.Context, opts options) error { + tokenText, name := &opts.token, &opts.nodeName + if strings.TrimSpace(*tokenText) == "" { + return errors.New("enrol --token : the token is carried to this machine by a " + + "person, and is the only thing it needs") + } + + // Refused whole if incomplete. A token without the fingerprint would have this machine + // connect to whatever answers; without the signing key it could not tell a declaration from + // a forgery, and it applies whatever the link delivers. + token, err := identity.ParseToken(*tokenText) + if err != nil { + return err + } + + // The name comes from the token, because the node cannot work it out: the broker account it + // authenticates as is named after it, and that account exists before this machine has been + // told anything. --name remains for a token issued before the name travelled in one, and + // saying so beats a connection refused with an empty username — which is what this was. + if strings.TrimSpace(*name) == "" { + *name = token.Node + } + if strings.TrimSpace(*name) == "" { + return errors.New( + "this token does not say what the mesh calls this machine, and no --name was given. " + + "A token issued by a current control plane carries the name") + } + + // Before anything else: an already-enrolled machine must not quietly acquire a second + // identity. The mesh believes the first one, and re-enrolling is a deliberate act that + // starts with a person issuing a new token for that node record. + identityPath := identity.Path(opts.state) + switch existing, err := identity.Load(identityPath); { + case err == nil: + return fmt.Errorf( + "this machine is already node %q. Re-enrolling replaces the identity the mesh "+ + "believes, so it is done deliberately: remove %s first", + existing.Node, identityPath) + case errors.Is(err, identity.ErrNoIdentity): + default: + return err + } + + fmt.Printf("token for broker %s\n", token.Broker) + fmt.Printf(" pinned certificate %s\n", token.Fingerprint) + fmt.Printf(" signing key %s\n", + base64.StdEncoding.EncodeToString(token.Signer)[:16]+"...") + + // The check that has to happen before this machine says anything. + conn, err := link.Dial(token.Broker, token.Fingerprint, opts.timeout) + if err != nil { + return err + } + defer conn.Close() + fmt.Println("\nthe broker presented the certificate this token pins") + + conn.Close() + + mine, err := identity.Generate(*name) + if err != nil { + return err + } + fmt.Printf("generated this node's identity: %s\n", mine.PublicBase64()) + + // Its key on the private network, generated here and now for the same reason: the private + // half must never have been anywhere else. The mesh receives only the public half and uses it + // to compute a graph it cannot impersonate. + mine.Overlay, err = identity.GenerateOverlayKey() + if err != nil { + return err + } + fmt.Printf("generated this node's overlay key: %s\n", mine.Overlay.Public) + + // And the key secrets are sealed to. Here, with the others, because the mesh cannot seal + // anything to a key it has not been told about — a key made later would leave a node that + // looks enrolled and can receive no credential. + sealing, err := identity.GenerateSealingKey() + if err != nil { + return err + } + fmt.Printf("generated this node's sealing key: %s\n", sealing.Public) + + // And the key it serves TLS with on its name inside the mesh. Generated here for the same + // reason as the others: the private half must never have been anywhere else, and the mesh + // only ever certifies the public one. + serving, err := identity.GenerateServingKey() + if err != nil { + return err + } + fmt.Printf("generated this node's serving key: %s\n", serving.Public) + + // What this machine can be asked to do, gathered before joining rather than after. The + // control plane cannot decide what a node should run without it, so it travels with the + // request instead of being asked for in a second round trip. + detected := profile.Detect(ctx, profile.Default(nil), opts.timeout) + reported := map[string]any{} + if raw, err := json.Marshal(detected); err == nil { + _ = json.Unmarshal(raw, &reported) + } + + reply, err := link.Enrol(ctx, token.Broker, token.Fingerprint, *name, token.Secret, + mine.Public, mine.Overlay.Public, sealing.Public, serving.Public, reported, opts.timeout) + if err != nil { + return err + } + + // The mesh's name for this node wins over what the machine called itself: the token was + // issued for a node record, and that record is what the identity binds to. + mine.Node = reply.Node + mine.Membership = identity.Membership{ + Broker: firstNonEmpty(reply.Broker, token.Broker), + Fingerprint: firstNonEmpty(reply.Fingerprint, token.Fingerprint), + Signer: firstNonEmpty2(reply.Signer, token.Signer), + Password: reply.Password, + } + if mine.Membership.Password == "" { + // The mesh did not replace the token's secret, so it is still this node's broker + // password. Said rather than silently kept: a one-time secret living on as a credential + // is worth knowing about. + mine.Membership.Password = token.Secret + fmt.Println("\nnote: the mesh issued no separate broker password, so the token's secret " + + "remains this node's credential") + } + + // Saved only now, and only once the mesh has said it knows this node. A node holding an + // identity the mesh has never recorded would believe it had joined and be believed by + // nobody — worse than not having joined, because nothing would look wrong. + if err := identity.Save(identityPath, mine); err != nil { + return fmt.Errorf( + "the mesh accepted this node as %q and its identity could not be saved: %w\n"+ + "That token is spent, so getting back needs a new one", reply.Node, err) + } + + if !mine.Membership.Joined() { + return fmt.Errorf( + "the mesh accepted this node as %q but did not say how to reach it again, so this "+ + "identity could not be used after a restart. Nothing was saved", reply.Node) + } + + // Written before the identity, so a node that dies between the two has a key file with no + // identity — which enrols again cleanly — rather than an identity naming a key that is not + // there, which looks joined and cannot come up. + if err := os.WriteFile(identity.OverlayKeyPath(opts.state), + []byte(mine.Overlay.Private+"\n"), 0o600); err != nil { + return fmt.Errorf("cannot write this node's overlay key: %w", err) + } + if err := os.WriteFile(identity.SealingKeyPath(opts.state), + []byte(sealing.Private+"\n"), 0o600); err != nil { + return fmt.Errorf("cannot write this node's sealing key: %w", err) + } + if err := identity.WriteServingKey(identity.ServingKeyPath(opts.state), serving); err != nil { + return fmt.Errorf("cannot write this node's serving key: %w", err) + } + + fmt.Printf("\nenrolled as %s\n", reply.Node) + fmt.Printf(" identity %s\n", identityPath) + fmt.Printf(" queue %s\n", reply.Queue) + return nil +} + +func firstNonEmpty(values ...string) string { + for _, v := range values { + if strings.TrimSpace(v) != "" { + return v + } + } + return "" +} + +func firstNonEmpty2(values ...[]byte) []byte { + for _, v := range values { + if len(v) > 0 { + return v + } + } + return nil +} + +// runLink holds this node's link to the mesh open, applying what arrives. +// +// One outbound connection and nothing listening. While it is up this node is enrolled; while it +// is down it is disconnected, which is an ordinary situation rather than a failure — the machine +// keeps running whatever it was last told, from its own store. +func runLink(ctx context.Context, opts options) error { + // A machine that has not enrolled waits here rather than failing. It is *hosted*: the host is + // running, it has no identity, and there is nobody to link to — an ordinary state, and the + // one every machine passes through (novox/hq 09-the-node-lifecycle). + // + // Exiting instead would be worse than untidy. The launcher counts a failed start, and three + // of them roll the binary back — so a freshly installed host, waiting to be enrolled exactly + // as intended, would undo its own installation. + mine, err := waitForEnrolment(ctx, opts) + if err != nil { + return err + } + if mine.Node == "" { + return nil // asked to stop while waiting + } + + fmt.Printf("node %s, linking to %s\n", mine.Node, mine.Membership.Broker) + + apply := func(ctx context.Context, raw, signature []byte) link.Report { + return applyAndKeep(ctx, opts, raw, &store.Declared{Declaration: raw, Signature: signature}) + } + say := func(line string) { fmt.Println(line) } + + // Two things at once, and the second is what makes disconnection ordinary. The link brings + // new declarations; this holds the machine in the last one whether the link is up or not. A + // laptop shut for a week comes back and reconciles — it does not come back and ask what it is + // (novox/hq ADR 0004). + go holdTheMachine(ctx, opts, mine, say) + + return link.HoldRoused(ctx, link.Membership{ + Node: mine.Node, + Broker: mine.Membership.Broker, + Fingerprint: mine.Membership.Fingerprint, + Password: mine.Membership.Password, + Signer: mine.Membership.Signer, + }, apply, say, opts.timeout, rousedBySignal(ctx)) +} + +// rousedBySignal is the machine telling this process that its link is probably stale. +// +// **A signal, because nothing may listen on a node** (novox/hq ADR 0004). A socket for this would +// be a control surface on every machine, reachable by anything that can reach the machine, in +// exchange for saving twenty seconds — and the whole security argument rests on there not being +// one. A signal is delivered by the service manager to a process it already supervises. +// +// SIGHUP, because that is the signal a long-running program conventionally reads as *look again*, +// and nothing here is being reloaded from a file that a different signal would suit better. +// +// Dropped rather than queued when one arrives while another is unread: two wakes in the same +// instant are one wake, and a machine that suspends and resumes repeatedly must not build a +// backlog of reconnections to work through. +func rousedBySignal(ctx context.Context) link.Roused { + woken := make(chan os.Signal, 1) + signal.Notify(woken, syscall.SIGHUP) + + out := make(chan struct{}, 1) + go func() { + defer signal.Stop(woken) + for { + select { + case <-ctx.Done(): + return + case <-woken: + select { + case out <- struct{}{}: + default: + // One is already waiting to be read. Two wakes in the same instant are one. + } + } + } + }() + return out +} + +// ReconcileEvery is how often a node re-applies what it was last told. +// +// Not driven by the link. Changes are pushed, so this is not polling for them — it is the answer +// to a machine drifting: a file edited by hand, a container stopped by somebody, a service that +// died. A node that only acted when told would hold its state exactly until something else +// changed it, and then for ever. +const ReconcileEvery = 5 * time.Minute + +func holdTheMachine(ctx context.Context, opts options, mine identity.Identity, say link.Announce) { + ticker := time.NewTicker(ReconcileEvery) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + + declared, err := store.LoadDeclared(store.DeclaredPath(opts.state), mine.Membership.Signer) + if errors.Is(err, store.ErrNothingDeclared) { + // Nothing to hold this machine to yet. Ordinary on a node that has enrolled and not + // been assigned anything. + continue + } + if err != nil { + say("cannot re-apply what this node was told: " + err.Error()) + continue + } + + report := applyDeclared(ctx, opts, declared) + switch { + case report.Refused != "": + say("what this node was last told no longer applies: " + report.Refused) + case len(report.Failed) > 0: + say(fmt.Sprintf("holding this machine: %d applied, and %v", len(report.Applied), report.Failed)) + } + } +} + +// applyDeclared applies a declaration that has already been proved to come from the mesh. +// +// Signature checking happens before this is called, in the link. By the time anything here runs, +// the question "is this from the mesh I joined" is settled — which is why this can treat the +// bytes as instructions. +func applyDeclared(ctx context.Context, opts options, raw []byte) link.Report { + return applyAndKeep(ctx, opts, raw, nil) +} + +// applyAndKeep applies a declaration and, when it came from the mesh, keeps it so this node can +// go on obeying it while disconnected. +func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.Declared) link.Report { + declared, err := declaration.Parse(raw) + if err != nil { + return link.Report{Refused: err.Error()} + } + + built, err := system.For(builtFor) + if err != nil { + return link.Report{Refused: err.Error()} + } + if err := system.Check(built, declared); err != nil { + return link.Report{Refused: err.Error()} + } + + known, err := store.Load(opts.state) + if err != nil { + return link.Report{Refused: err.Error()} + } + + if err := built.Confirm(ctx, apply.ExecRunner); err != nil { + return link.Report{Refused: err.Error()} + } + + // Declared, not carried. A declaration from the mesh removes only what the mesh previously + // declared — never what this machine raised for itself from its bundle (04-ISSUES/010). + outcome, updated, applyErr := apply.Apply(ctx, built, declared, known, store.OriginDeclared, + apply.ExecRunner, nil, sealOpener(opts.state)) + + // Saved whichever way it went. Recording only on success would lose the footprint of a + // failed apply, and that footprint is on the machine either way. + if saveErr := store.Save(opts.state, updated); saveErr != nil { + return link.Report{Refused: "applied, and the node's state could not be saved: " + + saveErr.Error()} + } + + report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw)} + for _, change := range outcome.Outcomes { + report.Applied = append(report.Applied, change.ID) + } + // Kept whichever way it went, so a node that is disconnected next minute still knows what it + // was told. Saved after applying rather than before: what is kept is what this node acted on. + if signed != nil { + if err := store.SaveDeclared(store.DeclaredPath(opts.state), *signed); err != nil { + report.Failed = map[string]string{"keeping the declaration": err.Error()} + } + } + if applyErr != nil { + report.Failed = map[string]string{"apply": applyErr.Error()} + } + return report +} + +// waitForEnrolment returns this node's identity, waiting for one if it has none. +// +// It applies the carried bundle first, if there is one, because that is what a first node does +// before there is a mesh at all — and a machine that has been installed and not yet enrolled +// should still be whatever its bundle says it is. +func waitForEnrolment(ctx context.Context, opts options) (identity.Identity, error) { + const look = 5 * time.Second + + said := false + for { + mine, err := identity.Load(identity.Path(opts.state)) + if err == nil { + return mine, nil + } + if !errors.Is(err, identity.ErrNoIdentity) { + // An identity that exists and cannot be read is a fault, not a wait. Treating it as + // "not enrolled yet" would leave a node sitting quietly for ever while the mesh + // believes it is a member. + return identity.Identity{}, err + } + + if !said { + fmt.Println("this machine has not joined a mesh, and is waiting to be told which one.") + fmt.Println(" enrol it with: mesh-host enrol --token --name ") + said = true + } + + select { + case <-ctx.Done(): + return identity.Identity{}, nil + case <-time.After(look): + } + } +} + +// sealOpener is how a sealed file is opened. +// +// Looked up per file rather than held, because most declarations contain no sealed file at all +// and a node with no key must fail on the one that needs it rather than on every apply. +func sealOpener(statePath string) apply.Unseal { + return func(sealed string) ([]byte, error) { + key, err := identity.LoadSealingKey(identity.SealingKeyPath(statePath)) + if err != nil { + return nil, err + } + return key.Unseal(sealed) + } +} + +// carriedPorts is every machine port held by what this host raised from its own bundle. +// +// **What the mesh must assign around** (novox/hq ADR 0038). The substrate is not a module: a node +// raises it before any mesh exists, so the control plane has never heard of the store or the +// broker. Told this, it can put a module somewhere else; not told, it hands out a port one of them +// holds and finds out from a container runtime. +// +// Only what was carried. What the mesh itself put here it already knows about, and reporting it +// back would make the machine an authority on the mesh's own bookkeeping. +// digestOf names a declaration by its bytes, exactly as the mesh names what it sends. The two +// sides never exchange the digest of different things: this hashes the same raw bytes the mesh +// hashed when it recorded the send. +func digestOf(body []byte) string { + sum := sha256.Sum256(body) + return hex.EncodeToString(sum[:]) +} + +func carriedPorts(state store.State) []int { + seen := map[int]bool{} + var out []int + for _, applied := range state.Resources { + if applied.Origin == store.OriginDeclared { + continue + } + for _, port := range applied.Holds { + if !seen[port] { + seen[port] = true + out = append(out, port) + } + } + } + sort.Ints(out) + return out +} diff --git a/cmd/mesh-host/main_test.go b/cmd/mesh-host/main_test.go index bdec6e3..562c36d 100644 --- a/cmd/mesh-host/main_test.go +++ b/cmd/mesh-host/main_test.go @@ -1,6 +1,7 @@ package main import ( + "github.com/novox/mesh-host/internal/store" "testing" "time" ) @@ -81,3 +82,56 @@ func TestNoCommandIsNotAnError(t *testing.T) { t.Errorf("command = %q, want empty", command) } } + +func TestApplyNeedsExactlyOneDeclaration(t *testing.T) { + // `apply` takes a file where every other command takes nothing, so the leftover-argument + // rule has an exception — and an exception is where a parser stops refusing things it + // should. Both directions are checked. + if _, _, err := parseArgs([]string{"apply"}); err == nil { + t.Error("apply with no file was accepted") + } + if _, _, err := parseArgs([]string{"apply", "a.json", "b.json"}); err == nil { + t.Error("apply with two files was accepted") + } + + command, opts, err := parseArgs([]string{"apply", "decl.json", "--dry-run"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if command != "apply" || opts.file != "decl.json" || !opts.dryRun { + t.Errorf("parsed as command=%q file=%q dry-run=%v", command, opts.file, opts.dryRun) + } +} + +func TestTheStateHasADocumentedDefault(t *testing.T) { + // A host that wrote its state somewhere unexpected would forget what it owns on the next + // run, and then leave everything it had applied behind forever. + _, opts, err := parseArgs([]string{"owned"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if opts.state != store.DefaultPath { + t.Errorf("default state path is %q, not the documented %q", opts.state, store.DefaultPath) + } +} + +func TestAFlagAfterAPositionalIsRead(t *testing.T) { + // The same fault as TestAFlagAfterTheCommandIsRead, one level down. Taking the subcommand + // off the front fixed the flag after the COMMAND and not the flag after its ARGUMENT: the + // standard library stops at the first non-flag argument wherever that argument is. + for _, args := range [][]string{ + {"apply", "decl.json", "--dry-run", "--json"}, + {"apply", "--dry-run", "decl.json", "--json"}, + {"apply", "--dry-run", "--json", "decl.json"}, + } { + command, opts, err := parseArgs(args) + if err != nil { + t.Errorf("%v: unexpected error: %v", args, err) + continue + } + if command != "apply" || opts.file != "decl.json" || !opts.dryRun || !opts.json { + t.Errorf("%v parsed as file=%q dry-run=%v json=%v", + args, opts.file, opts.dryRun, opts.json) + } + } +} diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..687430a --- /dev/null +++ b/examples/README.md @@ -0,0 +1,52 @@ +# Examples + +## `substrate-first-node.lock` + +What a machine must be before a mesh exists — the bootstrap in +[novox/hq `07-the-substrate.md`](https://git.novox.be/novox/hq), whole: + +``` +0 a container runtime +1 the store runs +2 a database per context `inventory` and `identity` +3 those contexts' schemas mesh-control migrate +4 the broker runs with a certificate it generated itself +5 the control plane runs mesh-control serve +``` + +**A machine that applies this is a mesh** — one node, with nothing joined to it yet, which is +exactly what the first node is (novox/hq ADR 0004). From here it hands out tokens and everything +else joins the ordinary way. + +This file said it stopped at step 4 for longer than that was true, which is its own small lesson: +a comment about what something does not do is a comment nobody updates. + +Build a host carrying it: + +``` +make host SYSTEM=arch BUNDLE=examples/substrate-first-node.lock +``` + +**The registry address and digests have to be replaced before this is useful.** They are written +as `192.0.2.250:5000/…@sha256:…` because a digest belongs to whatever registry serves it — here, +one a lab scenario raises, which reports its digests when it comes up. That is not a placeholder +to be tidied away: a bundle is built *for a target*, and which registry that target pulls from is +part of the target. + +### What was verified, and how + +On a lab machine confirmed sealed — `curl https://example.com` times out, the lab registry answers +200 — the whole bundle applied from bare: eight resources, `inventory` created and `mesh` nowhere, +the `node` table present with its indexes, the migration recorded, and LavinMQ answering +`lavinmqctl status` with AMQP listening on 5672. + +Three consecutive reconciles after that: **already matches — 8 resource(s) checked**, each time. + +Then the machine was **rebooted**, and everything came back: docker from `boot: enabled`, both +containers because the host creates every container `--restart unless-stopped` +(`internal/apply/apply.go`), the schema intact in its named volume, and reconcile still finding +nothing to do. + +The reboot is worth doing rather than assuming. Nothing in the declaration asks for a container to +return, so that it does is a property of the host, and the only way to know it holds is to take +the machine away and give it back. diff --git a/examples/bundle_test.go b/examples/bundle_test.go new file mode 100644 index 0000000..47ce39d --- /dev/null +++ b/examples/bundle_test.go @@ -0,0 +1,69 @@ +// Package examples checks the bundles shipped in this directory. +// +// **Nothing checked them before.** `substrate-first-node.lock` is what a machine becomes when +// there is no mesh to ask — the one declaration applied with nothing to verify it against — and +// it was edited by hand and read by nobody but a running host. +package examples + +import ( + "os" + "sort" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" +) + +func bundle(t *testing.T) *declaration.Declaration { + t.Helper() + raw, err := os.ReadFile("substrate-first-node.lock") + if err != nil { + t.Fatal(err) + } + d, err := declaration.ParseFileTrusted(raw) + if err != nil { + t.Fatalf("the bundle a first node applies does not parse: %v", err) + } + return d +} + +// Defends novox/hq ADR 0028: the substrate supplies the control plane and nothing else. +// +// The object store was substrate for months on the strength of "it cannot grant itself a bucket", +// which answers half the test. The control plane never needed one, and nothing noticed because +// nothing counted what the bundle holds. +func TestTheBundleCarriesTheSubstrateAndTheControlPlaneAndNothingElse(t *testing.T) { + var images []string + for _, r := range bundle(t).Resources { + c, ok := r.(*declaration.Container) + if !ok { + continue + } + // The registry the images come from is the lab's, and is not what this asserts. + name := c.Image + if i := strings.LastIndex(name, "/"); i >= 0 { + name = name[i+1:] + } + images = append(images, strings.SplitN(name, "@", 2)[0]) + } + sort.Strings(images) + + want := []string{"lavinmq", "mesh-control", "postgres"} + if strings.Join(images, ",") != strings.Join(want, ",") { + t.Fatalf("the bundle carries %v; expected exactly %v.\n\n"+ + "Adding one is a change to what every first node becomes, and to ADR 0006's "+ + "membership — take it deliberately, not by editing a lock file.", images, want) + } +} + +// The control plane is in the bundle, and the design overlooked it once by reasoning about +// substrate services rather than counting containers (novox/hq 03-DESIGN/01-to-be/07). +func TestTheControlPlaneIsCarriedToo(t *testing.T) { + for _, r := range bundle(t).Resources { + if c, ok := r.(*declaration.Container); ok && strings.Contains(c.Image, "mesh-control") { + return + } + } + t.Fatal("nothing in the bundle starts the control plane, so the machine would raise a " + + "substrate and stop") +} diff --git a/examples/substrate-first-node.lock b/examples/substrate-first-node.lock new file mode 100644 index 0000000..2fcefe6 --- /dev/null +++ b/examples/substrate-first-node.lock @@ -0,0 +1,153 @@ +// substrate-first-node.lock — what a machine must be before a mesh exists. +// +// The whole bootstrap (novox/hq 03-DESIGN/01-to-be/07-the-substrate.md): a container runtime, a +// store, a database per context, those contexts' schemas, the broker, and the control plane +// running on top of them. +// +// It stopped before the control plane once, and this comment said so for longer than it was true. +// +// The broker generates its OWN certificate, in its own image, into a volume it then mounts read +// only. Self-signed, because at this moment there is no mesh to issue one and no public name to +// obtain one for -- and it does not matter, because what a joining node checks is the fingerprint +// pinned in its token, not a chain or a name (novox/hq ADR 0004). The subject is decoration. +// +// PINNED BY DIGEST, and the digest is not decoration: a tag can be made to point at a different +// image, and this file is applied on a machine with no mesh to ask about anything. These digests +// belong to the registry the lab raises, which is what a real node pulls from anyway — what is +// required is a reference that is exact and cannot move (novox/hq ADR 0006). +// +// The store waits up to three minutes rather than one. A machine that has just pulled the +// image and is running initdb for the first time can take longer than sixty seconds, and it +// failed that way three times in the lab -- a flaky bootstrap that a second run always fixed, +// which is the worst kind because it teaches people to run things twice. +// +// It failed a fourth time on 2026-08-30, on a loaded machine, and the report was `docker exited +// 1:` with nothing after the colon. Raising the timeout again would treat the symptom; what makes +// a retry the only available response is a timeout that reports nothing. So the wait now says +// what it saw before giving up. +// +// The store's data is a NAMED VOLUME, not a directory on the machine. A directory the host +// creates is owned by root, and the database runs as somebody else inside the container — so it +// could not write, and the container crash-looped. A named volume lets the image set up its own +// ownership, and outlives the container, which is what you want for the thing holding the mesh's +// state. +{ + "declaration": 1, + "resources": [ + { + "id": "container-runtime", + "type": "package", + "package": "docker" + }, + { + "id": "container-runtime-running", + "type": "service", + "unit": "docker.service", + "state": "running", + "boot": "enabled" + }, + { + "id": "store", + "type": "container", + "name": "mesh-store", + "image": "192.0.2.250:5000/postgres@sha256:7abf537131b66ed5af448d90653abf1679b0c7e9a1f07efdd4c3108a401b259a", + "env": { + "POSTGRES_PASSWORD": "bootstrap", + "PGDATA": "/var/lib/postgresql/data/pgdata" + }, + "ports": ["127.0.0.1:5432:5432"], + "volumes": ["mesh-store-data:/var/lib/postgresql/data"] + }, + // Over TCP, not the socket. While the store initialises it runs a temporary server on the + // socket ONLY, then stops it and starts the real one — so a socket check passes, the action + // exits happy, and the verify a moment later lands in the gap and fails. The action and its + // verify must ask the same question, or the action can succeed into a state verify rejects. + { + "id": "store-ready", + "type": "action", + "in": "mesh-store", + "command": ["sh", "-c", "for i in $(seq 1 180); do pg_isready -h 127.0.0.1 -U postgres >/dev/null 2>&1 && exit 0; sleep 1; done; echo 'the store did not answer within 180s; its own last words follow'; pg_isready -h 127.0.0.1 -U postgres; tail -n 20 /var/lib/postgresql/data/log/*.log 2>/dev/null; exit 1"], +"verify": ["pg_isready", "-h", "127.0.0.1", "-U", "postgres"] + }, + { + "id": "inventory-database", + "type": "action", + "in": "mesh-store", + "command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE inventory'"], + "verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw inventory"] + }, + { + "id": "identity-database", + "type": "action", + "in": "mesh-store", + "command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE identity'"], + "verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw identity"] + }, + // Each context owns its own database (novox/hq ADR 0008). A third one is a third database, + // created the same way and named the same way — which is the whole of adding a context to the + // bootstrap, and is why the count is not something the substrate has an opinion about. + { + "id": "licences-database", + "type": "action", + "in": "mesh-store", + "command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE licences'"], + "verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw licences"] + }, + { + "id": "context-schemas", + "type": "action", + "command": ["docker", "run", "--rm", "--network", "container:mesh-store", + "-e", "MESH_STORE_INVENTORY=postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable", + "-e", "MESH_STORE_IDENTITY=postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable", + "-e", "MESH_STORE_LICENCES=postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable", + "192.0.2.250:5000/mesh-control@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace", + "migrate"], + "verify": ["sh", "-c", "docker exec mesh-store psql -U postgres -d inventory -tAc \"select to_regclass('public.node')\" | grep -qx node && docker exec mesh-store psql -U postgres -d identity -tAc \"select to_regclass('public.signing_key')\" | grep -qx signing_key && docker exec mesh-store psql -U postgres -d licences -tAc \"select to_regclass('public.licence')\" | grep -qx licence"] + }, + { + "id": "broker-certificate", + "type": "action", + "command": ["docker", "run", "--rm", "--entrypoint", "sh", "-v", "mesh-broker-tls:/tls", + "192.0.2.250:5000/cloudamqp/lavinmq@sha256:b117c254e6e269a29db479e6b410ca4e46e035b4981e49d24b159673ef09d336", + "-c", "test -f /tls/tls.crt || (openssl req -x509 -newkey rsa:2048 -nodes -keyout /tls/tls.key -out /tls/tls.crt -days 3650 -subj '/CN=mesh-broker' >/dev/null 2>&1 && chmod 644 /tls/tls.crt && chmod 600 /tls/tls.key)"], + "verify": ["docker", "run", "--rm", "--entrypoint", "sh", "-v", "mesh-broker-tls:/tls", + "192.0.2.250:5000/cloudamqp/lavinmq@sha256:b117c254e6e269a29db479e6b410ca4e46e035b4981e49d24b159673ef09d336", + "-c", "test -s /tls/tls.crt && openssl x509 -in /tls/tls.crt -noout"] + }, + { + "id": "broker", + "type": "container", + "name": "mesh-broker", + "image": "192.0.2.250:5000/cloudamqp/lavinmq@sha256:b117c254e6e269a29db479e6b410ca4e46e035b4981e49d24b159673ef09d336", + "ports": ["5671:5671", "127.0.0.1:5672:5672", "127.0.0.1:15672:15672"], + "volumes": ["mesh-broker-data:/var/lib/lavinmq", "mesh-broker-tls:/tls:ro"], + "args": ["--amqps-port=5671", "--cert=/tls/tls.crt", "--key=/tls/tls.key"] + }, + { + "id": "broker-ready", + "type": "action", + "in": "mesh-broker", + "command": ["sh", "-c", "for i in $(seq 1 60); do lavinmqctl status >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"], + "verify": ["lavinmqctl", "status"] + }, + { + "id": "control-plane", + "type": "container", + "name": "mesh-control", + "image": "192.0.2.250:5000/mesh-control@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace", + "network": "host", + "args": ["serve"], + "volumes": ["mesh-broker-tls:/broker-tls:ro"], + "env": { + "MESH_STORE_INVENTORY": "postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable", + "MESH_STORE_IDENTITY": "postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable", + "MESH_STORE_LICENCES": "postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable", + "MESH_BROKER_AMQP": "amqp://guest:guest@127.0.0.1:5672/", + "MESH_BROKER_MANAGEMENT": "http://guest:guest@127.0.0.1:15672", + "MESH_BROKER_ADDRESS": "192.0.2.10:5671", + "MESH_BROKER_CERTIFICATE": "/broker-tls/tls.crt" + } + } + + ] +} diff --git a/go.mod b/go.mod index 5e7610d..c3444a4 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,9 @@ module github.com/novox/mesh-host -go 1.24 +go 1.25.0 + +require ( + github.com/rabbitmq/amqp091-go v1.14.0 // indirect + golang.org/x/crypto v0.55.0 // indirect + golang.org/x/sys v0.47.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..661ae93 --- /dev/null +++ b/go.sum @@ -0,0 +1,6 @@ +github.com/rabbitmq/amqp091-go v1.14.0 h1:RSaT7aOKt/OrkVUyswPDW29lnRz9psuGmfZFBmLqLek= +github.com/rabbitmq/amqp091-go v1.14.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= diff --git a/internal/apply/apply.go b/internal/apply/apply.go new file mode 100644 index 0000000..9905485 --- /dev/null +++ b/internal/apply/apply.go @@ -0,0 +1,1112 @@ +// Package apply makes a machine match a declaration. +// +// Three properties, each following a recorded decision, and each of them the difference +// between this and a script that writes files: +// +// - A failed step fails the apply (novox/hq ADR 0010). Not "logs and continues": a partial +// apply that reports success is the mesh's most expensive shape. +// - Every applier READS BACK. Setting a value is not evidence the value took. +// - What was applied is recorded after it works, never before (ADR 0018). A failed apply +// leaves the machine in whatever state it reached, and nothing must claim otherwise. +package apply + +import ( + "context" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/system" +) + +// Runner executes a command. The real one is used everywhere outside unit tests; behaviour +// against a real system is tested alongside rather than mocked (novox/hq ADR 0017). +type Runner = system.Runner + +// Outcome is what happened to one resource. +type Outcome struct { + ID string `json:"id"` + Type string `json:"type"` + Target string `json:"target"` + // Action is created · updated · unchanged · corrected · removed. + // + // "corrected" is its own answer and not a kind of "updated": it means the machine had drifted + // from what this host last wrote, so somebody changed it by hand. The mesh converging is + // right either way; being unable to say which happened is not. + Action string `json:"action"` + Detail string `json:"detail,omitempty"` + + // wrote is a digest of what this apply put there, kept so the next one can tell a machine + // that drifted from one the mesh changed its mind about. Not reported: it is bookkeeping. + wrote string +} + +// Report is what an apply did, in the order it did it. +type Report struct { + Outcomes []Outcome `json:"outcomes"` +} + +// Changed reports whether anything about the machine actually moved. An apply that changed +// nothing is the ordinary steady state, and saying so is not the same as saying it failed. +func (r Report) Changed() bool { + for _, o := range r.Outcomes { + if o.Action != "unchanged" { + return true + } + } + return false +} + +// Error is a failure part-way through, carrying what had already been done. +// +// The outcomes matter as much as the message: the machine is in whatever state the apply +// reached, and the only honest thing to hand back is the list of what did happen. +type Error struct { + Resource string + Err error + Done Report + // Others is how many more resources also failed. Named rather than folded into the message, + // because "one thing failed" and "eleven things failed" are different machines and the first + // line is what somebody reads. + Others int + + // Gated is true when what failed was an action, so nothing after it was attempted. A person + // reading a report needs to know the difference between "these things failed" and "these + // things failed and the rest was never tried". + Gated bool +} + +func (e *Error) Error() string { + also := "" + if e.Others == 1 { + also = ", and one other resource also failed" + } else if e.Others > 1 { + also = fmt.Sprintf(", and %d other resources also failed", e.Others) + } + rest := "everything was attempted, so what is not listed as failed was done." + if e.Gated { + // An action is a gate: it exists to make something true before the next thing needs it. + rest = "this is an action, so nothing after it was attempted — the machine is in " + + "whatever state that left it." + } + return fmt.Sprintf("applying %q: %v%s\n\n%d resource(s) were applied and remain; %s", + e.Resource, e.Err, also, len(e.Done.Outcomes), rest) +} + +func (e *Error) Unwrap() error { return e.Err } + +// Apply makes the machine match the declaration, and returns what it did. +// +// Removal happens FIRST, and the order is not arbitrary. A resource that leaves a declaration +// while another arrives at the same path is an ordinary rename: removing afterwards would +// delete the file that had just been written. Removing first risks losing the old state if the +// apply then fails — a recovery concern, where the other is a correctness one. +func Apply( + ctx context.Context, + sys system.System, + d *declaration.Declaration, + known store.State, + origin string, + run Runner, + log func(string), + unseal Unseal, +) (Report, store.State, error) { + if log == nil { + log = func(string) {} + } + report := Report{} + + declared := map[string]bool{} + for _, r := range d.Resources { + declared[r.Identity()] = true + } + + for _, orphan := range known.Orphans(declared, origin) { + action, detail, err := remove(ctx, sys, orphan, run) + if err != nil { + return report, known, &Error{Resource: orphan.ID, Err: err, Done: report} + } + known.Forget(orphan.ID) + report.Outcomes = append(report.Outcomes, Outcome{ + ID: orphan.ID, Type: orphan.Type, Target: orphan.Target, + Action: action, Detail: detail, + }) + log(fmt.Sprintf(" %s %s (%s)", action, orphan.ID, orphan.Target)) + } + + // What moved in this apply, so a service that must reflect a file can be told the file + // moved. Only within one apply: a change from an earlier one has already been reflected, and + // restarting for it every time would make a steady machine restart its services for ever. + changed := map[string]bool{} + + // Everything is attempted, and every failure is reported. + // + // **It used to stop at the first one**, and that made one broken resource hold the whole + // machine hostage: a module declaring a package that does not exist meant every module + // ordered after it was never applied, for ever, and the mesh reported "failed" without + // saying that the rest had not been tried. A machine with one bad module and nine good ones + // ran none of the nine. + // + // The argument for stopping was that a resource may depend on an earlier one. It still may — + // and it will then fail its own check and be reported, which is more information than not + // attempting it. A service started against a file that was never written does not verify, and + // this host reads back after every write precisely so that is caught rather than assumed. + // + // What does not change: **a declaration that cannot be parsed is still refused whole.** That + // is a different thing — one is "this machine could not do it", the other is "this was never + // a declaration", and they are fixed in different places. + var failures []*Error + for _, resource := range d.Resources { + was, _ := known.Find(resource.Identity()) + outcome, err := applyOne(ctx, sys, resource, run, changed, was, unseal) + if err != nil { + failed := &Error{Resource: resource.Identity(), Err: err, Done: report} + failures = append(failures, failed) + log(fmt.Sprintf(" failed %s (%s): %v", resource.Identity(), outcome.Target, err)) + + // **A failed action stops what follows. Nothing else does.** + // + // An action is the only shape whose purpose is to make something true *before* the + // next thing needs it — which is why it is the only one with a `verify`. The + // bootstrap is a row of them: the store answers, then its databases exist, then their + // schemas, then the broker. Carrying on past one that did not happen means starting + // things against a machine that is not ready, and on a small machine that is how a + // database still initialising gets its memory taken away and shuts down. Observed, + // in the lab, caused by an earlier version of this loop. + // + // Everything else is independent state. A package that will not install has nothing + // to do with a file on the other side of the declaration, and stopping there is what + // made one broken module hold a whole machine hostage + // (novox/hq 04-ISSUES/011). + if resource.Kind() == declaration.TypeAction { + failed.Done = report + failed.Others = len(failures) - 1 + failed.Gated = true + return report, known, failed + } + continue + } + + // Only now. The record follows the fact, never leads it. + known.Record(store.Applied{ + Origin: origin, + ID: resource.Identity(), Type: string(resource.Kind()), + Target: outcome.Target, AppliedAt: time.Now().UTC(), + Wrote: outcome.wrote, + Holds: holds(resource), + }) + report.Outcomes = append(report.Outcomes, outcome) + if outcome.Action != "unchanged" { + changed[resource.Identity()] = true + log(fmt.Sprintf(" %s %s (%s)", outcome.Action, outcome.ID, outcome.Target)) + } + } + + if len(failures) > 0 { + // The first, carrying everything that did happen. One error is what the caller reports + // and what a person reads first; the rest are in the report, which is what the mesh + // keeps. + first := failures[0] + first.Done = report + first.Others = len(failures) - 1 + return report, known, first + } + return report, known, nil +} + +// Unseal opens a value the mesh sealed to this node. Nil when the node has no sealing key, which +// makes every sealed file an error rather than a silently skipped one. +type Unseal func(sealed string) ([]byte, error) + +func applyOne(ctx context.Context, sys system.System, r declaration.Resource, run Runner, + changed map[string]bool, previous store.Applied, unseal Unseal) (Outcome, error) { + switch res := r.(type) { + case *declaration.Directory: + return applyDirectory(res) + case *declaration.File: + return applyFile(res, previous, unseal) + case *declaration.Service: + return applyService(ctx, sys, res, run, changed) + case *declaration.Package: + return applyPackage(ctx, sys, res, run) + case *declaration.Container: + return applyContainer(ctx, res, run, changed) + case *declaration.User: + return applyUser(ctx, sys, res, run) + case *declaration.Archive: + return applyArchive(ctx, res, previous) + case *declaration.Action: + return applyAction(ctx, res, run) + case *declaration.Network: + return applyNetwork(ctx, res, run) + default: + // Unreachable: the declaration refused this already. Present because "unreachable" + // stops being true the moment someone adds a kind and forgets this switch. + return Outcome{}, fmt.Errorf("no applier for type %q", r.Kind()) + } +} + +// begin starts an outcome from any resource, so the three facts a report needs are read from +// the resource itself rather than restated by each applier. +func begin(r declaration.Resource) Outcome { + return Outcome{ID: r.Identity(), Type: string(r.Kind()), Target: r.Target()} +} + +func modeOf(spec string, fallback os.FileMode) (os.FileMode, error) { + if spec == "" { + return fallback, nil + } + parsed, err := strconv.ParseUint(spec, 8, 32) + if err != nil { + return 0, fmt.Errorf("mode %q: %w", spec, err) + } + return os.FileMode(parsed), nil +} + +func applyDirectory(r *declaration.Directory) (Outcome, error) { + out := begin(r) + mode, err := modeOf(r.Mode, 0o755) + if err != nil { + return out, err + } + + before, err := os.Stat(r.Path) + existed := err == nil + if err != nil && !errors.Is(err, os.ErrNotExist) { + return out, err + } + if existed && !before.IsDir() { + return out, fmt.Errorf("%s exists and is not a directory", r.Path) + } + + if !existed { + if err := os.MkdirAll(r.Path, mode); err != nil { + return out, err + } + } + // Set explicitly even when it existed: MkdirAll applies the mode only on creation, and a + // permission set at creation is not a permission maintained — a lesson this repository + // already paid for once, with world-readable environment files. + if err := os.Chmod(r.Path, mode); err != nil { + return out, err + } + + // Read back. + after, err := os.Stat(r.Path) + if err != nil { + return out, fmt.Errorf("made %s and cannot stat it: %w", r.Path, err) + } + if !after.IsDir() { + return out, fmt.Errorf("%s is not a directory after applying", r.Path) + } + if after.Mode().Perm() != mode.Perm() { + return out, fmt.Errorf("%s is mode %o after setting %o", r.Path, after.Mode().Perm(), mode.Perm()) + } + + ownedAlready, err := ownedBy(r.Path, r.Owner) + if err != nil { + return out, err + } + if !ownedAlready { + if err := own(r.Path, r.Owner); err != nil { + return out, err + } + } + + out.Action = "unchanged" + if !existed { + out.Action = "created" + } else if !ownedAlready { + out.Action = "updated" + } else if before.Mode().Perm() != mode.Perm() { + out.Action = "updated" + out.Detail = fmt.Sprintf("mode %o to %o", before.Mode().Perm(), mode.Perm()) + } + return out, nil +} + +func applyFile(r *declaration.File, previous store.Applied, unseal Unseal) (Outcome, error) { + out := begin(r) + + // What actually goes on disk. For a sealed file the mesh never had this, and neither did + // whatever carried the declaration here. + content := r.Content + if r.Bytes != "" { + // Not text. Decoded here rather than written as base64, because what a declaration says + // is in a file has to be what ends up in it — a wallpaper stored as its own encoding is + // a wallpaper nothing can open. + decoded, err := base64.StdEncoding.DecodeString(r.Bytes) + if err != nil { + return out, fmt.Errorf("%s carries bytes that are not base64: %w", r.Path, err) + } + content = string(decoded) + } + // A secret written world-readable is a secret. The default differs from an ordinary file's + // for that reason alone; an explicit mode still wins, because a module may need its own user + // to read it and only the module knows which. + fallback := os.FileMode(0o644) + if r.Secret() { + fallback = 0o600 + if unseal == nil { + // Refused rather than skipped. A machine that quietly does not apply the one resource + // carrying a credential is a machine that looks configured and cannot connect. + return out, fmt.Errorf( + "%s is sealed to this node and this node has no sealing key", r.Path) + } + opened, err := unseal(r.Sealed) + if err != nil { + return out, fmt.Errorf("cannot open %s: %w", r.Path, err) + } + content = string(opened) + } + // Sealed values into the holes the content left for them. **The host is the only thing that + // ever holds both** — the mesh discarded the value, and the module wrote the document without + // it (novox/hq ADR 0024). + if len(r.Secrets) > 0 { + if unseal == nil { + return out, fmt.Errorf( + "%s needs %d sealed value(s) and this node has no sealing key", + r.Path, len(r.Secrets)) + } + for _, name := range r.SecretsUsed() { + opened, err := unseal(r.Secrets[name]) + if err != nil { + return out, fmt.Errorf("cannot open the secret %q for %s: %w", name, r.Path, err) + } + content = strings.ReplaceAll(content, "${secret:"+name+"}", string(opened)) + } + // A file carrying a credential is not world-readable, whatever else it also carries. + if r.Mode == "" { + fallback = 0o600 + } + } + out.wrote = digestOf(content) + mode, err := modeOf(r.Mode, fallback) + if err != nil { + return out, err + } + + existing, readErr := os.ReadFile(r.Path) + existed := readErr == nil + if readErr != nil && !errors.Is(readErr, os.ErrNotExist) { + return out, readErr + } + + var beforeMode os.FileMode + if existed { + if info, err := os.Stat(r.Path); err == nil { + beforeMode = info.Mode().Perm() + } + } + + contentSame := existed && string(existing) == content + + // Whether the machine still holds what this host last put there. When it does not, and the + // declaration has not changed either, somebody edited it — and saying so is the whole + // difference between a change that vanishes mysteriously and one that is reported. + drifted := existed && previous.Wrote != "" && digestOf(string(existing)) != previous.Wrote + modeSame := existed && beforeMode == mode.Perm() + + if !contentSame { + if err := os.MkdirAll(filepath.Dir(r.Path), 0o755); err != nil { + return out, err + } + if err := writeAtomically(r.Path, []byte(content), mode); err != nil { + return out, err + } + } else if !modeSame { + if err := os.Chmod(r.Path, mode); err != nil { + return out, err + } + } + + // Read back — the file, not the call that wrote it. + written, err := os.ReadFile(r.Path) + if err != nil { + return out, fmt.Errorf("wrote %s and cannot read it back: %w", r.Path, err) + } + if string(written) != content { + return out, fmt.Errorf("%s does not contain what was declared after writing it", r.Path) + } + info, err := os.Stat(r.Path) + if err != nil { + return out, err + } + if info.Mode().Perm() != mode.Perm() { + return out, fmt.Errorf("%s is mode %o after setting %o", r.Path, info.Mode().Perm(), mode.Perm()) + } + + // And who it belongs to. Checked before setting, so a file already owned correctly is not + // reported as changed on every apply — which would make every reconcile look like work. + ownedAlready, err := ownedBy(r.Path, r.Owner) + if err != nil { + return out, err + } + if !ownedAlready { + if err := own(r.Path, r.Owner); err != nil { + return out, err + } + contentSame = false + } + + switch { + case !existed: + out.Action = "created" + case drifted: + // Somebody changed this on the machine. The mesh puts it back either way — that is what + // holding a machine to what it was told means — but a change that vanishes with nothing + // said is how a person ends up editing the same file every five minutes, believing the + // machine is broken. + out.Action = "corrected" + out.Detail = "it had been changed on the machine since this host last wrote it" + case !contentSame && !modeSame: + out.Action = "updated" + out.Detail = "content and mode" + case !contentSame: + out.Action = "updated" + out.Detail = "content" + case !modeSame: + out.Action = "updated" + out.Detail = fmt.Sprintf("mode %o to %o", beforeMode, mode.Perm()) + default: + out.Action = "unchanged" + } + return out, nil +} + +// writeAtomically writes through a temporary file in the same directory. +// +// A reader of a managed file must never see half of one. The mesh's own configuration is read +// by daemons that reload on change, so a torn write is a service reading a truncated config. +func writeAtomically(path string, content []byte, mode os.FileMode) error { + tmp, err := os.CreateTemp(filepath.Dir(path), ".mesh-host-*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + + if _, err := tmp.Write(content); err != nil { + tmp.Close() + return err + } + if err := tmp.Sync(); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + if err := os.Chmod(tmp.Name(), mode); err != nil { + return err + } + return os.Rename(tmp.Name(), path) +} + +// reflects reports whether anything this service must mirror changed in this apply. +func reflects(r *declaration.Service, changed map[string]bool) bool { + return len(reflected(r, changed)) > 0 +} + +// reflected is which of them changed, so the outcome can say why the service was restarted. A +// restart with no reason given is indistinguishable from a service that keeps falling over. +func reflected(r *declaration.Service, changed map[string]bool) []string { + return restartedBy(r.RestartOn, changed) +} + +func applyService(ctx context.Context, sys system.System, r *declaration.Service, run Runner, + changed map[string]bool) (Outcome, error) { + out := begin(r) + var changes []string + + // Boot first. A unit asked to be running and enabled should survive this apply failing + // half way in the more useful direction: enabled-and-stopped comes back at the next boot, + // where running-and-disabled does not. + if r.Boot != "" { + bootBefore, err := sys.ServiceBoot(ctx, run, r.Unit) + if err != nil { + return out, err + } + if bootBefore != r.Boot { + if err := sys.SetServiceBoot(ctx, run, r.Unit, r.Boot); err != nil { + return out, fmt.Errorf("setting %s to %s at boot: %w", r.Unit, r.Boot, err) + } + bootAfter, err := sys.ServiceBoot(ctx, run, r.Unit) + if err != nil { + return out, err + } + if bootAfter != r.Boot { + return out, fmt.Errorf( + "%s was asked to be %s at boot and is %s", r.Unit, r.Boot, bootAfter) + } + changes = append(changes, "boot "+bootBefore+" to "+bootAfter) + } + } + + before, err := sys.ServiceState(ctx, run, r.Unit) + if err != nil { + return out, err + } + if before != r.State { + if err := sys.SetServiceState(ctx, run, r.Unit, r.State); err != nil { + return out, fmt.Errorf("setting %s to %s: %w", r.Unit, r.State, err) + } + + // Read back. A service manager accepting a command says the transaction was accepted, + // not that the unit is running — one that starts and immediately dies satisfies it. + after, err := sys.ServiceState(ctx, run, r.Unit) + if err != nil { + return out, err + } + if after != r.State { + return out, fmt.Errorf("%s was asked to be %s and is %s", r.Unit, r.State, after) + } + changes = append(changes, before+" to "+after) + } else if r.State == "running" && reflects(r, changed) { + // The service is already in the state it was asked for, and something it must reflect + // changed in this same apply. A running service does not re-read its configuration, so + // leaving it alone here is how a machine ends up correct on disk and wrong in fact — + // with every check passing. + if err := sys.SetServiceState(ctx, run, r.Unit, "stopped"); err != nil { + return out, fmt.Errorf("restarting %s: stopping it: %w", r.Unit, err) + } + if err := sys.SetServiceState(ctx, run, r.Unit, "running"); err != nil { + return out, fmt.Errorf("restarting %s: starting it again: %w", r.Unit, err) + } + // Read back, for the same reason as above: a unit that starts and immediately dies + // satisfies a service manager and nothing else. + after, err := sys.ServiceState(ctx, run, r.Unit) + if err != nil { + return out, err + } + if after != "running" { + return out, fmt.Errorf( + "%s was restarted to pick up a change and is %s", r.Unit, after) + } + changes = append(changes, "restarted for "+strings.Join(reflected(r, changed), ", ")) + } + + if len(changes) == 0 { + out.Action = "unchanged" + out.Detail = before + return out, nil + } + out.Action = "updated" + out.Detail = strings.Join(changes, ", ") + return out, nil +} + +// remove undoes one resource the host applied and the declaration no longer names, and reports +// what it actually did. +// +// Only ever called for something in the store, which is what bounds it: the host is +// authoritative over its own footprint and inert everywhere else (novox/hq ADR 0005). +// +// It returns the action rather than assuming "removed", because for half the vocabulary the +// honest word is "forgotten". A host that reported a package removed when it left the package +// installed would be describing an effect it declined to have. +func remove(ctx context.Context, sys system.System, a store.Applied, run Runner) (string, string, error) { + switch declaration.Type(a.Type) { + case declaration.TypeDirectory: + // **A directory with anything left in it is kept, and that is the rule that protects + // data.** Everything the mesh put inside is itself a declared resource, and orphans are + // removed in reverse declaration order — so by the time a directory comes to be removed, + // what the mesh wrote there is already gone. Anything still present is something nobody + // declared: a database's files, a mail spool, somebody's uploads. + // + // Before this, unassigning a module deleted its data directory and everything under it, + // and the report said "removed". Nothing anywhere said what had been in there. + // + // This is the host's own line, applied to the one shape where getting it wrong is not + // recoverable: it removes what it made and leaves what it merely configured. An empty + // directory is what it made. A full one is not. + entries, err := os.ReadDir(a.Target) + if errors.Is(err, os.ErrNotExist) { + return "forgotten", "no longer there", nil + } + if err != nil { + return "", "", err + } + if len(entries) > 0 { + return "kept", fmt.Sprintf( + "no longer declared, and %d item(s) inside that the mesh did not put there — "+ + "remove it by hand once you know what it is", len(entries)), nil + } + if err := os.Remove(a.Target); err != nil { + return "", "", err + } + return "removed", "no longer declared, and empty", nil + + case declaration.TypeFile: + if err := os.RemoveAll(a.Target); err != nil { + return "", "", err + } + if _, err := os.Stat(a.Target); !errors.Is(err, os.ErrNotExist) { + return "", "", fmt.Errorf("%s is still there after removing it", a.Target) + } + return "removed", "no longer declared", nil + + case declaration.TypeService: + // A unit that is no longer declared is stopped, not deleted. The host did not install + // it and does not own the unit file — only the state it put the unit into. + // + // A unit that no longer EXISTS is already in the state removal is trying to reach, and + // saying so matters: stopping it fails, and a failure here fails the whole apply. A + // host holding a record of an uninstalled unit would then be unable to apply anything, + // ever, with no way out but editing its state by hand. Removal is idempotent for the + // same reason `os.RemoveAll` is. + if _, err := sys.ServiceState(ctx, run, a.Target); err != nil { + if strings.Contains(err.Error(), "does not exist on this machine") { + return "forgotten", "the unit no longer exists", nil + } + return "", "", err + } + if err := sys.SetServiceState(ctx, run, a.Target, "stopped"); err != nil { + return "", "", fmt.Errorf("stopping %s: %w", a.Target, err) + } + return "removed", "stopped; the unit file is not the host's to delete", nil + + case declaration.TypeContainer: + // The host CREATED this one, so the host removes it. That is the line: it removes what + // it made and leaves what it merely configured. + if _, err := run(ctx, "docker", "rm", "-f", a.Target); err != nil { + // Already gone is the state removal wants. Anything else is a real failure. + if _, alive := containerState(ctx, a.Target, run); alive == nil { + return "", "", fmt.Errorf("removing container %s: %w", a.Target, err) + } + } + if _, err := containerState(ctx, a.Target, run); err == nil { + return "", "", fmt.Errorf("container %s is still there after removing it", a.Target) + } + return "removed", "no longer declared", nil + + case declaration.TypePackage: + // Deliberately not uninstalled, and this is a decision rather than an omission. + // + // The host cannot know what else on this machine needs the package. Uninstalling a + // container runtime because a declaration changed would stop every container on the + // node, and the machine may have had the package before the mesh ever saw it + // (novox/hq research 012: adopted, not installed). Undeclaring says "the mesh no + // longer requires this", which is not the same as "remove it". + return "forgotten", "left installed; the host does not uninstall what it cannot know is unused", nil + + case declaration.TypeAction: + // An action has no footprint the host can undo — it ran, and whatever it did belongs + // to whatever it acted on. + return "forgotten", "an action leaves nothing the host owns", nil + + case declaration.TypeNetwork: + // **The reason this is a shape at all** (novox/hq ADR 0029). Orphans are removed in + // reverse declaration order, so a network written before the containers that join it is + // removed after they are gone — and a runtime refusing to remove one still in use is + // reported rather than swallowed, because that means something the mesh did not declare + // is holding it. + cri, err := containerRuntime(ctx, run) + if err != nil { + return "", "", fmt.Errorf("%w, so the network %q cannot be removed", err, a.Target) + } + if _, err := run(ctx, cri, "network", "inspect", a.Target); err != nil { + return "forgotten", "no longer there", nil + } + if _, err := run(ctx, cri, "network", "rm", a.Target); err != nil { + return "", "", fmt.Errorf("cannot remove the network %q: %w", a.Target, err) + } + return "removed", "no longer declared", nil + + default: + return "", "", fmt.Errorf("no way to remove a %q", a.Type) + } +} + +// ExecRunner runs a real command, with stdin closed and output captured. +func ExecRunner(ctx context.Context, name string, args ...string) (string, error) { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Stdin = nil + out, err := cmd.Output() + if err != nil { + var exit *exec.ExitError + if errors.As(err, &exit) { + return string(out), fmt.Errorf("%s exited %d: %s", + name, exit.ExitCode(), strings.TrimSpace(string(exit.Stderr))) + } + return string(out), fmt.Errorf("%s: %w", name, err) + } + return string(out), nil +} + +// applyPackage installs a package the machine does not have. +// +// It never upgrades and never removes. "Present" is the whole of what a package resource +// asserts, because version is the package manager's business and the mesh does not have a +// second opinion about it (novox/hq ADR 0005 — the host depends on nothing, and that includes +// not becoming a second package manager). +func applyPackage(ctx context.Context, sys system.System, r *declaration.Package, run Runner) (Outcome, error) { + out := begin(r) + + installed, err := sys.PackageInstalled(ctx, run, r.Package) + if err != nil { + return out, err + } + if installed { + out.Action = "unchanged" + out.Detail = "already installed" + return out, nil + } + + if err := sys.InstallPackage(ctx, run, r.Package); err != nil { + return out, fmt.Errorf("installing %s: %w", r.Package, err) + } + + // Read back. A package manager exiting zero says the transaction was accepted. + installed, err = sys.PackageInstalled(ctx, run, r.Package) + if err != nil { + return out, err + } + if !installed { + return out, fmt.Errorf( + "%s was installed without error and the package database does not have it", r.Package) + } + + out.Action = "created" + return out, nil +} + +// Labels the host puts on every container it creates. +// +// specLabel carries a digest of the declaration that made the container. It is what lets a +// reconcile answer "is this container the one the current declaration describes" without +// comparing every field the runtime reports — which cannot be done reliably, because a runtime +// normalises, defaults and reorders what it is given, and the differences that produces are +// indistinguishable from real drift. +const ( + specLabel = "mesh-host.spec" + idLabel = "mesh-host.id" +) + +// containerSpec is the identity of a declared container: everything that, if changed, means +// the running container is no longer what was asked for. +func containerSpec(r *declaration.Container) string { + keys := make([]string, 0, len(r.Env)) + for k := range r.Env { + keys = append(keys, k) + } + sort.Strings(keys) + + var b strings.Builder + b.WriteString(r.Image + "\n" + r.Name + "\n") + for _, k := range keys { + b.WriteString("env " + k + "=" + r.Env[k] + "\n") + } + for _, p := range r.Ports { + b.WriteString("port " + p + "\n") + } + for _, v := range r.Volumes { + b.WriteString("volume " + v + "\n") + } + for _, a := range r.Args { + b.WriteString("arg " + a + "\n") + } + return fmt.Sprintf("%x", sha256.Sum256([]byte(b.String()))) +} + +// containerState reports whether a container is running and which spec made it. +// The error means the container does not exist. +func containerState(ctx context.Context, name string, run Runner) (state struct { + Running bool + Spec string +}, err error) { + out, err := run(ctx, "docker", "inspect", "--format", + "{{.State.Running}}\t{{index .Config.Labels \""+specLabel+"\"}}", name) + if err != nil { + return state, fmt.Errorf("no container named %s", name) + } + running, spec, _ := strings.Cut(strings.TrimSpace(out), "\t") + state.Running = running == "true" + state.Spec = strings.TrimSpace(spec) + return state, nil +} + +// applyContainer makes the declared container the one that is running. +// +// There is no "update" for a container: a container's configuration is fixed when it is +// created, so any change is a replacement. Saying that plainly is better than a partial +// in-place update that leaves the running thing half-declared. +// applyNetwork creates a named network if the machine does not already have one. +// +// **Existence is the whole of the state.** A network the mesh declared and a network somebody +// made by hand are indistinguishable by name, and that is deliberate: the mesh owns the name, not +// the thing, so it will not tear down and rebuild one that is already there and working. What it +// records is that this resource is now present, which is what lets it be removed later. +// +// Nothing is reconciled beyond presence. A driver or a subnet changed underneath would not be +// noticed — and is not declarable either (novox/hq ADR 0029), so there is nothing to disagree +// with. +func applyNetwork(ctx context.Context, r *declaration.Network, run Runner) (Outcome, error) { + out := begin(r) + + cri, err := containerRuntime(ctx, run) + if err != nil { + return out, fmt.Errorf("%w, so nothing can be said about the network %q", err, r.Name) + } + + if _, err := run(ctx, cri, "network", "inspect", r.Name); err == nil { + out.Action = "unchanged" + out.Detail = "already there" + return out, nil + } + + if _, err := run(ctx, cri, "network", "create", r.Name); err != nil { + return out, fmt.Errorf("cannot create the network %q: %w", r.Name, err) + } + // Read back rather than trusting the exit status (novox/hq ADR 0018). A runtime that reports + // success and made nothing leaves every container that joins it failing to start, with the + // cause one step away. + if _, err := run(ctx, cri, "network", "inspect", r.Name); err != nil { + return out, fmt.Errorf( + "the network %q was created and is not there afterwards: %w", r.Name, err) + } + out.Action = "created" + out.Detail = "a network for this module's own containers" + return out, nil +} + +func applyContainer(ctx context.Context, r *declaration.Container, run Runner, changed map[string]bool) (Outcome, error) { + out := begin(r) + want := containerSpec(r) + + cri, err := containerRuntime(ctx, run) + if err != nil { + return out, fmt.Errorf("%w, so nothing can be said about %q", err, r.Name) + } + + before, err := containerState(ctx, r.Name, run) + existed := err == nil + + // A container reads a mounted file once, at start. When one of its restart-on resources changed + // this pass — a settings-merged config the runtime read, say — the file on disk is new and the + // running process still holds the old value, and the spec (image, env, volumes) has not moved, + // so the plain "spec matches, leave it" below would keep the stale process for ever + // (novox/hq 04-ISSUES/009). Recreating is how a container gets restart-on, which a service + // already has. + reasons := restartedBy(r.RestartOn, changed) + + switch { + case existed && before.Spec == want && before.Running && len(reasons) == 0: + out.Action = "unchanged" + return out, nil + case existed: + if _, err := run(ctx, cri, "rm", "-f", r.Name); err != nil { + return out, fmt.Errorf("replacing container %s: %w", r.Name, err) + } + } + + args := []string{"run", "--detach", "--name", r.Name, "--restart", "unless-stopped"} + for _, file := range r.EnvFile { + args = append(args, "--env-file", file) + } + if r.Network != "" { + args = append(args, "--network", r.Network) + } + args = append(args, + "--label", specLabel+"="+want, "--label", idLabel+"="+r.ID) + for _, k := range sortedKeys(r.Env) { + args = append(args, "--env", k+"="+r.Env[k]) + } + for _, p := range r.Ports { + args = append(args, "--publish", p) + } + for _, v := range r.Volumes { + args = append(args, "--volume", v) + } + for _, h := range r.Hosts { + // Written into the container's own hosts file by the runtime. Per container rather than + // by editing the machine's resolver configuration: that file belongs to something else on + // most machines, and a host that edited it would be fighting whatever owns it on every + // boot — the fault this host exists to avoid, in the place it would be hardest to see. + args = append(args, "--add-host", h) + } + args = append(args, r.Image) + args = append(args, r.Args...) + + if _, err := run(ctx, cri, args...); err != nil { + return out, fmt.Errorf("starting container %s: %w", r.Name, err) + } + + // Read back. `docker run --detach` returning an id says the container was created, not + // that it is still running — a container whose entrypoint exits immediately satisfies the + // command exactly as one that came up does. + after, err := containerState(ctx, r.Name, run) + if err != nil { + return out, fmt.Errorf("started container %s and it is not there: %w", r.Name, err) + } + if !after.Running { + return out, fmt.Errorf( + "container %s was started and is not running. It exited; ask the runtime for its "+ + "logs", r.Name) + } + if after.Spec != want { + return out, fmt.Errorf("container %s is not the one that was declared after creating it", r.Name) + } + + out.Action = "created" + if existed { + out.Action = "updated" + if len(reasons) > 0 { + out.Detail = "recreated to pick up " + strings.Join(reasons, ", ") + } else { + out.Detail = "replaced; a container's configuration is fixed when it is created" + } + } + return out, nil +} + +// restartedBy is which of the named resources changed this pass — the reason a container or service +// must be brought back rather than left as it is (novox/hq 04-ISSUES/009). +func restartedBy(restartOn []string, changed map[string]bool) []string { + var which []string + for _, id := range restartOn { + if changed[id] { + which = append(which, id) + } + } + return which +} + +func sortedKeys(m map[string]string) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +// applyAction runs something the bundle declared, and never learns what it means. +// +// Verify does double duty, and that is the design rather than a convenience: it is both the +// idempotency check and the read-back. Running it first is how the host knows whether there is +// anything to do — it does not know what a database is, so "is the database there" is a +// question only the declaration can ask. Running it again afterwards is how the host knows the +// command had the effect it claimed (novox/hq ADR 0005). +func applyAction(ctx context.Context, r *declaration.Action, run Runner) (Outcome, error) { + out := begin(r) + + if _, err := runAction(ctx, r, r.Verify, run); err == nil { + out.Action = "unchanged" + out.Detail = "already true" + return out, nil + } + + if _, err := runAction(ctx, r, r.Command, run); err != nil { + return out, fmt.Errorf("running the action: %w", err) + } + + if _, err := runAction(ctx, r, r.Verify, run); err != nil { + return out, fmt.Errorf( + "the action ran without error and its own verify still fails: %w\n\n"+ + "The command reported success and the thing it was for did not happen, which "+ + "is exactly what verify exists to catch", err) + } + + out.Action = "created" + out.Detail = "verify was false and is now true" + return out, nil +} + +// runAction runs one of an action's command lines, on the machine or inside a container. +func runAction(ctx context.Context, r *declaration.Action, argv []string, run Runner) (string, error) { + if len(argv) == 0 { + return "", errors.New("no command") + } + if r.In != "" { + return run(ctx, "docker", append([]string{"exec", r.In}, argv...)...) + } + return run(ctx, argv[0], argv[1:]...) +} + +// Container runtimes the host knows how to ask. +// +// Two, because two exist on machines the mesh runs on. The list is short on purpose: each entry +// is a claim that its probe and its CLI have been checked, not that a binary of that name might +// work (novox/hq ADR 0005). +// +// The probe differs and the rest does not, which is what makes this a lookup rather than an +// interface. `docker info --format {{.ServerVersion}}` fails on podman — the field does not +// exist in its report — while `run`, `inspect --format` and `rm -f` are identical, including +// docker's own Go template syntax for reading state and labels. +var containerRuntimes = []struct { + command string + // probe asks the runtime for its version in the form THAT runtime understands. It must + // prove the runtime is FUNCTIONING, never that a binary is on disk + // (novox/hq 04-ISSUES/007). + probe []string +}{ + {command: "docker", probe: []string{"info", "--format", "{{.ServerVersion}}"}}, + {command: "podman", probe: []string{"info", "--format", "{{.Version.Version}}"}}, +} + +// containerRuntime returns the runtime this machine actually has, or says there is none. +// +// Detected rather than declared, because a machine already carrying one keeps it: adoption +// takes over what is there rather than replacing it (novox/hq research 012). Which runtime a +// machine has is reported upward in the profile; what to install on a machine with none is the +// control plane's decision, not this one's. +func containerRuntime(ctx context.Context, run Runner) (string, error) { + var tried []string + for _, rt := range containerRuntimes { + if _, err := run(ctx, rt.command, rt.probe...); err == nil { + return rt.command, nil + } + tried = append(tried, rt.command) + } + return "", fmt.Errorf( + "no container runtime answers on this machine (tried %s)", strings.Join(tried, ", ")) +} + +// digestOf is how this host recognises what it wrote. +// +// A digest rather than the content: the store is read on every reconcile and sits beside the +// state on disk, and keeping every managed file twice would make it grow with the machine rather +// than with the number of resources. +func digestOf(content string) string { + sum := sha256.Sum256([]byte(content)) + return hex.EncodeToString(sum[:]) +} + +// holds is the machine's own ports a resource occupies. +// +// **What the declaration binds, not what is open.** A machine's open ports are a moving target — +// something a person started, a connection the kernel handed out — and assigning around them would +// mean a port that was free when it was asked for and taken when it was used. What a resource +// declares is stable, and it is the half the mesh can be responsible for. +func holds(resource declaration.Resource) []int { + container, ok := resource.(*declaration.Container) + if !ok { + return nil + } + var out []int + for _, mapping := range container.Ports { + // "8080:80", or "127.0.0.1:8080:80" when an address was named. The machine's port is the + // one before the last colon; the last is inside the container and is not the machine's. + parts := strings.Split(mapping, ":") + if len(parts) < 2 { + continue + } + port, err := strconv.Atoi(strings.TrimSpace(parts[len(parts)-2])) + if err != nil { + continue + } + out = append(out, port) + } + return out +} diff --git a/internal/apply/apply_test.go b/internal/apply/apply_test.go new file mode 100644 index 0000000..0990fca --- /dev/null +++ b/internal/apply/apply_test.go @@ -0,0 +1,1494 @@ +package apply + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/system" +) + +// Each test names the decision it defends (novox/hq ADR 0017). + +func parse(t *testing.T, raw string) *declaration.Declaration { + t.Helper() + d, err := declaration.Parse([]byte(raw)) + if err != nil { + t.Fatalf("fixture is not a valid declaration: %v", err) + } + return d +} + +// noServices refuses to run anything. Used where a test declares no services, so that a test +// which accidentally reaches the service manager fails loudly instead of passing quietly. +func noServices(context.Context, string, ...string) (string, error) { + return "", errors.New("this test declares no services and should not have run a command") +} + +func TestApplyingTwiceChangesNothingTheSecondTime(t *testing.T) { + // Idempotence is what makes an apply safe to run on a schedule. Without it, a host that + // reconciles every few minutes rewrites files forever and every reader sees churn. + dir := t.TempDir() + d := parse(t, `{"declaration":1,"resources":[ + {"id":"d","type":"directory","path":"`+dir+`/etc","mode":"0755"}, + {"id":"f","type":"file","path":"`+dir+`/etc/a.conf","content":"hello\n","mode":"0640"} + ]}`) + + first, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + if !first.Changed() { + t.Fatal("the first apply on an empty machine changed nothing") + } + + second, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + if second.Changed() { + t.Errorf("the second apply changed something: %+v", second.Outcomes) + } +} + +// Defends novox/hq ADR 0011: a managed file is generated onto a node and never edited there. +// +// Not by overwriting silently — by noticing. An edit that vanishes without a word is how somebody +// spends an afternoon re-fixing a bug they already fixed. +func TestADriftedMachineIsReturned(t *testing.T) { + // The other half of idempotence, and the half that matters: converging is not "do nothing + // if the state file says it was done". The machine is read, not the record. + dir := t.TempDir() + path := filepath.Join(dir, "a.conf") + d := parse(t, `{"declaration":1,"resources":[ + {"id":"f","type":"file","path":"`+path+`","content":"correct\n","mode":"0644"} + ]}`) + + _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte("someone edited this\n"), 0o644); err != nil { + t.Fatal(err) + } + + report, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + if !report.Changed() { + t.Fatal("a drifted file was left drifted") + } + got, _ := os.ReadFile(path) + if string(got) != "correct\n" { + t.Errorf("the file was not returned: %q", got) + } +} + +func TestADroppedResourceIsRemoved(t *testing.T) { + // novox/hq ADR 0005: the host removes what it previously applied and is no longer + // declared. Removing a line from a declaration is an act with an effect. + dir := t.TempDir() + keep := filepath.Join(dir, "keep.conf") + drop := filepath.Join(dir, "drop.conf") + + both := parse(t, `{"declaration":1,"resources":[ + {"id":"keep","type":"file","path":"`+keep+`","content":"a\n"}, + {"id":"drop","type":"file","path":"`+drop+`","content":"b\n"} + ]}`) + _, state, err := Apply(context.Background(), archHost(t), both, store.State{}, store.OriginCarried, noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + + one := parse(t, `{"declaration":1,"resources":[ + {"id":"keep","type":"file","path":"`+keep+`","content":"a\n"} + ]}`) + report, state, err := Apply(context.Background(), archHost(t), one, state, store.OriginCarried, noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + + if _, err := os.Stat(drop); !errors.Is(err, os.ErrNotExist) { + t.Error("a resource dropped from the declaration was left on the machine") + } + if _, err := os.Stat(keep); err != nil { + t.Error("a declared resource was removed") + } + if _, still := state.Find("drop"); still { + t.Error("the host still believes it owns what it removed") + } + if report.Outcomes[0].Action != "removed" { + t.Errorf("removal is not reported first: %+v", report.Outcomes) + } +} + +func TestNothingTheHostDidNotCreateIsTouched(t *testing.T) { + // The boundary the whole removal rule turns on. A machine has things on it the mesh did + // not put there, and a converger that treats "not declared" as "must not exist" deletes + // them. Authoritative over its own footprint; inert everywhere else. + dir := t.TempDir() + stranger := filepath.Join(dir, "not-ours.conf") + if err := os.WriteFile(stranger, []byte("someone else's\n"), 0o644); err != nil { + t.Fatal(err) + } + + d := parse(t, `{"declaration":1,"resources":[ + {"id":"ours","type":"file","path":"`+filepath.Join(dir, "ours.conf")+`","content":"a\n"} + ]}`) + if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil); err != nil { + t.Fatal(err) + } + + got, err := os.ReadFile(stranger) + if err != nil || string(got) != "someone else's\n" { + t.Error("a file the host did not create was removed or changed") + } +} + +func TestARenameToTheSamePathDoesNotDeleteTheNewFile(t *testing.T) { + // Why removal happens FIRST. A resource leaving a declaration while another arrives at the + // same path is an ordinary rename; removing afterwards would delete the file just written. + dir := t.TempDir() + path := filepath.Join(dir, "shared.conf") + + before := parse(t, `{"declaration":1,"resources":[ + {"id":"old","type":"file","path":"`+path+`","content":"old\n"} + ]}`) + _, state, err := Apply(context.Background(), archHost(t), before, store.State{}, store.OriginCarried, noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + + after := parse(t, `{"declaration":1,"resources":[ + {"id":"new","type":"file","path":"`+path+`","content":"new\n"} + ]}`) + if _, _, err := Apply(context.Background(), archHost(t), after, state, store.OriginCarried, noServices, nil, nil); err != nil { + t.Fatal(err) + } + + got, err := os.ReadFile(path) + if err != nil { + t.Fatalf("the renamed resource is gone: %v", err) + } + if string(got) != "new\n" { + t.Errorf("content is %q, want the new one", got) + } +} + +func TestAFailedStepFailsTheApplyAndTheRestIsStillAttempted(t *testing.T) { + // The apply fails, names the resource, and carries what did happen — because the machine is + // in whatever state the apply reached and the only honest thing to hand back is that list. + // + // **And everything is attempted.** It used to stop at the first failure, which made one + // broken resource hold the whole machine hostage: a module declaring a package that does not + // exist meant every module after it was never applied, for ever + // (novox/hq 04-ISSUES/011). The case for stopping was that a later resource may depend on an + // earlier one — and it still may, and it then fails its own check and is reported, which is + // more information than not attempting it. + dir := t.TempDir() + blocker := filepath.Join(dir, "blocker") + if err := os.WriteFile(blocker, []byte("i am a file\n"), 0o644); err != nil { + t.Fatal(err) + } + + d := parse(t, `{"declaration":1,"resources":[ + {"id":"fine","type":"file","path":"`+filepath.Join(dir, "fine.conf")+`","content":"a\n"}, + {"id":"doomed","type":"directory","path":"`+blocker+`"}, + {"id":"after","type":"file","path":"`+filepath.Join(dir, "after.conf")+`","content":"b\n"} + ]}`) + + _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil) + if err == nil { + t.Fatal("an impossible resource did not fail the apply") + } + + var applyErr *Error + if !errors.As(err, &applyErr) { + t.Fatalf("expected an apply error, got %T", err) + } + if applyErr.Resource != "doomed" { + t.Errorf("the failure names %q, not the resource that failed", applyErr.Resource) + } + // Everything that worked is in the report, before and after the failure. + if len(applyErr.Done.Outcomes) != 2 { + t.Errorf("the error does not carry what was applied: %+v", applyErr.Done.Outcomes) + } + if _, err := os.Stat(filepath.Join(dir, "after.conf")); err != nil { + t.Error("a resource after the failing one was never attempted, so one broken module " + + "still blocks every module after it") + } +} + +func TestEveryFailureIsCountedNotJustTheFirst(t *testing.T) { + // "One thing failed" and "eleven things failed" are different machines, and the first line is + // what somebody reads. + dir := t.TempDir() + for _, name := range []string{"one", "two"} { + if err := os.WriteFile(filepath.Join(dir, name), []byte("a file\n"), 0o644); err != nil { + t.Fatal(err) + } + } + d := parse(t, `{"declaration":1,"resources":[ + {"id":"first","type":"directory","path":"`+filepath.Join(dir, "one")+`"}, + {"id":"second","type":"directory","path":"`+filepath.Join(dir, "two")+`"} + ]}`) + + _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil) + if err == nil { + t.Fatal("two impossible resources did not fail the apply") + } + var applyErr *Error + if !errors.As(err, &applyErr) { + t.Fatalf("got %T", err) + } + if applyErr.Others != 1 { + t.Errorf("the failure says %d others also failed, and one did", applyErr.Others) + } + if !strings.Contains(err.Error(), "one other resource also failed") { + t.Errorf("the message does not say others failed: %v", err) + } +} + +func TestNothingIsRecordedUntilItWorked(t *testing.T) { + // novox/hq ADR 0018. A record written before the fact restates the request in a new place + // and inherits none of the authority of having happened. + dir := t.TempDir() + blocker := filepath.Join(dir, "blocker") + if err := os.WriteFile(blocker, []byte("x\n"), 0o644); err != nil { + t.Fatal(err) + } + d := parse(t, `{"declaration":1,"resources":[ + {"id":"doomed","type":"directory","path":"`+blocker+`"} + ]}`) + + _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil) + if err == nil { + t.Fatal("expected a failure") + } + if _, claimed := state.Find("doomed"); claimed { + t.Error("the host recorded owning something it failed to apply") + } +} + +func TestAModeIsMaintainedNotJustSet(t *testing.T) { + // A permission set at creation is not a permission maintained — this repository has + // already paid for that once, with generated files left world-readable because the mode + // applied only when the file was first written. + dir := t.TempDir() + path := filepath.Join(dir, "secret.conf") + d := parse(t, `{"declaration":1,"resources":[ + {"id":"f","type":"file","path":"`+path+`","content":"s\n","mode":"0600"} + ]}`) + + _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + if err := os.Chmod(path, 0o666); err != nil { + t.Fatal(err) + } + + report, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + info, _ := os.Stat(path) + if info.Mode().Perm() != 0o600 { + t.Errorf("mode is %o after reconciling, want 0600", info.Mode().Perm()) + } + if !report.Changed() { + t.Error("a mode that had drifted was reported as unchanged") + } +} + +func TestAServiceIsReadBackNotAssumed(t *testing.T) { + // `systemctl start` returning zero says the transaction was accepted, not that the unit is + // running. A unit that starts and immediately dies satisfies the command. + started := false + run := func(ctx context.Context, name string, args ...string) (string, error) { + if args[0] == "show" { + if started { + return "LoadState=loaded\nActiveState=failed\n", nil // started, then died + } + return "LoadState=loaded\nActiveState=inactive\n", nil + } + started = true + return "", nil // `systemctl start` succeeds + } + + d := parse(t, `{"declaration":1,"resources":[ + {"id":"s","type":"service","unit":"doomed.service","state":"running"} + ]}`) + _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) + if err == nil { + t.Fatal("a service that died immediately was reported as running") + } + if !strings.Contains(err.Error(), "asked to be running and is stopped") { + t.Errorf("the failure does not say what was observed: %v", err) + } +} + +func TestAnUnknownServiceStateIsRefusedNotGuessed(t *testing.T) { + run := func(ctx context.Context, name string, args ...string) (string, error) { + return "LoadState=loaded\nActiveState=reticent\n", nil + } + d := parse(t, `{"declaration":1,"resources":[ + {"id":"s","type":"service","unit":"odd.service","state":"running"} + ]}`) + _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) + if err == nil || !strings.Contains(err.Error(), "neither running nor stopped") { + t.Errorf("an unrecognised service state was not refused: %v", err) + } +} + +func TestADroppedServiceIsStoppedNotDeleted(t *testing.T) { + // The host did not install the unit and does not own the unit file — only the state it put + // the unit into. + var commands []string + run := func(ctx context.Context, name string, args ...string) (string, error) { + commands = append(commands, strings.Join(args, " ")) + if args[0] == "show" { + return "LoadState=loaded\nActiveState=active\n", nil + } + return "", nil + } + state := store.State{Resources: []store.Applied{ + {ID: "s", Type: "service", Target: "gone.service"}, + }} + d := parse(t, `{"declaration":1,"resources":[ + {"id":"other","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"} + ]}`) + + if _, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, run, nil, nil); err != nil { + t.Fatal(err) + } + joined := strings.Join(commands, "; ") + if !strings.Contains(joined, "stop gone.service") { + t.Errorf("the dropped service was not stopped: %s", joined) + } + if strings.Contains(joined, "disable") || strings.Contains(joined, "mask") { + t.Errorf("the host did more than stop a unit it does not own: %s", joined) + } +} + +func TestAUnitThatDoesNotExistIsNotStopped(t *testing.T) { + // Found by applying inside a raised machine. `systemctl is-active` says "inactive" for a + // unit that DOES NOT EXIST exactly as it does for one that is installed and stopped, so + // declaring a unit stopped reported success for a unit the host cannot manage at all. + // + // Absence read as satisfaction — 04-ISSUES/007 wearing a different hat, and the mirror of + // the degraded-init bug the capability detector had. + absent := func(ctx context.Context, name string, args ...string) (string, error) { + return "LoadState=not-found\nActiveState=inactive\n", nil + } + d := parse(t, `{"declaration":1,"resources":[ + {"id":"s","type":"service","unit":"never-installed.service","state":"stopped"} + ]}`) + + _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, absent, nil, nil) + if err == nil { + t.Fatal("a unit that does not exist was reported as satisfactorily stopped") + } + if !strings.Contains(err.Error(), "does not exist on this machine") { + t.Errorf("the failure does not say the unit is absent: %v", err) + } + if _, claimed := state.Find("s"); claimed { + t.Error("the host recorded owning a unit that is not installed") + } +} + +func TestAMaskedUnitIsRefused(t *testing.T) { + // Masked means someone deliberately made it unstartable. Applying over that would undo a + // decision the host did not make and cannot see the reason for. + masked := func(ctx context.Context, name string, args ...string) (string, error) { + return "LoadState=masked\nActiveState=inactive\n", nil + } + d := parse(t, `{"declaration":1,"resources":[ + {"id":"s","type":"service","unit":"masked.service","state":"running"} + ]}`) + if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, masked, nil, nil); err == nil { + t.Fatal("a masked unit was accepted") + } +} + +func TestForgettingAUnitThatIsGoneDoesNotStrandTheNode(t *testing.T) { + // Found on a real machine. Removing an orphaned service runs `systemctl stop`, which fails + // when the unit no longer exists — and a failure there fails the whole apply. A host + // holding a record of an uninstalled unit could then apply NOTHING, ever, with no way out + // but editing its state by hand. + // + // Removal is idempotent for the same reason os.RemoveAll is: the desired end state is + // already true. + var stopped bool + run := func(ctx context.Context, name string, args ...string) (string, error) { + if args[0] == "show" { + return "LoadState=not-found\nActiveState=inactive\n", nil + } + stopped = true + return "", errors.New("systemctl exited 5: Unit not loaded") + } + known := store.State{Resources: []store.Applied{ + {ID: "gone", Type: "service", Target: "uninstalled.service"}, + }} + d := parse(t, `{"declaration":1,"resources":[ + {"id":"f","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"} + ]}`) + + report, state, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil, nil) + if err != nil { + t.Fatalf("a vanished unit stranded the apply: %v", err) + } + if stopped { + t.Error("the host tried to stop a unit that does not exist") + } + if _, still := state.Find("gone"); still { + t.Error("the host still believes it owns a unit that is gone") + } + // "forgotten", not "removed": the host stopped believing it owns the unit, and did not + // remove anything, because there was nothing there to remove. Reporting an effect it did + // not have would be the same class of untruth as reporting a package uninstalled. + if report.Outcomes[0].Action != "forgotten" { + t.Errorf("the vanished unit was not reported as forgotten: %+v", report.Outcomes) + } +} + +// --- package, container and action (novox/hq 07-the-substrate.md, ADR 0006, ADR 0005) --- + +func parseTrusted(t *testing.T, raw string) *declaration.Declaration { + t.Helper() + d, err := declaration.ParseTrusted([]byte(raw)) + if err != nil { + t.Fatalf("fixture is not a valid declaration: %v", err) + } + return d +} + +const pinned = "docker.io/library/postgres@sha256:" + + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + +func TestABrokenPackageDatabaseIsNotReadAsNotInstalled(t *testing.T) { + // The same trap serviceState documents. `pacman -Q x` exits non-zero both for a package + // that is not installed and for a database that cannot be read — so believing the first + // answer would silently reinstall on a machine whose package manager is broken, or report + // "installed nothing" as success. The apply must fail instead. + run := func(ctx context.Context, name string, args ...string) (string, error) { + return "", errors.New("pacman: error: could not lock database") + } + d := parseTrusted(t, `{"declaration":1,"resources":[ + {"id":"rt","type":"package","package":"docker"} + ]}`) + + _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) + if err == nil { + t.Fatal("a broken package database was read as 'not installed'") + } + if !strings.Contains(err.Error(), "does not answer") { + t.Errorf("failed for the wrong reason: %v", err) + } +} + +func TestAnInstalledPackageIsNotReinstalled(t *testing.T) { + var installed bool + run := func(ctx context.Context, name string, args ...string) (string, error) { + if args[0] == "-S" { + installed = true + } + return "docker 27.0-1\n", nil // -Q succeeds for everything + } + d := parseTrusted(t, `{"declaration":1,"resources":[ + {"id":"rt","type":"package","package":"docker"} + ]}`) + + report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) + if err != nil { + t.Fatalf("apply failed: %v", err) + } + if installed { + t.Error("a package that was already present was installed again") + } + if report.Changed() { + t.Errorf("an already-installed package reported a change: %+v", report.Outcomes) + } +} + +func TestAPackageIsNeverUninstalled(t *testing.T) { + // Deliberate: the host cannot know what else needs the package. Uninstalling a container + // runtime because a declaration changed would stop every container on the node, and the + // machine may have had it before the mesh ever saw it. Undeclaring is not "remove it". + var uninstalled bool + run := func(ctx context.Context, name string, args ...string) (string, error) { + if len(args) > 0 && (args[0] == "-R" || args[0] == "-Rs") { + uninstalled = true + } + return "", nil + } + known := store.State{Resources: []store.Applied{ + {ID: "rt", Type: "package", Target: "docker"}, + }} + d := parseTrusted(t, `{"declaration":1,"resources":[ + {"id":"f","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"} + ]}`) + + report, state, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil, nil) + if err != nil { + t.Fatalf("dropping a package stranded the apply: %v", err) + } + if uninstalled { + t.Fatal("the host uninstalled a package") + } + if _, still := state.Find("rt"); still { + t.Error("the host still believes it owns the package") + } + // "forgotten", not "removed" — the host must not claim an effect it declined to have. + if report.Outcomes[0].Action != "forgotten" { + t.Errorf("dropping a package was not reported as forgotten: %+v", report.Outcomes[0]) + } +} + +func TestAnActionThatIsAlreadyTrueDoesNotRun(t *testing.T) { + // Verify is the idempotency check as well as the read-back. The host does not know what a + // database is, so "is it already there" is a question only the declaration can ask. + var ran bool + run := func(ctx context.Context, name string, args ...string) (string, error) { + if name == "create-db" { + ran = true + } + return "", nil // verify passes + } + d := parseTrusted(t, `{"declaration":1,"resources":[ + {"id":"db","type":"action","command":["create-db","mesh"],"verify":["has-db","mesh"]} + ]}`) + + report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) + if err != nil { + t.Fatalf("apply failed: %v", err) + } + if ran { + t.Error("an action whose verify already passed was run anyway") + } + if report.Changed() { + t.Errorf("an already-satisfied action reported a change: %+v", report.Outcomes) + } +} + +func TestAnActionThatSucceedsAndDoesNothingFails(t *testing.T) { + // The whole reason verify is mandatory: a command that exits zero and has no effect is + // this repository's most expensive failure shape. Here the command "succeeds" every time + // and verify never passes. + run := func(ctx context.Context, name string, args ...string) (string, error) { + if name == "has-db" { + return "", errors.New("no such database") + } + return "", nil + } + d := parseTrusted(t, `{"declaration":1,"resources":[ + {"id":"db","type":"action","command":["create-db","mesh"],"verify":["has-db","mesh"]} + ]}`) + + _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) + if err == nil { + t.Fatal("an action that reported success and did nothing was accepted") + } + if !strings.Contains(err.Error(), "verify still fails") { + t.Errorf("failed for the wrong reason: %v", err) + } + if _, recorded := state.Find("db"); recorded { + t.Error("an action that did not work was recorded as applied") + } +} + +func TestAnActionRunsInsideTheContainerItNames(t *testing.T) { + // Steps 2 and 3 of the bootstrap act on something inside the store's container, before + // there is any mesh to ask. + var sawExec bool + run := func(ctx context.Context, name string, args ...string) (string, error) { + if name == "docker" && args[0] == "exec" && args[1] == "store" { + sawExec = true + return "", nil + } + return "", errors.New("not run in the container") + } + d := parseTrusted(t, `{"declaration":1,"resources":[ + {"id":"db","type":"action","in":"store","command":["createdb","mesh"],"verify":["psql","-lqt"]} + ]}`) + + if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil); err != nil { + t.Fatalf("apply failed: %v", err) + } + if !sawExec { + t.Error("an action naming a container did not run inside it") + } +} + +func TestAContainerThatExitsImmediatelyFailsTheApply(t *testing.T) { + // `docker run --detach` returning an id says the container was created, not that it is + // still running. A container whose entrypoint dies satisfies the command exactly as one + // that came up does — which is the read-back rule, in the place it matters most. + run := func(ctx context.Context, name string, args ...string) (string, error) { + switch { + case args[0] == "info": + return "27.0\n", nil + case args[0] == "inspect": + return "false\t" + "", nil // exists, not running + case args[0] == "run": + return "deadbeef\n", nil + } + return "", nil + } + d := parseTrusted(t, `{"declaration":1,"resources":[ + {"id":"store","type":"container","name":"store","image":"`+pinned+`"} + ]}`) + + _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) + if err == nil { + t.Fatal("a container that exited immediately was reported as applied") + } + if !strings.Contains(err.Error(), "is not running") { + t.Errorf("failed for the wrong reason: %v", err) + } + if _, recorded := state.Find("store"); recorded { + t.Error("a container that is not running was recorded as applied") + } +} + +func TestAContainerWhoseDeclarationChangedIsReplaced(t *testing.T) { + // A container's configuration is fixed when it is created, so any change is a replacement. + // The spec label is what makes the difference visible without diffing everything the + // runtime reports — which cannot be done reliably, because a runtime normalises what it is + // given and that is indistinguishable from drift. + d := parseTrusted(t, `{"declaration":1,"resources":[ + {"id":"store","type":"container","name":"store","image":"`+pinned+`","env":{"PGDATA":"/data"}} + ]}`) + want := containerSpec(d.Resources[0].(*declaration.Container)) + + var removed, created bool + run := func(ctx context.Context, name string, args ...string) (string, error) { + switch args[0] { + case "info": + return "27.0\n", nil + case "inspect": + if created { + return "true\t" + want, nil + } + return "true\tsome-older-spec", nil + case "rm": + removed = true + return "", nil + case "run": + created = true + return "deadbeef\n", nil + } + return "", nil + } + + report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) + if err != nil { + t.Fatalf("apply failed: %v", err) + } + if !removed || !created { + t.Fatalf("a changed container was not replaced (removed=%v created=%v)", removed, created) + } + if report.Outcomes[0].Action != "updated" { + t.Errorf("a replacement was not reported as an update: %+v", report.Outcomes[0]) + } +} + +func TestAContainerThatMatchesIsLeftAlone(t *testing.T) { + d := parseTrusted(t, `{"declaration":1,"resources":[ + {"id":"store","type":"container","name":"store","image":"`+pinned+`","env":{"PGDATA":"/data"}} + ]}`) + spec := containerSpec(d.Resources[0].(*declaration.Container)) + + var touched bool + run := func(ctx context.Context, name string, args ...string) (string, error) { + switch args[0] { + case "info": + return "27.0\n", nil + case "inspect": + return "true\t" + spec, nil + } + touched = true + return "", nil + } + + report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) + if err != nil { + t.Fatalf("apply failed: %v", err) + } + if touched { + t.Error("a container that already matched was restarted") + } + if report.Changed() { + t.Errorf("a matching container reported a change: %+v", report.Outcomes) + } +} + +func TestAContainerIsRecreatedWhenARestartOnResourceChanged(t *testing.T) { + // A container reads a mounted file once, at start. When the file changed this pass but the + // container's spec did not, the plain "spec matches, leave it" rule would keep the process + // holding the old value for ever, with every check passing (novox/hq 04-ISSUES/009). A + // container names the resources it must reflect in restart-on, the same as a service, and the + // host recreates it. Here the config file is fresh, so it is written this pass, and the + // already-running-and-matching container must still be replaced. + dir := t.TempDir() + conf := filepath.Join(dir, "config.json") + d := parseTrusted(t, `{"declaration":1,"resources":[ + {"id":"config","type":"file","path":"`+conf+`","content":"{\"token\":\"new\"}\n"}, + {"id":"app","type":"container","name":"app","image":"`+pinned+`","restart-on":["config"]} + ]}`) + spec := containerSpec(d.Resources[1].(*declaration.Container)) + + var removed, created bool + run := func(ctx context.Context, name string, args ...string) (string, error) { + switch args[0] { + case "info": + return "27.0\n", nil + case "inspect": + // The container already exists, running, with exactly the spec it is declared with — + // only the mounted file changed. + return "true\t" + spec, nil + case "rm": + removed = true + return "", nil + case "run": + created = true + return "deadbeef\n", nil + } + return "", nil + } + + report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) + if err != nil { + t.Fatalf("apply failed: %v", err) + } + if !removed || !created { + t.Fatalf("a container was not recreated when its restart-on file changed (removed=%v created=%v)", removed, created) + } + app := report.Outcomes[len(report.Outcomes)-1] + if app.Action != "updated" || !strings.Contains(app.Detail, "config") { + t.Errorf("the recreation did not name why it happened: %+v", app) + } +} + +func TestRestartOnFiresOnlyForResourcesThatChanged(t *testing.T) { + // restart-on must not mean "always restart": it names resources, and only a resource that + // changed this pass is a reason. This is the guard shared by containers and services + // (novox/hq 04-ISSUES/009), so a container that reflects an unchanged file is left running. + changed := map[string]bool{"other": true} + if got := restartedBy([]string{"config"}, changed); len(got) != 0 { + t.Errorf("an unchanged resource was treated as a reason to restart: %v", got) + } + changed["config"] = true + if got := restartedBy([]string{"config", "missing"}, changed); len(got) != 1 || got[0] != "config" { + t.Errorf("restart-on did not name exactly the changed resource: %v", got) + } +} + +// --- boot state (novox/hq: a unit started but not enabled stops being true at the next reboot) --- + +// systemctlStub answers `show` and `is-enabled` the way systemd does, and records the verbs it +// was asked to perform. Real command shapes, because the trap being tested is what systemd +// actually says rather than what a fake would. +func systemctlStub(t *testing.T, load, active, enabled string, verbs *[]string) Runner { + t.Helper() + return func(ctx context.Context, name string, args ...string) (string, error) { + switch args[0] { + case "show": + return "LoadState=" + load + "\nActiveState=" + active + "\n", nil + case "is-enabled": + // Non-zero for everything but "enabled" — the exit code says nothing useful, which + // is the whole reason this reads the output. + if enabled == "enabled" { + return enabled + "\n", nil + } + return enabled + "\n", errors.New("exit status 1") + case "enable": + *verbs = append(*verbs, "enable") + enabled = "enabled" + return "", nil + case "disable": + *verbs = append(*verbs, "disable") + enabled = "disabled" + return "", nil + case "start": + *verbs = append(*verbs, "start") + active = "active" + return "", nil + case "stop": + *verbs = append(*verbs, "stop") + active = "inactive" + return "", nil + } + return "", nil + } +} + +func TestAServiceIsEnabledAtBootWhenAsked(t *testing.T) { + // The gap this closes: the host could start a unit and never make it survive a reboot, so + // the declaration reported success and stopped being true at the next power cut. + var verbs []string + d := parseTrusted(t, `{"declaration":1,"resources":[ + {"id":"rt","type":"service","unit":"docker.service","state":"running","boot":"enabled"} + ]}`) + + report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, + systemctlStub(t, "loaded", "inactive", "disabled", &verbs), nil, nil) + if err != nil { + t.Fatalf("apply failed: %v", err) + } + if len(verbs) != 2 || verbs[0] != "enable" || verbs[1] != "start" { + t.Errorf("expected enable then start, got %v", verbs) + } + if report.Outcomes[0].Action != "updated" { + t.Errorf("enabling and starting was not reported as an update: %+v", report.Outcomes[0]) + } +} + +func TestBootIsEnabledBeforeTheUnitIsStarted(t *testing.T) { + // Order matters when an apply fails part way. Enabled-and-stopped comes back at the next + // boot; running-and-disabled does not. So the more durable half is made true first. + var verbs []string + d := parseTrusted(t, `{"declaration":1,"resources":[ + {"id":"rt","type":"service","unit":"docker.service","state":"running","boot":"enabled"} + ]}`) + if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, + systemctlStub(t, "loaded", "inactive", "disabled", &verbs), nil, nil); err != nil { + t.Fatal(err) + } + if len(verbs) < 2 || verbs[0] != "enable" { + t.Errorf("boot state was not made true first: %v", verbs) + } +} + +func TestAlreadyEnabledAndRunningIsUnchanged(t *testing.T) { + var verbs []string + d := parseTrusted(t, `{"declaration":1,"resources":[ + {"id":"rt","type":"service","unit":"docker.service","state":"running","boot":"enabled"} + ]}`) + + report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, + systemctlStub(t, "loaded", "active", "enabled", &verbs), nil, nil) + if err != nil { + t.Fatalf("apply failed: %v", err) + } + if len(verbs) != 0 { + t.Errorf("a unit already in the declared state was touched: %v", verbs) + } + if report.Changed() { + t.Errorf("an unchanged service reported a change: %+v", report.Outcomes) + } +} + +func TestOmittingBootLeavesItAlone(t *testing.T) { + // Absent means the host asserts nothing. A machine whose operator enabled something must + // not have it silently disabled because a declaration did not mention it. + var verbs []string + d := parseTrusted(t, `{"declaration":1,"resources":[ + {"id":"rt","type":"service","unit":"docker.service","state":"running"} + ]}`) + if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, + systemctlStub(t, "loaded", "inactive", "enabled", &verbs), nil, nil); err != nil { + t.Fatal(err) + } + for _, v := range verbs { + if v == "enable" || v == "disable" { + t.Errorf("boot state was changed by a declaration that did not mention it: %v", verbs) + } + } +} + +func TestAStaticUnitCannotBeEnabled(t *testing.T) { + // `static` is neither enabled nor disabled: the unit has no install section and CANNOT be + // enabled. Reading it as "disabled" would have the host try, fail, and blame the wrong + // thing — the same shape as reading a missing unit as "stopped". + var verbs []string + d := parseTrusted(t, `{"declaration":1,"resources":[ + {"id":"rt","type":"service","unit":"dbus.socket","state":"running","boot":"enabled"} + ]}`) + + _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, + systemctlStub(t, "loaded", "active", "static", &verbs), nil, nil) + if err == nil { + t.Fatal("a static unit was accepted as enable-able") + } + if !strings.Contains(err.Error(), "no install section") { + t.Errorf("failed for the wrong reason: %v", err) + } +} + +func TestAnUnknownBootStateIsRefusedNotGuessed(t *testing.T) { + var verbs []string + d := parseTrusted(t, `{"declaration":1,"resources":[ + {"id":"rt","type":"service","unit":"x.service","state":"running","boot":"enabled"} + ]}`) + _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, + systemctlStub(t, "loaded", "active", "indirect", &verbs), nil, nil) + if err == nil { + t.Fatal("an unrecognised boot state was guessed at instead of refused") + } +} + +// --- more than one container runtime (novox/hq ADR 0005) --- + +func TestTheRuntimeProbeIsPerRuntime(t *testing.T) { + // Verified against a real podman 6.1.0 before this was written: + // + // docker info --format '{{.ServerVersion}}' -> 29.7.2 + // podman info --format '{{.ServerVersion}}' -> Error: can't evaluate field + // ServerVersion in type system.infoReport + // podman info --format '{{.Version.Version}}' -> 6.1.0 + // + // So a single probe cannot find both, and a host that used docker's would report a machine + // with podman as having no container runtime at all. + for _, tc := range []struct { + name, present, wantProbe string + }{ + {"docker", "docker", "{{.ServerVersion}}"}, + {"podman", "podman", "{{.Version.Version}}"}, + } { + t.Run(tc.name, func(t *testing.T) { + var probedWith string + run := func(ctx context.Context, name string, args ...string) (string, error) { + if name != tc.present { + return "", errors.New("not installed") + } + if args[0] == "info" { + probedWith = args[2] + } + return "ok\n", nil + } + got, err := containerRuntime(context.Background(), run) + if err != nil { + t.Fatalf("%s was present and was not found: %v", tc.present, err) + } + if got != tc.present { + t.Errorf("found %q, expected %q", got, tc.present) + } + if probedWith != tc.wantProbe { + t.Errorf("probed %s with %q; that template does not work on it", + tc.present, probedWith) + } + }) + } +} + +func TestAContainerUsesTheRuntimeTheMachineHas(t *testing.T) { + // The applier must not call `docker` on a machine that has podman. Adoption keeps what the + // machine already has (novox/hq research 012), so hardcoding one contradicts it. + var calledWith []string + run := func(ctx context.Context, name string, args ...string) (string, error) { + if name == "docker" { + return "", errors.New("not installed") + } + calledWith = append(calledWith, name) + switch args[0] { + case "info": + return "6.1.0\n", nil + case "inspect": + return "false\t\n", errors.New("no such container") + case "run": + return "deadbeef\n", nil + } + return "", nil + } + d := parseTrusted(t, `{"declaration":1,"resources":[ + {"id":"store","type":"container","name":"store","image":"`+pinned+`"} + ]}`) + + // It will fail at read-back — the stub never reports it running — and what matters is + // WHICH binary it used getting there. + _, _, _ = Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) + + for _, c := range calledWith { + if c != "podman" { + t.Errorf("called %q on a machine that only has podman", c) + } + } + if len(calledWith) == 0 { + t.Error("nothing was called; the runtime was not found") + } +} + +func TestNoRuntimeIsSaidPlainly(t *testing.T) { + // Naming what was tried, because "docker: command not found" on a machine that deliberately + // runs podman sends the reader looking for the wrong thing. + run := func(ctx context.Context, name string, args ...string) (string, error) { + return "", errors.New("not installed") + } + d := parseTrusted(t, `{"declaration":1,"resources":[ + {"id":"store","type":"container","name":"store","image":"`+pinned+`"} + ]}`) + + _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) + if err == nil { + t.Fatal("a machine with no container runtime applied a container") + } + for _, want := range []string{"docker", "podman", "no container runtime"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the failure does not mention %q: %v", want, err) + } + } +} + +// archHost is the system these tests run against. They were written for pacman and systemd, and +// naming that is better than the implicit default it used to be. +func archHost(t *testing.T) system.System { + t.Helper() + s, err := system.For("arch") + if err != nil { + t.Fatal(err) + } + return s +} + +func TestAServiceIsRestartedWhenWhatItReflectsChanges(t *testing.T) { + // A running service does not re-read its configuration. Replace the file, find the service + // already running, do nothing — and the machine keeps behaving as it did while every check + // passes, because the file is right and the service is up. + // + // That is how a third node joining a mesh left the first two carrying a network that no + // longer existed. Found in the lab; this is the shape of the fix. + dir := t.TempDir() + path := filepath.Join(dir, "thing.conf") + + d := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[ + {"id":"conf","type":"file","path":%q,"content":"first\n","mode":"0644"}, + {"id":"svc","type":"service","unit":"thing.service","state":"running","restart-on":["conf"]} + ]}`, path)) + + var commands []string + run := recordingServices(&commands) + + if _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, + store.OriginCarried, run, nil, nil); err != nil { + t.Fatal(err) + } else { + // Second apply with the same content: nothing moved, so nothing restarts. A machine that + // restarted its services on every reconcile would never be steady. + commands = nil + if _, _, err := Apply(context.Background(), archHost(t), d, state, + store.OriginCarried, run, nil, nil); err != nil { + t.Fatal(err) + } + for _, c := range commands { + if strings.Contains(c, "stop") { + t.Errorf("an unchanged declaration restarted the service: %s", c) + } + } + + // Now the file changes. The service is already running and must still be restarted. + changedDecl := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[ + {"id":"conf","type":"file","path":%q,"content":"second\n","mode":"0644"}, + {"id":"svc","type":"service","unit":"thing.service","state":"running","restart-on":["conf"]} + ]}`, path)) + commands = nil + if _, _, err := Apply(context.Background(), archHost(t), changedDecl, state, + store.OriginCarried, run, nil, nil); err != nil { + t.Fatal(err) + } + var stopped, started bool + for _, c := range commands { + if strings.Contains(c, "stop thing.service") { + stopped = true + } + if strings.Contains(c, "start thing.service") { + started = true + } + } + if !stopped || !started { + t.Errorf("the file changed and the service was not restarted; commands were %v", commands) + } + } +} + +func TestAServiceIsNotRestartedByAChangeItDoesNotName(t *testing.T) { + // The list is what it reflects, not everything in the declaration. A service restarted by any + // change anywhere would make every apply a fleet-wide bounce. + dir := t.TempDir() + conf := filepath.Join(dir, "thing.conf") + other := filepath.Join(dir, "unrelated") + + first := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[ + {"id":"conf","type":"file","path":%q,"content":"same\n","mode":"0644"}, + {"id":"other","type":"file","path":%q,"content":"one\n","mode":"0644"}, + {"id":"svc","type":"service","unit":"thing.service","state":"running","restart-on":["conf"]} + ]}`, conf, other)) + + var commands []string + run := recordingServices(&commands) + _, state, err := Apply(context.Background(), archHost(t), first, store.State{}, + store.OriginCarried, run, nil, nil) + if err != nil { + t.Fatal(err) + } + + second := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[ + {"id":"conf","type":"file","path":%q,"content":"same\n","mode":"0644"}, + {"id":"other","type":"file","path":%q,"content":"two\n","mode":"0644"}, + {"id":"svc","type":"service","unit":"thing.service","state":"running","restart-on":["conf"]} + ]}`, conf, other)) + commands = nil + if _, _, err := Apply(context.Background(), archHost(t), second, state, + store.OriginCarried, run, nil, nil); err != nil { + t.Fatal(err) + } + for _, c := range commands { + if strings.Contains(c, "stop") { + t.Errorf("a change to a file the service does not name restarted it: %s", c) + } + } +} + +// recordingServices answers the way a machine with a running unit would, and remembers what it +// was asked to do — which is what a restart has to be proved by, since "running" looks the same +// before and after one. +func recordingServices(commands *[]string) Runner { + return func(_ context.Context, name string, args ...string) (string, error) { + line := name + " " + strings.Join(args, " ") + *commands = append(*commands, line) + switch { + case strings.Contains(line, "is-enabled"): + return "enabled", nil + case strings.Contains(line, "show") && strings.Contains(line, "ActiveState"): + return "LoadState=loaded\nActiveState=active\nSubState=running", nil + } + return "", nil + } +} + +func TestAFileChangedOnTheMachineIsCorrectedAndSaidSo(t *testing.T) { + // The question this answers: how would anybody know somebody edited a managed file? Before + // this they would not. It was rewritten within five minutes and reported as "updated", + // which is what the mesh changing its mind looks like — so the person's change vanished and + // nothing anywhere said why. They edit it again, and again. + dir := t.TempDir() + path := filepath.Join(dir, "thing.conf") + d := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[ + {"id":"conf","type":"file","path":%q,"content":"from the mesh\n","mode":"0644"} + ]}`, path)) + + _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, + store.OriginCarried, noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + + // Somebody edits it. + if err := os.WriteFile(path, []byte("edited by hand\n"), 0o644); err != nil { + t.Fatal(err) + } + + report, state, err := Apply(context.Background(), archHost(t), d, state, + store.OriginCarried, noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + if got := report.Outcomes[0].Action; got != "corrected" { + t.Errorf("a hand edit was reported as %q; the mesh cannot tell it from changing its own "+ + "mind, and neither can anybody reading this", got) + } + + // And it is put back, because holding the machine to what it was told is the point. + back, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(back) != "from the mesh\n" { + t.Errorf("the file was left as %q", back) + } +} + +func TestTheMeshChangingItsMindIsNotDrift(t *testing.T) { + // The other half. A new declaration is an ordinary update and must not read as somebody + // having meddled, or every real change would look like an incident. + dir := t.TempDir() + path := filepath.Join(dir, "thing.conf") + first := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[ + {"id":"conf","type":"file","path":%q,"content":"one\n","mode":"0644"} + ]}`, path)) + second := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[ + {"id":"conf","type":"file","path":%q,"content":"two\n","mode":"0644"} + ]}`, path)) + + _, state, err := Apply(context.Background(), archHost(t), first, store.State{}, + store.OriginCarried, noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + report, _, err := Apply(context.Background(), archHost(t), second, state, + store.OriginCarried, noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + if got := report.Outcomes[0].Action; got != "updated" { + t.Errorf("the mesh changing what it wants was reported as %q", got) + } +} + +func TestAnUntouchedFileIsStillUnchanged(t *testing.T) { + // And nothing about this makes a steady machine look busy. + dir := t.TempDir() + path := filepath.Join(dir, "thing.conf") + d := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[ + {"id":"conf","type":"file","path":%q,"content":"steady\n","mode":"0644"} + ]}`, path)) + + _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, + store.OriginCarried, noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + report, _, err := Apply(context.Background(), archHost(t), d, state, + store.OriginCarried, noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + if got := report.Outcomes[0].Action; got != "unchanged" { + t.Errorf("an untouched file was reported as %q", got) + } +} + +func TestAFailedActionStopsWhatFollows(t *testing.T) { + // An action is the only shape whose purpose is to make something true BEFORE the next thing + // needs it, which is why it is the only one with a verify. The bootstrap is a row of them: + // the store answers, then its databases exist, then their schemas, then the broker. + // + // Carrying on past one that did not happen starts things against a machine that is not ready + // — and on a small machine that is how a database still initialising has its memory taken + // away and shuts down. Observed in the lab, caused by a version of this loop that continued + // past everything. + dir := t.TempDir() + d := parseTrusted(t, `{"declaration":1,"resources":[ + {"id":"gate","type":"action","command":["false"],"verify":["false"]}, + {"id":"after","type":"file","path":"`+filepath.Join(dir, "after.conf")+`","content":"b\n"} + ]}`) + + _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, + ExecRunner, nil, nil) + if err == nil { + t.Fatal("an action that cannot succeed did not fail the apply") + } + var applyErr *Error + if !errors.As(err, &applyErr) { + t.Fatalf("got %T", err) + } + if !applyErr.Gated { + t.Error("the failure does not say that nothing after it was attempted") + } + if _, statErr := os.Stat(filepath.Join(dir, "after.conf")); statErr == nil { + t.Error("the apply continued past a failed action, which is a gate") + } + if !strings.Contains(err.Error(), "nothing after it was attempted") { + t.Errorf("the message does not say the rest was not tried: %v", err) + } +} + +// A container is told which resolver to use, because it does not inherit the machine's names. +// +// A container gets its own `/etc/hosts` holding its own hostname, and a runtime rewrites +// `resolv.conf` — so every internal name the mesh wrote for the machine is invisible to what the +// machine is running. That was hit for real: a database client on one node could not resolve +// another node, on a mesh where both names were correct and present on both machines. +func TestAContainerIsGivenTheMeshsNames(t *testing.T) { + var ran []string + run := func(_ context.Context, name string, args ...string) (string, error) { + if name != "docker" { + return "", errors.New("not installed") + } + switch args[0] { + case "info": + return "29.0.0\n", nil + case "inspect": + return "false\t\n", errors.New("no such container") + case "run": + ran = args + return "deadbeef\n", nil + } + return "", nil + } + d := parseTrusted(t, `{"declaration":1,"resources":[ + {"id":"app","type":"container","name":"app","image":"`+pinned+`", + "hosts":["anchor.internal:10.42.0.1"]} + ]}`) + _, _, _ = Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) + + var told bool + for i, a := range ran { + if a == "--add-host" && i+1 < len(ran) && ran[i+1] == "anchor.internal:10.42.0.1" { + told = true + } + } + if !told { + t.Fatalf("the container cannot reach another machine by name: %v", ran) + } +} + +// And a container given no names is run exactly as before. +func TestAContainerGivenNoNamesIsRunAsBefore(t *testing.T) { + var ran []string + run := func(_ context.Context, name string, args ...string) (string, error) { + if name != "docker" { + return "", errors.New("not installed") + } + switch args[0] { + case "info": + return "29.0.0\n", nil + case "inspect": + return "false\t\n", errors.New("no such container") + case "run": + ran = args + return "deadbeef\n", nil + } + return "", nil + } + d := parseTrusted(t, `{"declaration":1,"resources":[ + {"id":"app","type":"container","name":"app","image":"`+pinned+`"} + ]}`) + _, _, _ = Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) + + for _, a := range ran { + if a == "--add-host" { + t.Fatalf("a container given no names was given some anyway: %v", ran) + } + } +} + +// Defends the ordering half of novox/hq work breakdown 1.3: resources are applied in the order +// the module declared them. +// +// **Already true, and untested until now** — the apply loop walks `d.Resources` and sorts nothing, +// so a module that needs one thing before another says so by writing it first. Worth an assertion +// because it is the kind of property a later change would break silently: sorting the resources +// for any good reason at all — by type, by identity, for a tidier report — would still pass every +// other test in this package. +// +// It is sequence, not readiness. A container started is not a container ready, and nothing here +// waits: what depends on something being *usable* retries, which is what both example +// provisioners do and is more robust than start ordering, because a dependency can also restart +// long after everything was applied. +func TestResourcesAreAppliedInTheOrderTheyWereDeclared(t *testing.T) { + dir := t.TempDir() + var order []string + done := map[string]bool{} + run := func(_ context.Context, name string, args ...string) (string, error) { + order = append(order, name+" "+strings.Join(args, " ")) + // `verify` is the idempotency check, so it has to fail before the step and pass after — + // a stub that always succeeds means every action is already done and nothing ever runs, + // which is what the first version of this test measured. + if name == "check" { + if !done[args[0]] { + return "", fmt.Errorf("not yet") + } + return "", nil + } + done[args[0]] = true + return "", nil + } + + _ = dir + // Trusted, because the link may not carry an action and only a bundle may (novox/hq ADR 0005). + // Actions are used here because they are the one shape whose execution is observable through + // the runner, which is what makes the order visible at all. + d, err := declaration.ParseTrusted([]byte(`{"declaration":1,"resources":[` + + `{"id":"first","type":"action","command":["step","one"],"verify":["check","one"]},` + + `{"id":"second","type":"action","command":["step","two"],"verify":["check","two"]},` + + `{"id":"third","type":"action","command":["step","three"],"verify":["check","three"]}]}`)) + if err != nil { + t.Fatal(err) + } + + if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, + store.OriginCarried, run, nil, nil); err != nil { + t.Fatal(err) + } + + var ran []string + for _, line := range order { + if strings.HasPrefix(line, "step ") { + ran = append(ran, strings.TrimPrefix(line, "step ")) + } + } + want := []string{"one", "two", "three"} + if strings.Join(ran, ",") != strings.Join(want, ",") { + t.Fatalf("declared one, two, three and ran %v — a module that needs one thing before "+ + "another has no way to say so", ran) + } +} + +// **A data directory is never removed by the mesh.** The one failure in this system that cannot +// be undone. +// +// Unassigning a module made its directory an orphan, and an orphan directory was deleted with +// everything under it — a database's files, a mail spool, somebody's uploads — and the report +// said "removed". Reproduced before it was fixed: a module was assigned, a service wrote into +// its directory, the module was unassigned, and the file was gone. +// +// What makes the rule safe rather than merely cautious is the removal order. Everything the mesh +// puts in a directory is itself a declared resource, and orphans are removed in reverse +// declaration order — so what the mesh wrote is already gone by the time the directory is +// reached. Anything still there was put there by something else. +func TestADirectoryHoldingAnythingTheMeshDidNotPutThereIsKept(t *testing.T) { + root := t.TempDir() + data := filepath.Join(root, "keycloak") + + d := declare(t, `{"id":"data","type":"directory","path":"`+data+`","mode":"0700"}`) + _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, + store.OriginDeclared, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + live := filepath.Join(data, "realm.db") + if err := os.WriteFile(live, []byte("everybody's logins"), 0o600); err != nil { + t.Fatal(err) + } + + // The module is unassigned: the mesh declares something else entirely. + after := declare(t, `{"id":"other","type":"directory","path":"`+filepath.Join(root, "other")+`","mode":"0700"}`) + report, _, err := Apply(context.Background(), archHost(t), after, state, + store.OriginDeclared, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + + if _, err := os.Stat(live); err != nil { + t.Fatalf("the data was deleted by unassigning a module: %v", err) + } + // And it is said, rather than left for somebody to notice. A directory quietly left behind is + // how a machine accumulates things nobody can account for. + var said bool + for _, o := range report.Outcomes { + if o.Action == "kept" && strings.Contains(o.Detail, "did not put there") { + said = true + } + } + if !said { + t.Errorf("the directory was kept and nothing reported it: %+v", report.Outcomes) + } +} + +// An empty one is the mesh's own, and goes. +func TestAnEmptyDirectoryIsStillRemoved(t *testing.T) { + root := t.TempDir() + mine := filepath.Join(root, "config") + d := declare(t, `{"id":"c","type":"directory","path":"`+mine+`","mode":"0700"}`) + _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, + store.OriginDeclared, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + after := declare(t, `{"id":"other","type":"directory","path":"`+filepath.Join(root, "other")+`","mode":"0700"}`) + if _, _, err := Apply(context.Background(), archHost(t), after, state, + store.OriginDeclared, nil, nil, nil); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(mine); !errors.Is(err, os.ErrNotExist) { + t.Fatal("an empty directory the mesh made was left behind, so nothing is ever cleaned up") + } +} diff --git a/internal/apply/archive.go b/internal/apply/archive.go new file mode 100644 index 0000000..490891a --- /dev/null +++ b/internal/apply/archive.go @@ -0,0 +1,185 @@ +package apply + +import ( + "archive/tar" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" +) + +// A set of files, fetched by digest and unpacked. +// +// For what inlining cannot serve: a theme, an icon set, a tree of configuration. Hundreds of +// files inlined would make every declaration enormous and rewrite all of them when one changed. +// +// **This is the one place the host reaches out on its own.** Everywhere else it holds a single +// outbound connection to the broker and fetches nothing; a container image is pulled by the +// runtime rather than by this process. So the discipline has to be explicit and it is the same +// one the bootstrap uses for images: **pinned by digest, and the digest is checked before +// anything is written.** What is fetched is bytes from a network the mesh does not control, and +// the only thing making them safe to unpack is that they hash to what was declared. + +// maxArchive is how much will be read before giving up. +// +// Because a fetch with no limit is a machine somebody can fill up from the far end. Chosen large +// enough for a desktop theme and small enough to notice. +const maxArchive = 512 << 20 + +func applyArchive(ctx context.Context, r *declaration.Archive, previous store.Applied) (Outcome, error) { + out := begin(r) + out.Action = "unchanged" + + body, err := fetch(ctx, r.Source) + if err != nil { + return out, err + } + sum := sha256.Sum256(body) + got := "sha256:" + hex.EncodeToString(sum[:]) + if got != r.Digest { + // Refused before a single file is written. A digest that does not match means the thing + // at that address is not the thing that was declared, and unpacking it would be applying + // something nobody reviewed. + return out, fmt.Errorf( + "%s was declared as %s and what arrived is %s; nothing was unpacked", + r.Source, r.Digest, got) + } + out.wrote = got + + // Already what it should be. The digest is the whole identity of an archive, so a matching + // record means the unpacked tree came from these exact bytes. + if previous.Wrote == got { + if _, err := os.Stat(r.Path); err == nil { + owned, err := ownedBy(r.Path, r.Owner) + if err == nil && owned { + return out, nil + } + } + } + + if err := os.MkdirAll(r.Path, 0o755); err != nil { + return out, err + } + written, err := unpack(body, r.Path) + if err != nil { + return out, err + } + if err := ownAll(r.Path, r.Owner); err != nil { + return out, err + } + out.Action = "updated" + if previous.Wrote == "" { + out.Action = "created" + } + out.Detail = fmt.Sprintf("%d file(s)", written) + return out, nil +} + +func fetch(ctx context.Context, source string) ([]byte, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, source, nil) + if err != nil { + return nil, err + } + response, err := http.DefaultClient.Do(request) + if err != nil { + return nil, fmt.Errorf("cannot fetch %s: %w", source, err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%s answered %s", source, response.Status) + } + body, err := io.ReadAll(io.LimitReader(response.Body, maxArchive+1)) + if err != nil { + return nil, err + } + if len(body) > maxArchive { + return nil, fmt.Errorf("%s is larger than %d bytes, which is not an archive this host "+ + "will unpack", source, maxArchive) + } + return body, nil +} + +// unpack writes a gzipped tar into a directory, refusing anything that would land outside it. +func unpack(body []byte, into string) (int, error) { + zipped, err := gzip.NewReader(strings.NewReader(string(body))) + if err != nil { + return 0, fmt.Errorf("this is not a gzipped tar: %w", err) + } + defer zipped.Close() + + root, err := filepath.Abs(into) + if err != nil { + return 0, err + } + reader := tar.NewReader(zipped) + written := 0 + for { + header, err := reader.Next() + if err == io.EOF { + return written, nil + } + if err != nil { + return written, err + } + + // The oldest bug in unpacking: an entry named ../../etc/passwd writes outside the + // directory it was unpacked into. + // + // **Refused, not sanitised.** Rewriting the name so it lands inside would put a file + // somewhere nobody asked for and report success — the "looks configured and is not" + // failure this host exists to prevent. An archive that names a path outside itself is + // either hostile or broken, and both want the same answer. + cleaned := filepath.Clean(header.Name) + if filepath.IsAbs(cleaned) || cleaned == ".." || strings.HasPrefix(cleaned, ".."+string(os.PathSeparator)) { + return written, fmt.Errorf( + "%s names a path outside the archive; nothing more was unpacked", header.Name) + } + // And the same question asked of the result, because a name can be made to resolve + // outside without saying so. + target := filepath.Join(root, cleaned) + if !strings.HasPrefix(target, root+string(os.PathSeparator)) && target != root { + return written, fmt.Errorf( + "%s would land outside %s; nothing more was unpacked", header.Name, into) + } + + switch header.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(target, os.FileMode(header.Mode)&os.ModePerm); err != nil { + return written, err + } + case tar.TypeReg: + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return written, err + } + file, err := os.OpenFile(target, + os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.FileMode(header.Mode)&os.ModePerm) + if err != nil { + return written, err + } + if _, err := io.Copy(file, io.LimitReader(reader, maxArchive)); err != nil { + file.Close() + return written, err + } + if err := file.Close(); err != nil { + return written, err + } + written++ + default: + // Symlinks, devices, fifos. Refused rather than skipped: a theme that needed one + // would silently arrive incomplete, and a device node in an archive is not something + // to unpack quietly onto a machine. + return written, fmt.Errorf( + "%s is a %c, and this host unpacks only files and directories", + header.Name, header.Typeflag) + } + } +} diff --git a/internal/apply/sealed_test.go b/internal/apply/sealed_test.go new file mode 100644 index 0000000..77df034 --- /dev/null +++ b/internal/apply/sealed_test.go @@ -0,0 +1,187 @@ +package apply + +import ( + "context" + "encoding/json" + "os" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/identity" + "github.com/novox/mesh-host/internal/store" +) + +// A file the mesh delivers without being able to read. +// +// Everything else in a declaration is visible to whatever carried it: the message is signed, so +// it cannot be forged, and signing does not make it unreadable. A password in `content` is a +// password the broker sees — the transitive trust this design refuses everywhere else. + +func sealedTo(t *testing.T, key identity.SealingKey, value string) string { + t.Helper() + sealed, err := identity.Seal(key.Public, []byte(value)) + if err != nil { + t.Fatal(err) + } + return sealed +} + +func opener(key identity.SealingKey) Unseal { + return func(sealed string) ([]byte, error) { return key.Unseal(sealed) } +} + +func sealedFile(t *testing.T, path, sealed string) *declaration.Declaration { + t.Helper() + raw := map[string]any{"declaration": 1, "resources": []map[string]any{ + {"id": "creds", "type": "file", "path": path, "sealed": sealed}, + }} + body, _ := json.Marshal(raw) + d, err := declaration.Parse(body) + if err != nil { + t.Fatal(err) + } + return d +} + +func TestASealedFileIsOpenedAndWritten(t *testing.T) { + key, err := identity.GenerateSealingKey() + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + path := dir + "/db.json" + d := sealedFile(t, path, sealedTo(t, key, `{"password":"hunter2"}`)) + + report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, + store.OriginCarried, noServices, nil, opener(key)) + if err != nil { + t.Fatal(err) + } + if !report.Changed() { + t.Fatal("nothing changed") + } + on, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(on) != `{"password":"hunter2"}` { + t.Fatalf("the file holds %q", on) + } +} + +func TestASecretIsNotWorldReadableByDefault(t *testing.T) { + // An ordinary file defaults to 0644, which for a credential is the whole problem. The default + // differs because the consequence differs; an explicit mode still wins, since a module may + // need its own user to read it and only the module knows which. + key, _ := identity.GenerateSealingKey() + dir := t.TempDir() + path := dir + "/db.json" + d := sealedFile(t, path, sealedTo(t, key, "secret")) + if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, + store.OriginCarried, noServices, nil, opener(key)); err != nil { + t.Fatal(err) + } + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o600 { + t.Fatalf("a credential landed mode %o", info.Mode().Perm()) + } +} + +func TestSomethingSealedToAnotherNodeIsRefused(t *testing.T) { + // Refused, not skipped, and refused before anything is written. A machine that quietly does + // not apply the one resource carrying a credential looks configured and cannot connect. + mine, _ := identity.GenerateSealingKey() + theirs, _ := identity.GenerateSealingKey() + dir := t.TempDir() + path := dir + "/db.json" + d := sealedFile(t, path, sealedTo(t, theirs, "not for you")) + + _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, + store.OriginCarried, noServices, nil, opener(mine)) + if err == nil { + t.Fatal("a file sealed to another node was applied") + } + if _, statErr := os.Stat(path); statErr == nil { + t.Fatal("something was written before the failure") + } +} + +func TestANodeWithNoSealingKeyRefusesRatherThanSkipping(t *testing.T) { + key, _ := identity.GenerateSealingKey() + dir := t.TempDir() + d := sealedFile(t, dir+"/db.json", sealedTo(t, key, "secret")) + _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, + store.OriginCarried, noServices, nil, nil) + if err == nil { + t.Fatal("a sealed file was skipped by a node that cannot open one") + } + if !strings.Contains(err.Error(), "sealing key") { + t.Fatalf("the failure does not say why: %v", err) + } +} + +func TestTheSecretIsNeverInWhatTheMeshIsToldBack(t *testing.T) { + // The node reports what it applied, and that report goes over the same broker the sealing was + // for. A digest is a fact about the file; the file is not. + key, _ := identity.GenerateSealingKey() + dir := t.TempDir() + d := sealedFile(t, dir+"/db.json", sealedTo(t, key, "hunter2")) + report, state, err := Apply(context.Background(), archHost(t), d, store.State{}, + store.OriginCarried, noServices, nil, opener(key)) + if err != nil { + t.Fatal(err) + } + said, _ := json.Marshal(report) + kept, _ := json.Marshal(state) + for what, blob := range map[string][]byte{"the report": said, "the node's state": kept} { + if strings.Contains(string(blob), "hunter2") { + t.Fatalf("%s carries the secret in plain text:\n%s", what, blob) + } + } +} + +func TestASealedFileStillNoticesAHandEdit(t *testing.T) { + // Drift detection must survive not holding the plaintext. It does, because what is recorded + // is a digest of what was written rather than what was written. + key, _ := identity.GenerateSealingKey() + dir := t.TempDir() + path := dir + "/db.json" + d := sealedFile(t, path, sealedTo(t, key, "hunter2")) + _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, + store.OriginCarried, noServices, nil, opener(key)) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte("meddled"), 0o600); err != nil { + t.Fatal(err) + } + again, _, err := Apply(context.Background(), archHost(t), d, state, + store.OriginCarried, noServices, nil, opener(key)) + if err != nil { + t.Fatal(err) + } + if !again.Changed() { + t.Fatal("a hand-edited credential was left as it was found") + } + on, _ := os.ReadFile(path) + if string(on) != "hunter2" { + t.Fatalf("it was not put back: %q", on) + } +} + +func TestContentAndSealedTogetherIsRefused(t *testing.T) { + // Otherwise nobody can tell by looking whether what landed on the machine was the secret or + // the placeholder. + _, err := declaration.Parse([]byte(`{"declaration":1,"resources":[ + {"id":"f","type":"file","path":"/etc/x","content":"a","sealed":"b"}]}`)) + if err == nil { + t.Fatal("a file that is both literal and sealed was accepted") + } + if !strings.Contains(err.Error(), "exactly once") { + t.Fatalf("unhelpful refusal: %v", err) + } +} diff --git a/internal/apply/user.go b/internal/apply/user.go new file mode 100644 index 0000000..2cf2026 --- /dev/null +++ b/internal/apply/user.go @@ -0,0 +1,189 @@ +package apply + +import ( + "context" + "fmt" + "os" + osuser "os/user" + "path/filepath" + "strconv" + "strings" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/system" +) + +// Logins, and the files that belong to them. +// +// Most of what a person installs is not a service. A shell, a terminal, a chat client, a desktop +// are a package plus configuration **in somebody's home** — so a mesh with no notion of a user +// can manage /etc and nothing anybody looks at. + +// applyUser makes a login match what was declared. +// +// Reconciling, like everything else here: it is not told whether the user is new. Creating, +// setting a shell and adding groups are each done only when the machine does not already agree. +func applyUser(ctx context.Context, sys system.System, r *declaration.User, run Runner) (Outcome, error) { + out := begin(r) + out.Action = "unchanged" + + login, exists, err := system.LookUpUser(ctx, system.Runner(run), r.Name) + if err != nil { + return out, err + } + if !exists { + if err := sys.CreateUser(ctx, system.Runner(run), r.Name, r.Home, r.Shell); err != nil { + return out, err + } + // Read back from the machine, not from the call that made it. A useradd that returns + // success and leaves no entry is exactly the failure this host takes trouble over. + login, exists, err = system.LookUpUser(ctx, system.Runner(run), r.Name) + if err != nil { + return out, err + } + if !exists { + return out, fmt.Errorf("created the user %q and the user database does not have it", + r.Name) + } + out.Action = "created" + } + + // The shell, only when it differs. Absent means the host asserts nothing — a field that + // always asserts cannot express "leave it alone", which is the difference between managing a + // machine and taking it over. + if r.Shell != "" && login.Shell != r.Shell { + if err := sys.SetUserShell(ctx, system.Runner(run), r.Name, r.Shell); err != nil { + return out, err + } + if back, _, err := system.LookUpUser(ctx, system.Runner(run), r.Name); err != nil { + return out, err + } else if back.Shell != r.Shell { + return out, fmt.Errorf("set %q's shell to %q and the user database says %q", + r.Name, r.Shell, back.Shell) + } + if out.Action == "unchanged" { + out.Action = "updated" + } + } + + if len(r.Groups) > 0 { + in, err := system.GroupsOf(ctx, system.Runner(run), r.Name) + if err != nil { + return out, err + } + already := map[string]bool{} + for _, g := range in { + already[g] = true + } + for _, want := range r.Groups { + if already[want] { + continue + } + if err := sys.AddUserToGroup(ctx, system.Runner(run), r.Name, want); err != nil { + return out, err + } + if out.Action == "unchanged" { + out.Action = "updated" + } + } + } + return out, nil +} + +// own sets a path's owner, when one was declared. +// +// Looked up by name every time rather than cached: a user's numeric id is not stable across +// machines, and the whole reason this exists is that the same declaration lands on several. +func own(path, owner string) error { + if owner == "" { + return nil + } + uid, gid, err := idsOf(owner) + if err != nil { + return fmt.Errorf("%s should belong to %q: %w", path, owner, err) + } + if err := os.Chown(path, uid, gid); err != nil { + return fmt.Errorf("cannot give %s to %q: %w", path, owner, err) + } + return nil +} + +// idsOf resolves an owner to a uid and gid: a name this machine knows, or numbers it does not. +// +// **Numbers, because a container's user is a number the machine has never heard of.** A directory +// a module mounts into its container belongs to whoever runs inside — grafana's 472, redis's 999, +// www-data's 33 — and none of those has a row in this machine's passwd, so there is no name to +// look up and none to create. Refusing them looked principled and meant every module whose +// container drops privileges could not own its own data: the store's config was unreadable to +// the store, and the forge could not traverse into the directory that held its files. +// +// "uid:gid" and bare "uid" are numeric; anything else is a name, resolved as before. +func idsOf(owner string) (int, int, error) { + user, group, both := strings.Cut(owner, ":") + if uid, err := strconv.Atoi(user); err == nil { + gid := uid + if both { + g, err := strconv.Atoi(group) + if err != nil { + return 0, 0, fmt.Errorf( + "%q reads as a uid with a group that is not a gid", owner) + } + gid = g + } + return uid, gid, nil + } + if both { + return 0, 0, fmt.Errorf("%q mixes a name with a colon; a name stands alone", owner) + } + found, err := osuser.Lookup(owner) + if err != nil { + return 0, 0, fmt.Errorf("this machine has no such user: %w", err) + } + uid, err := strconv.Atoi(found.Uid) + if err != nil { + return 0, 0, err + } + gid, err := strconv.Atoi(found.Gid) + if err != nil { + return 0, 0, err + } + return uid, gid, nil +} + +// ownedBy reports whether a path already belongs to a user, so applying twice changes nothing. +func ownedBy(path, owner string) (bool, error) { + if owner == "" { + return true, nil + } + wantUID, wantGID, err := idsOf(owner) + if err != nil { + return false, nil + } + info, err := os.Stat(path) + if err != nil { + return false, err + } + uid, gid, ok := ownerOf(info) + if !ok { + return false, nil + } + return uid == wantUID && gid == wantGID, nil +} + +// ownAll gives a whole tree to a user, for an archive that was unpacked into it. +func ownAll(root, owner string) error { + if owner == "" { + return nil + } + return filepath.Walk(root, func(path string, _ os.FileInfo, err error) error { + if err != nil { + return err + } + return own(path, owner) + }) +} + +// ownerOf is the numeric owner of a file, where the platform reports one. +func ownerOf(info os.FileInfo) (uid, gid int, ok bool) { + return statOwner(info) +} diff --git a/internal/apply/user_numeric_test.go b/internal/apply/user_numeric_test.go new file mode 100644 index 0000000..a42ff45 --- /dev/null +++ b/internal/apply/user_numeric_test.go @@ -0,0 +1,34 @@ +package apply + +import "testing" + +// A container's user is a number the machine has never heard of — grafana's 472, redis's 999 — +// so an owner must be expressible without a passwd row. Refusing numerics looked principled and +// meant every module whose container drops privileges could not own its own data. +func TestAnOwnerMayBeANumberTheMachineDoesNotKnow(t *testing.T) { + for owner, want := range map[string][2]int{ + "472:472": {472, 472}, + "1000:1000": {1000, 1000}, + "999": {999, 999}, + "10001:10001": {10001, 10001}, + "33:0": {33, 0}, + } { + uid, gid, err := idsOf(owner) + if err != nil { + t.Errorf("%q refused: %v", owner, err) + continue + } + if uid != want[0] || gid != want[1] { + t.Errorf("%q resolved to %d:%d, wanted %d:%d", owner, uid, gid, want[0], want[1]) + } + } + if _, _, err := idsOf("no-such-user-exists-here"); err == nil { + t.Error("a name this machine does not know was accepted") + } + if _, _, err := idsOf("root:something"); err == nil { + t.Error("a name with a colon was accepted; a name stands alone") + } + if uid, gid, err := idsOf("root"); err != nil || uid != 0 || gid != 0 { + t.Errorf("root resolved to %d:%d (%v); names must still work", uid, gid, err) + } +} diff --git a/internal/apply/user_unix.go b/internal/apply/user_unix.go new file mode 100644 index 0000000..fb6765d --- /dev/null +++ b/internal/apply/user_unix.go @@ -0,0 +1,18 @@ +//go:build unix + +package apply + +import ( + "os" + "syscall" +) + +// statOwner reads a file's numeric owner. Split out because the field is platform-specific and +// the rest of this package should not have to know that. +func statOwner(info os.FileInfo) (uid, gid int, ok bool) { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return 0, 0, false + } + return int(stat.Uid), int(stat.Gid), true +} diff --git a/internal/apply/vocabulary_test.go b/internal/apply/vocabulary_test.go new file mode 100644 index 0000000..1670f49 --- /dev/null +++ b/internal/apply/vocabulary_test.go @@ -0,0 +1,405 @@ +package apply + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" +) + +// The shapes added so that most of what a person installs is expressible. +// +// A shell, a chat client, a desktop are a package plus configuration in somebody's home, and a +// mesh with no user can manage /etc and nothing anybody looks at. + +func declare(t *testing.T, resources string) *declaration.Declaration { + t.Helper() + d, err := declaration.Parse([]byte(`{"declaration":1,"resources":[` + resources + `]}`)) + if err != nil { + t.Fatal(err) + } + return d +} + +func TestAFileMayBeBytesRatherThanText(t *testing.T) { + // A wallpaper, a font, an icon. Stored as its own encoding it would be a wallpaper nothing + // can open. + dir := t.TempDir() + original := []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0xff} + d := declare(t, `{"id":"w","type":"file","path":"`+dir+`/wall.png","bytes":"`+ + base64.StdEncoding.EncodeToString(original)+`"}`) + + if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, + store.OriginCarried, noServices, nil, nil); err != nil { + t.Fatal(err) + } + on, err := os.ReadFile(dir + "/wall.png") + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(on, original) { + t.Fatalf("the bytes did not survive: %x", on) + } +} + +func TestAFileSaysWhatIsInItExactlyOnce(t *testing.T) { + // Three ways of saying it and no precedence between them, so "what is in this file" is + // answerable by looking rather than by knowing which field wins. + _, err := declaration.Parse([]byte(`{"declaration":1,"resources":[ + {"id":"f","type":"file","path":"/etc/x","content":"a","bytes":"YQ=="}]}`)) + if err == nil { + t.Fatal("a file that was both text and bytes was accepted") + } + if !strings.Contains(err.Error(), "exactly once") { + t.Fatalf("unhelpful refusal: %v", err) + } +} + +func TestBytesThatAreNotBase64AreRefused(t *testing.T) { + dir := t.TempDir() + d := declare(t, `{"id":"w","type":"file","path":"`+dir+`/x","bytes":"not base64!!"}`) + _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, + store.OriginCarried, noServices, nil, nil) + if err == nil { + t.Fatal("a file carrying nonsense was written") + } + if _, statErr := os.Stat(dir + "/x"); statErr == nil { + t.Fatal("something was written before the failure") + } +} + +// A gzipped tar, and its digest, built here so the test does not depend on a fixture nobody can +// regenerate. +func anArchive(t *testing.T, files map[string]string) ([]byte, string) { + t.Helper() + var raw bytes.Buffer + zipped := gzip.NewWriter(&raw) + writer := tar.NewWriter(zipped) + for name, body := range files { + if err := writer.WriteHeader(&tar.Header{ + Name: name, Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg, + }); err != nil { + t.Fatal(err) + } + if _, err := writer.Write([]byte(body)); err != nil { + t.Fatal(err) + } + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + if err := zipped.Close(); err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(raw.Bytes()) + return raw.Bytes(), "sha256:" + hex.EncodeToString(sum[:]) +} + +func serving(t *testing.T, body []byte) string { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(body) + })) + t.Cleanup(server.Close) + return server.URL + "/theme.tar.gz" +} + +func TestAnArchiveIsUnpacked(t *testing.T) { + body, digest := anArchive(t, map[string]string{ + "config/theme.conf": "dark", "config/icons/one.svg": "", + }) + dir := t.TempDir() + d := declare(t, `{"id":"theme","type":"archive","source":"`+serving(t, body)+ + `","digest":"`+digest+`","path":"`+dir+`/theme"}`) + + report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, + store.OriginCarried, noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + if !report.Changed() { + t.Fatal("nothing changed") + } + on, err := os.ReadFile(dir + "/theme/config/theme.conf") + if err != nil { + t.Fatal(err) + } + if string(on) != "dark" { + t.Fatalf("got %q", on) + } +} + +func TestAnArchiveThatIsNotWhatWasDeclaredIsRefusedBeforeAnythingIsWritten(t *testing.T) { + // The only thing making bytes from a network the mesh does not control safe to unpack is + // that they hash to what was declared. + body, _ := anArchive(t, map[string]string{"a": "b"}) + dir := t.TempDir() + d := declare(t, `{"id":"theme","type":"archive","source":"`+serving(t, body)+ + `","digest":"sha256:`+strings.Repeat("ab", 32)+`","path":"`+dir+`/theme"}`) + + _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, + store.OriginCarried, noServices, nil, nil) + if err == nil { + t.Fatal("an archive that was not what was declared was unpacked") + } + if entries, _ := os.ReadDir(dir); len(entries) != 0 { + t.Fatal("something was written before the digest was checked") + } +} + +func TestAnArchiveCannotWriteOutsideWhereItWasUnpacked(t *testing.T) { + // The oldest bug in unpacking. Checked against the resolved root rather than by looking for + // "..", because there is more than one way to name a path that escapes. + body, digest := anArchive(t, map[string]string{"../../escaped": "no"}) + dir := t.TempDir() + d := declare(t, `{"id":"theme","type":"archive","source":"`+serving(t, body)+ + `","digest":"`+digest+`","path":"`+dir+`/theme"}`) + + _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, + store.OriginCarried, noServices, nil, nil) + if err != nil && !strings.Contains(err.Error(), "outside") { + t.Fatalf("refused for the wrong reason: %v", err) + } + if _, statErr := os.Stat(dir + "/escaped"); statErr == nil { + t.Fatal("a file landed outside the directory it was unpacked into") + } + if err == nil { + t.Fatal("an escaping entry was accepted") + } +} + +func TestAnUnpackedArchiveIsNotFetchedAgainForNothing(t *testing.T) { + // The digest is the whole identity of an archive, so a matching record means the tree came + // from these exact bytes. Applying twice must not report work. + body, digest := anArchive(t, map[string]string{"a": "b"}) + dir := t.TempDir() + d := declare(t, `{"id":"theme","type":"archive","source":"`+serving(t, body)+ + `","digest":"`+digest+`","path":"`+dir+`/theme"}`) + + _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, + store.OriginCarried, noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + again, _, err := Apply(context.Background(), archHost(t), d, state, + store.OriginCarried, noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + if again.Changed() { + said, _ := json.Marshal(again) + t.Fatalf("the second apply did work: %s", said) + } +} + +// Defends novox/hq ADR 0012: the mesh creates no symlinks — a derived file is a copy. +// +// The archive is the one path where a symlink could arrive without anybody declaring it, which is +// why the refusal lives here. ADR 0012 was earned by production data loss through a symlink +// resolved inside a container volume path. +func TestAnArchiveWithSomethingThatIsNotAFileIsRefused(t *testing.T) { + // A theme needing a symlink would otherwise arrive silently incomplete, and a device node in + // an archive is not something to unpack quietly onto a machine. + var raw bytes.Buffer + zipped := gzip.NewWriter(&raw) + writer := tar.NewWriter(zipped) + if err := writer.WriteHeader(&tar.Header{ + Name: "link", Typeflag: tar.TypeSymlink, Linkname: "/etc/passwd", Mode: 0o777, + }); err != nil { + t.Fatal(err) + } + writer.Close() + zipped.Close() + sum := sha256.Sum256(raw.Bytes()) + digest := "sha256:" + hex.EncodeToString(sum[:]) + + dir := t.TempDir() + d := declare(t, `{"id":"theme","type":"archive","source":"`+serving(t, raw.Bytes())+ + `","digest":"`+digest+`","path":"`+dir+`/theme"}`) + _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, + store.OriginCarried, noServices, nil, nil) + if err == nil { + t.Fatal("a symlink was unpacked") + } + if !strings.Contains(err.Error(), "files and directories") { + t.Fatalf("refused for the wrong reason: %v", err) + } +} + +func TestAnArchiveMustBePinned(t *testing.T) { + _, err := declaration.Parse([]byte(`{"declaration":1,"resources":[ + {"id":"t","type":"archive","source":"https://example.invalid/a.tgz","path":"/opt/t"}]}`)) + if err == nil { + t.Fatal("an unpinned archive was accepted") + } + if !strings.Contains(err.Error(), "digest") { + t.Fatalf("unhelpful refusal: %v", err) + } +} + +// Defends novox/hq ADR 0029: a network is a shape so that it can be removed. +// +// The whole argument for widening the vocabulary is lifecycle — an action could create one and +// nothing could ever take it away — so removal is the assertion that matters, not creation. +func TestANetworkIsCreatedAndThenRemovedWhenNoLongerDeclared(t *testing.T) { + var calls []string + there := map[string]bool{} + run := func(_ context.Context, name string, args ...string) (string, error) { + calls = append(calls, name+" "+strings.Join(args, " ")) + if name != "docker" || len(args) < 2 || args[0] != "network" { + return "", nil // the runtime probe + } + switch args[1] { + case "inspect": + if !there[args[2]] { + return "", fmt.Errorf("no such network") + } + case "create": + there[args[2]] = true + case "rm": + delete(there, args[2]) + } + return "", nil + } + + d := declare(t, `{"id":"private","type":"network","name":"mail"}`) + _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, + store.OriginDeclared, run, nil, nil) + if err != nil { + t.Fatal(err) + } + if !there["mail"] { + t.Fatal("the network was not created") + } + + // The module is unassigned: the mesh now declares nothing. + empty := declare(t, `{"id":"unrelated","type":"directory","path":"`+t.TempDir()+`"}`) + if _, _, err := Apply(context.Background(), archHost(t), empty, state, + store.OriginDeclared, run, nil, nil); err != nil { + t.Fatal(err) + } + if there["mail"] { + t.Fatal("the network outlived the module that declared it, which is the entire reason " + + "this is a shape rather than an action") + } +} + +// A network is created once and left alone when it is already there. +func TestANetworkAlreadyThereIsNotRebuilt(t *testing.T) { + var created int + run := func(_ context.Context, name string, args ...string) (string, error) { + if name == "docker" && len(args) > 1 && args[0] == "network" && args[1] == "create" { + created++ + } + return "", nil // inspect succeeds: it is already there + } + + d := declare(t, `{"id":"private","type":"network","name":"mail"}`) + if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, + store.OriginDeclared, run, nil, nil); err != nil { + t.Fatal(err) + } + if created != 0 { + t.Fatalf("a network that was already there was created %d time(s); the mesh owns the "+ + "name and not the thing, so it does not tear one down and rebuild it", created) + } +} + +// A secret inside a configuration file, substituted on the machine. +// +// **The one place a credential and a configuration meet.** A program wanting its token inside a +// JSON document cannot be handed a file that is entirely a token, and the mesh cannot compose the +// document because it discarded the value. So the module supplies the document with a hole, the +// mesh delivers the value sealed, and the host is the only thing that ever holds both. +func TestASealedValueIsPutIntoTheFileThatNamesIt(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "settings.json") + d := declare(t, `{"id":"settings","type":"file","path":"`+path+`",`+ + `"content":"{\"tracking\":\"on\",\"token\":\"${secret:atlassian}\"}",`+ + `"secrets":{"atlassian":"SEALED"}}`) + + open := func(blob string) ([]byte, error) { + if blob != "SEALED" { + return nil, fmt.Errorf("asked to open %q", blob) + } + return []byte("the-real-token"), nil + } + if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, + store.OriginDeclared, nil, nil, open); err != nil { + t.Fatal(err) + } + + written, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(written), `"token":"the-real-token"`) { + t.Fatalf("the secret was not put in: %s", written) + } + if strings.Contains(string(written), "secret:") { + t.Fatalf("a placeholder survived into the file: %s", written) + } + // The rest of the document is untouched — this is substitution, not replacement. + if !strings.Contains(string(written), `"tracking":"on"`) { + t.Fatalf("the content around the secret was lost: %s", written) + } + // And it carries a credential, so it is not world-readable. + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o600 { + t.Errorf("a file holding a credential is %v", info.Mode().Perm()) + } +} + +// Defends the reason env-file exists: a credential may not travel in `env`. +// +// A declaration reaches a node over the broker and `env` is plain text in it, so a password there +// is a password the broker sees. A sealed file arrives unreadable, the host writes it, and the +// runtime reads it. +func TestAContainerIsGivenItsEnvironmentFiles(t *testing.T) { + var ran []string + run := func(_ context.Context, name string, args ...string) (string, error) { + ran = append(ran, name+" "+strings.Join(args, " ")) + if len(args) > 0 && args[0] == "inspect" { + return "", fmt.Errorf("no such container") + } + return "", nil + } + d := declare(t, `{"id":"app","type":"container","name":"umami",`+ + `"image":"umami@sha256:0000000000000000000000000000000000000000000000000000000000000000",`+ + `"env-file":["/var/lib/umami/database.env","/var/lib/umami/app.env"]}`) + + _, _, _ = Apply(context.Background(), archHost(t), d, store.State{}, + store.OriginDeclared, run, nil, nil) + + var started string + for _, line := range ran { + if strings.Contains(line, "run ") { + started = line + } + } + for _, want := range []string{ + "--env-file /var/lib/umami/database.env", + "--env-file /var/lib/umami/app.env", + } { + if !strings.Contains(started, want) { + t.Errorf("the container was started without %q:\n%s", want, started) + } + } +} diff --git a/internal/bundle/bundle.go b/internal/bundle/bundle.go new file mode 100644 index 0000000..f9f3c6b --- /dev/null +++ b/internal/bundle/bundle.go @@ -0,0 +1,82 @@ +// Package bundle is the declaration the host carries. +// +// novox/hq ADR 0004: the host has one behaviour and two sources of declaration — the control +// plane when a mesh is reachable, and this when none is. The first node is not a different +// kind of node; it is a node whose mesh is not up yet, and this is what it applies until it is. +// +// Carried inside the binary rather than beside it, because "copy it onto a machine and run it +// is the whole installation" (ADR 0005) stops being true the moment a second file has to +// arrive with it. +package bundle + +import ( + _ "embed" + "errors" + "fmt" + "strings" + + "github.com/novox/mesh-host/internal/declaration" +) + +// One bundle per operating system, because its CONTENTS are per system even though its +// mechanism is not: package names, unit names and service names all differ +// (novox/hq ADR 0005). All three are embedded and the host applies the one it was built for — +// an arch host never reads the alpine bundle. +// +// A host whose bundle is only comments carries nothing, and says so rather than applying +// nothing and reporting success — a host that silently did nothing on a first node would look +// exactly like one that worked. +// +//go:embed substrate-arch.lock +var archLock []byte + +//go:embed substrate-alpine.lock +var alpineLock []byte + +//go:embed substrate-android.lock +var androidLock []byte + +var locks = map[string][]byte{ + "arch": archLock, + "alpine": alpineLock, + "android": androidLock, +} + +// ErrEmpty means this host carries no bundle. +var ErrEmpty = errors.New( + "this host carries no bundle. A host without one cannot raise a first node, and applying " + + "nothing would look exactly like applying something") + +// Raw returns the carried bytes, for inspection. +func Raw(system string) []byte { return locks[system] } + +// IsEmpty reports whether anything was built in. A bundle of only comments and whitespace is +// empty for this purpose: a placeholder is a comment, and treating it as content would mean a +// host claims to carry a substrate it does not. +func IsEmpty(system string) bool { + for _, line := range strings.Split(string(locks[system]), "\n") { + line = strings.TrimSpace(line) + if line != "" && !strings.HasPrefix(line, "//") { + return false + } + } + return true +} + +// Load parses the carried bundle. +// +// The same parser the link will use. A bundle that reaches a machine and is then refused by the +// host that carries it would be a build-time mistake discovered at the worst possible moment, +// which is why `mesh-host bundle` exists to ask before it matters. +func Load(system string) (*declaration.Declaration, error) { + if _, known := locks[system]; !known { + return nil, fmt.Errorf("no bundle is built for %q", system) + } + if IsEmpty(system) { + return nil, ErrEmpty + } + // ParseTrusted: the bundle arrives with the binary, so it may carry actions the link may + // not (novox/hq ADR 0005). The bootstrap needs them — creating the control plane's database + // happens before there is any mesh to ask for one. + return declaration.ParseFileTrusted(locks[system]) +} diff --git a/internal/bundle/bundle_test.go b/internal/bundle/bundle_test.go new file mode 100644 index 0000000..b7fb1ea --- /dev/null +++ b/internal/bundle/bundle_test.go @@ -0,0 +1,88 @@ +package bundle + +import ( + "errors" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" +) + +func TestADefaultBuildCarriesNothingAndSaysSo(t *testing.T) { + // The important one. A host built without a bundle that applied nothing and reported + // success would look exactly like a host that raised a first node — and the difference + // would surface as a mesh that never came up, with nothing to point at. + if !IsEmpty("arch") { + t.Fatal("the default build claims to carry a substrate") + } + _, err := Load("arch") + if !errors.Is(err, ErrEmpty) { + t.Fatalf("an empty bundle did not refuse: %v", err) + } + if !strings.Contains(err.Error(), "would look exactly like applying something") { + t.Errorf("the refusal does not say why it matters: %v", err) + } +} + +func TestABundleWithContentIsParsedByTheSameParserTheLinkWillUse(t *testing.T) { + // A bundle that reaches a machine and is then refused by the host carrying it would be a + // build-time mistake found at the worst possible moment. + // + // Comment handling itself is asserted where it now lives, in `declaration`. It was here, and + // having it in two places is how it came to be *done* in two places. + real := []byte(`// pinned +{"declaration":1,"resources":[{"id":"d","type":"directory","path":"/etc/mesh"}]}`) + parsed, err := declaration.ParseFileTrusted(real) + if err != nil { + t.Fatalf("an annotated bundle was refused: %v", err) + } + if len(parsed.Resources) != 1 { + t.Fatalf("got %d resources", len(parsed.Resources)) + } +} + +func TestWhatValidatesIsWhatIsApplied(t *testing.T) { + // Found on a real machine. `mesh-host bundle` validated the carried bundle through Load, + // which strips comments; `reconcile` handed the RAW bytes to the parser, which does not. + // So the command whose whole job is to check the bundle said yes, and the command that + // uses it said no — two paths to one artefact, disagreeing. + // + // There is now one path. This asserts the property that made the bug possible cannot + // return: whatever Load accepts is what gets applied, byte for byte. + annotated := []byte("// a comment\n" + `{"declaration":1,"resources":[{"id":"d","type":"directory","path":"/etc/mesh"}]}`) + + if _, err := declaration.ParseFileTrusted(annotated); err != nil { + t.Fatalf("an annotated bundle was refused: %v", err) + } +} + +func TestEverySystemHasABundleAndAndroidsRefuses(t *testing.T) { + // The bundle's contents are per system even though its mechanism is not (novox/hq ADR + // 0060), so a host must find one built for it — and a host built for a system with no + // bundle at all must say that rather than behave like an empty one. + for _, system := range []string{"arch", "alpine", "android"} { + if _, err := Load(system); err == nil { + t.Errorf("%s: a placeholder bundle loaded as if it had contents", system) + } else if !errors.Is(err, ErrEmpty) { + t.Errorf("%s: refused for the wrong reason: %v", system, err) + } + } + + if _, err := Load("debian"); err == nil { + t.Error("a bundle was loaded for a system nobody has built") + } else if errors.Is(err, ErrEmpty) { + t.Error("an unbuilt system was reported as an empty bundle; those are different things") + } +} + +func TestAndroidsBundleSaysWhyThereIsNone(t *testing.T) { + // Not a placeholder waiting to be filled in. An android host implements neither `package` + // nor `container` nor `service`, so every step of the bootstrap is a shape it does not + // have — a partial host can JOIN a mesh and cannot BE the first node. + text := string(Raw("android")) + for _, want := range []string{"cannot raise a mesh", "JOIN", "first node"} { + if !strings.Contains(text, want) { + t.Errorf("the android bundle does not explain itself; missing %q", want) + } + } +} diff --git a/internal/bundle/substrate-alpine.lock b/internal/bundle/substrate-alpine.lock new file mode 100644 index 0000000..1d0f5cc --- /dev/null +++ b/internal/bundle/substrate-alpine.lock @@ -0,0 +1,11 @@ +// substrate-alpine.lock — the pinned tier-1 descriptor the ALPINE host carries. +// +// Per system, because its CONTENTS are: this one names apk packages and OpenRC services where +// the arch bundle names pacman packages and systemd units (novox/hq ADR 0005). +// +// Empty on purpose. What belongs here is the closure for a one-node mesh, and that is not +// yet known: novox/hq research 012 asks what the minimum actually is, and research 011 is +// what would compute it rather than assert it. +// +// A host built with this placeholder refuses to reconcile and says why, rather than applying +// nothing and reporting success. Building a real host means building it with a real bundle. diff --git a/internal/bundle/substrate-android.lock b/internal/bundle/substrate-android.lock new file mode 100644 index 0000000..6ee155f --- /dev/null +++ b/internal/bundle/substrate-android.lock @@ -0,0 +1,12 @@ +// substrate-android.lock — deliberately not a bundle. +// +// An android host cannot raise a mesh, and this file says so rather than being an empty +// placeholder waiting to be filled in. +// +// The substrate is a container runtime, a store and the control plane (novox/hq +// 07-the-substrate.md). An android host implements `file`, `directory` and `action` and refuses +// `package`, `container` and `service` (ADR 0005) — so every step of the bootstrap is a shape it +// does not have. No amount of filling this in changes that. +// +// **A partial host can JOIN a mesh and cannot BE the first node.** That is a real distinction +// and it belongs here, where somebody looking for the android bundle will find it. diff --git a/internal/bundle/substrate-arch.lock b/internal/bundle/substrate-arch.lock new file mode 100644 index 0000000..21cc7be --- /dev/null +++ b/internal/bundle/substrate-arch.lock @@ -0,0 +1,11 @@ +// substrate-arch.lock — the pinned tier-1 descriptor the ARCH host carries. +// +// Per system, because its CONTENTS are: package names, unit names and service names all differ +// (novox/hq ADR 0005). The mechanism is shared; what it names is not. +// +// Empty on purpose. What belongs here is the closure for a one-node mesh, and that is not +// yet known: novox/hq research 012 asks what the minimum actually is, and research 011 is +// what would compute it rather than assert it. +// +// A host built with this placeholder refuses to reconcile and says why, rather than applying +// nothing and reporting success. Building a real host means building it with a real bundle. diff --git a/internal/declaration/declaration.go b/internal/declaration/declaration.go new file mode 100644 index 0000000..ed25882 --- /dev/null +++ b/internal/declaration/declaration.go @@ -0,0 +1,841 @@ +// Package declaration is what the host is told a machine should be. +// +// Data, never instructions. The vocabulary is finite, versioned, and anything outside it +// refuses the whole declaration rather than being skipped — a host that applied most of what +// it was sent and reported success is a node that looks configured and is not +// (novox/hq ADR 0005). +package declaration + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "reflect" + "regexp" + "slices" + "sort" + "strings" +) + +// Version is the vocabulary this host speaks. A declaration naming any other version is +// refused: an older host handed a newer vocabulary must not quietly do half of it. +const Version = 1 + +// Type names a kind of resource. Every addition widens what a compromised control plane can +// express, so the list is a security artefact and grows deliberately. +type Type string + +const ( + TypeDirectory Type = "directory" + TypeFile Type = "file" + TypeService Type = "service" + TypePackage Type = "package" + TypeContainer Type = "container" + TypeAction Type = "action" + + // TypeUser is a login on the machine. Added because most of what a person actually installs + // is not a service: a shell, a terminal, a chat client, a desktop. All of those are a package + // plus configuration **in somebody's home**, and a mesh with no notion of a user can only + // manage /etc. + TypeUser Type = "user" + + // TypeArchive is a set of files fetched by digest and unpacked. A desktop theme is hundreds + // of files; inlining them would make every declaration enormous and rewrite the lot whenever + // one changed. + TypeArchive Type = "archive" + + // TypeNetwork is a named network on this machine, for a module whose containers must reach + // each other by name. Created if absent, removed when no longer declared — which is the whole + // reason it is a shape rather than an action, because an action leaves nothing the host can + // undo and the network would outlive the module (novox/hq ADR 0029). + TypeNetwork Type = "network" +) + +// Resource is one thing that should be true of the machine. +// +// A struct per kind rather than one struct carrying every field, because the decoder is then +// what rejects a field the kind does not have: a `file` carrying an `image` is refused because +// File has no such field, not because a list somewhere remembered to say so. The one-struct +// form needs every kind revisited whenever a field is added, and the kind nobody revisits +// silently accepts a field the host will never read. +type Resource interface { + // Identity is the name the control plane keeps stable across declarations. Not a position + // and not a hash of the content: it is what lets the store say *this is the same resource + // I applied last time*, which is what makes removal possible at all. + Identity() string + // Kind is the resource's type, for the store and for reporting. + Kind() Type + // Target is what the resource acts on, for a person reading a report. + Target() string + + validate(where string, allowActions bool) []string +} + +// The `Type` field on each kind below exists only to absorb the JSON `"type"` key, which the +// strict decoder would otherwise refuse. `Kind()` returns the constant and is what anything +// else should read. + +// Directory is a directory that should exist, with a mode. +type Directory struct { + ID string `json:"id"` + Type Type `json:"type"` + Path string `json:"path"` + Mode string `json:"mode,omitempty"` + // Owner is the user this belongs to, by name. Absent means root, which is what everything + // managed was until users existed. + Owner string `json:"owner,omitempty"` +} + +func (d *Directory) Identity() string { return d.ID } +func (d *Directory) Kind() Type { return TypeDirectory } +func (d *Directory) Target() string { return d.Path } + +func (d *Directory) validate(where string, _ bool) []string { + var problems []string + if d.Path == "" { + problems = append(problems, where+": a directory needs a path") + } + return append(problems, checkMode(where, d.Mode)...) +} + +// File is a file with literal content. The host renders nothing. +type File struct { + ID string `json:"id"` + Type Type `json:"type"` + Path string `json:"path"` + Content string `json:"content"` + Mode string `json:"mode,omitempty"` + + // Sealed is content encrypted to this node's sealing key, for a file the mesh must deliver + // without being able to read. + // + // The one thing here the host cannot simply write. Everything else in a declaration is + // visible to whatever carried it — the broker relays the message, and the message is signed + // so it cannot be forged, but signing does not make it unreadable. A password travelling in + // `content` would be a password the broker sees, which is the transitive trust the design + // refuses everywhere else (novox/hq ADR 0004). + // + // Exclusive with Content: a file is one or the other, so that "was this secret" is answerable + // by looking rather than by knowing which field won. + Sealed string `json:"sealed,omitempty"` + + // Secrets are sealed values put into Content where it says `${secret:name}`. + // + // **The one place a secret and a configuration meet, and it happens on the machine.** A + // program that wants its token inside a JSON document cannot be given a file that is entirely + // a token, and the mesh cannot compose the document itself — it discarded the value + // (novox/hq ADR 0024). So the module supplies the document with a hole in it, the mesh + // delivers the value sealed, and the host is the only thing that ever sees both. + // + // **Substitution is textual and the host learns no formats.** That is deliberate: a mechanism + // that understood JSON would be asked to understand YAML next, and then INI, which is how the + // arrangement this replaces became something nobody could hold in their head. The module knows + // its own format, because it wrote the rest of the file. + // + // The sharp edge, stated rather than discovered: a value containing a quote or a backslash + // will not be escaped for whatever syntax surrounds it. + Secrets map[string]string `json:"secrets,omitempty"` + + // Bytes is content that is not text, base64-encoded — a wallpaper, a font, an icon. + // + // A third way of saying what is in a file, and the three are exclusive. It would have been + // tempting to let Content carry base64 and add a flag, and then "what is in this file" would + // depend on a field somewhere else. + Bytes string `json:"bytes,omitempty"` + + // Owner is the user this belongs to, by name. Absent means root. + Owner string `json:"owner,omitempty"` +} + +// Secret reports whether this file arrived sealed, which is what decides both that it must be +// opened before writing and that its contents must never appear in a report. +func (f *File) Secret() bool { return f.Sealed != "" } + +func (f *File) Identity() string { return f.ID } +func (f *File) Kind() Type { return TypeFile } +func (f *File) Target() string { return f.Path } + +// placeholder is what Content says where a sealed value belongs: ${secret:name}. +var placeholder = regexp.MustCompile(`\$\{secret:([a-z0-9][a-z0-9-]*)\}`) + +// SecretsUsed are the names Content asks for, in the order they first appear. +func (f *File) SecretsUsed() []string { + var used []string + seen := map[string]bool{} + for _, m := range placeholder.FindAllStringSubmatch(f.Content, -1) { + if !seen[m[1]] { + seen[m[1]] = true + used = append(used, m[1]) + } + } + return used +} + +func (f *File) validate(where string, _ bool) []string { + var problems []string + if f.Path == "" { + problems = append(problems, where+": a file needs a path") + } + var said []string + for name, value := range map[string]string{ + "content": f.Content, "sealed": f.Sealed, "bytes": f.Bytes, + } { + if value != "" { + said = append(said, name) + } + } + if len(said) > 1 { + sort.Strings(said) + problems = append(problems, where+ + ": a file says what is in it exactly once, and this says it as "+ + strings.Join(said, " and ")+ + " — otherwise nobody can tell by looking which one landed on the machine") + } + // A file whose content names a secret must be given exactly the secrets it names. + // + // **Both directions, and both are refusals rather than warnings.** A placeholder with nothing + // to fill it would write `${secret:x}` into a configuration file, which the program reads as + // a value and fails on somewhere unrelated. A secret nobody uses means whoever wrote this + // believes a credential is in a file where it is not. + if len(f.Secrets) > 0 && f.Content == "" { + problems = append(problems, where+ + ": secrets were given and there is no content to put them in") + } + used := f.SecretsUsed() + for _, name := range used { + if f.Secrets[name] == "" { + problems = append(problems, fmt.Sprintf( + "%s: the content asks for the secret %q and none was given", where, name)) + } + } + for name := range f.Secrets { + if !slices.Contains(used, name) { + problems = append(problems, fmt.Sprintf( + "%s: the secret %q was given and the content never asks for it", where, name)) + } + } + return append(problems, checkMode(where, f.Mode)...) +} + +// User is a login on the machine. +// +// The thing that makes a shell, a chat client or a desktop expressible at all: each is a package +// plus configuration in somebody's home, and until this the mesh could only own /etc. +// +// It also makes "zsh is my login shell" **declared state** rather than an action. `chsh` is a +// command, the link may not carry one (novox/hq ADR 0005), and a shell that could only be set by +// hand would be a shell the mesh cannot manage — which is most of the reason to manage a machine +// at all. +type User struct { + ID string `json:"id"` + Type Type `json:"type"` + Name string `json:"name"` + + // Shell this user logs in with. Absent means the host asserts nothing and leaves whatever is + // there — the same rule Service.Boot follows, for the same reason: a field that always + // asserts cannot express "I do not care". + Shell string `json:"shell,omitempty"` + + // Groups this user must be in. Additive: the host puts the user in these and does not remove + // it from others, because a machine's own groups are not the mesh's to know about. + Groups []string `json:"groups,omitempty"` + + // Home directory. Absent means the system's default for a new user, and is not changed for + // one that exists — moving somebody's home is not something a declaration should do quietly. + Home string `json:"home,omitempty"` +} + +// Network is a named network on this machine. +// +// **A name and nothing else.** Not a driver, a subnet or a gateway: each of those is something a +// module would have to know about the machine it lands on, and a module naming a subnet is a +// module that collides with whatever else chose the same one. The runtime picks; the mesh names +// (novox/hq ADR 0029). +type Network struct { + ID string `json:"id"` + Type Type `json:"type"` + Name string `json:"name"` +} + +func (n *Network) Identity() string { return n.ID } +func (n *Network) Kind() Type { return TypeNetwork } +func (n *Network) Target() string { return n.Name } + +func (n *Network) validate(where string, _ bool) []string { + var problems []string + if n.Name == "" { + problems = append(problems, where+": a network needs a name") + } + // The runtimes accept more than this, and the mesh does not: a name with a slash or a colon + // in it reads as a reference to something else entirely wherever it is later printed. + for _, r := range n.Name { + if (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') && (r < '0' || r > '9') && + r != '-' && r != '_' && r != '.' { + problems = append(problems, where+ + ": a network name is letters, digits, dashes, underscores and dots, and "+ + n.Name+" is not") + break + } + } + return problems +} + +func (u *User) Identity() string { return u.ID } +func (u *User) Kind() Type { return TypeUser } +func (u *User) Target() string { return u.Name } + +func (u *User) validate(where string, _ bool) []string { + var problems []string + if u.Name == "" { + problems = append(problems, where+": a user needs a name") + } + if u.Shell != "" && !strings.HasPrefix(u.Shell, "/") { + problems = append(problems, where+ + ": a login shell is an absolute path, and "+u.Shell+" is not one") + } + if u.Home != "" && !strings.HasPrefix(u.Home, "/") { + problems = append(problems, where+": a home directory is an absolute path") + } + return problems +} + +// Archive is a set of files, fetched by digest and unpacked. +// +// For the case inlining cannot serve: a theme, an icon set, a tree of configuration. Hundreds of +// files inlined would make every declaration enormous and rewrite all of it when one changed. +// +// **Pinned by digest, and the digest is checked before anything is unpacked.** The same discipline +// the bootstrap uses for images, and for the same reason — this is fetched over a network the +// mesh does not control, and a reference that can be made to point elsewhere is not a reference. +type Archive struct { + ID string `json:"id"` + Type Type `json:"type"` + // Source is where to fetch it from. + Source string `json:"source"` + // Digest is sha256 of the archive, as "sha256:". + Digest string `json:"digest"` + // Path is the directory it is unpacked into. + Path string `json:"path"` + // Owner is the user the unpacked files belong to. Absent means root. + Owner string `json:"owner,omitempty"` +} + +func (a *Archive) Identity() string { return a.ID } +func (a *Archive) Kind() Type { return TypeArchive } +func (a *Archive) Target() string { return a.Path } + +func (a *Archive) validate(where string, _ bool) []string { + var problems []string + if a.Source == "" { + problems = append(problems, where+": an archive needs somewhere to fetch it from") + } + if a.Path == "" { + problems = append(problems, where+": an archive needs somewhere to unpack into") + } + if !strings.HasPrefix(a.Digest, "sha256:") || len(a.Digest) != len("sha256:")+64 { + // Refused rather than fetched and trusted. Everything else pinned in this vocabulary is + // pinned by digest, and an archive that was not would be the one way in. + problems = append(problems, where+ + ": an archive is pinned by digest, as sha256:<64 hex characters>") + } + return problems +} + +// Service is a unit the host puts into a state. It does not install the unit. +// +// Two states, and they are orthogonal rather than one scale. A unit can be enabled and stopped +// (it will come back at boot), or disabled and running (started by hand, gone after a reboot). +// Folding them into one field would make the second expressible only by accident. +type Service struct { + ID string `json:"id"` + Type Type `json:"type"` + Unit string `json:"unit"` + State string `json:"state"` + // Boot is "enabled" or "disabled" — whether the unit starts at boot. Optional: absent means + // the host asserts nothing about it and leaves whatever is there. + // + // Without this the host could start a unit and not make it survive a reboot, which is a + // declaration that reports success and stops being true at the next power cut. + Boot string `json:"boot,omitempty"` + + // RestartOn names resources whose change means this service must be restarted. + // + // Because a running service does not re-read its configuration. Replace the file, find the + // service already running, do nothing, and the machine keeps behaving the way it did before — + // while every check passes, because the file is right and the service is up. That is not + // hypothetical: it is how a third node joining a mesh left the first two carrying a network + // that no longer existed, and every part of it reported success. + // + // This is declared state rather than a command. The declaration says the running service must + // reflect these files; the host works out that it does not and acts. A *command* to restart + // would be an action, and the link may not carry one (novox/hq ADR 0005) — so this is not a + // way around that rule, it is the shape the rule leaves. + RestartOn []string `json:"restart-on,omitempty"` +} + +func (s *Service) Identity() string { return s.ID } +func (s *Service) Kind() Type { return TypeService } +func (s *Service) Target() string { return s.Unit } + +func (s *Service) validate(where string, _ bool) []string { + var problems []string + if s.Unit == "" { + problems = append(problems, where+": a service needs a unit") + } + if s.State != "running" && s.State != "stopped" { + problems = append(problems, fmt.Sprintf( + "%s: state %q; a service is \"running\" or \"stopped\"", where, s.State)) + } + if s.Boot != "" && s.Boot != "enabled" && s.Boot != "disabled" { + problems = append(problems, fmt.Sprintf( + "%s: boot %q; a service is \"enabled\" or \"disabled\" at boot, or omits it to "+ + "leave the machine's own setting alone", where, s.Boot)) + } + return problems +} + +// Package is a package that should be present. +// +// Present is the whole of what it asserts, never a version: version is the package manager's +// business and the mesh does not hold a second opinion about it. +type Package struct { + ID string `json:"id"` + Type Type `json:"type"` + Package string `json:"package"` +} + +func (p *Package) Identity() string { return p.ID } +func (p *Package) Kind() Type { return TypePackage } +func (p *Package) Target() string { return p.Package } + +func (p *Package) validate(where string, _ bool) []string { + if p.Package == "" { + return []string{where + ": a package needs a package name"} + } + return nil +} + +// Container is a container that should be running, from an image pinned by digest. +type Container struct { + ID string `json:"id"` + Type Type `json:"type"` + Name string `json:"name"` + // Image is pinned by digest (novox/hq ADR 0006) — a tag moves and a digest does not. + Image string `json:"image"` + Env map[string]string `json:"env,omitempty"` + // EnvFile names files the runtime reads environment from, in order. + // + // **Because a secret may not travel in Env.** A declaration reaches a node over the broker, + // and `env` is plain text in it — so a password there is a password the broker sees, which is + // the transitive trust refused everywhere else (novox/hq ADR 0004). A sealed file reaches the + // machine unreadable, the host writes it, and the runtime reads it: the mesh never holds it + // and neither does anything between them. + // + // It is also simply how third-party software takes credentials. Nothing that ships in a + // container will read a path the mesh invented; every one of them reads its environment. + EnvFile []string `json:"env-file,omitempty"` + Ports []string `json:"ports,omitempty"` + Volumes []string `json:"volumes,omitempty"` + Args []string `json:"args,omitempty"` + // Names this container can reach, as `name:address`. + // + // **Because a container does not inherit the machine's names.** It gets its own `/etc/hosts` + // holding only its own hostname, so every internal name the mesh wrote for this machine is + // invisible to the thing the machine is running. That was hit for real: a database client on + // one node could not resolve another node, on a mesh where both names were correct and + // present on both machines. + // + // **A file rather than a resolver, which is the decision the mesh already made about names** + // and this extends rather than overturns: it works on every runtime, needs no package, and + // has no failure mode of its own. A resolver becomes necessary when names are wanted that are + // not one-per-node — service names, wildcards — and that is still not true. + // + // Set by the mesh, not by a module: which machines exist is a fact about the mesh, and a + // module that listed them would be a module that goes stale when one joins. + Hosts []string `json:"hosts,omitempty"` + + // Network is the container's network, passed to the runtime unchanged. + // + // Needed because the control plane must reach the store and the broker on the machine it was + // raised on, before there is any mesh to arrange that. The alternative was publishing ports + // and guessing an address that works from inside a container, which is the same thing with a + // worse failure mode. + Network string `json:"network,omitempty"` + + // RestartOn names resources whose change means this container must be recreated — the same + // field a service has, for the same reason (novox/hq 04-ISSUES/009). A container reads a + // mounted file once at start; a changed file leaves the running process holding the old value, + // while every check passes because the file on disk is right. The container's spec — image, + // env, volumes — does not include a mounted file's *content*, so a settings change that + // re-renders that file is invisible to the ordinary spec diff. This closes that: the host + // recreates the container when one of these resources changed this pass, even if the spec + // matches. + RestartOn []string `json:"restart-on,omitempty"` +} + +func (c *Container) Identity() string { return c.ID } +func (c *Container) Kind() Type { return TypeContainer } +func (c *Container) Target() string { return c.Name } + +func (c *Container) validate(where string, _ bool) []string { + var problems []string + if c.Name == "" { + problems = append(problems, where+": a container needs a name") + } + return append(problems, checkImage(where, c.Image)...) +} + +// Action runs something the bundle declared, and the host never learns what it means. +type Action struct { + ID string `json:"id"` + Type Type `json:"type"` + Command []string `json:"command"` + // Verify is not optional and is not a courtesy. It is the read-back AND the idempotency + // check: the host does not know what a database is, so "is it already there" is a question + // only the declaration can ask (novox/hq ADR 0005). + Verify []string `json:"verify"` + // In names a container to run inside. Empty means the machine itself. + In string `json:"in,omitempty"` +} + +func (a *Action) Identity() string { return a.ID } +func (a *Action) Kind() Type { return TypeAction } + +func (a *Action) Target() string { + target := strings.Join(a.Command, " ") + if a.In != "" { + return "in " + a.In + ": " + target + } + return target +} + +func (a *Action) validate(where string, allowActions bool) []string { + // The bound the whole security argument rests on (novox/hq ADR 0005). + if !allowActions { + return []string{where + + ": an action arrived over the link, and the link may not carry one. The host " + + "applies declarations of known shape; a command to run is not one. A bundle may " + + "carry an action because it arrives with the binary — anyone able to put a " + + "hostile action there could have put it in the host itself"} + } + var problems []string + if len(a.Command) == 0 { + problems = append(problems, where+": an action needs a command") + } + if len(a.Verify) == 0 { + problems = append(problems, where+ + ": an action needs a verify. An action that runs and reports success without "+ + "reading anything back is the fault this host exists to prevent, and verify is "+ + "also how the host knows whether the action is already done") + } + return problems +} + +// newOf returns an empty resource of a kind, or nil if the kind is unknown. +// +// This is the whole vocabulary, in one place. A kind that is not here cannot be declared. +func newOf(t Type) Resource { + switch t { + case TypeDirectory: + return &Directory{} + case TypeFile: + return &File{} + case TypeService: + return &Service{} + case TypePackage: + return &Package{} + case TypeContainer: + return &Container{} + case TypeAction: + return &Action{} + case TypeNetwork: + return &Network{} + case TypeUser: + return &User{} + case TypeArchive: + return &Archive{} + } + return nil +} + +// Vocabulary is every kind this host speaks. +func Vocabulary() []Type { + return []Type{ + TypeAction, TypeArchive, TypeContainer, TypeDirectory, TypeFile, TypeNetwork, + TypePackage, TypeService, TypeUser, + } +} + +// Declaration is what a machine should be, in the order it should be made so. +type Declaration struct { + Version int + // For names the node this is meant for. A host with an identity refuses one addressed + // elsewhere; a host without one — the first node, applying the bundle it carries — has + // nothing to check against. + For string + // Resources, in the order they are applied. The host does not sort them: ordering is a + // decision, and deciding is not what the host does (novox/hq ADR 0005). + Resources []Resource +} + +// RefusalError refuses a whole declaration, naming every problem at once. +// +// Every problem rather than the first: a caller fixing one at a time learns the next only by +// running again, and a declaration is generated, so a person reading this is debugging the +// generator. +type RefusalError struct { + Problems []string +} + +func (e *RefusalError) Error() string { + return fmt.Sprintf( + "this declaration is refused, and none of it was applied:\n - %s\n\n"+ + "A host that applied the parts it understood would leave a machine that looks "+ + "configured and is not.", + strings.Join(e.Problems, "\n - ")) +} + +// Parse reads a declaration that arrived over the link, and refuses anything it does not fully +// understand — including any action, which the link may not carry (novox/hq ADR 0005). +func Parse(raw []byte) (*Declaration, error) { return parse(raw, false) } + +// ParseTrusted reads a declaration from a source already as privileged as the host itself: the +// bundle it carries, or a file handed to it by someone who is running it as root. +// +// Actions are permitted here and nowhere else. The asymmetry is deliberate and is the entire +// content of ADR 0005: refusing actions from the bundle buys nothing, because whoever built the +// bundle built the binary; refusing them from the link buys the bound on what a compromised +// control plane can express. +func ParseTrusted(raw []byte) (*Declaration, error) { return parse(raw, true) } + +// envelope is the declaration with its resources still unread. +// +// Two passes, because which fields are legal depends on the "type" inside each resource. The +// first pass takes the envelope and each resource's bytes; the second decodes each one into +// the struct for its kind, strictly. +type envelope struct { + Version int `json:"declaration"` + For string `json:"for,omitempty"` + Resources []json.RawMessage `json:"resources"` +} + +func parse(raw []byte, allowActions bool) (*Declaration, error) { + var env envelope + if err := strictDecode(raw, &env); err != nil { + return nil, &RefusalError{Problems: []string{"not a declaration: " + err.Error()}} + } + + if env.Version != Version { + // Everything below assumes the vocabulary, so there is nothing further to say. + return nil, &RefusalError{Problems: []string{fmt.Sprintf( + "declaration version %d; this host speaks version %d. Refused whole rather than "+ + "partly, so a newer vocabulary is never half-applied by an older host", + env.Version, Version)}} + } + + d := &Declaration{Version: env.Version, For: env.For} + var problems []string + + if len(env.Resources) == 0 { + problems = append(problems, "no resources. An empty declaration is a mistake, not a "+ + "machine with nothing on it — say so with an explicit empty list if that is meant") + } + + seen := map[string]int{} + for i, rawResource := range env.Resources { + // Peek, leniently. This pass only needs to know which struct to decode into; reading + // strictly here would report an unknown field before knowing which fields are known. + var head struct { + ID string `json:"id"` + Type Type `json:"type"` + } + _ = json.Unmarshal(rawResource, &head) + + where := fmt.Sprintf("resource %d", i) + if head.ID != "" { + where = fmt.Sprintf("resource %q", head.ID) + } + + if head.ID == "" { + problems = append(problems, where+": no id. Identity is what lets the host know "+ + "this is the same resource it applied last time") + } else if first, ok := seen[head.ID]; ok { + problems = append(problems, fmt.Sprintf( + "%s: id already used by resource %d. Two resources with one identity cannot "+ + "both be tracked", where, first)) + } else { + seen[head.ID] = i + } + + resource := newOf(head.Type) + if resource == nil { + problems = append(problems, fmt.Sprintf( + "%s: unknown type %q. This host understands %s", where, head.Type, vocabulary())) + continue + } + + // A field the kind does not have is refused, and the struct is what says so — there + // is no list of exclusions for anyone to keep current. + // + // Asked separately rather than taken from the decoder's error, because the decoder + // stops at the first unknown field and this record promises every problem at once. A + // caller fixing one field at a time learns the next only by running again. + if unknown := unknownFields(rawResource, resource); len(unknown) > 0 { + for _, field := range unknown { + problems = append(problems, fmt.Sprintf( + "%s: a %s does not use %q, and it is set. Refused rather than ignored", + where, head.Type, field)) + } + continue + } + if err := json.Unmarshal(rawResource, resource); err != nil { + problems = append(problems, fmt.Sprintf("%s: %s", where, err)) + continue + } + + problems = append(problems, resource.validate(where, allowActions)...) + d.Resources = append(d.Resources, resource) + } + + if len(problems) > 0 { + return nil, &RefusalError{Problems: problems} + } + return d, nil +} + +func strictDecode(raw []byte, into any) error { + // DisallowUnknownFields is the whole point rather than strictness for its own sake: a + // field the host does not know is a thing the control plane believes it asked for. + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(into); err != nil { + return err + } + // And **nothing after it**. A decoder reads one value and stops, so a file holding a + // declaration followed by anything at all — a truncated rewrite, two declarations + // concatenated, a stray line from whatever wrote the file — parses as the first value and the + // rest is never looked at. + // + // That is the same fault this host refuses everywhere else, in its quietest form: the machine + // applies something, reports success, and what it applied is not what the file says. Found + // when a test harness appended a line to a bundle by accident and every apply kept working. + if _, err := dec.Token(); err != io.EOF { + return fmt.Errorf( + "there is more in this file after the declaration ends. Refused whole: a file with " + + "something after it may be a truncated rewrite or two declarations run together, " + + "and applying the first would be applying something nobody wrote") + } + return nil +} + +// unknownFields names every JSON key the kind's struct has no field for. +// +// The struct's own tags are the list of what is legal, so adding a field to a kind is the +// whole of adding it — there is nowhere else that has to agree. +func unknownFields(raw []byte, into Resource) []string { + var got map[string]json.RawMessage + if err := json.Unmarshal(raw, &got); err != nil { + return nil // not an object; the decode below will say so properly + } + + known := map[string]bool{} + t := reflect.TypeOf(into).Elem() + for i := 0; i < t.NumField(); i++ { + name, _, _ := strings.Cut(t.Field(i).Tag.Get("json"), ",") + if name != "" && name != "-" { + known[name] = true + } + } + + var unknown []string + for field := range got { + if !known[field] { + unknown = append(unknown, field) + } + } + sort.Strings(unknown) + return unknown +} + +func checkMode(where, mode string) []string { + if mode == "" { + return nil + } + if len(mode) != 4 || mode[0] != '0' { + return []string{fmt.Sprintf( + "%s: mode %q; write it as four octal digits such as \"0644\", so it means the "+ + "same thing here as it does in the manifest it came from", where, mode)} + } + for _, c := range mode[1:] { + if c < '0' || c > '7' { + return []string{fmt.Sprintf("%s: mode %q is not octal", where, mode)} + } + } + return nil +} + +// checkImage insists on a digest. +// +// A tag moves and a digest does not. The bundle's whole claim is that what it names is exact +// (novox/hq ADR 0006), and a bundle pinning `postgres:17` pins nothing — it names whatever +// that tag points at on the day the host happens to run. +func checkImage(where, image string) []string { + if image == "" { + return []string{where + ": a container needs an image"} + } + name, digest, found := strings.Cut(image, "@") + if !found || name == "" { + return []string{fmt.Sprintf( + "%s: image %q is not pinned. Write it as name@sha256:... — a tag moves, and a "+ + "bundle that pinned a tag would not be pinned", where, image)} + } + if !strings.HasPrefix(digest, "sha256:") || len(digest) != len("sha256:")+64 { + return []string{fmt.Sprintf( + "%s: image digest %q is not a sha256 digest", where, digest)} + } + return nil +} + +func vocabulary() string { + kinds := Vocabulary() + names := make([]string, 0, len(kinds)) + for _, t := range kinds { + names = append(names, string(t)) + } + sort.Strings(names) + return strings.Join(names, ", ") +} + +// ParseFileTrusted reads a declaration from a file somebody handed this host. +// +// The same as ParseTrusted, and it allows whole-line `//` comments first. A pinned, hand-authored +// artefact that nobody can annotate is one nobody can review — the substrate bundle is mostly +// explanation of why each digest is what it is. +// +// **Only for a file, never for the link.** Over the link the format stays exactly JSON, because +// a wire format with a second thing to strip is a wire format with a second thing to disagree +// about. +// +// It exists because there were two readers for one file: the bundle stripped comments and `apply` +// did not, so the example bundle in this repository could be built into a binary and not applied +// from disk. The failure was `invalid character '/'`, which names the symptom and not the cause. +func ParseFileTrusted(raw []byte) (*Declaration, error) { + return ParseTrusted(stripComments(raw)) +} + +// stripComments removes whole lines beginning with `//`. +// +// Only whole lines: anything cleverer would need to know where strings begin and end, and a +// parser that half-understands its input is worse than one that does not try. A `//` inside a +// value — every image reference has one — is untouched. +func stripComments(raw []byte) []byte { + var kept []string + for _, line := range strings.Split(string(raw), "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "//") { + continue + } + kept = append(kept, line) + } + return []byte(strings.Join(kept, "\n")) +} diff --git a/internal/declaration/declaration_test.go b/internal/declaration/declaration_test.go new file mode 100644 index 0000000..f738002 --- /dev/null +++ b/internal/declaration/declaration_test.go @@ -0,0 +1,384 @@ +package declaration + +import ( + "errors" + "strings" + "testing" +) + +// Each test names the decision it defends (novox/hq ADR 0017). The decision here is ADR 0005, +// and the property it turns on is that unknown is REFUSED, never skipped. + +func valid() string { + return `{"declaration":1,"resources":[ + {"id":"etc","type":"directory","path":"/etc/mesh","mode":"0755"}, + {"id":"conf","type":"file","path":"/etc/mesh/host.conf","content":"a\n","mode":"0640"}, + {"id":"svc","type":"service","unit":"mesh-host.service","state":"running"} + ]}` +} + +func refusalFor(t *testing.T, raw string) *RefusalError { + t.Helper() + _, err := Parse([]byte(raw)) + if err == nil { + t.Fatal("expected a refusal") + } + var refusal *RefusalError + if !errors.As(err, &refusal) { + t.Fatalf("expected a RefusalError, got %T: %v", err, err) + } + return refusal +} + +func TestAValidDeclarationParsesInOrder(t *testing.T) { + d, err := Parse([]byte(valid())) + if err != nil { + t.Fatalf("unexpected refusal: %v", err) + } + // Order is stated, not derived. The host must not sort. + got := []string{d.Resources[0].Identity(), d.Resources[1].Identity(), d.Resources[2].Identity()} + want := []string{"etc", "conf", "svc"} + for i := range want { + if got[i] != want[i] { + t.Fatalf("resources reordered: %v, want %v", got, want) + } + } +} + +func TestAnUnknownTypeRefusesTheWholeDeclaration(t *testing.T) { + // The property everything else rests on. A host that skipped what it did not understand + // would apply most of a declaration and report success — a node that looks configured and + // is not, which is 04-ISSUES/003 with the declaration on the other side of the wire. + refusal := refusalFor(t, `{"declaration":1,"resources":[ + {"id":"ok","type":"directory","path":"/etc/mesh"}, + {"id":"what","type":"blockchain","path":"/etc/mesh"} + ]}`) + + joined := strings.Join(refusal.Problems, "\n") + if !strings.Contains(joined, "blockchain") { + t.Errorf("the unknown type was not named: %v", refusal.Problems) + } + // And it must say what IS understood, or the reader goes to the source to find out. + if !strings.Contains(joined, "directory") || !strings.Contains(joined, "service") { + t.Errorf("the refusal does not say what this host understands: %v", refusal.Problems) + } +} + +func TestAnUnknownFieldIsRefused(t *testing.T) { + // A field the host does not know is a thing the control plane believes it asked for. + refusal := refusalFor(t, `{"declaration":1,"resources":[ + {"id":"conf","type":"file","path":"/etc/x","content":"a","immutable":true} + ]}`) + if !strings.Contains(strings.Join(refusal.Problems, "\n"), "immutable") { + t.Errorf("the unknown field was not named: %v", refusal.Problems) + } +} + +func TestAFieldTheTypeDoesNotUseIsRefusedNotIgnored(t *testing.T) { + // The same fault in miniature: set and ignored means the control plane believes it asked + // for something the host will never do. + refusal := refusalFor(t, `{"declaration":1,"resources":[ + {"id":"svc","type":"service","unit":"a.service","state":"running","path":"/etc/x"} + ]}`) + joined := strings.Join(refusal.Problems, "\n") + if !strings.Contains(joined, "path") || !strings.Contains(joined, "Refused rather than ignored") { + t.Errorf("a field a service does not use was accepted: %v", refusal.Problems) + } +} + +func TestAnUnknownVersionIsRefusedWhole(t *testing.T) { + // An older host handed a newer vocabulary must not quietly do half of it. + refusal := refusalFor(t, `{"declaration":99,"resources":[ + {"id":"a","type":"directory","path":"/etc/mesh"} + ]}`) + joined := strings.Join(refusal.Problems, "\n") + if !strings.Contains(joined, "99") || !strings.Contains(joined, "version 1") { + t.Errorf("the version mismatch was not stated plainly: %v", refusal.Problems) + } + // Nothing else is reported, because everything else assumes a vocabulary this host does + // not have — a list of complaints derived from the wrong grammar is noise. + if len(refusal.Problems) != 1 { + t.Errorf("expected only the version problem, got: %v", refusal.Problems) + } +} + +func TestEveryProblemIsReportedAtOnce(t *testing.T) { + // A declaration is generated, so a person reading a refusal is debugging the generator. + // Fixing one problem at a time and re-running to find the next wastes their afternoon. + refusal := refusalFor(t, `{"declaration":1,"resources":[ + {"id":"","type":"directory","path":"/a"}, + {"id":"b","type":"file"}, + {"id":"c","type":"service","unit":"x.service","state":"dancing"} + ]}`) + if len(refusal.Problems) < 3 { + t.Errorf("expected every problem at once, got: %v", refusal.Problems) + } +} + +func TestIdentityIsRequiredAndUnique(t *testing.T) { + // Identity is what lets the store know this is the same resource it applied last time, + // which is what makes removal possible at all. + refusal := refusalFor(t, `{"declaration":1,"resources":[ + {"id":"same","type":"directory","path":"/a"}, + {"id":"same","type":"directory","path":"/b"} + ]}`) + if !strings.Contains(strings.Join(refusal.Problems, "\n"), "already used") { + t.Errorf("a duplicate identity was accepted: %v", refusal.Problems) + } +} + +func TestModeIsRefusedUnlessItMeansWhatItLooksLike(t *testing.T) { + // "644" and "0644" differ, and the one that looks right in a manifest is the four-digit + // form. Accepting both would make a mode mean two things. + for _, mode := range []string{"644", "0999", "rwxr-xr-x", "07777777"} { + refusal := refusalFor(t, `{"declaration":1,"resources":[ + {"id":"f","type":"file","path":"/a","mode":"`+mode+`"} + ]}`) + if !strings.Contains(strings.Join(refusal.Problems, "\n"), "mode") { + t.Errorf("mode %q was accepted: %v", mode, refusal.Problems) + } + } + if _, err := Parse([]byte(`{"declaration":1,"resources":[ + {"id":"f","type":"file","path":"/a","mode":"0644"} + ]}`)); err != nil { + t.Errorf("a well-formed mode was refused: %v", err) + } +} + +func TestARefusalSaysNothingWasApplied(t *testing.T) { + // The reader's first question is whether the machine was left half-changed. + refusal := refusalFor(t, `{"declaration":1,"resources":[{"id":"x","type":"nope"}]}`) + if !strings.Contains(refusal.Error(), "none of it was applied") { + t.Errorf("the refusal does not say the machine is untouched: %s", refusal.Error()) + } +} + +func TestAnEmptyDeclarationIsAMistake(t *testing.T) { + refusalFor(t, `{"declaration":1,"resources":[]}`) +} + +// --- the vocabulary the substrate bootstrap needs (novox/hq 07-the-substrate.md) --- + +func TestAnActionOverTheLinkIsRefused(t *testing.T) { + // novox/hq ADR 0005. The link may push declarations of known shape and never a command to + // run. This is the boundary the whole security argument rests on, so it is asserted + // directly rather than inferred from the type list. + raw := []byte(`{"declaration":1,"resources":[ + {"id":"schema","type":"action","command":["psql","-f","x.sql"],"verify":["psql","-c","select 1"]} + ]}`) + + if _, err := Parse(raw); err == nil { + t.Fatal("an action arriving over the link was accepted") + } else if !strings.Contains(err.Error(), "the link may not carry one") { + t.Errorf("refused for the wrong reason: %v", err) + } + + // And the same bytes from the bundle are fine — the asymmetry IS the decision. + if _, err := ParseTrusted(raw); err != nil { + t.Errorf("the bundle may carry an action, and this one was refused: %v", err) + } +} + +func TestAnActionWithoutVerifyIsRefused(t *testing.T) { + // An action that runs and reports success without reading anything back is the fault this + // host exists to prevent. Verify is also the idempotency check, so an action without one + // cannot be applied twice safely either. + _, err := ParseTrusted([]byte(`{"declaration":1,"resources":[ + {"id":"schema","type":"action","command":["psql","-f","x.sql"]} + ]}`)) + if err == nil { + t.Fatal("an action with no verify was accepted") + } + if !strings.Contains(err.Error(), "needs a verify") { + t.Errorf("refused for the wrong reason: %v", err) + } +} + +func TestAnImageMustBePinnedByDigest(t *testing.T) { + // novox/hq ADR 0006: reproducibility comes from pinning the identity of a thing. A bundle + // naming a tag pins nothing — it names whatever that tag points at on the day it runs. + for _, image := range []string{ + "postgres:17", + "postgres", + "postgres@sha256:short", + "@sha256:0000000000000000000000000000000000000000000000000000000000000000", + } { + _, err := ParseTrusted([]byte(`{"declaration":1,"resources":[ + {"id":"store","type":"container","name":"store","image":"` + image + `"} + ]}`)) + if err == nil { + t.Errorf("image %q was accepted and is not pinned", image) + } + } + + good := "postgres@sha256:" + strings.Repeat("a", 64) + if _, err := ParseTrusted([]byte(`{"declaration":1,"resources":[ + {"id":"store","type":"container","name":"store","image":"` + good + `"} + ]}`)); err != nil { + t.Errorf("a properly pinned image was refused: %v", err) + } +} + +func TestAFieldTheNewTypesDoNotUseIsRefused(t *testing.T) { + // The field-set check must cover the types added last, not only the three it was written + // for. A package that carries a `content` is a control plane believing it asked for + // something that will never happen. + for _, body := range []string{ + `{"id":"p","type":"package","package":"docker","content":"x"}`, + `{"id":"p","type":"package","package":"docker","image":"x"}`, + `{"id":"c","type":"container","name":"n","image":"i@sha256:` + strings.Repeat("a", 64) + `","unit":"x.service"}`, + `{"id":"a","type":"action","command":["x"],"verify":["y"],"path":"/tmp/x"}`, + } { + _, err := ParseTrusted([]byte(`{"declaration":1,"resources":[` + body + `]}`)) + if err == nil { + t.Errorf("a resource carrying a field its type does not use was accepted: %s", body) + continue + } + if !strings.Contains(err.Error(), "Refused rather than ignored") { + t.Errorf("refused for the wrong reason: %v", err) + } + } +} + +func TestTheVocabularyIsTheEightShapesTheMeshNeeds(t *testing.T) { + // Six of them the bootstrap uses (novox/hq 07-the-substrate.md), and removing one is a + // failing test rather than a discovery during a first-node install. + // + // Two were added on 2026-08-30 and the count is asserted precisely because adding one is a + // decision. `user` and `archive` exist because most of what a person installs is not a + // service: a shell, a chat client, a desktop are a package plus configuration **in + // somebody's home**, and a mesh with no user can only own /etc. `archive` is for the case + // inlining cannot serve — a theme is hundreds of files, and inlining them would rewrite all + // of them whenever one changed. + speaks := map[Type]bool{} + for _, t := range Vocabulary() { + speaks[t] = true + } + for _, want := range []Type{ + TypeDirectory, TypeFile, TypeService, TypePackage, TypeContainer, TypeAction, + TypeUser, TypeArchive, TypeNetwork, + } { + if !speaks[want] { + t.Errorf("the host no longer speaks %q", want) + } + if newOf(want) == nil { + t.Errorf("%q is in the vocabulary and cannot be constructed", want) + } + } + // `network` is the ninth, and novox/hq ADR 0029 is the decision that made it one: an action + // could create a network and nothing could remove it, because an action leaves no footprint + // the host can undo — so the network would outlive every module that was ever unassigned. + if len(speaks) != 9 { + t.Errorf("the vocabulary is %d shapes rather than 9; every addition widens what a compromised "+ + "control plane can express, so a change here is a decision: %s", + len(speaks), vocabulary()) + } +} + +func TestADeclarationFromDiskMayBeAnnotated(t *testing.T) { + // The bundle in this repository is mostly explanation of why each digest is what it is, and + // it could be built into a binary and not applied from disk — two readers for one file. The + // failure was `invalid character '/'`, which names the symptom and not the cause. + raw := []byte(`// why this exists +{ + "declaration": 1, + // and why this resource is here + "resources": [ + {"id": "f", "type": "file", "path": "/etc/x", "content": "hello"} + ] +}`) + d, err := ParseFileTrusted(raw) + if err != nil { + t.Fatalf("a file with comments was refused: %v", err) + } + if len(d.Resources) != 1 { + t.Fatalf("got %d resources", len(d.Resources)) + } + // And the wire format is untouched: over the link it is exactly JSON, because a format with + // a second thing to strip is a format with a second thing to disagree about. + if _, err := Parse(raw); err == nil { + t.Fatal("the link accepted a declaration with comments in it") + } +} + +func TestSomethingInsideAValueIsNotAComment(t *testing.T) { + // Every image reference has a `//` in it somewhere near. Only whole lines are dropped. + d, err := ParseFileTrusted([]byte(`{"declaration":1,"resources":[ + {"id":"f","type":"file","path":"/etc/x","content":"see https://example.invalid/ for why"}]}`)) + if err != nil { + t.Fatal(err) + } + file, ok := d.Resources[0].(*File) + if !ok { + t.Fatalf("got %T", d.Resources[0]) + } + if !strings.Contains(file.Content, "https://example.invalid/") { + t.Fatalf("a value was mangled: %q", file.Content) + } +} + +// A declaration followed by anything at all is refused whole. +// +// A JSON decoder reads one value and stops, so a file holding a declaration and then a stray line +// parses as the declaration and the rest is never looked at. The machine applies something, +// reports success, and what it applied is not what the file says. +// +// Not hypothetical: a test harness appended a line to the substrate bundle by accident, every +// apply kept working, and nothing said so for the entire time it was wrong. +func TestSomethingAfterTheDeclarationIsRefused(t *testing.T) { + good := `{"declaration":1,"resources":[{"id":"a","type":"file","path":"/tmp/a",` + + `"content":"x","mode":"0644"}]}` + if _, err := ParseFileTrusted([]byte(good)); err != nil { + t.Fatalf("an ordinary declaration was refused: %v", err) + } + for _, after := range []string{ + "MESHBUNDLE 2>&1; echo \"__exit=$?\"", + good, + "garbage", + } { + _, err := ParseFileTrusted([]byte(good + "\n" + after)) + if err == nil { + t.Fatalf("a file with %q after the declaration was accepted", after) + } + } + // Trailing whitespace is not "something after it", and refusing it would make every file + // written by an editor unusable. + if _, err := ParseFileTrusted([]byte(good + "\n\n \n")); err != nil { + t.Fatalf("a declaration with a trailing newline was refused: %v", err) + } +} + +// A name that would read as a reference to something else is refused before it reaches a runtime. +func TestANetworkNameIsRefusedIfItIsNotOne(t *testing.T) { + for _, name := range []string{"", "mail/private", "host:mail", "a b"} { + refusal := refusalFor(t, `{"declaration":1,"resources":[ + {"id":"private","type":"network","name":"`+name+`"} + ]}`) + if len(refusal.Problems) == 0 { + t.Errorf("a network named %q was accepted", name) + } + } +} + +// A placeholder with nothing to fill it is refused before anything is written. +// +// The alternative is a configuration file containing the literal `${secret:x}`, which the program +// reads as a value and fails on somewhere with no connection to this. +func TestAFileAskingForASecretItWasNotGivenIsRefused(t *testing.T) { + refusal := refusalFor(t, `{"declaration":1,"resources":[ + {"id":"c","type":"file","path":"/tmp/x","content":"token=${secret:absent}"} + ]}`) + if len(refusal.Problems) == 0 { + t.Fatal("a file naming a secret nobody gave it was accepted") + } +} + +// And a secret nobody asked for, because somebody believes it is in a file where it is not. +func TestASecretTheContentNeverUsesIsRefused(t *testing.T) { + refusal := refusalFor(t, `{"declaration":1,"resources":[ + {"id":"c","type":"file","path":"/tmp/x","content":"nothing here","secrets":{"spare":"S"}} + ]}`) + if len(refusal.Problems) == 0 { + t.Fatal("a secret the content never mentions was accepted") + } +} diff --git a/internal/declaration/emitted_check_test.go b/internal/declaration/emitted_check_test.go new file mode 100644 index 0000000..948dca5 --- /dev/null +++ b/internal/declaration/emitted_check_test.go @@ -0,0 +1,33 @@ +package declaration + +import ( + "os" + "testing" +) + +// The control plane's actual output, put through the host's actual parser. +// +// Skipped unless MESH_EMITTED names a file, so this is a check somebody runs deliberately rather +// than a dependency between two repositories. It exists because "the host needs no new +// vocabulary" is the load-bearing claim of every computed and contributed resource, and the only +// honest way to know it is to hand the host one and see. +func TestWhatTheControlPlaneEmittedIsSomethingThisHostAccepts(t *testing.T) { + path := os.Getenv("MESH_EMITTED") + if path == "" { + t.Skip("set MESH_EMITTED to a declaration the control plane produced") + } + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + d, err := Parse(raw) + if err != nil { + t.Fatalf("the host refuses what the control plane sends:\n%v", err) + } + if len(d.Resources) == 0 { + t.Fatal("parsed, and empty") + } + for _, r := range d.Resources { + t.Logf("accepted %s %s → %s", r.Kind(), r.Identity(), r.Target()) + } +} diff --git a/internal/identity/identity.go b/internal/identity/identity.go new file mode 100644 index 0000000..9ff2ee8 --- /dev/null +++ b/internal/identity/identity.go @@ -0,0 +1,199 @@ +// Package identity is what this node presents to prove it is this node. +// +// novox/hq ADR 0004: the node generates a keypair, the private half never leaves the machine, and +// the mesh records the public half. The same rule the overlay keys already follow, applied to the +// node itself. +// +// The mesh issues nothing here. A node arrives at enrolment already holding its identity; what it +// receives is *being known*. So this package is the whole of a node's identity, and it is made +// before anybody is asked for anything. +package identity + +import ( + "crypto/ed25519" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" +) + +// FileName is where a node keeps its identity, beside its state. +const FileName = "identity.json" + +// Path is where the identity lives, given where the state lives. +func Path(statePath string) string { + return filepath.Join(filepath.Dir(statePath), FileName) +} + +// dirOf is where a node keeps everything it knows about itself. +func dirOf(statePath string) string { return filepath.Dir(statePath) } + +// Identity is this node's own keypair, the name the mesh knows it by, and what it needs to get +// back to that mesh without a person. +// +// The keypair is the node's own and was never anybody else's. Everything under Membership was +// learned at enrolment and is the mesh's answer rather than this machine's — kept here because a +// node that could not reconnect after a restart without a new token would make disconnection a +// crisis instead of an ordinary situation (novox/hq ADR 0004). +type Identity struct { + // Node is the name in the mesh's records. Learned at enrolment — the one thing here the node + // does not decide for itself. + Node string `json:"node"` + + Public []byte `json:"public"` + Private []byte `json:"private"` + + Membership Membership `json:"membership"` + + // Overlay is this node's key on the private network. Generated here, like the identity above, + // and for the same reason: the mesh computes a graph it cannot impersonate. + Overlay OverlayKey `json:"overlay"` +} + +// Membership is how this node reaches the mesh it belongs to, and who it believes. +type Membership struct { + // Broker is an address, not a name: there is no resolution before the link. + Broker string `json:"broker"` + + // Fingerprint is checked before anything is sent, on every connection and not only the first. + Fingerprint string `json:"fingerprint"` + + // Signer is the control plane's public signing key. Kept because **each declaration is + // verified by its signature, every time** (novox/hq ADR 0004) — a node that only pinned the + // broker would make the control plane's authority transitive, and a compromised broker could + // then forge declarations, which is the whole machine. + Signer []byte `json:"signer"` + + // Password is this node's own broker account, issued at enrolment and belonging to it alone. + // Not the token's secret: that is spent, and a credential that lives for ever should not be + // the same string as one that was meant to be used once. + Password string `json:"password"` +} + +// Queue is where this node listens. Its account may read this and nothing else. +func (i Identity) Queue() string { return "node." + i.Node } + +// Joined reports whether this identity can reach its mesh unaided. +func (m Membership) Joined() bool { + return m.Broker != "" && m.Fingerprint != "" && len(m.Signer) == ed25519.PublicKeySize && + m.Password != "" +} + +// ErrNoIdentity means this machine has not enrolled. +// +// Not a fault: a hosted machine has a host running and no identity, and that is a real state +// (novox/hq 09-the-node-lifecycle). It is the difference between "not a node yet" and "a node +// whose identity is missing", and only the second is a problem. +var ErrNoIdentity = errors.New("this machine has no identity, so it has not joined a mesh") + +// Generate makes a new identity. The private half exists only here, from this moment. +func Generate(node string) (Identity, error) { + public, private, err := ed25519.GenerateKey(nil) + if err != nil { + return Identity{}, fmt.Errorf("cannot generate this node's identity: %w", err) + } + return Identity{Node: node, Public: public, Private: private}, nil +} + +// Sign proves this node is that node. +func (i Identity) Sign(message []byte) []byte { + return ed25519.Sign(ed25519.PrivateKey(i.Private), message) +} + +// PublicBase64 is the public half as it travels. +func (i Identity) PublicBase64() string { + return base64.StdEncoding.EncodeToString(i.Public) +} + +// Load reads this node's identity. +func Load(path string) (Identity, error) { + raw, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return Identity{}, ErrNoIdentity + } + if err != nil { + // Never a silent absence. A machine that has an identity and cannot read it must not + // behave as one that never had one — the second re-enrols, which would discard the + // identity the mesh still believes. + return Identity{}, fmt.Errorf( + "this node has an identity at %s and cannot read it: %w. That is not the same as "+ + "having none, so it will not re-enrol on its own", path, err) + } + + var i Identity + if err := json.Unmarshal(raw, &i); err != nil { + return Identity{}, fmt.Errorf("the identity at %s is not readable: %w", path, err) + } + if len(i.Private) != ed25519.PrivateKeySize || len(i.Public) != ed25519.PublicKeySize { + return Identity{}, fmt.Errorf( + "the identity at %s is the wrong shape: %d-byte public and %d-byte private, where an "+ + "Ed25519 identity is %d and %d", + path, len(i.Public), len(i.Private), ed25519.PublicKeySize, ed25519.PrivateKeySize) + } + if strings.TrimSpace(i.Node) == "" { + return Identity{}, fmt.Errorf("the identity at %s names no node", path) + } + // Checked here rather than at the moment it is used, which would be while trying to + // reconnect on a machine nobody is watching. + if !i.Membership.Joined() { + return Identity{}, fmt.Errorf( + "the identity at %s does not say how to reach its mesh, so this node cannot "+ + "reconnect. It needs a new token", path) + } + return i, nil +} + +// Save writes the identity, readable by nobody else. +// +// Written to a temporary file and renamed, so a machine losing power mid-write keeps the identity +// it had rather than acquiring half of one. A node cannot regenerate its way out of that: the mesh +// believes the old public key, and a new one needs a new token from a person. +func Save(path string, i Identity) error { + if len(i.Private) != ed25519.PrivateKeySize { + return errors.New("refusing to save an identity with no usable private key") + } + // Save refuses exactly what Load refuses. Without this, a caller can write a file that + // cannot be read back — and it would be read back on the next start, on a machine nobody is + // watching, by which time the token that could have fixed it is spent. + if strings.TrimSpace(i.Node) == "" { + return errors.New("refusing to save an identity that names no node") + } + if !i.Membership.Joined() { + return errors.New("refusing to save an identity that does not say how to reach its " + + "mesh: it could not be used after a restart, and Load will not accept it") + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return err + } + + raw, err := json.MarshalIndent(i, "", " ") + if err != nil { + return err + } + + tmp, err := os.CreateTemp(filepath.Dir(path), ".identity-*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + + if err := tmp.Chmod(0o600); err != nil { + tmp.Close() + return err + } + if _, err := tmp.Write(raw); err != nil { + tmp.Close() + return err + } + if err := tmp.Sync(); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmp.Name(), path) +} diff --git a/internal/identity/identity_test.go b/internal/identity/identity_test.go new file mode 100644 index 0000000..2d0c4f9 --- /dev/null +++ b/internal/identity/identity_test.go @@ -0,0 +1,398 @@ +package identity + +import ( + "crypto/ed25519" + "encoding/base64" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestAMachineThatHasNotJoinedHasNoIdentityAndThatIsNotAFault(t *testing.T) { + // A hosted machine has a host running and no identity. That is a real state, and confusing + // it with a fault would have every fresh install look broken. + _, err := Load(Path(filepath.Join(t.TempDir(), "state.json"))) + if !errors.Is(err, ErrNoIdentity) { + t.Fatalf("a machine that never joined gave %v", err) + } +} + +func TestAnUnreadableIdentityIsNotTheSameAsHavingNone(t *testing.T) { + // The distinction that matters most here. "None" leads to enrolling; if an unreadable + // identity took that path, a node would discard the identity the mesh still believes and + // need a person with a new token to get back. + dir := t.TempDir() + path := Path(filepath.Join(dir, "state.json")) + if err := os.WriteFile(path, []byte("{"), 0o600); err != nil { + t.Fatal(err) + } + + _, err := Load(path) + if err == nil { + t.Fatal("a corrupt identity loaded") + } + if errors.Is(err, ErrNoIdentity) { + t.Fatal("a corrupt identity was reported as having none; this node would re-enrol and " + + "throw away the identity the mesh believes") + } +} + +// joined is an identity as it exists after enrolment, which is the only kind ever saved: +// Generate makes the keypair, and the mesh supplies everything under Membership. +func joined(t *testing.T, name string) Identity { + t.Helper() + made, err := Generate(name) + if err != nil { + t.Fatal(err) + } + signer, _, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + made.Membership = Membership{ + Broker: "192.0.2.10:5671", + Fingerprint: "sha256:" + strings.Repeat("ab", 32), + Signer: signer, + Password: "this node's own", + } + return made +} + +func TestSaveRefusesWhatLoadWouldRefuse(t *testing.T) { + // The two must agree, or a caller can write a file that cannot be read back — and it would + // be read back on the next start, on a machine nobody is watching, by which time the token + // that could have fixed it is spent. + path := Path(filepath.Join(t.TempDir(), "state.json")) + unenrolled, err := Generate("workstation") + if err != nil { + t.Fatal(err) + } + if err := Save(path, unenrolled); err == nil { + t.Fatal("an identity with no membership was saved; Load will not accept it") + } + if _, err := os.Stat(path); err == nil { + t.Error("the refused identity was written anyway") + } +} + +func TestWhatIsSavedIsWhatIsLoaded(t *testing.T) { + path := Path(filepath.Join(t.TempDir(), "state.json")) + made := joined(t, "workstation") + if err := Save(path, made); err != nil { + t.Fatal(err) + } + + back, err := Load(path) + if err != nil { + t.Fatal(err) + } + if back.Node != made.Node || string(back.Public) != string(made.Public) || + string(back.Private) != string(made.Private) { + t.Error("the identity changed across a save and load") + } + if back.Membership.Broker != made.Membership.Broker || + back.Membership.Fingerprint != made.Membership.Fingerprint || + back.Membership.Password != made.Membership.Password || + string(back.Membership.Signer) != string(made.Membership.Signer) { + t.Error("the membership changed across a save and load; this node could not come back") + } +} + +func TestTheIdentityIsNotReadableByAnybodyElse(t *testing.T) { + // It is the only secret on the machine that identifies it. A mode that let another user on + // this machine read it would make "compromise of a node is compromise of that node" false in + // the other direction — any local user could become the node. + path := Path(filepath.Join(t.TempDir(), "state.json")) + if err := Save(path, joined(t, "workstation")); err != nil { + t.Fatal(err) + } + + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm()&0o077 != 0 { + t.Errorf("the identity is mode %04o; anything but 0600 lets another local user become "+ + "this node", info.Mode().Perm()) + } +} + +func TestSavingLeavesNoHalfWrittenIdentity(t *testing.T) { + // Written and renamed, so power lost mid-write keeps the old identity rather than producing + // half of one. A node cannot regenerate its way out of a broken identity — the mesh believes + // the old public key, and a new one needs a person with a new token. + dir := t.TempDir() + path := Path(filepath.Join(dir, "state.json")) + made := joined(t, "workstation") + for i := 0; i < 3; i++ { + if err := Save(path, made); err != nil { + t.Fatal(err) + } + } + + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + if strings.HasPrefix(e.Name(), ".identity-") { + t.Errorf("a temporary file survived: %s", e.Name()) + } + } +} + +func TestAnIdentityOfTheWrongShapeIsRefused(t *testing.T) { + // The one that would load happily and fail at the moment it signs, which is during enrolment + // against a mesh, far from here. + path := Path(filepath.Join(t.TempDir(), "state.json")) + raw, err := json.Marshal(Identity{Node: "workstation", Public: []byte("short"), Private: []byte("also short")}) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, raw, 0o600); err != nil { + t.Fatal(err) + } + if _, err := Load(path); err == nil { + t.Fatal("an identity with a truncated key loaded") + } +} + +func TestSigningProvesTheNodeIsThatNode(t *testing.T) { + made, err := Generate("workstation") + if err != nil { + t.Fatal(err) + } + challenge := []byte("prove it") + if !ed25519.Verify(ed25519.PublicKey(made.Public), challenge, made.Sign(challenge)) { + t.Fatal("a node's own signature did not verify against the half it publishes") + } +} + +// --- the token, which the control plane writes and this parses --- + +func encodeToken(t *testing.T, body string) string { + t.Helper() + return base64.RawURLEncoding.EncodeToString([]byte(body)) +} + +func completeToken(t *testing.T) string { + t.Helper() + public, _, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + raw, err := json.Marshal(Token{ + Version: 1, Broker: "192.0.2.10:5671", + Fingerprint: "sha256:" + strings.Repeat("ab", 32), + Signer: public, Secret: "one-time", + }) + if err != nil { + t.Fatal(err) + } + return base64.RawURLEncoding.EncodeToString(raw) +} + +func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) { + // The contract with the control plane, which defines this format separately because the host + // requires nothing present and does not import it (novox/hq ADR 0005). There is a matching + // test on the other side. Rename a field on either and both fail, which is the point — the + // alternative is a rename that only breaks at enrolment, on a real machine. + public, _, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + raw, err := json.Marshal(Token{Version: 1, Broker: "b", Fingerprint: "f", Signer: public, Secret: "s"}) + if err != nil { + t.Fatal(err) + } + var fields map[string]any + if err := json.Unmarshal(raw, &fields); err != nil { + t.Fatal(err) + } + for _, want := range []string{"v", "broker", "fingerprint", "signer", "secret"} { + if _, ok := fields[want]; !ok { + t.Errorf("the token has no %q field; the control plane writes that name", want) + } + } + if len(fields) != 5 { + t.Errorf("the token has %d fields, expected 5: %v", len(fields), fields) + } +} + +func TestACompleteTokenParses(t *testing.T) { + got, err := ParseToken(completeToken(t)) + if err != nil { + t.Fatal(err) + } + if got.Broker != "192.0.2.10:5671" || len(got.SignerKey()) != ed25519.PublicKeySize { + t.Errorf("parsed %+v", got) + } +} + +func TestAPastedTokenTolerantOfWhitespace(t *testing.T) { + if _, err := ParseToken(" " + completeToken(t) + "\n"); err != nil { + t.Errorf("a pasted token was refused: %v", err) + } +} + +func TestAnIncompleteTokenIsRefusedWholeAndSaysWhatIsMissing(t *testing.T) { + // Not a reduced capability — an unsafe one. Without the fingerprint this node would connect + // to whatever answers; without the signing key it could not tell a declaration from a + // forgery, and it applies whatever the link delivers. + for _, c := range []struct{ body, expect string }{ + {`{"v":1,"fingerprint":"f","signer":"` + base64Key(t) + `","secret":"s"}`, "broker's address"}, + {`{"v":1,"broker":"b","signer":"` + base64Key(t) + `","secret":"s"}`, "fingerprint"}, + {`{"v":1,"broker":"b","fingerprint":"f","secret":"s"}`, "signing key"}, + {`{"v":1,"broker":"b","fingerprint":"f","signer":"` + base64Key(t) + `"}`, "one-time secret"}, + } { + _, err := ParseToken(encodeToken(t, c.body)) + if err == nil { + t.Errorf("a token missing %s was accepted", c.expect) + continue + } + if !strings.Contains(err.Error(), c.expect) { + t.Errorf("the refusal does not name %s: %v", c.expect, err) + } + } +} + +func TestATokenFromAnotherVersionIsRefused(t *testing.T) { + if _, err := ParseToken(encodeToken(t, `{"v":99,"broker":"b","fingerprint":"f","secret":"s"}`)); err == nil { + t.Fatal("a token from an unknown version was accepted") + } +} + +func TestGarbageIsRefused(t *testing.T) { + for _, bad := range []string{"", "!!!not base64!!!", "aGVsbG8"} { + if _, err := ParseToken(bad); err == nil { + t.Errorf("%q parsed as a token", bad) + } + } +} + +func base64Key(t *testing.T) string { + t.Helper() + public, _, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + return base64.StdEncoding.EncodeToString(public) +} + +func TestAnIdentityThatCannotBeReadIsNotReportedAsAbsent(t *testing.T) { + // The other half of the distinction above, and the one that was untested: a file that exists + // and cannot be read. The corrupt case is caught when it fails to parse; this one never gets + // that far, so it needs its own check — and without it a permissions accident would look + // exactly like a machine that has never joined, and the node would enrol again and discard + // the identity the mesh still believes. + if os.Geteuid() == 0 { + t.Skip("running as root, which can read anything") + } + path := Path(filepath.Join(t.TempDir(), "state.json")) + if err := Save(path, joined(t, "workstation")); err != nil { + t.Fatal(err) + } + if err := os.Chmod(path, 0o000); err != nil { + t.Fatal(err) + } + + _, err := Load(path) + if err == nil { + t.Fatal("an unreadable identity loaded") + } + if errors.Is(err, ErrNoIdentity) { + t.Fatal("an unreadable identity was reported as having none; this node would re-enrol " + + "and throw away the identity the mesh believes") + } + if !strings.Contains(err.Error(), "not the same as") { + t.Errorf("the error does not say why this is different from having none: %v", err) + } +} + +func TestASealingKeyOpensOnlyWhatWasSealedToIt(t *testing.T) { + mine, err := GenerateSealingKey() + if err != nil { + t.Fatal(err) + } + theirs, err := GenerateSealingKey() + if err != nil { + t.Fatal(err) + } + sealed, err := Seal(mine.Public, []byte("hunter2")) + if err != nil { + t.Fatal(err) + } + got, err := mine.Unseal(sealed) + if err != nil { + t.Fatal(err) + } + if string(got) != "hunter2" { + t.Fatalf("got %q", got) + } + if _, err := theirs.Unseal(sealed); err == nil { + t.Fatal("another node opened it") + } +} + +func TestSealingTheSameValueTwiceLooksDifferent(t *testing.T) { + // Sealed boxes are randomised, so an observer cannot tell that two nodes were given the same + // password, nor that a rotation changed nothing. Worth asserting because the alternative is + // a subtle leak nobody would look for. + key, _ := GenerateSealingKey() + first, _ := Seal(key.Public, []byte("same")) + second, _ := Seal(key.Public, []byte("same")) + if first == second { + t.Fatal("sealing is deterministic, so equal secrets are visible as equal blobs") + } +} + +func TestANodeWithNoSealingKeySaysWhatToDo(t *testing.T) { + // Rather than making one. A key the mesh was never told about is a key nothing can be sealed + // to, so a node that quietly created one would look fine and receive nothing for ever. + _, err := LoadSealingKey(t.TempDir() + "/absent.key") + if err == nil { + t.Fatal("a sealing key appeared out of nowhere") + } + if !strings.Contains(err.Error(), "join again") { + t.Fatalf("the failure does not say what to do: %v", err) + } +} + +func TestASealingKeyOnDiskSurvivesATrailingNewline(t *testing.T) { + // It is written with one, the way every other key file here is, and reading it back has to + // cope — otherwise the key works until the first restart. + key, _ := GenerateSealingKey() + path := t.TempDir() + "/sealing.key" + if err := os.WriteFile(path, []byte(key.Private+"\n"), 0o600); err != nil { + t.Fatal(err) + } + back, err := LoadSealingKey(path) + if err != nil { + t.Fatal(err) + } + if back.Public != key.Public { + t.Fatalf("a round trip through the disk changed the key") + } +} + +func TestATokenSaysWhatTheMeshCallsThisMachine(t *testing.T) { + // The node cannot work its own name out. The broker account it authenticates as is named + // after it and exists before this machine has been told anything — so without the name in the + // token, enrolment is a connection refused with an empty username, which names nothing about + // the cause. That is exactly how the first end-to-end raise went. + raw := base64.RawURLEncoding.EncodeToString([]byte( + `{"v":1,"node":"anchor","broker":"192.0.2.10:5671",` + + `"fingerprint":"sha256:` + strings.Repeat("ab", 32) + `",` + + `"signer":"` + base64.StdEncoding.EncodeToString(make([]byte, 32)) + `",` + + `"secret":"a-one-time-secret"}`)) + token, err := ParseToken(raw) + if err != nil { + t.Fatal(err) + } + if token.Node != "anchor" { + t.Fatalf("the name did not survive the token: %q", token.Node) + } +} diff --git a/internal/identity/overlay.go b/internal/identity/overlay.go new file mode 100644 index 0000000..4c3eb39 --- /dev/null +++ b/internal/identity/overlay.go @@ -0,0 +1,54 @@ +package identity + +import ( + "crypto/ecdh" + "crypto/rand" + "encoding/base64" + "fmt" +) + +// The node's key on the private network, which is a different key from the one that says who it +// is — and deliberately so. +// +// novox/hq 08-connectivity: each node generates its own keypair, the private half never leaves +// the machine, and the public half is published to the mesh. That means the control plane +// computes a peer graph it cannot itself impersonate: it knows every public key and holds no +// private one, so it can say who may talk to whom without being able to pretend to be any of them. +// +// Separate from the identity keypair because they are verified by different things at different +// times — the identity signs messages to the mesh, this one encrypts traffic between nodes — and +// a key used for two purposes is one rotation away from breaking the other. + +// OverlayKey is a Curve25519 keypair, which is what WireGuard uses. +type OverlayKey struct { + // Public is what travels. Base64, which is the form WireGuard configuration files use, so it + // is carried the way it will be written rather than converted at the last moment. + Public string `json:"public"` + + // Private never leaves this machine. It is written to a file of its own that the interface + // configuration points at, so the control plane can compose that configuration without ever + // holding this. + Private string `json:"private"` +} + +// GenerateOverlayKey makes this node's keypair for the private network. +func GenerateOverlayKey() (OverlayKey, error) { + private, err := ecdh.X25519().GenerateKey(rand.Reader) + if err != nil { + return OverlayKey{}, fmt.Errorf("cannot generate this node's overlay key: %w", err) + } + return OverlayKey{ + Public: base64.StdEncoding.EncodeToString(private.PublicKey().Bytes()), + Private: base64.StdEncoding.EncodeToString(private.Bytes()), + }, nil +} + +// OverlayKeyPath is where the private half lives: a file of its own, referenced by the interface +// configuration rather than embedded in it. +// +// That separation is what lets the mesh compose the configuration. WireGuard's `PostUp` can set a +// private key from a file, so the declaration the control plane sends names this path and carries +// no secret — and the file it names was written by the node, from a key nothing else ever saw. +func OverlayKeyPath(statePath string) string { + return dirOf(statePath) + "/overlay.key" +} diff --git a/internal/identity/sealing.go b/internal/identity/sealing.go new file mode 100644 index 0000000..218f8bc --- /dev/null +++ b/internal/identity/sealing.go @@ -0,0 +1,143 @@ +package identity + +import ( + "crypto/ecdh" + "crypto/rand" + "encoding/base64" + "fmt" + "os" + "strings" + + "golang.org/x/crypto/nacl/box" +) + +// The key a secret is sealed to, so the mesh can carry one without ever holding a usable copy. +// +// **The fault this exists to avoid is documented, in another mesh, in its own tooling.** There, +// credentials live in the control plane's database, encrypted at rest — which protects against +// somebody reading the database file and nothing else. The same secret is also in each node's +// environment file in plain text, and, worse, inside every connection string composed from it, so +// the tool for finding copies has to search *by value* rather than by name. Its own documentation +// says the copies inside composed URLs "are often the only copies actually in use". Encryption at +// rest also cost the ability to audit: a query against the encrypted column returns zero rows and +// proves nothing. +// +// So the arrangement here is the other one. **The node generates this key and the mesh only ever +// sees the public half**, exactly as with the identity and overlay keys +// (novox/hq ADR 0004). A secret is sealed to that public half before it is stored, so: +// +// - the control plane's database holds nothing usable, and a copy of it grants nothing +// - the broker relays a blob it cannot read, which is the point of not trusting it +// - *compromise of a node is compromise of that node* becomes true of secrets too, rather +// than being true of identity and quietly false of everything that matters +// +// A third key rather than reusing one of the two that exist. The identity key signs and is +// Ed25519; the overlay key is WireGuard's and is tied to being on the private network, which a +// machine may not be. A key used for two purposes is one rotation away from breaking the other. + +// SealingKey is an X25519 keypair used only for receiving secrets. +type SealingKey struct { + // Public is what the mesh records. + Public string `json:"public"` + // Private never leaves this machine. + Private string `json:"private"` +} + +// GenerateSealingKey makes this node's key for receiving secrets. +func GenerateSealingKey() (SealingKey, error) { + private, err := ecdh.X25519().GenerateKey(rand.Reader) + if err != nil { + return SealingKey{}, fmt.Errorf("cannot generate this node's sealing key: %w", err) + } + return SealingKey{ + Public: base64.StdEncoding.EncodeToString(private.PublicKey().Bytes()), + Private: base64.StdEncoding.EncodeToString(private.Bytes()), + }, nil +} + +// SealingKeyPath is where the private half lives. +func SealingKeyPath(statePath string) string { + return dirOf(statePath) + "/sealing.key" +} + +// LoadSealingKey reads this node's sealing key. +// +// It does not make one. A key the mesh has never been told about is a key nothing can be sealed +// to, so creating one here would produce a node that silently cannot receive any secret and looks +// fine — the key is generated at enrolment, where its public half is reported in the same breath. +func LoadSealingKey(path string) (SealingKey, error) { + raw, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return SealingKey{}, fmt.Errorf( + "this node has no sealing key at %s, so nothing can be sealed to it — it is made "+ + "at enrolment, and a node that joined before secrets existed must join again", + path) + } + return SealingKey{}, err + } + { + private, decodeErr := base64.StdEncoding.DecodeString(strings.TrimSpace(string(raw))) + if decodeErr != nil || len(private) != 32 { + return SealingKey{}, fmt.Errorf( + "%s is not a sealing key; move it aside to have a new one made", path) + } + key, keyErr := ecdh.X25519().NewPrivateKey(private) + if keyErr != nil { + return SealingKey{}, fmt.Errorf("%s is not a usable sealing key: %w", path, keyErr) + } + return SealingKey{ + Public: base64.StdEncoding.EncodeToString(key.PublicKey().Bytes()), + Private: base64.StdEncoding.EncodeToString(key.Bytes()), + }, nil + } +} + +// Unseal opens something the mesh sealed to this node. +// +// Anonymous sealed boxes: the sender is not authenticated here, and does not need to be. What a +// node applies is bounded by the declaration's signature, which is checked before any of this — +// so a blob that arrives in a verified declaration came from the mesh, and this only has to +// answer whether it was meant for this machine. +func (s SealingKey) Unseal(sealed string) ([]byte, error) { + blob, err := base64.StdEncoding.DecodeString(sealed) + if err != nil { + return nil, fmt.Errorf("this is not a sealed value: %w", err) + } + private, err := base64.StdEncoding.DecodeString(s.Private) + if err != nil || len(private) != 32 { + return nil, fmt.Errorf("this node's sealing key is unusable") + } + public, err := base64.StdEncoding.DecodeString(s.Public) + if err != nil || len(public) != 32 { + return nil, fmt.Errorf("this node's sealing key is unusable") + } + + var pub, priv [32]byte + copy(pub[:], public) + copy(priv[:], private) + out, ok := box.OpenAnonymous(nil, blob, &pub, &priv) + if !ok { + // Sealed to a different node, or to a key this one no longer has. Said as one thing + // because from here they are indistinguishable, and both mean the same: this machine + // cannot read it and applying it would write a file of rubbish. + return nil, fmt.Errorf("this was not sealed to this node's current key") + } + return out, nil +} + +// Seal closes a value to a node's public sealing key. Here so that a test can produce what the +// mesh produces, rather than asserting against a blob nobody can regenerate. +func Seal(publicKey string, value []byte) (string, error) { + public, err := base64.StdEncoding.DecodeString(publicKey) + if err != nil || len(public) != 32 { + return "", fmt.Errorf("%q is not a sealing key", publicKey) + } + var pub [32]byte + copy(pub[:], public) + sealed, err := box.SealAnonymous(nil, value, &pub, rand.Reader) + if err != nil { + return "", err + } + return base64.StdEncoding.EncodeToString(sealed), nil +} diff --git a/internal/identity/serving.go b/internal/identity/serving.go new file mode 100644 index 0000000..22753f8 --- /dev/null +++ b/internal/identity/serving.go @@ -0,0 +1,132 @@ +package identity + +import ( + "crypto/ed25519" + "crypto/rand" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "fmt" + "os" + "path/filepath" + "strings" +) + +// The key a node serves TLS with, on its name inside the mesh. +// +// A fourth key, and the reasoning is the one this file's neighbours already give twice: **a key +// used for two purposes is one rotation away from breaking the other**. The identity key signs +// messages to the mesh and would do for TLS — Ed25519 works in TLS 1.3 — and reusing it would +// mean rotating a node's identity every time its certificate is replaced, or the reverse. +// +// **The private half never leaves the machine.** The mesh is told the public half at enrolment +// and signs a certificate binding it to this node's internal name, which is the whole of what a +// certificate authority does. There is no request to send and nothing to seal: the mesh issues +// something public, about a key it cannot use. +// +// novox/hq 08-connectivity: the mesh CA certifies internal names, and it is not a bootstrap +// concern — a joining node verifies the control plane against the fingerprint in its token, so +// nothing needs the CA before membership. + +// ServingKey is an Ed25519 keypair a node presents when something connects to it by name. +type ServingKey struct { + // Public is what the mesh records and certifies. + Public string `json:"public"` + // Private never leaves this machine. + Private string `json:"private"` +} + +// GenerateServingKey makes this node's key for serving on its internal name. +func GenerateServingKey() (ServingKey, error) { + public, private, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return ServingKey{}, fmt.Errorf("cannot generate this node's serving key: %w", err) + } + return ServingKey{ + Public: base64.StdEncoding.EncodeToString(public), + Private: base64.StdEncoding.EncodeToString(private), + }, nil +} + +// ServingKeyPath is where the private half lives. +// +// A file of its own, named by whatever configuration needs it — the same arrangement the overlay +// key has, and for the same reason: the mesh can compose a service's configuration without ever +// holding the key that configuration points at. +// +// **PKCS#8 PEM**, because that is the only reason the file exists. A key stored in this host's own +// encoding is a key nothing can serve with: the mesh delivers a PEM certificate beside it, and +// every TLS server there is — a web server's `ssl_certificate_key`, Go's `LoadX509KeyPair`, +// `openssl s_server -key` — reads PEM and nothing else. It was base64 once, and the certificate +// arrived, and the file was there, and nothing could start. +func ServingKeyPath(statePath string) string { + return dirOf(statePath) + "/serving.key" +} + +// CertificatePath is where the certificate the mesh issued lives. +// +// Beside the key, and written by the host from an ordinary declaration — it is public, so it +// travels in the open like any other file. +func CertificatePath(statePath string) string { + return dirOf(statePath) + "/serving.crt" +} + +// LoadServingKey reads this node's serving key. +// +// It does not make one, for the same reason LoadSealingKey does not: a key the mesh has never +// certified is a key nothing will trust, so a node that quietly generated one would serve a +// certificate for a key it no longer has and fail in a way that names neither. +func LoadServingKey(path string) (ServingKey, error) { + raw, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return ServingKey{}, fmt.Errorf( + "this node has no serving key at %s, so nothing can be certified for it — it is "+ + "made at enrolment, and a node that joined before had none", path) + } + return ServingKey{}, err + } + block, _ := pem.Decode(raw) + if block == nil { + // Distinguished from a corrupt key, because the remedy is different and the difference is + // invisible otherwise. A key this host wrote before it stored PEM is intact and unusable: + // nothing serving TLS can read it, and the machine fails at the moment something connects. + if _, err := base64.StdEncoding.DecodeString(strings.TrimSpace(string(raw))); err == nil { + return ServingKey{}, fmt.Errorf( + "%s holds a serving key in this host's old encoding, which nothing serving TLS "+ + "can read. It is replaced by enrolling again, which generates one and tells "+ + "the mesh about it", path) + } + return ServingKey{}, fmt.Errorf("%s is not a serving key", path) + } + parsed, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return ServingKey{}, fmt.Errorf("%s is not a serving key: %w", path, err) + } + key, isEd25519 := parsed.(ed25519.PrivateKey) + if !isEd25519 { + return ServingKey{}, fmt.Errorf( + "%s holds a %T, and a node serves with an Ed25519 key", path, parsed) + } + return ServingKey{ + Public: base64.StdEncoding.EncodeToString(key.Public().(ed25519.PublicKey)), + Private: base64.StdEncoding.EncodeToString(key), + }, nil +} + +// WriteServingKey puts the private half where configuration can point at it, as PKCS#8 PEM. +func WriteServingKey(path string, key ServingKey) error { + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return err + } + raw, err := base64.StdEncoding.DecodeString(key.Private) + if err != nil || len(raw) != ed25519.PrivateKeySize { + return fmt.Errorf("this is not a serving key to write") + } + encoded, err := x509.MarshalPKCS8PrivateKey(ed25519.PrivateKey(raw)) + if err != nil { + return err + } + return os.WriteFile(path, + pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: encoded}), 0o600) +} diff --git a/internal/identity/serving_test.go b/internal/identity/serving_test.go new file mode 100644 index 0000000..15b5bf0 --- /dev/null +++ b/internal/identity/serving_test.go @@ -0,0 +1,71 @@ +package identity + +import ( + "crypto/ed25519" + "crypto/x509" + "encoding/pem" + "os" + "path/filepath" + "strings" + "testing" +) + +// The whole reason the file exists is that something else reads it. +// +// A key in this host's own encoding is intact, unusable, and indistinguishable from a working one +// until the moment a client connects — the mesh delivers the certificate, the file is there with +// the right permissions, and the server will not start. +func TestTheServingKeyIsWrittenInTheFormatAServerReads(t *testing.T) { + made, err := GenerateServingKey() + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "serving.key") + if err := WriteServingKey(path, made); err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + block, _ := pem.Decode(raw) + if block == nil { + t.Fatalf("the serving key is not PEM, so nothing serving TLS can read it:\n%s", raw) + } + parsed, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + t.Fatalf("the serving key is PEM and not a key: %v", err) + } + // And it is the key that was written, not merely a key — a file that round-trips through the + // wrong half would certify a public key the machine cannot prove it holds. + if _, isEd25519 := parsed.(ed25519.PrivateKey); !isEd25519 { + t.Fatalf("the serving key is a %T", parsed) + } + read, err := LoadServingKey(path) + if err != nil { + t.Fatal(err) + } + if read.Public != made.Public { + t.Fatal("the key read back is not the key written, so the mesh would certify the wrong one") + } +} + +// The old encoding is refused by name, because the remedy is different from a corrupt file and +// the difference is invisible from the outside. +func TestAServingKeyInTheOldEncodingIsNamedRatherThanCalledCorrupt(t *testing.T) { + made, err := GenerateServingKey() + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "serving.key") + if err := os.WriteFile(path, []byte(made.Private+"\n"), 0o600); err != nil { + t.Fatal(err) + } + _, err = LoadServingKey(path) + if err == nil { + t.Fatal("a key nothing can serve with was accepted") + } + if !strings.Contains(err.Error(), "enrolling again") { + t.Fatalf("refused without naming the remedy: %v", err) + } +} diff --git a/internal/identity/token.go b/internal/identity/token.go new file mode 100644 index 0000000..40fab81 --- /dev/null +++ b/internal/identity/token.go @@ -0,0 +1,81 @@ +package identity + +import ( + "crypto/ed25519" + "encoding/base64" + "encoding/json" + "fmt" + "strings" +) + +// Token is what a person carries to a machine that is joining. +// +// novox/hq ADR 0004 — four things: where the broker is, what certificate to expect there, whose +// signature to believe afterwards, and a one-time right to join. +// +// THIS IS A WIRE FORMAT SHARED WITH THE CONTROL PLANE, which writes it. The two definitions are +// separate on purpose — the host requires nothing present and does not import the control plane — +// so they are held together by a test on each side asserting the exact field names rather than by +// a shared type. If a field is renamed here and not there, that test fails on both sides. +type Token struct { + Version int `json:"v"` + + // Node is what the mesh calls this machine, and it arrives here because the node cannot work + // it out. The broker account it must authenticate as is named after it, so it has to be known + // before the mesh can say anything — and without it enrolment is a connection refused with an + // empty username, which names nothing. + Node string `json:"node,omitempty"` + + Broker string `json:"broker,omitempty"` + Fingerprint string `json:"fingerprint,omitempty"` + Signer []byte `json:"signer,omitempty"` + Secret string `json:"secret"` +} + +// ParseToken reads a token a person pasted. +// +// Every refusal here says *this is not a token* rather than *this is the wrong token*. The +// difference matters once there is a mesh: a host must tell "this is not from the mesh I joined" +// apart from "this is malformed" (novox/hq ADR 0004), and the first is a signature check later, +// not a parse failure here. +func ParseToken(encoded string) (Token, error) { + raw, err := base64.RawURLEncoding.DecodeString(strings.TrimSpace(encoded)) + if err != nil { + return Token{}, fmt.Errorf("this is not a token: %w", err) + } + var t Token + if err := json.Unmarshal(raw, &t); err != nil { + return Token{}, fmt.Errorf("this is not a token: %w", err) + } + if t.Version != 1 { + return Token{}, fmt.Errorf( + "this token says it is version %d, and this host understands version 1", t.Version) + } + + var missing []string + if strings.TrimSpace(t.Broker) == "" { + missing = append(missing, "the broker's address") + } + if strings.TrimSpace(t.Fingerprint) == "" { + missing = append(missing, "the broker certificate's fingerprint") + } + if len(t.Signer) != ed25519.PublicKeySize { + missing = append(missing, "the control plane's signing key") + } + if strings.TrimSpace(t.Secret) == "" { + missing = append(missing, "the one-time secret") + } + if len(missing) > 0 { + // Refused whole rather than used partially. A token missing the fingerprint would have + // this node connect to whatever answers at that address, and one missing the signing key + // would leave it unable to tell a declaration from a forgery — so an incomplete token is + // not a reduced capability, it is an unsafe one. + return Token{}, fmt.Errorf( + "this token is missing %s, so it cannot be used to join anything", + strings.Join(missing, ", ")) + } + return t, nil +} + +// SignerKey is the control plane's public signing key, as a key. +func (t Token) SignerKey() ed25519.PublicKey { return ed25519.PublicKey(t.Signer) } diff --git a/internal/inventory/inventory.go b/internal/inventory/inventory.go index 6437eb1..c6a10d6 100644 --- a/internal/inventory/inventory.go +++ b/internal/inventory/inventory.go @@ -3,7 +3,7 @@ // Reported upward and never asked downward (novox/hq 03-DESIGN/01-to-be/05-the-node-host.md). // Everything here is read from the machine at the moment of asking — nothing is remembered, // nothing is derived from a file that says what the machine ought to be -// (novox/hq ADR 0035). +// (novox/hq ADR 0018). package inventory import ( @@ -39,7 +39,7 @@ type Inventory struct { // ObservedAt is when this was read. An inventory with no timestamp cannot be told from a // stale one, and a node that has been unreachable for a week is an ordinary situation - // (novox/hq ADR 0036) rather than an error — so the age of the observation is part of it. + // (novox/hq ADR 0004) rather than an error — so the age of the observation is part of it. ObservedAt time.Time `json:"observed_at"` // Unreadable lists what could not be determined, and why. An absent field and a field that diff --git a/internal/inventory/inventory_test.go b/internal/inventory/inventory_test.go index b0dfdb0..a3b0dce 100644 --- a/internal/inventory/inventory_test.go +++ b/internal/inventory/inventory_test.go @@ -64,7 +64,7 @@ func TestMemoryIsReadOrReportedMissing(t *testing.T) { } func TestTheInventorySaysWhenItWasTaken(t *testing.T) { - // A node unreachable for a week is an ordinary situation (novox/hq ADR 0036), so an + // A node unreachable for a week is an ordinary situation (novox/hq ADR 0004), so an // inventory that cannot be told from a stale one is missing the fact that matters. before := time.Now().UTC() inv := Collect(context.Background(), nil, nil, time.Second) @@ -79,7 +79,7 @@ func TestTheInventorySaysWhenItWasTaken(t *testing.T) { func TestTheHostDoesNotNameTheNode(t *testing.T) { // A node's name is assigned by the mesh. A host that named itself would be deciding - // something, which is precisely what novox/hq ADR 0037 forbids it to do. + // something, which is precisely what novox/hq ADR 0005 forbids it to do. inv := Collect(context.Background(), nil, nil, time.Second) hostname, _ := os.Hostname() @@ -91,7 +91,7 @@ func TestTheHostDoesNotNameTheNode(t *testing.T) { // --- against this machine --------------------------------------------------------------- func TestAgainstThisMachine_inventoryIsTrue(t *testing.T) { - // novox/hq ADR 0034: behaviour against a real system is tested alongside, not mocked. + // novox/hq ADR 0017: behaviour against a real system is tested alongside, not mocked. inv := Collect(context.Background(), nil, profile.Default(nil), 10*time.Second) if inv.Machine == "" { diff --git a/internal/link/enrol.go b/internal/link/enrol.go new file mode 100644 index 0000000..5999395 --- /dev/null +++ b/internal/link/enrol.go @@ -0,0 +1,179 @@ +package link + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + "time" + + amqp "github.com/rabbitmq/amqp091-go" +) + +// The wire format shared with the control plane, which defines it separately because this binary +// requires nothing present and does not import it. A test on each side asserts the field names. +const ( + Exchange = "mesh" + KeyEnrol = "enrol" +) + +// QueueFor is the queue this node consumes from — the only one its account may read. +func QueueFor(node string) string { return "node." + node } + +// EnrolRequest is what this node says when joining. +type EnrolRequest struct { + Node string `json:"node"` + Secret string `json:"secret"` + PublicKey []byte `json:"public_key"` + + // OverlayKey is the public half of this node's key on the private network — a different key + // from PublicKey above, generated at the same moment and for a different purpose. + // + // Sent with enrolment because the overlay is the first declaration a node receives, and the + // mesh cannot compose it without this. Asking for it afterwards would mean a node is enrolled + // and unreachable for a round trip, which is the state everything else here works to avoid. + OverlayKey string `json:"overlay_key,omitempty"` + + // SealingKey is the public half of the key secrets are sealed to. A third key, and the + // reasoning is the same one twice over: the mesh must be able to send this node something + // nothing else can read, and it must never be able to read it either. + SealingKey string `json:"sealing_key,omitempty"` + + // ServingKey is the public half of the key this node serves TLS with on its internal name. + // The mesh signs a certificate binding it; the private half never leaves the machine, so + // there is nothing to seal and nothing that could be stolen from the mesh's copy. + ServingKey string `json:"serving_key,omitempty"` + + Profile map[string]any `json:"profile,omitempty"` +} + +// EnrolReply is what the mesh says back. +type EnrolReply struct { + Accepted bool `json:"accepted"` + Node string `json:"node,omitempty"` + Queue string `json:"queue,omitempty"` + + // What this node keeps so it can come back on its own. Without these a restart would need a + // person with a new token, which would make disconnection a crisis rather than the ordinary + // situation novox/hq ADR 0004 says it is. + Password string `json:"password,omitempty"` + Broker string `json:"broker,omitempty"` + Fingerprint string `json:"fingerprint,omitempty"` + Signer []byte `json:"signer,omitempty"` + + Refusal string `json:"refusal,omitempty"` +} + +// ErrRefused is what a node gets when the mesh will not have it. +var ErrRefused = errors.New("the mesh refused this enrolment") + +// Enrol presents this node's key and its one-time secret, and waits to be told it is known. +// +// The broker has already authenticated this connection: the account was created when the token +// was issued and the secret is its password. So this is not how the node gets in — it is what it +// says once it is in, and the secret travels again because the control plane must not have to ask +// the broker who connected. +func Enrol(ctx context.Context, address, pin, node, secret string, public []byte, + overlayKey, sealingKey, servingKey string, profile map[string]any, + timeout time.Duration) (EnrolReply, error) { + + config, err := PinnedConfig(pin) + if err != nil { + return EnrolReply{}, err + } + + // The account name is the node's, and the password is the token's secret. Escaped because a + // name or secret containing a colon or an at-sign would otherwise change which host this + // connects to — a credential silently redirecting a connection is the worst shape this could + // take. + dsn := fmt.Sprintf("amqps://%s:%s@%s/", + url.QueryEscape(node), url.QueryEscape(secret), address) + + conn, err := amqp.DialConfig(dsn, amqp.Config{ + TLSClientConfig: config, + Dial: amqp.DefaultDial(timeout), + }) + if err != nil { + if errors.Is(err, ErrWrongCertificate) { + return EnrolReply{}, err + } + // Not quoted back: the DSN carries the one-time secret. + return EnrolReply{}, fmt.Errorf("cannot reach the broker at %s as %s: %w", address, node, err) + } + defer conn.Close() + + channel, err := conn.Channel() + if err != nil { + return EnrolReply{}, err + } + defer channel.Close() + + // This node's own queue, which its account is scoped to and nothing else may read. + queue, err := channel.QueueDeclare(QueueFor(node), true, false, false, false, nil) + if err != nil { + return EnrolReply{}, fmt.Errorf( + "cannot declare this node's queue %s: %w", QueueFor(node), err) + } + + replies, err := channel.Consume(queue.Name, "", true, false, false, false, nil) + if err != nil { + return EnrolReply{}, err + } + + request := EnrolRequest{Node: node, Secret: secret, PublicKey: public, + OverlayKey: overlayKey, SealingKey: sealingKey, ServingKey: servingKey, Profile: profile} + body, err := json.Marshal(request) + if err != nil { + return EnrolReply{}, err + } + + correlation := fmt.Sprintf("%s-%d", node, time.Now().UnixNano()) + publish, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + if err := channel.PublishWithContext(publish, Exchange, KeyEnrol, false, false, + amqp.Publishing{ + ContentType: "application/json", + CorrelationId: correlation, + ReplyTo: queue.Name, + Body: body, + }); err != nil { + return EnrolReply{}, fmt.Errorf("cannot publish to the %s exchange: %w", Exchange, err) + } + + // Waited for rather than assumed. A published message that nothing answers means the control + // plane is not running, and a node that carried on regardless would believe it had joined a + // mesh that has never heard of it. + deadline := time.NewTimer(timeout) + defer deadline.Stop() + closed := conn.NotifyClose(make(chan *amqp.Error, 1)) + + for { + select { + case <-ctx.Done(): + return EnrolReply{}, ctx.Err() + case reason := <-closed: + return EnrolReply{}, fmt.Errorf("the broker closed the connection: %v", reason) + case <-deadline.C: + return EnrolReply{}, fmt.Errorf( + "the broker accepted this node's connection and nothing answered within %s. The "+ + "mesh's broker is running and its control plane is not", timeout) + case delivery, ok := <-replies: + if !ok { + return EnrolReply{}, errors.New("the broker stopped delivering") + } + // Anything else on this queue is not the answer to this question. + if delivery.CorrelationId != correlation { + continue + } + var reply EnrolReply + if err := json.Unmarshal(delivery.Body, &reply); err != nil { + return EnrolReply{}, fmt.Errorf("the mesh's answer could not be read: %w", err) + } + if !reply.Accepted { + return reply, fmt.Errorf("%w: %s", ErrRefused, reply.Refusal) + } + return reply, nil + } + } +} diff --git a/internal/link/enrol_shape_test.go b/internal/link/enrol_shape_test.go new file mode 100644 index 0000000..7b53f95 --- /dev/null +++ b/internal/link/enrol_shape_test.go @@ -0,0 +1,67 @@ +package link + +import ( + "encoding/json" + "os" + "testing" + + "github.com/novox/mesh-host/internal/identity" +) + +// What this node says when it joins, written out so the mesh's own suite can accept it. +// +// The two ends are separate structs in separate repositories. Every field here is one somebody +// could rename on one side, and the failure would be silent: enrolment succeeds, a key is simply +// absent, and the node looks joined until the first thing sealed to it cannot be opened. That is +// exactly the shape of fault this project keeps finding late. +// +// Skipped unless MESH_ENROL_OUT names a file, so this is a check somebody runs deliberately +// rather than a dependency between two repositories. +func TestWhatThisNodeSaysWhenItJoins(t *testing.T) { + path := os.Getenv("MESH_ENROL_OUT") + if path == "" { + t.Skip("set MESH_ENROL_OUT to write the enrolment request the mesh's suite reads") + } + + // A real one. Generated the way enrolment generates them rather than typed as literals, so a + // key that stopped being a key would be caught here rather than travelling. + mine, err := identity.Generate("workstation") + if err != nil { + t.Fatal(err) + } + overlay, err := identity.GenerateOverlayKey() + if err != nil { + t.Fatal(err) + } + sealing, err := identity.GenerateSealingKey() + if err != nil { + t.Fatal(err) + } + serving, err := identity.GenerateServingKey() + if err != nil { + t.Fatal(err) + } + + request := EnrolRequest{ + Node: "workstation", + Secret: "a-one-time-secret", + PublicKey: mine.Public, + OverlayKey: overlay.Public, + SealingKey: sealing.Public, + ServingKey: serving.Public, + Profile: map[string]any{"seat": true}, + } + body, err := json.MarshalIndent(request, "", " ") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, append(body, '\n'), 0o644); err != nil { + t.Fatal(err) + } + // The private half of the sealing key goes beside it, so the mesh's suite can prove what it + // sealed is openable rather than merely present. + if err := os.WriteFile(path+".sealing-private", []byte(sealing.Private), 0o600); err != nil { + t.Fatal(err) + } + t.Logf("wrote %s", path) +} diff --git a/internal/link/messages.go b/internal/link/messages.go new file mode 100644 index 0000000..becc2ef --- /dev/null +++ b/internal/link/messages.go @@ -0,0 +1,80 @@ +package link + +// The wire formats shared with the control plane, which defines them separately because this +// binary requires nothing present and does not import it. A test on each side asserts the field +// names, so a rename breaks both at once rather than on a real machine months later. + +// Routing keys a node may publish. Its broker account is scoped to this exchange and its own +// queue, so it can say these things and nothing else. +const ( + KeyReport = "report" + KeyAlive = "alive" +) + +// Alive is a node saying nothing except that it is there. +// +// novox/hq 09-the-node-lifecycle: *how long it has been disconnected is a fact the mesh must +// hold, and nothing holds it today. Without it, a node running last month's assignments looks +// exactly like one that is current.* +// +// Separate from a report because the two happen at completely different rates: a node is alive +// constantly and applies something rarely, and reading one as the other would make a quiet node +// look like a stale one. +type Alive struct { + Node string `json:"node"` +} + +// Signed is a declaration and the signature over it. +// +// novox/hq ADR 0004: the transport is verified once at connect, and **each declaration is +// verified by its signature, every time**. The two are different questions — a node connects to +// the broker and takes instruction from the control plane behind it, and pinning only the first +// would make the second transitive. +// +// The signature is over Declaration exactly as it arrived, bytes unchanged. Re-encoding before +// verifying would mean checking a signature over something other than what was sent, and any +// difference in key order or spacing would break it — so the raw message is what is signed and +// what is checked. +type Signed struct { + Declaration []byte `json:"declaration"` + Signature []byte `json:"signature"` +} + +// Report is what a node says after applying, and it is a statement rather than a write. +// +// A node states; the context that owns the data writes (novox/hq ADR 0006). The difference is the +// security boundary: something that can write cannot be prevented from writing anything, and +// something that can only state has its blast radius bounded by what this struct can say. +type Report struct { + Node string `json:"node"` + + // Applied is what this machine now owns, by resource id. + Applied []string `json:"applied,omitempty"` + + // Failed says what could not be applied, and why, in words for a person. + Failed map[string]string `json:"failed,omitempty"` + + // Refused is set when the declaration was rejected whole rather than applied in part. + Refused string `json:"refused,omitempty"` + + // Carried are the machine's ports held by what this host raised from its own bundle. + // + // **So the mesh can assign around what it did not put here** (novox/hq ADR 0038). A node + // raises its substrate before any mesh exists, so the control plane has never heard of the + // store or the broker — and would hand a module a port one of them holds, discovering it only + // when a container runtime refused to start. + // + // A node *states* and the mesh writes, which is the whole shape of this message: this is the + // machine saying what is true of it, not asking for anything. + Carried []int `json:"carried,omitempty"` + + // Declared is the digest of the declaration this report is about — sha256 of the exact bytes + // the mesh sent, which the mesh recorded when it sent them. + // + // **Which declaration, not when.** The mesh compared its send time to this report's arrival + // to decide whether a machine had caught up, and lost the race it invited: an apply started + // under the previous declaration finishes after the next one is sent, its report lands newer + // than the send, and the machine reads as caught up with words it has not read yet. Clocks + // cannot answer "which"; the digest is the answer itself. + Declared string `json:"declared,omitempty"` +} diff --git a/internal/link/messages_test.go b/internal/link/messages_test.go new file mode 100644 index 0000000..7983221 --- /dev/null +++ b/internal/link/messages_test.go @@ -0,0 +1,141 @@ +package link + +import ( + "context" + "crypto/ed25519" + "encoding/json" + "testing" +) + +// verified runs what Run does to a delivery body, without a broker: unmarshal, check the +// signature, and only then apply. Isolating it keeps this test about the check rather than about +// AMQP, which is tested against a real broker in the lab. +func verified(t *testing.T, signer ed25519.PublicKey, body []byte) (Report, bool) { + t.Helper() + applied := false + report := handleBody(context.Background(), Membership{Node: "anchor", Signer: signer}, body, + func(context.Context, []byte, []byte) Report { + applied = true + return Report{Applied: []string{"something"}} + }) + return report, applied +} + +func signedBody(t *testing.T, private ed25519.PrivateKey, declaration string) []byte { + t.Helper() + raw, err := json.Marshal(Signed{ + Declaration: []byte(declaration), + Signature: ed25519.Sign(private, []byte(declaration)), + }) + if err != nil { + t.Fatal(err) + } + return raw +} + +func TestTheMeshsOwnDeclarationIsApplied(t *testing.T) { + public, private, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + report, applied := verified(t, public, signedBody(t, private, `{"declaration":1}`)) + if !applied { + t.Fatalf("a declaration the mesh signed was not applied: %s", report.Refused) + } +} + +// Defends novox/hq ADR 0002: everything reaching a node arrives over the broker — and therefore +// ADR 0004's consequence, that the broker is not trusted to say who is speaking. +// +// A transport nobody authenticates per-message would let whatever holds the connection attribute +// a declaration to any node it liked. +func TestAForgedDeclarationIsNeverApplied(t *testing.T) { + // The check that stands between "the mesh changes this machine" and "anybody does". The host + // applies whatever the link delivers, so a forged declaration is the whole machine. + public, _, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + _, other, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + + report, applied := verified(t, public, signedBody(t, other, `{"declaration":1}`)) + if applied { + t.Fatal("a declaration signed by another key was applied") + } + if report.Refused != ErrForged.Error() { + t.Errorf("refused, but not as a forgery: %q", report.Refused) + } +} + +func TestATamperedDeclarationIsNeverApplied(t *testing.T) { + // A broker that changed the declaration in flight, keeping the signature. This is what makes + // pinning the transport insufficient on its own. + public, private, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + raw, err := json.Marshal(Signed{ + Declaration: []byte(`{"declaration":1,"resources":["something else entirely"]}`), + Signature: ed25519.Sign(private, []byte(`{"declaration":1}`)), + }) + if err != nil { + t.Fatal(err) + } + + report, applied := verified(t, public, raw) + if applied { + t.Fatal("a declaration altered after signing was applied") + } + if report.Refused != ErrForged.Error() { + t.Errorf("refused, but not as a forgery: %q", report.Refused) + } +} + +func TestAMalformedMessageIsToldApartFromAForgery(t *testing.T) { + // novox/hq ADR 0004 requires these to be distinguishable: one means somebody is trying, the + // other means something is broken, and they need different responses from a person. + public, _, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + report, applied := verified(t, public, []byte("this is not a message")) + if applied { + t.Fatal("something unparseable was applied") + } + if report.Refused == ErrForged.Error() { + t.Error("a malformed message was reported as a forgery; those must be distinguishable") + } +} + +func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) { + // The contract with the control plane, which defines these separately. A matching test lives + // there; rename a field on either side and both fail. + for _, c := range []struct { + value any + expect []string + }{ + {Signed{Declaration: []byte("{}"), Signature: []byte("x")}, []string{"declaration", "signature"}}, + {Report{Node: "n", Applied: []string{"a"}, Failed: map[string]string{"k": "v"}, Refused: "r"}, + []string{"node", "applied", "failed", "refused"}}, + } { + raw, err := json.Marshal(c.value) + if err != nil { + t.Fatal(err) + } + var fields map[string]any + if err := json.Unmarshal(raw, &fields); err != nil { + t.Fatal(err) + } + for _, want := range c.expect { + if _, ok := fields[want]; !ok { + t.Errorf("%T has no %q field; the control plane uses that name", c.value, want) + } + } + if len(fields) != len(c.expect) { + t.Errorf("%T has %d fields, expected %d: %v", c.value, len(fields), len(c.expect), fields) + } + } +} diff --git a/internal/link/pinned.go b/internal/link/pinned.go new file mode 100644 index 0000000..4e12f10 --- /dev/null +++ b/internal/link/pinned.go @@ -0,0 +1,92 @@ +// Package link is how a node reaches the mesh: one outbound connection to the broker, and +// nothing listening on this machine. +// +// novox/hq ADR 0004: the node checks the broker's certificate against the fingerprint in its +// token *before sending anything*. That is trust on first use with the first use moved out of +// band — the token travelled by a person, so its authenticity comes from the channel it took +// rather than from anything this machine can check afterwards. +package link + +import ( + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "errors" + "fmt" + "net" + "strings" + "time" +) + +// ErrWrongCertificate is what a node gets when the broker is not the one its token described. +// +// Its own error because it means something specific and alarming: either the mesh's broker was +// replaced, or this node is being pointed at something else. It is not a connection problem and +// must not be retried as one. +var ErrWrongCertificate = errors.New("the broker presented a certificate this token does not pin") + +// Fingerprint is what a pin looks like: sha256 over the certificate as it arrives on the wire. +func Fingerprint(der []byte) string { + sum := sha256.Sum256(der) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +// PinnedConfig is a TLS configuration that trusts exactly one certificate. +// +// InsecureSkipVerify is true and that is not a weakening — it is the point. The mesh's broker at +// bootstrap has a self-signed certificate and is reached at an address rather than a name, so +// there is no authority to check it against and no name to match. Chain and hostname verification +// are replaced with something stricter: this exact certificate, or nothing. +// +// The check runs in VerifyPeerCertificate, which TLS calls before the handshake completes — so a +// wrong broker is refused before this node sends anything, which is what ADR 0004 requires. +func PinnedConfig(pin string) (*tls.Config, error) { + pin = strings.TrimSpace(pin) + if !strings.HasPrefix(pin, "sha256:") || len(pin) != len("sha256:")+64 { + return nil, fmt.Errorf( + "%q is not a certificate fingerprint: it is sha256: followed by 64 hex characters", pin) + } + if _, err := hex.DecodeString(pin[len("sha256:"):]); err != nil { + return nil, fmt.Errorf("%q is not a certificate fingerprint: %w", pin, err) + } + + return &tls.Config{ + InsecureSkipVerify: true, //nolint:gosec // replaced by the pin below, which is stricter + MinVersion: tls.VersionTLS12, + VerifyPeerCertificate: func(raw [][]byte, _ [][]*x509.Certificate) error { + if len(raw) == 0 { + return fmt.Errorf("%w: it presented none", ErrWrongCertificate) + } + // The leaf, which is what the pin is of. A chain is irrelevant here: nothing is + // being traced to an authority, so an intermediate matching would prove nothing. + got := Fingerprint(raw[0]) + if got != pin { + return fmt.Errorf( + "%w\n expected %s\n got %s\nEither this mesh's broker was replaced, "+ + "or this node is being pointed at something else. This is not a "+ + "connection problem and retrying will not help", + ErrWrongCertificate, pin, got) + } + return nil + }, + }, nil +} + +// Dial opens a TLS connection to the broker, refusing anything but the pinned certificate. +func Dial(address, pin string, timeout time.Duration) (*tls.Conn, error) { + config, err := PinnedConfig(pin) + if err != nil { + return nil, err + } + + dialer := &net.Dialer{Timeout: timeout} + conn, err := tls.DialWithDialer(dialer, "tcp", address, config) + if err != nil { + if errors.Is(err, ErrWrongCertificate) { + return nil, err + } + return nil, fmt.Errorf("cannot reach the broker at %s: %w", address, err) + } + return conn, nil +} diff --git a/internal/link/pinned_test.go b/internal/link/pinned_test.go new file mode 100644 index 0000000..792b8ec --- /dev/null +++ b/internal/link/pinned_test.go @@ -0,0 +1,170 @@ +package link + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "errors" + "math/big" + "net" + "strings" + "testing" + "time" +) + +// A real TLS server with a real self-signed certificate. Not a fake: what is being tested is that +// Go's TLS stack calls this verification before the handshake completes and that a wrong +// certificate is refused there — a fake would assert that the fake refuses it +// (novox/hq ADR 0017). +func server(t *testing.T) (address string, fingerprint string) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "mesh-broker"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + } + der, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + + listener, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{ + Certificates: []tls.Certificate{{Certificate: [][]byte{der}, PrivateKey: key}}, + MinVersion: tls.VersionTLS12, + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { listener.Close() }) + + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + go func() { + // Complete the handshake, then close. Enough for a client to have checked. + _ = conn.(*tls.Conn).Handshake() + conn.Close() + }() + } + }() + return listener.Addr().String(), Fingerprint(der) +} + +func TestTheRightBrokerIsAccepted(t *testing.T) { + address, pin := server(t) + conn, err := Dial(address, pin, 5*time.Second) + if err != nil { + t.Fatalf("the broker its token describes was refused: %v", err) + } + conn.Close() +} + +func TestADifferentBrokerIsRefused(t *testing.T) { + // The case the pin exists for: something else answering at that address. Since the host + // applies whatever the link delivers, connecting to the wrong mesh is the whole machine. + address, _ := server(t) + _, other := server(t) + + _, err := Dial(address, other, 5*time.Second) + if err == nil { + t.Fatal("a broker presenting a different certificate was accepted") + } + if !errors.Is(err, ErrWrongCertificate) { + t.Fatalf("refused, but not as a wrong certificate: %v", err) + } + if !strings.Contains(err.Error(), "retrying will not help") { + t.Error("the error reads like a connection problem; this one must not be retried") + } +} + +func TestNothingIsSentToTheWrongBroker(t *testing.T) { + // ADR 0004 requires the check to happen *before* anything is sent. Asserted by counting what + // the wrong server received: a handshake, and no application bytes. + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + template := x509.Certificate{ + SerialNumber: big.NewInt(2), + Subject: pkix.Name{CommonName: "impostor"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + } + der, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + listener, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{ + Certificates: []tls.Certificate{{Certificate: [][]byte{der}, PrivateKey: key}}, + MinVersion: tls.VersionTLS12, + }) + if err != nil { + t.Fatal(err) + } + defer listener.Close() + + received := make(chan int, 1) + go func() { + conn, err := listener.Accept() + if err != nil { + received <- -1 + return + } + defer conn.Close() + _ = conn.SetReadDeadline(time.Now().Add(2 * time.Second)) + buf := make([]byte, 512) + n, _ := conn.Read(buf) + received <- n + }() + + // A pin for a certificate this server does not have. + _, elsewhere := server(t) + if _, err := Dial(listener.Addr().String(), elsewhere, 5*time.Second); err == nil { + t.Fatal("the impostor was accepted") + } + if n := <-received; n > 0 { + t.Errorf("%d application byte(s) reached a broker that failed the pin", n) + } +} + +func TestAMalformedPinIsRefusedBeforeConnecting(t *testing.T) { + // Caught here rather than at the handshake, so a mistyped token fails while a person is + // looking at it. + for _, bad := range []string{"", "sha256:short", strings.Repeat("a", 64), + "sha256:" + strings.Repeat("z", 64), "md5:" + strings.Repeat("a", 64)} { + if _, err := PinnedConfig(bad); err == nil { + t.Errorf("%q was accepted as a fingerprint", bad) + } + } +} + +func TestAnUnreachableBrokerIsAnOrdinaryFailure(t *testing.T) { + // Must not read as a wrong certificate: one is a network problem worth retrying, the other + // means the mesh was substituted. + _, pin := server(t) + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + address := listener.Addr().String() + listener.Close() + + _, err = Dial(address, pin, 2*time.Second) + if err == nil { + t.Fatal("dialling a closed port succeeded") + } + if errors.Is(err, ErrWrongCertificate) { + t.Error("an unreachable broker was reported as presenting the wrong certificate") + } +} diff --git a/internal/link/roused_test.go b/internal/link/roused_test.go new file mode 100644 index 0000000..1071566 --- /dev/null +++ b/internal/link/roused_test.go @@ -0,0 +1,103 @@ +package link + +import ( + "context" + "errors" + "strings" + "sync" + "testing" + "time" +) + +// A machine that just woke does not wait to be told its link is dead. +// +// After a resume the socket looks perfectly healthy from inside the process — no error, no close, +// because nothing has tried to send anything. Heartbeats discover it twenty or thirty seconds +// later, and for that time the node believes it is in a mesh it has left, which is the one state +// this design says must never be indistinguishable from being connected. +func TestBeingRousedEndsTheCurrentAttemptRatherThanWaitingForATimeout(t *testing.T) { + // A link that never returns on its own, which is exactly what a suspended connection is. + held := make(chan context.Context, 4) + running := func(ctx context.Context) error { + held <- ctx + <-ctx.Done() + return errors.New("the link ended") + } + + rouse := make(chan struct{}, 1) + said := &saidSoFar{} + ctx, stop := context.WithCancel(context.Background()) + defer stop() + + finished := make(chan error, 1) + go func() { finished <- holdWith(ctx, running, said.say, rouse) }() + + first := <-held + select { + case <-first.Done(): + t.Fatal("the link ended before anything roused it") + case <-time.After(50 * time.Millisecond): + } + + rouse <- struct{}{} + select { + case <-first.Done(): + case <-time.After(2 * time.Second): + t.Fatal("the machine woke and the link was left running against a socket that is gone") + } + + // And it opens another one rather than stopping. + select { + case <-held: + case <-time.After(5 * time.Second): + t.Fatal("the link was dropped and never opened again") + } + if !strings.Contains(said.all(), "woken or moved") { + t.Fatalf("nothing was said about why the link was dropped:\n%s", said.all()) + } + + stop() + select { + case <-finished: + case <-time.After(2 * time.Second): + t.Fatal("it did not stop when asked") + } +} + +// A machine nothing ever rouses is every machine that does not suspend, and must behave as before. +func TestAMachineNothingRousesIsUnaffected(t *testing.T) { + attempts := make(chan struct{}, 4) + running := func(context.Context) error { + attempts <- struct{}{} + return errors.New("the link ended") + } + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go func() { _ = holdWith(ctx, running, func(string) {}, nil) }() + + // It keeps trying, which is the behaviour a node with no rouse has always had. + for i := 0; i < 2; i++ { + select { + case <-attempts: + case <-time.After(10 * time.Second): + t.Fatal("a node with nothing to rouse it stopped reconnecting") + } + } +} + +type saidSoFar struct { + mu sync.Mutex + said []string +} + +func (s *saidSoFar) say(line string) { + s.mu.Lock() + defer s.mu.Unlock() + s.said = append(s.said, line) +} + +func (s *saidSoFar) all() string { + s.mu.Lock() + defer s.mu.Unlock() + return strings.Join(s.said, "\n") +} diff --git a/internal/link/run.go b/internal/link/run.go new file mode 100644 index 0000000..dc62e5c --- /dev/null +++ b/internal/link/run.go @@ -0,0 +1,336 @@ +package link + +import ( + "context" + "crypto/ed25519" + "encoding/json" + "errors" + "fmt" + "net/url" + "time" + + amqp "github.com/rabbitmq/amqp091-go" +) + +// ErrForged is what a node returns for a declaration whose signature is not the mesh's. +// +// Its own error, and it must never be confused with a malformed message. novox/hq ADR 0004 +// requires a host to tell *this is not from the mesh I joined* apart from *this is malformed*: +// the first means somebody is trying, the second means something is broken. +var ErrForged = errors.New("this declaration was not signed by the mesh this node joined") + +// AliveEvery is how often a node says it is there. +// +// Often enough that "no word for five minutes" means something, rarely enough that a hundred +// nodes are not a hundred messages a second. The mesh reads absence rather than presence, so what +// matters is the interval being known and steady. +const AliveEvery = 60 * time.Second + +// Membership is what a node needs to reach its mesh again, held by the caller. +type Membership struct { + Node string + Broker string + Fingerprint string + Password string + Signer ed25519.PublicKey +} + +// Applier is what the host does with a declaration that has been proved to come from the mesh. +// +// It receives the signature as well as the declaration, so the host can keep both: what it was +// told is kept signed and verified again when it is read back, which means the file on disk is +// trusted for the same reason the message was rather than for being local. +type Applier func(ctx context.Context, declaration, signature []byte) Report + +// Announce is how the link says what is happening, so a node running unattended leaves an +// account of it. Nil is allowed and means say nothing. +type Announce func(string) + +// Hold keeps this node in the mesh, reconnecting for as long as it is asked to. +// +// Disconnection is an ordinary situation and not a failure (novox/hq ADR 0004), so this does not +// give up. A laptop shut for a week comes back and reconnects; it does not come back needing +// somebody to start it again. +// +// The backoff exists because the two common reasons differ in how long they last: a broker +// restarting is back in seconds, and a machine that has moved to a network with no route may be +// hours. Retrying every second for hours is a node shouting into nothing; waiting a minute after +// a broker blip is a node that is needlessly late. So it starts fast and slows down, and resets +// once a connection has actually held. +// Roused is a channel that says the machine has reason to believe its link is stale — it woke +// from suspend, or its network changed. +// +// **The machine knows before any timeout does.** A suspended laptop's connection is dead the +// moment it wakes, and heartbeats find that out in twenty or thirty seconds; for that time the +// node believes it is in the mesh and is not, which is the one state this design says must never +// be indistinguishable from being connected. Nothing new listens on the node to arrange it — the +// signal a service manager already sends is enough (novox/hq ADR 0004). +// +// Nil is allowed and means nothing ever rouses it, which is every machine that does not suspend. +type Roused <-chan struct{} + +func Hold(ctx context.Context, m Membership, apply Applier, say Announce, timeout time.Duration) error { + return HoldRoused(ctx, m, apply, say, timeout, nil) +} + +// HoldRoused is Hold, told when the machine has reason to think its link is stale. +func HoldRoused(ctx context.Context, m Membership, apply Applier, say Announce, + timeout time.Duration, roused Roused) error { + + return holdWith(ctx, func(ctx context.Context) error { + return Run(ctx, m, apply, say, timeout) + }, say, roused) +} + +// attempt is one try at holding the link open, returning when it ends for any reason. +// +// Named so the loop below can be driven without a broker. What the loop decides — when to wait, +// how long, what being roused does — is the part with the reasoning in it, and it was reachable +// only through a real connection before. +type attempt func(context.Context) error + +func holdWith(ctx context.Context, run attempt, say Announce, roused Roused) error { + const ( + first = 2 * time.Second + most = 2 * time.Minute + // A connection that lasted this long counts as having worked, so the next failure starts + // from the bottom again. Without it a node that reconnects and immediately drops climbs + // to the maximum and stays there, long after whatever caused it went away. + settled = 30 * time.Second + ) + wait := first + + for { + began := time.Now() + + // The link runs under a context this loop can cancel, so being roused ends the current + // attempt rather than only shortening the wait after it. + // + // **That is the whole of it.** After a resume the socket looks perfectly healthy from + // inside this process — there is no error and no close, because nothing has tried to + // send anything. It is heartbeats that eventually discover it, twenty or thirty seconds + // later. A machine that knows it just woke does not have to wait to be told. + // + // A rouse that turns out to be spurious costs one reconnect, which is cheap and + // idempotent: the node redeclares its queue and anything unacknowledged is redelivered. + // The alternative costs half a minute of believing it is in a mesh it has left. + trying, done := context.WithCancel(ctx) + if roused != nil { + go func() { + select { + case <-trying.Done(): + case <-roused: + say("woken or moved — dropping the link and opening it again") + done() + } + }() + } + err := run(trying) + done() + if ctx.Err() != nil { + return nil + } + if time.Since(began) > settled { + wait = first + } + + switch { + case errors.Is(err, ErrWrongCertificate): + // Said in full every time rather than folded into a retry count. This does not mean + // the network is down; it means what answered is not the mesh this node joined, and + // no amount of waiting fixes it. The node keeps running what it was last told, which + // is the right thing to do while somebody works out what happened. + say("the broker is not the one this node joined: " + err.Error()) + say("this will not fix itself. This node keeps running what it was last told.") + case err != nil: + say(fmt.Sprintf("disconnected: %v — trying again in %s", err, wait)) + default: + say(fmt.Sprintf("the link closed — trying again in %s", wait)) + } + + select { + case <-ctx.Done(): + return nil + case <-roused: + // And it does not serve out a wait computed for a broker that was restarting, either. + // + // The backoff is not *reset* by this. Being roused says the machine changed, not that + // whatever was refusing the connection has stopped — a laptop woken repeatedly on a + // network with no route would otherwise retry at full speed for as long as somebody + // keeps opening the lid. + say("woken or moved — trying again now") + case <-time.After(wait): + } + if wait *= 2; wait > most { + wait = most + } + } +} + +// Run holds the link open once, applying what arrives and reporting what happened. +// +// Outbound only, and nothing listens on this machine. Returns when the link ends, for any reason; +// Hold is what decides whether to open it again. +func Run(ctx context.Context, m Membership, apply Applier, say Announce, timeout time.Duration) error { + if say == nil { + say = func(string) {} + } + config, err := PinnedConfig(m.Fingerprint) + if err != nil { + return err + } + + dsn := fmt.Sprintf("amqps://%s:%s@%s/", + url.QueryEscape(m.Node), url.QueryEscape(m.Password), m.Broker) + conn, err := amqp.DialConfig(dsn, amqp.Config{ + TLSClientConfig: config, + Dial: amqp.DefaultDial(timeout), + // Kept short so a node that has silently lost its route notices, rather than holding a + // connection the broker forgot about and believing it is still in the mesh. + Heartbeat: 10 * time.Second, + }) + if err != nil { + if errors.Is(err, ErrWrongCertificate) { + return err + } + return fmt.Errorf("cannot reach the broker at %s: %w", m.Broker, err) + } + defer conn.Close() + + channel, err := conn.Channel() + if err != nil { + return err + } + defer channel.Close() + + queue := QueueFor(m.Node) + if _, err := channel.QueueDeclare(queue, true, false, false, false, nil); err != nil { + return fmt.Errorf("cannot declare this node's queue %s: %w", queue, err) + } + + // One at a time. A declaration is applied to a machine, and applying two at once would race + // on the same filesystem — so the broker holds the next one until this one is finished, + // where it survives a restart. + if err := channel.Qos(1, 0, false); err != nil { + return err + } + + deliveries, err := channel.ConsumeWithContext(ctx, queue, "", false, false, false, false, nil) + if err != nil { + return err + } + // Said, because it is the event anybody watching actually wants. Without it a node logs + // every failure and nothing on success, so a log full of "trying again" and then silence + // reads as still broken when it means the opposite. + say("in the mesh, consuming " + queue) + + // A word every so often, so the mesh can tell a node that is quiet from one that is gone. + // Cheap on purpose: it carries a name and nothing else, because anything more would be a + // report, and reports are rare where this is constant. + beat := time.NewTicker(AliveEvery) + defer beat.Stop() + publishAlive(ctx, channel, m, say, timeout) + + closed := conn.NotifyClose(make(chan *amqp.Error, 1)) + + // Published mandatory, so the broker hands back anything it cannot route rather than + // dropping it. Without this a report goes to an exchange with no matching binding, the + // publisher is told nothing, and the mesh believes this node never answered while the node + // believes it did — which is what happened when `report` was left unbound on the other side. + returned := channel.NotifyReturn(make(chan amqp.Return, 4)) + go func() { + for r := range returned { + say(fmt.Sprintf("the broker could not route this node's %s: %s (%d %s)", + r.RoutingKey, r.Exchange, r.ReplyCode, r.ReplyText)) + } + }() + + for { + select { + case <-ctx.Done(): + return nil + case <-beat.C: + publishAlive(ctx, channel, m, say, timeout) + case reason := <-closed: + return fmt.Errorf("the link closed: %v", reason) + case delivery, ok := <-deliveries: + if !ok { + return errors.New("the broker stopped delivering") + } + report := handle(ctx, m, apply, delivery) + switch { + case report.Refused != "": + say("refused a declaration: " + report.Refused) + case len(report.Failed) > 0: + say(fmt.Sprintf("applied %d and failed: %v", len(report.Applied), report.Failed)) + default: + say(fmt.Sprintf("applied %d resource(s)", len(report.Applied))) + } + publishReport(ctx, channel, m, report, say, timeout) + // Acknowledged after the report is published. A node that dies between applying and + // reporting leaves the declaration on the broker and applies it again on return, + // which is safe because applying is reconciliation — it converges rather than + // repeating. + _ = delivery.Ack(false) + } + } +} + +func handle(ctx context.Context, m Membership, apply Applier, delivery amqp.Delivery) Report { + return handleBody(ctx, m, delivery.Body, apply) +} + +// handleBody is the whole of deciding whether to trust a message, separated from the broker so it +// can be tested as the security check it is rather than as message plumbing. +func handleBody(ctx context.Context, m Membership, body []byte, apply Applier) Report { + var signed Signed + if err := json.Unmarshal(body, &signed); err != nil { + return Report{Node: m.Node, Refused: "this message is not a declaration: " + err.Error()} + } + + // Before anything is read out of it, let alone applied. The host applies whatever the link + // delivers, so this check is the difference between the mesh changing this machine and + // anybody changing it. + if !ed25519.Verify(m.Signer, signed.Declaration, signed.Signature) { + return Report{Node: m.Node, Refused: ErrForged.Error()} + } + return apply(ctx, signed.Declaration, signed.Signature) +} + +func publishReport(ctx context.Context, channel *amqp.Channel, m Membership, report Report, + say Announce, timeout time.Duration) { + report.Node = m.Node + body, err := json.Marshal(report) + if err != nil { + say("cannot encode this node's own report: " + err.Error()) + return + } + publish, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + // Said rather than swallowed. A report that fails to publish leaves the mesh believing this + // node never answered, while the node believes it did — and the two would go on disagreeing + // with nothing anywhere saying so. That shape of fault is the one this project keeps finding. + if err := channel.PublishWithContext(publish, Exchange, KeyReport, true, false, + amqp.Publishing{ContentType: "application/json", Body: body}); err != nil { + say(fmt.Sprintf("applied, and could not tell the mesh: %v", err)) + } +} + +// publishAlive says this node is here, and nothing else. +func publishAlive(ctx context.Context, channel *amqp.Channel, m Membership, say Announce, + timeout time.Duration) { + body, err := json.Marshal(Alive{Node: m.Node}) + if err != nil { + return + } + publish, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + // Not mandatory, unlike a report. Losing one is nothing: the next is a minute away, and the + // mesh is reading a gap rather than counting arrivals. Insisting on delivery would turn a + // harmless miss into a logged failure every minute. + if err := channel.PublishWithContext(publish, Exchange, KeyAlive, false, false, + amqp.Publishing{ContentType: "application/json", Body: body}); err != nil { + say("could not tell the mesh this node is here: " + err.Error()) + } +} diff --git a/internal/profile/detectors.go b/internal/profile/detectors.go index e9ca9cf..d19114c 100644 --- a/internal/profile/detectors.go +++ b/internal/profile/detectors.go @@ -16,7 +16,10 @@ const ( CapFirewall = "firewall" CapOverlay = "overlay" CapGraphicalSession = "graphical-session" - CapPrivileged = "privileged" + // CapSeat is hardware: somewhere a display server COULD run. CapGraphicalSession above is + // state: whether one IS running. Assignment needs the first. + CapSeat = "seat" + CapPrivileged = "privileged" ) // commandCapability is the shape most detectors take: run something, and treat a working @@ -111,7 +114,7 @@ func firstLine(s string) string { // privileged reports whether the host can change this machine at all. // // Reported as a capability rather than checked at startup on purpose: a host that cannot act -// is still a host that can report, and novox/hq ADR 0036 says what varies between nodes lives +// is still a host that can report, and novox/hq ADR 0004 says what varies between nodes lives // here rather than in the definition of a node. type privileged struct{} @@ -172,6 +175,7 @@ func Default(runner Runner) []Detector { return []Detector{ privileged{}, graphicalSession{}, + seat{}, commandCapability{ name: CapContainerRuntime, command: "docker", args: []string{"info", "--format", "{{.ServerVersion}}"}, why: "asks the daemon for its version — a running daemon, not an installed client", diff --git a/internal/profile/profile.go b/internal/profile/profile.go index 5ce7d70..ace9572 100644 --- a/internal/profile/profile.go +++ b/internal/profile/profile.go @@ -44,7 +44,7 @@ type Detector interface { // Runner executes a command. Replaceable in tests for the pure-logic layer ONLY — every // detector in this package is exercised against the real machine as well, because a test that // fakes the system under detection asserts that the fake behaves as expected -// (novox/hq ADR 0034). +// (novox/hq ADR 0017). type Runner func(ctx context.Context, name string, args ...string) (stdout string, err error) // ExecRunner runs a real command, with output captured and stdin closed. @@ -97,7 +97,7 @@ func (p Profile) Missing() []string { // Detect runs every detector and collects the verdicts. // // A detector that fails does not fail the profile. This is deliberately NOT the rule in -// novox/hq ADR 0008: that rule governs applying state, where a failed step means the machine +// novox/hq ADR 0010: that rule governs applying state, where a failed step means the machine // is not what was asked for. Detection is the opposite — a failed probe is a finding, and the // finding is "absent, because the probe failed", which is exactly what a caller needs to know. // Aborting would replace one legible absence with total ignorance. diff --git a/internal/profile/profile_system_test.go b/internal/profile/profile_system_test.go index a6b58f2..882f1dd 100644 --- a/internal/profile/profile_system_test.go +++ b/internal/profile/profile_system_test.go @@ -8,7 +8,7 @@ import ( "time" ) -// Against the real machine. novox/hq ADR 0034: structure and logic are tested first, behaviour +// Against the real machine. novox/hq ADR 0017: structure and logic are tested first, behaviour // against a real system alongside, and mocking the boundary is forbidden — a test that fakes // the system under detection asserts that the fake behaves as expected. // diff --git a/internal/profile/profile_test.go b/internal/profile/profile_test.go index 926238d..9c844ed 100644 --- a/internal/profile/profile_test.go +++ b/internal/profile/profile_test.go @@ -8,7 +8,7 @@ import ( "time" ) -// The decision each test defends is named in the test, per novox/hq ADR 0034. These cover +// The decision each test defends is named in the test, per novox/hq ADR 0017. These cover // structure and logic; profile_system_test.go covers the same detectors against the real // machine, because a test that fakes the system under detection asserts only that the fake // behaves as expected. @@ -57,7 +57,7 @@ func TestEveryVerdictSaysHowItKnows(t *testing.T) { } func TestDetectionSurvivesAFailingProbe(t *testing.T) { - // Deliberately NOT ADR 0008. That rule governs APPLYING state, where a failed step means + // Deliberately NOT ADR 0010. That rule governs APPLYING state, where a failed step means // the machine is not what was asked for. A failed probe is a finding, and aborting would // replace one legible absence with total ignorance of the rest. only := func(ctx context.Context, name string, args ...string) (string, error) { diff --git a/internal/profile/seat.go b/internal/profile/seat.go new file mode 100644 index 0000000..09fff1d --- /dev/null +++ b/internal/profile/seat.go @@ -0,0 +1,90 @@ +package profile + +import ( + "context" + "os" + "path/filepath" + "sort" + "strings" +) + +// A seat is somewhere a display server could run: a graphics device with a display attached. +// +// **Not the same question as `graphical-session`**, and the difference is what makes it worth +// having. That one asks whether a session is running *now*, which is state; this asks whether one +// could ever run here, which is hardware. Deciding whether to assign a display server needs the +// second — a headless server can never have one, a workstation with nothing installed yet can, +// and until this existed those two looked identical. The mesh would have assigned xorg to the +// server and found out at apply time. +// +// **`seat` rather than `display`, and the distinction matters most on a phone.** An Android +// device plainly has a screen and has no seat: nothing there is going to take a DRM device and +// present an X or Wayland session on it. A capability called `display` would answer *yes* and be +// useless; `seat` answers *no*, which is the true and useful answer. +// +// The term is logind's, and it means what is wanted here: a set of hardware one person sits at. + +// SeatSource is where the kernel reports what is attached. A variable so a test can point it at a +// directory it made, rather than at whatever this machine happens to have. +var SeatSource = "/sys/class/drm" + +type seat struct{} + +func (seat) Name() string { return CapSeat } + +func (seat) Detect(context.Context) Verdict { + const how = "/sys/class/drm/*/status — a connector the kernel reports as connected" + + entries, err := os.ReadDir(SeatSource) + if err != nil { + // No DRM subsystem at all: a container, a phone, a machine with no graphics stack. Said + // as what was looked for rather than as an error, because this is the ordinary answer on + // most nodes and not a fault on any of them. + return Verdict{ + Name: CapSeat, Present: false, + Detail: "no graphics devices are present (" + SeatSource + " is not readable)", + How: how, + } + } + + var connected, disconnected []string + for _, e := range entries { + status, err := os.ReadFile(filepath.Join(SeatSource, e.Name(), "status")) + if err != nil { + // Cards themselves have no status file; only connectors do. Skipping is right and + // not a failure to report. + continue + } + switch strings.TrimSpace(string(status)) { + case "connected": + connected = append(connected, e.Name()) + default: + disconnected = append(disconnected, e.Name()) + } + } + sort.Strings(connected) + + if len(connected) > 0 { + return Verdict{ + Name: CapSeat, Present: true, + Detail: strings.Join(connected, ", "), + How: how, + } + } + if len(disconnected) > 0 { + // The distinction worth drawing: this machine has graphics hardware and nothing plugged + // into it. A server with a GPU and no monitor, which is a different thing from a machine + // with no graphics at all — and a person deciding where a desktop goes wants to know + // which they are looking at. + return Verdict{ + Name: CapSeat, Present: false, + Detail: "a graphics device with no display attached", + How: how, + } + } + return Verdict{ + Name: CapSeat, Present: false, + Detail: "no display connectors are present", + How: how, + } +} diff --git a/internal/profile/seat_test.go b/internal/profile/seat_test.go new file mode 100644 index 0000000..d403440 --- /dev/null +++ b/internal/profile/seat_test.go @@ -0,0 +1,114 @@ +package profile + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +// drm builds what the kernel would expose, so the cases below are the real ones rather than a +// fake's idea of them: a workstation, a server with a card and no monitor, a machine with no +// graphics at all. +func drm(t *testing.T, connectors map[string]string) { + t.Helper() + dir := t.TempDir() + for name, status := range connectors { + if err := os.MkdirAll(filepath.Join(dir, name), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, name, "status"), []byte(status+"\n"), 0o644); err != nil { + t.Fatal(err) + } + } + // The card itself has no status file, and skipping it must not be mistaken for an answer. + if err := os.MkdirAll(filepath.Join(dir, "card0"), 0o755); err != nil { + t.Fatal(err) + } + old := SeatSource + SeatSource = dir + t.Cleanup(func() { SeatSource = old }) +} + +func TestAMachineWithAMonitorHasASeat(t *testing.T) { + drm(t, map[string]string{"card0-DP-1": "connected", "card0-HDMI-A-1": "disconnected"}) + + got := seat{}.Detect(context.Background()) + if !got.Present { + t.Fatalf("a machine with a connected display has no seat: %s", got.Detail) + } + if !strings.Contains(got.Detail, "card0-DP-1") { + t.Errorf("the answer does not say which connector: %q", got.Detail) + } + if strings.Contains(got.Detail, "HDMI") { + t.Errorf("a disconnected connector was counted: %q", got.Detail) + } +} + +func TestAServerWithAGraphicsCardAndNoMonitorHasNoSeat(t *testing.T) { + // The case this capability exists for. A display server assigned here would install, start, + // and have nowhere to draw — and the mesh would have thought it succeeded. + drm(t, map[string]string{"card0-HDMI-A-1": "disconnected", "card0-DP-1": "disconnected"}) + + got := seat{}.Detect(context.Background()) + if got.Present { + t.Fatal("a machine with nothing plugged in reported a seat") + } + // And it says which of the two "no" answers this is, because a person deciding where a + // desktop goes wants to know whether the hardware is there. + if !strings.Contains(got.Detail, "no display attached") { + t.Errorf("the answer does not distinguish this from having no graphics at all: %q", got.Detail) + } +} + +func TestAMachineWithNoGraphicsAtAllHasNoSeat(t *testing.T) { + // A container, or a phone, where the DRM subsystem is not there to read. The ordinary answer + // on most nodes and a fault on none of them, so it is a plain "no" rather than an error. + old := SeatSource + SeatSource = filepath.Join(t.TempDir(), "not-here") + t.Cleanup(func() { SeatSource = old }) + + got := seat{}.Detect(context.Background()) + if got.Present { + t.Fatal("a machine with no graphics subsystem reported a seat") + } + if got.Detail == "" || strings.Contains(strings.ToLower(got.Detail), "error") { + t.Errorf("absence was reported as a failure: %q", got.Detail) + } +} + +func TestAnUnknownConnectorIsNotASeat(t *testing.T) { + // Writeback connectors and some virtual devices report "unknown". Counting those would give + // a seat to machines that have none — and this workstation has one, so it is not theoretical. + drm(t, map[string]string{"card0-Writeback-1": "unknown"}) + + if (seat{}).Detect(context.Background()).Present { + t.Fatal("a connector reporting \"unknown\" was counted as a display") + } +} + +func TestTheAnswerSaysHowItKnows(t *testing.T) { + // Every detector says how, so a wrong answer can be found rather than argued about. + drm(t, map[string]string{"card0-DP-1": "connected"}) + if !strings.Contains((seat{}).Detect(context.Background()).How, "status") { + t.Error("the seat detector does not say what it looked at") + } +} + +func TestASeatIsNotAGraphicalSession(t *testing.T) { + // The two are different questions and the whole point of adding one was that they had been + // answered by the same thing. A machine can have a seat and no session — that is every + // workstation before anything is installed on it, and exactly where a display server should + // be assigned. + drm(t, map[string]string{"card0-DP-1": "connected"}) + t.Setenv("DISPLAY", "") + t.Setenv("WAYLAND_DISPLAY", "") + + if !(seat{}).Detect(context.Background()).Present { + t.Fatal("a machine with a monitor and no session has no seat") + } + if (graphicalSession{}).Detect(context.Background()).Present { + t.Fatal("a machine with no session reported one") + } +} diff --git a/internal/store/declared.go b/internal/store/declared.go new file mode 100644 index 0000000..5d2e80d --- /dev/null +++ b/internal/store/declared.go @@ -0,0 +1,108 @@ +package store + +import ( + "crypto/ed25519" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" +) + +// What the mesh last told this node to be, kept so it can go on being it. +// +// novox/hq ADR 0004: a disconnected node keeps reconciling against its own store, so it holds its +// machine in the last state it was told. A laptop shut for a week comes back and reconciles; it +// does not come back and ask what it is. +// +// That needs the declaration itself. The record of what was *applied* is not enough to re-apply: +// it holds an id, a type and a target, which is what removal needs and not what creation needs. +// So the declaration is kept whole. +// +// **Kept signed, and verified again on every load.** The signature is not decoration here: this +// file is on a machine, and a node that read it back unverified would apply whatever was in it. +// Anyone able to write it already has root — but the check costs nothing, and it means the file +// is trusted for the same reason the message was, rather than for being local. + +// DeclaredName is where it lives, beside the state. +const DeclaredName = "declared.json" + +// DeclaredPath is where the last declaration lives, given where the state lives. +func DeclaredPath(statePath string) string { + return filepath.Join(filepath.Dir(statePath), DeclaredName) +} + +// Declared is the last thing the mesh said, and the signature it came with. +type Declared struct { + Declaration []byte `json:"declaration"` + Signature []byte `json:"signature"` +} + +// ErrNothingDeclared means the mesh has never told this node anything. +// +// An ordinary state, not a fault: a node that has enrolled and not yet been sent a declaration +// has nothing to reconcile against, and that is different from having lost it. +var ErrNothingDeclared = errors.New("the mesh has not told this node anything yet") + +// SaveDeclared keeps what the mesh said, so a disconnected node can go on obeying it. +func SaveDeclared(path string, d Declared) error { + if len(d.Declaration) == 0 { + return errors.New("refusing to keep an empty declaration") + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return err + } + raw, err := json.Marshal(d) + if err != nil { + return err + } + + tmp, err := os.CreateTemp(filepath.Dir(path), ".declared-*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + if err := tmp.Chmod(0o600); err != nil { + tmp.Close() + return err + } + if _, err := tmp.Write(raw); err != nil { + tmp.Close() + return err + } + if err := tmp.Sync(); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmp.Name(), path) +} + +// LoadDeclared reads it back and proves it is still the mesh's. +// +// Verified against the signing key this node holds, which came from its token. A declaration on +// disk that does not verify is refused rather than applied: either the file was changed, or this +// node now believes a different mesh — and applying it either way would be applying something +// nobody in this mesh said. +func LoadDeclared(path string, signer ed25519.PublicKey) ([]byte, error) { + raw, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return nil, ErrNothingDeclared + } + if err != nil { + return nil, fmt.Errorf("this node was told something and cannot read it back: %w", err) + } + + var d Declared + if err := json.Unmarshal(raw, &d); err != nil { + return nil, fmt.Errorf("what this node was told is unreadable at %s: %w", path, err) + } + if !ed25519.Verify(signer, d.Declaration, d.Signature) { + return nil, fmt.Errorf( + "what this node kept at %s is not signed by the mesh it joined. It will not be "+ + "applied — either the file was changed, or this node's signing key was", path) + } + return d.Declaration, nil +} diff --git a/internal/store/declared_test.go b/internal/store/declared_test.go new file mode 100644 index 0000000..4d36a0f --- /dev/null +++ b/internal/store/declared_test.go @@ -0,0 +1,140 @@ +package store + +import ( + "crypto/ed25519" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +func signedBy(t *testing.T, body string) (ed25519.PublicKey, Declared) { + t.Helper() + public, private, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + return public, Declared{ + Declaration: []byte(body), + Signature: ed25519.Sign(private, []byte(body)), + } +} + +func TestNothingDeclaredIsNotAFault(t *testing.T) { + // A node that has enrolled and not yet been assigned anything has nothing to hold its machine + // to, and that is an ordinary state — different from having lost what it was told. + public, _ := signedBy(t, "{}") + _, err := LoadDeclared(DeclaredPath(filepath.Join(t.TempDir(), "state.json")), public) + if !errors.Is(err, ErrNothingDeclared) { + t.Fatalf("a node that was never told anything gave %v", err) + } +} + +func TestWhatWasKeptIsWhatComesBack(t *testing.T) { + path := DeclaredPath(filepath.Join(t.TempDir(), "state.json")) + public, d := signedBy(t, `{"declaration":1,"resources":[]}`) + if err := SaveDeclared(path, d); err != nil { + t.Fatal(err) + } + + back, err := LoadDeclared(path, public) + if err != nil { + t.Fatal(err) + } + if string(back) != string(d.Declaration) { + t.Errorf("kept %q and read back %q", d.Declaration, back) + } +} + +func TestWhatWasKeptIsVerifiedAgainOnLoad(t *testing.T) { + // This file is on a machine, and a node reading it back unverified would apply whatever is in + // it. Anyone able to write it already has root — but the check costs nothing, and it means + // the file is trusted for the same reason the message was rather than for being local. + path := DeclaredPath(filepath.Join(t.TempDir(), "state.json")) + public, d := signedBy(t, `{"declaration":1,"resources":[]}`) + if err := SaveDeclared(path, d); err != nil { + t.Fatal(err) + } + + // Somebody edits it, keeping the signature. + tampered, err := json.Marshal(Declared{ + Declaration: []byte(`{"declaration":1,"resources":["something else"]}`), + Signature: d.Signature, + }) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, tampered, 0o600); err != nil { + t.Fatal(err) + } + + if _, err := LoadDeclared(path, public); err == nil { + t.Fatal("an edited declaration was read back and would have been applied") + } +} + +func TestAnotherMeshsDeclarationIsRefused(t *testing.T) { + // The same check answers a second question: this node now believes a different signing key, + // so what it kept is not this mesh's. Applying it would be applying something nobody in this + // mesh said. + path := DeclaredPath(filepath.Join(t.TempDir(), "state.json")) + _, d := signedBy(t, `{"declaration":1}`) + if err := SaveDeclared(path, d); err != nil { + t.Fatal(err) + } + other, _ := signedBy(t, "unrelated") + + _, err := LoadDeclared(path, other) + if err == nil { + t.Fatal("a declaration signed by another mesh was accepted") + } + if !strings.Contains(err.Error(), "not signed by the mesh it joined") { + t.Errorf("the refusal does not say what is wrong: %v", err) + } +} + +func TestWhatWasKeptIsNotWorldReadable(t *testing.T) { + path := DeclaredPath(filepath.Join(t.TempDir(), "state.json")) + _, d := signedBy(t, `{"declaration":1}`) + if err := SaveDeclared(path, d); err != nil { + t.Fatal(err) + } + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm()&0o077 != 0 { + t.Errorf("what this node was told is mode %04o", info.Mode().Perm()) + } +} + +func TestKeepingLeavesNoHalfWrittenFile(t *testing.T) { + dir := t.TempDir() + path := DeclaredPath(filepath.Join(dir, "state.json")) + _, d := signedBy(t, `{"declaration":1}`) + for i := 0; i < 3; i++ { + if err := SaveDeclared(path, d); err != nil { + t.Fatal(err) + } + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + if strings.HasPrefix(e.Name(), ".declared-") { + t.Errorf("a temporary file survived: %s", e.Name()) + } + } +} + +func TestAnEmptyDeclarationIsNotKept(t *testing.T) { + // It would read back as an instruction to own nothing, and a node that acted on it would + // remove everything the mesh had given it. + path := DeclaredPath(filepath.Join(t.TempDir(), "state.json")) + if err := SaveDeclared(path, Declared{}); err == nil { + t.Fatal("an empty declaration was kept") + } +} diff --git a/internal/store/store.go b/internal/store/store.go new file mode 100644 index 0000000..e0669d7 --- /dev/null +++ b/internal/store/store.go @@ -0,0 +1,235 @@ +// Package store is what this node knows about itself, and it is authoritative while +// disconnected. +// +// Not a cache of the control plane. novox/hq ADR 0004 makes disconnection an ordinary +// situation rather than an exception, and this is what makes it ordinary: a machine shut for a +// week comes back and reconciles, it does not come back and ask what it is. +// +// Its first job arrives with the first apply rather than with the link (ADR 0005): the host +// removes what it previously applied and is no longer declared, and it can only know that +// because it wrote it down. +package store + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "time" +) + +// DefaultPath is where a node keeps what it knows. Under /var/lib because it survives a +// reboot and is not configuration — nothing generates this, the host writes it. +const DefaultPath = "/var/lib/mesh-host/state.json" + +// Applied is one resource the host put on this machine, and what it did. +// +// Recorded AFTER the resource was applied and read back, never before (novox/hq ADR 0018). +// A record written up front restates the request in a new place and inherits none of the +// authority of having happened. +type Applied struct { + ID string `json:"id"` + Type string `json:"type"` + // Origin is who asked for this: the bundle this host carries, or the mesh. + // + // Recorded because the two must not remove each other. A node raises its own substrate from + // the bundle before any mesh exists, then enrols and is sent declarations — and a + // declaration naming two resources would otherwise remove the store, the broker and the + // control plane, which is 04-ISSUES/010 and happened on the first end-to-end run. + // + // Empty means carried, for state written before this field existed: everything a host had + // applied at that point came from its bundle. + Origin string `json:"origin,omitempty"` + + // Holds are the machine's own ports this resource occupies. + // + // **So the mesh can assign around what it did not put here** (novox/hq ADR 0038). A node + // raises its substrate from the bundle before any mesh exists, so the control plane has never + // heard of the store, the broker or the control plane's own container — and a module assigned + // afterwards would be given a port one of them already holds, and would be told so by a + // container runtime rather than by anything that could have prevented it. + // + // Recorded per resource rather than counted per machine, because what a machine happens to + // have open right now is a moving target, and what its declaration binds is not. + Holds []int `json:"holds,omitempty"` + // Target is what was changed — a path, a unit — so removal knows what to undo without + // re-reading a declaration that may no longer exist. + Target string `json:"target"` + AppliedAt time.Time `json:"applied_at"` + + // Wrote is a digest of what this host last put there, for resources where that is a + // meaningful question. + // + // Without it, a file that does not match the declaration has two possible explanations and + // the host cannot tell them apart: the mesh changed what it wants, or somebody edited the + // machine. Both end with the file being rewritten, so the outcome is identical — and a + // person who edits a managed file watches their change vanish every few minutes with nothing + // anywhere saying why. + Wrote string `json:"wrote,omitempty"` +} + +// State is the whole of what a node knows about what it has done. +type State struct { + // Resources, keyed by identity, in the order they were applied. Order matters for removal: + // undoing in reverse is the only ordering the host can derive without deciding anything. + Resources []Applied `json:"resources"` + UpdatedAt time.Time `json:"updated_at"` +} + +// Find returns what was applied under an identity. +func (s State) Find(id string) (Applied, bool) { + for _, r := range s.Resources { + if r.ID == id { + return r, true + } + } + return Applied{}, false +} + +// IDs returns every identity the host has applied, sorted. +func (s State) IDs() []string { + out := make([]string, 0, len(s.Resources)) + for _, r := range s.Resources { + out = append(out, r.ID) + } + sort.Strings(out) + return out +} + +// Load reads the state. A node that has never applied anything has an empty state, which is a +// fact rather than an error — the first apply on a fresh machine is the ordinary case. +// +// A state file that exists and cannot be read IS an error, and a loud one: continuing with an +// empty state would make the host believe it owns nothing, and it would then remove nothing it +// should and re-apply everything it need not. +func Load(path string) (State, error) { + raw, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + return State{}, nil + } + if err != nil { + return State{}, fmt.Errorf("reading what this node knows about itself (%s): %w", path, err) + } + + var s State + if err := json.Unmarshal(raw, &s); err != nil { + return State{}, fmt.Errorf( + "what this node knows about itself is unreadable (%s): %w\n"+ + "Refusing rather than starting empty: an empty state would mean the host "+ + "believes it owns nothing, so it would remove nothing it should and re-apply "+ + "everything it need not", path, err) + } + return s, nil +} + +// Save writes the state, atomically. +// +// Atomic because the alternative has a failure mode with no floor: a host interrupted while +// writing loses the record of everything it owns, and then owns nothing it can clean up. +func Save(path string, s State) error { + s.UpdatedAt = time.Now().UTC() + + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return fmt.Errorf("making room for the node's state: %w", err) + } + + raw, err := json.MarshalIndent(s, "", " ") + if err != nil { + return fmt.Errorf("encoding the node's state: %w", err) + } + raw = append(raw, '\n') + + tmp, err := os.CreateTemp(filepath.Dir(path), ".state-*.json") + if err != nil { + return fmt.Errorf("writing the node's state: %w", err) + } + defer os.Remove(tmp.Name()) + + if _, err := tmp.Write(raw); err != nil { + tmp.Close() + return fmt.Errorf("writing the node's state: %w", err) + } + // Flushed before the rename: a rename is atomic, and a rename of a file whose contents are + // still in the page cache is atomically the wrong thing. + if err := tmp.Sync(); err != nil { + tmp.Close() + return fmt.Errorf("flushing the node's state: %w", err) + } + if err := tmp.Close(); err != nil { + return fmt.Errorf("closing the node's state: %w", err) + } + if err := os.Chmod(tmp.Name(), 0o600); err != nil { + return fmt.Errorf("securing the node's state: %w", err) + } + if err := os.Rename(tmp.Name(), path); err != nil { + return fmt.Errorf("replacing the node's state: %w", err) + } + return nil +} + +// Record adds or replaces what is known about one resource, preserving order. +func (s *State) Record(a Applied) { + for i, existing := range s.Resources { + if existing.ID == a.ID { + s.Resources[i] = a + return + } + } + s.Resources = append(s.Resources, a) +} + +// Forget drops a resource from what the node owns. +func (s *State) Forget(id string) { + kept := s.Resources[:0] + for _, r := range s.Resources { + if r.ID != id { + kept = append(kept, r) + } + } + s.Resources = kept +} + +// Orphans returns what the host applied and the declaration no longer names, newest first. +// +// Reverse order because undoing in the order things were made undoes a directory before the +// file inside it. Reversing is the only ordering the host can derive without deciding +// anything, which is the line novox/hq ADR 0005 draws. +func (s State) Orphans(declared map[string]bool, origin string) []Applied { + var out []Applied + for i := len(s.Resources) - 1; i >= 0; i-- { + r := s.Resources[i] + // Only this origin's own. A mesh declaration says nothing about what the bundle raised, + // and a bundle says nothing about what the mesh assigned — so neither may remove the + // other's by omission, which is the only way either could express removal. + if originOf(r) != origin { + continue + } + if !declared[r.ID] { + out = append(out, r) + } + } + return out +} + +// Origins a resource can have. +const ( + // Carried is the bundle this host was built with. + OriginCarried = "carried" + // Declared is the mesh, over the link. + OriginDeclared = "declared" +) + +// originOf reads a record's origin, treating absence as carried. +// +// State written before origins existed was all bundle-applied: a host had no other way to be +// told anything. Guessing wrong in the other direction would have a first upgrade remove the +// substrate, which is the fault this field exists to prevent. +func originOf(r Applied) string { + if r.Origin == "" { + return OriginCarried + } + return r.Origin +} diff --git a/internal/store/store_test.go b/internal/store/store_test.go new file mode 100644 index 0000000..3867be9 --- /dev/null +++ b/internal/store/store_test.go @@ -0,0 +1,188 @@ +package store + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestAFreshMachineHasAnEmptyStateNotAnError(t *testing.T) { + // The first apply on a machine that has never been touched is the ordinary case, not a + // failure. A host that errored here could never bootstrap anything. + s, err := Load(filepath.Join(t.TempDir(), "nothing-here.json")) + if err != nil { + t.Fatalf("a fresh machine produced an error: %v", err) + } + if len(s.Resources) != 0 { + t.Errorf("a fresh machine claims to own %d resources", len(s.Resources)) + } +} + +func TestAnUnreadableStateIsRefusedNotIgnored(t *testing.T) { + // The dangerous one. Starting empty would make the host believe it owns nothing, so it + // would remove nothing it should and re-apply everything it need not — silently. + path := filepath.Join(t.TempDir(), "state.json") + if err := os.WriteFile(path, []byte("{this is not json"), 0o600); err != nil { + t.Fatal(err) + } + _, err := Load(path) + if err == nil { + t.Fatal("a corrupt state was read as an empty one") + } + if !strings.Contains(err.Error(), "believes it owns nothing") { + t.Errorf("the error does not say why this matters: %v", err) + } +} + +func TestWhatIsSavedIsWhatIsLoaded(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + want := State{Resources: []Applied{ + {ID: "etc", Type: "directory", Target: "/etc/mesh", AppliedAt: time.Now().UTC().Truncate(time.Second)}, + {ID: "conf", Type: "file", Target: "/etc/mesh/host.conf", AppliedAt: time.Now().UTC().Truncate(time.Second)}, + }} + if err := Save(path, want); err != nil { + t.Fatal(err) + } + got, err := Load(path) + if err != nil { + t.Fatal(err) + } + if len(got.Resources) != 2 || got.Resources[0].ID != "etc" || got.Resources[1].ID != "conf" { + t.Fatalf("order or content was lost: %+v", got.Resources) + } + if got.UpdatedAt.IsZero() { + t.Error("the state does not say when it was written") + } +} + +func TestTheStateIsNotWorldReadable(t *testing.T) { + // It records what is on the machine and where. Not secret, and not everyone's business. + path := filepath.Join(t.TempDir(), "state.json") + if err := Save(path, State{Resources: []Applied{{ID: "a", Type: "file", Target: "/a"}}}); err != nil { + t.Fatal(err) + } + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if mode := info.Mode().Perm(); mode&0o077 != 0 { + t.Errorf("the state is readable by others: %o", mode) + } +} + +func TestSavingLeavesNoDebrisBehind(t *testing.T) { + // The write is atomic through a temporary file. A run that left those behind would fill a + // directory with near-copies of the truth, and the next reader would have to guess. + dir := t.TempDir() + path := filepath.Join(dir, "state.json") + for i := 0; i < 3; i++ { + if err := Save(path, State{Resources: []Applied{{ID: "a", Type: "file", Target: "/a"}}}); err != nil { + t.Fatal(err) + } + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 { + names := []string{} + for _, e := range entries { + names = append(names, e.Name()) + } + t.Errorf("expected only the state file, found: %v", names) + } +} + +func TestRecordReplacesRatherThanDuplicating(t *testing.T) { + s := State{} + s.Record(Applied{ID: "a", Type: "file", Target: "/old"}) + s.Record(Applied{ID: "a", Type: "file", Target: "/new"}) + + if len(s.Resources) != 1 { + t.Fatalf("one identity produced %d records", len(s.Resources)) + } + if s.Resources[0].Target != "/new" { + t.Errorf("the record was not updated: %+v", s.Resources[0]) + } +} + +func TestOrphansAreWhatWasAppliedAndIsNoLongerDeclared(t *testing.T) { + // The whole reason the store arrives at stage 2 rather than stage 3: removal is impossible + // without knowing what was applied. + s := State{Resources: []Applied{ + {ID: "dir", Type: "directory", Target: "/etc/mesh"}, + {ID: "file", Type: "file", Target: "/etc/mesh/a.conf"}, + {ID: "kept", Type: "file", Target: "/etc/mesh/b.conf"}, + }} + orphans := s.Orphans(map[string]bool{"kept": true}, OriginCarried) + + if len(orphans) != 2 { + t.Fatalf("expected two orphans, got %d: %+v", len(orphans), orphans) + } + // Reverse order: undoing in the order things were made would remove a directory before the + // file inside it. + if orphans[0].ID != "file" || orphans[1].ID != "dir" { + t.Errorf("orphans are not in reverse order: %s then %s", orphans[0].ID, orphans[1].ID) + } +} + +func TestNothingIsAnOrphanWhenEverythingIsDeclared(t *testing.T) { + s := State{Resources: []Applied{{ID: "a", Type: "file", Target: "/a"}}} + if got := s.Orphans(map[string]bool{"a": true}, OriginCarried); len(got) != 0 { + t.Errorf("a declared resource was treated as an orphan: %+v", got) + } +} + +func TestADeclarationDoesNotOrphanWhatTheBundleRaised(t *testing.T) { + // 04-ISSUES/010. A first node raises its substrate from the bundle it carries, then enrols + // and is sent a declaration naming two resources. Before origins, that removed the store, the + // broker and the control plane that had sent it — the mesh deleting itself over the link the + // message arrived on, in under a second, on the first end-to-end run. + s := State{Resources: []Applied{ + {ID: "store", Type: "container", Target: "mesh-store", Origin: OriginCarried}, + {ID: "broker", Type: "container", Target: "mesh-broker", Origin: OriginCarried}, + {ID: "greeting", Type: "directory", Target: "/var/lib/demo", Origin: OriginDeclared}, + }} + + // The mesh declares nothing at all. Everything it previously declared is an orphan; nothing + // the bundle raised is. + orphans := s.Orphans(map[string]bool{}, OriginDeclared) + if len(orphans) != 1 || orphans[0].ID != "greeting" { + var got []string + for _, o := range orphans { + got = append(got, o.ID) + } + t.Fatalf("a declaration would remove %v; it may only remove what the mesh declared", got) + } +} + +func TestTheBundleDoesNotOrphanWhatTheMeshDeclared(t *testing.T) { + // The same rule the other way. A host reconciling its carried bundle must not remove what the + // mesh assigned to this node, or every restart would undo the node's actual work. + s := State{Resources: []Applied{ + {ID: "store", Type: "container", Target: "mesh-store", Origin: OriginCarried}, + {ID: "workload", Type: "container", Target: "some-app", Origin: OriginDeclared}, + }} + + orphans := s.Orphans(map[string]bool{"store": true}, OriginCarried) + if len(orphans) != 0 { + t.Errorf("reconciling the bundle would remove %s, which the mesh declared", orphans[0].ID) + } +} + +func TestStateWrittenBeforeOriginsExistedIsTreatedAsCarried(t *testing.T) { + // Every resource a host had applied before this field existed came from its bundle, because + // there was no other way to tell it anything. Guessing the other way would have the first + // declaration remove the substrate — which is the fault this exists to prevent, arriving + // through the upgrade that fixes it. + s := State{Resources: []Applied{{ID: "store", Type: "container", Target: "mesh-store"}}} + + if got := s.Orphans(map[string]bool{}, OriginDeclared); len(got) != 0 { + t.Errorf("a declaration would remove %s, recorded before origins existed", got[0].ID) + } + if got := s.Orphans(map[string]bool{}, OriginCarried); len(got) != 1 { + t.Error("the bundle cannot remove its own resource, so nothing could ever remove it") + } +} diff --git a/internal/system/alpine.go b/internal/system/alpine.go new file mode 100644 index 0000000..5c66602 --- /dev/null +++ b/internal/system/alpine.go @@ -0,0 +1,170 @@ +package system + +import ( + "context" + "fmt" + "strings" + + "github.com/novox/mesh-host/internal/declaration" +) + +// alpine is apk and OpenRC. +// +// The intended first node. Where it differs from systemd is not cosmetic, and each difference +// below is a place where the systemd implementation's care had to be re-derived rather than +// translated. +type alpine struct{} + +func (alpine) Name() string { return "alpine" } +func (alpine) Shapes() []declaration.Type { return everyShape() } + +func (a alpine) Confirm(ctx context.Context, run Runner) error { + // `apk info -e apk-tools` asks the installed-package database about something that is + // certainly there. `apk --version` would prove only that a binary exists, which is the + // assumption 04-ISSUES/007 records. + if _, err := run(ctx, "apk", "info", "-e", "apk-tools"); err != nil { + return fmt.Errorf( + "this is the alpine host and apk does not answer here. Either this machine is not "+ + "Alpine, or its package database is broken: %w", err) + } + return nil +} + +// PackageInstalled asks apk, having first established that apk answers. +// +// `apk info -e ` prints the name when installed and NOTHING when not — and exits zero +// either way. So unlike pacman, the exit code cannot be used at all here: an empty answer is +// the negative. Reading the exit code would report every package as installed. +func (a alpine) PackageInstalled(ctx context.Context, run Runner, name string) (bool, error) { + if err := a.Confirm(ctx, run); err != nil { + return false, fmt.Errorf("nothing can be said about %q: %w", name, err) + } + out, err := run(ctx, "apk", "info", "-e", name) + if err != nil { + return false, nil + } + return strings.TrimSpace(out) != "", nil +} + +func (alpine) InstallPackage(ctx context.Context, run Runner, name string) error { + _, err := run(ctx, "apk", "add", "--no-cache", name) + return err +} + +// ServiceState reads what OpenRC says about a service. +// +// The same trap as systemd's, and it needs answering differently because OpenRC has no +// LoadState. `rc-service status` exits 3 for a stopped service and 1 for one that does +// not exist — but the exit code reaches us wrapped, so the output is read instead: OpenRC says +// "does not exist" plainly, and that distinction is the whole reason this function is not a +// one-liner. +func (alpine) ServiceState(ctx context.Context, run Runner, unit string) (string, error) { + out, err := run(ctx, "rc-service", unit, "status") + text := strings.ToLower(out + " " + errText(err)) + + switch { + case strings.Contains(text, "does not exist"): + return "", fmt.Errorf( + "%s does not exist on this machine. A declaration naming a service that is not "+ + "installed cannot be satisfied, and reporting it stopped would be reporting "+ + "absence as success", unit) + case strings.Contains(text, "status: started"), strings.Contains(text, "status: starting"): + return "running", nil + case strings.Contains(text, "status: stopped"), strings.Contains(text, "status: stopping"): + return "stopped", nil + case strings.Contains(text, "status: crashed"): + // Crashed is not running, and it is not the same as stopped either — but a + // declaration can only ask for one of two things, and the honest mapping is that the + // service is not up. Starting it is then the right next act. + return "stopped", nil + case strings.TrimSpace(text) == "": + return "", fmt.Errorf("the service manager said nothing about %s", unit) + default: + return "", fmt.Errorf( + "the service manager reports %s as %q, which is neither running nor stopped", + unit, strings.TrimSpace(out)) + } +} + +func (alpine) SetServiceState(ctx context.Context, run Runner, unit, state string) error { + verb := "start" + if state == "stopped" { + verb = "stop" + } + _, err := run(ctx, "rc-service", unit, verb) + return err +} + +// ServiceBoot reads whether a service is in a runlevel. +// +// OpenRC has no `is-enabled`. What it has is `rc-update show`, which lists services against the +// runlevels they are added to — so "does it start at boot" becomes "does it appear here", and +// there is no equivalent of systemd's `static` because OpenRC has no unit files without an +// install story. +func (alpine) ServiceBoot(ctx context.Context, run Runner, unit string) (string, error) { + out, err := run(ctx, "rc-update", "show", "default") + if err != nil { + return "", fmt.Errorf("cannot read which services start at boot: %w", err) + } + for _, line := range strings.Split(out, "\n") { + // A line looks like ` docker | default`. The name is the first field. + name, _, _ := strings.Cut(strings.TrimSpace(line), "|") + if strings.TrimSpace(name) == unit { + return "enabled", nil + } + } + return "disabled", nil +} + +func (alpine) SetServiceBoot(ctx context.Context, run Runner, unit, boot string) error { + verb := "add" + if boot == "disabled" { + verb = "del" + } + _, err := run(ctx, "rc-update", verb, unit, "default") + return err +} + +func errText(err error) string { + if err == nil { + return "" + } + return err.Error() +} + +// CreateUser makes a login with busybox adduser, whose flags are not useradd's. +// +// `-D` is "do not ask for a password", which is what makes it usable without a terminal. A login +// created this way has no password and cannot be logged into over the network with one, which is +// correct: what the mesh manages is what a login owns, never a way to become it. +func (alpine) CreateUser(ctx context.Context, run Runner, name, home, shell string) error { + args := []string{"-D"} + if home != "" { + args = append(args, "-h", home) + } + if shell != "" { + args = append(args, "-s", shell) + } + if _, err := run(ctx, "adduser", append(args, name)...); err != nil { + return fmt.Errorf("cannot create the user %q: %w", name, err) + } + return nil +} + +func (alpine) SetUserShell(ctx context.Context, run Runner, name, shell string) error { + // busybox has no usermod. `sed`-ing /etc/passwd is what the distribution's own tooling does, + // and chsh is the one command that exists for it everywhere. + if _, err := run(ctx, "chsh", "-s", shell, name); err != nil { + return fmt.Errorf("cannot set %q's shell to %q: %w", name, shell, err) + } + return nil +} + +// AddUserToGroup uses addgroup, which on busybox takes the user and the group and is additive by +// construction — there is no form of it that replaces the set. +func (alpine) AddUserToGroup(ctx context.Context, run Runner, name, group string) error { + if _, err := run(ctx, "addgroup", name, group); err != nil { + return fmt.Errorf("cannot put %q in the group %q: %w", name, group, err) + } + return nil +} diff --git a/internal/system/android.go b/internal/system/android.go new file mode 100644 index 0000000..b74e10b --- /dev/null +++ b/internal/system/android.go @@ -0,0 +1,100 @@ +package system + +import ( + "context" + "fmt" + + "github.com/novox/mesh-host/internal/declaration" +) + +// android is a partial host, and being partial is the point. +// +// It implements `file`, `directory` and `action` — the shapes that need only a filesystem and a +// way to run something — and refuses the other three. That is not a broken host: a declaration +// naming a shape this host does not implement is refused whole, the same treatment an unknown +// type gets, and the profile tells the control plane which shapes exist so it never sends one +// it cannot do (novox/hq ADR 0005). +// +// What it cannot do, and why: +// +// - **package** — there is no package manager an ordinary app may drive. Installing software +// on Android means the framework installing an APK, which is not something a process asks +// for on its own behalf. +// - **service** — Android's init reads .rc files from the system partition, which needs root +// and an unlocked bootloader. On a normal device nothing can register with it. +// - **container** — no container runtime, and no kernel access to give one. +// +// **This host is EPISODIC** (novox/hq ADR 0005). Everywhere else an init runs the launcher at +// boot and the launcher supervises the host. Android grants neither: nothing to register with +// without root, and nothing worth supervising, because a supervisor would be killed alongside +// what it supervises. +// +// So it runs when the platform allows and is killed when the platform wants the memory — and +// that is **disconnection**, which ADR 0004 already made an ordinary situation rather than an +// exception. It needs no keep-alive and no new mechanism: the store is already authoritative +// while disconnected, reconcile already happens on start, and the mesh already reports *last +// heard from* rather than alarming on silence. +// +// It also **cannot be the first node** — every step of raising a substrate is a shape it +// refuses — and its bundle says so rather than being an empty placeholder. +type android struct{} + +func (android) Name() string { return "android" } +func (android) Shapes() []declaration.Type { return portableShapes() } + +func (android) Confirm(ctx context.Context, run Runner) error { + // Ask the property service, which exists on every Android and nowhere else. A file path + // check would pass inside a chroot; this asks something only Android answers. + if _, err := run(ctx, "getprop", "ro.build.version.sdk"); err != nil { + return fmt.Errorf( + "this is the android host and the property service does not answer here. Either "+ + "this is not Android, or it is a container without it: %w", err) + } + return nil +} + +// The four below are unreachable through the ordinary path: Check refuses a declaration naming +// these shapes before anything is applied. They are here so that "unreachable" fails loudly if +// it ever stops being true, rather than a nil applier being called. + +func (a android) PackageInstalled(context.Context, Runner, string) (bool, error) { + return false, fmt.Errorf("%w: package (there is no package manager an app may drive)", ErrUnsupported) +} + +func (a android) InstallPackage(context.Context, Runner, string) error { + return fmt.Errorf("%w: package", ErrUnsupported) +} + +func (a android) ServiceState(context.Context, Runner, string) (string, error) { + return "", fmt.Errorf("%w: service (init is not reachable without root)", ErrUnsupported) +} + +func (a android) SetServiceState(context.Context, Runner, string, string) error { + return fmt.Errorf("%w: service", ErrUnsupported) +} + +func (a android) ServiceBoot(context.Context, Runner, string) (string, error) { + return "", fmt.Errorf("%w: service", ErrUnsupported) +} + +func (a android) SetServiceBoot(context.Context, Runner, string, string) error { + return fmt.Errorf("%w: service", ErrUnsupported) +} + +// Users are one of the shapes this host refuses. +// +// Android's user database belongs to the framework and is not something an ordinary app may +// write. Refused with a reason rather than attempted, the same as package, service and container +// above — and the profile says so, so the control plane never sends one. +func (android) CreateUser(context.Context, Runner, string, string, string) error { + return fmt.Errorf("this host implements no users: Android's user database belongs to the " + + "framework and is not writable by an ordinary process") +} + +func (android) SetUserShell(context.Context, Runner, string, string) error { + return fmt.Errorf("this host implements no users") +} + +func (android) AddUserToGroup(context.Context, Runner, string, string) error { + return fmt.Errorf("this host implements no users") +} diff --git a/internal/system/arch.go b/internal/system/arch.go new file mode 100644 index 0000000..80ce687 --- /dev/null +++ b/internal/system/arch.go @@ -0,0 +1,221 @@ +package system + +import ( + "context" + "fmt" + "strings" + + "github.com/novox/mesh-host/internal/declaration" +) + +// arch is pacman and systemd. +type arch struct{} + +func (arch) Name() string { return "arch" } +func (arch) Shapes() []declaration.Type { return everyShape() } + +func (a arch) Confirm(ctx context.Context, run Runner) error { + if _, err := run(ctx, "pacman", "-Q", "pacman"); err != nil { + return fmt.Errorf( + "this is the arch host and pacman does not answer here. Either this machine is not "+ + "Arch, or its package database is broken: %w", err) + } + return nil +} + +// PackageInstalled asks the package database, having first established that it answers. +// +// The two-step is the trap this file exists to remember. `pacman -Q name` exits non-zero for a +// package that is not installed AND for a database that cannot be read, so believing the first +// answer reports a broken package manager as "nothing is installed" — absence read as fact. +// Proving the tool answers about something that certainly exists separates them. +func (a arch) PackageInstalled(ctx context.Context, run Runner, name string) (bool, error) { + if err := a.Confirm(ctx, run); err != nil { + return false, fmt.Errorf("nothing can be said about %q: %w", name, err) + } + if _, err := run(ctx, "pacman", "-Q", name); err != nil { + return false, nil + } + return true, nil +} + +func (arch) InstallPackage(ctx context.Context, run Runner, name string) error { + out, err := run(ctx, "pacman", "-S", "--noconfirm", "--needed", name) + if err == nil { + return nil + } + + // **The package manager's own words, and a name for the case that looks like a bug in the + // declaration and is not.** A stale index asks the mirrors for a version they have already + // superseded and gets a 404 from every one of them — so the package exists, the declaration is + // correct, and the machine's idea of what exists is old (novox/hq 04-ISSUES/002). + // + // **It is not fixed by syncing here.** `pacman -Sy ` installs a package built against + // libraries this machine does not have: a partial upgrade, which Arch does not support and + // which breaks the machine in a way that surfaces much later as something unrelated. The + // remedy is a full upgrade, and it is a decision about the whole machine rather than + // something to do silently in the middle of applying one resource. + // + // So this says which of the two it is looking at. A declaration that is wrong and a machine + // that is out of date fail identically otherwise, and they are fixed in completely different + // places. + if staleIndex(out) { + return fmt.Errorf( + "%s could not be fetched from any mirror, which is what a stale package index looks "+ + "like: this machine is asking for a version the mirrors have replaced. The "+ + "package and the declaration are probably both fine. It is fixed by upgrading "+ + "the machine, not by this host syncing one package — that would be a partial "+ + "upgrade, which this distribution does not support.\n\n%s", + name, strings.TrimSpace(out)) + } + return fmt.Errorf("%w\n\n%s", err, strings.TrimSpace(out)) +} + +// staleIndex reports whether a failed install looks like the machine's view being old rather than +// the package being wrong. +// +// By what the package manager said, because there is nothing else to go on: the exit code is the +// same for both. +func staleIndex(out string) bool { + said := strings.ToLower(out) + if !strings.Contains(said, "failed retrieving file") && !strings.Contains(said, "404") { + return false + } + // Every mirror, not one. A single mirror failing is an ordinary transient thing and retrying + // is the answer; every one of them saying the file is gone is the index being old. + return strings.Contains(said, "error") || strings.Count(said, "404") > 1 +} + +// ServiceState reads what systemd says about a unit. +// +// Two traps, and both were hit before this read what it now reads. +// +// The exit code is not the answer: `is-active` exits non-zero for every state except active. +// +// And "inactive" does not mean stopped. `systemctl is-active` says "inactive" for a unit that +// DOES NOT EXIST exactly as it does for one installed and stopped, so declaring a unit stopped +// reported success for a unit the host cannot manage at all. LoadState is what separates them, +// so LoadState is what is read — and it is the thing an interface spanning systemd and OpenRC +// would have had to drop. +func (arch) ServiceState(ctx context.Context, run Runner, unit string) (string, error) { + out, _ := run(ctx, "systemctl", "show", unit, + "--property=LoadState", "--property=ActiveState") + + var load, active string + for _, line := range strings.Split(out, "\n") { + key, value, found := strings.Cut(strings.TrimSpace(line), "=") + if !found { + continue + } + switch key { + case "LoadState": + load = value + case "ActiveState": + active = value + } + } + + switch load { + case "": + return "", fmt.Errorf("the service manager said nothing about %s", unit) + case "not-found": + return "", fmt.Errorf( + "%s does not exist on this machine. A declaration naming a unit that is not "+ + "installed cannot be satisfied, and reporting it stopped would be reporting "+ + "absence as success", unit) + case "masked": + return "", fmt.Errorf("%s is masked, so its state cannot be declared", unit) + case "error", "bad-setting": + return "", fmt.Errorf("%s is installed but its unit file cannot be loaded (%s)", unit, load) + } + + switch active { + case "active", "activating", "reloading": + return "running", nil + case "inactive", "failed", "deactivating": + return "stopped", nil + default: + return "", fmt.Errorf( + "the service manager reports %s as %q, which is neither running nor stopped", unit, active) + } +} + +func (arch) SetServiceState(ctx context.Context, run Runner, unit, state string) error { + verb := "start" + if state == "stopped" { + verb = "stop" + } + _, err := run(ctx, "systemctl", verb, unit) + return err +} + +// ServiceBoot reads whether a unit starts at boot. +// +// `is-enabled` has more than two answers, and `static` is the one that matters: the unit has no +// install section and CANNOT be enabled. Reading it as "disabled" would have the host try, fail, +// and blame the wrong thing — the same shape as reading a missing unit as "stopped". +func (arch) ServiceBoot(ctx context.Context, run Runner, unit string) (string, error) { + out, _ := run(ctx, "systemctl", "is-enabled", unit) + switch state := strings.TrimSpace(out); state { + case "enabled", "enabled-runtime", "alias": + return "enabled", nil + case "disabled": + return "disabled", nil + case "": + return "", fmt.Errorf("the service manager said nothing about whether %s starts at boot", unit) + case "static": + return "", fmt.Errorf( + "%s is static — it has no install section, so it cannot be enabled or disabled. "+ + "Something else pulls it in, and that is what a declaration should name", unit) + case "masked", "masked-runtime": + return "", fmt.Errorf("%s is masked, so its boot state cannot be declared", unit) + default: + return "", fmt.Errorf( + "the service manager reports %s as %q at boot, which is neither enabled nor disabled", + unit, state) + } +} + +func (arch) SetServiceBoot(ctx context.Context, run Runner, unit, boot string) error { + verb := "enable" + if boot == "disabled" { + verb = "disable" + } + _, err := run(ctx, "systemctl", verb, unit) + return err +} + +// CreateUser makes a login with useradd. +// +// `--create-home` because a user whose home does not exist is a user nothing can be delivered +// to, and delivering a shell's configuration is most of why the mesh knows about users at all. +func (arch) CreateUser(ctx context.Context, run Runner, name, home, shell string) error { + args := []string{"--create-home"} + if home != "" { + args = append(args, "--home-dir", home) + } + if shell != "" { + args = append(args, "--shell", shell) + } + if _, err := run(ctx, "useradd", append(args, name)...); err != nil { + return fmt.Errorf("cannot create the user %q: %w", name, err) + } + return nil +} + +func (arch) SetUserShell(ctx context.Context, run Runner, name, shell string) error { + if _, err := run(ctx, "usermod", "--shell", shell, name); err != nil { + return fmt.Errorf("cannot set %q's shell to %q: %w", name, shell, err) + } + return nil +} + +// AddUserToGroup appends, and `--append` is the whole point: without it usermod REPLACES the +// user's supplementary groups, so a declaration naming one group would silently remove every +// other — including the ones that make a login able to use a machine at all. +func (arch) AddUserToGroup(ctx context.Context, run Runner, name, group string) error { + if _, err := run(ctx, "usermod", "--append", "--groups", group, name); err != nil { + return fmt.Errorf("cannot put %q in the group %q: %w", name, group, err) + } + return nil +} diff --git a/internal/system/shapes_test.go b/internal/system/shapes_test.go new file mode 100644 index 0000000..2305929 --- /dev/null +++ b/internal/system/shapes_test.go @@ -0,0 +1,31 @@ +package system + +import ( + "testing" + + "github.com/novox/mesh-host/internal/declaration" +) + +// Every shape in the vocabulary is implemented by a host that claims to do everything. +// +// **Added because adding a shape and forgetting this was silent here and loud there.** The +// `network` shape parsed, validated, applied and removed, and a full host still refused every +// declaration containing one — correctly, because it does not do half a declaration. The count +// test above passed throughout: it checks what the language has, not what a host can do. +func TestAFullHostImplementsEveryShapeTheLanguageHas(t *testing.T) { + does := map[declaration.Type]bool{} + for _, s := range All() { + if s.Name() != "arch" { + continue + } + for _, shape := range s.Shapes() { + does[shape] = true + } + } + for _, shape := range declaration.Vocabulary() { + if !does[shape] { + t.Errorf("the language has %q and a full host cannot apply it, so every declaration "+ + "carrying one is refused whole", shape) + } + } +} diff --git a/internal/system/system.go b/internal/system/system.go new file mode 100644 index 0000000..9508982 --- /dev/null +++ b/internal/system/system.go @@ -0,0 +1,208 @@ +// Package system is the part of the host that differs between operating systems. +// +// novox/hq ADR 0005. A machine has apk because it is Alpine; the package manager, the service +// manager and the packaging format arrive together as one decision somebody made at install +// time. So they are not independent knobs — they are one implementation, named after the system +// it belongs to. +// +// Everything else in the host is shared: the declaration vocabulary, the store, the apply loop, +// the read-back discipline, the refusal model, the link. What lives here is two appliers' worth +// of difference and the probes that go with them. +// +// Not abstracted behind a lowest common denominator, deliberately. `systemctl show` reports a +// LoadState that separates *not installed* from *stopped*, and OpenRC has no equivalent — an +// interface spanning both would have to drop it, and dropping it is how absence gets reported +// as success. Each system says what it can say. +package system + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/novox/mesh-host/internal/declaration" +) + +// Runner executes a command. The real one runs a process; tests pass one that records what was +// asked for, because what is being tested is which commands each system issues. +type Runner func(ctx context.Context, name string, args ...string) (string, error) + +// ErrUnsupported is what a system returns for a shape it cannot implement. +// +// Not an error in the ordinary sense — an Android host declining to install packages is +// correct, not broken. It is refused at the declaration rather than attempted and failed, so a +// control plane learns the difference from the profile instead of from a stack trace. +var ErrUnsupported = errors.New("this host does not implement that") + +// System is one operating system's half of the host. +type System interface { + // Name is what this host was built for: "arch", "alpine", "android". + Name() string + + // Shapes are the declaration types this host can apply. Anything else is refused whole. + Shapes() []declaration.Type + + // Confirm proves this is the system the host was built for. + // + // A host installed on the wrong machine must say so, not discover it by calling a package + // manager that is not there. The failure is legible exactly once, at start. + Confirm(ctx context.Context, run Runner) error + + PackageInstalled(ctx context.Context, run Runner, name string) (bool, error) + InstallPackage(ctx context.Context, run Runner, name string) error + + // ServiceState is "running" or "stopped". A unit that does not exist is an error, never + // "stopped" — reporting absence as satisfaction is the fault this host exists to prevent. + ServiceState(ctx context.Context, run Runner, unit string) (string, error) + SetServiceState(ctx context.Context, run Runner, unit, state string) error + + // ServiceBoot is "enabled" or "disabled" — whether the unit starts at boot. + ServiceBoot(ctx context.Context, run Runner, unit string) (string, error) + SetServiceBoot(ctx context.Context, run Runner, unit, boot string) error + + // CreateUser makes a login. Home and shell may be empty, meaning the system's own defaults — + // a declaration that says nothing about them must not impose an opinion. + CreateUser(ctx context.Context, run Runner, name, home, shell string) error + // SetUserShell changes an existing login's shell, which is what makes "zsh is my shell" + // declared state rather than a command the link may not carry. + SetUserShell(ctx context.Context, run Runner, name, shell string) error + // AddUserToGroup is additive and never removes. A machine's own groups are not the mesh's to + // know about, and a declaration that pruned them would take away what somebody set by hand. + AddUserToGroup(ctx context.Context, run Runner, name, group string) error +} + +// Login is what the machine's user database says about a login. +type Login struct { + Home string + Shell string +} + +// LookUpUser reads a login from the user database. +// +// Shared rather than per-system: `getent passwd` gives the same seven colon-separated fields +// everywhere this host runs, and a second implementation would be a second thing to get wrong in +// the same way. +// +// **Absent is an answer, an error is not.** A user database that cannot be read must not be +// reported as "no such user" — that is absence read as fact, the exact confusion this package +// takes trouble over elsewhere. `getent` exits 2 for "not found" and other codes for failures, so +// the two are distinguished rather than collapsed. +func LookUpUser(ctx context.Context, run Runner, name string) (Login, bool, error) { + out, err := run(ctx, "getent", "passwd", name) + if err != nil { + // getent's own convention: 2 means the key was not found, which is the only failure that + // means "no such user". + if strings.Contains(err.Error(), "exit status 2") { + return Login{}, false, nil + } + return Login{}, false, fmt.Errorf( + "the user database did not answer about %q, so nothing can be said about it: %w", + name, err) + } + fields := strings.Split(strings.TrimSpace(out), ":") + if len(fields) < 7 { + return Login{}, false, fmt.Errorf("the user database gave %q for %q, which is not a passwd entry", + strings.TrimSpace(out), name) + } + return Login{Home: fields[5], Shell: fields[6]}, true, nil +} + +// GroupsOf is every group a login is in. +func GroupsOf(ctx context.Context, run Runner, name string) ([]string, error) { + out, err := run(ctx, "id", "-nG", name) + if err != nil { + return nil, err + } + return strings.Fields(out), nil +} + +// Supports reports whether this host can apply a shape. +func Supports(s System, t declaration.Type) bool { + for _, shape := range s.Shapes() { + if shape == t { + return true + } + } + return false +} + +// Check refuses a declaration naming a shape this host cannot apply. +// +// Refused whole and before anything is applied, which is the same treatment an unknown type +// gets (novox/hq ADR 0005) — a host that applied the parts it understood would leave a machine +// that looks configured and is not. The reason differs and the outcome does not. +func Check(s System, d *declaration.Declaration) error { + var problems []string + seen := map[declaration.Type]bool{} + for _, r := range d.Resources { + t := r.Kind() + if Supports(s, t) || seen[t] { + continue + } + seen[t] = true + problems = append(problems, fmt.Sprintf( + "resource %q is a %s, and the %s host does not implement that shape. This host "+ + "applies %s", + r.Identity(), t, s.Name(), shapeList(s))) + } + if len(problems) > 0 { + return &declaration.RefusalError{Problems: problems} + } + return nil +} + +func shapeList(s System) string { + names := make([]string, 0, len(s.Shapes())) + for _, t := range s.Shapes() { + names = append(names, string(t)) + } + return strings.Join(names, ", ") +} + +// everyShape is what a host on a full operating system can apply. +func everyShape() []declaration.Type { + return []declaration.Type{ + declaration.TypeDirectory, declaration.TypeFile, declaration.TypeService, + declaration.TypePackage, declaration.TypeContainer, declaration.TypeAction, + declaration.TypeUser, declaration.TypeArchive, + // A network needs the same runtime a container does, so a host that can run one can make + // the other. Not in portableShapes for exactly that reason. + declaration.TypeNetwork, + } +} + +// portableShapes need only a filesystem and a way to run something. +// +// The floor. A host that can do nothing else can still do these, which is what makes a partial +// host a real thing rather than a broken one (novox/hq ADR 0005). +func portableShapes() []declaration.Type { + return []declaration.Type{ + declaration.TypeDirectory, declaration.TypeFile, declaration.TypeAction, + // An archive is a file that arrives in a bundle rather than in the declaration. It needs + // only a filesystem and a way to fetch, so a partial host can do it; a user needs a user + // database it is allowed to write, which it does not have. + declaration.TypeArchive, + } +} + +// For returns the system with this name, or an error naming the ones that exist. +func For(name string) (System, error) { + for _, s := range All() { + if s.Name() == name { + return s, nil + } + } + var names []string + for _, s := range All() { + names = append(names, s.Name()) + } + return nil, fmt.Errorf( + "this host was built for %q, which is not a system it knows. Built hosts are: %s", + name, strings.Join(names, ", ")) +} + +// All is every system the host can be built for. +func All() []System { + return []System{arch{}, alpine{}, android{}} +} diff --git a/internal/system/system_test.go b/internal/system/system_test.go new file mode 100644 index 0000000..41702ee --- /dev/null +++ b/internal/system/system_test.go @@ -0,0 +1,416 @@ +package system + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" +) + +// recorder answers a fixed map of commands and remembers what it was asked. +// +// What is being tested is which commands each system issues and how it reads the answers, so +// the commands are real command shapes taken from the tools themselves. +type recorder struct { + answers map[string]string // first two argv words -> stdout + fails map[string]bool + calls []string +} + +func (r *recorder) run(ctx context.Context, name string, args ...string) (string, error) { + r.calls = append(r.calls, strings.TrimSpace(name+" "+strings.Join(args, " "))) + + // Two keys, most specific first. `systemctl show` and `systemctl is-enabled` need telling + // apart, while `rc-service docker status` puts the UNIT where the verb would be — so a + // binary-only key is needed too. + keys := []string{name} + if len(args) > 0 { + keys = []string{name + " " + args[0], name} + } + for _, key := range keys { + if r.fails[key] { + return r.answers[key], errors.New("exit status 1") + } + if out, ok := r.answers[key]; ok { + return out, nil + } + } + return "", errors.New("exit status 127: not found") +} + +func sys(t *testing.T, name string) System { + t.Helper() + s, err := For(name) + if err != nil { + t.Fatal(err) + } + return s +} + +func TestEverySystemIsNamedAndReachable(t *testing.T) { + for _, want := range []string{"arch", "alpine", "android"} { + if _, err := For(want); err != nil { + t.Errorf("the %s host cannot be built: %v", want, err) + } + } + if _, err := For("debian"); err == nil { + t.Error("a system nobody has written was returned instead of refused") + } else if !strings.Contains(err.Error(), "arch") { + t.Errorf("the refusal does not say which hosts exist: %v", err) + } + // A host built without -X main.builtFor must refuse rather than default to something. + if _, err := For(""); err == nil { + t.Error("a host built for nothing was accepted") + } +} + +// --- what each system can do ----------------------------------------------------------------- + +func TestAndroidRefusesTheShapesItCannotDo(t *testing.T) { + // The point of a partial host: refused whole, before anything is applied, naming what this + // host does implement. Not attempted-and-failed half way through. + d, err := declaration.ParseTrusted([]byte(`{"declaration":1,"resources":[ + {"id":"f","type":"file","path":"/data/x","content":"a\n"}, + {"id":"p","type":"package","package":"docker"} + ]}`)) + if err != nil { + t.Fatal(err) + } + + refusal := Check(sys(t, "android"), d) + if refusal == nil { + t.Fatal("the android host accepted a package") + } + for _, want := range []string{"package", "android", "file", "directory", "action"} { + if !strings.Contains(refusal.Error(), want) { + t.Errorf("the refusal does not mention %q: %v", want, refusal) + } + } +} + +func TestAndroidAcceptsThePortableShapes(t *testing.T) { + // file, directory and action need only a filesystem and a way to run something. A host that + // can do nothing else can still do these, which is what makes a partial host a real one. + d, err := declaration.ParseTrusted([]byte(`{"declaration":1,"resources":[ + {"id":"d","type":"directory","path":"/data/mesh"}, + {"id":"f","type":"file","path":"/data/mesh/x","content":"a\n"}, + {"id":"a","type":"action","command":["true"],"verify":["true"]} + ]}`)) + if err != nil { + t.Fatal(err) + } + if err := Check(sys(t, "android"), d); err != nil { + t.Errorf("the android host refused a portable declaration: %v", err) + } +} + +func TestArchAndAlpineDoEveryShape(t *testing.T) { + for _, name := range []string{"arch", "alpine"} { + s := sys(t, name) + for _, shape := range everyShape() { + if !Supports(s, shape) { + t.Errorf("the %s host does not implement %s", name, shape) + } + } + } +} + +// --- confirming the machine is the one the host was built for -------------------------------- + +func TestAHostOnTheWrongMachineSaysSo(t *testing.T) { + // Installing the arch host on Alpine must fail once, at the start, rather than later inside + // a package manager that is not there. + alpineMachine := &recorder{answers: map[string]string{"apk info": "apk-tools-2.14.0\n"}} + if err := sys(t, "arch").Confirm(context.Background(), alpineMachine.run); err == nil { + t.Fatal("the arch host confirmed itself on an Alpine machine") + } else if !strings.Contains(err.Error(), "not Arch") { + t.Errorf("the failure does not say what is wrong: %v", err) + } + + archMachine := &recorder{answers: map[string]string{"pacman -Q": "pacman 7.0.0-1\n"}} + if err := sys(t, "alpine").Confirm(context.Background(), archMachine.run); err == nil { + t.Fatal("the alpine host confirmed itself on an Arch machine") + } +} + +// --- packages --------------------------------------------------------------------------------- + +func TestApkReportsAbsenceByEmptyOutputNotByExitCode(t *testing.T) { + // The difference that matters between apk and pacman, and it is invisible until it bites. + // + // pacman -Q missing -> exits NON-ZERO + // apk info -e missing -> exits ZERO and prints NOTHING + // + // So reading apk's exit code the way pacman's is read reports every package as installed. + r := &recorder{answers: map[string]string{ + "apk info": "", // installed-check for a package that is not there + }} + // apk-tools is what Confirm asks about; both go through the same key, so the empty answer + // stands in for "not installed" while the call still succeeds. + installed, err := sys(t, "alpine").PackageInstalled(context.Background(), r.run, "docker") + if err == nil && installed { + t.Error("apk's empty output was read as 'installed'") + } +} + +func TestApkFindsAnInstalledPackage(t *testing.T) { + r := &recorder{answers: map[string]string{"apk info": "docker-24.0.7-r0\n"}} + installed, err := sys(t, "alpine").PackageInstalled(context.Background(), r.run, "docker") + if err != nil { + t.Fatalf("could not ask: %v", err) + } + if !installed { + t.Error("an installed package was reported missing") + } +} + +func TestABrokenPackageDatabaseIsNotReadAsNotInstalled(t *testing.T) { + // Both systems, same trap: a package manager that cannot answer must not read as "nothing + // is installed", or the host reinstalls on a machine whose database is broken. + for _, name := range []string{"arch", "alpine"} { + r := &recorder{} // answers nothing; every command fails + if _, err := sys(t, name).PackageInstalled(context.Background(), r.run, "docker"); err == nil { + t.Errorf("%s: a broken package database was read as 'not installed'", name) + } + } +} + +// --- services ---------------------------------------------------------------------------------- + +func TestAServiceThatDoesNotExistIsNeverReportedStopped(t *testing.T) { + // The most important thing both service managers must get right, and they say it + // differently: systemd through LoadState=not-found, OpenRC in prose. + for _, tc := range []struct{ name, key, out string }{ + {"arch", "systemctl show", "LoadState=not-found\nActiveState=inactive\n"}, + {"alpine", "rc-service", " * rc-service: service `nope' does not exist\n"}, + } { + r := &recorder{answers: map[string]string{tc.key: tc.out}} + state, err := sys(t, tc.name).ServiceState(context.Background(), r.run, "nope") + if err == nil { + t.Errorf("%s: a service that does not exist was reported as %q", tc.name, state) + continue + } + // Asserted on the DIAGNOSIS, not on "does not exist" — the fall-through error echoes + // the raw output, which contains that phrase, so matching it passed even with the + // distinction removed. Only the correct branch explains why absence is not stopped. + if !strings.Contains(err.Error(), "absence as success") { + t.Errorf("%s: failed for the wrong reason: %v", tc.name, err) + } + } +} + +func TestOpenRCStateIsReadFromItsOwnWords(t *testing.T) { + for _, tc := range []struct{ out, want string }{ + {" * status: started\n", "running"}, + {" * status: stopped\n", "stopped"}, + {" * status: crashed\n", "stopped"}, // not up, so starting it is the right next act + } { + r := &recorder{answers: map[string]string{"rc-service": tc.out}} + got, err := sys(t, "alpine").ServiceState(context.Background(), r.run, "docker") + if err != nil { + t.Errorf("%q: %v", tc.out, err) + continue + } + if got != tc.want { + t.Errorf("%q read as %q, expected %q", tc.out, got, tc.want) + } + } +} + +func TestOpenRCBootStateComesFromTheRunlevel(t *testing.T) { + // OpenRC has no `is-enabled`. What it has is the runlevel listing, so "starts at boot" + // becomes "appears here". + r := &recorder{answers: map[string]string{ + "rc-update show": " docker | default\n sshd | default\n", + }} + s := sys(t, "alpine") + + got, err := s.ServiceBoot(context.Background(), r.run, "docker") + if err != nil || got != "enabled" { + t.Errorf("a service in the default runlevel read as %q (%v)", got, err) + } + got, err = s.ServiceBoot(context.Background(), r.run, "chronyd") + if err != nil || got != "disabled" { + t.Errorf("a service not in any runlevel read as %q (%v)", got, err) + } +} + +func TestEachSystemUsesItsOwnCommands(t *testing.T) { + // The whole point of ADR 0005: the alpine host must never reach for systemctl, and the arch + // host must never reach for rc-service. + for _, tc := range []struct{ name, forbidden string }{ + {"arch", "rc-service"}, + {"arch", "apk"}, + {"alpine", "systemctl"}, + {"alpine", "pacman"}, + } { + r := &recorder{answers: map[string]string{ + "systemctl show": "LoadState=loaded\nActiveState=active\n", + "rc-service": " * status: started\n", + "pacman -Q": "pacman 7.0.0\n", + "apk info": "apk-tools-2.14\n", + "rc-update show": " docker | default\n", + "systemctl is-enabled": "enabled\n", + }} + s := sys(t, tc.name) + _, _ = s.ServiceState(context.Background(), r.run, "docker") + _, _ = s.ServiceBoot(context.Background(), r.run, "docker") + _, _ = s.PackageInstalled(context.Background(), r.run, "docker") + + for _, call := range r.calls { + if strings.HasPrefix(call, tc.forbidden) { + t.Errorf("the %s host called %q", tc.name, call) + } + } + } +} + +func TestAndroidsUnreachableAppliersFailLoudly(t *testing.T) { + // Check refuses these shapes before an applier is reached, so these are unreachable — and + // they say so rather than returning a zero value, in case "unreachable" ever stops being + // true. + s := sys(t, "android") + r := &recorder{} + if _, err := s.PackageInstalled(context.Background(), r.run, "x"); !errors.Is(err, ErrUnsupported) { + t.Errorf("android's package applier did not report it as unsupported: %v", err) + } + if _, err := s.ServiceState(context.Background(), r.run, "x"); !errors.Is(err, ErrUnsupported) { + t.Errorf("android's service applier did not report it as unsupported: %v", err) + } +} + +func TestALoginThatIsNotThereIsAnAnswerAndABrokenDatabaseIsNot(t *testing.T) { + // The distinction this package takes trouble over everywhere else, applied to users. A user + // database that cannot be read must not be reported as "no such user" — absence read as + // fact is the fault the whole host exists to prevent. + notFound := func(context.Context, string, ...string) (string, error) { + return "", errors.New("exit status 2") + } + if _, exists, err := LookUpUser(context.Background(), notFound, "nobody"); err != nil { + t.Fatalf("a missing user was reported as a failure: %v", err) + } else if exists { + t.Fatal("a missing user was reported as present") + } + + broken := func(context.Context, string, ...string) (string, error) { + return "", errors.New("exit status 71: cannot read /etc/passwd") + } + if _, exists, err := LookUpUser(context.Background(), broken, "somebody"); err == nil { + t.Fatal("a broken user database was reported as an answer") + } else if exists { + t.Fatal("a broken user database reported a user as present") + } +} + +func TestALoginIsReadFromThePasswdEntry(t *testing.T) { + answering := func(context.Context, string, ...string) (string, error) { + return "worker:x:1001:1001:,,,:/home/worker:/usr/bin/zsh\n", nil + } + login, exists, err := LookUpUser(context.Background(), answering, "worker") + if err != nil || !exists { + t.Fatalf("exists=%v err=%v", exists, err) + } + if login.Home != "/home/worker" || login.Shell != "/usr/bin/zsh" { + t.Fatalf("got %+v", login) + } +} + +func TestSomethingThatIsNotAPasswdEntryIsRefused(t *testing.T) { + // Rather than read as a login with empty fields, which would have the host decide the shell + // differs and set it on every apply for ever. + nonsense := func(context.Context, string, ...string) (string, error) { + return "who knows\n", nil + } + if _, _, err := LookUpUser(context.Background(), nonsense, "worker"); err == nil { + t.Fatal("nonsense was read as a login") + } +} + +func TestAPartialHostRefusesUsersAndAllowsArchives(t *testing.T) { + // An archive needs a filesystem and a way to fetch; a user needs a user database this host is + // allowed to write, which Android does not have. + speaks := map[declaration.Type]bool{} + for _, shape := range (android{}).Shapes() { + speaks[shape] = true + } + if !speaks[declaration.TypeArchive] { + t.Error("a partial host refuses archives, which need only a filesystem") + } + if speaks[declaration.TypeUser] { + t.Error("a partial host claims to manage users") + } + if err := (android{}).CreateUser(context.Background(), nil, "a", "", ""); err == nil { + t.Error("a partial host created a user") + } +} + +// A stale index and a wrong declaration fail identically, and are fixed in completely different +// places. +// +// novox/hq 04-ISSUES/002: the package exists, the declaration is correct, and the machine is +// asking the mirrors for a version they have already replaced. Reported as a generic install +// failure it sends somebody to check the manifest, which is the one thing that is right. +func TestAStaleIndexIsNamedRatherThanReportedAsAFailedInstall(t *testing.T) { + said := "error: failed retrieving file 'dnsmasq-2.90-1-x86_64.pkg.tar.zst' from mirror.one : " + + "The requested URL returned error: 404\n" + + "error: failed retrieving file 'dnsmasq-2.90-1-x86_64.pkg.tar.zst' from mirror.two : " + + "The requested URL returned error: 404\n" + + "error: failed to commit transaction (failed to retrieve some files)" + run := func(context.Context, string, ...string) (string, error) { + return said, errors.New("exit status 1") + } + err := (arch{}).InstallPackage(context.Background(), run, "dnsmasq") + if err == nil { + t.Fatal("an install that failed reported success") + } + for _, want := range []string{"stale package index", "upgrading the machine", "partial upgrade"} { + if !strings.Contains(err.Error(), want) { + t.Fatalf("the failure does not say %q, so it reads as a wrong declaration:\n%v", want, err) + } + } + // And the package manager's own words, which were being thrown away entirely. + if !strings.Contains(err.Error(), "404") { + t.Fatalf("what the package manager said was discarded:\n%v", err) + } +} + +// An ordinary failure is not dressed up as a stale index: saying "upgrade the machine" about a +// package that does not exist sends somebody to do something large and useless. +func TestAnOrdinaryInstallFailureIsNotCalledAStaleIndex(t *testing.T) { + run := func(context.Context, string, ...string) (string, error) { + return "error: target not found: nosuchpackage", errors.New("exit status 1") + } + err := (arch{}).InstallPackage(context.Background(), run, "nosuchpackage") + if err == nil { + t.Fatal("an install that failed reported success") + } + if strings.Contains(err.Error(), "stale package index") { + t.Fatalf("a package that does not exist was called a stale index:\n%v", err) + } + if !strings.Contains(err.Error(), "target not found") { + t.Fatalf("what the package manager said was discarded:\n%v", err) + } +} + +// One mirror failing is transient and retrying is the answer. Every mirror saying the file is gone +// is the index being old. +func TestOneMirrorFailingIsNotAStaleIndex(t *testing.T) { + run := func(context.Context, string, ...string) (string, error) { + return "warning: failed retrieving file 'x.pkg.tar.zst' from mirror.one : timeout", + errors.New("exit status 1") + } + err := (arch{}).InstallPackage(context.Background(), run, "x") + if err != nil && strings.Contains(err.Error(), "stale package index") { + t.Fatalf("one mirror timing out was called a stale index:\n%v", err) + } +} + +// And an install that works still works. +func TestAnInstallThatSucceedsSaysNothing(t *testing.T) { + run := func(context.Context, string, ...string) (string, error) { return "installed", nil } + if err := (arch{}).InstallPackage(context.Background(), run, "dnsmasq"); err != nil { + t.Fatalf("a successful install reported a failure: %v", err) + } +} diff --git a/internal/upgrade/upgrade.go b/internal/upgrade/upgrade.go new file mode 100644 index 0000000..a8d2647 --- /dev/null +++ b/internal/upgrade/upgrade.go @@ -0,0 +1,159 @@ +// Package upgrade is how the host survives replacing itself. +// +// novox/hq ADR 0005. Two facts, and neither is the host judging its own health: +// +// - whether the executable this process started from has been replaced on disk, which is how +// it knows to stand aside for a new one; +// - which version last got as far as a completed reconcile, which is what a rollback outside +// this binary reads when this binary will not start. +// +// The second is written for a reader that is not the host. A binary that cannot start cannot be +// its own recovery, so what it leaves behind has to be plain enough for a shell script. +package upgrade + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" +) + +// Files the launcher reads and this binary writes. Next to the store, because they are node +// state of exactly the same kind. +const ( + KnownGoodName = "known-good" + AttemptsName = "start-attempts" +) + +// Self is the executable this process started from, remembered. +// +// Identity is taken once, at start, and compared later. The obvious alternative — asking +// /proc/self/exe whether it is marked deleted — was tried and is worse in two ways: it is Linux +// procfs behaviour rather than a fact about files, and it catches only *unlink*, so a binary +// swapped by rename onto the same path reads as untouched. Remembering what we started from +// needs no special filesystem and misses neither case. +type Self struct { + path string + info os.FileInfo +} + +// Current captures the running executable's identity. +// +// path is what os.Executable() returned; a test passes one it can manipulate, because the +// boundary being tested is the filesystem and a fake would assert that the fake behaves as +// expected (novox/hq ADR 0017). +func Current(path string) (Self, error) { + info, err := os.Stat(path) + if err != nil { + return Self{}, fmt.Errorf( + "cannot stat %s, so this host cannot tell whether it is later replaced: %w", path, err) + } + return Self{path: path, info: info}, nil +} + +// Path is where the executable was when this process started. +func (s Self) Path() string { return s.path } + +// Replaced reports whether a different file is at that path now, or none. +// +// Never a silent false: a host that cannot read its own image says so rather than assuming it is +// current, which is the shape of every fault this repository catalogues. +func (s Self) Replaced() (bool, error) { + if s.info == nil { + return false, errors.New("this host never captured its own identity, so it cannot tell " + + "whether it has been replaced") + } + + now, err := os.Stat(s.path) + if errors.Is(err, os.ErrNotExist) { + // Removed rather than upgraded. Still not what is running, and saying "unchanged" + // would leave the host claiming a version that is no longer installed. + return true, nil + } + if err != nil { + return false, err + } + + return !os.SameFile(s.info, now), nil +} + +// KnownGoodPath is where the marker lives, given where the store lives. +func KnownGoodPath(statePath string) string { + return filepath.Join(filepath.Dir(statePath), KnownGoodName) +} + +// AttemptsPath is where the launcher counts starts that have not yet worked. +func AttemptsPath(statePath string) string { + return filepath.Join(filepath.Dir(statePath), AttemptsName) +} + +// ClearAttempts tells the launcher this start worked. +// +// Written at the same moment as known-good and for the same reason: a completed reconcile is +// the evidence, and it is the only evidence either of them has. Without this the counter only +// ever climbs, so a node that has been up for months rolls itself back on its third ordinary +// restart — a healthy machine undone by its own recovery. +func ClearAttempts(path string) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + return os.WriteFile(path, []byte("0\n"), 0o644) +} + +// RecordKnownGood marks a version as one that started and completed a reconcile. +// +// Written atomically and as one bare line. The reader is a shell script running on a machine +// where the host is failing to start, so the format is the least it can be: no JSON, no +// escaping, nothing that needs a parser to be present and working. +func RecordKnownGood(path, version string) error { + if strings.TrimSpace(version) == "" { + return errors.New("refusing to record an empty version as known-good: a rollback " + + "reading it would install nothing and report success") + } + if strings.ContainsAny(version, "\n\r") { + return fmt.Errorf("refusing to record %q as known-good: it must be one line", version) + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + + tmp, err := os.CreateTemp(filepath.Dir(path), ".known-good-*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + + if _, err := fmt.Fprintln(tmp, version); err != nil { + tmp.Close() + return err + } + if err := tmp.Sync(); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + if err := os.Chmod(tmp.Name(), 0o644); err != nil { + return err + } + return os.Rename(tmp.Name(), path) +} + +// ReadKnownGood returns the recorded version, or "" if there has never been one. +// +// Absence is not an error. A machine whose host has never completed a reconcile has no version +// to go back to, and that is a real state rather than a fault: the node was never working, so +// the failure belongs to the installation and not to an upgrade. A rollback that guessed here +// would become a second fault. +func ReadKnownGood(path string) (string, error) { + raw, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return "", nil + } + if err != nil { + return "", err + } + return strings.TrimSpace(string(raw)), nil +} diff --git a/internal/upgrade/upgrade_test.go b/internal/upgrade/upgrade_test.go new file mode 100644 index 0000000..ecf3e63 --- /dev/null +++ b/internal/upgrade/upgrade_test.go @@ -0,0 +1,193 @@ +package upgrade + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// started puts a binary on disk and captures it the way the host does at start. +// +// Against the real filesystem rather than a fake one. What is being tested is how the operating +// system behaves when a file is replaced under a running process, and a fake would assert that +// the fake behaves as expected (novox/hq ADR 0017). +func started(t *testing.T) (Self, string) { + t.Helper() + binary := filepath.Join(t.TempDir(), "mesh-host") + if err := os.WriteFile(binary, []byte("version one"), 0o755); err != nil { + t.Fatal(err) + } + self, err := Current(binary) + if err != nil { + t.Fatal(err) + } + return self, binary +} + +func TestAnUntouchedBinaryIsNotReplaced(t *testing.T) { + // The case that runs every ten minutes forever. A false positive here is a node that exits + // and restarts on every reconcile — a restart loop dressed as an upgrade. + self, _ := started(t) + + replaced, err := self.Replaced() + if err != nil { + t.Fatalf("could not tell: %v", err) + } + if replaced { + t.Error("an untouched binary was reported as replaced; this host would restart forever") + } +} + +func TestRewritingTheSameFileIsNotAReplacement(t *testing.T) { + // Touching content in place keeps the inode, and a package manager does not install this + // way — but something else on the machine might. The claim is about identity, not content. + self, binary := started(t) + + f, err := os.OpenFile(binary, os.O_WRONLY, 0o755) + if err != nil { + t.Fatal(err) + } + if _, err := f.WriteString("same inode, new bytes"); err != nil { + t.Fatal(err) + } + f.Close() + + replaced, err := self.Replaced() + if err != nil { + t.Fatalf("could not tell: %v", err) + } + if replaced { + t.Error("writing through the same inode was reported as a replacement") + } +} + +func TestInstallingOverTheBinaryIsAReplacement(t *testing.T) { + // What a package manager actually does: write a new file and rename it over the old one. + // The running process keeps the old inode; the path now holds a different file. + self, binary := started(t) + + next := binary + ".new" + if err := os.WriteFile(next, []byte("version two"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Rename(next, binary); err != nil { + t.Fatal(err) + } + + replaced, err := self.Replaced() + if err != nil { + t.Fatalf("could not tell: %v", err) + } + if !replaced { + t.Error("a binary replaced by rename was not noticed; this host would keep running the " + + "old version and report the new one") + } +} + +func TestRemovingTheBinaryIsAReplacement(t *testing.T) { + // A package removed rather than upgraded. Nothing is at the path, and the honest answer is + // still "not what I am running" — reporting unchanged would leave the host claiming a + // version that is no longer installed. + self, binary := started(t) + if err := os.Remove(binary); err != nil { + t.Fatal(err) + } + + replaced, err := self.Replaced() + if err != nil { + t.Fatalf("could not tell: %v", err) + } + if !replaced { + t.Error("a removed binary was reported as unchanged") + } +} + +func TestNotBeingAbleToTellIsAnError(t *testing.T) { + // Never a silent false. A host that cannot read its own image must say so rather than + // assume it is current, which is the shape of every fault this repository catalogues. + if _, err := Current(filepath.Join(t.TempDir(), "no-such-binary")); err == nil { + t.Fatal("capturing a nonexistent executable returned an identity instead of an error") + } + // And a Self that was never captured must refuse rather than answer. + if _, err := (Self{}).Replaced(); err == nil { + t.Fatal("an uncaptured Self answered instead of refusing") + } +} + +func TestKnownGoodRoundTrips(t *testing.T) { + path := KnownGoodPath(filepath.Join(t.TempDir(), "state.json")) + + if err := RecordKnownGood(path, "1.4.2"); err != nil { + t.Fatalf("could not record: %v", err) + } + got, err := ReadKnownGood(path) + if err != nil { + t.Fatalf("could not read back: %v", err) + } + if got != "1.4.2" { + t.Errorf("recorded 1.4.2 and read back %q", got) + } +} + +func TestKnownGoodIsOneBareLine(t *testing.T) { + // The reader is a shell script on a machine where the host is failing to start. It must not + // need a JSON parser, and it must not need to strip anything but a newline. + path := KnownGoodPath(filepath.Join(t.TempDir(), "state.json")) + if err := RecordKnownGood(path, "1.4.2"); err != nil { + t.Fatal(err) + } + + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(raw) != "1.4.2\n" { + t.Errorf("known-good is %q; a rollback script reads this with `cat`, so it is one bare "+ + "line and nothing else", string(raw)) + } + if strings.ContainsAny(string(raw), "{}\"") { + t.Error("known-good contains structure; it must be readable without a parser") + } +} + +func TestNeverHavingBeenGoodIsNotAnError(t *testing.T) { + // A machine whose host has never completed a reconcile has nothing to go back to. That is a + // real state — the node was never working — and a rollback must be able to tell it apart + // from a read failure, because guessing a version is how recovery becomes a second fault. + path := KnownGoodPath(filepath.Join(t.TempDir(), "state.json")) + + got, err := ReadKnownGood(path) + if err != nil { + t.Fatalf("absence was reported as a failure: %v", err) + } + if got != "" { + t.Errorf("expected no known-good version, got %q", got) + } +} + +func TestAnEmptyVersionIsRefused(t *testing.T) { + // An empty known-good would make the rollback script install nothing and report success — + // the exact failure the rollback exists to prevent, relocated into the rollback. + path := KnownGoodPath(filepath.Join(t.TempDir(), "state.json")) + if err := RecordKnownGood(path, ""); err == nil { + t.Fatal("an empty version was accepted as known-good") + } +} + +func TestRecordingAgainReplacesRatherThanAppends(t *testing.T) { + path := KnownGoodPath(filepath.Join(t.TempDir(), "state.json")) + for _, v := range []string{"1.4.2", "1.4.3", "1.5.0"} { + if err := RecordKnownGood(path, v); err != nil { + t.Fatal(err) + } + } + + got, err := ReadKnownGood(path) + if err != nil { + t.Fatal(err) + } + if got != "1.5.0" { + t.Errorf("after three recordings the file says %q; it holds the last one, not a history", got) + } +} diff --git a/packaging/launch_test.sh b/packaging/launch_test.sh new file mode 100755 index 0000000..014e246 --- /dev/null +++ b/packaging/launch_test.sh @@ -0,0 +1,191 @@ +#!/bin/sh +# Tests for nox-mesh-host-launch. +# +# The counter is the whole mechanism and it is the part to get wrong: never cleared and a node +# rolls back on a healthy boot; cleared too eagerly and it never rolls back at all. So the +# counter is what most of these assert. +set -eu +cd "$(dirname "$0")" +LAUNCH="$PWD/nox-mesh-host-launch" +PASS=0; FAIL=0 + +setup() { + WORK="$(mktemp -d)" + export MESH_HOST_STATE_DIR="$WORK/state" + export MESH_HOST_LIBEXEC="$WORK/libexec" + export MESH_HOST_BIN="$WORK/bin/nox-mesh-host" + export MESH_HOST_START_LIMIT=3 + export MESH_HOST_BACKOFF=0 + export MESH_HOST_RUN_ONCE=1 + mkdir -p "$MESH_HOST_STATE_DIR" "$MESH_HOST_LIBEXEC" "$WORK/bin" + + # A host that records being started. It exits immediately, which is what the launcher's + # exec makes indistinguishable from a host that ran for a week — the launcher is gone by + # then either way. + cat > "$MESH_HOST_BIN" <<'STUB' +#!/bin/sh +echo "$@" >> "$MESH_HOST_STATE_DIR/host.starts" +exit "${STUB_HOST_EXIT:-1}" +STUB + cat > "$MESH_HOST_LIBEXEC/rollback" <<'STUB' +#!/bin/sh +echo rolled-back >> "$MESH_HOST_STATE_DIR/rollback.calls" +[ -n "${STUB_ROLLBACK_FAILS:-}" ] && exit 1 +echo "$(cat "$MESH_HOST_STATE_DIR/known-good" 2>/dev/null)" > "$MESH_HOST_STATE_DIR/rollback-attempted" +exit 0 +STUB + chmod +x "$MESH_HOST_BIN" "$MESH_HOST_LIBEXEC/rollback" + unset STUB_ROLLBACK_FAILS || true +} + +# `|| true` on every launcher call above: a launcher that exits non-zero is something to +# ASSERT, not something to abort on. With `set -e` and a bare call, removing a guard from the +# launcher killed this script at the first corrupt-counter case and silently skipped the rest — +# reporting a full pass over tests that never ran. +check() { if [ "$3" = "$4" ]; then PASS=$((PASS+1)); printf ' ok %s\n' "$1" + else FAIL=$((FAIL+1)); printf ' FAIL %s\n %s\n got: %s\n expected: %s\n' "$1" "$2" "$3" "$4"; fi; } + +count() { cat "$MESH_HOST_STATE_DIR/start-attempts" 2>/dev/null || echo MISSING; } +started() { [ -f "$MESH_HOST_STATE_DIR/host.starts" ] && echo yes || echo no; } +rolled() { [ -f "$MESH_HOST_STATE_DIR/rollback.calls" ] && echo yes || echo no; } + +# --- the ordinary start ----------------------------------------------------------------------- +setup +"$LAUNCH" >/dev/null 2>&1 || true +check "starts the host" "the common case, every boot" "$(started)" "yes" +check "counts the attempt" "the counter is what decides a rollback later" "$(count)" "1" +check "does not roll back" "a first start is not a failure" "$(rolled)" "no" + +# --- failures below the limit ----------------------------------------------------------------- +setup +i=1; while [ $i -le 3 ]; do "$LAUNCH" >/dev/null 2>&1 || true; i=$((i+1)); done +check "three starts do not trigger a rollback" "the limit is exceeded, not reached" "$(rolled)" "no" +check "counts them all" "" "$(count)" "3" + +# --- past the limit --------------------------------------------------------------------------- +setup +echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good" +i=1; while [ $i -le 4 ]; do "$LAUNCH" >/dev/null 2>&1 || true; i=$((i+1)); done +check "the fourth start rolls back" "three failures is a binary that does not work" "$(rolled)" "yes" +check "and still starts the host" "the rolled-back version has to be run" "$(started)" "yes" +# Below the limit, not exactly zero. The rollback resets it and the rolled-back version then +# fails once here, so 1 is right — the property is that it did NOT inherit a count already at +# the limit, which would halt the new version on its first attempt. +check "resets the counter after rolling back" "the new version deserves its own attempts, or it halts at once" \ + "$([ "$(count)" -lt 3 ] && echo below-limit || echo "at-limit($(count))")" "below-limit" + +# --- the host clears the counter on success ---------------------------------------------------- +setup +i=1; while [ $i -le 2 ]; do "$LAUNCH" >/dev/null 2>&1 || true; i=$((i+1)); done +printf '0\n' > "$MESH_HOST_STATE_DIR/start-attempts" # what the host does on a completed reconcile +i=1; while [ $i -le 3 ]; do "$LAUNCH" >/dev/null 2>&1 || true; i=$((i+1)); done +check "a cleared counter prevents a rollback" "a node up for months must not roll back on a healthy boot" \ + "$(rolled)" "no" + +# --- rolled back once already ------------------------------------------------------------------- +setup +echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good" +echo "1.4.2" > "$MESH_HOST_STATE_DIR/rollback-attempted" +i=1; while [ $i -le 4 ]; do "$LAUNCH" >/dev/null 2>&1 || true; i=$((i+1)); done +check "does not roll back twice" "the previous version failing too means the machine, not the binary" \ + "$(rolled)" "no" +check "halts instead" "" "$([ -f "$MESH_HOST_STATE_DIR/halted" ] && echo halted || echo running)" "halted" + +# --- halted stays halted -------------------------------------------------------------------------- +setup +echo "rolled back and still failing" > "$MESH_HOST_STATE_DIR/halted" +"$LAUNCH" >/dev/null 2>&1 || true +check "a halted node does not start the host" "nothing further is tried automatically" "$(started)" "no" +set +e; "$LAUNCH" >/dev/null 2>&1; RC=$?; set -e +check "a halted node exits zero" "a supervisor loop that is slow and visible beats a crash loop" "$RC" "0" + +# --- the rollback itself fails ---------------------------------------------------------------------- +setup +echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good" +STUB_ROLLBACK_FAILS=1; export STUB_ROLLBACK_FAILS +i=1; while [ $i -le 4 ]; do "$LAUNCH" >/dev/null 2>&1 || true; i=$((i+1)); done +check "a failed rollback halts" "restarting into the same failure would loop forever" \ + "$([ -f "$MESH_HOST_STATE_DIR/halted" ] && echo halted || echo running)" "halted" +# Counted, not "was it ever started": the first three attempts DID start it, correctly, and +# only the fourth must not. An earlier version of this asserted the host was never started and +# failed for that reason rather than for a fault. +check "and does not start it on the halting attempt" "three starts, not four" \ + "$(wc -l < "$MESH_HOST_STATE_DIR/host.starts" 2>/dev/null || echo 0)" "3" + +# --- a corrupt counter ------------------------------------------------------------------------------ +# +# The values here are chosen because they DISCRIMINATE. An earlier version used +# "not-a-number", which shell arithmetic happens to evaluate to 0 — so the test passed with the +# guard removed and proved nothing. These two do not: +# +# 5x shell arithmetic errors, and under `set -e` the launcher dies without starting the host +# 0x10 is read as HEX 16 — past the limit, so a healthy node would roll back for no reason +for corrupt in "5x" "0x10" "1 2" ""; do + setup + echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good" + printf '%s\n' "$corrupt" > "$MESH_HOST_STATE_DIR/start-attempts" + "$LAUNCH" >/dev/null 2>&1 || true + # The exact number is not the property — "1 2" legitimately recovers a leading 1, while + # "5x" is rejected to 0. What must hold for every one of them is that the launcher + # survives its own state and does not read it as "past the limit". + check "corrupt counter [$corrupt]: starts the host" "the launcher must not die on its own state" \ + "$(started)" "yes" + check "corrupt counter [$corrupt]: does not roll back" "a healthy node must not roll back on a bad counter" \ + "$(rolled)" "no" + check "corrupt counter [$corrupt]: counter is a sane integer" "it is written back for the next start to read" \ + "$(count | grep -cE '^[0-9]+$')" "1" +done + +# --- the loop, and shutting down ------------------------------------------------------------ +# +# These need the launcher to actually run as a supervisor rather than one iteration, so they do +# not set MESH_HOST_RUN_ONCE. + +# A host that exits 0 has upgraded itself and stood aside (novox/hq ADR 0005). The launcher must +# start it again — and must NOT count it, because it did not fail. +setup +unset MESH_HOST_RUN_ONCE +cat > "$MESH_HOST_BIN" <<'STUB' +#!/bin/sh +echo start >> "$MESH_HOST_STATE_DIR/host.starts" +# Exit 0 three times, then hang so the launcher stops looping and can be killed. +if [ "$(wc -l < "$MESH_HOST_STATE_DIR/host.starts")" -lt 3 ]; then exit 0; fi +sleep 30 +STUB +chmod +x "$MESH_HOST_BIN" +"$LAUNCH" >/dev/null 2>&1 & +LP=$! +sleep 1 +check "a clean exit restarts the host" "that is how it stands aside for a new binary" \ + "$([ "$(wc -l < "$MESH_HOST_STATE_DIR/host.starts" 2>/dev/null || echo 0)" -ge 3 ] && echo looped || echo stopped)" "looped" +# No counter file at all: nothing has failed, so nothing has been counted. +check "a clean exit is not counted as a failure" "it finished, it did not fail" "$(count)" "MISSING" + +# Shutting down: the signal must reach the host, and the launcher must wait for it rather than +# exiting and leaving the host to be killed mid-apply. +kill -TERM "$LP" 2>/dev/null +sleep 1 +check "SIGTERM stops the launcher" "a supervisor that ignores shutdown hangs the machine" \ + "$(kill -0 "$LP" 2>/dev/null && echo running || echo stopped)" "stopped" +check "and does not leave the host running" "the child must go down with it" \ + "$(pgrep -f "$MESH_HOST_BIN" >/dev/null 2>&1 && echo orphaned || echo reaped)" "reaped" + +# A crash IS counted, and the launcher keeps going. +setup +unset MESH_HOST_RUN_ONCE +export MESH_HOST_BACKOFF=0 +cat > "$MESH_HOST_BIN" <<'STUB' +#!/bin/sh +echo start >> "$MESH_HOST_STATE_DIR/host.starts" +if [ "$(wc -l < "$MESH_HOST_STATE_DIR/host.starts")" -lt 2 ]; then exit 3; fi +sleep 30 +STUB +chmod +x "$MESH_HOST_BIN" +"$LAUNCH" >/dev/null 2>&1 & +LP=$! +sleep 1 +check "a crash is counted" "unlike a clean exit, which is not" "$(count)" "1" +kill -TERM "$LP" 2>/dev/null; sleep 1; pkill -f "$MESH_HOST_BIN" 2>/dev/null || true + +printf '\nlaunch: %d passed, %d failed\n' "$PASS" "$FAIL" +[ "$FAIL" -eq 0 ] diff --git a/packaging/nox-mesh-host-launch b/packaging/nox-mesh-host-launch new file mode 100755 index 0000000..b2a5495 --- /dev/null +++ b/packaging/nox-mesh-host-launch @@ -0,0 +1,129 @@ +#!/bin/sh +# Supervise the host: start it, watch it, and decide what to do when it stops. +# +# novox/hq ADR 0005. The init is asked for ONE thing — run this at boot — and everything else +# lives here, in a script that can be tested. Whether to restart, how long to wait, when to give +# up, when to roll back: all of it is policy, and policy in a unit file can only be read and +# hoped for. +# +# It does NOT exec the host. Exec would replace this process, and then only the init could +# restart anything — which is the arrangement this exists to remove. The cost of staying is +# signal handling, below. +# +# POSIX sh. `set -e` is deliberately absent: this script's whole job is to inspect exit codes, +# and -e would make it exit on the first one it is meant to handle. +set -u + +STATE_DIR="${MESH_HOST_STATE_DIR:-/var/lib/mesh-host}" +LIBEXEC="${MESH_HOST_LIBEXEC:-/usr/lib/nox-mesh-host}" +HOST="${MESH_HOST_BIN:-/usr/bin/nox-mesh-host}" +LIMIT="${MESH_HOST_START_LIMIT:-3}" +BACKOFF="${MESH_HOST_BACKOFF:-5}" +ONCE="${MESH_HOST_RUN_ONCE:-}" # tests run one iteration; nothing else sets this + +ATTEMPTS="$STATE_DIR/start-attempts" +HALTED="$STATE_DIR/halted" + +say() { echo "nox-mesh-host-launch: $*" >&2; } + +child= +stopping= + +# The machine is shutting down. Pass it on and wait for the host to finish — a supervisor that +# exits while its child is still running leaves the host to be killed rather than to stop, and +# an apply interrupted that way is exactly the half-configured machine this project is about. +on_term() { + stopping=yes + if [ -n "$child" ]; then + say "stopping: passing the signal to the host" + kill -TERM "$child" 2>/dev/null + fi +} +trap on_term TERM INT + +mkdir -p "$STATE_DIR" + +while :; do + if [ -n "$stopping" ]; then + exit 0 + fi + + if [ -e "$HALTED" ]; then + say "halted: $(cat "$HALTED" 2>/dev/null || echo 'reason not recorded')" + say "not starting the host. this node needs a person." + exit 0 + fi + + # Consecutive failed starts, not starts. Cleared by the host itself when it completes a + # reconcile, which is the only evidence either this or known-good has. + # + # Read the FIRST FIELD, then insist it is a plain integer. + # + # Stripping whitespace instead concatenates, and that is not hypothetical: a counter + # holding "1 2" became "12", past the limit, so a healthy node rolled itself back. An + # unreadable counter must fail towards "start normally", never towards "give up". + count=0 + if [ -s "$ATTEMPTS" ]; then + read -r count _ < "$ATTEMPTS" 2>/dev/null || count=0 + fi + case "${count:-}" in + '' | *[!0-9]*) count=0 ;; + esac + + if [ "$count" -ge "$LIMIT" ]; then + if [ -e "$STATE_DIR/rollback-attempted" ]; then + say "the host failed $count times after a rollback. the previous version does not" + say "start either, so this is the machine and not the binary." + printf 'rolled back and still failing\n' > "$HALTED" + exit 0 + fi + + say "the host failed $count times. rolling back." + if "$LIBEXEC/rollback"; then + # Fresh count for the version just installed: it deserves its own attempts, and + # without this it inherits a count already over the limit and halts at once. + # + # The variable too, not only the file. Resetting one and not the other made the + # next failure count from the OLD value — so the rolled-back version got one + # attempt instead of three. + count=0 + printf '%s\n' "$count" > "$ATTEMPTS" + else + say "rollback failed. halting rather than restarting into the same failure." + printf 'rollback failed\n' > "$HALTED" + exit 0 + fi + fi + + "$HOST" run & + child=$! + status=0 + wait "$child" || status=$? + child= + + if [ -n "$stopping" ]; then + exit 0 + fi + + # A signal the host did not survive, and we are not shutting down: treat it as a crash. + case "$status" in + 0) + # Exited cleanly. That is how the host stands aside for a new binary after an + # upgrade (novox/hq ADR 0005) — so loop and run whatever is now on disk. + # + # Deliberately NOT counted, and this is the whole reason the counter is + # incremented here rather than before the start: counting attempts meant a host + # that upgraded itself three times rolled itself back, having worked perfectly + # every time. + say "the host exited cleanly; starting it again" + continue + ;; + esac + + count=$((count + 1)) + printf '%s\n' "$count" > "$ATTEMPTS" + + say "the host exited $status ($count consecutive); restarting in ${BACKOFF}s" + [ -n "$ONCE" ] && exit "$status" + sleep "$BACKOFF" +done diff --git a/packaging/nox-mesh-host-network.sh b/packaging/nox-mesh-host-network.sh new file mode 100755 index 0000000..31eff2c --- /dev/null +++ b/packaging/nox-mesh-host-network.sh @@ -0,0 +1,22 @@ +#!/bin/sh +# Rouse the host when this machine's network changes. +# +# Installed as a NetworkManager dispatcher script (/etc/NetworkManager/dispatcher.d) and as a +# networkd-dispatcher one. Both hand the interface and the event as arguments; both are ignored +# beyond the event, because *which* interface changed does not matter — what matters is that a +# connection opened over the old route is now pointing at nothing, and that is true whichever +# interface it was. +# +# A machine that moves from wifi to ethernet holds a socket that looks perfectly healthy from +# inside the process: no error, no close, because nothing has tried to send anything. Heartbeats +# find it twenty or thirty seconds later. The machine knew immediately. +set -eu + +event="${2:-}" +case "$event" in + up|dhcp4-change|dhcp6-change|connectivity-change|routes-change) + # Only events that can change where packets go. `down` is deliberately not one: the link is + # already gone, reconnecting will fail, and the backoff exists for exactly that. + systemctl start --no-block nox-mesh-host-roused.service 2>/dev/null || true + ;; +esac diff --git a/packaging/nox-mesh-host-resume.service b/packaging/nox-mesh-host-resume.service new file mode 100644 index 0000000..bba7d2b --- /dev/null +++ b/packaging/nox-mesh-host-resume.service @@ -0,0 +1,14 @@ +# Rouse the host when this machine wakes. +# +# After sleep.target rather than before: the point is to act once the machine is back, and a +# signal sent on the way down would be read by a process that is about to be frozen with it. +[Unit] +Description=Rouse the Novox Mesh host after resume +After=suspend.target hibernate.target hybrid-sleep.target suspend-then-hibernate.target + +[Service] +Type=oneshot +ExecStart=/bin/sh -c 'systemctl start --no-block nox-mesh-host-roused.service' + +[Install] +WantedBy=suspend.target hibernate.target hybrid-sleep.target suspend-then-hibernate.target diff --git a/packaging/nox-mesh-host-rollback b/packaging/nox-mesh-host-rollback new file mode 100755 index 0000000..f0608a3 --- /dev/null +++ b/packaging/nox-mesh-host-rollback @@ -0,0 +1,65 @@ +#!/bin/sh +# Put the host back on the last version that worked. +# +# novox/hq ADR 0005. This runs when nox-mesh-host will not start, so it shares no code with it +# and calls none of it: a binary that cannot start cannot be its own recovery. POSIX sh, no +# bashisms, nothing that has to be installed. +# +# It is deliberately dull. Everything it does is one of: read a file, run the package manager, +# ask the service manager to try again. +set -eu + +STATE_DIR="${MESH_HOST_STATE_DIR:-/var/lib/mesh-host}" +PKG_CACHE="${MESH_HOST_PKG_CACHE:-/var/cache/pacman/pkg}" +PACKAGE="${MESH_HOST_PACKAGE:-nox-mesh-host}" + +KNOWN_GOOD="$STATE_DIR/known-good" +ATTEMPTED="$STATE_DIR/rollback-attempted" + +say() { echo "nox-mesh-host-rollback: $*" >&2; } + +# Roll back once. A second failure is a different diagnosis: the previously working binary also +# does not run, so the binary is not the problem — the machine is. Rolling back again would flap +# between two versions forever and bury the actual cause under a loop. +if [ -e "$ATTEMPTED" ]; then + say "already rolled back once, to $(cat "$ATTEMPTED" 2>/dev/null || echo unknown)." + say "the previous version also failed to start, so this is the machine and not the binary." + say "not rolling back again. this node needs a person." + exit 0 +fi + +# A machine whose host never completed a reconcile has no version to go back to. That is a real +# state rather than a fault: the node was never working, so the failure belongs to the +# installation. Guessing a version here is how a recovery becomes a second fault. +if [ ! -s "$KNOWN_GOOD" ]; then + say "no known-good version recorded — this host has never completed a reconcile." + say "there is nothing to roll back to. this is an installation failure, not an upgrade one." + exit 0 +fi + +VERSION="$(tr -d '[:space:]' < "$KNOWN_GOOD")" +if [ -z "$VERSION" ]; then + say "known-good is empty. refusing to guess." + exit 0 +fi + +PKG="$(ls "$PKG_CACHE"/"$PACKAGE"-"$VERSION"-*.pkg.tar.* 2>/dev/null | head -n 1 || true)" +if [ -z "$PKG" ]; then + say "known-good is $VERSION and no package for it is in $PKG_CACHE." + say "the cache was cleaned, or that version was never installed from here." + say "cannot roll back. this node needs a person." + exit 1 +fi + +say "rolling back to $VERSION ($PKG)" +printf '%s\n' "$VERSION" > "$ATTEMPTED" + +if ! pacman -U --noconfirm "$PKG"; then + say "the package manager refused to install $PKG." + exit 1 +fi + +# Deliberately does NOT start anything. The launcher called this and will exec the host next, +# so starting it here would run two. novox/hq ADR 0005 moved that responsibility; this script +# installs a version and says so, and nothing else. +say "rolled back to $VERSION. the launcher will start it." diff --git a/packaging/nox-mesh-host-roused.service b/packaging/nox-mesh-host-roused.service new file mode 100644 index 0000000..807cf77 --- /dev/null +++ b/packaging/nox-mesh-host-roused.service @@ -0,0 +1,17 @@ +# Tell the running host that this machine's link is probably stale. +# +# **A laptop knows it just woke; the link does not.** After a resume the socket looks perfectly +# healthy from inside the process — no error, no close, because nothing has tried to send +# anything. Heartbeats discover it twenty or thirty seconds later, and for that time the node +# believes it is in a mesh it has left. +# +# A signal rather than anything that listens: nothing may listen on a node (novox/hq ADR 0004), +# and a socket for this would be a control surface on every machine in exchange for saving twenty +# seconds. +[Unit] +Description=Tell the Novox Mesh host its link may be stale + +[Service] +Type=oneshot +# Nothing to do if the host is not running: this is a hint to a process, not a way to start one. +ExecStart=/bin/sh -c 'systemctl is-active --quiet nox-mesh-host.service && systemctl kill --signal=SIGHUP nox-mesh-host.service || true' diff --git a/packaging/nox-mesh-host.openrc b/packaging/nox-mesh-host.openrc new file mode 100644 index 0000000..cc740f0 --- /dev/null +++ b/packaging/nox-mesh-host.openrc @@ -0,0 +1,8 @@ +#!/sbin/openrc-run +# The Alpine equivalent of the systemd unit beside this. Four lines of the same two facts: +# run the launcher, and bring it back if it dies. Everything else is in the launcher, which is +# what makes a second init transcription rather than a port (novox/hq ADR 0005). +name="nox-mesh-host" +command="/usr/lib/nox-mesh-host/launch" +supervisor="supervise-daemon" +depend() { need net; } diff --git a/packaging/nox-mesh-host.service b/packaging/nox-mesh-host.service new file mode 100644 index 0000000..c9c6df1 --- /dev/null +++ b/packaging/nox-mesh-host.service @@ -0,0 +1,17 @@ +[Unit] +Description=Novox Mesh node host +After=network-online.target +Wants=network-online.target + +# One line of policy: run the launcher at boot (novox/hq ADR 0005). Restarting the host, +# backing off, giving up and rolling back are all the launcher's, where they can be tested. +# Restart= here is a backstop for the launcher itself being killed, not the mechanism. +[Service] +ExecStart=/usr/lib/nox-mesh-host/launch +Restart=always +RestartSec=5s +StateDirectory=mesh-host +KillMode=mixed + +[Install] +WantedBy=multi-user.target diff --git a/packaging/rollback_test.sh b/packaging/rollback_test.sh new file mode 100755 index 0000000..27e71ba --- /dev/null +++ b/packaging/rollback_test.sh @@ -0,0 +1,96 @@ +#!/bin/sh +# Tests for nox-mesh-host-rollback. +# +# It runs on a machine where the host will not start, which is the one moment nobody can afford +# it to be wrong — and the one moment it is hardest to debug. So it is tested here, against a +# real filesystem, with a stub package manager that records what it was asked to do. +set -eu +cd "$(dirname "$0")" +SCRIPT="$PWD/nox-mesh-host-rollback" +PASS=0; FAIL=0 + +setup() { + WORK="$(mktemp -d)" + export MESH_HOST_STATE_DIR="$WORK/state" + export MESH_HOST_PKG_CACHE="$WORK/cache" + export MESH_HOST_PACKAGE="nox-mesh-host" + mkdir -p "$MESH_HOST_STATE_DIR" "$MESH_HOST_PKG_CACHE" "$WORK/bin" + + # Stubs on PATH. Not mocks of the script's own logic — the boundary is real commands, and + # these record the calls so a test can assert what the script asked the machine to do. + cat > "$WORK/bin/pacman" <<'STUB' +#!/bin/sh +echo "$@" >> "$MESH_HOST_STATE_DIR/pacman.calls" +[ -n "${STUB_PACMAN_FAILS:-}" ] && exit 1 +exit 0 +STUB + cat > "$WORK/bin/systemctl" <<'STUB' +#!/bin/sh +echo "$@" >> "$MESH_HOST_STATE_DIR/systemctl.calls" +exit 0 +STUB + chmod +x "$WORK/bin/pacman" "$WORK/bin/systemctl" + PATH="$WORK/bin:$PATH"; export PATH + unset STUB_PACMAN_FAILS || true +} + +check() { # name, condition-description, actual, expected + if [ "$3" = "$4" ]; then PASS=$((PASS+1)); printf ' ok %s\n' "$1" + else FAIL=$((FAIL+1)); printf ' FAIL %s\n %s\n got: %s\n expected: %s\n' "$1" "$2" "$3" "$4"; fi +} + +# --- a normal rollback --------------------------------------------------------------------- +setup +echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good" +touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst" +"$SCRIPT" >/dev/null 2>&1 +check "installs the known-good version" "pacman is asked to install the cached package" \ + "$(grep -c 'nox-mesh-host-1.4.2' "$MESH_HOST_STATE_DIR/pacman.calls" 2>/dev/null || echo 0)" "1" +# It installs and stops. The launcher execs the host next, and starting it here would run two +# (novox/hq ADR 0005). +check "does not start anything itself" "the launcher owns starting" \ + "$([ -f "$MESH_HOST_STATE_DIR/systemctl.calls" ] && echo started || echo not-started)" "not-started" +check "records that it rolled back" "the attempted marker holds the version" \ + "$(cat "$MESH_HOST_STATE_DIR/rollback-attempted" 2>/dev/null || echo MISSING)" "1.4.2" + +# --- it rolls back only once --------------------------------------------------------------- +setup +echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good" +echo "1.4.2" > "$MESH_HOST_STATE_DIR/rollback-attempted" +touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst" +"$SCRIPT" >/dev/null 2>&1 +check "does not roll back twice" "a second failure is the machine, not the binary" \ + "$([ -f "$MESH_HOST_STATE_DIR/pacman.calls" ] && echo called || echo not-called)" "not-called" + +# --- nothing to roll back to --------------------------------------------------------------- +setup +set +e; "$SCRIPT" >/dev/null 2>&1; RC=$?; set -e +check "no known-good: does nothing" "a host that never reconciled has no version to return to" \ + "$([ -f "$MESH_HOST_STATE_DIR/pacman.calls" ] && echo called || echo not-called)" "not-called" +# The exit code is asserted from a real run, not from a literal. An earlier version of this +# compared "0" to "0" and could not fail — which hid an injected fault that made the script die +# here instead of returning cleanly. +check "no known-good: exits zero" "an installation failure is not a rollback failure" "$RC" "0" + +setup +printf ' \n' > "$MESH_HOST_STATE_DIR/known-good" +"$SCRIPT" >/dev/null 2>&1 +check "blank known-good: refuses to guess" "installing nothing and reporting success is the fault this prevents" \ + "$([ -f "$MESH_HOST_STATE_DIR/pacman.calls" ] && echo called || echo not-called)" "not-called" + +# --- the cache was cleaned ------------------------------------------------------------------ +setup +echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good" +set +e; "$SCRIPT" >/dev/null 2>&1; RC=$?; set -e +check "missing package: fails loudly" "cannot roll back, and says so rather than reporting success" "$RC" "1" + +# --- the package manager refuses ------------------------------------------------------------- +setup +echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good" +touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst" +STUB_PACMAN_FAILS=1 ; export STUB_PACMAN_FAILS +set +e; "$SCRIPT" >/dev/null 2>&1; RC=$?; set -e +check "pacman fails: exits non-zero" "a failed rollback is a failure the launcher must see" "$RC" "1" + +printf '\nrollback: %d passed, %d failed\n' "$PASS" "$FAIL" +[ "$FAIL" -eq 0 ] diff --git a/packaging/roused_test.sh b/packaging/roused_test.sh new file mode 100755 index 0000000..00df906 --- /dev/null +++ b/packaging/roused_test.sh @@ -0,0 +1,42 @@ +#!/bin/sh +# The dispatcher script rouses on the events that change where packets go, and on nothing else. +# +# Checked by running it with a stub systemctl on PATH, because the thing worth testing is which +# events it acts on — a script that rouses on `down` would reconnect into a network that is gone, +# and one that rouses on nothing is the timeout it was written to avoid. +set -eu + +here=$(cd "$(dirname "$0")" && pwd) +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +mkdir -p "$work/bin" +cat > "$work/bin/systemctl" <<'STUB' +#!/bin/sh +echo "$@" >> "$ROUSED_LOG" +STUB +chmod +x "$work/bin/systemctl" +export PATH="$work/bin:$PATH" +export ROUSED_LOG="$work/rousings" +: > "$ROUSED_LOG" + +for event in up dhcp4-change connectivity-change routes-change; do + sh "$here/nox-mesh-host-network.sh" wlan0 "$event" +done +count=$(grep -c "nox-mesh-host-roused" "$ROUSED_LOG" || true) +if [ "$count" -ne 4 ]; then + echo "FAIL: four events that change where packets go roused $count time(s)" >&2 + exit 1 +fi + +: > "$ROUSED_LOG" +for event in down pre-up hostname ""; do + sh "$here/nox-mesh-host-network.sh" wlan0 "$event" +done +if [ -s "$ROUSED_LOG" ]; then + echo "FAIL: an event that does not change where packets go roused the host:" >&2 + cat "$ROUSED_LOG" >&2 + exit 1 +fi + +echo "the dispatcher rouses on route changes and nothing else"