Files
mesh-host/internal/apply/apply_test.go
T
jschoubben 827ce481f2 Somebody editing a managed file is now visible instead of mysterious
Asked how the mesh would know if somebody edited their hosts file. It would
not. The file was rewritten within five minutes and the outcome said
"updated" -- which is exactly what the mesh changing its own mind looks like.
So the change vanished, nothing anywhere said why, and the obvious thing to do
is edit it again.

The host now records a digest of what it wrote, which is enough to tell the two
apart on the next pass:

  the file matches the declaration          unchanged
  it matches what was last written          updated -- the mesh changed its mind
  it matches neither                        corrected -- somebody changed it here

The machine is put back either way, because holding it to what it was told is
the point. What changes is that it says so.

A digest rather than the content: the store is read on every reconcile and sits
beside the state on disk, and keeping every managed file twice would make it
grow with the size of the machine rather than with the number of resources.
2026-08-29 22:55:26 +02:00

1154 lines
42 KiB
Go

package apply
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/store"
"github.com/novox/mesh-host/internal/system"
)
// Each test names the decision it defends (novox/hq ADR 0017).
func parse(t *testing.T, raw string) *declaration.Declaration {
t.Helper()
d, err := declaration.Parse([]byte(raw))
if err != nil {
t.Fatalf("fixture is not a valid declaration: %v", err)
}
return d
}
// noServices refuses to run anything. Used where a test declares no services, so that a test
// which accidentally reaches the service manager fails loudly instead of passing quietly.
func noServices(context.Context, string, ...string) (string, error) {
return "", errors.New("this test declares no services and should not have run a command")
}
func TestApplyingTwiceChangesNothingTheSecondTime(t *testing.T) {
// Idempotence is what makes an apply safe to run on a schedule. Without it, a host that
// reconciles every few minutes rewrites files forever and every reader sees churn.
dir := t.TempDir()
d := parse(t, `{"declaration":1,"resources":[
{"id":"d","type":"directory","path":"`+dir+`/etc","mode":"0755"},
{"id":"f","type":"file","path":"`+dir+`/etc/a.conf","content":"hello\n","mode":"0640"}
]}`)
first, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil)
if err != nil {
t.Fatal(err)
}
if !first.Changed() {
t.Fatal("the first apply on an empty machine changed nothing")
}
second, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil)
if err != nil {
t.Fatal(err)
}
if second.Changed() {
t.Errorf("the second apply changed something: %+v", second.Outcomes)
}
}
func TestADriftedMachineIsReturned(t *testing.T) {
// The other half of idempotence, and the half that matters: converging is not "do nothing
// if the state file says it was done". The machine is read, not the record.
dir := t.TempDir()
path := filepath.Join(dir, "a.conf")
d := parse(t, `{"declaration":1,"resources":[
{"id":"f","type":"file","path":"`+path+`","content":"correct\n","mode":"0644"}
]}`)
_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte("someone edited this\n"), 0o644); err != nil {
t.Fatal(err)
}
report, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil)
if err != nil {
t.Fatal(err)
}
if !report.Changed() {
t.Fatal("a drifted file was left drifted")
}
got, _ := os.ReadFile(path)
if string(got) != "correct\n" {
t.Errorf("the file was not returned: %q", got)
}
}
func TestADroppedResourceIsRemoved(t *testing.T) {
// novox/hq ADR 0005: the host removes what it previously applied and is no longer
// declared. Removing a line from a declaration is an act with an effect.
dir := t.TempDir()
keep := filepath.Join(dir, "keep.conf")
drop := filepath.Join(dir, "drop.conf")
both := parse(t, `{"declaration":1,"resources":[
{"id":"keep","type":"file","path":"`+keep+`","content":"a\n"},
{"id":"drop","type":"file","path":"`+drop+`","content":"b\n"}
]}`)
_, state, err := Apply(context.Background(), archHost(t), both, store.State{}, store.OriginCarried, noServices, nil)
if err != nil {
t.Fatal(err)
}
one := parse(t, `{"declaration":1,"resources":[
{"id":"keep","type":"file","path":"`+keep+`","content":"a\n"}
]}`)
report, state, err := Apply(context.Background(), archHost(t), one, state, store.OriginCarried, noServices, nil)
if err != nil {
t.Fatal(err)
}
if _, err := os.Stat(drop); !errors.Is(err, os.ErrNotExist) {
t.Error("a resource dropped from the declaration was left on the machine")
}
if _, err := os.Stat(keep); err != nil {
t.Error("a declared resource was removed")
}
if _, still := state.Find("drop"); still {
t.Error("the host still believes it owns what it removed")
}
if report.Outcomes[0].Action != "removed" {
t.Errorf("removal is not reported first: %+v", report.Outcomes)
}
}
func TestNothingTheHostDidNotCreateIsTouched(t *testing.T) {
// The boundary the whole removal rule turns on. A machine has things on it the mesh did
// not put there, and a converger that treats "not declared" as "must not exist" deletes
// them. Authoritative over its own footprint; inert everywhere else.
dir := t.TempDir()
stranger := filepath.Join(dir, "not-ours.conf")
if err := os.WriteFile(stranger, []byte("someone else's\n"), 0o644); err != nil {
t.Fatal(err)
}
d := parse(t, `{"declaration":1,"resources":[
{"id":"ours","type":"file","path":"`+filepath.Join(dir, "ours.conf")+`","content":"a\n"}
]}`)
if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil); err != nil {
t.Fatal(err)
}
got, err := os.ReadFile(stranger)
if err != nil || string(got) != "someone else's\n" {
t.Error("a file the host did not create was removed or changed")
}
}
func TestARenameToTheSamePathDoesNotDeleteTheNewFile(t *testing.T) {
// Why removal happens FIRST. A resource leaving a declaration while another arrives at the
// same path is an ordinary rename; removing afterwards would delete the file just written.
dir := t.TempDir()
path := filepath.Join(dir, "shared.conf")
before := parse(t, `{"declaration":1,"resources":[
{"id":"old","type":"file","path":"`+path+`","content":"old\n"}
]}`)
_, state, err := Apply(context.Background(), archHost(t), before, store.State{}, store.OriginCarried, noServices, nil)
if err != nil {
t.Fatal(err)
}
after := parse(t, `{"declaration":1,"resources":[
{"id":"new","type":"file","path":"`+path+`","content":"new\n"}
]}`)
if _, _, err := Apply(context.Background(), archHost(t), after, state, store.OriginCarried, noServices, nil); err != nil {
t.Fatal(err)
}
got, err := os.ReadFile(path)
if err != nil {
t.Fatalf("the renamed resource is gone: %v", err)
}
if string(got) != "new\n" {
t.Errorf("content is %q, want the new one", got)
}
}
func TestAFailedStepFailsTheApply(t *testing.T) {
// novox/hq ADR 0010. And the error carries what HAD been done, because the machine is in
// whatever state the apply reached and the only honest thing to hand back is that list.
dir := t.TempDir()
blocker := filepath.Join(dir, "blocker")
if err := os.WriteFile(blocker, []byte("i am a file\n"), 0o644); err != nil {
t.Fatal(err)
}
d := parse(t, `{"declaration":1,"resources":[
{"id":"fine","type":"file","path":"`+filepath.Join(dir, "fine.conf")+`","content":"a\n"},
{"id":"doomed","type":"directory","path":"`+blocker+`"},
{"id":"never","type":"file","path":"`+filepath.Join(dir, "never.conf")+`","content":"b\n"}
]}`)
_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil)
if err == nil {
t.Fatal("an impossible resource did not fail the apply")
}
var applyErr *Error
if !errors.As(err, &applyErr) {
t.Fatalf("expected an apply error, got %T", err)
}
if applyErr.Resource != "doomed" {
t.Errorf("the failure names %q, not the resource that failed", applyErr.Resource)
}
if len(applyErr.Done.Outcomes) != 1 {
t.Errorf("the error does not carry what was already applied: %+v", applyErr.Done.Outcomes)
}
// And nothing after the failure ran.
if _, err := os.Stat(filepath.Join(dir, "never.conf")); !errors.Is(err, os.ErrNotExist) {
t.Error("the apply continued past a failure")
}
}
func TestNothingIsRecordedUntilItWorked(t *testing.T) {
// novox/hq ADR 0018. A record written before the fact restates the request in a new place
// and inherits none of the authority of having happened.
dir := t.TempDir()
blocker := filepath.Join(dir, "blocker")
if err := os.WriteFile(blocker, []byte("x\n"), 0o644); err != nil {
t.Fatal(err)
}
d := parse(t, `{"declaration":1,"resources":[
{"id":"doomed","type":"directory","path":"`+blocker+`"}
]}`)
_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil)
if err == nil {
t.Fatal("expected a failure")
}
if _, claimed := state.Find("doomed"); claimed {
t.Error("the host recorded owning something it failed to apply")
}
}
func TestAModeIsMaintainedNotJustSet(t *testing.T) {
// A permission set at creation is not a permission maintained — this repository has
// already paid for that once, with generated files left world-readable because the mode
// applied only when the file was first written.
dir := t.TempDir()
path := filepath.Join(dir, "secret.conf")
d := parse(t, `{"declaration":1,"resources":[
{"id":"f","type":"file","path":"`+path+`","content":"s\n","mode":"0600"}
]}`)
_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil)
if err != nil {
t.Fatal(err)
}
if err := os.Chmod(path, 0o666); err != nil {
t.Fatal(err)
}
report, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil)
if err != nil {
t.Fatal(err)
}
info, _ := os.Stat(path)
if info.Mode().Perm() != 0o600 {
t.Errorf("mode is %o after reconciling, want 0600", info.Mode().Perm())
}
if !report.Changed() {
t.Error("a mode that had drifted was reported as unchanged")
}
}
func TestAServiceIsReadBackNotAssumed(t *testing.T) {
// `systemctl start` returning zero says the transaction was accepted, not that the unit is
// running. A unit that starts and immediately dies satisfies the command.
started := false
run := func(ctx context.Context, name string, args ...string) (string, error) {
if args[0] == "show" {
if started {
return "LoadState=loaded\nActiveState=failed\n", nil // started, then died
}
return "LoadState=loaded\nActiveState=inactive\n", nil
}
started = true
return "", nil // `systemctl start` succeeds
}
d := parse(t, `{"declaration":1,"resources":[
{"id":"s","type":"service","unit":"doomed.service","state":"running"}
]}`)
_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil)
if err == nil {
t.Fatal("a service that died immediately was reported as running")
}
if !strings.Contains(err.Error(), "asked to be running and is stopped") {
t.Errorf("the failure does not say what was observed: %v", err)
}
}
func TestAnUnknownServiceStateIsRefusedNotGuessed(t *testing.T) {
run := func(ctx context.Context, name string, args ...string) (string, error) {
return "LoadState=loaded\nActiveState=reticent\n", nil
}
d := parse(t, `{"declaration":1,"resources":[
{"id":"s","type":"service","unit":"odd.service","state":"running"}
]}`)
_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil)
if err == nil || !strings.Contains(err.Error(), "neither running nor stopped") {
t.Errorf("an unrecognised service state was not refused: %v", err)
}
}
func TestADroppedServiceIsStoppedNotDeleted(t *testing.T) {
// The host did not install the unit and does not own the unit file — only the state it put
// the unit into.
var commands []string
run := func(ctx context.Context, name string, args ...string) (string, error) {
commands = append(commands, strings.Join(args, " "))
if args[0] == "show" {
return "LoadState=loaded\nActiveState=active\n", nil
}
return "", nil
}
state := store.State{Resources: []store.Applied{
{ID: "s", Type: "service", Target: "gone.service"},
}}
d := parse(t, `{"declaration":1,"resources":[
{"id":"other","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"}
]}`)
if _, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, run, nil); err != nil {
t.Fatal(err)
}
joined := strings.Join(commands, "; ")
if !strings.Contains(joined, "stop gone.service") {
t.Errorf("the dropped service was not stopped: %s", joined)
}
if strings.Contains(joined, "disable") || strings.Contains(joined, "mask") {
t.Errorf("the host did more than stop a unit it does not own: %s", joined)
}
}
func TestAUnitThatDoesNotExistIsNotStopped(t *testing.T) {
// Found by applying inside a raised machine. `systemctl is-active` says "inactive" for a
// unit that DOES NOT EXIST exactly as it does for one that is installed and stopped, so
// declaring a unit stopped reported success for a unit the host cannot manage at all.
//
// Absence read as satisfaction — 04-ISSUES/007 wearing a different hat, and the mirror of
// the degraded-init bug the capability detector had.
absent := func(ctx context.Context, name string, args ...string) (string, error) {
return "LoadState=not-found\nActiveState=inactive\n", nil
}
d := parse(t, `{"declaration":1,"resources":[
{"id":"s","type":"service","unit":"never-installed.service","state":"stopped"}
]}`)
_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, absent, nil)
if err == nil {
t.Fatal("a unit that does not exist was reported as satisfactorily stopped")
}
if !strings.Contains(err.Error(), "does not exist on this machine") {
t.Errorf("the failure does not say the unit is absent: %v", err)
}
if _, claimed := state.Find("s"); claimed {
t.Error("the host recorded owning a unit that is not installed")
}
}
func TestAMaskedUnitIsRefused(t *testing.T) {
// Masked means someone deliberately made it unstartable. Applying over that would undo a
// decision the host did not make and cannot see the reason for.
masked := func(ctx context.Context, name string, args ...string) (string, error) {
return "LoadState=masked\nActiveState=inactive\n", nil
}
d := parse(t, `{"declaration":1,"resources":[
{"id":"s","type":"service","unit":"masked.service","state":"running"}
]}`)
if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, masked, nil); err == nil {
t.Fatal("a masked unit was accepted")
}
}
func TestForgettingAUnitThatIsGoneDoesNotStrandTheNode(t *testing.T) {
// Found on a real machine. Removing an orphaned service runs `systemctl stop`, which fails
// when the unit no longer exists — and a failure there fails the whole apply. A host
// holding a record of an uninstalled unit could then apply NOTHING, ever, with no way out
// but editing its state by hand.
//
// Removal is idempotent for the same reason os.RemoveAll is: the desired end state is
// already true.
var stopped bool
run := func(ctx context.Context, name string, args ...string) (string, error) {
if args[0] == "show" {
return "LoadState=not-found\nActiveState=inactive\n", nil
}
stopped = true
return "", errors.New("systemctl exited 5: Unit not loaded")
}
known := store.State{Resources: []store.Applied{
{ID: "gone", Type: "service", Target: "uninstalled.service"},
}}
d := parse(t, `{"declaration":1,"resources":[
{"id":"f","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"}
]}`)
report, state, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil)
if err != nil {
t.Fatalf("a vanished unit stranded the apply: %v", err)
}
if stopped {
t.Error("the host tried to stop a unit that does not exist")
}
if _, still := state.Find("gone"); still {
t.Error("the host still believes it owns a unit that is gone")
}
// "forgotten", not "removed": the host stopped believing it owns the unit, and did not
// remove anything, because there was nothing there to remove. Reporting an effect it did
// not have would be the same class of untruth as reporting a package uninstalled.
if report.Outcomes[0].Action != "forgotten" {
t.Errorf("the vanished unit was not reported as forgotten: %+v", report.Outcomes)
}
}
// --- package, container and action (novox/hq 07-the-substrate.md, ADR 0006, ADR 0005) ---
func parseTrusted(t *testing.T, raw string) *declaration.Declaration {
t.Helper()
d, err := declaration.ParseTrusted([]byte(raw))
if err != nil {
t.Fatalf("fixture is not a valid declaration: %v", err)
}
return d
}
const pinned = "docker.io/library/postgres@sha256:" +
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
func TestABrokenPackageDatabaseIsNotReadAsNotInstalled(t *testing.T) {
// The same trap serviceState documents. `pacman -Q x` exits non-zero both for a package
// that is not installed and for a database that cannot be read — so believing the first
// answer would silently reinstall on a machine whose package manager is broken, or report
// "installed nothing" as success. The apply must fail instead.
run := func(ctx context.Context, name string, args ...string) (string, error) {
return "", errors.New("pacman: error: could not lock database")
}
d := parseTrusted(t, `{"declaration":1,"resources":[
{"id":"rt","type":"package","package":"docker"}
]}`)
_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil)
if err == nil {
t.Fatal("a broken package database was read as 'not installed'")
}
if !strings.Contains(err.Error(), "does not answer") {
t.Errorf("failed for the wrong reason: %v", err)
}
}
func TestAnInstalledPackageIsNotReinstalled(t *testing.T) {
var installed bool
run := func(ctx context.Context, name string, args ...string) (string, error) {
if args[0] == "-S" {
installed = true
}
return "docker 27.0-1\n", nil // -Q succeeds for everything
}
d := parseTrusted(t, `{"declaration":1,"resources":[
{"id":"rt","type":"package","package":"docker"}
]}`)
report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil)
if err != nil {
t.Fatalf("apply failed: %v", err)
}
if installed {
t.Error("a package that was already present was installed again")
}
if report.Changed() {
t.Errorf("an already-installed package reported a change: %+v", report.Outcomes)
}
}
func TestAPackageIsNeverUninstalled(t *testing.T) {
// Deliberate: the host cannot know what else needs the package. Uninstalling a container
// runtime because a declaration changed would stop every container on the node, and the
// machine may have had it before the mesh ever saw it. Undeclaring is not "remove it".
var uninstalled bool
run := func(ctx context.Context, name string, args ...string) (string, error) {
if len(args) > 0 && (args[0] == "-R" || args[0] == "-Rs") {
uninstalled = true
}
return "", nil
}
known := store.State{Resources: []store.Applied{
{ID: "rt", Type: "package", Target: "docker"},
}}
d := parseTrusted(t, `{"declaration":1,"resources":[
{"id":"f","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"}
]}`)
report, state, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil)
if err != nil {
t.Fatalf("dropping a package stranded the apply: %v", err)
}
if uninstalled {
t.Fatal("the host uninstalled a package")
}
if _, still := state.Find("rt"); still {
t.Error("the host still believes it owns the package")
}
// "forgotten", not "removed" — the host must not claim an effect it declined to have.
if report.Outcomes[0].Action != "forgotten" {
t.Errorf("dropping a package was not reported as forgotten: %+v", report.Outcomes[0])
}
}
func TestAnActionThatIsAlreadyTrueDoesNotRun(t *testing.T) {
// Verify is the idempotency check as well as the read-back. The host does not know what a
// database is, so "is it already there" is a question only the declaration can ask.
var ran bool
run := func(ctx context.Context, name string, args ...string) (string, error) {
if name == "create-db" {
ran = true
}
return "", nil // verify passes
}
d := parseTrusted(t, `{"declaration":1,"resources":[
{"id":"db","type":"action","command":["create-db","mesh"],"verify":["has-db","mesh"]}
]}`)
report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil)
if err != nil {
t.Fatalf("apply failed: %v", err)
}
if ran {
t.Error("an action whose verify already passed was run anyway")
}
if report.Changed() {
t.Errorf("an already-satisfied action reported a change: %+v", report.Outcomes)
}
}
func TestAnActionThatSucceedsAndDoesNothingFails(t *testing.T) {
// The whole reason verify is mandatory: a command that exits zero and has no effect is
// this repository's most expensive failure shape. Here the command "succeeds" every time
// and verify never passes.
run := func(ctx context.Context, name string, args ...string) (string, error) {
if name == "has-db" {
return "", errors.New("no such database")
}
return "", nil
}
d := parseTrusted(t, `{"declaration":1,"resources":[
{"id":"db","type":"action","command":["create-db","mesh"],"verify":["has-db","mesh"]}
]}`)
_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil)
if err == nil {
t.Fatal("an action that reported success and did nothing was accepted")
}
if !strings.Contains(err.Error(), "verify still fails") {
t.Errorf("failed for the wrong reason: %v", err)
}
if _, recorded := state.Find("db"); recorded {
t.Error("an action that did not work was recorded as applied")
}
}
func TestAnActionRunsInsideTheContainerItNames(t *testing.T) {
// Steps 2 and 3 of the bootstrap act on something inside the store's container, before
// there is any mesh to ask.
var sawExec bool
run := func(ctx context.Context, name string, args ...string) (string, error) {
if name == "docker" && args[0] == "exec" && args[1] == "store" {
sawExec = true
return "", nil
}
return "", errors.New("not run in the container")
}
d := parseTrusted(t, `{"declaration":1,"resources":[
{"id":"db","type":"action","in":"store","command":["createdb","mesh"],"verify":["psql","-lqt"]}
]}`)
if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil); err != nil {
t.Fatalf("apply failed: %v", err)
}
if !sawExec {
t.Error("an action naming a container did not run inside it")
}
}
func TestAContainerThatExitsImmediatelyFailsTheApply(t *testing.T) {
// `docker run --detach` returning an id says the container was created, not that it is
// still running. A container whose entrypoint dies satisfies the command exactly as one
// that came up does — which is the read-back rule, in the place it matters most.
run := func(ctx context.Context, name string, args ...string) (string, error) {
switch {
case args[0] == "info":
return "27.0\n", nil
case args[0] == "inspect":
return "false\t" + "", nil // exists, not running
case args[0] == "run":
return "deadbeef\n", nil
}
return "", nil
}
d := parseTrusted(t, `{"declaration":1,"resources":[
{"id":"store","type":"container","name":"store","image":"`+pinned+`"}
]}`)
_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil)
if err == nil {
t.Fatal("a container that exited immediately was reported as applied")
}
if !strings.Contains(err.Error(), "is not running") {
t.Errorf("failed for the wrong reason: %v", err)
}
if _, recorded := state.Find("store"); recorded {
t.Error("a container that is not running was recorded as applied")
}
}
func TestAContainerWhoseDeclarationChangedIsReplaced(t *testing.T) {
// A container's configuration is fixed when it is created, so any change is a replacement.
// The spec label is what makes the difference visible without diffing everything the
// runtime reports — which cannot be done reliably, because a runtime normalises what it is
// given and that is indistinguishable from drift.
d := parseTrusted(t, `{"declaration":1,"resources":[
{"id":"store","type":"container","name":"store","image":"`+pinned+`","env":{"PGDATA":"/data"}}
]}`)
want := containerSpec(d.Resources[0].(*declaration.Container))
var removed, created bool
run := func(ctx context.Context, name string, args ...string) (string, error) {
switch args[0] {
case "info":
return "27.0\n", nil
case "inspect":
if created {
return "true\t" + want, nil
}
return "true\tsome-older-spec", nil
case "rm":
removed = true
return "", nil
case "run":
created = true
return "deadbeef\n", nil
}
return "", nil
}
report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil)
if err != nil {
t.Fatalf("apply failed: %v", err)
}
if !removed || !created {
t.Fatalf("a changed container was not replaced (removed=%v created=%v)", removed, created)
}
if report.Outcomes[0].Action != "updated" {
t.Errorf("a replacement was not reported as an update: %+v", report.Outcomes[0])
}
}
func TestAContainerThatMatchesIsLeftAlone(t *testing.T) {
d := parseTrusted(t, `{"declaration":1,"resources":[
{"id":"store","type":"container","name":"store","image":"`+pinned+`","env":{"PGDATA":"/data"}}
]}`)
spec := containerSpec(d.Resources[0].(*declaration.Container))
var touched bool
run := func(ctx context.Context, name string, args ...string) (string, error) {
switch args[0] {
case "info":
return "27.0\n", nil
case "inspect":
return "true\t" + spec, nil
}
touched = true
return "", nil
}
report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil)
if err != nil {
t.Fatalf("apply failed: %v", err)
}
if touched {
t.Error("a container that already matched was restarted")
}
if report.Changed() {
t.Errorf("a matching container reported a change: %+v", report.Outcomes)
}
}
// --- boot state (novox/hq: a unit started but not enabled stops being true at the next reboot) ---
// systemctlStub answers `show` and `is-enabled` the way systemd does, and records the verbs it
// was asked to perform. Real command shapes, because the trap being tested is what systemd
// actually says rather than what a fake would.
func systemctlStub(t *testing.T, load, active, enabled string, verbs *[]string) Runner {
t.Helper()
return func(ctx context.Context, name string, args ...string) (string, error) {
switch args[0] {
case "show":
return "LoadState=" + load + "\nActiveState=" + active + "\n", nil
case "is-enabled":
// Non-zero for everything but "enabled" — the exit code says nothing useful, which
// is the whole reason this reads the output.
if enabled == "enabled" {
return enabled + "\n", nil
}
return enabled + "\n", errors.New("exit status 1")
case "enable":
*verbs = append(*verbs, "enable")
enabled = "enabled"
return "", nil
case "disable":
*verbs = append(*verbs, "disable")
enabled = "disabled"
return "", nil
case "start":
*verbs = append(*verbs, "start")
active = "active"
return "", nil
case "stop":
*verbs = append(*verbs, "stop")
active = "inactive"
return "", nil
}
return "", nil
}
}
func TestAServiceIsEnabledAtBootWhenAsked(t *testing.T) {
// The gap this closes: the host could start a unit and never make it survive a reboot, so
// the declaration reported success and stopped being true at the next power cut.
var verbs []string
d := parseTrusted(t, `{"declaration":1,"resources":[
{"id":"rt","type":"service","unit":"docker.service","state":"running","boot":"enabled"}
]}`)
report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried,
systemctlStub(t, "loaded", "inactive", "disabled", &verbs), nil)
if err != nil {
t.Fatalf("apply failed: %v", err)
}
if len(verbs) != 2 || verbs[0] != "enable" || verbs[1] != "start" {
t.Errorf("expected enable then start, got %v", verbs)
}
if report.Outcomes[0].Action != "updated" {
t.Errorf("enabling and starting was not reported as an update: %+v", report.Outcomes[0])
}
}
func TestBootIsEnabledBeforeTheUnitIsStarted(t *testing.T) {
// Order matters when an apply fails part way. Enabled-and-stopped comes back at the next
// boot; running-and-disabled does not. So the more durable half is made true first.
var verbs []string
d := parseTrusted(t, `{"declaration":1,"resources":[
{"id":"rt","type":"service","unit":"docker.service","state":"running","boot":"enabled"}
]}`)
if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried,
systemctlStub(t, "loaded", "inactive", "disabled", &verbs), nil); err != nil {
t.Fatal(err)
}
if len(verbs) < 2 || verbs[0] != "enable" {
t.Errorf("boot state was not made true first: %v", verbs)
}
}
func TestAlreadyEnabledAndRunningIsUnchanged(t *testing.T) {
var verbs []string
d := parseTrusted(t, `{"declaration":1,"resources":[
{"id":"rt","type":"service","unit":"docker.service","state":"running","boot":"enabled"}
]}`)
report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried,
systemctlStub(t, "loaded", "active", "enabled", &verbs), nil)
if err != nil {
t.Fatalf("apply failed: %v", err)
}
if len(verbs) != 0 {
t.Errorf("a unit already in the declared state was touched: %v", verbs)
}
if report.Changed() {
t.Errorf("an unchanged service reported a change: %+v", report.Outcomes)
}
}
func TestOmittingBootLeavesItAlone(t *testing.T) {
// Absent means the host asserts nothing. A machine whose operator enabled something must
// not have it silently disabled because a declaration did not mention it.
var verbs []string
d := parseTrusted(t, `{"declaration":1,"resources":[
{"id":"rt","type":"service","unit":"docker.service","state":"running"}
]}`)
if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried,
systemctlStub(t, "loaded", "inactive", "enabled", &verbs), nil); err != nil {
t.Fatal(err)
}
for _, v := range verbs {
if v == "enable" || v == "disable" {
t.Errorf("boot state was changed by a declaration that did not mention it: %v", verbs)
}
}
}
func TestAStaticUnitCannotBeEnabled(t *testing.T) {
// `static` is neither enabled nor disabled: the unit has no install section and CANNOT be
// enabled. Reading it as "disabled" would have the host try, fail, and blame the wrong
// thing — the same shape as reading a missing unit as "stopped".
var verbs []string
d := parseTrusted(t, `{"declaration":1,"resources":[
{"id":"rt","type":"service","unit":"dbus.socket","state":"running","boot":"enabled"}
]}`)
_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried,
systemctlStub(t, "loaded", "active", "static", &verbs), nil)
if err == nil {
t.Fatal("a static unit was accepted as enable-able")
}
if !strings.Contains(err.Error(), "no install section") {
t.Errorf("failed for the wrong reason: %v", err)
}
}
func TestAnUnknownBootStateIsRefusedNotGuessed(t *testing.T) {
var verbs []string
d := parseTrusted(t, `{"declaration":1,"resources":[
{"id":"rt","type":"service","unit":"x.service","state":"running","boot":"enabled"}
]}`)
_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried,
systemctlStub(t, "loaded", "active", "indirect", &verbs), nil)
if err == nil {
t.Fatal("an unrecognised boot state was guessed at instead of refused")
}
}
// --- more than one container runtime (novox/hq ADR 0005) ---
func TestTheRuntimeProbeIsPerRuntime(t *testing.T) {
// Verified against a real podman 6.1.0 before this was written:
//
// docker info --format '{{.ServerVersion}}' -> 29.7.2
// podman info --format '{{.ServerVersion}}' -> Error: can't evaluate field
// ServerVersion in type system.infoReport
// podman info --format '{{.Version.Version}}' -> 6.1.0
//
// So a single probe cannot find both, and a host that used docker's would report a machine
// with podman as having no container runtime at all.
for _, tc := range []struct {
name, present, wantProbe string
}{
{"docker", "docker", "{{.ServerVersion}}"},
{"podman", "podman", "{{.Version.Version}}"},
} {
t.Run(tc.name, func(t *testing.T) {
var probedWith string
run := func(ctx context.Context, name string, args ...string) (string, error) {
if name != tc.present {
return "", errors.New("not installed")
}
if args[0] == "info" {
probedWith = args[2]
}
return "ok\n", nil
}
got, err := containerRuntime(context.Background(), run)
if err != nil {
t.Fatalf("%s was present and was not found: %v", tc.present, err)
}
if got != tc.present {
t.Errorf("found %q, expected %q", got, tc.present)
}
if probedWith != tc.wantProbe {
t.Errorf("probed %s with %q; that template does not work on it",
tc.present, probedWith)
}
})
}
}
func TestAContainerUsesTheRuntimeTheMachineHas(t *testing.T) {
// The applier must not call `docker` on a machine that has podman. Adoption keeps what the
// machine already has (novox/hq research 012), so hardcoding one contradicts it.
var calledWith []string
run := func(ctx context.Context, name string, args ...string) (string, error) {
if name == "docker" {
return "", errors.New("not installed")
}
calledWith = append(calledWith, name)
switch args[0] {
case "info":
return "6.1.0\n", nil
case "inspect":
return "false\t\n", errors.New("no such container")
case "run":
return "deadbeef\n", nil
}
return "", nil
}
d := parseTrusted(t, `{"declaration":1,"resources":[
{"id":"store","type":"container","name":"store","image":"`+pinned+`"}
]}`)
// It will fail at read-back — the stub never reports it running — and what matters is
// WHICH binary it used getting there.
_, _, _ = Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil)
for _, c := range calledWith {
if c != "podman" {
t.Errorf("called %q on a machine that only has podman", c)
}
}
if len(calledWith) == 0 {
t.Error("nothing was called; the runtime was not found")
}
}
func TestNoRuntimeIsSaidPlainly(t *testing.T) {
// Naming what was tried, because "docker: command not found" on a machine that deliberately
// runs podman sends the reader looking for the wrong thing.
run := func(ctx context.Context, name string, args ...string) (string, error) {
return "", errors.New("not installed")
}
d := parseTrusted(t, `{"declaration":1,"resources":[
{"id":"store","type":"container","name":"store","image":"`+pinned+`"}
]}`)
_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil)
if err == nil {
t.Fatal("a machine with no container runtime applied a container")
}
for _, want := range []string{"docker", "podman", "no container runtime"} {
if !strings.Contains(err.Error(), want) {
t.Errorf("the failure does not mention %q: %v", want, err)
}
}
}
// archHost is the system these tests run against. They were written for pacman and systemd, and
// naming that is better than the implicit default it used to be.
func archHost(t *testing.T) system.System {
t.Helper()
s, err := system.For("arch")
if err != nil {
t.Fatal(err)
}
return s
}
func TestAServiceIsRestartedWhenWhatItReflectsChanges(t *testing.T) {
// A running service does not re-read its configuration. Replace the file, find the service
// already running, do nothing — and the machine keeps behaving as it did while every check
// passes, because the file is right and the service is up.
//
// That is how a third node joining a mesh left the first two carrying a network that no
// longer existed. Found in the lab; this is the shape of the fix.
dir := t.TempDir()
path := filepath.Join(dir, "thing.conf")
d := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
{"id":"conf","type":"file","path":%q,"content":"first\n","mode":"0644"},
{"id":"svc","type":"service","unit":"thing.service","state":"running","restart-on":["conf"]}
]}`, path))
var commands []string
run := recordingServices(&commands)
if _, state, err := Apply(context.Background(), archHost(t), d, store.State{},
store.OriginCarried, run, nil); err != nil {
t.Fatal(err)
} else {
// Second apply with the same content: nothing moved, so nothing restarts. A machine that
// restarted its services on every reconcile would never be steady.
commands = nil
if _, _, err := Apply(context.Background(), archHost(t), d, state,
store.OriginCarried, run, nil); err != nil {
t.Fatal(err)
}
for _, c := range commands {
if strings.Contains(c, "stop") {
t.Errorf("an unchanged declaration restarted the service: %s", c)
}
}
// Now the file changes. The service is already running and must still be restarted.
changedDecl := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
{"id":"conf","type":"file","path":%q,"content":"second\n","mode":"0644"},
{"id":"svc","type":"service","unit":"thing.service","state":"running","restart-on":["conf"]}
]}`, path))
commands = nil
if _, _, err := Apply(context.Background(), archHost(t), changedDecl, state,
store.OriginCarried, run, nil); err != nil {
t.Fatal(err)
}
var stopped, started bool
for _, c := range commands {
if strings.Contains(c, "stop thing.service") {
stopped = true
}
if strings.Contains(c, "start thing.service") {
started = true
}
}
if !stopped || !started {
t.Errorf("the file changed and the service was not restarted; commands were %v", commands)
}
}
}
func TestAServiceIsNotRestartedByAChangeItDoesNotName(t *testing.T) {
// The list is what it reflects, not everything in the declaration. A service restarted by any
// change anywhere would make every apply a fleet-wide bounce.
dir := t.TempDir()
conf := filepath.Join(dir, "thing.conf")
other := filepath.Join(dir, "unrelated")
first := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
{"id":"conf","type":"file","path":%q,"content":"same\n","mode":"0644"},
{"id":"other","type":"file","path":%q,"content":"one\n","mode":"0644"},
{"id":"svc","type":"service","unit":"thing.service","state":"running","restart-on":["conf"]}
]}`, conf, other))
var commands []string
run := recordingServices(&commands)
_, state, err := Apply(context.Background(), archHost(t), first, store.State{},
store.OriginCarried, run, nil)
if err != nil {
t.Fatal(err)
}
second := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
{"id":"conf","type":"file","path":%q,"content":"same\n","mode":"0644"},
{"id":"other","type":"file","path":%q,"content":"two\n","mode":"0644"},
{"id":"svc","type":"service","unit":"thing.service","state":"running","restart-on":["conf"]}
]}`, conf, other))
commands = nil
if _, _, err := Apply(context.Background(), archHost(t), second, state,
store.OriginCarried, run, nil); err != nil {
t.Fatal(err)
}
for _, c := range commands {
if strings.Contains(c, "stop") {
t.Errorf("a change to a file the service does not name restarted it: %s", c)
}
}
}
// recordingServices answers the way a machine with a running unit would, and remembers what it
// was asked to do — which is what a restart has to be proved by, since "running" looks the same
// before and after one.
func recordingServices(commands *[]string) Runner {
return func(_ context.Context, name string, args ...string) (string, error) {
line := name + " " + strings.Join(args, " ")
*commands = append(*commands, line)
switch {
case strings.Contains(line, "is-enabled"):
return "enabled", nil
case strings.Contains(line, "show") && strings.Contains(line, "ActiveState"):
return "LoadState=loaded\nActiveState=active\nSubState=running", nil
}
return "", nil
}
}
func TestAFileChangedOnTheMachineIsCorrectedAndSaidSo(t *testing.T) {
// The question this answers: how would anybody know somebody edited a managed file? Before
// this they would not. It was rewritten within five minutes and reported as "updated",
// which is what the mesh changing its mind looks like — so the person's change vanished and
// nothing anywhere said why. They edit it again, and again.
dir := t.TempDir()
path := filepath.Join(dir, "thing.conf")
d := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
{"id":"conf","type":"file","path":%q,"content":"from the mesh\n","mode":"0644"}
]}`, path))
_, state, err := Apply(context.Background(), archHost(t), d, store.State{},
store.OriginCarried, noServices, nil)
if err != nil {
t.Fatal(err)
}
// Somebody edits it.
if err := os.WriteFile(path, []byte("edited by hand\n"), 0o644); err != nil {
t.Fatal(err)
}
report, state, err := Apply(context.Background(), archHost(t), d, state,
store.OriginCarried, noServices, nil)
if err != nil {
t.Fatal(err)
}
if got := report.Outcomes[0].Action; got != "corrected" {
t.Errorf("a hand edit was reported as %q; the mesh cannot tell it from changing its own "+
"mind, and neither can anybody reading this", got)
}
// And it is put back, because holding the machine to what it was told is the point.
back, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if string(back) != "from the mesh\n" {
t.Errorf("the file was left as %q", back)
}
}
func TestTheMeshChangingItsMindIsNotDrift(t *testing.T) {
// The other half. A new declaration is an ordinary update and must not read as somebody
// having meddled, or every real change would look like an incident.
dir := t.TempDir()
path := filepath.Join(dir, "thing.conf")
first := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
{"id":"conf","type":"file","path":%q,"content":"one\n","mode":"0644"}
]}`, path))
second := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
{"id":"conf","type":"file","path":%q,"content":"two\n","mode":"0644"}
]}`, path))
_, state, err := Apply(context.Background(), archHost(t), first, store.State{},
store.OriginCarried, noServices, nil)
if err != nil {
t.Fatal(err)
}
report, _, err := Apply(context.Background(), archHost(t), second, state,
store.OriginCarried, noServices, nil)
if err != nil {
t.Fatal(err)
}
if got := report.Outcomes[0].Action; got != "updated" {
t.Errorf("the mesh changing what it wants was reported as %q", got)
}
}
func TestAnUntouchedFileIsStillUnchanged(t *testing.T) {
// And nothing about this makes a steady machine look busy.
dir := t.TempDir()
path := filepath.Join(dir, "thing.conf")
d := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
{"id":"conf","type":"file","path":%q,"content":"steady\n","mode":"0644"}
]}`, path))
_, state, err := Apply(context.Background(), archHost(t), d, store.State{},
store.OriginCarried, noServices, nil)
if err != nil {
t.Fatal(err)
}
report, _, err := Apply(context.Background(), archHost(t), d, state,
store.OriginCarried, noServices, nil)
if err != nil {
t.Fatal(err)
}
if got := report.Outcomes[0].Action; got != "unchanged" {
t.Errorf("an untouched file was reported as %q", got)
}
}