package apply import ( "context" "errors" "os" "path/filepath" "strings" "testing" "github.com/novox/mesh-host/internal/declaration" "github.com/novox/mesh-host/internal/store" ) // Each test names the decision it defends (novox/hq ADR 0034). func parse(t *testing.T, raw string) *declaration.Declaration { t.Helper() d, err := declaration.Parse([]byte(raw)) if err != nil { t.Fatalf("fixture is not a valid declaration: %v", err) } return d } // noServices refuses to run anything. Used where a test declares no services, so that a test // which accidentally reaches the service manager fails loudly instead of passing quietly. func noServices(context.Context, string, ...string) (string, error) { return "", errors.New("this test declares no services and should not have run a command") } func TestApplyingTwiceChangesNothingTheSecondTime(t *testing.T) { // Idempotence is what makes an apply safe to run on a schedule. Without it, a host that // reconciles every few minutes rewrites files forever and every reader sees churn. dir := t.TempDir() d := parse(t, `{"declaration":1,"resources":[ {"id":"d","type":"directory","path":"`+dir+`/etc","mode":"0755"}, {"id":"f","type":"file","path":"`+dir+`/etc/a.conf","content":"hello\n","mode":"0640"} ]}`) first, state, err := Apply(context.Background(), d, store.State{}, noServices, nil) if err != nil { t.Fatal(err) } if !first.Changed() { t.Fatal("the first apply on an empty machine changed nothing") } second, _, err := Apply(context.Background(), d, state, noServices, nil) if err != nil { t.Fatal(err) } if second.Changed() { t.Errorf("the second apply changed something: %+v", second.Outcomes) } } func TestADriftedMachineIsReturned(t *testing.T) { // The other half of idempotence, and the half that matters: converging is not "do nothing // if the state file says it was done". The machine is read, not the record. dir := t.TempDir() path := filepath.Join(dir, "a.conf") d := parse(t, `{"declaration":1,"resources":[ {"id":"f","type":"file","path":"`+path+`","content":"correct\n","mode":"0644"} ]}`) _, state, err := Apply(context.Background(), d, store.State{}, noServices, nil) if err != nil { t.Fatal(err) } if err := os.WriteFile(path, []byte("someone edited this\n"), 0o644); err != nil { t.Fatal(err) } report, _, err := Apply(context.Background(), d, state, noServices, nil) if err != nil { t.Fatal(err) } if !report.Changed() { t.Fatal("a drifted file was left drifted") } got, _ := os.ReadFile(path) if string(got) != "correct\n" { t.Errorf("the file was not returned: %q", got) } } func TestADroppedResourceIsRemoved(t *testing.T) { // novox/hq ADR 0043: the host removes what it previously applied and is no longer // declared. Removing a line from a declaration is an act with an effect. dir := t.TempDir() keep := filepath.Join(dir, "keep.conf") drop := filepath.Join(dir, "drop.conf") both := parse(t, `{"declaration":1,"resources":[ {"id":"keep","type":"file","path":"`+keep+`","content":"a\n"}, {"id":"drop","type":"file","path":"`+drop+`","content":"b\n"} ]}`) _, state, err := Apply(context.Background(), both, store.State{}, noServices, nil) if err != nil { t.Fatal(err) } one := parse(t, `{"declaration":1,"resources":[ {"id":"keep","type":"file","path":"`+keep+`","content":"a\n"} ]}`) report, state, err := Apply(context.Background(), one, state, noServices, nil) if err != nil { t.Fatal(err) } if _, err := os.Stat(drop); !errors.Is(err, os.ErrNotExist) { t.Error("a resource dropped from the declaration was left on the machine") } if _, err := os.Stat(keep); err != nil { t.Error("a declared resource was removed") } if _, still := state.Find("drop"); still { t.Error("the host still believes it owns what it removed") } if report.Outcomes[0].Action != "removed" { t.Errorf("removal is not reported first: %+v", report.Outcomes) } } func TestNothingTheHostDidNotCreateIsTouched(t *testing.T) { // The boundary the whole removal rule turns on. A machine has things on it the mesh did // not put there, and a converger that treats "not declared" as "must not exist" deletes // them. Authoritative over its own footprint; inert everywhere else. dir := t.TempDir() stranger := filepath.Join(dir, "not-ours.conf") if err := os.WriteFile(stranger, []byte("someone else's\n"), 0o644); err != nil { t.Fatal(err) } d := parse(t, `{"declaration":1,"resources":[ {"id":"ours","type":"file","path":"`+filepath.Join(dir, "ours.conf")+`","content":"a\n"} ]}`) if _, _, err := Apply(context.Background(), d, store.State{}, noServices, nil); err != nil { t.Fatal(err) } got, err := os.ReadFile(stranger) if err != nil || string(got) != "someone else's\n" { t.Error("a file the host did not create was removed or changed") } } func TestARenameToTheSamePathDoesNotDeleteTheNewFile(t *testing.T) { // Why removal happens FIRST. A resource leaving a declaration while another arrives at the // same path is an ordinary rename; removing afterwards would delete the file just written. dir := t.TempDir() path := filepath.Join(dir, "shared.conf") before := parse(t, `{"declaration":1,"resources":[ {"id":"old","type":"file","path":"`+path+`","content":"old\n"} ]}`) _, state, err := Apply(context.Background(), before, store.State{}, noServices, nil) if err != nil { t.Fatal(err) } after := parse(t, `{"declaration":1,"resources":[ {"id":"new","type":"file","path":"`+path+`","content":"new\n"} ]}`) if _, _, err := Apply(context.Background(), after, state, noServices, nil); err != nil { t.Fatal(err) } got, err := os.ReadFile(path) if err != nil { t.Fatalf("the renamed resource is gone: %v", err) } if string(got) != "new\n" { t.Errorf("content is %q, want the new one", got) } } func TestAFailedStepFailsTheApply(t *testing.T) { // novox/hq ADR 0008. And the error carries what HAD been done, because the machine is in // whatever state the apply reached and the only honest thing to hand back is that list. dir := t.TempDir() blocker := filepath.Join(dir, "blocker") if err := os.WriteFile(blocker, []byte("i am a file\n"), 0o644); err != nil { t.Fatal(err) } d := parse(t, `{"declaration":1,"resources":[ {"id":"fine","type":"file","path":"`+filepath.Join(dir, "fine.conf")+`","content":"a\n"}, {"id":"doomed","type":"directory","path":"`+blocker+`"}, {"id":"never","type":"file","path":"`+filepath.Join(dir, "never.conf")+`","content":"b\n"} ]}`) _, _, err := Apply(context.Background(), d, store.State{}, noServices, nil) if err == nil { t.Fatal("an impossible resource did not fail the apply") } var applyErr *Error if !errors.As(err, &applyErr) { t.Fatalf("expected an apply error, got %T", err) } if applyErr.Resource != "doomed" { t.Errorf("the failure names %q, not the resource that failed", applyErr.Resource) } if len(applyErr.Done.Outcomes) != 1 { t.Errorf("the error does not carry what was already applied: %+v", applyErr.Done.Outcomes) } // And nothing after the failure ran. if _, err := os.Stat(filepath.Join(dir, "never.conf")); !errors.Is(err, os.ErrNotExist) { t.Error("the apply continued past a failure") } } func TestNothingIsRecordedUntilItWorked(t *testing.T) { // novox/hq ADR 0035. A record written before the fact restates the request in a new place // and inherits none of the authority of having happened. dir := t.TempDir() blocker := filepath.Join(dir, "blocker") if err := os.WriteFile(blocker, []byte("x\n"), 0o644); err != nil { t.Fatal(err) } d := parse(t, `{"declaration":1,"resources":[ {"id":"doomed","type":"directory","path":"`+blocker+`"} ]}`) _, state, err := Apply(context.Background(), d, store.State{}, noServices, nil) if err == nil { t.Fatal("expected a failure") } if _, claimed := state.Find("doomed"); claimed { t.Error("the host recorded owning something it failed to apply") } } func TestAModeIsMaintainedNotJustSet(t *testing.T) { // A permission set at creation is not a permission maintained — this repository has // already paid for that once, with generated files left world-readable because the mode // applied only when the file was first written. dir := t.TempDir() path := filepath.Join(dir, "secret.conf") d := parse(t, `{"declaration":1,"resources":[ {"id":"f","type":"file","path":"`+path+`","content":"s\n","mode":"0600"} ]}`) _, state, err := Apply(context.Background(), d, store.State{}, noServices, nil) if err != nil { t.Fatal(err) } if err := os.Chmod(path, 0o666); err != nil { t.Fatal(err) } report, _, err := Apply(context.Background(), d, state, noServices, nil) if err != nil { t.Fatal(err) } info, _ := os.Stat(path) if info.Mode().Perm() != 0o600 { t.Errorf("mode is %o after reconciling, want 0600", info.Mode().Perm()) } if !report.Changed() { t.Error("a mode that had drifted was reported as unchanged") } } func TestAServiceIsReadBackNotAssumed(t *testing.T) { // `systemctl start` returning zero says the transaction was accepted, not that the unit is // running. A unit that starts and immediately dies satisfies the command. started := false run := func(ctx context.Context, name string, args ...string) (string, error) { if args[0] == "show" { if started { return "LoadState=loaded\nActiveState=failed\n", nil // started, then died } return "LoadState=loaded\nActiveState=inactive\n", nil } started = true return "", nil // `systemctl start` succeeds } d := parse(t, `{"declaration":1,"resources":[ {"id":"s","type":"service","unit":"doomed.service","state":"running"} ]}`) _, _, err := Apply(context.Background(), d, store.State{}, run, nil) if err == nil { t.Fatal("a service that died immediately was reported as running") } if !strings.Contains(err.Error(), "asked to be running and is stopped") { t.Errorf("the failure does not say what was observed: %v", err) } } func TestAnUnknownServiceStateIsRefusedNotGuessed(t *testing.T) { run := func(ctx context.Context, name string, args ...string) (string, error) { return "LoadState=loaded\nActiveState=reticent\n", nil } d := parse(t, `{"declaration":1,"resources":[ {"id":"s","type":"service","unit":"odd.service","state":"running"} ]}`) _, _, err := Apply(context.Background(), d, store.State{}, run, nil) if err == nil || !strings.Contains(err.Error(), "neither running nor stopped") { t.Errorf("an unrecognised service state was not refused: %v", err) } } func TestADroppedServiceIsStoppedNotDeleted(t *testing.T) { // The host did not install the unit and does not own the unit file — only the state it put // the unit into. var commands []string run := func(ctx context.Context, name string, args ...string) (string, error) { commands = append(commands, strings.Join(args, " ")) if args[0] == "show" { return "LoadState=loaded\nActiveState=active\n", nil } return "", nil } state := store.State{Resources: []store.Applied{ {ID: "s", Type: "service", Target: "gone.service"}, }} d := parse(t, `{"declaration":1,"resources":[ {"id":"other","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"} ]}`) if _, _, err := Apply(context.Background(), d, state, run, nil); err != nil { t.Fatal(err) } joined := strings.Join(commands, "; ") if !strings.Contains(joined, "stop gone.service") { t.Errorf("the dropped service was not stopped: %s", joined) } if strings.Contains(joined, "disable") || strings.Contains(joined, "mask") { t.Errorf("the host did more than stop a unit it does not own: %s", joined) } } func TestAUnitThatDoesNotExistIsNotStopped(t *testing.T) { // Found by applying inside a raised machine. `systemctl is-active` says "inactive" for a // unit that DOES NOT EXIST exactly as it does for one that is installed and stopped, so // declaring a unit stopped reported success for a unit the host cannot manage at all. // // Absence read as satisfaction — 04-ISSUES/007 wearing a different hat, and the mirror of // the degraded-init bug the capability detector had. absent := func(ctx context.Context, name string, args ...string) (string, error) { return "LoadState=not-found\nActiveState=inactive\n", nil } d := parse(t, `{"declaration":1,"resources":[ {"id":"s","type":"service","unit":"never-installed.service","state":"stopped"} ]}`) _, state, err := Apply(context.Background(), d, store.State{}, absent, nil) if err == nil { t.Fatal("a unit that does not exist was reported as satisfactorily stopped") } if !strings.Contains(err.Error(), "does not exist on this machine") { t.Errorf("the failure does not say the unit is absent: %v", err) } if _, claimed := state.Find("s"); claimed { t.Error("the host recorded owning a unit that is not installed") } } func TestAMaskedUnitIsRefused(t *testing.T) { // Masked means someone deliberately made it unstartable. Applying over that would undo a // decision the host did not make and cannot see the reason for. masked := func(ctx context.Context, name string, args ...string) (string, error) { return "LoadState=masked\nActiveState=inactive\n", nil } d := parse(t, `{"declaration":1,"resources":[ {"id":"s","type":"service","unit":"masked.service","state":"running"} ]}`) if _, _, err := Apply(context.Background(), d, store.State{}, masked, nil); err == nil { t.Fatal("a masked unit was accepted") } } func TestForgettingAUnitThatIsGoneDoesNotStrandTheNode(t *testing.T) { // Found on a real machine. Removing an orphaned service runs `systemctl stop`, which fails // when the unit no longer exists — and a failure there fails the whole apply. A host // holding a record of an uninstalled unit could then apply NOTHING, ever, with no way out // but editing its state by hand. // // Removal is idempotent for the same reason os.RemoveAll is: the desired end state is // already true. var stopped bool run := func(ctx context.Context, name string, args ...string) (string, error) { if args[0] == "show" { return "LoadState=not-found\nActiveState=inactive\n", nil } stopped = true return "", errors.New("systemctl exited 5: Unit not loaded") } known := store.State{Resources: []store.Applied{ {ID: "gone", Type: "service", Target: "uninstalled.service"}, }} d := parse(t, `{"declaration":1,"resources":[ {"id":"f","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"} ]}`) report, state, err := Apply(context.Background(), d, known, run, nil) if err != nil { t.Fatalf("a vanished unit stranded the apply: %v", err) } if stopped { t.Error("the host tried to stop a unit that does not exist") } if _, still := state.Find("gone"); still { t.Error("the host still believes it owns a unit that is gone") } // "forgotten", not "removed": the host stopped believing it owns the unit, and did not // remove anything, because there was nothing there to remove. Reporting an effect it did // not have would be the same class of untruth as reporting a package uninstalled. if report.Outcomes[0].Action != "forgotten" { t.Errorf("the vanished unit was not reported as forgotten: %+v", report.Outcomes) } } // --- package, container and action (novox/hq 07-the-substrate.md, ADR 0046, ADR 0047) --- func parseTrusted(t *testing.T, raw string) *declaration.Declaration { t.Helper() d, err := declaration.ParseTrusted([]byte(raw)) if err != nil { t.Fatalf("fixture is not a valid declaration: %v", err) } return d } const pinned = "docker.io/library/postgres@sha256:" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" func TestABrokenPackageDatabaseIsNotReadAsNotInstalled(t *testing.T) { // The same trap serviceState documents. `pacman -Q x` exits non-zero both for a package // that is not installed and for a database that cannot be read — so believing the first // answer would silently reinstall on a machine whose package manager is broken, or report // "installed nothing" as success. The apply must fail instead. run := func(ctx context.Context, name string, args ...string) (string, error) { return "", errors.New("pacman: error: could not lock database") } d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"rt","type":"package","package":"docker"} ]}`) _, _, err := Apply(context.Background(), d, store.State{}, run, nil) if err == nil { t.Fatal("a broken package database was read as 'not installed'") } if !strings.Contains(err.Error(), "does not answer") { t.Errorf("failed for the wrong reason: %v", err) } } func TestAnInstalledPackageIsNotReinstalled(t *testing.T) { var installed bool run := func(ctx context.Context, name string, args ...string) (string, error) { if args[0] == "-S" { installed = true } return "docker 27.0-1\n", nil // -Q succeeds for everything } d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"rt","type":"package","package":"docker"} ]}`) report, _, err := Apply(context.Background(), d, store.State{}, run, nil) if err != nil { t.Fatalf("apply failed: %v", err) } if installed { t.Error("a package that was already present was installed again") } if report.Changed() { t.Errorf("an already-installed package reported a change: %+v", report.Outcomes) } } func TestAPackageIsNeverUninstalled(t *testing.T) { // Deliberate: the host cannot know what else needs the package. Uninstalling a container // runtime because a declaration changed would stop every container on the node, and the // machine may have had it before the mesh ever saw it. Undeclaring is not "remove it". var uninstalled bool run := func(ctx context.Context, name string, args ...string) (string, error) { if len(args) > 0 && (args[0] == "-R" || args[0] == "-Rs") { uninstalled = true } return "", nil } known := store.State{Resources: []store.Applied{ {ID: "rt", Type: "package", Target: "docker"}, }} d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"f","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"} ]}`) report, state, err := Apply(context.Background(), d, known, run, nil) if err != nil { t.Fatalf("dropping a package stranded the apply: %v", err) } if uninstalled { t.Fatal("the host uninstalled a package") } if _, still := state.Find("rt"); still { t.Error("the host still believes it owns the package") } // "forgotten", not "removed" — the host must not claim an effect it declined to have. if report.Outcomes[0].Action != "forgotten" { t.Errorf("dropping a package was not reported as forgotten: %+v", report.Outcomes[0]) } } func TestAnActionThatIsAlreadyTrueDoesNotRun(t *testing.T) { // Verify is the idempotency check as well as the read-back. The host does not know what a // database is, so "is it already there" is a question only the declaration can ask. var ran bool run := func(ctx context.Context, name string, args ...string) (string, error) { if name == "create-db" { ran = true } return "", nil // verify passes } d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"db","type":"action","command":["create-db","mesh"],"verify":["has-db","mesh"]} ]}`) report, _, err := Apply(context.Background(), d, store.State{}, run, nil) if err != nil { t.Fatalf("apply failed: %v", err) } if ran { t.Error("an action whose verify already passed was run anyway") } if report.Changed() { t.Errorf("an already-satisfied action reported a change: %+v", report.Outcomes) } } func TestAnActionThatSucceedsAndDoesNothingFails(t *testing.T) { // The whole reason verify is mandatory: a command that exits zero and has no effect is // this repository's most expensive failure shape. Here the command "succeeds" every time // and verify never passes. run := func(ctx context.Context, name string, args ...string) (string, error) { if name == "has-db" { return "", errors.New("no such database") } return "", nil } d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"db","type":"action","command":["create-db","mesh"],"verify":["has-db","mesh"]} ]}`) _, state, err := Apply(context.Background(), d, store.State{}, run, nil) if err == nil { t.Fatal("an action that reported success and did nothing was accepted") } if !strings.Contains(err.Error(), "verify still fails") { t.Errorf("failed for the wrong reason: %v", err) } if _, recorded := state.Find("db"); recorded { t.Error("an action that did not work was recorded as applied") } } func TestAnActionRunsInsideTheContainerItNames(t *testing.T) { // Steps 2 and 3 of the bootstrap act on something inside the store's container, before // there is any mesh to ask. var sawExec bool run := func(ctx context.Context, name string, args ...string) (string, error) { if name == "docker" && args[0] == "exec" && args[1] == "store" { sawExec = true return "", nil } return "", errors.New("not run in the container") } d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"db","type":"action","in":"store","command":["createdb","mesh"],"verify":["psql","-lqt"]} ]}`) if _, _, err := Apply(context.Background(), d, store.State{}, run, nil); err != nil { t.Fatalf("apply failed: %v", err) } if !sawExec { t.Error("an action naming a container did not run inside it") } } func TestAContainerThatExitsImmediatelyFailsTheApply(t *testing.T) { // `docker run --detach` returning an id says the container was created, not that it is // still running. A container whose entrypoint dies satisfies the command exactly as one // that came up does — which is the read-back rule, in the place it matters most. run := func(ctx context.Context, name string, args ...string) (string, error) { switch { case args[0] == "version": return "27.0\n", nil case args[0] == "inspect": return "false\t" + "", nil // exists, not running case args[0] == "run": return "deadbeef\n", nil } return "", nil } d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"store","type":"container","name":"store","image":"`+pinned+`"} ]}`) _, state, err := Apply(context.Background(), d, store.State{}, run, nil) if err == nil { t.Fatal("a container that exited immediately was reported as applied") } if !strings.Contains(err.Error(), "is not running") { t.Errorf("failed for the wrong reason: %v", err) } if _, recorded := state.Find("store"); recorded { t.Error("a container that is not running was recorded as applied") } } func TestAContainerWhoseDeclarationChangedIsReplaced(t *testing.T) { // A container's configuration is fixed when it is created, so any change is a replacement. // The spec label is what makes the difference visible without diffing everything the // runtime reports — which cannot be done reliably, because a runtime normalises what it is // given and that is indistinguishable from drift. d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"store","type":"container","name":"store","image":"`+pinned+`","env":{"PGDATA":"/data"}} ]}`) want := containerSpec(d.Resources[0].(*declaration.Container)) var removed, created bool run := func(ctx context.Context, name string, args ...string) (string, error) { switch args[0] { case "version": return "27.0\n", nil case "inspect": if created { return "true\t" + want, nil } return "true\tsome-older-spec", nil case "rm": removed = true return "", nil case "run": created = true return "deadbeef\n", nil } return "", nil } report, _, err := Apply(context.Background(), d, store.State{}, run, nil) if err != nil { t.Fatalf("apply failed: %v", err) } if !removed || !created { t.Fatalf("a changed container was not replaced (removed=%v created=%v)", removed, created) } if report.Outcomes[0].Action != "updated" { t.Errorf("a replacement was not reported as an update: %+v", report.Outcomes[0]) } } func TestAContainerThatMatchesIsLeftAlone(t *testing.T) { d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"store","type":"container","name":"store","image":"`+pinned+`","env":{"PGDATA":"/data"}} ]}`) spec := containerSpec(d.Resources[0].(*declaration.Container)) var touched bool run := func(ctx context.Context, name string, args ...string) (string, error) { switch args[0] { case "version": return "27.0\n", nil case "inspect": return "true\t" + spec, nil } touched = true return "", nil } report, _, err := Apply(context.Background(), d, store.State{}, run, nil) if err != nil { t.Fatalf("apply failed: %v", err) } if touched { t.Error("a container that already matched was restarted") } if report.Changed() { t.Errorf("a matching container reported a change: %+v", report.Outcomes) } }