package apply import ( "context" "errors" "fmt" "os" "path/filepath" "strings" "testing" "github.com/novox/mesh-host/internal/declaration" "github.com/novox/mesh-host/internal/store" "github.com/novox/mesh-host/internal/system" ) // Each test names the decision it defends (novox/hq ADR 0017). func parse(t *testing.T, raw string) *declaration.Declaration { t.Helper() d, err := declaration.Parse([]byte(raw)) if err != nil { t.Fatalf("fixture is not a valid declaration: %v", err) } return d } // noServices refuses to run anything. Used where a test declares no services, so that a test // which accidentally reaches the service manager fails loudly instead of passing quietly. func noServices(context.Context, string, ...string) (string, error) { return "", errors.New("this test declares no services and should not have run a command") } func TestApplyingTwiceChangesNothingTheSecondTime(t *testing.T) { // Idempotence is what makes an apply safe to run on a schedule. Without it, a host that // reconciles every few minutes rewrites files forever and every reader sees churn. dir := t.TempDir() d := parse(t, `{"declaration":1,"resources":[ {"id":"d","type":"directory","path":"`+dir+`/etc","mode":"0755"}, {"id":"f","type":"file","path":"`+dir+`/etc/a.conf","content":"hello\n","mode":"0640"} ]}`) first, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil) if err != nil { t.Fatal(err) } if !first.Changed() { t.Fatal("the first apply on an empty machine changed nothing") } second, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil, nil) if err != nil { t.Fatal(err) } if second.Changed() { t.Errorf("the second apply changed something: %+v", second.Outcomes) } } func TestADriftedMachineIsReturned(t *testing.T) { // The other half of idempotence, and the half that matters: converging is not "do nothing // if the state file says it was done". The machine is read, not the record. dir := t.TempDir() path := filepath.Join(dir, "a.conf") d := parse(t, `{"declaration":1,"resources":[ {"id":"f","type":"file","path":"`+path+`","content":"correct\n","mode":"0644"} ]}`) _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil) if err != nil { t.Fatal(err) } if err := os.WriteFile(path, []byte("someone edited this\n"), 0o644); err != nil { t.Fatal(err) } report, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil, nil) if err != nil { t.Fatal(err) } if !report.Changed() { t.Fatal("a drifted file was left drifted") } got, _ := os.ReadFile(path) if string(got) != "correct\n" { t.Errorf("the file was not returned: %q", got) } } func TestADroppedResourceIsRemoved(t *testing.T) { // novox/hq ADR 0005: the host removes what it previously applied and is no longer // declared. Removing a line from a declaration is an act with an effect. dir := t.TempDir() keep := filepath.Join(dir, "keep.conf") drop := filepath.Join(dir, "drop.conf") both := parse(t, `{"declaration":1,"resources":[ {"id":"keep","type":"file","path":"`+keep+`","content":"a\n"}, {"id":"drop","type":"file","path":"`+drop+`","content":"b\n"} ]}`) _, state, err := Apply(context.Background(), archHost(t), both, store.State{}, store.OriginCarried, noServices, nil, nil) if err != nil { t.Fatal(err) } one := parse(t, `{"declaration":1,"resources":[ {"id":"keep","type":"file","path":"`+keep+`","content":"a\n"} ]}`) report, state, err := Apply(context.Background(), archHost(t), one, state, store.OriginCarried, noServices, nil, nil) if err != nil { t.Fatal(err) } if _, err := os.Stat(drop); !errors.Is(err, os.ErrNotExist) { t.Error("a resource dropped from the declaration was left on the machine") } if _, err := os.Stat(keep); err != nil { t.Error("a declared resource was removed") } if _, still := state.Find("drop"); still { t.Error("the host still believes it owns what it removed") } if report.Outcomes[0].Action != "removed" { t.Errorf("removal is not reported first: %+v", report.Outcomes) } } func TestNothingTheHostDidNotCreateIsTouched(t *testing.T) { // The boundary the whole removal rule turns on. A machine has things on it the mesh did // not put there, and a converger that treats "not declared" as "must not exist" deletes // them. Authoritative over its own footprint; inert everywhere else. dir := t.TempDir() stranger := filepath.Join(dir, "not-ours.conf") if err := os.WriteFile(stranger, []byte("someone else's\n"), 0o644); err != nil { t.Fatal(err) } d := parse(t, `{"declaration":1,"resources":[ {"id":"ours","type":"file","path":"`+filepath.Join(dir, "ours.conf")+`","content":"a\n"} ]}`) if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil); err != nil { t.Fatal(err) } got, err := os.ReadFile(stranger) if err != nil || string(got) != "someone else's\n" { t.Error("a file the host did not create was removed or changed") } } func TestARenameToTheSamePathDoesNotDeleteTheNewFile(t *testing.T) { // Why removal happens FIRST. A resource leaving a declaration while another arrives at the // same path is an ordinary rename; removing afterwards would delete the file just written. dir := t.TempDir() path := filepath.Join(dir, "shared.conf") before := parse(t, `{"declaration":1,"resources":[ {"id":"old","type":"file","path":"`+path+`","content":"old\n"} ]}`) _, state, err := Apply(context.Background(), archHost(t), before, store.State{}, store.OriginCarried, noServices, nil, nil) if err != nil { t.Fatal(err) } after := parse(t, `{"declaration":1,"resources":[ {"id":"new","type":"file","path":"`+path+`","content":"new\n"} ]}`) if _, _, err := Apply(context.Background(), archHost(t), after, state, store.OriginCarried, noServices, nil, nil); err != nil { t.Fatal(err) } got, err := os.ReadFile(path) if err != nil { t.Fatalf("the renamed resource is gone: %v", err) } if string(got) != "new\n" { t.Errorf("content is %q, want the new one", got) } } func TestAFailedStepFailsTheApplyAndTheRestIsStillAttempted(t *testing.T) { // The apply fails, names the resource, and carries what did happen — because the machine is // in whatever state the apply reached and the only honest thing to hand back is that list. // // **And everything is attempted.** It used to stop at the first failure, which made one // broken resource hold the whole machine hostage: a module declaring a package that does not // exist meant every module after it was never applied, for ever // (novox/hq 04-ISSUES/011). The case for stopping was that a later resource may depend on an // earlier one — and it still may, and it then fails its own check and is reported, which is // more information than not attempting it. dir := t.TempDir() blocker := filepath.Join(dir, "blocker") if err := os.WriteFile(blocker, []byte("i am a file\n"), 0o644); err != nil { t.Fatal(err) } d := parse(t, `{"declaration":1,"resources":[ {"id":"fine","type":"file","path":"`+filepath.Join(dir, "fine.conf")+`","content":"a\n"}, {"id":"doomed","type":"directory","path":"`+blocker+`"}, {"id":"after","type":"file","path":"`+filepath.Join(dir, "after.conf")+`","content":"b\n"} ]}`) _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil) if err == nil { t.Fatal("an impossible resource did not fail the apply") } var applyErr *Error if !errors.As(err, &applyErr) { t.Fatalf("expected an apply error, got %T", err) } if applyErr.Resource != "doomed" { t.Errorf("the failure names %q, not the resource that failed", applyErr.Resource) } // Everything that worked is in the report, before and after the failure. if len(applyErr.Done.Outcomes) != 2 { t.Errorf("the error does not carry what was applied: %+v", applyErr.Done.Outcomes) } if _, err := os.Stat(filepath.Join(dir, "after.conf")); err != nil { t.Error("a resource after the failing one was never attempted, so one broken module " + "still blocks every module after it") } } func TestEveryFailureIsCountedNotJustTheFirst(t *testing.T) { // "One thing failed" and "eleven things failed" are different machines, and the first line is // what somebody reads. dir := t.TempDir() for _, name := range []string{"one", "two"} { if err := os.WriteFile(filepath.Join(dir, name), []byte("a file\n"), 0o644); err != nil { t.Fatal(err) } } d := parse(t, `{"declaration":1,"resources":[ {"id":"first","type":"directory","path":"`+filepath.Join(dir, "one")+`"}, {"id":"second","type":"directory","path":"`+filepath.Join(dir, "two")+`"} ]}`) _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil) if err == nil { t.Fatal("two impossible resources did not fail the apply") } var applyErr *Error if !errors.As(err, &applyErr) { t.Fatalf("got %T", err) } if applyErr.Others != 1 { t.Errorf("the failure says %d others also failed, and one did", applyErr.Others) } if !strings.Contains(err.Error(), "one other resource also failed") { t.Errorf("the message does not say others failed: %v", err) } } func TestNothingIsRecordedUntilItWorked(t *testing.T) { // novox/hq ADR 0018. A record written before the fact restates the request in a new place // and inherits none of the authority of having happened. dir := t.TempDir() blocker := filepath.Join(dir, "blocker") if err := os.WriteFile(blocker, []byte("x\n"), 0o644); err != nil { t.Fatal(err) } d := parse(t, `{"declaration":1,"resources":[ {"id":"doomed","type":"directory","path":"`+blocker+`"} ]}`) _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil) if err == nil { t.Fatal("expected a failure") } if _, claimed := state.Find("doomed"); claimed { t.Error("the host recorded owning something it failed to apply") } } func TestAModeIsMaintainedNotJustSet(t *testing.T) { // A permission set at creation is not a permission maintained — this repository has // already paid for that once, with generated files left world-readable because the mode // applied only when the file was first written. dir := t.TempDir() path := filepath.Join(dir, "secret.conf") d := parse(t, `{"declaration":1,"resources":[ {"id":"f","type":"file","path":"`+path+`","content":"s\n","mode":"0600"} ]}`) _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil) if err != nil { t.Fatal(err) } if err := os.Chmod(path, 0o666); err != nil { t.Fatal(err) } report, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil, nil) if err != nil { t.Fatal(err) } info, _ := os.Stat(path) if info.Mode().Perm() != 0o600 { t.Errorf("mode is %o after reconciling, want 0600", info.Mode().Perm()) } if !report.Changed() { t.Error("a mode that had drifted was reported as unchanged") } } func TestAServiceIsReadBackNotAssumed(t *testing.T) { // `systemctl start` returning zero says the transaction was accepted, not that the unit is // running. A unit that starts and immediately dies satisfies the command. started := false run := func(ctx context.Context, name string, args ...string) (string, error) { if args[0] == "show" { if started { return "LoadState=loaded\nActiveState=failed\n", nil // started, then died } return "LoadState=loaded\nActiveState=inactive\n", nil } started = true return "", nil // `systemctl start` succeeds } d := parse(t, `{"declaration":1,"resources":[ {"id":"s","type":"service","unit":"doomed.service","state":"running"} ]}`) _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) if err == nil { t.Fatal("a service that died immediately was reported as running") } if !strings.Contains(err.Error(), "asked to be running and is stopped") { t.Errorf("the failure does not say what was observed: %v", err) } } func TestAnUnknownServiceStateIsRefusedNotGuessed(t *testing.T) { run := func(ctx context.Context, name string, args ...string) (string, error) { return "LoadState=loaded\nActiveState=reticent\n", nil } d := parse(t, `{"declaration":1,"resources":[ {"id":"s","type":"service","unit":"odd.service","state":"running"} ]}`) _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) if err == nil || !strings.Contains(err.Error(), "neither running nor stopped") { t.Errorf("an unrecognised service state was not refused: %v", err) } } func TestADroppedServiceIsStoppedNotDeleted(t *testing.T) { // The host did not install the unit and does not own the unit file — only the state it put // the unit into. var commands []string run := func(ctx context.Context, name string, args ...string) (string, error) { commands = append(commands, strings.Join(args, " ")) if args[0] == "show" { return "LoadState=loaded\nActiveState=active\n", nil } return "", nil } state := store.State{Resources: []store.Applied{ {ID: "s", Type: "service", Target: "gone.service"}, }} d := parse(t, `{"declaration":1,"resources":[ {"id":"other","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"} ]}`) if _, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, run, nil, nil); err != nil { t.Fatal(err) } joined := strings.Join(commands, "; ") if !strings.Contains(joined, "stop gone.service") { t.Errorf("the dropped service was not stopped: %s", joined) } if strings.Contains(joined, "disable") || strings.Contains(joined, "mask") { t.Errorf("the host did more than stop a unit it does not own: %s", joined) } } func TestAUnitThatDoesNotExistIsNotStopped(t *testing.T) { // Found by applying inside a raised machine. `systemctl is-active` says "inactive" for a // unit that DOES NOT EXIST exactly as it does for one that is installed and stopped, so // declaring a unit stopped reported success for a unit the host cannot manage at all. // // Absence read as satisfaction — 04-ISSUES/007 wearing a different hat, and the mirror of // the degraded-init bug the capability detector had. absent := func(ctx context.Context, name string, args ...string) (string, error) { return "LoadState=not-found\nActiveState=inactive\n", nil } d := parse(t, `{"declaration":1,"resources":[ {"id":"s","type":"service","unit":"never-installed.service","state":"stopped"} ]}`) _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, absent, nil, nil) if err == nil { t.Fatal("a unit that does not exist was reported as satisfactorily stopped") } if !strings.Contains(err.Error(), "does not exist on this machine") { t.Errorf("the failure does not say the unit is absent: %v", err) } if _, claimed := state.Find("s"); claimed { t.Error("the host recorded owning a unit that is not installed") } } func TestAMaskedUnitIsRefused(t *testing.T) { // Masked means someone deliberately made it unstartable. Applying over that would undo a // decision the host did not make and cannot see the reason for. masked := func(ctx context.Context, name string, args ...string) (string, error) { return "LoadState=masked\nActiveState=inactive\n", nil } d := parse(t, `{"declaration":1,"resources":[ {"id":"s","type":"service","unit":"masked.service","state":"running"} ]}`) if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, masked, nil, nil); err == nil { t.Fatal("a masked unit was accepted") } } func TestForgettingAUnitThatIsGoneDoesNotStrandTheNode(t *testing.T) { // Found on a real machine. Removing an orphaned service runs `systemctl stop`, which fails // when the unit no longer exists — and a failure there fails the whole apply. A host // holding a record of an uninstalled unit could then apply NOTHING, ever, with no way out // but editing its state by hand. // // Removal is idempotent for the same reason os.RemoveAll is: the desired end state is // already true. var stopped bool run := func(ctx context.Context, name string, args ...string) (string, error) { if args[0] == "show" { return "LoadState=not-found\nActiveState=inactive\n", nil } stopped = true return "", errors.New("systemctl exited 5: Unit not loaded") } known := store.State{Resources: []store.Applied{ {ID: "gone", Type: "service", Target: "uninstalled.service"}, }} d := parse(t, `{"declaration":1,"resources":[ {"id":"f","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"} ]}`) report, state, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil, nil) if err != nil { t.Fatalf("a vanished unit stranded the apply: %v", err) } if stopped { t.Error("the host tried to stop a unit that does not exist") } if _, still := state.Find("gone"); still { t.Error("the host still believes it owns a unit that is gone") } // "forgotten", not "removed": the host stopped believing it owns the unit, and did not // remove anything, because there was nothing there to remove. Reporting an effect it did // not have would be the same class of untruth as reporting a package uninstalled. if report.Outcomes[0].Action != "forgotten" { t.Errorf("the vanished unit was not reported as forgotten: %+v", report.Outcomes) } } // --- package, container and action (novox/hq 07-the-substrate.md, ADR 0006, ADR 0005) --- func parseTrusted(t *testing.T, raw string) *declaration.Declaration { t.Helper() d, err := declaration.ParseTrusted([]byte(raw)) if err != nil { t.Fatalf("fixture is not a valid declaration: %v", err) } return d } const pinned = "docker.io/library/postgres@sha256:" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" func TestABrokenPackageDatabaseIsNotReadAsNotInstalled(t *testing.T) { // The same trap serviceState documents. `pacman -Q x` exits non-zero both for a package // that is not installed and for a database that cannot be read — so believing the first // answer would silently reinstall on a machine whose package manager is broken, or report // "installed nothing" as success. The apply must fail instead. run := func(ctx context.Context, name string, args ...string) (string, error) { return "", errors.New("pacman: error: could not lock database") } d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"rt","type":"package","package":"docker"} ]}`) _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) if err == nil { t.Fatal("a broken package database was read as 'not installed'") } if !strings.Contains(err.Error(), "does not answer") { t.Errorf("failed for the wrong reason: %v", err) } } func TestAnInstalledPackageIsNotReinstalled(t *testing.T) { var installed bool run := func(ctx context.Context, name string, args ...string) (string, error) { if args[0] == "-S" { installed = true } return "docker 27.0-1\n", nil // -Q succeeds for everything } d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"rt","type":"package","package":"docker"} ]}`) report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) if err != nil { t.Fatalf("apply failed: %v", err) } if installed { t.Error("a package that was already present was installed again") } if report.Changed() { t.Errorf("an already-installed package reported a change: %+v", report.Outcomes) } } func TestAPackageIsNeverUninstalled(t *testing.T) { // Deliberate: the host cannot know what else needs the package. Uninstalling a container // runtime because a declaration changed would stop every container on the node, and the // machine may have had it before the mesh ever saw it. Undeclaring is not "remove it". var uninstalled bool run := func(ctx context.Context, name string, args ...string) (string, error) { if len(args) > 0 && (args[0] == "-R" || args[0] == "-Rs") { uninstalled = true } return "", nil } known := store.State{Resources: []store.Applied{ {ID: "rt", Type: "package", Target: "docker"}, }} d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"f","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"} ]}`) report, state, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil, nil) if err != nil { t.Fatalf("dropping a package stranded the apply: %v", err) } if uninstalled { t.Fatal("the host uninstalled a package") } if _, still := state.Find("rt"); still { t.Error("the host still believes it owns the package") } // "forgotten", not "removed" — the host must not claim an effect it declined to have. if report.Outcomes[0].Action != "forgotten" { t.Errorf("dropping a package was not reported as forgotten: %+v", report.Outcomes[0]) } } func TestAnActionThatIsAlreadyTrueDoesNotRun(t *testing.T) { // Verify is the idempotency check as well as the read-back. The host does not know what a // database is, so "is it already there" is a question only the declaration can ask. var ran bool run := func(ctx context.Context, name string, args ...string) (string, error) { if name == "create-db" { ran = true } return "", nil // verify passes } d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"db","type":"action","command":["create-db","mesh"],"verify":["has-db","mesh"]} ]}`) report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) if err != nil { t.Fatalf("apply failed: %v", err) } if ran { t.Error("an action whose verify already passed was run anyway") } if report.Changed() { t.Errorf("an already-satisfied action reported a change: %+v", report.Outcomes) } } func TestAnActionThatSucceedsAndDoesNothingFails(t *testing.T) { // The whole reason verify is mandatory: a command that exits zero and has no effect is // this repository's most expensive failure shape. Here the command "succeeds" every time // and verify never passes. run := func(ctx context.Context, name string, args ...string) (string, error) { if name == "has-db" { return "", errors.New("no such database") } return "", nil } d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"db","type":"action","command":["create-db","mesh"],"verify":["has-db","mesh"]} ]}`) _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) if err == nil { t.Fatal("an action that reported success and did nothing was accepted") } if !strings.Contains(err.Error(), "verify still fails") { t.Errorf("failed for the wrong reason: %v", err) } if _, recorded := state.Find("db"); recorded { t.Error("an action that did not work was recorded as applied") } } func TestAnActionRunsInsideTheContainerItNames(t *testing.T) { // Steps 2 and 3 of the bootstrap act on something inside the store's container, before // there is any mesh to ask. var sawExec bool run := func(ctx context.Context, name string, args ...string) (string, error) { if name == "docker" && args[0] == "exec" && args[1] == "store" { sawExec = true return "", nil } return "", errors.New("not run in the container") } d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"db","type":"action","in":"store","command":["createdb","mesh"],"verify":["psql","-lqt"]} ]}`) if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil); err != nil { t.Fatalf("apply failed: %v", err) } if !sawExec { t.Error("an action naming a container did not run inside it") } } func TestAContainerThatExitsImmediatelyFailsTheApply(t *testing.T) { // `docker run --detach` returning an id says the container was created, not that it is // still running. A container whose entrypoint dies satisfies the command exactly as one // that came up does — which is the read-back rule, in the place it matters most. run := func(ctx context.Context, name string, args ...string) (string, error) { switch { case args[0] == "info": return "27.0\n", nil case args[0] == "inspect": return "false\t" + "", nil // exists, not running case args[0] == "run": return "deadbeef\n", nil } return "", nil } d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"store","type":"container","name":"store","image":"`+pinned+`"} ]}`) _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) if err == nil { t.Fatal("a container that exited immediately was reported as applied") } if !strings.Contains(err.Error(), "is not running") { t.Errorf("failed for the wrong reason: %v", err) } if _, recorded := state.Find("store"); recorded { t.Error("a container that is not running was recorded as applied") } } func TestAContainerWhoseDeclarationChangedIsReplaced(t *testing.T) { // A container's configuration is fixed when it is created, so any change is a replacement. // The spec label is what makes the difference visible without diffing everything the // runtime reports — which cannot be done reliably, because a runtime normalises what it is // given and that is indistinguishable from drift. d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"store","type":"container","name":"store","image":"`+pinned+`","env":{"PGDATA":"/data"}} ]}`) want := containerSpec(d.Resources[0].(*declaration.Container)) var removed, created bool run := func(ctx context.Context, name string, args ...string) (string, error) { switch args[0] { case "info": return "27.0\n", nil case "inspect": if created { return "true\t" + want, nil } return "true\tsome-older-spec", nil case "rm": removed = true return "", nil case "run": created = true return "deadbeef\n", nil } return "", nil } report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) if err != nil { t.Fatalf("apply failed: %v", err) } if !removed || !created { t.Fatalf("a changed container was not replaced (removed=%v created=%v)", removed, created) } if report.Outcomes[0].Action != "updated" { t.Errorf("a replacement was not reported as an update: %+v", report.Outcomes[0]) } } func TestAContainerThatMatchesIsLeftAlone(t *testing.T) { d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"store","type":"container","name":"store","image":"`+pinned+`","env":{"PGDATA":"/data"}} ]}`) spec := containerSpec(d.Resources[0].(*declaration.Container)) var touched bool run := func(ctx context.Context, name string, args ...string) (string, error) { switch args[0] { case "info": return "27.0\n", nil case "inspect": return "true\t" + spec, nil } touched = true return "", nil } report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) if err != nil { t.Fatalf("apply failed: %v", err) } if touched { t.Error("a container that already matched was restarted") } if report.Changed() { t.Errorf("a matching container reported a change: %+v", report.Outcomes) } } // --- boot state (novox/hq: a unit started but not enabled stops being true at the next reboot) --- // systemctlStub answers `show` and `is-enabled` the way systemd does, and records the verbs it // was asked to perform. Real command shapes, because the trap being tested is what systemd // actually says rather than what a fake would. func systemctlStub(t *testing.T, load, active, enabled string, verbs *[]string) Runner { t.Helper() return func(ctx context.Context, name string, args ...string) (string, error) { switch args[0] { case "show": return "LoadState=" + load + "\nActiveState=" + active + "\n", nil case "is-enabled": // Non-zero for everything but "enabled" — the exit code says nothing useful, which // is the whole reason this reads the output. if enabled == "enabled" { return enabled + "\n", nil } return enabled + "\n", errors.New("exit status 1") case "enable": *verbs = append(*verbs, "enable") enabled = "enabled" return "", nil case "disable": *verbs = append(*verbs, "disable") enabled = "disabled" return "", nil case "start": *verbs = append(*verbs, "start") active = "active" return "", nil case "stop": *verbs = append(*verbs, "stop") active = "inactive" return "", nil } return "", nil } } func TestAServiceIsEnabledAtBootWhenAsked(t *testing.T) { // The gap this closes: the host could start a unit and never make it survive a reboot, so // the declaration reported success and stopped being true at the next power cut. var verbs []string d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"rt","type":"service","unit":"docker.service","state":"running","boot":"enabled"} ]}`) report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, systemctlStub(t, "loaded", "inactive", "disabled", &verbs), nil, nil) if err != nil { t.Fatalf("apply failed: %v", err) } if len(verbs) != 2 || verbs[0] != "enable" || verbs[1] != "start" { t.Errorf("expected enable then start, got %v", verbs) } if report.Outcomes[0].Action != "updated" { t.Errorf("enabling and starting was not reported as an update: %+v", report.Outcomes[0]) } } func TestBootIsEnabledBeforeTheUnitIsStarted(t *testing.T) { // Order matters when an apply fails part way. Enabled-and-stopped comes back at the next // boot; running-and-disabled does not. So the more durable half is made true first. var verbs []string d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"rt","type":"service","unit":"docker.service","state":"running","boot":"enabled"} ]}`) if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, systemctlStub(t, "loaded", "inactive", "disabled", &verbs), nil, nil); err != nil { t.Fatal(err) } if len(verbs) < 2 || verbs[0] != "enable" { t.Errorf("boot state was not made true first: %v", verbs) } } func TestAlreadyEnabledAndRunningIsUnchanged(t *testing.T) { var verbs []string d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"rt","type":"service","unit":"docker.service","state":"running","boot":"enabled"} ]}`) report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, systemctlStub(t, "loaded", "active", "enabled", &verbs), nil, nil) if err != nil { t.Fatalf("apply failed: %v", err) } if len(verbs) != 0 { t.Errorf("a unit already in the declared state was touched: %v", verbs) } if report.Changed() { t.Errorf("an unchanged service reported a change: %+v", report.Outcomes) } } func TestOmittingBootLeavesItAlone(t *testing.T) { // Absent means the host asserts nothing. A machine whose operator enabled something must // not have it silently disabled because a declaration did not mention it. var verbs []string d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"rt","type":"service","unit":"docker.service","state":"running"} ]}`) if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, systemctlStub(t, "loaded", "inactive", "enabled", &verbs), nil, nil); err != nil { t.Fatal(err) } for _, v := range verbs { if v == "enable" || v == "disable" { t.Errorf("boot state was changed by a declaration that did not mention it: %v", verbs) } } } func TestAStaticUnitCannotBeEnabled(t *testing.T) { // `static` is neither enabled nor disabled: the unit has no install section and CANNOT be // enabled. Reading it as "disabled" would have the host try, fail, and blame the wrong // thing — the same shape as reading a missing unit as "stopped". var verbs []string d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"rt","type":"service","unit":"dbus.socket","state":"running","boot":"enabled"} ]}`) _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, systemctlStub(t, "loaded", "active", "static", &verbs), nil, nil) if err == nil { t.Fatal("a static unit was accepted as enable-able") } if !strings.Contains(err.Error(), "no install section") { t.Errorf("failed for the wrong reason: %v", err) } } func TestAnUnknownBootStateIsRefusedNotGuessed(t *testing.T) { var verbs []string d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"rt","type":"service","unit":"x.service","state":"running","boot":"enabled"} ]}`) _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, systemctlStub(t, "loaded", "active", "indirect", &verbs), nil, nil) if err == nil { t.Fatal("an unrecognised boot state was guessed at instead of refused") } } // --- more than one container runtime (novox/hq ADR 0005) --- func TestTheRuntimeProbeIsPerRuntime(t *testing.T) { // Verified against a real podman 6.1.0 before this was written: // // docker info --format '{{.ServerVersion}}' -> 29.7.2 // podman info --format '{{.ServerVersion}}' -> Error: can't evaluate field // ServerVersion in type system.infoReport // podman info --format '{{.Version.Version}}' -> 6.1.0 // // So a single probe cannot find both, and a host that used docker's would report a machine // with podman as having no container runtime at all. for _, tc := range []struct { name, present, wantProbe string }{ {"docker", "docker", "{{.ServerVersion}}"}, {"podman", "podman", "{{.Version.Version}}"}, } { t.Run(tc.name, func(t *testing.T) { var probedWith string run := func(ctx context.Context, name string, args ...string) (string, error) { if name != tc.present { return "", errors.New("not installed") } if args[0] == "info" { probedWith = args[2] } return "ok\n", nil } got, err := containerRuntime(context.Background(), run) if err != nil { t.Fatalf("%s was present and was not found: %v", tc.present, err) } if got != tc.present { t.Errorf("found %q, expected %q", got, tc.present) } if probedWith != tc.wantProbe { t.Errorf("probed %s with %q; that template does not work on it", tc.present, probedWith) } }) } } func TestAContainerUsesTheRuntimeTheMachineHas(t *testing.T) { // The applier must not call `docker` on a machine that has podman. Adoption keeps what the // machine already has (novox/hq research 012), so hardcoding one contradicts it. var calledWith []string run := func(ctx context.Context, name string, args ...string) (string, error) { if name == "docker" { return "", errors.New("not installed") } calledWith = append(calledWith, name) switch args[0] { case "info": return "6.1.0\n", nil case "inspect": return "false\t\n", errors.New("no such container") case "run": return "deadbeef\n", nil } return "", nil } d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"store","type":"container","name":"store","image":"`+pinned+`"} ]}`) // It will fail at read-back — the stub never reports it running — and what matters is // WHICH binary it used getting there. _, _, _ = Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) for _, c := range calledWith { if c != "podman" { t.Errorf("called %q on a machine that only has podman", c) } } if len(calledWith) == 0 { t.Error("nothing was called; the runtime was not found") } } func TestNoRuntimeIsSaidPlainly(t *testing.T) { // Naming what was tried, because "docker: command not found" on a machine that deliberately // runs podman sends the reader looking for the wrong thing. run := func(ctx context.Context, name string, args ...string) (string, error) { return "", errors.New("not installed") } d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"store","type":"container","name":"store","image":"`+pinned+`"} ]}`) _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) if err == nil { t.Fatal("a machine with no container runtime applied a container") } for _, want := range []string{"docker", "podman", "no container runtime"} { if !strings.Contains(err.Error(), want) { t.Errorf("the failure does not mention %q: %v", want, err) } } } // archHost is the system these tests run against. They were written for pacman and systemd, and // naming that is better than the implicit default it used to be. func archHost(t *testing.T) system.System { t.Helper() s, err := system.For("arch") if err != nil { t.Fatal(err) } return s } func TestAServiceIsRestartedWhenWhatItReflectsChanges(t *testing.T) { // A running service does not re-read its configuration. Replace the file, find the service // already running, do nothing — and the machine keeps behaving as it did while every check // passes, because the file is right and the service is up. // // That is how a third node joining a mesh left the first two carrying a network that no // longer existed. Found in the lab; this is the shape of the fix. dir := t.TempDir() path := filepath.Join(dir, "thing.conf") d := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[ {"id":"conf","type":"file","path":%q,"content":"first\n","mode":"0644"}, {"id":"svc","type":"service","unit":"thing.service","state":"running","restart-on":["conf"]} ]}`, path)) var commands []string run := recordingServices(&commands) if _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil); err != nil { t.Fatal(err) } else { // Second apply with the same content: nothing moved, so nothing restarts. A machine that // restarted its services on every reconcile would never be steady. commands = nil if _, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, run, nil, nil); err != nil { t.Fatal(err) } for _, c := range commands { if strings.Contains(c, "stop") { t.Errorf("an unchanged declaration restarted the service: %s", c) } } // Now the file changes. The service is already running and must still be restarted. changedDecl := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[ {"id":"conf","type":"file","path":%q,"content":"second\n","mode":"0644"}, {"id":"svc","type":"service","unit":"thing.service","state":"running","restart-on":["conf"]} ]}`, path)) commands = nil if _, _, err := Apply(context.Background(), archHost(t), changedDecl, state, store.OriginCarried, run, nil, nil); err != nil { t.Fatal(err) } var stopped, started bool for _, c := range commands { if strings.Contains(c, "stop thing.service") { stopped = true } if strings.Contains(c, "start thing.service") { started = true } } if !stopped || !started { t.Errorf("the file changed and the service was not restarted; commands were %v", commands) } } } func TestAServiceIsNotRestartedByAChangeItDoesNotName(t *testing.T) { // The list is what it reflects, not everything in the declaration. A service restarted by any // change anywhere would make every apply a fleet-wide bounce. dir := t.TempDir() conf := filepath.Join(dir, "thing.conf") other := filepath.Join(dir, "unrelated") first := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[ {"id":"conf","type":"file","path":%q,"content":"same\n","mode":"0644"}, {"id":"other","type":"file","path":%q,"content":"one\n","mode":"0644"}, {"id":"svc","type":"service","unit":"thing.service","state":"running","restart-on":["conf"]} ]}`, conf, other)) var commands []string run := recordingServices(&commands) _, state, err := Apply(context.Background(), archHost(t), first, store.State{}, store.OriginCarried, run, nil, nil) if err != nil { t.Fatal(err) } second := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[ {"id":"conf","type":"file","path":%q,"content":"same\n","mode":"0644"}, {"id":"other","type":"file","path":%q,"content":"two\n","mode":"0644"}, {"id":"svc","type":"service","unit":"thing.service","state":"running","restart-on":["conf"]} ]}`, conf, other)) commands = nil if _, _, err := Apply(context.Background(), archHost(t), second, state, store.OriginCarried, run, nil, nil); err != nil { t.Fatal(err) } for _, c := range commands { if strings.Contains(c, "stop") { t.Errorf("a change to a file the service does not name restarted it: %s", c) } } } // recordingServices answers the way a machine with a running unit would, and remembers what it // was asked to do — which is what a restart has to be proved by, since "running" looks the same // before and after one. func recordingServices(commands *[]string) Runner { return func(_ context.Context, name string, args ...string) (string, error) { line := name + " " + strings.Join(args, " ") *commands = append(*commands, line) switch { case strings.Contains(line, "is-enabled"): return "enabled", nil case strings.Contains(line, "show") && strings.Contains(line, "ActiveState"): return "LoadState=loaded\nActiveState=active\nSubState=running", nil } return "", nil } } func TestAFileChangedOnTheMachineIsCorrectedAndSaidSo(t *testing.T) { // The question this answers: how would anybody know somebody edited a managed file? Before // this they would not. It was rewritten within five minutes and reported as "updated", // which is what the mesh changing its mind looks like — so the person's change vanished and // nothing anywhere said why. They edit it again, and again. dir := t.TempDir() path := filepath.Join(dir, "thing.conf") d := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[ {"id":"conf","type":"file","path":%q,"content":"from the mesh\n","mode":"0644"} ]}`, path)) _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil) if err != nil { t.Fatal(err) } // Somebody edits it. if err := os.WriteFile(path, []byte("edited by hand\n"), 0o644); err != nil { t.Fatal(err) } report, state, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil, nil) if err != nil { t.Fatal(err) } if got := report.Outcomes[0].Action; got != "corrected" { t.Errorf("a hand edit was reported as %q; the mesh cannot tell it from changing its own "+ "mind, and neither can anybody reading this", got) } // And it is put back, because holding the machine to what it was told is the point. back, err := os.ReadFile(path) if err != nil { t.Fatal(err) } if string(back) != "from the mesh\n" { t.Errorf("the file was left as %q", back) } } func TestTheMeshChangingItsMindIsNotDrift(t *testing.T) { // The other half. A new declaration is an ordinary update and must not read as somebody // having meddled, or every real change would look like an incident. dir := t.TempDir() path := filepath.Join(dir, "thing.conf") first := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[ {"id":"conf","type":"file","path":%q,"content":"one\n","mode":"0644"} ]}`, path)) second := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[ {"id":"conf","type":"file","path":%q,"content":"two\n","mode":"0644"} ]}`, path)) _, state, err := Apply(context.Background(), archHost(t), first, store.State{}, store.OriginCarried, noServices, nil, nil) if err != nil { t.Fatal(err) } report, _, err := Apply(context.Background(), archHost(t), second, state, store.OriginCarried, noServices, nil, nil) if err != nil { t.Fatal(err) } if got := report.Outcomes[0].Action; got != "updated" { t.Errorf("the mesh changing what it wants was reported as %q", got) } } func TestAnUntouchedFileIsStillUnchanged(t *testing.T) { // And nothing about this makes a steady machine look busy. dir := t.TempDir() path := filepath.Join(dir, "thing.conf") d := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[ {"id":"conf","type":"file","path":%q,"content":"steady\n","mode":"0644"} ]}`, path)) _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil) if err != nil { t.Fatal(err) } report, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil, nil) if err != nil { t.Fatal(err) } if got := report.Outcomes[0].Action; got != "unchanged" { t.Errorf("an untouched file was reported as %q", got) } } func TestAFailedActionStopsWhatFollows(t *testing.T) { // An action is the only shape whose purpose is to make something true BEFORE the next thing // needs it, which is why it is the only one with a verify. The bootstrap is a row of them: // the store answers, then its databases exist, then their schemas, then the broker. // // Carrying on past one that did not happen starts things against a machine that is not ready // — and on a small machine that is how a database still initialising has its memory taken // away and shuts down. Observed in the lab, caused by a version of this loop that continued // past everything. dir := t.TempDir() d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"gate","type":"action","command":["false"],"verify":["false"]}, {"id":"after","type":"file","path":"`+filepath.Join(dir, "after.conf")+`","content":"b\n"} ]}`) _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, ExecRunner, nil, nil) if err == nil { t.Fatal("an action that cannot succeed did not fail the apply") } var applyErr *Error if !errors.As(err, &applyErr) { t.Fatalf("got %T", err) } if !applyErr.Gated { t.Error("the failure does not say that nothing after it was attempted") } if _, statErr := os.Stat(filepath.Join(dir, "after.conf")); statErr == nil { t.Error("the apply continued past a failed action, which is a gate") } if !strings.Contains(err.Error(), "nothing after it was attempted") { t.Errorf("the message does not say the rest was not tried: %v", err) } } // A container is told which resolver to use, because it does not inherit the machine's names. // // A container gets its own `/etc/hosts` holding its own hostname, and a runtime rewrites // `resolv.conf` — so every internal name the mesh wrote for the machine is invisible to what the // machine is running. That was hit for real: a database client on one node could not resolve // another node, on a mesh where both names were correct and present on both machines. func TestAContainerIsToldWhichResolverToUse(t *testing.T) { var ran []string run := func(_ context.Context, name string, args ...string) (string, error) { if name != "docker" { return "", errors.New("not installed") } switch args[0] { case "info": return "29.0.0\n", nil case "inspect": return "false\t\n", errors.New("no such container") case "run": ran = args return "deadbeef\n", nil } return "", nil } d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"app","type":"container","name":"app","image":"`+pinned+`", "nameservers":["10.42.0.1"]} ]}`) _, _, _ = Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) var told bool for i, a := range ran { if a == "--dns" && i+1 < len(ran) && ran[i+1] == "10.42.0.1" { told = true } } if !told { t.Fatalf("the container was not told where to resolve names: %v", ran) } } // And a container that was told nothing is run exactly as before: most containers resolve // whatever the machine resolves, and passing an empty flag would be a change of behaviour // dressed as a default. func TestAContainerToldNothingIsRunAsBefore(t *testing.T) { var ran []string run := func(_ context.Context, name string, args ...string) (string, error) { if name != "docker" { return "", errors.New("not installed") } switch args[0] { case "info": return "29.0.0\n", nil case "inspect": return "false\t\n", errors.New("no such container") case "run": ran = args return "deadbeef\n", nil } return "", nil } d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"app","type":"container","name":"app","image":"`+pinned+`"} ]}`) _, _, _ = Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) for _, a := range ran { if a == "--dns" { t.Fatalf("a container that was told nothing was given a resolver anyway: %v", ran) } } }