package apply import ( "context" "encoding/json" "fmt" "os" "path/filepath" "strings" "testing" "github.com/novox/mesh-host/internal/declaration" "github.com/novox/mesh-host/internal/store" ) // Defends novox/hq ADR 0102: a file the mesh shares with software it did not install is written // into, never over, and a service that re-reads its configuration is reloaded, not restarted. func intoDecl(t *testing.T, path, keys string) string { t.Helper() return fmt.Sprintf(`{"declaration":1,"resources":[ {"id":"networking.registry-trust","type":"file","path":%q,"into":"json","content":%q} ]}`, path, keys) } func readObject(t *testing.T, path string) map[string]any { t.Helper() raw, err := os.ReadFile(path) if err != nil { t.Fatal(err) } var o map[string]any if err := json.Unmarshal(raw, &o); err != nil { t.Fatalf("%s is not a JSON object: %v\n%s", path, err, raw) } return o } // The machine's own runtime settings, the way a predecessor leaves them. const machinesOwn = `{"data-root":"/srv/docker","log-opts":{"max-size":"10m"},"insecure-registries":["192.0.2.7:5000"]}` func TestWritingIntoKeepsEveryKeyTheMachineHad(t *testing.T) { path := filepath.Join(t.TempDir(), "daemon.json") if err := os.WriteFile(path, []byte(machinesOwn), 0o600); err != nil { t.Fatal(err) } d := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000"]}`)) report, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, nil, nil, nil) if err != nil { t.Fatal(err) } o := readObject(t, path) if o["data-root"] != "/srv/docker" { t.Errorf("the machine's data directory was not kept: %v", o) } if fmt.Sprint(o["log-opts"]) != "map[max-size:10m]" { t.Errorf("the machine's logging settings were not kept: %v", o) } if fmt.Sprint(o["insecure-registries"]) != "[192.0.2.7:5000 10.42.0.1:5000]" { t.Errorf("the mesh's member was not added beside the machine's own: %v", o) } if info, _ := os.Stat(path); info.Mode().Perm() != 0o600 { t.Errorf("the machine's file mode was changed to %o", info.Mode().Perm()) } if got := report.Outcomes[0].Action; got != "updated" { t.Errorf("writing into was reported as %q", got) } // Again, with nothing changed: nothing to do. report, state, err = Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, nil, nil, nil) if err != nil { t.Fatal(err) } if got := report.Outcomes[0].Action; got != "unchanged" { t.Errorf("a second apply was %q", got) } // Undeclared: the key goes back to what the machine had, and the file stays. empty := somethingElse(t) report, _, err = Apply(context.Background(), archHost(t), empty, state, store.OriginDeclared, nil, nil, nil) if err != nil { t.Fatal(err) } o = readObject(t, path) if fmt.Sprint(o["insecure-registries"]) != "[192.0.2.7:5000]" || o["data-root"] != "/srv/docker" { t.Errorf("undeclaring did not give the machine back what it had: %v", o) } if got := report.Outcomes[0].Action; got != "restored" { t.Errorf("undeclaring was reported as %q", got) } } func TestAFileWrittenIntoThatWasNotThereIsRemovedWhenOnlyTheMeshsKeysAreLeft(t *testing.T) { path := filepath.Join(t.TempDir(), "daemon.json") d := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000"]}`)) report, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, nil, nil, nil) if err != nil { t.Fatal(err) } if got := report.Outcomes[0].Action; got != "created" { t.Errorf("writing into a file that was not there was %q", got) } // Somebody else adds a key of their own: the file is no longer only the mesh's. o := readObject(t, path) o["debug"] = true raw, _ := json.Marshal(o) _ = os.WriteFile(path, raw, 0o644) empty := somethingElse(t) if _, _, err := Apply(context.Background(), archHost(t), empty, state, store.OriginDeclared, nil, nil, nil); err != nil { t.Fatal(err) } o = readObject(t, path) if _, still := o["insecure-registries"]; still || o["debug"] != true { t.Errorf("undeclaring should remove the mesh's key and keep the other: %v", o) } // Without the other key, the file the mesh created goes. path2 := filepath.Join(t.TempDir(), "daemon.json") d2 := parse(t, intoDecl(t, path2, `{"insecure-registries":["10.42.0.1:5000"]}`)) _, state2, err := Apply(context.Background(), archHost(t), d2, store.State{}, store.OriginDeclared, nil, nil, nil) if err != nil { t.Fatal(err) } if _, _, err := Apply(context.Background(), archHost(t), empty, state2, store.OriginDeclared, nil, nil, nil); err != nil { t.Fatal(err) } if _, err := os.Stat(path2); !os.IsNotExist(err) { t.Errorf("a file the mesh created, holding only its keys, was left behind") } } func TestAKeyNoLongerDeclaredGoesBackAndANewOneIsRemembered(t *testing.T) { path := filepath.Join(t.TempDir(), "daemon.json") _ = os.WriteFile(path, []byte(machinesOwn), 0o644) first := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000"]}`)) _, state, err := Apply(context.Background(), archHost(t), first, store.State{}, store.OriginDeclared, nil, nil, nil) if err != nil { t.Fatal(err) } second := parse(t, intoDecl(t, path, `{"registry-mirrors":["http://10.42.0.1:5000"]}`)) if _, _, err := Apply(context.Background(), archHost(t), second, state, store.OriginDeclared, nil, nil, nil); err != nil { t.Fatal(err) } o := readObject(t, path) if fmt.Sprint(o["insecure-registries"]) != "[192.0.2.7:5000]" { t.Errorf("a key the mesh stopped declaring was not given back: %v", o) } if fmt.Sprint(o["registry-mirrors"]) != "[http://10.42.0.1:5000]" { t.Errorf("the newly declared key was not written: %v", o) } } func TestAFileThatIsNotAnObjectIsRefusedAndLeftAlone(t *testing.T) { path := filepath.Join(t.TempDir(), "daemon.json") _ = os.WriteFile(path, []byte("# not json at all\n"), 0o644) d := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000"]}`)) if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, nil, nil, nil); err == nil { t.Fatal("writing into a file that is not a JSON object was not refused") } raw, _ := os.ReadFile(path) if string(raw) != "# not json at all\n" { t.Errorf("a file the mesh could not write into was changed: %q", raw) } } func TestAFileWrittenIntoIsNeverHeldOnAnAdoptedNode(t *testing.T) { path := filepath.Join(t.TempDir(), "daemon.json") _ = os.WriteFile(path, []byte(machinesOwn), 0o644) d := adopted(t, `{"taken":[],"untaken":{"networking":["networking.registry-trust"]}}`, fmt.Sprintf(`{"id":"networking.registry-trust","type":"file","path":%q,"into":"json","content":%q}`, path, `{"insecure-registries":["10.42.0.1:5000"]}`)) m := &machine{} report, state := applyAdopted(t, d, store.State{}, m, t.TempDir()) if got := outcomeOf(report, "networking.registry-trust").Action; got == "held" { t.Fatal("a file written into was held, though it replaces nothing that was found") } if len(state.Held) != 0 { t.Errorf("something was held: %+v", state.Held) } o := readObject(t, path) if o["data-root"] != "/srv/docker" || fmt.Sprint(o["insecure-registries"]) != "[192.0.2.7:5000 10.42.0.1:5000]" { t.Errorf("the adopted node's file was not written into: %v", o) } } func TestAServiceIsReloadedNotRestartedForWhatItReloadsOn(t *testing.T) { path := filepath.Join(t.TempDir(), "daemon.json") d := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[ {"id":"trust","type":"file","path":%q,"into":"json","content":%q}, {"id":"runtime","type":"service","unit":"docker.service","state":"running","reload-on":["trust"]} ]}`, path, `{"insecure-registries":["10.42.0.1:5000"]}`)) var commands []string if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, recordingServices(&commands), nil, nil); err != nil { t.Fatal(err) } joined := strings.Join(commands, "\n") if !strings.Contains(joined, "systemctl reload docker.service") { t.Errorf("the runtime was not reloaded; commands were %v", commands) } if strings.Contains(joined, "stop docker.service") || strings.Contains(joined, "restart docker.service") { t.Errorf("the runtime was stopped, which stops every container on the machine; commands were %v", commands) } } // somethingElse is a declaration that no longer holds the file: only an unrelated directory. func somethingElse(t *testing.T) *declaration.Declaration { t.Helper() return parse(t, fmt.Sprintf(`{"declaration":1,"resources":[ {"id":"other","type":"directory","path":%q} ]}`, filepath.Join(t.TempDir(), "other"))) } func TestAListIsAddedToNeverReplaced(t *testing.T) { // The predecessor's own trusted registries are kept; the mesh adds its own and, undeclared, // takes back only what it added (novox/hq ADR 0102). path := filepath.Join(t.TempDir(), "daemon.json") _ = os.WriteFile(path, []byte(`{"insecure-registries":["192.0.2.7:5000","10.42.0.9:5000"]}`), 0o644) // First the mesh's own member alone, so there is a record of this file. first := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000"]}`)) _, state, err := Apply(context.Background(), archHost(t), first, store.State{}, store.OriginDeclared, nil, nil, nil) if err != nil { t.Fatal(err) } // Now 10.42.0.9 is declared too, and was already the machine's: it is never the mesh's to remove. d := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000","10.42.0.9:5000"]}`)) _, state, err = Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, nil, nil, nil) if err != nil { t.Fatal(err) } if got := fmt.Sprint(readObject(t, path)["insecure-registries"]); got != "[192.0.2.7:5000 10.42.0.9:5000 10.42.0.1:5000]" { t.Fatalf("the list after writing into it: %s", got) } rec, _ := state.Find("networking.registry-trust") if added := rec.Into.Added["insecure-registries"]; len(added) != 1 || canonical(added[0]) != `"10.42.0.1:5000"` { t.Errorf("recorded as added: %s", added) } // The predecessor adds a member of its own: not the mesh's drift. o := readObject(t, path) o["insecure-registries"] = append(o["insecure-registries"].([]any), "198.51.100.3:5000") raw, _ := json.Marshal(o) _ = os.WriteFile(path, raw, 0o644) report, state, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, nil, nil, nil) if err != nil { t.Fatal(err) } if got := report.Outcomes[0].Action; got != "unchanged" { t.Errorf("a member the machine added was taken for drift: %q", got) } // Somebody takes the mesh's member out: that is drift, and it is put back. o = readObject(t, path) o["insecure-registries"] = []any{"192.0.2.7:5000", "10.42.0.9:5000", "198.51.100.3:5000"} raw, _ = json.Marshal(o) _ = os.WriteFile(path, raw, 0o644) report, state, err = Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, nil, nil, nil) if err != nil { t.Fatal(err) } if got := report.Outcomes[0].Action; got != "corrected" { t.Errorf("the mesh's member removed by hand was %q", got) } // Undeclared: only the member the mesh added goes. if _, _, err := Apply(context.Background(), archHost(t), somethingElse(t), state, store.OriginDeclared, nil, nil, nil); err != nil { t.Fatal(err) } if got := fmt.Sprint(readObject(t, path)["insecure-registries"]); got != "[192.0.2.7:5000 10.42.0.9:5000 198.51.100.3:5000]" { t.Errorf("undeclaring took more than the mesh added: %s", got) } } func TestAListTheMeshCreatedGoesWhenEmptied(t *testing.T) { path := filepath.Join(t.TempDir(), "daemon.json") _ = os.WriteFile(path, []byte(`{"data-root":"/srv/docker"}`), 0o644) d := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000"]}`)) _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, nil, nil, nil) if err != nil { t.Fatal(err) } if _, _, err := Apply(context.Background(), archHost(t), somethingElse(t), state, store.OriginDeclared, nil, nil, nil); err != nil { t.Fatal(err) } if o := readObject(t, path); fmt.Sprint(o) != "map[data-root:/srv/docker]" { t.Errorf("the key the mesh created was not removed: %v", o) } } func TestAHoldFromAWholeFileDoesNotKeepOutAnIntoWrite(t *testing.T) { // Declared whole before, the runtime's file was held; declared into now, it is written into. path := filepath.Join(t.TempDir(), "daemon.json") _ = os.WriteFile(path, []byte(machinesOwn), 0o644) known := store.State{Held: []store.Held{{ID: "networking.registry-trust", Module: "networking", Kind: "file", Target: path}}} d := adopted(t, `{"taken":[],"untaken":{"networking":["networking.registry-trust"]}}`, fmt.Sprintf(`{"id":"networking.registry-trust","type":"file","path":%q,"into":"json","content":%q}`, path, `{"insecure-registries":["10.42.0.1:5000"]}`)) report, state := applyAdopted(t, d, known, &machine{}, t.TempDir()) if got := outcomeOf(report, "networking.registry-trust").Action; got != "updated" { t.Errorf("the file was %q, not written into", got) } if len(state.Held) != 0 { t.Errorf("the old hold outlived the into declaration: %+v", state.Held) } if fmt.Sprint(readObject(t, path)["insecure-registries"]) != "[192.0.2.7:5000 10.42.0.1:5000]" { t.Errorf("the mesh's member was not written in: %v", readObject(t, path)) } } func TestAWriteWithNoRecordOfItIsTheMeshsOwn(t *testing.T) { // A host that wrote into the file and died before saving its state comes back with no record // of it. What is there is then exactly what the mesh declares — and remembered as the // machine's it would never be given back (novox/hq ADR 0102). path := filepath.Join(t.TempDir(), "daemon.json") _ = os.WriteFile(path, []byte(`{"data-root":"/srv/docker","insecure-registries":["192.0.2.7:5000"]}`), 0o644) d := parse(t, intoDecl(t, path, `{"live-restore":true,"insecure-registries":["10.42.0.1:5000"]}`)) if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, nil, nil, nil); err != nil { t.Fatal(err) } // The crash: the state was never saved, so the next apply knows nothing of this file. _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, nil, nil, nil) if err != nil { t.Fatal(err) } rec, _ := state.Find("networking.registry-trust") if _, asTheMachines := rec.Into.Before["live-restore"]; asTheMachines { t.Error("the mesh's own key was remembered as the machine's") } if _, _, err := Apply(context.Background(), archHost(t), somethingElse(t), state, store.OriginDeclared, nil, nil, nil); err != nil { t.Fatal(err) } o := readObject(t, path) if _, still := o["live-restore"]; still { t.Errorf("undeclaring left the mesh's key behind: %v", o) } if fmt.Sprint(o["insecure-registries"]) != "[192.0.2.7:5000]" || o["data-root"] != "/srv/docker" { t.Errorf("the machine did not get its file back: %v", o) } }