Write into a file the machine shares instead of over it, and reload a service that re-reads its configuration instead of restarting it (hq ADR 0102)

internal/apply/apply.go

@@ -50,6 +50,8 @@ type Outcome struct {
 	// wrote is a digest of what this apply put there, kept so the next one can tell a machine
 	// that drifted from one the mesh changed its mind about. Not reported: it is bookkeeping.
 	wrote string
+	// into is what a file written into held before the mesh's keys (novox/hq ADR 0102).
+	into *store.Into
 }
 
 // Report is what an apply did, in the order it did it.
@@ -298,6 +300,7 @@ func ApplyKeeping(
 			ID:     resource.Identity(), Type: string(resource.Kind()),
 			Target: outcome.Target, AppliedAt: time.Now().UTC(),
 			Wrote: outcome.wrote,
+			Into:  outcome.into,
 			Holds: holds(resource),
 		})
 		// Its module has been taken, and what was held for it is now the mesh's.
@@ -489,6 +492,9 @@ func applyAccess(r *declaration.Access) (Outcome, error) {
 }
 
 func applyFile(r *declaration.File, previous store.Applied, unseal Unseal) (Outcome, error) {
+	if r.Into != "" {
+		return applyInto(r, previous)
+	}
 	out := begin(r)
 
 	// What actually goes on disk. For a sealed file the mesh never had this, and neither did
@@ -688,6 +694,11 @@ func reflected(r *declaration.Service, changed map[string]bool) []string {
 	return restartedBy(r.RestartOn, changed)
 }
 
+// serviceReloader is a service manager that can tell a running unit to read its configuration again.
+type serviceReloader interface {
+	ReloadService(ctx context.Context, run system.Runner, unit string) error
+}
+
 // unitReloader is a service manager that caches unit files and must be told to read them again.
 type unitReloader interface {
 	ReloadUnits(ctx context.Context, run system.Runner) error
@@ -774,6 +785,25 @@ func applyService(ctx context.Context, sys system.System, r *declaration.Service
 				"%s was restarted to pick up a change and is %s", r.Unit, after)
 		}
 		changes = append(changes, "restarted for "+strings.Join(reflected(r, changed), ", "))
+	} else if r.State == "running" && len(restartedBy(r.ReloadOn, changed)) > 0 {
+		// Told to read its configuration again, not stopped: for a service whose restart would
+		// stop what it runs — every container, for the container runtime (novox/hq ADR 0102).
+		reloader, ok := sys.(serviceReloader)
+		if !ok {
+			return out, fmt.Errorf("%s must be reloaded for %s and this machine's service manager "+
+				"cannot reload a unit", r.Unit, strings.Join(restartedBy(r.ReloadOn, changed), ", "))
+		}
+		if err := reloader.ReloadService(ctx, run, r.Unit); err != nil {
+			return out, fmt.Errorf("reloading %s: %w", r.Unit, err)
+		}
+		after, err := sys.ServiceState(ctx, run, r.Unit)
+		if err != nil {
+			return out, err
+		}
+		if after != "running" {
+			return out, fmt.Errorf("%s was reloaded to pick up a change and is %s", r.Unit, after)
+		}
+		changes = append(changes, "reloaded for "+strings.Join(restartedBy(r.ReloadOn, changed), ", "))
 	}
 
 	if len(changes) == 0 {
@@ -828,6 +858,9 @@ func remove(ctx context.Context, sys system.System, a store.Applied, run Runner)
 		return "removed", "no longer declared, and empty", nil
 
 	case declaration.TypeFile:
+		if a.Into != nil {
+			return removeInto(a)
+		}
 		if err := os.RemoveAll(a.Target); err != nil {
 			return "", "", err
 		}

internal/apply/hold.go

@@ -46,9 +46,13 @@ func KeepIn(dir string) Keep {
 }
 
 // holdable is whether a resource is one a predecessor can already have on the machine: a file at
-// a path, or a container under a name.
+// a path, or a container under a name. A file written into is not: it replaces nothing that was
+// found, only adds the mesh's keys beside it (novox/hq ADR 0102).
 func holdable(r declaration.Resource) bool {
-	return r.Kind() == declaration.TypeFile || r.Kind() == declaration.TypeContainer
+	if f, ok := r.(*declaration.File); ok {
+		return f.Into == ""
+	}
+	return r.Kind() == declaration.TypeContainer
 }
 
 // found is whether a declared file or container is present on the machine with no record of this

internal/apply/into.go

@@ -0,0 +1,235 @@
+package apply
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"slices"
+	"sort"
+
+	"github.com/novox/mesh-host/internal/declaration"
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// A file written into, never over (novox/hq ADR 0102).
+//
+// **The file is the machine's; the mesh owns keys in it.** The container runtime's configuration
+// is the case that needed it: the mesh states one fact there — its registry is trusted over the
+// private network — and writing the file whole replaced everything the machine had set, down to
+// where the runtime keeps its data. So the host reads what is there, sets only the declared keys,
+// keeps every other key as it found it, and records what each of its keys held before. Undeclared,
+// each key goes back, and a file the mesh created goes only if nothing but its keys is left.
+
+// applyInto writes a file's declared keys into the object already at its path.
+func applyInto(r *declaration.File, previous store.Applied) (Outcome, error) {
+	out := begin(r)
+	if r.Into != declaration.IntoJSON {
+		return out, fmt.Errorf("%s: into %q is not a format this host writes into", r.Path, r.Into)
+	}
+	var declared map[string]json.RawMessage
+	if err := json.Unmarshal([]byte(r.Content), &declared); err != nil {
+		return out, fmt.Errorf("%s: the keys to write are not a JSON object: %w", r.Path, err)
+	}
+
+	existing, err := os.ReadFile(r.Path)
+	existed := err == nil
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return out, err
+	}
+	object := map[string]json.RawMessage{}
+	if existed && len(bytes.TrimSpace(existing)) > 0 {
+		if err := json.Unmarshal(existing, &object); err != nil || object == nil {
+			// Refused, never replaced: a file the host cannot read as an object is a file it
+			// cannot write into without losing whatever it is.
+			return out, fmt.Errorf("%s is not a JSON object, so the mesh cannot write its keys into it "+
+				"without replacing what is there; it was left as it is", r.Path)
+		}
+	}
+
+	rec := store.Into{Format: declaration.IntoJSON, Before: map[string]json.RawMessage{}}
+	if previous.Into != nil {
+		rec.Created = previous.Into.Created
+		for k, v := range previous.Into.Before {
+			rec.Before[k] = v
+		}
+		rec.Absent = slices.Clone(previous.Into.Absent)
+	} else {
+		rec.Created = !existed
+	}
+	tracked := func(k string) bool { _, ok := rec.Before[k]; return ok || slices.Contains(rec.Absent, k) }
+
+	// Drift: the machine no longer holds what this host last set in its keys.
+	drifted := previous.Wrote != "" && existed && digestOf(keysOf(object, keysTracked(rec))) != previous.Wrote
+
+	// Keys the mesh set before and no longer declares go back to what they held.
+	for _, k := range keysTracked(rec) {
+		if _, still := declared[k]; still {
+			continue
+		}
+		giveBack(object, &rec, k)
+	}
+	// Declared keys: remember what each held the first time, then set it.
+	for _, k := range keysIn(declared) {
+		if !tracked(k) {
+			if v, had := object[k]; had {
+				rec.Before[k] = v
+			} else {
+				rec.Absent = append(rec.Absent, k)
+			}
+		}
+		object[k] = declared[k]
+	}
+
+	want, err := render(object)
+	if err != nil {
+		return out, err
+	}
+	same := existed && canonical(existing) == canonical(want)
+	if !same {
+		mode := os.FileMode(0o644)
+		if info, err := os.Stat(r.Path); err == nil {
+			mode = info.Mode().Perm() // the machine's file keeps the machine's mode
+		} else if r.Mode != "" {
+			if m, err := modeOf(r.Mode, mode); err == nil {
+				mode = m
+			}
+		}
+		if err := os.MkdirAll(filepath.Dir(r.Path), 0o755); err != nil {
+			return out, err
+		}
+		if err := writeAtomically(r.Path, want, mode); err != nil {
+			return out, err
+		}
+	}
+	// Read back: every declared key holds what was declared.
+	written, err := os.ReadFile(r.Path)
+	if err != nil {
+		return out, fmt.Errorf("wrote into %s and cannot read it back: %w", r.Path, err)
+	}
+	var check map[string]json.RawMessage
+	if err := json.Unmarshal(written, &check); err != nil {
+		return out, fmt.Errorf("%s is not a JSON object after writing into it: %w", r.Path, err)
+	}
+	for k, v := range declared {
+		if canonical(check[k]) != canonical(v) {
+			return out, fmt.Errorf("%s does not hold the declared %q after writing into it", r.Path, k)
+		}
+	}
+
+	if len(rec.Before) == 0 {
+		rec.Before = nil
+	}
+	out.into = &rec
+	out.wrote = digestOf(keysOf(check, keysIn(declared)))
+	switch {
+	case !existed:
+		out.Action = "created"
+		out.Detail = "written into; the file was not there"
+	case same:
+		out.Action = "unchanged"
+	case drifted:
+		out.Action = "corrected"
+		out.Detail = "the mesh's keys had been changed on the machine; the rest of the file was kept"
+	default:
+		out.Action = "updated"
+		out.Detail = "the mesh's keys written in; every other key kept as it was"
+	}
+	return out, nil
+}
+
+// removeInto gives back what a file written into held before the mesh's keys.
+func removeInto(a store.Applied) (string, string, error) {
+	existing, err := os.ReadFile(a.Target)
+	if errors.Is(err, os.ErrNotExist) {
+		return "forgotten", "no longer there", nil
+	}
+	if err != nil {
+		return "", "", err
+	}
+	object := map[string]json.RawMessage{}
+	if len(bytes.TrimSpace(existing)) > 0 {
+		if err := json.Unmarshal(existing, &object); err != nil || object == nil {
+			return "kept", "no longer a JSON object, so the mesh's keys were left in it; " +
+				"remove them by hand", nil
+		}
+	}
+	rec := *a.Into
+	for _, k := range keysTracked(rec) {
+		giveBack(object, &rec, k)
+	}
+	if a.Into.Created && len(object) == 0 {
+		if err := os.Remove(a.Target); err != nil {
+			return "", "", err
+		}
+		return "removed", "no longer declared; the mesh had created it and nothing else was in it", nil
+	}
+	want, err := render(object)
+	if err != nil {
+		return "", "", err
+	}
+	info, err := os.Stat(a.Target)
+	if err != nil {
+		return "", "", err
+	}
+	if err := writeAtomically(a.Target, want, info.Mode().Perm()); err != nil {
+		return "", "", err
+	}
+	return "restored", "no longer declared; the mesh's keys were given back what they held", nil
+}
+
+func giveBack(object map[string]json.RawMessage, rec *store.Into, k string) {
+	if v, had := rec.Before[k]; had {
+		object[k] = v
+		delete(rec.Before, k)
+		return
+	}
+	delete(object, k)
+	rec.Absent = slices.DeleteFunc(rec.Absent, func(a string) bool { return a == k })
+}
+
+func keysTracked(rec store.Into) []string {
+	var keys []string
+	for k := range rec.Before {
+		keys = append(keys, k)
+	}
+	keys = append(keys, rec.Absent...)
+	sort.Strings(keys)
+	return slices.Compact(keys)
+}
+
+func keysOf(object map[string]json.RawMessage, keys []string) string {
+	var b bytes.Buffer
+	for _, k := range keys {
+		b.WriteString(k + "=" + canonical(object[k]) + "\n")
+	}
+	return b.String()
+}
+
+func keysIn(m map[string]json.RawMessage) []string {
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	return keys
+}
+
+// canonical is a JSON value compacted, so formatting is not mistaken for a change.
+func canonical(raw []byte) string {
+	var b bytes.Buffer
+	if err := json.Compact(&b, raw); err != nil {
+		return string(raw)
+	}
+	return b.String()
+}
+
+func render(object map[string]json.RawMessage) ([]byte, error) {
+	b, err := json.MarshalIndent(object, "", "  ")
+	if err != nil {
+		return nil, err
+	}
+	return append(b, '\n'), nil
+}

internal/apply/into_test.go

@@ -0,0 +1,213 @@
+package apply
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-host/internal/declaration"
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// Defends novox/hq ADR 0102: a file the mesh shares with software it did not install is written
+// into, never over, and a service that re-reads its configuration is reloaded, not restarted.
+
+func intoDecl(t *testing.T, path, keys string) string {
+	t.Helper()
+	return fmt.Sprintf(`{"declaration":1,"resources":[
+	  {"id":"networking.registry-trust","type":"file","path":%q,"into":"json","content":%q}
+	]}`, path, keys)
+}
+
+func readObject(t *testing.T, path string) map[string]any {
+	t.Helper()
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var o map[string]any
+	if err := json.Unmarshal(raw, &o); err != nil {
+		t.Fatalf("%s is not a JSON object: %v\n%s", path, err, raw)
+	}
+	return o
+}
+
+// The machine's own runtime settings, the way a predecessor leaves them.
+const machinesOwn = `{"data-root":"/srv/docker","log-opts":{"max-size":"10m"},"insecure-registries":["192.0.2.7:5000"]}`
+
+func TestWritingIntoKeepsEveryKeyTheMachineHad(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "daemon.json")
+	if err := os.WriteFile(path, []byte(machinesOwn), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	d := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000"]}`))
+	report, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, nil, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	o := readObject(t, path)
+	if o["data-root"] != "/srv/docker" {
+		t.Errorf("the machine's data directory was not kept: %v", o)
+	}
+	if fmt.Sprint(o["log-opts"]) != "map[max-size:10m]" {
+		t.Errorf("the machine's logging settings were not kept: %v", o)
+	}
+	if fmt.Sprint(o["insecure-registries"]) != "[10.42.0.1:5000]" {
+		t.Errorf("the mesh's key was not written: %v", o)
+	}
+	if info, _ := os.Stat(path); info.Mode().Perm() != 0o600 {
+		t.Errorf("the machine's file mode was changed to %o", info.Mode().Perm())
+	}
+	if got := report.Outcomes[0].Action; got != "updated" {
+		t.Errorf("writing into was reported as %q", got)
+	}
+
+	// Again, with nothing changed: nothing to do.
+	report, state, err = Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, nil, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := report.Outcomes[0].Action; got != "unchanged" {
+		t.Errorf("a second apply was %q", got)
+	}
+
+	// Undeclared: the key goes back to what the machine had, and the file stays.
+	empty := somethingElse(t)
+	report, _, err = Apply(context.Background(), archHost(t), empty, state, store.OriginDeclared, nil, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	o = readObject(t, path)
+	if fmt.Sprint(o["insecure-registries"]) != "[192.0.2.7:5000]" || o["data-root"] != "/srv/docker" {
+		t.Errorf("undeclaring did not give the machine back what it had: %v", o)
+	}
+	if got := report.Outcomes[0].Action; got != "restored" {
+		t.Errorf("undeclaring was reported as %q", got)
+	}
+}
+
+func TestAFileWrittenIntoThatWasNotThereIsRemovedWhenOnlyTheMeshsKeysAreLeft(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "daemon.json")
+	d := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000"]}`))
+	report, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, nil, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := report.Outcomes[0].Action; got != "created" {
+		t.Errorf("writing into a file that was not there was %q", got)
+	}
+	// Somebody else adds a key of their own: the file is no longer only the mesh's.
+	o := readObject(t, path)
+	o["debug"] = true
+	raw, _ := json.Marshal(o)
+	_ = os.WriteFile(path, raw, 0o644)
+
+	empty := somethingElse(t)
+	if _, _, err := Apply(context.Background(), archHost(t), empty, state, store.OriginDeclared, nil, nil, nil); err != nil {
+		t.Fatal(err)
+	}
+	o = readObject(t, path)
+	if _, still := o["insecure-registries"]; still || o["debug"] != true {
+		t.Errorf("undeclaring should remove the mesh's key and keep the other: %v", o)
+	}
+
+	// Without the other key, the file the mesh created goes.
+	path2 := filepath.Join(t.TempDir(), "daemon.json")
+	d2 := parse(t, intoDecl(t, path2, `{"insecure-registries":["10.42.0.1:5000"]}`))
+	_, state2, err := Apply(context.Background(), archHost(t), d2, store.State{}, store.OriginDeclared, nil, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, _, err := Apply(context.Background(), archHost(t), empty, state2, store.OriginDeclared, nil, nil, nil); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Stat(path2); !os.IsNotExist(err) {
+		t.Errorf("a file the mesh created, holding only its keys, was left behind")
+	}
+}
+
+func TestAKeyNoLongerDeclaredGoesBackAndANewOneIsRemembered(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "daemon.json")
+	_ = os.WriteFile(path, []byte(machinesOwn), 0o644)
+	first := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000"]}`))
+	_, state, err := Apply(context.Background(), archHost(t), first, store.State{}, store.OriginDeclared, nil, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	second := parse(t, intoDecl(t, path, `{"registry-mirrors":["http://10.42.0.1:5000"]}`))
+	if _, _, err := Apply(context.Background(), archHost(t), second, state, store.OriginDeclared, nil, nil, nil); err != nil {
+		t.Fatal(err)
+	}
+	o := readObject(t, path)
+	if fmt.Sprint(o["insecure-registries"]) != "[192.0.2.7:5000]" {
+		t.Errorf("a key the mesh stopped declaring was not given back: %v", o)
+	}
+	if fmt.Sprint(o["registry-mirrors"]) != "[http://10.42.0.1:5000]" {
+		t.Errorf("the newly declared key was not written: %v", o)
+	}
+}
+
+func TestAFileThatIsNotAnObjectIsRefusedAndLeftAlone(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "daemon.json")
+	_ = os.WriteFile(path, []byte("# not json at all\n"), 0o644)
+	d := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000"]}`))
+	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, nil, nil, nil); err == nil {
+		t.Fatal("writing into a file that is not a JSON object was not refused")
+	}
+	raw, _ := os.ReadFile(path)
+	if string(raw) != "# not json at all\n" {
+		t.Errorf("a file the mesh could not write into was changed: %q", raw)
+	}
+}
+
+func TestAFileWrittenIntoIsNeverHeldOnAnAdoptedNode(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "daemon.json")
+	_ = os.WriteFile(path, []byte(machinesOwn), 0o644)
+	d := adopted(t, `{"taken":[],"untaken":{"networking":["networking.registry-trust"]}}`,
+		fmt.Sprintf(`{"id":"networking.registry-trust","type":"file","path":%q,"into":"json","content":%q}`,
+			path, `{"insecure-registries":["10.42.0.1:5000"]}`))
+	m := &machine{}
+	report, state := applyAdopted(t, d, store.State{}, m, t.TempDir())
+	if got := outcomeOf(report, "networking.registry-trust").Action; got == "held" {
+		t.Fatal("a file written into was held, though it replaces nothing that was found")
+	}
+	if len(state.Held) != 0 {
+		t.Errorf("something was held: %+v", state.Held)
+	}
+	o := readObject(t, path)
+	if o["data-root"] != "/srv/docker" || fmt.Sprint(o["insecure-registries"]) != "[10.42.0.1:5000]" {
+		t.Errorf("the adopted node's file was not written into: %v", o)
+	}
+}
+
+func TestAServiceIsReloadedNotRestartedForWhatItReloadsOn(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "daemon.json")
+	d := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
+	  {"id":"trust","type":"file","path":%q,"into":"json","content":%q},
+	  {"id":"runtime","type":"service","unit":"docker.service","state":"running","reload-on":["trust"]}
+	]}`, path, `{"insecure-registries":["10.42.0.1:5000"]}`))
+	var commands []string
+	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared,
+		recordingServices(&commands), nil, nil); err != nil {
+		t.Fatal(err)
+	}
+	joined := strings.Join(commands, "\n")
+	if !strings.Contains(joined, "systemctl reload docker.service") {
+		t.Errorf("the runtime was not reloaded; commands were %v", commands)
+	}
+	if strings.Contains(joined, "stop docker.service") || strings.Contains(joined, "restart docker.service") {
+		t.Errorf("the runtime was stopped, which stops every container on the machine; commands were %v", commands)
+	}
+}
+
+// somethingElse is a declaration that no longer holds the file: only an unrelated directory.
+func somethingElse(t *testing.T) *declaration.Declaration {
+	t.Helper()
+	return parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
+	  {"id":"other","type":"directory","path":%q}
+	]}`, filepath.Join(t.TempDir(), "other")))
+}