Add the mesh's members to a list found in a file written into, and take back only those (hq ADR 0102)

internal/apply/into.go

@@ -49,20 +49,28 @@ func applyInto(r *declaration.File, previous store.Applied) (Outcome, error) {
 		}
 	}
 
-	rec := store.Into{Format: declaration.IntoJSON, Before: map[string]json.RawMessage{}}
+	rec := store.Into{Format: declaration.IntoJSON, Before: map[string]json.RawMessage{},
+		Added: map[string][]json.RawMessage{}}
 	if previous.Into != nil {
 		rec.Created = previous.Into.Created
 		for k, v := range previous.Into.Before {
 			rec.Before[k] = v
 		}
 		rec.Absent = slices.Clone(previous.Into.Absent)
+		for k, v := range previous.Into.Added {
+			rec.Added[k] = slices.Clone(v)
+		}
 	} else {
 		rec.Created = !existed
 	}
-	tracked := func(k string) bool { _, ok := rec.Before[k]; return ok || slices.Contains(rec.Absent, k) }
+	tracked := func(k string) bool {
+		_, before := rec.Before[k]
+		_, added := rec.Added[k]
+		return before || added || slices.Contains(rec.Absent, k)
+	}
 
 	// Drift: the machine no longer holds what this host last set in its keys.
-	drifted := previous.Wrote != "" && existed && digestOf(keysOf(object, keysTracked(rec))) != previous.Wrote
+	drifted := previous.Wrote != "" && existed && digestOf(viewOf(object, rec, keysTracked(rec))) != previous.Wrote
 
 	// Keys the mesh set before and no longer declares go back to what they held.
 	for _, k := range keysTracked(rec) {
@@ -71,8 +79,28 @@ func applyInto(r *declaration.File, previous store.Applied) (Outcome, error) {
 		}
 		giveBack(object, &rec, k)
 	}
-	// Declared keys: remember what each held the first time, then set it.
+	// Declared keys: remember what each held the first time, then set it. A list is the
+	// machine's too — a predecessor's own trusted registries, say — so the mesh adds its members
+	// to it rather than replacing it, and remembers exactly which it added.
 	for _, k := range keysIn(declared) {
+		_, scalar := rec.Before[k]
+		if isList(declared[k]) && !scalar {
+			current, had := object[k]
+			if had && !isList(current) {
+				return out, fmt.Errorf("%s: the mesh adds to the list %q, and the machine holds something "+
+					"other than a list there; it was left as it is", r.Path, k)
+			}
+			if !tracked(k) && !had {
+				rec.Absent = append(rec.Absent, k)
+			}
+			merged, added, err := addMembers(current, declared[k], rec.Added[k])
+			if err != nil {
+				return out, fmt.Errorf("%s: %q: %w", r.Path, k, err)
+			}
+			rec.Added[k] = added
+			object[k] = merged
+			continue
+		}
 		if !tracked(k) {
 			if v, had := object[k]; had {
 				rec.Before[k] = v
@@ -114,6 +142,19 @@ func applyInto(r *declaration.File, previous store.Applied) (Outcome, error) {
 		return out, fmt.Errorf("%s is not a JSON object after writing into it: %w", r.Path, err)
 	}
 	for k, v := range declared {
+		if _, list := rec.Added[k]; list {
+			members, _ := membersOf(v)
+			have, err := membersOf(check[k])
+			if err != nil {
+				return out, fmt.Errorf("%s does not hold a list at %q after writing into it", r.Path, k)
+			}
+			for _, m := range members {
+				if !hasMember(have, m) {
+					return out, fmt.Errorf("%s does not hold the declared %s in %q after writing into it", r.Path, m, k)
+				}
+			}
+			continue
+		}
 		if canonical(check[k]) != canonical(v) {
 			return out, fmt.Errorf("%s does not hold the declared %q after writing into it", r.Path, k)
 		}
@@ -122,8 +163,11 @@ func applyInto(r *declaration.File, previous store.Applied) (Outcome, error) {
 	if len(rec.Before) == 0 {
 		rec.Before = nil
 	}
+	if len(rec.Added) == 0 {
+		rec.Added = nil
+	}
 	out.into = &rec
-	out.wrote = digestOf(keysOf(check, keysIn(declared)))
+	out.wrote = digestOf(viewOf(check, rec, keysIn(declared)))
 	switch {
 	case !existed:
 		out.Action = "created"
@@ -181,6 +225,23 @@ func removeInto(a store.Applied) (string, string, error) {
 }
 
 func giveBack(object map[string]json.RawMessage, rec *store.Into, k string) {
+	if added, list := rec.Added[k]; list {
+		// Only the members the mesh added go; the list and everything else in it stay, unless
+		// the mesh made the key and nothing is left in it.
+		wasAbsent := slices.Contains(rec.Absent, k)
+		if current, had := object[k]; had && isList(current) {
+			have, _ := membersOf(current)
+			have = slices.DeleteFunc(have, func(m json.RawMessage) bool { return hasMember(added, m) })
+			if len(have) == 0 && wasAbsent {
+				delete(object, k)
+			} else {
+				object[k] = listOf(have)
+			}
+		}
+		delete(rec.Added, k)
+		rec.Absent = slices.DeleteFunc(rec.Absent, func(a string) bool { return a == k })
+		return
+	}
 	if v, had := rec.Before[k]; had {
 		object[k] = v
 		delete(rec.Before, k)
@@ -195,19 +256,99 @@ func keysTracked(rec store.Into) []string {
 	for k := range rec.Before {
 		keys = append(keys, k)
 	}
+	for k := range rec.Added {
+		keys = append(keys, k)
+	}
 	keys = append(keys, rec.Absent...)
 	sort.Strings(keys)
 	return slices.Compact(keys)
 }
 
-func keysOf(object map[string]json.RawMessage, keys []string) string {
+// viewOf is what the mesh holds itself to in a file written into: each scalar key's value, and for
+// a list only whether each member the mesh added is still there — what the machine keeps beside
+// them is not the mesh's to judge.
+func viewOf(object map[string]json.RawMessage, rec store.Into, keys []string) string {
 	var b bytes.Buffer
 	for _, k := range keys {
+		if added, list := rec.Added[k]; list {
+			have, _ := membersOf(object[k])
+			b.WriteString(k + " holds")
+			for _, m := range added {
+				fmt.Fprintf(&b, " %s=%v", canonical(m), hasMember(have, m))
+			}
+			b.WriteString("\n")
+			continue
+		}
 		b.WriteString(k + "=" + canonical(object[k]) + "\n")
 	}
 	return b.String()
 }
 
+func isList(raw json.RawMessage) bool {
+	t := bytes.TrimSpace(raw)
+	return len(t) > 0 && t[0] == '['
+}
+
+func membersOf(raw json.RawMessage) ([]json.RawMessage, error) {
+	if len(bytes.TrimSpace(raw)) == 0 {
+		return nil, nil
+	}
+	var members []json.RawMessage
+	if err := json.Unmarshal(raw, &members); err != nil {
+		return nil, err
+	}
+	return members, nil
+}
+
+func hasMember(list []json.RawMessage, m json.RawMessage) bool {
+	for _, have := range list {
+		if canonical(have) == canonical(m) {
+			return true
+		}
+	}
+	return false
+}
+
+func listOf(members []json.RawMessage) json.RawMessage {
+	if members == nil {
+		members = []json.RawMessage{}
+	}
+	raw, _ := json.Marshal(members)
+	return raw
+}
+
+// addMembers adds the declared members to the machine's list, dropping only members the mesh
+// added before and no longer declares. It returns the list and exactly which members the mesh
+// added — a declared member the machine already had is the machine's, and is never recorded.
+func addMembers(current, declared json.RawMessage, addedBefore []json.RawMessage) (json.RawMessage,
+	[]json.RawMessage, error) {
+	have, err := membersOf(current)
+	if err != nil {
+		return nil, nil, err
+	}
+	want, err := membersOf(declared)
+	if err != nil {
+		return nil, nil, err
+	}
+	added := []json.RawMessage{}
+	for _, a := range addedBefore {
+		if hasMember(want, a) {
+			added = append(added, a)
+			continue
+		}
+		have = slices.DeleteFunc(have, func(m json.RawMessage) bool { return canonical(m) == canonical(a) })
+	}
+	for _, m := range want {
+		if !hasMember(have, m) {
+			have = append(have, m)
+			if !hasMember(added, m) {
+				added = append(added, m)
+			}
+		}
+	}
+	return listOf(have), added, nil
+}
+
 func keysIn(m map[string]json.RawMessage) []string {
 	keys := make([]string, 0, len(m))
 	for k := range m {

internal/apply/into_test.go

@@ -56,8 +56,8 @@ func TestWritingIntoKeepsEveryKeyTheMachineHad(t *testing.T) {
 	if fmt.Sprint(o["log-opts"]) != "map[max-size:10m]" {
 		t.Errorf("the machine's logging settings were not kept: %v", o)
 	}
-	if fmt.Sprint(o["insecure-registries"]) != "[10.42.0.1:5000]" {
+	if fmt.Sprint(o["insecure-registries"]) != "[192.0.2.7:5000 10.42.0.1:5000]" {
-		t.Errorf("the mesh's key was not written: %v", o)
+		t.Errorf("the mesh's member was not added beside the machine's own: %v", o)
 	}
 	if info, _ := os.Stat(path); info.Mode().Perm() != 0o600 {
 		t.Errorf("the machine's file mode was changed to %o", info.Mode().Perm())
@@ -179,7 +179,7 @@ func TestAFileWrittenIntoIsNeverHeldOnAnAdoptedNode(t *testing.T) {
 		t.Errorf("something was held: %+v", state.Held)
 	}
 	o := readObject(t, path)
-	if o["data-root"] != "/srv/docker" || fmt.Sprint(o["insecure-registries"]) != "[10.42.0.1:5000]" {
+	if o["data-root"] != "/srv/docker" || fmt.Sprint(o["insecure-registries"]) != "[192.0.2.7:5000 10.42.0.1:5000]" {
 		t.Errorf("the adopted node's file was not written into: %v", o)
 	}
 }
@@ -211,3 +211,73 @@ func somethingElse(t *testing.T) *declaration.Declaration {
 	  {"id":"other","type":"directory","path":%q}
 	]}`, filepath.Join(t.TempDir(), "other")))
 }
+
+func TestAListIsAddedToNeverReplaced(t *testing.T) {
+	// The predecessor's own trusted registries are kept; the mesh adds its own and, undeclared,
+	// takes back only what it added (novox/hq ADR 0102).
+	path := filepath.Join(t.TempDir(), "daemon.json")
+	_ = os.WriteFile(path, []byte(`{"insecure-registries":["192.0.2.7:5000","10.42.0.9:5000"]}`), 0o644)
+	// 10.42.0.9 is declared too, and was already the machine's: it is never the mesh's to remove.
+	d := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000","10.42.0.9:5000"]}`))
+	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, nil, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := fmt.Sprint(readObject(t, path)["insecure-registries"]); got != "[192.0.2.7:5000 10.42.0.9:5000 10.42.0.1:5000]" {
+		t.Fatalf("the list after writing into it: %s", got)
+	}
+	rec, _ := state.Find("networking.registry-trust")
+	if added := rec.Into.Added["insecure-registries"]; len(added) != 1 || canonical(added[0]) != `"10.42.0.1:5000"` {
+		t.Errorf("recorded as added: %s", added)
+	}
+
+	// The predecessor adds a member of its own: not the mesh's drift.
+	o := readObject(t, path)
+	o["insecure-registries"] = append(o["insecure-registries"].([]any), "198.51.100.3:5000")
+	raw, _ := json.Marshal(o)
+	_ = os.WriteFile(path, raw, 0o644)
+	report, state, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, nil, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := report.Outcomes[0].Action; got != "unchanged" {
+		t.Errorf("a member the machine added was taken for drift: %q", got)
+	}
+
+	// Somebody takes the mesh's member out: that is drift, and it is put back.
+	o = readObject(t, path)
+	o["insecure-registries"] = []any{"192.0.2.7:5000", "10.42.0.9:5000", "198.51.100.3:5000"}
+	raw, _ = json.Marshal(o)
+	_ = os.WriteFile(path, raw, 0o644)
+	report, state, err = Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, nil, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := report.Outcomes[0].Action; got != "corrected" {
+		t.Errorf("the mesh's member removed by hand was %q", got)
+	}
+
+	// Undeclared: only the member the mesh added goes.
+	if _, _, err := Apply(context.Background(), archHost(t), somethingElse(t), state, store.OriginDeclared, nil, nil, nil); err != nil {
+		t.Fatal(err)
+	}
+	if got := fmt.Sprint(readObject(t, path)["insecure-registries"]); got != "[192.0.2.7:5000 10.42.0.9:5000 198.51.100.3:5000]" {
+		t.Errorf("undeclaring took more than the mesh added: %s", got)
+	}
+}
+
+func TestAListTheMeshCreatedGoesWhenEmptied(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "daemon.json")
+	_ = os.WriteFile(path, []byte(`{"data-root":"/srv/docker"}`), 0o644)
+	d := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000"]}`))
+	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, nil, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, _, err := Apply(context.Background(), archHost(t), somethingElse(t), state, store.OriginDeclared, nil, nil, nil); err != nil {
+		t.Fatal(err)
+	}
+	if o := readObject(t, path); fmt.Sprint(o) != "map[data-root:/srv/docker]" {
+		t.Errorf("the key the mesh created was not removed: %v", o)
+	}
+}

internal/store/store.go

@@ -82,6 +82,10 @@ type Into struct {
 	Before  map[string]json.RawMessage `json:"before,omitempty"`
 	Absent  []string                   `json:"absent,omitempty"`
 	Created bool                       `json:"created,omitempty"`
+	// Added is, for each key whose declared value is a list, exactly the members the mesh added
+	// to the machine's list — never a member that was already there. Undeclared, only these go,
+	// and drift is judged on these alone (novox/hq ADR 0102).
+	Added map[string][]json.RawMessage `json:"added,omitempty"`
 }
 
 // State is the whole of what a node knows about what it has done.