internal/apply/apply.go

@@ -20,6 +20,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -230,6 +231,18 @@ func ApplyKeeping(
 			protecting = append(protecting, orphan)
 			continue
 		}
+		// **A module the mesh left out is not a module the mesh removed** (novox/hq ADR 0163, rule
+		// 6): its resources are absent because a setting stored for it cannot compose, and the
+		// mesh said so by name. What the host wrote for it stays as it is, recorded, until the
+		// module is declared again or unassigned.
+		if module, left := d.LeftOutModuleOf(orphan.ID); left {
+			report.Outcomes = append(report.Outcomes, Outcome{
+				ID: orphan.ID, Type: orphan.Type, Target: orphan.Target,
+				Action: "unchanged", Detail: "kept: " + module + " was left out of this declaration by the mesh, not removed",
+			})
+			log(fmt.Sprintf("  kept %s (%s): %s was left out of this declaration by the mesh, not removed", orphan.ID, orphan.Target, module))
+			continue
+		}
 		orphans = append(orphans, orphan)
 	}
 	ordered := d.Resources
@@ -270,6 +283,11 @@ func ApplyKeeping(
 			if declared[h.ID] {
 				continue
 			}
+			if slices.Contains(d.LeftOut, h.Module) {
+				// Left out, not unassigned (ADR 0163, rule 6): still held for the module, as the
+				// mesh asked.
+				continue
+			}
 			known.Release(h.ID)
 			report.Outcomes = append(report.Outcomes, Outcome{ID: h.ID, Type: h.Kind, Target: h.Target,
 				Action: "forgotten", Detail: "no longer declared; left as found"})
@@ -1526,6 +1544,11 @@ func containerSpecReading(r *declaration.Container, declares, reads map[string]s
 	if r.IP != "" {
 		b.WriteString("ip " + r.IP + "\n")
 	}
+	// The networks it also joins are part of what it is (ADR 0163, rule 4): kept or let go, the
+	// container is recreated, and a neighbour's reach changes with it.
+	for _, n := range r.Networks {
+		b.WriteString("also-on " + n + "\n")
+	}
 	// The cadence is part of what was declared, so a changed schedule is a changed spec — the marker
 	// moves and the install is reported "updated" and re-established. Added only when present, so no
 	// ordinary container's or run-once step's digest moves for a field it does not set.
@@ -1766,6 +1789,15 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner,
 	if after.Spec != want {
 		return out, fmt.Errorf("container %s is not the one that was declared after creating it", r.Name)
 	}
+	// The found networks a per-machine setting keeps for it (novox/hq ADR 0163, rule 4), joined
+	// once it runs: a runtime starts a container on one network, and the others are connected.
+	// Refused, not skipped, when one cannot be joined — a neighbour that was promised to keep
+	// reaching this container by name would silently not.
+	for _, n := range r.Networks {
+		if _, err := run(ctx, cri, "network", "connect", n, r.Name); err != nil {
+			return out, fmt.Errorf("container %s could not join the kept network %s: %w", r.Name, n, err)
+		}
+	}
 
 	out.Action = "created"
 	if existed {

internal/apply/left_out_test.go

@@ -0,0 +1,136 @@
+package apply
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-host/internal/declaration"
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// A taken container keeps a found network by a per-machine setting (novox/hq ADR 0163, rule 4):
+// joined once it runs, part of its spec, and refused when it cannot be joined.
+func TestAContainerJoinsTheNetworksItKeeps(t *testing.T) {
+	var ran []string
+	connectFails := false
+	run := func(_ context.Context, name string, args ...string) (string, error) {
+		if name != "docker" {
+			return "", errors.New("not installed")
+		}
+		ran = append(ran, strings.Join(args, " "))
+		switch args[0] {
+		case "info":
+			return "29.0.0\n", nil
+		case "container":
+			if len(ran) > 2 {
+				return "true\t" + specOfLast, nil
+			}
+			return "false\t\n", errors.New("no such container")
+		case "run":
+			return "deadbeef\n", nil
+		case "network":
+			if connectFails {
+				return "", errors.New("network predecessor_default not found")
+			}
+		}
+		return "", nil
+	}
+	d := parseTrusted(t, `{"declaration":1,"resources":[
+	  {"id":"app","type":"container","name":"app","image":"`+pinned+`",
+	   "networks":["predecessor_default"]}
+	]}`)
+	specOfLast = containerSpec(d.Resources[0].(*declaration.Container), inputs{})
+	alone := *d.Resources[0].(*declaration.Container)
+	alone.Networks = nil
+	if specOfLast == containerSpec(&alone, inputs{}) {
+		t.Fatal("the kept network is not part of the container's spec: kept or let go, the container would be left alone")
+	}
+	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	joined := false
+	for i, line := range ran {
+		if line == "network connect predecessor_default app" {
+			joined = true
+			if ran[i-1] != "container inspect --format {{.State.Running}}\t{{index .Config.Labels \""+specLabel+"\"}} app" &&
+				!strings.HasPrefix(ran[i-1], "container inspect") {
+				t.Errorf("joined before the container was read back as running: %v", ran)
+			}
+		}
+	}
+	if !joined || report.Outcomes[0].Action != "created" {
+		t.Fatalf("the container did not join the kept network: %v\n%+v", ran, report.Outcomes)
+	}
+
+	connectFails, ran = true, nil
+	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil); err == nil ||
+		!strings.Contains(err.Error(), "could not join the kept network predecessor_default") {
+		t.Fatalf("a network that cannot be joined was passed over: %v", err)
+	}
+}
+
+var specOfLast string
+
+// A module the mesh left out of a declaration is not a module the mesh removed (novox/hq ADR 0163,
+// rule 6): what the host wrote for it stays, recorded and said; what it holds for it stays held.
+// A module simply absent is removed as it always was.
+func TestALeftOutModuleIsNeitherRemovedNorForgotten(t *testing.T) {
+	var removed []string
+	gone := map[string]bool{}
+	run := func(_ context.Context, name string, args ...string) (string, error) {
+		if name != "docker" {
+			return "", nil
+		}
+		switch args[0] {
+		case "info":
+			return "29.0.0\n", nil
+		case "rm":
+			removed = append(removed, args[len(args)-1])
+			gone[args[len(args)-1]] = true
+		case "container":
+			if gone[args[len(args)-1]] {
+				return "", errors.New("no such container")
+			}
+			return "true\tspec", nil
+		}
+		return "", nil
+	}
+	known := store.State{
+		Resources: []store.Applied{
+			{ID: "web.server", Type: "container", Target: "web", Origin: store.OriginDeclared},
+			{ID: "old.server", Type: "container", Target: "old", Origin: store.OriginDeclared},
+		},
+		Held: []store.Held{{ID: "web.page", Module: "web", Kind: "file", Target: "/srv/web/index.html"}},
+	}
+	d := parse(t, `{"declaration":1,"left_out":["web"],"resources":[
+	  {"id":"notes.conf","type":"file","path":"`+t.TempDir()+`/notes.conf","content":"x"}
+	]}`)
+	report, state, err := Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(removed) != 1 || removed[0] != "old" {
+		t.Fatalf("removed %v; only the module that is absent goes", removed)
+	}
+	if _, kept := state.At("container", "web"); !kept {
+		t.Fatal("the left-out module's record was forgotten")
+	}
+	if _, held := state.HeldAt("web.page"); !held {
+		t.Fatal("the left-out module's hold was released")
+	}
+	said := false
+	for _, o := range report.Outcomes {
+		if o.ID == "web.server" && o.Action == "unchanged" && strings.Contains(o.Detail, "web was left out of this declaration by the mesh") {
+			said = true
+		}
+		if o.ID == "web.server" && o.Action != "unchanged" {
+			t.Errorf("the left-out module's container was %s", o.Action)
+		}
+	}
+	if !said {
+		t.Fatalf("keeping the left-out module's container was not said: %+v", report.Outcomes)
+	}
+}

internal/bootstrap/genesis_forge_test.go

@@ -0,0 +1,79 @@
+package bootstrap
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+// What genesis raises, it raises as the module that succeeds it declares — name, data directory
+// and image — so the module adopts it by the found rule that already exists (novox/hq ADR 0163,
+// rule 7; issue 090). The network is the one difference left: the bootstrap forge runs on the
+// machine's network to reach the store on its loopback, and a take says so.
+func TestGenesisRaisesTheForgeAsTheModuleDeclaresIt(t *testing.T) {
+	var ran [][]string
+	run := func(_ context.Context, name string, args ...string) (string, error) {
+		if name == "docker" && args[0] == "container" {
+			return "", nil // not raised yet
+		}
+		ran = append(ran, append([]string{name}, args...))
+		return "", nil
+	}
+	if err := raiseGiteaServer(context.Background(), run, time.Second, "pw", DefaultPorts(), quietly); err != nil {
+		t.Fatal(err)
+	}
+	var raised []string
+	for _, r := range ran {
+		if r[0] == "docker" && r[1] == "run" {
+			raised = r
+		}
+	}
+	line := strings.Join(raised, " ")
+	for _, want := range []string{"--name gitea ", "--volume " + giteaDataDir + ":/data", " " + giteaImage} {
+		if !strings.Contains(line+" ", want) {
+			t.Errorf("the forge is not raised with %q: %s", want, line)
+		}
+	}
+	if giteaBootstrap != ForgeModule {
+		t.Errorf("the bootstrap forge is %q and the module names its container %q", giteaBootstrap, ForgeModule)
+	}
+
+	// Against the module's own manifest, where the catalogue is checked out beside this repository.
+	var manifest []byte
+	for _, candidate := range []string{"../../../mesh-catalog/modules/gitea/module.json", "../../../../../mesh-catalog/modules/gitea/module.json"} {
+		if raw, err := os.ReadFile(filepath.Clean(candidate)); err == nil {
+			manifest = raw
+			break
+		}
+	}
+	if manifest == nil {
+		t.Skip("the catalogue is not beside this checkout; the module's pin is not compared")
+	}
+	var m struct {
+		Resources []struct {
+			ID, Type, Name, Image string
+			Volumes               []string
+		} `json:"resources"`
+	}
+	if err := json.Unmarshal(manifest, &m); err != nil {
+		t.Fatal(err)
+	}
+	for _, r := range m.Resources {
+		if r.Type != "container" || r.ID != "server" {
+			continue
+		}
+		if r.Name != giteaBootstrap {
+			t.Errorf("the module names its container %q; genesis raises %q", r.Name, giteaBootstrap)
+		}
+		if r.Image != giteaImage {
+			t.Errorf("the module pins %s; genesis raises %s — the two must move together", r.Image, giteaImage)
+		}
+		if len(r.Volumes) != 1 || !strings.HasSuffix(r.Volumes[0], ":/data") {
+			t.Errorf("the module mounts %v; genesis mounts %s:/data", r.Volumes, giteaDataDir)
+		}
+	}
+}

internal/bootstrap/inuse_test.go

@@ -91,7 +91,7 @@ func TestARerunOfGenesisIsNotAMachineInUse(t *testing.T) {
 	if err := store.Save(o.State, store.State{Resources: []store.Applied{{ID: "store", Type: "container", Target: "mesh-store"}}}); err != nil {
 		t.Fatal(err)
 	}
-	m := inUseRunner{ps: "mesh-gitea-server\t\n", ss: servingSockets}
+	m := inUseRunner{ps: giteaBootstrap + "\t\n", ss: servingSockets}
 	if err := RefuseAMachineInUse(context.Background(), o, m.run, quietly); err != nil {
 		t.Errorf("what an earlier genesis raised was counted as a machine in use: %v", err)
 	}

internal/bootstrap/phase_packages.go

@@ -25,11 +25,24 @@ const (
 	// foundationStore is the foundation's postgres container — the mesh's own memory, raised from the
 	// bundle. gitea's bootstrap database lives here too, so a mesh runs one postgres (issue 051).
 	foundationStore = "mesh-store"
-	// giteaBootstrap is the gitea server raised directly at genesis, before gitea is a module.
-	giteaBootstrap = "mesh-gitea-server"
-	// giteaImage is the same upstream image the gitea module runs, pinned identically so the module
-	// adopts the running server rather than replacing it.
-	giteaImage = "gitea/gitea@sha256:dfc61e347c8b582df918f4556401bf2cecdfbdb56c5282ae9488dd76fca3e41c"
+	// giteaBootstrap is the gitea server raised directly at genesis, before gitea is a module —
+	// under the name the gitea MODULE declares for its container, so the module finds it and holds
+	// it rather than raising a second forge beside it (novox/hq ADR 0163, rule 7; issue 090).
+	giteaBootstrap = "gitea"
+	// giteaImage is the image the gitea module declares for that container, pinned to the same
+	// digest, so taking the module over is not a downgrade and not an upgrade. **Moves with the
+	// module's pin**: the two are compared by a take, and a difference is said there — but a
+	// genesis that raised an older image than the module declares would be taken over as an
+	// upgrade on first push, which a forge holding the mesh's packages must not have done to it
+	// unannounced. Checked in TestGenesisRaisesTheForgeAsTheModuleDeclaresIt against the module's
+	// manifest where the catalogue is beside this checkout.
+	giteaImage = "gitea/gitea@sha256:87a67ee09d3ae0d1df5fda5dcda3e2a1f9236a45b0a59025d6e00e46adc43bef"
+	// giteaDataDir is where the module's `data` directory resolves on a machine with the default
+	// layout (<data root>/<module>/<id>, novox/hq ADR 0112): mounted at /data as the module mounts
+	// it, so the repositories, attachments and indexes the bootstrap forge accumulates are the
+	// module's the day it is taken — before this, the forge had no volume and its data was the
+	// container's, lost with it.
+	giteaDataDir = "/var/lib/gitea/data"
 	// packagesOrg is the npm owner: every module consumes `@novox/*` from this gitea org.
 	packagesOrg = "novox"
 	// packagesTeam is the org team whose members may read and write the org's packages.
@@ -262,9 +275,13 @@ func raiseGiteaServer(ctx context.Context, run Runner, timeout time.Duration, db
 		"run", "-d", "--name", giteaBootstrap,
 		// Host network, like the control plane: it reaches the foundation store on the machine's
 		// loopback (where the store publishes 5432) and answers on the machine's own 3000, which is
-		// where mesh-bootstrap and the builder's build containers look for it.
+		// where mesh-bootstrap and the builder's build containers look for it. The module runs
+		// bridged and publishes its ports; that is the one difference a take still has to say
+		// (ADR 0163, rule 7) — the data, the name and the image are the module's already.
 		"--network", "host",
 		"--restart", "unless-stopped",
+		// The module's data directory, so what the forge accumulates is the module's when taken.
+		"--volume", giteaDataDir + ":/data",
 	}, env...)
 	args = append(args, giteaImage)
 

internal/declaration/declaration.go

@@ -940,6 +940,13 @@ type Container struct {
 	// its siblings can name before any of them can resolve anything.
 	Dns []string `json:"dns,omitempty"`
 
+	// Networks are networks this container also joins once created, by name — a found network a
+	// per-machine setting keeps for a taken container (novox/hq ADR 0163, rule 4), so a
+	// neighbour that resolves it there keeps resolving it until the neighbour is taken too.
+	// Joined after creation, because a runtime starts a container on one network; part of the
+	// container's spec, so a network kept or let go recreates it.
+	Networks []string `json:"networks,omitempty"`
+
 	// IP is this container's address on its network, passed to the runtime unchanged.
 	//
 	// Only meaningful on a user-defined network, and refused by the runtime elsewhere. Exists for
@@ -1032,6 +1039,16 @@ func (c *Container) validate(where string, _ bool) []string {
 				"static address anywhere but a user-defined one")
 		}
 	}
+	for _, n := range c.Networks {
+		problems = append(problems, (&Network{Name: n}).validate(where+": networks", false)...)
+		if n == c.Network {
+			problems = append(problems, where+": networks names "+n+", which is already the container's network")
+		}
+	}
+	if len(c.Networks) > 0 && (c.RunOnce || c.Schedule != "") {
+		problems = append(problems, where+": networks is for a container that keeps running; a step "+
+			"runs and exits, and joins nothing afterwards")
+	}
 	return append(problems, checkImage(where, c.Image)...)
 }
 
@@ -1150,6 +1167,28 @@ type Declaration struct {
 	// backlog longer than the batch, or a slow broker, applied a declaration the mesh had already
 	// superseded.
 	Sequence int64
+
+	// LeftOut names the modules of this machine's set the mesh left out of this declaration,
+	// because a setting stored for one cannot compose with its definition (novox/hq ADR 0163,
+	// rule 6). A machine is told everything or nothing about what it IS told; this is what it is
+	// not told, said. The host keeps what it holds for a left-out module and touches none of
+	// what it wrote for it — its resources are absent from the declaration, and absence would
+	// otherwise read as removal.
+	LeftOut []string
+}
+
+// LeftOutModuleOf says which left-out module a recorded resource belongs to, if any: its id is the
+// module's name, a dot, and the module's own id for it. A module's name may contain a dot, so the
+// longest left-out name that prefixes the id wins; a false match keeps a thing an apply would
+// otherwise remove, which is the conservative mistake.
+func (d *Declaration) LeftOutModuleOf(id string) (string, bool) {
+	best := ""
+	for _, m := range d.LeftOut {
+		if strings.HasPrefix(id, m+".") && len(m) > len(best) {
+			best = m
+		}
+	}
+	return best, best != ""
 }
 
 // Adoption is a node's mode, as the controller records it: the node is adopted, and these are
@@ -1290,6 +1329,8 @@ type envelope struct {
 	// Sequence is optional on the wire, so a controller that does not send one is still
 	// understood: absent reads as zero, which is "no ordering claimed" rather than "first".
 	Sequence int64 `json:"sequence,omitempty"`
+	// LeftOut is optional on the wire too, and absent when nothing was left out (ADR 0163).
+	LeftOut []string `json:"left_out,omitempty"`
 }
 
 func parse(raw []byte, allowActions bool) (*Declaration, error) {
@@ -1306,8 +1347,20 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) {
 			env.Version, Version)}}
 	}
 
-	d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption, Sequence: env.Sequence}
+	d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption, Sequence: env.Sequence,
+		LeftOut: env.LeftOut}
 	var problems []string
+	if len(env.LeftOut) > 0 && allowActions {
+		// The bundle is carried with the binary and leaves nothing out: which module a setting
+		// stopped composing for is the mesh's record (ADR 0163).
+		problems = append(problems, "a carried bundle says modules were left out, and only the "+
+			"mesh can say that")
+	}
+	for _, m := range env.LeftOut {
+		if strings.TrimSpace(m) == "" {
+			problems = append(problems, "left_out names a module with no name")
+		}
+	}
 
 	if len(env.Resources) == 0 && !env.OwnsNothing {
 		problems = append(problems, "no resources. An empty declaration is a mistake, not a "+

internal/declaration/declaration_test.go

@@ -464,3 +464,43 @@ func TestAnExplicitlyEmptyDeclarationIsAccepted(t *testing.T) {
 		t.Fatalf("an unmarked empty declaration must still be refused; got %v", err)
 	}
 }
+
+// A container's kept networks are names, not its own network, and not for a step (novox/hq ADR
+// 0163, rule 4); and the mesh may say which modules it left out, which a carried bundle may not.
+func TestKeptNetworksAndLeftOutModulesAreReadStrictly(t *testing.T) {
+	pinnedImage := "postgres@sha256:" + strings.Repeat("a", 64)
+	d, err := Parse([]byte(`{"declaration":1,"left_out":["web"],"resources":[
+	  {"id":"app","type":"container","name":"app","image":"` + pinnedImage + `","networks":["predecessor_default"]}
+	]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := d.Resources[0].(*Container).Networks; len(got) != 1 || got[0] != "predecessor_default" {
+		t.Fatalf("the kept network was not read: %v", got)
+	}
+	if m, left := d.LeftOutModuleOf("web.server"); !left || m != "web" {
+		t.Fatalf("web.server is not web's: %q %v", m, left)
+	}
+	if _, left := d.LeftOutModuleOf("webapp.server"); left {
+		t.Fatal("webapp.server was taken for web's")
+	}
+	for name, raw := range map[string]string{
+		"a bad network name": `{"declaration":1,"resources":[
+		  {"id":"app","type":"container","name":"app","image":"` + pinnedImage + `","networks":["a/b"]}]}`,
+		"its own network": `{"declaration":1,"resources":[
+		  {"id":"app","type":"container","name":"app","image":"` + pinnedImage + `","network":"own","networks":["own"]}]}`,
+		"a step": `{"declaration":1,"resources":[
+		  {"id":"app","type":"container","name":"app","image":"` + pinnedImage + `","run-once":true,"networks":["x"]}]}`,
+		"a nameless module": `{"declaration":1,"left_out":[""],"resources":[
+		  {"id":"app","type":"container","name":"app","image":"` + pinnedImage + `"}]}`,
+	} {
+		if _, err := Parse([]byte(raw)); err == nil {
+			t.Errorf("%s was accepted", name)
+		}
+	}
+	if _, err := ParseTrusted([]byte(`{"declaration":1,"left_out":["web"],"resources":[
+	  {"id":"app","type":"container","name":"app","image":"` + pinnedImage + `"}]}`)); err == nil ||
+		!strings.Contains(err.Error(), "only the mesh can say that") {
+		t.Fatalf("a carried bundle leaving modules out was accepted: %v", err)
+	}
+}