apply: container and network resources, over the container runtime

Extends the applier past the filesystem to the two types the workloads
need: a container and the private network it joins. The workloads are
the bulk of what a cutover re-declares (research 009), so this is what
makes a workload manifest actually appliable.

- container: run/reconcile/remove over the runtime. Up to date means a
  container that is ours (a spec-hash label matches this exact
  declaration) AND running; anything else — a changed spec, a stopped
  container, or a foreign one the old control plane left by that name —
  is recreated into ours. Safe because a container carries no state:
  its data is in bind-mounted directories declared separately, and
  recreating it never touches them. Read-back asks the runtime whether
  it is actually running on the declared spec, because 'started' only
  means the runtime returned.
- network: create if absent, adopt if present, remove only what it
  created.
- The runtime is driven through a Runner, faked in unit tests and
  exercised for real in a smoke test that stands a container up, proves
  idempotency, and tears it down — skipped, never failed, where the
  runtime is absent.

The store's per-resource reference generalises from a path to a ref:
a path for files and directories, a name for containers and networks.

Verified end to end through the binary: a container on a bind mount,
then dropped from the declaration — the container is removed and the
data directory survives, which is the migration property itself.

Still deferred: sealed secrets, and package/service/archive/user/action
— refused whole until built, never half-applied.

Claude-Session: https://claude.ai/code/session_01LrgweAeERJYBg88c5cKDzF
This commit is contained in:
2026-09-02 22:48:27 +02:00
parent 7414da96ed
commit c80f671203
7 changed files with 524 additions and 20 deletions
+7 -5
View File
@@ -1,9 +1,11 @@
// Command mesh-host is tier 0 of the Novox Mesh: the one thing installed by hand, and the
// only thing that changes a machine.
//
// Stage 1 (novox/hq 03-DESIGN/01-to-be/05-the-node-host.md) is profile and inventory only —
// the host reads what this machine can do and what it is, and reports it. It applies nothing,
// connects to nothing, and listens on nothing.
// It reports (profile, inventory) and it applies (novox/hq 03-DESIGN/01-to-be/05-the-node-host.md).
// `apply` takes a declaration and makes this machine match it, from a local file, with no mesh
// present — the network-free resource types first (directories, files, containers, networks).
// It still connects to nothing and listens on nothing: the link to the control plane, sealed
// secrets, and the remaining resource types are designed and not yet built.
package main
import (
@@ -173,10 +175,10 @@ func runApply(opts options) error {
return writeJSON(res)
}
for _, r := range res.Applied {
fmt.Printf(" applied %-10s %s\n", r.Type, r.Path)
fmt.Printf(" applied %-10s %s\n", r.Type, r.Ref)
}
for _, r := range res.Removed {
fmt.Printf(" removed %-10s %s\n", r.Type, r.Path)
fmt.Printf(" removed %-10s %s\n", r.Type, r.Ref)
}
fmt.Printf("\n%d applied, %d removed\n", len(res.Applied), len(res.Removed))
return nil
+1 -1
View File
@@ -68,7 +68,7 @@ func Apply(decl Declaration, identity string, store *Store, appliers map[string]
return Result{}, fmt.Errorf("applying %s %q: %w", r.Type, r.ID, err)
}
applied = append(applied, Record{
ID: r.ID, Type: r.Type, Path: r.Path(),
ID: r.ID, Type: r.Type, Ref: r.ref(),
Created: created || priorCreated[r.ID],
})
}
+219
View File
@@ -0,0 +1,219 @@
package apply
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"os/exec"
"strings"
)
// containerRuntime is the runtime the mesh uses. Named rather than assumed, so the one place that
// would change for podman is here — and profile already detects that a container-runtime is
// present before any container resource is placed on a node.
const containerRuntime = "docker"
// specLabel carries a hash of the desired container spec. It is what lets a re-apply tell an
// up-to-date container from one that must be recreated, without parsing the runtime's own view of
// env, ports and mounts and hoping the comparison matches field for field. The host wrote the
// hash; the host trusts the hash.
const specLabel = "mesh-host.spec"
// Runner executes a container-runtime command. Replaceable in tests, and exercised against the
// real runtime as well (novox/hq ADR 0034): a test that only fakes the runtime asserts the fake
// behaves as expected, so the real path is smoke-tested too.
type Runner func(name string, args ...string) (string, error)
func execRunner(name string, args ...string) (string, error) {
out, err := exec.Command(name, args...).CombinedOutput()
if err != nil {
return string(out), fmt.Errorf("%s %s: %w: %s",
name, strings.Join(args, " "), err, strings.TrimSpace(string(out)))
}
return string(out), nil
}
// --- network ---
type networkApplier struct{ run Runner }
func (networkApplier) Type() string { return "network" }
func (n networkApplier) Apply(r Resource) (bool, error) {
name := r.stringField("name")
if _, err := n.run(containerRuntime, "network", "inspect", name); err == nil {
return false, nil // already present — adopted, not created
}
if _, err := n.run(containerRuntime, "network", "create", name); err != nil {
return false, fmt.Errorf("creating network %s: %w", name, err)
}
// Read back: the network must now be inspectable, or the create did not take.
if _, err := n.run(containerRuntime, "network", "inspect", name); err != nil {
return true, fmt.Errorf("network %s did not take: %w", name, err)
}
return true, nil
}
func (n networkApplier) Remove(rec Record) error {
if _, err := n.run(containerRuntime, "network", "rm", rec.Ref); err != nil {
if alreadyGone(err) {
return nil
}
return fmt.Errorf("removing network %s: %w", rec.Ref, err)
}
return nil
}
// --- container ---
type containerApplier struct{ run Runner }
func (containerApplier) Type() string { return "container" }
func (c containerApplier) Apply(r Resource) (bool, error) {
name := r.stringField("name")
image := r.stringField("image")
if image == "" {
return false, fmt.Errorf("container %q names no image", name)
}
hash := specHash(r)
exists, running, curHash := c.inspect(name)
// A container is up to date only when it is ours (its spec label matches this exact spec) AND
// it is running. A foreign container by the same name — one the old control plane started, with
// no label — does not match, and is recreated into ours. That is safe: a container carries no
// state, its data lives in bind-mounted directories declared separately, and recreating it does
// not touch them.
upToDate := exists && running && curHash == hash
if upToDate {
return false, nil // present and correct; this apply created nothing
}
if exists {
if _, err := c.run(containerRuntime, "rm", "-f", name); err != nil {
return false, fmt.Errorf("replacing container %s: %w", name, err)
}
}
if _, err := c.run(containerRuntime, runArgs(r, hash)...); err != nil {
return true, fmt.Errorf("starting container %s: %w", name, err)
}
// Read back: the container must now be running, on this exact spec. "Service started" only
// means the runtime returned — the host asks whether it is actually up (novox/hq
// troubleshooting/service-started-is-not-ready).
exists, running, curHash = c.inspect(name)
if !exists || !running {
return true, fmt.Errorf("container %s did not come up", name)
}
if curHash != hash {
return true, fmt.Errorf("container %s came up on a spec that is not the one declared", name)
}
return true, nil
}
func (c containerApplier) Remove(rec Record) error {
if _, err := c.run(containerRuntime, "rm", "-f", rec.Ref); err != nil {
if alreadyGone(err) {
return nil
}
return fmt.Errorf("removing container %s: %w", rec.Ref, err)
}
return nil
}
// alreadyGone reports whether a removal failed only because the thing was not there — which is
// success, not failure. The runtime phrases it variously ("No such container", "no such object",
// "not found") across versions, so the match is lenient and case-insensitive.
func alreadyGone(err error) bool {
msg := strings.ToLower(err.Error())
return strings.Contains(msg, "no such") || strings.Contains(msg, "not found")
}
// inspect reports whether a container by this name exists, whether it is running, and the spec
// hash it was labelled with (empty for a container the host did not label).
func (c containerApplier) inspect(name string) (exists, running bool, hash string) {
out, err := c.run(containerRuntime, "inspect", "-f",
"{{.State.Running}}|{{index .Config.Labels \""+specLabel+"\"}}", name)
if err != nil {
return false, false, ""
}
parts := strings.SplitN(strings.TrimSpace(out), "|", 2)
running = parts[0] == "true"
if len(parts) == 2 && parts[1] != "<no value>" {
hash = parts[1]
}
return true, running, hash
}
// runArgs builds the `docker run` invocation for r, labelled with its spec hash.
func runArgs(r Resource, hash string) []string {
args := []string{"run", "-d", "--name", r.stringField("name"), "--label", specLabel + "=" + hash}
if net := r.stringField("network"); net != "" {
args = append(args, "--network", net)
}
for _, ef := range r.stringSlice("env-file") {
args = append(args, "--env-file", ef)
}
// Env is emitted in the JSON-sorted order specHash also uses, so the invocation is stable.
env := r.stringMap("env")
for _, k := range sortedKeys(env) {
args = append(args, "-e", k+"="+env[k])
}
for _, p := range r.stringSlice("ports") {
args = append(args, "-p", p)
}
for _, v := range r.stringSlice("volumes") {
args = append(args, "-v", v)
}
for _, h := range r.stringSlice("hosts") {
args = append(args, "--add-host", h)
}
args = append(args, r.stringField("image"))
args = append(args, r.stringSlice("args")...)
return args
}
// specHash is a stable fingerprint of everything that decides whether a running container matches
// what is declared. Marshalled through a struct so the field set is explicit, and json.Marshal
// sorts map keys, so the same declaration always hashes the same.
func specHash(r Resource) string {
type spec struct {
Name string `json:"name"`
Image string `json:"image"`
Network string `json:"network"`
Env map[string]string `json:"env"`
EnvFile []string `json:"env_file"`
Ports []string `json:"ports"`
Volumes []string `json:"volumes"`
Hosts []string `json:"hosts"`
Args []string `json:"args"`
}
b, _ := json.Marshal(spec{
Name: r.stringField("name"),
Image: r.stringField("image"),
Network: r.stringField("network"),
Env: r.stringMap("env"),
EnvFile: r.stringSlice("env-file"),
Ports: r.stringSlice("ports"),
Volumes: r.stringSlice("volumes"),
Hosts: r.stringSlice("hosts"),
Args: r.stringSlice("args"),
})
sum := sha256.Sum256(b)
return hex.EncodeToString(sum[:])[:16]
}
func sortedKeys(m map[string]string) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
// small n; insertion order does not matter, only that it is stable and sorted
for i := 1; i < len(out); i++ {
for j := i; j > 0 && out[j-1] > out[j]; j-- {
out[j-1], out[j] = out[j], out[j-1]
}
}
return out
}
+235
View File
@@ -0,0 +1,235 @@
package apply
import (
"fmt"
"os/exec"
"strings"
"testing"
)
// fakeRuntime scripts the container runtime: it records every invocation and answers `inspect`
// from a small in-memory model of which containers exist, whether they run, and their spec label.
type fakeRuntime struct {
calls []string
exists map[string]bool
running map[string]bool
label map[string]string
failOnRun bool
}
func newFakeRuntime() *fakeRuntime {
return &fakeRuntime{
exists: map[string]bool{},
running: map[string]bool{},
label: map[string]string{},
}
}
func (f *fakeRuntime) run(name string, args ...string) (string, error) {
f.calls = append(f.calls, name+" "+strings.Join(args, " "))
switch {
case len(args) >= 1 && args[0] == "inspect":
container := args[len(args)-1]
if !f.exists[container] {
return "", fmt.Errorf("Error: No such object: %s", container)
}
return fmt.Sprintf("%v|%s", f.running[container], f.label[container]), nil
case len(args) >= 1 && args[0] == "run":
if f.failOnRun {
return "", fmt.Errorf("simulated run failure")
}
name, hash := parseRunNameAndLabel(args)
f.exists[name] = true
f.running[name] = true
f.label[name] = hash
return name, nil
case len(args) >= 2 && args[0] == "rm":
container := args[len(args)-1]
delete(f.exists, container)
delete(f.running, container)
delete(f.label, container)
return "", nil
case len(args) >= 2 && args[0] == "network" && args[1] == "inspect":
return "", fmt.Errorf("Error: No such network")
case len(args) >= 2 && args[0] == "network" && args[1] == "create":
return "", nil
}
return "", nil
}
func parseRunNameAndLabel(args []string) (name, hash string) {
for i := 0; i < len(args)-1; i++ {
switch args[i] {
case "--name":
name = args[i+1]
case "--label":
if v, ok := strings.CutPrefix(args[i+1], specLabel+"="); ok {
hash = v
}
}
}
return name, hash
}
func containerResource(t *testing.T, name, image string) Resource {
t.Helper()
d := mustParse(t, `{"version":1,"resources":[
{"id":"`+name+`","type":"container","name":"`+name+`","image":"`+image+`",
"network":"n","env":{"A":"1","B":"2"},"ports":["3000"],
"volumes":["/services/x/data:/data"]}]}`)
return d.Resources[0]
}
// A container that does not exist is created; applying the same declaration again finds it up to
// date and starts nothing.
func TestContainerApplyCreatesThenIsIdempotent(t *testing.T) {
f := newFakeRuntime()
c := containerApplier{run: f.run}
r := containerResource(t, "gitea", "gitea/gitea@sha256:abc")
created, err := c.Apply(r)
if err != nil {
t.Fatal(err)
}
if !created {
t.Fatal("first apply did not report creating the container")
}
if countCalls(f.calls, "run") != 1 {
t.Fatalf("expected exactly one run, got calls: %v", f.calls)
}
f.calls = nil
created, err = c.Apply(r)
if err != nil {
t.Fatal(err)
}
if created {
t.Fatal("second apply recreated an up-to-date container")
}
if countCalls(f.calls, "run") != 0 {
t.Fatalf("idempotent apply still ran the container: %v", f.calls)
}
}
// A container whose spec changed is recreated: the old one removed, a new one started.
func TestContainerRecreatedWhenSpecChanges(t *testing.T) {
f := newFakeRuntime()
c := containerApplier{run: f.run}
if _, err := c.Apply(containerResource(t, "gitea", "gitea/gitea@sha256:old")); err != nil {
t.Fatal(err)
}
f.calls = nil
// A new image is a new spec hash.
if _, err := c.Apply(containerResource(t, "gitea", "gitea/gitea@sha256:new")); err != nil {
t.Fatal(err)
}
if countCalls(f.calls, "rm") != 1 || countCalls(f.calls, "run") != 1 {
t.Fatalf("a changed spec should remove and recreate; calls: %v", f.calls)
}
}
// A foreign container by the same name — no spec label — is taken over, because a container holds
// no state; its data is in bind mounts declared separately.
func TestForeignContainerIsTakenOver(t *testing.T) {
f := newFakeRuntime()
f.exists["gitea"] = true
f.running["gitea"] = true
f.label["gitea"] = "" // started by something else, unlabelled
c := containerApplier{run: f.run}
created, err := c.Apply(containerResource(t, "gitea", "gitea/gitea@sha256:abc"))
if err != nil {
t.Fatal(err)
}
if !created {
t.Fatal("taking over a foreign container should count as creating ours")
}
if countCalls(f.calls, "rm") != 1 {
t.Fatalf("the foreign container should have been replaced: %v", f.calls)
}
}
// runArgs carries every declared facet through to the invocation, env in sorted order.
func TestRunArgsAreComplete(t *testing.T) {
r := containerResource(t, "gitea", "img@sha256:abc")
args := strings.Join(runArgs(r, "hash123"), " ")
for _, want := range []string{
"run -d --name gitea --label mesh-host.spec=hash123",
"--network n", "--env-file", "-e A=1 -e B=2", "-p 3000",
"-v /services/x/data:/data", "img@sha256:abc",
} {
if want == "--env-file" {
continue // this resource declares none; the others must all be present
}
if !strings.Contains(args, want) {
t.Errorf("run args missing %q\n got: %s", want, args)
}
}
}
// A container resource that names no image is refused rather than started blank.
func TestContainerWithoutImageIsRefused(t *testing.T) {
// Built directly: the shape allows omitting image, but the applier must not.
d := mustParse(t, `{"version":1,"resources":[
{"id":"x","type":"container","name":"x"}]}`)
f := newFakeRuntime()
if _, err := (containerApplier{run: f.run}).Apply(d.Resources[0]); err == nil {
t.Fatal("a container with no image was started")
}
}
func countCalls(calls []string, verb string) int {
n := 0
for _, c := range calls {
// docker <verb> ... — verb is the first arg after the runtime name.
fields := strings.Fields(c)
if len(fields) >= 2 && fields[1] == verb {
n++
}
}
return n
}
// Smoke test against the real runtime: a container really comes up, is idempotent, and is
// removed — the read-back path this whole design rests on, exercised for real. Skipped where the
// runtime or its image is not available, never failed for the environment (novox/hq ADR 0034).
func TestContainerAgainstRealRuntime(t *testing.T) {
if _, err := exec.LookPath(containerRuntime); err != nil {
t.Skipf("%s not installed", containerRuntime)
}
if out, err := exec.Command(containerRuntime, "run", "--rm", "alpine", "true").CombinedOutput(); err != nil {
t.Skipf("cannot run a probe container (no image/daemon): %s", strings.TrimSpace(string(out)))
}
name := "mesh-host-apply-smoke"
_ = exec.Command(containerRuntime, "rm", "-f", name).Run()
t.Cleanup(func() { _ = exec.Command(containerRuntime, "rm", "-f", name).Run() })
r := mustParse(t, `{"version":1,"resources":[
{"id":"c","type":"container","name":"`+name+`","image":"alpine","args":["sleep","30"]}]}`).Resources[0]
c := containerApplier{run: execRunner}
created, err := c.Apply(r)
if err != nil {
t.Fatalf("real apply failed: %v", err)
}
if !created {
t.Fatal("first real apply did not create the container")
}
// Idempotent: the running container is left alone.
created, err = c.Apply(r)
if err != nil {
t.Fatalf("real re-apply failed: %v", err)
}
if created {
t.Fatal("real re-apply recreated an up-to-date container")
}
// Removal really removes it.
if err := c.Remove(Record{ID: "c", Type: "container", Ref: name, Created: true}); err != nil {
t.Fatalf("real remove failed: %v", err)
}
if out, _ := exec.Command(containerRuntime, "inspect", name).CombinedOutput(); !strings.Contains(strings.ToLower(string(out)), "no such") {
t.Fatalf("container still present after removal: %s", out)
}
}
+42 -6
View File
@@ -53,12 +53,21 @@ type Resource struct {
Fields map[string]json.RawMessage
}
// Path is the host-owned filesystem path this resource lives at. Every type this host applies
// so far is addressed by a path, and the store needs it to remove the resource later.
// Path is the host-owned filesystem path a file or directory resource lives at.
func (r Resource) Path() string {
return r.stringField("path")
}
// ref is what the store records to find this resource again for removal: a filesystem path for
// files and directories, a name for containers and networks. Every resource is addressed by one
// or the other, and the parser refuses a resource that has neither.
func (r Resource) ref() string {
if p := r.stringField("path"); p != "" {
return p
}
return r.stringField("name")
}
func (r Resource) stringField(key string) string {
raw, ok := r.Fields[key]
if !ok {
@@ -71,6 +80,29 @@ func (r Resource) stringField(key string) string {
return s
}
// stringSlice reads a field that is a JSON array of strings — ports, volumes, args, hosts.
func (r Resource) stringSlice(key string) []string {
raw, ok := r.Fields[key]
if !ok {
return nil
}
var out []string
_ = json.Unmarshal(raw, &out)
return out
}
// stringMap reads a field that is a JSON object of string→string — a container's env. Keys are
// returned sorted by the caller when order matters, so a container's spec hash is stable.
func (r Resource) stringMap(key string) map[string]string {
raw, ok := r.Fields[key]
if !ok {
return nil
}
var out map[string]string
_ = json.Unmarshal(raw, &out)
return out
}
// shape is the set of field keys a resource type may carry, beyond the common id and type.
// This is the host's half of a wire contract whose other half is the control plane's catalogue
// (novox/mesh-control examples/modules/modules_test.go). Duplicated deliberately, because the
@@ -84,6 +116,8 @@ func (r Resource) stringField(key string) string {
var shapes = map[string][]string{
"directory": {"path", "mode", "owner"},
"file": {"path", "content", "mode", "owner"},
"network": {"name"},
"container": {"name", "image", "env", "env-file", "ports", "volumes", "args", "hosts", "network"},
}
// Parse reads a declaration and refuses anything it does not fully understand.
@@ -167,11 +201,13 @@ func parseResource(raw json.RawMessage) (Resource, error) {
}
}
if _, hasPath := extra["path"]; !hasPath {
return Resource{}, fmt.Errorf("%q is a %s and names no path", id, typ)
res := Resource{ID: id, Type: typ, Fields: extra}
if res.ref() == "" {
return Resource{}, fmt.Errorf(
"%q is a %s and names neither a path nor a name — the store would have no way to find "+
"it again", id, typ)
}
return Resource{ID: id, Type: typ, Fields: extra}, nil
return res, nil
}
func decodeString(raw json.RawMessage) string {
+15 -5
View File
@@ -26,11 +26,21 @@ type Applier interface {
Remove(rec Record) error
}
// Appliers is the set of types this host can apply, keyed by type name.
// Appliers is the set of types this host can apply, keyed by type name, using the real container
// runtime.
func Appliers() map[string]Applier {
return appliersWith(execRunner)
}
// appliersWith builds the applier set against a given runtime runner. The filesystem appliers
// ignore it; the container and network ones drive the runtime through it, which is where a test
// substitutes a fake.
func appliersWith(run Runner) map[string]Applier {
return map[string]Applier{
"directory": directoryApplier{},
"file": fileApplier{},
"network": networkApplier{run: run},
"container": containerApplier{run: run},
}
}
@@ -82,12 +92,12 @@ func (directoryApplier) Remove(rec Record) error {
// os.Remove, never RemoveAll: it fails on a non-empty directory, and that failure is the
// point. A directory the host created but that now holds something is not the host's to
// delete — data outlives the mesh that declared it (ADR 0030).
err := os.Remove(rec.Path)
err := os.Remove(rec.Ref)
if os.IsNotExist(err) {
return nil
}
if err != nil {
return fmt.Errorf("removing directory %s (left in place): %w", rec.Path, err)
return fmt.Errorf("removing directory %s (left in place): %w", rec.Ref, err)
}
return nil
}
@@ -148,12 +158,12 @@ func (fileApplier) Apply(r Resource) (bool, error) {
}
func (fileApplier) Remove(rec Record) error {
err := os.Remove(rec.Path)
err := os.Remove(rec.Ref)
if os.IsNotExist(err) {
return nil
}
if err != nil {
return fmt.Errorf("removing file %s: %w", rec.Path, err)
return fmt.Errorf("removing file %s: %w", rec.Ref, err)
}
return nil
}
+5 -3
View File
@@ -28,9 +28,11 @@ type Store struct {
// removed, which is the same rule that ADR 0018 exists to enforce: never act on a path you did
// not create.
type Record struct {
ID string `json:"id"`
Type string `json:"type"`
Path string `json:"path"`
ID string `json:"id"`
Type string `json:"type"`
// Ref is how the resource is found again for removal: a filesystem path for files and
// directories, a container or network name for those.
Ref string `json:"ref"`
Created bool `json:"created"`
}