No container is given the mesh's names; it resolves them

novox/hq ADR 0148, step 3. Every container got the whole roster as
--add-host entries at creation and nothing re-read them (issues 109,
135); once the roster was in the digest so that could be caught, one
name moving anywhere replaced every container in the mesh (issue 151).
A container resolves through its machine's resolver, which the resolver
module tells the runtime about once per machine. A module's own hosts
entries stay exactly as declared.

Also brings the resolver tests up to the catalogue as it now is: the
runtime is reloaded (never restarted) and given live-restore, and the
resolver answers by address, not by interface (issue 110).

cmd/mesh-controller/push.go

@@ -338,6 +338,12 @@ func pushCommand(ctx context.Context, args []string) error {
 	sentDigest := map[string]string{}
 	defer release()
 	for _, s := range sending {
+		// Numbered under the hold, one higher than the last, before the body exists — the number is
+		// inside the signed bytes, so a replayed older declaration cannot borrow a newer one's
+		// (novox/hq 04-ISSUES/107).
+		if err := number(ctx, inv, &s); err != nil {
+			return err
+		}
 		body, err := s.declared.Body()
 		if err != nil {
 			return err
@@ -564,6 +570,9 @@ func sendRound(ctx context.Context, open *stores, names []string,
 		return compose(held, node)
 	})
 	for _, s := range sending {
+		if err := number(ctx, open.inventory, &s); err != nil {
+			return refused, err
+		}
 		body, err := s.declared.Body()
 		if err != nil {
 			return refused, err
@@ -645,6 +654,9 @@ func sendTo(ctx context.Context, open *stores, names []string) error {
 	defer server.Close()
 
 	for _, s := range sending {
+		if err := number(ctx, inv, &s); err != nil {
+			return err
+		}
 		body, err := s.declared.Body()
 		if err != nil {
 			return err
@@ -694,6 +706,12 @@ func wouldSend(ctx context.Context, open *stores,
 		if err != nil {
 			continue
 		}
+		// Composed with the number the machine was LAST sent, so this is byte for byte what it was
+		// sent when nothing else changed. A fresh number here would make every machine read as
+		// behind for ever (novox/hq 04-ISSUES/107).
+		if declared.Sequence, err = open.inventory.Sequence(ctx, n.ID); err != nil {
+			return nil, err
+		}
 		body, err := declared.Body()
 		if err != nil {
 			return nil, err
@@ -827,3 +845,17 @@ func seatHolders(ctx context.Context, inv *inventory.Inventory) (map[string]brok
 	}
 	return out, nil
 }
+
+// number gives one send the next sequence for its node (novox/hq 04-ISSUES/107).
+func number(ctx context.Context, inv *inventory.Inventory, s *readyNode) error {
+	record, err := inv.NodeByName(ctx, s.node)
+	if err != nil {
+		return err
+	}
+	seq, err := inv.NextSequence(ctx, record.ID)
+	if err != nil {
+		return err
+	}
+	s.declared.Sequence = seq
+	return nil
+}

cmd/mesh-controller/sendable.go

@@ -18,6 +18,10 @@ import (
 // make a machine look out of date for ever, or send something `plan` never showed.
 type sendable struct {
 	Resources []map[string]any
+	// Sequence orders this send against every other to the same node: one higher each time, taken
+	// under the node's hold just before the body is made (novox/hq 04-ISSUES/107). Zero is not sent
+	// at all, which a host reads as "no order claimed" — the shape of every declaration before this.
+	Sequence int64
 	// Adoption is nil for a converged node, and then the body is byte for byte what it was before
 	// adoption existed: an older host parses the envelope strictly and would refuse the key.
 	Adoption *adoptionEnvelope
@@ -38,6 +42,9 @@ func (s sendable) Body() ([]byte, error) {
 	if s.Adoption != nil {
 		envelope["adoption"] = s.Adoption
 	}
+	if s.Sequence > 0 {
+		envelope["sequence"] = s.Sequence
+	}
 	// An empty declaration is deliberate here — the node owns nothing the mesh put there
 	// (novox/hq issue 127) — and the host refuses an empty body unless it is told the emptiness
 	// is meant, so a truncated or mis-composed body is never mistaken for "own nothing".

cmd/mesh-controller/sequence_test.go

@@ -0,0 +1,77 @@
+package main
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+// A declaration's only identity was the digest of its bytes; the controller already held a per-node
+// lock and recorded each send, so the order existed and was thrown away at the wire (novox/hq
+// 04-ISSUES/107).
+
+func TestASendCarriesItsNumberInsideTheSignedBytes(t *testing.T) {
+	body, err := sendable{Resources: []map[string]any{{"id": "x", "type": "file"}}, Sequence: 7}.Body()
+	if err != nil {
+		t.Fatal(err)
+	}
+	var env map[string]any
+	if err := json.Unmarshal(body, &env); err != nil {
+		t.Fatal(err)
+	}
+	if got, _ := env["sequence"].(float64); got != 7 {
+		t.Fatalf("the body carries sequence %v, wanted 7", env["sequence"])
+	}
+}
+
+func TestAnUnnumberedSendIsByteForByteWhatItWasBefore(t *testing.T) {
+	// Zero is not sent at all. A host reads absence as "no order claimed" — the shape of every
+	// declaration before this — so an older host, or the read-only comparison against a machine
+	// sent nothing since sends were numbered, sees exactly the bytes it always saw.
+	body, err := sendable{Resources: []map[string]any{{"id": "x", "type": "file"}}}.Body()
+	if err != nil {
+		t.Fatal(err)
+	}
+	var env map[string]any
+	if err := json.Unmarshal(body, &env); err != nil {
+		t.Fatal(err)
+	}
+	if _, present := env["sequence"]; present {
+		t.Fatalf("a send numbered zero put a sequence on the wire: %s", body)
+	}
+}
+
+func TestEachSendToANodeIsOneHigherAndReadable(t *testing.T) {
+	open := aMesh(t)
+	record, err := open.inventory.NodeByName(t.Context(), "anchor")
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Sent nothing since numbering existed: what it would be sent is composed with zero, which is
+	// not on the wire, which is what it was actually sent.
+	if n, err := open.inventory.Sequence(t.Context(), record.ID); err != nil || n != 0 {
+		t.Fatalf("a fresh node reads sequence %d, %v", n, err)
+	}
+	first, err := open.inventory.NextSequence(t.Context(), record.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	second, err := open.inventory.NextSequence(t.Context(), record.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if first != 1 || second != 2 {
+		t.Fatalf("two sends were numbered %d and %d", first, second)
+	}
+	// And the read path sees the last one taken, so the comparison composes what was sent.
+	if n, err := open.inventory.Sequence(t.Context(), record.ID); err != nil || n != 2 {
+		t.Fatalf("after two sends the node reads sequence %d, %v", n, err)
+	}
+	// Another node counts on its own.
+	other, err := open.inventory.NodeByName(t.Context(), "laptop")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n, err := open.inventory.NextSequence(t.Context(), other.ID); err != nil || n != 1 {
+		t.Fatalf("a second node's first send was numbered %d, %v", n, err)
+	}
+}

internal/builder/binary_name_test.go

@@ -0,0 +1,44 @@
+package builder
+
+import (
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// The name a machine runs a binary by is not always the name of the package that built it. The host's
+// command is cmd/mesh-host and every machine runs it as nox-mesh-host — the path it is installed at,
+// the name in its unit, and the name its launcher looks for inside a delivered version.
+//
+// A bundle carrying the package's name was delivered to a machine correctly, reported "created … 1
+// file(s)", and was invisible to the launcher (novox/hq 04-ISSUES/142). Found by reading the delivered
+// directory rather than by trusting the line that said it worked.
+
+func TestACompiledArtifactNamesTheBinaryAMachineWillRun(t *testing.T) {
+	got := binaryName(catalogue.Artifact{
+		Name: "host-arch", From: "cmd/mesh-host", Binary: "nox-mesh-host",
+	})
+	if got != "nox-mesh-host" {
+		t.Fatalf("the binary is named %q, and the launcher looks for nox-mesh-host", got)
+	}
+}
+
+func TestSayingNothingKeepsWhatTheCompilerWouldHaveChosen(t *testing.T) {
+	// go build names its output after the package, so an artifact that says nothing gets the same
+	// thing it got before this existed.
+	if got := binaryName(catalogue.Artifact{Name: "host-arch", From: "cmd/mesh-host"}); got != "mesh-host" {
+		t.Fatalf("an artifact naming no binary produced %q", got)
+	}
+	if got := binaryName(catalogue.Artifact{Name: "host-arch", From: "./cmd/agent/"}); got != "agent" {
+		t.Fatalf("a from with slashes produced %q", got)
+	}
+}
+
+func TestABundleBuiltFromTheModuleRootFallsBackToItsArtifactName(t *testing.T) {
+	// A single-command repository names no package, and `go build -o <dir>` would then write a file
+	// named after the module directory — which is not something the manifest states. The artifact's
+	// own name is what the manifest does state.
+	if got := binaryName(catalogue.Artifact{Name: "tool"}); got != "tool" {
+		t.Fatalf("a bundle built from the root produced %q", got)
+	}
+}

internal/builder/builder.go

@@ -877,8 +877,32 @@ func compile(ctx context.Context, run Runner, tree string, chain Toolchain,
 		base,
 	}
 	invocation = append(invocation, chain.Compile...)
+	// **One `-ldflags`, composed here.** A repeated flag is not a merged one: the Go command takes
+	// the last and drops the first, so passing the toolchain's flags and then the system stamp as a
+	// second `-ldflags` produced a binary that knew its system and had lost `-s -w` — half again the
+	// size, with its debug info (novox/hq 04-ISSUES/161).
+	//
+	// What it was built for is the one thing taken from the artifact, and ADR 0142 says why: the
+	// target is a property of the artifact rather than of the recipe. A host with no system refuses
+	// every declaration before it applies anything.
+	linker := append([]string(nil), chain.LinkerFlags...)
+	if chain.SystemStamp != "" && strings.TrimSpace(a.System) != "" {
+		linker = append(linker, "-X", chain.SystemStamp+"="+strings.TrimSpace(a.System))
+	}
+	if len(linker) > 0 {
+		invocation = append(invocation, "-ldflags", strings.Join(linker, " "))
+	}
 	if chain.OutputFlag != "" {
-		invocation = append(invocation, chain.OutputFlag, out)
+		// A compiler pointed at a package is told the file to write, not the directory: the name a
+		// machine runs it by is not always the name of the package that built it. The host's command
+		// is `cmd/mesh-host` and every machine runs it as `nox-mesh-host` — so a bundle carrying the
+		// package's name lands correctly, reports success, and is invisible to whatever looks for it
+		// (novox/hq 04-ISSUES/142).
+		target := out
+		if chain.Unit == UnitPackage {
+			target = filepath.Join(out, binaryName(a))
+		}
+		invocation = append(invocation, chain.OutputFlag, target)
 	}
 	// What to compile. Named by the module rather than discovered, so adding a file does not
 	// silently change what a build produces.
@@ -1067,3 +1091,15 @@ func readBy(manifest catalogue.Manifest) []catalogue.ArtifactContext {
 	})
 	return out
 }
+
+// binaryName is what a compiled bundle's executable is called: what the artifact says, or the name of
+// the package it is built from, which is what a compiler would have chosen anyway.
+func binaryName(a catalogue.Artifact) string {
+	if name := strings.TrimSpace(a.Binary); name != "" {
+		return name
+	}
+	if from := strings.Trim(a.From, "./"); from != "" {
+		return filepath.Base(from)
+	}
+	return a.Name
+}

internal/builder/system_stamp_test.go

@@ -0,0 +1,76 @@
+package builder
+
+import (
+	"strings"
+	"testing"
+)
+
+// A host built without knowing its system refuses every declaration before applying anything —
+// safely, totally, and with nothing reporting it. The mesh built one, delivered it, started it, and
+// it would have refused the first thing it was asked to do (novox/hq 04-ISSUES/161).
+
+func TestTheGoToolchainStampsTheArtifactsSystem(t *testing.T) {
+	chain, err := ToolchainFor("go")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if chain.SystemStamp != "main.builtFor" {
+		t.Fatalf("the go toolchain fills %q", chain.SystemStamp)
+	}
+}
+
+func TestALanguageWithNoPinnedSystemStampsNothing(t *testing.T) {
+	// Interpreted output is not pinned to a system, and a manifest declaring one for it is already
+	// refused. Nothing to fill.
+	for _, language := range []string{"typescript", "python"} {
+		chain, err := ToolchainFor(language)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if chain.SystemStamp != "" {
+			t.Fatalf("%s fills %q, and its output is not pinned to a system",
+				language, chain.SystemStamp)
+		}
+	}
+}
+
+func TestTheStampIsTheOneThingTakenFromTheArtifact(t *testing.T) {
+	// The toolchain accepts nothing else from the module — anything it could override it would be
+	// writing a Dockerfile to override. The system is the stated exception, because a compiled
+	// binary is per system and the artifact is what declares one (ADR 0142).
+	chain, err := ToolchainFor("go")
+	if err != nil {
+		t.Fatal(err)
+	}
+	joined := strings.Join(chain.Compile, " ")
+	if strings.Contains(joined, "${") || strings.Contains(joined, "%s") {
+		t.Fatalf("the compile line takes something from the module: %q", joined)
+	}
+}
+
+func TestTheLinkerIsToldOnceNotTwice(t *testing.T) {
+	// A repeated flag is not a merged one: the Go command takes the last -ldflags and drops the
+	// first. Passing the toolchain's flags and then the stamp separately produced a binary that knew
+	// its system and had lost -s -w — 12.2MB against 8.5MB, with its debug info (04-ISSUES/161).
+	chain, err := ToolchainFor("go")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, arg := range chain.Compile {
+		if arg == "-ldflags" {
+			t.Fatal("the compile line carries -ldflags, so composing one here makes two")
+		}
+	}
+	if len(chain.LinkerFlags) == 0 {
+		t.Fatal("the go toolchain passes no linker flags, so the binary keeps its debug info")
+	}
+	var stripped bool
+	for _, f := range chain.LinkerFlags {
+		if f == "-s" {
+			stripped = true
+		}
+	}
+	if !stripped {
+		t.Fatalf("the go toolchain does not strip: %v", chain.LinkerFlags)
+	}
+}

internal/builder/toolchain.go

@@ -49,6 +49,26 @@ type Toolchain struct {
 	// is named as it will be FOUND, inside the unpacked bundle, so the source is the same path with
 	// the output directory taken off the front and this on the end.
 	SourceExt string
+	// LinkerFlags are passed to the linker as one flag, together with the system stamp below.
+	//
+	// **Separate from Compile because a repeated flag is not a merged one.** They were in the compile
+	// line, and appending the stamp as a second `-ldflags` meant the Go command took the last and
+	// dropped the first — so the binary gained its system and lost `-s -w`, growing by half and
+	// carrying its debug info. The mistake was believing a comment rather than reading the file it
+	// produced (novox/hq 04-ISSUES/161).
+	LinkerFlags []string
+	// SystemStamp is the variable this language's linker fills with the artifact's declared system,
+	// for a language whose binaries are pinned to one at link time (novox/hq ADR 0005).
+	//
+	// **The one thing a toolchain takes from the artifact, and 0142 says why**: the target is a
+	// property of the artifact rather than of the recipe, because a compiled binary is per system
+	// and a toolchain that accepted it from the module would be accepting a build instruction. This
+	// is the narrow exception, named here rather than inferred.
+	//
+	// Empty for a language that compiles to nothing pinned. A host built without it refuses every
+	// declaration before applying anything — safely, totally, and with nothing reporting it
+	// (novox/hq 04-ISSUES/161).
+	SystemStamp string
 }
 
 // What a toolchain is pointed at.
@@ -114,14 +134,20 @@ var toolchains = []Toolchain{
 		// rather than from the linker: two builds of one commit produce the same bytes.
 		Compile: []string{
 			"env", "CGO_ENABLED=0", "GOFLAGS=-trimpath",
-			"go", "build", "-ldflags", "-s -w",
+			"go", "build",
 		},
-		OutputFlag: "-o",
+		// Stripped of symbols and debug info: what a machine holds is a file it runs, not one it
+		// debugs, and the difference measured 12.2MB against 8.5MB.
+		LinkerFlags: []string{"-s", "-w"},
+		OutputFlag:  "-o",
 		// Pointed at the package the artifact is built `from`, compiled whole. Go writes the binary
 		// into the output directory, named after the package — so the bundle a machine unpacks is a
 		// directory holding one executable, which is what the delivery mechanism expects
 		// (novox/hq ADR 0141).
 		Unit: UnitPackage,
+		// The mesh's own Go components read the system they were built for from this variable, and
+		// refuse to touch a machine without one.
+		SystemStamp: "main.builtFor",
 	},
 	{
 		Language: "python",

internal/catalogue/declaration.go

@@ -618,17 +618,15 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri
 		// merging earlier would throw away the files it still needs.
 		resources = append(append([]map[string]any{}, first...), resources...)
 
-		// Every container is given the mesh's names. Not a choice a module makes: a module that
+		// No container is given the mesh's names (novox/hq ADR 0148). It used to be: every
-		// listed them would go stale the day a machine joins, and one that did not would be a
+		// container got the whole roster as `--add-host` entries at creation, and a name that
-		// module whose containers cannot reach anything by name.
+		// moved afterwards was wrong inside it for as long as it ran (issues 109, 135) — and once
-		//
+		// the roster was made part of a container's identity so that could be caught, one name
-		// A container that was given names of its own keeps them and gets the mesh's beside them:
+		// moving anywhere replaced every container in the mesh (issue 151). A container resolves a
-		// the mesh does not know what else a workload needs to reach, and taking something away
+		// mesh name through its machine's resolver at the moment it asks, which the runtime is
-		// to add something is not what "also" means.
+		// told once per machine, as a file, by the resolver's own module. The names a module
-		if len(with.Names) > 0 {
+		// declares for itself are its own and stay exactly as written: they are part of what the
-			resources = withMeshNames(resources, with.Names)
+		// module is, and the mesh does not know what they mean.
-		}
-
 		// What this module may name from inside one of its own files. Gathered once per module
 		// rather than per file, because it is a fact about the module.
 		sealed, err := sealedFor(m, r.Needs, with)
@@ -1482,43 +1480,6 @@ func (r Resolution) servedOnThisMachine(provision string, with Rendering) (map[s
 	return nil, false, nil
 }
 
-// withMeshNames gives every container in a set the mesh's names.
-//
-// Copied rather than edited in place: these maps come from a module's manifest, and mutating one
-// would change what the catalogue holds for every other machine running that module.
-//
-// A host-network container gets the names too. It was once skipped, on the belief that it "shares
-// the machine's hosts file already" — but it does not: `docker run --network host` still gives the
-// container its own /etc/hosts (localhost and its own id only), so every `<node>.internal` name the
-// mesh wrote for the machine is invisible inside it, and a client that dials one gets EAI_AGAIN. The
-// remedy is the same `--add-host` every other container gets — the runtime accepts it with
-// `--network host` (verified), and without it a host-network consumer cannot reach a provider by the
-// `.internal` address the mesh hands it as `${bound:...:at}`.
-func withMeshNames(resources []map[string]any, names map[string]string) []map[string]any {
-	out := make([]map[string]any, 0, len(resources))
-	for _, r := range resources {
-		if r["type"] != "container" {
-			out = append(out, r)
-			continue
-		}
-
-		copied := map[string]any{}
-		for k, v := range r {
-			copied[k] = v
-		}
-		var given []any
-		if already, ok := copied["hosts"].([]any); ok {
-			given = append(given, already...)
-		}
-		for _, name := range sortedKeys(names) {
-			given = append(given, name+":"+names[name])
-		}
-		copied["hosts"] = given
-		out = append(out, copied)
-	}
-	return out
-}
-
 // pinned refuses an image that is not really pinned, on its way to a machine.
 //
 // **Here and not at parse** (novox/hq 04-ISSUES/025). A manifest in a repository names artifacts

internal/catalogue/manifest.go

@@ -597,6 +597,16 @@ type Artifact struct {
 	// Empty for every other kind, which do not compile.
 	Language string `json:"language,omitempty"`
 
+	// Binary is what the compiled executable is called, for a bundle in a language that compiles to
+	// one. Empty means the package's own name, which is what a compiler does by default.
+	//
+	// **Because the name a machine runs it by is not always the name of the package that built it.**
+	// The host's command is `cmd/mesh-host` and every machine runs it as `nox-mesh-host` — the path
+	// it is installed at, the name in its unit, and the name its launcher looks for inside a
+	// delivered version. A bundle that carried the package's name was delivered correctly, reported
+	// success, and was invisible to the launcher (novox/hq 04-ISSUES/142).
+	Binary string `json:"binary,omitempty"`
+
 	// Entrypoints are the compiled files a tool host should load from this module, relative to the
 	// bundle's root.
 	//

internal/catalogue/names_test.go

@@ -1,6 +1,7 @@
 package catalogue
 
 import (
+	"reflect"
 	"strings"
 	"testing"
 )
@@ -30,36 +31,38 @@ func namesOf(r map[string]any) []string {
 	return out
 }
 
-// A container does not inherit the machine's names, so the mesh gives them to it.
+// No mesh name is written into a container (novox/hq ADR 0148). It resolves them through its
+// machine's resolver at the moment it asks, so a name that moves is answered differently by the
+// next lookup, in every container, with nothing recreated.
 //
-// It gets its own hosts file holding only its own hostname — every internal name the mesh wrote
+// Checked the way the record says: the declaration a container gets does not move when the mesh's
-// for the machine is invisible to what the machine runs. A database client on one node could not
+// roster does. A roster with one machine and a roster with three produce the same container, byte
-// resolve another node, on a mesh where both names were correct and present on both machines.
+// for byte, so the digest a host computes from it cannot move either — which is what stopped one
-func TestEveryContainerIsGivenTheMeshsNames(t *testing.T) {
+// name moving from replacing every container in the mesh (issue 151).
-	got := containersOf(t, Resolution{Node: "laptop", Modules: []Manifest{{
+func TestAContainerIsTheSameWhateverTheMeshsRosterSays(t *testing.T) {
-		Module: "app",
+	module := Manifest{Module: "app", Resources: []map[string]any{{"id": "web", "type": "container",
-		Resources: []map[string]any{{"id": "web", "type": "container", "name": "web",
+		"name": "web", "image": "registry.example/web@sha256:" + strings.Repeat("a", 64)}}}
-			"image": "registry.example/web@sha256:" + strings.Repeat("a", 64)}},
+	one := containersOf(t, Resolution{Node: "laptop", Modules: []Manifest{module}},
-	}}}, Rendering{Names: map[string]string{
+		Rendering{Names: map[string]string{"laptop.internal": "10.42.0.2"}})
-		"anchor.internal": "10.42.0.1", "laptop.internal": "10.42.0.2",
+	three := containersOf(t, Resolution{Node: "laptop", Modules: []Manifest{module}},
-	}})
+		Rendering{Names: map[string]string{
-	if len(got) != 1 {
+			"anchor.internal": "10.42.0.1", "laptop.internal": "10.42.0.2", "git.example.tld": "10.42.0.1",
-		t.Fatalf("expected one container, got %d", len(got))
+		}})
+	if len(one) != 1 || len(three) != 1 {
+		t.Fatalf("expected one container each, got %d and %d", len(one), len(three))
 	}
-	given := namesOf(got[0])
+	if given := namesOf(three[0]); len(given) != 0 {
-	if len(given) != 2 {
+		t.Fatalf("the mesh's names were copied into the container: %v", given)
-		t.Fatalf("the container was given %d name(s): %v", len(given), given)
 	}
-	if given[0] != "anchor.internal:10.42.0.1" {
+	if !reflect.DeepEqual(one[0], three[0]) {
-		t.Fatalf("the name is not in the form a runtime writes: %v", given)
+		t.Fatalf("the container moved with the roster:\n%v\n%v", one[0], three[0])
 	}
 }
 
-// A container that named its own keeps them and gets the mesh's beside them.
+// The names a module declares for itself are its own: part of what the module is, kept exactly as
-//
+// written, and the mesh does not know what they mean. They are the one thing in a container's
-// The mesh does not know what else a workload needs to reach, and taking something away in order
+// hosts that does move its identity, because they do not move when the mesh's roster does.
-// to add something is not what "also" means.
+func TestAContainersOwnNamesAreKeptAsWritten(t *testing.T) {
-func TestAContainersOwnNamesAreKept(t *testing.T) {
 	got := containersOf(t, Resolution{Node: "laptop", Modules: []Manifest{{
 		Module: "app",
 		Resources: []map[string]any{{"id": "web", "type": "container", "name": "web",
@@ -68,62 +71,15 @@ func TestAContainersOwnNamesAreKept(t *testing.T) {
 	}}}, Rendering{Names: map[string]string{"anchor.internal": "10.42.0.1"}})
 
 	given := namesOf(got[0])
-	if len(given) != 2 || given[0] != "something.else:203.0.113.9" {
+	if len(given) != 1 || given[0] != "something.else:203.0.113.9" {
-		t.Fatalf("the container's own names were lost: %v", given)
+		t.Fatalf("the container's own names were not kept as written: %v", given)
 	}
 }
 
-// A container on the machine's own network gets the names too — it does NOT share the machine's
+// No resource is given a `hosts` key it did not declare. A file or a service carrying one is a
-// hosts file. `docker run --network host` still gives the container its own /etc/hosts (localhost
+// declaration the host refuses outright — it takes no unknown field — so an invented key breaks
-// and its own id only), so every `<node>.internal` name the mesh wrote is invisible inside it, and a
+// the whole machine rather than one resource.
-// client that dials one gets EAI_AGAIN. It gets the same `--add-host` entries every other container
+func TestNothingIsGivenNamesItDidNotDeclare(t *testing.T) {
-// gets (the runtime accepts them with `--network host`), so a host-network consumer can reach a
-// provider by the `.internal` address the mesh hands it.
-func TestAContainerOnTheMachinesNetworkIsGivenTheNamesToo(t *testing.T) {
-	got := containersOf(t, Resolution{Node: "anchor", Modules: []Manifest{{
-		Module: "control",
-		Resources: []map[string]any{{"id": "c", "type": "container", "name": "c",
-			"image": "registry.example/c@sha256:" + strings.Repeat("a", 64), "network": "host"}},
-	}}}, Rendering{Names: map[string]string{"anchor.internal": "10.42.0.1"}})
-
-	given := namesOf(got[0])
-	if len(given) != 1 || given[0] != "anchor.internal:10.42.0.1" {
-		t.Fatalf("a host-networked container was not given the mesh's names: %v", got[0])
-	}
-}
-
-// A mesh with no private network gives nothing, rather than a name with no address behind it.
-func TestAMeshWithNoNamesGivesNone(t *testing.T) {
-	got := containersOf(t, Resolution{Node: "alone", Modules: []Manifest{{
-		Module: "app",
-		Resources: []map[string]any{{"id": "web", "type": "container", "name": "web",
-			"image": "registry.example/web@sha256:" + strings.Repeat("a", 64)}},
-	}}}, Rendering{})
-	if len(namesOf(got[0])) != 0 {
-		t.Fatalf("names were invented for a mesh that has none: %v", got[0])
-	}
-}
-
-// The catalogue's copy is not edited: these maps come from a manifest, and mutating one would
-// change what every other machine running that module is given.
-func TestGivingNamesDoesNotChangeTheCatalogue(t *testing.T) {
-	held := map[string]any{"id": "web", "type": "container", "name": "web",
-		"image": "registry.example/web@sha256:" + strings.Repeat("a", 64)}
-	module := Manifest{Module: "app", Resources: []map[string]any{held}}
-
-	for _, node := range []string{"one", "two"} {
-		containersOf(t, Resolution{Node: node, Modules: []Manifest{module}},
-			Rendering{Names: map[string]string{"anchor.internal": "10.42.0.1"}})
-	}
-	if _, changed := held["hosts"]; changed {
-		t.Fatal("the manifest the catalogue holds was edited, so every machine now carries this")
-	}
-}
-
-// Only containers. A file or a service given a `hosts` key is a declaration the host refuses
-// outright — it takes no unknown field — so getting this wrong breaks the whole machine rather
-// than one resource, and breaks it for something that was never about names.
-func TestNothingButAContainerIsGivenNames(t *testing.T) {
 	out, err := Resolution{Node: "laptop", Modules: []Manifest{{
 		Module: "app",
 		Resources: []map[string]any{
@@ -137,11 +93,8 @@ func TestNothingButAContainerIsGivenNames(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, r := range out {
-		if r["type"] == "container" {
-			continue
-		}
 		if _, given := r["hosts"]; given {
-			t.Fatalf("a %v was given names, which the host will refuse: %v", r["type"], r)
+			t.Fatalf("a %v was given names it never declared: %v", r["type"], r)
 		}
 	}
 }

internal/catalogue/port_into.go

@@ -98,8 +98,7 @@ func portInto(resource map[string]any, module string, listens []Listening, with
 
 		// **A fresh map, and only when something changes.** This map came out of the module's
 		// manifest and the resource around it is a shallow copy, so filling a value in place would
-		// change what the catalogue holds for every other machine running the module — the trap
+		// change what the catalogue holds for every other machine running the module.
-		// withMeshNames is written to avoid, one field along.
 		var filled map[string]any
 		for _, key := range named {
 			written, ok := env[key].(string)

internal/catalogue/resolver_manifests_test.go

@@ -48,7 +48,7 @@ func TestTheResolverForwardsToFixedUpstreamsAndNeverReadsResolvConf(t *testing.T
 	}
 	for _, want := range []string{
 		"\nno-resolv\n", "\nserver=1.1.1.1\n", "\nserver=8.8.8.8\n",
-		"\nlisten-address=127.0.0.1\n", "\ninterface=mesh0\n", "\nbind-dynamic\n",
+		"\nlisten-address=127.0.0.1\n", "\nlisten-address=${machine:address}\n", "\nbind-dynamic\n",
 		"\ndomain-needed\n", "\nbogus-priv\n",
 		"\nconf-file=" + m.Facts["node-zones"].Path + "\n",
 	} {
@@ -56,6 +56,14 @@ func TestTheResolverForwardsToFixedUpstreamsAndNeverReadsResolvConf(t *testing.T
 			t.Errorf("the resolver's configuration lacks %q:\n%s", strings.TrimSpace(want), config)
 		}
 	}
+	// By address and never by interface: dnsmasq admits a query by the interface it arrives on
+	// when told one, and a container's query to the private address arrives on the runtime's
+	// bridge — `interface=mesh0` dropped every such query, silently (novox/hq issue 110).
+	for _, line := range strings.Split(config, "\n") {
+		if strings.HasPrefix(line, "interface=") {
+			t.Errorf("the resolver answers by interface, so a container's query on a bridge is dropped: %s", line)
+		}
+	}
 	// Not .53 or .54, which systemd-resolved holds; and not .55 any more, which was a convention
 	// beside the one every machine already followed — the predecessor's resolv.conf says .1.
 	for _, taken := range []string{"127.0.0.53", "127.0.0.54", "127.0.0.55"} {
@@ -137,23 +145,39 @@ func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) {
 		t.Errorf("the daemon does not restart on its configuration and the machines file both: %v", service["restart-on"])
 	}
 
-	// The runtime's own file, written into (novox/hq ADR 0102) with the one key this module states.
+	// The runtime's own file, written into (novox/hq ADR 0102) with the keys this module states:
+	// where containers resolve, and that a restart keeps them running — because the runtime reads
+	// `dns` only when it starts, and the one restart that needs is the operator's (issue 110).
 	runtime := ids["dnsmasq.runtime-dns"]
 	if runtime == nil || runtime["path"] != "/etc/docker/daemon.json" || runtime["into"] != "json" {
 		t.Fatalf("the runtime's dns is not written into its file: %v", runtime)
 	}
-	var keys map[string][]string
+	var keys map[string]any
 	if err := json.Unmarshal([]byte(runtime["content"].(string)), &keys); err != nil {
 		t.Fatalf("the runtime's keys are not JSON: %v", err)
 	}
-	if len(keys) != 1 || len(keys["dns"]) != 1 || keys["dns"][0] != "10.42.0.1" {
+	dns, _ := keys["dns"].([]any)
-		t.Errorf("the runtime is pointed at %v; containers resolve at this machine's own private-network address, and nothing else is written", keys)
+	if len(keys) != 2 || len(dns) != 1 || dns[0] != "10.42.0.1" || keys["live-restore"] != true {
+		t.Errorf("the runtime is given %v; containers resolve at this machine's own private-network address, a restart keeps them, and nothing else is written", keys)
 	}
+	// The runtime is reloaded when that file changes, and never restarted: a restart stops every
+	// container on the machine (ADR 0102), and a reload is what turns live-restore on.
+	var reloaded bool
 	for _, r := range out {
-		if r["type"] == "service" && r["unit"] == "docker.service" && r["id"] != "" &&
+		if r["type"] != "service" || r["unit"] != "docker.service" {
-			strings.HasPrefix(r["id"].(string), "dnsmasq.") {
+			continue
-			t.Errorf("the resolver orders the runtime restarted or reloaded, which stops every container (ADR 0102) or does nothing for dns: %v", r)
 		}
+		if _, restarts := r["restart-on"]; restarts {
+			t.Errorf("the resolver orders the runtime restarted, which stops every container (ADR 0102): %v", r)
+		}
+		for _, on := range asStrings(r["reload-on"]) {
+			if on == "dnsmasq.runtime-dns" {
+				reloaded = true
+			}
+		}
+	}
+	if !reloaded {
+		t.Errorf("the runtime is not reloaded when its file changes, so live-restore never takes effect")
 	}
 
 	resolv := ids["resolv-conf.resolv"]

internal/catalogue/routing_test.go

@@ -198,37 +198,33 @@ func TestTheApexLabelComposesToTheBarePrivateAddress(t *testing.T) {
 	}
 }
 
+// novox/hq ADR 0066 propagate, by ADR 0148's means: a granted route name is published into the
+// machine's roster mapped to the node that serves it, beside the `<node>.internal` names, and the
+// machine's resolver answers it to every container. Nothing is written into the container itself —
+// a routed name that moved would otherwise be wrong inside every container until each was recreated.
 func TestARoutedNameResolvesToTheServingNode(t *testing.T) {
-	// novox/hq ADR 0066 propagate: a granted route name is published into internal resolution,
+	names := map[string]string{
-	// mapped to the node that serves it, alongside the `<node>.internal` names — so every
+		"anchor.internal": "10.42.0.1",
-	// container, and an in-mesh ACME validator, resolves a routed name to the proxy that serves it.
+		"git.example.tld": "10.42.0.1",
-	// The names map is what withMeshNames writes into every container as `--add-host`; a route name
+	}
-	// mapped to the serving node's address rides the same mechanism.
 	got := containersOf(t, Resolution{Node: "anchor", Modules: []Manifest{{
 		Module: "app",
 		Resources: []map[string]any{{"id": "web", "type": "container", "name": "web",
 			"image": "registry.example/web@sha256:" + strings.Repeat("a", 64)}},
-	}}}, Rendering{Names: map[string]string{
+	}}}, Rendering{Names: names})
-		"anchor.internal": "10.42.0.1",
-		"git.example.tld": "10.42.0.1",
-	}})
 	if len(got) != 1 {
 		t.Fatalf("expected one container, got %d", len(got))
 	}
-	given := namesOf(got[0])
+	if given := namesOf(got[0]); len(given) != 0 {
-	var sawNode, sawRoute bool
+		t.Fatalf("the routed name was copied into the container, where it would go stale: %v", given)
-	for _, h := range given {
+	}
-		if h == "anchor.internal:10.42.0.1" {
+	var sawRoute bool
-			sawNode = true
+	for _, e := range entriesFrom(names, nil, "internal") {
-		}
+		if e.Name == "git.example.tld" && e.Address == "10.42.0.1" {
-		if h == "git.example.tld:10.42.0.1" {
 			sawRoute = true
 		}
 	}
-	if !sawNode {
-		t.Fatalf("the container lost the mesh's node names: %v", given)
-	}
 	if !sawRoute {
-		t.Fatalf("the routed name was not published to the serving node: %v", given)
+		t.Fatalf("the routed name is not in the roster the machine's resolver answers from")
 	}
 }

internal/inventory/migrations/0046-each-send-is-numbered.sql

@@ -0,0 +1,11 @@
+-- The order of what a machine was sent, so a host can tell an older declaration from a newer.
+--
+-- novox/hq 04-ISSUES/107. A declaration's only identity was the digest of its bytes. The host could
+-- say "this is not the last one" and could not say "this is older", so a backlog drained out of
+-- order applied a declaration the mesh had already superseded. The controller already holds a
+-- per-node lock while it composes and records each send — the order existed and was thrown away at
+-- the wire.
+--
+-- One counter per node, taken under that hold, one higher per send. Null is a machine sent nothing
+-- since this existed, which is not the same as a machine sent nothing.
+alter table node add column sequence bigint;

internal/inventory/nodes.go

@@ -1005,3 +1005,34 @@ func (i *Inventory) RecordHostVersion(ctx context.Context, id, version string) e
 		`update node set host_version = $2, last_seen = now() where id = $1`, id, version)
 	return err
 }
+
+// NextSequence takes the next number for a declaration to this node, one higher than the last it
+// was sent (novox/hq 04-ISSUES/107).
+//
+// **One statement, so two composers cannot take the same number.** The caller holds the node while it
+// composes and sends, so in practice there is one; the increment is atomic anyway, because a rule
+// that is true only while a lock is held somewhere else is a rule nobody can see from here.
+func (i *Inventory) NextSequence(ctx context.Context, id string) (int64, error) {
+	var n int64
+	err := i.store.Pool().QueryRow(ctx,
+		`update node set sequence = coalesce(sequence, 0) + 1 where id = $1 returning sequence`, id).Scan(&n)
+	if err != nil {
+		return 0, fmt.Errorf("taking the next sequence for %s: %w", id, err)
+	}
+	return n, nil
+}
+
+// Sequence is the number of the last declaration this node was sent, and zero for one sent nothing
+// since sends were numbered. Read, not taken: what the mesh WOULD send is composed with this, so it
+// is byte for byte what it DID send when nothing else changed — a comparison that took a fresh number
+// would read every machine as behind for ever (novox/hq 04-ISSUES/107).
+func (i *Inventory) Sequence(ctx context.Context, id string) (int64, error) {
+	var n *int64
+	if err := i.store.Pool().QueryRow(ctx, `select sequence from node where id = $1`, id).Scan(&n); err != nil {
+		return 0, fmt.Errorf("reading the sequence of %s: %w", id, err)
+	}
+	if n == nil {
+		return 0, nil
+	}
+	return *n, nil
+}