From 0e9035d4792b850a866a8aefe74b9850199192c8 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 17 Sep 2026 23:05:23 +0200 Subject: [PATCH 1/3] The network carries the registry trust (ADR 0082, issues 042/048) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Being on the private network is what grants a machine the right to pull from the mesh's artifact store, so the module that puts a machine on the network writes the runtime's trust — a merged /etc/docker/daemon.json naming the store's internal name under insecure-registries, and a docker.service restart when that fact first lands. The registry speaks plain HTTP because every path to it is already inside the overlay's encryption; the provider is found, not configured — whichever module serves artifact-store, on whichever machine holds it — and with no store on the network nothing is written, which is genesis. https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- cmd/mesh-controller/network.go | 45 ++++++++++++++++++++++++ internal/overlay/generator.go | 33 +++++++++++++++++- internal/overlay/generator_test.go | 56 ++++++++++++++++++++++++++++++ 3 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 internal/overlay/generator_test.go diff --git a/cmd/mesh-controller/network.go b/cmd/mesh-controller/network.go index 7c561b8..e883298 100644 --- a/cmd/mesh-controller/network.go +++ b/cmd/mesh-controller/network.go @@ -152,6 +152,15 @@ func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool, return overlay.Empty(), nil } g, err := overlay.From(nodes, overlayCIDR(), "") + if g != nil { + // The artifact store, as this network reaches it. Found rather than configured: the + // provider is whichever module offers it, on whichever machine holds that module — and if + // nothing does yet (genesis raises the registry before the catalogue knows it), there is + // no trust to write and nothing is written (novox/hq ADR 0082). + if at, port, found := artifactStoreOnNetwork(ctx, inv, on); found { + g.TrustRegistry(overlay.InternalName(at) + ":" + port) + } + } if err != nil && len(refused) > 0 { // The network is missing something, and some machines could not be resolved at all. Those // are almost always the same fact: a node that does not resolve contributes nothing, so @@ -383,3 +392,39 @@ func namesInTheMesh(ctx context.Context, inv *inventory.Inventory) (map[string]s } return out, nil } + +// artifactStoreOnNetwork is the machine and port the mesh's artifact store answers on, when a +// module providing it is assigned to a machine that is on the private network. +func artifactStoreOnNetwork(ctx context.Context, inv *inventory.Inventory, + on map[string]bool) (node, port string, found bool) { + + shelf, err := inv.Catalogue(ctx) + if err != nil || shelf == nil { + return "", "", false + } + providers := map[string]string{} // module -> served port + for name, m := range shelf { + served, offers := m.Serves[catalogue.ArtifactStoreProvision] + if !offers { + continue + } + if p, ok := served["port"]; ok { + providers[name] = fmt.Sprintf("%v", p) + } + } + if len(providers) == 0 { + return "", "", false + } + for machine := range on { + assigned, err := inv.Assigned(ctx, machine) + if err != nil { + continue + } + for _, a := range assigned { + if p, ok := providers[a]; ok { + return machine, p, true + } + } + } + return "", "", false +} diff --git a/internal/overlay/generator.go b/internal/overlay/generator.go index 3b10895..54995ed 100644 --- a/internal/overlay/generator.go +++ b/internal/overlay/generator.go @@ -83,8 +83,17 @@ type Generator struct { // keyPath is where each node keeps the private half it generated. Named rather than carried: // the mesh has never seen it and never will. keyPath string + // registry is the mesh's artifact store as the network reaches it (host:port), or empty when + // the mesh has none. Being on the network is what grants a machine the right to pull from it + // (novox/hq ADR 0082), so the module that puts a machine on the network is what writes the + // runtime's trust — the same reasoning that has it write /etc/hosts. + registry string } +// TrustRegistry names the artifact store this network's machines pull from in the clear — +// the overlay is the transport security (ADR 0082). +func (g *Generator) TrustRegistry(hostPort string) { g.registry = hostPort } + // From builds a generator over the machines that are part of the network. // // The nodes given are the ones assigned the module — not every node the mesh knows. A machine @@ -123,7 +132,29 @@ func (g *Generator) Resources(node string) ([]map[string]any, bool, error) { if err := json.Unmarshal(raw, &parsed); err != nil { return nil, false, err } - return parsed.Resources, true, nil + resources := parsed.Resources + if g.registry != "" { + trust, err := json.Marshal(map[string]any{"insecure-registries": []string{g.registry}}) + if err != nil { + return nil, false, err + } + resources = append(resources, + map[string]any{ + // Merged, not owned: the runtime's daemon file is the machine's, and this states + // one fact into it. The registry speaks plain HTTP because every path to it is + // already inside the overlay's encryption (ADR 0082) — this line is the runtime + // being told what the mesh already means. + "id": "registry-trust", "type": "file", "path": "/etc/docker/daemon.json", + "content": string(trust) + "\n", "mode": "0644", "merge": "json", + }, + map[string]any{ + // The runtime reloads nothing for this setting, so it is restarted when the fact + // changes — once, at joining, before the machine runs anything that would mind. + "id": "registry-trust-reload", "type": "service", "unit": "docker.service", + "state": "running", "restart-on": []string{"registry-trust"}, + }) + } + return resources, true, nil } // Nodes are the machines this generator was built over, so a caller can say who is on the network. diff --git a/internal/overlay/generator_test.go b/internal/overlay/generator_test.go new file mode 100644 index 0000000..0cc935a --- /dev/null +++ b/internal/overlay/generator_test.go @@ -0,0 +1,56 @@ +package overlay + +import ( + "strings" + "testing" +) + +func TestTheNetworkCarriesRegistryTrust(t *testing.T) { + // novox/hq ADR 0082: being on the network is what grants a machine the right to pull from the + // mesh's artifact store in the clear, so the network module writes the runtime's trust — and + // writes nothing when the mesh has no store to trust. + nodes := []Node{ + {Name: "anchor", Site: "lab", Hub: true, Endpoint: "192.0.2.10:51820", Key: "k1", Address: "10.42.0.1"}, + {Name: "node2", Site: "lab", Key: "k2", Address: "10.42.0.2"}, + } + g, err := From(nodes, "10.42.0.0/16", "") + if err != nil { + t.Fatal(err) + } + plain, _, err := g.Resources("node2") + if err != nil { + t.Fatal(err) + } + for _, r := range plain { + if r["id"] == "registry-trust" { + t.Fatal("trust was written with no artifact store to trust") + } + } + + g.TrustRegistry("anchor.internal:5000") + trusted, part, err := g.Resources("node2") + if err != nil || !part { + t.Fatalf("resources: %v part=%v", err, part) + } + var file, service map[string]any + for _, r := range trusted { + switch r["id"] { + case "registry-trust": + file = r + case "registry-trust-reload": + service = r + } + } + if file == nil || service == nil { + t.Fatalf("the trust file or its reload is missing: %v", trusted) + } + if file["path"] != "/etc/docker/daemon.json" || file["merge"] != "json" { + t.Fatalf("the trust is not a merged daemon.json: %v", file) + } + if content, _ := file["content"].(string); !strings.Contains(content, `"anchor.internal:5000"`) { + t.Fatalf("the trust does not name the store: %v", file["content"]) + } + if service["unit"] != "docker.service" { + t.Fatalf("the reload does not restart the runtime: %v", service) + } +} -- 2.54.0 From bc289cbe04d411df270b5132580fe3d9f2b2dd69 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 17 Sep 2026 23:35:12 +0200 Subject: [PATCH 2/3] The restart-on rename reads both list shapes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A resource composed in code carries restart-on as []string; the rename only read []any, so the overlay's registry-trust reload kept its bare reference, pointed at nothing, and the runtime was never restarted — the trust was on disk and not in the daemon, with every check passing. Diagnosed on the built-store-cross-node bed, run 8 (issues 042/048). --- internal/catalogue/declaration.go | 41 ++++++++++++++++----- internal/catalogue/reflects_renamed_test.go | 32 ++++++++++++++++ 2 files changed, 63 insertions(+), 10 deletions(-) create mode 100644 internal/catalogue/reflects_renamed_test.go diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index 2204a62..76339dc 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -484,16 +484,7 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) { // the daemon's. Written `.`, and a dot is what marks it as already // answered: prefixing it again would point at nothing, silently, and the daemon would // serve the old names for ever while everything reported success. - if reflects, ok := resource["restart-on"].([]any); ok { - var renamed []any - for _, id := range reflects { - named := fmt.Sprint(id) - if strings.Contains(named, ".") { - renamed = append(renamed, named) - continue - } - renamed = append(renamed, m.Module+"."+named) - } + if renamed := reflectsRenamed(m.Module, resource["restart-on"]); renamed != nil { copied["restart-on"] = renamed } out = append(out, copied) @@ -566,6 +557,36 @@ type Contribution struct { // Named after both. Named after the machine alone, two modules on one node wrote to one path: the // second overwrote the first, and the provisioner — reading a directory — saw one consumer where // there were two. +// reflectsRenamed is a resource's restart-on list under the module's prefix, or nil when it has +// none. It reads both the shape JSON parsing produces ([]any) and the shape code composing +// resources natively produces ([]string): a reference that was skipped because its list arrived +// in the other shape would point at nothing — silently, with the service never restarting and +// every check passing, which is how a runtime kept serving without the registry trust its +// daemon file already carried. +func reflectsRenamed(module string, reflects any) []any { + var names []string + switch v := reflects.(type) { + case []any: + for _, id := range v { + names = append(names, fmt.Sprint(id)) + } + case []string: + names = v + } + if names == nil { + return nil + } + var renamed []any + for _, named := range names { + if strings.Contains(named, ".") { + renamed = append(renamed, named) + continue + } + renamed = append(renamed, module+"."+named) + } + return renamed +} + func grantPath(directory, consumer, module string) string { return strings.TrimRight(directory, "/") + "/" + consumer + "." + module + ".secret" } diff --git a/internal/catalogue/reflects_renamed_test.go b/internal/catalogue/reflects_renamed_test.go new file mode 100644 index 0000000..e5bee0b --- /dev/null +++ b/internal/catalogue/reflects_renamed_test.go @@ -0,0 +1,32 @@ +package catalogue + +import ( + "reflect" + "testing" +) + +func TestReflectsRenamedReadsBothShapes(t *testing.T) { + // The list arrives as []any when the resource was parsed from JSON, and as []string when it + // was composed in code — the overlay's registry trust is the second kind. A shape that was + // skipped would leave the reference unprefixed, pointing at nothing, and the service would + // never restart while every check passed (novox/hq issues 042/048, the run-8 diagnosis). + parsed := reflectsRenamed("mesh-wireguard", []any{"registry-trust"}) + if !reflect.DeepEqual(parsed, []any{"mesh-wireguard.registry-trust"}) { + t.Fatalf("parsed shape: %v", parsed) + } + composed := reflectsRenamed("mesh-wireguard", []string{"registry-trust"}) + if !reflect.DeepEqual(composed, []any{"mesh-wireguard.registry-trust"}) { + t.Fatalf("composed shape: %v", composed) + } + + // A name already under a module keeps it: that is how a service reflects a file another + // module put on the machine. + kept := reflectsRenamed("resolver", []string{"mesh-wireguard.fact-node-names", "own-config"}) + if !reflect.DeepEqual(kept, []any{"mesh-wireguard.fact-node-names", "resolver.own-config"}) { + t.Fatalf("dotted name was not kept: %v", kept) + } + + if got := reflectsRenamed("any", nil); got != nil { + t.Fatalf("no list should rename to nothing, got %v", got) + } +} -- 2.54.0 From 98aea8b25ca0d906577b57c3213e1dbee82b2a6e Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 18 Sep 2026 00:14:49 +0200 Subject: [PATCH 3/3] An artifact-store lookup failure refuses the compose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit artifactStoreOnNetwork collapsed a failed inventory read into 'no store', so a hiccup composed a declaration without the registry trust, delivered by a push that reported success — and nothing recomposed the machine until the next push. Seen once in three fresh runs of the built-store-cross-node bed (run 11). Refused loudly instead: 'no store' now only ever means the mesh has none. --- cmd/mesh-controller/network.go | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/cmd/mesh-controller/network.go b/cmd/mesh-controller/network.go index e883298..79ddc8c 100644 --- a/cmd/mesh-controller/network.go +++ b/cmd/mesh-controller/network.go @@ -157,7 +157,16 @@ func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool, // provider is whichever module offers it, on whichever machine holds that module — and if // nothing does yet (genesis raises the registry before the catalogue knows it), there is // no trust to write and nothing is written (novox/hq ADR 0082). - if at, port, found := artifactStoreOnNetwork(ctx, inv, on); found { + // + // Refused rather than composed without it when the question could not be answered: a + // declaration missing the trust because a lookup failed is a machine that cannot pull, + // delivered by a push that reported success — and nothing recomposes it until the next + // push (the shape of novox/hq issues 042/048, reappearing as a race). + at, port, found, storeErr := artifactStoreOnNetwork(ctx, inv, on) + if storeErr != nil { + return nil, fmt.Errorf("finding the artifact store this network reaches: %w", storeErr) + } + if found { g.TrustRegistry(overlay.InternalName(at) + ":" + port) } } @@ -395,12 +404,17 @@ func namesInTheMesh(ctx context.Context, inv *inventory.Inventory) (map[string]s // artifactStoreOnNetwork is the machine and port the mesh's artifact store answers on, when a // module providing it is assigned to a machine that is on the private network. +// +// A lookup failure is an error, never "not found": collapsing the two composed a declaration +// without the trust whenever the inventory hiccuped, delivered by a push that reported success — +// and nothing recomposed the machine until the next push. "No store" must mean the mesh has none, +// not that the question went unanswered. func artifactStoreOnNetwork(ctx context.Context, inv *inventory.Inventory, - on map[string]bool) (node, port string, found bool) { + on map[string]bool) (node, port string, found bool, err error) { shelf, err := inv.Catalogue(ctx) - if err != nil || shelf == nil { - return "", "", false + if err != nil { + return "", "", false, fmt.Errorf("reading the catalogue: %w", err) } providers := map[string]string{} // module -> served port for name, m := range shelf { @@ -413,18 +427,18 @@ func artifactStoreOnNetwork(ctx context.Context, inv *inventory.Inventory, } } if len(providers) == 0 { - return "", "", false + return "", "", false, nil } for machine := range on { assigned, err := inv.Assigned(ctx, machine) if err != nil { - continue + return "", "", false, fmt.Errorf("reading what %s is assigned: %w", machine, err) } for _, a := range assigned { if p, ok := providers[a]; ok { - return machine, p, true + return machine, p, true, nil } } } - return "", "", false + return "", "", false, nil } -- 2.54.0