The network carries the registry trust (ADR 0082, issues 042/048/062) #29

Merged
jschoubben merged 3 commits from feat/the-network-carries-registry-trust into main 2026-09-17 23:00:37 +00:00
5 changed files with 210 additions and 11 deletions
+59
View File
@@ -152,6 +152,24 @@ func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool,
return overlay.Empty(), nil return overlay.Empty(), nil
} }
g, err := overlay.From(nodes, overlayCIDR(), "") g, err := overlay.From(nodes, overlayCIDR(), "")
if g != nil {
// The artifact store, as this network reaches it. Found rather than configured: the
// provider is whichever module offers it, on whichever machine holds that module — and if
// nothing does yet (genesis raises the registry before the catalogue knows it), there is
// no trust to write and nothing is written (novox/hq ADR 0082).
//
// Refused rather than composed without it when the question could not be answered: a
// declaration missing the trust because a lookup failed is a machine that cannot pull,
// delivered by a push that reported success — and nothing recomposes it until the next
// push (the shape of novox/hq issues 042/048, reappearing as a race).
at, port, found, storeErr := artifactStoreOnNetwork(ctx, inv, on)
if storeErr != nil {
return nil, fmt.Errorf("finding the artifact store this network reaches: %w", storeErr)
}
if found {
g.TrustRegistry(overlay.InternalName(at) + ":" + port)
}
}
if err != nil && len(refused) > 0 { if err != nil && len(refused) > 0 {
// The network is missing something, and some machines could not be resolved at all. Those // The network is missing something, and some machines could not be resolved at all. Those
// are almost always the same fact: a node that does not resolve contributes nothing, so // are almost always the same fact: a node that does not resolve contributes nothing, so
@@ -383,3 +401,44 @@ func namesInTheMesh(ctx context.Context, inv *inventory.Inventory) (map[string]s
} }
return out, nil return out, nil
} }
// artifactStoreOnNetwork is the machine and port the mesh's artifact store answers on, when a
// module providing it is assigned to a machine that is on the private network.
//
// A lookup failure is an error, never "not found": collapsing the two composed a declaration
// without the trust whenever the inventory hiccuped, delivered by a push that reported success —
// and nothing recomposed the machine until the next push. "No store" must mean the mesh has none,
// not that the question went unanswered.
func artifactStoreOnNetwork(ctx context.Context, inv *inventory.Inventory,
on map[string]bool) (node, port string, found bool, err error) {
shelf, err := inv.Catalogue(ctx)
if err != nil {
return "", "", false, fmt.Errorf("reading the catalogue: %w", err)
}
providers := map[string]string{} // module -> served port
for name, m := range shelf {
served, offers := m.Serves[catalogue.ArtifactStoreProvision]
if !offers {
continue
}
if p, ok := served["port"]; ok {
providers[name] = fmt.Sprintf("%v", p)
}
}
if len(providers) == 0 {
return "", "", false, nil
}
for machine := range on {
assigned, err := inv.Assigned(ctx, machine)
if err != nil {
return "", "", false, fmt.Errorf("reading what %s is assigned: %w", machine, err)
}
for _, a := range assigned {
if p, ok := providers[a]; ok {
return machine, p, true, nil
}
}
}
return "", "", false, nil
}
+31 -10
View File
@@ -484,16 +484,7 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) {
// the daemon's. Written `<module>.<id>`, and a dot is what marks it as already // the daemon's. Written `<module>.<id>`, and a dot is what marks it as already
// answered: prefixing it again would point at nothing, silently, and the daemon would // answered: prefixing it again would point at nothing, silently, and the daemon would
// serve the old names for ever while everything reported success. // serve the old names for ever while everything reported success.
if reflects, ok := resource["restart-on"].([]any); ok { if renamed := reflectsRenamed(m.Module, resource["restart-on"]); renamed != nil {
var renamed []any
for _, id := range reflects {
named := fmt.Sprint(id)
if strings.Contains(named, ".") {
renamed = append(renamed, named)
continue
}
renamed = append(renamed, m.Module+"."+named)
}
copied["restart-on"] = renamed copied["restart-on"] = renamed
} }
out = append(out, copied) out = append(out, copied)
@@ -566,6 +557,36 @@ type Contribution struct {
// Named after both. Named after the machine alone, two modules on one node wrote to one path: the // Named after both. Named after the machine alone, two modules on one node wrote to one path: the
// second overwrote the first, and the provisioner — reading a directory — saw one consumer where // second overwrote the first, and the provisioner — reading a directory — saw one consumer where
// there were two. // there were two.
// reflectsRenamed is a resource's restart-on list under the module's prefix, or nil when it has
// none. It reads both the shape JSON parsing produces ([]any) and the shape code composing
// resources natively produces ([]string): a reference that was skipped because its list arrived
// in the other shape would point at nothing — silently, with the service never restarting and
// every check passing, which is how a runtime kept serving without the registry trust its
// daemon file already carried.
func reflectsRenamed(module string, reflects any) []any {
var names []string
switch v := reflects.(type) {
case []any:
for _, id := range v {
names = append(names, fmt.Sprint(id))
}
case []string:
names = v
}
if names == nil {
return nil
}
var renamed []any
for _, named := range names {
if strings.Contains(named, ".") {
renamed = append(renamed, named)
continue
}
renamed = append(renamed, module+"."+named)
}
return renamed
}
func grantPath(directory, consumer, module string) string { func grantPath(directory, consumer, module string) string {
return strings.TrimRight(directory, "/") + "/" + consumer + "." + module + ".secret" return strings.TrimRight(directory, "/") + "/" + consumer + "." + module + ".secret"
} }
@@ -0,0 +1,32 @@
package catalogue
import (
"reflect"
"testing"
)
func TestReflectsRenamedReadsBothShapes(t *testing.T) {
// The list arrives as []any when the resource was parsed from JSON, and as []string when it
// was composed in code — the overlay's registry trust is the second kind. A shape that was
// skipped would leave the reference unprefixed, pointing at nothing, and the service would
// never restart while every check passed (novox/hq issues 042/048, the run-8 diagnosis).
parsed := reflectsRenamed("mesh-wireguard", []any{"registry-trust"})
if !reflect.DeepEqual(parsed, []any{"mesh-wireguard.registry-trust"}) {
t.Fatalf("parsed shape: %v", parsed)
}
composed := reflectsRenamed("mesh-wireguard", []string{"registry-trust"})
if !reflect.DeepEqual(composed, []any{"mesh-wireguard.registry-trust"}) {
t.Fatalf("composed shape: %v", composed)
}
// A name already under a module keeps it: that is how a service reflects a file another
// module put on the machine.
kept := reflectsRenamed("resolver", []string{"mesh-wireguard.fact-node-names", "own-config"})
if !reflect.DeepEqual(kept, []any{"mesh-wireguard.fact-node-names", "resolver.own-config"}) {
t.Fatalf("dotted name was not kept: %v", kept)
}
if got := reflectsRenamed("any", nil); got != nil {
t.Fatalf("no list should rename to nothing, got %v", got)
}
}
+32 -1
View File
@@ -83,8 +83,17 @@ type Generator struct {
// keyPath is where each node keeps the private half it generated. Named rather than carried: // keyPath is where each node keeps the private half it generated. Named rather than carried:
// the mesh has never seen it and never will. // the mesh has never seen it and never will.
keyPath string keyPath string
// registry is the mesh's artifact store as the network reaches it (host:port), or empty when
// the mesh has none. Being on the network is what grants a machine the right to pull from it
// (novox/hq ADR 0082), so the module that puts a machine on the network is what writes the
// runtime's trust — the same reasoning that has it write /etc/hosts.
registry string
} }
// TrustRegistry names the artifact store this network's machines pull from in the clear —
// the overlay is the transport security (ADR 0082).
func (g *Generator) TrustRegistry(hostPort string) { g.registry = hostPort }
// From builds a generator over the machines that are part of the network. // From builds a generator over the machines that are part of the network.
// //
// The nodes given are the ones assigned the module — not every node the mesh knows. A machine // The nodes given are the ones assigned the module — not every node the mesh knows. A machine
@@ -123,7 +132,29 @@ func (g *Generator) Resources(node string) ([]map[string]any, bool, error) {
if err := json.Unmarshal(raw, &parsed); err != nil { if err := json.Unmarshal(raw, &parsed); err != nil {
return nil, false, err return nil, false, err
} }
return parsed.Resources, true, nil resources := parsed.Resources
if g.registry != "" {
trust, err := json.Marshal(map[string]any{"insecure-registries": []string{g.registry}})
if err != nil {
return nil, false, err
}
resources = append(resources,
map[string]any{
// Merged, not owned: the runtime's daemon file is the machine's, and this states
// one fact into it. The registry speaks plain HTTP because every path to it is
// already inside the overlay's encryption (ADR 0082) — this line is the runtime
// being told what the mesh already means.
"id": "registry-trust", "type": "file", "path": "/etc/docker/daemon.json",
"content": string(trust) + "\n", "mode": "0644", "merge": "json",
},
map[string]any{
// The runtime reloads nothing for this setting, so it is restarted when the fact
// changes — once, at joining, before the machine runs anything that would mind.
"id": "registry-trust-reload", "type": "service", "unit": "docker.service",
"state": "running", "restart-on": []string{"registry-trust"},
})
}
return resources, true, nil
} }
// Nodes are the machines this generator was built over, so a caller can say who is on the network. // Nodes are the machines this generator was built over, so a caller can say who is on the network.
+56
View File
@@ -0,0 +1,56 @@
package overlay
import (
"strings"
"testing"
)
func TestTheNetworkCarriesRegistryTrust(t *testing.T) {
// novox/hq ADR 0082: being on the network is what grants a machine the right to pull from the
// mesh's artifact store in the clear, so the network module writes the runtime's trust — and
// writes nothing when the mesh has no store to trust.
nodes := []Node{
{Name: "anchor", Site: "lab", Hub: true, Endpoint: "192.0.2.10:51820", Key: "k1", Address: "10.42.0.1"},
{Name: "node2", Site: "lab", Key: "k2", Address: "10.42.0.2"},
}
g, err := From(nodes, "10.42.0.0/16", "")
if err != nil {
t.Fatal(err)
}
plain, _, err := g.Resources("node2")
if err != nil {
t.Fatal(err)
}
for _, r := range plain {
if r["id"] == "registry-trust" {
t.Fatal("trust was written with no artifact store to trust")
}
}
g.TrustRegistry("anchor.internal:5000")
trusted, part, err := g.Resources("node2")
if err != nil || !part {
t.Fatalf("resources: %v part=%v", err, part)
}
var file, service map[string]any
for _, r := range trusted {
switch r["id"] {
case "registry-trust":
file = r
case "registry-trust-reload":
service = r
}
}
if file == nil || service == nil {
t.Fatalf("the trust file or its reload is missing: %v", trusted)
}
if file["path"] != "/etc/docker/daemon.json" || file["merge"] != "json" {
t.Fatalf("the trust is not a merged daemon.json: %v", file)
}
if content, _ := file["content"].(string); !strings.Contains(content, `"anchor.internal:5000"`) {
t.Fatalf("the trust does not name the store: %v", file["content"])
}
if service["unit"] != "docker.service" {
t.Fatalf("the reload does not restart the runtime: %v", service)
}
}