From 5062c36fc9efe159aa9706c0ca2c873351ef1ce0 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 14 Sep 2026 21:11:37 +0200 Subject: [PATCH 01/17] A binding answered on this very node still carries an address MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two sibling branches resolve a provision answered by the consumer's own machine. The one for a node-scoped provider falls back to loopback when the machine is on no private network, with a comment saying why and a test holding it. The one for a mesh-scoped provider passed node.At straight through, and nothing noticed because nothing had yet composed a host out of it. The mesh's own artifact store is mesh-scoped and sits on the same machine as the builder that pushes to it. Give the builder the address from its binding and it gets MESH_REGISTRY=:5000 — a name with no host, written into its environment without complaint. It surfaces much later as ":5000/mesh-tools/build" is not a valid repository/tag which is a message about a tag for a fault in how a binding was resolved, on a machine several steps from the decision. A machine off the private network still reaches itself, which is what the neighbouring branch already said. The test fails without the fix, showing the empty address rather than only the symptom. Claude-Session: https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- internal/catalogue/resolve.go | 13 +++++- internal/catalogue/resolve_loopback_test.go | 52 +++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 internal/catalogue/resolve_loopback_test.go diff --git a/internal/catalogue/resolve.go b/internal/catalogue/resolve.go index b92f350..55c00b2 100644 --- a/internal/catalogue/resolve.go +++ b/internal/catalogue/resolve.go @@ -287,8 +287,19 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world if satisfied[want] && !isModule(catalogue, want) { if brokered[want] { // Answered here, and still a need: the provider is this node. + // + // **The same loopback fallback as the branch below, and it was missing here.** A + // machine with no private network has no `at`, and this branch passed that through + // — so a consumer whose file says `${bound::at}:${bound:...:port}` was + // handed `:5000`, a name with no host, written into its environment without + // complaint. The sibling case a few lines down had the fallback and the reasoning + // for it; only this one did not. A machine off the network still reaches itself. + at := node.At + if at == "" { + at = "127.0.0.1" + } needs = append(needs, Needed{ - Name: want, From: node.Name, At: node.At, + Name: want, From: node.Name, At: at, Serves: servedHere(catalogue, chosen, want), For: because[want]}) } else if served := servedHere(catalogue, chosen, want); len(served) > 0 { // Answered here with no credential to mint, but the provider serves facts the diff --git a/internal/catalogue/resolve_loopback_test.go b/internal/catalogue/resolve_loopback_test.go new file mode 100644 index 0000000..d9ca22c --- /dev/null +++ b/internal/catalogue/resolve_loopback_test.go @@ -0,0 +1,52 @@ +package catalogue + +import "testing" + +// A MESH-SCOPED provider on the consumer's own node must deliver a usable address too. +// +// The sibling case — a node-scoped provider minting no credential — has had this fallback and a +// test for it for some time. This branch did not, and the difference is invisible until something +// puts the address into a string: the mesh's own artifact store is mesh-scoped and lives on the +// same machine as the builder that pushes to it, so the builder's environment was written as +// `MESH_REGISTRY=:5000`. Nothing refused it. The runtime did, several steps later, with +// `":5000/mesh-tools/build" is not a valid repository/tag` — a message about a tag, for a fault in +// how a binding was resolved. +// +// A machine off the private network still reaches itself. +func TestAMeshScopedProviderOnThisNodeStillCarriesAnAddress(t *testing.T) { + provider := Manifest{Module: "registry", + Provides: []Offer{{Name: "artifact-store", Scope: ScopeMesh}}, + Serves: map[string]map[string]any{ + "artifact-store": {"port": 5000}}} + consumer := Manifest{Module: "builder", Requires: []string{"artifact-store"}} + + // No `At`: a mesh with no private network raised, which is every mesh before somebody places + // its nodes on one — including the moment just after it is installed. + got, err := Resolve( + map[string]Manifest{"registry": provider, "builder": consumer}, + []string{"builder", "registry"}, + Node{Name: "anchor"}, + World{}) + if err != nil { + t.Fatalf("a mesh-scoped provision answered on this very node was refused: %v", err) + } + + var found *Needed + for i := range got.Needs { + if got.Needs[i].Name == "artifact-store" && got.Needs[i].For == "builder" { + found = &got.Needs[i] + } + } + if found == nil { + t.Fatalf("the store was not delivered to the builder at all: %+v", got.Needs) + } + if found.At == "" { + t.Fatalf("the binding carries no address, so anything composing a host from it gets none: %+v", found) + } + if found.At != "127.0.0.1" { + t.Fatalf("off the private network the address must fall back to loopback, got %q", found.At) + } + if got, want := plainly(found.Serves["port"]), "5000"; got != want { + t.Fatalf("the port was not delivered: got %q want %q", got, want) + } +} -- 2.54.0 From dda001d64b49779d08a977f2bb6310fa8dcdb223 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 15 Sep 2026 00:54:31 +0200 Subject: [PATCH 02/17] The firewall opens the port the mesh itself runs on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rules are derived from what modules declare they listen on, and the substrate is not a module. So the broker's port — the one every machine dials to enrol and to receive every declaration it is ever sent — appeared in no ruleset the mesh has ever generated. Nothing caught it because a mesh of one never dials its own broker across the network: the ruleset looks complete right up until a second machine tries to join a firewalled anchor and is refused by the packet filter, during enrolment, before the mesh can report anything about it. Assigning the firewall before joining machines is both the natural order and the one that breaks. It is a floor for the same reason ssh is. A machine nobody can reach cannot be repaired; a machine the mesh cannot reach cannot be managed. Neither is a thing any module asks for and neither may be derived away. From anywhere rather than from the private network, deliberately: a node enrols BEFORE it has an address on that network, so narrowing the rule to it would close the door being knocked on. The port is read from the broker this control plane was told about, so the address handed out in a token and the port a machine must accept on stay one fact. A mesh never told about a broker gets no such rule, rather than a broken one — and cannot issue tokens either, which is where that surfaces. Closes novox/hq 04-ISSUES/052. Claude-Session: https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- cmd/mesh-control/plan.go | 18 ++++++- internal/catalogue/declaration.go | 8 +++- internal/catalogue/filtering.go | 27 ++++++++++- .../catalogue/filtering_substrate_test.go | 48 +++++++++++++++++++ internal/catalogue/filtering_test.go | 28 +++++------ internal/catalogue/print_rehearsal_test.go | 2 +- 6 files changed, 113 insertions(+), 18 deletions(-) create mode 100644 internal/catalogue/filtering_substrate_test.go diff --git a/cmd/mesh-control/plan.go b/cmd/mesh-control/plan.go index 260816a..a37e1a9 100644 --- a/cmd/mesh-control/plan.go +++ b/cmd/mesh-control/plan.go @@ -9,9 +9,12 @@ import ( "sort" "strings" + "github.com/novox/mesh-control/internal/broker" "github.com/novox/mesh-control/internal/catalogue" "github.com/novox/mesh-control/internal/inventory" "github.com/novox/mesh-control/internal/licences" + "net" + "strconv" ) // working out what one machine should be. @@ -447,9 +450,22 @@ func declarationWith(ctx context.Context, open *stores, node string, names[name] = at } + // The ports the mesh itself needs open, which no module declares. Read from the broker this + // control plane was told about rather than written down twice: the address a node is handed in + // its token and the port its machine must accept on are the same fact. + var substrate []int + if b, err := broker.FromEnvironment(); err == nil { + if _, port, err := net.SplitHostPort(b.Address); err == nil { + if n, err := strconv.Atoi(port); err == nil { + substrate = append(substrate, n) + } + } + } + return plan.Declaration(catalogue.Rendering{ Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports, - Certificate: certificate, Authority: authority, Mesh: private, Names: names}) + Certificate: certificate, Authority: authority, Mesh: private, Names: names, + Substrate: substrate}) } // routeNamesInTheMesh is every routed name and the address of the node that serves it (novox/hq diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index e91c3da..3b4a1c5 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -89,6 +89,12 @@ type Rendering struct { // a fact about the mesh, and resolution answers questions about one machine. Mesh []string + // Substrate is the ports the mesh itself needs reachable on every machine, which no module + // declares because the substrate is not a module (novox/hq 04-ISSUES/051 and 052). The broker + // is the one that matters: a machine dials it to enrol, and a firewall derived only from + // modules closes it. + Substrate []int + // Names is every machine's internal name and its address, for containers to be given. // // **A container does not inherit the machine's names**, so every internal name the mesh wrote @@ -207,7 +213,7 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) { if err != nil { return nil, err } - filtering := AsNftables(rules, with.Mesh, r.PublicDomain != "") + filtering := AsNftables(rules, with.Mesh, r.PublicDomain != "", with.Substrate) var out []map[string]any for _, m := range r.Modules { diff --git a/internal/catalogue/filtering.go b/internal/catalogue/filtering.go index e9dbef7..ba1d6dc 100644 --- a/internal/catalogue/filtering.go +++ b/internal/catalogue/filtering.go @@ -216,7 +216,20 @@ const SSHPort = 22 // // `outward` says this machine is reachable from outside the mesh, which is the only thing that // decides whether ssh is answered there as well as on the private network. -func AsNftables(rules []Rule, mesh []string, outward bool) string { +// +// `substrate` is the ports the MESH ITSELF needs reachable, which no module declares. +// +// **Everything else in this file is derived from what modules say they listen on, and the +// substrate is not a module** (novox/hq 04-ISSUES/051). So the broker — the port every machine +// dials to enrol and to receive every declaration it is ever sent — was absent from the ruleset, +// and nothing noticed: a mesh of one never dials its own broker across the network. The first +// machine to join a firewalled anchor is refused by the packet filter during enrolment, before +// the mesh can report anything about it. +// +// It is a floor for the same reason ssh is. A machine nobody can reach is a machine nobody can +// repair; a machine the mesh cannot reach is a machine the mesh cannot manage. Neither is a thing +// any module asks for, and neither may be derived away. +func AsNftables(rules []Rule, mesh []string, outward bool, substrate []int) string { var b strings.Builder b.WriteString("# Computed by the mesh from what is assigned to this node.\n") b.WriteString("# Edits are lost on the next declaration; change a module's listens instead.\n\n") @@ -279,6 +292,18 @@ func AsNftables(rules []Rule, mesh []string, outward bool) string { b.WriteString(fmt.Sprintf("\t\ttcp dport %d accept\n", SSHPort)) } + // The mesh's own ports, from anywhere. + // + // Not narrowed to the private network, because the machines that need this most are the ones + // not on it yet: a node enrols BEFORE it has an address here, over the ordinary network, and a + // rule allowing only the private network would close the door being knocked on. + if len(substrate) > 0 { + b.WriteString("\n\t\t# the mesh's own — never derived, never closed\n") + for _, port := range substrate { + b.WriteString(fmt.Sprintf("\t\ttcp dport %d accept\n", port)) + } + } + if len(rules) > 0 { b.WriteString("\n") } diff --git a/internal/catalogue/filtering_substrate_test.go b/internal/catalogue/filtering_substrate_test.go new file mode 100644 index 0000000..7df5680 --- /dev/null +++ b/internal/catalogue/filtering_substrate_test.go @@ -0,0 +1,48 @@ +package catalogue + +import ( + "strings" + "testing" +) + +// The mesh's own ports survive a ruleset derived from modules that do not mention them. +// +// **The firewall is computed from what modules declare they listen on, and the substrate is not a +// module** (novox/hq 04-ISSUES/052). So the broker's port — the one every machine dials to enrol +// and to receive every declaration it is ever sent — was absent from every ruleset the mesh ever +// generated, and nothing caught it: a mesh of one never dials its own broker across the network, +// so the ruleset looks complete right up until a second machine tries to join and is refused by +// the packet filter, during enrolment, before the mesh can report anything about it. +func TestTheBrokersPortIsOpenedThoughNoModuleDeclaresIt(t *testing.T) { + const brokerPort = 5671 + + // A machine on the private network, with one ordinary module rule, and nothing that mentions + // the broker — which is every machine. + rules := []Rule{{Port: 8080, From: FromMesh, Because: []string{"some-module"}}} + out := AsNftables(rules, []string{"10.42.0.1"}, false, []int{brokerPort}) + + if !strings.Contains(out, "tcp dport 5671 accept") { + t.Fatalf("the broker's port is not opened, so no machine could enrol:\n%s", out) + } + + // From anywhere, deliberately: a node enrols BEFORE it has an address on the private network, + // so a rule narrowed to that network would close the door being knocked on. + for _, line := range strings.Split(out, "\n") { + if strings.Contains(line, "5671") && strings.Contains(line, "saddr") { + t.Fatalf("the broker's port is narrowed to the private network, which a machine that "+ + "has not yet enrolled is not on:\n%s", line) + } + } +} + +// And a mesh that was never told about a broker still gets a ruleset, rather than an empty one or +// a panic. A control plane in that state cannot issue tokens either, which is where it surfaces. +func TestNoBrokerMeansNoSubstrateRuleRatherThanNoRuleset(t *testing.T) { + out := AsNftables(nil, []string{"10.42.0.1"}, false, nil) + if !strings.Contains(out, "table inet mesh") { + t.Fatalf("no ruleset at all:\n%s", out) + } + if strings.Contains(out, "never derived, never closed\n\t\ttcp dport") { + t.Fatalf("a substrate rule was written for a mesh with no broker:\n%s", out) + } +} diff --git a/internal/catalogue/filtering_test.go b/internal/catalogue/filtering_test.go index ddfd9fb..22a4e3f 100644 --- a/internal/catalogue/filtering_test.go +++ b/internal/catalogue/filtering_test.go @@ -79,7 +79,7 @@ func TestTwoModulesWantingOnePortAreBothNamed(t *testing.T) { t.Fatalf("a module that wanted this port open is not named: %+v", rules[0]) } // The consequence, which is the reason this matters: removing web must not read as closing 443. - nft := AsNftables(rules, nil, false) + nft := AsNftables(rules, nil, false, nil) if !strings.Contains(nft, "web") || !strings.Contains(nft, "board") { t.Fatalf("the rendered rule set does not name both sources:\n%s", nft) } @@ -107,7 +107,7 @@ func TestAPortOpenToEveryoneIsNotAlsoRestrictedToTheMesh(t *testing.T) { func TestWhatNoModuleDeclaredIsClosed(t *testing.T) { nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{ {Module: "web", Listens: []Listening{{Port: 443, From: FromEverywhere}}}, - }}, nil), []string{"198.51.100.2"}, false) + }}, nil), []string{"198.51.100.2"}, false, nil) // Naming the chain, not just the policy: the forward chain drops too, and an assertion on // "policy drop" alone passes while the input chain accepts everything. It did, once, here. if !strings.Contains(nft, "type filter hook input priority filter; policy drop;") { @@ -134,7 +134,7 @@ func TestWhatNoModuleDeclaredIsClosed(t *testing.T) { // `flush ruleset` would do the first and not the second: it empties every table on the machine, // including the ones the container runtime writes for its bridges. func TestReloadingReplacesOnlyTheMeshsOwnRules(t *testing.T) { - nft := AsNftables(nil, nil, false) + nft := AsNftables(nil, nil, false, nil) if strings.Contains(nft, "flush ruleset") { t.Fatalf("loading the rule set empties every table on the machine:\n%s", nft) } @@ -160,7 +160,7 @@ func TestReloadingReplacesOnlyTheMeshsOwnRules(t *testing.T) { // So the chain exists and denies by default, and the runtime's own networks are allowed explicitly // — which is how the system being replaced has been doing it on these machines for months. func TestWhatIsForwardedIsGovernedToo(t *testing.T) { - nft := AsNftables(nil, []string{"198.51.100.2"}, false) + nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil) if !strings.Contains(nft, "hook forward priority filter; policy drop") { t.Fatalf("forwarded traffic is not governed, so container ports are open:\n%s", nft) } @@ -168,7 +168,7 @@ func TestWhatIsForwardedIsGovernedToo(t *testing.T) { // And containers keep working, which is the whole reason the chain was left out before. func TestTheRuntimesOwnNetworksKeepWorking(t *testing.T) { - nft := AsNftables(nil, []string{"198.51.100.2"}, false) + nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil) for _, network := range []string{"172.16.0.0/12", "192.168.128.0/17"} { if !strings.Contains(nft, "ip saddr "+network+" accept") { t.Fatalf("%s is not allowed, so denying by default stops every container:\n%s", network, nft) @@ -183,7 +183,7 @@ func TestTheRuntimesOwnNetworksKeepWorking(t *testing.T) { func TestAPublishedPortIsMatchedByWhatWasAskedFor(t *testing.T) { nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{ {Module: "web", Listens: []Listening{{Port: 8080, From: FromEverywhere}}}, - }}, nil), []string{"198.51.100.2"}, false) + }}, nil), []string{"198.51.100.2"}, false, nil) if !strings.Contains(nft, "ct original proto-dst 8080 accept") { t.Fatalf("the forwarded rule does not match the port a client asked for:\n%s", nft) } @@ -193,7 +193,7 @@ func TestAPublishedPortIsMatchedByWhatWasAskedFor(t *testing.T) { func TestAMeshScopedPortIsMeshScopedWhenForwarded(t *testing.T) { nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{ {Module: "store", Listens: []Listening{{Port: 5432, From: FromMesh}}}, - }}, nil), []string{"198.51.100.2"}, false) + }}, nil), []string{"198.51.100.2"}, false, nil) if !strings.Contains(nft, "ip saddr { 198.51.100.2 } ct original proto-dst 5432 accept") { t.Fatalf("a mesh-only port is reachable from anywhere once forwarded:\n%s", nft) } @@ -203,7 +203,7 @@ func TestAMeshScopedPortIsMeshScopedWhenForwarded(t *testing.T) { func TestFromTheMeshIsTheNodesTheMeshKnows(t *testing.T) { nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{ {Module: "store", Listens: []Listening{{Port: 5432, From: FromMesh}}}, - }}, nil), []string{"198.51.100.2", "198.51.100.3"}, false) + }}, nil), []string{"198.51.100.2", "198.51.100.3"}, false, nil) if !strings.Contains(nft, "ip saddr { 198.51.100.2, 198.51.100.3 } tcp dport 5432 accept") { t.Fatalf("a mesh-scoped port was not restricted to the mesh's addresses:\n%s", nft) } @@ -213,7 +213,7 @@ func TestFromTheMeshIsTheNodesTheMeshKnows(t *testing.T) { func TestAMeshPortOnANodeWithNoMeshIsClosedAndSaysSo(t *testing.T) { nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{ {Module: "store", Listens: []Listening{{Port: 5432, From: FromMesh}}}, - }}, nil), nil, false) + }}, nil), nil, false, nil) if strings.Contains(nft, "dport 5432 accept") { t.Fatalf("a port meant for the mesh was opened to everything:\n%s", nft) } @@ -226,7 +226,7 @@ func TestAMeshPortOnANodeWithNoMeshIsClosedAndSaysSo(t *testing.T) { func TestAMachineScopedPortIsNotOpened(t *testing.T) { nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{ {Module: "cache", Listens: []Listening{{Port: 6379, From: FromMachine}}}, - }}, nil), []string{"198.51.100.2"}, false) + }}, nil), []string{"198.51.100.2"}, false, nil) if strings.Contains(nft, "dport 6379 accept") { t.Fatalf("a port for this machine only was opened to the network:\n%s", nft) } @@ -268,7 +268,7 @@ func TestAskingForTheRuleSetWithNowhereToPutItIsRefused(t *testing.T) { func TestAMeshOnBothAddressFamiliesRendersBoth(t *testing.T) { nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{ {Module: "store", Listens: []Listening{{Port: 5432, From: FromMesh}}}, - }}, nil), []string{"198.51.100.2", "2001:db8::2"}, false) + }}, nil), []string{"198.51.100.2", "2001:db8::2"}, false, nil) if !strings.Contains(nft, "ip saddr { 198.51.100.2 } tcp dport 5432 accept") { t.Fatalf("the machines with v4 addresses were dropped:\n%s", nft) } @@ -672,7 +672,7 @@ func TestExposureRefusesAPortNotListenedOnAndABadSource(t *testing.T) { // loading the rules lives on conntrack until it drops, and then the machine is reached from a // rescue console (novox/hq issue 047). func TestSSHIsOpenFromTheMeshEvenWhenNothingIsAssigned(t *testing.T) { - nft := AsNftables(nil, []string{"198.51.100.2", "198.51.100.3"}, false) + nft := AsNftables(nil, []string{"198.51.100.2", "198.51.100.3"}, false, nil) if !strings.Contains(nft, "ip saddr { 198.51.100.2, 198.51.100.3 } tcp dport 22 accept") { t.Fatalf("ssh is not open to the mesh, so a machine can lock everyone out:\n%s", nft) } @@ -685,7 +685,7 @@ func TestSSHIsOpenFromTheMeshEvenWhenNothingIsAssigned(t *testing.T) { // And from outside as well, on a machine that faces outward — because that is the way in when the // private network is the thing that broke. func TestSSHIsOpenFromOutsideOnAMachineThatFacesIt(t *testing.T) { - nft := AsNftables(nil, []string{"198.51.100.2"}, true) + nft := AsNftables(nil, []string{"198.51.100.2"}, true, nil) if !strings.Contains(nft, "\t\ttcp dport 22 accept") { t.Fatalf("a machine reachable from outside does not answer ssh there:\n%s", nft) } @@ -697,7 +697,7 @@ func TestSSHIsOpenFromOutsideOnAMachineThatFacesIt(t *testing.T) { // to narrow the rule to, so narrowing it shuts the port entirely — on the first machine anybody // adopts, reached over the network, closed by the act of adopting it. func TestSSHIsNeverLeftWithoutARule(t *testing.T) { - nft := AsNftables(nil, nil, false) + nft := AsNftables(nil, nil, false, nil) if !strings.Contains(nft, "tcp dport 22 accept") { t.Fatalf("a machine with no mesh addresses has no ssh rule, so adopting it locks it:\n%s", nft) } diff --git a/internal/catalogue/print_rehearsal_test.go b/internal/catalogue/print_rehearsal_test.go index 71e3dbb..65fbb58 100644 --- a/internal/catalogue/print_rehearsal_test.go +++ b/internal/catalogue/print_rehearsal_test.go @@ -12,5 +12,5 @@ func TestPrintRehearsalRuleset(t *testing.T) { rules := mustFilter(t, Resolution{Modules: []Manifest{ {Module: "pub", Listens: []Listening{{Port: 8099, From: FromMesh, Why: "the thing it serves"}}}, }}, nil) - t.Log("\n" + AsNftables(rules, []string{"192.0.2.20"}, true)) + t.Log("\n" + AsNftables(rules, []string{"192.0.2.20"}, true, nil)) } -- 2.54.0 From 2b82872ac38c431dadf0abb403e3878abe4ff98c Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 15 Sep 2026 01:22:34 +0200 Subject: [PATCH 03/17] A build keeps what it was told MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The builder announces a build with the resolved manifest, the path inside the repository, and every artifact it stood on. The control plane received all of it and kept none of it. That was survivable while the catalogue heard the same announcement directly. It stops being survivable the moment the catalogue was not there to hear it — which on a fresh mesh is always, and always for the same modules: the shared base, the store the catalogue runs on, and the catalogue itself are each necessarily built BEFORE the catalogue exists to hear about them. The graph's foundation is the part the graph never sees. Replaying those builds needs what they said, not a summary. Without the manifest there are no requires/provides edges; without `against` there are no build edges, which are the ones that answer "a base moved, what must be rebuilt". A replay carrying neither would restore the module list and leave the question the catalogue exists for still wrong, while looking fixed. Kept null rather than empty where a build predates this, so a replay can say it is holding nothing instead of inventing an empty declaration for a module that certainly had one. And `built_against`, not `built_on`: that column exists and means the machine, which is a different fact about a different subject. Toward novox/hq 04-ISSUES/050. Claude-Session: https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- cmd/mesh-control/build.go | 6 ++++ internal/inventory/builds.go | 36 +++++++++++++++++-- .../0022-a-build-keeps-what-it-was-told.sql | 26 ++++++++++++++ 3 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 internal/inventory/migrations/0022-a-build-keeps-what-it-was-told.sql diff --git a/cmd/mesh-control/build.go b/cmd/mesh-control/build.go index dc72584..008adee 100644 --- a/cmd/mesh-control/build.go +++ b/cmd/mesh-control/build.go @@ -72,6 +72,12 @@ func buildFrom(result link.BuildResult) inventory.Build { kept := inventory.Build{ ID: result.ID, Repository: result.Repository, Ref: result.Ref, Commit: result.Commit, On: result.On, Failed: result.Failed, + // **What the announcement carries, kept rather than discarded** (novox/hq 04-ISSUES/050). + // The catalogue turns the manifest into requires/provides edges and `against` into build + // edges, and it is not always listening when a build happens — on a fresh mesh it cannot + // be, for exactly the modules it needs most. Keeping them is what makes a replay able to + // rebuild the graph rather than a list of names. + Path: result.Path, Manifest: result.Manifest, Against: result.Against, } for _, made := range result.Made { kept.Made = append(kept.Made, inventory.Artifact{ diff --git a/internal/inventory/builds.go b/internal/inventory/builds.go index 9990993..ef07fdb 100644 --- a/internal/inventory/builds.go +++ b/internal/inventory/builds.go @@ -23,6 +23,18 @@ type Build struct { Commit string // On is the machine that did it. On string + // Path is where inside the repository the module lives (novox/hq ADR 0069). + Path string + // Manifest is the declaration the builder resolved, as it announced it. + // + // **Kept because the catalogue may not have been listening.** The announcement carries this + // and the catalogue turns it into the module's requires/provides edges. On a fresh mesh the + // modules built before the catalogue exists are exactly the ones it most needs, so the mesh + // has to be able to say afterwards what they declared (novox/hq 04-ISSUES/050). + Manifest []byte + // Against is every artifact this build stood on, as references rather than module names — + // what makes a build edge derived rather than declared (ADR 0009). + Against []string // Failed is the builder's own words, empty when it worked. Failed string Made []Artifact @@ -49,15 +61,21 @@ func (i *Inventory) RecordBuild(ctx context.Context, b Build) error { if err != nil { return err } + against, err := json.Marshal(b.Against) + if err != nil { + return err + } var module *string if b.Module != "" { module = &b.Module } _, err = i.store.Pool().Exec(ctx, - `insert into build (id, repository, ref, module, commit_hash, built_on, failed, made) - values ($1, $2, $3, $4, $5, $6, $7, $8) + `insert into build (id, repository, ref, module, commit_hash, built_on, failed, made, + source_path, manifest, built_against) + values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) on conflict (id) do nothing`, - b.ID, b.Repository, b.Ref, module, b.Commit, b.On, b.Failed, made) + b.ID, b.Repository, b.Ref, module, b.Commit, b.On, b.Failed, made, + b.Path, manifestOrNil(b.Manifest), against) return err } @@ -144,3 +162,15 @@ func (i *Inventory) Held(ctx context.Context) (map[string]string, error) { } return held, rows.Err() } + +// manifestOrNil keeps the difference between "declared nothing" and "predates this being kept". +// +// A build recorded before the mesh kept manifests has no manifest, and that is not the same as one +// whose manifest was empty. A replay can then say which it is holding instead of inventing an +// empty declaration for a module that certainly had one. +func manifestOrNil(raw []byte) any { + if len(raw) == 0 { + return nil + } + return raw +} diff --git a/internal/inventory/migrations/0022-a-build-keeps-what-it-was-told.sql b/internal/inventory/migrations/0022-a-build-keeps-what-it-was-told.sql new file mode 100644 index 0000000..e86f3ef --- /dev/null +++ b/internal/inventory/migrations/0022-a-build-keeps-what-it-was-told.sql @@ -0,0 +1,26 @@ +-- What a build result carried and the mesh threw away. +-- +-- novox/hq 04-ISSUES/050. The builder announces a build with the resolved manifest, the path +-- inside the repository, and every artifact it was built against. The control plane receives all +-- of it and kept none of it: `build` held the repository, the ref, the commit and what was made. +-- +-- That was survivable while the catalogue heard the same announcement directly. It stops being +-- survivable the moment the catalogue was not there to hear it — which on a fresh mesh is always, +-- and always for the same modules. The shared base, the store the catalogue itself runs on, and +-- the catalogue: each is necessarily built BEFORE the catalogue exists to hear about it, so the +-- graph's foundation is the part the graph never sees. +-- +-- Replaying those builds needs what they said, not a summary of it. Without the manifest there +-- are no requires/provides edges; without `against` there are no build edges, which are the ones +-- that answer "a base moved, what must be rebuilt". A replay carrying neither would restore the +-- module list and leave the question the catalogue exists for still wrong, while looking fixed. +-- +-- Empty and null-free, so every build recorded before this keeps exactly the meaning it had: a +-- row with no manifest is one that predates this, and a replay says so rather than inventing an +-- empty declaration. +-- +-- `built_against` rather than `built_on`: that column already exists and means the MACHINE that +-- did the build, which is a different fact about a different subject. +alter table build add column source_path text not null default ''; +alter table build add column manifest jsonb; +alter table build add column built_against jsonb; -- 2.54.0 From 3ae7b88c6d798ebb7842ddb3021d332db402c020 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 15 Sep 2026 01:26:58 +0200 Subject: [PATCH 04/17] A catalogue asks for what it was not there to hear MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Its event queue is durable, so a running catalogue misses nothing. What it cannot have is what was announced before it first ran — and on a fresh mesh that is never arbitrary: the shared base, the store the catalogue runs on, and the catalogue itself are each necessarily built BEFORE a catalogue exists to hear about them. The graph's foundation is the part it never sees. So it says it is catching up, and the control plane re-announces what it recorded, oldest first, marked as a replay. Oldest first because a graph is built in the order things happened: registering a module that stands on a base before the base would point an edge at a version nothing has seen, and the shape of a fresh mesh guarantees the base is both first and the one that was missed. The replayer hands announcements back rather than publishing them, because the wire belongs to the link package and a replay building its own events could drift from what the builder emits — the one thing it must match exactly, since the catalogue has a single handler for both. Its own queue and its own consumer: two consumers on one queue split its messages, and a catch-up request going to whichever half was not listening is a gap that looks like a working mesh. Toward novox/hq 04-ISSUES/050. Claude-Session: https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- cmd/mesh-control/push.go | 5 +++ cmd/mesh-control/upgrades.go | 31 +++++++++++++++++ internal/inventory/builds.go | 61 ++++++++++++++++++++++++++++++++ internal/link/events.go | 50 +++++++++++++++++++++++++++ internal/link/serve.go | 67 ++++++++++++++++++++++++++++++++++++ 5 files changed, 214 insertions(+) diff --git a/cmd/mesh-control/push.go b/cmd/mesh-control/push.go index e4f0c93..8eb13f2 100644 --- a/cmd/mesh-control/push.go +++ b/cmd/mesh-control/push.go @@ -92,6 +92,11 @@ func serve(ctx context.Context) error { if err := server.Follows(following{open}); err != nil { return err } + // And a catalogue that has just started, asking for what it missed. The same type answers + // both: what a build meant and what the builds were are two questions about one record. + if err := server.Answers(following{open}); err != nil { + return err + } return server.Serve(ctx) } diff --git a/cmd/mesh-control/upgrades.go b/cmd/mesh-control/upgrades.go index f4ba58c..2823d0a 100644 --- a/cmd/mesh-control/upgrades.go +++ b/cmd/mesh-control/upgrades.go @@ -163,3 +163,34 @@ func sayUpgrade(module string, u inventory.Upgrade) string { return fmt.Sprintf("when %s moves, the machines running it are sent the new version one at "+ "a time, stopping at the first that fails", module) } + +// Announceable is every build this mesh recorded, in the shape the builder announces one. +// +// **The catalogue asks for this when it starts, and the answer is the graph's foundation** +// (novox/hq 04-ISSUES/050). A durable queue keeps what arrived after it existed, so a running +// catalogue misses nothing — but the modules built before it first ran were announced to a queue +// that did not exist, and on a fresh mesh those are always the same three: the shared base, the +// store the catalogue runs on, and the catalogue itself. +func (f following) Announceable(ctx context.Context) ([]link.Announcement, error) { + builds, err := f.open.inventory.Announceable(ctx) + if err != nil { + return nil, err + } + out := make([]link.Announcement, 0, len(builds)) + for _, b := range builds { + a := link.Announcement{ + Module: b.Module, Commit: b.Commit, Repository: b.Repository, + Path: b.Path, Ref: b.Ref, Against: b.Against, + } + if len(b.Manifest) > 0 { + a.Manifest = b.Manifest + } + for _, made := range b.Made { + a.Made = append(a.Made, link.MadeArtifact{ + Name: made.Name, Kind: made.Kind, Reference: made.Reference, + }) + } + out = append(out, a) + } + return out, nil +} diff --git a/internal/inventory/builds.go b/internal/inventory/builds.go index ef07fdb..037a531 100644 --- a/internal/inventory/builds.go +++ b/internal/inventory/builds.go @@ -3,6 +3,7 @@ package inventory import ( "context" "encoding/json" + "sort" "time" ) @@ -174,3 +175,63 @@ func manifestOrNil(raw []byte) any { } return raw } + +// Announceable is every build worth telling a catalogue about, oldest first. +// +// **Oldest first, because a graph is built in the order things happened.** Registering a module +// that stands on a base before the base itself would make the edge point at a version the +// catalogue has not seen, and the shape of a fresh mesh guarantees that order matters: the base is +// always first and always the one that was missed. +// +// Only builds that succeeded and know what they built. A failure produced no module-version, and +// announcing one would put something in the graph that was never made — the same rule the builder +// follows when it decides whether to announce at all. +// +// One row per module and commit: a module built twice at the same commit is one fact, and the +// latest row is the one whose artifacts are current. +func (i *Inventory) Announceable(ctx context.Context) ([]Build, error) { + rows, err := i.store.Pool().Query(ctx, + `select distinct on (module, commit_hash) + id, repository, ref, module, commit_hash, built_on, failed, made, + source_path, manifest, built_against, at + from build + where failed = '' and module is not null and module <> '' and commit_hash <> '' + order by module, commit_hash, at desc`) + if err != nil { + return nil, err + } + defer rows.Close() + + var out []Build + for rows.Next() { + var b Build + var made []byte + var manifest, against []byte + if err := rows.Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit, + &b.On, &b.Failed, &made, &b.Path, &manifest, &against, &b.At); err != nil { + return nil, err + } + if err := json.Unmarshal(made, &b.Made); err != nil { + return nil, err + } + // Null rather than empty is a build recorded before the mesh kept these, and saying so is + // the point of keeping them nullable: the replay carries nothing rather than an empty + // declaration for a module that certainly had one. + if len(manifest) > 0 { + b.Manifest = manifest + } + if len(against) > 0 { + if err := json.Unmarshal(against, &b.Against); err != nil { + return nil, err + } + } + out = append(out, b) + } + if err := rows.Err(); err != nil { + return nil, err + } + // Sorted here rather than in the query, because `distinct on` fixes the ordering it needs and + // the order that matters to a catalogue is a different one. + sort.Slice(out, func(a, b int) bool { return out[a].At.Before(out[b].At) }) + return out, nil +} diff --git a/internal/link/events.go b/internal/link/events.go index 7cb62fb..3d461a3 100644 --- a/internal/link/events.go +++ b/internal/link/events.go @@ -80,10 +80,60 @@ const KeyModuleBuilt = "module.builder.built" // two would eventually disagree (novox/hq ADR 0072). const KeyModuleUpgraded = "module.mesh-catalog.upgraded" +// KeyCatchingUp is the catalogue saying it has just started and may have missed things. +// +// **A durable queue only keeps what arrived after it existed.** The catalogue's own queue is +// durable, so nothing is lost once it is running — but the modules built before it first ran were +// announced to a queue that did not exist yet, and on a fresh mesh those are, necessarily, the +// shared base, the store the catalogue runs on, and the catalogue itself. The graph's foundation +// is the part it never hears about (novox/hq 04-ISSUES/050). +// +// So it asks, and the control plane answers with what it recorded. Asking rather than being told +// because only the catalogue knows it has a gap; the control plane cannot tell a fresh catalogue +// from one that is merely quiet. +const KeyCatchingUp = "module.mesh-catalog.catching-up" + +// CatchUpQueue is where that lands. Durable, for the same reason the upgrade queue is: a catalogue +// that started while the control plane was restarting is exactly the one with a gap to fill. +const CatchUpQueue = "control.catchup" + // UpgradeQueue is where those land. Durable and named, not a temporary queue: an upgrade announced // while the control plane is restarting is exactly the one that must not be missed. const UpgradeQueue = "control.upgrades" +// Replayer answers a catalogue that says it has just started. +// +// It is handed every build the mesh recorded, oldest first, and re-announces each. The catalogue +// registers them as history: a replayed build changed nothing in the world, so announcing it as an +// upgrade would have the mesh act on news that is years old. +type Replayer interface { + // Announceable is every build worth re-announcing, oldest first. + // + // It hands them back rather than publishing them: the wire belongs to this package, and a + // replay that built its own announcements could drift from what the builder emits — which is + // the one thing it must match exactly, because the catalogue has a single handler for both. + Announceable(ctx context.Context) ([]Announcement, error) +} + +// Announcement is a build, in the shape the builder announces one. +// +// The field names are the wire's, not Go's, because a catalogue reads these and a rename here is +// an event nobody handles. +type Announcement struct { + Module string `json:"module"` + Commit string `json:"commit"` + Repository string `json:"repository"` + Path string `json:"path"` + Ref string `json:"ref"` + Manifest json.RawMessage `json:"manifest,omitempty"` + Against []string `json:"against,omitempty"` + Made []MadeArtifact `json:"made,omitempty"` + // Replay says this is history rather than news: it was built once, and this is the mesh + // telling a catalogue that missed it. A consumer registers it and announces nothing — an + // upgrade that happened months ago is not one anything should act on now. + Replay bool `json:"replay,omitempty"` +} + // Upgraded is what the catalogue says when a module's current version moves. type Upgraded struct { Module string `json:"module"` diff --git a/internal/link/serve.go b/internal/link/serve.go index fef6712..2d402e7 100644 --- a/internal/link/serve.go +++ b/internal/link/serve.go @@ -51,6 +51,7 @@ type Server struct { recorder Recorder log *log.Logger upgrader Upgrader + replayer Replayer } // Records tells the server where to keep build results. @@ -89,6 +90,21 @@ func (s *Server) Follows(u Upgrader) error { return nil } +// Answers binds the queue a catalogue's catch-up request arrives on. +// +// **Not bound unless something is listening**, for the same reason upgrades are not: a durable +// queue with no consumer fills quietly and the first symptom is a broker out of disk. +func (s *Server) Answers(r Replayer) error { + if _, err := s.channel.QueueDeclare(CatchUpQueue, true, false, false, false, nil); err != nil { + return fmt.Errorf("cannot declare the %s queue: %w", CatchUpQueue, err) + } + if err := s.channel.QueueBind(CatchUpQueue, KeyCatchingUp, EventsExchange, false, nil); err != nil { + return fmt.Errorf("cannot bind %s to %s/%s: %w", CatchUpQueue, EventsExchange, KeyCatchingUp, err) + } + s.replayer = r + return nil +} + // Connect opens the control plane's own connection to the broker. func Connect(enroller Enroller, listener Listener) (*Server, error) { url := strings.TrimSpace(os.Getenv(AMQPVar)) @@ -187,6 +203,18 @@ func (s *Server) Serve(ctx context.Context) error { } } + // Its own queue and its own consumer, for the reason above: two consumers on one queue split + // its messages, and a catch-up request going to whichever half was not listening is a gap that + // looks like a working mesh. + var catchups <-chan amqp.Delivery + if s.replayer != nil { + catchups, err = s.channel.ConsumeWithContext(ctx, CatchUpQueue, "control-plane-catchup", + false, false, false, false, nil) + if err != nil { + return err + } + } + closed := s.conn.NotifyClose(make(chan *amqp.Error, 1)) s.log.Printf("consuming %s, bound to %s/{%s,%s,%s,%s}", ControlQueue, Exchange, KeyEnrol, KeyReport, KeyAlive, KeyBuilt) @@ -198,6 +226,14 @@ func (s *Server) Serve(ctx context.Context) error { select { case <-ctx.Done(): return nil + case delivery, ok := <-catchups: + if !ok { + if catchups != nil { + return errors.New("the broker stopped delivering catch-up requests") + } + continue + } + s.catchingUp(ctx, delivery) case delivery, ok := <-upgrades: // A nil channel blocks for ever, so this case simply never fires when nothing is // listening for upgrades. Closed is different, and means the broker stopped. @@ -379,6 +415,37 @@ func (s *Server) handleBuilt(ctx context.Context, delivery amqp.Delivery) { // none of those get better by being handed the same message again. Requeuing would put a poison // message at the head of a durable queue and stop every upgrade behind it, which turns one module // nobody can push into a mesh that stops following its own catalogue. +// catchingUp answers a catalogue that has just started and may have missed builds. +// +// Acknowledged before the work, deliberately: a replay that fails is not one that succeeds by +// being handed the same request again, and the catalogue asks every time it starts. Requeueing a +// poison request would stop every later catch-up behind it. +func (s *Server) catchingUp(ctx context.Context, delivery amqp.Delivery) { + defer func() { _ = delivery.Ack(false) }() + if s.replayer == nil { + s.log.Printf("a catalogue asked to catch up and this control plane has nothing to replay") + return + } + announcements, err := s.replayer.Announceable(ctx) + if err != nil { + s.log.Printf("a catalogue asked to catch up and the mesh could not read its builds: %v", err) + return + } + sent := 0 + for _, a := range announcements { + a.Replay = true + if err := EmitEvent(ctx, s.channel, KeyModuleBuilt, "control-plane", "", a); err != nil { + // Said and abandoned rather than retried: the catalogue asks again every time it + // starts, and half a graph delivered twice is no better than half delivered once. + s.log.Printf("replaying %s at %s failed, and the rest is abandoned: %v", + a.Module, short(a.Commit), err) + return + } + sent++ + } + s.log.Printf("a catalogue asked to catch up; re-announced %d build(s)", sent) +} + func (s *Server) upgraded(ctx context.Context, delivery amqp.Delivery) { defer func() { _ = delivery.Ack(false) }() var u Upgraded -- 2.54.0 From 4b7bd1b3e29eabeb4d700899790d0371d8d82b54 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 15 Sep 2026 01:59:07 +0200 Subject: [PATCH 05/17] A module says what it is written in, and needs no Dockerfile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bundle recipe: the one that both builds and packs. An archive packs a directory as it stands, so shipping compiled output meant compiling somewhere first — which meant a Dockerfile repeating the same incantation in every module. Two base arguments with no defaults, a working directory chosen so the SDK resolves upward, the compiler invoked by absolute path because the usual symlink is resolved away when the base image is assembled, a second stage, an environment variable naming the entrypoints. Most of the catalogue is unconverted and that is why; two conversions done in one session were each wrong twice with a working example open in the next window. A bundle says a language and a list of entrypoints. The mesh knows what the language implies. Anything a module could override there it would be writing a Dockerfile to override, so a toolchain is deliberately not configurable. Declared rather than inferred, both of them: guessing the language from which files are present makes a build depend on a directory listing, and guessing the entrypoints makes it change meaning when somebody adds a helper. A toolchain the mesh does not hold is refused before anything is compiled, naming what to build first — the same treatment a missing base already gets, because it is the same question and somebody can answer it. A language the mesh does not build is refused saying what would have worked, since the author is usually one word away. The list of languages is closed and adding to it is a decision. Every language is another implementation of the contracts every module shares, and those change rarely and cascade when they do (ADR 0039) — a mesh whose SDKs disagree about the envelope fails by ignoring messages rather than by failing to compile. Claude-Session: https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- internal/builder/builder.go | 89 ++++++++++++++++++++++- internal/builder/bundle_test.go | 109 +++++++++++++++++++++++++++++ internal/builder/toolchain.go | 99 ++++++++++++++++++++++++++ internal/builder/toolchain_test.go | 56 +++++++++++++++ internal/catalogue/build.go | 47 ++++++++++--- internal/catalogue/manifest.go | 35 +++++++++ 6 files changed, 424 insertions(+), 11 deletions(-) create mode 100644 internal/builder/bundle_test.go create mode 100644 internal/builder/toolchain.go create mode 100644 internal/builder/toolchain_test.go diff --git a/internal/builder/builder.go b/internal/builder/builder.go index 8658b65..fe6c754 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -129,7 +129,7 @@ func Build(ctx context.Context, run Runner, publish Publisher, // logs can be compared. sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name }) for _, a := range artifacts { - made, err := one(ctx, run, publish, manifest.Module, within, commit, a, args) + made, err := one(ctx, run, publish, manifest.Module, within, commit, a, args, held) if err != nil { return Result{}, err } @@ -215,7 +215,8 @@ func against(within string, manifest catalogue.Manifest) []string { const ManifestName = "module.json" func one(ctx context.Context, run Runner, publish Publisher, - module, tree, commit string, a catalogue.Artifact, args []string) (catalogue.Built, error) { + module, tree, commit string, a catalogue.Artifact, args []string, + held map[string]string) (catalogue.Built, error) { switch a.Kind { case catalogue.ArtifactUpstream: @@ -252,6 +253,46 @@ func one(ctx context.Context, run Runner, publish Publisher, } return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil + case catalogue.ArtifactBundle: + // **The one recipe that both builds and packs.** Everything else either produces an image + // or packs what is already there; this compiles the module's own code first, in a + // toolchain the mesh chose from what the module said it was written in, and packs the + // result. + // + // The compiler runs in a container rather than on the build machine, for the reason every + // other build does: what a build needs installed is the toolchain's business, and a build + // machine that accumulated one toolchain per language would be a machine nobody could + // reproduce. + chain, err := ToolchainFor(a.Language) + if err != nil { + return catalogue.Built{}, fmt.Errorf("%s: %s: %w", module, a.Name, err) + } + base, ok := held[chain.Base+"/"+chain.Artifact] + if !ok { + // Named, not pinned: the mesh answers with the copy it holds. Refused before anything + // is built, saying which module has to exist first, rather than failing inside a + // compile with a message about an image (novox/hq 04-ISSUES/044). + return catalogue.Built{}, fmt.Errorf( + "%s: %s is written in %s, which is compiled by %s's %q artifact, and this mesh "+ + "holds no copy of it. Build %s first", + module, a.Name, chain.Language, chain.Base, chain.Artifact, chain.Base) + } + compiled, err := compile(ctx, run, tree, chain, base, a) + if err != nil { + return catalogue.Built{}, fmt.Errorf("%s: compiling %s failed: %w", module, a.Name, err) + } + body, err := pack(compiled) + if err != nil { + return catalogue.Built{}, fmt.Errorf("%s: packing %s failed: %w", module, a.Name, err) + } + sum := sha256.Sum256(body) + digest := "sha256:" + hex.EncodeToString(sum[:]) + where, err := publish.PublishArchive(ctx, module+"/"+a.Name, body, digest) + if err != nil { + return catalogue.Built{}, err + } + return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: where, Digest: digest}, nil + case catalogue.ArtifactArchive: body, err := pack(filepath.Join(tree, a.From)) if err != nil { @@ -406,3 +447,47 @@ func standingOn(manifest catalogue.Manifest, held map[string]string) ([]string, } return args, nil } + +// compile runs a module's own code through its toolchain, and says where the result is. +// +// **In the module's own directory, under the path the toolchain expects.** A module is compiled +// where its dependencies resolve upward into the base's own library directory, so what it is +// compiled against is exactly what it will run against — the reason every hand-written Dockerfile +// had to choose a working directory carefully, and the reason none of them has to now. +func compile(ctx context.Context, run Runner, tree string, chain Toolchain, + base string, a catalogue.Artifact) (string, error) { + + // Where inside the toolchain the module's source is mounted, and where its output lands. Fixed + // rather than configurable: a module that could move this would be describing its own build. + const within = "/app/modules/module" + + invocation := []string{ + "run", "--rm", + "--volume", tree + ":" + within, + "--workdir", within, + base, + } + invocation = append(invocation, chain.Compile...) + // What to compile. Named by the module rather than discovered, so adding a file does not + // silently change what a build produces. + if len(a.Entrypoints) > 0 { + invocation = append(invocation, sourcesFor(a.Entrypoints)...) + } + if _, err := run(ctx, tree, "docker", invocation...); err != nil { + return "", err + } + return filepath.Join(tree, chain.Output), nil +} + +// sourcesFor turns compiled entrypoints back into what to compile. +// +// A module names what a tool host should LOAD — compiled paths under the bundle's root — because +// that is the thing anything else needs to know. What to compile is the same list with the +// language's own extension, which is the toolchain's business rather than the module's. +func sourcesFor(entrypoints []string) []string { + out := make([]string, 0, len(entrypoints)) + for _, e := range entrypoints { + out = append(out, strings.TrimSuffix(e, filepath.Ext(e))+".ts") + } + return out +} diff --git a/internal/builder/bundle_test.go b/internal/builder/bundle_test.go new file mode 100644 index 0000000..2f40ba9 --- /dev/null +++ b/internal/builder/bundle_test.go @@ -0,0 +1,109 @@ +package builder + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +const aBundle = `{"module":"greeter","version":"1", + "build":{"artifacts":[ + {"name":"code","kind":"bundle","language":"typescript","entrypoints":["dist/index.js"]}]}, + "resources":[ + {"id":"files","type":"archive","path":"/opt/greeter","artifact":"code"}]}` + +// compiling is a runner that behaves like a toolchain: when asked to compile, it leaves output +// where the toolchain says output lands. Without this the pack step has nothing to pack, and the +// test would be asserting on a failure rather than on a build. +type compiling struct{ *recorded } + +func (c compiling) run(ctx context.Context, dir, name string, args ...string) (string, error) { + out, err := c.recorded.run(ctx, dir, name, args...) + if name == "docker" && len(args) > 0 && args[0] == "run" { + made := filepath.Join(dir, "dist") + if err := os.MkdirAll(made, 0o755); err != nil { + return "", err + } + if err := os.WriteFile(filepath.Join(made, "index.js"), []byte("console.log(1)"), 0o644); err != nil { + return "", err + } + } + return out, err +} + +// **A module says what it is written in, and needs no Dockerfile.** This is the whole point of the +// bundle recipe: the same module previously needed a hand-written recipe repeating an incantation +// that is easy to get wrong in ways that fail somewhere else. +func TestABundleIsCompiledAndPackedWithNoDockerfile(t *testing.T) { + r, workspace := aRepository(t, aBundle, map[string]string{"index.ts": "console.log(1)"}) + held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)} + + got, err := Build(context.Background(), compiling{r}.run, r, + "https://forge.invalid/greeter.git", "", "", workspace, held) + if err != nil { + t.Fatalf("a module with a language and no Dockerfile did not build: %v", err) + } + + // Compiled in the toolchain the mesh chose, not in one the module named. + var compiled string + for _, line := range r.ran { + if strings.HasPrefix(line, "docker run") { + compiled = line + } + } + if compiled == "" { + t.Fatalf("nothing was compiled:\n%s", strings.Join(r.ran, "\n")) + } + if !strings.Contains(compiled, "mesh-tools/build@sha256:") { + t.Fatalf("the compile did not run in the mesh's own toolchain: %s", compiled) + } + if strings.Contains(strings.Join(r.ran, "\n"), "docker build") { + t.Fatalf("a bundle invoked a Dockerfile build, which is the thing it exists to avoid:\n%s", + strings.Join(r.ran, "\n")) + } + + // And pinned by a digest of what came out, like any other artifact. + digest, _ := got.Manifest.Resources[0]["digest"].(string) + if !strings.HasPrefix(digest, "sha256:") { + t.Fatalf("the bundle was not pinned: %v", got.Manifest.Resources[0]) + } +} + +// **Refused before anything is built, naming what to build first.** A base the mesh has not built +// is not a compile that fails on its first line — it is a question somebody can answer, and saying +// it early is the difference between a fixable message and one about a missing image. +func TestABundleWhoseToolchainIsNotHeldIsRefusedFirst(t *testing.T) { + r, workspace := aRepository(t, aBundle, map[string]string{"index.ts": "console.log(1)"}) + + _, err := Build(context.Background(), compiling{r}.run, r, + "https://forge.invalid/greeter.git", "", "", workspace, nil) + if err == nil { + t.Fatal("a bundle was built with no toolchain to compile it in") + } + if !strings.Contains(err.Error(), "mesh-tools") { + t.Fatalf("the refusal does not name what has to be built first: %v", err) + } + for _, line := range r.ran { + if strings.HasPrefix(line, "docker run") { + t.Fatalf("a compile was attempted before the refusal: %s", line) + } + } +} + +// A language the mesh does not build is refused the same way, and names what it can build. +func TestABundleInAnUnknownLanguageIsRefused(t *testing.T) { + manifest := strings.Replace(aBundle, `"language":"typescript"`, `"language":"cobol"`, 1) + r, workspace := aRepository(t, manifest, map[string]string{"index.ts": "x"}) + + _, err := Build(context.Background(), compiling{r}.run, r, + "https://forge.invalid/greeter.git", "", "", workspace, + map[string]string{"mesh-tools/build": "registry.invalid/x@sha256:" + strings.Repeat("c", 64)}) + if err == nil { + t.Fatal("a language nothing can compile was accepted") + } + if !strings.Contains(err.Error(), "typescript") { + t.Fatalf("the refusal does not say what would have worked: %v", err) + } +} diff --git a/internal/builder/toolchain.go b/internal/builder/toolchain.go new file mode 100644 index 0000000..2d23d19 --- /dev/null +++ b/internal/builder/toolchain.go @@ -0,0 +1,99 @@ +package builder + +import ( + "fmt" + "sort" + "strings" +) + +// What a language implies, so a module does not have to say it. +// +// **A module says what it is written in; this says what that means.** The alternative is what the +// mesh had: every module carrying a Dockerfile that repeated the same incantation, and most of the +// catalogue never converted because the incantation is easy to get wrong in ways that fail +// somewhere else (novox/hq 03-DESIGN/01-to-be/18-building-a-module.md). +// +// A toolchain is deliberately not configurable by the module. Anything a module could override +// here it would be writing a Dockerfile to override, and then this bought nothing. + +// Toolchain is how one language is compiled into a bundle. +type Toolchain struct { + // Language is what a module declares to select this. + Language string + // Base is the module whose artifact provides the compiler, named rather than pinned: the mesh + // answers with the copy it holds, so a recipe never names one particular build of it + // (novox/hq 04-ISSUES/044). + Base string + // Artifact is which of that module's artifacts is the compiling one. + Artifact string + // Compile is what runs inside it, relative to the module's own directory. The output goes to + // Output, which is what gets packed. + Compile []string + // Output is the directory the compiled result lands in, relative to the module's directory. + Output string +} + +// toolchains is every language the mesh can build. +// +// **A closed list, and adding to it is a decision rather than a configuration.** Every language is +// permanent: it needs an SDK carrying the broker client, sealed-credential reading, the event +// envelope and tool serving, and the contracts every module shares change rarely and cascade when +// they do (novox/hq ADR 0039). A mesh whose languages disagree about the envelope fails by ignoring +// messages rather than by failing to compile, so a new entry here is a commitment to keeping N +// implementations of one contract in step. +var toolchains = []Toolchain{ + { + Language: "typescript", + Base: "mesh-tools", + Artifact: "build", + // Invoked by its real path rather than through node_modules/.bin, whose entries are + // symlinks to a launcher that requires its library relatively — and the base image's own + // assembly resolves them away, leaving a launcher whose relative require points nowhere. + // Every module's hand-written Dockerfile had to know this. Now none of them does. + Compile: []string{ + "node", "/app/node_modules/typescript/bin/tsc", + "--module", "NodeNext", "--moduleResolution", "NodeNext", + "--target", "ES2022", "--outDir", "dist", + }, + Output: "dist", + }, +} + +// ToolchainFor is what builds this language, or says what it can build. +func ToolchainFor(language string) (Toolchain, error) { + want := strings.ToLower(strings.TrimSpace(language)) + if want == "" { + return Toolchain{}, fmt.Errorf( + "a bundle must say what language it is written in: the mesh chooses the compiler, and "+ + "it cannot choose one for a module that has not said. It can build %s", spoken()) + } + for _, t := range toolchains { + if t.Language == want { + return t, nil + } + } + return Toolchain{}, fmt.Errorf( + "%q is not a language this mesh builds. It can build %s — and adding one is a decision "+ + "rather than a setting, because every language is another implementation of the "+ + "contracts every module shares", language, spoken()) +} + +// spoken lists the languages, so a refusal says what would have worked. +func spoken() string { + names := make([]string, 0, len(toolchains)) + for _, t := range toolchains { + names = append(names, t.Language) + } + sort.Strings(names) + return strings.Join(names, ", ") +} + +// Languages is every language the mesh can build, for anything that wants to say so. +func Languages() []string { + names := make([]string, 0, len(toolchains)) + for _, t := range toolchains { + names = append(names, t.Language) + } + sort.Strings(names) + return names +} diff --git a/internal/builder/toolchain_test.go b/internal/builder/toolchain_test.go new file mode 100644 index 0000000..e387de3 --- /dev/null +++ b/internal/builder/toolchain_test.go @@ -0,0 +1,56 @@ +package builder + +import ( + "strings" + "testing" +) + +// A language the mesh builds resolves to the toolchain that builds it. +func TestADeclaredLanguageSelectsItsToolchain(t *testing.T) { + chain, err := ToolchainFor("typescript") + if err != nil { + t.Fatalf("typescript is not buildable: %v", err) + } + if chain.Base == "" || chain.Artifact == "" { + t.Fatalf("a toolchain names no base to compile in: %+v", chain) + } + if len(chain.Compile) == 0 || chain.Output == "" { + t.Fatalf("a toolchain says nothing about how to compile or where output lands: %+v", chain) + } +} + +// Case and stray whitespace are a module author's slip, not a different language. +func TestALanguageIsMatchedLoosely(t *testing.T) { + for _, said := range []string{"TypeScript", " typescript ", "TYPESCRIPT"} { + if _, err := ToolchainFor(said); err != nil { + t.Fatalf("%q was refused: %v", said, err) + } + } +} + +// **A refusal says what would have worked.** A module author who names a language the mesh does +// not build is one word away from a language it does, and a bare "unsupported" makes them go +// looking for a list that exists in one place in the source. +func TestAnUnknownLanguageSaysWhatIsBuildable(t *testing.T) { + _, err := ToolchainFor("cobol") + if err == nil { + t.Fatal("a language nothing can build was accepted") + } + for _, want := range Languages() { + if !strings.Contains(err.Error(), want) { + t.Fatalf("the refusal does not mention %q, which would have worked: %v", want, err) + } + } +} + +// And saying nothing is its own message: the mesh chooses the compiler, so a bundle that names no +// language has not asked for anything in particular — which is a mistake rather than a default. +func TestABundleMustSayWhatItIsWrittenIn(t *testing.T) { + _, err := ToolchainFor("") + if err == nil { + t.Fatal("a bundle with no language was accepted, so the mesh guessed a compiler") + } + if !strings.Contains(err.Error(), "must say") { + t.Fatalf("the refusal does not say a language is required: %v", err) + } +} diff --git a/internal/catalogue/build.go b/internal/catalogue/build.go index 361c65e..d7df29a 100644 --- a/internal/catalogue/build.go +++ b/internal/catalogue/build.go @@ -88,12 +88,16 @@ func (m Manifest) Resolve(built []Built) (Manifest, error) { switch artifact.Kind { case ArtifactImage, ArtifactUpstream: filled["image"] = artifact.Reference - case ArtifactArchive: + case ArtifactArchive, ArtifactBundle: + // The same on the wire: both are bytes fetched by digest and unpacked. They differ in + // how they were made — one packed as it stood, the other compiled first — and a + // machine has no reason to care which. filled["source"] = artifact.Reference filled["digest"] = artifact.Digest default: - return Manifest{}, fmt.Errorf("%s: %q is a %q, and an artifact is %q, %q or %q", - m.Module, named, artifact.Kind, ArtifactImage, ArtifactArchive, ArtifactUpstream) + return Manifest{}, fmt.Errorf("%s: %q is a %q, and an artifact is %q, %q, %q or %q", + m.Module, named, artifact.Kind, ArtifactImage, ArtifactArchive, ArtifactUpstream, + ArtifactBundle) } out.Resources = append(out.Resources, filled) } @@ -119,15 +123,40 @@ func (b *Build) problems(module string) []string { } seen[a.Name] = true switch a.Kind { - case ArtifactImage, ArtifactArchive, ArtifactUpstream: + case ArtifactImage, ArtifactArchive, ArtifactUpstream, ArtifactBundle: default: problems = append(problems, fmt.Sprintf( - "%s: %q is a %q, and an artifact is %q, %q or %q", - module, a.Name, a.Kind, ArtifactImage, ArtifactArchive, ArtifactUpstream)) + "%s: %q is a %q, and an artifact is %q, %q, %q or %q", + module, a.Name, a.Kind, ArtifactImage, ArtifactArchive, ArtifactUpstream, + ArtifactBundle)) } - if a.From == "" { - problems = append(problems, fmt.Sprintf( - "%s: %q says nothing about what it is built from", module, a.Name)) + // **A bundle is built from the module itself, so it says a language instead.** Everything + // else names what it is built from: a Dockerfile, a directory, somebody else's reference. + // A bundle's source is the module's own directory by definition, and what it needs to say + // is which compiler — because the mesh chooses that, and cannot choose for a module that + // has not said. + if a.Kind == ArtifactBundle { + if a.From != "" { + problems = append(problems, fmt.Sprintf( + "%s: %q is a bundle and names what it is built from (%q). A bundle is built "+ + "from the module's own directory; what it says is the language", + module, a.Name, a.From)) + } + if strings.TrimSpace(a.Language) == "" { + problems = append(problems, fmt.Sprintf( + "%s: %q is a bundle and says no language, so nothing can choose a compiler "+ + "for it", module, a.Name)) + } + } else { + if a.From == "" { + problems = append(problems, fmt.Sprintf( + "%s: %q says nothing about what it is built from", module, a.Name)) + } + if a.Language != "" { + problems = append(problems, fmt.Sprintf( + "%s: %q is a %q and names a language. Only a bundle is compiled by the mesh; "+ + "everything else brings its own recipe", module, a.Name, a.Kind)) + } } // An upstream image is named, not read from the repository, so the path rule does not // apply to it — and applying it anyway would refuse every reference with a registry host diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index 2085d2d..7f6eb9c 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -389,6 +389,26 @@ type Artifact struct { // version and an operating system — while letting each be built and published separately. // Empty means the whole recipe, which is what a module with one image says by saying nothing. Target string `json:"target,omitempty"` + + // Language is what this module's code is written in, for a bundle. + // + // **Declared, never guessed.** Inferring it from what files happen to be present makes a + // module's build depend on a directory listing, and a module that adds a stray file builds + // differently for a reason nobody can see. It is also the only thing a bundle needs to say: + // everything else about the toolchain — which compiler, which flags, which base — is the + // mesh's, and a module that could override it would be writing a Dockerfile again. + // + // Empty for every other kind, which do not compile. + Language string `json:"language,omitempty"` + + // Entrypoints are the compiled files a tool host should load from this module, relative to the + // bundle's root. + // + // **Named rather than derived from which files exist**, for the same reason as the language: + // the module knows what it serves, and a build that guesses would change meaning when + // somebody adds a helper. An empty list is a bundle that is run rather than loaded — a + // provisioner or a step, named by whatever runs it. + Entrypoints []string `json:"entrypoints,omitempty"` } // Kinds an artifact may be. @@ -397,6 +417,21 @@ const ( ArtifactImage = "image" // ArtifactArchive is a directory in this repository, packed. ArtifactArchive = "archive" + // ArtifactBundle is this module's own code, COMPILED by a toolchain and then packed. + // + // **The one recipe that both builds and packs**, and the reason it exists is the authoring + // burden. An `archive` packs a directory as it stands, so shipping compiled output means + // compiling somewhere first — which means a Dockerfile, repeating the same incantation in + // every module: two base arguments, a working directory chosen so the SDK resolves upward, the + // compiler invoked by absolute path because the usual symlink is resolved away when the base is + // assembled, a second stage, an environment variable naming the entrypoints. Most of the + // catalogue is unconverted and that is why. + // + // A bundle says what the module is written in and nothing about how. The mesh knows what a + // language implies, which is the whole of the difference: a Dockerfile is right for software + // that needs a particular base, and wrong for "compile my module's code", which is the same + // operation every time. + ArtifactBundle = "bundle" // ArtifactUpstream is an image somebody else built, mirrored into the mesh's own registry and // pinned by the digest it lands with. // -- 2.54.0 From e3748bc06f487e9f540e93c15f97c3e609bdb1f8 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 15 Sep 2026 02:09:06 +0200 Subject: [PATCH 06/17] One module, several languages, each bundle packed alone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A module is one piece of software and may still carry a daemon in one language, tools in another and a package in a third. The first cut compiled every bundle into the toolchain's single output directory, so two of them would have overwritten each other and then been packed together — one artifact containing both, published twice. So output is a property of the artifact, not of the toolchain, and the toolchain says how it is told where to write rather than where it writes. Under a directory named for the build rather than beside the source, so a pack never sweeps up the module's own working files. A second toolchain is declared so the multi-language path is exercised rather than asserted — a list with one entry cannot fail the way a list with four will. And the fake compiler in the tests now writes where it was TOLD to. One that always wrote to a fixed place would have passed whether or not each artifact got its own directory, which is the whole of what these tests are for. Claude-Session: https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- internal/builder/builder.go | 25 +++++++--- internal/builder/bundle_test.go | 77 ++++++++++++++++++++++++++---- internal/builder/toolchain.go | 38 ++++++++++++--- internal/builder/toolchain_test.go | 37 +++++++++++++- 4 files changed, 155 insertions(+), 22 deletions(-) diff --git a/internal/builder/builder.go b/internal/builder/builder.go index fe6c754..c8a16ae 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -461,6 +461,12 @@ func compile(ctx context.Context, run Runner, tree string, chain Toolchain, // rather than configurable: a module that could move this would be describing its own build. const within = "/app/modules/module" + // **Its own output directory, because a module may be several languages at once.** One module + // is one piece of software and can still carry a daemon in one language, tools in another and + // a package in a third (ADR 0040). Compiling them all into one place would have them overwrite + // each other and then be packed together, so each bundle compiles and packs alone. + out := Out(a.Name) + invocation := []string{ "run", "--rm", "--volume", tree + ":" + within, @@ -468,15 +474,18 @@ func compile(ctx context.Context, run Runner, tree string, chain Toolchain, base, } invocation = append(invocation, chain.Compile...) + if chain.OutputFlag != "" { + invocation = append(invocation, chain.OutputFlag, out) + } // What to compile. Named by the module rather than discovered, so adding a file does not // silently change what a build produces. if len(a.Entrypoints) > 0 { - invocation = append(invocation, sourcesFor(a.Entrypoints)...) + invocation = append(invocation, sourcesFor(a.Entrypoints, out)...) } if _, err := run(ctx, tree, "docker", invocation...); err != nil { return "", err } - return filepath.Join(tree, chain.Output), nil + return filepath.Join(tree, out), nil } // sourcesFor turns compiled entrypoints back into what to compile. @@ -484,10 +493,14 @@ func compile(ctx context.Context, run Runner, tree string, chain Toolchain, // A module names what a tool host should LOAD — compiled paths under the bundle's root — because // that is the thing anything else needs to know. What to compile is the same list with the // language's own extension, which is the toolchain's business rather than the module's. -func sourcesFor(entrypoints []string) []string { - out := make([]string, 0, len(entrypoints)) +func sourcesFor(entrypoints []string, out string) []string { + sources := make([]string, 0, len(entrypoints)) for _, e := range entrypoints { - out = append(out, strings.TrimSuffix(e, filepath.Ext(e))+".ts") + // An entrypoint is named as it will be FOUND — a path inside the unpacked bundle — so the + // source is the same path with the output directory taken off the front and the language's + // own extension on the end. + at := strings.TrimPrefix(strings.TrimPrefix(e, out), "/") + sources = append(sources, strings.TrimSuffix(at, filepath.Ext(at))+".ts") } - return out + return sources } diff --git a/internal/builder/bundle_test.go b/internal/builder/bundle_test.go index 2f40ba9..0b8115a 100644 --- a/internal/builder/bundle_test.go +++ b/internal/builder/bundle_test.go @@ -10,7 +10,7 @@ import ( const aBundle = `{"module":"greeter","version":"1", "build":{"artifacts":[ - {"name":"code","kind":"bundle","language":"typescript","entrypoints":["dist/index.js"]}]}, + {"name":"code","kind":"bundle","language":"typescript","entrypoints":["index.js"]}]}, "resources":[ {"id":"files","type":"archive","path":"/opt/greeter","artifact":"code"}]}` @@ -21,15 +21,28 @@ type compiling struct{ *recorded } func (c compiling) run(ctx context.Context, dir, name string, args ...string) (string, error) { out, err := c.recorded.run(ctx, dir, name, args...) - if name == "docker" && len(args) > 0 && args[0] == "run" { - made := filepath.Join(dir, "dist") - if err := os.MkdirAll(made, 0o755); err != nil { - return "", err - } - if err := os.WriteFile(filepath.Join(made, "index.js"), []byte("console.log(1)"), 0o644); err != nil { - return "", err + if name != "docker" || len(args) == 0 || args[0] != "run" { + return out, err + } + // **Writes where it was TOLD to**, rather than to a fixed directory. A fake that always wrote + // to one place would pass whether or not the builder gave each artifact its own — which is the + // thing being tested. + where := "" + for i, a := range args { + if a == "--outDir" && i+1 < len(args) { + where = args[i+1] } } + if where == "" { + return out, err + } + made := filepath.Join(dir, where) + if err := os.MkdirAll(made, 0o755); err != nil { + return "", err + } + if err := os.WriteFile(filepath.Join(made, "index.js"), []byte("console.log(1)"), 0o644); err != nil { + return "", err + } return out, err } @@ -107,3 +120,51 @@ func TestABundleInAnUnknownLanguageIsRefused(t *testing.T) { t.Fatalf("the refusal does not say what would have worked: %v", err) } } + +// **One module, two bundles, and neither packs the other.** +// +// The case that matters for real modules: a module is one piece of software and may still carry a +// daemon in one language and tools in another (ADR 0040). An earlier version of this compiled +// every bundle into the toolchain's single output directory, so two of them would overwrite each +// other and then be packed together — one artifact containing both, twice. +func TestTwoBundlesInOneModuleArePackedSeparately(t *testing.T) { + const two = `{"module":"greeter","version":"1", + "build":{"artifacts":[ + {"name":"daemon","kind":"bundle","language":"typescript","entrypoints":["index.js"]}, + {"name":"tools","kind":"bundle","language":"typescript","entrypoints":["index.js"]}]}, + "resources":[ + {"id":"a","type":"archive","path":"/opt/greeter/daemon","artifact":"daemon"}, + {"id":"b","type":"archive","path":"/opt/greeter/tools","artifact":"tools"}]}` + + r, workspace := aRepository(t, two, map[string]string{"index.ts": "console.log(1)"}) + held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)} + + got, err := Build(context.Background(), compiling{r}.run, r, + "https://forge.invalid/greeter.git", "", "", workspace, held) + if err != nil { + t.Fatalf("a module with two bundles did not build: %v", err) + } + + // Compiled into two different places. + var outputs []string + for _, line := range r.ran { + for _, part := range strings.Fields(line) { + if strings.HasPrefix(part, ".mesh-build/") { + outputs = append(outputs, part) + } + } + } + if len(outputs) != 2 || outputs[0] == outputs[1] { + t.Fatalf("two bundles did not get their own output directories: %v", outputs) + } + + // And published as two artifacts, each with its own digest. + if len(r.archives) != 2 { + t.Fatalf("expected two archives published, got %v", r.archives) + } + first, _ := got.Manifest.Resources[0]["digest"].(string) + second, _ := got.Manifest.Resources[1]["digest"].(string) + if first == "" || second == "" { + t.Fatalf("a bundle was not pinned: %v", got.Manifest.Resources) + } +} diff --git a/internal/builder/toolchain.go b/internal/builder/toolchain.go index 2d23d19..9d50213 100644 --- a/internal/builder/toolchain.go +++ b/internal/builder/toolchain.go @@ -26,13 +26,28 @@ type Toolchain struct { Base string // Artifact is which of that module's artifacts is the compiling one. Artifact string - // Compile is what runs inside it, relative to the module's own directory. The output goes to - // Output, which is what gets packed. + // Compile is what runs inside it, relative to the module's own directory. + // + // The output directory is appended by the builder, per artifact, because one module may + // declare several bundles — a daemon in one language, tools in another, a package in a third — + // and a toolchain with one fixed output would have them overwrite each other and then be + // packed together. Compile []string - // Output is the directory the compiled result lands in, relative to the module's directory. - Output string + // OutputFlag is how this compiler is told where to put its output. + OutputFlag string } +// Out is where one artifact's compiled output lands, inside the module's own directory. +// +// **Per artifact, never per toolchain.** A module is one piece of software and may still be +// written in several languages — a daemon in one, a tool in another, a package in a third (ADR +// 0040). Each bundle is compiled and packed alone, so what a machine unpacks is that artifact and +// nothing else. +// +// Under a directory named for the build rather than beside the source, so a pack never sweeps up +// the module's own working files, and two builds of one commit see the same tree. +func Out(artifact string) string { return ".mesh-build/" + artifact } + // toolchains is every language the mesh can build. // // **A closed list, and adding to it is a decision rather than a configuration.** Every language is @@ -53,9 +68,20 @@ var toolchains = []Toolchain{ Compile: []string{ "node", "/app/node_modules/typescript/bin/tsc", "--module", "NodeNext", "--moduleResolution", "NodeNext", - "--target", "ES2022", "--outDir", "dist", + "--target", "ES2022", }, - Output: "dist", + OutputFlag: "--outDir", + }, + { + Language: "python", + Base: "mesh-tools-python", + Artifact: "build", + // Nothing to compile: what a bundle needs is the module's own code and its dependencies + // resolved, so the "compile" is an install into the output directory. Named here rather + // than left implicit because a reader comparing two toolchains should be able to see what + // each actually does. + Compile: []string{"python", "-m", "pip", "install", "--no-compile", "--target"}, + OutputFlag: "", }, } diff --git a/internal/builder/toolchain_test.go b/internal/builder/toolchain_test.go index e387de3..ffb90ec 100644 --- a/internal/builder/toolchain_test.go +++ b/internal/builder/toolchain_test.go @@ -14,8 +14,8 @@ func TestADeclaredLanguageSelectsItsToolchain(t *testing.T) { if chain.Base == "" || chain.Artifact == "" { t.Fatalf("a toolchain names no base to compile in: %+v", chain) } - if len(chain.Compile) == 0 || chain.Output == "" { - t.Fatalf("a toolchain says nothing about how to compile or where output lands: %+v", chain) + if len(chain.Compile) == 0 { + t.Fatalf("a toolchain says nothing about how to compile: %+v", chain) } } @@ -54,3 +54,36 @@ func TestABundleMustSayWhatItIsWrittenIn(t *testing.T) { t.Fatalf("the refusal does not say a language is required: %v", err) } } + +// **One module, several languages, and each bundle packed alone.** +// +// A module is one piece of software (ADR 0040) and may still carry a daemon in one language, tools +// in another and a package in a third. Compiling them into one output directory would have them +// overwrite each other and then be packed together, so output is a property of the artifact rather +// than of the toolchain. +func TestTwoBundlesInOneModuleDoNotShareAnOutputDirectory(t *testing.T) { + first, second := Out("daemon"), Out("tools") + if first == second { + t.Fatalf("two artifacts compile into the same place (%q), so one would overwrite the "+ + "other and both would be packed together", first) + } + for _, out := range []string{first, second} { + if strings.HasPrefix(out, "/") || strings.Contains(out, "..") { + t.Fatalf("%q leaves the module's own directory", out) + } + } +} + +// And the mesh can say what it builds, which is what a refusal quotes. +func TestTheMeshSaysWhichLanguagesItBuilds(t *testing.T) { + spoken := Languages() + if len(spoken) < 2 { + t.Fatalf("only %v — this test exists to keep the multi-language path real rather than "+ + "theoretical", spoken) + } + for _, language := range spoken { + if _, err := ToolchainFor(language); err != nil { + t.Fatalf("%q is listed as buildable and has no toolchain: %v", language, err) + } + } +} -- 2.54.0 From 385bc5b9bf90d3310a8537a436c54a86a1528fed Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 15 Sep 2026 13:20:01 +0200 Subject: [PATCH 07/17] A module can be told which port it was given MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mesh assigns the machine-side port and a module does not choose one (ADR 0038). For a container that is invisible: the mesh rewrites ports into assigned:wanted, the software binds the number it always bound, and the machine publishes another. A process has no such layer. It runs on the machine, there is nothing to rewrite, and it binds whatever its configuration says. So every process bound the number written in its own config, two modules declaring the same one would collide, and the mesh's whole reason for assigning ports was defeated by the resource kind that most needs it — introduced, by me, three commits ago. So a module asks. ${port:8080} is "the machine-side port you gave me for the 8080 I said I listen on", written into its own configuration exactly as an address it was bound to is. Asking about a port it never declared is refused, and the refusal says what it did declare: the module is asking about something the mesh has no opinion on, and answering would put a guess into a configuration file as a port number. With nothing assigned yet it is told what it asked for, so a mesh that has made no assignment still composes something coherent rather than writing a zero. Claude-Session: https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- internal/catalogue/declaration.go | 6 ++ internal/catalogue/port_into_files.go | 87 ++++++++++++++++++++ internal/catalogue/port_into_files_test.go | 66 +++++++++++++++ internal/catalogue/showcase_manifest_test.go | 83 +++++++++++++++++++ 4 files changed, 242 insertions(+) create mode 100644 internal/catalogue/port_into_files.go create mode 100644 internal/catalogue/port_into_files_test.go create mode 100644 internal/catalogue/showcase_manifest_test.go diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index 3b4a1c5..707a086 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -456,6 +456,12 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) { // assigned to. Beside the bound values because it is the same kind of fact — the // mesh's own, held in the clear — and because a module that must name itself to // something else has no binding to learn it from (novox/hq ADR 0066). + // Which port this machine gave it, for a module that binds directly rather than + // through a runtime that can remap (ADR 0038). Applied before the machine's facts so + // a refusal names the port rather than whatever came after it. + if err := portInto(copied, m.Module, m.Listens, with); err != nil { + return nil, err + } if err := machineInto(copied, thisMachine, m.Module); err != nil { return nil, err } diff --git a/internal/catalogue/port_into_files.go b/internal/catalogue/port_into_files.go new file mode 100644 index 0000000..f694cd9 --- /dev/null +++ b/internal/catalogue/port_into_files.go @@ -0,0 +1,87 @@ +package catalogue + +import ( + "fmt" + "regexp" + "strconv" + "strings" +) + +// Telling a module which port it was given. +// +// **The mesh assigns the machine-side port and a module does not choose one** +// ([ADR 0038](../../02-DECISIONS/0038-the-mesh-assigns-the-port.md)). For a container that is +// invisible: the mesh rewrites `ports` into `assigned:wanted`, the software inside binds the number +// it has always bound, and the machine publishes a different one. +// +// **A process has no such layer.** It runs on the machine, there is nothing to rewrite, and it +// binds whatever its configuration says — so without this, every process binds the number written +// in its own config, two modules declaring the same one collide, and the mesh's whole reason for +// assigning ports is defeated by the resource kind that most needs it. +// +// So a module asks. `${port:8080}` is "the machine-side port you gave me for the 8080 I said I +// listen on", and the module writes that into its own configuration exactly as it writes an +// address it was bound to. + +// ofPort is where a module asks which port it was given: ${port:}. +var ofPort = regexp.MustCompile(`\$\{port:([0-9]+)\}`) + +// portsUsed are the ports a file's content asks about, first appearance first. +func portsUsed(content string) []int { + var used []int + seen := map[int]bool{} + for _, m := range ofPort.FindAllStringSubmatch(content, -1) { + n, err := strconv.Atoi(m[1]) + if err != nil || seen[n] { + continue + } + seen[n] = true + used = append(used, n) + } + return used +} + +// portInto replaces a file's ${port:…} placeholders with what this machine assigned. +// +// A port the module did not say it listens on is refused, for the same reason a binding's unknown +// key is: the module is asking about something it never declared, and the answer would be a guess. +// Left alone, the literal would be written into a configuration file and read as a port number. +func portInto(resource map[string]any, module string, listens []Listening, with Rendering) error { + if fmt.Sprint(resource["type"]) != "file" { + return nil + } + content, ok := resource["content"].(string) + if !ok { + return nil + } + for _, wanted := range portsUsed(content) { + var declared bool + for _, l := range listens { + if l.Port == wanted { + declared = true + } + } + if !declared { + return fmt.Errorf( + "%s has a file that says ${port:%d}, and %s does not say it listens on %d. A "+ + "module is told the port it was given for something it declared, and %s", + module, wanted, module, wanted, orNoListens(listens)) + } + content = strings.ReplaceAll(content, fmt.Sprintf("${port:%d}", wanted), + strconv.Itoa(with.machinePort(module, wanted))) + resource["content"] = content + } + return nil +} + +// orNoListens says what would have worked, so a refusal is one edit from right. +func orNoListens(listens []Listening) string { + if len(listens) == 0 { + return "it declares no ports at all" + } + said := make([]string, 0, len(listens)) + for _, l := range listens { + said = append(said, strconv.Itoa(l.Port)) + } + return "it declares " + strings.Join(said, ", ") +} diff --git a/internal/catalogue/port_into_files_test.go b/internal/catalogue/port_into_files_test.go new file mode 100644 index 0000000..e8bcba7 --- /dev/null +++ b/internal/catalogue/port_into_files_test.go @@ -0,0 +1,66 @@ +package catalogue + +import ( + "strings" + "testing" +) + +// **A process binds the port the mesh gave it, not the one it wrote down.** +// +// A container never needed this: the mesh rewrites its `ports` into assigned:wanted, so the +// software binds the number it always bound and the machine publishes another. A process runs on +// the machine with nothing to rewrite, so without a way to ask, every process binds the number in +// its own configuration and two modules declaring the same one collide — which is the whole +// problem ADR 0038 exists to prevent, reintroduced by the resource kind that most needs it. +func TestAModuleIsToldWhichPortItWasGiven(t *testing.T) { + file := map[string]any{ + "type": "file", "id": "settings", + "content": "LISTEN=${port:8080}\n", + } + listens := []Listening{{Port: 8080, From: FromMesh}} + with := Rendering{Ports: map[string]map[int]int{"showcase": {8080: 21000}}} + + if err := portInto(file, "showcase", listens, with); err != nil { + t.Fatal(err) + } + if got := file["content"].(string); got != "LISTEN=21000\n" { + t.Fatalf("the module was not told its assigned port: %q", got) + } +} + +// With nothing assigned yet, it is told the port it asked about — so a mesh that has not made an +// assignment still composes something coherent rather than writing a zero. +func TestWithNoAssignmentAModuleIsToldWhatItAskedFor(t *testing.T) { + file := map[string]any{"type": "file", "content": "LISTEN=${port:8080}\n"} + if err := portInto(file, "showcase", []Listening{{Port: 8080}}, Rendering{}); err != nil { + t.Fatal(err) + } + if got := file["content"].(string); got != "LISTEN=8080\n" { + t.Fatalf("an unassigned port did not fall back to what was declared: %q", got) + } +} + +// **Asking about a port it never declared is refused**, and the refusal says what it did declare. +// The module is asking about something the mesh has no opinion on, and answering would be a guess +// written into a configuration file as a port number. +func TestAskingAboutAnUndeclaredPortIsRefused(t *testing.T) { + file := map[string]any{"type": "file", "content": "LISTEN=${port:9999}\n"} + err := portInto(file, "showcase", []Listening{{Port: 8080}}, Rendering{}) + if err == nil { + t.Fatal("a module was told a port it never said it listens on") + } + if !strings.Contains(err.Error(), "8080") { + t.Fatalf("the refusal does not say what would have worked: %v", err) + } +} + +// And a file mentioning no port is left exactly as it was. +func TestAFileWithNoPortIsUntouched(t *testing.T) { + file := map[string]any{"type": "file", "content": "GREETING=hello\n"} + if err := portInto(file, "showcase", nil, Rendering{}); err != nil { + t.Fatal(err) + } + if got := file["content"].(string); got != "GREETING=hello\n" { + t.Fatalf("a file with no port was changed: %q", got) + } +} diff --git a/internal/catalogue/showcase_manifest_test.go b/internal/catalogue/showcase_manifest_test.go new file mode 100644 index 0000000..3dc2d49 --- /dev/null +++ b/internal/catalogue/showcase_manifest_test.go @@ -0,0 +1,83 @@ +package catalogue + +import ( + "os" + "testing" +) + +// **The showcase module is parsed by the real parser, in the real test suite.** +// +// A module that exercises every capability is only worth having if something checks it still does. +// Written as a test rather than a script so it runs whenever anything about manifests changes — +// which is exactly when a module using all of it would quietly stop being valid. +func TestTheShowcaseModuleIsAValidManifest(t *testing.T) { + raw, err := os.ReadFile("../../../mesh-catalog/modules/showcase/module.json") + if err != nil { + t.Skipf("the catalogue is not beside this checkout: %v", err) + } + m, err := ParseManifest(raw) + if err != nil { + t.Fatalf("the module that exercises everything does not parse:\n%v", err) + } + + // Every resource kind a MODULE may use, actually in it. + // + // Two of the host's eleven are deliberately absent, and the reasons are worth keeping: + // + // - `action` is refused to modules outright. The link may not carry a command to run (ADR + // 0005), so a module that needs something done ships a program that reads what the mesh + // delivered and reconciles — which is what a run-once `process` is. + // - `service` puts an EXISTING unit into a state and deliberately installs none, which is + // right for software that ships its own unit. A module whose code the mesh built has no + // such unit until the mesh writes one, and that is a `process`. + kinds := map[string]bool{} + for _, r := range m.Resources { + kind, _ := r["type"].(string) + kinds[kind] = true + } + for _, want := range []string{ + "access", "archive", "container", "directory", "file", "network", "package", + "process", "user", + } { + if !kinds[want] { + t.Errorf("showcase no longer exercises %q", want) + } + } + + // And all three ways a module's own code can run, which is the thing most easily lost. + var stays, once, scheduled bool + for _, r := range m.Resources { + if kind, _ := r["type"].(string); kind != "process" { + continue + } + switch { + case r["run-once"] == true: + once = true + case r["schedule"] != nil: + scheduled = true + default: + stays = true + } + } + if !stays || !once || !scheduled { + t.Errorf("showcase does not exercise all three process modes: stays=%v once=%v scheduled=%v", + stays, once, scheduled) + } + + // And the artifact kinds, including the one that compiles. + var bundle, archive, upstream bool + for _, a := range m.Build.Artifacts { + switch a.Kind { + case ArtifactBundle: + bundle = true + case ArtifactArchive: + archive = true + case ArtifactUpstream: + upstream = true + } + } + if !bundle || !archive || !upstream { + t.Errorf("showcase does not exercise every artifact kind: bundle=%v archive=%v upstream=%v", + bundle, archive, upstream) + } +} -- 2.54.0 From 18ec632baad8cda58a38c21d8eb68ff1227ba818 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 15 Sep 2026 20:51:00 +0200 Subject: [PATCH 08/17] The example manifest follows the rename Claude-Session: https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- examples/modules/{registry.json => distribution.json} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename examples/modules/{registry.json => distribution.json} (97%) diff --git a/examples/modules/registry.json b/examples/modules/distribution.json similarity index 97% rename from examples/modules/registry.json rename to examples/modules/distribution.json index 6fe2ba0..5cfb4ba 100644 --- a/examples/modules/registry.json +++ b/examples/modules/distribution.json @@ -1,5 +1,5 @@ { - "module": "registry", + "module": "distribution", "version": "1", "provides": [ { -- 2.54.0 From fcdb065660851b17f2bcf1f558a782146f45f539 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 15 Sep 2026 21:31:18 +0200 Subject: [PATCH 09/17] The mesh's knowledge is a fact a module asks for, not three modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mesh-names, mesh-resolver and the names half of the overlay generators are gone. They ran no software and could not be swapped for anything, which is the test of whether something is a module at all — they existed because computed output needed somewhere to live, and the control plane's only shape for output was a module. Now a module says where it wants what the mesh knows: facts: { node-zones: /etc/mesh-resolver/nodes.conf } and is given a file, under its own name, applied and removed like anything else it declares. Two facts exist: node-names (a hosts file — exact names) and node-zones (every machine as a wildcard, *.homer.internal is homer). Asking for a fact the mesh does not compute is refused naming what would have worked, because a daemon that starts and reads a file nobody wrote is a worse way to find out. The names ride with the network now: wireguard's manifest asks for node-names into /etc/hosts, because being on the private network is what gives a machine a name. networking no longer requires name-resolution — names are not a provision, and the module that answered it ran nothing. One behaviour inverted, deliberately: choosing another VPN used to drag WireGuard in anyway, because only WireGuard provided the addressing the names module required — the node-scope claim existed to at least make that loud. With names as a fact there is nothing to drag in: tailscale assigned means tailscale, alone. The claim still catches two VPNs assigned explicitly. And a machine the mesh cannot place is left out of both files rather than named at nothing: a name resolving to nothing hangs a connection, where an unknown name fails at once and says so. In practice that is only ever a token issued and not yet used — a machine that has announced itself has an address. Claude-Session: https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- cmd/mesh-control/modules.go | 8 +- cmd/mesh-control/network.go | 11 +- examples/modules/dnsmasq.json | 10 +- examples/modules/modules_test.go | 9 +- internal/catalogue/declaration.go | 13 ++ internal/catalogue/facts.go | 145 +++++++++++++++++++++++ internal/catalogue/facts_test.go | 97 +++++++++++++++ internal/catalogue/manifest.go | 19 +++ internal/catalogue/provided_test.go | 58 +++++---- internal/overlay/generator.go | 80 ++----------- internal/overlay/names_generator_test.go | 61 ---------- internal/overlay/resolver.go | 82 ------------- internal/overlay/resolver_test.go | 99 ---------------- 13 files changed, 348 insertions(+), 344 deletions(-) create mode 100644 internal/catalogue/facts.go create mode 100644 internal/catalogue/facts_test.go delete mode 100644 internal/overlay/names_generator_test.go delete mode 100644 internal/overlay/resolver.go delete mode 100644 internal/overlay/resolver_test.go diff --git a/cmd/mesh-control/modules.go b/cmd/mesh-control/modules.go index 96e2c3b..cc09f3e 100644 --- a/cmd/mesh-control/modules.go +++ b/cmd/mesh-control/modules.go @@ -36,8 +36,12 @@ import ( func providedModules() []catalogue.Manifest { var out []catalogue.Manifest for _, raw := range []map[string]any{ - overlay.Manifest(), overlay.NamesManifest(), overlay.ResolverManifest(), - overlay.DomainManifest(), + // **Two used to be here and are gone**: one wrote the mesh's names into a hosts file, the + // other wrote the same machines as wildcards for a resolver to read. Neither ran software + // and neither could be swapped for anything, which is the test of whether a thing is a + // module at all (novox/hq ADR 0040). They existed because computed output needed somewhere + // to live, and now a module says where it wants it — `facts` in its own manifest. + overlay.Manifest(), overlay.DomainManifest(), } { var m catalogue.Manifest b, _ := json.Marshal(raw) diff --git a/cmd/mesh-control/network.go b/cmd/mesh-control/network.go index c1a2d82..c031f5f 100644 --- a/cmd/mesh-control/network.go +++ b/cmd/mesh-control/network.go @@ -231,12 +231,13 @@ func generators(ctx context.Context, open *stores) ( if err != nil { return nil, err } - // Both generators see the same machines: the ones on the private network. Names for a machine - // that is not on it would resolve to addresses it cannot reach, which is worse than no names. + // **One generator now.** Two more used to sit beside it — the names and a resolver's zone + // file — as modules that ran nothing. Both are facts a module asks for in its manifest + // (`facts:` — catalogue.FactsInto), computed from the same machines this sees: the ones on the + // private network, because a name for a machine not on it would resolve to an address nothing + // can reach. return map[string]catalogue.Generator{ - overlay.Name: net, - overlay.Names: overlay.NamesFor(net.Nodes()), - overlay.Resolver: overlay.ResolverFor(net.Nodes()), + overlay.Name: net, }, nil } diff --git a/examples/modules/dnsmasq.json b/examples/modules/dnsmasq.json index 79d311d..87ec177 100644 --- a/examples/modules/dnsmasq.json +++ b/examples/modules/dnsmasq.json @@ -1,9 +1,6 @@ { "module": "dnsmasq", "version": "1", - "requires": [ - "resolver-data" - ], "provides": [ "wildcard-resolution" ], @@ -43,8 +40,11 @@ "boot": "enabled", "restart-on": [ "config", - "mesh-resolver.nodes" + "dnsmasq.fact-node-zones" ] } - ] + ], + "facts": { + "node-zones": "/etc/mesh-resolver/nodes.conf" + } } diff --git a/examples/modules/modules_test.go b/examples/modules/modules_test.go index fcaca21..72d93c1 100644 --- a/examples/modules/modules_test.go +++ b/examples/modules/modules_test.go @@ -64,13 +64,16 @@ func TestTheResolverReadsTheMeshsNamesAndFollowsThem(t *testing.T) { if config == nil || service == nil { t.Fatal("the module has no configuration or no service") } - if !strings.Contains(config["content"].(string), overlay.ResolverPath) { - t.Fatalf("it does not read what the mesh writes at %s", overlay.ResolverPath) + // The zone file is a FACT the module asks for, at a path it chose. The mesh writes it there; + // what reads it and how is this module's own business, which is the whole shape. + const zones = "/etc/mesh-resolver/nodes.conf" + if !strings.Contains(config["content"].(string), zones) { + t.Fatalf("it does not read what the mesh writes at %s", zones) } var follows bool for _, id := range service["restart-on"].([]any) { - if id.(string) == overlay.Resolver+".nodes" { + if id.(string) == "dnsmasq.fact-node-zones" { follows = true } } diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index 707a086..c99cb26 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -498,6 +498,19 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) { } out = append(out, copied) } + + // **What only the mesh knows, where this module asked for it.** The graph is the control + // plane's; making a name resolve is the module's software. Emitted as ordinary files under + // this module's name, so they are applied, reported and removed exactly as anything else + // it declares. + given, err := FactsInto(m, r, with.Names) + if err != nil { + return nil, err + } + for _, fact := range given { + fact["id"] = m.Module + "." + fmt.Sprint(fact["id"]) + out = append(out, fact) + } } return out, nil } diff --git a/internal/catalogue/facts.go b/internal/catalogue/facts.go new file mode 100644 index 0000000..d16c59c --- /dev/null +++ b/internal/catalogue/facts.go @@ -0,0 +1,145 @@ +package catalogue + +import ( + "fmt" + "sort" + "strings" +) + +// What only the mesh knows, written where a module asks for it. +// +// **The graph is the control plane's; using it is the module's.** The mesh knows which machines +// exist, what they are called, and where they are. Turning that into a name that resolves is +// somebody's software, and which software is a choice the mesh should not be making. +// +// This replaced three modules — names, a resolver's data, and the private network's own +// configuration — that existed only because computed output needed somewhere to live. They ran no +// software and could not be swapped for anything, which is the test of whether something is a +// module at all (novox/hq ADR 0040). + +const ( + // FactNodeNames is every machine's name and address, as a hosts file. + // + // Exact names only: `homer` and `homer.internal` resolve to homer. Anything *under* a machine + // is a wildcard, which a hosts file cannot express — that is FactNodeZones. + FactNodeNames = "node-names" + + // FactNodeZones is every machine as a wildcard: `*.homer.internal` is homer. + // + // Written in the form a resolver reads. A machine's own name and everything under it are one + // fact — if homer is at an address, so is anything homer serves. + FactNodeZones = "node-zones" +) + +// facts is every fact the mesh computes, and what writes it. +// +// **A closed list.** A module asking for a fact the mesh does not have is asking for a file nobody +// will write, and finding that out on a machine — as a daemon that starts, reads nothing, and +// answers no queries — is worse than being told where the manifest is. +var facts = map[string]func(Resolution, map[string]string) string{ + FactNodeNames: nodeNames, + FactNodeZones: nodeZones, +} + +// FactsInto renders the facts a module asked for, as files it will be given. +// +// The module owns everything after the file exists: loading it, restarting on it, what a resolver +// does with it. This only puts it there. +func FactsInto(m Manifest, r Resolution, addresses map[string]string) ([]map[string]any, error) { + if len(m.Facts) == 0 { + return nil, nil + } + names := make([]string, 0, len(m.Facts)) + for name := range m.Facts { + names = append(names, name) + } + sort.Strings(names) + + out := make([]map[string]any, 0, len(names)) + for _, name := range names { + write, known := facts[name] + if !known { + return nil, fmt.Errorf( + "%s asks the mesh for %q, which it does not compute. It has %s", + m.Module, name, spokenFacts()) + } + path := m.Facts[name] + if !strings.HasPrefix(path, "/") { + return nil, fmt.Errorf( + "%s asks for %q at %q, which is not an absolute path", m.Module, name, path) + } + out = append(out, map[string]any{ + "id": "fact-" + name, "type": "file", "path": path, "mode": "0644", + "content": write(r, addresses), + }) + } + return out, nil +} + +// spokenFacts lists them, so a refusal says what would have worked. +func spokenFacts() string { + names := make([]string, 0, len(facts)) + for name := range facts { + names = append(names, name) + } + sort.Strings(names) + return strings.Join(names, ", ") +} + +// nodeNames is every machine's name and address, as a hosts file. +// +// **A machine with no address is left out.** The mesh has a record for it — somebody added it — +// and does not yet know where it is, which is the ordinary state between adding a machine and it +// joining. Writing the name anyway would give a name that resolves to nothing, and a connection to +// that hangs; leaving it out fails at once and says the name is unknown. +func nodeNames(r Resolution, addresses map[string]string) string { + var b strings.Builder + b.WriteString("# Generated by the mesh. Do not edit — this file is replaced whenever a machine\n") + b.WriteString("# joins or leaves, and an edit would survive until then and vanish.\n\n") + // The floor every Linux expects, and which removing would break things that have nothing to do + // with the mesh. + b.WriteString("127.0.0.1\tlocalhost\n") + b.WriteString("::1\t\tlocalhost ip6-localhost ip6-loopback\n") + if r.Node != "" { + fmt.Fprintf(&b, "127.0.1.1\t%s\n", r.Node) + } + b.WriteString("\n") + for _, name := range sortedNames(addresses) { + at := addresses[name] + // Its mesh name resolves to its address on the private network rather than to loopback, + // so a service binding the name it was given stays reachable from everywhere else. + fmt.Fprintf(&b, "%s\t%s.internal\t%s", at, name, name) + if name == r.Node { + b.WriteString("\t# this machine") + } + b.WriteString("\n") + } + return b.String() +} + +// nodeZones is every machine as a wildcard, in the form a resolver reads. +// +// `*.homer.internal` is homer, which is the whole rule: if homer is at an address, so is anything +// homer serves. A module wanting this runs the resolver; the mesh only says what is true. +func nodeZones(_ Resolution, addresses map[string]string) string { + var b strings.Builder + b.WriteString("# Generated by the mesh. Do not edit — this file is replaced whenever a machine\n") + b.WriteString("# joins or leaves, and an edit would survive until then and vanish.\n\n") + for _, name := range sortedNames(addresses) { + fmt.Fprintf(&b, "address=/%s.internal/%s\n", name, addresses[name]) + } + return b.String() +} + +func sortedNames(addresses map[string]string) []string { + out := make([]string, 0, len(addresses)) + for name, at := range addresses { + // See nodeNames: a machine the mesh cannot place is left out rather than named at nothing. + if at == "" { + continue + } + out = append(out, name) + } + sort.Strings(out) + return out +} diff --git a/internal/catalogue/facts_test.go b/internal/catalogue/facts_test.go new file mode 100644 index 0000000..b543768 --- /dev/null +++ b/internal/catalogue/facts_test.go @@ -0,0 +1,97 @@ +package catalogue + +import ( + "strings" + "testing" +) + +var threeMachines = map[string]string{"homer": "10.42.0.1", "marge": "10.42.0.2", "bart": ""} + +// **`*.homer.internal` is homer. That is the whole rule.** +func TestEveryMachineIsAWildcardUnderItsOwnName(t *testing.T) { + out := nodeZones(Resolution{Node: "homer"}, threeMachines) + for _, want := range []string{ + "address=/homer.internal/10.42.0.1", + "address=/marge.internal/10.42.0.2", + } { + if !strings.Contains(out, want) { + t.Fatalf("missing %q:\n%s", want, out) + } + } +} + +// A machine the mesh has a record for and cannot place is left out of both. +// +// **Not an oversight — the alternative is worse.** A name written with no address resolves to +// nothing, and a connection to that hangs. Leaving it out fails at once and says the name is +// unknown, which is a thing somebody can act on. +func TestAMachineWithNoAddressIsNotNamed(t *testing.T) { + for _, out := range []string{ + nodeNames(Resolution{Node: "homer"}, threeMachines), + nodeZones(Resolution{Node: "homer"}, threeMachines), + } { + if strings.Contains(out, "bart") { + t.Fatalf("a machine with no address was named, so its name resolves to nothing:\n%s", out) + } + } +} + +// A machine's own mesh name points at its address on the private network, not at loopback — or a +// service binding the name it was given is unreachable from everywhere else. +func TestAMachinesOwnNameIsItsMeshAddress(t *testing.T) { + out := nodeNames(Resolution{Node: "homer"}, threeMachines) + var line string + for _, l := range strings.Split(out, "\n") { + if strings.Contains(l, "homer.internal") { + line = l + } + } + if !strings.HasPrefix(line, "10.42.0.1") { + t.Fatalf("a machine's own mesh name is not its mesh address: %q", line) + } + // And the loopback floor is still there, or things with nothing to do with the mesh break. + if !strings.Contains(out, "127.0.0.1\tlocalhost") { + t.Fatalf("the loopback floor was removed:\n%s", out) + } +} + +// A module says where it wants a fact, and is given a file. +func TestAModuleIsGivenTheFactsItAskedFor(t *testing.T) { + m := Manifest{Module: "dnsmasq", Facts: map[string]string{FactNodeZones: "/etc/mesh/zones.conf"}} + given, err := FactsInto(m, Resolution{Node: "homer"}, threeMachines) + if err != nil { + t.Fatal(err) + } + if len(given) != 1 { + t.Fatalf("expected one file, got %d", len(given)) + } + if given[0]["path"] != "/etc/mesh/zones.conf" || given[0]["type"] != "file" { + t.Fatalf("not written where it was asked for: %v", given[0]) + } + if !strings.Contains(given[0]["content"].(string), "homer.internal") { + t.Fatalf("the file does not hold the fact: %v", given[0]["content"]) + } +} + +// **Asking for a fact the mesh does not have is refused here, not on a machine.** A daemon that +// starts, reads a file nobody wrote, and answers no queries is a much worse way to find out. +func TestAskingForAFactTheMeshDoesNotHaveIsRefused(t *testing.T) { + m := Manifest{Module: "dnsmasq", Facts: map[string]string{"the-weather": "/etc/weather"}} + _, err := FactsInto(m, Resolution{}, nil) + if err == nil { + t.Fatal("a module asked for something nobody computes and was given nothing, silently") + } + for _, known := range []string{FactNodeNames, FactNodeZones} { + if !strings.Contains(err.Error(), known) { + t.Fatalf("the refusal does not say what would have worked: %v", err) + } + } +} + +// And a relative path is refused, or a module decides where the mesh writes on a machine. +func TestAFactMustBeAskedForAtAnAbsolutePath(t *testing.T) { + m := Manifest{Module: "dnsmasq", Facts: map[string]string{FactNodeNames: "etc/hosts"}} + if _, err := FactsInto(m, Resolution{}, nil); err == nil { + t.Fatal("a relative path was accepted") + } +} diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index 7f6eb9c..03ba034 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -315,6 +315,25 @@ type Manifest struct { // that could only see its own ports would write a rule set that closed everything else. Filtering *Filtering `json:"filtering,omitempty"` + // Facts are things only the mesh knows, written where this module asks for them. + // + // **The graph is the control plane's; how a machine uses it is the module's.** The mesh knows + // which machines exist, what they are called and where they are. Making a name resolve, or a + // peer reachable, is somebody's software — dnsmasq, a resolver, a VPN — and the mesh has no + // business shipping one, choosing which, or knowing its configuration language. + // + // So a module says *put the node names here* and owns everything after that. The same shape as + // `filtering`, generalised: a fact, and a path. + // + // It replaces three modules that existed only because computed output needed somewhere to + // live — they ran no software, could not be swapped for anything, and appeared in the graph as + // modules while being a data channel wearing a costume. + // + // Keyed by fact name; the names are a closed list, because a module asking for one the mesh + // does not compute is asking for something nobody will write, and finding that out on a machine + // is worse than being told here. + Facts map[string]string `json:"facts,omitempty"` + // Certificate is where this module wants a certificate for its machine's name inside the // mesh, and where the key that goes with it can be found. // diff --git a/internal/catalogue/provided_test.go b/internal/catalogue/provided_test.go index b6ea4be..12d9a09 100644 --- a/internal/catalogue/provided_test.go +++ b/internal/catalogue/provided_test.go @@ -19,7 +19,7 @@ func provided(t *testing.T) map[string]catalogue.Manifest { t.Helper() out := map[string]catalogue.Manifest{} for _, raw := range []map[string]any{ - overlay.Manifest(), overlay.NamesManifest(), overlay.DomainManifest(), + overlay.Manifest(), overlay.DomainManifest(), } { b, err := json.Marshal(raw) if err != nil { @@ -44,22 +44,51 @@ func TestTheShippedNetworkingModulesResolveOnTheirOwn(t *testing.T) { for _, m := range got.Modules { have = append(have, m.Module) } - for _, want := range []string{overlay.Domain, overlay.Name, overlay.Names} { + for _, want := range []string{overlay.Domain, overlay.Name} { if !strings.Contains(strings.Join(have, " "), want) { t.Fatalf("%s did not bring in %s: %v", overlay.Domain, want, have) } } + + // **The names come WITH the network now, not from a third module.** Being on the private + // network is what gives a machine a name, so the provider asks for the node-names fact and + // there is nothing else to bring in. A module that ran nothing used to be here. + for _, m := range got.Modules { + if m.Module == overlay.Name && m.Facts["node-names"] == "" { + t.Fatalf("the network's provider does not ask for the names: %+v", m.Facts) + } + } } -func TestTheShippedWireGuardModuleClaimsBeingTheNetwork(t *testing.T) { - // Without this, a person who chose another VPN gets WireGuard as well, dragged in by the - // names, and is not told. The claim is the only thing that catches it. +func TestAnotherVPNSatisfiesNetworkingWithoutDraggingWireGuardIn(t *testing.T) { + // **This inverted, and the inversion is the improvement.** The names used to be a module that + // required the mesh's own addressing, which only WireGuard provided — so choosing another VPN + // dragged WireGuard in anyway, and the node-scoped claim existed to at least make that + // collision loud. With the names a fact rather than a provision, a person who chose tailscale + // gets tailscale, and there is nothing left to collide. shipped := provided(t) - _, err := catalogue.Resolve( + got, err := catalogue.Resolve( withTailscale(shipped), []string{overlay.Domain, "tailscale"}, catalogue.Node{Name: "workstation", Site: "house"}, catalogue.World{}) + if err != nil { + t.Fatalf("choosing another VPN was refused: %v", err) + } + for _, m := range got.Modules { + if m.Module == overlay.Name { + t.Fatalf("the other VPN was chosen and WireGuard came anyway: %v", got.Modules) + } + } +} +func TestTwoVPNsAssignedTogetherStillCollide(t *testing.T) { + // The claim still guards the case it was always for: both assigned EXPLICITLY, which is a + // machine with two private networks and a coin toss about which one a peer reaches it on. + shipped := provided(t) + _, err := catalogue.Resolve( + withTailscale(shipped), + []string{overlay.Name, "tailscale"}, + catalogue.Node{Name: "workstation", Site: "house"}, catalogue.World{}) if err == nil { t.Fatal("a machine was given two private networks and nobody was told") } @@ -68,20 +97,9 @@ func TestTheShippedWireGuardModuleClaimsBeingTheNetwork(t *testing.T) { } } -func TestTheShippedNamesModuleNeedsTheMeshsOwnAddresses(t *testing.T) { - // Over a VPN whose addresses the mesh does not hand out, it has no names to write. Refusing - // is what stops a machine getting a hosts file that means nothing on it. - shipped := provided(t) - delete(shipped, overlay.Name) - _, err := catalogue.Resolve(shipped, []string{overlay.Names}, catalogue.Node{Name: "workstation", Site: "house"}, catalogue.World{}) - - if err == nil { - t.Fatal("the mesh's names resolved with nothing handing out the mesh's addresses") - } - if !strings.Contains(err.Error(), overlay.Addressing) { - t.Fatalf("the refusal does not name what is missing: %v", err) - } -} +// The names-need-addressing test went with the names module: names are a fact now, and a machine +// the mesh cannot place is simply left out of the file (facts_test.go) — which is the same +// protection, enforced where the file is written rather than by a provision refusing. func withTailscale(shelf map[string]catalogue.Manifest) map[string]catalogue.Manifest { out := map[string]catalogue.Manifest{} diff --git a/internal/overlay/generator.go b/internal/overlay/generator.go index 5e50899..d065f73 100644 --- a/internal/overlay/generator.go +++ b/internal/overlay/generator.go @@ -138,30 +138,6 @@ func (g *Generator) Resources(node string) ([]map[string]any, bool, error) { // private network the peers would be written by something else and this would be unchanged. type NameGenerator struct{ nodes []Node } -// NamesFor builds the name generator over the machines on the private network. -func NamesFor(nodes []Node) *NameGenerator { return &NameGenerator{nodes: nodes} } - -// Resources is the one file. -func (g *NameGenerator) Resources(node string) ([]map[string]any, bool, error) { - var found bool - for _, n := range g.nodes { - if n.Name == node { - found = true - } - } - if !found { - return nil, false, nil - } - hosts, err := Hosts(g.nodes, node) - if err != nil { - return nil, false, err - } - return []map[string]any{{ - "id": "mesh-names", "type": "file", "path": HostsPath, - "mode": "0644", "content": hosts, - }}, true, nil -} - // Nodes are the machines this generator was built over, so a caller can say who is on the network. func (g *Generator) Nodes() []Node { return g.nodes } @@ -179,55 +155,25 @@ func Manifest() map[string]any { "version": "1", "computed": Name, "provides": []string{Requirement, Addressing}, - "claims": []map[string]any{{"name": TheNetwork, "scope": "node"}}, - } -} - -// NamesManifest is the module that gives machines names on the private network. -// -// It requires the network rather than providing it, which is the whole reason it is separate: a -// name resolves to an address on the private wire, so having names without being on it would -// point every machine at somewhere it cannot reach. -func NamesManifest() map[string]any { - return map[string]any{ - "module": Names, - "version": "1", - "computed": Names, - "provides": []string{Resolution}, - "requires": []string{Addressing}, - } -} - -// ResolverManifest is what a resolver on this machine must know: every name under every machine. -// -// **It writes the data and runs no daemon.** A resolver is third-party software, and third-party -// software runs *on* the mesh rather than being *of* it -// ([ADR 0001](novox/hq)) — the mesh has no business shipping one, choosing which one, or knowing -// its configuration language. What only the mesh can know is which machines exist and where they -// are, so that is what it computes. -// -// So a module that runs a resolver requires what this provides, and reads one file. Swapping the -// daemon changes that module and nothing here. -// -// **Separate from names rather than part of them**, because a machine with no container runtime -// can still have a hosts file. Folding them together would take exact names away from a machine -// that cannot run a daemon, to give it a wildcard it cannot use either. -func ResolverManifest() map[string]any { - return map[string]any{ - "module": Resolver, - "version": "1", - "computed": Resolver, - "requires": []string{Resolution}, - "provides": []string{ResolverData}, + // Being on the private network is what gives a machine a name, so the module that puts it + // there is what writes them. Asked for rather than generated by a module of its own: the + // mesh knows which machines exist and where; writing that into a hosts file is not a thing + // that needs a module to run nowhere. + "facts": map[string]string{"node-names": "/etc/hosts"}, + "claims": []map[string]any{{"name": TheNetwork, "scope": "node"}}, } } // DomainManifest is the module that means "get the network working". func DomainManifest() map[string]any { return map[string]any{ - "module": Domain, - "version": "1", - "requires": []string{Requirement, Resolution}, + "module": Domain, + "version": "1", + // **Only the network now.** It used to require name-resolution as well, answered by a + // module that wrote a hosts file and ran nothing. Names are not a provision — they are a + // fact the mesh computes, and whatever puts a machine on the private network writes them, + // because a mesh name IS an address on that network. + "requires": []string{Requirement}, } } diff --git a/internal/overlay/names_generator_test.go b/internal/overlay/names_generator_test.go deleted file mode 100644 index 205cf48..0000000 --- a/internal/overlay/names_generator_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package overlay - -import ( - "strings" - "testing" -) - -// Names are their own module. -// -// They used to arrive inside the WireGuard declaration, on the argument that a machine with peers -// and no names is half on the network. True, and the wrong place to fix it: names would be -// identical over a different private network, so bundling them made one module out of two things. - -func TestAMachineNotOnTheNetworkGetsNoNames(t *testing.T) { - // Names resolve to addresses on the private wire. Giving them to a machine that is not on it - // would point every lookup somewhere it cannot reach — worse than having no names at all. - g := NamesFor([]Node{at("anchor", "dc", "10.42.0.1", "198.51.100.10:51820", true)}) - _, part, err := g.Resources("laptop") - if err != nil { - t.Fatal(err) - } - if part { - t.Fatal("a machine that is not on the private network was given the mesh's names") - } -} - -func TestTheNamesAreOneFileAndSayWhoIsAsking(t *testing.T) { - g := NamesFor([]Node{ - at("anchor", "dc", "10.42.0.1", "198.51.100.10:51820", true), - at("workstation", "house", "10.42.0.2", "", false), - }) - out, part, err := g.Resources("workstation") - if err != nil || !part { - t.Fatalf("part=%v err=%v", part, err) - } - if len(out) != 1 || out[0]["path"] != HostsPath { - t.Fatalf("got %v", out) - } - content := out[0]["content"].(string) - if !strings.Contains(content, "anchor.internal") { - t.Fatalf("another machine on the network has no name here:\n%s", content) - } - if !strings.Contains(content, "this machine") { - t.Fatalf("the file does not say which machine it is on:\n%s", content) - } -} - -func TestTheWireGuardDeclarationNoLongerCarriesTheNames(t *testing.T) { - // The split, asserted. Two modules, so a machine can have the peers from one and the names - // from another — which is what makes a second VPN possible at all. - raw, err := Declaration( - at("workstation", "house", "10.42.0.2", "", false), - []Peer{{Name: "anchor", Key: "PUB", Allowed: "10.42.0.0/16", - Endpoint: "198.51.100.10:51820"}}, "") - if err != nil { - t.Fatal(err) - } - if strings.Contains(string(raw), HostsPath) { - t.Fatalf("the WireGuard declaration still writes %s", HostsPath) - } -} diff --git a/internal/overlay/resolver.go b/internal/overlay/resolver.go deleted file mode 100644 index fbdef37..0000000 --- a/internal/overlay/resolver.go +++ /dev/null @@ -1,82 +0,0 @@ -package overlay - -import ( - "fmt" - "sort" - "strings" -) - -// A resolver answers every name under a node, not just the node. -// -// **Services are named under the machine they run on** — `postgres.novox.internal`, -// `plex.ace.internal`. The first label is the service and the rest is the node, so what has to -// resolve is *anything* under a node's name, going to that node's address. A reverse proxy there -// routes by the name it was asked for, which is a separate concern and stays separate. -// -// **This is what a hosts file cannot do.** It answers exact names; a wildcard would mean writing -// down every service name in advance, which is the enumeration the arrangement exists to avoid. -// novox/hq 08-connectivity named exactly this as the trigger for needing a resolver rather than a -// file, and it is the first thing to meet it. -// -// What is generated is the data, not the daemon's configuration language. One line per node, -// in the form dnsmasq reads because that is what the module runs — and if a mesh runs something -// else, this is the shape it translates from rather than a second thing to compute. - -// ResolverPath is where the mesh writes what a node must answer. -const ResolverPath = "/etc/mesh-resolver/nodes.conf" - -// Wildcards is one line per node: everything under its name, and the name itself. -// -// A machine with no address is left out. A wildcard pointing at nothing is worse than no wildcard: -// every name under it would resolve and then hang, where an unresolvable name fails at once and -// says which name it was. -func Wildcards(nodes []Node) string { - var b strings.Builder - b.WriteString("# Generated by the mesh. Do not edit — it is replaced whenever a machine\n") - b.WriteString("# joins or leaves, and an edit would survive until then and vanish.\n") - b.WriteString("#\n") - b.WriteString("# Each line answers the node's own name AND everything under it, so a service\n") - b.WriteString("# is reached at .." + Suffix() + " without the mesh being told\n") - b.WriteString("# the service exists. What routes it there once it arrives is the proxy's.\n\n") - - named := make([]Node, 0, len(nodes)) - for _, n := range nodes { - if strings.TrimSpace(n.Address) == "" { - continue - } - named = append(named, n) - } - sort.Slice(named, func(i, j int) bool { return named[i].Name < named[j].Name }) - - for _, n := range named { - fmt.Fprintf(&b, "address=/%s/%s\n", InternalName(n.Name), n.Address) - } - if len(named) == 0 { - b.WriteString("# No machine in this mesh has an address on the private network.\n") - } - return b.String() -} - -// ResolverGenerator answers what one node's resolver must know. -type ResolverGenerator struct{ nodes []Node } - -// ResolverFor builds it over the machines on the private network. -func ResolverFor(nodes []Node) *ResolverGenerator { return &ResolverGenerator{nodes: nodes} } - -// Resources is the one file. The daemon that reads it is the module's, not the mesh's. -func (g *ResolverGenerator) Resources(node string) ([]map[string]any, bool, error) { - var here bool - for _, n := range g.nodes { - if n.Name == node { - here = true - } - } - if !here { - // Assigned and not yet on the network. Ordinary and brief. - return nil, false, nil - } - return []map[string]any{{ - "id": "nodes", "type": "file", "path": ResolverPath, - "content": Wildcards(g.nodes), "mode": "0644", - }}, true, nil -} diff --git a/internal/overlay/resolver_test.go b/internal/overlay/resolver_test.go deleted file mode 100644 index c59f44a..0000000 --- a/internal/overlay/resolver_test.go +++ /dev/null @@ -1,99 +0,0 @@ -package overlay - -import ( - "strings" - "testing" -) - -// Services are named under the machine they run on, so what must resolve is anything under a -// node's name — not the node's name alone. -// -// This is what a hosts file cannot do: it answers exact names, and a wildcard there would mean -// writing down every service in advance, which is the enumeration the arrangement exists to -// avoid. -func TestEverythingUnderANodesNameGoesToThatNode(t *testing.T) { - written := Wildcards([]Node{ - {Name: "novox", Address: "10.42.0.1"}, - {Name: "ace", Address: "10.42.0.2"}, - }) - for _, want := range []string{ - "address=/novox.internal/10.42.0.1", - "address=/ace.internal/10.42.0.2", - } { - if !strings.Contains(written, want) { - t.Fatalf("missing %q:\n%s", want, written) - } - } - - // Sorted, because this file is compared against its last version on every apply and a set - // that reorders itself would rewrite it — and restart what reads it — for no change. - if strings.Index(written, "/ace.") > strings.Index(written, "/novox.") { - t.Fatalf("the machines are not in a stable order:\n%s", written) - } -} - -// A machine with no address is left out. -// -// A wildcard pointing at nothing is worse than no wildcard: every name under it resolves and then -// hangs, where an unresolvable name fails at once and says which name it was. -func TestAMachineWithNoAddressGetsNoWildcard(t *testing.T) { - written := Wildcards([]Node{ - {Name: "novox", Address: "10.42.0.1"}, - {Name: "unplaced"}, - }) - if strings.Contains(written, "unplaced") { - t.Fatalf("a machine with no address was given a wildcard:\n%s", written) - } - if !strings.Contains(written, "novox.internal") { - t.Fatalf("the machine that does have one lost it:\n%s", written) - } -} - -// A mesh where nobody is on the private network says so rather than producing an empty file that -// reads as "nothing was generated". -func TestAMeshWithNoAddressesSaysSo(t *testing.T) { - written := Wildcards(nil) - if !strings.Contains(written, "No machine in this mesh has an address") { - t.Fatalf("an empty answer is indistinguishable from a failure to answer:\n%s", written) - } -} - -// The suffix a mesh chose is used, not a hardcoded one. -func TestTheMeshsOwnSuffixIsUsed(t *testing.T) { - t.Setenv(SuffixVar, "mesh.example") - written := Wildcards([]Node{{Name: "novox", Address: "10.42.0.1"}}) - if !strings.Contains(written, "address=/novox.mesh.example/10.42.0.1") { - t.Fatalf("the mesh's own suffix was not used:\n%s", written) - } -} - -// A machine not on the network is given no resolver data, which is an answer rather than an -// error: a node assigned the module before it is placed is in exactly that state. -func TestAMachineNotOnTheNetworkGetsNoResolverData(t *testing.T) { - _, part, err := ResolverFor([]Node{{Name: "novox", Address: "10.42.0.1"}}).Resources("stranger") - if err != nil { - t.Fatal(err) - } - if part { - t.Fatal("a machine not on the network was given the mesh's resolver data") - } -} - -// And a machine on it gets the whole set, including itself: a service on this machine reached by -// its own mesh name must arrive the same way it would from anywhere else. -func TestAMachineGetsTheWholeSetIncludingItself(t *testing.T) { - got, part, err := ResolverFor([]Node{ - {Name: "novox", Address: "10.42.0.1"}, - {Name: "ace", Address: "10.42.0.2"}, - }).Resources("novox") - if err != nil { - t.Fatal(err) - } - if !part || len(got) != 1 { - t.Fatalf("expected one file for a machine on the network, got %d (part=%v)", len(got), part) - } - content, _ := got[0]["content"].(string) - if !strings.Contains(content, "novox.internal") || !strings.Contains(content, "ace.internal") { - t.Fatalf("the machine was not given the whole mesh:\n%s", content) - } -} -- 2.54.0 From b8cacbaf4d4e0658f234a023b105355144ff333d Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 15 Sep 2026 21:33:20 +0200 Subject: [PATCH 10/17] What remains of names.go is the naming MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hosts-file writing moved to the node-names fact; what stays is the suffix (configurable, MESH_INTERNAL_SUFFIX, defaulting to .internal which IANA reserved for exactly this), a node's internal name, and the rule for what a node may be called. Several things compose an internal name, and one of them writing the suffix differently would be a name nothing answers to. First attempt at this rewrote the file from memory and silently dropped the configurable suffix. Restored from the original instead — deleting most of a file is git surgery, not paraphrase. Claude-Session: https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- internal/overlay/generator.go | 28 +++------ internal/overlay/names.go | 56 ++---------------- internal/overlay/names_test.go | 105 --------------------------------- 3 files changed, 12 insertions(+), 177 deletions(-) delete mode 100644 internal/overlay/names_test.go diff --git a/internal/overlay/generator.go b/internal/overlay/generator.go index d065f73..ac6f88b 100644 --- a/internal/overlay/generator.go +++ b/internal/overlay/generator.go @@ -29,19 +29,14 @@ import ( // how a mesh ends up unable to have a second one. const Requirement = "private-network" -// Name is the module that answers it with WireGuard, and Names is the one that gives the machines -// names. Two modules rather than one, because they are two different things: names would be the -// same over any private network, and they are only bundled here by an accident of both being -// computed. -const ( - Name = "mesh-wireguard" - Names = "mesh-names" -) - -// Resolution is what a module asks for when it needs to reach other machines by name. Separate -// from Requirement because they are separate jobs: one is whether packets arrive, the other is -// whether a name means anything. A machine can want the first without the second. -const Resolution = "name-resolution" +// Name is the module that answers it with WireGuard. +// +// **The names went with it.** A mesh-names module used to sit beside this — it wrote /etc/hosts +// and ran nothing, which is not a module. Being on the private network is what gives a machine a +// name, so this module asks for the `node-names` fact and the mesh writes the file. The +// name-resolution provision went the same way: names are facts the mesh computes, not something a +// module that runs nowhere can provide. +const Name = "mesh-wireguard" // Addressing is the mesh handing out addresses on the private network itself. // @@ -131,13 +126,6 @@ func (g *Generator) Resources(node string) ([]map[string]any, bool, error) { return parsed.Resources, true, nil } -// NameGenerator answers what one node's hosts file is. -// -// Separate from the interface and the peers because it is a separate concern. A machine's names -// come from the mesh knowing every machine, not from how the packets travel — over a different -// private network the peers would be written by something else and this would be unchanged. -type NameGenerator struct{ nodes []Node } - // Nodes are the machines this generator was built over, so a caller can say who is on the network. func (g *Generator) Nodes() []Node { return g.nodes } diff --git a/internal/overlay/names.go b/internal/overlay/names.go index a495f56..a3b0b03 100644 --- a/internal/overlay/names.go +++ b/internal/overlay/names.go @@ -1,10 +1,8 @@ package overlay import ( - "fmt" "os" "regexp" - "sort" "strings" ) @@ -49,53 +47,7 @@ var nodeName = regexp.MustCompile(`^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$`) // InternalName is a node's name inside the mesh. func InternalName(node string) string { return node + "." + Suffix() } -// Hosts writes the name file for one node. -// -// Every node in the mesh, including this one. Including itself because a machine referring to -// itself by its mesh name should get its overlay address rather than a loopback — otherwise a -// service that binds to the name it was given ends up unreachable from everywhere else. -// -// The machine's own loopback lines come first and are not the mesh's to have an opinion about, -// but they have to be here: this file is generated whole, so anything left out is removed. -func Hosts(nodes []Node, self string) (string, error) { - var b strings.Builder - b.WriteString("# Generated by the mesh. Do not edit — this file is replaced whenever a node\n") - b.WriteString("# joins or leaves, and an edit would survive until then and vanish.\n\n") - - // The floor every Linux expects, and which removing would break things that have nothing to - // do with the mesh. - b.WriteString("127.0.0.1\tlocalhost\n") - b.WriteString("::1\t\tlocalhost ip6-localhost ip6-loopback\n") - if self != "" { - fmt.Fprintf(&b, "127.0.1.1\t%s\n", self) - } - - named := make([]Node, 0, len(nodes)) - for _, n := range nodes { - if n.Address == "" { - // A node with no address on the network has no name here. Writing one that resolves - // to nothing is worse than not writing it: a connection to an address that does not - // answer hangs, where a name that does not resolve fails at once and says so. - continue - } - if !nodeName.MatchString(n.Name) { - return "", fmt.Errorf( - "%q cannot be a mesh name: it becomes a hostname, so it is lower-case letters, "+ - "digits and dashes", n.Name) - } - named = append(named, n) - } - sort.Slice(named, func(i, j int) bool { return named[i].Name < named[j].Name }) - - if len(named) > 0 { - fmt.Fprintf(&b, "\n# the mesh, %d node(s)\n", len(named)) - } - for _, n := range named { - line := fmt.Sprintf("%s\t%s\t%s", n.Address, InternalName(n.Name), n.Name) - if n.Name == self { - line += "\t# this machine" - } - b.WriteString(line + "\n") - } - return b.String(), nil -} +// **What remains of a larger file.** The rest wrote /etc/hosts — that is the `node-names` fact now +// (catalogue.FactsInto), computed where the graph lives instead of by a module that ran nothing. +// The naming stays here, because several things compose a node's internal name and one of them +// writing the suffix differently would be a name nothing answers to. diff --git a/internal/overlay/names_test.go b/internal/overlay/names_test.go deleted file mode 100644 index 00103f7..0000000 --- a/internal/overlay/names_test.go +++ /dev/null @@ -1,105 +0,0 @@ -package overlay - -import ( - "strings" - "testing" -) - -func hostsFor(t *testing.T, self string, nodes ...Node) string { - t.Helper() - out, err := Hosts(nodes, self) - if err != nil { - t.Fatal(err) - } - return out -} - -func TestEveryNodeWithAPlaceGetsAName(t *testing.T) { - got := hostsFor(t, "laptop", - Node{Name: "anchor", Address: "10.42.0.1"}, - Node{Name: "laptop", Address: "10.42.0.2"}) - - for _, want := range []string{"10.42.0.1\tanchor.internal\tanchor", "10.42.0.2\tlaptop.internal\tlaptop"} { - if !strings.Contains(got, want) { - t.Errorf("no entry for %q in:\n%s", want, got) - } - } -} - -func TestANodeSeesItselfAtItsOverlayAddress(t *testing.T) { - // Not at a loopback. A service that binds to the name the machine was given would otherwise - // listen somewhere nothing else can reach, and the failure appears on every other node rather - // than this one. - got := hostsFor(t, "laptop", Node{Name: "laptop", Address: "10.42.0.2"}) - if !strings.Contains(got, "10.42.0.2\tlaptop.internal") { - t.Error("a node does not resolve its own mesh name to its overlay address") - } -} - -func TestTheMachinesOwnLoopbackSurvives(t *testing.T) { - // This file is generated whole, so anything left out is removed. Dropping localhost would - // break things that have nothing to do with the mesh, on a machine the mesh was asked to - // improve. - got := hostsFor(t, "laptop", Node{Name: "laptop", Address: "10.42.0.2"}) - if !strings.Contains(got, "127.0.0.1\tlocalhost") { - t.Error("localhost is missing; this file replaces the machine's own") - } - if !strings.Contains(got, "::1") { - t.Error("the IPv6 loopback is missing") - } -} - -func TestANodeWithNoAddressGetsNoName(t *testing.T) { - // A name resolving to nothing is worse than no name: a connection to an address that does not - // answer hangs, where a name that does not resolve fails at once and says which name it was. - got := hostsFor(t, "laptop", - Node{Name: "laptop", Address: "10.42.0.2"}, - Node{Name: "newcomer", Address: ""}) - if strings.Contains(got, "newcomer") { - t.Error("a node with no address on the network was given a name") - } -} - -func TestANameThatCannotBeAHostnameIsRefused(t *testing.T) { - // Refused here, where a person is looking, rather than written into a file that every - // machine then reads and disagrees about. - for _, bad := range []string{"Anchor", "my node", "under_score", "-leading", "trailing-"} { - if _, err := Hosts([]Node{{Name: bad, Address: "10.42.0.1"}}, ""); err == nil { - t.Errorf("%q was accepted as a mesh name", bad) - } - } -} - -func TestTheOrderIsStable(t *testing.T) { - // The file is rewritten whenever anything changes, and a file whose lines move for no reason - // makes every reconcile look like a change — which means a service that reflects it restarts - // for ever. - a := hostsFor(t, "", Node{Name: "b", Address: "10.42.0.2"}, Node{Name: "a", Address: "10.42.0.1"}) - b := hostsFor(t, "", Node{Name: "a", Address: "10.42.0.1"}, Node{Name: "b", Address: "10.42.0.2"}) - if a != b { - t.Error("the same mesh produced two different files depending on the order it was read in") - } -} - -func TestTheSuffixIsReservedForThis(t *testing.T) { - // `.internal` was reserved by IANA in 2024 for exactly this. A name under it can never - // collide with a public one, so an internal name that leaks into a public resolver fails - // rather than reaching a stranger's machine. - if InternalName("anchor") != "anchor.internal" { - t.Errorf("internal names end in %q", Suffix()) - } -} - -func TestTheSuffixCanBeChosen(t *testing.T) { - t.Setenv(SuffixVar, ".mesh") - if InternalName("anchor") != "anchor.mesh" { - t.Errorf("got %q", InternalName("anchor")) - } -} - -func TestTheFileSaysItIsGenerated(t *testing.T) { - got := hostsFor(t, "laptop", Node{Name: "laptop", Address: "10.42.0.2"}) - if !strings.Contains(got, "Do not edit") { - t.Error("a generated file does not say so") - } -} -- 2.54.0 From 9070d2502c988677ca1174ac69b750c00daf109e Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 15 Sep 2026 22:26:35 +0200 Subject: [PATCH 11/17] The builder narrates every step, and every command it runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A build was silent from clone to publish, so a build in progress, one that failed quietly, and a request that never arrived all looked identical — which cost a long diagnosis against a running mesh chasing "the handler never fired". Now: the handler announces a request the instant it lands. Build logs each phase — clone, commit, manifest, bases, each artifact starting and finishing with what it produced, resolve, done — through a Log callback that is nil-safe, so the tests that pass none still build. And the Command runner echoes every command before it runs, with where and how long it took, because on a hang the last line is exactly the command it is stuck inside: "git clone waiting on a network that will not answer" rather than "the builder did nothing". The unreadable-request path prints to stdout now too, not stderr, so it shows in docker logs without splitting streams — the split is what hid it. Claude-Session: https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- cmd/mesh-builder/main.go | 13 ++++- cmd/mesh-builder/once.go | 3 +- internal/builder/builder.go | 96 +++++++++++++++++++++++++++++++- internal/builder/builder_test.go | 20 +++---- internal/builder/bundle_test.go | 8 +-- 5 files changed, 120 insertions(+), 20 deletions(-) diff --git a/cmd/mesh-builder/main.go b/cmd/mesh-builder/main.go index 2289726..bb68faa 100644 --- a/cmd/mesh-builder/main.go +++ b/cmd/mesh-builder/main.go @@ -160,12 +160,18 @@ func run() error { func answer(ctx context.Context, channel *amqp.Channel, publisher builder.Publisher, on, workspace string, delivery amqp.Delivery) { + // **First thing, and to stdout.** A build request that arrives and produces no visible line + // until it either finishes or fails is indistinguishable from one that never arrived — which + // cost a long diagnosis against a running mesh, chasing "the handler never fired" when the + // truth was only that the handler said nothing until the end. + fmt.Printf("a build request arrived (%d bytes)\n", len(delivery.Body)) + var request link.BuildRequest if err := json.Unmarshal(delivery.Body, &request); err != nil { // Unreadable. Acknowledged and dropped rather than requeued: a message this builder // cannot parse will not become parseable by being delivered again, and requeueing it // would put it in front of every real request for ever. - fmt.Fprintf(os.Stderr, "a request could not be read and was dropped: %v\n", err) + fmt.Printf("a request could not be read and was dropped: %v\n", err) _ = delivery.Ack(false) return } @@ -184,7 +190,10 @@ func answer(ctx context.Context, channel *amqp.Channel, publisher builder.Publis fmt.Println() built, err := builder.Build(ctx, builder.Command, publisher, - request.Repository, request.Path, request.Ref, workspace, request.Held) + request.Repository, request.Path, request.Ref, workspace, request.Held, + func(step, message string) { + fmt.Printf(" [%s] %s\n", step, message) + }) if err != nil { // A failure is a result. A build that fails and says nothing is indistinguishable from a // builder that is not running, and those want completely different responses. diff --git a/cmd/mesh-builder/once.go b/cmd/mesh-builder/once.go index 2be3bdb..953cf47 100644 --- a/cmd/mesh-builder/once.go +++ b/cmd/mesh-builder/once.go @@ -84,7 +84,8 @@ func buildOnce(ctx context.Context, args []string) error { } fmt.Fprintln(os.Stderr) - built, buildErr := builder.Build(ctx, builder.Command, publisher, repository, *path, *ref, where, bases) + built, buildErr := builder.Build(ctx, builder.Command, publisher, repository, *path, *ref, where, bases, + func(step, message string) { fmt.Printf(" [%s] %s\n", step, message) }) if buildErr != nil { return buildErr } diff --git a/internal/builder/builder.go b/internal/builder/builder.go index c8a16ae..d950905 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -14,6 +14,7 @@ import ( "regexp" "sort" "strings" + "time" "github.com/novox/mesh-control/internal/catalogue" ) @@ -67,7 +68,10 @@ type Result struct { // archive failed would otherwise leave half of itself in the store under a digest the mesh never // records — reachable, unreferenced, and indistinguishable from something in use. func Build(ctx context.Context, run Runner, publish Publisher, - repository, path, ref, workspace string, held map[string]string) (Result, error) { + repository, path, ref, workspace string, held map[string]string, log Log) (Result, error) { + + say := logging(log) + say("clone", "%s%s at %s", repository, describePath(path), refOrHead(ref)) // Made rather than required. A builder that fails because the directory it was told to work // in does not exist is a builder that needs a setup step nobody documented. @@ -82,8 +86,10 @@ func Build(ctx context.Context, run Runner, publish Publisher, // that reuses a working tree can succeed because of something a previous build left behind, // and that is a build nobody can reproduce. if _, err := run(ctx, workspace, "git", "clone", "--quiet", repository, tree); err != nil { + say("clone", "FAILED: %v", err) return Result{}, fmt.Errorf("cannot clone %s: %w", repository, err) } + say("clone", "done") if ref != "" { if _, err := run(ctx, tree, "git", "checkout", "--quiet", ref); err != nil { return Result{}, fmt.Errorf("%s has no %s: %w", repository, ref, err) @@ -94,6 +100,7 @@ func Build(ctx context.Context, run Runner, publish Publisher, return Result{}, err } commit = strings.TrimSpace(commit) + say("commit", "%s", short(commit)) // A module is a repository and a path within it (novox/hq ADR 0069). The ordinary case is an // empty path, meaning the repository's root; a repository holding several modules names each @@ -112,8 +119,10 @@ func Build(ctx context.Context, run Runner, publish Publisher, } manifest, err := catalogue.ParseManifest(raw) if err != nil { + say("manifest", "INVALID: %v", err) return Result{}, err } + say("manifest", "%s v%s — %d artifact(s)", manifest.Module, manifest.Version, artifactCount(manifest)) var built []catalogue.Built if manifest.Build != nil { @@ -122,29 +131,86 @@ func Build(ctx context.Context, run Runner, publish Publisher, // fix it rather than inside a build that stops on its own first line. args, err := standingOn(manifest, held) if err != nil { + say("bases", "UNMET: %v", err) return Result{}, err } + if len(args) > 0 { + say("bases", "%d resolved from what the mesh holds", len(args)/2) + } artifacts := append([]catalogue.Artifact{}, manifest.Build.Artifacts...) // Ordered, so two builds of one commit do the same work in the same sequence and their // logs can be compared. sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name }) for _, a := range artifacts { - made, err := one(ctx, run, publish, manifest.Module, within, commit, a, args, held) + say("artifact", "%s (%s%s) — starting", a.Name, a.Kind, langSuffix(a)) + made, err := one(ctx, run, publish, manifest.Module, within, commit, a, args, held, say) if err != nil { + say("artifact", "%s FAILED: %v", a.Name, err) return Result{}, err } + say("artifact", "%s done — %s", a.Name, describeMade(made)) built = append(built, made) } } resolved, err := manifest.Resolve(built) if err != nil { + say("resolve", "FAILED: %v", err) return Result{}, err } + say("done", "%s at %s — %d artifact(s) pinned", manifest.Module, short(commit), len(built)) return Result{Manifest: resolved, Commit: commit, Built: built, Against: against(within, manifest)}, nil } +// Log is where a build says what it is doing, step by step. Nil is silent — the tests pass none, +// and a build with nowhere to speak must still build. +type Log func(step, message string) + +func logging(log Log) func(step, format string, args ...any) { + if log == nil { + return func(string, string, ...any) {} + } + return func(step, format string, args ...any) { + log(step, fmt.Sprintf(format, args...)) + } +} + +func describePath(path string) string { + if path == "" { + return "" + } + return " at " + path +} + +func refOrHead(ref string) string { + if ref == "" { + return "HEAD" + } + return ref +} + +func artifactCount(m catalogue.Manifest) int { + if m.Build == nil { + return 0 + } + return len(m.Build.Artifacts) +} + +func langSuffix(a catalogue.Artifact) string { + if a.Language != "" { + return ", " + a.Language + } + return "" +} + +func describeMade(made catalogue.Built) string { + if made.Digest != "" { + return made.Kind + " " + short(strings.TrimPrefix(made.Digest, "sha256:")) + } + return made.Kind + " " + made.Reference +} + // inside resolves a module's path within a clone, and refuses one that leaves it. // // **A build reads only its own tree.** A path of `../../etc` would otherwise make a build read — @@ -216,16 +282,18 @@ const ManifestName = "module.json" func one(ctx context.Context, run Runner, publish Publisher, module, tree, commit string, a catalogue.Artifact, args []string, - held map[string]string) (catalogue.Built, error) { + held map[string]string, say func(step, format string, args ...any)) (catalogue.Built, error) { switch a.Kind { case catalogue.ArtifactUpstream: // Mirrored, not built. Pulled by the reference the module names and pushed under a name // of the mesh's own, so what a machine fetches is pinned by a digest this registry // assigned rather than by a tag somebody else can move. + say("mirror", "pulling %s", a.From) if _, err := run(ctx, tree, "docker", "pull", a.From); err != nil { return catalogue.Built{}, fmt.Errorf("%s: cannot fetch %s: %w", module, a.From, err) } + say("mirror", "publishing under the mesh's own name") reference, err := publish.PublishImage(ctx, a.From, module+"/"+a.Name) if err != nil { return catalogue.Built{}, err @@ -244,9 +312,11 @@ func one(ctx context.Context, run Runner, publish Publisher, invocation = append(invocation, "--target", a.Target) } invocation = append(invocation, ".") + say("image", "docker build -f %s", a.From) if _, err := run(ctx, tree, "docker", invocation...); err != nil { return catalogue.Built{}, fmt.Errorf("%s: building %s failed: %w", module, a.Name, err) } + say("image", "built, publishing") reference, err := publish.PublishImage(ctx, local, module+"/"+a.Name) if err != nil { return catalogue.Built{}, err @@ -277,10 +347,12 @@ func one(ctx context.Context, run Runner, publish Publisher, "holds no copy of it. Build %s first", module, a.Name, chain.Language, chain.Base, chain.Artifact, chain.Base) } + say("bundle", "compiling %s in %s's toolchain", a.Language, chain.Base) compiled, err := compile(ctx, run, tree, chain, base, a) if err != nil { return catalogue.Built{}, fmt.Errorf("%s: compiling %s failed: %w", module, a.Name, err) } + say("bundle", "compiled, packing") body, err := pack(compiled) if err != nil { return catalogue.Built{}, fmt.Errorf("%s: packing %s failed: %w", module, a.Name, err) @@ -401,16 +473,31 @@ func short(commit string) string { // Command is a Runner that actually runs things. func Command(ctx context.Context, dir, name string, args ...string) (string, error) { + // **Every command is echoed before it runs**, with where. On a build that hangs, the last line + // is exactly the command it is inside — which is the difference between "the builder did + // nothing" and "git clone is waiting on a network that will not answer". Silent on success is + // what made an empty workspace unreadable. + started := timeNow() + fmt.Printf(" $ (%s) %s %s\n", short(filepath.Base(dir)), name, strings.Join(args, " ")) cmd := exec.CommandContext(ctx, name, args...) cmd.Dir = dir out, err := cmd.CombinedOutput() if err != nil { + fmt.Printf(" ! %s %s failed after %s\n", name, args[0], since(started)) return string(out), fmt.Errorf("%s %s: %w\n%s", name, strings.Join(args, " "), err, strings.TrimSpace(string(out))) } + fmt.Printf(" ✓ %s %s (%s)\n", name, firstArg(args), since(started)) return string(out), nil } +func firstArg(args []string) string { + if len(args) == 0 { + return "" + } + return args[0] +} + var _ io.Writer = (*stringWriter)(nil) // standingOn turns the bases a module named into build arguments for what this mesh holds. @@ -504,3 +591,6 @@ func sourcesFor(entrypoints []string, out string) []string { } return sources } + +func timeNow() time.Time { return time.Now() } +func since(t time.Time) string { return time.Since(t).Round(time.Millisecond).String() } diff --git a/internal/builder/builder_test.go b/internal/builder/builder_test.go index 7f1c2c3..a97a93f 100644 --- a/internal/builder/builder_test.go +++ b/internal/builder/builder_test.go @@ -108,7 +108,7 @@ func TestABuildProducesAManifestThePinsAreIn(t *testing.T) { r, workspace := aRepository(t, withBoth, map[string]string{ "Dockerfile": "FROM scratch", "files/theme.conf": "dark", }) - got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, nil) if err != nil { t.Fatal(err) } @@ -134,7 +134,7 @@ func TestTwoBuildsOfOneCommitProduceOneDigest(t *testing.T) { }) // A year apart, so a packer carrying timestamps cannot accidentally agree. r.stamped = time.Date(2020+i, time.March, 3, 4, 5, 6, 0, time.UTC) - got, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, nil) if err != nil { t.Fatal(err) } @@ -154,7 +154,7 @@ func TestNothingIsPublishedUntilEverythingIsBuilt(t *testing.T) { // unreferenced, and indistinguishable from something in use. r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch"}) // `files` is missing, so packing the archive fails — after the image would have been pushed. - _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil) + _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, nil) if err == nil { t.Fatal("a build with a missing input succeeded") } @@ -166,7 +166,7 @@ func TestNothingIsPublishedUntilEverythingIsBuilt(t *testing.T) { func TestARepositoryWithNoManifestSaysSo(t *testing.T) { workspace := t.TempDir() r := &recorded{contents: map[string]string{"README.md": "nothing to see"}} - _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil) + _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, nil) if err == nil { t.Fatal("a repository with nothing saying what it is was built") } @@ -179,7 +179,7 @@ func TestAModuleThatBuildsNothingStillProducesAManifest(t *testing.T) { // Most of what a person installs is configuration. r, workspace := aRepository(t, `{"module":"shell","version":"1","resources":[ {"id":"rc","type":"file","path":"/etc/zsh/zshrc","content":"setopt"}]}`, nil) - got, err := Build(context.Background(), r.run, r, "https://forge.invalid/shell.git", "", "", workspace, nil) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/shell.git", "", "", workspace, nil, nil) if err != nil { t.Fatal(err) } @@ -209,7 +209,7 @@ func TestTheTreeIsFreshEveryTime(t *testing.T) { if err := os.WriteFile(leftover, []byte("stale"), 0o644); err != nil { t.Fatal(err) } - if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil); err != nil { + if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, nil); err != nil { t.Fatal(err) } if _, err := os.Stat(leftover); err == nil { @@ -222,7 +222,7 @@ func TestABuildThatCannotPushFails(t *testing.T) { "Dockerfile": "FROM scratch", "files/a": "b", }) r.failPush = true - if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil); err == nil { + if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, nil); err == nil { t.Fatal("a build that could publish nothing reported success") } } @@ -236,7 +236,7 @@ func TestAnUpstreamImageIsMirroredRatherThanBuilt(t *testing.T) { "resources":[{"id":"db","type":"container","name":"mesh-postgres","artifact":"store"}]}` r, workspace := aRepository(t, mirrors, nil) - got, err := Build(context.Background(), r.run, r, "https://forge.invalid/postgres.git", "", "", workspace, nil) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/postgres.git", "", "", workspace, nil, nil) if err != nil { t.Fatal(err) } @@ -302,7 +302,7 @@ func TestAModuleIsBuiltFromItsPathWithinTheRepository(t *testing.T) { "modules/other/" + ManifestName: `{"module":"other","version":"1"}`, }} got, err := Build(context.Background(), r.run, r, - "https://forge.invalid/catalogue.git", "modules/shell", "", t.TempDir(), nil) + "https://forge.invalid/catalogue.git", "modules/shell", "", t.TempDir(), nil, nil) if err != nil { t.Fatal(err) } @@ -321,7 +321,7 @@ func TestAPathThatLeavesTheRepositoryIsRefused(t *testing.T) { for _, escaping := range []string{"../../etc", "/etc"} { r := &recorded{contents: map[string]string{ManifestName: withBoth}} _, err := Build(context.Background(), r.run, r, - "https://forge.invalid/x.git", escaping, "", t.TempDir(), nil) + "https://forge.invalid/x.git", escaping, "", t.TempDir(), nil, nil) if err == nil { t.Fatalf("%q was accepted as a module's path", escaping) } diff --git a/internal/builder/bundle_test.go b/internal/builder/bundle_test.go index 0b8115a..ffbce41 100644 --- a/internal/builder/bundle_test.go +++ b/internal/builder/bundle_test.go @@ -54,7 +54,7 @@ func TestABundleIsCompiledAndPackedWithNoDockerfile(t *testing.T) { held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)} got, err := Build(context.Background(), compiling{r}.run, r, - "https://forge.invalid/greeter.git", "", "", workspace, held) + "https://forge.invalid/greeter.git", "", "", workspace, held, nil) if err != nil { t.Fatalf("a module with a language and no Dockerfile did not build: %v", err) } @@ -91,7 +91,7 @@ func TestABundleWhoseToolchainIsNotHeldIsRefusedFirst(t *testing.T) { r, workspace := aRepository(t, aBundle, map[string]string{"index.ts": "console.log(1)"}) _, err := Build(context.Background(), compiling{r}.run, r, - "https://forge.invalid/greeter.git", "", "", workspace, nil) + "https://forge.invalid/greeter.git", "", "", workspace, nil, nil) if err == nil { t.Fatal("a bundle was built with no toolchain to compile it in") } @@ -112,7 +112,7 @@ func TestABundleInAnUnknownLanguageIsRefused(t *testing.T) { _, err := Build(context.Background(), compiling{r}.run, r, "https://forge.invalid/greeter.git", "", "", workspace, - map[string]string{"mesh-tools/build": "registry.invalid/x@sha256:" + strings.Repeat("c", 64)}) + map[string]string{"mesh-tools/build": "registry.invalid/x@sha256:" + strings.Repeat("c", 64)}, nil) if err == nil { t.Fatal("a language nothing can compile was accepted") } @@ -140,7 +140,7 @@ func TestTwoBundlesInOneModuleArePackedSeparately(t *testing.T) { held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)} got, err := Build(context.Background(), compiling{r}.run, r, - "https://forge.invalid/greeter.git", "", "", workspace, held) + "https://forge.invalid/greeter.git", "", "", workspace, held, nil) if err != nil { t.Fatalf("a module with two bundles did not build: %v", err) } -- 2.54.0 From 2fb700d61b76449298c27e7f7d12f063019a7f23 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 15 Sep 2026 23:01:04 +0200 Subject: [PATCH 12/17] Builder diagnostics go to stderr; stdout is the result alone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The logging added a line to stdout, and the genesis path parses the builder's stdout as JSON — so the first log line broke the parse with "invalid character 'c'", the c from "[clone]". A build that had worked stopped working because of a print statement. The installer's runner captures stdout alone (cmd.Output), and the contract was already stdout=result, stderr=everything else. The fix is to honour it: every builder diagnostic — the step log, the per-command echo, the module path's own lines — goes to stderr. Stdout carries only once.go's result JSON. And a unit test now fails if any fmt.Print to stdout appears in the two builder command files, except the three that belong there: the result, --version, and --help. A guard, because this was invisible until a 20-minute run hit it, and the same class of mistake should fail in milliseconds next time. Claude-Session: https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- cmd/mesh-builder/main.go | 18 +++++++------- cmd/mesh-builder/once.go | 2 +- cmd/mesh-builder/stdout_test.go | 44 +++++++++++++++++++++++++++++++++ internal/builder/builder.go | 6 ++--- 4 files changed, 57 insertions(+), 13 deletions(-) create mode 100644 cmd/mesh-builder/stdout_test.go diff --git a/cmd/mesh-builder/main.go b/cmd/mesh-builder/main.go index bb68faa..2ca09b0 100644 --- a/cmd/mesh-builder/main.go +++ b/cmd/mesh-builder/main.go @@ -139,7 +139,7 @@ func run() error { return err } - fmt.Printf("building for the mesh, publishing to %s\n", registry) + fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry) publisher := builder.Registry{Address: registry, Run: builder.Command} for { @@ -164,14 +164,14 @@ func answer(ctx context.Context, channel *amqp.Channel, publisher builder.Publis // until it either finishes or fails is indistinguishable from one that never arrived — which // cost a long diagnosis against a running mesh, chasing "the handler never fired" when the // truth was only that the handler said nothing until the end. - fmt.Printf("a build request arrived (%d bytes)\n", len(delivery.Body)) + fmt.Fprintf(os.Stderr, "a build request arrived (%d bytes)\n", len(delivery.Body)) var request link.BuildRequest if err := json.Unmarshal(delivery.Body, &request); err != nil { // Unreadable. Acknowledged and dropped rather than requeued: a message this builder // cannot parse will not become parseable by being delivered again, and requeueing it // would put it in front of every real request for ever. - fmt.Printf("a request could not be read and was dropped: %v\n", err) + fmt.Fprintf(os.Stderr, "a request could not be read and was dropped: %v\n", err) _ = delivery.Ack(false) return } @@ -180,19 +180,19 @@ func answer(ctx context.Context, channel *amqp.Channel, publisher builder.Publis ID: request.ID, Repository: request.Repository, Path: request.Path, Ref: request.Ref, On: on, } - fmt.Printf("building %s", request.Repository) + fmt.Fprintf(os.Stderr, "building %s", request.Repository) if request.Path != "" { - fmt.Printf(" at %s", request.Path) + fmt.Fprintf(os.Stderr, " at %s", request.Path) } if request.Ref != "" { - fmt.Printf(" at %s", request.Ref) + fmt.Fprintf(os.Stderr, " at %s", request.Ref) } - fmt.Println() + fmt.Fprintln(os.Stderr) built, err := builder.Build(ctx, builder.Command, publisher, request.Repository, request.Path, request.Ref, workspace, request.Held, func(step, message string) { - fmt.Printf(" [%s] %s\n", step, message) + fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message) }) if err != nil { // A failure is a result. A build that fails and says nothing is indistinguishable from a @@ -212,7 +212,7 @@ func answer(ctx context.Context, channel *amqp.Channel, publisher builder.Publis }) } result.Against = built.Against - fmt.Printf(" built %s from %s\n", built.Manifest.Module, short(built.Commit)) + fmt.Fprintf(os.Stderr, " built %s from %s\n", built.Manifest.Module, short(built.Commit)) } } diff --git a/cmd/mesh-builder/once.go b/cmd/mesh-builder/once.go index 953cf47..e6f7196 100644 --- a/cmd/mesh-builder/once.go +++ b/cmd/mesh-builder/once.go @@ -85,7 +85,7 @@ func buildOnce(ctx context.Context, args []string) error { fmt.Fprintln(os.Stderr) built, buildErr := builder.Build(ctx, builder.Command, publisher, repository, *path, *ref, where, bases, - func(step, message string) { fmt.Printf(" [%s] %s\n", step, message) }) + func(step, message string) { fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message) }) if buildErr != nil { return buildErr } diff --git a/cmd/mesh-builder/stdout_test.go b/cmd/mesh-builder/stdout_test.go new file mode 100644 index 0000000..1d1bfd5 --- /dev/null +++ b/cmd/mesh-builder/stdout_test.go @@ -0,0 +1,44 @@ +package main + +import ( + "os" + "strings" + "testing" +) + +// **The genesis path writes its result to stdout and nothing else there.** The installer captures +// stdout alone and parses it as JSON; one stray log line makes that fail with "invalid character" +// — exactly what a builder fmt.Printf caused, breaking a build that had worked. This keeps +// diagnostics on stderr so a future print cannot repeat it silently. +func TestBuilderDiagnosticsStayOffStdout(t *testing.T) { + allowed := map[string]bool{ + "string(body)": true, // once.go: the result JSON, which IS stdout + "version)": true, // --version + `"stopping")`: true, // the loop.s shutdown line + "usage)": true, // --help text, for a human + } + for _, file := range []string{"once.go", "main.go"} { + src, err := os.ReadFile(file) + if err != nil { + t.Fatal(err) + } + for i, raw := range strings.Split(string(src), "\n") { + line := strings.TrimSpace(raw) + if !strings.HasPrefix(line, "fmt.Printf(") && + !strings.HasPrefix(line, "fmt.Println(") && + !strings.HasPrefix(line, "fmt.Print(") { + continue + } + ok := false + for token := range allowed { + if strings.Contains(line, token) { + ok = true + } + } + if !ok { + t.Errorf("%s:%d writes a diagnostic to stdout, which the installer parses as JSON:\n %s", + file, i+1, line) + } + } + } +} diff --git a/internal/builder/builder.go b/internal/builder/builder.go index d950905..54342ed 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -478,16 +478,16 @@ func Command(ctx context.Context, dir, name string, args ...string) (string, err // nothing" and "git clone is waiting on a network that will not answer". Silent on success is // what made an empty workspace unreadable. started := timeNow() - fmt.Printf(" $ (%s) %s %s\n", short(filepath.Base(dir)), name, strings.Join(args, " ")) + fmt.Fprintf(os.Stderr, " $ (%s) %s %s\n", short(filepath.Base(dir)), name, strings.Join(args, " ")) cmd := exec.CommandContext(ctx, name, args...) cmd.Dir = dir out, err := cmd.CombinedOutput() if err != nil { - fmt.Printf(" ! %s %s failed after %s\n", name, args[0], since(started)) + fmt.Fprintf(os.Stderr, " ! %s %s failed after %s\n", name, args[0], since(started)) return string(out), fmt.Errorf("%s %s: %w\n%s", name, strings.Join(args, " "), err, strings.TrimSpace(string(out))) } - fmt.Printf(" ✓ %s %s (%s)\n", name, firstArg(args), since(started)) + fmt.Fprintf(os.Stderr, " ✓ %s %s (%s)\n", name, firstArg(args), since(started)) return string(out), nil } -- 2.54.0 From 4b9bc50aad2a342ea3c90231291f2ca5d115890a Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 16 Sep 2026 10:27:26 +0200 Subject: [PATCH 13/17] The builder resolves the SDK from the mesh registry, and can publish packages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A new 'package' artifact kind builds a module's own code on a public base image and publishes it to the mesh's package registry by version (hq ADR 0076) — the SDK above all, which the toolchain is built from and so cannot be built in the toolchain. The credential a build needs to resolve or publish packages is rendered as an .npmrc (basic auth, hq ADR 0048) and given to an image build as a buildkit secret, never a layer, so a token is not baked into the toolchain image. Claude-Session: https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- cmd/mesh-builder/main.go | 98 ++++++++++++++- cmd/mesh-builder/once.go | 6 +- internal/builder/builder.go | 102 +++++++++++++++- internal/builder/builder_test.go | 20 +-- internal/builder/bundle_test.go | 8 +- internal/builder/packages.go | 127 +++++++++++++++++++ internal/builder/packages_test.go | 196 ++++++++++++++++++++++++++++++ internal/catalogue/build.go | 12 +- internal/catalogue/build_test.go | 38 ++++++ internal/catalogue/manifest.go | 7 ++ 10 files changed, 588 insertions(+), 26 deletions(-) create mode 100644 internal/builder/packages.go create mode 100644 internal/builder/packages_test.go diff --git a/cmd/mesh-builder/main.go b/cmd/mesh-builder/main.go index 2ca09b0..9a5c9e2 100644 --- a/cmd/mesh-builder/main.go +++ b/cmd/mesh-builder/main.go @@ -55,6 +55,11 @@ is dialled except the broker. MESH_BROKER_FILE a file the mesh sealed to this machine holding the same MESH_REGISTRY host:port to publish artifacts to, when the mesh has not said MESH_BINDING a file the mesh wrote saying where the artifact store is + MESH_PACKAGE_BINDING a file the mesh wrote saying where the package registry is + MESH_NPM_TOKEN_FILE a file the mesh sealed holding the token for it + MESH_NPM_REGISTRY a package registry URL, when the mesh has not said (a person, the bootstrap) + MESH_NPM_TOKEN the token for it, likewise + MESH_NPM_SCOPE the scope it answers for (default: @novox) MESH_WORKSPACE where to clone and build (default: a temporary directory) It also builds one module and stops, which is how a mesh is raised — before there is a @@ -189,11 +194,18 @@ func answer(ctx context.Context, channel *amqp.Channel, publisher builder.Publis } fmt.Fprintln(os.Stderr) - built, err := builder.Build(ctx, builder.Command, publisher, - request.Repository, request.Path, request.Ref, workspace, request.Held, - func(step, message string) { - fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message) - }) + npmrc, err := packagesFrom() + var built builder.Result + if err == nil { + // The package-registry credential is a build input, so it is resolved before the clone: a + // build that could not have resolved its dependencies is refused in front of the reason, + // not after a clone that then fails at npm ci. + built, err = builder.Build(ctx, builder.Command, publisher, + request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc, + func(step, message string) { + fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message) + }) + } if err != nil { // A failure is a result. A build that fails and says nothing is indistinguishable from a // builder that is not running, and those want completely different responses. @@ -279,6 +291,82 @@ func moduleOf(manifest json.RawMessage) string { return named.Module } +// packagesFrom is where a build resolves the mesh's own published packages — the SDK above all +// (novox/hq ADR 0076, issue 053). +// +// Preferably from the mesh: a package-registry binding names the endpoint the way the artifact +// store's binding does, and a sealed token file the credential the way the broker's does. The +// environment variables remain for a builder run by a person, and for the bootstrap, where there is +// no registry yet — there the result is disabled and a build that needs no mesh-published dependency +// builds anyway. +func packagesFrom() (builder.Npmrc, error) { + scope := strings.TrimSpace(os.Getenv("MESH_NPM_SCOPE")) + if scope == "" { + scope = "@novox" + } + + registry := strings.TrimSpace(os.Getenv("MESH_NPM_REGISTRY")) + var username string + if path := strings.TrimSpace(os.Getenv("MESH_PACKAGE_BINDING")); path != "" { + raw, err := os.ReadFile(path) + if err != nil { + return builder.Npmrc{}, fmt.Errorf("cannot read what the mesh said about the package registry: %w", err) + } + var told struct { + From string `json:"from"` + At string `json:"at"` + As string `json:"as"` + Serves map[string]any `json:"serves"` + } + if err := json.Unmarshal(raw, &told); err != nil { + return builder.Npmrc{}, fmt.Errorf("%s is not a binding: %w", path, err) + } + if told.At == "" { + return builder.Npmrc{}, fmt.Errorf( + "%s says the package registry is on %q and gives no address for it", path, told.From) + } + // Composed from what the provider serves, so nothing here knows gitea's URL shape from + // another registry's: it states its port, the path its registry answers on, and the scheme. + scheme := "https" + if s, ok := told.Serves["scheme"]; ok { + scheme = fmt.Sprintf("%v", s) + } + port, ok := told.Serves["port"] + if !ok { + return builder.Npmrc{}, fmt.Errorf("%s says nothing about which port the package registry answers on", path) + } + npmPath, ok := told.Serves["npm-path"] + if !ok { + return builder.Npmrc{}, fmt.Errorf("%s says nothing about the path the package registry answers on", path) + } + registry = fmt.Sprintf("%s://%s:%v%v", scheme, told.At, port, npmPath) + username = told.As + } + + // The credential the mesh sealed to this machine. The mesh authenticates the ordinary way — a + // generated password the provider only applies (novox/hq ADR 0048) — so with a username this is + // a password (basic auth); without one it is a bearer token a provider minted. + secret := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN")) + if path := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN_FILE")); path != "" { + raw, err := os.ReadFile(path) + if err != nil { + return builder.Npmrc{}, fmt.Errorf("cannot read this builder's package-registry credential: %w", err) + } + secret = strings.TrimSpace(string(raw)) + } + if u := strings.TrimSpace(os.Getenv("MESH_NPM_USER")); u != "" { + username = u + } + + if registry == "" && secret == "" { + return builder.Npmrc{}, nil + } + if username != "" { + return builder.Npmrc{Scope: scope, Registry: registry, Username: username, Password: secret}, nil + } + return builder.Npmrc{Scope: scope, Registry: registry, Token: secret}, nil +} + func short(commit string) string { if len(commit) > 8 { return commit[:8] diff --git a/cmd/mesh-builder/once.go b/cmd/mesh-builder/once.go index e6f7196..8423c67 100644 --- a/cmd/mesh-builder/once.go +++ b/cmd/mesh-builder/once.go @@ -84,7 +84,11 @@ func buildOnce(ctx context.Context, args []string) error { } fmt.Fprintln(os.Stderr) - built, buildErr := builder.Build(ctx, builder.Command, publisher, repository, *path, *ref, where, bases, + npmrc, err := packagesFrom() + if err != nil { + return err + } + built, buildErr := builder.Build(ctx, builder.Command, publisher, repository, *path, *ref, where, bases, npmrc, func(step, message string) { fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message) }) if buildErr != nil { return buildErr diff --git a/internal/builder/builder.go b/internal/builder/builder.go index 54342ed..a7cddcc 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -6,6 +6,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "encoding/json" "fmt" "io" "os" @@ -68,7 +69,7 @@ type Result struct { // archive failed would otherwise leave half of itself in the store under a digest the mesh never // records — reachable, unreferenced, and indistinguishable from something in use. func Build(ctx context.Context, run Runner, publish Publisher, - repository, path, ref, workspace string, held map[string]string, log Log) (Result, error) { + repository, path, ref, workspace string, held map[string]string, npmrc Npmrc, log Log) (Result, error) { say := logging(log) say("clone", "%s%s at %s", repository, describePath(path), refOrHead(ref)) @@ -124,6 +125,22 @@ func Build(ctx context.Context, run Runner, publish Publisher, } say("manifest", "%s v%s — %d artifact(s)", manifest.Module, manifest.Version, artifactCount(manifest)) + // A build-time credential, written where a build can mount it but never where it can be copied + // into an image or committed: under the workspace, beside the clone, not inside it. Absent when + // this mesh has no package registry yet, which is the bootstrap case (novox/hq ADR 0076). + var npmrcPath string + if npmrc.Enabled() { + content, err := npmrc.File() + if err != nil { + return Result{}, err + } + npmrcPath = filepath.Join(workspace, "npmrc") + if err := os.WriteFile(npmrcPath, []byte(content), 0o600); err != nil { + return Result{}, fmt.Errorf("cannot write the package-registry credential for the build: %w", err) + } + say("packages", "resolving %s from the mesh's package registry", npmrc.Scope) + } + var built []catalogue.Built if manifest.Build != nil { // What this module said it stands on, answered with what this mesh actually holds. Done @@ -143,7 +160,7 @@ func Build(ctx context.Context, run Runner, publish Publisher, sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name }) for _, a := range artifacts { say("artifact", "%s (%s%s) — starting", a.Name, a.Kind, langSuffix(a)) - made, err := one(ctx, run, publish, manifest.Module, within, commit, a, args, held, say) + made, err := one(ctx, run, publish, manifest.Module, within, commit, a, args, held, npmrcPath, say) if err != nil { say("artifact", "%s FAILED: %v", a.Name, err) return Result{}, err @@ -282,7 +299,7 @@ const ManifestName = "module.json" func one(ctx context.Context, run Runner, publish Publisher, module, tree, commit string, a catalogue.Artifact, args []string, - held map[string]string, say func(step, format string, args ...any)) (catalogue.Built, error) { + held map[string]string, npmrc string, say func(step, format string, args ...any)) (catalogue.Built, error) { switch a.Kind { case catalogue.ArtifactUpstream: @@ -311,6 +328,12 @@ func one(ctx context.Context, run Runner, publish Publisher, if a.Target != "" { invocation = append(invocation, "--target", a.Target) } + if npmrc != "" { + // Given to the build as a buildkit secret, so a RUN that needs the package registry mounts + // it at that step and it is in no image layer. A Dockerfile that does not ask for it is + // unaffected; the secret is simply not read (novox/hq ADR 0076). + invocation = append(invocation, "--secret", "id=npmrc,src="+npmrc) + } invocation = append(invocation, ".") say("image", "docker build -f %s", a.From) if _, err := run(ctx, tree, "docker", invocation...); err != nil { @@ -365,6 +388,19 @@ func one(ctx context.Context, run Runner, publish Publisher, } return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: where, Digest: digest}, nil + case catalogue.ArtifactPackage: + // Built and published on a public base, to the mesh's package registry, by version + // (novox/hq ADR 0076). Not an image, not an archive: nothing on a machine references it, so + // there is no Publisher call — the container itself publishes, with the credential the + // build was handed. + say("package", "building and publishing %s (%s)", a.Name, a.Language) + reference, err := publishPackage(ctx, run, module, tree, a, npmrc, say) + if err != nil { + return catalogue.Built{}, fmt.Errorf("%s: publishing %s failed: %w", module, a.Name, err) + } + say("package", "published %s", reference) + return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil + case catalogue.ArtifactArchive: body, err := pack(filepath.Join(tree, a.From)) if err != nil { @@ -541,6 +577,66 @@ func standingOn(manifest catalogue.Manifest, held map[string]string) ([]string, // where its dependencies resolve upward into the base's own library directory, so what it is // compiled against is exactly what it will run against — the reason every hand-written Dockerfile // had to choose a working directory carefully, and the reason none of them has to now. +// publishPackage builds a `package` artifact on a public base image and publishes it to the mesh's +// package registry by version. The credential arrives as an .npmrc file the build was handed +// (novox/hq ADR 0076); it is mounted read-only into the container rather than baked, because a +// package build produces no image to leak it into. The reference returned is name@version, read from +// the module's own package.json — the same two fields npm publishes under. +func publishPackage(ctx context.Context, run Runner, module, dir string, a catalogue.Artifact, + npmrc string, say func(step, format string, args ...any)) (string, error) { + + recipe, ok := packageRecipes[a.Language] + if !ok { + return "", fmt.Errorf( + "a package written in %q cannot be built: no public toolchain is known for it", a.Language) + } + if npmrc == "" { + // A package with nowhere to be published is not built. Said here rather than failing inside + // npm publish with a message about a registry that is simply absent. + return "", fmt.Errorf( + "%s is a package and this build was given no package registry to publish it to", a.Name) + } + + raw, err := os.ReadFile(filepath.Join(dir, "package.json")) + if err != nil { + return "", fmt.Errorf("a package is published by name and version, and %s has no package.json: %w", module, err) + } + var pkg struct { + Name string `json:"name"` + Version string `json:"version"` + } + if err := json.Unmarshal(raw, &pkg); err != nil { + return "", fmt.Errorf("%s's package.json is not readable: %w", module, err) + } + if pkg.Name == "" || pkg.Version == "" { + return "", fmt.Errorf("%s's package.json names no %s to publish under", + module, either(pkg.Name == "", "name", "version")) + } + + const within = "/app/module" + invocation := []string{ + "run", "--rm", + "--volume", dir + ":" + within, + // Read-only, so a build cannot alter the credential, and at /root where npm reads it. + "--volume", npmrc + ":/root/.npmrc:ro", + "--workdir", within, + recipe.Base, + "sh", "-c", recipe.Script, + } + if _, err := run(ctx, dir, "docker", invocation...); err != nil { + return "", err + } + return pkg.Name + "@" + pkg.Version, nil +} + +// either names whichever of two fields is the missing one, for a message that says which. +func either(first bool, a, b string) string { + if first { + return a + } + return b +} + func compile(ctx context.Context, run Runner, tree string, chain Toolchain, base string, a catalogue.Artifact) (string, error) { diff --git a/internal/builder/builder_test.go b/internal/builder/builder_test.go index a97a93f..6938d71 100644 --- a/internal/builder/builder_test.go +++ b/internal/builder/builder_test.go @@ -108,7 +108,7 @@ func TestABuildProducesAManifestThePinsAreIn(t *testing.T) { r, workspace := aRepository(t, withBoth, map[string]string{ "Dockerfile": "FROM scratch", "files/theme.conf": "dark", }) - got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, nil) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, Npmrc{}, nil) if err != nil { t.Fatal(err) } @@ -134,7 +134,7 @@ func TestTwoBuildsOfOneCommitProduceOneDigest(t *testing.T) { }) // A year apart, so a packer carrying timestamps cannot accidentally agree. r.stamped = time.Date(2020+i, time.March, 3, 4, 5, 6, 0, time.UTC) - got, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, nil) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, nil) if err != nil { t.Fatal(err) } @@ -154,7 +154,7 @@ func TestNothingIsPublishedUntilEverythingIsBuilt(t *testing.T) { // unreferenced, and indistinguishable from something in use. r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch"}) // `files` is missing, so packing the archive fails — after the image would have been pushed. - _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, nil) + _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, nil) if err == nil { t.Fatal("a build with a missing input succeeded") } @@ -166,7 +166,7 @@ func TestNothingIsPublishedUntilEverythingIsBuilt(t *testing.T) { func TestARepositoryWithNoManifestSaysSo(t *testing.T) { workspace := t.TempDir() r := &recorded{contents: map[string]string{"README.md": "nothing to see"}} - _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, nil) + _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, nil) if err == nil { t.Fatal("a repository with nothing saying what it is was built") } @@ -179,7 +179,7 @@ func TestAModuleThatBuildsNothingStillProducesAManifest(t *testing.T) { // Most of what a person installs is configuration. r, workspace := aRepository(t, `{"module":"shell","version":"1","resources":[ {"id":"rc","type":"file","path":"/etc/zsh/zshrc","content":"setopt"}]}`, nil) - got, err := Build(context.Background(), r.run, r, "https://forge.invalid/shell.git", "", "", workspace, nil, nil) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/shell.git", "", "", workspace, nil, Npmrc{}, nil) if err != nil { t.Fatal(err) } @@ -209,7 +209,7 @@ func TestTheTreeIsFreshEveryTime(t *testing.T) { if err := os.WriteFile(leftover, []byte("stale"), 0o644); err != nil { t.Fatal(err) } - if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, nil); err != nil { + if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, nil); err != nil { t.Fatal(err) } if _, err := os.Stat(leftover); err == nil { @@ -222,7 +222,7 @@ func TestABuildThatCannotPushFails(t *testing.T) { "Dockerfile": "FROM scratch", "files/a": "b", }) r.failPush = true - if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, nil); err == nil { + if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, nil); err == nil { t.Fatal("a build that could publish nothing reported success") } } @@ -236,7 +236,7 @@ func TestAnUpstreamImageIsMirroredRatherThanBuilt(t *testing.T) { "resources":[{"id":"db","type":"container","name":"mesh-postgres","artifact":"store"}]}` r, workspace := aRepository(t, mirrors, nil) - got, err := Build(context.Background(), r.run, r, "https://forge.invalid/postgres.git", "", "", workspace, nil, nil) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/postgres.git", "", "", workspace, nil, Npmrc{}, nil) if err != nil { t.Fatal(err) } @@ -302,7 +302,7 @@ func TestAModuleIsBuiltFromItsPathWithinTheRepository(t *testing.T) { "modules/other/" + ManifestName: `{"module":"other","version":"1"}`, }} got, err := Build(context.Background(), r.run, r, - "https://forge.invalid/catalogue.git", "modules/shell", "", t.TempDir(), nil, nil) + "https://forge.invalid/catalogue.git", "modules/shell", "", t.TempDir(), nil, Npmrc{}, nil) if err != nil { t.Fatal(err) } @@ -321,7 +321,7 @@ func TestAPathThatLeavesTheRepositoryIsRefused(t *testing.T) { for _, escaping := range []string{"../../etc", "/etc"} { r := &recorded{contents: map[string]string{ManifestName: withBoth}} _, err := Build(context.Background(), r.run, r, - "https://forge.invalid/x.git", escaping, "", t.TempDir(), nil, nil) + "https://forge.invalid/x.git", escaping, "", t.TempDir(), nil, Npmrc{}, nil) if err == nil { t.Fatalf("%q was accepted as a module's path", escaping) } diff --git a/internal/builder/bundle_test.go b/internal/builder/bundle_test.go index ffbce41..5cf8846 100644 --- a/internal/builder/bundle_test.go +++ b/internal/builder/bundle_test.go @@ -54,7 +54,7 @@ func TestABundleIsCompiledAndPackedWithNoDockerfile(t *testing.T) { held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)} got, err := Build(context.Background(), compiling{r}.run, r, - "https://forge.invalid/greeter.git", "", "", workspace, held, nil) + "https://forge.invalid/greeter.git", "", "", workspace, held, Npmrc{}, nil) if err != nil { t.Fatalf("a module with a language and no Dockerfile did not build: %v", err) } @@ -91,7 +91,7 @@ func TestABundleWhoseToolchainIsNotHeldIsRefusedFirst(t *testing.T) { r, workspace := aRepository(t, aBundle, map[string]string{"index.ts": "console.log(1)"}) _, err := Build(context.Background(), compiling{r}.run, r, - "https://forge.invalid/greeter.git", "", "", workspace, nil, nil) + "https://forge.invalid/greeter.git", "", "", workspace, nil, Npmrc{}, nil) if err == nil { t.Fatal("a bundle was built with no toolchain to compile it in") } @@ -112,7 +112,7 @@ func TestABundleInAnUnknownLanguageIsRefused(t *testing.T) { _, err := Build(context.Background(), compiling{r}.run, r, "https://forge.invalid/greeter.git", "", "", workspace, - map[string]string{"mesh-tools/build": "registry.invalid/x@sha256:" + strings.Repeat("c", 64)}, nil) + map[string]string{"mesh-tools/build": "registry.invalid/x@sha256:" + strings.Repeat("c", 64)}, Npmrc{}, nil) if err == nil { t.Fatal("a language nothing can compile was accepted") } @@ -140,7 +140,7 @@ func TestTwoBundlesInOneModuleArePackedSeparately(t *testing.T) { held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)} got, err := Build(context.Background(), compiling{r}.run, r, - "https://forge.invalid/greeter.git", "", "", workspace, held, nil) + "https://forge.invalid/greeter.git", "", "", workspace, held, Npmrc{}, nil) if err != nil { t.Fatalf("a module with two bundles did not build: %v", err) } diff --git a/internal/builder/packages.go b/internal/builder/packages.go new file mode 100644 index 0000000..d30cacd --- /dev/null +++ b/internal/builder/packages.go @@ -0,0 +1,127 @@ +package builder + +import ( + "encoding/base64" + "fmt" + "net/url" + "strings" +) + +// Npmrc is what a build needs to resolve the mesh's own shared library — and any other package the +// mesh publishes — from the mesh's package registry rather than from a git URL (novox/hq ADR 0076, +// issue 053). +// +// It is a build-time credential, not a runtime one. A module compiled inside the toolchain image +// resolves the SDK there, once, when that image is built; the running container never speaks to the +// package registry. So this is given to the *builder*, the way the artifact store is +// (`whereToPublish`), and reaches a build as a secret rather than a layer — see Secret. +// +// The registry's exact URL shape is the provider's business, not the builder's: it arrives whole, +// either from the binding the mesh writes (a package-registry provider's `serves` facts) or from the +// environment when a person runs a build by hand. Nothing here knows gitea from verdaccio. +type Npmrc struct { + // Scope is the npm scope the registry answers for, e.g. "@novox". A build resolves only this + // scope from the mesh; everything else resolves the ordinary way, so a mesh with no internet + // still cannot pull the public registry's version of a name the mesh also publishes. + Scope string + // Registry is the full base URL a client uses for this scope, e.g. + // "https:///api/packages//npm/". Trailing slash tolerated either way. + Registry string + // Token authenticates to the registry as a bearer token, when a provider mints one. Left empty + // when the mesh authenticates the ordinary way it authenticates everything — a generated + // password it applies and seals — for which see Username and Password. + Token string + // Username and Password authenticate by basic auth, which is what gitea and verdaccio both + // accept and what lets the credential be a mesh-generated password the provider's provisioner + // applies and the mesh seals to the consumer — the same shape a database password takes. The + // username is the consumer's mesh identity. Ignored when Token is set. + Username string + Password string +} + +// Enabled reports whether there is a registry to resolve against at all. A bootstrap build that +// runs before any package registry exists has none, and must still build whatever needs no +// mesh-published dependency. +func (n Npmrc) Enabled() bool { + return strings.TrimSpace(n.Scope) != "" && strings.TrimSpace(n.Registry) != "" +} + +// File renders the .npmrc a build mounts. Two lines: which registry answers for the scope, and the +// token to present to it. The auth line is keyed by the registry URL with its scheme removed, which +// is how npm matches a stored credential to a request. +// +// It returns an error rather than a malformed file, because an .npmrc that npm parses but points +// nowhere fails much later, inside a build, as a package that cannot be found. +func (n Npmrc) File() (string, error) { + scope := strings.TrimSpace(n.Scope) + if !strings.HasPrefix(scope, "@") { + return "", fmt.Errorf("a package-registry scope is written with its leading @, not %q", scope) + } + reg := strings.TrimSpace(n.Registry) + if !strings.HasPrefix(reg, "http://") && !strings.HasPrefix(reg, "https://") { + return "", fmt.Errorf("a package registry is reached over http(s), and %q is neither", reg) + } + if !strings.HasSuffix(reg, "/") { + // npm's per-scope registry key is matched by prefix, and the auth key below is derived from + // it; a missing trailing slash makes the two disagree and the token is never sent. + reg += "/" + } + parsed, err := url.Parse(reg) + if err != nil { + return "", fmt.Errorf("%q is not a usable registry URL: %w", reg, err) + } + // The auth key is the URL without its scheme, e.g. "//host/api/packages/owner/npm/". + authKey := "//" + parsed.Host + parsed.EscapedPath() + + var auth string + switch { + case strings.TrimSpace(n.Token) != "": + auth = fmt.Sprintf("%s:_authToken=%s\n", authKey, strings.TrimSpace(n.Token)) + case strings.TrimSpace(n.Username) != "" && n.Password != "": + // npm reads the password base64-encoded, and always-auth so it presents the credential to + // reads as well as writes — a private registry answers neither without it. + enc := base64.StdEncoding.EncodeToString([]byte(n.Password)) + auth = fmt.Sprintf("%s:username=%s\n%s:_password=%s\n%s:always-auth=true\n", + authKey, strings.TrimSpace(n.Username), authKey, enc, authKey) + default: + return "", fmt.Errorf( + "the package registry at %s was given neither a token nor a username and password", reg) + } + return fmt.Sprintf("%s:registry=%s\n%s", scope, reg, auth), nil +} + +// packageRecipe is how a `package` artifact is built and published: on a PUBLIC base image, never +// the mesh toolchain, because the toolchain is built from the package this produces (the SDK). The +// script builds the module, then publishes it to the mesh's package registry unless that exact +// version is already there — so a re-run of genesis, which must be safe, does not fail on a version +// it published a moment ago. +type packageRecipe struct { + Base string + Script string +} + +var packageRecipes = map[string]packageRecipe{ + "typescript": { + Base: "node:22-bookworm-slim", + Script: `set -e +npm install --no-audit --no-fund +npm run build +name="$(node -p "require('./package.json').name")" +ver="$(node -p "require('./package.json').version")" +if npm view "$name@$ver" version >/dev/null 2>&1; then + echo "mesh-builder: $name@$ver is already published, leaving it" +else + npm publish +fi`, + }, +} + +// PackageLanguages is the languages a package artifact can be written in, for a manifest check that +// wants to refuse one it cannot build before a build starts. +func PackageLanguages() []string { + out := make([]string, 0, len(packageRecipes)) + for l := range packageRecipes { + out = append(out, l) + } + return out +} diff --git a/internal/builder/packages_test.go b/internal/builder/packages_test.go new file mode 100644 index 0000000..a26a287 --- /dev/null +++ b/internal/builder/packages_test.go @@ -0,0 +1,196 @@ +package builder + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestNpmrcRendersRegistryAndTokenForTheScope(t *testing.T) { + n := Npmrc{ + Scope: "@novox", + Registry: "https://forge.invalid/api/packages/novox/npm/", + Token: "a-token", + } + got, err := n.File() + if err != nil { + t.Fatalf("a complete credential did not render: %v", err) + } + if !strings.Contains(got, "@novox:registry=https://forge.invalid/api/packages/novox/npm/") { + t.Fatalf("the scope's registry line is missing:\n%s", got) + } + // The auth line is keyed by the URL without its scheme, or npm never sends the token. + if !strings.Contains(got, "//forge.invalid/api/packages/novox/npm/:_authToken=a-token") { + t.Fatalf("the auth line does not match the registry key:\n%s", got) + } +} + +func TestNpmrcAddsATrailingSlashSoTheAuthKeyMatches(t *testing.T) { + n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm", Token: "t"} + got, err := n.File() + if err != nil { + t.Fatal(err) + } + if !strings.Contains(got, "registry=https://forge.invalid/api/packages/novox/npm/\n") { + t.Fatalf("a missing trailing slash was not normalised:\n%s", got) + } +} + +func TestNpmrcRefusesTheHalfConfigured(t *testing.T) { + cases := map[string]Npmrc{ + "scope without @": {Scope: "novox", Registry: "https://x.invalid/", Token: "t"}, + "registry not http": {Scope: "@novox", Registry: "ftp://x.invalid/", Token: "t"}, + "no token": {Scope: "@novox", Registry: "https://x.invalid/", Token: ""}, + } + for name, n := range cases { + if _, err := n.File(); err == nil { + t.Fatalf("%s rendered an .npmrc rather than refusing", name) + } + } +} + +func TestNpmrcDisabledUntilThereIsARegistry(t *testing.T) { + if (Npmrc{}).Enabled() { + t.Fatal("an empty credential reported itself usable") + } + if (Npmrc{Scope: "@novox"}).Enabled() { + t.Fatal("a scope with no registry reported itself usable") + } + if !(Npmrc{Scope: "@novox", Registry: "https://x.invalid/"}).Enabled() { + t.Fatal("a scope and a registry did not count as usable") + } +} + +// The credential reaches an image build as a buildkit secret and never as a file inside the build +// context, because a token copied into a layer is a token published (novox/hq ADR 0076). +func TestAnImageBuildGetsTheCredentialAsASecretNotALayer(t *testing.T) { + r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch", "files/x": "y"}) + n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm/", Token: "t"} + if _, err := Build(context.Background(), r.run, r, + "https://forge.invalid/meshboard.git", "", "", workspace, nil, n, nil); err != nil { + t.Fatalf("the build failed: %v", err) + } + + var build string + for _, line := range r.ran { + if strings.HasPrefix(line, "docker build") { + build = line + } + } + if build == "" { + t.Fatal("no docker build ran") + } + if !strings.Contains(build, "--secret id=npmrc,src=") { + t.Fatalf("the build was not given the credential as a secret: %s", build) + } + + // The .npmrc lives under the workspace, beside the clone, never inside the source tree that is + // the docker context. + tree := filepath.Join(workspace, "source") + src := strings.SplitN(strings.SplitN(build, "--secret id=npmrc,src=", 2)[1], " ", 2)[0] + if strings.HasPrefix(src, tree+string(os.PathSeparator)) { + t.Fatalf("the credential file %s is inside the build context %s", src, tree) + } + if _, err := os.Stat(src); err != nil { + t.Fatalf("the credential file the build was pointed at does not exist: %v", err) + } +} + +func TestAnImageBuildWithoutACredentialGetsNoSecret(t *testing.T) { + r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch", "files/x": "y"}) + if _, err := Build(context.Background(), r.run, r, + "https://forge.invalid/meshboard.git", "", "", workspace, nil, Npmrc{}, nil); err != nil { + t.Fatalf("the build failed: %v", err) + } + for _, line := range r.ran { + if strings.HasPrefix(line, "docker build") && strings.Contains(line, "--secret") { + t.Fatalf("a build with no credential was still given a secret: %s", line) + } + } +} + +const aPackage = `{"module":"mesh-sdk","version":"1", + "build":{"artifacts":[{"name":"lib","kind":"package","language":"typescript"}]}, + "resources":[]}` + +// A package is compiled on a public base and published to the mesh's package registry by version, +// with nothing pushed to the artifact store and the credential mounted, not baked (novox/hq ADR 0076). +func TestAPackageIsBuiltOnAPublicBaseAndPublishedByVersion(t *testing.T) { + r, workspace := aRepository(t, aPackage, map[string]string{ + "package.json": `{"name":"@novox/mesh-sdk","version":"0.1.0"}`, + }) + n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm/", Token: "t"} + got, err := Build(context.Background(), r.run, r, + "https://forge.invalid/mesh-sdk.git", "", "", workspace, nil, n, nil) + if err != nil { + t.Fatalf("the package did not build: %v", err) + } + if len(got.Built) != 1 || got.Built[0].Reference != "@novox/mesh-sdk@0.1.0" { + t.Fatalf("a package is published by name and version, got %+v", got.Built) + } + if len(r.images) != 0 || len(r.archives) != 0 { + t.Fatal("a package was pushed to the artifact store, which is not where packages live") + } + var ran string + for _, line := range r.ran { + if strings.HasPrefix(line, "docker run") { + ran = line + } + } + if ran == "" { + t.Fatal("nothing ran to build the package") + } + if !strings.Contains(ran, "node:22-bookworm-slim") { + t.Fatalf("a package was not built on a public base: %s", ran) + } + if !strings.Contains(ran, ":/root/.npmrc:ro") { + t.Fatalf("the credential was not mounted read-only for the publish: %s", ran) + } +} + +func TestAPackageWithNoRegistryIsRefused(t *testing.T) { + r, workspace := aRepository(t, aPackage, map[string]string{ + "package.json": `{"name":"@novox/mesh-sdk","version":"0.1.0"}`, + }) + _, err := Build(context.Background(), r.run, r, + "https://forge.invalid/mesh-sdk.git", "", "", workspace, nil, Npmrc{}, nil) + if err == nil { + t.Fatal("a package built with no registry to publish to, silently") + } +} + +func TestNpmrcRendersBasicAuthWhenGivenAUserAndPassword(t *testing.T) { + n := Npmrc{ + Scope: "@novox", + Registry: "http://forge.invalid:3000/api/packages/novox/npm/", + Username: "mesh_anchor_builder", + Password: "s3cret", + } + got, err := n.File() + if err != nil { + t.Fatalf("basic-auth credential did not render: %v", err) + } + key := "//forge.invalid:3000/api/packages/novox/npm/" + if !strings.Contains(got, key+":username=mesh_anchor_builder\n") { + t.Fatalf("username line missing:\n%s", got) + } + // npm reads the password base64-encoded. + if !strings.Contains(got, key+":_password=czNjcmV0\n") { + t.Fatalf("base64 password line missing or wrong:\n%s", got) + } + if !strings.Contains(got, key+":always-auth=true\n") { + t.Fatalf("always-auth missing, so reads would go unauthenticated:\n%s", got) + } + if strings.Contains(got, "_authToken") { + t.Fatalf("a token line was rendered for a basic-auth credential:\n%s", got) + } +} + +func TestNpmrcRefusesWhenGivenNeitherTokenNorPassword(t *testing.T) { + n := Npmrc{Scope: "@novox", Registry: "http://x.invalid/npm/", Username: "u"} + if _, err := n.File(); err == nil { + t.Fatal("a username with no password rendered an .npmrc") + } +} diff --git a/internal/catalogue/build.go b/internal/catalogue/build.go index d7df29a..9ff4336 100644 --- a/internal/catalogue/build.go +++ b/internal/catalogue/build.go @@ -86,6 +86,12 @@ func (m Manifest) Resolve(built []Built) (Manifest, error) { } delete(filled, "artifact") switch artifact.Kind { + case ArtifactPackage: + // A package is not a resource on any machine; it is consumed by other builds. A + // resource that names one is a manifest error, named here rather than shipped. + return Manifest{}, fmt.Errorf( + "%s: %v uses %q, which is a package — a build input, not a resource a machine runs", + m.Module, r["id"], named) case ArtifactImage, ArtifactUpstream: filled["image"] = artifact.Reference case ArtifactArchive, ArtifactBundle: @@ -123,19 +129,19 @@ func (b *Build) problems(module string) []string { } seen[a.Name] = true switch a.Kind { - case ArtifactImage, ArtifactArchive, ArtifactUpstream, ArtifactBundle: + case ArtifactImage, ArtifactArchive, ArtifactUpstream, ArtifactBundle, ArtifactPackage: default: problems = append(problems, fmt.Sprintf( "%s: %q is a %q, and an artifact is %q, %q, %q or %q", module, a.Name, a.Kind, ArtifactImage, ArtifactArchive, ArtifactUpstream, - ArtifactBundle)) + ArtifactBundle+", "+ArtifactPackage)) } // **A bundle is built from the module itself, so it says a language instead.** Everything // else names what it is built from: a Dockerfile, a directory, somebody else's reference. // A bundle's source is the module's own directory by definition, and what it needs to say // is which compiler — because the mesh chooses that, and cannot choose for a module that // has not said. - if a.Kind == ArtifactBundle { + if a.Kind == ArtifactBundle || a.Kind == ArtifactPackage { if a.From != "" { problems = append(problems, fmt.Sprintf( "%s: %q is a bundle and names what it is built from (%q). A bundle is built "+ diff --git a/internal/catalogue/build_test.go b/internal/catalogue/build_test.go index cba3ed2..34a8694 100644 --- a/internal/catalogue/build_test.go +++ b/internal/catalogue/build_test.go @@ -142,3 +142,41 @@ func TestAnArtifactOfAnUnknownKindIsRefused(t *testing.T) { t.Fatal("an artifact of an unknown kind was accepted") } } + +func TestAPackageParsesLikeABundleAndNeedsALanguage(t *testing.T) { + // The SDK's shape: a package built from the module's own directory, naming a language. + m, err := ParseManifest([]byte(`{"module":"mesh-sdk","version":"1","slug":"sdk", + "build":{"artifacts":[{"name":"lib","kind":"package","language":"typescript"}]}, + "resources":[]}`)) + if err != nil { + t.Fatalf("the SDK's package manifest did not parse: %v", err) + } + if m.Build.Artifacts[0].Kind != ArtifactPackage { + t.Fatalf("expected a package artifact, got %q", m.Build.Artifacts[0].Kind) + } + + // A package that names what it is built from is refused, exactly as a bundle is: it is built + // from the module's own directory. + if _, err := ParseManifest([]byte(`{"module":"a","version":"1","build":{"artifacts":[ + {"name":"lib","kind":"package","language":"typescript","from":"Dockerfile"}]}}`)); err == nil { + t.Fatal("a package naming a source was accepted") + } + // A package with no language cannot choose a toolchain. + if _, err := ParseManifest([]byte(`{"module":"a","version":"1","build":{"artifacts":[ + {"name":"lib","kind":"package"}]}}`)); err == nil { + t.Fatal("a package with no language was accepted") + } +} + +func TestAResourceNamingAPackageIsRefused(t *testing.T) { + m, err := ParseManifest([]byte(`{"module":"a","version":"1","slug":"a", + "build":{"artifacts":[{"name":"lib","kind":"package","language":"typescript"}]}, + "resources":[{"id":"svc","type":"container","name":"a","artifact":"lib"}]}`)) + if err != nil { + t.Fatalf("parse: %v", err) + } + _, err = m.Resolve([]Built{{Name: "lib", Kind: ArtifactPackage, Reference: "@novox/a@1.0.0"}}) + if err == nil { + t.Fatal("a resource backed by a package was accepted; a package is not a resource") + } +} diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index 03ba034..38585f6 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -463,6 +463,13 @@ const ( // Mirroring is what the bootstrap already does by hand: the lab stocks upstream images into // the registry a first node pulls from. This makes that a thing a module can say. ArtifactUpstream = "upstream" + + // ArtifactPackage is this module's own code, compiled and published to the mesh's package + // registry by version, for other modules to consume when they are built — the SDK above all + // (novox/hq ADR 0076). Like a bundle it is built from the module's own directory and names a + // language; unlike a bundle it is not a resource on any machine, it is a build input. It is + // compiled on a PUBLIC base, never the mesh toolchain, because the toolchain is built from it. + ArtifactPackage = "package" ) // ArtifactStoreProvision is the name a module offers when it is the mesh's store for what modules -- 2.54.0 From ae55bd7bdee22c94fbf1ab6fd4d5a9dab8d6da05 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 16 Sep 2026 10:33:07 +0200 Subject: [PATCH 14/17] Builds that need the registry run on the host network An image build with an npm credential, and a package publish, join the host network so 127.0.0.1 reaches the registry where the binding names it. Claude-Session: https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- internal/builder/builder.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/builder/builder.go b/internal/builder/builder.go index a7cddcc..32895a1 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -333,6 +333,9 @@ func one(ctx context.Context, run Runner, publish Publisher, // it at that step and it is in no image layer. A Dockerfile that does not ask for it is // unaffected; the secret is simply not read (novox/hq ADR 0076). invocation = append(invocation, "--secret", "id=npmrc,src="+npmrc) + // Host network for the build, so a RUN reaching the package registry finds it where the + // binding says it is — the machine's own loopback, where the registry answers. + invocation = append(invocation, "--network", "host") } invocation = append(invocation, ".") say("image", "docker build -f %s", a.From) @@ -616,6 +619,8 @@ func publishPackage(ctx context.Context, run Runner, module, dir string, a catal const within = "/app/module" invocation := []string{ "run", "--rm", + // Host network, so the publish reaches the registry at the address the binding names. + "--network", "host", "--volume", dir + ":" + within, // Read-only, so a build cannot alter the credential, and at /root where npm reads it. "--volume", npmrc + ":/root/.npmrc:ro", -- 2.54.0 From 6a7e701629f46eedb25cabe781fa95ba6ea0a0b0 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 16 Sep 2026 12:45:18 +0200 Subject: [PATCH 15/17] Write the package credential only into a build that asks for it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A per-run .npmrc in every build context put a changing credential in COPY . . of modules that resolve no mesh package — a non-deterministic image (a needless rollout every build, which recreated the control plane) and a credential in a build stage. Now it is written only for a package artifact or an image whose Dockerfile names .npmrc. Claude-Session: https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- internal/builder/builder.go | 39 ++++++++++++++------ internal/builder/packages_test.go | 60 +++++++++++++++++++++---------- 2 files changed, 71 insertions(+), 28 deletions(-) diff --git a/internal/builder/builder.go b/internal/builder/builder.go index 32895a1..a9f7acf 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -125,16 +125,19 @@ func Build(ctx context.Context, run Runner, publish Publisher, } say("manifest", "%s v%s — %d artifact(s)", manifest.Module, manifest.Version, artifactCount(manifest)) - // A build-time credential, written where a build can mount it but never where it can be copied - // into an image or committed: under the workspace, beside the clone, not inside it. Absent when - // this mesh has no package registry yet, which is the bootstrap case (novox/hq ADR 0076). + // A build-time credential, written into the build context as .npmrc, but ONLY for a module that + // asks for it: a `package` artifact (which publishes), or an image whose Dockerfile COPYs .npmrc. + // Writing it into every context would put a per-run credential in `COPY . .` of modules that + // never resolve a mesh package — making their image non-deterministic (a needless rollout every + // build) and leaking the credential into a build stage. Absent entirely with no registry, which + // is the bootstrap case (novox/hq ADR 0076). var npmrcPath string - if npmrc.Enabled() { + if npmrc.Enabled() && manifest.Build != nil && wantsPackages(manifest, within) { content, err := npmrc.File() if err != nil { return Result{}, err } - npmrcPath = filepath.Join(workspace, "npmrc") + npmrcPath = filepath.Join(within, ".npmrc") if err := os.WriteFile(npmrcPath, []byte(content), 0o600); err != nil { return Result{}, fmt.Errorf("cannot write the package-registry credential for the build: %w", err) } @@ -297,6 +300,24 @@ func against(within string, manifest catalogue.Manifest) []string { // setting somebody has to find. const ManifestName = "module.json" +// wantsPackages reports whether this module's build resolves anything from the mesh's package +// registry, so the credential is written into its context only then. A package artifact always +// does; an image does when its Dockerfile names .npmrc — the file it would COPY to authenticate. +func wantsPackages(manifest catalogue.Manifest, within string) bool { + for _, a := range manifest.Build.Artifacts { + switch a.Kind { + case catalogue.ArtifactPackage: + return true + case catalogue.ArtifactImage: + raw, err := os.ReadFile(filepath.Join(within, a.From)) + if err == nil && strings.Contains(string(raw), ".npmrc") { + return true + } + } + } + return false +} + func one(ctx context.Context, run Runner, publish Publisher, module, tree, commit string, a catalogue.Artifact, args []string, held map[string]string, npmrc string, say func(step, format string, args ...any)) (catalogue.Built, error) { @@ -329,12 +350,10 @@ func one(ctx context.Context, run Runner, publish Publisher, invocation = append(invocation, "--target", a.Target) } if npmrc != "" { - // Given to the build as a buildkit secret, so a RUN that needs the package registry mounts - // it at that step and it is in no image layer. A Dockerfile that does not ask for it is - // unaffected; the secret is simply not read (novox/hq ADR 0076). - invocation = append(invocation, "--secret", "id=npmrc,src="+npmrc) // Host network for the build, so a RUN reaching the package registry finds it where the - // binding says it is — the machine's own loopback, where the registry answers. + // binding says it is — the machine's own loopback, where the registry answers. The + // credential itself is in the context as .npmrc, COPY'd by a stage that is not published; + // buildkit is not required, because this machine's docker may not carry buildx. invocation = append(invocation, "--network", "host") } invocation = append(invocation, ".") diff --git a/internal/builder/packages_test.go b/internal/builder/packages_test.go index a26a287..8a8ae8c 100644 --- a/internal/builder/packages_test.go +++ b/internal/builder/packages_test.go @@ -63,10 +63,12 @@ func TestNpmrcDisabledUntilThereIsARegistry(t *testing.T) { } } -// The credential reaches an image build as a buildkit secret and never as a file inside the build -// context, because a token copied into a layer is a token published (novox/hq ADR 0076). -func TestAnImageBuildGetsTheCredentialAsASecretNotALayer(t *testing.T) { - r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch", "files/x": "y"}) +// The credential reaches an image build as an .npmrc inside the build context — for a Dockerfile to +// COPY in a stage it does not publish — and the build runs on the host network so a RUN resolving the +// registry reaches it where the binding says (novox/hq ADR 0076). Not a buildkit secret, because this +// machine's docker may carry no buildx. +func TestAnImageBuildGetsTheCredentialInTheContextAndHostNetwork(t *testing.T) { + r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch\nCOPY .npmrc ./", "files/x": "y"}) n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm/", Token: "t"} if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, n, nil); err != nil { @@ -82,33 +84,35 @@ func TestAnImageBuildGetsTheCredentialAsASecretNotALayer(t *testing.T) { if build == "" { t.Fatal("no docker build ran") } - if !strings.Contains(build, "--secret id=npmrc,src=") { - t.Fatalf("the build was not given the credential as a secret: %s", build) + if strings.Contains(build, "--secret") { + t.Fatalf("the build used a buildkit secret, which this path avoids: %s", build) } - - // The .npmrc lives under the workspace, beside the clone, never inside the source tree that is - // the docker context. + if !strings.Contains(build, "--network host") { + t.Fatalf("the build was not given the host network to reach the registry: %s", build) + } + // The .npmrc is written into the build context (the source tree), where a Dockerfile COPYs it. tree := filepath.Join(workspace, "source") - src := strings.SplitN(strings.SplitN(build, "--secret id=npmrc,src=", 2)[1], " ", 2)[0] - if strings.HasPrefix(src, tree+string(os.PathSeparator)) { - t.Fatalf("the credential file %s is inside the build context %s", src, tree) - } - if _, err := os.Stat(src); err != nil { - t.Fatalf("the credential file the build was pointed at does not exist: %v", err) + npmrc := filepath.Join(tree, ".npmrc") + if _, err := os.Stat(npmrc); err != nil { + t.Fatalf("the credential was not written into the build context: %v", err) } } -func TestAnImageBuildWithoutACredentialGetsNoSecret(t *testing.T) { +func TestAnImageBuildWithoutACredentialGetsNoHostNetwork(t *testing.T) { r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch", "files/x": "y"}) if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, Npmrc{}, nil); err != nil { t.Fatalf("the build failed: %v", err) } for _, line := range r.ran { - if strings.HasPrefix(line, "docker build") && strings.Contains(line, "--secret") { - t.Fatalf("a build with no credential was still given a secret: %s", line) + if strings.HasPrefix(line, "docker build") && (strings.Contains(line, "--secret") || strings.Contains(line, "--network host")) { + t.Fatalf("a build with no credential was still given build-network or a secret: %s", line) } } + tree := filepath.Join(workspace, "source") + if _, err := os.Stat(filepath.Join(tree, ".npmrc")); err == nil { + t.Fatal("an .npmrc was written into a build that has no credential") + } } const aPackage = `{"module":"mesh-sdk","version":"1", @@ -194,3 +198,23 @@ func TestNpmrcRefusesWhenGivenNeitherTokenNorPassword(t *testing.T) { t.Fatal("a username with no password rendered an .npmrc") } } + +func TestAnImageThatDoesNotAskForTheCredentialDoesNotGetIt(t *testing.T) { + // A Dockerfile with no .npmrc reference (like the control plane's) must build clean: no .npmrc + // in its context, no host network — so its image stays deterministic and the credential does not + // leak into a build that never resolves a mesh package. + r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch\nCOPY . .", "files/x": "y"}) + n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm/", Token: "t"} + if _, err := Build(context.Background(), r.run, r, + "https://forge.invalid/meshboard.git", "", "", workspace, nil, n, nil); err != nil { + t.Fatalf("the build failed: %v", err) + } + for _, line := range r.ran { + if strings.HasPrefix(line, "docker build") && strings.Contains(line, "--network host") { + t.Fatalf("a build that does not ask for the credential got the host network: %s", line) + } + } + if _, err := os.Stat(filepath.Join(workspace, "source", ".npmrc")); err == nil { + t.Fatal("an .npmrc was written into a build that does not reference it") + } +} -- 2.54.0 From 366840fc0dfbe9a35978adc27bb2bccfc7410c8e Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 16 Sep 2026 13:23:01 +0200 Subject: [PATCH 16/17] push --wait: optionally block until a named node applies what it was sent Opt-in (default 0, unchanged behaviour). A named push can wait until the node reports it applied exactly this declaration, so an interactive push means "the node is now what it was told". Not made the default: a push that recreates the control plane would kill the waiting command running inside it. Claude-Session: https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- cmd/mesh-control/push.go | 55 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/cmd/mesh-control/push.go b/cmd/mesh-control/push.go index 8eb13f2..1def023 100644 --- a/cmd/mesh-control/push.go +++ b/cmd/mesh-control/push.go @@ -168,6 +168,11 @@ func pushCommand(ctx context.Context, args []string) error { // eventually watches. behind := set.Bool("behind", false, "only machines whose last declaration was refused or partly failed") + // For a named node, wait until it reports applying exactly what it was sent, so `push ` + // means "this node is now what it was told" — a command right after does not race the apply + // (novox/hq ADR 0010). 0 waits for nothing, which is the old fire-and-forget. + wait := set.Duration("wait", 0, + "for a named node, how long to wait for it to report applying what it was sent (0: do not wait)") positionals, err := parseAround(set, args) if err != nil { return err @@ -292,6 +297,7 @@ func pushCommand(ctx context.Context, args []string) error { return declarationWith(ctx, open, node, plan, settings, gens, Allocating) }) + sentDigest := map[string]string{} for _, s := range sending { body, err := json.Marshal(map[string]any{"declaration": 1, "resources": s.resources}) if err != nil { @@ -306,15 +312,62 @@ func pushCommand(ctx context.Context, args []string) error { if err != nil { return err } - if err := inv.RecordSent(ctx, record.ID, digestOf(body)); err != nil { + digest := digestOf(body) + if err := inv.RecordSent(ctx, record.ID, digest); err != nil { return err } + sentDigest[s.node] = digest fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.resources)) } fmt.Printf("\n%d node(s) told\n", len(sending)) + + // A named node is a request to make THAT node current now, so it waits for the node to say it + // applied exactly this. A whole-mesh or --behind push does not wait: it is a sweep, and blocking + // on the slowest machine would hold back the report on all the others. + if *wait > 0 && len(args) == 1 { + if err := waitForApplied(ctx, inv, args[0], sentDigest[args[0]], *wait); err != nil { + return err + } + } return couldNotBeResolved(refusals, len(sending)) } +// waitForApplied blocks until the node reports it applied exactly the declaration just sent, or the +// wait runs out. A report of failure or refusal for that same declaration ends the wait at once — +// there is nothing to wait for, and the reason is the node's own. +func waitForApplied(ctx context.Context, inv *inventory.Inventory, node, digest string, wait time.Duration) error { + if digest == "" { + return nil // nothing was sent to this node + } + deadline := time.Now().Add(wait) + for { + doing, said, err := inv.DoingOf(ctx, node) + if err != nil { + return err + } + if said && doing.Declared == digest { + switch doing.Outcome { + case inventory.OutcomeApplied: + fmt.Printf("%s applied it\n", node) + return nil + case inventory.OutcomeFailed: + return fmt.Errorf("%s applied what it was sent but %d resource(s) failed", node, len(doing.Failed)) + case inventory.OutcomeRefused: + return fmt.Errorf("%s refused what it was sent: %s", node, doing.Refused) + } + } + if time.Now().After(deadline) { + return fmt.Errorf("%s did not report applying what it was sent within %s "+ + "(it may still be converging; check `status`)", node, wait) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(500 * time.Millisecond): + } + } +} + // readyNode is one machine and the declaration it would be sent. type readyNode struct { node string -- 2.54.0 From c3b88b914896b7b4f004d3f92774ee7000384730 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 16 Sep 2026 18:40:40 +0200 Subject: [PATCH 17/17] Rename mesh-control -> mesh-controller, substrate -> foundation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One name per thing, per the HQ glossary: the module/container/image/binary/repo becomes mesh-controller, the seat the-controller, and the store+broker pair the foundation (embedded base bundles, default template and example lock renamed with their go:embed directives). No behaviour change — a pure vocabulary rename. Claude-Session: https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- Dockerfile | 6 ++--- Makefile | 8 +++---- README.md | 24 +++++++++---------- cmd/mesh-builder/main.go | 4 ++-- cmd/mesh-builder/once.go | 2 +- cmd/{mesh-control => mesh-controller}/acts.go | 2 +- .../acts_test.go | 2 +- cmd/{mesh-control => mesh-controller}/api.go | 0 .../api_test.go | 0 .../board.go | 0 .../board_test.go | 2 +- .../build.go | 8 +++---- .../licence.go | 0 cmd/{mesh-control => mesh-controller}/main.go | 18 +++++++------- .../mesh_for_test.go | 8 +++---- .../modules.go | 12 +++++----- .../network.go | 6 ++--- .../network_test.go | 2 +- .../nodes.go | 8 +++---- .../nodes_test.go | 0 cmd/{mesh-control => mesh-controller}/plan.go | 14 +++++------ cmd/{mesh-control => mesh-controller}/push.go | 8 +++---- .../push_test.go | 0 .../readable.go | 0 .../readable_test.go | 2 +- .../rotate.go | 0 .../secret.go | 0 .../secret_test.go | 0 .../status.go | 4 ++-- .../status_test.go | 0 .../stores.go | 8 +++---- .../upgrades.go | 4 ++-- examples/modules/modules_test.go | 4 ++-- examples/postgres-provisioner/where_test.go | 2 +- go.mod | 2 +- internal/broker/agreement_test.go | 4 ++-- internal/broker/management.go | 6 ++--- internal/broker/module_account_test.go | 2 +- internal/builder/builder.go | 2 +- internal/builder/builder_test.go | 2 +- internal/builder/standing_on_test.go | 2 +- internal/catalogue/bootstrap_cycle_test.go | 2 +- internal/catalogue/co_located_test.go | 2 +- internal/catalogue/declaration.go | 10 ++++---- internal/catalogue/filtering.go | 10 ++++---- ...e_test.go => filtering_foundation_test.go} | 6 ++--- internal/catalogue/manifest.go | 2 +- internal/catalogue/provided_test.go | 4 ++-- internal/identity/identity.go | 2 +- internal/identity/identity_test.go | 2 +- internal/inventory/buildinput_test.go | 2 +- internal/inventory/catalogue.go | 2 +- internal/inventory/catalogue_test.go | 2 +- internal/inventory/forget_test.go | 2 +- internal/inventory/fortest.go | 2 +- internal/inventory/inventory.go | 2 +- .../0017-what-a-machine-already-holds.sql | 2 +- internal/inventory/nodes.go | 2 +- internal/inventory/ports.go | 2 +- internal/inventory/ports_test.go | 2 +- internal/inventory/secrets.go | 2 +- internal/inventory/secrets_test.go | 2 +- internal/licences/adapters/adapters.go | 2 +- internal/licences/fortest.go | 2 +- internal/licences/licences.go | 6 ++--- internal/licences/refresh_test.go | 4 ++-- internal/licences/submitrefresh_test.go | 2 +- internal/link/enrol_shape_test.go | 8 +++---- internal/link/enrolment.go | 6 ++--- internal/link/heard_test.go | 6 ++--- internal/link/protocol.go | 2 +- internal/overlay/declaration.go | 2 +- internal/overlay/generator.go | 2 +- internal/overlay/opens_test.go | 2 +- internal/secrets/sealedbox_xcheck_test.go | 2 +- .../testdata/module-sealedbox-fixture.json | 2 +- module.json | 24 +++++++++---------- 77 files changed, 157 insertions(+), 157 deletions(-) rename cmd/{mesh-control => mesh-controller}/acts.go (99%) rename cmd/{mesh-control => mesh-controller}/acts_test.go (98%) rename cmd/{mesh-control => mesh-controller}/api.go (100%) rename cmd/{mesh-control => mesh-controller}/api_test.go (100%) rename cmd/{mesh-control => mesh-controller}/board.go (100%) rename cmd/{mesh-control => mesh-controller}/board_test.go (99%) rename cmd/{mesh-control => mesh-controller}/build.go (98%) rename cmd/{mesh-control => mesh-controller}/licence.go (100%) rename cmd/{mesh-control => mesh-controller}/main.go (93%) rename cmd/{mesh-control => mesh-controller}/mesh_for_test.go (94%) rename cmd/{mesh-control => mesh-controller}/modules.go (97%) rename cmd/{mesh-control => mesh-controller}/network.go (98%) rename cmd/{mesh-control => mesh-controller}/network_test.go (97%) rename cmd/{mesh-control => mesh-controller}/nodes.go (98%) rename cmd/{mesh-control => mesh-controller}/nodes_test.go (100%) rename cmd/{mesh-control => mesh-controller}/plan.go (99%) rename cmd/{mesh-control => mesh-controller}/push.go (98%) rename cmd/{mesh-control => mesh-controller}/push_test.go (100%) rename cmd/{mesh-control => mesh-controller}/readable.go (100%) rename cmd/{mesh-control => mesh-controller}/readable_test.go (98%) rename cmd/{mesh-control => mesh-controller}/rotate.go (100%) rename cmd/{mesh-control => mesh-controller}/secret.go (100%) rename cmd/{mesh-control => mesh-controller}/secret_test.go (100%) rename cmd/{mesh-control => mesh-controller}/status.go (98%) rename cmd/{mesh-control => mesh-controller}/status_test.go (100%) rename cmd/{mesh-control => mesh-controller}/stores.go (95%) rename cmd/{mesh-control => mesh-controller}/upgrades.go (98%) rename internal/catalogue/{filtering_substrate_test.go => filtering_foundation_test.go} (91%) diff --git a/Dockerfile b/Dockerfile index 4e398c0..9bc18a5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,13 +22,13 @@ COPY . . ARG VERSION=development RUN CGO_ENABLED=0 go build -trimpath \ -ldflags "-s -w -X main.version=${VERSION}" \ - -o /mesh-control ./cmd/mesh-control + -o /mesh-controller ./cmd/mesh-controller FROM scratch -COPY --from=build /mesh-control /mesh-control +COPY --from=build /mesh-controller /mesh-controller # Numeric because there is no /etc/passwd to look a name up in. Nothing here needs to be root: # it opens outbound connections and writes nothing to its own filesystem. USER 65534:65534 -ENTRYPOINT ["/mesh-control"] +ENTRYPOINT ["/mesh-controller"] diff --git a/Makefile b/Makefile index bb6bc6f..2e21062 100644 --- a/Makefile +++ b/Makefile @@ -12,20 +12,20 @@ LDFLAGS := -s -w -X main.version=$(VERSION) # touches a database anybody else is using. Override PG_PORT if this one is taken -- the first # port chosen was already serving something that had been up for six days. PG_PORT ?= 55532 -PG_CONTAINER ?= mesh-control-check +PG_CONTAINER ?= mesh-controller-check PG_IMAGE ?= postgres:17-alpine export MESH_TEST_POSTGRES ?= postgres://postgres:check@127.0.0.1:$(PG_PORT)/postgres?sslmode=disable .PHONY: build image check test vet fmt postgres postgres-stop clean build: - CGO_ENABLED=0 go build -trimpath -ldflags '$(LDFLAGS)' -o build/mesh-control ./cmd/mesh-control + CGO_ENABLED=0 go build -trimpath -ldflags '$(LDFLAGS)' -o build/mesh-controller ./cmd/mesh-controller # Tagged 'development' as well as by version, because the lab places images by name and a # scenario naming a version would have to be edited on every build. The version tag is what a # real bundle pins. -IMAGE ?= mesh-control:$(VERSION) -DEV_TAG ?= mesh-control:development +IMAGE ?= mesh-controller:$(VERSION) +DEV_TAG ?= mesh-controller:development image: docker build --build-arg VERSION=$(VERSION) -t $(IMAGE) -t $(DEV_TAG) . diff --git a/README.md b/README.md index 287af4a..427ed83 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# mesh-control +# mesh-controller **Tier 2 of Novox Mesh — the control plane.** Everything that needs to know about more than one node. @@ -31,17 +31,17 @@ argument that is not settled there. | the interface every surface speaks to | not built; its shape is not decided | ``` -mesh-control migrate bring each context's schema up to date -mesh-control node add create a node record -mesh-control node list the nodes this mesh knows about -mesh-control token issue --node a one-time right to join, for an existing record -mesh-control token issue --new create the record and issue for it -mesh-control identity show this control plane's signing key -mesh-control broker show where the broker is, and what to expect there -mesh-control version what this binary is +mesh-controller migrate bring each context's schema up to date +mesh-controller node add create a node record +mesh-controller node list the nodes this mesh knows about +mesh-controller token issue --node a one-time right to join, for an existing record +mesh-controller token issue --new create the record and issue for it +mesh-controller identity show this control plane's signing key +mesh-controller broker show where the broker is, and what to expect there +mesh-controller version what this binary is ``` -`migrate` is **step 3 of the substrate bootstrap** — the step the first node cannot get past, run +`migrate` is **step 3 of the foundation bootstrap** — the step the first node cannot get past, run against a database raised moments earlier from the bundle the host carries. ### Tokens, and what they are missing @@ -176,7 +176,7 @@ without its neighbour checked out is a repository nobody can build. **What this mesh sends, read by the host that receives it:** ``` -mesh-control: ./build/mesh-control plan --json > /tmp/d.json +mesh-controller: ./build/mesh-controller plan --json > /tmp/d.json mesh-host: MESH_EMITTED=/tmp/d.json go test ./internal/declaration/ -v ``` @@ -184,7 +184,7 @@ mesh-host: MESH_EMITTED=/tmp/d.json go test ./internal/declaration/ -v ``` mesh-host: MESH_ENROL_OUT=/tmp/enrol.json go test ./internal/link/ -mesh-control: MESH_ENROL=/tmp/enrol.json make check +mesh-controller: MESH_ENROL=/tmp/enrol.json make check ``` The second does more than compare shapes: it seals something to the key that arrived and opens it diff --git a/cmd/mesh-builder/main.go b/cmd/mesh-builder/main.go index 9a5c9e2..5ec294a 100644 --- a/cmd/mesh-builder/main.go +++ b/cmd/mesh-builder/main.go @@ -32,8 +32,8 @@ import ( amqp "github.com/rabbitmq/amqp091-go" - "github.com/novox/mesh-control/internal/builder" - "github.com/novox/mesh-control/internal/link" + "github.com/novox/mesh-controller/internal/builder" + "github.com/novox/mesh-controller/internal/link" ) // version is set at build time. diff --git a/cmd/mesh-builder/once.go b/cmd/mesh-builder/once.go index 8423c67..1ea9da7 100644 --- a/cmd/mesh-builder/once.go +++ b/cmd/mesh-builder/once.go @@ -9,7 +9,7 @@ import ( "os" "strings" - "github.com/novox/mesh-control/internal/builder" + "github.com/novox/mesh-controller/internal/builder" ) // buildOnce is the builder doing one build and stopping, with no broker and no mesh. diff --git a/cmd/mesh-control/acts.go b/cmd/mesh-controller/acts.go similarity index 99% rename from cmd/mesh-control/acts.go rename to cmd/mesh-controller/acts.go index 7911024..4a0abe8 100644 --- a/cmd/mesh-control/acts.go +++ b/cmd/mesh-controller/acts.go @@ -6,7 +6,7 @@ import ( "sort" "strings" - "github.com/novox/mesh-control/internal/catalogue" + "github.com/novox/mesh-controller/internal/catalogue" ) // The things the mesh can be asked to do, separated from how it was asked. diff --git a/cmd/mesh-control/acts_test.go b/cmd/mesh-controller/acts_test.go similarity index 98% rename from cmd/mesh-control/acts_test.go rename to cmd/mesh-controller/acts_test.go index 7cbc642..a2e99a6 100644 --- a/cmd/mesh-control/acts_test.go +++ b/cmd/mesh-controller/acts_test.go @@ -4,7 +4,7 @@ import ( "strings" "testing" - "github.com/novox/mesh-control/internal/catalogue" + "github.com/novox/mesh-controller/internal/catalogue" ) // What an assignment says about the machines it was not about. diff --git a/cmd/mesh-control/api.go b/cmd/mesh-controller/api.go similarity index 100% rename from cmd/mesh-control/api.go rename to cmd/mesh-controller/api.go diff --git a/cmd/mesh-control/api_test.go b/cmd/mesh-controller/api_test.go similarity index 100% rename from cmd/mesh-control/api_test.go rename to cmd/mesh-controller/api_test.go diff --git a/cmd/mesh-control/board.go b/cmd/mesh-controller/board.go similarity index 100% rename from cmd/mesh-control/board.go rename to cmd/mesh-controller/board.go diff --git a/cmd/mesh-control/board_test.go b/cmd/mesh-controller/board_test.go similarity index 99% rename from cmd/mesh-control/board_test.go rename to cmd/mesh-controller/board_test.go index 85462c3..ea8029d 100644 --- a/cmd/mesh-control/board_test.go +++ b/cmd/mesh-controller/board_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/novox/mesh-control/internal/inventory" + "github.com/novox/mesh-controller/internal/inventory" ) // Refused and failed stay distinct all the way to the page. diff --git a/cmd/mesh-control/build.go b/cmd/mesh-controller/build.go similarity index 98% rename from cmd/mesh-control/build.go rename to cmd/mesh-controller/build.go index 008adee..1506400 100644 --- a/cmd/mesh-control/build.go +++ b/cmd/mesh-controller/build.go @@ -12,10 +12,10 @@ import ( "strings" "time" - "github.com/novox/mesh-control/internal/broker" - "github.com/novox/mesh-control/internal/catalogue" - "github.com/novox/mesh-control/internal/inventory" - "github.com/novox/mesh-control/internal/link" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" ) // asking a build machine for a module, and what came back. diff --git a/cmd/mesh-control/licence.go b/cmd/mesh-controller/licence.go similarity index 100% rename from cmd/mesh-control/licence.go rename to cmd/mesh-controller/licence.go diff --git a/cmd/mesh-control/main.go b/cmd/mesh-controller/main.go similarity index 93% rename from cmd/mesh-control/main.go rename to cmd/mesh-controller/main.go index f69e90b..bcfb385 100644 --- a/cmd/mesh-control/main.go +++ b/cmd/mesh-controller/main.go @@ -1,9 +1,9 @@ -// Command mesh-control is the control plane: everything that needs to know about more than one +// Command mesh-controller is the control plane: everything that needs to know about more than one // node (novox/hq ADR 0006). // // It runs as one process holding several contexts, each owning its own store. Today it holds one, // `inventory`, and does one thing with it — brings its schema up to date, which is step 3 of the -// bootstrap in novox/hq 07-the-substrate and the step the first node cannot get past without. +// bootstrap in novox/hq 07-the-foundation and the step the first node cannot get past without. package main import ( @@ -14,11 +14,11 @@ import ( "os/signal" "syscall" - "github.com/novox/mesh-control/internal/identity" - "github.com/novox/mesh-control/internal/inventory" - "github.com/novox/mesh-control/internal/licences" - "github.com/novox/mesh-control/internal/link" - "github.com/novox/mesh-control/internal/store" + "github.com/novox/mesh-controller/internal/identity" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/licences" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/store" ) // version is stamped at link time. Unset in a development build, and it says so rather than @@ -40,7 +40,7 @@ var held = []struct { func main() { if err := run(); err != nil { - fmt.Fprintf(os.Stderr, "mesh-control: %v\n", err) + fmt.Fprintf(os.Stderr, "mesh-controller: %v\n", err) os.Exit(1) } } @@ -119,7 +119,7 @@ func run() error { } func usage() { - fmt.Fprint(os.Stderr, `mesh-control — the control plane + fmt.Fprint(os.Stderr, `mesh-controller — the control plane migrate bring each context's schema up to date node add create a node record diff --git a/cmd/mesh-control/mesh_for_test.go b/cmd/mesh-controller/mesh_for_test.go similarity index 94% rename from cmd/mesh-control/mesh_for_test.go rename to cmd/mesh-controller/mesh_for_test.go index 43ff46f..35f3627 100644 --- a/cmd/mesh-control/mesh_for_test.go +++ b/cmd/mesh-controller/mesh_for_test.go @@ -8,10 +8,10 @@ import ( "fmt" "testing" - "github.com/novox/mesh-control/internal/catalogue" - "github.com/novox/mesh-control/internal/inventory" - "github.com/novox/mesh-control/internal/licences" - "github.com/novox/mesh-control/internal/overlay" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/licences" + "github.com/novox/mesh-controller/internal/overlay" ) // A mesh a command can be run against. diff --git a/cmd/mesh-control/modules.go b/cmd/mesh-controller/modules.go similarity index 97% rename from cmd/mesh-control/modules.go rename to cmd/mesh-controller/modules.go index cc09f3e..4a12a0b 100644 --- a/cmd/mesh-control/modules.go +++ b/cmd/mesh-controller/modules.go @@ -12,10 +12,10 @@ import ( "sort" "strings" - "github.com/novox/mesh-control/internal/broker" - "github.com/novox/mesh-control/internal/catalogue" - "github.com/novox/mesh-control/internal/inventory" - "github.com/novox/mesh-control/internal/overlay" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/overlay" ) // the catalogue: what exists, what is assigned, and how it is configured. @@ -257,7 +257,7 @@ func moduleCommand(ctx context.Context, args []string) error { if err != nil { return err } - // The substrate owns the bus; make sure it exists before a module binds onto it. + // The foundation owns the bus; make sure it exists before a module binds onto it. if err := management.EnsureEventExchanges(ctx); err != nil { return err } @@ -271,7 +271,7 @@ func moduleCommand(ctx context.Context, args []string) error { if err != nil { return err } - // A consumer's queue, with its dead-letter, is the substrate's to declare — its own account + // A consumer's queue, with its dead-letter, is the foundation's to declare — its own account // may not (ADR 0043). Made now, so it exists before the module binds onto it. if len(m.Consumes) > 0 { if err := management.EnsureModuleQueue(ctx, *forNode, module); err != nil { diff --git a/cmd/mesh-control/network.go b/cmd/mesh-controller/network.go similarity index 98% rename from cmd/mesh-control/network.go rename to cmd/mesh-controller/network.go index c031f5f..7c561b8 100644 --- a/cmd/mesh-control/network.go +++ b/cmd/mesh-controller/network.go @@ -10,9 +10,9 @@ import ( "strings" "time" - "github.com/novox/mesh-control/internal/catalogue" - "github.com/novox/mesh-control/internal/inventory" - "github.com/novox/mesh-control/internal/overlay" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/overlay" ) // the private network: who is on it, where, and what they are called. diff --git a/cmd/mesh-control/network_test.go b/cmd/mesh-controller/network_test.go similarity index 97% rename from cmd/mesh-control/network_test.go rename to cmd/mesh-controller/network_test.go index 3c354c6..789fe8a 100644 --- a/cmd/mesh-control/network_test.go +++ b/cmd/mesh-controller/network_test.go @@ -5,7 +5,7 @@ import ( "strings" "testing" - "github.com/novox/mesh-control/internal/inventory" + "github.com/novox/mesh-controller/internal/inventory" ) // placementOf is what the mesh holds about where one node is. diff --git a/cmd/mesh-control/nodes.go b/cmd/mesh-controller/nodes.go similarity index 98% rename from cmd/mesh-control/nodes.go rename to cmd/mesh-controller/nodes.go index 6d021d0..e6dcfe0 100644 --- a/cmd/mesh-control/nodes.go +++ b/cmd/mesh-controller/nodes.go @@ -8,10 +8,10 @@ import ( "strings" "time" - "github.com/novox/mesh-control/internal/broker" - "github.com/novox/mesh-control/internal/inventory" - "github.com/novox/mesh-control/internal/link" - "github.com/novox/mesh-control/internal/token" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/token" ) // what a machine is, and what it is allowed to be told. diff --git a/cmd/mesh-control/nodes_test.go b/cmd/mesh-controller/nodes_test.go similarity index 100% rename from cmd/mesh-control/nodes_test.go rename to cmd/mesh-controller/nodes_test.go diff --git a/cmd/mesh-control/plan.go b/cmd/mesh-controller/plan.go similarity index 99% rename from cmd/mesh-control/plan.go rename to cmd/mesh-controller/plan.go index a37e1a9..5aa92b6 100644 --- a/cmd/mesh-control/plan.go +++ b/cmd/mesh-controller/plan.go @@ -9,10 +9,10 @@ import ( "sort" "strings" - "github.com/novox/mesh-control/internal/broker" - "github.com/novox/mesh-control/internal/catalogue" - "github.com/novox/mesh-control/internal/inventory" - "github.com/novox/mesh-control/internal/licences" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/licences" "net" "strconv" ) @@ -453,11 +453,11 @@ func declarationWith(ctx context.Context, open *stores, node string, // The ports the mesh itself needs open, which no module declares. Read from the broker this // control plane was told about rather than written down twice: the address a node is handed in // its token and the port its machine must accept on are the same fact. - var substrate []int + var foundation []int if b, err := broker.FromEnvironment(); err == nil { if _, port, err := net.SplitHostPort(b.Address); err == nil { if n, err := strconv.Atoi(port); err == nil { - substrate = append(substrate, n) + foundation = append(foundation, n) } } } @@ -465,7 +465,7 @@ func declarationWith(ctx context.Context, open *stores, node string, return plan.Declaration(catalogue.Rendering{ Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports, Certificate: certificate, Authority: authority, Mesh: private, Names: names, - Substrate: substrate}) + Foundation: foundation}) } // routeNamesInTheMesh is every routed name and the address of the node that serves it (novox/hq diff --git a/cmd/mesh-control/push.go b/cmd/mesh-controller/push.go similarity index 98% rename from cmd/mesh-control/push.go rename to cmd/mesh-controller/push.go index 1def023..656d683 100644 --- a/cmd/mesh-control/push.go +++ b/cmd/mesh-controller/push.go @@ -12,10 +12,10 @@ import ( "strings" "time" - "github.com/novox/mesh-control/internal/broker" - "github.com/novox/mesh-control/internal/catalogue" - "github.com/novox/mesh-control/internal/inventory" - "github.com/novox/mesh-control/internal/link" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" ) // reportUnhostable says which of a node's assigned modules the machine cannot run, once per push. diff --git a/cmd/mesh-control/push_test.go b/cmd/mesh-controller/push_test.go similarity index 100% rename from cmd/mesh-control/push_test.go rename to cmd/mesh-controller/push_test.go diff --git a/cmd/mesh-control/readable.go b/cmd/mesh-controller/readable.go similarity index 100% rename from cmd/mesh-control/readable.go rename to cmd/mesh-controller/readable.go diff --git a/cmd/mesh-control/readable_test.go b/cmd/mesh-controller/readable_test.go similarity index 98% rename from cmd/mesh-control/readable_test.go rename to cmd/mesh-controller/readable_test.go index 2091d2a..fd7dedc 100644 --- a/cmd/mesh-control/readable_test.go +++ b/cmd/mesh-controller/readable_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/novox/mesh-control/internal/inventory" + "github.com/novox/mesh-controller/internal/inventory" ) // The shape something other than a person reads. diff --git a/cmd/mesh-control/rotate.go b/cmd/mesh-controller/rotate.go similarity index 100% rename from cmd/mesh-control/rotate.go rename to cmd/mesh-controller/rotate.go diff --git a/cmd/mesh-control/secret.go b/cmd/mesh-controller/secret.go similarity index 100% rename from cmd/mesh-control/secret.go rename to cmd/mesh-controller/secret.go diff --git a/cmd/mesh-control/secret_test.go b/cmd/mesh-controller/secret_test.go similarity index 100% rename from cmd/mesh-control/secret_test.go rename to cmd/mesh-controller/secret_test.go diff --git a/cmd/mesh-control/status.go b/cmd/mesh-controller/status.go similarity index 98% rename from cmd/mesh-control/status.go rename to cmd/mesh-controller/status.go index 86bd83a..2ebdd1f 100644 --- a/cmd/mesh-control/status.go +++ b/cmd/mesh-controller/status.go @@ -8,8 +8,8 @@ import ( "strings" "time" - "github.com/novox/mesh-control/internal/inventory" - "github.com/novox/mesh-control/internal/overlay" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/overlay" ) // is anything broken, is anything not answering, is anything out of date. diff --git a/cmd/mesh-control/status_test.go b/cmd/mesh-controller/status_test.go similarity index 100% rename from cmd/mesh-control/status_test.go rename to cmd/mesh-controller/status_test.go diff --git a/cmd/mesh-control/stores.go b/cmd/mesh-controller/stores.go similarity index 95% rename from cmd/mesh-control/stores.go rename to cmd/mesh-controller/stores.go index 5c467b2..8a46256 100644 --- a/cmd/mesh-control/stores.go +++ b/cmd/mesh-controller/stores.go @@ -5,10 +5,10 @@ import ( "fmt" "time" - "github.com/novox/mesh-control/internal/identity" - "github.com/novox/mesh-control/internal/inventory" - "github.com/novox/mesh-control/internal/licences" - "github.com/novox/mesh-control/internal/store" + "github.com/novox/mesh-controller/internal/identity" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/licences" + "github.com/novox/mesh-controller/internal/store" ) // reaching each context's store, which no other context may touch. diff --git a/cmd/mesh-control/upgrades.go b/cmd/mesh-controller/upgrades.go similarity index 98% rename from cmd/mesh-control/upgrades.go rename to cmd/mesh-controller/upgrades.go index 2823d0a..63b5769 100644 --- a/cmd/mesh-control/upgrades.go +++ b/cmd/mesh-controller/upgrades.go @@ -7,8 +7,8 @@ import ( "fmt" "strings" - "github.com/novox/mesh-control/internal/inventory" - "github.com/novox/mesh-control/internal/link" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" ) // following acts on what the catalogue announces. diff --git a/examples/modules/modules_test.go b/examples/modules/modules_test.go index 72d93c1..bd60453 100644 --- a/examples/modules/modules_test.go +++ b/examples/modules/modules_test.go @@ -10,8 +10,8 @@ import ( "strings" "testing" - "github.com/novox/mesh-control/internal/catalogue" - "github.com/novox/mesh-control/internal/overlay" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/overlay" ) // The examples are manifests, so the thing to check is that the catalogue accepts them. diff --git a/examples/postgres-provisioner/where_test.go b/examples/postgres-provisioner/where_test.go index 534afae..1097ae2 100644 --- a/examples/postgres-provisioner/where_test.go +++ b/examples/postgres-provisioner/where_test.go @@ -6,7 +6,7 @@ import ( "strings" "testing" - "github.com/novox/mesh-control/internal/catalogue" + "github.com/novox/mesh-controller/internal/catalogue" ) // The password comes from a file, because that is how the mesh delivers one. diff --git a/go.mod b/go.mod index 2cee974..a827fad 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/novox/mesh-control +module github.com/novox/mesh-controller go 1.25.0 diff --git a/internal/broker/agreement_test.go b/internal/broker/agreement_test.go index f2360e8..00f692a 100644 --- a/internal/broker/agreement_test.go +++ b/internal/broker/agreement_test.go @@ -4,8 +4,8 @@ import ( "regexp" "testing" - "github.com/novox/mesh-control/internal/broker" - "github.com/novox/mesh-control/internal/link" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/link" ) // The names are written twice, so a test keeps them agreeing. diff --git a/internal/broker/management.go b/internal/broker/management.go index 0277d61..684d0a1 100644 --- a/internal/broker/management.go +++ b/internal/broker/management.go @@ -68,7 +68,7 @@ const ExchangeName = "mesh" const BuildQueueName = "builds" // The events bus (novox/hq ADR 0042): one topic exchange every event rides, a second for tool -// RPC kept apart, and a dead-letter home for a poison event. The substrate owns these — a module's +// RPC kept apart, and a dead-letter home for a poison event. The foundation owns these — a module's // account cannot declare them, only bind its own queue to the events one. const ( EventsExchangeName = "mesh.events" @@ -151,7 +151,7 @@ func (m *Management) CreateModuleAccount(ctx context.Context, node, module, pass } // EnsureModuleQueue declares a consuming module's queue with its dead-letter exchange, idempotently. -// The substrate declares it because a scoped module account may not: the broker refuses a queue with +// The foundation declares it because a scoped module account may not: the broker refuses a queue with // a dead-letter exchange to a non-administrator (novox/hq ADR 0043), so a consumer passively checks // the queue the mesh made rather than declaring its own. func (m *Management) EnsureModuleQueue(ctx context.Context, node, module string) error { @@ -166,7 +166,7 @@ func (m *Management) EnsureModuleQueue(ctx context.Context, node, module string) } // EnsureEventExchanges declares the bus's exchanges and the dead-letter home, idempotently. The -// substrate owns them (a module's account may not declare an exchange), and a dead-letter exchange +// foundation owns them (a module's account may not declare an exchange), and a dead-letter exchange // with no queue behind it drops what it receives — so a durable queue bound to `#` retains a poison // event for inspection, which is the whole reason the trail exists. func (m *Management) EnsureEventExchanges(ctx context.Context) error { diff --git a/internal/broker/module_account_test.go b/internal/broker/module_account_test.go index 05b4f7d..2f401fc 100644 --- a/internal/broker/module_account_test.go +++ b/internal/broker/module_account_test.go @@ -9,7 +9,7 @@ import ( "strings" "testing" - "github.com/novox/mesh-control/internal/broker" + "github.com/novox/mesh-controller/internal/broker" ) // The audit logger consumes everything and emits nothing. Its account must let it declare and read diff --git a/internal/builder/builder.go b/internal/builder/builder.go index a9f7acf..a174d48 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -17,7 +17,7 @@ import ( "strings" "time" - "github.com/novox/mesh-control/internal/catalogue" + "github.com/novox/mesh-controller/internal/catalogue" ) // Turning a repository into artifacts the mesh can pin. diff --git a/internal/builder/builder_test.go b/internal/builder/builder_test.go index 6938d71..c04ba87 100644 --- a/internal/builder/builder_test.go +++ b/internal/builder/builder_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/novox/mesh-control/internal/catalogue" + "github.com/novox/mesh-controller/internal/catalogue" ) // A repository becoming artifacts the mesh can pin. diff --git a/internal/builder/standing_on_test.go b/internal/builder/standing_on_test.go index b4d3e1f..5cee3cd 100644 --- a/internal/builder/standing_on_test.go +++ b/internal/builder/standing_on_test.go @@ -4,7 +4,7 @@ import ( "strings" "testing" - "github.com/novox/mesh-control/internal/catalogue" + "github.com/novox/mesh-controller/internal/catalogue" ) // A module naming a base the mesh has not built is refused, and the refusal names what is missing. diff --git a/internal/catalogue/bootstrap_cycle_test.go b/internal/catalogue/bootstrap_cycle_test.go index e37ab7f..9cbbd67 100644 --- a/internal/catalogue/bootstrap_cycle_test.go +++ b/internal/catalogue/bootstrap_cycle_test.go @@ -51,7 +51,7 @@ func TestAModuleThatDoesNotProvideTheStoreStillBuilds(t *testing.T) { // A mapping that names an address still names the port, and the machine side is still the middle. // -// The substrate bundle writes "127.0.0.1:5432:5432" today, so the shape is not hypothetical. +// The foundation bundle writes "127.0.0.1:5432:5432" today, so the shape is not hypothetical. // Found in review: the first cut split on the first colon, read "127.0.0.1" as the machine port, // failed to parse it, and silently skipped the mapping — which put the filter back on the // declared port, the exact fault MachineSide was written to end. diff --git a/internal/catalogue/co_located_test.go b/internal/catalogue/co_located_test.go index 7fac677..e667917 100644 --- a/internal/catalogue/co_located_test.go +++ b/internal/catalogue/co_located_test.go @@ -62,7 +62,7 @@ func routeProxy() Manifest { // A co-located provider's served VALUES reach its consumer, not just its served keys. // // The mesh walks a provider on ANOTHER machine and settles what it serves with that node's settings -// layers before offering it (cmd/mesh-control plan.go, theRestOfTheMesh). A provider on the +// layers before offering it (cmd/mesh-controller plan.go, theRestOfTheMesh). A provider on the // consumer's own machine was never settled at all: resolve.go's servedHere and declaration.go's // here() both read the manifest and stop there. So a served value the operator supplied — the one // kind of value a manifest cannot carry, because it is different on every mesh — arrived as the diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index c99cb26..2204a62 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -89,11 +89,11 @@ type Rendering struct { // a fact about the mesh, and resolution answers questions about one machine. Mesh []string - // Substrate is the ports the mesh itself needs reachable on every machine, which no module - // declares because the substrate is not a module (novox/hq 04-ISSUES/051 and 052). The broker + // Foundation is the ports the mesh itself needs reachable on every machine, which no module + // declares because the foundation is not a module (novox/hq 04-ISSUES/051 and 052). The broker // is the one that matters: a machine dials it to enrol, and a firewall derived only from // modules closes it. - Substrate []int + Foundation []int // Names is every machine's internal name and its address, for containers to be given. // @@ -213,7 +213,7 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) { if err != nil { return nil, err } - filtering := AsNftables(rules, with.Mesh, r.PublicDomain != "", with.Substrate) + filtering := AsNftables(rules, with.Mesh, r.PublicDomain != "", with.Foundation) var out []map[string]any for _, m := range r.Modules { @@ -821,7 +821,7 @@ func here(r Resolution, requirement string, with Rendering) (*Needed, error) { // // **The one derivation, so the two arrangements cannot disagree.** For a provider elsewhere the // control plane walks that node, reads its manifest with that machine's port assignments, and -// settles the result with that node's settings layers before offering it (cmd/mesh-control plan.go, +// settles the result with that node's settings layers before offering it (cmd/mesh-controller plan.go, // theRestOfTheMesh). A provider on the consumer's own machine never passes through that walk, so // every step of it has to be repeated here — and each step that was not repeated was a promise the // co-located arrangement quietly broke: first the port (novox/hq 04-ISSUES/038), then the settled diff --git a/internal/catalogue/filtering.go b/internal/catalogue/filtering.go index ba1d6dc..773a434 100644 --- a/internal/catalogue/filtering.go +++ b/internal/catalogue/filtering.go @@ -217,10 +217,10 @@ const SSHPort = 22 // `outward` says this machine is reachable from outside the mesh, which is the only thing that // decides whether ssh is answered there as well as on the private network. // -// `substrate` is the ports the MESH ITSELF needs reachable, which no module declares. +// `foundation` is the ports the MESH ITSELF needs reachable, which no module declares. // // **Everything else in this file is derived from what modules say they listen on, and the -// substrate is not a module** (novox/hq 04-ISSUES/051). So the broker — the port every machine +// foundation is not a module** (novox/hq 04-ISSUES/051). So the broker — the port every machine // dials to enrol and to receive every declaration it is ever sent — was absent from the ruleset, // and nothing noticed: a mesh of one never dials its own broker across the network. The first // machine to join a firewalled anchor is refused by the packet filter during enrolment, before @@ -229,7 +229,7 @@ const SSHPort = 22 // It is a floor for the same reason ssh is. A machine nobody can reach is a machine nobody can // repair; a machine the mesh cannot reach is a machine the mesh cannot manage. Neither is a thing // any module asks for, and neither may be derived away. -func AsNftables(rules []Rule, mesh []string, outward bool, substrate []int) string { +func AsNftables(rules []Rule, mesh []string, outward bool, foundation []int) string { var b strings.Builder b.WriteString("# Computed by the mesh from what is assigned to this node.\n") b.WriteString("# Edits are lost on the next declaration; change a module's listens instead.\n\n") @@ -297,9 +297,9 @@ func AsNftables(rules []Rule, mesh []string, outward bool, substrate []int) stri // Not narrowed to the private network, because the machines that need this most are the ones // not on it yet: a node enrols BEFORE it has an address here, over the ordinary network, and a // rule allowing only the private network would close the door being knocked on. - if len(substrate) > 0 { + if len(foundation) > 0 { b.WriteString("\n\t\t# the mesh's own — never derived, never closed\n") - for _, port := range substrate { + for _, port := range foundation { b.WriteString(fmt.Sprintf("\t\ttcp dport %d accept\n", port)) } } diff --git a/internal/catalogue/filtering_substrate_test.go b/internal/catalogue/filtering_foundation_test.go similarity index 91% rename from internal/catalogue/filtering_substrate_test.go rename to internal/catalogue/filtering_foundation_test.go index 7df5680..96d4263 100644 --- a/internal/catalogue/filtering_substrate_test.go +++ b/internal/catalogue/filtering_foundation_test.go @@ -7,7 +7,7 @@ import ( // The mesh's own ports survive a ruleset derived from modules that do not mention them. // -// **The firewall is computed from what modules declare they listen on, and the substrate is not a +// **The firewall is computed from what modules declare they listen on, and the foundation is not a // module** (novox/hq 04-ISSUES/052). So the broker's port — the one every machine dials to enrol // and to receive every declaration it is ever sent — was absent from every ruleset the mesh ever // generated, and nothing caught it: a mesh of one never dials its own broker across the network, @@ -37,12 +37,12 @@ func TestTheBrokersPortIsOpenedThoughNoModuleDeclaresIt(t *testing.T) { // And a mesh that was never told about a broker still gets a ruleset, rather than an empty one or // a panic. A control plane in that state cannot issue tokens either, which is where it surfaces. -func TestNoBrokerMeansNoSubstrateRuleRatherThanNoRuleset(t *testing.T) { +func TestNoBrokerMeansNoFoundationRuleRatherThanNoRuleset(t *testing.T) { out := AsNftables(nil, []string{"10.42.0.1"}, false, nil) if !strings.Contains(out, "table inet mesh") { t.Fatalf("no ruleset at all:\n%s", out) } if strings.Contains(out, "never derived, never closed\n\t\ttcp dport") { - t.Fatalf("a substrate rule was written for a mesh with no broker:\n%s", out) + t.Fatalf("a foundation rule was written for a mesh with no broker:\n%s", out) } } diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index 38585f6..79646e1 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -706,7 +706,7 @@ func ParseManifest(raw []byte) (Manifest, error) { // that provides the store and also builds something asks the mesh to put an artifact into the // thing that artifact is needed to create. // - // It is the question the substrate record asks of every candidate — can it grant itself the + // It is the question the foundation record asks of every candidate — can it grant itself the // thing it provides? The store cannot create its own database, the broker cannot create its // own virtual host, and a registry cannot grant itself a repository. Such a module names its // image, exactly as the bundle names the three a first node starts from. diff --git a/internal/catalogue/provided_test.go b/internal/catalogue/provided_test.go index 12d9a09..17719e6 100644 --- a/internal/catalogue/provided_test.go +++ b/internal/catalogue/provided_test.go @@ -5,8 +5,8 @@ import ( "strings" "testing" - "github.com/novox/mesh-control/internal/catalogue" - "github.com/novox/mesh-control/internal/overlay" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/overlay" ) // The manifests the control plane actually ships, resolved. diff --git a/internal/identity/identity.go b/internal/identity/identity.go index 2d64b7b..c38d88c 100644 --- a/internal/identity/identity.go +++ b/internal/identity/identity.go @@ -19,7 +19,7 @@ import ( "time" "github.com/jackc/pgx/v5" - "github.com/novox/mesh-control/internal/store" + "github.com/novox/mesh-controller/internal/store" ) // Name is what this context is called: its database and its credential are named after it. diff --git a/internal/identity/identity_test.go b/internal/identity/identity_test.go index 577681e..7aa1582 100644 --- a/internal/identity/identity_test.go +++ b/internal/identity/identity_test.go @@ -13,7 +13,7 @@ import ( "time" "github.com/jackc/pgx/v5" - "github.com/novox/mesh-control/internal/store" + "github.com/novox/mesh-controller/internal/store" ) func fresh(t *testing.T) *Identity { diff --git a/internal/inventory/buildinput_test.go b/internal/inventory/buildinput_test.go index fd67283..6bb6e97 100644 --- a/internal/inventory/buildinput_test.go +++ b/internal/inventory/buildinput_test.go @@ -3,7 +3,7 @@ package inventory import ( "testing" - "github.com/novox/mesh-control/internal/catalogue" + "github.com/novox/mesh-controller/internal/catalogue" ) // The image every module is compiled on top of is built and never run. diff --git a/internal/inventory/catalogue.go b/internal/inventory/catalogue.go index 6d23461..71527d3 100644 --- a/internal/inventory/catalogue.go +++ b/internal/inventory/catalogue.go @@ -8,7 +8,7 @@ import ( "strings" "github.com/jackc/pgx/v5" - "github.com/novox/mesh-control/internal/catalogue" + "github.com/novox/mesh-controller/internal/catalogue" ) // ErrNoSuchModule is what the mesh says about a module it has never been told about. diff --git a/internal/inventory/catalogue_test.go b/internal/inventory/catalogue_test.go index 13dbf43..1a6cce0 100644 --- a/internal/inventory/catalogue_test.go +++ b/internal/inventory/catalogue_test.go @@ -6,7 +6,7 @@ import ( "strings" "testing" - "github.com/novox/mesh-control/internal/catalogue" + "github.com/novox/mesh-controller/internal/catalogue" ) func manifest(name string, provides, requires []string) catalogue.Manifest { diff --git a/internal/inventory/forget_test.go b/internal/inventory/forget_test.go index 31e3b02..a8a8ca3 100644 --- a/internal/inventory/forget_test.go +++ b/internal/inventory/forget_test.go @@ -5,7 +5,7 @@ import ( "strings" "testing" - "github.com/novox/mesh-control/internal/catalogue" + "github.com/novox/mesh-controller/internal/catalogue" ) // What removing a module takes with it, and what re-registering one does not. diff --git a/internal/inventory/fortest.go b/internal/inventory/fortest.go index 6536f8f..dddca8f 100644 --- a/internal/inventory/fortest.go +++ b/internal/inventory/fortest.go @@ -10,7 +10,7 @@ import ( "github.com/jackc/pgx/v5" - "github.com/novox/mesh-control/internal/store" + "github.com/novox/mesh-controller/internal/store" ) // ForTest is a fresh inventory in a database of its own, dropped when the test ends. diff --git a/internal/inventory/inventory.go b/internal/inventory/inventory.go index 34fe44e..44b6c14 100644 --- a/internal/inventory/inventory.go +++ b/internal/inventory/inventory.go @@ -10,7 +10,7 @@ package inventory import ( "embed" - "github.com/novox/mesh-control/internal/store" + "github.com/novox/mesh-controller/internal/store" ) // Name is what this context is called: its database, and the environment variable holding the diff --git a/internal/inventory/migrations/0017-what-a-machine-already-holds.sql b/internal/inventory/migrations/0017-what-a-machine-already-holds.sql index 14de2cf..44f30fe 100644 --- a/internal/inventory/migrations/0017-what-a-machine-already-holds.sql +++ b/internal/inventory/migrations/0017-what-a-machine-already-holds.sql @@ -1,6 +1,6 @@ -- Ports a machine holds that the mesh did not assign. -- --- novox/hq ADR 0038. The substrate is not a module: a node raises it from the bundle it carries +-- novox/hq ADR 0038. The foundation is not a module: a node raises it from the bundle it carries -- before any mesh exists, so the control plane has never heard of the store or the broker. Told -- what they hold, it can put a module somewhere else; not told, it hands out a port one of them -- has and finds out from a container runtime three layers down. diff --git a/internal/inventory/nodes.go b/internal/inventory/nodes.go index 65a34d4..ed6d60b 100644 --- a/internal/inventory/nodes.go +++ b/internal/inventory/nodes.go @@ -13,7 +13,7 @@ import ( "time" "github.com/jackc/pgx/v5" - "github.com/novox/mesh-control/internal/store" + "github.com/novox/mesh-controller/internal/store" ) // Inventory is this context, holding the store it exclusively owns. diff --git a/internal/inventory/ports.go b/internal/inventory/ports.go index 8ac354e..3e031c4 100644 --- a/internal/inventory/ports.go +++ b/internal/inventory/ports.go @@ -120,7 +120,7 @@ func (i *Inventory) assignPort( if err != nil { return Assigned{}, err } - // And what the machine itself says it already holds — the substrate it raised before there + // And what the machine itself says it already holds — the foundation it raised before there // was a mesh to ask (novox/hq ADR 0038). Not assignments: nothing here chose them, and // nothing here can move them. carried, err := i.carriedOn(ctx, nodeID) diff --git a/internal/inventory/ports_test.go b/internal/inventory/ports_test.go index 808876e..ef2c8fa 100644 --- a/internal/inventory/ports_test.go +++ b/internal/inventory/ports_test.go @@ -4,7 +4,7 @@ import ( "errors" "testing" - "github.com/novox/mesh-control/internal/catalogue" + "github.com/novox/mesh-controller/internal/catalogue" ) func aNodeWithModules(t *testing.T, modules ...string) (*Inventory, string) { diff --git a/internal/inventory/secrets.go b/internal/inventory/secrets.go index 8672542..9756459 100644 --- a/internal/inventory/secrets.go +++ b/internal/inventory/secrets.go @@ -7,7 +7,7 @@ import ( "github.com/jackc/pgx/v5" - "github.com/novox/mesh-control/internal/secrets" + "github.com/novox/mesh-controller/internal/secrets" ) // Where sealed secrets live. diff --git a/internal/inventory/secrets_test.go b/internal/inventory/secrets_test.go index 309c47f..9a92f07 100644 --- a/internal/inventory/secrets_test.go +++ b/internal/inventory/secrets_test.go @@ -5,7 +5,7 @@ import ( "crypto/ecdh" "crypto/rand" "encoding/base64" - "github.com/novox/mesh-control/internal/catalogue" + "github.com/novox/mesh-controller/internal/catalogue" "strings" "testing" diff --git a/internal/licences/adapters/adapters.go b/internal/licences/adapters/adapters.go index 6f8e27b..dfe4da2 100644 --- a/internal/licences/adapters/adapters.go +++ b/internal/licences/adapters/adapters.go @@ -23,7 +23,7 @@ import ( "strings" "sync" - "github.com/novox/mesh-control/internal/secrets" + "github.com/novox/mesh-controller/internal/secrets" ) // Shape is how a vendor's credential behaves, and the switch the ADR 0050 carve-out turns on. diff --git a/internal/licences/fortest.go b/internal/licences/fortest.go index 10d9276..193376c 100644 --- a/internal/licences/fortest.go +++ b/internal/licences/fortest.go @@ -13,7 +13,7 @@ import ( "github.com/jackc/pgx/v5" - "github.com/novox/mesh-control/internal/store" + "github.com/novox/mesh-controller/internal/store" ) // ForTest is a fresh licence store in a database of its own, dropped when the test ends. diff --git a/internal/licences/licences.go b/internal/licences/licences.go index face562..b0f49e0 100644 --- a/internal/licences/licences.go +++ b/internal/licences/licences.go @@ -20,9 +20,9 @@ import ( "time" "github.com/jackc/pgx/v5" - "github.com/novox/mesh-control/internal/licences/adapters" - "github.com/novox/mesh-control/internal/secrets" - "github.com/novox/mesh-control/internal/store" + "github.com/novox/mesh-controller/internal/licences/adapters" + "github.com/novox/mesh-controller/internal/secrets" + "github.com/novox/mesh-controller/internal/store" ) // Name is what this context is called: its database and its credential are named after it. diff --git a/internal/licences/refresh_test.go b/internal/licences/refresh_test.go index 6478fda..b7600bb 100644 --- a/internal/licences/refresh_test.go +++ b/internal/licences/refresh_test.go @@ -10,8 +10,8 @@ import ( "golang.org/x/crypto/nacl/box" - "github.com/novox/mesh-control/internal/licences/adapters" - "github.com/novox/mesh-control/internal/secrets" + "github.com/novox/mesh-controller/internal/licences/adapters" + "github.com/novox/mesh-controller/internal/secrets" ) // nodeKeyPair is a node's key as the node would hold it: the public half the mesh seals to, and an diff --git a/internal/licences/submitrefresh_test.go b/internal/licences/submitrefresh_test.go index e12cb19..0e32567 100644 --- a/internal/licences/submitrefresh_test.go +++ b/internal/licences/submitrefresh_test.go @@ -4,7 +4,7 @@ import ( "strings" "testing" - "github.com/novox/mesh-control/internal/secrets" + "github.com/novox/mesh-controller/internal/secrets" ) // SubmitRefresh is the boundary a manager NODE crosses to publish a refresh it performed: the control diff --git a/internal/link/enrol_shape_test.go b/internal/link/enrol_shape_test.go index 6257950..e28070b 100644 --- a/internal/link/enrol_shape_test.go +++ b/internal/link/enrol_shape_test.go @@ -11,9 +11,9 @@ import ( "golang.org/x/crypto/nacl/box" - "github.com/novox/mesh-control/internal/catalogue" - "github.com/novox/mesh-control/internal/inventory" - "github.com/novox/mesh-control/internal/link" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" ) // What a node says when it joins, as that node's own code writes it. @@ -25,7 +25,7 @@ import ( // Skipped unless MESH_ENROL names the file the host's suite wrote: // // mesh-host: MESH_ENROL_OUT=/tmp/enrol.json go test ./internal/link/ -// mesh-control: MESH_ENROL=/tmp/enrol.json make check +// mesh-controller: MESH_ENROL=/tmp/enrol.json make check // // **What this does not cover**, said so nobody reads more into a pass than is there: the full // enrolment path also issues a broker account, and that needs a broker. What is checked here is diff --git a/internal/link/enrolment.go b/internal/link/enrolment.go index dc7eef2..a7c05f8 100644 --- a/internal/link/enrolment.go +++ b/internal/link/enrolment.go @@ -9,9 +9,9 @@ import ( "fmt" "sort" - "github.com/novox/mesh-control/internal/broker" - "github.com/novox/mesh-control/internal/identity" - "github.com/novox/mesh-control/internal/inventory" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/identity" + "github.com/novox/mesh-controller/internal/inventory" ) // Enrolment is what actually happens when a node presents a token: the token is spent, the key is diff --git a/internal/link/heard_test.go b/internal/link/heard_test.go index c9b438d..01fb10b 100644 --- a/internal/link/heard_test.go +++ b/internal/link/heard_test.go @@ -5,8 +5,8 @@ import ( "strings" "testing" - "github.com/novox/mesh-control/internal/inventory" - "github.com/novox/mesh-control/internal/link" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" ) // Turning what a node said into what the mesh keeps. @@ -152,7 +152,7 @@ func TestABareAliveDoesNotWipeTheDeclarationThatSaysANodeIsCurrent(t *testing.T) func TestAFailureDoesNotBecomeTheAccountOfWhatTheMachineHolds(t *testing.T) { // A partial list is not an account of what the machine holds. Recording one as though it // were would tell a rebuilding node to remove what it still has — which is the fault that - // destroyed a substrate once (novox/hq 04-ISSUES/010). + // destroyed a foundation once (novox/hq 04-ISSUES/010). inv := inventory.ForTest(t) ctx := context.Background() node, err := inv.AddNode(ctx, "workstation") diff --git a/internal/link/protocol.go b/internal/link/protocol.go index a4bea31..28c37c4 100644 --- a/internal/link/protocol.go +++ b/internal/link/protocol.go @@ -88,7 +88,7 @@ type Report struct { // Carried are the machine's ports held by what that host raised from its own bundle. // - // **The half the mesh cannot know** (novox/hq ADR 0038). The substrate is not a module — a + // **The half the mesh cannot know** (novox/hq ADR 0038). The foundation is not a module — a // node raises it before any mesh exists — so without being told, the mesh assigns a module a // port the store or the broker already holds, and hears about it from a container runtime. // diff --git a/internal/overlay/declaration.go b/internal/overlay/declaration.go index 481024c..0039fb4 100644 --- a/internal/overlay/declaration.go +++ b/internal/overlay/declaration.go @@ -127,7 +127,7 @@ func config(node Node, peers []Peer, keyPath string) string { b.WriteString("PostDown = sysctl -q -w net.ipv4.ip_forward=0\n") // And past the machine's own firewall, which on any node with a container runtime is - // closed. Docker sets the FORWARD policy to DROP and inserts its chains, so the substrate + // closed. Docker sets the FORWARD policy to DROP and inserts its chains, so the foundation // this mesh installs at tier 1 silently breaks the network it builds at tier 2: every // spoke reaches the hub, no spoke reaches any other, and every part of it reports // success. Found in the lab; nothing about it is visible from the mesh's own state. diff --git a/internal/overlay/generator.go b/internal/overlay/generator.go index ac6f88b..3b10895 100644 --- a/internal/overlay/generator.go +++ b/internal/overlay/generator.go @@ -5,7 +5,7 @@ import ( "fmt" "strconv" - "github.com/novox/mesh-control/internal/catalogue" + "github.com/novox/mesh-controller/internal/catalogue" ) // The private network as a module rather than as code beside the module system. diff --git a/internal/overlay/opens_test.go b/internal/overlay/opens_test.go index f764f19..06f4c93 100644 --- a/internal/overlay/opens_test.go +++ b/internal/overlay/opens_test.go @@ -4,7 +4,7 @@ import ( "strings" "testing" - "github.com/novox/mesh-control/internal/catalogue" + "github.com/novox/mesh-controller/internal/catalogue" ) func networkOf(t *testing.T, nodes []Node) *Generator { diff --git a/internal/secrets/sealedbox_xcheck_test.go b/internal/secrets/sealedbox_xcheck_test.go index cbc3dc3..856cb53 100644 --- a/internal/secrets/sealedbox_xcheck_test.go +++ b/internal/secrets/sealedbox_xcheck_test.go @@ -15,7 +15,7 @@ import ( // **Why it must hold.** The refresh token is sealed to the manager node — at adoption and after each // rotation — by the manager MODULE, in TypeScript (mesh-catalog anthropic-manager/sealedbox.ts). The // HOST then unseals it with Go's box.OpenAnonymous (mesh-host identity.SealingKey.Unseal) to mount the -// cleartext, and mesh-control seals every other credential with box.SealAnonymous (secrets.Seal). If +// cleartext, and mesh-controller seals every other credential with box.SealAnonymous (secrets.Seal). If // the TS seal and the Go box disagreed by a byte, the host would refuse the refresh token as a value // it cannot open — silently, as a manager that never gets its credential. So this is load-bearing, and // it is pinned here rather than trusted. diff --git a/internal/secrets/testdata/module-sealedbox-fixture.json b/internal/secrets/testdata/module-sealedbox-fixture.json index bd50c37..6823c09 100644 --- a/internal/secrets/testdata/module-sealedbox-fixture.json +++ b/internal/secrets/testdata/module-sealedbox-fixture.json @@ -1,5 +1,5 @@ { - "_comment": "Produced by mesh-catalog anthropic-manager sealedbox.ts (crypto_box_seal). Proves that value the module seals to a node's public key opens under Go box.OpenAnonymous — the host's Unseal and mesh-control secrets.Seal/Open. Regenerate with the module's compiled seal().", + "_comment": "Produced by mesh-catalog anthropic-manager sealedbox.ts (crypto_box_seal). Proves that value the module seals to a node's public key opens under Go box.OpenAnonymous — the host's Unseal and mesh-controller secrets.Seal/Open. Regenerate with the module's compiled seal().", "managerPublicKey": "rJZ9OSnuCcU5MNi8iV0EK8c5nYN+Cx5A+q+miIIIoUc=", "managerPrivateKey": "wGHx9hpbO1pyvLiw8oGwi31LBce3HscDiGhXpNU+wl4=", "plaintext": "rt-a-refresh-token-only-the-manager-may-read", diff --git a/module.json b/module.json index a6b7c36..a3b7a9d 100644 --- a/module.json +++ b/module.json @@ -1,5 +1,5 @@ { - "module": "mesh-control", + "module": "mesh-controller", "version": "1", "slug": "control", "capabilities": [ @@ -7,42 +7,42 @@ ], "claims": [ { - "name": "the-control-plane", + "name": "the-controller", "scope": "mesh" } ], "own-secrets": { - "inventory": "/var/lib/mesh/mesh-control/inventory", - "identity": "/var/lib/mesh/mesh-control/identity", - "licences": "/var/lib/mesh/mesh-control/licences", - "broker": "/var/lib/mesh/mesh-control/broker", - "broker-management": "/var/lib/mesh/mesh-control/broker-management", - "broker-address": "/var/lib/mesh/mesh-control/broker-address" + "inventory": "/var/lib/mesh/mesh-controller/inventory", + "identity": "/var/lib/mesh/mesh-controller/identity", + "licences": "/var/lib/mesh/mesh-controller/licences", + "broker": "/var/lib/mesh/mesh-controller/broker", + "broker-management": "/var/lib/mesh/mesh-controller/broker-management", + "broker-address": "/var/lib/mesh/mesh-controller/broker-address" }, "resources": [ { "id": "mesh-state", "type": "directory", - "path": "/var/lib/mesh/mesh-control", + "path": "/var/lib/mesh/mesh-controller", "mode": "0700" }, { "id": "control-env", "type": "file", - "path": "/var/lib/mesh/mesh-control/control.env", + "path": "/var/lib/mesh/mesh-controller/control.env", "mode": "0600", "content": "MESH_STORE_INVENTORY=${secret:inventory}\nMESH_STORE_IDENTITY=${secret:identity}\nMESH_STORE_LICENCES=${secret:licences}\nMESH_BROKER_AMQP=${secret:broker}\nMESH_BROKER_MANAGEMENT=${secret:broker-management}\nMESH_BROKER_ADDRESS=${secret:broker-address}\n" }, { "id": "server", "type": "container", - "name": "mesh-control", + "name": "mesh-controller", "network": "host", "args": [ "serve" ], "env-file": [ - "/var/lib/mesh/mesh-control/control.env" + "/var/lib/mesh/mesh-controller/control.env" ], "env": { "MESH_BROKER_CERTIFICATE": "/broker-tls/tls.crt" -- 2.54.0