diff --git a/internal/catalogue/adoption.go b/internal/catalogue/adoption.go deleted file mode 100644 index 72bc5645..00000000 --- a/internal/catalogue/adoption.go +++ /dev/null @@ -1,242 +0,0 @@ -package catalogue - -import ( - "fmt" - "sort" - "strconv" - "strings" -) - -// What an adopted node is declared in place of a filter (novox/hq ADR 0100). -// -// On an adopted node the firewall found on the machine stays in force: the mesh loads no table -// that drops by default or holds an accept. What the mesh needs reachable is declared as -// openings, which the host converges through the found firewall in its own terms; and the mesh -// guards its own foundation ports itself, in a table that only refuses. - -// AdoptionPrefix is the id prefix of what the mesh declares of its own on an adopted node. It is -// never a module's, so none of it is ever held as found. -const AdoptionPrefix = "adoption." - -// Where an opening admits from, on the wire. -const ( - OpeningFromEverywhere = "everywhere" - OpeningFromMesh = "mesh" -) - -// The two paths a packet reaches a port by: received by the machine, or forwarded to a -// container that publishes it. -const ( - PathIncoming = "incoming" - PathForwarded = "forwarded" -) - -// Guard resources: the refusal-only table, the unit that loads it, and that unit running. -const ( - GuardPath = "/etc/mesh/guard.nft" - GuardUnit = "mesh-guard.service" - // GuardUnitPath is where the unit is written. - GuardUnitPath = "/etc/systemd/system/" + GuardUnit -) - -// GuardID, GuardUnitID and GuardRunningID are the guard's resource identities. The installer -// raises the same three on an adopted genesis, so the first push finds them already there. -func GuardID() string { return AdoptionPrefix + "guard" } -func GuardUnitID() string { return AdoptionPrefix + "guard-unit" } -func GuardRunningID() string { return AdoptionPrefix + "guard-running" } - -// GuardPackageID is the tool that loads the guard, declared first: a node joining adopted has -// no filter module and may have no nft at all, and a table nothing can load guards nothing. -func GuardPackageID() string { return AdoptionPrefix + "guard-package" } - -// GuardPackage is the package that carries nft. -const GuardPackage = "nftables" - -// OpeningID is an opening's resource identity: its protocol, port and path say what it is. -func OpeningID(protocol string, port int, path string) string { - return fmt.Sprintf("%sopening-%s-%d-%s", AdoptionPrefix, protocol, port, path) -} - -// Openings are what the mesh needs reachable on an adopted node, from the same inputs as the -// filter it would load were the node converged, each from where that filter would admit it. -// -// `rules` is Filtering's answer — every module's listens, the hub's port, the per-node exposure — -// and `foundation` is the ports the mesh itself needs, from everywhere. A rule for this machine -// only opens nothing. `published` maps a machine port a container publishes to the container's -// port: a published port is forwarded, not received, so its opening names the forwarded path and -// the port the packet is forwarded to. -func Openings(rules []Rule, foundation []int, published map[string]map[int]int) []map[string]any { - type key struct { - protocol string - port int - } - from := map[key]string{} - var order []key - widen := func(k key, f string) { - was, seen := from[k] - if !seen { - order = append(order, k) - } - if !seen || was != OpeningFromEverywhere { - from[k] = f - } - } - for _, rule := range rules { - switch rule.From { - case FromEverywhere: - widen(key{rule.Protocol, rule.Port}, OpeningFromEverywhere) - case FromMesh: - widen(key{rule.Protocol, rule.Port}, OpeningFromMesh) - case FromLocalNetwork: - // The private network's part only (novox/hq ADR 0340). An adopted node's found firewall - // stays, and it is what admits the machine's local network; an opening has no word for a - // local network, and widening it to everywhere would open the port to the internet. - widen(key{rule.Protocol, rule.Port}, OpeningFromMesh) - } - } - for _, port := range foundation { - widen(key{"tcp", port}, OpeningFromEverywhere) - } - sort.Slice(order, func(a, b int) bool { - if order[a].port != order[b].port { - return order[a].port < order[b].port - } - return order[a].protocol < order[b].protocol - }) - out := make([]map[string]any, 0, len(order)) - for _, k := range order { - opening := map[string]any{"type": "opening", "port": k.port, "protocol": k.protocol, - "from": from[k]} - if to, forwarded := published[k.protocol][k.port]; forwarded { - opening["id"] = OpeningID(k.protocol, k.port, PathForwarded) - opening["path"] = PathForwarded - opening["to"] = to - } else { - opening["id"] = OpeningID(k.protocol, k.port, PathIncoming) - opening["path"] = PathIncoming - } - out = append(out, opening) - } - return out -} - -// Published is every port the given containers publish on the machine, by protocol and machine -// port, mapped to the container's own port. A mapping bound to loopback is left out: nothing off -// the machine reaches it, forwarded or not. -func Published(resources []map[string]any) map[string]map[int]int { - out := map[string]map[int]int{} - for _, r := range resources { - if fmt.Sprint(r["type"]) != "container" { - continue - } - listed, _ := r["ports"].([]any) - for _, entry := range listed { - written := strings.TrimSpace(fmt.Sprint(entry)) - protocol := "tcp" - if cut := strings.LastIndex(written, "/"); cut >= 0 { - protocol = written[cut+1:] - } - // Indexed from the end, so an IPv6 address's own colons never shift the ports. - outer, inner, address, ok := mapping(written) - if !ok { - continue - } - switch strings.Trim(address, "[]") { - case "127.0.0.1", "localhost", "::1": - continue - } - if out[protocol] == nil { - out[protocol] = map[int]int{} - } - out[protocol][outer] = inner - } - } - return out -} - -// AsGuard renders the mesh's refusal-only table for the given machine ports. -// -// It passes everything by default and holds nothing but a refusal, so it cannot close anything -// the machine serves; and it is the mesh's own table, so the found firewall reloading does not -// touch it. It refuses only packets addressed to this machine, and the ports except from the -// machine itself — its loopback and the container runtime's own networks — and from the private -// network, known by the interface a packet arrives on and never by its source address. At -// prerouting, ahead of the runtime's destination translation, so it matches the port the packet -// was sent to; in the inet family, so both address families. -// -// **The machine's own interfaces are named, where the derived filter names address ranges.** The -// filter accepts the container runtime's networks by CIDR; this excludes its bridges by name — lo, -// docker0, the br-* a compose network gets, and the mesh's own mesh0. A runtime whose bridge is -// named anything else (a podman or libvirt bridge, or a docker network created with a fixed name) -// would have its containers' traffic to a guarded port refused, which reads as the port being -// down. Names rather than addresses is deliberate: a source address can be claimed by whoever -// sends the packet, and this table exists to refuse what the found firewall never sees. Widening -// it means adding names here and in the installer's copy together, which the golden test holds to -// one text. -// -// The same text the installer raises on an adopted genesis; a test holds both to it. -func AsGuard(ports []int) string { - sorted := append([]int{}, ports...) - sort.Ints(sorted) - listed := make([]string, len(sorted)) - for i, p := range sorted { - listed[i] = strconv.Itoa(p) - } - var b strings.Builder - b.WriteString("table inet mesh_guard {}\n") - b.WriteString("delete table inet mesh_guard\n") - b.WriteString("table inet mesh_guard {\n") - b.WriteString("\tchain prerouting {\n") - b.WriteString("\t\ttype filter hook prerouting priority raw; policy accept;\n") - // Only packets addressed to this machine: traffic it routes for others — a predecessor's hub, - // say — is never the guard's business (novox/hq ADR 0103). - fmt.Fprintf(&b, "\t\tfib daddr type local iifname != \"lo\" iifname != \"docker0\" "+ - "iifname != \"br-*\" iifname != \"mesh0\" tcp dport { %s } drop\n", - strings.Join(listed, ", ")) - b.WriteString("\t}\n") - b.WriteString("}\n") - return b.String() -} - -// GuardUnitText is the unit that loads the guard. Stopping it deletes only its own table: never -// a flush, which would take the container runtime's rules and the found firewall with it. -func GuardUnitText() string { - return "[Unit]\n" + - "Description=The mesh's guard: refuses its own ports from outside (novox/hq ADR 0100)\n" + - // Early, before the network is up, and without the default dependencies that would - // order it after the network; stopped at shutdown like any unit. - "DefaultDependencies=no\n" + - "Wants=network-pre.target\n" + - "Before=network-pre.target shutdown.target\n" + - "Conflicts=shutdown.target\n" + - "\n" + - "[Service]\n" + - "Type=oneshot\n" + - "RemainAfterExit=yes\n" + - "ExecStart=nft -f " + GuardPath + "\n" + - "ExecReload=nft -f " + GuardPath + "\n" + - "ExecStop=nft delete table inet mesh_guard\n" + - "\n" + - "[Install]\n" + - "WantedBy=multi-user.target\n" -} - -// GuardResources are the guard as four resources of the existing kinds: the tool that loads it, -// the table, the unit, and the unit running — reloaded when the table changes, so the new table -// replaces the old in one `nft -f` through the unit's ExecReload with no moment unguarded, and -// restarted only when the unit itself changes. Nothing when there is nothing to guard: an empty -// set is not a table nft loads. -func GuardResources(ports []int) []map[string]any { - if len(ports) == 0 { - return nil - } - return []map[string]any{ - {"id": GuardPackageID(), "type": "package", "package": GuardPackage}, - {"id": GuardID(), "type": "file", "path": GuardPath, "content": AsGuard(ports), - "mode": "0644"}, - {"id": GuardUnitID(), "type": "file", "path": GuardUnitPath, "content": GuardUnitText(), - "mode": "0644"}, - {"id": GuardRunningID(), "type": "service", "unit": GuardUnit, "state": "running", - "boot": "enabled", "restart-on": []any{GuardUnitID()}, "reload-on": []any{GuardID()}}, - } -} diff --git a/internal/catalogue/artifacts.go b/internal/catalogue/artifacts.go deleted file mode 100644 index e9a010d8..00000000 --- a/internal/catalogue/artifacts.go +++ /dev/null @@ -1,145 +0,0 @@ -package catalogue - -import ( - "fmt" - "strings" -) - -// What the mesh built, named by what it is rather than by where it was pushed. -// -// **An image is recorded by its digest and its path; the registry's address is a route to it** -// (novox/hq 04-ISSUES/102). A build used to be recorded as `://@sha256:…` -// — the reference the builder pushed to, kept whole — and every declaration carried that literal. -// Move the registry's port, or the registry, and every fresh pull of a mesh image fails: a new -// node, a recreate after eviction. The digest is the identity; the address is the node's setting -// for the module that serves the artifact store, and it is read from there when a reference is -// composed, never written into a record. -// -// So a kept reference has a scheme of the mesh's own, named after the provision that answers it: -// -// artifact-store:///@sha256: an image -// artifact-store:////blobs/sha256: an archive -// -// No runtime knows the scheme. That is the point of it being one: a reference that leaks to a -// machine uncomposed is refused by the runtime as malformed, in front of whoever sent it, rather -// than pulled from a public registry that happens to have a repository by that name — which is -// what an address-less `/@sha256:…` would be. - -// ArtifactStoreScheme marks a reference to something in the mesh's artifact store, kept without -// the store's address. -const ArtifactStoreScheme = ArtifactStoreProvision + "://" - -// Recorded is a reference as the mesh records it: the artifact store's address, if the builder wrote -// one, taken off. -// -// For a reference the builder announced — one it pushed to the store — which is the only kind -// this is called on. An image the builder named `/@sha256:…` is kept as its path; an -// archive it named `http:///v2//blobs/` likewise. A reference with no address -// in it — an image id from a genesis build that had nowhere to publish, a package version — is -// what it was. -func Recorded(reference string) string { - if strings.HasPrefix(reference, ArtifactStoreScheme) { - return reference - } - if rest, isURL := strings.CutPrefix(reference, "http://"); isURL { - if _, path, ok := strings.Cut(rest, "/v2/"); ok && strings.Contains(path, "/blobs/") { - return ArtifactStoreScheme + path - } - return reference - } - host, path, ok := strings.Cut(reference, "/") - if !ok || !isRegistryHost(host) || !strings.Contains(path, "@sha256:") { - return reference - } - return ArtifactStoreScheme + path -} - -// isRegistryHost is the runtime's own rule for reading the first component of a reference as a -// registry rather than as a namespace: it has a dot or a port in it, or it is localhost. -func isRegistryHost(component string) bool { - return component == "localhost" || strings.ContainsAny(component, ".:") -} - -// InArtifactStore reports whether a reference is a kept one, and what it names there. -func InArtifactStore(reference string) (path string, kept bool) { - return strings.CutPrefix(reference, ArtifactStoreScheme) -} - -// Routed is a kept reference as a machine fetches it, through the artifact store at `address` -// (host:port). A reference that is not a kept one is what it was. -func Routed(reference, address string) string { - path, kept := InArtifactStore(reference) - if !kept { - return reference - } - if strings.Contains(path, "/blobs/") { - return "http://" + address + "/v2/" + path - } - return address + "/" + path -} - -// Rerouted is a reference the mesh recorded, whichever way it was recorded, as a machine fetches -// it now: a kept one composed with the store's address, and one recorded before references were -// kept without their address — the builder's own `/@sha256:…` — re-routed to where -// the store is now. Only for references that are the mesh's own: everything a build record -// holds is, by construction. -func Rerouted(reference, address string) string { - return Routed(Recorded(reference), address) -} - -// artifactsInto composes the artifact store's address into a resource's `image` and `source`. -// -// **Composed here, at the last moment before a machine, and stored nowhere.** A kept reference is -// routed through the store as this network reaches it now. A reference recorded with an address -// before references were kept without one is re-routed the same way — but only when the mesh -// built it (`with.Built` names every `/` it has), because a module may run an -// image from a public registry under its own name and that one is exactly where it says. -// -// A kept reference with no store to route it through is refused: sent as it is, the runtime would -// refuse the scheme on the machine, one push away from the reason. -// -// **What this does not reach: the images genesis pinned.** The installer builds the control plane -// and the builder before the mesh exists, pushes them itself and pins their manifests to -// `:/mesh-controller@…` and `:/mesh-builder@…` — single-segment -// repositories with no build record, so `with.Built` does not name them and they are left as -// written until each is rebuilt through the mesh, which records it by digest and path. Until then -// a registry that moves strands exactly those two on a recreate, and the control plane's is the -// one that cannot be repaired through the mesh. Rebuild both through `build` before moving the -// store (novox/hq 04-ISSUES/102, finding F4). -func artifactsInto(resource map[string]any, module string, with Rendering) error { - for _, key := range []string{"image", "source"} { - written, ok := resource[key].(string) - if !ok { - continue - } - if _, kept := InArtifactStore(written); kept { - if with.ArtifactStore == "" { - return fmt.Errorf( - "%s's %v names %s, which is in the mesh's artifact store, and this mesh has no "+ - "artifact store on its network to fetch it from — nothing assigned offers "+ - "%s, or the machine offering it is not on the private network", - module, resource["id"], written, ArtifactStoreProvision) - } - resource[key] = Routed(written, with.ArtifactStore) - continue - } - if with.ArtifactStore == "" { - continue - } - recorded := Recorded(written) - if recorded == written { - continue - } - path, _ := InArtifactStore(recorded) - repository := path - if at := strings.IndexAny(path, "@"); at >= 0 { - repository = path[:at] - } else if blobs := strings.Index(path, "/blobs/"); blobs >= 0 { - repository = path[:blobs] - } - if with.Built[repository] { - resource[key] = Routed(recorded, with.ArtifactStore) - } - } - return nil -} diff --git a/internal/catalogue/backup_placement.go b/internal/catalogue/backup_placement.go deleted file mode 100644 index ddd35178..00000000 --- a/internal/catalogue/backup_placement.go +++ /dev/null @@ -1,53 +0,0 @@ -package catalogue - -import ( - "fmt" - "regexp" -) - -// What the machine's modules back up is placement (design 57 L3, issue 794). The holder of node-backup names it as -// `${backup:lines}` (what to run and which paths to keep) and `${backup:data}` (every item with its class and -// protection): the text `${contribution:node-backup:backup}` and `${contribution:node-backup:data}` render, from the -// same function, derived from each placed module's declared data (novox/hq ADR 0233). A handed holder has the text in -// its plan's placement for its client to fill; one not handed has the controller fill it where it stands. - -// BackupNamespace is the placeholder kind of the machine's backup lines and data. -const BackupNamespace = "backup" - -// ofBackup is where a holder places them: ${backup:}. Loose inside the braces, so a misspelt key is found and -// refused rather than written out as text. -var ofBackup = regexp.MustCompile(`\$\{backup:([^}]*)\}`) - -// backupKinds is the contribution kind each key of the backup placement renders. -var backupKinds = map[string]string{"lines": BackupKindBackup, "data": BackupKindData} - -// backupPlacer is the machine's modules the backup placement is composed over, as the old contributions are. -type backupPlacer struct { - modules []Manifest - with Rendering - facts map[string]string - caps map[string]bool - unplaced *[]string -} - -// placed is the text of one key for a holder, or what is wrong with naming it. -func (b backupPlacer) placed(holder Manifest, key string, item bool) (string, string) { - kind, known := backupKinds[key] - switch { - case item: - return "", "an item is kept per item; the machine's backup lines are placed only in a file" - case !known: - return "", fmt.Sprintf("the backup placement has `lines` and `data`, not %q", key) - case !holder.ClaimsSeat(BackupSeat): - return "", fmt.Sprintf("%s does not claim %s; the machine's backup lines are placed by its holder alone "+ - "(novox/hq ADR 0212)", holder.Module, BackupSeat) - } - text, left, err := seatContributions(b.modules, holder, BackupSeat+":"+kind, b.with, b.facts, b.caps) - if b.unplaced != nil { - *b.unplaced = append(*b.unplaced, left...) - } - if err != nil { - return "", err.Error() - } - return text, "" -} diff --git a/internal/catalogue/bound.go b/internal/catalogue/bound.go deleted file mode 100644 index 12941a4a..00000000 --- a/internal/catalogue/bound.go +++ /dev/null @@ -1,186 +0,0 @@ -package catalogue - -import ( - "fmt" - "slices" - "strings" -) - -// A consumer of a provision that keeps its data stays bound where its data is (novox/hq ADR 0232). -// -// **What a resolution chooses is not where a consumer's data is.** Resolving answers "which provider -// would I pick now", from the seats' holders, the pins and what is assigned where, and every one of -// those can change under a consumer without anybody meaning to move it. For a resolver that is the -// point: any provider answers alike. For a database it is the consumer's whole state: on 2026-10-05 -// one change to how a seat's holder answers (issue 258) re-bound five database consumers on one -// machine to the store on another, each was given a fresh, empty database there, and nothing warned -// for twenty hours (issue 273). Nothing was lost only because the old provider kept everything. -// -// So the mesh records where each such consumer was last sent (the store's `binding` table), and a -// resolution that would answer it from anywhere else keeps the recorded provider instead and says so. -// **Only a pin moves it**, because a pin is a person: the one act that says "answer this machine's -// consumers from there", taken knowing the data must go first. - -// KeptBinding is one consumer this resolution would have moved, and did not. -type KeptBinding struct { - // Machine is where the consumer runs, and Consumer the module there that is bound. - Machine string - Consumer string - // Provision is what it is bound for. - Provision string - // Bound is the provider it was recorded at, and still is; Would is the one the resolution chose. - Bound Chosen - Would Chosen -} - -// String is the condition's sentence: the move, where the data is, and the act that confirms it. -func (k KeptBinding) String() string { - return fmt.Sprintf("would move %s's %s from %s to %s — its data is on %s; kept there. "+ - "`pin %s %s %s %s` to confirm a move (and move the data first)", - k.Consumer, k.Provision, k.Bound, k.Would, k.Bound, k.Machine, k.Provision, k.Would.Node, k.Would.Module) -} - -// keepBound holds every need for a provision that keeps its consumers' data at the provider its -// consumer was bound to, where the resolution chose another. -// -// A need with no record is a binding being made, and is left as resolved: it is recorded when it is -// first sent. A pin naming the provider the resolution chose is a person moving it, and is left too. -// Otherwise the recorded provider answers, if it still provides the provision — beside the consumer, -// or offered from elsewhere — and the move is returned as kept. **One that no longer does is refused, -// never answered by the provider chosen**: answering it there is exactly the silent move this exists -// to stop, and the consumer's data is still wherever it was. -func keepBound(needs []Needed, catalogue map[string]Manifest, node Node, world World, - keeps map[string]bool, here func(string) bool) ([]Needed, []KeptBinding, []string) { - var kept []KeptBinding - var problems []string - refused := map[[2]string]bool{} - out := make([]Needed, 0, len(needs)) - for _, n := range needs { - if n.ByRecord || !keeps[n.Name] { - out = append(out, n) - continue - } - n.KeepsData = true - bound, recorded := world.Bound[n.For][n.Name] - chose := Chosen{Node: n.From, Module: n.Module} - if !recorded || sameProvider(bound, chose) { - out = append(out, n) - continue - } - if pin, pinned := world.Pinned[n.Name]; pinned && pin.matches(Provider{Node: chose.Node, Module: chose.Module}) { - out = append(out, n) - continue - } - held, ok, why := boundNeed(n, bound, catalogue, node, world, here) - if !ok { - if key := [2]string{n.For, n.Name}; !refused[key] { - refused[key] = true - problems = append(problems, fmt.Sprintf( - "%s on %s is bound to %s for %q, which keeps its data, and %s — it would move to %s, "+ - "which holds none of it. Move its data and say so with `pin %s %s %s %s`, or give "+ - "%s back what it provided", - n.For, node.Name, bound, n.Name, why, chose, node.Name, n.Name, chose.Node, chose.Module, - bound.Node)) - } - continue - } - out = append(out, held) - kept = append(kept, KeptBinding{Machine: node.Name, Consumer: n.For, Provision: n.Name, Bound: bound, Would: chose}) - } - return out, kept, problems -} - -// sameProvider is whether a recorded provider is the one chosen. A record naming no module (none -// is written without one, but a person's hand might) matches any module on its node. -func sameProvider(bound, chose Chosen) bool { - return bound.Node == chose.Node && (bound.Module == "" || bound.Module == chose.Module) -} - -// boundNeed is the need answered by the recorded provider, or why it cannot be. -func boundNeed(n Needed, bound Chosen, catalogue map[string]Manifest, node Node, world World, - here func(string) bool) (Needed, bool, string) { - held := Needed{Name: n.Name, For: n.For, Local: n.Local, KeepsData: true} - if bound.Node == node.Name { - m, known := catalogue[bound.Module] - if !known || !here(bound.Module) || !providesAt(m, n.Name, ScopeMesh) { - return Needed{}, false, fmt.Sprintf("%s no longer runs on %s", bound.Module, node.Name) - } - at := node.At - if at == "" { - at = "127.0.0.1" - } - held.From, held.At, held.Module = node.Name, at, m.Module - held.Serves, held.SharedOwn, held.Identity = servedByOne(m, n.Name), sharedByOne(m, n.Name), m.IdentityBoundOf(n.Name) - held.CredentialForm, held.CredentialTool = m.CredentialFormOf(n.Name), m.CredentialToolOf(n.Name) - return held, true, "" - } - matching := bound.among(world.Offered[n.Name]) - if len(matching) != 1 { - return Needed{}, false, fmt.Sprintf("%s no longer provides it", bound) - } - p := matching[0] - if node.At == "" || p.At == "" { - return Needed{}, false, fmt.Sprintf("%s and %s are not both on the private network", node.Name, p.Node) - } - identity := DefaultIdentityBound - if pm, known := catalogue[p.Module]; known { - held.SharedOwn, _ = pm.SharedCredentialOf(n.Name) - held.CredentialForm, held.CredentialTool = pm.CredentialFormOf(n.Name), pm.CredentialToolOf(n.Name) - identity = pm.IdentityBoundOf(n.Name) - } - held.From, held.At, held.Module, held.Serves, held.Identity = p.Node, p.At, p.Module, p.Serves, identity - held.LeftOut = p.LeftOut - return held, true, "" -} - -// Unbound is one consumer that still asks for a provision and whose own resolution binds it to -// another provider than the one a pair credential on record was made with (novox/hq issue 274). -// -// **A provider is granted exactly the consumers bound to it.** The credential from the old provider -// stays on record — it is the key to a login that provider keeps, disabled, with the consumer's data, -// until a person deletes it with `cleanup delete` (ADR 0230), and a pin back must find it — but it is -// no longer granted, so the provider stops being asked for it and retires it. Said on every plan and -// push of the provider, so a credential the mesh keeps and does not use is never kept silently. -type Unbound struct { - // Provision is what was required, Provider the machine the credential on record is from. - Provision string `json:"provision"` - Provider string `json:"provider"` - // Consumer is the machine, Module the module on it that requires it, Local the credential's - // name inside it where it keeps several (ADR 0094). - Consumer string `json:"consumer"` - Module string `json:"module"` - Local string `json:"local,omitempty"` - // BoundTo is every provider the consumer's resolution binds this credential to now; empty when - // it binds it nowhere — the module asks for the provision under other local names. - BoundTo []string `json:"bound_to,omitempty"` -} - -func (u Unbound) String() string { - who := u.Module - if u.Local != "" { - who += " (as " + u.Local + ")" - } - now := "is bound to no provider under that name" - if len(u.BoundTo) > 0 { - now = "is bound to " + strings.Join(u.BoundTo, ", ") - } - return fmt.Sprintf("%s on %s %s for %s, not to %s — %s no longer grants it, so it retires that "+ - "login and keeps its data until `cleanup delete` (ADR 0230); the credential from %s stays on "+ - "record while that login does", who, u.Consumer, now, u.Provision, u.Provider, u.Provider, u.Provider) -} - -// BindsFrom is the providers this resolution binds a pair credential's need to — the provision, the -// module that requires it and the credential's local name — answered by a machine rather than by a -// record. None means this machine states no such binding. -func (r Resolution) BindsFrom(provision, module, local string) []string { - var from []string - for _, n := range r.Needs { - if n.ByRecord || n.Name != provision || n.For != module || n.Local != local { - continue - } - if !slices.Contains(from, n.From) { - from = append(from, n.From) - } - } - return from -} diff --git a/internal/catalogue/bound_into_files.go b/internal/catalogue/bound_into_files.go deleted file mode 100644 index e04caad4..00000000 --- a/internal/catalogue/bound_into_files.go +++ /dev/null @@ -1,192 +0,0 @@ -package catalogue - -import ( - "fmt" - "regexp" - "sort" - "strings" -) - -// The half of a connection that is not secret, put where the program reading it can find it. -// -// **The asymmetry this removes was backwards** (novox/hq 04-ISSUES/023). A sealed credential can -// be placed inside any configuration file a module writes: the module leaves a hole, the mesh -// delivers the value sealed beside it, and the host — the only thing that sees both — fills it in. -// The host and the port and the name to present are ordinary facts the mesh holds in the clear, -// and they were the ones stuck: readable only inside a JSON binding, which a program reading -// `KEY=value` cannot use. -// -// So the same shape, and simpler. These values are not secret, so **the control plane substitutes -// them itself** before the declaration is sent. Nothing new reaches the host, which learns no -// formats and gains no fields. -// -// **It stays name-agnostic** ([ADR 0027]). The mesh does not learn what a `postgres-database` is: -// `at`, `as`, `from` and `address` (the providing machine's private address, novox/hq ADR 0194) are -// facts about any provision at all, and everything else comes from what the provider said it -// serves — whose keys are agreed by the requirement's name, not by this file. - -// bound is where a module says a value from one of its bindings belongs: -// ${bound::}, and for an optional requirement the value it takes while absent, -// ${bound::|default:} (novox/hq ADR 0330 §4). A default is never empty: an -// absent requirement is never rendered as an empty string. -var bound = regexp.MustCompile(`\$\{bound:([a-z0-9][a-z0-9.-]*[a-z0-9]):([a-z0-9][a-z0-9_-]*)(?:\|default:([^}\n]+))?\}`) - -// knownFor is everything a module may name from one of its bindings. -// -// Three facts the mesh states about any provision, plus whatever the provider said it serves. A -// module may not reach a binding it does not have — the same boundary as a secret, for the same -// reason. -func knownFor(m Manifest, needs []Needed, node string) (map[string]map[string]string, error) { - out := map[string]map[string]string{} - for _, want := range m.Wants() { - for i := range needs { - n := needs[i] - if n.Name != want || n.For != m.Module { - continue - } - as := ConsumerIdentity(node, IdentitySource(m.Slug, m.Module)) - values := map[string]string{ - "at": n.At, - "from": n.From, - "as": as, - } - // What the provider derives for this consumer rather than for all of them - // (novox/hq ADR 0202). Filled here, the one place a provision and the module - // requiring it are both in hand. - served, err := ServedTo(n.Serves, as) - if err != nil { - return nil, fmt.Errorf("%s requires %s: %w", m.Module, want, err) - } - for key, value := range served { - // The provider's own vocabulary. Rendered plainly: a port is 5432, not 5432.000000, - // which is what a float would write and what a connection string would refuse. - values[key] = plainly(value) - } - out[want] = values - } - } - return out, nil -} - -// withOwnNames adds a module's own composed names to what it may name from one binding: -// `${bound::name}` and `:internal-name`, and for several contributions to one requirement -// `:name-` / `:internal-name-`. Set over anything the provider serves under those keys: -// what the module is called is the mesh's statement, not the provider's. -func withOwnNames(values map[string]string, own map[string]any) { - for _, key := range []string{"name", "internal-name"} { - if v, ok := own[key].(string); ok { - values[key] = v - } - } - many, _ := own["names"].(map[string]any) - for local, raw := range many { - names, _ := raw.(map[string]any) - for _, key := range []string{"name", "internal-name"} { - if v, ok := names[key].(string); ok { - values[key+"-"+local] = v - } - } - } -} - -// plainly renders a served value as a program would expect to read it. -func plainly(value any) string { - switch v := value.(type) { - case string: - return v - case float64: - if v == float64(int64(v)) { - return fmt.Sprintf("%d", int64(v)) - } - return strings.TrimRight(strings.TrimRight(fmt.Sprintf("%f", v), "0"), ".") - case bool: - return fmt.Sprintf("%t", v) - case nil: - return "" - default: - return fmt.Sprint(v) - } -} - -// boundInto replaces a file's ${bound:…} placeholders with what the mesh knows. -// -// A placeholder naming something the module does not require, or a key the provider does not -// serve, is refused. Left as it was, the literal `${bound:x:y}` would be written into a -// configuration file and read as a value — a connection to a host called `${bound:x:y}`, failing -// somewhere that names neither the module nor the mesh. -// -// **An optional requirement left absent takes the default its placeholder names** (novox/hq ADR -// 0330 §4), and one that names none is refused rather than rendered empty. The module check refuses -// that already; this is the same rule where the file is made, for a manifest registered before it. -func boundInto(resource map[string]any, known map[string]map[string]string, module string, absent map[string]bool) error { - if fmt.Sprint(resource["type"]) != "file" { - return nil - } - content, ok := resource["content"].(string) - if !ok { - return nil - } - var refused error - filled := bound.ReplaceAllStringFunc(content, func(token string) string { - if refused != nil { - return token - } - m := bound.FindStringSubmatch(token) - provision, key, fallback := m[1], m[2], m[3] - values, has := known[provision] - if !has { - if absent[provision] { - if fallback != "" { - return fallback - } - refused = fmt.Errorf( - "%s has a file that says ${bound:%s:%s}, and %q is optional and absent here: it names no "+ - "value for then (${bound:%s:%s|default:}), so it would be written empty "+ - "(novox/hq ADR 0330 §4)", module, provision, key, provision, provision, key) - return token - } - refused = fmt.Errorf( - "%s has a file that says ${bound:%s:%s}, and %s does not require %q. It may name %s", - module, provision, key, module, provision, orNothing(namesOfBindings(known))) - return token - } - value, said := values[key] - if !said { - refused = fmt.Errorf( - "%s asks its %s binding for %q, and what answers it says %s", - module, provision, key, orNothing(namesOfKeys(values))) - return token - } - return value - }) - if refused != nil { - return refused - } - resource["content"] = filled - return nil -} - -func namesOfBindings(known map[string]map[string]string) []string { - var names []string - for name := range known { - names = append(names, fmt.Sprintf("%q", name)) - } - sort.Strings(names) - return names -} - -func namesOfKeys(values map[string]string) []string { - var names []string - for key := range values { - names = append(names, fmt.Sprintf("%q", key)) - } - sort.Strings(names) - return names -} - -func orNothing(names []string) string { - if len(names) == 0 { - return "nothing" - } - return join(names) -} diff --git a/internal/catalogue/build.go b/internal/catalogue/build.go deleted file mode 100644 index e36de7f5..00000000 --- a/internal/catalogue/build.go +++ /dev/null @@ -1,651 +0,0 @@ -package catalogue - -import ( - "fmt" - "net" - "net/url" - "path" - "regexp" - "sort" - "strings" -) - -// Turning a manifest that names artifacts into one that names digests. -// -// **Two documents, deliberately.** The manifest in a repository says *this resource uses the -// archive called `config`*; the manifest the mesh holds says *this resource is sha256:…*. A digest -// is not knowable until something is built, so a repository carrying one would be a repository -// whose file is wrong the moment anybody edits anything — and the mesh would be pinning a value -// nobody could have checked. -// -// So the built manifest is **derived**, and the record of which commit it was derived from is what -// makes "is this current?" answerable without building (novox/hq ADR 0009). - -// Built is one artifact after it exists: where it is and what it hashes to. -type Built struct { - // Name is what the manifest called it. - Name string - // Kind is "image" or "archive". - Kind string - // Reference is what a machine uses to get it — an image reference for an image, a URL for an - // archive. Both already carry the digest for an image; an archive carries it separately. - Reference string - // Digest is "sha256:", for an archive. An image reference already ends in one. - Digest string - // Launchers are, for a bundle in an interpreted language, the executable the build wrote beside - // each entrypoint, by entrypoint (novox/hq ADR 0193): what the node's runtime starts to serve it. - Launchers map[string]string - // Texts are, for the archive a module packs its templates into (TemplatesArtifact), each file's text as the - // build read it back from the packed archive, by its path inside it (novox/hq ADR 0349 §2). Never carried on - // the wire or kept: Resolve reads them into the files' content, which is what the mesh holds. - Texts map[string]string `json:"-"` -} - -// Resolve fills a manifest's resources in from what was built. -// -// Every resource naming an artifact is rewritten to name the thing itself, and the `artifact` key -// is removed — because it is a build-time word and the host has never heard of it. A resource -// naming an artifact nothing produced is refused: it would otherwise reach a machine with an -// empty image or an unpinned archive, which is the shape of failure that looks like success. -func (m Manifest) Resolve(built []Built) (Manifest, error) { - if m.Build == nil && len(built) == 0 { - return m, nil - } - - by := map[string]Built{} - for _, b := range built { - by[b.Name] = b - } - // Declared and not produced is a build that did not do what the manifest asked, and saying so - // here beats a machine reporting it later. - var missing []string - if m.Build != nil { - for _, a := range m.Build.Artifacts { - if _, ok := by[a.Name]; !ok { - missing = append(missing, a.Name) - } - } - } - if len(missing) > 0 { - sort.Strings(missing) - return Manifest{}, fmt.Errorf( - "%s says it builds %s and the build did not produce %s — the build did not do what "+ - "the manifest asked, which is a different fault from a resource asking for the "+ - "wrong thing", - m.Module, strings.Join(missing, " and "), oneOrOther(len(missing))) - } - - // **Each template's text, as the build read it back from its templates archive** (novox/hq ADR 0349 §2): into - // the file's content, so what the mesh holds and composes is what the same text inline would be. - var handed *Handed - if m.Templated() { - name := TemplatesArtifact(m) - texts := by[name].Texts - if name == "" || texts == nil { - return Manifest{}, fmt.Errorf("%s names templates, and the build carried no text read back from the "+ - "archive it packs %s/ into (%q): the builder that built it does not read templates", - m.Module, m.TemplatesDir(), name) - } - dir := m.TemplatesDir() - filled, err := fillTemplates(m.Module, dir, m.Resources, func(template string) (string, bool) { - t, ok := texts[strings.TrimPrefix(template, dir+"/")] - return t, ok - }, fmt.Sprintf("the archive %q", name)) - if err != nil { - return Manifest{}, err - } - items, err := fillItemTemplates(m, m.Items, func(template string) (string, bool) { - t, ok := texts[strings.TrimPrefix(template, dir+"/")] - return t, ok - }, fmt.Sprintf("the archive %q", name)) - if err != nil { - return Manifest{}, err - } - // And, for a moved module, where its templates are and which one each file and item is filled from - // (handed.go): read while its resources and items still name their templates. - handed = handedBy(m, by[name]) - m.Resources = filled - m.Items = items - } - - out := m - out.Handed = handed - out.Build = nil - // Read with the build, like Build: the manifest the mesh holds carries each file's text (issue 731). - out.Templates = "" - out.Resources = nil - // What the build compiled, kept on the resolved manifest (novox/hq ADR 0175): a tools bundle is - // named by no resource of the module's own — the node's runtime loads it — so this is the only - // place the mesh would otherwise not have it. In artifact order, so two resolutions of one - // build compare equal. - out.Bundles = nil - if m.Build != nil { - run := runByAResource(m) - for _, a := range m.Build.Artifacts { - if a.Kind != ArtifactBundle { - continue - } - made := by[a.Name] - // What the runtime loads: what the artifact said, else every entrypoint of a module - // that declares tools, else nothing (the field's own rule; see Artifact.Loads). - // - // **Never, unasked, a bundle one of the module's own resources runs** (novox/hq issue 213). - // A process the host runs is the module's service, not its tools: the controller declares - // the verbs it answers as `tools` and serves them itself, and its bundle would otherwise - // have been launched a second time by the node's runtime, as an MCP child it is not. - loads := append([]string(nil), a.Loads...) - if a.Loads == nil && len(m.Tools) > 0 && !run[a.Name] { - loads = append([]string(nil), a.Entrypoints...) - // A bundle compiled to a binary has no entrypoints: the binary is what it is, and what - // the runtime starts to serve it (novox/hq ADR 0193). So a Go tools bundle is served - // as Go — the runtime execs it — exactly as a TypeScript one is through its launcher. - if bin := BinaryOf(a); bin != "" { - loads = []string{bin} - } - } - // **Kept, never routed** (ADR 0155): the builder publishes to the store at the address - // it reached it by, and a manifest carrying that address names an installation — - // registration refused node-tools for exactly this on 2026-10-02. The build record - // already keeps the store-relative form; the resolved manifest keeps the same, and - // composition routes it through the store a machine reaches (Routed). - out.Bundles = append(out.Bundles, Bundle{ - Name: a.Name, Source: Recorded(made.Reference), Digest: made.Digest, - Language: a.Language, Entrypoints: append([]string(nil), a.Entrypoints...), - Loads: loads, Env: copyWords(a.Env), Launchers: copyWords(made.Launchers), - Binary: BinaryOf(a), - }) - } - sort.Slice(out.Bundles, func(i, j int) bool { return out.Bundles[i].Name < out.Bundles[j].Name }) - } - for _, r := range m.Resources { - named, _ := r["artifact"].(string) - if named == "" { - out.Resources = append(out.Resources, r) - continue - } - artifact, ok := by[named] - if !ok { - return Manifest{}, fmt.Errorf( - "%s: %v uses the artifact %q, and this module builds no such thing", - m.Module, r["id"], named) - } - - filled := map[string]any{} - for k, v := range r { - filled[k] = v - } - delete(filled, "artifact") - switch artifact.Kind { - case ArtifactPackage: - // A package is not a resource on any machine; it is consumed by other builds. A - // resource that names one is a manifest error, named here rather than shipped. - return Manifest{}, fmt.Errorf( - "%s: %v uses %q, which is a package — a build input, not a resource a machine runs", - m.Module, r["id"], named) - case ArtifactImage, ArtifactUpstream: - filled["image"] = artifact.Reference - // An image is not unpacked anywhere, so it has no directory to be named for its - // version and `${version}` has nothing to mean. Refused rather than left as literal - // text in a path, which is how it would reach a machine and be created as a directory - // called `${version}`. - for key, value := range filled { - if text, isText := value.(string); isText && strings.Contains(text, versionRef) { - return Manifest{}, fmt.Errorf( - "%s: %v says %s in %q, and %q is an image — an image is not unpacked, so "+ - "it has no versioned place. %s is for an archive or a bundle", - m.Module, r["id"], versionRef, key, named, versionRef) - } - } - case ArtifactArchive, ArtifactBundle: - // The same on the wire: both are bytes fetched by digest and unpacked. They differ in - // how they were made — one packed as it stood, the other compiled first — and a - // machine has no reason to care which. - // Kept, not routed, for the reason the bundles above are (ADR 0155). - filled["source"] = Recorded(artifact.Reference) - filled["digest"] = artifact.Digest - // **And `${version}`, so a resource can name a place that is this build's alone** - // (novox/hq ADR 0141, 04-ISSUES/142). A component is unpacked into a directory named - // for its version so it can read its own version from its path — and until this, - // nothing could compose that path: an archive named a fixed one in the manifest and - // nothing interpolated the build into it, so nothing could ask for - // `…/versions//` and every machine took a hand-placed fallback. - // - // The version is the artifact's own digest, short. Not the commit: two builds of one - // commit are meant to be the same bytes (the toolchains are `-trimpath` for this), and - // a content-addressed version means an unchanged build resolves to the path it already - // had — so re-composing a declaration moves nothing, where a commit would move the - // path of an identical binary and recreate everything that reads it. - for key, value := range filled { - filled[key] = withVersion(value, versionOf(artifact.Digest)) - } - default: - return Manifest{}, fmt.Errorf("%s: %q is a %q, and an artifact is %q, %q, %q or %q", - m.Module, named, artifact.Kind, ArtifactImage, ArtifactArchive, ArtifactUpstream, - ArtifactBundle) - } - out.Resources = append(out.Resources, filled) - } - return out, nil -} - -// withVersion is a resource's value with `${version}` filled: in a string, and in each string of a map — -// a process's env, where the controller is told which build it is (novox/hq issue 352). Anything else is -// left as it is. -func withVersion(value any, version string) any { - switch v := value.(type) { - case string: - if strings.Contains(v, versionRef) { - return strings.ReplaceAll(v, versionRef, version) - } - case map[string]any: - out := make(map[string]any, len(v)) - for k, x := range v { - out[k] = withVersion(x, version) - } - return out - case map[string]string: - out := make(map[string]string, len(v)) - for k, x := range v { - out[k] = strings.ReplaceAll(x, versionRef, version) - } - return out - } - return value -} - -// checkBuild is the manifest's own account of what it builds. -func (b *Build) problems(module string) []string { - if b == nil { - return nil - } - var problems []string - seen := map[string]bool{} - for _, a := range b.Artifacts { - if a.Name == "" { - problems = append(problems, module+" builds an artifact with no name") - continue - } - if seen[a.Name] { - problems = append(problems, fmt.Sprintf( - "%s builds two artifacts called %q, and a resource naming it could mean either", - module, a.Name)) - } - seen[a.Name] = true - switch a.Kind { - case ArtifactImage, ArtifactArchive, ArtifactUpstream, ArtifactBundle, ArtifactPackage: - default: - problems = append(problems, fmt.Sprintf( - "%s: %q is a %q, and an artifact is %q, %q, %q or %q", - module, a.Name, a.Kind, ArtifactImage, ArtifactArchive, ArtifactUpstream, - ArtifactBundle+", "+ArtifactPackage)) - } - // **A bundle is built from the module itself, so it says a language instead.** Everything - // else names what it is built from: a Dockerfile, a directory, somebody else's reference. - // A bundle's source is the module's own directory by definition, and what it needs to say - // is which compiler — because the mesh chooses that, and cannot choose for a module that - // has not said. - if len(a.External) > 0 && (a.Kind != ArtifactBundle || a.Language != "typescript") { - problems = append(problems, fmt.Sprintf( - "%s: %q names packages it keeps external, and only a TypeScript bundle is bundled into "+ - "one file with some kept out (novox/hq ADR 0193)", module, a.Name)) - } - if len(a.Env) > 0 && a.Kind != ArtifactBundle { - problems = append(problems, fmt.Sprintf( - "%s: %q is a %q and says what it is given (env). Only a bundle the node's runtime "+ - "serves is given words (novox/hq ADR 0192); a container says its own environment", - module, a.Name, a.Kind)) - } - if a.Kind == ArtifactBundle || a.Kind == ArtifactPackage { - // **Except for a language that compiles to a binary, where it names which one** - // (novox/hq 04-ISSUES/142). A bundle in an interpreted language is the module's own - // directory compiled whole, and naming a source would be describing its own build. A - // repository written in a compiled language holds several commands — the host and its - // bootstrap live in one, and the mesh needs the host — and "the module's own directory" - // is then not a package at all. So the compiled case may say which package, and says - // the module root by saying nothing. - if a.From != "" && !compilesToABinary(a.Language) { - problems = append(problems, fmt.Sprintf( - "%s: %q is a bundle and names what it is built from (%q). A bundle is built "+ - "from the module's own directory; what it says is the language", - module, a.Name, a.From)) - } - if strings.TrimSpace(a.Language) == "" { - problems = append(problems, fmt.Sprintf( - "%s: %q is a bundle and says no language, so nothing can choose a compiler "+ - "for it", module, a.Name)) - } - problems = append(problems, bundleEnvProblems(module, a)...) - // What the runtime loads is among what was compiled (ADR 0175): a name here that is - // not an entrypoint is a file the bundle does not contain, and the runtime would - // fail to import it on every machine rather than here. - for _, load := range a.Loads { - found := false - for _, e := range a.Entrypoints { - found = found || e == load - } - // A bundle compiled to a binary is one executable: the runtime loads that or nothing - // (novox/hq ADR 0193). - if bin := BinaryOf(a); bin != "" { - found = load == bin - } - if !found { - problems = append(problems, fmt.Sprintf( - "%s: %q says the runtime loads %q, which is not among its entrypoints — "+ - "what is loaded is compiled, so it is named there too", module, a.Name, load)) - } - } - // **A system, for a language that compiles to a binary** (novox/hq ADR 0142). A binary - // is pinned to one operating system at link time so a host refuses to touch a machine - // it was not built for (novox/hq ADR 0005); an artifact that says nothing would be - // compiled for whatever the build machine happened to be, which reads as portable and - // is not. - if compiled := compilesToABinary(a.Language); compiled && strings.TrimSpace(a.System) == "" { - problems = append(problems, fmt.Sprintf( - "%s: %q is compiled to a binary and says no system, so it would be built for "+ - "whatever the build machine happens to be. Declare one artifact per "+ - "system: %s", module, a.Name, spokenSystems())) - } else if !compiled && strings.TrimSpace(a.System) != "" { - problems = append(problems, fmt.Sprintf( - "%s: %q names the system %q and is written in %q, which compiles to code that "+ - "runs anywhere — a system that decides nothing reads as though it did", - module, a.Name, a.System, a.Language)) - } else if compiled && !knownSystem(a.System) { - problems = append(problems, fmt.Sprintf( - "%s: %q is built for %q, and a system is %s", - module, a.Name, a.System, spokenSystems())) - } - } else { - if a.From == "" && a.URL == "" { - problems = append(problems, fmt.Sprintf( - "%s: %q says nothing about what it is built from", module, a.Name)) - } - if a.Language != "" { - problems = append(problems, fmt.Sprintf( - "%s: %q is a %q and names a language. Only a bundle is compiled by the mesh; "+ - "everything else brings its own recipe", module, a.Name, a.Kind)) - } - } - if a.Compiles != "" { - if a.Kind != ArtifactImage { - problems = append(problems, fmt.Sprintf( - "%s: %q is a %q and names a Go package it compiles. Only an image's recipe is "+ - "told which; a bundle names its package in from (novox/hq ADR 0267)", module, a.Name, a.Kind)) - } else if c := strings.TrimSpace(a.Compiles); strings.HasPrefix(c, "/") || c == ".." || - strings.HasPrefix(c, "../") || strings.Contains(c, "/../") || strings.HasSuffix(c, "/..") { - problems = append(problems, fmt.Sprintf( - "%s: %q compiles %q, which leaves the tree it is built in", module, a.Name, a.Compiles)) - } - } - // An upstream image is named, not read from the repository, so the path rule does not - // apply to it — and applying it anyway would refuse every reference with a registry host - // in it. - if a.Kind != ArtifactUpstream && - (strings.HasPrefix(a.From, "/") || strings.Contains(a.From, "..")) { - // A build reads its own repository and nothing else. A path leaving it would make - // what gets built depend on whatever happens to be on the machine building it. - problems = append(problems, fmt.Sprintf( - "%s: %q is built from %q, which is outside its own repository", - module, a.Name, a.From)) - } - if a.Kind == ArtifactUpstream && !strings.Contains(a.From, ":") { - // Without a tag or digest, what gets mirrored is whatever `latest` means today, and - // a module pinned to that is not pinned. - problems = append(problems, fmt.Sprintf( - "%s: %q mirrors %q, which names no tag or digest", module, a.Name, a.From)) - } - problems = append(problems, upstreamFileProblems(module, a)...) - } - return problems -} - -// reservedSuffixes are names no public vendor publishes under: an installation's own and documentation's. -var reservedSuffixes = []string{".internal", ".local", ".lan", ".home", ".corp", ".localhost", ".home.arpa", - ".test", ".invalid", ".example", ".intranet", ".private"} - -// notAPublicHost says why a host is not a public vendor's name, or "". -func notAPublicHost(host string) string { - h := strings.ToLower(strings.TrimSuffix(host, ".")) - if net.ParseIP(h) != nil { - return "an address rather than a name" - } - if !strings.Contains(h, ".") { - return "a name with no domain" - } - for _, s := range reservedSuffixes { - if strings.HasSuffix(h, s) { - return "a name under " + s + ", which is nobody's public domain" - } - } - return "" -} - -// sha256Hex is a pinned sha256 as a manifest writes it: 64 lower-case hexadecimal characters. -var sha256Hex = regexp.MustCompile(`^[0-9a-f]{64}$`) - -// upstreamFileProblems is what is wrong with an archive pinned by URL (novox/hq ADR 0316 §2, §6): it is an -// archive, fetched over https, pinned by a sha256 and a version, and it names no directory of the repository -// besides; a file name or a directory within is a plain path that stays where it is put. -func upstreamFileProblems(module string, a Artifact) []string { - pinned := a.URL != "" || a.SHA256 != "" || a.Version != "" || a.File != "" || a.Within != "" - if !pinned { - return nil - } - var problems []string - say := func(format string, args ...any) { - problems = append(problems, fmt.Sprintf("%s: %q ", module, a.Name)+fmt.Sprintf(format, args...)) - } - if a.Kind != ArtifactArchive { - say("is a %q and names an upstream file (url, sha256, version, file, within): only an archive is "+ - "mirrored from one (novox/hq ADR 0316)", a.Kind) - return problems - } - if a.URL == "" { - say("names a sha256, a version, a file or a directory within, and no url to fetch it from") - return problems - } - if a.From != "" { - say("is built from %q and fetched from %s: an archive is one or the other", a.From, a.URL) - } - if u, err := url.Parse(a.URL); err != nil || u.Scheme != "https" || u.Host == "" { - say("is fetched from %q, and an upstream file is fetched over https from a named host", a.URL) - } else if why := notAPublicHost(u.Hostname()); why != "" { - say("is fetched from %s, %s: an upstream file is somebody else's release, published under a public name", a.URL, why) - } - if !sha256Hex.MatchString(a.SHA256) { - say("is fetched from %s without a pinned sha256 (64 lower-case hexadecimal characters): what that "+ - "address serves changes under the mesh, and only the digest says it is what was reviewed", a.URL) - } - if strings.TrimSpace(a.Version) == "" { - say("pins %s and says no version, so a person reading the pin cannot tell which release it is", a.URL) - } - if a.File != "" && a.Within != "" { - say("names a file (%q) and a directory within (%q): a file is kept whole, a directory is taken out "+ - "of an archive", a.File, a.Within) - } - for field, p := range map[string]string{"file": a.File, "within": a.Within} { - if p == "" { - continue - } - clean := path.Clean(p) - if clean != p || strings.HasPrefix(p, "/") || clean == "." || clean == ".." || strings.HasPrefix(clean, "../") || - (field == "file" && strings.Contains(p, "/")) { - say("names %s %q, which is not a plain relative %s", field, p, map[string]string{ - "file": "file name", "within": "directory"}[field]) - } - } - return problems -} - -// oneOrOther keeps the message readable for one artifact and for several, because a message that -// says "neither" about one thing reads as a bug in the message. -func oneOrOther(n int) string { - if n == 1 { - return "it" - } - return "them" -} - -// Systems the mesh builds binaries for, which is the set a host may be pinned to (novox/hq ADR 0005). -// -// **A closed list, and the host's own, not the compiler's.** These are not the values a Go toolchain -// would call an operating system — the difference between two of them is a C library, not a kernel. -// They are what a machine reports itself to be and what a host is linked to refuse, so the list that -// matters is the one the host understands. -var systems = []string{"alpine", "android", "arch"} - -// knownSystem is whether the mesh builds for it. -func knownSystem(system string) bool { - want := strings.ToLower(strings.TrimSpace(system)) - for _, s := range systems { - if s == want { - return true - } - } - return false -} - -// spokenSystems is the list as a refusal says it, so a reader is one edit from right. -func spokenSystems() string { - return strings.Join(systems, ", ") -} - -// compilesToABinary is whether this language's bundle is a binary for one operating system rather -// than code that runs wherever its interpreter does. -// -// **Asked of the language, not of the artifact.** A module says what it is written in; what that -// implies is the mesh's to know, exactly as the compiler is (novox/hq ADR 0142). Asking the artifact -// would let two artifacts in one language disagree about whether they are portable. -func compilesToABinary(language string) bool { - switch strings.ToLower(strings.TrimSpace(language)) { - case "go": - return true - default: - return false - } -} - -// versionRef is how a resource names the version of the artifact it uses: ${version}. -// -// No artifact name in it, because the resource already says which artifact it is for — a second -// name would be a second thing to keep in step with the first. -const versionRef = "${version}" - -// versionOf is an artifact's version as a path names it: its digest, short. -// -// **Content-addressed on purpose.** The alternative is the commit a build came from, and two builds -// of one commit are meant to produce the same bytes — every toolchain here is `-trimpath` for that -// reason. A commit-named path would move for an identical binary, and everything reading that path -// would be recreated for a change that is not one. A digest-named path moves exactly when the bytes -// do. -// -// Twelve hex characters: enough that two of this mesh's builds will not collide, short enough to -// read in a path and in a journal line. The `sha256:` prefix goes, because a directory name carrying -// a colon is a directory name people quote wrong. -func versionOf(digest string) string { - hex := strings.TrimPrefix(strings.TrimSpace(digest), "sha256:") - if len(hex) > 12 { - return hex[:12] - } - return hex -} - -// bundleEnvWords are the words the runtime sets for itself; a bundle that named one would be -// telling the runtime what it is, which is the mesh's to say (novox/hq ADR 0192). -var bundleEnvWords = map[string]bool{ - RuntimeToolModules: true, RuntimeBrokerFile: true, RuntimeOperatorAccount: true, - RuntimeOperatorHome: true, RuntimeToolEnv: true, RuntimeAgentAccount: true, RuntimeAgentHome: true, -} - -// bundleEnvProblems says what is wrong with what a bundle says it is given (novox/hq ADR 0192): -// a value is a path or a constant written with the references a container's environment may use -// for a place or a port, and never a secret's content or another module's binding — a secret -// reaches a tool as a file whose path is named. -func bundleEnvProblems(module string, a Artifact) []string { - if len(a.Env) == 0 { - return nil - } - var problems []string - for _, word := range sortedKeys(a.Env) { - value := a.Env[word] - if bundleEnvWords[word] { - problems = append(problems, fmt.Sprintf( - "%s: %q gives itself %s, which the node's runtime sets for itself; a bundle is "+ - "given its own words beside the runtime's, never in place of them (novox/hq ADR 0192)", - module, a.Name, word)) - } - rest := ofPort.ReplaceAllString(dirRef.ReplaceAllString(value, ""), "") - if strings.Contains(rest, "${") { - problems = append(problems, fmt.Sprintf( - "%s: %q gives %s the value %q. A bundle's word is a path or a constant, written with "+ - "${dir:…} and ${port:…} only; a secret reaches a tool as a file the mesh places, "+ - "named by its path, never as its content (novox/hq ADR 0192)", - module, a.Name, word, value)) - } - } - return problems -} - -func copyWords(in map[string]string) map[string]string { - if len(in) == 0 { - return nil - } - out := make(map[string]string, len(in)) - for k, v := range in { - out[k] = v - } - return out -} - -// BinaryOf is what a bundle compiled to a binary is called once built: what the artifact names, else -// the package it is built from, else the artifact's own name (novox/hq 04-ISSUES/142). Empty for a -// language that does not compile to one. The builder writes the binary under this name, and the -// composer runs it by it, so both ask here. -func BinaryOf(a Artifact) string { - if !compilesToABinary(a.Language) { - return "" - } - if name := strings.TrimSpace(a.Binary); name != "" { - return name - } - if from := strings.Trim(a.From, "./"); from != "" { - return path.Base(from) - } - return a.Name -} - -// runByAResource is the artifacts one of a module's own resources names — a process that runs it, -// a step, an archive that unpacks it — by name. -func runByAResource(m Manifest) map[string]bool { - named := map[string]bool{} - for _, r := range m.Resources { - if a, ok := r["artifact"].(string); ok && a != "" { - named[a] = true - } - } - return named -} - -// undeliveredBundles says which of a module's bundles nothing would ever put on a machine (novox/hq -// 04-ISSUES/216). A bundle reaches a machine three ways: the node's runtime serves it (it says -// `loads`, or its module declares `tools`), a resource names it (a process, a step, an archive), or -// it is the runtime itself. One reached by none of them was built, recorded and pushed as success, -// and was simply absent — seven modules' tools went missing that way on 2026-10-03. Refused here, -// naming the field that would deliver it. -func undeliveredBundles(m Manifest) []string { - if m.Build == nil || m.Module == RuntimeModule { - return nil - } - named := runByAResource(m) - var problems []string - for _, a := range m.Build.Artifacts { - if a.Kind != ArtifactBundle || named[a.Name] || len(a.Loads) > 0 || len(m.Tools) > 0 { - continue - } - problems = append(problems, fmt.Sprintf( - "%s: the bundle %q would be built and never reach a machine: nothing loads it, runs it or "+ - "unpacks it. A tools bundle says `loads` (the entrypoints the node's runtime serves) or its "+ - "module lists its `tools`; a daemon or a step is a resource naming it (novox/hq 04-ISSUES/216)", - m.Module, a.Name)) - } - return problems -} diff --git a/internal/catalogue/busy.go b/internal/catalogue/busy.go deleted file mode 100644 index 18cbbf02..00000000 --- a/internal/catalogue/busy.go +++ /dev/null @@ -1,124 +0,0 @@ -package catalogue - -import ( - "fmt" - "sort" - "strings" - "time" -) - -// A long-running container may say it is busy (novox/hq ADR 0335 §1). -// -// **Beside its health, on the container**: a command run inside it, by its shell, as an exec health check -// is, that exits 0 while the container holds work a replacement would lose and non-zero when it is idle; -// and how long one ask may take. Being asked is also its notice that it is about to be replaced: from the -// first ask it takes no new work. The node-engine asks before it replaces the container, leaves it running -// while it says busy, and replaces it on the first pass it says idle, or at 15 minutes — the engine's bound, -// never the module's. -// -// Sent only to a node-engine that reads it (link.BusyContract): an older, strict engine refuses a field it -// does not know, whole, and to it the container is replaced at once, as before. - -// BusyField is the resource field. -const BusyField = "busy" - -// The bounds of one ask (ADR 0335), the node-engine's own, held here near the author. -const ( - BusyTimeoutDefault = 10 * time.Second - BusyTimeoutCeiling = time.Minute -) - -// Busy is one container's busy, read. -type Busy struct { - Command string - Timeout time.Duration -} - -// ReadBusy reads a resource's `busy` field, its default applied. False when it carries none. -func ReadBusy(r map[string]any) (Busy, bool, []string) { - raw, present := r[BusyField] - if !present { - return Busy{}, false, nil - } - id := fmt.Sprint(r["id"]) - fields, ok := raw.(map[string]any) - if !ok { - return Busy{}, true, []string{fmt.Sprintf("%s: busy is %T; it is an object with a command", id, raw)} - } - var problems []string - keys := make([]string, 0, len(fields)) - for k := range fields { - keys = append(keys, k) - } - sort.Strings(keys) - for _, k := range keys { - if k != "command" && k != "timeout" { - problems = append(problems, fmt.Sprintf("%s: its busy says %q; busy has a command and a timeout", id, k)) - } - } - b := Busy{Timeout: BusyTimeoutDefault} - if v, has := fields["command"]; has { - s, ok := v.(string) - if !ok { - problems = append(problems, fmt.Sprintf("%s: its busy's command is %T; it is text", id, v)) - } - b.Command = strings.TrimSpace(s) - } - if v, has := fields["timeout"]; has { - s, _ := v.(string) - d, err := time.ParseDuration(strings.TrimSpace(s)) - switch { - case err != nil: - problems = append(problems, fmt.Sprintf("%s: its busy's timeout is %v; it is a duration such as \"10s\"", id, v)) - case d <= 0 || d > BusyTimeoutCeiling: - problems = append(problems, fmt.Sprintf("%s: an ask of its busy takes more than nothing and at most %s, not %s", - id, BusyTimeoutCeiling, d)) - default: - b.Timeout = d - } - } - return b, true, problems -} - -// busyProblems is everything wrong with a manifest's `busy` fields, in the manifest's words: a command on -// one line, a timeout within its bounds, and only on a container that stays up. -func busyProblems(m Manifest) []string { - var problems []string - for _, r := range m.Resources { - b, has, read := ReadBusy(r) - if !has { - continue - } - where := m.Module + ": " + fmt.Sprint(r["id"]) - for _, p := range read { - problems = append(problems, m.Module+": "+p) - } - if fmt.Sprint(r["type"]) != "container" || !LongRunning(r) { - problems = append(problems, fmt.Sprintf("%s declares busy and is not a container that stays up: busy is "+ - "asked of a running container before it is replaced (ADR 0335)", where)) - } - if b.Command == "" { - problems = append(problems, where+"'s busy names no command: one run inside the container, exiting 0 "+ - "while it is busy") - } - if strings.ContainsAny(b.Command, "\n\r") { - problems = append(problems, where+"'s busy command is one line") - } - } - return problems -} - -// busyInto composes a container's `busy` into the node-engine's words, or takes it away: for an engine that -// reads it, the command and its timeout written out; for one that does not, the field is not sent, and the -// container is replaced at once, as before. -func busyInto(resource map[string]any, with Rendering) { - if _, has := resource[BusyField]; !has { - return - } - if !with.ReadsBusy { - delete(resource, BusyField) - return - } - b, _, _ := ReadBusy(resource) - resource[BusyField] = map[string]any{"command": b.Command, "timeout": b.Timeout.String()} -} diff --git a/internal/catalogue/chosen.go b/internal/catalogue/chosen.go deleted file mode 100644 index 1fb8233c..00000000 --- a/internal/catalogue/chosen.go +++ /dev/null @@ -1,100 +0,0 @@ -package catalogue - -import ( - "sort" -) - -// Chosen is the provider somebody named for a provision: the module, and the node it runs on. Both, -// always (novox/hq #258) — a provision comes from a module, and the same module on two machines is -// two answers, so neither half alone says which. Module is empty only on a record made before this -// was asked, and such a record is honoured exactly as long as it is unambiguous. -type Chosen struct { - Node string - Module string -} - -func (c Chosen) String() string { - if c.Module == "" { - return c.Node - } - return c.Node + "/" + c.Module -} - -// matches is whether this provider is the one chosen. -func (c Chosen) matches(p Provider) bool { - return p.Node == c.Node && (c.Module == "" || p.Module == c.Module) -} - -// among is every offered provider the choice names — one, when the choice is whole. -func (c Chosen) among(where []Provider) []Provider { - var out []Provider - for _, p := range where { - if c.matches(p) { - out = append(out, p) - } - } - return out -} - -// nameOf is how a refusal names a provider: the node and the module on it. -func nameOf(p Provider) string { - return Chosen{Node: p.Node, Module: p.Module}.String() -} - -// providerNames is every provider named, sorted, for a refusal to list. -func providerNames(where []Provider) []string { - out := make([]string, 0, len(where)) - for _, p := range where { - out = append(out, nameOf(p)) - } - sort.Strings(out) - return out -} - -// providersHere is which modules in this node's own set offer a provision, sorted. -func providersHere(catalogue map[string]Manifest, here func(string) bool, want string) []string { - var out []string - for name, m := range catalogue { - if !here(name) { - continue - } - for _, o := range m.Offers() { - if o == want { - out = append(out, name) - break - } - } - } - sort.Strings(out) - return out -} - -// servedByOne is what one provider beside the consumer says a consumer needs to know, or nothing. -// -// Serving is *whether* a need is created at all when the provider is on this same machine (novox/hq -// 04-ISSUES/038's sibling): a need never created is a binding the consumer never gets. The manifest -// alone answers that; the values are settled later, with the node's settings. -func servedByOne(m Manifest, want string) map[string]any { - if _, ok := m.Serves[want]; ok { - return ServedOn(m, want, nil) - } - return nil -} - -// sharedByOne is the own secret that provider names as its credential (ADR 0158), or "" when it -// gives each consumer its own. -func sharedByOne(m Manifest, want string) string { - if own, shared := m.SharedCredentialOf(want); shared { - return own - } - return "" -} - -func oneOf(list []string, s string) bool { - for _, x := range list { - if x == s { - return true - } - } - return false -} diff --git a/internal/catalogue/collections.go b/internal/catalogue/collections.go deleted file mode 100644 index c2a65622..00000000 --- a/internal/catalogue/collections.go +++ /dev/null @@ -1,635 +0,0 @@ -package catalogue - -import ( - "fmt" - "path" - "regexp" - "slices" - "sort" - "strconv" - "strings" -) - -// Collections (novox/hq ADR 0349 clause 13, design 57 L1, issue 684): a provision with many providers. -// -// **A collector requires a collection; a contributor provides an item from a template.** A module taking additions -// from others names a collection in a file of its own, `${collected:}`, and declares it under -// `collections`, with the comment its tool writes a line in. Each contributor ships its item as a template of its -// own repository and declares it under `items`: `{"collection": …, "template": "files/"}`. Its build packs the -// template with the module's others, and the mesh holds its text as it holds a file's (template.go). -// -// **The node's provisioner client fills both.** An item is filled on the node by the client of the node its -// contributor is placed on, from the same owners as a file, and recorded in mesh-provisioner_collect; a collector's -// file is filled by the same client from every item of its collection recorded for that node, so an item goes when -// its contributor leaves. So a module contributing an item is a moved module (handed.go), and the controller renders -// no item: the plan of a contributor names its items and their templates, and the plan of a collector names each -// collection it reads and its comment. Neither carries a contributor's text. -// -// **The three old namespaces read as collections.** `${shell::}`, `${environment:}` and -// `${contribution::}` name, in a handed collector's file, the collection of the same name — -// `shell:zsh:normal` — with the comment the controller wrote before each module's piece. An item is contributed to -// one by that name, by a module the old form would have let contribute it. Each file is filled one way, never -// both (ADR 0349 clause 17): a handed collector is filled from items alone and an unhanded one from the old forms -// alone, and a piece of the other form on the same machine is said as not placed, never dropped quietly. - -// Item is one item a module contributes to a collection: its template in the module's repository, or, in the -// manifest the mesh holds, the template's text, read back from the build's templates archive like a file's. -// -// **Attributes, a capability and a name** (design 57 L3, issue 777): an item may carry `attributes` (string keys and -// values; `order`, a whole number 0–99 defaulting to 50, orders the items of one placeholder) and `if-capability`, -// the capability a machine must report for the item to be placed on it. A module may give several items to one -// collection, each then with a `name` unique within the module and the collection. A collector takes only the items -// whose attributes match, `${collected::=,…}`, and all of them without the selector. -type Item struct { - Collection string `json:"collection"` - Name string `json:"name,omitempty"` - Template string `json:"template,omitempty"` - Content string `json:"content,omitempty"` - Attributes map[string]string `json:"attributes,omitempty"` - IfCapability string `json:"if-capability,omitempty"` -} - -// ItemKey is an item's key among its module's: the collection, and `/` for a named item. -func ItemKey(collection, name string) string { - if name == "" { - return collection - } - return collection + "/" + name -} - -func (it Item) key() string { return ItemKey(it.Collection, it.Name) } - -// order is the item's place among the items of one placeholder: its `order` attribute, or OrderDefault. -func (it Item) order() int { - if n, err := strconv.Atoi(it.Attributes[OrderAttribute]); err == nil && n >= 0 && n <= 99 { - return n - } - return OrderDefault -} - -// OrderAttribute is the attribute that orders items. -const OrderAttribute = "order" - -// matches is whether the item's attributes carry every pair of the selector. -func (it Item) matches(where map[string]string) bool { - for k, v := range where { - if got, has := it.Attributes[k]; !has || got != v { - return false - } - } - return true -} - -// itemsInOrder is the module's items by order, then name; the order the old form placed a module's pieces in. -func itemsInOrder(items []Item) []Item { - out := append([]Item(nil), items...) - sort.SliceStable(out, func(a, b int) bool { - if oa, ob := out[a].order(), out[b].order(); oa != ob { - return oa < ob - } - return out[a].Name < out[b].Name - }) - return out -} - -// Collection is one collection a module collects, and the comment its tool writes a line in: each contributor's item -// is introduced by ` `, and by nothing without one. -type Collection struct { - Name string `json:"name"` - Comment string `json:"comment,omitempty"` -} - -// CollectedNamespace is a collection's placeholder namespace, `${collected:}`. -const CollectedNamespace = "collected" - -// EnvironmentCollection is the one collection of the session environment (design 57 L3, issue 795): its items are -// typed by attributes, a variable (`kind=variable`, `name`) or a PATH entry (`kind=path`, `at=start|end`), and the -// template's text is the value or the entry. A collector names the rendering, `${collected:environment:posix}` or -// `${collected:environment:systemd}`, which is not a selector. -const EnvironmentCollection = "environment" - -// oldCollectionNamespaces are the namespaces read as collections of their own name. -var oldCollectionNamespaces = []string{"shell", "environment", "contribution"} - -var ( - // ofCollected is where a collector places a collection. Loose inside the braces, so a misspelt name is refused. - ofCollected = regexp.MustCompile(`\$\{collected:([^}]*)\}`) - // collectionName is words joined by colons, so an old namespace's placeholder names its collection as it reads. - collectionName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*(:[a-z0-9][a-z0-9-]*)*$`) - // itemName, attributeKey and attributeValue are the words an item's name and attributes are made of; none holds - // a colon, a comma or an equals sign, which tell a selector from a collection's name. - itemName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`) - attributeKey = regexp.MustCompile(`^[a-z][a-z0-9-]*$`) - attributeValue = regexp.MustCompile(`^[a-z0-9][a-z0-9_.-]*$`) -) - -// splitSelector is a placeholder's name without its selector, and the selector: `widgets:bar=top,place=left` is -// `widgets` and `bar=top,place=left`. A collection's name holds no `=`, so the selector begins at the last colon -// before the first one. -func splitSelector(name string) (collection, selector string) { - eq := strings.Index(name, "=") - if eq < 0 { - return name, "" - } - colon := strings.LastIndex(name[:eq], ":") - if colon < 0 { - return name, "" - } - return name[:colon], name[colon+1:] -} - -// selectorOf is a selector's pairs, or why it is none. -func selectorOf(selector string) (map[string]string, error) { - out := map[string]string{} - if selector == "" { - return out, nil - } - for _, pair := range strings.Split(selector, ",") { - k, v, ok := strings.Cut(pair, "=") - switch { - case !ok || !attributeKey.MatchString(k) || !attributeValue.MatchString(v): - return nil, fmt.Errorf("%q in %q is not =", pair, selector) - case out[k] != "": - return nil, fmt.Errorf("%q names %s twice", selector, k) - } - out[k] = v - } - return out, nil -} - -// collectionOfPlaceholder is the collection a placeholder's namespace and key name, and whether they name one. -func collectionOfPlaceholder(namespace, key string) (string, bool) { - key, _ = splitSelector(key) - switch { - case namespace == CollectedNamespace && strings.HasPrefix(key, EnvironmentCollection+":"): - // The environment is one collection, and the rest of the name picks the rendering (issue 795). - return EnvironmentCollection, true - case namespace == CollectedNamespace: - return key, true - case namespace == "environment": - return EnvironmentCollection, true - case slices.Contains(oldCollectionNamespaces, namespace): - return namespace + ":" + key, true - } - return "", false -} - -// oldCollection is an old namespace's collection by its name: its comment, or why no item may be contributed to it. -// ok is false for a name that is not an old namespace's. -// -// forItem is whether an item is to be contributed to it: an item may be contributed to a kind received as data, -// which a collector may place as a collection only where its shape says its pieces are text (design 57 L3). -func oldCollection(name string, forItem bool) (comment string, old bool, wrong error) { - namespace, key, _ := strings.Cut(name, ":") - if !slices.Contains(oldCollectionNamespaces, namespace) { - return "", false, nil - } - switch namespace { - case "shell": - target, slot, two := strings.Cut(key, ":") - if !two || !oneOf(contributionTargets(), target) || !oneOf(knownSlots, slot) { - return "", true, fmt.Errorf("%s is no slot: shell code is shell::, the shell one of %s, "+ - "the session's file one of %s or the power moment one of %s, and the slot one of %s", name, - strings.Join(knownShells, ", "), strings.Join(sessionFiles, ", "), strings.Join(powerMoments, ", "), - strings.Join(knownSlots, ", ")) - } - // shellCode's line before each module's code. - return "#", true, nil - case "environment": - if key == "" { - // The collection itself, which an item is contributed to; a placeholder names a rendering. - return "#", true, nil - } - if key != EnvironmentPOSIX && key != EnvironmentSystemd { - return "", true, fmt.Errorf("%s is no format: the environment is environment:%s or environment:%s", - name, EnvironmentPOSIX, EnvironmentSystemd) - } - return "#", true, nil - } - seat, kind, _ := strings.Cut(key, ":") - s, r, ok := receivable(seat, kind) - switch { - case !ok && s.Name == "": - return "", true, fmt.Errorf("%s names the seat %q, which the mesh does not define", name, seat) - case !ok: - return "", true, fmt.Errorf("%s: %s; %s", name, s.Name, kindsOf(s)) - case r.Shape != nil && (forItem || collectableAsText(r)): - return r.Comment, true, nil - case r.Shape != nil: - return "", true, fmt.Errorf("%s is received by %s as data in its own shape, which no item of text is "+ - "(novox/hq ADR 0255)", name, s.Name) - case r.Dirs: - return "", true, fmt.Errorf("%s names its contributors' directories, which the holder reads as paths "+ - "the controller fills; it is not yet a collection (design 57 L1)", name) - case s.Name != canonicalSeat(seat): - return "", true, fmt.Errorf("%s names the seat by another name; a collection names it %s", name, s.Name) - } - return r.Comment, true, nil -} - -// placerOfOld is the seat whose holder alone may collect an old namespace's collection (ADR 0203, 0204, 0208, -// 0211, 0212): the same seat that places it the old way. -func placerOfOld(name string) string { - namespace, key, _ := strings.Cut(name, ":") - switch namespace { - case "shell": - target, _, _ := strings.Cut(key, ":") - return placerOf(target) - case "environment": - return EnvironmentSeat - } - seat, _, _ := strings.Cut(key, ":") - return canonicalSeat(seat) -} - -// collectionProblems is what is wrong with a module's collections and items, from the manifest alone. -func collectionProblems(m Manifest) []string { - var problems []string - declared := map[string]bool{} - for i, c := range m.Collections { - switch { - case !collectionName.MatchString(c.Name): - problems = append(problems, fmt.Sprintf("%s's collection %d is %q, which is not a collection's name: "+ - "words of lower-case letters, digits and hyphens, joined by colons", m.Module, i+1, c.Name)) - case declared[c.Name]: - problems = append(problems, fmt.Sprintf("%s collects %s twice", m.Module, c.Name)) - } - if _, old, _ := oldCollection(c.Name, false); old { - problems = append(problems, fmt.Sprintf("%s collects %s, which is read from its placeholder: a module "+ - "places it as ${%s} where it holds the seat that places it, and declares nothing", m.Module, c.Name, - c.Name)) - } - if strings.ContainsAny(c.Comment, "\n\r") { - problems = append(problems, fmt.Sprintf("%s's collection %s has a comment holding a line break", - m.Module, c.Name)) - } - declared[c.Name] = true - } - // A collection declared and placed nowhere is a word read by nothing; judged where every file's text is here. - if !m.Templated() { - named := map[string]bool{} - for _, r := range m.Resources { - if text, ok := r["content"].(string); ok { - for _, f := range ofCollected.FindAllStringSubmatch(text, -1) { - base, _ := splitSelector(f[1]) - named[base] = true - } - } - } - for _, c := range m.Collections { - if !named[c.Name] && collectionName.MatchString(c.Name) { - problems = append(problems, fmt.Sprintf("%s collects %s and no file of its places it as "+ - "${collected:%s}", m.Module, c.Name, c.Name)) - } - } - } - contributed := map[string]bool{} - for i, it := range m.Items { - where := fmt.Sprintf("%s's item %d", m.Module, i+1) - if !collectionName.MatchString(it.Collection) { - problems = append(problems, fmt.Sprintf("%s is to %q, which is not a collection's name", where, - it.Collection)) - } else if _, old, wrong := oldCollection(it.Collection, true); old && wrong != nil { - problems = append(problems, fmt.Sprintf("%s: %v", where, wrong)) - } - switch { - case it.Name != "" && !itemName.MatchString(it.Name): - problems = append(problems, fmt.Sprintf("%s: the name %q is not a name: lower-case letters, digits and "+ - "hyphens", where, it.Name)) - case contributed[it.key()]: - problems = append(problems, fmt.Sprintf("%s contributes to %s twice; a module contributes one item to "+ - "a collection, or several each with a name of its own", m.Module, it.key())) - case it.Name == "" && slices.ContainsFunc(m.Items, func(o Item) bool { - return o.Collection == it.Collection && o.Name != "" - }): - problems = append(problems, fmt.Sprintf("%s contributes several items to %s and %s has no name; each "+ - "of several items has a name of its own", m.Module, it.Collection, where)) - } - contributed[it.key()] = true - problems = append(problems, itemAttributeProblems(m.Module, where, it)...) - if it.Collection == EnvironmentCollection { - problems = append(problems, environmentItemProblems(where, it)...) - } - switch { - case it.Template == "" && it.Content == "": - problems = append(problems, fmt.Sprintf("%s names no template: an item is a template of the module's "+ - "own, \"template\": \"%s/\" (novox/hq ADR 0349 §13)", where, m.TemplatesDir())) - case it.Template != "" && it.Content != "": - problems = append(problems, fmt.Sprintf("%s names its template and says its content too", where)) - case it.Template != "": - if wrong := itemTemplateProblem(m, it.Template); wrong != "" { - problems = append(problems, fmt.Sprintf("%s: %s", where, wrong)) - } - // **Filled by the provisioner's client alone**: the controller renders no item. - if !m.Moved { - problems = append(problems, fmt.Sprintf("%s: an item is filled by the provisioner's client of the "+ - "node its contributor is placed on, so a module contributing one is marked moved "+ - "(\"moved\": true; novox/hq ADR 0349 §13, §17)", where)) - } - } - if it.Content != "" { - problems = append(problems, itemTextProblems(m.Module, it)...) - } - } - return problems -} - -// itemAttributeProblems is what is wrong with an item's attributes and capability. The attributes of an item of a kind -// received as data are the fields its shape places by (and `order`), each with a value the shape takes. -func itemAttributeProblems(module, where string, it Item) []string { - var problems []string - for _, k := range sortedKeys(it.Attributes) { - v := it.Attributes[k] - switch { - case !attributeKey.MatchString(k): - problems = append(problems, fmt.Sprintf("%s: the attribute %q is not a name: lower-case letters, digits "+ - "and hyphens", where, k)) - continue - case !attributeFits(it.Collection, k, v): - problems = append(problems, fmt.Sprintf("%s: %s=%q is not a value: lower-case letters, digits, hyphens, "+ - "dots and underscores (an environment variable's name, letters, digits and underscores)", where, k, v)) - continue - } - if n, err := strconv.Atoi(v); k == OrderAttribute && (err != nil || n < 0 || n > 99) { - problems = append(problems, fmt.Sprintf("%s: order=%q is not a whole number from 0 to 99", where, v)) - } - } - if namespace, key, _ := strings.Cut(it.Collection, ":"); namespace == "contribution" { - seat, kind, _ := strings.Cut(key, ":") - if _, r, ok := receivable(seat, kind); ok && r.Shape != nil { - for _, k := range sortedKeys(it.Attributes) { - var place *PlaceField - for i := range r.Shape.Places { - if r.Shape.Places[i].Field == k { - place = &r.Shape.Places[i] - } - } - switch { - case k == OrderAttribute: - case place == nil: - problems = append(problems, fmt.Sprintf("%s: %s is placed by %s, and %q is none of them", where, - it.Collection, placeFields(r), k)) - case !oneOf(place.Values, it.Attributes[k]): - problems = append(problems, fmt.Sprintf("%s: %s=%s; %s is one of %s", where, k, it.Attributes[k], k, - strings.Join(place.Values, ", "))) - } - } - } - } - switch { - case it.IfCapability == "": - case !capabilityName.MatchString(it.IfCapability): - problems = append(problems, fmt.Sprintf("%s: if-capability %q is not a capability's name", where, it.IfCapability)) - case !oneOf(KnownCapabilities, it.IfCapability): - problems = append(problems, fmt.Sprintf("%s: if-capability %q is one no machine reports; the node-engine "+ - "detects %s", where, it.IfCapability, strings.Join(KnownCapabilities, ", "))) - } - return problems -} - -// AttributeFits is whether an item's attribute is of the shape its collection allows: words of lower case, except -// the environment's `name`, which is a variable's name (design 57 L3, issue 795). The same rule is mesh-catalog's -// provisioner client's, and the inventory's. -func attributeFits(collection, k, v string) bool { - if collection == EnvironmentCollection && k == "name" { - return variableName.MatchString(v) - } - return attributeKey.MatchString(k) && attributeValue.MatchString(v) -} - -// environmentItemProblems is what is wrong with an item of the environment collection: typed by `kind`, a variable -// with its `name`, a PATH entry `at` the start or the end, and no attribute besides `order`. -func environmentItemProblems(where string, it Item) []string { - var problems []string - for _, k := range sortedKeys(it.Attributes) { - if !slices.Contains([]string{"kind", "name", "at", OrderAttribute}, k) { - problems = append(problems, fmt.Sprintf("%s: %s is typed by kind, name and at, and %q is none of them", - where, it.Collection, k)) - } - } - switch kind := it.Attributes["kind"]; kind { - case "variable": - if it.Attributes["name"] == "" || it.Attributes["at"] != "" { - problems = append(problems, fmt.Sprintf("%s: a variable names itself with name= and has no at", where)) - } else if it.Attributes["name"] == "PATH" { - problems = append(problems, fmt.Sprintf("%s: PATH is not a variable; a PATH entry is kind=path", where)) - } - case "path": - if it.Attributes["name"] != "" || (it.Attributes["at"] != PathAtStart && it.Attributes["at"] != PathAtEnd) { - problems = append(problems, fmt.Sprintf("%s: a PATH entry is at=%s or at=%s and has no name", where, - PathAtStart, PathAtEnd)) - } - default: - problems = append(problems, fmt.Sprintf("%s: an item of %s is kind=variable or kind=path, not %q", where, - it.Collection, kind)) - } - return problems -} - -// itemTemplateProblem is what is wrong with an item's template path, or "". -func itemTemplateProblem(m Manifest, name string) string { - dir := m.TemplatesDir() - if strings.HasPrefix(name, "/") || path.Clean(name) != name || !strings.HasPrefix(name, dir+"/") || - strings.Contains(name, "\\") { - return fmt.Sprintf("the template %q is not a file under %s/ of its own repository: a template is %q, a "+ - "plain relative path, and the build packs only %s/", name, dir, dir+"/", dir) - } - return "" -} - -// itemTextProblems is what an item's text may not name: a secret or a bound value, since an item is kept in a -// bucket and no secret is (ADR 0349 clause 7), and a collection, since an item is not a collector. And every -// placeholder no pass fills, as in a file's content. -func itemTextProblems(module string, it Item) []string { - var problems []string - for _, f := range templatePlaceholder.FindAllStringSubmatch(it.Content, -1) { - namespace, key := f[1], f[2] - switch _, collection := collectionOfPlaceholder(namespace, key); { - case namespace == "secret" || namespace == "bound": - problems = append(problems, fmt.Sprintf("%s's item of %s names %s: an item is kept in a bucket, where "+ - "no secret is (novox/hq ADR 0349 §7)", module, it.Collection, f[0])) - case collection: - problems = append(problems, fmt.Sprintf("%s's item of %s names %s: an item is not a collector", - module, it.Collection, f[0])) - } - } - return append(problems, unconsumedPlaceholders(module, map[string]any{"id": "item:" + it.key(), - "type": "file", "content": it.Content})...) -} - -// isOld is whether a collection's name is an old namespace's. -func isOld(name string) bool { - _, old, _ := oldCollection(name, true) - return old -} - -// collectedPlaceholderProblems is what is wrong with one resource's ${collected:…}: placed only in a file's content, -// and naming a collection the module declares it collects. -func collectedPlaceholderProblems(m Manifest, r map[string]any) []string { - var problems []string - for _, field := range sortedKeys(r) { - v, ok := r[field].(string) - if !ok { - continue - } - for _, f := range ofCollected.FindAllStringSubmatch(v, -1) { - base, selector := splitSelector(f[1]) - if _, err := selectorOf(selector); err != nil { - problems = append(problems, fmt.Sprintf("%s's resource %v names %s: %v", m.Module, r["id"], f[0], err)) - continue - } - switch { - case field != "content": - problems = append(problems, fmt.Sprintf("%s's resource %v names %s in its %s; a collection is placed "+ - "only in a file's content", m.Module, r["id"], f[0], field)) - case isOld(base): - if _, _, wrong := oldCollection(base, false); wrong != nil { - problems = append(problems, fmt.Sprintf("%s's resource %v names %s: %v", m.Module, r["id"], f[0], wrong)) - } - case !slices.ContainsFunc(m.Collections, func(c Collection) bool { return c.Name == base }): - problems = append(problems, fmt.Sprintf("%s's resource %v names %s, and %s does not collect %q: "+ - "declare it under collections (novox/hq ADR 0349 §13)", m.Module, r["id"], f[0], m.Module, base)) - } - } - } - return problems -} - -// collectionComment is the comment of a collection a module's file names: declared, or the old namespace's. -func collectionComment(m Manifest, name string) (string, error) { - if comment, old, wrong := oldCollection(name, false); old { - return comment, wrong - } - for _, c := range m.Collections { - if c.Name == name { - return c.Comment, nil - } - } - return "", fmt.Errorf("%s names ${collected:%s} and does not collect it", m.Module, name) -} - -// itemsLeftOut says each item of a collection a file is filled with the old way that the controller could not render -// there (heldItemText): the collector is not handed to the provisioner on this machine (ADR 0349 clause 17). Said, -// never dropped quietly. -func itemsLeftOut(modules []Manifest, holder Manifest, collection string, facts map[string]string) []string { - return itemsLeftOutFor(modules, holder, collection, nil, facts, nil) -} - -// itemsLeftOutFor is itemsLeftOut for a placeholder that selects by attributes (where), on a machine with these -// capabilities: an item the selector does not take, or whose capability the machine lacks, is not left out but not -// wanted here. -func itemsLeftOutFor(modules []Manifest, holder Manifest, collection string, where map[string]string, - facts map[string]string, caps map[string]bool) []string { - var out []string - for _, m := range inModuleOrder(modules) { - for _, it := range itemsInOrder(m.Items) { - if it.Collection != collection || !it.matches(where) || (it.IfCapability != "" && !caps[it.IfCapability]) { - continue - } - if _, rendered := heldText(it, facts); rendered && renderedOldWay(collection) { - continue - } - out = append(out, fmt.Sprintf("%s's item of %s is not placed: %s is filled the old way on this "+ - "machine, from contributions, and an item only by the provisioner's client (design 57 L1)", - m.Module, it.key(), holder.Module)) - } - } - return out -} - -// oldFormsLeftOut says each piece of an old form a handed collector's collections would have held: the collector is -// filled from items alone, and the piece's module contributes none (ADR 0349 clause 17). -func oldFormsLeftOut(modules []Manifest, holder Manifest, collections []PlanCollection) []string { - var out []string - for _, c := range collections { - namespace, key, _ := strings.Cut(c.Name, ":") - for _, m := range inModuleOrder(modules) { - old := false - switch namespace { - case "shell": - target, slot, _ := strings.Cut(key, ":") - old = slices.ContainsFunc(m.Shell, func(s ShellCode) bool { return s.For == target && s.Slot == slot }) - case "environment": - old = m.Environment != nil && (len(m.Environment.Variables) > 0 || len(m.Environment.Path) > 0) - case "contribution": - seat, kind, _ := strings.Cut(key, ":") - old = slices.ContainsFunc(m.allContributions(), func(s SeatContribution) bool { - return canonicalSeat(s.Seat) == canonicalSeat(seat) && s.Kind == kind - }) - } - if old { - out = append(out, fmt.Sprintf("%s's %s is not placed: %s is filled by the provisioner's client on "+ - "this machine, from items, and %s contributes it the old way (design 57 L1)", m.Module, c.Name, - holder.Module, m.Module)) - } - } - } - return out -} - -// heldItemText is a moved contributor's items of an old namespace's collection as the controller renders them into a -// collector not handed on this machine (design 57 L2, issue 681): the text the mesh holds, read back from the -// contributor's templates archive, with the machine's facts filled, each ending its line. As a moved module's file -// keeps its text for a machine that cannot take it handed (handed.go), an item keeps its place in a collector filled -// the old way, so the file stays what its old form made it. Only `shell:` and `contribution:` collections, which the -// controller renders as text; false for a module with no such item, or an item whose text the mesh does not hold or -// which names anything but a machine fact this machine has, which is then said as not placed. -func heldItemText(m Manifest, collection string, facts map[string]string) (string, bool) { - items := heldItems(m, collection, nil, facts, nil) - var b strings.Builder - for _, h := range items { - b.WriteString(h.text) - } - return b.String(), len(items) > 0 -} - -// renderedOldWay is whether the controller renders a collection's items into a collector not handed. -func renderedOldWay(collection string) bool { - namespace, _, _ := strings.Cut(collection, ":") - return namespace == "shell" || namespace == "contribution" || namespace == EnvironmentCollection -} - -// heldItem is one item of a module rendered the old way. -type heldItem struct { - item Item - text string -} - -// heldItems is a module's items of the collection that the selector takes and the machine's capabilities allow, by -// order then name, each rendered as heldText does; the ones it cannot render are left out (itemsLeftOutFor says so). -func heldItems(m Manifest, collection string, where map[string]string, facts map[string]string, - caps map[string]bool) []heldItem { - if !renderedOldWay(collection) { - return nil - } - var out []heldItem - for _, it := range itemsInOrder(m.Items) { - if it.Collection != collection || !it.matches(where) || (it.IfCapability != "" && !caps[it.IfCapability]) { - continue - } - if text, ok := heldText(it, facts); ok { - out = append(out, heldItem{item: it, text: text}) - } - } - return out -} - -// heldText is an item's text with the machine's facts filled, ending its line; false for an item whose text the mesh -// does not hold or which names anything but a machine fact this machine has. -func heldText(it Item, facts map[string]string) (string, bool) { - if it.Content == "" { - return "", false - } - text := it.Content - for _, f := range templatePlaceholder.FindAllStringSubmatch(text, -1) { - value, has := facts[f[2]] - if f[1] != "machine" || !has { - return "", false - } - text = strings.ReplaceAll(text, f[0], value) - } - if !strings.HasSuffix(text, "\n") { - text += "\n" - } - return text, true -} diff --git a/internal/catalogue/config.go b/internal/catalogue/config.go deleted file mode 100644 index 0a54370b..00000000 --- a/internal/catalogue/config.go +++ /dev/null @@ -1,504 +0,0 @@ -package catalogue - -import ( - "encoding/json" - "fmt" - "regexp" - "slices" - "strings" -) - -// A module declares its config (novox/hq ADR 0348 §1, §4, §5, §6, §8, §11; design 57 package K1, issue 660). -// -// **One concept, two spellings for a while.** "Config" replaces "settings": the manifest says `config` for -// `settings`, `takes-config` for `takes-settings` and `${config:}` for `${setting:}`. The old names stay -// accepted as aliases until every module has moved (ADR 0348 §1), and a module uses one form or the other, never -// both. The config form is read into the very fields the settings form fills — Settings, a mergeable file's -// takes-settings and `${setting:}` — so composition has one path and `${config:}` fills exactly as -// `${setting:}` does: a module rewritten from settings to config composes byte-identically. It is written -// back in the form it was read in, so what the mesh holds is what its author wrote. -// -// **A moved module is held to the whole declaration.** Under `config` every value has a type (number, string or -// boolean; an object or a list only when trusted), a description including its unit, its validation (enum, -// minimum and maximum, pattern), whether it is trusted, optionally the predefined key it falls back to, and either -// a default or `required`; it lands in a file or is read at runtime (`runtime: true`); every mergeable file names -// the keys it takes; a `reload-on` or `restart-on` names a file the module declares. A module not yet moved keeps -// today's form and today's rules (SettingProblems, TakenSettingsProblems) until the catalogue's list of such -// modules is empty (ADR 0348 §11). -// -// **A required value nothing sets** is the operator's own: it composes as an operator's setting nobody gave does -// today, refused by name at composition (UnsetSettingError). Who reports it once mesh-config answers values is the -// provisioner's (ADR 0349), not this file's. - -// ConfigField is the manifest key a module declares its config values under. -const ConfigField = "config" - -// TakesConfigField is the key a mergeable file names the config values it takes by. -const TakesConfigField = "takes-config" - -// SettingsField is the old name of ConfigField, read as its alias. -const SettingsField = "settings" - -// configRef is how a file asks for a config value: ${config:}, the same keys as settingRef. -var configRef = regexp.MustCompile(`\$\{config:([a-z0-9][a-z0-9_.-]*)\}`) - -// takesConfigKey and takesSettingsKey are the two spellings as a key of a JSON object. A string value is never -// followed by a colon, and a quote inside a string is escaped, so these match only keys. -var ( - takesConfigKey = regexp.MustCompile(`"` + TakesConfigField + `"(\s*):`) - takesSettingsKey = regexp.MustCompile(`"` + TakesSettingsField + `"(\s*):`) -) - -// The types a config value may have (ADR 0348 §4). -const ( - ConfigNumber = "number" - ConfigString = "string" - ConfigBoolean = "boolean" - ConfigObject = "object" - ConfigList = "list" -) - -// accountFacts are what the mesh places for a module's account, never config values (ADR 0348 §11). -var accountFacts = map[string]bool{"puid": true, "pgid": true} - -// configForms is which of the two forms a manifest was written in, by the words of each it uses. -type configForms struct { - config, settings []string -} - -// DeclaresConfig is whether the manifest is written in the config form: `config`, `takes-config` or -// `${config:}` (ADR 0348 §1, §11). -func (m Manifest) DeclaresConfig() bool { return len(m.forms.config) > 0 } - -// toSettingsForm is raw with the config form's placeholders and takes-config read as the settings form's, and -// which words of each form it used. `config` itself is a top-level key, renamed where the keys are read -// (UnmarshalJSON). -func toSettingsForm(raw []byte) ([]byte, configForms) { - var f configForms - if configRef.Match(raw) { - f.config = append(f.config, "${config:…}") - } - if takesConfigKey.Match(raw) { - f.config = append(f.config, TakesConfigField) - } - if settingRef.Match(raw) { - f.settings = append(f.settings, "${setting:…}") - } - if takesSettingsKey.Match(raw) { - f.settings = append(f.settings, TakesSettingsField) - } - if len(f.config) == 0 { - return raw, f - } - raw = configRef.ReplaceAllFunc(raw, func(ref []byte) []byte { - return []byte("${setting:" + string(configRef.FindSubmatch(ref)[1]) + "}") - }) - raw = takesConfigKey.ReplaceAllFunc(raw, func(key []byte) []byte { - return []byte(`"` + TakesSettingsField + `"` + string(takesConfigKey.FindSubmatch(key)[1]) + ":") - }) - return raw, f -} - -// toConfigForm is a manifest written in the settings form's words, as JSON, in the config form's: what -// toSettingsForm and UnmarshalJSON read, written back. -func toConfigForm(raw []byte) ([]byte, error) { - var keys map[string]json.RawMessage - if err := json.Unmarshal(raw, &keys); err != nil { - return nil, err - } - if settings, has := keys[SettingsField]; has { - keys[ConfigField] = settings - delete(keys, SettingsField) - var err error - if raw, err = json.Marshal(keys); err != nil { - return nil, err - } - } - raw = settingRef.ReplaceAllFunc(raw, func(ref []byte) []byte { - return []byte("${config:" + string(settingRef.FindSubmatch(ref)[1]) + "}") - }) - raw = takesSettingsKey.ReplaceAllFunc(raw, func(key []byte) []byte { - return []byte(`"` + TakesConfigField + `"` + string(takesSettingsKey.FindSubmatch(key)[1]) + ":") - }) - return raw, nil -} - -// inConfigWords is a refusal said of a module in the config form in its words: `${config:}` and -// takes-config, where the rules shared with the settings form say `${setting:}` and takes-settings. -func inConfigWords(problem string) string { - problem = settingRef.ReplaceAllStringFunc(problem, func(ref string) string { - return "${config:" + settingRef.FindStringSubmatch(ref)[1] + "}" - }) - return strings.ReplaceAll(problem, TakesSettingsField, TakesConfigField) -} - -// MixedFormProblems is a manifest using both forms, said with the words of each (ADR 0348 §1). -func MixedFormProblems(m Manifest) []string { - if len(m.forms.config) == 0 || len(m.forms.settings) == 0 { - return nil - } - return []string{fmt.Sprintf("%s mixes the config form (%s) with the settings form (%s): a module uses one form "+ - "or the other, never both; write %s, %s and ${config:} throughout (novox/hq ADR 0348 §1)", - m.Module, strings.Join(m.forms.config, ", "), strings.Join(m.forms.settings, ", "), - ConfigField, TakesConfigField)} -} - -// configDeclaration is the config form's declaration as written: nothing of the settings form's, and nothing -// unsaid. -type configDeclaration struct { - Type string `json:"type,omitempty"` - Description string `json:"description,omitempty"` - Enum []any `json:"enum,omitempty"` - Minimum *float64 `json:"minimum,omitempty"` - Maximum *float64 `json:"maximum,omitempty"` - Pattern string `json:"pattern,omitempty"` - Trusted bool `json:"trusted,omitempty"` - Predefined string `json:"predefined,omitempty"` - Default any `json:"default,omitempty"` - Required bool `json:"required,omitempty"` - Runtime bool `json:"runtime,omitempty"` -} - -// MarshalJSON writes a settings declaration exactly as before, and a config declaration without the settings -// form's kind and why, and without a default it does not have. -func (d SettingDeclaration) MarshalJSON() ([]byte, error) { - type plain SettingDeclaration - if d.Kind != "" || d.Why != "" { - return json.Marshal(plain(d)) - } - return json.Marshal(configDeclaration{Type: d.Type, Description: d.Description, Enum: d.Enum, - Minimum: d.Minimum, Maximum: d.Maximum, Pattern: d.Pattern, Trusted: d.Trusted, Predefined: d.Predefined, - Default: d.Default, Required: d.Required, Runtime: d.Runtime}) -} - -// configShaped is whether a declaration says anything of the config form's. -func configShaped(d SettingDeclaration) bool { - return d.Type != "" || d.Description != "" || len(d.Enum) > 0 || d.Minimum != nil || d.Maximum != nil || - d.Pattern != "" || d.Trusted || d.Predefined != "" || d.Required || d.Runtime -} - -// TypedSettingProblems is a declaration under `settings` that says what only `config` says: a value with a type, -// a description and its validation is declared in the config form (ADR 0348 §1, §4). -func TypedSettingProblems(m Manifest) []string { - var problems []string - for _, key := range sortedSettingKeys(m.Settings) { - if configShaped(m.Settings[key]) { - problems = append(problems, fmt.Sprintf("%s's setting %q says its type, description or validation "+ - "under %s: a value declared with those is declared under config, with ${config:} and "+ - "takes-config, never under %s (novox/hq ADR 0348 §1, §4)", m.Module, key, SettingsField, SettingsField)) - } - } - return problems -} - -// ConfigProblems is every way a module in the config form declares its config wrongly, each said by name with -// what to do (ADR 0348 How it is checked, design 57 package K1). -func ConfigProblems(m Manifest) []string { - if !m.DeclaresConfig() { - return nil - } - var problems []string - placed := configPlaced(m) - fallsBack := fileFallsBack(m) - renamed := renamedTaken(m) - for _, key := range sortedSettingKeys(m.Settings) { - d := m.Settings[key] - say := func(format string, args ...any) { - problems = append(problems, fmt.Sprintf("%s's config value %q: ", m.Module, key)+fmt.Sprintf(format, args...)) - } - if !settingRef.MatchString("${setting:" + key + "}") { - say("not a usable key: lower-case letters, digits, dots, dashes and underscores") - continue - } - if meshWords[key] || accountFacts[key] { - say("%q is placement, which the mesh decides, never a config value (novox/hq ADR 0348 §11)", key) - continue - } - if d.Kind != "" || d.Why != "" { - say("kind and why are the settings form's; under config a value says its type and its description " + - "(novox/hq ADR 0348 §1, §4)") - } - switch d.Type { - case ConfigNumber, ConfigString, ConfigBoolean: - case ConfigObject, ConfigList: - if !d.Trusted { - say("is %s and not trusted: an object or a list decides more than one value can say, so it is "+ - "trusted, \"trusted\": true, or split into numbers, strings and booleans (novox/hq ADR 0348 §4)", d.Type) - } - case "": - say("no type: say \"type\": number, string or boolean, or object or list when trusted (novox/hq ADR 0348 §4)") - default: - say("type %q is none of number, string, boolean, object and list (novox/hq ADR 0348 §4)", d.Type) - } - if strings.TrimSpace(d.Description) == "" { - say("no description: say in a sentence what the value is, including its unit (novox/hq ADR 0348 §4)") - } - switch { - case d.Required && d.Default != nil: - say("both a default and required: a required value is the operator's own and has no default; give " + - "one or the other (novox/hq ADR 0348 §4)") - case !d.Required && d.Default == nil: - say("neither a default nor required: give a default, or say \"required\": true for a value that is " + - "the operator's own (novox/hq ADR 0348 §4)") - } - if as := renamed[key]; as != "" && d.Required { - say("is laid into its file as %q, and only when it is set, so the file and its code fall back when "+ - "it is not: it is never required; give it the file's default, the empty value of its type "+ - "(novox/hq issue 751)", as) - } - for _, wrong := range validationProblems(d) { - say("%s", wrong) - } - if word := operatorsWord(key); word != "" && d.Default != nil && !fallsBack[key] { - say("a key naming %q is the operator's value: declare it \"required\": true, with no default, "+ - "unless a mergeable file takes it and no placeholder asks for it, where the file falls back "+ - "(novox/hq ADR 0348 §4, ADR 0112, issue 735)", word) - } - if d.Predefined != "" { - p, listed := m.predefined[d.Predefined] - switch { - case m.predefined == nil: - say("falls back to the predefined key %q, and the list of predefined keys (%s at the root of "+ - "the catalogue's repository) is not present where it is judged: a value falls back only to a "+ - "key on that list; judge it in the catalogue, or leave \"predefined\" out (novox/hq ADR 0348 §7)", - d.Predefined, PredefinedFile) - case !listed: - say("falls back to %q, which is not on the list of predefined keys: name a key on it, or leave "+ - "\"predefined\" out (novox/hq ADR 0348 §7)", d.Predefined) - default: - if p.Type != d.Type { - say("falls back to the predefined key %q, which is a %s, and this value is a %s: a value falls "+ - "back only to a key of its own type (novox/hq ADR 0348 §4)", d.Predefined, p.Type, d.Type) - } - if p.Trusted && !d.Trusted { - say("falls back to the trusted predefined key %q and is not trusted itself: a value that falls "+ - "back to a trusted key is trusted, \"trusted\": true (novox/hq ADR 0348 §8)", d.Predefined) - } - } - } - // A template's text is not here yet. Resolve fills it from the build, WithTemplatesFrom from the tree; neither - // judges again itself, but WithTemplatesFrom parses its result again, and a resolved build is parsed again - // where it is registered, and this is judged then. - if !placed[key] && !d.Runtime && !m.Templated() { - say("placed nowhere: no file says ${config:%s} or takes it under %s, and it is not marked "+ - "\"runtime\": true for a value the module's code reads itself (novox/hq ADR 0348 §5)", key, TakesConfigField) - } - } - used := settingKeysUsedBy(m) - for _, key := range sortedKeys(used) { - if _, declared := m.Settings[key]; !declared { - problems = append(problems, fmt.Sprintf("%s says ${config:%s} and declares no such value under %s: "+ - "declare %q with its type, description and a default or required (novox/hq ADR 0348 §4, §5)", - m.Module, key, ConfigField, key)) - } - } - problems = append(problems, configTakesProblems(m)...) - problems = append(problems, reflectsProblems(m)...) - return problems -} - -// configPlaced is every value a file, a unit, a directory's owner, a contribution or a served fact asks for, and -// every value a mergeable file takes. -func configPlaced(m Manifest) map[string]bool { - placed := settingKeysUsedBy(m) - for _, r := range m.Resources { - for _, k := range takenKeyList(r) { - placed[k] = true - } - } - return placed -} - -// fileFallsBack is every value a mergeable file takes and no placeholder asks for. Unset, such a value is simply -// not laid over the file: the file's own content, or the module's code reading the file, falls back. So one named -// after the operator's value (a url, a user, an address) is not `required`, and its default is the file's: its -// value in the content, else the empty value of its type, never laid over the file (the operator's decision on -// issue 664, issue 735). A placeholder has no such fallback: its default would be filled in where the operator's -// value belongs (ADR 0112). -func fileFallsBack(m Manifest) map[string]bool { - asked := settingKeysUsedBy(m) - out := map[string]bool{} - for _, r := range m.Resources { - if how, _ := r["merge"].(string); how == "" { - continue - } - for _, k := range takenKeyList(r) { - if !asked[k] { - out[k] = true - } - } - } - return out -} - -// renamedTaken is every value a mergeable file takes under a name of its own, with that name (issue 751). -func renamedTaken(m Manifest) map[string]string { - out := map[string]string{} - for _, r := range m.Resources { - if how, _ := r["merge"].(string); how == "" { - continue - } - for _, e := range takenEntries(r[TakesSettingsField]) { - if e.renamed { - out[e.key] = e.as - } - } - } - return out -} - -// validationProblems is what is wrong with a declaration's validation, and its default against it. -func validationProblems(d SettingDeclaration) []string { - var problems []string - isNumber := d.Type == ConfigNumber - if len(d.Enum) > 0 && d.Type != ConfigNumber && d.Type != ConfigString { - problems = append(problems, fmt.Sprintf("an enum is for a number or a string, and this is a %s", d.Type)) - } - if (d.Minimum != nil || d.Maximum != nil) && !isNumber { - problems = append(problems, fmt.Sprintf("a minimum or maximum is for a number, and this is a %s", d.Type)) - } - if d.Minimum != nil && d.Maximum != nil && *d.Minimum > *d.Maximum { - problems = append(problems, fmt.Sprintf("its minimum %v is above its maximum %v", *d.Minimum, *d.Maximum)) - } - var pattern *regexp.Regexp - if d.Pattern != "" { - if d.Type != ConfigString { - problems = append(problems, fmt.Sprintf("a pattern is for a string, and this is a %s", d.Type)) - } - var err error - if pattern, err = regexp.Compile(`^(?:` + d.Pattern + `)$`); err != nil { - problems = append(problems, fmt.Sprintf("its pattern %q is not a regular expression: %v", d.Pattern, err)) - } - } - for _, v := range d.Enum { - if !ofType(v, d.Type) { - problems = append(problems, fmt.Sprintf("its enum holds %v, which is not a %s", v, d.Type)) - } - } - if d.Default == nil { - return problems - } - if !ofType(d.Default, d.Type) { - if d.Type != "" { - problems = append(problems, fmt.Sprintf("its default %v is not a %s", d.Default, d.Type)) - } - return problems - } - if len(d.Enum) > 0 && !slices.ContainsFunc(d.Enum, func(v any) bool { return v == d.Default }) { - problems = append(problems, fmt.Sprintf("its default %v is not one of %v", d.Default, d.Enum)) - } - if n, ok := d.Default.(float64); ok { - if d.Minimum != nil && n < *d.Minimum { - problems = append(problems, fmt.Sprintf("its default %v is below its minimum %v", n, *d.Minimum)) - } - if d.Maximum != nil && n > *d.Maximum { - problems = append(problems, fmt.Sprintf("its default %v is above its maximum %v", n, *d.Maximum)) - } - } - if s, ok := d.Default.(string); ok && pattern != nil && !pattern.MatchString(s) { - problems = append(problems, fmt.Sprintf("its default %q does not match its pattern %q", s, d.Pattern)) - } - return problems -} - -// ofType is whether a value read from JSON is of a config type. -func ofType(v any, kind string) bool { - switch v.(type) { - case float64: - return kind == ConfigNumber - case string: - return kind == ConfigString - case bool: - return kind == ConfigBoolean - case map[string]any: - return kind == ConfigObject - case []any: - return kind == ConfigList - } - return false -} - -// configTakesProblems is what ADR 0348 §11 adds for a module in the config form: an account fact in -// takes-config (the mesh's own words are refused for both forms by TakenSettingsProblems), and a mergeable file -// naming no keys where none of the module's does (where one does, TakenSettingsProblems names every one that -// does not). -func configTakesProblems(m Manifest) []string { - var problems, silent []string - naming := false - for _, r := range m.Resources { - if how, _ := r["merge"].(string); how == "" { - continue - } - if _, said := r[TakesSettingsField]; !said { - silent = append(silent, fmt.Sprint(r["id"])) - continue - } - naming = true - for _, k := range takenKeyList(r) { - _, declared := m.Settings[k] - switch { - case accountFacts[k]: - problems = append(problems, fmt.Sprintf("%s's %v names %q in %s, and %q is an account fact the "+ - "mesh places, never a config value (novox/hq ADR 0348 §11)", m.Module, r["id"], k, - TakesConfigField, k)) - case meshWords[k]: - // Refused for both forms by TakenSettingsProblems, in its words. - case !declared: - problems = append(problems, fmt.Sprintf("%s's %v names %q in %s and declares no such value under "+ - "%s: a value a file takes is declared like every other, with its type, description and a "+ - "default or required, or the config verb could never set it (novox/hq ADR 0348 §4, §11)", - m.Module, r["id"], k, TakesConfigField, ConfigField)) - } - } - } - if !naming { - for _, id := range silent { - problems = append(problems, fmt.Sprintf("%s's mergeable %s names no %s: a module that declares config "+ - "names the keys each mergeable file takes, an empty list for none (novox/hq ADR 0348 §11)", - m.Module, id, TakesConfigField)) - } - } - return problems -} - -// meshMadeIDs are the resources the mesh composes into a module's own set, which its `restart-on` may name -// without declaring them; meshMadePrefixes are those named after what they are for. -var ( - meshMadeIDs = []string{ComposedJailsID(), FilteringID(), CertificateID(), AuthorityID(), BusUsersID()} - meshMadePrefixes = []string{NeedID(""), SecretID(""), strings.TrimSuffix(GrantID("", ""), "-"), BoundID(""), ReceivedID(""), AccessID("/")} -) - -// reflectsProblems is a `reload-on` or `restart-on` naming a file the module does not declare (ADR 0348 §6): not -// one of its own resources, not one the mesh composes for it, and not another module's, which is named -// `.`. -func reflectsProblems(m Manifest) []string { - own := map[string]bool{} - for _, r := range m.Resources { - own[fmt.Sprint(r["id"])] = true - } - meshMade := func(id string) bool { - if slices.Contains(meshMadeIDs, id) { - return true - } - for _, p := range meshMadePrefixes { - if strings.HasPrefix(id, p) && len(id) > len(p) { - return true - } - } - return false - } - var problems []string - for _, r := range m.Resources { - for _, field := range []string{"reload-on", "restart-on"} { - for _, id := range stringsOf(r[field]) { - if strings.Contains(id, ".") || own[id] || meshMade(id) { - continue - } - problems = append(problems, fmt.Sprintf("%s's %v names %q in its %s, and %s declares no such "+ - "file: name a file of its own, or another module's as . (novox/hq ADR 0348 §6)", - m.Module, r["id"], id, field, m.Module)) - } - } - } - return problems -} diff --git a/internal/catalogue/config_move.go b/internal/catalogue/config_move.go deleted file mode 100644 index e902b91f..00000000 --- a/internal/catalogue/config_move.go +++ /dev/null @@ -1,40 +0,0 @@ -package catalogue - -import "strings" - -// What the move of config reads of a module's declaration (novox/hq ADR 0348 §9, design 57 K5, issue 664): where a -// layer's key goes, how two layers' values become one, and whether a value is one its declaration takes. - -// TakenAs is each name a mergeable file takes a config value under that is not the value's config key, with that -// key (issue 751): a layer set before the module moved may still say the file's name, and the move sets it under -// the config key. -func TakenAs(m Manifest) map[string]string { - out := map[string]string{} - for key, as := range renamedTaken(m) { - out[as] = key - } - return out -} - -// AskedKeys is every value a placeholder of the module asks for: a file, a unit, a directory's owner, a -// contribution or a served fact. Such a value is taken whole from the last layer that sets it, where a mergeable -// file lays one object over another key by key. -func AskedKeys(m Manifest) map[string]bool { return settingKeysUsedBy(m) } - -// LaidOver is a value laid over another as a later layer is laid over an earlier: two objects key by key, anything -// else replaced whole. -func LaidOver(under, over any) any { return mergeValue(under, over) } - -// ValueProblems is what is wrong with a value for a declaration: its type, and its enum, bounds or pattern. -func ValueProblems(d SettingDeclaration, v any) []string { - if v == nil { - return []string{"no value"} - } - d.Default = v - d.Required = false - var problems []string - for _, p := range validationProblems(d) { - problems = append(problems, strings.ReplaceAll(p, "its default", "the value")) - } - return problems -} diff --git a/internal/catalogue/consumer_into_serves.go b/internal/catalogue/consumer_into_serves.go deleted file mode 100644 index e5d47916..00000000 --- a/internal/catalogue/consumer_into_serves.go +++ /dev/null @@ -1,338 +0,0 @@ -package catalogue - -import ( - "fmt" - "regexp" - "sort" - "strings" -) - -// What a provider derives for one consumer, said once in the provider's definition and delivered -// to both ends (novox/hq ADR 0202, issue 124). -// -// A `serves` block is otherwise literal: the same values for every consumer. Where the provider -// *names the resource* — a bucket, a database, a vhost — the name is derived from who is asking, -// and before this the mesh had no channel for it. The provider recomputed it in its own code and -// every consumer transcribed it into its own definition by hand, which is a copy of somebody -// else's rule kept in agreement by nobody. One of three transcriptions was wrong for months. -// -// **The mesh learns no protocol here; it spells its own name in an alphabet it already knows.** -// The only fact a served value may name is the identity the mesh itself minted for the consumer, -// in one of two alphabets: as it was minted, and as a DNS label. Everything a provider wants -// around it — a prefix, a suffix, a separator — it writes around the placeholder, because a -// served value is a string. - -// consumerFact is `${consumer:}` or `${consumer::}`. -var consumerFact = regexp.MustCompile(`\$\{consumer:([a-z][a-z0-9-]*)(?::([a-z][a-z0-9-]*))?\}`) - -// consumerFacts are what a served value may name about the consumer it is being derived for. -// One entry, deliberately: the identity is the one thing about a consumer the mesh itself chose, -// so it is the one thing the mesh can hand to a provider without either end guessing. -var consumerFacts = []string{"as"} - -// consumerAlphabets are the ways the mesh will write that identity. `dns` is the mesh's own -// identifier with its separator written `-` instead of `_` — the whole of the difference between -// the alphabet the mesh mints in and the one buckets, vhosts and hostnames accept. -var consumerAlphabets = []string{"dns"} - -// ServedTo fills a provider's served values for one consumer. -// -// `as` is the identity the mesh minted for that consumer — the same string it is told to present -// as a login. Values with no placeholder are returned exactly as they were, and a block with no -// placeholder at all is returned unchanged, so this costs nothing for the providers that derive -// nothing. -// -// Only strings carry placeholders. A number, a boolean or a nested object is a value the provider -// stated outright, and is left alone. -func ServedTo(serves map[string]any, as string) (map[string]any, error) { - if len(serves) == 0 { - return serves, nil - } - var out map[string]any - for _, key := range sortedAnyKeys(serves) { - text, ok := serves[key].(string) - if !ok || !strings.Contains(text, "${consumer:") { - continue - } - filled, err := consumerInto(text, as) - if err != nil { - return nil, fmt.Errorf("the value served as %q: %w", key, err) - } - if out == nil { - // Copied only once something actually changes: the caller's map is the manifest's, - // and a provider that derives nothing must not have it rewritten underneath it. - out = make(map[string]any, len(serves)) - for k, v := range serves { - out[k] = v - } - } - out[key] = filled - } - if out == nil { - return serves, nil - } - return out, nil -} - -// consumerInto replaces every `${consumer:…}` in one value. -// -// **A fact or an alphabet the mesh does not have is refused, not left standing.** Written through, -// the literal `${consumer:as}` would reach a configuration file and be read as a bucket name, -// failing somewhere that names neither the module nor the mesh — the same reasoning `${bound:…}` -// is refused by (boundInto). -func consumerInto(value, as string) (string, error) { - var failed error - out := consumerFact.ReplaceAllStringFunc(value, func(match string) string { - parts := consumerFact.FindStringSubmatch(match) - fact, alphabet := parts[1], parts[2] - if fact != "as" { - if failed == nil { - failed = fmt.Errorf( - "says %s, and the mesh states %s about a consumer", match, orNothing(consumerFacts)) - } - return match - } - switch alphabet { - case "": - return as - case "dns": - return asDNSLabel(as) - default: - if failed == nil { - failed = fmt.Errorf( - "says %s, and the mesh writes an identity as %s", match, orNothing(consumerAlphabets)) - } - return match - } - }) - if failed != nil { - return "", failed - } - return out, nil -} - -// asDNSLabel writes a minted identity as a DNS label. -// -// The mesh's identities are already lower-case letters, digits and `_` (ConsumerIdentity), and -// already inside the bound of the provision serving them: a provider that serves one as a DNS label -// bounds it at 63 or less (CheckServes, novox/hq ADR 0225), and one that serves it at all without -// saying is held to twenty (IdentityBoundOf). So -// this is the separator and nothing else — no lower-casing of what is already lower case, no -// truncation to a limit the identity is already inside, no padding of a name that is already long -// enough. Each of those would be the mesh guessing at a rule it has not been given. -func asDNSLabel(as string) string { - return strings.ReplaceAll(as, "_", "-") -} - -// CheckServes refuses a `serves` block that names a consumer fact or an alphabet the mesh does not -// have, when the definition is parsed rather than when a consumer is resolved. -// -// A provision nobody consumes yet still has its rule read: a definition that would be refused the -// first time somebody required it is a definition that is wrong now. -func CheckServes(m Manifest) []string { - var problems []string - for _, provision := range sortedServes(m.Serves) { - for _, key := range sortedAnyKeys(m.Serves[provision]) { - text, ok := m.Serves[provision][key].(string) - if !ok { - // A provider's own name is filled only in a value of its own (novox/hq ADR 0346): one - // nested in a list or an object would be served as the placeholder, unsaid. - problems = append(problems, nestedOwnNames(m, provision, key, m.Serves[provision][key])...) - continue - } - // A probe identity, because what is checked is the shape of the statement and not - // what any consumer is called. - if _, err := consumerInto(text, "mesh_node_module"); err != nil { - problems = append(problems, fmt.Sprintf( - "%s serves %s, and the value it serves as %q %s", m.Module, provision, key, err)) - } - // And the provider's own names it may serve (novox/hq ADR 0346). - problems = append(problems, checkServedOwnNames(m, provision, key, text)...) - // A label longer than DNS keeps is not truncated here (asDNSLabel), so the offer's own - // bound has to keep the identity inside one (ADR 0225). - if strings.Contains(text, "${consumer:as:dns}") { - if b := m.IdentityBoundOf(provision); !b.Bounded() || b.Max > dnsLabelLimit { - problems = append(problems, fmt.Sprintf( - "%s serves %s's consumers their identity as a DNS label in %q, and bounds "+ - "that identity at %s: a label keeps %d — state an `identity` of at most %d", - m.Module, provision, key, boundWords(b), dnsLabelLimit, dnsLabelLimit)) - } - } - } - } - return problems -} - -// dnsLabelLimit is the longest DNS label (RFC 1035 §2.3.4). -const dnsLabelLimit = 63 - -func boundWords(b IdentityBound) string { - if !b.Bounded() { - return "nothing" - } - return fmt.Sprintf("%d", b.Max) -} - -func sortedServes(serves map[string]map[string]any) []string { - out := make([]string, 0, len(serves)) - for k := range serves { - out = append(out, k) - } - sort.Strings(out) - return out -} - -func sortedAnyKeys(values map[string]any) []string { - out := make([]string, 0, len(values)) - for k := range values { - out = append(out, k) - } - sort.Strings(out) - return out -} - -// derivedFor is what the provider on this machine derives for one consumer of one provision -// (novox/hq ADR 0202). -// -// Settled first, then derived: an operator may set a prefix on what the provider serves and the -// mesh still fills the consumer's half of it ([ADR 0174]). Only the keys that actually name the -// consumer are returned — the rest of a `serves` block is the same for every consumer and is -// already in the provider's own definition, so repeating it here would be a second copy to go -// stale. -// -// The first module in the resolved order that says it serves the provision answers, which is the -// choice servedOnThisMachine makes for the consumer's half. Nothing serving it on this machine is -// not an error: a contribution can reach a machine whose provider is a record or an adapter, and -// then there is nothing derived to tell. -func (r Resolution) derivedFor(provision, as, consumer, local string, settings SettingsBy) (map[string]any, error) { - for _, m := range r.Modules { - serves, said := m.Serves[provision] - if !said { - continue - } - var names map[string]any - for key, value := range serves { - if text, ok := value.(string); ok && strings.Contains(text, "${consumer:") { - if names == nil { - names = map[string]any{} - } - names[key] = value - } - } - if names == nil { - return nil, nil - } - // **A consumer that keeps several holders of this provision is refused** — this is issue - // 124's own failure one case to the side, and it would be just as quiet. - // - // Each holder gets its own login, `…_` (ADR 0094), and a provider derives from the - // login, so it would make one resource per holder. The consumer's side has no such - // dimension: one binding file per provision, one `${bound::}`, both - // derived from the un-suffixed identity. So the provider would create the holder's - // resource and the consumer would be configured against a name nothing made — it would - // authenticate successfully and be refused on every object, which reads like a credential - // fault and is not one. - // - // Lifting this means giving the consumer's side a local dimension. That is a decision, - // not an omission, and until it is taken the mesh says so rather than guessing. - if local != "" { - return nil, fmt.Errorf( - "%s keeps several holders of %s (this one is %q), and %s derives %s for each "+ - "consumer from the login the mesh minted. Each holder has its own login, and a "+ - "consumer is told one value per requirement — so the two ends would name "+ - "different things and nothing would compare them (novox/hq ADR 0202)", - consumer, local, provision, m.Module, orNothing(sortedAnyKeys(names))) - } - settled, err := Settle(names, WithDefaults(m, settings[m.Module])) - if err != nil { - return nil, fmt.Errorf("%s serving %s: %w", m.Module, provision, err) - } - derived, err := ServedTo(settled, as) - if err != nil { - return nil, fmt.Errorf("%s serving %s to %s: %w", m.Module, provision, as, err) - } - return derived, nil - } - return nil, nil -} - -// notTranscribed refuses a consumer's file that writes out the value its provider derives for it, -// instead of asking for it (novox/hq ADR 0202, issue 124). -// -// **What would have caught the one wrong instance.** The object store's three consumers each wrote -// their bucket into their own configuration by hand. One of them named a predecessor's bucket, and -// nothing compared it to what the provider would actually create: the module would have -// authenticated successfully and been refused on every object, which reads like a credential fault -// and is not one. It looked authoritative for months. -// -// The test is exact and costs one string search: a definition whose file already contains the -// value the mesh is about to derive for it has written down somebody else's rule. It cannot be a -// coincidence — a derived value carries the identity the mesh minted for this very consumer on -// this very machine, which nothing else would spell out — and it cannot be checked afterwards, -// because after substitution every consumer's file contains it legitimately. -// -// Only values that actually name the consumer are judged. A provider that serves a constant under -// the same key serves the same constant to everyone, and a consumer repeating it is redundant -// rather than wrong. -func notTranscribed(resource map[string]any, known map[string]map[string]string, module string) error { - if fmt.Sprint(resource["type"]) != "file" { - return nil - } - content, ok := resource["content"].(string) - if !ok || content == "" { - return nil - } - for _, provision := range sortedKnown(known) { - values := known[provision] - identity := values["as"] - if identity == "" { - continue - } - for _, key := range sortedStringKeys(values) { - if key == "as" { - // The login is not derived from itself, and a consumer that must present it in a - // connection string legitimately has it from `${bound:…}` — which is what it will - // be after substitution, so this would judge the substitution, not the module. - continue - } - value := values[key] - if value == "" || !namesTheConsumer(value, identity) { - continue - } - if !strings.Contains(content, value) { - continue - } - return fmt.Errorf( - "%s writes %q into %v, and that is exactly what %s derives for it — a definition "+ - "keeping its own copy of somebody else's naming rule is one that can disagree "+ - "with it, silently. Say ${bound:%s:%s} and be told", - module, value, resource["id"], provision, provision, key) - } - } - return nil -} - -// namesTheConsumer is whether a derived value was built from this consumer's identity — in the -// alphabet it was minted in, or as a DNS label. A value that does not contain it was not derived -// from it, whatever else it may be. -func namesTheConsumer(value, identity string) bool { - return strings.Contains(value, identity) || strings.Contains(value, asDNSLabel(identity)) -} - -func sortedKnown(known map[string]map[string]string) []string { - out := make([]string, 0, len(known)) - for k := range known { - out = append(out, k) - } - sort.Strings(out) - return out -} - -func sortedStringKeys(values map[string]string) []string { - out := make([]string, 0, len(values)) - for k := range values { - out = append(out, k) - } - sort.Strings(out) - return out -} diff --git a/internal/catalogue/consumers.go b/internal/catalogue/consumers.go deleted file mode 100644 index 46a64e0e..00000000 --- a/internal/catalogue/consumers.go +++ /dev/null @@ -1,110 +0,0 @@ -package catalogue - -import ( - "fmt" - "regexp" - "slices" -) - -// A provider's consumers' credentials, in a file of the provider's own (novox/hq ADR 0373, issue 679). -// -// A provider in the tool runner may not ask the vault (ADR 0349 clause 11, ADR 0370 clause 2), so its template names -// `${consumers:}` and its machine's provisioner client fills it: one entry per consumer bound to the -// provider for that provision, each carrying the credential the vault keeps for the pair. Only the client fills it, -// so the placeholder is a provider's alone, for a provision it provides, in a file only the provider reads (mode 0600 -// or 0400, owned at runtime, so a change on disk is written back); and a module not handed to the client on a machine -// is refused there, never written the placeholder's text, nor an empty list the provider would read as no consumers. - -// ConsumersNamespace is a provider's consumers' credentials' placeholder namespace, `${consumers:}`. -const ConsumersNamespace = "consumers" - -// ofConsumers is where a provider places its consumers' credentials. Loose inside the braces, so a misspelt name is -// refused rather than passed over. -var ofConsumers = regexp.MustCompile(`\$\{consumers:([^}]*)\}`) - -// consumersPlaceholderProblems is what is wrong with one resource's ${consumers:…}: placed only in a file's content, -// for a provision the module provides, in a file only the provider reads and the client writes back. -func consumersPlaceholderProblems(m Manifest, r map[string]any) []string { - var problems []string - for _, field := range sortedKeys(r) { - v, ok := r[field].(string) - if !ok { - continue - } - for _, f := range ofConsumers.FindAllStringSubmatch(v, -1) { - switch { - case field != "content" || fmt.Sprint(r["type"]) != "file": - problems = append(problems, fmt.Sprintf("%s's resource %v names %s in its %s; a provider's consumers "+ - "are placed only in a file's content (novox/hq ADR 0373)", m.Module, r["id"], f[0], field)) - continue - case !slices.ContainsFunc(m.Provides, func(o Offer) bool { return o.Name == f[1] }): - problems = append(problems, fmt.Sprintf("%s's resource %v names %s, and %s does not provide %q: only a "+ - "provider's template holds its consumers' credentials (novox/hq ADR 0373)", m.Module, r["id"], f[0], - m.Module, f[1])) - } - if mode := fmt.Sprint(r["mode"]); mode != "0600" && mode != "0400" { - problems = append(problems, fmt.Sprintf("%s's resource %v holds its consumers' credentials (%s) with "+ - "mode %q: the provider alone reads it, 0600 or 0400 (novox/hq ADR 0373)", m.Module, r["id"], f[0], - r["mode"])) - } - if owner := ownershipOf(r); owner != RuntimeOwned { - problems = append(problems, fmt.Sprintf("%s's resource %v holds its consumers' credentials (%s) and is "+ - "%s at runtime: it is owned, written back whenever it differs (novox/hq ADR 0373)", m.Module, - r["id"], f[0], owner)) - } - } - } - return problems -} - -// consumersHandedOnly says what a file naming ${consumers:…} is where it would be written the old way: the provisioner's -// client alone fills it, so on a machine that does not hand the module it is **left out** of the declaration, never -// written as that text or as an empty list the provider would read as no consumers, and never the reason the module is -// refused: a provider not handed there keeps working as it did, from its grant files (novox/hq ADR 0373, issue 780). -// A moved module judged from its repository's tree (WithTemplatesFrom: `module check`), on a machine -// that would hand it, has no templates archive to hand and is judged as it composes; there the placeholder is judged -// and taken out of the text, since what is composed so is never sent (judged answers whether the file is so judged). -func consumersHandedOnly(m Manifest, r map[string]any, with Rendering) (judged, leftOut bool) { - content, _ := r["content"].(string) - switch { - case ofConsumers.FindString(content) == "": - return false, false - case m.Moved && m.Handed == nil && with.Hands && with.Provisioner != "": - return true, false - } - return false, true -} - -// consumersLeftOutLine is the plan's one line for a file left out because the module is not handed. -func consumersLeftOutLine(m Manifest, r map[string]any) string { - return fmt.Sprintf("%s's %v names ${consumers:…}, which the provisioner's client alone fills: left out, handed only "+ - "(novox/hq ADR 0373)", m.Module, r["id"]) -} - -// consumersNotHanded is consumersHandedOnly as a refusal, for a caller that has no way to leave the file out. -func consumersNotHanded(m Manifest, r map[string]any, with Rendering) (judged bool, err error) { - judged, left := consumersHandedOnly(m, r, with) - if left { - return false, fmt.Errorf("%s's resource %v names %s, which the provisioner's client alone fills, and %s is not "+ - "handed to it on this machine: it would be written as that text, or empty, and the provider would read no "+ - "consumers (novox/hq ADR 0373)", m.Module, r["id"], ofConsumers.FindString(fmt.Sprint(r["content"])), m.Module) - } - return judged, nil -} - -// consumersOfTemplate is what is wrong with a handed file's template naming ${consumers:…}, judged on the file's -// resource as if its text were inline. -func consumersOfTemplate(m Manifest, id, text string) []string { - for _, r := range m.Resources { - if fmt.Sprint(r["id"]) != id { - continue - } - withText := map[string]any{} - for k, v := range r { - withText[k] = v - } - withText["content"] = text - return consumersPlaceholderProblems(m, withText) - } - return nil -} diff --git a/internal/catalogue/core_credentials.go b/internal/catalogue/core_credentials.go deleted file mode 100644 index f11b70c0..00000000 --- a/internal/catalogue/core_credentials.go +++ /dev/null @@ -1,33 +0,0 @@ -package catalogue - -import ( - "encoding/json" - "slices" -) - -// The core's own credentials (novox/hq ADR 0349 §16, issue 755). The bus, the node-engine, the controller and -// mesh-provisioner need their files before any provisioner client runs, so their files stay placement and their -// credentials travel in the declaration as values the vault sealed to the machine they run on, which the controller -// carries and cannot open; the node-engine opens them there (mesh-host's VaultBoxContract). - -// CoreCredentialModules are the modules whose own secrets are the core's credentials: the bus, mesh-provisioner and -// its client. The node-engine and the controller hold no own secret of a module's. **Fixed here, in the controller's -// code, as the clause says** — and the vault keeps the same list (mesh-vault's CoreModules), answering the controller -// for these alone. -var CoreCredentialModules = []string{"mesh-provisioner", "mesh-provisioner-client", "nats"} - -// CarriesVaultSealed says whether module's own secrets are the core's credentials. -func CarriesVaultSealed(module string) bool { return slices.Contains(CoreCredentialModules, module) } - -// VaultSealedField is the field of a file the vault sealed to its machine (mesh-host's File.VaultSealed). -const VaultSealedField = "vault-sealed" - -// ForeseenVaultBox stands for a core credential's box in a composition asked ahead of the send, when none is kept -// yet: the send asks the vault for it. Never sent. -var ForeseenVaultBox = json.RawMessage(`{"v":1,"eph":"","iv":"","tag":"","ct":"","foreseen":"asked of the vault by the next send"}`) - -// vaultSealedOf is the vault's box for one of module's own secrets, if the rendering was given one. -func (with Rendering) vaultSealedOf(module, name string) (json.RawMessage, bool) { - box, ok := with.VaultSealed[module][name] - return box, ok && len(box) > 0 -} diff --git a/internal/catalogue/cron.go b/internal/catalogue/cron.go deleted file mode 100644 index fbb6d299..00000000 --- a/internal/catalogue/cron.go +++ /dev/null @@ -1,92 +0,0 @@ -package catalogue - -// A five-field cron validator, enough to refuse a malformed schedule near its author -// (novox/hq ADR 0053). -// -// **Validation only, and written rather than pulled in.** The control plane's part in a scheduled -// step is small and exact: carry the field to the host unchanged, and refuse a `schedule` that is -// not a well-formed cron here rather than on the machine — the same near-versus-far discipline the -// manifest keeps for an action a module may not declare and a run-once that is not a boolean. What -// *when to run it again* means is the host's to evaluate; the control plane only checks the string -// is one it could. A cron library would be a dependency carried for evaluation nobody does here. -// -// The ordinary five fields — minute, hour, day-of-month, month, day-of-week — with `*`, lists -// (`,`), ranges (`-`) and steps (`/`). No names (jan, mon): a numeric cron is what the host -// evaluates, and refusing a name here is refusing it near whoever wrote it. - -import ( - "fmt" - "strconv" - "strings" -) - -// validateCron reports why a string is not a five-field cron expression, or nil if it is one. -func validateCron(expr string) error { - fields := strings.Fields(expr) - if len(fields) != 5 { - return fmt.Errorf( - "a schedule is a five-field cron expression (minute hour day-of-month month "+ - "day-of-week), and this has %d field(s)", len(fields)) - } - bounds := []struct { - name string - min, max int - }{ - {"minute", 0, 59}, - {"hour", 0, 23}, - {"day-of-month", 1, 31}, - {"month", 1, 12}, - {"day-of-week", 0, 7}, // 0 and 7 are both Sunday, the convention every cron keeps - } - for i, b := range bounds { - if err := validateCronField(fields[i], b.min, b.max); err != nil { - return fmt.Errorf("%s field: %w", b.name, err) - } - } - return nil -} - -// validateCronField checks one field's elements are within range and well-formed. -func validateCronField(spec string, min, max int) error { - if spec == "" { - return fmt.Errorf("is empty") - } - for _, part := range strings.Split(spec, ",") { - if part == "" { - return fmt.Errorf("%q has an empty element between commas", spec) - } - - rangePart := part - if slash := strings.IndexByte(part, '/'); slash >= 0 { - rangePart = part[:slash] - n, err := strconv.Atoi(part[slash+1:]) - if err != nil || n < 1 { - return fmt.Errorf("step in %q is not a positive number", part) - } - } - - switch { - case rangePart == "*": - // Any value; nothing to bound. - case strings.IndexByte(rangePart, '-') >= 0: - dash := strings.IndexByte(rangePart, '-') - lo, errLo := strconv.Atoi(rangePart[:dash]) - hi, errHi := strconv.Atoi(rangePart[dash+1:]) - if errLo != nil || errHi != nil { - return fmt.Errorf("range %q is not two numbers", rangePart) - } - if lo < min || hi > max || lo > hi { - return fmt.Errorf("range %q is outside the allowed %d-%d", part, min, max) - } - default: - v, err := strconv.Atoi(rangePart) - if err != nil { - return fmt.Errorf("%q is not a number", rangePart) - } - if v < min || v > max { - return fmt.Errorf("%q is outside the allowed range %d-%d", part, min, max) - } - } - } - return nil -} diff --git a/internal/catalogue/data.go b/internal/catalogue/data.go deleted file mode 100644 index 5a43203c..00000000 --- a/internal/catalogue/data.go +++ /dev/null @@ -1,791 +0,0 @@ -package catalogue - -import ( - "bytes" - "encoding/json" - "fmt" - "regexp" - "sort" - "strconv" - "strings" - "time" -) - -// A module declares the data it holds, and the mesh protects and watches it from that declaration -// (novox/hq ADR 0233). -// -// **One section, `data`, for every kind of data a module keeps:** its own, by directory — a store's -// files, a mail spool, an application's uploads — or by an operator's path it was given; what it keeps -// for its consumers, by the provision they reach it through; and what of its own lives with a -// provider, by the provision it requires. Each entry has a class — how precious it is — and, for its -// own data, how it is protected; everything the mesh does is derived from those, never written per -// module: -// -// - a binding to a consumer's data does not move (ADR 0232) where the provision's class keeps data; -// - the backup holder's lines are composed from it — a module no longer writes them by hand; -// - what an unassignment leaves behind of an irreplaceable or valuable item is retired, listed by -// `cleanup list` and deleted only by `cleanup delete` (ADR 0230); -// - the self-check measures every item and says when one shrinks, disappears, stops being written, -// goes without its backup, sits on a degraded array, or is replaced by an empty copy of itself — -// urgent for what is irreplaceable, a warning for what is valuable. - -// The classes: how precious the data is. A fixed vocabulary, ranked by the operator (2026-10-06): a -// class is what the protections are derived from, so a new one is a decision, not a manifest's choice. -const ( - // ClassIrreplaceable is what must never be lost: the operator names it (the media library, the - // photo sites' storage). Protected by a backup or by a declared redundancy — one is required — - // retired rather than removed, and every alert about it is urgent. - ClassIrreplaceable = "irreplaceable" - // ClassValuable is anybody's work that would be painful to lose: in the nightly backup by default, - // retired rather than removed, and every alert about it a warning. - ClassValuable = "valuable" - // ClassRebuildable can be made again from something kept elsewhere — a clone, an index, a - // download, a night's dump — at a cost in time, not in data. In the nightly backup by default; - // forgotten when its module goes, and never alerted on. - ClassRebuildable = "rebuildable" - // ClassCache may be emptied at any moment with nothing lost but speed: never backed up, never - // measured, never alerted on. - ClassCache = "cache" - // ClassNone is a provision that keeps nothing of its consumers' (consumers only): a resolver, a - // certificate authority, an artifact store. - ClassNone = "none" -) - -// classRank orders the classes by how precious they are, so the stricter of two is chosen. -var classRank = map[string]int{ClassNone: 0, ClassCache: 1, ClassRebuildable: 2, ClassValuable: 3, ClassIrreplaceable: 4} - -// StricterClass is the more precious of two classes. -func StricterClass(a, b string) string { - if classRank[b] > classRank[a] { - return b - } - return a -} - -// Retires is whether a class's data is retired, not forgotten, when its machine no longer declares it. -func Retires(class string) bool { return class == ClassIrreplaceable || class == ClassValuable } - -// Watched is whether a class's data raises conditions: irreplaceable and valuable. -func Watched(class string) bool { return Retires(class) } - -// DefaultBackupWithin is how old the last good backup of an item may be before the self-check says so -// (novox/hq ADR 0214: a machine with data and no good backup in 48 hours). -const DefaultBackupWithin = 48 * time.Hour - -// Data is a manifest's `data` section. -type Data struct { - // Own is the data this module keeps itself. - Own []DataItem `json:"own,omitempty"` - // Consumers is what it keeps for its consumers, per provision it grants. - Consumers map[string]ConsumerData `json:"consumers,omitempty"` - // KeptBy is what of its own lives with the provider of a provision it requires — its rows, its - // objects — and how precious that is. The provider's protections follow the stricter of its own - // class for its consumers and this. - KeptBy map[string]KeptData `json:"kept-by,omitempty"` -} - -// DataItem is one piece of a module's own data. -type DataItem struct { - // ID names it within the module: what `data`, `cleanup` and a condition call it. - ID string `json:"id"` - // Path is one of the module's directories, `${dir:}`, or an operator's path it was given, - // `${access:}`, or a path beneath either. Never a machine path (novox/hq ADR 0112). - Path string `json:"path"` - Class string `json:"class"` - // Backup is how it is copied: "copy" (the holder reads it as it stands), "none", or a dump — a - // command that writes a consistent copy into another item, which is copied. Unsaid, anything but a - // cache is copied, unless it says it is protected by redundancy instead. - Backup *DataBackup `json:"backup,omitempty"` - // Redundancy says it is protected by the redundancy of the storage it lives on rather than by a - // copy, and why that is enough: the media library on an array there is no room to copy. The - // backup holder then watches that array, and a degraded one is said. - Redundancy string `json:"redundancy,omitempty"` - // Within is how old its last good backup may be; unsaid, DefaultBackupWithin. - Within string `json:"within,omitempty"` - // Measure is how the backup holder measures it: `walk` (the default — every file, at most daily and - // bounded, for small items), `dataset` (a ZFS dataset's own counters, hourly, nothing walked) or - // `shallow` (its top-level entries only, no size). A large item never says walk. - Measure string `json:"measure,omitempty"` - // Active is how long it may go unwritten before that is a fault — for data something is - // expected to write all the time. Unsaid, a quiet item is not a fault. - Active string `json:"active,omitempty"` - // Why is a line for the reviewer: why this class. - Why string `json:"why,omitempty"` -} - -// ConsumerData is what a provider keeps for the consumers of one provision. -type ConsumerData struct { - Class string `json:"class"` - // In is where it lives: one of the module's own items (whose protection covers it), or a - // provision this module requires (whose provider keeps it, as its consumer). Unsaid only for none - // and cache. - In string `json:"in,omitempty"` - Why string `json:"why,omitempty"` -} - -// KeptData is how precious what a module keeps with a provider is. -type KeptData struct { - Class string `json:"class"` - Why string `json:"why,omitempty"` -} - -// DataBackup is how an item is copied. -type DataBackup struct { - Copy bool - None bool - // Dump is the command writing a consistent copy into the item Into. - Dump string - Into string -} - -// UnmarshalJSON reads "copy", "none" or {"dump": "...", "into": ""}. -func (b *DataBackup) UnmarshalJSON(raw []byte) error { - var word string - if err := json.Unmarshal(raw, &word); err == nil { - switch word { - case "copy": - *b = DataBackup{Copy: true} - case "none": - *b = DataBackup{None: true} - default: - return fmt.Errorf("a data item's backup is \"copy\", \"none\" or {dump, into}, not %q", word) - } - return nil - } - var full struct { - Dump string `json:"dump"` - Into string `json:"into"` - } - dec := json.NewDecoder(bytes.NewReader(raw)) - dec.DisallowUnknownFields() - if err := dec.Decode(&full); err != nil { - return fmt.Errorf("a data item's backup is \"copy\", \"none\" or {dump, into}: %w", typedUnknown(err)) - } - *b = DataBackup{Dump: full.Dump, Into: full.Into} - return nil -} - -// MarshalJSON writes it back in the form it was written. -func (b DataBackup) MarshalJSON() ([]byte, error) { - switch { - case b.Copy: - return json.Marshal("copy") - case b.None: - return json.Marshal("none") - } - return json.Marshal(struct { - Dump string `json:"dump"` - Into string `json:"into"` - }{b.Dump, b.Into}) -} - -// IsDump is whether the item is copied by a dump. -func (b *DataBackup) IsDump() bool { return b != nil && b.Dump != "" } - -// BackedUp is whether the holder keeps restore points of this item: anything but a cache by default — -// the nightly backup is the standard plan — unless it says "none", or says it is protected by -// redundancy and says nothing of a backup. -func (it DataItem) BackedUp() bool { - switch { - case it.Backup == nil: - return it.Class != ClassCache && it.Redundancy == "" - case it.Backup.None: - return false - } - return true -} - -// Protection is how the item is protected, in one word: "backup", "redundancy", both joined by "+", -// or "none". -func (it DataItem) Protection() string { - var by []string - if it.BackedUp() { - by = append(by, "backup") - } - if it.Redundancy != "" { - by = append(by, "redundancy") - } - if len(by) == 0 { - return "none" - } - return strings.Join(by, "+") -} - -// The ways an item is measured. -const ( - MeasureWalk = "walk" - MeasureDataset = "dataset" - MeasureShallow = "shallow" -) - -// MeasuredBy is how the item is measured, with the default applied. -func (it DataItem) MeasuredBy() string { - if it.Measure == "" { - return MeasureWalk - } - return it.Measure -} - -// OwnedByModule is whether the item is in one of the module's own directories — the mesh's to retire — -// rather than an operator's path it was given, which the mesh never retires and never deletes. -func (it DataItem) OwnedByModule() bool { return strings.HasPrefix(it.Path, "${dir:") } - -// BackupWithin is the item's bound on its last good backup. -func (it DataItem) BackupWithin() time.Duration { - if d, err := ParseDataDuration(it.Within); err == nil && d > 0 { - return d - } - return DefaultBackupWithin -} - -// ActiveWithin is how long the item may go unwritten; zero when quiet is not a fault. -func (it DataItem) ActiveWithin() time.Duration { - d, _ := ParseDataDuration(it.Active) - return d -} - -// ParseDataDuration reads "48h", "90m" or "7d"; empty is zero. -func ParseDataDuration(s string) (time.Duration, error) { - s = strings.TrimSpace(s) - if s == "" { - return 0, nil - } - if days, ok := strings.CutSuffix(s, "d"); ok { - n, err := strconv.Atoi(days) - if err != nil || n <= 0 { - return 0, fmt.Errorf("%q is not a number of days", s) - } - return time.Duration(n) * 24 * time.Hour, nil - } - d, err := time.ParseDuration(s) - if err != nil || d <= 0 { - return 0, fmt.Errorf("%q is not a duration (48h, 90m, 7d)", s) - } - return d, nil -} - -// DataItems is the module's own data, in the order declared. -func (m Manifest) DataItems() []DataItem { - if m.Data == nil { - return nil - } - return m.Data.Own -} - -// DataItem is one own item by id. -func (m Manifest) DataItem(id string) (DataItem, bool) { - for _, it := range m.DataItems() { - if it.ID == id { - return it, true - } - } - return DataItem{}, false -} - -// ConsumerDataOf is what this module says it keeps for a provision's consumers, and whether it says. -func (m Manifest) ConsumerDataOf(provision string) (ConsumerData, bool) { - if m.Data == nil { - return ConsumerData{}, false - } - c, ok := m.Data.Consumers[provision] - return c, ok -} - -// KeptByOf is how precious what this module keeps with a provision's provider is, and whether it says. -func (m Manifest) KeptByOf(provision string) (KeptData, bool) { - if m.Data == nil { - return KeptData{}, false - } - k, ok := m.Data.KeptBy[provision] - return k, ok -} - -// keepsByClass is whether a class keeps something a binding must not move away from. -func keepsByClass(class string) bool { - return class == ClassIrreplaceable || class == ClassValuable || class == ClassRebuildable -} - -// dataID is what an item's id may be: it is a token in a condition's key and a word on a line. -var dataID = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`) - -// dataPathRef is an item's path: one of the module's directories or accesses, and optionally a path -// beneath it. -var dataPathRef = regexp.MustCompile(`^\$\{(dir|access):([a-z0-9][a-z0-9-]*)\}(/[^\s$]*)?$`) - -// dataProblems is what is wrong with the `data` section itself, from the manifest alone: judged at -// registration as every other per-manifest problem is. Whether a module declares what it should is -// the catalogue check's (DataProblems), because a module already running was written before it. -func (m Manifest) dataProblems() []string { - if m.Data == nil { - return nil - } - var problems []string - say := func(format string, args ...any) { - problems = append(problems, fmt.Sprintf("%s's data: ", m.Module)+fmt.Sprintf(format, args...)) - } - dirs := map[string]bool{} - for _, r := range m.Resources { - if fmt.Sprint(r["type"]) == "directory" { - dirs[fmt.Sprint(r["id"])] = true - } - } - accesses := map[string]bool{} - for _, a := range m.Accesses { - if a.ID != "" { - accesses[a.ID] = true - } - } - ids := map[string]DataItem{} - paths := map[string]string{} - for i, it := range m.Data.Own { - label := it.ID - if label == "" { - label = fmt.Sprintf("item %d", i+1) - } - if !dataID.MatchString(it.ID) { - say("%s has no usable id: lower-case letters, digits and dashes", label) - } else if _, twice := ids[it.ID]; twice { - say("%s is declared twice", it.ID) - } - ids[it.ID] = it - ref := dataPathRef.FindStringSubmatch(it.Path) - switch { - case ref == nil: - say("%s's path %q is not one of the module's directories or accesses: ${dir:} or ${access:}, "+ - "or a path beneath one (a definition names no machine path, novox/hq ADR 0112)", label, it.Path) - case ref[1] == "dir" && !dirs[ref[2]]: - say("%s's path names ${dir:%s}, and %s declares no directory %q", label, ref[2], m.Module, ref[2]) - case ref[1] == "access" && !accesses[ref[2]]: - say("%s's path names ${access:%s}, and %s declares no access %q", label, ref[2], m.Module, ref[2]) - case strings.Contains(it.Path, ".."): - say("%s's path %q climbs out of its directory", label, it.Path) - } - if other, twice := paths[it.Path]; twice && it.Path != "" { - say("%s and %s name the same path %s", other, label, it.Path) - } - paths[it.Path] = label - switch it.Class { - case ClassIrreplaceable, ClassValuable, ClassRebuildable, ClassCache: - case ClassNone: - say("%s is class none, which only a provision keeping nothing of its consumers' is; own data "+ - "that is disposable is cache", label) - default: - say("%s's class %q is not one the mesh protects by: irreplaceable, valuable, rebuildable or cache", - label, it.Class) - } - if it.Class == ClassIrreplaceable && !it.BackedUp() && strings.TrimSpace(it.Redundancy) == "" { - say("%s is irreplaceable and is neither backed up nor said to be protected by redundancy; the only "+ - "copy of something is protected one way or the other (novox/hq ADR 0233) — copy it, dump it into "+ - "another item, or say `redundancy` with why that is enough", label) - } - if it.Class == ClassCache && it.Backup != nil && !it.Backup.None { - say("%s is a cache and asks to be backed up; a cache is disposable, or it is not a cache", label) - } - if it.Class == ClassCache && it.Redundancy != "" { - say("%s is a cache and says it is protected by redundancy; a cache is not protected", label) - } - switch it.Measure { - case "", MeasureWalk, MeasureDataset, MeasureShallow: - default: - say("%s's measure %q is walk, dataset or shallow", label, it.Measure) - } - if _, err := ParseDataDuration(it.Within); err != nil { - say("%s's within: %v", label, err) - } - if _, err := ParseDataDuration(it.Active); err != nil { - say("%s's active: %v", label, err) - } - if it.Within != "" && !it.BackedUp() { - say("%s states within, and is not backed up", label) - } - if it.Active != "" && !Watched(it.Class) { - say("%s is %s and expects writes; only irreplaceable and valuable data is watched", label, it.Class) - } - } - // Dumps go into an item of the same module, declared, and not into themselves. - for _, it := range m.Data.Own { - if it.Backup == nil || it.Backup.Copy || it.Backup.None { - continue - } - switch into, ok := ids[it.Backup.Into]; { - case strings.TrimSpace(it.Backup.Dump) == "": - say("%s's backup names no dump command", it.ID) - case it.Backup.Into == "": - say("%s's dump says no item it writes into", it.ID) - case !ok: - say("%s's dump writes into %q, which is not one of the module's data items", it.ID, it.Backup.Into) - case into.ID == it.ID: - say("%s's dump writes into itself; a dump is copied from where it lands", it.ID) - case into.Class == ClassCache: - say("%s's dump writes into %s, a cache; what is copied must not be disposable", it.ID, into.ID) - } - if it.Backup.Dump != "" { - declared := map[string]string{} - for d := range dirs { - declared[d] = "" - } - if _, err := dirFill(it.Backup.Dump, declared, m.Module); err != nil { - say("%s's dump: %v", it.ID, err) - } - } - } - provided := map[string]bool{} - for _, o := range m.Provides { - provided[o.Name] = true - } - wants := map[string]bool{} - for _, w := range m.Wants() { - wants[w] = true - } - for _, provision := range sortedKeys(m.Data.Consumers) { - c := m.Data.Consumers[provision] - if !provided[provision] { - say("says what it keeps for the consumers of %q, which it does not provide", provision) - } - switch c.Class { - case ClassIrreplaceable, ClassValuable, ClassRebuildable, ClassCache, ClassNone: - default: - say("its consumers' %s is class %q; one of irreplaceable, valuable, rebuildable, cache or none", provision, c.Class) - } - switch { - case c.Class == ClassNone && c.In != "": - say("its consumers' %s keeps nothing and says where it is kept (%s)", provision, c.In) - case keepsByClass(c.Class) && c.In == "": - say("its consumers' %s is %s and says nowhere it lives: `in` names one of the module's items, or a "+ - "provision it requires", provision, c.Class) - case c.In != "": - if own, ok := ids[c.In]; ok { - if c.Class == ClassIrreplaceable && !own.BackedUp() && own.Redundancy == "" { - say("its consumers' %s is irreplaceable and lives in %s, which is not protected", provision, c.In) - } - if classRank[own.Class] < classRank[c.Class] { - say("its consumers' %s is %s and lives in %s, which is only %s", provision, c.Class, c.In, own.Class) - } - } else if !wants[c.In] { - say("its consumers' %s lives in %q, which is neither one of its data items nor a provision it "+ - "requires", provision, c.In) - } - } - } - for _, provision := range sortedKeys(m.Data.KeptBy) { - k := m.Data.KeptBy[provision] - if !wants[provision] { - say("says it keeps data with the provider of %q, which it does not require", provision) - } - switch k.Class { - case ClassIrreplaceable, ClassValuable, ClassRebuildable, ClassCache: - default: - say("what it keeps with %s is class %q; one of irreplaceable, valuable, rebuildable or cache", provision, k.Class) - } - } - return problems -} - -// KeepsConsumerData is whether this module, providing a provision, keeps what each consumer writes -// there (novox/hq ADR 0232, ADR 0233): whether a consumer bound to it is bound to its data. -// -// **What the data section says, it gets**: irreplaceable, valuable or rebuildable keeps; cache and none -// do not — a cache lost in a move costs speed, not data. Before the section, an offer said it with -// `keeps-consumer-data`, which is still read; unsaid in both, a provider that grants each consumer a -// credential of its own keeps that consumer's data (ADR 0232 §1). The catalogue check refuses the -// unsaid case for a provider that grants (DataProblems), so the inference is what an older definition -// on the shelf gets, never a new one. -func (m Manifest) KeepsConsumerData(provision string) bool { - if c, ok := m.ConsumerDataOf(provision); ok { - return keepsByClass(c.Class) - } - for _, o := range m.Provides { - if o.Name == provision && o.KeepsConsumerData != nil { - return *o.KeepsConsumerData - } - } - _, grants := m.Grants[provision] - return grants -} - -// NeedsBackupHolder is whether a module's data needs the machine's backup holder to be there: it keeps -// something irreplaceable — backed up by the holder, or protected by an array the holder watches. A -// module keeping only valuable or rebuildable data is backed up where a holder is, and refused nowhere -// for want of one: the standard plan, not a requirement. -func (m Manifest) NeedsBackupHolder() bool { - for _, it := range m.DataItems() { - if it.Class == ClassIrreplaceable { - return true - } - } - return false -} - -// --- derived: the backup holder's lines --------------------------------------------------------- - -// The node-backup seat's kinds (novox/hq to-be 43, ADR 0233): `backup` is what to run and which paths -// to keep, `data` every item with its class and protection, for the holder to measure and watch. -const ( - BackupKindBackup = "backup" - BackupKindData = "data" -) - -// derivedContributions is what a module's data section gives the backup holder: its backup lines and -// its items. Every item is listed, so a valuable one on a machine is measured whether or not it is -// copied; a cache is listed and not measured. -func (m Manifest) derivedContributions() []SeatContribution { - items := m.DataItems() - if len(items) == 0 { - return nil - } - var lines []string - kept := map[string]bool{} - keep := func(path string) { - if !kept[path] { - kept[path] = true - lines = append(lines, "path "+path) - } - } - for _, it := range items { - if !it.BackedUp() { - continue - } - if it.Backup.IsDump() { - lines = append(lines, "run "+strings.TrimSpace(it.Backup.Dump)) - if into, ok := m.DataItem(it.Backup.Into); ok { - keep(into.Path) - } - continue - } - keep(it.Path) - } - var described []string - for _, it := range items { - covered := "-" - switch { - case it.Backup.IsDump(): - if into, ok := m.DataItem(it.Backup.Into); ok { - covered = into.Path - } - case it.BackedUp(): - covered = it.Path - } - described = append(described, fmt.Sprintf("item %s %s %s %s %s %s", it.ID, it.Class, it.Path, covered, - it.Protection(), it.MeasuredBy())) - } - var out []SeatContribution - if len(lines) > 0 { - out = append(out, SeatContribution{Seat: BackupSeat, Kind: BackupKindBackup, Content: strings.Join(lines, "\n") + "\n"}) - } - return append(out, SeatContribution{Seat: BackupSeat, Kind: BackupKindData, Content: strings.Join(described, "\n") + "\n"}) -} - -// allContributions is every contribution a module makes to a seat's holder: what it wrote, and what -// its data section derives. **One list**: a module that declares its data has its backup lines -// composed from it, and a backup line it still writes by hand is not placed — the catalogue check -// refuses it — so the holder never copies two lists that disagree. -func (m Manifest) allContributions() []SeatContribution { - if m.Data == nil { - return m.Contributions - } - derived := m.derivedContributions() - out := make([]SeatContribution, 0, len(m.Contributions)+len(derived)) - for _, c := range m.Contributions { - if s, known := SeatNamed(c.Seat); known && s.Name == BackupSeat { - continue - } - out = append(out, c) - } - return append(out, derived...) -} - -// --- the catalogue check ------------------------------------------------------------------------ - -// DataProblems is what the catalogue check refuses about the data a module declares (novox/hq ADR -// 0233), judged over the manifests given together: -// -// - a provider that grants a provision says what it keeps for that provision's consumers; -// - nobody says it on the offer any more (`keeps-consumer-data`): the data section is the one place; -// - nobody writes a backup line by hand: it is derived from the data section; -// - a directory a container writes, mounted whole, is a data item of some class; -// - consumer data said to live in a required provision lives where that provision's provider keeps -// its consumers' data, as preciously, when the provider is given; -// - data a consumer keeps with a provider as irreplaceable is protected there, when the provider is -// given. -// -// **The check's, not registration's**, as ADR 0214's backup rule was: a module already on the shelf -// was written before the rule, and refusing it there would refuse the very providers whose data the -// rule protects. Each module meets it where it is written. -func DataProblems(shelf Shelf) []string { - var problems []string - for _, name := range shelfOrder(shelf) { - m := shelf[name] - for _, provision := range sortedKeys(m.Grants) { - if _, said := m.ConsumerDataOf(provision); !said { - problems = append(problems, fmt.Sprintf( - "%s grants %s and does not say what it keeps for its consumers: data.consumers.%s with a "+ - "class — irreplaceable, valuable, rebuildable, cache, or none (novox/hq ADR 0233)", name, provision, provision)) - } - } - for _, o := range m.Provides { - if o.KeepsConsumerData != nil { - problems = append(problems, fmt.Sprintf( - "%s says keeps-consumer-data on its offer of %s; it is said once, in data.consumers.%s, with a "+ - "class (novox/hq ADR 0233)", name, o.Name, o.Name)) - } - } - for i, c := range m.Contributions { - if s, known := SeatNamed(c.Seat); known && s.Name == BackupSeat { - problems = append(problems, fmt.Sprintf( - "%s's contribution %d is a backup line written by hand; backups are derived from the data "+ - "section — declare the data, with its class and how it is protected (novox/hq ADR 0233)", name, i+1)) - } - } - for _, dir := range writtenDirectories(m) { - if !coversDirectory(m, dir) { - problems = append(problems, fmt.Sprintf( - "%s mounts its directory %q into a container to be written, and declares no data in it: "+ - "data.own with a class — cache if it is disposable (novox/hq ADR 0233)", name, dir)) - } - } - if m.Data == nil { - continue - } - for _, provision := range sortedKeys(m.Data.Consumers) { - c := m.Data.Consumers[provision] - if c.In == "" { - continue - } - if _, own := m.DataItem(c.In); own { - continue - } - for _, pname := range shelfOrder(shelf) { - p := shelf[pname] - pc, said := p.ConsumerDataOf(c.In) - if !said || !providesName(p, c.In) { - continue - } - if classRank[pc.Class] < classRank[c.Class] { - problems = append(problems, fmt.Sprintf( - "%s keeps its consumers' %s, %s, in %s — and %s keeps its consumers' %s as %s, "+ - "so it is not protected as %s", name, provision, c.Class, c.In, pname, c.In, pc.Class, c.Class)) - } - } - } - for _, provision := range sortedKeys(m.Data.KeptBy) { - k := m.Data.KeptBy[provision] - if k.Class != ClassIrreplaceable { - continue - } - for _, pname := range shelfOrder(shelf) { - p := shelf[pname] - pc, said := p.ConsumerDataOf(provision) - if !said || !providesName(p, provision) { - continue - } - if !keepsByClass(pc.Class) { - problems = append(problems, fmt.Sprintf( - "%s keeps irreplaceable data with %s, and %s keeps its consumers' %s as %s — nothing of it is "+ - "protected there", name, provision, pname, provision, pc.Class)) - continue - } - if in, own := p.DataItem(pc.In); own && !in.BackedUp() && in.Redundancy == "" { - problems = append(problems, fmt.Sprintf( - "%s keeps irreplaceable data with %s, and %s keeps it in %s, which is neither backed up nor "+ - "on declared redundancy", name, provision, pname, pc.In)) - } - } - } - } - return problems -} - -func providesName(m Manifest, provision string) bool { - for _, o := range m.Provides { - if o.Name == provision { - return true - } - } - return false -} - -// writtenDirectories are the module's directories a container mounts whole and may write: a volume -// `${dir:}:` without `:ro`, or a directory stated by an absolute path mounted the same -// way. A file beneath a directory, or a read-only mount, is configuration the mesh wrote. -func writtenDirectories(m Manifest) []string { - absolute := map[string]string{} - for _, r := range m.Resources { - if fmt.Sprint(r["type"]) != "directory" { - continue - } - if p, ok := r["path"].(string); ok && strings.HasPrefix(p, "/") { - absolute[strings.TrimRight(p, "/")] = fmt.Sprint(r["id"]) - } - } - seen := map[string]bool{} - for _, r := range m.Resources { - if fmt.Sprint(r["type"]) != "container" { - continue - } - vols, _ := r["volumes"].([]any) - for _, v := range vols { - s, _ := v.(string) - mount := volumeMount.FindStringSubmatch(s) - if mount == nil || volumeReadOnly(mount[3]) { - continue - } - src := strings.TrimRight(mount[1], "/") - if ref := dirPlain.FindStringSubmatch(src); ref != nil { - seen[ref[1]] = true - } else if id, ok := absolute[src]; ok { - seen[id] = true - } - } - } - out := make([]string, 0, len(seen)) - for id := range seen { - out = append(out, id) - } - sort.Strings(out) - return out -} - -// volumeMount is a container's volume, `:[:]`, its source possibly a -// `${dir:}` — whose own colon is not the separator. -var volumeMount = regexp.MustCompile(`^(\$\{(?:dir|access):[a-z0-9][a-z0-9-]*\}[^:]*|[^:]+):([^:]+)(?::(.*))?$`) - -// volumeReadOnly is whether a volume's options mount it read-only. -func volumeReadOnly(options string) bool { - for _, o := range strings.Split(options, ",") { - if strings.TrimSpace(o) == "ro" { - return true - } - } - return false -} - -// dirPlain is a whole directory, `${dir:}` and nothing after it. -var dirPlain = regexp.MustCompile(`^\$\{dir:([a-z0-9][a-z0-9-]*)\}$`) - -// coversDirectory is whether a data item names the directory, a path within it, or a directory it -// is placed beneath. -func coversDirectory(m Manifest, dir string) bool { - parents := map[string]string{} - for _, r := range m.Resources { - if fmt.Sprint(r["type"]) != "directory" { - continue - } - if p, ok := r["path"].(string); ok { - if ref := dirRef.FindStringSubmatch(p); ref != nil && strings.HasPrefix(p, "${dir:") { - parents[fmt.Sprint(r["id"])] = ref[1] - } - } - } - for _, it := range m.DataItems() { - ref := dataPathRef.FindStringSubmatch(it.Path) - if ref == nil || ref[1] != "dir" { - continue - } - for d, hops := dir, 0; d != "" && hops < 4; d, hops = parents[d], hops+1 { - if ref[2] == d { - return true - } - } - } - return false -} diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go deleted file mode 100644 index a2d75976..00000000 --- a/internal/catalogue/declaration.go +++ /dev/null @@ -1,2819 +0,0 @@ -package catalogue - -// Turning a resolution into the declaration a node is sent. -// -// Separate from resolving because they answer different questions. Resolving asks *what should -// this machine run*; this asks *what does that look like as resources*, and the second is where -// settings are applied, generators are called, contributions are collected and credentials are -// placed. Both lived in one file until it was doing four jobs at once — which is the shape the -// system this replaces failed in, one import at a time. - -import ( - "encoding/json" - "fmt" - "sort" - "strconv" - "strings" - "time" -) - -// SettingsBy is the layers that apply to each module, keyed by module name. -type SettingsBy map[string][]Layer - -// Generator works out a module's resources for one node, where they cannot be written in advance. -type Generator interface { - // Resources for this node. Absent means the node is not part of whatever this generates, - // which is an ordinary answer rather than a failure — a machine assigned the module before it - // has an address on the network is in exactly that state. - Resources(node string) ([]map[string]any, bool, error) -} - -// OpensPorts is a generator that also says what its resources accept connections on. -// -// **Separate from Generator, because most generators have nothing to say here** and requiring an -// empty method of each would be a cost paid everywhere for one caller. -// -// It exists because a static field cannot express this. A hub accepts connections from every node -// at other sites; a machine that is not a hub dials out and needs nothing open — and they are the -// same module. `listens` in a manifest is one answer for every machine that runs it, so the -// machine that most needs filtering, the one facing the public internet, was the one whose rule -// set would have closed its own overlay. -type OpensPorts interface { - // Listens is what this node accepts on because of what was computed for it. Nothing is the - // ordinary answer: most machines running a computed module open no port at all. - Listens(node string) ([]Listening, error) -} - -// Grant is one consumer's credential, on the machine that must create it. -type Grant struct { - // Provision is what was required. - Provision string - // Consumer is the node that will use it. - Consumer string - // From is the module on that machine which asked. - // - // **Part of who this credential is for, not a label** (novox/hq 04-ISSUES/022). Together with - // Consumer it names one consumer; the node alone does not, because a machine routinely runs - // several modules wanting the same thing. It is also what the provider names the role or the - // bucket or the client after, so withdrawing one consumer does not take another's away. - From string - // Values are what that module contributed — the name it wants, and anything else the - // provision's own vocabulary defines. - Values map[string]any - // Local is the name the credential goes by inside the consumer where it keeps several for one - // provision (ADR 0094); empty for the ordinary one. The provider sees it as a holder of its own. - Local string - // Slug is the consumer module's identity slug, if it declared one — carried on the grant so the - // provider side derives the same login the consumer does, even across nodes where the consumer's - // manifest is not in view (novox/hq ADR 0049). Empty means "use the module name". - Slug string - // At is where the consuming machine is on the private network, empty if it is not on one. - // - // Passed in with the grant because it is a fact about another machine, and resolution answers - // questions about one. A provider that must reach back to its consumer — a reverse proxy is - // the whole reason this exists — otherwise has to know how the mesh names machines. - At string - // Sealed is the credential, closed to the providing node. - Sealed string -} - -// Rendering is everything needed to turn a resolution into the declaration a node is sent. -type Rendering struct { - // Certificate is what the mesh issued for this machine's internal name, and the mesh's own - // certificate. Both public — the key they belong to never left the machine. - Certificate string - Authority string - - // Needed is each module's own secrets, sealed to this node, keyed by module and then by the - // name the module gave it. - Needed map[string]map[string]string - // Foreseen is each own secret not made yet that the next send WILL make, keyed like Needed: a - // composition asked ahead of the send (the self-check's D1) composes it with ForeseenSealed in - // its place and lists it in Composed.Foreseen, rather than failing for a value only a send - // makes. Nil on every composition that is sent, and on every other question, which then refuse - // a secret not made with a *NotMadeError. - Foreseen map[string]map[string]bool - // VaultSealed is, for a module on the core's list (CoreCredentialModules), each own secret as the vault sealed it - // to this machine (the mesh's sealed box, as the vault answered it), keyed like Needed (novox/hq ADR 0349 §16, - // issue 755). Given only for a machine whose engine opens the box (VaultBoxContract): its file then carries the - // box and never the controller's own sealed value. Nil everywhere else, and the composition is as it was. - VaultSealed map[string]map[string]json.RawMessage - // NotPlaced is each core module left out of this machine's declaration because the vault gave nothing for its - // credential and no box was kept, with why (issue 755): the machine keeps what it holds of it, and a module not - // yet there is not placed. Nil everywhere else. - NotPlaced map[string]string - - // Mesh is every node's address on the private network, which is what a rule saying "from the - // mesh" resolves to. Passed in for the same reason grants are: who else is on the network is - // a fact about the mesh, and resolution answers questions about one machine. - Mesh []string - // Suffix is what a machine's internal name ends in, as the control plane composed Names — - // `internal` unless the operator chose another — so a fact writing those names does not - // compose it a second time. - Suffix string - - // BusUsers is the mesh's composed user list, for the module holding `mesh-broker`. Empty on - // every other node, and on this one until the controller has composed it. - // - // **Only the users, never the server's own settings**: those are the module's, in its image and - // its mounts (Manifest.BusUsers). - BusUsers string - // BusMembership is this machine's membership for the bus the mesh is moving to, sealed to it - // (design 28, task 5.2). Empty for a machine not being moved. Written as a file the host reads - // after the declaration has applied, so the bus it names is standing before the machine leaves - // the one it is on. - BusMembership string - - // MeshRange is the private network's CIDR (the range node addresses are allocated from), for a - // module that must name the whole mesh rather than one machine — an intrusion filter that must - // never ban a tunnel peer, say. A per-mesh value the module cannot know, so it is carried here - // and offered as ${machine:mesh-range}, the same way one machine's address is. - MeshRange string - - // Accounts is each machine's operator account, by the same internal name Names uses (novox/hq - // to-be 29). What an ssh Host block's `User` line is composed from; empty for a machine no - // operator account is known on. - Accounts map[string]string - - // Kept is every operator-sealed secret in the mesh, for a module that `keeps` them. Nil when - // nothing on this node keeps them, or the mesh has no operator key. - Kept *KeptExport - - // OutwardLinks is the links this machine reported as facing outside it, which the filter is - // written around (novox/hq ADR 0140). Empty means the machine has not said, and the mesh - // composes no filter for it rather than writing a rule around a link with no name. - OutwardLinks []string - - // LocalNetworks is this machine's local networks (novox/hq ADR 0340): the networks it reported on a - // link facing outside it that lie wholly in a private range (JudgeLocalNetworks), which a port - // reached from `local-network` is admitted from beside the private network. Empty when it reported - // none, and the rule set then says so rather than widening the port or dropping it. - LocalNetworks []string - - // TunnelInterface is the interface the mesh's private network runs on, named here rather than - // imported because the overlay package rests on this one. Traffic arriving on it is the mesh's, - // not this machine's own guest, so the filter admits it only by a rule. - TunnelInterface string - - // Foundation is the ports the mesh itself needs reachable on every machine, which no module - // declares because the foundation is not a module (novox/hq 04-ISSUES/051 and 052). The broker - // is the one that matters: a machine dials it to enrol, and a firewall derived only from - // modules closes it. - Foundation []int - - // Names is every machine's internal name and its address, for containers to be given. - // - // **A container does not inherit the machine's names**, so every internal name the mesh wrote - // is invisible to what the machine runs. Given here rather than looked up on the machine, - // because which machines exist is a fact about the mesh. - Names map[string]string - - // Machines is only the machines, by the same internal name — the subset of Names that is a - // node of this mesh rather than a name it was told to serve. Both matter and they are not the - // same set: a container's hosts wants every name, so a routed name resolves to the proxy that - // serves it, while a resolver told the mesh's suffix is authoritative for it answers from what - // it is given and forwards nothing — so a routed name written there is a name nobody asks for, - // standing beside the machines and looking as real as they do. - Machines map[string]string - - // Zones is every zone a module in the mesh answers itself, where it is answered (novox/hq ADR - // 0199): the mesh's resolver forwards each one there. - Zones []ZoneAt - - // Holders is, for each replicated mesh seat, every machine holding it, by internal name and - // private address (novox/hq ADR 0223) — the same shape as Machines. What a machine's resolver - // file lists: every holder of the mesh's resolver, this machine first if it is one. - Holders map[string]map[string]string - - Settings SettingsBy - Generators map[string]Generator - // Grants are the credentials this node must create, for the provisions it offers. Passed in - // rather than resolved, because who consumes a node is a fact about the rest of the mesh and - // resolution answers questions about one machine. - Grants []Grant - // Withheld is every consumer left out of Grants because its identity overflows the provision's - // bound (novox/hq ADR 0225). Composed into nothing; carried so the machine's declaration can say - // whom it does not serve, and why, beside what it does. - Withheld []Overflow - // Unbound is every consumer whose pair credential from this node is on record and whose own - // resolution binds it elsewhere (novox/hq issue 274): never granted, carried to be said. - Unbound []Unbound - - // Ports is where this machine puts what each module needs reachable, by module and by the - // port the software itself uses (novox/hq ADR 0038). - // - // **The one place the number now lives.** A module used to write it three times — for the rule - // set, for what a consumer is told, and for what the runtime publishes — and nothing checked - // that the three agreed. They are all derived from this. - Ports map[string]map[int]int - - // Adopted says the node is adopted (novox/hq ADR 0100): the firewall found on it stays in - // force, so no module that loads a filter is declared there, and what the mesh needs - // reachable is declared as openings, with its own ports guarded by a table that only refuses. - Adopted bool - - // Given is the machine ports this node was given for its modules' ports, by module and by the - // port the software uses (novox/hq ADR 0100) — the foundation's ports, as genesis chose them. - // They win over anything the mesh would assign and over a manifest's own long-form mapping. - Given map[string]map[int]int - - // Taken is the modules taken on this adopted node (novox/hq ADR 0100). The guard is derived - // from these only (ADR 0103): a port of a module assigned but not taken may still be the - // predecessor's. - Taken map[string]bool - - // Seats is where this machine put each mesh-scoped seat's holder, by seat and by the port the - // holder's software uses (novox/hq 04-ISSUES/102) — read from the node's settings and - // assignments for whichever module claims the seat, whether or not it is in this node's set. - // What ${seat:…} answers with; see seat_into.go for why the answer may be absent. - Seats map[string]map[int]int - - // ArtifactStore is the mesh's artifact store as this network reaches it (host:port) — the - // node holding it and the port that node put it on — or empty when the mesh has none on its - // network yet. Composed into every image and archive the mesh built, at this moment and never - // stored (novox/hq 04-ISSUES/102). - ArtifactStore string - - // SeatReach is where this machine reaches the holder of each mesh-scoped seat it may be asked - // about (host:port), by seat: what ${seat::reach} answers with (novox/hq ADR 0222). Absent - // when no holder is reachable yet; see seat_into.go for which seats are answered. - SeatReach map[string]string - - // Built is every `/` the mesh has built. What tells a reference recorded - // with an address — before references were kept without one — from an image a module runs - // straight from a public registry. - Built map[string]bool - - // DataRoot is where this node keeps the directories the mesh places for its modules - // (novox/hq ADR 0112, to-be 27): a directory resource that states no path resolves to - // //, and ${dir:} names that place from the module's own files, - // mounts and environment. A node setting fixed at installation; empty means the default, - // /var/lib — see dir_into.go. - DataRoot string - - // ReadsHealth says this machine's node-engine reads a resource's `health` (novox/hq ADR 0240 Phase B: - // its statement's contract is link.ReadinessContract or later). An older engine parses strictly and - // refuses a field it does not know, whole — so to it the field is not sent, and what it runs is - // judged by liveness alone. - ReadsHealth bool - - // JudgesRoot says this machine's node-engine judges a user's declared `root` (novox/hq ADR 0266: its - // statement's contract is link.RootContract or later). To an older, strict engine the field is not - // sent, and the account it names is not judged — which the self-check says, as not judged. - JudgesRoot bool - - // ReadsBusy says this machine's node-engine reads a container's `busy` (novox/hq ADR 0335: its statement's - // contract is link.BusyContract or later). To an older, strict engine the field is not sent, and the - // container is replaced at once, as before. - ReadsBusy bool - - // Hands says this machine's node-engine writes a filled file on the provisioner client's word alone (its - // statement's contract is link.HandedContract or later, design 57 P7), and Provisioner is the account the - // provisioner's client runs as here, empty where none runs (novox/hq ADR 0349 §4). With both, a moved module's - // templated files are handed to the client (handed.go); without either, it composes as before. - Hands bool - Provisioner string -} - -// RootField is a user resource's field saying the account must never become root without a person -// (novox/hq ADR 0266). -const RootField = "root" - -// rootInto composes a user's `root` for the node-engine: taken away when it asserts nothing (empty — a -// machine where agents run as the operator) or when the engine is older than the field and parses -// strictly; kept as "never" otherwise. -func rootInto(resource map[string]any, with Rendering) { - if resource["type"] != "user" { - return - } - value, has := resource[RootField] - if !has { - return - } - if s, _ := value.(string); s == "" || !with.JudgesRoot { - delete(resource, RootField) - } -} - -// machinePort is where a module's port lives on this machine, or the port itself when the mesh has -// not been asked. Unassigned is not an error here: a module with no `listens` never needed one, -// and a caller composing a declaration without a store still gets something coherent. -func (r Rendering) machinePort(module string, wanted int) int { - if at, given := r.Given[module][wanted]; given { - return at - } - if at, known := r.Ports[module][wanted]; known { - return at - } - return wanted -} - -// Declaration is everything the resolved modules put on the node, with settings applied. -// -// Resource identities are prefixed with the module they came from. Two modules may reasonably -// both call something "config", and without this the second would silently replace the first — -// the node applying one of them and reporting success. -func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) { - composed, err := r.Compose(with) - if err != nil { - return nil, err - } - return composed.Resources, nil -} - -// Composed is a declaration's resources and which module each came from. -// -// Owner is kept beside the resources because a resource id cannot be split back into its module: -// a module's name may itself contain a dot. What the mesh adds of its own — an opening, the guard — -// has no owner. -// -// Received is what each module on the machine is given for each requirement it receives — the same -// contributions its received file is written from, kept beside it so the mesh can also issue them -// on the bus in the module's membership (novox/hq ADR 0167). By module, then requirement. -type Composed struct { - Resources []map[string]any - Owner map[string]string - Received map[string]map[string][]Contribution - // LeftOut is every module of this machine's set that was left out of its declaration, and - // why (novox/hq ADR 0163, rule 6): a setting stored for it that its definition can no longer - // compose. Its held things are kept and its containers untouched — the machine is told so — - // and it is told everything else. - LeftOut map[string]string - // Foreseen is every own secret composed with ForeseenSealed in its place (Rendering.Foreseen), - // as `module/name`, sorted: what the next send will make. Never set on a declaration that is - // sent — a placeholder in a sealed file is a credential the process cannot read. - Foreseen []string - // Unplaced is every contribution the holder's template could not render into something, each - // naming its module and why (novox/hq ADR 0255): that piece is left out and the machine is told - // everything else. Push and plan say it; the merge gate refuses a change that adds one. - Unplaced []string - // HandedOnly is each file left out because only the provisioner's client fills it and the module is not handed on - // this machine (a provider's consumers file, ADR 0373), said once in the plan. Not an unplaced contribution, which - // the merge gate refuses: a module not handed composes as it did. - HandedOnly []string - // Filled is the path of every file handed to the provisioner, in the order composed: the declaration's - // filled files (novox/hq ADR 0349 §4, §5). Plans is each handed module's plan on this machine, by module. - // Both empty where nothing is handed, and the declaration is then what it was. - Filled []string - Plans map[string]Plan -} - -// ForeseenSealed is what stands for an own secret a send will make, in a composition asked ahead of -// the send. Not a sealed value: nothing composed with it may be sent. -const ForeseenSealed = "foreseen: made by the next send" - -// NotMadeError is a module's own secret the composition was given no value for (novox/hq issue 275): -// typed, so that a caller can tell "a send would make this, and none has yet" from a composition that -// fails for any other reason without reading the words. -type NotMadeError struct { - Module, Name string -} - -func (e *NotMadeError) Error() string { - return fmt.Sprintf("%s needs a secret called %q and none was made for it", e.Module, e.Name) -} - -// LeftOut is which of this machine's modules a declaration composed with these settings leaves -// out, and why (novox/hq ADR 0163, rule 6): each whose stored settings its definition can no longer -// compose. Empty when every module composes. The same judgement SetSettings makes before storing. -func (r Resolution) LeftOut(settings SettingsBy, adopted bool) map[string]string { - out := map[string]string{} - for module, why := range r.LeftOutBecause(settings, adopted) { - out[module] = why.Error() - } - return out -} - -// LeftOutBecause is LeftOut with each reason as the error it was, so a reader can tell a setting nobody -// gave (an *UnsetSettingError) from any other cause and say it as the operator's to give (novox/hq issue -// 380). An UnreadManifestError for a stored manifest this controller cannot read whole. -func (r Resolution) LeftOutBecause(settings SettingsBy, adopted bool) map[string]error { - out := map[string]error{} - for _, m := range r.Modules { - if why := UnknownFieldReason(m); why != "" { - out[m.Module] = &UnreadManifestError{Module: m.Module, said: why} - continue - } - if err := JudgeSettings(m, settings[m.Module], adopted); err != nil { - out[m.Module] = err - } - } - return out -} - -// UnreadManifestError is a module left out because its stored manifest has a key this controller does not know -// (novox/hq ADR 0262): said once for the whole mesh, by the catalogue's own condition, not per machine. -type UnreadManifestError struct { - Module string - said string -} - -func (e *UnreadManifestError) Error() string { return e.said } - -// Compose is Declaration with the owner of every resource said. -func (r Resolution) Compose(with Rendering) (Composed, error) { - owner := map[string]string{} - received := map[string]map[string][]Contribution{} - leftOut := map[string]string{} - var foreseen, unplaced, handedOnly []string - handed := &handedFiles{plans: map[string]Plan{}} - resources, err := r.compose(with, owner, received, leftOut, &foreseen, &unplaced, &handedOnly, handed) - if err != nil { - return Composed{}, err - } - sort.Strings(foreseen) - if with.BusMembership != "" { - // The machine's own, not any module's: how it reaches the mesh from now on. Sealed like a - // secret and placed where the host looks for exactly this (design 28, task 5.2). - resources = append(resources, map[string]any{ - "id": BusMembershipID(), "type": "file", "path": BusMembershipPath, - "sealed": with.BusMembership, "mode": "0600", - }) - } - sort.Strings(unplaced) - sort.Strings(handedOnly) - out := Composed{Resources: resources, Owner: owner, Received: received, LeftOut: leftOut, - Foreseen: foreseen, Unplaced: unplaced, HandedOnly: handedOnly, Filled: handed.paths} - if plans := handed.finished(r.Needs); len(plans) > 0 { - out.Plans = plans - } - return out, nil -} - -// BusMembershipID names the resource carrying a machine's membership for the new bus, and -// BusMembershipPath is where the host reads it — the same constant on both sides. -func BusMembershipID() string { return "bus-membership" } - -const BusMembershipPath = "/var/lib/mesh/membership-next.json" - -func (r Resolution) compose(with Rendering, owner map[string]string, - received map[string]map[string][]Contribution, leftOut map[string]string, - foreseen, unplaced, handedOnly *[]string, handed *handedFiles) ([]map[string]any, error) { - // **A setting is judged where it is stored, and an impossible one costs a module, not a - // machine** (novox/hq ADR 0163, rule 6). A definition that moved under a stored setting makes - // this module uncomposable; it is left out of the declaration — its held things kept, its - // containers untouched, the machine told so by name — and the machine is told everything else. - // Before placing, because a placement is a setting too. - left := r.LeftOut(with.Settings, with.Adopted) - kept := make([]Manifest, 0, len(r.Modules)) - var stillBackedUp []Manifest - for module, why := range with.NotPlaced { - if _, already := left[module]; !already { - left[module] = why - } - } - for _, m := range r.Modules { - if why, isLeft := left[m.Module]; isLeft { - if leftOut != nil { - leftOut[m.Module] = why - } - // **Its data is still copied** (novox/hq ADR 0262): a module left out runs nothing new, and - // the data it already holds on the machine is the reason to keep copying it. Only its data, - // as the backup holder's lines are derived from it, and the directories they name. - stillBackedUp = append(stillBackedUp, backupView(m)) - continue - } - kept = append(kept, m) - } - r.Modules = kept - - // Every manifest is placed first (novox/hq ADR 0112): the maps naming where its bindings, - // credentials and contributions land are resolved against this node's directories, so every - // reader below — the binding files, the sealed secrets, the grant paths a contribution - // names — sees a concrete place and none learns the vocabulary. - // Into a fresh slice, never the caller's: one resolution may compose for many nodes, and a - // slice element written in place would carry the first node's places into the second's. - placed := make([]Manifest, len(r.Modules)) - for i, m := range r.Modules { - var err error - if placed[i], err = placedManifest(m, with); err != nil { - return nil, err - } - } - r.Modules = placed - - // Where each provision's credentials land, so a contribution can name the file rather than - // carry a value the mesh does not have. - directories := map[string]string{} - for _, m := range r.Modules { - for provision, where := range m.Grants { - directories[provision] = where - } - } - given, err := r.contributions(with.Settings, with.Grants, directories) - if err != nil { - return nil, err - } - - // A workload on THIS machine contributes the port it declared, and the machine may have - // published it somewhere else (novox/hq ADR 0066). The mirror of the redirect below: 038 fixed - // what a consumer is TOLD about a provider, and this is what a workload TELLS the provider about - // itself — gitea declaring 3000, published as 20000:3000, and the co-located proxy dialling 3000, - // where nothing listens, for every request. - // - // **The contributing module's assignment, never the provider's.** The port belongs to the - // workload; using the map of whatever answers the requirement would move a route to wherever the - // proxy happens to be published. A contribution carried here from another machine is left exactly - // as it is — its port is that machine's to assign, and this map knows nothing about it. - for provision := range given { - for i := range given[provision] { - said := &given[provision][i] - if said.Node != "" && said.Node != r.Node { - continue - } - said.Values = atMachinePort(said.Values, said.From, with.Ports) - } - } - - // A provider answered on this same machine never passed through the walk that works out what a - // provider on ANOTHER machine serves (novox/hq 04-ISSUES/038, ADR 0066). That walk does two - // things — it derives the served facts with the provider node's port assignments, and it settles - // them with that node's settings layers — and the same-node paths (resolve.go's servedHere, and - // here() below) did neither, because both run while resolving, before either is known. - // - // The port was the first half to be noticed (038). The second half is worse and quieter: a served - // value the OPERATOR supplied — a certificate authority's root, which is the one kind of value a - // manifest cannot carry, because it differs on every mesh — arrived as the manifest's empty - // default. A proxy then wrote an empty CA bundle, fell back to the system trust store, could not - // verify the internal authority, and stopped issuing with nothing saying why. - // - // So it is re-derived here, in full, exactly as the cross-node path derives it. **After the - // closure is known**, which also removes an order dependence: the resolver built these needs - // mid-walk from whichever modules had been chosen by then, so what a co-located binding carried - // depended on the order somebody happened to assign things in. - for i := range r.Needs { - if r.Needs[i].ByRecord || r.Needs[i].From != r.Node { - continue - } - serves, answered, err := r.servedOnThisMachine(r.Needs[i].Name, with) - if err != nil { - return nil, err - } - if answered { - r.Needs[i].Serves = serves - } - } - - // Once, from every module's listens -- not per module. A module receiving only its own ports - // would write a rule set that closed every other module on the machine. Each module's per-node - // exposure settings override its listens' source first (novox/hq ADR 0046). - rules, err := r.Rules(with) - if err != nil { - return nil, err - } - // **A machine that has not said which links face outside is sent no filter** (novox/hq ADR - // 0140). The whole chain is written around those links: with none, the rule that lets this - // machine's own guests keep working would name an empty set, which nftables refuses, and a rule - // set that does not load is a machine filtering nothing while its unit reports success. Refused - // here, where a person reads it, rather than on the machine — and the machine keeps the filter - // it already has. - if filters := r.filtersHere(); filters != "" && len(with.OutwardLinks) == 0 { - return nil, fmt.Errorf( - "%s cannot be sent a filter: it has not reported which of its links face outside, and "+ - "every rule in the chain is written around them. It reports that on each apply; "+ - "`node show %s` says whether it has. Until then %s is not sent, and the machine "+ - "keeps the filter it has", r.Node, r.Node, filters) - } - filtering := AsNftablesWithLocal(rules, with.Mesh, r.PublicDomain != "", with.Foundation, - with.OutwardLinks, with.TunnelInterface, with.LocalNetworks) - - // **A variable two modules set is refused whether or not anything places it** (novox/hq ADR - // 0203 §5): the account has one environment, and a machine whose holder arrives later should not - // be the moment two modules are found to disagree about it. - if err := variablesSetOnce(r.Modules); err != nil { - return nil, err - } - - generated, err := r.generatedHere(with) - if err != nil { - return nil, err - } - - var out []map[string]any - for _, m := range r.Modules { - if with.Adopted && m.Filtering != nil { - // Nothing of a module that loads a filter, on an adopted node: its table would drop - // by default and hold accepts, and the found firewall stays in force. Every resource, - // not only the rule set — its service must not run, and a node returned to adopted - // stops it by the ordinary removal of what is no longer declared. - continue - } - resources := m.Resources - - // What the mesh computes for this module goes FIRST, before the module's own resources. - // - // **Order is stated, not derived — the host does not sort** (novox/hq ADR 0005), so - // whatever the mesh writes down is the order a machine applies. A module's service or - // container routinely depends on one of these files; nothing here ever depends on a - // module's resources, because none of it is computed from them. - // - // Appended, this was wrong in a way that only showed on the first apply and then healed: - // the service started before its certificate or its rule set existed, failed, and the next - // reconcile fixed it. A fault that repairs itself on the second attempt is worse than one - // that does not, because what gets remembered is that it works. - var first []map[string]any - if f := m.Filtering; f != nil { - first = append(first, map[string]any{ - "id": FilteringID(), "type": "file", "path": f.Into, - "content": filtering, "mode": "0600", - }) - } - // The node's fail2ban jails, composed from every module it runs (novox/hq to-be 31), written - // where the intrusion-prevention holder owns them. Like the rule set above: gathered from all - // modules, written by the one that holds the role. - if j := m.Jailing; j != nil { - first = append(first, jailsInto(r.Modules, j)...) - } - if c := m.Certificate; c != nil { - if with.Certificate == "" { - // Asked for and not issued. Refused rather than skipped: a module that serves TLS - // with no certificate does not start, and the reason is somewhere else entirely. - return nil, fmt.Errorf( - "%s wants a certificate for this machine and none was issued", m.Module) - } - first = append(first, map[string]any{ - "id": CertificateID(), "type": "file", "path": c.Into, - // Public. It travels in the open like any other file, because it is a statement - // about a key rather than the key. - "content": with.Certificate, "mode": "0644", - }) - if c.Authority != "" { - first = append(first, map[string]any{ - "id": AuthorityID(), "type": "file", "path": c.Authority, - "content": with.Authority, "mode": "0644", - }) - } - } - if m.BusUsers != "" { - // **The claim authorises it, not the field.** This file holds every user's password - // hash, so a module that could ask for it could read every credential on the bus. - // Checked from this manifest alone, which is the cheapest check there is: whether some - // other module also claims the seat is resolution's business elsewhere, and one holder - // mesh-wide is already guaranteed. - if !m.ClaimsSeat("mesh-broker") { - return nil, fmt.Errorf( - "%s asks for the mesh's user list and does not claim mesh-broker. That file "+ - "holds every user's password hash, so the seat is what authorises it", - m.Module) - } - if with.BusUsers == "" { - // Asked for and not composed. Refused rather than skipped, for the reason a - // certificate is: a bus with no user list refuses every connection in the mesh, and - // an empty file would look like a configuration problem on the machine. - return nil, fmt.Errorf( - "%s holds mesh-broker and the mesh composed no user list, so the bus would "+ - "refuse every connection", m.Module) - } - first = append(first, map[string]any{ - "id": BusUsersID(), "type": "file", "path": m.BusUsers, - "content": with.BusUsers, - // Readable by the server and nothing else. Hashes rather than passwords, so this is - // not a set of working credentials — but a list of every user in the mesh is worth - // keeping to the one process that needs it. - "mode": "0600", - }) - } - // **A secret family's members, as given** (novox/hq ADR 0283): one file each at the family's path, and - // nothing for a member not given — the mesh never makes one, and the module says it waits for it. - for _, name := range sortedKeys(with.Needed[m.Module]) { - own, family, ok := m.OwnSecrets.Lookup(name) - if !ok || family == "" || with.Needed[m.Module][name] == "" { - continue - } - // A per-secret owner (novox/hq ADR 0436 §1) stands in for the module's secrets-owner, when - // the manifest named one for this secret (or the family it belongs to). - owner := m.SecretsOwner - if own.Owner != "" { - owner = own.Owner - } - first = append(first, ownedBy(owner, map[string]any{ - "id": NeedID(name), "type": "file", "path": own.Path, "sealed": with.Needed[m.Module][name], - })) - } - // **A handed module's own secrets are the vault's** (design 57 V4, issue 647): each file holding one is - // declared placed and without a value, filled by the machine's provisioner client from the vault; the - // controller makes and seals none of them here. - handsSecrets := with.Handing(m) - for _, name := range sortedKeys(m.OwnSecrets.Plain()) { - if handsSecrets { - owner := m.SecretsOwner - if m.Module == RuntimeModule && r.Account != "" { - owner = r.Account - } - if o := m.OwnSecrets[name].Owner; o != "" { - owner = o - } - first = append(first, ownedBy(owner, map[string]any{ - "id": NeedID(name), "type": "file", "path": m.OwnSecrets[name].Path, HandedSecretField: name, - })) - continue - } - // **A core credential is the vault's box** (ADR 0349 §16, issue 755): carried as the vault sealed it to this - // machine, which the controller cannot open; the controller's own sealed value is not sent with it. - if box, ok := with.vaultSealedOf(m.Module, name); ok { - if string(box) == string(ForeseenVaultBox) { - *foreseen = append(*foreseen, m.Module+"/"+name) - } - owner := m.SecretsOwner - if m.Module == RuntimeModule && r.Account != "" { - owner = r.Account - } - if o := m.OwnSecrets[name].Owner; o != "" { - owner = o - } - first = append(first, ownedBy(owner, map[string]any{ - "id": NeedID(name), "type": "file", "path": m.OwnSecrets[name].Path, VaultSealedField: box, - })) - continue - } - sealed := with.Needed[m.Module][name] - if sealed == "" && with.Foreseen[m.Module][name] { - // Not made, and the next send makes it: composed with a stand-in so that whatever - // else this composition would refuse is still found (novox/hq issue 275). - sealed = ForeseenSealed - *foreseen = append(*foreseen, m.Module+"/"+name) - } - if sealed == "" && m.OwnSecrets[name].IssuedBy == IssuedOutside { - // **Not given, and the mesh never makes it** (novox/hq issue 401, ADR 0228): no file, as for a - // family's member (ADR 0283), so the module reads its absence as not given. - continue - } - if sealed == "" { - // Declared and not made. Refused rather than skipped: a module whose own - // credential is silently absent starts, fails to authenticate, and the reason is - // three layers away from the machine reporting it. - return nil, &NotMadeError{Module: m.Module, Name: name} - } - // The runtime's credential belongs to the account the runtime runs as (novox/hq ADR 0175, - // to-be 38 WP3): its process is composed `user: ` where the node has one, and a - // root-owned 0600 file is one that process cannot read. Composed here rather than said in - // the manifest, because a manifest cannot say ${machine:account} safely — a node with no - // account has nothing to resolve it to, and then the runtime runs as root and the file - // stays root's. - owner := m.SecretsOwner - if m.Module == RuntimeModule && r.Account != "" { - owner = r.Account - } - if o := m.OwnSecrets[name].Owner; o != "" { - owner = o - } - first = append(first, ownedBy(owner, map[string]any{ - "id": NeedID(name), "type": "file", "path": m.OwnSecrets[name].Path, "sealed": sealed, - })) - } - // This module's tools bundles, where the machine runs the node's tool runtime (novox/hq - // ADR 0175, to-be 38 WP2). Mesh-computed like everything above it, and before the module's - // own resources for the same reason: the runtime's process names the files inside these - // and is restarted when one changes, so they are on the machine before it is. - if r.runtimeHere() { - first = append(first, bundleArchives(m)...) - } - // Operator-owned paths this module is granted use of (novox/hq ADR 0051). Written before - // the module's own resources, and so before the container that mounts them: the host must - // find each present — refusing clearly if the operator has not provided it — before it - // starts anything that depends on it. The mesh creates, chowns and reconciles none of it; - // an `access` resource says only *this path must exist, and this module reaches it*. - // Where each is on THIS machine is the assignment's (novox/hq issue 153): placed by id - // where the operator said, the definition's default otherwise, refused where neither. - accesses, accessPaths, err := accessesFor(m, with.Settings[m.Module]) - if err != nil { - return nil, err - } - for _, a := range accesses { - first = append(first, map[string]any{ - "id": AccessID(a.Path), "type": "access", "path": a.Path, "mode": a.Mode, - }) - } - for _, to := range m.SecretRequirements() { - for _, file := range m.SecretFiles(to) { - var found *Needed - for i, n := range r.Needs { - // **This module's need, not the provision's** (novox/hq 04-ISSUES/022). Matching - // on the name alone, every consumer of a provision took whichever credential - // happened to be last in the list — so on a node with two of them, one module - // would be given the other's password and fail to authenticate with a valid - // credential belonging to somebody else. And this file's local name, where the - // module keeps several (ADR 0094). - if n.Name == to && n.For == m.Module && n.Local == file.Local { - found = &r.Needs[i] - } - } - // **A credential the vault keeps for this holder alone** (issue 757): where the module is handed, its - // file is declared placed and without a value, and the machine's client fills it from the vault. - if found != nil && found.IssuedToHolder && with.Handing(m) { - first = append(first, ownedBy(m.SecretsOwner, map[string]any{ - "id": SecretID(SecretLocal(to, file.Local)), "type": "file", "path": file.Path, - HandedProvisionField: PlanProvision{Name: SecretLocal(to, file.Local), Provision: to, - Local: file.Local, Provider: found.From, ProviderModule: found.Module}, - })) - continue - } - if found != nil && found.ByRecord && found.Sealed == "" && !found.Manager { - // Answered by a record whose key has not been supplied since this consumer was - // put on it. **Refused, not skipped.** The mesh discarded the plaintext when the - // key was accepted and cannot seal another, so a machine that resolved cleanly - // would receive no file at all and fail at whatever tried to read it — which is - // the outcome ADR 0024 exists to avoid, arrived at politely. - // - // The manager holder is the one exception (novox/hq ADR 0050): an empty refresh token - // is a licence whose manager has not adopted one yet, a real waiting state rather than - // a lost key. It falls through to the skip below — its bound facts (carrying the - // manager's public key) are still delivered, which is what adoption needs to seal the - // first refresh token. - return nil, fmt.Errorf( - "%s on this machine uses the licence %q and no key has been sealed to it. "+ - "The mesh cannot make one; supply it again with `licence key %s`", - m.Module, found.From, found.From) - } - if found == nil || found.Sealed == "" { - // Answered on this machine, or answered by a node the mesh could not seal to. - // Nothing to write either way, and writing an empty credential file would be - // worse than none: something would read it and fail authenticating. - continue - } - first = append(first, ownedBy(m.SecretsOwner, map[string]any{ - "id": SecretID(SecretLocal(to, file.Local)), "type": "file", "path": file.Path, - "sealed": found.Sealed, - })) - } - } - for _, to := range sortedKeys(m.Grants) { - for _, g := range with.Grants { - if g.Provision != to { - continue - } - if g.From == "" { - // Nothing on that machine asks for this any more. Skipped here rather than - // where grants are gathered, so the rule holds whoever gathers them. - // - // **This is how a credential is withdrawn.** The provisioner removes what - // nobody asks for, and it can only do that if the mesh stops asking — a - // consumer that was unassigned would otherwise keep a working login for ever, - // and nothing would say so. - continue - } - first = append(first, ownedBy(r.provisionsAs(m), map[string]any{ - // One file per holder — the consumer's module with its local name after it - // where it keeps several (ADR 0094); the lab found two files with one id. - "id": GrantID(to, g.Consumer+"."+holderAs(g.From, g.Local)), - "type": "file", - "path": grantPath(m.Grants[to], g.Consumer, holderAs(g.From, g.Local)), - "sealed": g.Sealed, - })) - } - } - for _, to := range sortedKeys(m.Binds) { - var found *Needed - for i, n := range r.Needs { - if n.Name == to && n.For == m.Module { - found = &r.Needs[i] - } - } - if found == nil { - // Answered on this same machine rather than from elsewhere in the mesh. - // - // **Still written.** It used to be skipped, reasoning that "it is on this node" - // is a fact nobody needs — and that is true of the *location* and false of - // everything beside it. A binding also carries what the provider said a consumer - // must know, which is the port; a consumer cannot invent that, and got an absent - // file with no explanation. A build machine sharing a node with the registry it - // pushes to sat in a loop saying it could not read its own binding. - here, err := here(r, to, with) - if err != nil { - return nil, err - } - if here == nil { - // Nothing in this node's set offers it either, so there is genuinely nothing - // to say. Resolution has already refused anything unanswerable, so this is a - // requirement met by the module itself. - continue - } - found = here - } - own, err := r.ownNames(m, to, with.Settings[m.Module]) - if err != nil { - return nil, err - } - as := ConsumerIdentity(r.Node, IdentitySource(m.Slug, m.Module)) - // What the provider derives for THIS consumer, filled here where the consumer is - // known (novox/hq ADR 0202). The same fill knownFor does below, so the binding file - // and the module's `${bound:…}` substitutions cannot say different things. - told := *found - told.Serves, err = ServedTo(told.Serves, as) - if err != nil { - return nil, fmt.Errorf("%s is told about %s: %w", m.Module, to, err) - } - file, err := boundFile(told, m.Binds[to], as, own) - if err != nil { - return nil, err - } - first = append(first, file) - } - for _, to := range sortedKeys(m.Receives) { - file, err := receivedFile(to, m.Receives[to], given[to]) - if err != nil { - return nil, err - } - first = append(first, file) - if received[m.Module] == nil { - received[m.Module] = map[string][]Contribution{} - } - // Empty rather than absent when nobody contributed, for the reason the file is - // written empty: "nothing asked" and "never told" want different responses. - received[m.Module][to] = append([]Contribution{}, given[to]...) - } - if m.Keeps != "" && with.Kept != nil { - file, err := keptFile(m.Keeps, with.Kept) - if err != nil { - return nil, err - } - // Read by the module as its secrets' owner when it names one: the vault, as its own account (issue 723). - first = append(first, ownedBy(m.SecretsOwner, file)) - } - if m.Computed != "" { - mine, part := generated[m.Module] - if !part { - // Assigned, and not yet part of what this generates. Nothing to put on the - // machine, which is different from an error: a node given the network module - // before it has an address is in exactly that state, briefly. - continue - } - resources = mine - } - - // Now, and not before: a module whose resources are computed replaces them wholesale, and - // merging earlier would throw away the files it still needs. - // - // **Except the module's own accounts, which go before even those** (novox/hq issue 213). What - // the mesh computes may belong to one: a module whose code runs as an account it declares has - // its secrets written owned by that account, and a file given to a user the machine does not - // have yet fails — so on the first apply the secrets were refused, the process started without - // them, and the second apply healed it, which is the fault the paragraph above describes. - // An account depends on nothing the mesh computes. - accounts, rest := accountsFirst(resources) - resources = append(append(accounts, first...), rest...) - // **A file delivered only while its optional requirement is bound is left out while it is - // absent** (novox/hq ADR 0330 §5), and the field is dropped from the rest: the node-engine - // parses strictly and knows no such field. - absent := r.AbsentFor(m.Module) - resources = withoutAbsent(resources, absent) - - // No container is given the mesh's names (novox/hq ADR 0148). It used to be: every - // container got the whole roster as `--add-host` entries at creation, and a name that - // moved afterwards was wrong inside it for as long as it ran (issues 109, 135) — and once - // the roster was made part of a container's identity so that could be caught, one name - // moving anywhere replaced every container in the mesh (issue 151). A container resolves a - // mesh name through its machine's resolver at the moment it asks, which the runtime is - // told once per machine, as a file, by the resolver's own module. The names a module - // declares for itself are its own and stay exactly as written: they are part of what the - // module is, and the mesh does not know what they mean. - // What this module may name from inside one of its own files. Gathered once per module - // rather than per file, because it is a fact about the module. - sealed, err := sealedFor(m, r.Needs, with) - if err != nil { - return nil, err - } - // And what its bindings say, for the half of a connection that is not secret. - known, err := knownFor(m, r.Needs, r.Node) - if err != nil { - return nil, err - } - // A requirement answered on this same machine is not in r.Needs — its binding file is - // written from `here` (above) — and so `${bound:…}` could not name it, though the file - // beside it said the same facts. Filled from the same answer, so the two cannot disagree. - for _, want := range m.Wants() { - if _, has := known[want]; has { - continue - } - answered, err := here(r, want, with) - if err != nil { - return nil, err - } - if answered == nil { - continue - } - local := *answered - local.For = m.Module - here, err := knownFor(m, []Needed{local}, r.Node) - if err != nil { - return nil, err - } - for provision, values := range here { - known[provision] = values - } - } - // And the providing machine's private address, beside its name (novox/hq ADR 0194). A name is - // what nearly every consumer wants; the one that cannot use it is a machine's resolver - // configuration, which must reach the resolver before it can resolve anything — the resolver's - // own name included. Absent when the machine has no address yet, so a file naming it is refused - // rather than written with a blank where an address belongs. - for _, values := range known { - if address := with.Machines[values["at"]]; address != "" { - values["address"] = address - } - } - // And what the module is called through each requirement it contributes to (novox/hq - // 04-ISSUES/122) — the same composition its binding file carries. - for provision, values := range known { - own, err := r.ownNames(m, provision, with.Settings[m.Module]) - if err != nil { - return nil, err - } - withOwnNames(values, own) - } - // And where this node places the directories the module declared without a path - // (novox/hq ADR 0112) — resolved once per module, named by ${dir:…} from any resource. - // — and, on an adopted machine, where the assignment says they already are, with the - // owner the data already has (novox/hq issue 153). Malformed placements are refused here. - placed, err := Places(m, with.Settings[m.Module]) - if err != nil { - return nil, err - } - dirs := dirsFor(m, with) - // And the machine underneath, which no binding of its own can tell it. - thisMachine := machineFacts(r, with.Names, with.MeshRange) - - // **A definition that already holds the answer transcribed it** (novox/hq ADR 0202). - // Judged over what the module itself declares, and before anything is substituted: the - // mesh's own generated files — the binding, the contributions — legitimately carry the - // derived value, and after substitution so does every consumer's file, so this is the one - // moment the two can be told apart. - for _, own := range m.Resources { - if err := notTranscribed(own, known, m.Module); err != nil { - return nil, err - } - } - // **And every placeholder no pass fills where it stands is refused** (novox/hq issue 231): - // judged here, over the definition as written and before any pass, by the function the - // catalogue check runs — so a manifest registered by an older binary is judged too, and - // what a setting or a binding later puts into a file is its software's text, never swept. - for _, own := range m.Resources { - if problems := unconsumedPlaceholders(m.Module, own); len(problems) > 0 { - return nil, fmt.Errorf("%s", problems[0]) - } - } - - // Which of this module's files carry a secret, for the rule that a container may not read - // one of them as its environment without saying so (ADR 0086, issue 041). - secretFiles := secretFilesOf(resources) - - // Which of this module's resources its preparation runs before, if it prepares anything. - prepareBefore := preparationTarget(m) - - // Which found networks this machine's setting keeps for each of its containers (novox/hq - // ADR 0163, rule 4); judged above, so an invalid one is not here. - keptNetworks, _ := KeptNetworks(m, with.Settings[m.Module], with.Adopted) - - // **A moved module's templated files are the provisioner's** (novox/hq ADR 0349 §4, §6, design 57 P6): on a - // machine that can take them, each is declared placed and without its text, and no pass below fills it. - handing, err := handed.begin(r.Node, m, dirs, accessPaths, thisMachine, with, r.Capabilities, r.Needs, - backupPlacer{append(append([]Manifest(nil), r.Modules...), stillBackedUp...), with, thisMachine, r.Capabilities, - unplaced}) - if err != nil { - return nil, err - } - // **What the mesh calls the module through each provision it requires** (issue 792): the names its binding - // file carries, composed by the same call, handed to the provider through the plan's entry. - if _, ok := handed.plans[m.Module]; ok && handing != nil { - for _, to := range requirementsContributedTo(m) { - own, err := r.ownNames(m, to, with.Settings[m.Module]) - if err != nil { - return nil, err - } - handed.name(m.Module, to, own) - } - } - // A handed collector is filled from items alone: each old form its collections would have held is said - // (design 57 L1). - if plan, ok := handed.plans[m.Module]; ok && handing != nil && len(plan.Collections) > 0 { - *unplaced = append(*unplaced, oldFormsLeftOut(append(append([]Manifest(nil), r.Modules...), - stillBackedUp...), m, plan.Collections)...) - } - for _, unsettled := range resources { - if pr, ok := unsettled[HandedProvisionField].(PlanProvision); ok { - if handing == nil { - return nil, fmt.Errorf("%s: its credential of %s is handed to the provisioner and the module is "+ - "not: nothing would fill it", m.Module, pr.Provision) - } - marked := map[string]any{} - for k, v := range unsettled { - marked[k] = v - } - delete(marked, HandedProvisionField) - marked[HandedSecretField] = pr.Name - file, sec, err := handedSecretFile(marked, m, dirs, accessPaths, thisMachine, placed) - if err != nil { - return nil, err - } - file["id"] = m.Module + "." + fmt.Sprint(unsettled["id"]) - owner[fmt.Sprint(file["id"])] = m.Module - pr.Path = sec.Path - if err := handed.addProvisionFile(m.Module, pr); err != nil { - return nil, err - } - out = append(out, file) - continue - } - if unsettled[HandedSecretField] != nil { - if handing == nil { - return nil, fmt.Errorf("%s: its own secret %v is handed to the provisioner and the module is "+ - "not: nothing would fill it", m.Module, unsettled[HandedSecretField]) - } - file, sec, err := handedSecretFile(unsettled, m, dirs, accessPaths, thisMachine, placed) - if err != nil { - return nil, err - } - file["id"] = m.Module + "." + fmt.Sprint(unsettled["id"]) - owner[fmt.Sprint(file["id"])] = m.Module - if err := handed.addSecretFile(m.Module, sec); err != nil { - return nil, err - } - out = append(out, file) - continue - } - if handing != nil && fmt.Sprint(unsettled["type"]) == "file" { - if text, _ := unsettled["content"].(string); len(secretsUsed(text)) > 0 && - !handing[fmt.Sprint(unsettled["id"])] { - return nil, fmt.Errorf("%s: its file %v names a secret in its text, and a handed module's "+ - "secrets reach only the files its provisioner client fills: keep it in a template "+ - "(design 57 V4)", m.Module, unsettled["id"]) - } - } - if handing[fmt.Sprint(unsettled["id"])] { - file, line, err := handedFile(unsettled, m, dirs, accessPaths, thisMachine, placed, with) - if err != nil { - return nil, err - } - file["id"] = m.Module + "." + fmt.Sprint(unsettled["id"]) - owner[fmt.Sprint(file["id"])] = m.Module - if err := handed.add(m.Module, line); err != nil { - return nil, err - } - out = append(out, file) - continue - } - if fmt.Sprint(unsettled["type"]) == "file" { - if _, left := consumersHandedOnly(m, unsettled, with); left { - *handedOnly = append(*handedOnly, consumersLeftOutLine(m, unsettled)) - continue - } - } - resource, err := ApplySettings(unsettled, with.Settings[m.Module]) - if err != nil { - return nil, err - } - copied := map[string]any{} - for k, v := range resource { - copied[k] = v - } - if networks, keeps := keptNetworks[fmt.Sprint(copied["id"])]; keeps { - // The container also joins the found network the setting names, so a neighbour - // that resolves it there keeps resolving it. Passed to the host as its own field, - // which it joins after the container is made. - joins := make([]any, 0, len(networks)) - for _, n := range networks { - joins = append(joins, n) - } - copied["networks"] = joins - } - if err := refuseSecretsInEnvironment(copied, secretFiles, m.Module); err != nil { - return nil, err - } - // Said in the catalogue, not on the machine: the host parses strictly and knows no - // such field, and the reason is for a reader of the manifest. - delete(copied, SecretsInEnvironment) - delete(copied, NamesOnPurpose) - delete(copied, TrustedField) - // A process's bus stays controller-side (novox/hq ADR 0436 §1): never composed onto the - // node-engine's process resource, whose wire declaration is mesh-host's own, vendored, with a - // strict decoder that refuses an unknown key. - delete(copied, ProcessBusField) - // **An operator's value, from the assignment** (novox/hq ADR 0112, ADR 0155): what a - // definition may not carry because it is true of one installation only. Filled from - // the same layers a mergeable file takes, and refused when no layer set it. - if err := unsetInConfigWords(settingInto(copied, WithDefaults(m, with.Settings[m.Module]), m.Module), m); err != nil { - return nil, err - } - // **Placed before anything reads a path.** A pathless directory receives the path - // this node resolves for it, and every ${dir:…} — in paths, mounts, content and - // environment — becomes that path, so what follows sees only concrete places - // (novox/hq ADR 0112). The host receives paths exactly as it always has. - if err := dirInto(copied, dirs, m.Module); err != nil { - return nil, err - } - // The operator's data the same way: ${access:…} becomes where this node keeps it, - // and a placed directory takes the owner the assignment said (issue 153). - if err := accessInto(copied, accessPaths, m.Module); err != nil { - return nil, err - } - ownerInto(copied, placed) - // **After settings, and that is the whole reason it is here.** A module's file - // content is where a setting lands, so a placeholder may only exist once the setting - // has been put in — filling secrets first would look at content that is not yet what - // the machine receives. - if err := intoFile(copied, sealed, outsideNotGiven(m, with.Needed[m.Module]), m.Module); err != nil { - return nil, err - } - // **Substituted here, not on the machine.** A bound value is not secret — the mesh - // holds it in the clear — so there is nothing for the host to be the only witness of, - // and sending it already filled in means the host learns no new field. - if err := boundInto(copied, known, m.Module, absent); err != nil { - return nil, err - } - // And what the module could not have written: the machine it turned out to be - // assigned to. Beside the bound values because it is the same kind of fact — the - // mesh's own, held in the clear — and because a module that must name itself to - // something else has no binding to learn it from (novox/hq ADR 0066). - // Which port this machine gave it, for a module that binds directly rather than - // through a runtime that can remap (ADR 0038). Applied before the machine's facts so - // a refusal names the port rather than whatever came after it. - if err := portInto(copied, m.Module, m.Listens, with); err != nil { - return nil, err - } - // And where this machine put the foundation's servers, for the one module that - // reaches them by seat rather than by binding (novox/hq 04-ISSUES/102). - if err := seatInto(copied, m.Module, with); err != nil { - return nil, err - } - if err := machineInto(copied, thisMachine, m.Module); err != nil { - return nil, err - } - if err := pinned(copied, m.Module); err != nil { - return nil, err - } - // And the other half of the same question: an image nobody has published yet is named - // by an artifact rather than by a placeholder digest, and is just as unrunnable. - if err := built(copied, m.Module); err != nil { - return nil, err - } - // What the mesh built is kept by digest and path; the store's address is this - // network's now, composed here and never recorded (novox/hq 04-ISSUES/102). - if err := artifactsInto(copied, m.Module, with); err != nil { - return nil, err - } - publishedOn(copied, m.Module, with) - // How it is ready, in the node-engine's words: its endpoint as the port this machine - // published it on — or not sent at all to an engine older than the field (ADR 0240). - healthInto(copied, m, with) - // And whether it is busy (novox/hq ADR 0335): sent only to an engine that asks it. - busyInto(copied, with) - // And a user's `root` (novox/hq ADR 0266): sent only when it asserts something, to an engine - // that judges it. - rootInto(copied, with) - // The account's environment and every module's shell code, where this module holds the - // seat that places them (novox/hq ADR 0203, ADR 0204). Gathered from every module on - // the node, as the jails are, and **last of every placeholder pass**: shell code is a - // shell's own syntax, full of `${…}` no pass above should ever be shown. - contributing := append(append([]Manifest(nil), r.Modules...), stillBackedUp...) - if err := contributionsInto(copied, m, contributing, thisMachine, with, r.Capabilities, unplaced); err != nil { - return nil, err - } - copied["id"] = m.Module + "." + fmt.Sprint(resource["id"]) - // A service saying what it reflects names resources within its own module, so those - // are prefixed too or they would point at nothing. - // - // **Unless it already names one.** A module may reflect a file another module put on - // the machine — the case this exists for is a resolver restarting when the mesh - // rewrites the names, which are computed by the mesh and belong to its module, not to - // the daemon's. Written `.`, and a dot is what marks it as already - // answered: prefixing it again would point at nothing, silently, and the daemon would - // serve the old names for ever while everything reported success. - if renamed := reflectsRenamed(m.Module, resource["restart-on"]); renamed != nil { - copied["restart-on"] = renamed - } - // And what it is reloaded on, by the same rule (novox/hq ADR 0102): an id left - // unprefixed matches nothing, and the service is never reloaded. - if renamed := reflectsRenamed(m.Module, resource["reload-on"]); renamed != nil { - copied["reload-on"] = renamed - } - // And which of its module's containers a scheduled step holds still (novox/hq ADR 0189). - // **The loudest of the three when it is missed.** An unprefixed `restart-on` matches - // nothing and a service quietly never restarts; an unprefixed `while-stopped` names a - // container the declaration does not contain, and the host refuses the whole - // declaration — so the machine takes nothing at all, for every push, until this is - // right. That is what it did on the control node (2026-10-04). - if renamed := reflectsRenamed(m.Module, resource[WhileStopped]); renamed != nil { - copied[WhileStopped] = renamed - } - // And what a process replaces (novox/hq issue 213): a resource of this module's that it - // no longer declares, named as the host recorded it, or the host hands nothing over and - // removes it first. - if renamed := reflectsRenamed(m.Module, resource["replaces"]); renamed != nil { - copied["replaces"] = renamed - } - // **What reads one of this module's own secrets is restarted when it changes** (novox/hq - // issue 203, issue 206). A credential is re-issued by the mesh, and a container that - // mounted the old file keeps the old one open: the build machine ran for an hour on a - // credential the mesh had replaced, because its manifest restarted it on its - // environment file and nobody had thought to name the credential too. Composed here so - // no manifest has to say it, for a container or a daemon that names the secret's path. - if reads := secretsReadBy(copied, m, with.Needed[m.Module]); len(reads) > 0 { - copied["restart-on"] = withRestartOn(copied["restart-on"], reads) - } - // **A version prepares its state before it runs** (novox/hq ADR 0135). Derived from the - // module's own resource rather than declared beside it: what prepares the state is the - // module's own code, so what it is given has to be what that code is given — and a - // second resource written by hand is a second copy to drift from the first. Placed - // immediately before it, because a run-once step stops everything the declaration - // places after it (ADR 0052), which is how a version whose preparation failed does not - // serve. - if prepareBefore != "" && fmt.Sprint(resource["id"]) == prepareBefore { - step := prepared(copied) - owner[fmt.Sprint(step["id"])] = m.Module - out = append(out, step) - } - owner[fmt.Sprint(copied["id"])] = m.Module - out = append(out, copied) - } - - // **What only the mesh knows, where this module asked for it.** The graph is the control - // plane's; making a name resolve is the module's software. Emitted as ordinary files under - // this module's name, so they are applied, reported and removed exactly as anything else - // it declares. - // The resolver file is the node-resolver holder's where one is held here, and the uplink's - // holder steps back from it (novox/hq ADR 0247). - given, err := FactsFrom(stepsBack(m, r.Modules), r, with) - if err != nil { - return nil, err - } - for _, fact := range given { - fact["id"] = m.Module + "." + fmt.Sprint(fact["id"]) - owner[fmt.Sprint(fact["id"])] = m.Module - out = append(out, fact) - } - } - // The node's tool runtime, last (novox/hq ADR 0175, to-be 38 WP2.3): one process loading every - // bundle delivered above and holding the credential sealed above, so both exist before it starts - // — the order written here is the order the machine applies. - if r.runtimeHere() { - process, err := r.runtimeProcess(with) - if err != nil { - return nil, err - } - owner[fmt.Sprint(process["id"])] = RuntimeModule - out = append(out, process) - // And a runtime of its own for each module of its own account, after it (novox/hq ADR 0259 §8). - owns, err := r.ownRuntimes(with) - if err != nil { - return nil, err - } - for _, p := range owns { - id := fmt.Sprint(p["id"]) - owner[id] = strings.TrimSuffix(id, "."+OwnRuntimeID()) - out = append(out, p) - } - // What each module's bundles are given is read as the account the runtime runs as — not what a - // module of its own account is given, which its own account reads. - words := map[string]map[string]string{} - for _, m := range r.Modules { - if m.RunsAs != "" { - continue - } - w, err := bundleWords(m, with) - if err != nil { - return nil, err - } - words[m.Module] = w - } - // And a file a module's words name is one the runtime is restarted for when it changes. - if named := givenTo(out, owner, words, r.Account); len(named) > 0 { - restarts, _ := process["restart-on"].([]any) - seen := map[string]bool{} - for _, id := range restarts { - seen[fmt.Sprint(id)] = true - } - for _, id := range named { - if !seen[id] { - restarts = append(restarts, id) - seen[id] = true - } - } - process["restart-on"] = restarts - } - // **And what a module of its own account is given, its own account reads** (issue 723): the files and - // directories its words name that say no owner, given to the account it runs as, never the operator's. - for _, m := range r.Modules { - if m.RunsAs == "" { - continue - } - w, err := bundleWords(m, with) - if err != nil { - return nil, err - } - givenTo(out, owner, map[string]map[string]string{m.Module: w}, m.RunsAs) - } - } - if with.Adopted { - // First, before anything a module declares: what the mesh needs reachable, then its guard. - // The order a machine applies is the order written here. - ours := Openings(rules, with.Foundation, Published(out)) - ours = append(ours, GuardResources(r.guarded(out, owner, rules, with))...) - out = append(ours, out...) - } - return out, nil -} - -// guarded is what the mesh's guard refuses on an adopted node (novox/hq ADR 0103): derived, and -// for taken modules only. -// -// For each taken module, every machine port its containers publish that the filter would admit -// from the private network only — a published port is forwarded, not received, so a found -// firewall filtering only what it receives never sees it — together with the ports the module's -// manifest guards explicitly (the store's port, the broker's management port), wherever the -// machine put them. A port of a module assigned but not taken is not guarded: it may still be the -// predecessor's, serving the predecessor's other machines. The foundation's ports are admitted -// from everywhere and are never guarded. -func (r Resolution) guarded(out []map[string]any, owner map[string]string, rules []Rule, - with Rendering) []int { - meshOnly := map[int]bool{} - fromEverywhere := map[int]bool{} - for _, rule := range rules { - if rule.Protocol != "tcp" { - continue - } - switch rule.From { - case FromMesh: - meshOnly[rule.Port] = true - case FromEverywhere: - fromEverywhere[rule.Port] = true - } - } - for _, port := range with.Foundation { - delete(meshOnly, port) - fromEverywhere[port] = true - } - seen := map[int]bool{} - var ports []int - guard := func(at int) { - if !seen[at] { - seen[at] = true - ports = append(ports, at) - } - } - for _, m := range r.Modules { - if !with.Taken[m.Module] { - continue - } - var mine []map[string]any - for _, resource := range out { - if owner[fmt.Sprint(resource["id"])] == m.Module { - mine = append(mine, resource) - } - } - for outer := range Published(mine)["tcp"] { - if meshOnly[outer] { - guard(outer) - } - } - for _, want := range m.Guards { - at := with.machinePort(m.Module, want) - for _, resource := range mine { - if fmt.Sprint(resource["type"]) != "container" { - continue - } - listed, _ := resource["ports"].([]any) - for _, entry := range listed { - outer, inner, _, ok := mapping(fmt.Sprint(entry)) - if ok && inner == want { - at = outer - } - } - } - // Not what this node is told to open to everyone: a per-node exposure setting that - // widens a guarded port is the operator saying so, and declaring an opening for it - // and a guard dropping it would be one statement refusing the other. - if !fromEverywhere[at] { - guard(at) - } - } - } - sort.Ints(ports) - return ports -} - -// mapping reads a container's port mapping — `[address:]outer:inner[/protocol]`, the address -// possibly an IPv6 one in brackets — indexing from the end, so an address's own colons never -// shift the ports. Not ok for a short form or anything that is not a mapping. -func mapping(written string) (outer, inner int, address string, ok bool) { - written = strings.TrimSpace(written) - if cut := strings.LastIndex(written, "/"); cut >= 0 { - written = written[:cut] - } - parts := strings.Split(written, ":") - if len(parts) < 2 { - return 0, 0, "", false - } - inner, err := strconv.Atoi(parts[len(parts)-1]) - if err != nil { - return 0, 0, "", false - } - outer, err = strconv.Atoi(parts[len(parts)-2]) - if err != nil { - return 0, 0, "", false - } - return outer, inner, strings.Join(parts[:len(parts)-2], ":"), true -} - -// filtersHere is the module on this node that loads the machine's packet filter, or empty when none -// does. Named rather than counted: a refusal that says which module is one step from acted on. -func (r Resolution) filtersHere() string { - for _, m := range r.Modules { - if m.Filtering != nil { - return m.Module - } - } - return "" -} - -// Rules is the rule set this node's filter is derived from: every module's listens, what was -// computed for this machine, and each module's per-node exposure. The same answer whether the node -// is adopted or converged — the one loads it as a filter, the other declares it as openings. -func (r Resolution) Rules(with Rendering) ([]Rule, error) { - // A module whose settings cannot compose is left out of the declaration (novox/hq ADR 0163, - // rule 6), and out of the filter with it: nothing of it is declared, so nothing of it is let - // through. - left := r.LeftOut(with.Settings, with.Adopted) - exposure := map[string]map[int]string{} - for _, m := range r.Modules { - if _, isLeft := left[m.Module]; isLeft { - continue - } - e, err := Exposure(m, with.Settings[m.Module]) - if err != nil { - return nil, err - } - // And how far each endpoint reaches, which says the same thing to the filter and more - // besides (novox/hq ADR 0138). Folded in here rather than beside: the filter has one - // question — from where — and a reach answers it, so giving it two inputs would let them - // disagree. Reaches refuses a port that both name, so this cannot silently prefer one. - reaches, err := Reaches(m, with.Settings[m.Module]) - if err != nil { - return nil, err - } - // **Only for an endpoint the proxy does not serve.** A routed endpoint's port is how the - // proxy reaches it and nothing else (ADR 0045), so `public` there asks for a public name and - // says nothing about the port — opening it to the world as well would undo the arrangement - // the proxy exists for, and would silently reopen a port an operator had narrowed. - // - // Found by trying to express a real module: one whose routed name must be public and whose - // machine-side port must not be. Under one value for both, there was no way to say it. - routed := RoutedPorts(m) - for port, reach := range reaches { - if routed[port] { - continue - } - source, ok := FilterSource(reach) - if !ok { - return nil, fmt.Errorf("%s: %q is not a reach the filter can read", m.Module, reach) - } - if e == nil { - e = map[int]string{} - } - e[port] = source - } - if e != nil { - exposure[m.Module] = e - } - } - return r.Filtering(with.Generators, with.Ports, exposure) -} - -// Contribution is one module telling the answer to a requirement what it needs from it. -type Contribution struct { - // From is the module that said it, so the provider and a person reading the file can tell - // which route belongs to what. - From string `json:"from"` - // Node is the machine it said it from, empty when that is this one. - // - // A provision answered from anywhere in the mesh has consumers on other machines, and the - // provider has to know who they are — a database told to create a password and not who for - // cannot do anything with it. Contributions were node-local until this, which meant the one - // case that most needed them was the one they did not reach. - Node string `json:"node,omitempty"` - // At is where that machine is on the private network, empty when it is not on one or when it - // is this machine. - // - // The mesh knows it and a provider should not have to derive it. A reverse proxy is told - // *send traffic to this consumer* and has to open a connection — so without this every - // provider that reaches back to a consumer would have to know how the mesh names machines, - // which is a convention leaking into every module that implements a provision. - At string `json:"at,omitempty"` - // As is who this consumer is: what the provider should call the login it creates. - // - // **Said by the mesh rather than invented by the provisioner** (novox/hq 04-ISSUES/023). It - // used to be neither — the provisioner made a name, and the consumer, which has to present it - // to authenticate, had no way to learn it. One derivation reaches both ends, so they agree by - // construction. - // Omitted when there is none. A contribution that is not a credential grant — a module - // offering something to another on its own machine — has nobody to be identified to, and a - // field that is always present and usually empty teaches a reader to ignore it. - As string `json:"as,omitempty"` - // Secret is the file on this machine holding that consumer's credential, sealed to it. - // - // Named rather than carried, for the same reason the private network's key is: the mesh - // discarded the value and could not put it here if it wanted to. What is here is where to - // find it. - Secret string `json:"secret,omitempty"` - // Values are the module's own, with settings applied. What the keys mean is agreed by the - // requirement's name — everything providing `reverse-proxy` understands the same shape, which - // is what makes swapping one for another cost nothing. - Values map[string]any `json:"values"` - // Derived is what this provider's own definition said it derives for this consumer, already - // derived (novox/hq ADR 0202). - // - // **The provider is told, rather than recomputing it.** A served value may name the consumer's - // identity — a bucket named for who is asking, a database prefixed with it — and before this - // the rule lived twice: once in the provisioner's code, once transcribed into every consumer's - // definition. The mesh fills the provider's own statement here and delivers the same filled - // value to the consumer, so the two cannot disagree: there is no second computation to - // disagree with. - // - // Only the keys that are per-consumer. The rest of what the provider serves is the same for - // everyone and is in its own definition, where it already is. - Derived map[string]any `json:"derived,omitempty"` -} - -// provisionsAs is the account that reads what the mesh writes for this provider: the one secret -// per consumer it must open to set that consumer's password (novox/hq issue 225). -// -// **A root-owned 0600 file is one that process cannot read**, which is the same sentence already -// written above for a module's own secrets — and the grant secret is the other kind of secret -// the mesh writes for a module, so it is the same rule. -// -// Which account depends on where the module's code runs. A module whose code is a bundle is run -// by the node's tool runtime, as the node's account ([ADR 0198](0198)); one still in a container -// is whatever it declares as its secrets owner. Nothing names these paths, so the rule that -// claims a bundle's other files by the words that name them (givenTo) cannot reach them: the -// harness composes a grant secret's path from the contributions file, not from a word. -// -// Empty is root, which is what it was and what a module with no bundle and no declared owner -// still wants. -func (r Resolution) provisionsAs(m Manifest) string { - // A module of its own account reads them as that account (ADR 0259 §8, issue 723). - if m.RunsAs != "" { - return m.RunsAs - } - if len(m.Bundles) > 0 && r.Account != "" { - return r.Account - } - return m.SecretsOwner -} - -// grantPath is where one consumer's sealed credential lands on the providing machine. -// -// Suffixed, so the directory can also hold whatever the module writing it keeps there and so a -// node named like something else in that directory cannot collide with it. -// One file per consumer, and a consumer is a module on a machine (novox/hq 04-ISSUES/022). -// -// Named after both. Named after the machine alone, two modules on one node wrote to one path: the -// second overwrote the first, and the provisioner — reading a directory — saw one consumer where -// there were two. -// reflectsRenamed is a resource's restart-on list under the module's prefix, or nil when it has -// none. It reads both the shape JSON parsing produces ([]any) and the shape code composing -// resources natively produces ([]string): a reference that was skipped because its list arrived -// in the other shape would point at nothing — silently, with the service never restarting and -// every check passing, which is how a runtime kept serving without the registry trust its -// daemon file already carried. -func reflectsRenamed(module string, reflects any) []any { - var names []string - switch v := reflects.(type) { - case []any: - for _, id := range v { - names = append(names, fmt.Sprint(id)) - } - case []string: - names = v - } - if names == nil { - return nil - } - var renamed []any - for _, named := range names { - if strings.Contains(named, ".") { - renamed = append(renamed, named) - continue - } - renamed = append(renamed, module+"."+named) - } - return renamed -} - -func grantPath(directory, consumer, module string) string { - return strings.TrimRight(directory, "/") + "/" + consumer + "." + module + ".secret" -} - -// holderAs is a consumer's name at the provider with a local name after it, where it keeps several -// credentials for one provision (ADR 0094); the name alone otherwise. -func holderAs(as, local string) string { - if local == "" { - return as - } - return as + "_" + local -} - -// contributions collects what every module in this set contributes, by requirement. -// -// Ordered by contributing module, because the result becomes a file on a machine and a file whose -// lines move about is a file that looks changed when nothing changed. -func (r Resolution) contributions(settings SettingsBy, grants []Grant, - directories map[string]string) (map[string][]Contribution, error) { - out := map[string][]Contribution{} - modules := append([]Manifest{}, r.Modules...) - sort.Slice(modules, func(i, j int) bool { return modules[i].Module < modules[j].Module }) - - // What consumers on other machines asked for. Merged in with this machine's own, because from - // the provider's side they are the same thing — somebody wanting something — and a provider - // that had to read two lists would be a provider that reads one of them. - sorted := append([]Grant{}, grants...) - sort.Slice(sorted, func(i, j int) bool { - if sorted[i].Provision != sorted[j].Provision { - return sorted[i].Provision < sorted[j].Provision - } - if sorted[i].Consumer != sorted[j].Consumer { - return sorted[i].Consumer < sorted[j].Consumer - } - if sorted[i].From != sorted[j].From { - return sorted[i].From < sorted[j].From - } - return sorted[i].Local < sorted[j].Local - }) - // A consumer already carried by the grants loop, keyed (provision, module). When provider and - // consumer are co-located, `grantsFor` enumerates the same-node consumer too, so without this the - // module would be emitted a second time by the m.Contributes loop below — once full (with the - // grant's secret/as) and once partial — which is the duplicate seen in a co-located mesh.json. - granted := map[string]map[string]bool{} - for _, g := range sorted { - if g.From == "" { - // As above: nothing on that machine asks for this any more, so the provider is not - // told about it and withdraws the login on its next pass. - continue - } - as := holderAs(ConsumerIdentity(g.Consumer, IdentitySource(g.Slug, g.From)), g.Local) - derived, err := r.derivedFor(g.Provision, as, g.From, g.Local, settings) - if err != nil { - return nil, err - } - out[g.Provision] = append(out[g.Provision], Contribution{ - From: g.From, Node: g.Consumer, At: g.At, Values: g.Values, Derived: derived, - // One holder per local name: the identity the consumer is known by, and the local name - // after it where the module keeps several (ADR 0094). Not a login any backend checks — - // a secret is not a login — so the identity limit does not apply to the suffix. - As: as, - Secret: grantPath(directories[g.Provision], g.Consumer, holderAs(g.From, g.Local)), - }) - if granted[g.Provision] == nil { - granted[g.Provision] = map[string]bool{} - } - granted[g.Provision][g.From] = true - } - for _, m := range modules { - for _, to := range sortedKeys(m.Contributes) { - // The grants loop already emitted this module's contribution to this provision, with its - // minted secret — emitting the partial copy again would duplicate it. A contribution with - // no grant (a reverse-proxy route names a host, not a credential) is not in `granted`, so - // it still reaches the provider from here. - if granted[to][m.Module] { - continue - } - // Settings reach a contribution the same way they reach a file. A route's hostname is - // exactly the kind of thing that differs between one mesh and the next, and a module - // that could not have it set would have to be edited to be reused. - values, err := r.composed(m, to, m.Contributes[to], settings[m.Module], - m.Module+" contributing to "+to) - if err != nil { - return nil, err - } - out[to] = append(out[to], Contribution{From: m.Module, Values: values}) - } - // Several contributions to one requirement (ADR 0094's sibling for `contributes`): an - // object store's data API and its console are two different public names from one module, - // not one. Never in `granted` — a route names a host, not a credential — so every local - // name always reaches the provider from here. - for _, to := range sortedKeys(m.ContributesMany) { - for _, local := range sortedKeys(m.ContributesMany[to]) { - values, err := r.composed(m, to, m.ContributesMany[to][local], settings[m.Module], - m.Module+" contributing "+local+" to "+to) - if err != nil { - return nil, err - } - out[to] = append(out[to], Contribution{From: m.Module, Values: values}) - } - } - } - return out, nil -} - -// composed is one contribution as its provider receives it: settled with this node's settings, its -// endpoint's port filled in, and its names composed from the label. -// -// **One function, because two readers must agree.** The provider is told the names in its received -// file; the contributing module is told the same names in its own binding (novox/hq 04-ISSUES/122). -// Composing them twice, in two places, is how the proxy would come to serve one name while the -// module wrote another into its configuration. -func (r Resolution) composed(m Manifest, to string, raw map[string]any, layers []Layer, what string) ( - map[string]any, error) { - // Overridden, not merged: a setting changes a key the contribution declares and adds none. - // The provider reads the contribution as a contract, and a setting made for one of this - // module's files is no part of it (novox/hq 04-ISSUES/173). - values, err := overridden(raw, WithDefaults(m, layers), what) - if err != nil { - return nil, fmt.Errorf("%s: %w", what, err) - } - reaches, err := Reaches(m, layers) - if err != nil { - return nil, fmt.Errorf("%s: %w", what, err) - } - blocks, err := Endpoints(m, layers) - if err != nil { - return nil, fmt.Errorf("%s: %w", what, err) - } - portOfEndpoint(values, endpointPorts(m)) - composeName(values, r.PublicDomain, servingAt(r, to), reaches, endpointPorts(m), blocks) - return values, nil -} - -// ownNames is what a module is known by through what it contributes to one requirement — the names -// the mesh composed for it, and nothing else of the contribution. -// -// **The half a module could not learn** (novox/hq 04-ISSUES/122). A module contributes a label, the -// mesh joins it with this node's domains, and the provider serves the result — and the module itself -// was never told. Software that must know its own address (a login redirect, a canonical URL, an -// issuer) had it written into the manifest as a literal, which is a domain in a definition and wrong -// on every other machine. `${bound::name}` is the answer, from the same composition the -// provider receives. -// -// Several contributions to one requirement are keyed by their local name under `names`. -func (r Resolution) ownNames(m Manifest, to string, layers []Layer) (map[string]any, error) { - pick := func(values map[string]any) map[string]any { - names := map[string]any{} - for _, key := range []string{"name", "internal-name"} { - if v, ok := values[key].(string); ok && v != "" { - names[key] = v - } - } - return names - } - out := map[string]any{} - if raw, ok := m.Contributes[to]; ok { - values, err := r.composed(m, to, raw, layers, m.Module+" contributing to "+to) - if err != nil { - return nil, err - } - for k, v := range pick(values) { - out[k] = v - } - } - if locals := m.ContributesMany[to]; len(locals) > 0 { - many := map[string]any{} - for _, local := range sortedKeys(locals) { - values, err := r.composed(m, to, locals[local], layers, - m.Module+" contributing "+local+" to "+to) - if err != nil { - return nil, err - } - if names := pick(values); len(names) > 0 { - many[local] = names - } - } - if len(many) > 0 { - out["names"] = many - } - } - return out, nil -} - -// composeName joins a contribution's label with a node's public domain, and separately with its -// private one, in place (novox/hq ADR 0056). -// -// **The whole of what the mesh does with a route's name: join two given strings — twice.** A -// contribution carries a `label` — the subdomain its operator chose — and the node carries its -// public domain and its own private-network address; the granted names are `