From b280b9de1006999b42e228839a0286c183d8d867 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 08:12:38 +0000 Subject: [PATCH 01/45] Wait on the watch having handled the entry, not 2 s twice, in the pull-key watch test (issue 1337) The unit level ran at 4.4 s of its 5 s bound; this one test took 4.00 s of internal/link's 4.37 s by waiting a fixed 2 s for two things that must not happen. It now waits on the fake watch having handled the entry (two filler entries pass the one-slot channel only after the real one was handled) and on the watches having stopped after a move. Broken in turn (a verdict alone moving the check, a move told twice) it fails. Unit level at this commit: 2454 tests (6 skipped), 2.84 s (bound 5 s); internal/link 0.36 s. Five slowest tests: TestNoPlanIsWrittenButThroughSavePlansMoving 0.64 s, TestAResolverThatAnswersNothingIsHeldAndSaidInMachineNames 0.48 s, TestNoCodeNamesATableOfTheChain 0.31 s, TestEveryUplinkListsBothResolversItsOwnFirst 0.29 s, TestEveryWordingIsPlain 0.28 s. No test moved to the integration level. --- internal/link/pull_buckets_test.go | 44 ++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/internal/link/pull_buckets_test.go b/internal/link/pull_buckets_test.go index 2e27f187..7a98a939 100644 --- a/internal/link/pull_buckets_test.go +++ b/internal/link/pull_buckets_test.go @@ -6,6 +6,7 @@ import ( "errors" "reflect" "strings" + "sync" "testing" "time" @@ -116,11 +117,30 @@ func contains(s, sub string) bool { return strings.Contains(s, sub) } // fakeWatch is one bucket's watch of the key, fed by the test. type fakeWatch struct { updates chan jetstream.KeyValueEntry - stopped bool + done chan struct{} // closed when the watch is stopped + once sync.Once +} + +func newFakeWatch() *fakeWatch { + return &fakeWatch{updates: make(chan jetstream.KeyValueEntry, 1), done: make(chan struct{})} +} + +// handled returns once the watch has taken the entry fed before it and finished with it. The channel holds one entry, +// so the second filler (an entry the watch passes over) can only be sent after the first was taken, which the watch +// does only after it handled the entry before it. It needs a watch that is still listening: after a move it is not. +func (f *fakeWatch) handled(t *testing.T) { + t.Helper() + for range 2 { + select { + case f.updates <- fakeEntry{value: "not a key"}: + case <-time.After(2 * time.Second): + t.Fatal("the watch took no entry") + } + } } func (f *fakeWatch) Updates() <-chan jetstream.KeyValueEntry { return f.updates } -func (f *fakeWatch) Stop() error { f.stopped = true; return nil } +func (f *fakeWatch) Stop() error { f.once.Do(func() { close(f.done) }); return nil } type fakeEntry struct { jetstream.KeyValueEntry @@ -138,7 +158,7 @@ func TestARunningChecksWatchFindsItsKeyInTheGatesBucketOrTodays(t *testing.T) { run := func(exist ...string) (map[string]*fakeWatch, chan PullKey, func(), error) { watches := map[string]*fakeWatch{} for _, b := range exist { - watches[b] = &fakeWatch{updates: make(chan jetstream.KeyValueEntry, 1)} + watches[b] = newFakeWatch() } moved := make(chan PullKey, 2) stop, err := watchPullIn(t.Context(), req.Check.Pulls, func(_ context.Context, bucket string) (pullWatcher, error) { @@ -182,14 +202,28 @@ func TestARunningChecksWatchFindsItsKeyInTheGatesBucketOrTodays(t *testing.T) { t.Fatal(err) } watches["mesh-delivery-gate_pulls"].updates <- fakeEntry{value: `{"head":"old","trunk":"t","verdict":"pass"}`} - if _, ok := wait(moved); ok { + watches["mesh-delivery-gate_pulls"].handled(t) + select { + case <-moved: t.Error("a verdict alone moved the check") + default: } + // Both buckets hear the move; the second watch is done (stopped) once it has handled its entry, and by then a + // second telling would be in the channel. watches["mesh-delivery-gate_pulls"].updates <- fakeEntry{value: `{"head":"newer","trunk":"t"}`} watches["mesh-delivery_pulls"].updates <- fakeEntry{value: `{"head":"newer","trunk":"t"}`} wait(moved) - if _, ok := wait(moved); ok { + for b, w := range watches { + select { + case <-w.done: + case <-time.After(2 * time.Second): + t.Errorf("the watch of %s did not end after the move", b) + } + } + select { + case <-moved: t.Error("a move was told twice") + default: } stop() if _, _, _, err := run(); err == nil { From f64e00e83cfb4790c6f276cb1d9350424b81d216 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 08:19:32 +0000 Subject: [PATCH 02/45] Vendor mesh-sdk's catalogue, overlay, validator, declaration and the new delivery parts, copied from its main 14fd5cb6 (issue 1381) The files go/mod vendor writes for the pin that follows in the next commit, copied on the forge from mesh-sdk at 14fd5cb68fd1 (each equals the blob there; the vendored tree excludes tests). Split from the pin because one commit cannot carry both a patch and copies. --- .../novox/mesh-sdk/go/catalogue/adoption.go | 242 ++ .../novox/mesh-sdk/go/catalogue/artifacts.go | 145 + .../mesh-sdk/go/catalogue/backup_placement.go | 53 + .../novox/mesh-sdk/go/catalogue/bound.go | 186 + .../mesh-sdk/go/catalogue/bound_into_files.go | 192 + .../novox/mesh-sdk/go/catalogue/build.go | 651 ++++ .../novox/mesh-sdk/go/catalogue/busy.go | 124 + .../novox/mesh-sdk/go/catalogue/chosen.go | 100 + .../mesh-sdk/go/catalogue/collections.go | 635 ++++ .../novox/mesh-sdk/go/catalogue/config.go | 504 +++ .../mesh-sdk/go/catalogue/config_move.go | 40 + .../go/catalogue/consumer_into_serves.go | 338 ++ .../novox/mesh-sdk/go/catalogue/consumers.go | 110 + .../mesh-sdk/go/catalogue/core_credentials.go | 33 + .../novox/mesh-sdk/go/catalogue/cron.go | 92 + .../novox/mesh-sdk/go/catalogue/data.go | 791 ++++ .../mesh-sdk/go/catalogue/declaration.go | 2819 ++++++++++++++ .../mesh-sdk/go/catalogue/delivery_seat.go | 140 + .../novox/mesh-sdk/go/catalogue/dir_into.go | 373 ++ .../novox/mesh-sdk/go/catalogue/doc.go | 7 + .../mesh-sdk/go/catalogue/environment_into.go | 731 ++++ .../novox/mesh-sdk/go/catalogue/events.go | 199 + .../novox/mesh-sdk/go/catalogue/files.go | 147 + .../novox/mesh-sdk/go/catalogue/filtering.go | 1147 ++++++ .../go/catalogue/graphical_session.go | 247 ++ .../novox/mesh-sdk/go/catalogue/handed.go | 802 ++++ .../novox/mesh-sdk/go/catalogue/health.go | 495 +++ .../novox/mesh-sdk/go/catalogue/identity.go | 220 ++ .../mesh-sdk/go/catalogue/installation.go | 236 ++ .../novox/mesh-sdk/go/catalogue/jails_into.go | 72 + .../mesh-sdk/go/catalogue/localnetwork.go | 154 + .../novox/mesh-sdk/go/catalogue/logs.go | 105 + .../go/catalogue/machine_into_files.go | 177 + .../novox/mesh-sdk/go/catalogue/manifest.go | 3263 +++++++++++++++++ .../novox/mesh-sdk/go/catalogue/naming.go | 17 + .../mesh-sdk/go/catalogue/node_resolver.go | 119 + .../go/catalogue/optional_requirements.go | 282 ++ .../novox/mesh-sdk/go/catalogue/placement.go | 616 ++++ .../novox/mesh-sdk/go/catalogue/port_into.go | 168 + .../novox/mesh-sdk/go/catalogue/predefined.go | 263 ++ .../mesh-sdk/go/catalogue/process_bus.go | 189 + .../novox/mesh-sdk/go/catalogue/reads.go | 220 ++ .../novox/mesh-sdk/go/catalogue/recreates.go | 332 ++ .../novox/mesh-sdk/go/catalogue/resolve.go | 1459 ++++++++ .../novox/mesh-sdk/go/catalogue/roster.go | 287 ++ .../novox/mesh-sdk/go/catalogue/runtime.go | 479 +++ .../go/catalogue/seat_contributions.go | 291 ++ .../novox/mesh-sdk/go/catalogue/seat_data.go | 518 +++ .../go/catalogue/seat_dependencies.go | 392 ++ .../novox/mesh-sdk/go/catalogue/seat_into.go | 222 ++ .../novox/mesh-sdk/go/catalogue/seat_pass.go | 85 + .../novox/mesh-sdk/go/catalogue/seat_paths.go | 315 ++ .../novox/mesh-sdk/go/catalogue/seats.go | 951 +++++ .../mesh-sdk/go/catalogue/seats_declared.go | 587 +++ .../go/catalogue/secrets_into_files.go | 238 ++ .../mesh-sdk/go/catalogue/served_own_names.go | 202 + .../mesh-sdk/go/catalogue/setting_defaults.go | 261 ++ .../mesh-sdk/go/catalogue/setting_into.go | 367 ++ .../mesh-sdk/go/catalogue/setting_takes.go | 337 ++ .../novox/mesh-sdk/go/catalogue/settings.go | 655 ++++ .../novox/mesh-sdk/go/catalogue/shares.go | 99 + .../novox/mesh-sdk/go/catalogue/state.go | 169 + .../novox/mesh-sdk/go/catalogue/streams.go | 632 ++++ .../novox/mesh-sdk/go/catalogue/template.go | 454 +++ .../mesh-sdk/go/catalogue/terminal_keys.go | 188 + .../go/catalogue/unconsumed_placeholder.go | 178 + .../novox/mesh-sdk/go/catalogue/unhealthy.go | 55 + .../mesh-sdk/go/catalogue/unknown_field.go | 206 ++ .../novox/mesh-sdk/go/catalogue/upgrade.go | 120 + .../novox/mesh-sdk/go/catalogue/verbs.go | 764 ++++ .../novox/mesh-sdk/go/catalogue/zones.go | 117 + .../novox/mesh-sdk/go/declaration/busy.go | 87 + .../novox/mesh-sdk/go/declaration/cron.go | 182 + .../mesh-sdk/go/declaration/declaration.go | 2610 +++++++++++++ .../novox/mesh-sdk/go/declaration/health.go | 188 + .../novox/mesh-sdk/go/delivery/builder.go | 172 + .../novox/mesh-sdk/go/delivery/buildrecord.go | 42 + .../novox/mesh-sdk/go/delivery/checker.go | 295 ++ .../novox/mesh-sdk/go/delivery/delivery.go | 57 +- .../mesh-sdk/go/delivery/fingerprint2.go | 105 + .../novox/mesh-sdk/go/delivery/gate.go | 126 +- .../novox/mesh-sdk/go/delivery/manifests.go | 85 + .../novox/mesh-sdk/go/delivery/source.go | 236 +- .../novox/mesh-sdk/go/delivery/trunk.go | 32 + .../novox/mesh-sdk/go/overlay/declaration.go | 197 + .../novox/mesh-sdk/go/overlay/doc.go | 5 + .../novox/mesh-sdk/go/overlay/generator.go | 195 + .../novox/mesh-sdk/go/overlay/graph.go | 231 ++ .../novox/mesh-sdk/go/overlay/names.go | 39 + .../novox/mesh-sdk/go/validate/validate.go | 42 + 90 files changed, 33281 insertions(+), 49 deletions(-) create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/adoption.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/artifacts.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/backup_placement.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/bound.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/bound_into_files.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/build.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/busy.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/chosen.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/collections.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/config.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/config_move.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/consumer_into_serves.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/consumers.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/core_credentials.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/cron.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/data.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/declaration.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/delivery_seat.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/dir_into.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/doc.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/environment_into.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/events.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/files.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/filtering.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/graphical_session.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/handed.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/health.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/identity.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/installation.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/jails_into.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/localnetwork.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/logs.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/machine_into_files.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/manifest.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/naming.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/node_resolver.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/optional_requirements.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/placement.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/port_into.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/predefined.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/process_bus.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/reads.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/recreates.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/resolve.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/roster.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/runtime.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/seat_contributions.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/seat_data.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/seat_dependencies.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/seat_into.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/seat_pass.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/seat_paths.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/seats.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/seats_declared.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/secrets_into_files.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/served_own_names.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/setting_defaults.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/setting_into.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/setting_takes.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/settings.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/shares.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/state.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/streams.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/template.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/terminal_keys.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/unconsumed_placeholder.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/unhealthy.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/unknown_field.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/upgrade.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/verbs.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/zones.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/declaration/busy.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/declaration/cron.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/declaration/declaration.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/declaration/health.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/delivery/builder.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/delivery/buildrecord.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/delivery/checker.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/delivery/fingerprint2.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/delivery/manifests.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/delivery/trunk.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/overlay/declaration.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/overlay/doc.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/overlay/generator.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/overlay/graph.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/overlay/names.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/validate/validate.go diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/adoption.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/adoption.go new file mode 100644 index 00000000..72bc5645 --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/adoption.go @@ -0,0 +1,242 @@ +package catalogue + +import ( + "fmt" + "sort" + "strconv" + "strings" +) + +// What an adopted node is declared in place of a filter (novox/hq ADR 0100). +// +// On an adopted node the firewall found on the machine stays in force: the mesh loads no table +// that drops by default or holds an accept. What the mesh needs reachable is declared as +// openings, which the host converges through the found firewall in its own terms; and the mesh +// guards its own foundation ports itself, in a table that only refuses. + +// AdoptionPrefix is the id prefix of what the mesh declares of its own on an adopted node. It is +// never a module's, so none of it is ever held as found. +const AdoptionPrefix = "adoption." + +// Where an opening admits from, on the wire. +const ( + OpeningFromEverywhere = "everywhere" + OpeningFromMesh = "mesh" +) + +// The two paths a packet reaches a port by: received by the machine, or forwarded to a +// container that publishes it. +const ( + PathIncoming = "incoming" + PathForwarded = "forwarded" +) + +// Guard resources: the refusal-only table, the unit that loads it, and that unit running. +const ( + GuardPath = "/etc/mesh/guard.nft" + GuardUnit = "mesh-guard.service" + // GuardUnitPath is where the unit is written. + GuardUnitPath = "/etc/systemd/system/" + GuardUnit +) + +// GuardID, GuardUnitID and GuardRunningID are the guard's resource identities. The installer +// raises the same three on an adopted genesis, so the first push finds them already there. +func GuardID() string { return AdoptionPrefix + "guard" } +func GuardUnitID() string { return AdoptionPrefix + "guard-unit" } +func GuardRunningID() string { return AdoptionPrefix + "guard-running" } + +// GuardPackageID is the tool that loads the guard, declared first: a node joining adopted has +// no filter module and may have no nft at all, and a table nothing can load guards nothing. +func GuardPackageID() string { return AdoptionPrefix + "guard-package" } + +// GuardPackage is the package that carries nft. +const GuardPackage = "nftables" + +// OpeningID is an opening's resource identity: its protocol, port and path say what it is. +func OpeningID(protocol string, port int, path string) string { + return fmt.Sprintf("%sopening-%s-%d-%s", AdoptionPrefix, protocol, port, path) +} + +// Openings are what the mesh needs reachable on an adopted node, from the same inputs as the +// filter it would load were the node converged, each from where that filter would admit it. +// +// `rules` is Filtering's answer — every module's listens, the hub's port, the per-node exposure — +// and `foundation` is the ports the mesh itself needs, from everywhere. A rule for this machine +// only opens nothing. `published` maps a machine port a container publishes to the container's +// port: a published port is forwarded, not received, so its opening names the forwarded path and +// the port the packet is forwarded to. +func Openings(rules []Rule, foundation []int, published map[string]map[int]int) []map[string]any { + type key struct { + protocol string + port int + } + from := map[key]string{} + var order []key + widen := func(k key, f string) { + was, seen := from[k] + if !seen { + order = append(order, k) + } + if !seen || was != OpeningFromEverywhere { + from[k] = f + } + } + for _, rule := range rules { + switch rule.From { + case FromEverywhere: + widen(key{rule.Protocol, rule.Port}, OpeningFromEverywhere) + case FromMesh: + widen(key{rule.Protocol, rule.Port}, OpeningFromMesh) + case FromLocalNetwork: + // The private network's part only (novox/hq ADR 0340). An adopted node's found firewall + // stays, and it is what admits the machine's local network; an opening has no word for a + // local network, and widening it to everywhere would open the port to the internet. + widen(key{rule.Protocol, rule.Port}, OpeningFromMesh) + } + } + for _, port := range foundation { + widen(key{"tcp", port}, OpeningFromEverywhere) + } + sort.Slice(order, func(a, b int) bool { + if order[a].port != order[b].port { + return order[a].port < order[b].port + } + return order[a].protocol < order[b].protocol + }) + out := make([]map[string]any, 0, len(order)) + for _, k := range order { + opening := map[string]any{"type": "opening", "port": k.port, "protocol": k.protocol, + "from": from[k]} + if to, forwarded := published[k.protocol][k.port]; forwarded { + opening["id"] = OpeningID(k.protocol, k.port, PathForwarded) + opening["path"] = PathForwarded + opening["to"] = to + } else { + opening["id"] = OpeningID(k.protocol, k.port, PathIncoming) + opening["path"] = PathIncoming + } + out = append(out, opening) + } + return out +} + +// Published is every port the given containers publish on the machine, by protocol and machine +// port, mapped to the container's own port. A mapping bound to loopback is left out: nothing off +// the machine reaches it, forwarded or not. +func Published(resources []map[string]any) map[string]map[int]int { + out := map[string]map[int]int{} + for _, r := range resources { + if fmt.Sprint(r["type"]) != "container" { + continue + } + listed, _ := r["ports"].([]any) + for _, entry := range listed { + written := strings.TrimSpace(fmt.Sprint(entry)) + protocol := "tcp" + if cut := strings.LastIndex(written, "/"); cut >= 0 { + protocol = written[cut+1:] + } + // Indexed from the end, so an IPv6 address's own colons never shift the ports. + outer, inner, address, ok := mapping(written) + if !ok { + continue + } + switch strings.Trim(address, "[]") { + case "127.0.0.1", "localhost", "::1": + continue + } + if out[protocol] == nil { + out[protocol] = map[int]int{} + } + out[protocol][outer] = inner + } + } + return out +} + +// AsGuard renders the mesh's refusal-only table for the given machine ports. +// +// It passes everything by default and holds nothing but a refusal, so it cannot close anything +// the machine serves; and it is the mesh's own table, so the found firewall reloading does not +// touch it. It refuses only packets addressed to this machine, and the ports except from the +// machine itself — its loopback and the container runtime's own networks — and from the private +// network, known by the interface a packet arrives on and never by its source address. At +// prerouting, ahead of the runtime's destination translation, so it matches the port the packet +// was sent to; in the inet family, so both address families. +// +// **The machine's own interfaces are named, where the derived filter names address ranges.** The +// filter accepts the container runtime's networks by CIDR; this excludes its bridges by name — lo, +// docker0, the br-* a compose network gets, and the mesh's own mesh0. A runtime whose bridge is +// named anything else (a podman or libvirt bridge, or a docker network created with a fixed name) +// would have its containers' traffic to a guarded port refused, which reads as the port being +// down. Names rather than addresses is deliberate: a source address can be claimed by whoever +// sends the packet, and this table exists to refuse what the found firewall never sees. Widening +// it means adding names here and in the installer's copy together, which the golden test holds to +// one text. +// +// The same text the installer raises on an adopted genesis; a test holds both to it. +func AsGuard(ports []int) string { + sorted := append([]int{}, ports...) + sort.Ints(sorted) + listed := make([]string, len(sorted)) + for i, p := range sorted { + listed[i] = strconv.Itoa(p) + } + var b strings.Builder + b.WriteString("table inet mesh_guard {}\n") + b.WriteString("delete table inet mesh_guard\n") + b.WriteString("table inet mesh_guard {\n") + b.WriteString("\tchain prerouting {\n") + b.WriteString("\t\ttype filter hook prerouting priority raw; policy accept;\n") + // Only packets addressed to this machine: traffic it routes for others — a predecessor's hub, + // say — is never the guard's business (novox/hq ADR 0103). + fmt.Fprintf(&b, "\t\tfib daddr type local iifname != \"lo\" iifname != \"docker0\" "+ + "iifname != \"br-*\" iifname != \"mesh0\" tcp dport { %s } drop\n", + strings.Join(listed, ", ")) + b.WriteString("\t}\n") + b.WriteString("}\n") + return b.String() +} + +// GuardUnitText is the unit that loads the guard. Stopping it deletes only its own table: never +// a flush, which would take the container runtime's rules and the found firewall with it. +func GuardUnitText() string { + return "[Unit]\n" + + "Description=The mesh's guard: refuses its own ports from outside (novox/hq ADR 0100)\n" + + // Early, before the network is up, and without the default dependencies that would + // order it after the network; stopped at shutdown like any unit. + "DefaultDependencies=no\n" + + "Wants=network-pre.target\n" + + "Before=network-pre.target shutdown.target\n" + + "Conflicts=shutdown.target\n" + + "\n" + + "[Service]\n" + + "Type=oneshot\n" + + "RemainAfterExit=yes\n" + + "ExecStart=nft -f " + GuardPath + "\n" + + "ExecReload=nft -f " + GuardPath + "\n" + + "ExecStop=nft delete table inet mesh_guard\n" + + "\n" + + "[Install]\n" + + "WantedBy=multi-user.target\n" +} + +// GuardResources are the guard as four resources of the existing kinds: the tool that loads it, +// the table, the unit, and the unit running — reloaded when the table changes, so the new table +// replaces the old in one `nft -f` through the unit's ExecReload with no moment unguarded, and +// restarted only when the unit itself changes. Nothing when there is nothing to guard: an empty +// set is not a table nft loads. +func GuardResources(ports []int) []map[string]any { + if len(ports) == 0 { + return nil + } + return []map[string]any{ + {"id": GuardPackageID(), "type": "package", "package": GuardPackage}, + {"id": GuardID(), "type": "file", "path": GuardPath, "content": AsGuard(ports), + "mode": "0644"}, + {"id": GuardUnitID(), "type": "file", "path": GuardUnitPath, "content": GuardUnitText(), + "mode": "0644"}, + {"id": GuardRunningID(), "type": "service", "unit": GuardUnit, "state": "running", + "boot": "enabled", "restart-on": []any{GuardUnitID()}, "reload-on": []any{GuardID()}}, + } +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/artifacts.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/artifacts.go new file mode 100644 index 00000000..e9a010d8 --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/artifacts.go @@ -0,0 +1,145 @@ +package catalogue + +import ( + "fmt" + "strings" +) + +// What the mesh built, named by what it is rather than by where it was pushed. +// +// **An image is recorded by its digest and its path; the registry's address is a route to it** +// (novox/hq 04-ISSUES/102). A build used to be recorded as `://@sha256:…` +// — the reference the builder pushed to, kept whole — and every declaration carried that literal. +// Move the registry's port, or the registry, and every fresh pull of a mesh image fails: a new +// node, a recreate after eviction. The digest is the identity; the address is the node's setting +// for the module that serves the artifact store, and it is read from there when a reference is +// composed, never written into a record. +// +// So a kept reference has a scheme of the mesh's own, named after the provision that answers it: +// +// artifact-store:///@sha256: an image +// artifact-store:////blobs/sha256: an archive +// +// No runtime knows the scheme. That is the point of it being one: a reference that leaks to a +// machine uncomposed is refused by the runtime as malformed, in front of whoever sent it, rather +// than pulled from a public registry that happens to have a repository by that name — which is +// what an address-less `/@sha256:…` would be. + +// ArtifactStoreScheme marks a reference to something in the mesh's artifact store, kept without +// the store's address. +const ArtifactStoreScheme = ArtifactStoreProvision + "://" + +// Recorded is a reference as the mesh records it: the artifact store's address, if the builder wrote +// one, taken off. +// +// For a reference the builder announced — one it pushed to the store — which is the only kind +// this is called on. An image the builder named `/@sha256:…` is kept as its path; an +// archive it named `http:///v2//blobs/` likewise. A reference with no address +// in it — an image id from a genesis build that had nowhere to publish, a package version — is +// what it was. +func Recorded(reference string) string { + if strings.HasPrefix(reference, ArtifactStoreScheme) { + return reference + } + if rest, isURL := strings.CutPrefix(reference, "http://"); isURL { + if _, path, ok := strings.Cut(rest, "/v2/"); ok && strings.Contains(path, "/blobs/") { + return ArtifactStoreScheme + path + } + return reference + } + host, path, ok := strings.Cut(reference, "/") + if !ok || !isRegistryHost(host) || !strings.Contains(path, "@sha256:") { + return reference + } + return ArtifactStoreScheme + path +} + +// isRegistryHost is the runtime's own rule for reading the first component of a reference as a +// registry rather than as a namespace: it has a dot or a port in it, or it is localhost. +func isRegistryHost(component string) bool { + return component == "localhost" || strings.ContainsAny(component, ".:") +} + +// InArtifactStore reports whether a reference is a kept one, and what it names there. +func InArtifactStore(reference string) (path string, kept bool) { + return strings.CutPrefix(reference, ArtifactStoreScheme) +} + +// Routed is a kept reference as a machine fetches it, through the artifact store at `address` +// (host:port). A reference that is not a kept one is what it was. +func Routed(reference, address string) string { + path, kept := InArtifactStore(reference) + if !kept { + return reference + } + if strings.Contains(path, "/blobs/") { + return "http://" + address + "/v2/" + path + } + return address + "/" + path +} + +// Rerouted is a reference the mesh recorded, whichever way it was recorded, as a machine fetches +// it now: a kept one composed with the store's address, and one recorded before references were +// kept without their address — the builder's own `/@sha256:…` — re-routed to where +// the store is now. Only for references that are the mesh's own: everything a build record +// holds is, by construction. +func Rerouted(reference, address string) string { + return Routed(Recorded(reference), address) +} + +// artifactsInto composes the artifact store's address into a resource's `image` and `source`. +// +// **Composed here, at the last moment before a machine, and stored nowhere.** A kept reference is +// routed through the store as this network reaches it now. A reference recorded with an address +// before references were kept without one is re-routed the same way — but only when the mesh +// built it (`with.Built` names every `/` it has), because a module may run an +// image from a public registry under its own name and that one is exactly where it says. +// +// A kept reference with no store to route it through is refused: sent as it is, the runtime would +// refuse the scheme on the machine, one push away from the reason. +// +// **What this does not reach: the images genesis pinned.** The installer builds the control plane +// and the builder before the mesh exists, pushes them itself and pins their manifests to +// `:/mesh-controller@…` and `:/mesh-builder@…` — single-segment +// repositories with no build record, so `with.Built` does not name them and they are left as +// written until each is rebuilt through the mesh, which records it by digest and path. Until then +// a registry that moves strands exactly those two on a recreate, and the control plane's is the +// one that cannot be repaired through the mesh. Rebuild both through `build` before moving the +// store (novox/hq 04-ISSUES/102, finding F4). +func artifactsInto(resource map[string]any, module string, with Rendering) error { + for _, key := range []string{"image", "source"} { + written, ok := resource[key].(string) + if !ok { + continue + } + if _, kept := InArtifactStore(written); kept { + if with.ArtifactStore == "" { + return fmt.Errorf( + "%s's %v names %s, which is in the mesh's artifact store, and this mesh has no "+ + "artifact store on its network to fetch it from — nothing assigned offers "+ + "%s, or the machine offering it is not on the private network", + module, resource["id"], written, ArtifactStoreProvision) + } + resource[key] = Routed(written, with.ArtifactStore) + continue + } + if with.ArtifactStore == "" { + continue + } + recorded := Recorded(written) + if recorded == written { + continue + } + path, _ := InArtifactStore(recorded) + repository := path + if at := strings.IndexAny(path, "@"); at >= 0 { + repository = path[:at] + } else if blobs := strings.Index(path, "/blobs/"); blobs >= 0 { + repository = path[:blobs] + } + if with.Built[repository] { + resource[key] = Routed(recorded, with.ArtifactStore) + } + } + return nil +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/backup_placement.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/backup_placement.go new file mode 100644 index 00000000..ddd35178 --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/backup_placement.go @@ -0,0 +1,53 @@ +package catalogue + +import ( + "fmt" + "regexp" +) + +// What the machine's modules back up is placement (design 57 L3, issue 794). The holder of node-backup names it as +// `${backup:lines}` (what to run and which paths to keep) and `${backup:data}` (every item with its class and +// protection): the text `${contribution:node-backup:backup}` and `${contribution:node-backup:data}` render, from the +// same function, derived from each placed module's declared data (novox/hq ADR 0233). A handed holder has the text in +// its plan's placement for its client to fill; one not handed has the controller fill it where it stands. + +// BackupNamespace is the placeholder kind of the machine's backup lines and data. +const BackupNamespace = "backup" + +// ofBackup is where a holder places them: ${backup:}. Loose inside the braces, so a misspelt key is found and +// refused rather than written out as text. +var ofBackup = regexp.MustCompile(`\$\{backup:([^}]*)\}`) + +// backupKinds is the contribution kind each key of the backup placement renders. +var backupKinds = map[string]string{"lines": BackupKindBackup, "data": BackupKindData} + +// backupPlacer is the machine's modules the backup placement is composed over, as the old contributions are. +type backupPlacer struct { + modules []Manifest + with Rendering + facts map[string]string + caps map[string]bool + unplaced *[]string +} + +// placed is the text of one key for a holder, or what is wrong with naming it. +func (b backupPlacer) placed(holder Manifest, key string, item bool) (string, string) { + kind, known := backupKinds[key] + switch { + case item: + return "", "an item is kept per item; the machine's backup lines are placed only in a file" + case !known: + return "", fmt.Sprintf("the backup placement has `lines` and `data`, not %q", key) + case !holder.ClaimsSeat(BackupSeat): + return "", fmt.Sprintf("%s does not claim %s; the machine's backup lines are placed by its holder alone "+ + "(novox/hq ADR 0212)", holder.Module, BackupSeat) + } + text, left, err := seatContributions(b.modules, holder, BackupSeat+":"+kind, b.with, b.facts, b.caps) + if b.unplaced != nil { + *b.unplaced = append(*b.unplaced, left...) + } + if err != nil { + return "", err.Error() + } + return text, "" +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/bound.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/bound.go new file mode 100644 index 00000000..12941a4a --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/bound.go @@ -0,0 +1,186 @@ +package catalogue + +import ( + "fmt" + "slices" + "strings" +) + +// A consumer of a provision that keeps its data stays bound where its data is (novox/hq ADR 0232). +// +// **What a resolution chooses is not where a consumer's data is.** Resolving answers "which provider +// would I pick now", from the seats' holders, the pins and what is assigned where, and every one of +// those can change under a consumer without anybody meaning to move it. For a resolver that is the +// point: any provider answers alike. For a database it is the consumer's whole state: on 2026-10-05 +// one change to how a seat's holder answers (issue 258) re-bound five database consumers on one +// machine to the store on another, each was given a fresh, empty database there, and nothing warned +// for twenty hours (issue 273). Nothing was lost only because the old provider kept everything. +// +// So the mesh records where each such consumer was last sent (the store's `binding` table), and a +// resolution that would answer it from anywhere else keeps the recorded provider instead and says so. +// **Only a pin moves it**, because a pin is a person: the one act that says "answer this machine's +// consumers from there", taken knowing the data must go first. + +// KeptBinding is one consumer this resolution would have moved, and did not. +type KeptBinding struct { + // Machine is where the consumer runs, and Consumer the module there that is bound. + Machine string + Consumer string + // Provision is what it is bound for. + Provision string + // Bound is the provider it was recorded at, and still is; Would is the one the resolution chose. + Bound Chosen + Would Chosen +} + +// String is the condition's sentence: the move, where the data is, and the act that confirms it. +func (k KeptBinding) String() string { + return fmt.Sprintf("would move %s's %s from %s to %s — its data is on %s; kept there. "+ + "`pin %s %s %s %s` to confirm a move (and move the data first)", + k.Consumer, k.Provision, k.Bound, k.Would, k.Bound, k.Machine, k.Provision, k.Would.Node, k.Would.Module) +} + +// keepBound holds every need for a provision that keeps its consumers' data at the provider its +// consumer was bound to, where the resolution chose another. +// +// A need with no record is a binding being made, and is left as resolved: it is recorded when it is +// first sent. A pin naming the provider the resolution chose is a person moving it, and is left too. +// Otherwise the recorded provider answers, if it still provides the provision — beside the consumer, +// or offered from elsewhere — and the move is returned as kept. **One that no longer does is refused, +// never answered by the provider chosen**: answering it there is exactly the silent move this exists +// to stop, and the consumer's data is still wherever it was. +func keepBound(needs []Needed, catalogue map[string]Manifest, node Node, world World, + keeps map[string]bool, here func(string) bool) ([]Needed, []KeptBinding, []string) { + var kept []KeptBinding + var problems []string + refused := map[[2]string]bool{} + out := make([]Needed, 0, len(needs)) + for _, n := range needs { + if n.ByRecord || !keeps[n.Name] { + out = append(out, n) + continue + } + n.KeepsData = true + bound, recorded := world.Bound[n.For][n.Name] + chose := Chosen{Node: n.From, Module: n.Module} + if !recorded || sameProvider(bound, chose) { + out = append(out, n) + continue + } + if pin, pinned := world.Pinned[n.Name]; pinned && pin.matches(Provider{Node: chose.Node, Module: chose.Module}) { + out = append(out, n) + continue + } + held, ok, why := boundNeed(n, bound, catalogue, node, world, here) + if !ok { + if key := [2]string{n.For, n.Name}; !refused[key] { + refused[key] = true + problems = append(problems, fmt.Sprintf( + "%s on %s is bound to %s for %q, which keeps its data, and %s — it would move to %s, "+ + "which holds none of it. Move its data and say so with `pin %s %s %s %s`, or give "+ + "%s back what it provided", + n.For, node.Name, bound, n.Name, why, chose, node.Name, n.Name, chose.Node, chose.Module, + bound.Node)) + } + continue + } + out = append(out, held) + kept = append(kept, KeptBinding{Machine: node.Name, Consumer: n.For, Provision: n.Name, Bound: bound, Would: chose}) + } + return out, kept, problems +} + +// sameProvider is whether a recorded provider is the one chosen. A record naming no module (none +// is written without one, but a person's hand might) matches any module on its node. +func sameProvider(bound, chose Chosen) bool { + return bound.Node == chose.Node && (bound.Module == "" || bound.Module == chose.Module) +} + +// boundNeed is the need answered by the recorded provider, or why it cannot be. +func boundNeed(n Needed, bound Chosen, catalogue map[string]Manifest, node Node, world World, + here func(string) bool) (Needed, bool, string) { + held := Needed{Name: n.Name, For: n.For, Local: n.Local, KeepsData: true} + if bound.Node == node.Name { + m, known := catalogue[bound.Module] + if !known || !here(bound.Module) || !providesAt(m, n.Name, ScopeMesh) { + return Needed{}, false, fmt.Sprintf("%s no longer runs on %s", bound.Module, node.Name) + } + at := node.At + if at == "" { + at = "127.0.0.1" + } + held.From, held.At, held.Module = node.Name, at, m.Module + held.Serves, held.SharedOwn, held.Identity = servedByOne(m, n.Name), sharedByOne(m, n.Name), m.IdentityBoundOf(n.Name) + held.CredentialForm, held.CredentialTool = m.CredentialFormOf(n.Name), m.CredentialToolOf(n.Name) + return held, true, "" + } + matching := bound.among(world.Offered[n.Name]) + if len(matching) != 1 { + return Needed{}, false, fmt.Sprintf("%s no longer provides it", bound) + } + p := matching[0] + if node.At == "" || p.At == "" { + return Needed{}, false, fmt.Sprintf("%s and %s are not both on the private network", node.Name, p.Node) + } + identity := DefaultIdentityBound + if pm, known := catalogue[p.Module]; known { + held.SharedOwn, _ = pm.SharedCredentialOf(n.Name) + held.CredentialForm, held.CredentialTool = pm.CredentialFormOf(n.Name), pm.CredentialToolOf(n.Name) + identity = pm.IdentityBoundOf(n.Name) + } + held.From, held.At, held.Module, held.Serves, held.Identity = p.Node, p.At, p.Module, p.Serves, identity + held.LeftOut = p.LeftOut + return held, true, "" +} + +// Unbound is one consumer that still asks for a provision and whose own resolution binds it to +// another provider than the one a pair credential on record was made with (novox/hq issue 274). +// +// **A provider is granted exactly the consumers bound to it.** The credential from the old provider +// stays on record — it is the key to a login that provider keeps, disabled, with the consumer's data, +// until a person deletes it with `cleanup delete` (ADR 0230), and a pin back must find it — but it is +// no longer granted, so the provider stops being asked for it and retires it. Said on every plan and +// push of the provider, so a credential the mesh keeps and does not use is never kept silently. +type Unbound struct { + // Provision is what was required, Provider the machine the credential on record is from. + Provision string `json:"provision"` + Provider string `json:"provider"` + // Consumer is the machine, Module the module on it that requires it, Local the credential's + // name inside it where it keeps several (ADR 0094). + Consumer string `json:"consumer"` + Module string `json:"module"` + Local string `json:"local,omitempty"` + // BoundTo is every provider the consumer's resolution binds this credential to now; empty when + // it binds it nowhere — the module asks for the provision under other local names. + BoundTo []string `json:"bound_to,omitempty"` +} + +func (u Unbound) String() string { + who := u.Module + if u.Local != "" { + who += " (as " + u.Local + ")" + } + now := "is bound to no provider under that name" + if len(u.BoundTo) > 0 { + now = "is bound to " + strings.Join(u.BoundTo, ", ") + } + return fmt.Sprintf("%s on %s %s for %s, not to %s — %s no longer grants it, so it retires that "+ + "login and keeps its data until `cleanup delete` (ADR 0230); the credential from %s stays on "+ + "record while that login does", who, u.Consumer, now, u.Provision, u.Provider, u.Provider, u.Provider) +} + +// BindsFrom is the providers this resolution binds a pair credential's need to — the provision, the +// module that requires it and the credential's local name — answered by a machine rather than by a +// record. None means this machine states no such binding. +func (r Resolution) BindsFrom(provision, module, local string) []string { + var from []string + for _, n := range r.Needs { + if n.ByRecord || n.Name != provision || n.For != module || n.Local != local { + continue + } + if !slices.Contains(from, n.From) { + from = append(from, n.From) + } + } + return from +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/bound_into_files.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/bound_into_files.go new file mode 100644 index 00000000..e04caad4 --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/bound_into_files.go @@ -0,0 +1,192 @@ +package catalogue + +import ( + "fmt" + "regexp" + "sort" + "strings" +) + +// The half of a connection that is not secret, put where the program reading it can find it. +// +// **The asymmetry this removes was backwards** (novox/hq 04-ISSUES/023). A sealed credential can +// be placed inside any configuration file a module writes: the module leaves a hole, the mesh +// delivers the value sealed beside it, and the host — the only thing that sees both — fills it in. +// The host and the port and the name to present are ordinary facts the mesh holds in the clear, +// and they were the ones stuck: readable only inside a JSON binding, which a program reading +// `KEY=value` cannot use. +// +// So the same shape, and simpler. These values are not secret, so **the control plane substitutes +// them itself** before the declaration is sent. Nothing new reaches the host, which learns no +// formats and gains no fields. +// +// **It stays name-agnostic** ([ADR 0027]). The mesh does not learn what a `postgres-database` is: +// `at`, `as`, `from` and `address` (the providing machine's private address, novox/hq ADR 0194) are +// facts about any provision at all, and everything else comes from what the provider said it +// serves — whose keys are agreed by the requirement's name, not by this file. + +// bound is where a module says a value from one of its bindings belongs: +// ${bound::}, and for an optional requirement the value it takes while absent, +// ${bound::|default:} (novox/hq ADR 0330 §4). A default is never empty: an +// absent requirement is never rendered as an empty string. +var bound = regexp.MustCompile(`\$\{bound:([a-z0-9][a-z0-9.-]*[a-z0-9]):([a-z0-9][a-z0-9_-]*)(?:\|default:([^}\n]+))?\}`) + +// knownFor is everything a module may name from one of its bindings. +// +// Three facts the mesh states about any provision, plus whatever the provider said it serves. A +// module may not reach a binding it does not have — the same boundary as a secret, for the same +// reason. +func knownFor(m Manifest, needs []Needed, node string) (map[string]map[string]string, error) { + out := map[string]map[string]string{} + for _, want := range m.Wants() { + for i := range needs { + n := needs[i] + if n.Name != want || n.For != m.Module { + continue + } + as := ConsumerIdentity(node, IdentitySource(m.Slug, m.Module)) + values := map[string]string{ + "at": n.At, + "from": n.From, + "as": as, + } + // What the provider derives for this consumer rather than for all of them + // (novox/hq ADR 0202). Filled here, the one place a provision and the module + // requiring it are both in hand. + served, err := ServedTo(n.Serves, as) + if err != nil { + return nil, fmt.Errorf("%s requires %s: %w", m.Module, want, err) + } + for key, value := range served { + // The provider's own vocabulary. Rendered plainly: a port is 5432, not 5432.000000, + // which is what a float would write and what a connection string would refuse. + values[key] = plainly(value) + } + out[want] = values + } + } + return out, nil +} + +// withOwnNames adds a module's own composed names to what it may name from one binding: +// `${bound::name}` and `:internal-name`, and for several contributions to one requirement +// `:name-` / `:internal-name-`. Set over anything the provider serves under those keys: +// what the module is called is the mesh's statement, not the provider's. +func withOwnNames(values map[string]string, own map[string]any) { + for _, key := range []string{"name", "internal-name"} { + if v, ok := own[key].(string); ok { + values[key] = v + } + } + many, _ := own["names"].(map[string]any) + for local, raw := range many { + names, _ := raw.(map[string]any) + for _, key := range []string{"name", "internal-name"} { + if v, ok := names[key].(string); ok { + values[key+"-"+local] = v + } + } + } +} + +// plainly renders a served value as a program would expect to read it. +func plainly(value any) string { + switch v := value.(type) { + case string: + return v + case float64: + if v == float64(int64(v)) { + return fmt.Sprintf("%d", int64(v)) + } + return strings.TrimRight(strings.TrimRight(fmt.Sprintf("%f", v), "0"), ".") + case bool: + return fmt.Sprintf("%t", v) + case nil: + return "" + default: + return fmt.Sprint(v) + } +} + +// boundInto replaces a file's ${bound:…} placeholders with what the mesh knows. +// +// A placeholder naming something the module does not require, or a key the provider does not +// serve, is refused. Left as it was, the literal `${bound:x:y}` would be written into a +// configuration file and read as a value — a connection to a host called `${bound:x:y}`, failing +// somewhere that names neither the module nor the mesh. +// +// **An optional requirement left absent takes the default its placeholder names** (novox/hq ADR +// 0330 §4), and one that names none is refused rather than rendered empty. The module check refuses +// that already; this is the same rule where the file is made, for a manifest registered before it. +func boundInto(resource map[string]any, known map[string]map[string]string, module string, absent map[string]bool) error { + if fmt.Sprint(resource["type"]) != "file" { + return nil + } + content, ok := resource["content"].(string) + if !ok { + return nil + } + var refused error + filled := bound.ReplaceAllStringFunc(content, func(token string) string { + if refused != nil { + return token + } + m := bound.FindStringSubmatch(token) + provision, key, fallback := m[1], m[2], m[3] + values, has := known[provision] + if !has { + if absent[provision] { + if fallback != "" { + return fallback + } + refused = fmt.Errorf( + "%s has a file that says ${bound:%s:%s}, and %q is optional and absent here: it names no "+ + "value for then (${bound:%s:%s|default:}), so it would be written empty "+ + "(novox/hq ADR 0330 §4)", module, provision, key, provision, provision, key) + return token + } + refused = fmt.Errorf( + "%s has a file that says ${bound:%s:%s}, and %s does not require %q. It may name %s", + module, provision, key, module, provision, orNothing(namesOfBindings(known))) + return token + } + value, said := values[key] + if !said { + refused = fmt.Errorf( + "%s asks its %s binding for %q, and what answers it says %s", + module, provision, key, orNothing(namesOfKeys(values))) + return token + } + return value + }) + if refused != nil { + return refused + } + resource["content"] = filled + return nil +} + +func namesOfBindings(known map[string]map[string]string) []string { + var names []string + for name := range known { + names = append(names, fmt.Sprintf("%q", name)) + } + sort.Strings(names) + return names +} + +func namesOfKeys(values map[string]string) []string { + var names []string + for key := range values { + names = append(names, fmt.Sprintf("%q", key)) + } + sort.Strings(names) + return names +} + +func orNothing(names []string) string { + if len(names) == 0 { + return "nothing" + } + return join(names) +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/build.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/build.go new file mode 100644 index 00000000..e36de7f5 --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/build.go @@ -0,0 +1,651 @@ +package catalogue + +import ( + "fmt" + "net" + "net/url" + "path" + "regexp" + "sort" + "strings" +) + +// Turning a manifest that names artifacts into one that names digests. +// +// **Two documents, deliberately.** The manifest in a repository says *this resource uses the +// archive called `config`*; the manifest the mesh holds says *this resource is sha256:…*. A digest +// is not knowable until something is built, so a repository carrying one would be a repository +// whose file is wrong the moment anybody edits anything — and the mesh would be pinning a value +// nobody could have checked. +// +// So the built manifest is **derived**, and the record of which commit it was derived from is what +// makes "is this current?" answerable without building (novox/hq ADR 0009). + +// Built is one artifact after it exists: where it is and what it hashes to. +type Built struct { + // Name is what the manifest called it. + Name string + // Kind is "image" or "archive". + Kind string + // Reference is what a machine uses to get it — an image reference for an image, a URL for an + // archive. Both already carry the digest for an image; an archive carries it separately. + Reference string + // Digest is "sha256:", for an archive. An image reference already ends in one. + Digest string + // Launchers are, for a bundle in an interpreted language, the executable the build wrote beside + // each entrypoint, by entrypoint (novox/hq ADR 0193): what the node's runtime starts to serve it. + Launchers map[string]string + // Texts are, for the archive a module packs its templates into (TemplatesArtifact), each file's text as the + // build read it back from the packed archive, by its path inside it (novox/hq ADR 0349 §2). Never carried on + // the wire or kept: Resolve reads them into the files' content, which is what the mesh holds. + Texts map[string]string `json:"-"` +} + +// Resolve fills a manifest's resources in from what was built. +// +// Every resource naming an artifact is rewritten to name the thing itself, and the `artifact` key +// is removed — because it is a build-time word and the host has never heard of it. A resource +// naming an artifact nothing produced is refused: it would otherwise reach a machine with an +// empty image or an unpinned archive, which is the shape of failure that looks like success. +func (m Manifest) Resolve(built []Built) (Manifest, error) { + if m.Build == nil && len(built) == 0 { + return m, nil + } + + by := map[string]Built{} + for _, b := range built { + by[b.Name] = b + } + // Declared and not produced is a build that did not do what the manifest asked, and saying so + // here beats a machine reporting it later. + var missing []string + if m.Build != nil { + for _, a := range m.Build.Artifacts { + if _, ok := by[a.Name]; !ok { + missing = append(missing, a.Name) + } + } + } + if len(missing) > 0 { + sort.Strings(missing) + return Manifest{}, fmt.Errorf( + "%s says it builds %s and the build did not produce %s — the build did not do what "+ + "the manifest asked, which is a different fault from a resource asking for the "+ + "wrong thing", + m.Module, strings.Join(missing, " and "), oneOrOther(len(missing))) + } + + // **Each template's text, as the build read it back from its templates archive** (novox/hq ADR 0349 §2): into + // the file's content, so what the mesh holds and composes is what the same text inline would be. + var handed *Handed + if m.Templated() { + name := TemplatesArtifact(m) + texts := by[name].Texts + if name == "" || texts == nil { + return Manifest{}, fmt.Errorf("%s names templates, and the build carried no text read back from the "+ + "archive it packs %s/ into (%q): the builder that built it does not read templates", + m.Module, m.TemplatesDir(), name) + } + dir := m.TemplatesDir() + filled, err := fillTemplates(m.Module, dir, m.Resources, func(template string) (string, bool) { + t, ok := texts[strings.TrimPrefix(template, dir+"/")] + return t, ok + }, fmt.Sprintf("the archive %q", name)) + if err != nil { + return Manifest{}, err + } + items, err := fillItemTemplates(m, m.Items, func(template string) (string, bool) { + t, ok := texts[strings.TrimPrefix(template, dir+"/")] + return t, ok + }, fmt.Sprintf("the archive %q", name)) + if err != nil { + return Manifest{}, err + } + // And, for a moved module, where its templates are and which one each file and item is filled from + // (handed.go): read while its resources and items still name their templates. + handed = handedBy(m, by[name]) + m.Resources = filled + m.Items = items + } + + out := m + out.Handed = handed + out.Build = nil + // Read with the build, like Build: the manifest the mesh holds carries each file's text (issue 731). + out.Templates = "" + out.Resources = nil + // What the build compiled, kept on the resolved manifest (novox/hq ADR 0175): a tools bundle is + // named by no resource of the module's own — the node's runtime loads it — so this is the only + // place the mesh would otherwise not have it. In artifact order, so two resolutions of one + // build compare equal. + out.Bundles = nil + if m.Build != nil { + run := runByAResource(m) + for _, a := range m.Build.Artifacts { + if a.Kind != ArtifactBundle { + continue + } + made := by[a.Name] + // What the runtime loads: what the artifact said, else every entrypoint of a module + // that declares tools, else nothing (the field's own rule; see Artifact.Loads). + // + // **Never, unasked, a bundle one of the module's own resources runs** (novox/hq issue 213). + // A process the host runs is the module's service, not its tools: the controller declares + // the verbs it answers as `tools` and serves them itself, and its bundle would otherwise + // have been launched a second time by the node's runtime, as an MCP child it is not. + loads := append([]string(nil), a.Loads...) + if a.Loads == nil && len(m.Tools) > 0 && !run[a.Name] { + loads = append([]string(nil), a.Entrypoints...) + // A bundle compiled to a binary has no entrypoints: the binary is what it is, and what + // the runtime starts to serve it (novox/hq ADR 0193). So a Go tools bundle is served + // as Go — the runtime execs it — exactly as a TypeScript one is through its launcher. + if bin := BinaryOf(a); bin != "" { + loads = []string{bin} + } + } + // **Kept, never routed** (ADR 0155): the builder publishes to the store at the address + // it reached it by, and a manifest carrying that address names an installation — + // registration refused node-tools for exactly this on 2026-10-02. The build record + // already keeps the store-relative form; the resolved manifest keeps the same, and + // composition routes it through the store a machine reaches (Routed). + out.Bundles = append(out.Bundles, Bundle{ + Name: a.Name, Source: Recorded(made.Reference), Digest: made.Digest, + Language: a.Language, Entrypoints: append([]string(nil), a.Entrypoints...), + Loads: loads, Env: copyWords(a.Env), Launchers: copyWords(made.Launchers), + Binary: BinaryOf(a), + }) + } + sort.Slice(out.Bundles, func(i, j int) bool { return out.Bundles[i].Name < out.Bundles[j].Name }) + } + for _, r := range m.Resources { + named, _ := r["artifact"].(string) + if named == "" { + out.Resources = append(out.Resources, r) + continue + } + artifact, ok := by[named] + if !ok { + return Manifest{}, fmt.Errorf( + "%s: %v uses the artifact %q, and this module builds no such thing", + m.Module, r["id"], named) + } + + filled := map[string]any{} + for k, v := range r { + filled[k] = v + } + delete(filled, "artifact") + switch artifact.Kind { + case ArtifactPackage: + // A package is not a resource on any machine; it is consumed by other builds. A + // resource that names one is a manifest error, named here rather than shipped. + return Manifest{}, fmt.Errorf( + "%s: %v uses %q, which is a package — a build input, not a resource a machine runs", + m.Module, r["id"], named) + case ArtifactImage, ArtifactUpstream: + filled["image"] = artifact.Reference + // An image is not unpacked anywhere, so it has no directory to be named for its + // version and `${version}` has nothing to mean. Refused rather than left as literal + // text in a path, which is how it would reach a machine and be created as a directory + // called `${version}`. + for key, value := range filled { + if text, isText := value.(string); isText && strings.Contains(text, versionRef) { + return Manifest{}, fmt.Errorf( + "%s: %v says %s in %q, and %q is an image — an image is not unpacked, so "+ + "it has no versioned place. %s is for an archive or a bundle", + m.Module, r["id"], versionRef, key, named, versionRef) + } + } + case ArtifactArchive, ArtifactBundle: + // The same on the wire: both are bytes fetched by digest and unpacked. They differ in + // how they were made — one packed as it stood, the other compiled first — and a + // machine has no reason to care which. + // Kept, not routed, for the reason the bundles above are (ADR 0155). + filled["source"] = Recorded(artifact.Reference) + filled["digest"] = artifact.Digest + // **And `${version}`, so a resource can name a place that is this build's alone** + // (novox/hq ADR 0141, 04-ISSUES/142). A component is unpacked into a directory named + // for its version so it can read its own version from its path — and until this, + // nothing could compose that path: an archive named a fixed one in the manifest and + // nothing interpolated the build into it, so nothing could ask for + // `…/versions//` and every machine took a hand-placed fallback. + // + // The version is the artifact's own digest, short. Not the commit: two builds of one + // commit are meant to be the same bytes (the toolchains are `-trimpath` for this), and + // a content-addressed version means an unchanged build resolves to the path it already + // had — so re-composing a declaration moves nothing, where a commit would move the + // path of an identical binary and recreate everything that reads it. + for key, value := range filled { + filled[key] = withVersion(value, versionOf(artifact.Digest)) + } + default: + return Manifest{}, fmt.Errorf("%s: %q is a %q, and an artifact is %q, %q, %q or %q", + m.Module, named, artifact.Kind, ArtifactImage, ArtifactArchive, ArtifactUpstream, + ArtifactBundle) + } + out.Resources = append(out.Resources, filled) + } + return out, nil +} + +// withVersion is a resource's value with `${version}` filled: in a string, and in each string of a map — +// a process's env, where the controller is told which build it is (novox/hq issue 352). Anything else is +// left as it is. +func withVersion(value any, version string) any { + switch v := value.(type) { + case string: + if strings.Contains(v, versionRef) { + return strings.ReplaceAll(v, versionRef, version) + } + case map[string]any: + out := make(map[string]any, len(v)) + for k, x := range v { + out[k] = withVersion(x, version) + } + return out + case map[string]string: + out := make(map[string]string, len(v)) + for k, x := range v { + out[k] = strings.ReplaceAll(x, versionRef, version) + } + return out + } + return value +} + +// checkBuild is the manifest's own account of what it builds. +func (b *Build) problems(module string) []string { + if b == nil { + return nil + } + var problems []string + seen := map[string]bool{} + for _, a := range b.Artifacts { + if a.Name == "" { + problems = append(problems, module+" builds an artifact with no name") + continue + } + if seen[a.Name] { + problems = append(problems, fmt.Sprintf( + "%s builds two artifacts called %q, and a resource naming it could mean either", + module, a.Name)) + } + seen[a.Name] = true + switch a.Kind { + case ArtifactImage, ArtifactArchive, ArtifactUpstream, ArtifactBundle, ArtifactPackage: + default: + problems = append(problems, fmt.Sprintf( + "%s: %q is a %q, and an artifact is %q, %q, %q or %q", + module, a.Name, a.Kind, ArtifactImage, ArtifactArchive, ArtifactUpstream, + ArtifactBundle+", "+ArtifactPackage)) + } + // **A bundle is built from the module itself, so it says a language instead.** Everything + // else names what it is built from: a Dockerfile, a directory, somebody else's reference. + // A bundle's source is the module's own directory by definition, and what it needs to say + // is which compiler — because the mesh chooses that, and cannot choose for a module that + // has not said. + if len(a.External) > 0 && (a.Kind != ArtifactBundle || a.Language != "typescript") { + problems = append(problems, fmt.Sprintf( + "%s: %q names packages it keeps external, and only a TypeScript bundle is bundled into "+ + "one file with some kept out (novox/hq ADR 0193)", module, a.Name)) + } + if len(a.Env) > 0 && a.Kind != ArtifactBundle { + problems = append(problems, fmt.Sprintf( + "%s: %q is a %q and says what it is given (env). Only a bundle the node's runtime "+ + "serves is given words (novox/hq ADR 0192); a container says its own environment", + module, a.Name, a.Kind)) + } + if a.Kind == ArtifactBundle || a.Kind == ArtifactPackage { + // **Except for a language that compiles to a binary, where it names which one** + // (novox/hq 04-ISSUES/142). A bundle in an interpreted language is the module's own + // directory compiled whole, and naming a source would be describing its own build. A + // repository written in a compiled language holds several commands — the host and its + // bootstrap live in one, and the mesh needs the host — and "the module's own directory" + // is then not a package at all. So the compiled case may say which package, and says + // the module root by saying nothing. + if a.From != "" && !compilesToABinary(a.Language) { + problems = append(problems, fmt.Sprintf( + "%s: %q is a bundle and names what it is built from (%q). A bundle is built "+ + "from the module's own directory; what it says is the language", + module, a.Name, a.From)) + } + if strings.TrimSpace(a.Language) == "" { + problems = append(problems, fmt.Sprintf( + "%s: %q is a bundle and says no language, so nothing can choose a compiler "+ + "for it", module, a.Name)) + } + problems = append(problems, bundleEnvProblems(module, a)...) + // What the runtime loads is among what was compiled (ADR 0175): a name here that is + // not an entrypoint is a file the bundle does not contain, and the runtime would + // fail to import it on every machine rather than here. + for _, load := range a.Loads { + found := false + for _, e := range a.Entrypoints { + found = found || e == load + } + // A bundle compiled to a binary is one executable: the runtime loads that or nothing + // (novox/hq ADR 0193). + if bin := BinaryOf(a); bin != "" { + found = load == bin + } + if !found { + problems = append(problems, fmt.Sprintf( + "%s: %q says the runtime loads %q, which is not among its entrypoints — "+ + "what is loaded is compiled, so it is named there too", module, a.Name, load)) + } + } + // **A system, for a language that compiles to a binary** (novox/hq ADR 0142). A binary + // is pinned to one operating system at link time so a host refuses to touch a machine + // it was not built for (novox/hq ADR 0005); an artifact that says nothing would be + // compiled for whatever the build machine happened to be, which reads as portable and + // is not. + if compiled := compilesToABinary(a.Language); compiled && strings.TrimSpace(a.System) == "" { + problems = append(problems, fmt.Sprintf( + "%s: %q is compiled to a binary and says no system, so it would be built for "+ + "whatever the build machine happens to be. Declare one artifact per "+ + "system: %s", module, a.Name, spokenSystems())) + } else if !compiled && strings.TrimSpace(a.System) != "" { + problems = append(problems, fmt.Sprintf( + "%s: %q names the system %q and is written in %q, which compiles to code that "+ + "runs anywhere — a system that decides nothing reads as though it did", + module, a.Name, a.System, a.Language)) + } else if compiled && !knownSystem(a.System) { + problems = append(problems, fmt.Sprintf( + "%s: %q is built for %q, and a system is %s", + module, a.Name, a.System, spokenSystems())) + } + } else { + if a.From == "" && a.URL == "" { + problems = append(problems, fmt.Sprintf( + "%s: %q says nothing about what it is built from", module, a.Name)) + } + if a.Language != "" { + problems = append(problems, fmt.Sprintf( + "%s: %q is a %q and names a language. Only a bundle is compiled by the mesh; "+ + "everything else brings its own recipe", module, a.Name, a.Kind)) + } + } + if a.Compiles != "" { + if a.Kind != ArtifactImage { + problems = append(problems, fmt.Sprintf( + "%s: %q is a %q and names a Go package it compiles. Only an image's recipe is "+ + "told which; a bundle names its package in from (novox/hq ADR 0267)", module, a.Name, a.Kind)) + } else if c := strings.TrimSpace(a.Compiles); strings.HasPrefix(c, "/") || c == ".." || + strings.HasPrefix(c, "../") || strings.Contains(c, "/../") || strings.HasSuffix(c, "/..") { + problems = append(problems, fmt.Sprintf( + "%s: %q compiles %q, which leaves the tree it is built in", module, a.Name, a.Compiles)) + } + } + // An upstream image is named, not read from the repository, so the path rule does not + // apply to it — and applying it anyway would refuse every reference with a registry host + // in it. + if a.Kind != ArtifactUpstream && + (strings.HasPrefix(a.From, "/") || strings.Contains(a.From, "..")) { + // A build reads its own repository and nothing else. A path leaving it would make + // what gets built depend on whatever happens to be on the machine building it. + problems = append(problems, fmt.Sprintf( + "%s: %q is built from %q, which is outside its own repository", + module, a.Name, a.From)) + } + if a.Kind == ArtifactUpstream && !strings.Contains(a.From, ":") { + // Without a tag or digest, what gets mirrored is whatever `latest` means today, and + // a module pinned to that is not pinned. + problems = append(problems, fmt.Sprintf( + "%s: %q mirrors %q, which names no tag or digest", module, a.Name, a.From)) + } + problems = append(problems, upstreamFileProblems(module, a)...) + } + return problems +} + +// reservedSuffixes are names no public vendor publishes under: an installation's own and documentation's. +var reservedSuffixes = []string{".internal", ".local", ".lan", ".home", ".corp", ".localhost", ".home.arpa", + ".test", ".invalid", ".example", ".intranet", ".private"} + +// notAPublicHost says why a host is not a public vendor's name, or "". +func notAPublicHost(host string) string { + h := strings.ToLower(strings.TrimSuffix(host, ".")) + if net.ParseIP(h) != nil { + return "an address rather than a name" + } + if !strings.Contains(h, ".") { + return "a name with no domain" + } + for _, s := range reservedSuffixes { + if strings.HasSuffix(h, s) { + return "a name under " + s + ", which is nobody's public domain" + } + } + return "" +} + +// sha256Hex is a pinned sha256 as a manifest writes it: 64 lower-case hexadecimal characters. +var sha256Hex = regexp.MustCompile(`^[0-9a-f]{64}$`) + +// upstreamFileProblems is what is wrong with an archive pinned by URL (novox/hq ADR 0316 §2, §6): it is an +// archive, fetched over https, pinned by a sha256 and a version, and it names no directory of the repository +// besides; a file name or a directory within is a plain path that stays where it is put. +func upstreamFileProblems(module string, a Artifact) []string { + pinned := a.URL != "" || a.SHA256 != "" || a.Version != "" || a.File != "" || a.Within != "" + if !pinned { + return nil + } + var problems []string + say := func(format string, args ...any) { + problems = append(problems, fmt.Sprintf("%s: %q ", module, a.Name)+fmt.Sprintf(format, args...)) + } + if a.Kind != ArtifactArchive { + say("is a %q and names an upstream file (url, sha256, version, file, within): only an archive is "+ + "mirrored from one (novox/hq ADR 0316)", a.Kind) + return problems + } + if a.URL == "" { + say("names a sha256, a version, a file or a directory within, and no url to fetch it from") + return problems + } + if a.From != "" { + say("is built from %q and fetched from %s: an archive is one or the other", a.From, a.URL) + } + if u, err := url.Parse(a.URL); err != nil || u.Scheme != "https" || u.Host == "" { + say("is fetched from %q, and an upstream file is fetched over https from a named host", a.URL) + } else if why := notAPublicHost(u.Hostname()); why != "" { + say("is fetched from %s, %s: an upstream file is somebody else's release, published under a public name", a.URL, why) + } + if !sha256Hex.MatchString(a.SHA256) { + say("is fetched from %s without a pinned sha256 (64 lower-case hexadecimal characters): what that "+ + "address serves changes under the mesh, and only the digest says it is what was reviewed", a.URL) + } + if strings.TrimSpace(a.Version) == "" { + say("pins %s and says no version, so a person reading the pin cannot tell which release it is", a.URL) + } + if a.File != "" && a.Within != "" { + say("names a file (%q) and a directory within (%q): a file is kept whole, a directory is taken out "+ + "of an archive", a.File, a.Within) + } + for field, p := range map[string]string{"file": a.File, "within": a.Within} { + if p == "" { + continue + } + clean := path.Clean(p) + if clean != p || strings.HasPrefix(p, "/") || clean == "." || clean == ".." || strings.HasPrefix(clean, "../") || + (field == "file" && strings.Contains(p, "/")) { + say("names %s %q, which is not a plain relative %s", field, p, map[string]string{ + "file": "file name", "within": "directory"}[field]) + } + } + return problems +} + +// oneOrOther keeps the message readable for one artifact and for several, because a message that +// says "neither" about one thing reads as a bug in the message. +func oneOrOther(n int) string { + if n == 1 { + return "it" + } + return "them" +} + +// Systems the mesh builds binaries for, which is the set a host may be pinned to (novox/hq ADR 0005). +// +// **A closed list, and the host's own, not the compiler's.** These are not the values a Go toolchain +// would call an operating system — the difference between two of them is a C library, not a kernel. +// They are what a machine reports itself to be and what a host is linked to refuse, so the list that +// matters is the one the host understands. +var systems = []string{"alpine", "android", "arch"} + +// knownSystem is whether the mesh builds for it. +func knownSystem(system string) bool { + want := strings.ToLower(strings.TrimSpace(system)) + for _, s := range systems { + if s == want { + return true + } + } + return false +} + +// spokenSystems is the list as a refusal says it, so a reader is one edit from right. +func spokenSystems() string { + return strings.Join(systems, ", ") +} + +// compilesToABinary is whether this language's bundle is a binary for one operating system rather +// than code that runs wherever its interpreter does. +// +// **Asked of the language, not of the artifact.** A module says what it is written in; what that +// implies is the mesh's to know, exactly as the compiler is (novox/hq ADR 0142). Asking the artifact +// would let two artifacts in one language disagree about whether they are portable. +func compilesToABinary(language string) bool { + switch strings.ToLower(strings.TrimSpace(language)) { + case "go": + return true + default: + return false + } +} + +// versionRef is how a resource names the version of the artifact it uses: ${version}. +// +// No artifact name in it, because the resource already says which artifact it is for — a second +// name would be a second thing to keep in step with the first. +const versionRef = "${version}" + +// versionOf is an artifact's version as a path names it: its digest, short. +// +// **Content-addressed on purpose.** The alternative is the commit a build came from, and two builds +// of one commit are meant to produce the same bytes — every toolchain here is `-trimpath` for that +// reason. A commit-named path would move for an identical binary, and everything reading that path +// would be recreated for a change that is not one. A digest-named path moves exactly when the bytes +// do. +// +// Twelve hex characters: enough that two of this mesh's builds will not collide, short enough to +// read in a path and in a journal line. The `sha256:` prefix goes, because a directory name carrying +// a colon is a directory name people quote wrong. +func versionOf(digest string) string { + hex := strings.TrimPrefix(strings.TrimSpace(digest), "sha256:") + if len(hex) > 12 { + return hex[:12] + } + return hex +} + +// bundleEnvWords are the words the runtime sets for itself; a bundle that named one would be +// telling the runtime what it is, which is the mesh's to say (novox/hq ADR 0192). +var bundleEnvWords = map[string]bool{ + RuntimeToolModules: true, RuntimeBrokerFile: true, RuntimeOperatorAccount: true, + RuntimeOperatorHome: true, RuntimeToolEnv: true, RuntimeAgentAccount: true, RuntimeAgentHome: true, +} + +// bundleEnvProblems says what is wrong with what a bundle says it is given (novox/hq ADR 0192): +// a value is a path or a constant written with the references a container's environment may use +// for a place or a port, and never a secret's content or another module's binding — a secret +// reaches a tool as a file whose path is named. +func bundleEnvProblems(module string, a Artifact) []string { + if len(a.Env) == 0 { + return nil + } + var problems []string + for _, word := range sortedKeys(a.Env) { + value := a.Env[word] + if bundleEnvWords[word] { + problems = append(problems, fmt.Sprintf( + "%s: %q gives itself %s, which the node's runtime sets for itself; a bundle is "+ + "given its own words beside the runtime's, never in place of them (novox/hq ADR 0192)", + module, a.Name, word)) + } + rest := ofPort.ReplaceAllString(dirRef.ReplaceAllString(value, ""), "") + if strings.Contains(rest, "${") { + problems = append(problems, fmt.Sprintf( + "%s: %q gives %s the value %q. A bundle's word is a path or a constant, written with "+ + "${dir:…} and ${port:…} only; a secret reaches a tool as a file the mesh places, "+ + "named by its path, never as its content (novox/hq ADR 0192)", + module, a.Name, word, value)) + } + } + return problems +} + +func copyWords(in map[string]string) map[string]string { + if len(in) == 0 { + return nil + } + out := make(map[string]string, len(in)) + for k, v := range in { + out[k] = v + } + return out +} + +// BinaryOf is what a bundle compiled to a binary is called once built: what the artifact names, else +// the package it is built from, else the artifact's own name (novox/hq 04-ISSUES/142). Empty for a +// language that does not compile to one. The builder writes the binary under this name, and the +// composer runs it by it, so both ask here. +func BinaryOf(a Artifact) string { + if !compilesToABinary(a.Language) { + return "" + } + if name := strings.TrimSpace(a.Binary); name != "" { + return name + } + if from := strings.Trim(a.From, "./"); from != "" { + return path.Base(from) + } + return a.Name +} + +// runByAResource is the artifacts one of a module's own resources names — a process that runs it, +// a step, an archive that unpacks it — by name. +func runByAResource(m Manifest) map[string]bool { + named := map[string]bool{} + for _, r := range m.Resources { + if a, ok := r["artifact"].(string); ok && a != "" { + named[a] = true + } + } + return named +} + +// undeliveredBundles says which of a module's bundles nothing would ever put on a machine (novox/hq +// 04-ISSUES/216). A bundle reaches a machine three ways: the node's runtime serves it (it says +// `loads`, or its module declares `tools`), a resource names it (a process, a step, an archive), or +// it is the runtime itself. One reached by none of them was built, recorded and pushed as success, +// and was simply absent — seven modules' tools went missing that way on 2026-10-03. Refused here, +// naming the field that would deliver it. +func undeliveredBundles(m Manifest) []string { + if m.Build == nil || m.Module == RuntimeModule { + return nil + } + named := runByAResource(m) + var problems []string + for _, a := range m.Build.Artifacts { + if a.Kind != ArtifactBundle || named[a.Name] || len(a.Loads) > 0 || len(m.Tools) > 0 { + continue + } + problems = append(problems, fmt.Sprintf( + "%s: the bundle %q would be built and never reach a machine: nothing loads it, runs it or "+ + "unpacks it. A tools bundle says `loads` (the entrypoints the node's runtime serves) or its "+ + "module lists its `tools`; a daemon or a step is a resource naming it (novox/hq 04-ISSUES/216)", + m.Module, a.Name)) + } + return problems +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/busy.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/busy.go new file mode 100644 index 00000000..18cbbf02 --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/busy.go @@ -0,0 +1,124 @@ +package catalogue + +import ( + "fmt" + "sort" + "strings" + "time" +) + +// A long-running container may say it is busy (novox/hq ADR 0335 §1). +// +// **Beside its health, on the container**: a command run inside it, by its shell, as an exec health check +// is, that exits 0 while the container holds work a replacement would lose and non-zero when it is idle; +// and how long one ask may take. Being asked is also its notice that it is about to be replaced: from the +// first ask it takes no new work. The node-engine asks before it replaces the container, leaves it running +// while it says busy, and replaces it on the first pass it says idle, or at 15 minutes — the engine's bound, +// never the module's. +// +// Sent only to a node-engine that reads it (link.BusyContract): an older, strict engine refuses a field it +// does not know, whole, and to it the container is replaced at once, as before. + +// BusyField is the resource field. +const BusyField = "busy" + +// The bounds of one ask (ADR 0335), the node-engine's own, held here near the author. +const ( + BusyTimeoutDefault = 10 * time.Second + BusyTimeoutCeiling = time.Minute +) + +// Busy is one container's busy, read. +type Busy struct { + Command string + Timeout time.Duration +} + +// ReadBusy reads a resource's `busy` field, its default applied. False when it carries none. +func ReadBusy(r map[string]any) (Busy, bool, []string) { + raw, present := r[BusyField] + if !present { + return Busy{}, false, nil + } + id := fmt.Sprint(r["id"]) + fields, ok := raw.(map[string]any) + if !ok { + return Busy{}, true, []string{fmt.Sprintf("%s: busy is %T; it is an object with a command", id, raw)} + } + var problems []string + keys := make([]string, 0, len(fields)) + for k := range fields { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + if k != "command" && k != "timeout" { + problems = append(problems, fmt.Sprintf("%s: its busy says %q; busy has a command and a timeout", id, k)) + } + } + b := Busy{Timeout: BusyTimeoutDefault} + if v, has := fields["command"]; has { + s, ok := v.(string) + if !ok { + problems = append(problems, fmt.Sprintf("%s: its busy's command is %T; it is text", id, v)) + } + b.Command = strings.TrimSpace(s) + } + if v, has := fields["timeout"]; has { + s, _ := v.(string) + d, err := time.ParseDuration(strings.TrimSpace(s)) + switch { + case err != nil: + problems = append(problems, fmt.Sprintf("%s: its busy's timeout is %v; it is a duration such as \"10s\"", id, v)) + case d <= 0 || d > BusyTimeoutCeiling: + problems = append(problems, fmt.Sprintf("%s: an ask of its busy takes more than nothing and at most %s, not %s", + id, BusyTimeoutCeiling, d)) + default: + b.Timeout = d + } + } + return b, true, problems +} + +// busyProblems is everything wrong with a manifest's `busy` fields, in the manifest's words: a command on +// one line, a timeout within its bounds, and only on a container that stays up. +func busyProblems(m Manifest) []string { + var problems []string + for _, r := range m.Resources { + b, has, read := ReadBusy(r) + if !has { + continue + } + where := m.Module + ": " + fmt.Sprint(r["id"]) + for _, p := range read { + problems = append(problems, m.Module+": "+p) + } + if fmt.Sprint(r["type"]) != "container" || !LongRunning(r) { + problems = append(problems, fmt.Sprintf("%s declares busy and is not a container that stays up: busy is "+ + "asked of a running container before it is replaced (ADR 0335)", where)) + } + if b.Command == "" { + problems = append(problems, where+"'s busy names no command: one run inside the container, exiting 0 "+ + "while it is busy") + } + if strings.ContainsAny(b.Command, "\n\r") { + problems = append(problems, where+"'s busy command is one line") + } + } + return problems +} + +// busyInto composes a container's `busy` into the node-engine's words, or takes it away: for an engine that +// reads it, the command and its timeout written out; for one that does not, the field is not sent, and the +// container is replaced at once, as before. +func busyInto(resource map[string]any, with Rendering) { + if _, has := resource[BusyField]; !has { + return + } + if !with.ReadsBusy { + delete(resource, BusyField) + return + } + b, _, _ := ReadBusy(resource) + resource[BusyField] = map[string]any{"command": b.Command, "timeout": b.Timeout.String()} +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/chosen.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/chosen.go new file mode 100644 index 00000000..1fb8233c --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/chosen.go @@ -0,0 +1,100 @@ +package catalogue + +import ( + "sort" +) + +// Chosen is the provider somebody named for a provision: the module, and the node it runs on. Both, +// always (novox/hq #258) — a provision comes from a module, and the same module on two machines is +// two answers, so neither half alone says which. Module is empty only on a record made before this +// was asked, and such a record is honoured exactly as long as it is unambiguous. +type Chosen struct { + Node string + Module string +} + +func (c Chosen) String() string { + if c.Module == "" { + return c.Node + } + return c.Node + "/" + c.Module +} + +// matches is whether this provider is the one chosen. +func (c Chosen) matches(p Provider) bool { + return p.Node == c.Node && (c.Module == "" || p.Module == c.Module) +} + +// among is every offered provider the choice names — one, when the choice is whole. +func (c Chosen) among(where []Provider) []Provider { + var out []Provider + for _, p := range where { + if c.matches(p) { + out = append(out, p) + } + } + return out +} + +// nameOf is how a refusal names a provider: the node and the module on it. +func nameOf(p Provider) string { + return Chosen{Node: p.Node, Module: p.Module}.String() +} + +// providerNames is every provider named, sorted, for a refusal to list. +func providerNames(where []Provider) []string { + out := make([]string, 0, len(where)) + for _, p := range where { + out = append(out, nameOf(p)) + } + sort.Strings(out) + return out +} + +// providersHere is which modules in this node's own set offer a provision, sorted. +func providersHere(catalogue map[string]Manifest, here func(string) bool, want string) []string { + var out []string + for name, m := range catalogue { + if !here(name) { + continue + } + for _, o := range m.Offers() { + if o == want { + out = append(out, name) + break + } + } + } + sort.Strings(out) + return out +} + +// servedByOne is what one provider beside the consumer says a consumer needs to know, or nothing. +// +// Serving is *whether* a need is created at all when the provider is on this same machine (novox/hq +// 04-ISSUES/038's sibling): a need never created is a binding the consumer never gets. The manifest +// alone answers that; the values are settled later, with the node's settings. +func servedByOne(m Manifest, want string) map[string]any { + if _, ok := m.Serves[want]; ok { + return ServedOn(m, want, nil) + } + return nil +} + +// sharedByOne is the own secret that provider names as its credential (ADR 0158), or "" when it +// gives each consumer its own. +func sharedByOne(m Manifest, want string) string { + if own, shared := m.SharedCredentialOf(want); shared { + return own + } + return "" +} + +func oneOf(list []string, s string) bool { + for _, x := range list { + if x == s { + return true + } + } + return false +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/collections.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/collections.go new file mode 100644 index 00000000..c2a65622 --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/collections.go @@ -0,0 +1,635 @@ +package catalogue + +import ( + "fmt" + "path" + "regexp" + "slices" + "sort" + "strconv" + "strings" +) + +// Collections (novox/hq ADR 0349 clause 13, design 57 L1, issue 684): a provision with many providers. +// +// **A collector requires a collection; a contributor provides an item from a template.** A module taking additions +// from others names a collection in a file of its own, `${collected:}`, and declares it under +// `collections`, with the comment its tool writes a line in. Each contributor ships its item as a template of its +// own repository and declares it under `items`: `{"collection": …, "template": "files/"}`. Its build packs the +// template with the module's others, and the mesh holds its text as it holds a file's (template.go). +// +// **The node's provisioner client fills both.** An item is filled on the node by the client of the node its +// contributor is placed on, from the same owners as a file, and recorded in mesh-provisioner_collect; a collector's +// file is filled by the same client from every item of its collection recorded for that node, so an item goes when +// its contributor leaves. So a module contributing an item is a moved module (handed.go), and the controller renders +// no item: the plan of a contributor names its items and their templates, and the plan of a collector names each +// collection it reads and its comment. Neither carries a contributor's text. +// +// **The three old namespaces read as collections.** `${shell::}`, `${environment:}` and +// `${contribution::}` name, in a handed collector's file, the collection of the same name — +// `shell:zsh:normal` — with the comment the controller wrote before each module's piece. An item is contributed to +// one by that name, by a module the old form would have let contribute it. Each file is filled one way, never +// both (ADR 0349 clause 17): a handed collector is filled from items alone and an unhanded one from the old forms +// alone, and a piece of the other form on the same machine is said as not placed, never dropped quietly. + +// Item is one item a module contributes to a collection: its template in the module's repository, or, in the +// manifest the mesh holds, the template's text, read back from the build's templates archive like a file's. +// +// **Attributes, a capability and a name** (design 57 L3, issue 777): an item may carry `attributes` (string keys and +// values; `order`, a whole number 0–99 defaulting to 50, orders the items of one placeholder) and `if-capability`, +// the capability a machine must report for the item to be placed on it. A module may give several items to one +// collection, each then with a `name` unique within the module and the collection. A collector takes only the items +// whose attributes match, `${collected::=,…}`, and all of them without the selector. +type Item struct { + Collection string `json:"collection"` + Name string `json:"name,omitempty"` + Template string `json:"template,omitempty"` + Content string `json:"content,omitempty"` + Attributes map[string]string `json:"attributes,omitempty"` + IfCapability string `json:"if-capability,omitempty"` +} + +// ItemKey is an item's key among its module's: the collection, and `/` for a named item. +func ItemKey(collection, name string) string { + if name == "" { + return collection + } + return collection + "/" + name +} + +func (it Item) key() string { return ItemKey(it.Collection, it.Name) } + +// order is the item's place among the items of one placeholder: its `order` attribute, or OrderDefault. +func (it Item) order() int { + if n, err := strconv.Atoi(it.Attributes[OrderAttribute]); err == nil && n >= 0 && n <= 99 { + return n + } + return OrderDefault +} + +// OrderAttribute is the attribute that orders items. +const OrderAttribute = "order" + +// matches is whether the item's attributes carry every pair of the selector. +func (it Item) matches(where map[string]string) bool { + for k, v := range where { + if got, has := it.Attributes[k]; !has || got != v { + return false + } + } + return true +} + +// itemsInOrder is the module's items by order, then name; the order the old form placed a module's pieces in. +func itemsInOrder(items []Item) []Item { + out := append([]Item(nil), items...) + sort.SliceStable(out, func(a, b int) bool { + if oa, ob := out[a].order(), out[b].order(); oa != ob { + return oa < ob + } + return out[a].Name < out[b].Name + }) + return out +} + +// Collection is one collection a module collects, and the comment its tool writes a line in: each contributor's item +// is introduced by ` `, and by nothing without one. +type Collection struct { + Name string `json:"name"` + Comment string `json:"comment,omitempty"` +} + +// CollectedNamespace is a collection's placeholder namespace, `${collected:}`. +const CollectedNamespace = "collected" + +// EnvironmentCollection is the one collection of the session environment (design 57 L3, issue 795): its items are +// typed by attributes, a variable (`kind=variable`, `name`) or a PATH entry (`kind=path`, `at=start|end`), and the +// template's text is the value or the entry. A collector names the rendering, `${collected:environment:posix}` or +// `${collected:environment:systemd}`, which is not a selector. +const EnvironmentCollection = "environment" + +// oldCollectionNamespaces are the namespaces read as collections of their own name. +var oldCollectionNamespaces = []string{"shell", "environment", "contribution"} + +var ( + // ofCollected is where a collector places a collection. Loose inside the braces, so a misspelt name is refused. + ofCollected = regexp.MustCompile(`\$\{collected:([^}]*)\}`) + // collectionName is words joined by colons, so an old namespace's placeholder names its collection as it reads. + collectionName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*(:[a-z0-9][a-z0-9-]*)*$`) + // itemName, attributeKey and attributeValue are the words an item's name and attributes are made of; none holds + // a colon, a comma or an equals sign, which tell a selector from a collection's name. + itemName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`) + attributeKey = regexp.MustCompile(`^[a-z][a-z0-9-]*$`) + attributeValue = regexp.MustCompile(`^[a-z0-9][a-z0-9_.-]*$`) +) + +// splitSelector is a placeholder's name without its selector, and the selector: `widgets:bar=top,place=left` is +// `widgets` and `bar=top,place=left`. A collection's name holds no `=`, so the selector begins at the last colon +// before the first one. +func splitSelector(name string) (collection, selector string) { + eq := strings.Index(name, "=") + if eq < 0 { + return name, "" + } + colon := strings.LastIndex(name[:eq], ":") + if colon < 0 { + return name, "" + } + return name[:colon], name[colon+1:] +} + +// selectorOf is a selector's pairs, or why it is none. +func selectorOf(selector string) (map[string]string, error) { + out := map[string]string{} + if selector == "" { + return out, nil + } + for _, pair := range strings.Split(selector, ",") { + k, v, ok := strings.Cut(pair, "=") + switch { + case !ok || !attributeKey.MatchString(k) || !attributeValue.MatchString(v): + return nil, fmt.Errorf("%q in %q is not =", pair, selector) + case out[k] != "": + return nil, fmt.Errorf("%q names %s twice", selector, k) + } + out[k] = v + } + return out, nil +} + +// collectionOfPlaceholder is the collection a placeholder's namespace and key name, and whether they name one. +func collectionOfPlaceholder(namespace, key string) (string, bool) { + key, _ = splitSelector(key) + switch { + case namespace == CollectedNamespace && strings.HasPrefix(key, EnvironmentCollection+":"): + // The environment is one collection, and the rest of the name picks the rendering (issue 795). + return EnvironmentCollection, true + case namespace == CollectedNamespace: + return key, true + case namespace == "environment": + return EnvironmentCollection, true + case slices.Contains(oldCollectionNamespaces, namespace): + return namespace + ":" + key, true + } + return "", false +} + +// oldCollection is an old namespace's collection by its name: its comment, or why no item may be contributed to it. +// ok is false for a name that is not an old namespace's. +// +// forItem is whether an item is to be contributed to it: an item may be contributed to a kind received as data, +// which a collector may place as a collection only where its shape says its pieces are text (design 57 L3). +func oldCollection(name string, forItem bool) (comment string, old bool, wrong error) { + namespace, key, _ := strings.Cut(name, ":") + if !slices.Contains(oldCollectionNamespaces, namespace) { + return "", false, nil + } + switch namespace { + case "shell": + target, slot, two := strings.Cut(key, ":") + if !two || !oneOf(contributionTargets(), target) || !oneOf(knownSlots, slot) { + return "", true, fmt.Errorf("%s is no slot: shell code is shell::, the shell one of %s, "+ + "the session's file one of %s or the power moment one of %s, and the slot one of %s", name, + strings.Join(knownShells, ", "), strings.Join(sessionFiles, ", "), strings.Join(powerMoments, ", "), + strings.Join(knownSlots, ", ")) + } + // shellCode's line before each module's code. + return "#", true, nil + case "environment": + if key == "" { + // The collection itself, which an item is contributed to; a placeholder names a rendering. + return "#", true, nil + } + if key != EnvironmentPOSIX && key != EnvironmentSystemd { + return "", true, fmt.Errorf("%s is no format: the environment is environment:%s or environment:%s", + name, EnvironmentPOSIX, EnvironmentSystemd) + } + return "#", true, nil + } + seat, kind, _ := strings.Cut(key, ":") + s, r, ok := receivable(seat, kind) + switch { + case !ok && s.Name == "": + return "", true, fmt.Errorf("%s names the seat %q, which the mesh does not define", name, seat) + case !ok: + return "", true, fmt.Errorf("%s: %s; %s", name, s.Name, kindsOf(s)) + case r.Shape != nil && (forItem || collectableAsText(r)): + return r.Comment, true, nil + case r.Shape != nil: + return "", true, fmt.Errorf("%s is received by %s as data in its own shape, which no item of text is "+ + "(novox/hq ADR 0255)", name, s.Name) + case r.Dirs: + return "", true, fmt.Errorf("%s names its contributors' directories, which the holder reads as paths "+ + "the controller fills; it is not yet a collection (design 57 L1)", name) + case s.Name != canonicalSeat(seat): + return "", true, fmt.Errorf("%s names the seat by another name; a collection names it %s", name, s.Name) + } + return r.Comment, true, nil +} + +// placerOfOld is the seat whose holder alone may collect an old namespace's collection (ADR 0203, 0204, 0208, +// 0211, 0212): the same seat that places it the old way. +func placerOfOld(name string) string { + namespace, key, _ := strings.Cut(name, ":") + switch namespace { + case "shell": + target, _, _ := strings.Cut(key, ":") + return placerOf(target) + case "environment": + return EnvironmentSeat + } + seat, _, _ := strings.Cut(key, ":") + return canonicalSeat(seat) +} + +// collectionProblems is what is wrong with a module's collections and items, from the manifest alone. +func collectionProblems(m Manifest) []string { + var problems []string + declared := map[string]bool{} + for i, c := range m.Collections { + switch { + case !collectionName.MatchString(c.Name): + problems = append(problems, fmt.Sprintf("%s's collection %d is %q, which is not a collection's name: "+ + "words of lower-case letters, digits and hyphens, joined by colons", m.Module, i+1, c.Name)) + case declared[c.Name]: + problems = append(problems, fmt.Sprintf("%s collects %s twice", m.Module, c.Name)) + } + if _, old, _ := oldCollection(c.Name, false); old { + problems = append(problems, fmt.Sprintf("%s collects %s, which is read from its placeholder: a module "+ + "places it as ${%s} where it holds the seat that places it, and declares nothing", m.Module, c.Name, + c.Name)) + } + if strings.ContainsAny(c.Comment, "\n\r") { + problems = append(problems, fmt.Sprintf("%s's collection %s has a comment holding a line break", + m.Module, c.Name)) + } + declared[c.Name] = true + } + // A collection declared and placed nowhere is a word read by nothing; judged where every file's text is here. + if !m.Templated() { + named := map[string]bool{} + for _, r := range m.Resources { + if text, ok := r["content"].(string); ok { + for _, f := range ofCollected.FindAllStringSubmatch(text, -1) { + base, _ := splitSelector(f[1]) + named[base] = true + } + } + } + for _, c := range m.Collections { + if !named[c.Name] && collectionName.MatchString(c.Name) { + problems = append(problems, fmt.Sprintf("%s collects %s and no file of its places it as "+ + "${collected:%s}", m.Module, c.Name, c.Name)) + } + } + } + contributed := map[string]bool{} + for i, it := range m.Items { + where := fmt.Sprintf("%s's item %d", m.Module, i+1) + if !collectionName.MatchString(it.Collection) { + problems = append(problems, fmt.Sprintf("%s is to %q, which is not a collection's name", where, + it.Collection)) + } else if _, old, wrong := oldCollection(it.Collection, true); old && wrong != nil { + problems = append(problems, fmt.Sprintf("%s: %v", where, wrong)) + } + switch { + case it.Name != "" && !itemName.MatchString(it.Name): + problems = append(problems, fmt.Sprintf("%s: the name %q is not a name: lower-case letters, digits and "+ + "hyphens", where, it.Name)) + case contributed[it.key()]: + problems = append(problems, fmt.Sprintf("%s contributes to %s twice; a module contributes one item to "+ + "a collection, or several each with a name of its own", m.Module, it.key())) + case it.Name == "" && slices.ContainsFunc(m.Items, func(o Item) bool { + return o.Collection == it.Collection && o.Name != "" + }): + problems = append(problems, fmt.Sprintf("%s contributes several items to %s and %s has no name; each "+ + "of several items has a name of its own", m.Module, it.Collection, where)) + } + contributed[it.key()] = true + problems = append(problems, itemAttributeProblems(m.Module, where, it)...) + if it.Collection == EnvironmentCollection { + problems = append(problems, environmentItemProblems(where, it)...) + } + switch { + case it.Template == "" && it.Content == "": + problems = append(problems, fmt.Sprintf("%s names no template: an item is a template of the module's "+ + "own, \"template\": \"%s/\" (novox/hq ADR 0349 §13)", where, m.TemplatesDir())) + case it.Template != "" && it.Content != "": + problems = append(problems, fmt.Sprintf("%s names its template and says its content too", where)) + case it.Template != "": + if wrong := itemTemplateProblem(m, it.Template); wrong != "" { + problems = append(problems, fmt.Sprintf("%s: %s", where, wrong)) + } + // **Filled by the provisioner's client alone**: the controller renders no item. + if !m.Moved { + problems = append(problems, fmt.Sprintf("%s: an item is filled by the provisioner's client of the "+ + "node its contributor is placed on, so a module contributing one is marked moved "+ + "(\"moved\": true; novox/hq ADR 0349 §13, §17)", where)) + } + } + if it.Content != "" { + problems = append(problems, itemTextProblems(m.Module, it)...) + } + } + return problems +} + +// itemAttributeProblems is what is wrong with an item's attributes and capability. The attributes of an item of a kind +// received as data are the fields its shape places by (and `order`), each with a value the shape takes. +func itemAttributeProblems(module, where string, it Item) []string { + var problems []string + for _, k := range sortedKeys(it.Attributes) { + v := it.Attributes[k] + switch { + case !attributeKey.MatchString(k): + problems = append(problems, fmt.Sprintf("%s: the attribute %q is not a name: lower-case letters, digits "+ + "and hyphens", where, k)) + continue + case !attributeFits(it.Collection, k, v): + problems = append(problems, fmt.Sprintf("%s: %s=%q is not a value: lower-case letters, digits, hyphens, "+ + "dots and underscores (an environment variable's name, letters, digits and underscores)", where, k, v)) + continue + } + if n, err := strconv.Atoi(v); k == OrderAttribute && (err != nil || n < 0 || n > 99) { + problems = append(problems, fmt.Sprintf("%s: order=%q is not a whole number from 0 to 99", where, v)) + } + } + if namespace, key, _ := strings.Cut(it.Collection, ":"); namespace == "contribution" { + seat, kind, _ := strings.Cut(key, ":") + if _, r, ok := receivable(seat, kind); ok && r.Shape != nil { + for _, k := range sortedKeys(it.Attributes) { + var place *PlaceField + for i := range r.Shape.Places { + if r.Shape.Places[i].Field == k { + place = &r.Shape.Places[i] + } + } + switch { + case k == OrderAttribute: + case place == nil: + problems = append(problems, fmt.Sprintf("%s: %s is placed by %s, and %q is none of them", where, + it.Collection, placeFields(r), k)) + case !oneOf(place.Values, it.Attributes[k]): + problems = append(problems, fmt.Sprintf("%s: %s=%s; %s is one of %s", where, k, it.Attributes[k], k, + strings.Join(place.Values, ", "))) + } + } + } + } + switch { + case it.IfCapability == "": + case !capabilityName.MatchString(it.IfCapability): + problems = append(problems, fmt.Sprintf("%s: if-capability %q is not a capability's name", where, it.IfCapability)) + case !oneOf(KnownCapabilities, it.IfCapability): + problems = append(problems, fmt.Sprintf("%s: if-capability %q is one no machine reports; the node-engine "+ + "detects %s", where, it.IfCapability, strings.Join(KnownCapabilities, ", "))) + } + return problems +} + +// AttributeFits is whether an item's attribute is of the shape its collection allows: words of lower case, except +// the environment's `name`, which is a variable's name (design 57 L3, issue 795). The same rule is mesh-catalog's +// provisioner client's, and the inventory's. +func attributeFits(collection, k, v string) bool { + if collection == EnvironmentCollection && k == "name" { + return variableName.MatchString(v) + } + return attributeKey.MatchString(k) && attributeValue.MatchString(v) +} + +// environmentItemProblems is what is wrong with an item of the environment collection: typed by `kind`, a variable +// with its `name`, a PATH entry `at` the start or the end, and no attribute besides `order`. +func environmentItemProblems(where string, it Item) []string { + var problems []string + for _, k := range sortedKeys(it.Attributes) { + if !slices.Contains([]string{"kind", "name", "at", OrderAttribute}, k) { + problems = append(problems, fmt.Sprintf("%s: %s is typed by kind, name and at, and %q is none of them", + where, it.Collection, k)) + } + } + switch kind := it.Attributes["kind"]; kind { + case "variable": + if it.Attributes["name"] == "" || it.Attributes["at"] != "" { + problems = append(problems, fmt.Sprintf("%s: a variable names itself with name= and has no at", where)) + } else if it.Attributes["name"] == "PATH" { + problems = append(problems, fmt.Sprintf("%s: PATH is not a variable; a PATH entry is kind=path", where)) + } + case "path": + if it.Attributes["name"] != "" || (it.Attributes["at"] != PathAtStart && it.Attributes["at"] != PathAtEnd) { + problems = append(problems, fmt.Sprintf("%s: a PATH entry is at=%s or at=%s and has no name", where, + PathAtStart, PathAtEnd)) + } + default: + problems = append(problems, fmt.Sprintf("%s: an item of %s is kind=variable or kind=path, not %q", where, + it.Collection, kind)) + } + return problems +} + +// itemTemplateProblem is what is wrong with an item's template path, or "". +func itemTemplateProblem(m Manifest, name string) string { + dir := m.TemplatesDir() + if strings.HasPrefix(name, "/") || path.Clean(name) != name || !strings.HasPrefix(name, dir+"/") || + strings.Contains(name, "\\") { + return fmt.Sprintf("the template %q is not a file under %s/ of its own repository: a template is %q, a "+ + "plain relative path, and the build packs only %s/", name, dir, dir+"/", dir) + } + return "" +} + +// itemTextProblems is what an item's text may not name: a secret or a bound value, since an item is kept in a +// bucket and no secret is (ADR 0349 clause 7), and a collection, since an item is not a collector. And every +// placeholder no pass fills, as in a file's content. +func itemTextProblems(module string, it Item) []string { + var problems []string + for _, f := range templatePlaceholder.FindAllStringSubmatch(it.Content, -1) { + namespace, key := f[1], f[2] + switch _, collection := collectionOfPlaceholder(namespace, key); { + case namespace == "secret" || namespace == "bound": + problems = append(problems, fmt.Sprintf("%s's item of %s names %s: an item is kept in a bucket, where "+ + "no secret is (novox/hq ADR 0349 §7)", module, it.Collection, f[0])) + case collection: + problems = append(problems, fmt.Sprintf("%s's item of %s names %s: an item is not a collector", + module, it.Collection, f[0])) + } + } + return append(problems, unconsumedPlaceholders(module, map[string]any{"id": "item:" + it.key(), + "type": "file", "content": it.Content})...) +} + +// isOld is whether a collection's name is an old namespace's. +func isOld(name string) bool { + _, old, _ := oldCollection(name, true) + return old +} + +// collectedPlaceholderProblems is what is wrong with one resource's ${collected:…}: placed only in a file's content, +// and naming a collection the module declares it collects. +func collectedPlaceholderProblems(m Manifest, r map[string]any) []string { + var problems []string + for _, field := range sortedKeys(r) { + v, ok := r[field].(string) + if !ok { + continue + } + for _, f := range ofCollected.FindAllStringSubmatch(v, -1) { + base, selector := splitSelector(f[1]) + if _, err := selectorOf(selector); err != nil { + problems = append(problems, fmt.Sprintf("%s's resource %v names %s: %v", m.Module, r["id"], f[0], err)) + continue + } + switch { + case field != "content": + problems = append(problems, fmt.Sprintf("%s's resource %v names %s in its %s; a collection is placed "+ + "only in a file's content", m.Module, r["id"], f[0], field)) + case isOld(base): + if _, _, wrong := oldCollection(base, false); wrong != nil { + problems = append(problems, fmt.Sprintf("%s's resource %v names %s: %v", m.Module, r["id"], f[0], wrong)) + } + case !slices.ContainsFunc(m.Collections, func(c Collection) bool { return c.Name == base }): + problems = append(problems, fmt.Sprintf("%s's resource %v names %s, and %s does not collect %q: "+ + "declare it under collections (novox/hq ADR 0349 §13)", m.Module, r["id"], f[0], m.Module, base)) + } + } + } + return problems +} + +// collectionComment is the comment of a collection a module's file names: declared, or the old namespace's. +func collectionComment(m Manifest, name string) (string, error) { + if comment, old, wrong := oldCollection(name, false); old { + return comment, wrong + } + for _, c := range m.Collections { + if c.Name == name { + return c.Comment, nil + } + } + return "", fmt.Errorf("%s names ${collected:%s} and does not collect it", m.Module, name) +} + +// itemsLeftOut says each item of a collection a file is filled with the old way that the controller could not render +// there (heldItemText): the collector is not handed to the provisioner on this machine (ADR 0349 clause 17). Said, +// never dropped quietly. +func itemsLeftOut(modules []Manifest, holder Manifest, collection string, facts map[string]string) []string { + return itemsLeftOutFor(modules, holder, collection, nil, facts, nil) +} + +// itemsLeftOutFor is itemsLeftOut for a placeholder that selects by attributes (where), on a machine with these +// capabilities: an item the selector does not take, or whose capability the machine lacks, is not left out but not +// wanted here. +func itemsLeftOutFor(modules []Manifest, holder Manifest, collection string, where map[string]string, + facts map[string]string, caps map[string]bool) []string { + var out []string + for _, m := range inModuleOrder(modules) { + for _, it := range itemsInOrder(m.Items) { + if it.Collection != collection || !it.matches(where) || (it.IfCapability != "" && !caps[it.IfCapability]) { + continue + } + if _, rendered := heldText(it, facts); rendered && renderedOldWay(collection) { + continue + } + out = append(out, fmt.Sprintf("%s's item of %s is not placed: %s is filled the old way on this "+ + "machine, from contributions, and an item only by the provisioner's client (design 57 L1)", + m.Module, it.key(), holder.Module)) + } + } + return out +} + +// oldFormsLeftOut says each piece of an old form a handed collector's collections would have held: the collector is +// filled from items alone, and the piece's module contributes none (ADR 0349 clause 17). +func oldFormsLeftOut(modules []Manifest, holder Manifest, collections []PlanCollection) []string { + var out []string + for _, c := range collections { + namespace, key, _ := strings.Cut(c.Name, ":") + for _, m := range inModuleOrder(modules) { + old := false + switch namespace { + case "shell": + target, slot, _ := strings.Cut(key, ":") + old = slices.ContainsFunc(m.Shell, func(s ShellCode) bool { return s.For == target && s.Slot == slot }) + case "environment": + old = m.Environment != nil && (len(m.Environment.Variables) > 0 || len(m.Environment.Path) > 0) + case "contribution": + seat, kind, _ := strings.Cut(key, ":") + old = slices.ContainsFunc(m.allContributions(), func(s SeatContribution) bool { + return canonicalSeat(s.Seat) == canonicalSeat(seat) && s.Kind == kind + }) + } + if old { + out = append(out, fmt.Sprintf("%s's %s is not placed: %s is filled by the provisioner's client on "+ + "this machine, from items, and %s contributes it the old way (design 57 L1)", m.Module, c.Name, + holder.Module, m.Module)) + } + } + } + return out +} + +// heldItemText is a moved contributor's items of an old namespace's collection as the controller renders them into a +// collector not handed on this machine (design 57 L2, issue 681): the text the mesh holds, read back from the +// contributor's templates archive, with the machine's facts filled, each ending its line. As a moved module's file +// keeps its text for a machine that cannot take it handed (handed.go), an item keeps its place in a collector filled +// the old way, so the file stays what its old form made it. Only `shell:` and `contribution:` collections, which the +// controller renders as text; false for a module with no such item, or an item whose text the mesh does not hold or +// which names anything but a machine fact this machine has, which is then said as not placed. +func heldItemText(m Manifest, collection string, facts map[string]string) (string, bool) { + items := heldItems(m, collection, nil, facts, nil) + var b strings.Builder + for _, h := range items { + b.WriteString(h.text) + } + return b.String(), len(items) > 0 +} + +// renderedOldWay is whether the controller renders a collection's items into a collector not handed. +func renderedOldWay(collection string) bool { + namespace, _, _ := strings.Cut(collection, ":") + return namespace == "shell" || namespace == "contribution" || namespace == EnvironmentCollection +} + +// heldItem is one item of a module rendered the old way. +type heldItem struct { + item Item + text string +} + +// heldItems is a module's items of the collection that the selector takes and the machine's capabilities allow, by +// order then name, each rendered as heldText does; the ones it cannot render are left out (itemsLeftOutFor says so). +func heldItems(m Manifest, collection string, where map[string]string, facts map[string]string, + caps map[string]bool) []heldItem { + if !renderedOldWay(collection) { + return nil + } + var out []heldItem + for _, it := range itemsInOrder(m.Items) { + if it.Collection != collection || !it.matches(where) || (it.IfCapability != "" && !caps[it.IfCapability]) { + continue + } + if text, ok := heldText(it, facts); ok { + out = append(out, heldItem{item: it, text: text}) + } + } + return out +} + +// heldText is an item's text with the machine's facts filled, ending its line; false for an item whose text the mesh +// does not hold or which names anything but a machine fact this machine has. +func heldText(it Item, facts map[string]string) (string, bool) { + if it.Content == "" { + return "", false + } + text := it.Content + for _, f := range templatePlaceholder.FindAllStringSubmatch(text, -1) { + value, has := facts[f[2]] + if f[1] != "machine" || !has { + return "", false + } + text = strings.ReplaceAll(text, f[0], value) + } + if !strings.HasSuffix(text, "\n") { + text += "\n" + } + return text, true +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/config.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/config.go new file mode 100644 index 00000000..0a54370b --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/config.go @@ -0,0 +1,504 @@ +package catalogue + +import ( + "encoding/json" + "fmt" + "regexp" + "slices" + "strings" +) + +// A module declares its config (novox/hq ADR 0348 §1, §4, §5, §6, §8, §11; design 57 package K1, issue 660). +// +// **One concept, two spellings for a while.** "Config" replaces "settings": the manifest says `config` for +// `settings`, `takes-config` for `takes-settings` and `${config:}` for `${setting:}`. The old names stay +// accepted as aliases until every module has moved (ADR 0348 §1), and a module uses one form or the other, never +// both. The config form is read into the very fields the settings form fills — Settings, a mergeable file's +// takes-settings and `${setting:}` — so composition has one path and `${config:}` fills exactly as +// `${setting:}` does: a module rewritten from settings to config composes byte-identically. It is written +// back in the form it was read in, so what the mesh holds is what its author wrote. +// +// **A moved module is held to the whole declaration.** Under `config` every value has a type (number, string or +// boolean; an object or a list only when trusted), a description including its unit, its validation (enum, +// minimum and maximum, pattern), whether it is trusted, optionally the predefined key it falls back to, and either +// a default or `required`; it lands in a file or is read at runtime (`runtime: true`); every mergeable file names +// the keys it takes; a `reload-on` or `restart-on` names a file the module declares. A module not yet moved keeps +// today's form and today's rules (SettingProblems, TakenSettingsProblems) until the catalogue's list of such +// modules is empty (ADR 0348 §11). +// +// **A required value nothing sets** is the operator's own: it composes as an operator's setting nobody gave does +// today, refused by name at composition (UnsetSettingError). Who reports it once mesh-config answers values is the +// provisioner's (ADR 0349), not this file's. + +// ConfigField is the manifest key a module declares its config values under. +const ConfigField = "config" + +// TakesConfigField is the key a mergeable file names the config values it takes by. +const TakesConfigField = "takes-config" + +// SettingsField is the old name of ConfigField, read as its alias. +const SettingsField = "settings" + +// configRef is how a file asks for a config value: ${config:}, the same keys as settingRef. +var configRef = regexp.MustCompile(`\$\{config:([a-z0-9][a-z0-9_.-]*)\}`) + +// takesConfigKey and takesSettingsKey are the two spellings as a key of a JSON object. A string value is never +// followed by a colon, and a quote inside a string is escaped, so these match only keys. +var ( + takesConfigKey = regexp.MustCompile(`"` + TakesConfigField + `"(\s*):`) + takesSettingsKey = regexp.MustCompile(`"` + TakesSettingsField + `"(\s*):`) +) + +// The types a config value may have (ADR 0348 §4). +const ( + ConfigNumber = "number" + ConfigString = "string" + ConfigBoolean = "boolean" + ConfigObject = "object" + ConfigList = "list" +) + +// accountFacts are what the mesh places for a module's account, never config values (ADR 0348 §11). +var accountFacts = map[string]bool{"puid": true, "pgid": true} + +// configForms is which of the two forms a manifest was written in, by the words of each it uses. +type configForms struct { + config, settings []string +} + +// DeclaresConfig is whether the manifest is written in the config form: `config`, `takes-config` or +// `${config:}` (ADR 0348 §1, §11). +func (m Manifest) DeclaresConfig() bool { return len(m.forms.config) > 0 } + +// toSettingsForm is raw with the config form's placeholders and takes-config read as the settings form's, and +// which words of each form it used. `config` itself is a top-level key, renamed where the keys are read +// (UnmarshalJSON). +func toSettingsForm(raw []byte) ([]byte, configForms) { + var f configForms + if configRef.Match(raw) { + f.config = append(f.config, "${config:…}") + } + if takesConfigKey.Match(raw) { + f.config = append(f.config, TakesConfigField) + } + if settingRef.Match(raw) { + f.settings = append(f.settings, "${setting:…}") + } + if takesSettingsKey.Match(raw) { + f.settings = append(f.settings, TakesSettingsField) + } + if len(f.config) == 0 { + return raw, f + } + raw = configRef.ReplaceAllFunc(raw, func(ref []byte) []byte { + return []byte("${setting:" + string(configRef.FindSubmatch(ref)[1]) + "}") + }) + raw = takesConfigKey.ReplaceAllFunc(raw, func(key []byte) []byte { + return []byte(`"` + TakesSettingsField + `"` + string(takesConfigKey.FindSubmatch(key)[1]) + ":") + }) + return raw, f +} + +// toConfigForm is a manifest written in the settings form's words, as JSON, in the config form's: what +// toSettingsForm and UnmarshalJSON read, written back. +func toConfigForm(raw []byte) ([]byte, error) { + var keys map[string]json.RawMessage + if err := json.Unmarshal(raw, &keys); err != nil { + return nil, err + } + if settings, has := keys[SettingsField]; has { + keys[ConfigField] = settings + delete(keys, SettingsField) + var err error + if raw, err = json.Marshal(keys); err != nil { + return nil, err + } + } + raw = settingRef.ReplaceAllFunc(raw, func(ref []byte) []byte { + return []byte("${config:" + string(settingRef.FindSubmatch(ref)[1]) + "}") + }) + raw = takesSettingsKey.ReplaceAllFunc(raw, func(key []byte) []byte { + return []byte(`"` + TakesConfigField + `"` + string(takesSettingsKey.FindSubmatch(key)[1]) + ":") + }) + return raw, nil +} + +// inConfigWords is a refusal said of a module in the config form in its words: `${config:}` and +// takes-config, where the rules shared with the settings form say `${setting:}` and takes-settings. +func inConfigWords(problem string) string { + problem = settingRef.ReplaceAllStringFunc(problem, func(ref string) string { + return "${config:" + settingRef.FindStringSubmatch(ref)[1] + "}" + }) + return strings.ReplaceAll(problem, TakesSettingsField, TakesConfigField) +} + +// MixedFormProblems is a manifest using both forms, said with the words of each (ADR 0348 §1). +func MixedFormProblems(m Manifest) []string { + if len(m.forms.config) == 0 || len(m.forms.settings) == 0 { + return nil + } + return []string{fmt.Sprintf("%s mixes the config form (%s) with the settings form (%s): a module uses one form "+ + "or the other, never both; write %s, %s and ${config:} throughout (novox/hq ADR 0348 §1)", + m.Module, strings.Join(m.forms.config, ", "), strings.Join(m.forms.settings, ", "), + ConfigField, TakesConfigField)} +} + +// configDeclaration is the config form's declaration as written: nothing of the settings form's, and nothing +// unsaid. +type configDeclaration struct { + Type string `json:"type,omitempty"` + Description string `json:"description,omitempty"` + Enum []any `json:"enum,omitempty"` + Minimum *float64 `json:"minimum,omitempty"` + Maximum *float64 `json:"maximum,omitempty"` + Pattern string `json:"pattern,omitempty"` + Trusted bool `json:"trusted,omitempty"` + Predefined string `json:"predefined,omitempty"` + Default any `json:"default,omitempty"` + Required bool `json:"required,omitempty"` + Runtime bool `json:"runtime,omitempty"` +} + +// MarshalJSON writes a settings declaration exactly as before, and a config declaration without the settings +// form's kind and why, and without a default it does not have. +func (d SettingDeclaration) MarshalJSON() ([]byte, error) { + type plain SettingDeclaration + if d.Kind != "" || d.Why != "" { + return json.Marshal(plain(d)) + } + return json.Marshal(configDeclaration{Type: d.Type, Description: d.Description, Enum: d.Enum, + Minimum: d.Minimum, Maximum: d.Maximum, Pattern: d.Pattern, Trusted: d.Trusted, Predefined: d.Predefined, + Default: d.Default, Required: d.Required, Runtime: d.Runtime}) +} + +// configShaped is whether a declaration says anything of the config form's. +func configShaped(d SettingDeclaration) bool { + return d.Type != "" || d.Description != "" || len(d.Enum) > 0 || d.Minimum != nil || d.Maximum != nil || + d.Pattern != "" || d.Trusted || d.Predefined != "" || d.Required || d.Runtime +} + +// TypedSettingProblems is a declaration under `settings` that says what only `config` says: a value with a type, +// a description and its validation is declared in the config form (ADR 0348 §1, §4). +func TypedSettingProblems(m Manifest) []string { + var problems []string + for _, key := range sortedSettingKeys(m.Settings) { + if configShaped(m.Settings[key]) { + problems = append(problems, fmt.Sprintf("%s's setting %q says its type, description or validation "+ + "under %s: a value declared with those is declared under config, with ${config:} and "+ + "takes-config, never under %s (novox/hq ADR 0348 §1, §4)", m.Module, key, SettingsField, SettingsField)) + } + } + return problems +} + +// ConfigProblems is every way a module in the config form declares its config wrongly, each said by name with +// what to do (ADR 0348 How it is checked, design 57 package K1). +func ConfigProblems(m Manifest) []string { + if !m.DeclaresConfig() { + return nil + } + var problems []string + placed := configPlaced(m) + fallsBack := fileFallsBack(m) + renamed := renamedTaken(m) + for _, key := range sortedSettingKeys(m.Settings) { + d := m.Settings[key] + say := func(format string, args ...any) { + problems = append(problems, fmt.Sprintf("%s's config value %q: ", m.Module, key)+fmt.Sprintf(format, args...)) + } + if !settingRef.MatchString("${setting:" + key + "}") { + say("not a usable key: lower-case letters, digits, dots, dashes and underscores") + continue + } + if meshWords[key] || accountFacts[key] { + say("%q is placement, which the mesh decides, never a config value (novox/hq ADR 0348 §11)", key) + continue + } + if d.Kind != "" || d.Why != "" { + say("kind and why are the settings form's; under config a value says its type and its description " + + "(novox/hq ADR 0348 §1, §4)") + } + switch d.Type { + case ConfigNumber, ConfigString, ConfigBoolean: + case ConfigObject, ConfigList: + if !d.Trusted { + say("is %s and not trusted: an object or a list decides more than one value can say, so it is "+ + "trusted, \"trusted\": true, or split into numbers, strings and booleans (novox/hq ADR 0348 §4)", d.Type) + } + case "": + say("no type: say \"type\": number, string or boolean, or object or list when trusted (novox/hq ADR 0348 §4)") + default: + say("type %q is none of number, string, boolean, object and list (novox/hq ADR 0348 §4)", d.Type) + } + if strings.TrimSpace(d.Description) == "" { + say("no description: say in a sentence what the value is, including its unit (novox/hq ADR 0348 §4)") + } + switch { + case d.Required && d.Default != nil: + say("both a default and required: a required value is the operator's own and has no default; give " + + "one or the other (novox/hq ADR 0348 §4)") + case !d.Required && d.Default == nil: + say("neither a default nor required: give a default, or say \"required\": true for a value that is " + + "the operator's own (novox/hq ADR 0348 §4)") + } + if as := renamed[key]; as != "" && d.Required { + say("is laid into its file as %q, and only when it is set, so the file and its code fall back when "+ + "it is not: it is never required; give it the file's default, the empty value of its type "+ + "(novox/hq issue 751)", as) + } + for _, wrong := range validationProblems(d) { + say("%s", wrong) + } + if word := operatorsWord(key); word != "" && d.Default != nil && !fallsBack[key] { + say("a key naming %q is the operator's value: declare it \"required\": true, with no default, "+ + "unless a mergeable file takes it and no placeholder asks for it, where the file falls back "+ + "(novox/hq ADR 0348 §4, ADR 0112, issue 735)", word) + } + if d.Predefined != "" { + p, listed := m.predefined[d.Predefined] + switch { + case m.predefined == nil: + say("falls back to the predefined key %q, and the list of predefined keys (%s at the root of "+ + "the catalogue's repository) is not present where it is judged: a value falls back only to a "+ + "key on that list; judge it in the catalogue, or leave \"predefined\" out (novox/hq ADR 0348 §7)", + d.Predefined, PredefinedFile) + case !listed: + say("falls back to %q, which is not on the list of predefined keys: name a key on it, or leave "+ + "\"predefined\" out (novox/hq ADR 0348 §7)", d.Predefined) + default: + if p.Type != d.Type { + say("falls back to the predefined key %q, which is a %s, and this value is a %s: a value falls "+ + "back only to a key of its own type (novox/hq ADR 0348 §4)", d.Predefined, p.Type, d.Type) + } + if p.Trusted && !d.Trusted { + say("falls back to the trusted predefined key %q and is not trusted itself: a value that falls "+ + "back to a trusted key is trusted, \"trusted\": true (novox/hq ADR 0348 §8)", d.Predefined) + } + } + } + // A template's text is not here yet. Resolve fills it from the build, WithTemplatesFrom from the tree; neither + // judges again itself, but WithTemplatesFrom parses its result again, and a resolved build is parsed again + // where it is registered, and this is judged then. + if !placed[key] && !d.Runtime && !m.Templated() { + say("placed nowhere: no file says ${config:%s} or takes it under %s, and it is not marked "+ + "\"runtime\": true for a value the module's code reads itself (novox/hq ADR 0348 §5)", key, TakesConfigField) + } + } + used := settingKeysUsedBy(m) + for _, key := range sortedKeys(used) { + if _, declared := m.Settings[key]; !declared { + problems = append(problems, fmt.Sprintf("%s says ${config:%s} and declares no such value under %s: "+ + "declare %q with its type, description and a default or required (novox/hq ADR 0348 §4, §5)", + m.Module, key, ConfigField, key)) + } + } + problems = append(problems, configTakesProblems(m)...) + problems = append(problems, reflectsProblems(m)...) + return problems +} + +// configPlaced is every value a file, a unit, a directory's owner, a contribution or a served fact asks for, and +// every value a mergeable file takes. +func configPlaced(m Manifest) map[string]bool { + placed := settingKeysUsedBy(m) + for _, r := range m.Resources { + for _, k := range takenKeyList(r) { + placed[k] = true + } + } + return placed +} + +// fileFallsBack is every value a mergeable file takes and no placeholder asks for. Unset, such a value is simply +// not laid over the file: the file's own content, or the module's code reading the file, falls back. So one named +// after the operator's value (a url, a user, an address) is not `required`, and its default is the file's: its +// value in the content, else the empty value of its type, never laid over the file (the operator's decision on +// issue 664, issue 735). A placeholder has no such fallback: its default would be filled in where the operator's +// value belongs (ADR 0112). +func fileFallsBack(m Manifest) map[string]bool { + asked := settingKeysUsedBy(m) + out := map[string]bool{} + for _, r := range m.Resources { + if how, _ := r["merge"].(string); how == "" { + continue + } + for _, k := range takenKeyList(r) { + if !asked[k] { + out[k] = true + } + } + } + return out +} + +// renamedTaken is every value a mergeable file takes under a name of its own, with that name (issue 751). +func renamedTaken(m Manifest) map[string]string { + out := map[string]string{} + for _, r := range m.Resources { + if how, _ := r["merge"].(string); how == "" { + continue + } + for _, e := range takenEntries(r[TakesSettingsField]) { + if e.renamed { + out[e.key] = e.as + } + } + } + return out +} + +// validationProblems is what is wrong with a declaration's validation, and its default against it. +func validationProblems(d SettingDeclaration) []string { + var problems []string + isNumber := d.Type == ConfigNumber + if len(d.Enum) > 0 && d.Type != ConfigNumber && d.Type != ConfigString { + problems = append(problems, fmt.Sprintf("an enum is for a number or a string, and this is a %s", d.Type)) + } + if (d.Minimum != nil || d.Maximum != nil) && !isNumber { + problems = append(problems, fmt.Sprintf("a minimum or maximum is for a number, and this is a %s", d.Type)) + } + if d.Minimum != nil && d.Maximum != nil && *d.Minimum > *d.Maximum { + problems = append(problems, fmt.Sprintf("its minimum %v is above its maximum %v", *d.Minimum, *d.Maximum)) + } + var pattern *regexp.Regexp + if d.Pattern != "" { + if d.Type != ConfigString { + problems = append(problems, fmt.Sprintf("a pattern is for a string, and this is a %s", d.Type)) + } + var err error + if pattern, err = regexp.Compile(`^(?:` + d.Pattern + `)$`); err != nil { + problems = append(problems, fmt.Sprintf("its pattern %q is not a regular expression: %v", d.Pattern, err)) + } + } + for _, v := range d.Enum { + if !ofType(v, d.Type) { + problems = append(problems, fmt.Sprintf("its enum holds %v, which is not a %s", v, d.Type)) + } + } + if d.Default == nil { + return problems + } + if !ofType(d.Default, d.Type) { + if d.Type != "" { + problems = append(problems, fmt.Sprintf("its default %v is not a %s", d.Default, d.Type)) + } + return problems + } + if len(d.Enum) > 0 && !slices.ContainsFunc(d.Enum, func(v any) bool { return v == d.Default }) { + problems = append(problems, fmt.Sprintf("its default %v is not one of %v", d.Default, d.Enum)) + } + if n, ok := d.Default.(float64); ok { + if d.Minimum != nil && n < *d.Minimum { + problems = append(problems, fmt.Sprintf("its default %v is below its minimum %v", n, *d.Minimum)) + } + if d.Maximum != nil && n > *d.Maximum { + problems = append(problems, fmt.Sprintf("its default %v is above its maximum %v", n, *d.Maximum)) + } + } + if s, ok := d.Default.(string); ok && pattern != nil && !pattern.MatchString(s) { + problems = append(problems, fmt.Sprintf("its default %q does not match its pattern %q", s, d.Pattern)) + } + return problems +} + +// ofType is whether a value read from JSON is of a config type. +func ofType(v any, kind string) bool { + switch v.(type) { + case float64: + return kind == ConfigNumber + case string: + return kind == ConfigString + case bool: + return kind == ConfigBoolean + case map[string]any: + return kind == ConfigObject + case []any: + return kind == ConfigList + } + return false +} + +// configTakesProblems is what ADR 0348 §11 adds for a module in the config form: an account fact in +// takes-config (the mesh's own words are refused for both forms by TakenSettingsProblems), and a mergeable file +// naming no keys where none of the module's does (where one does, TakenSettingsProblems names every one that +// does not). +func configTakesProblems(m Manifest) []string { + var problems, silent []string + naming := false + for _, r := range m.Resources { + if how, _ := r["merge"].(string); how == "" { + continue + } + if _, said := r[TakesSettingsField]; !said { + silent = append(silent, fmt.Sprint(r["id"])) + continue + } + naming = true + for _, k := range takenKeyList(r) { + _, declared := m.Settings[k] + switch { + case accountFacts[k]: + problems = append(problems, fmt.Sprintf("%s's %v names %q in %s, and %q is an account fact the "+ + "mesh places, never a config value (novox/hq ADR 0348 §11)", m.Module, r["id"], k, + TakesConfigField, k)) + case meshWords[k]: + // Refused for both forms by TakenSettingsProblems, in its words. + case !declared: + problems = append(problems, fmt.Sprintf("%s's %v names %q in %s and declares no such value under "+ + "%s: a value a file takes is declared like every other, with its type, description and a "+ + "default or required, or the config verb could never set it (novox/hq ADR 0348 §4, §11)", + m.Module, r["id"], k, TakesConfigField, ConfigField)) + } + } + } + if !naming { + for _, id := range silent { + problems = append(problems, fmt.Sprintf("%s's mergeable %s names no %s: a module that declares config "+ + "names the keys each mergeable file takes, an empty list for none (novox/hq ADR 0348 §11)", + m.Module, id, TakesConfigField)) + } + } + return problems +} + +// meshMadeIDs are the resources the mesh composes into a module's own set, which its `restart-on` may name +// without declaring them; meshMadePrefixes are those named after what they are for. +var ( + meshMadeIDs = []string{ComposedJailsID(), FilteringID(), CertificateID(), AuthorityID(), BusUsersID()} + meshMadePrefixes = []string{NeedID(""), SecretID(""), strings.TrimSuffix(GrantID("", ""), "-"), BoundID(""), ReceivedID(""), AccessID("/")} +) + +// reflectsProblems is a `reload-on` or `restart-on` naming a file the module does not declare (ADR 0348 §6): not +// one of its own resources, not one the mesh composes for it, and not another module's, which is named +// `.`. +func reflectsProblems(m Manifest) []string { + own := map[string]bool{} + for _, r := range m.Resources { + own[fmt.Sprint(r["id"])] = true + } + meshMade := func(id string) bool { + if slices.Contains(meshMadeIDs, id) { + return true + } + for _, p := range meshMadePrefixes { + if strings.HasPrefix(id, p) && len(id) > len(p) { + return true + } + } + return false + } + var problems []string + for _, r := range m.Resources { + for _, field := range []string{"reload-on", "restart-on"} { + for _, id := range stringsOf(r[field]) { + if strings.Contains(id, ".") || own[id] || meshMade(id) { + continue + } + problems = append(problems, fmt.Sprintf("%s's %v names %q in its %s, and %s declares no such "+ + "file: name a file of its own, or another module's as . (novox/hq ADR 0348 §6)", + m.Module, r["id"], id, field, m.Module)) + } + } + } + return problems +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/config_move.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/config_move.go new file mode 100644 index 00000000..e902b91f --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/config_move.go @@ -0,0 +1,40 @@ +package catalogue + +import "strings" + +// What the move of config reads of a module's declaration (novox/hq ADR 0348 §9, design 57 K5, issue 664): where a +// layer's key goes, how two layers' values become one, and whether a value is one its declaration takes. + +// TakenAs is each name a mergeable file takes a config value under that is not the value's config key, with that +// key (issue 751): a layer set before the module moved may still say the file's name, and the move sets it under +// the config key. +func TakenAs(m Manifest) map[string]string { + out := map[string]string{} + for key, as := range renamedTaken(m) { + out[as] = key + } + return out +} + +// AskedKeys is every value a placeholder of the module asks for: a file, a unit, a directory's owner, a +// contribution or a served fact. Such a value is taken whole from the last layer that sets it, where a mergeable +// file lays one object over another key by key. +func AskedKeys(m Manifest) map[string]bool { return settingKeysUsedBy(m) } + +// LaidOver is a value laid over another as a later layer is laid over an earlier: two objects key by key, anything +// else replaced whole. +func LaidOver(under, over any) any { return mergeValue(under, over) } + +// ValueProblems is what is wrong with a value for a declaration: its type, and its enum, bounds or pattern. +func ValueProblems(d SettingDeclaration, v any) []string { + if v == nil { + return []string{"no value"} + } + d.Default = v + d.Required = false + var problems []string + for _, p := range validationProblems(d) { + problems = append(problems, strings.ReplaceAll(p, "its default", "the value")) + } + return problems +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/consumer_into_serves.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/consumer_into_serves.go new file mode 100644 index 00000000..e5d47916 --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/consumer_into_serves.go @@ -0,0 +1,338 @@ +package catalogue + +import ( + "fmt" + "regexp" + "sort" + "strings" +) + +// What a provider derives for one consumer, said once in the provider's definition and delivered +// to both ends (novox/hq ADR 0202, issue 124). +// +// A `serves` block is otherwise literal: the same values for every consumer. Where the provider +// *names the resource* — a bucket, a database, a vhost — the name is derived from who is asking, +// and before this the mesh had no channel for it. The provider recomputed it in its own code and +// every consumer transcribed it into its own definition by hand, which is a copy of somebody +// else's rule kept in agreement by nobody. One of three transcriptions was wrong for months. +// +// **The mesh learns no protocol here; it spells its own name in an alphabet it already knows.** +// The only fact a served value may name is the identity the mesh itself minted for the consumer, +// in one of two alphabets: as it was minted, and as a DNS label. Everything a provider wants +// around it — a prefix, a suffix, a separator — it writes around the placeholder, because a +// served value is a string. + +// consumerFact is `${consumer:}` or `${consumer::}`. +var consumerFact = regexp.MustCompile(`\$\{consumer:([a-z][a-z0-9-]*)(?::([a-z][a-z0-9-]*))?\}`) + +// consumerFacts are what a served value may name about the consumer it is being derived for. +// One entry, deliberately: the identity is the one thing about a consumer the mesh itself chose, +// so it is the one thing the mesh can hand to a provider without either end guessing. +var consumerFacts = []string{"as"} + +// consumerAlphabets are the ways the mesh will write that identity. `dns` is the mesh's own +// identifier with its separator written `-` instead of `_` — the whole of the difference between +// the alphabet the mesh mints in and the one buckets, vhosts and hostnames accept. +var consumerAlphabets = []string{"dns"} + +// ServedTo fills a provider's served values for one consumer. +// +// `as` is the identity the mesh minted for that consumer — the same string it is told to present +// as a login. Values with no placeholder are returned exactly as they were, and a block with no +// placeholder at all is returned unchanged, so this costs nothing for the providers that derive +// nothing. +// +// Only strings carry placeholders. A number, a boolean or a nested object is a value the provider +// stated outright, and is left alone. +func ServedTo(serves map[string]any, as string) (map[string]any, error) { + if len(serves) == 0 { + return serves, nil + } + var out map[string]any + for _, key := range sortedAnyKeys(serves) { + text, ok := serves[key].(string) + if !ok || !strings.Contains(text, "${consumer:") { + continue + } + filled, err := consumerInto(text, as) + if err != nil { + return nil, fmt.Errorf("the value served as %q: %w", key, err) + } + if out == nil { + // Copied only once something actually changes: the caller's map is the manifest's, + // and a provider that derives nothing must not have it rewritten underneath it. + out = make(map[string]any, len(serves)) + for k, v := range serves { + out[k] = v + } + } + out[key] = filled + } + if out == nil { + return serves, nil + } + return out, nil +} + +// consumerInto replaces every `${consumer:…}` in one value. +// +// **A fact or an alphabet the mesh does not have is refused, not left standing.** Written through, +// the literal `${consumer:as}` would reach a configuration file and be read as a bucket name, +// failing somewhere that names neither the module nor the mesh — the same reasoning `${bound:…}` +// is refused by (boundInto). +func consumerInto(value, as string) (string, error) { + var failed error + out := consumerFact.ReplaceAllStringFunc(value, func(match string) string { + parts := consumerFact.FindStringSubmatch(match) + fact, alphabet := parts[1], parts[2] + if fact != "as" { + if failed == nil { + failed = fmt.Errorf( + "says %s, and the mesh states %s about a consumer", match, orNothing(consumerFacts)) + } + return match + } + switch alphabet { + case "": + return as + case "dns": + return asDNSLabel(as) + default: + if failed == nil { + failed = fmt.Errorf( + "says %s, and the mesh writes an identity as %s", match, orNothing(consumerAlphabets)) + } + return match + } + }) + if failed != nil { + return "", failed + } + return out, nil +} + +// asDNSLabel writes a minted identity as a DNS label. +// +// The mesh's identities are already lower-case letters, digits and `_` (ConsumerIdentity), and +// already inside the bound of the provision serving them: a provider that serves one as a DNS label +// bounds it at 63 or less (CheckServes, novox/hq ADR 0225), and one that serves it at all without +// saying is held to twenty (IdentityBoundOf). So +// this is the separator and nothing else — no lower-casing of what is already lower case, no +// truncation to a limit the identity is already inside, no padding of a name that is already long +// enough. Each of those would be the mesh guessing at a rule it has not been given. +func asDNSLabel(as string) string { + return strings.ReplaceAll(as, "_", "-") +} + +// CheckServes refuses a `serves` block that names a consumer fact or an alphabet the mesh does not +// have, when the definition is parsed rather than when a consumer is resolved. +// +// A provision nobody consumes yet still has its rule read: a definition that would be refused the +// first time somebody required it is a definition that is wrong now. +func CheckServes(m Manifest) []string { + var problems []string + for _, provision := range sortedServes(m.Serves) { + for _, key := range sortedAnyKeys(m.Serves[provision]) { + text, ok := m.Serves[provision][key].(string) + if !ok { + // A provider's own name is filled only in a value of its own (novox/hq ADR 0346): one + // nested in a list or an object would be served as the placeholder, unsaid. + problems = append(problems, nestedOwnNames(m, provision, key, m.Serves[provision][key])...) + continue + } + // A probe identity, because what is checked is the shape of the statement and not + // what any consumer is called. + if _, err := consumerInto(text, "mesh_node_module"); err != nil { + problems = append(problems, fmt.Sprintf( + "%s serves %s, and the value it serves as %q %s", m.Module, provision, key, err)) + } + // And the provider's own names it may serve (novox/hq ADR 0346). + problems = append(problems, checkServedOwnNames(m, provision, key, text)...) + // A label longer than DNS keeps is not truncated here (asDNSLabel), so the offer's own + // bound has to keep the identity inside one (ADR 0225). + if strings.Contains(text, "${consumer:as:dns}") { + if b := m.IdentityBoundOf(provision); !b.Bounded() || b.Max > dnsLabelLimit { + problems = append(problems, fmt.Sprintf( + "%s serves %s's consumers their identity as a DNS label in %q, and bounds "+ + "that identity at %s: a label keeps %d — state an `identity` of at most %d", + m.Module, provision, key, boundWords(b), dnsLabelLimit, dnsLabelLimit)) + } + } + } + } + return problems +} + +// dnsLabelLimit is the longest DNS label (RFC 1035 §2.3.4). +const dnsLabelLimit = 63 + +func boundWords(b IdentityBound) string { + if !b.Bounded() { + return "nothing" + } + return fmt.Sprintf("%d", b.Max) +} + +func sortedServes(serves map[string]map[string]any) []string { + out := make([]string, 0, len(serves)) + for k := range serves { + out = append(out, k) + } + sort.Strings(out) + return out +} + +func sortedAnyKeys(values map[string]any) []string { + out := make([]string, 0, len(values)) + for k := range values { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// derivedFor is what the provider on this machine derives for one consumer of one provision +// (novox/hq ADR 0202). +// +// Settled first, then derived: an operator may set a prefix on what the provider serves and the +// mesh still fills the consumer's half of it ([ADR 0174]). Only the keys that actually name the +// consumer are returned — the rest of a `serves` block is the same for every consumer and is +// already in the provider's own definition, so repeating it here would be a second copy to go +// stale. +// +// The first module in the resolved order that says it serves the provision answers, which is the +// choice servedOnThisMachine makes for the consumer's half. Nothing serving it on this machine is +// not an error: a contribution can reach a machine whose provider is a record or an adapter, and +// then there is nothing derived to tell. +func (r Resolution) derivedFor(provision, as, consumer, local string, settings SettingsBy) (map[string]any, error) { + for _, m := range r.Modules { + serves, said := m.Serves[provision] + if !said { + continue + } + var names map[string]any + for key, value := range serves { + if text, ok := value.(string); ok && strings.Contains(text, "${consumer:") { + if names == nil { + names = map[string]any{} + } + names[key] = value + } + } + if names == nil { + return nil, nil + } + // **A consumer that keeps several holders of this provision is refused** — this is issue + // 124's own failure one case to the side, and it would be just as quiet. + // + // Each holder gets its own login, `…_` (ADR 0094), and a provider derives from the + // login, so it would make one resource per holder. The consumer's side has no such + // dimension: one binding file per provision, one `${bound::}`, both + // derived from the un-suffixed identity. So the provider would create the holder's + // resource and the consumer would be configured against a name nothing made — it would + // authenticate successfully and be refused on every object, which reads like a credential + // fault and is not one. + // + // Lifting this means giving the consumer's side a local dimension. That is a decision, + // not an omission, and until it is taken the mesh says so rather than guessing. + if local != "" { + return nil, fmt.Errorf( + "%s keeps several holders of %s (this one is %q), and %s derives %s for each "+ + "consumer from the login the mesh minted. Each holder has its own login, and a "+ + "consumer is told one value per requirement — so the two ends would name "+ + "different things and nothing would compare them (novox/hq ADR 0202)", + consumer, local, provision, m.Module, orNothing(sortedAnyKeys(names))) + } + settled, err := Settle(names, WithDefaults(m, settings[m.Module])) + if err != nil { + return nil, fmt.Errorf("%s serving %s: %w", m.Module, provision, err) + } + derived, err := ServedTo(settled, as) + if err != nil { + return nil, fmt.Errorf("%s serving %s to %s: %w", m.Module, provision, as, err) + } + return derived, nil + } + return nil, nil +} + +// notTranscribed refuses a consumer's file that writes out the value its provider derives for it, +// instead of asking for it (novox/hq ADR 0202, issue 124). +// +// **What would have caught the one wrong instance.** The object store's three consumers each wrote +// their bucket into their own configuration by hand. One of them named a predecessor's bucket, and +// nothing compared it to what the provider would actually create: the module would have +// authenticated successfully and been refused on every object, which reads like a credential fault +// and is not one. It looked authoritative for months. +// +// The test is exact and costs one string search: a definition whose file already contains the +// value the mesh is about to derive for it has written down somebody else's rule. It cannot be a +// coincidence — a derived value carries the identity the mesh minted for this very consumer on +// this very machine, which nothing else would spell out — and it cannot be checked afterwards, +// because after substitution every consumer's file contains it legitimately. +// +// Only values that actually name the consumer are judged. A provider that serves a constant under +// the same key serves the same constant to everyone, and a consumer repeating it is redundant +// rather than wrong. +func notTranscribed(resource map[string]any, known map[string]map[string]string, module string) error { + if fmt.Sprint(resource["type"]) != "file" { + return nil + } + content, ok := resource["content"].(string) + if !ok || content == "" { + return nil + } + for _, provision := range sortedKnown(known) { + values := known[provision] + identity := values["as"] + if identity == "" { + continue + } + for _, key := range sortedStringKeys(values) { + if key == "as" { + // The login is not derived from itself, and a consumer that must present it in a + // connection string legitimately has it from `${bound:…}` — which is what it will + // be after substitution, so this would judge the substitution, not the module. + continue + } + value := values[key] + if value == "" || !namesTheConsumer(value, identity) { + continue + } + if !strings.Contains(content, value) { + continue + } + return fmt.Errorf( + "%s writes %q into %v, and that is exactly what %s derives for it — a definition "+ + "keeping its own copy of somebody else's naming rule is one that can disagree "+ + "with it, silently. Say ${bound:%s:%s} and be told", + module, value, resource["id"], provision, provision, key) + } + } + return nil +} + +// namesTheConsumer is whether a derived value was built from this consumer's identity — in the +// alphabet it was minted in, or as a DNS label. A value that does not contain it was not derived +// from it, whatever else it may be. +func namesTheConsumer(value, identity string) bool { + return strings.Contains(value, identity) || strings.Contains(value, asDNSLabel(identity)) +} + +func sortedKnown(known map[string]map[string]string) []string { + out := make([]string, 0, len(known)) + for k := range known { + out = append(out, k) + } + sort.Strings(out) + return out +} + +func sortedStringKeys(values map[string]string) []string { + out := make([]string, 0, len(values)) + for k := range values { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/consumers.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/consumers.go new file mode 100644 index 00000000..46a64e0e --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/consumers.go @@ -0,0 +1,110 @@ +package catalogue + +import ( + "fmt" + "regexp" + "slices" +) + +// A provider's consumers' credentials, in a file of the provider's own (novox/hq ADR 0373, issue 679). +// +// A provider in the tool runner may not ask the vault (ADR 0349 clause 11, ADR 0370 clause 2), so its template names +// `${consumers:}` and its machine's provisioner client fills it: one entry per consumer bound to the +// provider for that provision, each carrying the credential the vault keeps for the pair. Only the client fills it, +// so the placeholder is a provider's alone, for a provision it provides, in a file only the provider reads (mode 0600 +// or 0400, owned at runtime, so a change on disk is written back); and a module not handed to the client on a machine +// is refused there, never written the placeholder's text, nor an empty list the provider would read as no consumers. + +// ConsumersNamespace is a provider's consumers' credentials' placeholder namespace, `${consumers:}`. +const ConsumersNamespace = "consumers" + +// ofConsumers is where a provider places its consumers' credentials. Loose inside the braces, so a misspelt name is +// refused rather than passed over. +var ofConsumers = regexp.MustCompile(`\$\{consumers:([^}]*)\}`) + +// consumersPlaceholderProblems is what is wrong with one resource's ${consumers:…}: placed only in a file's content, +// for a provision the module provides, in a file only the provider reads and the client writes back. +func consumersPlaceholderProblems(m Manifest, r map[string]any) []string { + var problems []string + for _, field := range sortedKeys(r) { + v, ok := r[field].(string) + if !ok { + continue + } + for _, f := range ofConsumers.FindAllStringSubmatch(v, -1) { + switch { + case field != "content" || fmt.Sprint(r["type"]) != "file": + problems = append(problems, fmt.Sprintf("%s's resource %v names %s in its %s; a provider's consumers "+ + "are placed only in a file's content (novox/hq ADR 0373)", m.Module, r["id"], f[0], field)) + continue + case !slices.ContainsFunc(m.Provides, func(o Offer) bool { return o.Name == f[1] }): + problems = append(problems, fmt.Sprintf("%s's resource %v names %s, and %s does not provide %q: only a "+ + "provider's template holds its consumers' credentials (novox/hq ADR 0373)", m.Module, r["id"], f[0], + m.Module, f[1])) + } + if mode := fmt.Sprint(r["mode"]); mode != "0600" && mode != "0400" { + problems = append(problems, fmt.Sprintf("%s's resource %v holds its consumers' credentials (%s) with "+ + "mode %q: the provider alone reads it, 0600 or 0400 (novox/hq ADR 0373)", m.Module, r["id"], f[0], + r["mode"])) + } + if owner := ownershipOf(r); owner != RuntimeOwned { + problems = append(problems, fmt.Sprintf("%s's resource %v holds its consumers' credentials (%s) and is "+ + "%s at runtime: it is owned, written back whenever it differs (novox/hq ADR 0373)", m.Module, + r["id"], f[0], owner)) + } + } + } + return problems +} + +// consumersHandedOnly says what a file naming ${consumers:…} is where it would be written the old way: the provisioner's +// client alone fills it, so on a machine that does not hand the module it is **left out** of the declaration, never +// written as that text or as an empty list the provider would read as no consumers, and never the reason the module is +// refused: a provider not handed there keeps working as it did, from its grant files (novox/hq ADR 0373, issue 780). +// A moved module judged from its repository's tree (WithTemplatesFrom: `module check`), on a machine +// that would hand it, has no templates archive to hand and is judged as it composes; there the placeholder is judged +// and taken out of the text, since what is composed so is never sent (judged answers whether the file is so judged). +func consumersHandedOnly(m Manifest, r map[string]any, with Rendering) (judged, leftOut bool) { + content, _ := r["content"].(string) + switch { + case ofConsumers.FindString(content) == "": + return false, false + case m.Moved && m.Handed == nil && with.Hands && with.Provisioner != "": + return true, false + } + return false, true +} + +// consumersLeftOutLine is the plan's one line for a file left out because the module is not handed. +func consumersLeftOutLine(m Manifest, r map[string]any) string { + return fmt.Sprintf("%s's %v names ${consumers:…}, which the provisioner's client alone fills: left out, handed only "+ + "(novox/hq ADR 0373)", m.Module, r["id"]) +} + +// consumersNotHanded is consumersHandedOnly as a refusal, for a caller that has no way to leave the file out. +func consumersNotHanded(m Manifest, r map[string]any, with Rendering) (judged bool, err error) { + judged, left := consumersHandedOnly(m, r, with) + if left { + return false, fmt.Errorf("%s's resource %v names %s, which the provisioner's client alone fills, and %s is not "+ + "handed to it on this machine: it would be written as that text, or empty, and the provider would read no "+ + "consumers (novox/hq ADR 0373)", m.Module, r["id"], ofConsumers.FindString(fmt.Sprint(r["content"])), m.Module) + } + return judged, nil +} + +// consumersOfTemplate is what is wrong with a handed file's template naming ${consumers:…}, judged on the file's +// resource as if its text were inline. +func consumersOfTemplate(m Manifest, id, text string) []string { + for _, r := range m.Resources { + if fmt.Sprint(r["id"]) != id { + continue + } + withText := map[string]any{} + for k, v := range r { + withText[k] = v + } + withText["content"] = text + return consumersPlaceholderProblems(m, withText) + } + return nil +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/core_credentials.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/core_credentials.go new file mode 100644 index 00000000..f11b70c0 --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/core_credentials.go @@ -0,0 +1,33 @@ +package catalogue + +import ( + "encoding/json" + "slices" +) + +// The core's own credentials (novox/hq ADR 0349 §16, issue 755). The bus, the node-engine, the controller and +// mesh-provisioner need their files before any provisioner client runs, so their files stay placement and their +// credentials travel in the declaration as values the vault sealed to the machine they run on, which the controller +// carries and cannot open; the node-engine opens them there (mesh-host's VaultBoxContract). + +// CoreCredentialModules are the modules whose own secrets are the core's credentials: the bus, mesh-provisioner and +// its client. The node-engine and the controller hold no own secret of a module's. **Fixed here, in the controller's +// code, as the clause says** — and the vault keeps the same list (mesh-vault's CoreModules), answering the controller +// for these alone. +var CoreCredentialModules = []string{"mesh-provisioner", "mesh-provisioner-client", "nats"} + +// CarriesVaultSealed says whether module's own secrets are the core's credentials. +func CarriesVaultSealed(module string) bool { return slices.Contains(CoreCredentialModules, module) } + +// VaultSealedField is the field of a file the vault sealed to its machine (mesh-host's File.VaultSealed). +const VaultSealedField = "vault-sealed" + +// ForeseenVaultBox stands for a core credential's box in a composition asked ahead of the send, when none is kept +// yet: the send asks the vault for it. Never sent. +var ForeseenVaultBox = json.RawMessage(`{"v":1,"eph":"","iv":"","tag":"","ct":"","foreseen":"asked of the vault by the next send"}`) + +// vaultSealedOf is the vault's box for one of module's own secrets, if the rendering was given one. +func (with Rendering) vaultSealedOf(module, name string) (json.RawMessage, bool) { + box, ok := with.VaultSealed[module][name] + return box, ok && len(box) > 0 +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/cron.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/cron.go new file mode 100644 index 00000000..fbb6d299 --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/cron.go @@ -0,0 +1,92 @@ +package catalogue + +// A five-field cron validator, enough to refuse a malformed schedule near its author +// (novox/hq ADR 0053). +// +// **Validation only, and written rather than pulled in.** The control plane's part in a scheduled +// step is small and exact: carry the field to the host unchanged, and refuse a `schedule` that is +// not a well-formed cron here rather than on the machine — the same near-versus-far discipline the +// manifest keeps for an action a module may not declare and a run-once that is not a boolean. What +// *when to run it again* means is the host's to evaluate; the control plane only checks the string +// is one it could. A cron library would be a dependency carried for evaluation nobody does here. +// +// The ordinary five fields — minute, hour, day-of-month, month, day-of-week — with `*`, lists +// (`,`), ranges (`-`) and steps (`/`). No names (jan, mon): a numeric cron is what the host +// evaluates, and refusing a name here is refusing it near whoever wrote it. + +import ( + "fmt" + "strconv" + "strings" +) + +// validateCron reports why a string is not a five-field cron expression, or nil if it is one. +func validateCron(expr string) error { + fields := strings.Fields(expr) + if len(fields) != 5 { + return fmt.Errorf( + "a schedule is a five-field cron expression (minute hour day-of-month month "+ + "day-of-week), and this has %d field(s)", len(fields)) + } + bounds := []struct { + name string + min, max int + }{ + {"minute", 0, 59}, + {"hour", 0, 23}, + {"day-of-month", 1, 31}, + {"month", 1, 12}, + {"day-of-week", 0, 7}, // 0 and 7 are both Sunday, the convention every cron keeps + } + for i, b := range bounds { + if err := validateCronField(fields[i], b.min, b.max); err != nil { + return fmt.Errorf("%s field: %w", b.name, err) + } + } + return nil +} + +// validateCronField checks one field's elements are within range and well-formed. +func validateCronField(spec string, min, max int) error { + if spec == "" { + return fmt.Errorf("is empty") + } + for _, part := range strings.Split(spec, ",") { + if part == "" { + return fmt.Errorf("%q has an empty element between commas", spec) + } + + rangePart := part + if slash := strings.IndexByte(part, '/'); slash >= 0 { + rangePart = part[:slash] + n, err := strconv.Atoi(part[slash+1:]) + if err != nil || n < 1 { + return fmt.Errorf("step in %q is not a positive number", part) + } + } + + switch { + case rangePart == "*": + // Any value; nothing to bound. + case strings.IndexByte(rangePart, '-') >= 0: + dash := strings.IndexByte(rangePart, '-') + lo, errLo := strconv.Atoi(rangePart[:dash]) + hi, errHi := strconv.Atoi(rangePart[dash+1:]) + if errLo != nil || errHi != nil { + return fmt.Errorf("range %q is not two numbers", rangePart) + } + if lo < min || hi > max || lo > hi { + return fmt.Errorf("range %q is outside the allowed %d-%d", part, min, max) + } + default: + v, err := strconv.Atoi(rangePart) + if err != nil { + return fmt.Errorf("%q is not a number", rangePart) + } + if v < min || v > max { + return fmt.Errorf("%q is outside the allowed range %d-%d", part, min, max) + } + } + } + return nil +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/data.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/data.go new file mode 100644 index 00000000..5a43203c --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/data.go @@ -0,0 +1,791 @@ +package catalogue + +import ( + "bytes" + "encoding/json" + "fmt" + "regexp" + "sort" + "strconv" + "strings" + "time" +) + +// A module declares the data it holds, and the mesh protects and watches it from that declaration +// (novox/hq ADR 0233). +// +// **One section, `data`, for every kind of data a module keeps:** its own, by directory — a store's +// files, a mail spool, an application's uploads — or by an operator's path it was given; what it keeps +// for its consumers, by the provision they reach it through; and what of its own lives with a +// provider, by the provision it requires. Each entry has a class — how precious it is — and, for its +// own data, how it is protected; everything the mesh does is derived from those, never written per +// module: +// +// - a binding to a consumer's data does not move (ADR 0232) where the provision's class keeps data; +// - the backup holder's lines are composed from it — a module no longer writes them by hand; +// - what an unassignment leaves behind of an irreplaceable or valuable item is retired, listed by +// `cleanup list` and deleted only by `cleanup delete` (ADR 0230); +// - the self-check measures every item and says when one shrinks, disappears, stops being written, +// goes without its backup, sits on a degraded array, or is replaced by an empty copy of itself — +// urgent for what is irreplaceable, a warning for what is valuable. + +// The classes: how precious the data is. A fixed vocabulary, ranked by the operator (2026-10-06): a +// class is what the protections are derived from, so a new one is a decision, not a manifest's choice. +const ( + // ClassIrreplaceable is what must never be lost: the operator names it (the media library, the + // photo sites' storage). Protected by a backup or by a declared redundancy — one is required — + // retired rather than removed, and every alert about it is urgent. + ClassIrreplaceable = "irreplaceable" + // ClassValuable is anybody's work that would be painful to lose: in the nightly backup by default, + // retired rather than removed, and every alert about it a warning. + ClassValuable = "valuable" + // ClassRebuildable can be made again from something kept elsewhere — a clone, an index, a + // download, a night's dump — at a cost in time, not in data. In the nightly backup by default; + // forgotten when its module goes, and never alerted on. + ClassRebuildable = "rebuildable" + // ClassCache may be emptied at any moment with nothing lost but speed: never backed up, never + // measured, never alerted on. + ClassCache = "cache" + // ClassNone is a provision that keeps nothing of its consumers' (consumers only): a resolver, a + // certificate authority, an artifact store. + ClassNone = "none" +) + +// classRank orders the classes by how precious they are, so the stricter of two is chosen. +var classRank = map[string]int{ClassNone: 0, ClassCache: 1, ClassRebuildable: 2, ClassValuable: 3, ClassIrreplaceable: 4} + +// StricterClass is the more precious of two classes. +func StricterClass(a, b string) string { + if classRank[b] > classRank[a] { + return b + } + return a +} + +// Retires is whether a class's data is retired, not forgotten, when its machine no longer declares it. +func Retires(class string) bool { return class == ClassIrreplaceable || class == ClassValuable } + +// Watched is whether a class's data raises conditions: irreplaceable and valuable. +func Watched(class string) bool { return Retires(class) } + +// DefaultBackupWithin is how old the last good backup of an item may be before the self-check says so +// (novox/hq ADR 0214: a machine with data and no good backup in 48 hours). +const DefaultBackupWithin = 48 * time.Hour + +// Data is a manifest's `data` section. +type Data struct { + // Own is the data this module keeps itself. + Own []DataItem `json:"own,omitempty"` + // Consumers is what it keeps for its consumers, per provision it grants. + Consumers map[string]ConsumerData `json:"consumers,omitempty"` + // KeptBy is what of its own lives with the provider of a provision it requires — its rows, its + // objects — and how precious that is. The provider's protections follow the stricter of its own + // class for its consumers and this. + KeptBy map[string]KeptData `json:"kept-by,omitempty"` +} + +// DataItem is one piece of a module's own data. +type DataItem struct { + // ID names it within the module: what `data`, `cleanup` and a condition call it. + ID string `json:"id"` + // Path is one of the module's directories, `${dir:}`, or an operator's path it was given, + // `${access:}`, or a path beneath either. Never a machine path (novox/hq ADR 0112). + Path string `json:"path"` + Class string `json:"class"` + // Backup is how it is copied: "copy" (the holder reads it as it stands), "none", or a dump — a + // command that writes a consistent copy into another item, which is copied. Unsaid, anything but a + // cache is copied, unless it says it is protected by redundancy instead. + Backup *DataBackup `json:"backup,omitempty"` + // Redundancy says it is protected by the redundancy of the storage it lives on rather than by a + // copy, and why that is enough: the media library on an array there is no room to copy. The + // backup holder then watches that array, and a degraded one is said. + Redundancy string `json:"redundancy,omitempty"` + // Within is how old its last good backup may be; unsaid, DefaultBackupWithin. + Within string `json:"within,omitempty"` + // Measure is how the backup holder measures it: `walk` (the default — every file, at most daily and + // bounded, for small items), `dataset` (a ZFS dataset's own counters, hourly, nothing walked) or + // `shallow` (its top-level entries only, no size). A large item never says walk. + Measure string `json:"measure,omitempty"` + // Active is how long it may go unwritten before that is a fault — for data something is + // expected to write all the time. Unsaid, a quiet item is not a fault. + Active string `json:"active,omitempty"` + // Why is a line for the reviewer: why this class. + Why string `json:"why,omitempty"` +} + +// ConsumerData is what a provider keeps for the consumers of one provision. +type ConsumerData struct { + Class string `json:"class"` + // In is where it lives: one of the module's own items (whose protection covers it), or a + // provision this module requires (whose provider keeps it, as its consumer). Unsaid only for none + // and cache. + In string `json:"in,omitempty"` + Why string `json:"why,omitempty"` +} + +// KeptData is how precious what a module keeps with a provider is. +type KeptData struct { + Class string `json:"class"` + Why string `json:"why,omitempty"` +} + +// DataBackup is how an item is copied. +type DataBackup struct { + Copy bool + None bool + // Dump is the command writing a consistent copy into the item Into. + Dump string + Into string +} + +// UnmarshalJSON reads "copy", "none" or {"dump": "...", "into": ""}. +func (b *DataBackup) UnmarshalJSON(raw []byte) error { + var word string + if err := json.Unmarshal(raw, &word); err == nil { + switch word { + case "copy": + *b = DataBackup{Copy: true} + case "none": + *b = DataBackup{None: true} + default: + return fmt.Errorf("a data item's backup is \"copy\", \"none\" or {dump, into}, not %q", word) + } + return nil + } + var full struct { + Dump string `json:"dump"` + Into string `json:"into"` + } + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(&full); err != nil { + return fmt.Errorf("a data item's backup is \"copy\", \"none\" or {dump, into}: %w", typedUnknown(err)) + } + *b = DataBackup{Dump: full.Dump, Into: full.Into} + return nil +} + +// MarshalJSON writes it back in the form it was written. +func (b DataBackup) MarshalJSON() ([]byte, error) { + switch { + case b.Copy: + return json.Marshal("copy") + case b.None: + return json.Marshal("none") + } + return json.Marshal(struct { + Dump string `json:"dump"` + Into string `json:"into"` + }{b.Dump, b.Into}) +} + +// IsDump is whether the item is copied by a dump. +func (b *DataBackup) IsDump() bool { return b != nil && b.Dump != "" } + +// BackedUp is whether the holder keeps restore points of this item: anything but a cache by default — +// the nightly backup is the standard plan — unless it says "none", or says it is protected by +// redundancy and says nothing of a backup. +func (it DataItem) BackedUp() bool { + switch { + case it.Backup == nil: + return it.Class != ClassCache && it.Redundancy == "" + case it.Backup.None: + return false + } + return true +} + +// Protection is how the item is protected, in one word: "backup", "redundancy", both joined by "+", +// or "none". +func (it DataItem) Protection() string { + var by []string + if it.BackedUp() { + by = append(by, "backup") + } + if it.Redundancy != "" { + by = append(by, "redundancy") + } + if len(by) == 0 { + return "none" + } + return strings.Join(by, "+") +} + +// The ways an item is measured. +const ( + MeasureWalk = "walk" + MeasureDataset = "dataset" + MeasureShallow = "shallow" +) + +// MeasuredBy is how the item is measured, with the default applied. +func (it DataItem) MeasuredBy() string { + if it.Measure == "" { + return MeasureWalk + } + return it.Measure +} + +// OwnedByModule is whether the item is in one of the module's own directories — the mesh's to retire — +// rather than an operator's path it was given, which the mesh never retires and never deletes. +func (it DataItem) OwnedByModule() bool { return strings.HasPrefix(it.Path, "${dir:") } + +// BackupWithin is the item's bound on its last good backup. +func (it DataItem) BackupWithin() time.Duration { + if d, err := ParseDataDuration(it.Within); err == nil && d > 0 { + return d + } + return DefaultBackupWithin +} + +// ActiveWithin is how long the item may go unwritten; zero when quiet is not a fault. +func (it DataItem) ActiveWithin() time.Duration { + d, _ := ParseDataDuration(it.Active) + return d +} + +// ParseDataDuration reads "48h", "90m" or "7d"; empty is zero. +func ParseDataDuration(s string) (time.Duration, error) { + s = strings.TrimSpace(s) + if s == "" { + return 0, nil + } + if days, ok := strings.CutSuffix(s, "d"); ok { + n, err := strconv.Atoi(days) + if err != nil || n <= 0 { + return 0, fmt.Errorf("%q is not a number of days", s) + } + return time.Duration(n) * 24 * time.Hour, nil + } + d, err := time.ParseDuration(s) + if err != nil || d <= 0 { + return 0, fmt.Errorf("%q is not a duration (48h, 90m, 7d)", s) + } + return d, nil +} + +// DataItems is the module's own data, in the order declared. +func (m Manifest) DataItems() []DataItem { + if m.Data == nil { + return nil + } + return m.Data.Own +} + +// DataItem is one own item by id. +func (m Manifest) DataItem(id string) (DataItem, bool) { + for _, it := range m.DataItems() { + if it.ID == id { + return it, true + } + } + return DataItem{}, false +} + +// ConsumerDataOf is what this module says it keeps for a provision's consumers, and whether it says. +func (m Manifest) ConsumerDataOf(provision string) (ConsumerData, bool) { + if m.Data == nil { + return ConsumerData{}, false + } + c, ok := m.Data.Consumers[provision] + return c, ok +} + +// KeptByOf is how precious what this module keeps with a provision's provider is, and whether it says. +func (m Manifest) KeptByOf(provision string) (KeptData, bool) { + if m.Data == nil { + return KeptData{}, false + } + k, ok := m.Data.KeptBy[provision] + return k, ok +} + +// keepsByClass is whether a class keeps something a binding must not move away from. +func keepsByClass(class string) bool { + return class == ClassIrreplaceable || class == ClassValuable || class == ClassRebuildable +} + +// dataID is what an item's id may be: it is a token in a condition's key and a word on a line. +var dataID = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`) + +// dataPathRef is an item's path: one of the module's directories or accesses, and optionally a path +// beneath it. +var dataPathRef = regexp.MustCompile(`^\$\{(dir|access):([a-z0-9][a-z0-9-]*)\}(/[^\s$]*)?$`) + +// dataProblems is what is wrong with the `data` section itself, from the manifest alone: judged at +// registration as every other per-manifest problem is. Whether a module declares what it should is +// the catalogue check's (DataProblems), because a module already running was written before it. +func (m Manifest) dataProblems() []string { + if m.Data == nil { + return nil + } + var problems []string + say := func(format string, args ...any) { + problems = append(problems, fmt.Sprintf("%s's data: ", m.Module)+fmt.Sprintf(format, args...)) + } + dirs := map[string]bool{} + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) == "directory" { + dirs[fmt.Sprint(r["id"])] = true + } + } + accesses := map[string]bool{} + for _, a := range m.Accesses { + if a.ID != "" { + accesses[a.ID] = true + } + } + ids := map[string]DataItem{} + paths := map[string]string{} + for i, it := range m.Data.Own { + label := it.ID + if label == "" { + label = fmt.Sprintf("item %d", i+1) + } + if !dataID.MatchString(it.ID) { + say("%s has no usable id: lower-case letters, digits and dashes", label) + } else if _, twice := ids[it.ID]; twice { + say("%s is declared twice", it.ID) + } + ids[it.ID] = it + ref := dataPathRef.FindStringSubmatch(it.Path) + switch { + case ref == nil: + say("%s's path %q is not one of the module's directories or accesses: ${dir:} or ${access:}, "+ + "or a path beneath one (a definition names no machine path, novox/hq ADR 0112)", label, it.Path) + case ref[1] == "dir" && !dirs[ref[2]]: + say("%s's path names ${dir:%s}, and %s declares no directory %q", label, ref[2], m.Module, ref[2]) + case ref[1] == "access" && !accesses[ref[2]]: + say("%s's path names ${access:%s}, and %s declares no access %q", label, ref[2], m.Module, ref[2]) + case strings.Contains(it.Path, ".."): + say("%s's path %q climbs out of its directory", label, it.Path) + } + if other, twice := paths[it.Path]; twice && it.Path != "" { + say("%s and %s name the same path %s", other, label, it.Path) + } + paths[it.Path] = label + switch it.Class { + case ClassIrreplaceable, ClassValuable, ClassRebuildable, ClassCache: + case ClassNone: + say("%s is class none, which only a provision keeping nothing of its consumers' is; own data "+ + "that is disposable is cache", label) + default: + say("%s's class %q is not one the mesh protects by: irreplaceable, valuable, rebuildable or cache", + label, it.Class) + } + if it.Class == ClassIrreplaceable && !it.BackedUp() && strings.TrimSpace(it.Redundancy) == "" { + say("%s is irreplaceable and is neither backed up nor said to be protected by redundancy; the only "+ + "copy of something is protected one way or the other (novox/hq ADR 0233) — copy it, dump it into "+ + "another item, or say `redundancy` with why that is enough", label) + } + if it.Class == ClassCache && it.Backup != nil && !it.Backup.None { + say("%s is a cache and asks to be backed up; a cache is disposable, or it is not a cache", label) + } + if it.Class == ClassCache && it.Redundancy != "" { + say("%s is a cache and says it is protected by redundancy; a cache is not protected", label) + } + switch it.Measure { + case "", MeasureWalk, MeasureDataset, MeasureShallow: + default: + say("%s's measure %q is walk, dataset or shallow", label, it.Measure) + } + if _, err := ParseDataDuration(it.Within); err != nil { + say("%s's within: %v", label, err) + } + if _, err := ParseDataDuration(it.Active); err != nil { + say("%s's active: %v", label, err) + } + if it.Within != "" && !it.BackedUp() { + say("%s states within, and is not backed up", label) + } + if it.Active != "" && !Watched(it.Class) { + say("%s is %s and expects writes; only irreplaceable and valuable data is watched", label, it.Class) + } + } + // Dumps go into an item of the same module, declared, and not into themselves. + for _, it := range m.Data.Own { + if it.Backup == nil || it.Backup.Copy || it.Backup.None { + continue + } + switch into, ok := ids[it.Backup.Into]; { + case strings.TrimSpace(it.Backup.Dump) == "": + say("%s's backup names no dump command", it.ID) + case it.Backup.Into == "": + say("%s's dump says no item it writes into", it.ID) + case !ok: + say("%s's dump writes into %q, which is not one of the module's data items", it.ID, it.Backup.Into) + case into.ID == it.ID: + say("%s's dump writes into itself; a dump is copied from where it lands", it.ID) + case into.Class == ClassCache: + say("%s's dump writes into %s, a cache; what is copied must not be disposable", it.ID, into.ID) + } + if it.Backup.Dump != "" { + declared := map[string]string{} + for d := range dirs { + declared[d] = "" + } + if _, err := dirFill(it.Backup.Dump, declared, m.Module); err != nil { + say("%s's dump: %v", it.ID, err) + } + } + } + provided := map[string]bool{} + for _, o := range m.Provides { + provided[o.Name] = true + } + wants := map[string]bool{} + for _, w := range m.Wants() { + wants[w] = true + } + for _, provision := range sortedKeys(m.Data.Consumers) { + c := m.Data.Consumers[provision] + if !provided[provision] { + say("says what it keeps for the consumers of %q, which it does not provide", provision) + } + switch c.Class { + case ClassIrreplaceable, ClassValuable, ClassRebuildable, ClassCache, ClassNone: + default: + say("its consumers' %s is class %q; one of irreplaceable, valuable, rebuildable, cache or none", provision, c.Class) + } + switch { + case c.Class == ClassNone && c.In != "": + say("its consumers' %s keeps nothing and says where it is kept (%s)", provision, c.In) + case keepsByClass(c.Class) && c.In == "": + say("its consumers' %s is %s and says nowhere it lives: `in` names one of the module's items, or a "+ + "provision it requires", provision, c.Class) + case c.In != "": + if own, ok := ids[c.In]; ok { + if c.Class == ClassIrreplaceable && !own.BackedUp() && own.Redundancy == "" { + say("its consumers' %s is irreplaceable and lives in %s, which is not protected", provision, c.In) + } + if classRank[own.Class] < classRank[c.Class] { + say("its consumers' %s is %s and lives in %s, which is only %s", provision, c.Class, c.In, own.Class) + } + } else if !wants[c.In] { + say("its consumers' %s lives in %q, which is neither one of its data items nor a provision it "+ + "requires", provision, c.In) + } + } + } + for _, provision := range sortedKeys(m.Data.KeptBy) { + k := m.Data.KeptBy[provision] + if !wants[provision] { + say("says it keeps data with the provider of %q, which it does not require", provision) + } + switch k.Class { + case ClassIrreplaceable, ClassValuable, ClassRebuildable, ClassCache: + default: + say("what it keeps with %s is class %q; one of irreplaceable, valuable, rebuildable or cache", provision, k.Class) + } + } + return problems +} + +// KeepsConsumerData is whether this module, providing a provision, keeps what each consumer writes +// there (novox/hq ADR 0232, ADR 0233): whether a consumer bound to it is bound to its data. +// +// **What the data section says, it gets**: irreplaceable, valuable or rebuildable keeps; cache and none +// do not — a cache lost in a move costs speed, not data. Before the section, an offer said it with +// `keeps-consumer-data`, which is still read; unsaid in both, a provider that grants each consumer a +// credential of its own keeps that consumer's data (ADR 0232 §1). The catalogue check refuses the +// unsaid case for a provider that grants (DataProblems), so the inference is what an older definition +// on the shelf gets, never a new one. +func (m Manifest) KeepsConsumerData(provision string) bool { + if c, ok := m.ConsumerDataOf(provision); ok { + return keepsByClass(c.Class) + } + for _, o := range m.Provides { + if o.Name == provision && o.KeepsConsumerData != nil { + return *o.KeepsConsumerData + } + } + _, grants := m.Grants[provision] + return grants +} + +// NeedsBackupHolder is whether a module's data needs the machine's backup holder to be there: it keeps +// something irreplaceable — backed up by the holder, or protected by an array the holder watches. A +// module keeping only valuable or rebuildable data is backed up where a holder is, and refused nowhere +// for want of one: the standard plan, not a requirement. +func (m Manifest) NeedsBackupHolder() bool { + for _, it := range m.DataItems() { + if it.Class == ClassIrreplaceable { + return true + } + } + return false +} + +// --- derived: the backup holder's lines --------------------------------------------------------- + +// The node-backup seat's kinds (novox/hq to-be 43, ADR 0233): `backup` is what to run and which paths +// to keep, `data` every item with its class and protection, for the holder to measure and watch. +const ( + BackupKindBackup = "backup" + BackupKindData = "data" +) + +// derivedContributions is what a module's data section gives the backup holder: its backup lines and +// its items. Every item is listed, so a valuable one on a machine is measured whether or not it is +// copied; a cache is listed and not measured. +func (m Manifest) derivedContributions() []SeatContribution { + items := m.DataItems() + if len(items) == 0 { + return nil + } + var lines []string + kept := map[string]bool{} + keep := func(path string) { + if !kept[path] { + kept[path] = true + lines = append(lines, "path "+path) + } + } + for _, it := range items { + if !it.BackedUp() { + continue + } + if it.Backup.IsDump() { + lines = append(lines, "run "+strings.TrimSpace(it.Backup.Dump)) + if into, ok := m.DataItem(it.Backup.Into); ok { + keep(into.Path) + } + continue + } + keep(it.Path) + } + var described []string + for _, it := range items { + covered := "-" + switch { + case it.Backup.IsDump(): + if into, ok := m.DataItem(it.Backup.Into); ok { + covered = into.Path + } + case it.BackedUp(): + covered = it.Path + } + described = append(described, fmt.Sprintf("item %s %s %s %s %s %s", it.ID, it.Class, it.Path, covered, + it.Protection(), it.MeasuredBy())) + } + var out []SeatContribution + if len(lines) > 0 { + out = append(out, SeatContribution{Seat: BackupSeat, Kind: BackupKindBackup, Content: strings.Join(lines, "\n") + "\n"}) + } + return append(out, SeatContribution{Seat: BackupSeat, Kind: BackupKindData, Content: strings.Join(described, "\n") + "\n"}) +} + +// allContributions is every contribution a module makes to a seat's holder: what it wrote, and what +// its data section derives. **One list**: a module that declares its data has its backup lines +// composed from it, and a backup line it still writes by hand is not placed — the catalogue check +// refuses it — so the holder never copies two lists that disagree. +func (m Manifest) allContributions() []SeatContribution { + if m.Data == nil { + return m.Contributions + } + derived := m.derivedContributions() + out := make([]SeatContribution, 0, len(m.Contributions)+len(derived)) + for _, c := range m.Contributions { + if s, known := SeatNamed(c.Seat); known && s.Name == BackupSeat { + continue + } + out = append(out, c) + } + return append(out, derived...) +} + +// --- the catalogue check ------------------------------------------------------------------------ + +// DataProblems is what the catalogue check refuses about the data a module declares (novox/hq ADR +// 0233), judged over the manifests given together: +// +// - a provider that grants a provision says what it keeps for that provision's consumers; +// - nobody says it on the offer any more (`keeps-consumer-data`): the data section is the one place; +// - nobody writes a backup line by hand: it is derived from the data section; +// - a directory a container writes, mounted whole, is a data item of some class; +// - consumer data said to live in a required provision lives where that provision's provider keeps +// its consumers' data, as preciously, when the provider is given; +// - data a consumer keeps with a provider as irreplaceable is protected there, when the provider is +// given. +// +// **The check's, not registration's**, as ADR 0214's backup rule was: a module already on the shelf +// was written before the rule, and refusing it there would refuse the very providers whose data the +// rule protects. Each module meets it where it is written. +func DataProblems(shelf Shelf) []string { + var problems []string + for _, name := range shelfOrder(shelf) { + m := shelf[name] + for _, provision := range sortedKeys(m.Grants) { + if _, said := m.ConsumerDataOf(provision); !said { + problems = append(problems, fmt.Sprintf( + "%s grants %s and does not say what it keeps for its consumers: data.consumers.%s with a "+ + "class — irreplaceable, valuable, rebuildable, cache, or none (novox/hq ADR 0233)", name, provision, provision)) + } + } + for _, o := range m.Provides { + if o.KeepsConsumerData != nil { + problems = append(problems, fmt.Sprintf( + "%s says keeps-consumer-data on its offer of %s; it is said once, in data.consumers.%s, with a "+ + "class (novox/hq ADR 0233)", name, o.Name, o.Name)) + } + } + for i, c := range m.Contributions { + if s, known := SeatNamed(c.Seat); known && s.Name == BackupSeat { + problems = append(problems, fmt.Sprintf( + "%s's contribution %d is a backup line written by hand; backups are derived from the data "+ + "section — declare the data, with its class and how it is protected (novox/hq ADR 0233)", name, i+1)) + } + } + for _, dir := range writtenDirectories(m) { + if !coversDirectory(m, dir) { + problems = append(problems, fmt.Sprintf( + "%s mounts its directory %q into a container to be written, and declares no data in it: "+ + "data.own with a class — cache if it is disposable (novox/hq ADR 0233)", name, dir)) + } + } + if m.Data == nil { + continue + } + for _, provision := range sortedKeys(m.Data.Consumers) { + c := m.Data.Consumers[provision] + if c.In == "" { + continue + } + if _, own := m.DataItem(c.In); own { + continue + } + for _, pname := range shelfOrder(shelf) { + p := shelf[pname] + pc, said := p.ConsumerDataOf(c.In) + if !said || !providesName(p, c.In) { + continue + } + if classRank[pc.Class] < classRank[c.Class] { + problems = append(problems, fmt.Sprintf( + "%s keeps its consumers' %s, %s, in %s — and %s keeps its consumers' %s as %s, "+ + "so it is not protected as %s", name, provision, c.Class, c.In, pname, c.In, pc.Class, c.Class)) + } + } + } + for _, provision := range sortedKeys(m.Data.KeptBy) { + k := m.Data.KeptBy[provision] + if k.Class != ClassIrreplaceable { + continue + } + for _, pname := range shelfOrder(shelf) { + p := shelf[pname] + pc, said := p.ConsumerDataOf(provision) + if !said || !providesName(p, provision) { + continue + } + if !keepsByClass(pc.Class) { + problems = append(problems, fmt.Sprintf( + "%s keeps irreplaceable data with %s, and %s keeps its consumers' %s as %s — nothing of it is "+ + "protected there", name, provision, pname, provision, pc.Class)) + continue + } + if in, own := p.DataItem(pc.In); own && !in.BackedUp() && in.Redundancy == "" { + problems = append(problems, fmt.Sprintf( + "%s keeps irreplaceable data with %s, and %s keeps it in %s, which is neither backed up nor "+ + "on declared redundancy", name, provision, pname, pc.In)) + } + } + } + } + return problems +} + +func providesName(m Manifest, provision string) bool { + for _, o := range m.Provides { + if o.Name == provision { + return true + } + } + return false +} + +// writtenDirectories are the module's directories a container mounts whole and may write: a volume +// `${dir:}:` without `:ro`, or a directory stated by an absolute path mounted the same +// way. A file beneath a directory, or a read-only mount, is configuration the mesh wrote. +func writtenDirectories(m Manifest) []string { + absolute := map[string]string{} + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) != "directory" { + continue + } + if p, ok := r["path"].(string); ok && strings.HasPrefix(p, "/") { + absolute[strings.TrimRight(p, "/")] = fmt.Sprint(r["id"]) + } + } + seen := map[string]bool{} + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) != "container" { + continue + } + vols, _ := r["volumes"].([]any) + for _, v := range vols { + s, _ := v.(string) + mount := volumeMount.FindStringSubmatch(s) + if mount == nil || volumeReadOnly(mount[3]) { + continue + } + src := strings.TrimRight(mount[1], "/") + if ref := dirPlain.FindStringSubmatch(src); ref != nil { + seen[ref[1]] = true + } else if id, ok := absolute[src]; ok { + seen[id] = true + } + } + } + out := make([]string, 0, len(seen)) + for id := range seen { + out = append(out, id) + } + sort.Strings(out) + return out +} + +// volumeMount is a container's volume, `:[:]`, its source possibly a +// `${dir:}` — whose own colon is not the separator. +var volumeMount = regexp.MustCompile(`^(\$\{(?:dir|access):[a-z0-9][a-z0-9-]*\}[^:]*|[^:]+):([^:]+)(?::(.*))?$`) + +// volumeReadOnly is whether a volume's options mount it read-only. +func volumeReadOnly(options string) bool { + for _, o := range strings.Split(options, ",") { + if strings.TrimSpace(o) == "ro" { + return true + } + } + return false +} + +// dirPlain is a whole directory, `${dir:}` and nothing after it. +var dirPlain = regexp.MustCompile(`^\$\{dir:([a-z0-9][a-z0-9-]*)\}$`) + +// coversDirectory is whether a data item names the directory, a path within it, or a directory it +// is placed beneath. +func coversDirectory(m Manifest, dir string) bool { + parents := map[string]string{} + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) != "directory" { + continue + } + if p, ok := r["path"].(string); ok { + if ref := dirRef.FindStringSubmatch(p); ref != nil && strings.HasPrefix(p, "${dir:") { + parents[fmt.Sprint(r["id"])] = ref[1] + } + } + } + for _, it := range m.DataItems() { + ref := dataPathRef.FindStringSubmatch(it.Path) + if ref == nil || ref[1] != "dir" { + continue + } + for d, hops := dir, 0; d != "" && hops < 4; d, hops = parents[d], hops+1 { + if ref[2] == d { + return true + } + } + } + return false +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/declaration.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/declaration.go new file mode 100644 index 00000000..a2d75976 --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/declaration.go @@ -0,0 +1,2819 @@ +package catalogue + +// Turning a resolution into the declaration a node is sent. +// +// Separate from resolving because they answer different questions. Resolving asks *what should +// this machine run*; this asks *what does that look like as resources*, and the second is where +// settings are applied, generators are called, contributions are collected and credentials are +// placed. Both lived in one file until it was doing four jobs at once — which is the shape the +// system this replaces failed in, one import at a time. + +import ( + "encoding/json" + "fmt" + "sort" + "strconv" + "strings" + "time" +) + +// SettingsBy is the layers that apply to each module, keyed by module name. +type SettingsBy map[string][]Layer + +// Generator works out a module's resources for one node, where they cannot be written in advance. +type Generator interface { + // Resources for this node. Absent means the node is not part of whatever this generates, + // which is an ordinary answer rather than a failure — a machine assigned the module before it + // has an address on the network is in exactly that state. + Resources(node string) ([]map[string]any, bool, error) +} + +// OpensPorts is a generator that also says what its resources accept connections on. +// +// **Separate from Generator, because most generators have nothing to say here** and requiring an +// empty method of each would be a cost paid everywhere for one caller. +// +// It exists because a static field cannot express this. A hub accepts connections from every node +// at other sites; a machine that is not a hub dials out and needs nothing open — and they are the +// same module. `listens` in a manifest is one answer for every machine that runs it, so the +// machine that most needs filtering, the one facing the public internet, was the one whose rule +// set would have closed its own overlay. +type OpensPorts interface { + // Listens is what this node accepts on because of what was computed for it. Nothing is the + // ordinary answer: most machines running a computed module open no port at all. + Listens(node string) ([]Listening, error) +} + +// Grant is one consumer's credential, on the machine that must create it. +type Grant struct { + // Provision is what was required. + Provision string + // Consumer is the node that will use it. + Consumer string + // From is the module on that machine which asked. + // + // **Part of who this credential is for, not a label** (novox/hq 04-ISSUES/022). Together with + // Consumer it names one consumer; the node alone does not, because a machine routinely runs + // several modules wanting the same thing. It is also what the provider names the role or the + // bucket or the client after, so withdrawing one consumer does not take another's away. + From string + // Values are what that module contributed — the name it wants, and anything else the + // provision's own vocabulary defines. + Values map[string]any + // Local is the name the credential goes by inside the consumer where it keeps several for one + // provision (ADR 0094); empty for the ordinary one. The provider sees it as a holder of its own. + Local string + // Slug is the consumer module's identity slug, if it declared one — carried on the grant so the + // provider side derives the same login the consumer does, even across nodes where the consumer's + // manifest is not in view (novox/hq ADR 0049). Empty means "use the module name". + Slug string + // At is where the consuming machine is on the private network, empty if it is not on one. + // + // Passed in with the grant because it is a fact about another machine, and resolution answers + // questions about one. A provider that must reach back to its consumer — a reverse proxy is + // the whole reason this exists — otherwise has to know how the mesh names machines. + At string + // Sealed is the credential, closed to the providing node. + Sealed string +} + +// Rendering is everything needed to turn a resolution into the declaration a node is sent. +type Rendering struct { + // Certificate is what the mesh issued for this machine's internal name, and the mesh's own + // certificate. Both public — the key they belong to never left the machine. + Certificate string + Authority string + + // Needed is each module's own secrets, sealed to this node, keyed by module and then by the + // name the module gave it. + Needed map[string]map[string]string + // Foreseen is each own secret not made yet that the next send WILL make, keyed like Needed: a + // composition asked ahead of the send (the self-check's D1) composes it with ForeseenSealed in + // its place and lists it in Composed.Foreseen, rather than failing for a value only a send + // makes. Nil on every composition that is sent, and on every other question, which then refuse + // a secret not made with a *NotMadeError. + Foreseen map[string]map[string]bool + // VaultSealed is, for a module on the core's list (CoreCredentialModules), each own secret as the vault sealed it + // to this machine (the mesh's sealed box, as the vault answered it), keyed like Needed (novox/hq ADR 0349 §16, + // issue 755). Given only for a machine whose engine opens the box (VaultBoxContract): its file then carries the + // box and never the controller's own sealed value. Nil everywhere else, and the composition is as it was. + VaultSealed map[string]map[string]json.RawMessage + // NotPlaced is each core module left out of this machine's declaration because the vault gave nothing for its + // credential and no box was kept, with why (issue 755): the machine keeps what it holds of it, and a module not + // yet there is not placed. Nil everywhere else. + NotPlaced map[string]string + + // Mesh is every node's address on the private network, which is what a rule saying "from the + // mesh" resolves to. Passed in for the same reason grants are: who else is on the network is + // a fact about the mesh, and resolution answers questions about one machine. + Mesh []string + // Suffix is what a machine's internal name ends in, as the control plane composed Names — + // `internal` unless the operator chose another — so a fact writing those names does not + // compose it a second time. + Suffix string + + // BusUsers is the mesh's composed user list, for the module holding `mesh-broker`. Empty on + // every other node, and on this one until the controller has composed it. + // + // **Only the users, never the server's own settings**: those are the module's, in its image and + // its mounts (Manifest.BusUsers). + BusUsers string + // BusMembership is this machine's membership for the bus the mesh is moving to, sealed to it + // (design 28, task 5.2). Empty for a machine not being moved. Written as a file the host reads + // after the declaration has applied, so the bus it names is standing before the machine leaves + // the one it is on. + BusMembership string + + // MeshRange is the private network's CIDR (the range node addresses are allocated from), for a + // module that must name the whole mesh rather than one machine — an intrusion filter that must + // never ban a tunnel peer, say. A per-mesh value the module cannot know, so it is carried here + // and offered as ${machine:mesh-range}, the same way one machine's address is. + MeshRange string + + // Accounts is each machine's operator account, by the same internal name Names uses (novox/hq + // to-be 29). What an ssh Host block's `User` line is composed from; empty for a machine no + // operator account is known on. + Accounts map[string]string + + // Kept is every operator-sealed secret in the mesh, for a module that `keeps` them. Nil when + // nothing on this node keeps them, or the mesh has no operator key. + Kept *KeptExport + + // OutwardLinks is the links this machine reported as facing outside it, which the filter is + // written around (novox/hq ADR 0140). Empty means the machine has not said, and the mesh + // composes no filter for it rather than writing a rule around a link with no name. + OutwardLinks []string + + // LocalNetworks is this machine's local networks (novox/hq ADR 0340): the networks it reported on a + // link facing outside it that lie wholly in a private range (JudgeLocalNetworks), which a port + // reached from `local-network` is admitted from beside the private network. Empty when it reported + // none, and the rule set then says so rather than widening the port or dropping it. + LocalNetworks []string + + // TunnelInterface is the interface the mesh's private network runs on, named here rather than + // imported because the overlay package rests on this one. Traffic arriving on it is the mesh's, + // not this machine's own guest, so the filter admits it only by a rule. + TunnelInterface string + + // Foundation is the ports the mesh itself needs reachable on every machine, which no module + // declares because the foundation is not a module (novox/hq 04-ISSUES/051 and 052). The broker + // is the one that matters: a machine dials it to enrol, and a firewall derived only from + // modules closes it. + Foundation []int + + // Names is every machine's internal name and its address, for containers to be given. + // + // **A container does not inherit the machine's names**, so every internal name the mesh wrote + // is invisible to what the machine runs. Given here rather than looked up on the machine, + // because which machines exist is a fact about the mesh. + Names map[string]string + + // Machines is only the machines, by the same internal name — the subset of Names that is a + // node of this mesh rather than a name it was told to serve. Both matter and they are not the + // same set: a container's hosts wants every name, so a routed name resolves to the proxy that + // serves it, while a resolver told the mesh's suffix is authoritative for it answers from what + // it is given and forwards nothing — so a routed name written there is a name nobody asks for, + // standing beside the machines and looking as real as they do. + Machines map[string]string + + // Zones is every zone a module in the mesh answers itself, where it is answered (novox/hq ADR + // 0199): the mesh's resolver forwards each one there. + Zones []ZoneAt + + // Holders is, for each replicated mesh seat, every machine holding it, by internal name and + // private address (novox/hq ADR 0223) — the same shape as Machines. What a machine's resolver + // file lists: every holder of the mesh's resolver, this machine first if it is one. + Holders map[string]map[string]string + + Settings SettingsBy + Generators map[string]Generator + // Grants are the credentials this node must create, for the provisions it offers. Passed in + // rather than resolved, because who consumes a node is a fact about the rest of the mesh and + // resolution answers questions about one machine. + Grants []Grant + // Withheld is every consumer left out of Grants because its identity overflows the provision's + // bound (novox/hq ADR 0225). Composed into nothing; carried so the machine's declaration can say + // whom it does not serve, and why, beside what it does. + Withheld []Overflow + // Unbound is every consumer whose pair credential from this node is on record and whose own + // resolution binds it elsewhere (novox/hq issue 274): never granted, carried to be said. + Unbound []Unbound + + // Ports is where this machine puts what each module needs reachable, by module and by the + // port the software itself uses (novox/hq ADR 0038). + // + // **The one place the number now lives.** A module used to write it three times — for the rule + // set, for what a consumer is told, and for what the runtime publishes — and nothing checked + // that the three agreed. They are all derived from this. + Ports map[string]map[int]int + + // Adopted says the node is adopted (novox/hq ADR 0100): the firewall found on it stays in + // force, so no module that loads a filter is declared there, and what the mesh needs + // reachable is declared as openings, with its own ports guarded by a table that only refuses. + Adopted bool + + // Given is the machine ports this node was given for its modules' ports, by module and by the + // port the software uses (novox/hq ADR 0100) — the foundation's ports, as genesis chose them. + // They win over anything the mesh would assign and over a manifest's own long-form mapping. + Given map[string]map[int]int + + // Taken is the modules taken on this adopted node (novox/hq ADR 0100). The guard is derived + // from these only (ADR 0103): a port of a module assigned but not taken may still be the + // predecessor's. + Taken map[string]bool + + // Seats is where this machine put each mesh-scoped seat's holder, by seat and by the port the + // holder's software uses (novox/hq 04-ISSUES/102) — read from the node's settings and + // assignments for whichever module claims the seat, whether or not it is in this node's set. + // What ${seat:…} answers with; see seat_into.go for why the answer may be absent. + Seats map[string]map[int]int + + // ArtifactStore is the mesh's artifact store as this network reaches it (host:port) — the + // node holding it and the port that node put it on — or empty when the mesh has none on its + // network yet. Composed into every image and archive the mesh built, at this moment and never + // stored (novox/hq 04-ISSUES/102). + ArtifactStore string + + // SeatReach is where this machine reaches the holder of each mesh-scoped seat it may be asked + // about (host:port), by seat: what ${seat::reach} answers with (novox/hq ADR 0222). Absent + // when no holder is reachable yet; see seat_into.go for which seats are answered. + SeatReach map[string]string + + // Built is every `/` the mesh has built. What tells a reference recorded + // with an address — before references were kept without one — from an image a module runs + // straight from a public registry. + Built map[string]bool + + // DataRoot is where this node keeps the directories the mesh places for its modules + // (novox/hq ADR 0112, to-be 27): a directory resource that states no path resolves to + // //, and ${dir:} names that place from the module's own files, + // mounts and environment. A node setting fixed at installation; empty means the default, + // /var/lib — see dir_into.go. + DataRoot string + + // ReadsHealth says this machine's node-engine reads a resource's `health` (novox/hq ADR 0240 Phase B: + // its statement's contract is link.ReadinessContract or later). An older engine parses strictly and + // refuses a field it does not know, whole — so to it the field is not sent, and what it runs is + // judged by liveness alone. + ReadsHealth bool + + // JudgesRoot says this machine's node-engine judges a user's declared `root` (novox/hq ADR 0266: its + // statement's contract is link.RootContract or later). To an older, strict engine the field is not + // sent, and the account it names is not judged — which the self-check says, as not judged. + JudgesRoot bool + + // ReadsBusy says this machine's node-engine reads a container's `busy` (novox/hq ADR 0335: its statement's + // contract is link.BusyContract or later). To an older, strict engine the field is not sent, and the + // container is replaced at once, as before. + ReadsBusy bool + + // Hands says this machine's node-engine writes a filled file on the provisioner client's word alone (its + // statement's contract is link.HandedContract or later, design 57 P7), and Provisioner is the account the + // provisioner's client runs as here, empty where none runs (novox/hq ADR 0349 §4). With both, a moved module's + // templated files are handed to the client (handed.go); without either, it composes as before. + Hands bool + Provisioner string +} + +// RootField is a user resource's field saying the account must never become root without a person +// (novox/hq ADR 0266). +const RootField = "root" + +// rootInto composes a user's `root` for the node-engine: taken away when it asserts nothing (empty — a +// machine where agents run as the operator) or when the engine is older than the field and parses +// strictly; kept as "never" otherwise. +func rootInto(resource map[string]any, with Rendering) { + if resource["type"] != "user" { + return + } + value, has := resource[RootField] + if !has { + return + } + if s, _ := value.(string); s == "" || !with.JudgesRoot { + delete(resource, RootField) + } +} + +// machinePort is where a module's port lives on this machine, or the port itself when the mesh has +// not been asked. Unassigned is not an error here: a module with no `listens` never needed one, +// and a caller composing a declaration without a store still gets something coherent. +func (r Rendering) machinePort(module string, wanted int) int { + if at, given := r.Given[module][wanted]; given { + return at + } + if at, known := r.Ports[module][wanted]; known { + return at + } + return wanted +} + +// Declaration is everything the resolved modules put on the node, with settings applied. +// +// Resource identities are prefixed with the module they came from. Two modules may reasonably +// both call something "config", and without this the second would silently replace the first — +// the node applying one of them and reporting success. +func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) { + composed, err := r.Compose(with) + if err != nil { + return nil, err + } + return composed.Resources, nil +} + +// Composed is a declaration's resources and which module each came from. +// +// Owner is kept beside the resources because a resource id cannot be split back into its module: +// a module's name may itself contain a dot. What the mesh adds of its own — an opening, the guard — +// has no owner. +// +// Received is what each module on the machine is given for each requirement it receives — the same +// contributions its received file is written from, kept beside it so the mesh can also issue them +// on the bus in the module's membership (novox/hq ADR 0167). By module, then requirement. +type Composed struct { + Resources []map[string]any + Owner map[string]string + Received map[string]map[string][]Contribution + // LeftOut is every module of this machine's set that was left out of its declaration, and + // why (novox/hq ADR 0163, rule 6): a setting stored for it that its definition can no longer + // compose. Its held things are kept and its containers untouched — the machine is told so — + // and it is told everything else. + LeftOut map[string]string + // Foreseen is every own secret composed with ForeseenSealed in its place (Rendering.Foreseen), + // as `module/name`, sorted: what the next send will make. Never set on a declaration that is + // sent — a placeholder in a sealed file is a credential the process cannot read. + Foreseen []string + // Unplaced is every contribution the holder's template could not render into something, each + // naming its module and why (novox/hq ADR 0255): that piece is left out and the machine is told + // everything else. Push and plan say it; the merge gate refuses a change that adds one. + Unplaced []string + // HandedOnly is each file left out because only the provisioner's client fills it and the module is not handed on + // this machine (a provider's consumers file, ADR 0373), said once in the plan. Not an unplaced contribution, which + // the merge gate refuses: a module not handed composes as it did. + HandedOnly []string + // Filled is the path of every file handed to the provisioner, in the order composed: the declaration's + // filled files (novox/hq ADR 0349 §4, §5). Plans is each handed module's plan on this machine, by module. + // Both empty where nothing is handed, and the declaration is then what it was. + Filled []string + Plans map[string]Plan +} + +// ForeseenSealed is what stands for an own secret a send will make, in a composition asked ahead of +// the send. Not a sealed value: nothing composed with it may be sent. +const ForeseenSealed = "foreseen: made by the next send" + +// NotMadeError is a module's own secret the composition was given no value for (novox/hq issue 275): +// typed, so that a caller can tell "a send would make this, and none has yet" from a composition that +// fails for any other reason without reading the words. +type NotMadeError struct { + Module, Name string +} + +func (e *NotMadeError) Error() string { + return fmt.Sprintf("%s needs a secret called %q and none was made for it", e.Module, e.Name) +} + +// LeftOut is which of this machine's modules a declaration composed with these settings leaves +// out, and why (novox/hq ADR 0163, rule 6): each whose stored settings its definition can no longer +// compose. Empty when every module composes. The same judgement SetSettings makes before storing. +func (r Resolution) LeftOut(settings SettingsBy, adopted bool) map[string]string { + out := map[string]string{} + for module, why := range r.LeftOutBecause(settings, adopted) { + out[module] = why.Error() + } + return out +} + +// LeftOutBecause is LeftOut with each reason as the error it was, so a reader can tell a setting nobody +// gave (an *UnsetSettingError) from any other cause and say it as the operator's to give (novox/hq issue +// 380). An UnreadManifestError for a stored manifest this controller cannot read whole. +func (r Resolution) LeftOutBecause(settings SettingsBy, adopted bool) map[string]error { + out := map[string]error{} + for _, m := range r.Modules { + if why := UnknownFieldReason(m); why != "" { + out[m.Module] = &UnreadManifestError{Module: m.Module, said: why} + continue + } + if err := JudgeSettings(m, settings[m.Module], adopted); err != nil { + out[m.Module] = err + } + } + return out +} + +// UnreadManifestError is a module left out because its stored manifest has a key this controller does not know +// (novox/hq ADR 0262): said once for the whole mesh, by the catalogue's own condition, not per machine. +type UnreadManifestError struct { + Module string + said string +} + +func (e *UnreadManifestError) Error() string { return e.said } + +// Compose is Declaration with the owner of every resource said. +func (r Resolution) Compose(with Rendering) (Composed, error) { + owner := map[string]string{} + received := map[string]map[string][]Contribution{} + leftOut := map[string]string{} + var foreseen, unplaced, handedOnly []string + handed := &handedFiles{plans: map[string]Plan{}} + resources, err := r.compose(with, owner, received, leftOut, &foreseen, &unplaced, &handedOnly, handed) + if err != nil { + return Composed{}, err + } + sort.Strings(foreseen) + if with.BusMembership != "" { + // The machine's own, not any module's: how it reaches the mesh from now on. Sealed like a + // secret and placed where the host looks for exactly this (design 28, task 5.2). + resources = append(resources, map[string]any{ + "id": BusMembershipID(), "type": "file", "path": BusMembershipPath, + "sealed": with.BusMembership, "mode": "0600", + }) + } + sort.Strings(unplaced) + sort.Strings(handedOnly) + out := Composed{Resources: resources, Owner: owner, Received: received, LeftOut: leftOut, + Foreseen: foreseen, Unplaced: unplaced, HandedOnly: handedOnly, Filled: handed.paths} + if plans := handed.finished(r.Needs); len(plans) > 0 { + out.Plans = plans + } + return out, nil +} + +// BusMembershipID names the resource carrying a machine's membership for the new bus, and +// BusMembershipPath is where the host reads it — the same constant on both sides. +func BusMembershipID() string { return "bus-membership" } + +const BusMembershipPath = "/var/lib/mesh/membership-next.json" + +func (r Resolution) compose(with Rendering, owner map[string]string, + received map[string]map[string][]Contribution, leftOut map[string]string, + foreseen, unplaced, handedOnly *[]string, handed *handedFiles) ([]map[string]any, error) { + // **A setting is judged where it is stored, and an impossible one costs a module, not a + // machine** (novox/hq ADR 0163, rule 6). A definition that moved under a stored setting makes + // this module uncomposable; it is left out of the declaration — its held things kept, its + // containers untouched, the machine told so by name — and the machine is told everything else. + // Before placing, because a placement is a setting too. + left := r.LeftOut(with.Settings, with.Adopted) + kept := make([]Manifest, 0, len(r.Modules)) + var stillBackedUp []Manifest + for module, why := range with.NotPlaced { + if _, already := left[module]; !already { + left[module] = why + } + } + for _, m := range r.Modules { + if why, isLeft := left[m.Module]; isLeft { + if leftOut != nil { + leftOut[m.Module] = why + } + // **Its data is still copied** (novox/hq ADR 0262): a module left out runs nothing new, and + // the data it already holds on the machine is the reason to keep copying it. Only its data, + // as the backup holder's lines are derived from it, and the directories they name. + stillBackedUp = append(stillBackedUp, backupView(m)) + continue + } + kept = append(kept, m) + } + r.Modules = kept + + // Every manifest is placed first (novox/hq ADR 0112): the maps naming where its bindings, + // credentials and contributions land are resolved against this node's directories, so every + // reader below — the binding files, the sealed secrets, the grant paths a contribution + // names — sees a concrete place and none learns the vocabulary. + // Into a fresh slice, never the caller's: one resolution may compose for many nodes, and a + // slice element written in place would carry the first node's places into the second's. + placed := make([]Manifest, len(r.Modules)) + for i, m := range r.Modules { + var err error + if placed[i], err = placedManifest(m, with); err != nil { + return nil, err + } + } + r.Modules = placed + + // Where each provision's credentials land, so a contribution can name the file rather than + // carry a value the mesh does not have. + directories := map[string]string{} + for _, m := range r.Modules { + for provision, where := range m.Grants { + directories[provision] = where + } + } + given, err := r.contributions(with.Settings, with.Grants, directories) + if err != nil { + return nil, err + } + + // A workload on THIS machine contributes the port it declared, and the machine may have + // published it somewhere else (novox/hq ADR 0066). The mirror of the redirect below: 038 fixed + // what a consumer is TOLD about a provider, and this is what a workload TELLS the provider about + // itself — gitea declaring 3000, published as 20000:3000, and the co-located proxy dialling 3000, + // where nothing listens, for every request. + // + // **The contributing module's assignment, never the provider's.** The port belongs to the + // workload; using the map of whatever answers the requirement would move a route to wherever the + // proxy happens to be published. A contribution carried here from another machine is left exactly + // as it is — its port is that machine's to assign, and this map knows nothing about it. + for provision := range given { + for i := range given[provision] { + said := &given[provision][i] + if said.Node != "" && said.Node != r.Node { + continue + } + said.Values = atMachinePort(said.Values, said.From, with.Ports) + } + } + + // A provider answered on this same machine never passed through the walk that works out what a + // provider on ANOTHER machine serves (novox/hq 04-ISSUES/038, ADR 0066). That walk does two + // things — it derives the served facts with the provider node's port assignments, and it settles + // them with that node's settings layers — and the same-node paths (resolve.go's servedHere, and + // here() below) did neither, because both run while resolving, before either is known. + // + // The port was the first half to be noticed (038). The second half is worse and quieter: a served + // value the OPERATOR supplied — a certificate authority's root, which is the one kind of value a + // manifest cannot carry, because it differs on every mesh — arrived as the manifest's empty + // default. A proxy then wrote an empty CA bundle, fell back to the system trust store, could not + // verify the internal authority, and stopped issuing with nothing saying why. + // + // So it is re-derived here, in full, exactly as the cross-node path derives it. **After the + // closure is known**, which also removes an order dependence: the resolver built these needs + // mid-walk from whichever modules had been chosen by then, so what a co-located binding carried + // depended on the order somebody happened to assign things in. + for i := range r.Needs { + if r.Needs[i].ByRecord || r.Needs[i].From != r.Node { + continue + } + serves, answered, err := r.servedOnThisMachine(r.Needs[i].Name, with) + if err != nil { + return nil, err + } + if answered { + r.Needs[i].Serves = serves + } + } + + // Once, from every module's listens -- not per module. A module receiving only its own ports + // would write a rule set that closed every other module on the machine. Each module's per-node + // exposure settings override its listens' source first (novox/hq ADR 0046). + rules, err := r.Rules(with) + if err != nil { + return nil, err + } + // **A machine that has not said which links face outside is sent no filter** (novox/hq ADR + // 0140). The whole chain is written around those links: with none, the rule that lets this + // machine's own guests keep working would name an empty set, which nftables refuses, and a rule + // set that does not load is a machine filtering nothing while its unit reports success. Refused + // here, where a person reads it, rather than on the machine — and the machine keeps the filter + // it already has. + if filters := r.filtersHere(); filters != "" && len(with.OutwardLinks) == 0 { + return nil, fmt.Errorf( + "%s cannot be sent a filter: it has not reported which of its links face outside, and "+ + "every rule in the chain is written around them. It reports that on each apply; "+ + "`node show %s` says whether it has. Until then %s is not sent, and the machine "+ + "keeps the filter it has", r.Node, r.Node, filters) + } + filtering := AsNftablesWithLocal(rules, with.Mesh, r.PublicDomain != "", with.Foundation, + with.OutwardLinks, with.TunnelInterface, with.LocalNetworks) + + // **A variable two modules set is refused whether or not anything places it** (novox/hq ADR + // 0203 §5): the account has one environment, and a machine whose holder arrives later should not + // be the moment two modules are found to disagree about it. + if err := variablesSetOnce(r.Modules); err != nil { + return nil, err + } + + generated, err := r.generatedHere(with) + if err != nil { + return nil, err + } + + var out []map[string]any + for _, m := range r.Modules { + if with.Adopted && m.Filtering != nil { + // Nothing of a module that loads a filter, on an adopted node: its table would drop + // by default and hold accepts, and the found firewall stays in force. Every resource, + // not only the rule set — its service must not run, and a node returned to adopted + // stops it by the ordinary removal of what is no longer declared. + continue + } + resources := m.Resources + + // What the mesh computes for this module goes FIRST, before the module's own resources. + // + // **Order is stated, not derived — the host does not sort** (novox/hq ADR 0005), so + // whatever the mesh writes down is the order a machine applies. A module's service or + // container routinely depends on one of these files; nothing here ever depends on a + // module's resources, because none of it is computed from them. + // + // Appended, this was wrong in a way that only showed on the first apply and then healed: + // the service started before its certificate or its rule set existed, failed, and the next + // reconcile fixed it. A fault that repairs itself on the second attempt is worse than one + // that does not, because what gets remembered is that it works. + var first []map[string]any + if f := m.Filtering; f != nil { + first = append(first, map[string]any{ + "id": FilteringID(), "type": "file", "path": f.Into, + "content": filtering, "mode": "0600", + }) + } + // The node's fail2ban jails, composed from every module it runs (novox/hq to-be 31), written + // where the intrusion-prevention holder owns them. Like the rule set above: gathered from all + // modules, written by the one that holds the role. + if j := m.Jailing; j != nil { + first = append(first, jailsInto(r.Modules, j)...) + } + if c := m.Certificate; c != nil { + if with.Certificate == "" { + // Asked for and not issued. Refused rather than skipped: a module that serves TLS + // with no certificate does not start, and the reason is somewhere else entirely. + return nil, fmt.Errorf( + "%s wants a certificate for this machine and none was issued", m.Module) + } + first = append(first, map[string]any{ + "id": CertificateID(), "type": "file", "path": c.Into, + // Public. It travels in the open like any other file, because it is a statement + // about a key rather than the key. + "content": with.Certificate, "mode": "0644", + }) + if c.Authority != "" { + first = append(first, map[string]any{ + "id": AuthorityID(), "type": "file", "path": c.Authority, + "content": with.Authority, "mode": "0644", + }) + } + } + if m.BusUsers != "" { + // **The claim authorises it, not the field.** This file holds every user's password + // hash, so a module that could ask for it could read every credential on the bus. + // Checked from this manifest alone, which is the cheapest check there is: whether some + // other module also claims the seat is resolution's business elsewhere, and one holder + // mesh-wide is already guaranteed. + if !m.ClaimsSeat("mesh-broker") { + return nil, fmt.Errorf( + "%s asks for the mesh's user list and does not claim mesh-broker. That file "+ + "holds every user's password hash, so the seat is what authorises it", + m.Module) + } + if with.BusUsers == "" { + // Asked for and not composed. Refused rather than skipped, for the reason a + // certificate is: a bus with no user list refuses every connection in the mesh, and + // an empty file would look like a configuration problem on the machine. + return nil, fmt.Errorf( + "%s holds mesh-broker and the mesh composed no user list, so the bus would "+ + "refuse every connection", m.Module) + } + first = append(first, map[string]any{ + "id": BusUsersID(), "type": "file", "path": m.BusUsers, + "content": with.BusUsers, + // Readable by the server and nothing else. Hashes rather than passwords, so this is + // not a set of working credentials — but a list of every user in the mesh is worth + // keeping to the one process that needs it. + "mode": "0600", + }) + } + // **A secret family's members, as given** (novox/hq ADR 0283): one file each at the family's path, and + // nothing for a member not given — the mesh never makes one, and the module says it waits for it. + for _, name := range sortedKeys(with.Needed[m.Module]) { + own, family, ok := m.OwnSecrets.Lookup(name) + if !ok || family == "" || with.Needed[m.Module][name] == "" { + continue + } + // A per-secret owner (novox/hq ADR 0436 §1) stands in for the module's secrets-owner, when + // the manifest named one for this secret (or the family it belongs to). + owner := m.SecretsOwner + if own.Owner != "" { + owner = own.Owner + } + first = append(first, ownedBy(owner, map[string]any{ + "id": NeedID(name), "type": "file", "path": own.Path, "sealed": with.Needed[m.Module][name], + })) + } + // **A handed module's own secrets are the vault's** (design 57 V4, issue 647): each file holding one is + // declared placed and without a value, filled by the machine's provisioner client from the vault; the + // controller makes and seals none of them here. + handsSecrets := with.Handing(m) + for _, name := range sortedKeys(m.OwnSecrets.Plain()) { + if handsSecrets { + owner := m.SecretsOwner + if m.Module == RuntimeModule && r.Account != "" { + owner = r.Account + } + if o := m.OwnSecrets[name].Owner; o != "" { + owner = o + } + first = append(first, ownedBy(owner, map[string]any{ + "id": NeedID(name), "type": "file", "path": m.OwnSecrets[name].Path, HandedSecretField: name, + })) + continue + } + // **A core credential is the vault's box** (ADR 0349 §16, issue 755): carried as the vault sealed it to this + // machine, which the controller cannot open; the controller's own sealed value is not sent with it. + if box, ok := with.vaultSealedOf(m.Module, name); ok { + if string(box) == string(ForeseenVaultBox) { + *foreseen = append(*foreseen, m.Module+"/"+name) + } + owner := m.SecretsOwner + if m.Module == RuntimeModule && r.Account != "" { + owner = r.Account + } + if o := m.OwnSecrets[name].Owner; o != "" { + owner = o + } + first = append(first, ownedBy(owner, map[string]any{ + "id": NeedID(name), "type": "file", "path": m.OwnSecrets[name].Path, VaultSealedField: box, + })) + continue + } + sealed := with.Needed[m.Module][name] + if sealed == "" && with.Foreseen[m.Module][name] { + // Not made, and the next send makes it: composed with a stand-in so that whatever + // else this composition would refuse is still found (novox/hq issue 275). + sealed = ForeseenSealed + *foreseen = append(*foreseen, m.Module+"/"+name) + } + if sealed == "" && m.OwnSecrets[name].IssuedBy == IssuedOutside { + // **Not given, and the mesh never makes it** (novox/hq issue 401, ADR 0228): no file, as for a + // family's member (ADR 0283), so the module reads its absence as not given. + continue + } + if sealed == "" { + // Declared and not made. Refused rather than skipped: a module whose own + // credential is silently absent starts, fails to authenticate, and the reason is + // three layers away from the machine reporting it. + return nil, &NotMadeError{Module: m.Module, Name: name} + } + // The runtime's credential belongs to the account the runtime runs as (novox/hq ADR 0175, + // to-be 38 WP3): its process is composed `user: ` where the node has one, and a + // root-owned 0600 file is one that process cannot read. Composed here rather than said in + // the manifest, because a manifest cannot say ${machine:account} safely — a node with no + // account has nothing to resolve it to, and then the runtime runs as root and the file + // stays root's. + owner := m.SecretsOwner + if m.Module == RuntimeModule && r.Account != "" { + owner = r.Account + } + if o := m.OwnSecrets[name].Owner; o != "" { + owner = o + } + first = append(first, ownedBy(owner, map[string]any{ + "id": NeedID(name), "type": "file", "path": m.OwnSecrets[name].Path, "sealed": sealed, + })) + } + // This module's tools bundles, where the machine runs the node's tool runtime (novox/hq + // ADR 0175, to-be 38 WP2). Mesh-computed like everything above it, and before the module's + // own resources for the same reason: the runtime's process names the files inside these + // and is restarted when one changes, so they are on the machine before it is. + if r.runtimeHere() { + first = append(first, bundleArchives(m)...) + } + // Operator-owned paths this module is granted use of (novox/hq ADR 0051). Written before + // the module's own resources, and so before the container that mounts them: the host must + // find each present — refusing clearly if the operator has not provided it — before it + // starts anything that depends on it. The mesh creates, chowns and reconciles none of it; + // an `access` resource says only *this path must exist, and this module reaches it*. + // Where each is on THIS machine is the assignment's (novox/hq issue 153): placed by id + // where the operator said, the definition's default otherwise, refused where neither. + accesses, accessPaths, err := accessesFor(m, with.Settings[m.Module]) + if err != nil { + return nil, err + } + for _, a := range accesses { + first = append(first, map[string]any{ + "id": AccessID(a.Path), "type": "access", "path": a.Path, "mode": a.Mode, + }) + } + for _, to := range m.SecretRequirements() { + for _, file := range m.SecretFiles(to) { + var found *Needed + for i, n := range r.Needs { + // **This module's need, not the provision's** (novox/hq 04-ISSUES/022). Matching + // on the name alone, every consumer of a provision took whichever credential + // happened to be last in the list — so on a node with two of them, one module + // would be given the other's password and fail to authenticate with a valid + // credential belonging to somebody else. And this file's local name, where the + // module keeps several (ADR 0094). + if n.Name == to && n.For == m.Module && n.Local == file.Local { + found = &r.Needs[i] + } + } + // **A credential the vault keeps for this holder alone** (issue 757): where the module is handed, its + // file is declared placed and without a value, and the machine's client fills it from the vault. + if found != nil && found.IssuedToHolder && with.Handing(m) { + first = append(first, ownedBy(m.SecretsOwner, map[string]any{ + "id": SecretID(SecretLocal(to, file.Local)), "type": "file", "path": file.Path, + HandedProvisionField: PlanProvision{Name: SecretLocal(to, file.Local), Provision: to, + Local: file.Local, Provider: found.From, ProviderModule: found.Module}, + })) + continue + } + if found != nil && found.ByRecord && found.Sealed == "" && !found.Manager { + // Answered by a record whose key has not been supplied since this consumer was + // put on it. **Refused, not skipped.** The mesh discarded the plaintext when the + // key was accepted and cannot seal another, so a machine that resolved cleanly + // would receive no file at all and fail at whatever tried to read it — which is + // the outcome ADR 0024 exists to avoid, arrived at politely. + // + // The manager holder is the one exception (novox/hq ADR 0050): an empty refresh token + // is a licence whose manager has not adopted one yet, a real waiting state rather than + // a lost key. It falls through to the skip below — its bound facts (carrying the + // manager's public key) are still delivered, which is what adoption needs to seal the + // first refresh token. + return nil, fmt.Errorf( + "%s on this machine uses the licence %q and no key has been sealed to it. "+ + "The mesh cannot make one; supply it again with `licence key %s`", + m.Module, found.From, found.From) + } + if found == nil || found.Sealed == "" { + // Answered on this machine, or answered by a node the mesh could not seal to. + // Nothing to write either way, and writing an empty credential file would be + // worse than none: something would read it and fail authenticating. + continue + } + first = append(first, ownedBy(m.SecretsOwner, map[string]any{ + "id": SecretID(SecretLocal(to, file.Local)), "type": "file", "path": file.Path, + "sealed": found.Sealed, + })) + } + } + for _, to := range sortedKeys(m.Grants) { + for _, g := range with.Grants { + if g.Provision != to { + continue + } + if g.From == "" { + // Nothing on that machine asks for this any more. Skipped here rather than + // where grants are gathered, so the rule holds whoever gathers them. + // + // **This is how a credential is withdrawn.** The provisioner removes what + // nobody asks for, and it can only do that if the mesh stops asking — a + // consumer that was unassigned would otherwise keep a working login for ever, + // and nothing would say so. + continue + } + first = append(first, ownedBy(r.provisionsAs(m), map[string]any{ + // One file per holder — the consumer's module with its local name after it + // where it keeps several (ADR 0094); the lab found two files with one id. + "id": GrantID(to, g.Consumer+"."+holderAs(g.From, g.Local)), + "type": "file", + "path": grantPath(m.Grants[to], g.Consumer, holderAs(g.From, g.Local)), + "sealed": g.Sealed, + })) + } + } + for _, to := range sortedKeys(m.Binds) { + var found *Needed + for i, n := range r.Needs { + if n.Name == to && n.For == m.Module { + found = &r.Needs[i] + } + } + if found == nil { + // Answered on this same machine rather than from elsewhere in the mesh. + // + // **Still written.** It used to be skipped, reasoning that "it is on this node" + // is a fact nobody needs — and that is true of the *location* and false of + // everything beside it. A binding also carries what the provider said a consumer + // must know, which is the port; a consumer cannot invent that, and got an absent + // file with no explanation. A build machine sharing a node with the registry it + // pushes to sat in a loop saying it could not read its own binding. + here, err := here(r, to, with) + if err != nil { + return nil, err + } + if here == nil { + // Nothing in this node's set offers it either, so there is genuinely nothing + // to say. Resolution has already refused anything unanswerable, so this is a + // requirement met by the module itself. + continue + } + found = here + } + own, err := r.ownNames(m, to, with.Settings[m.Module]) + if err != nil { + return nil, err + } + as := ConsumerIdentity(r.Node, IdentitySource(m.Slug, m.Module)) + // What the provider derives for THIS consumer, filled here where the consumer is + // known (novox/hq ADR 0202). The same fill knownFor does below, so the binding file + // and the module's `${bound:…}` substitutions cannot say different things. + told := *found + told.Serves, err = ServedTo(told.Serves, as) + if err != nil { + return nil, fmt.Errorf("%s is told about %s: %w", m.Module, to, err) + } + file, err := boundFile(told, m.Binds[to], as, own) + if err != nil { + return nil, err + } + first = append(first, file) + } + for _, to := range sortedKeys(m.Receives) { + file, err := receivedFile(to, m.Receives[to], given[to]) + if err != nil { + return nil, err + } + first = append(first, file) + if received[m.Module] == nil { + received[m.Module] = map[string][]Contribution{} + } + // Empty rather than absent when nobody contributed, for the reason the file is + // written empty: "nothing asked" and "never told" want different responses. + received[m.Module][to] = append([]Contribution{}, given[to]...) + } + if m.Keeps != "" && with.Kept != nil { + file, err := keptFile(m.Keeps, with.Kept) + if err != nil { + return nil, err + } + // Read by the module as its secrets' owner when it names one: the vault, as its own account (issue 723). + first = append(first, ownedBy(m.SecretsOwner, file)) + } + if m.Computed != "" { + mine, part := generated[m.Module] + if !part { + // Assigned, and not yet part of what this generates. Nothing to put on the + // machine, which is different from an error: a node given the network module + // before it has an address is in exactly that state, briefly. + continue + } + resources = mine + } + + // Now, and not before: a module whose resources are computed replaces them wholesale, and + // merging earlier would throw away the files it still needs. + // + // **Except the module's own accounts, which go before even those** (novox/hq issue 213). What + // the mesh computes may belong to one: a module whose code runs as an account it declares has + // its secrets written owned by that account, and a file given to a user the machine does not + // have yet fails — so on the first apply the secrets were refused, the process started without + // them, and the second apply healed it, which is the fault the paragraph above describes. + // An account depends on nothing the mesh computes. + accounts, rest := accountsFirst(resources) + resources = append(append(accounts, first...), rest...) + // **A file delivered only while its optional requirement is bound is left out while it is + // absent** (novox/hq ADR 0330 §5), and the field is dropped from the rest: the node-engine + // parses strictly and knows no such field. + absent := r.AbsentFor(m.Module) + resources = withoutAbsent(resources, absent) + + // No container is given the mesh's names (novox/hq ADR 0148). It used to be: every + // container got the whole roster as `--add-host` entries at creation, and a name that + // moved afterwards was wrong inside it for as long as it ran (issues 109, 135) — and once + // the roster was made part of a container's identity so that could be caught, one name + // moving anywhere replaced every container in the mesh (issue 151). A container resolves a + // mesh name through its machine's resolver at the moment it asks, which the runtime is + // told once per machine, as a file, by the resolver's own module. The names a module + // declares for itself are its own and stay exactly as written: they are part of what the + // module is, and the mesh does not know what they mean. + // What this module may name from inside one of its own files. Gathered once per module + // rather than per file, because it is a fact about the module. + sealed, err := sealedFor(m, r.Needs, with) + if err != nil { + return nil, err + } + // And what its bindings say, for the half of a connection that is not secret. + known, err := knownFor(m, r.Needs, r.Node) + if err != nil { + return nil, err + } + // A requirement answered on this same machine is not in r.Needs — its binding file is + // written from `here` (above) — and so `${bound:…}` could not name it, though the file + // beside it said the same facts. Filled from the same answer, so the two cannot disagree. + for _, want := range m.Wants() { + if _, has := known[want]; has { + continue + } + answered, err := here(r, want, with) + if err != nil { + return nil, err + } + if answered == nil { + continue + } + local := *answered + local.For = m.Module + here, err := knownFor(m, []Needed{local}, r.Node) + if err != nil { + return nil, err + } + for provision, values := range here { + known[provision] = values + } + } + // And the providing machine's private address, beside its name (novox/hq ADR 0194). A name is + // what nearly every consumer wants; the one that cannot use it is a machine's resolver + // configuration, which must reach the resolver before it can resolve anything — the resolver's + // own name included. Absent when the machine has no address yet, so a file naming it is refused + // rather than written with a blank where an address belongs. + for _, values := range known { + if address := with.Machines[values["at"]]; address != "" { + values["address"] = address + } + } + // And what the module is called through each requirement it contributes to (novox/hq + // 04-ISSUES/122) — the same composition its binding file carries. + for provision, values := range known { + own, err := r.ownNames(m, provision, with.Settings[m.Module]) + if err != nil { + return nil, err + } + withOwnNames(values, own) + } + // And where this node places the directories the module declared without a path + // (novox/hq ADR 0112) — resolved once per module, named by ${dir:…} from any resource. + // — and, on an adopted machine, where the assignment says they already are, with the + // owner the data already has (novox/hq issue 153). Malformed placements are refused here. + placed, err := Places(m, with.Settings[m.Module]) + if err != nil { + return nil, err + } + dirs := dirsFor(m, with) + // And the machine underneath, which no binding of its own can tell it. + thisMachine := machineFacts(r, with.Names, with.MeshRange) + + // **A definition that already holds the answer transcribed it** (novox/hq ADR 0202). + // Judged over what the module itself declares, and before anything is substituted: the + // mesh's own generated files — the binding, the contributions — legitimately carry the + // derived value, and after substitution so does every consumer's file, so this is the one + // moment the two can be told apart. + for _, own := range m.Resources { + if err := notTranscribed(own, known, m.Module); err != nil { + return nil, err + } + } + // **And every placeholder no pass fills where it stands is refused** (novox/hq issue 231): + // judged here, over the definition as written and before any pass, by the function the + // catalogue check runs — so a manifest registered by an older binary is judged too, and + // what a setting or a binding later puts into a file is its software's text, never swept. + for _, own := range m.Resources { + if problems := unconsumedPlaceholders(m.Module, own); len(problems) > 0 { + return nil, fmt.Errorf("%s", problems[0]) + } + } + + // Which of this module's files carry a secret, for the rule that a container may not read + // one of them as its environment without saying so (ADR 0086, issue 041). + secretFiles := secretFilesOf(resources) + + // Which of this module's resources its preparation runs before, if it prepares anything. + prepareBefore := preparationTarget(m) + + // Which found networks this machine's setting keeps for each of its containers (novox/hq + // ADR 0163, rule 4); judged above, so an invalid one is not here. + keptNetworks, _ := KeptNetworks(m, with.Settings[m.Module], with.Adopted) + + // **A moved module's templated files are the provisioner's** (novox/hq ADR 0349 §4, §6, design 57 P6): on a + // machine that can take them, each is declared placed and without its text, and no pass below fills it. + handing, err := handed.begin(r.Node, m, dirs, accessPaths, thisMachine, with, r.Capabilities, r.Needs, + backupPlacer{append(append([]Manifest(nil), r.Modules...), stillBackedUp...), with, thisMachine, r.Capabilities, + unplaced}) + if err != nil { + return nil, err + } + // **What the mesh calls the module through each provision it requires** (issue 792): the names its binding + // file carries, composed by the same call, handed to the provider through the plan's entry. + if _, ok := handed.plans[m.Module]; ok && handing != nil { + for _, to := range requirementsContributedTo(m) { + own, err := r.ownNames(m, to, with.Settings[m.Module]) + if err != nil { + return nil, err + } + handed.name(m.Module, to, own) + } + } + // A handed collector is filled from items alone: each old form its collections would have held is said + // (design 57 L1). + if plan, ok := handed.plans[m.Module]; ok && handing != nil && len(plan.Collections) > 0 { + *unplaced = append(*unplaced, oldFormsLeftOut(append(append([]Manifest(nil), r.Modules...), + stillBackedUp...), m, plan.Collections)...) + } + for _, unsettled := range resources { + if pr, ok := unsettled[HandedProvisionField].(PlanProvision); ok { + if handing == nil { + return nil, fmt.Errorf("%s: its credential of %s is handed to the provisioner and the module is "+ + "not: nothing would fill it", m.Module, pr.Provision) + } + marked := map[string]any{} + for k, v := range unsettled { + marked[k] = v + } + delete(marked, HandedProvisionField) + marked[HandedSecretField] = pr.Name + file, sec, err := handedSecretFile(marked, m, dirs, accessPaths, thisMachine, placed) + if err != nil { + return nil, err + } + file["id"] = m.Module + "." + fmt.Sprint(unsettled["id"]) + owner[fmt.Sprint(file["id"])] = m.Module + pr.Path = sec.Path + if err := handed.addProvisionFile(m.Module, pr); err != nil { + return nil, err + } + out = append(out, file) + continue + } + if unsettled[HandedSecretField] != nil { + if handing == nil { + return nil, fmt.Errorf("%s: its own secret %v is handed to the provisioner and the module is "+ + "not: nothing would fill it", m.Module, unsettled[HandedSecretField]) + } + file, sec, err := handedSecretFile(unsettled, m, dirs, accessPaths, thisMachine, placed) + if err != nil { + return nil, err + } + file["id"] = m.Module + "." + fmt.Sprint(unsettled["id"]) + owner[fmt.Sprint(file["id"])] = m.Module + if err := handed.addSecretFile(m.Module, sec); err != nil { + return nil, err + } + out = append(out, file) + continue + } + if handing != nil && fmt.Sprint(unsettled["type"]) == "file" { + if text, _ := unsettled["content"].(string); len(secretsUsed(text)) > 0 && + !handing[fmt.Sprint(unsettled["id"])] { + return nil, fmt.Errorf("%s: its file %v names a secret in its text, and a handed module's "+ + "secrets reach only the files its provisioner client fills: keep it in a template "+ + "(design 57 V4)", m.Module, unsettled["id"]) + } + } + if handing[fmt.Sprint(unsettled["id"])] { + file, line, err := handedFile(unsettled, m, dirs, accessPaths, thisMachine, placed, with) + if err != nil { + return nil, err + } + file["id"] = m.Module + "." + fmt.Sprint(unsettled["id"]) + owner[fmt.Sprint(file["id"])] = m.Module + if err := handed.add(m.Module, line); err != nil { + return nil, err + } + out = append(out, file) + continue + } + if fmt.Sprint(unsettled["type"]) == "file" { + if _, left := consumersHandedOnly(m, unsettled, with); left { + *handedOnly = append(*handedOnly, consumersLeftOutLine(m, unsettled)) + continue + } + } + resource, err := ApplySettings(unsettled, with.Settings[m.Module]) + if err != nil { + return nil, err + } + copied := map[string]any{} + for k, v := range resource { + copied[k] = v + } + if networks, keeps := keptNetworks[fmt.Sprint(copied["id"])]; keeps { + // The container also joins the found network the setting names, so a neighbour + // that resolves it there keeps resolving it. Passed to the host as its own field, + // which it joins after the container is made. + joins := make([]any, 0, len(networks)) + for _, n := range networks { + joins = append(joins, n) + } + copied["networks"] = joins + } + if err := refuseSecretsInEnvironment(copied, secretFiles, m.Module); err != nil { + return nil, err + } + // Said in the catalogue, not on the machine: the host parses strictly and knows no + // such field, and the reason is for a reader of the manifest. + delete(copied, SecretsInEnvironment) + delete(copied, NamesOnPurpose) + delete(copied, TrustedField) + // A process's bus stays controller-side (novox/hq ADR 0436 §1): never composed onto the + // node-engine's process resource, whose wire declaration is mesh-host's own, vendored, with a + // strict decoder that refuses an unknown key. + delete(copied, ProcessBusField) + // **An operator's value, from the assignment** (novox/hq ADR 0112, ADR 0155): what a + // definition may not carry because it is true of one installation only. Filled from + // the same layers a mergeable file takes, and refused when no layer set it. + if err := unsetInConfigWords(settingInto(copied, WithDefaults(m, with.Settings[m.Module]), m.Module), m); err != nil { + return nil, err + } + // **Placed before anything reads a path.** A pathless directory receives the path + // this node resolves for it, and every ${dir:…} — in paths, mounts, content and + // environment — becomes that path, so what follows sees only concrete places + // (novox/hq ADR 0112). The host receives paths exactly as it always has. + if err := dirInto(copied, dirs, m.Module); err != nil { + return nil, err + } + // The operator's data the same way: ${access:…} becomes where this node keeps it, + // and a placed directory takes the owner the assignment said (issue 153). + if err := accessInto(copied, accessPaths, m.Module); err != nil { + return nil, err + } + ownerInto(copied, placed) + // **After settings, and that is the whole reason it is here.** A module's file + // content is where a setting lands, so a placeholder may only exist once the setting + // has been put in — filling secrets first would look at content that is not yet what + // the machine receives. + if err := intoFile(copied, sealed, outsideNotGiven(m, with.Needed[m.Module]), m.Module); err != nil { + return nil, err + } + // **Substituted here, not on the machine.** A bound value is not secret — the mesh + // holds it in the clear — so there is nothing for the host to be the only witness of, + // and sending it already filled in means the host learns no new field. + if err := boundInto(copied, known, m.Module, absent); err != nil { + return nil, err + } + // And what the module could not have written: the machine it turned out to be + // assigned to. Beside the bound values because it is the same kind of fact — the + // mesh's own, held in the clear — and because a module that must name itself to + // something else has no binding to learn it from (novox/hq ADR 0066). + // Which port this machine gave it, for a module that binds directly rather than + // through a runtime that can remap (ADR 0038). Applied before the machine's facts so + // a refusal names the port rather than whatever came after it. + if err := portInto(copied, m.Module, m.Listens, with); err != nil { + return nil, err + } + // And where this machine put the foundation's servers, for the one module that + // reaches them by seat rather than by binding (novox/hq 04-ISSUES/102). + if err := seatInto(copied, m.Module, with); err != nil { + return nil, err + } + if err := machineInto(copied, thisMachine, m.Module); err != nil { + return nil, err + } + if err := pinned(copied, m.Module); err != nil { + return nil, err + } + // And the other half of the same question: an image nobody has published yet is named + // by an artifact rather than by a placeholder digest, and is just as unrunnable. + if err := built(copied, m.Module); err != nil { + return nil, err + } + // What the mesh built is kept by digest and path; the store's address is this + // network's now, composed here and never recorded (novox/hq 04-ISSUES/102). + if err := artifactsInto(copied, m.Module, with); err != nil { + return nil, err + } + publishedOn(copied, m.Module, with) + // How it is ready, in the node-engine's words: its endpoint as the port this machine + // published it on — or not sent at all to an engine older than the field (ADR 0240). + healthInto(copied, m, with) + // And whether it is busy (novox/hq ADR 0335): sent only to an engine that asks it. + busyInto(copied, with) + // And a user's `root` (novox/hq ADR 0266): sent only when it asserts something, to an engine + // that judges it. + rootInto(copied, with) + // The account's environment and every module's shell code, where this module holds the + // seat that places them (novox/hq ADR 0203, ADR 0204). Gathered from every module on + // the node, as the jails are, and **last of every placeholder pass**: shell code is a + // shell's own syntax, full of `${…}` no pass above should ever be shown. + contributing := append(append([]Manifest(nil), r.Modules...), stillBackedUp...) + if err := contributionsInto(copied, m, contributing, thisMachine, with, r.Capabilities, unplaced); err != nil { + return nil, err + } + copied["id"] = m.Module + "." + fmt.Sprint(resource["id"]) + // A service saying what it reflects names resources within its own module, so those + // are prefixed too or they would point at nothing. + // + // **Unless it already names one.** A module may reflect a file another module put on + // the machine — the case this exists for is a resolver restarting when the mesh + // rewrites the names, which are computed by the mesh and belong to its module, not to + // the daemon's. Written `.`, and a dot is what marks it as already + // answered: prefixing it again would point at nothing, silently, and the daemon would + // serve the old names for ever while everything reported success. + if renamed := reflectsRenamed(m.Module, resource["restart-on"]); renamed != nil { + copied["restart-on"] = renamed + } + // And what it is reloaded on, by the same rule (novox/hq ADR 0102): an id left + // unprefixed matches nothing, and the service is never reloaded. + if renamed := reflectsRenamed(m.Module, resource["reload-on"]); renamed != nil { + copied["reload-on"] = renamed + } + // And which of its module's containers a scheduled step holds still (novox/hq ADR 0189). + // **The loudest of the three when it is missed.** An unprefixed `restart-on` matches + // nothing and a service quietly never restarts; an unprefixed `while-stopped` names a + // container the declaration does not contain, and the host refuses the whole + // declaration — so the machine takes nothing at all, for every push, until this is + // right. That is what it did on the control node (2026-10-04). + if renamed := reflectsRenamed(m.Module, resource[WhileStopped]); renamed != nil { + copied[WhileStopped] = renamed + } + // And what a process replaces (novox/hq issue 213): a resource of this module's that it + // no longer declares, named as the host recorded it, or the host hands nothing over and + // removes it first. + if renamed := reflectsRenamed(m.Module, resource["replaces"]); renamed != nil { + copied["replaces"] = renamed + } + // **What reads one of this module's own secrets is restarted when it changes** (novox/hq + // issue 203, issue 206). A credential is re-issued by the mesh, and a container that + // mounted the old file keeps the old one open: the build machine ran for an hour on a + // credential the mesh had replaced, because its manifest restarted it on its + // environment file and nobody had thought to name the credential too. Composed here so + // no manifest has to say it, for a container or a daemon that names the secret's path. + if reads := secretsReadBy(copied, m, with.Needed[m.Module]); len(reads) > 0 { + copied["restart-on"] = withRestartOn(copied["restart-on"], reads) + } + // **A version prepares its state before it runs** (novox/hq ADR 0135). Derived from the + // module's own resource rather than declared beside it: what prepares the state is the + // module's own code, so what it is given has to be what that code is given — and a + // second resource written by hand is a second copy to drift from the first. Placed + // immediately before it, because a run-once step stops everything the declaration + // places after it (ADR 0052), which is how a version whose preparation failed does not + // serve. + if prepareBefore != "" && fmt.Sprint(resource["id"]) == prepareBefore { + step := prepared(copied) + owner[fmt.Sprint(step["id"])] = m.Module + out = append(out, step) + } + owner[fmt.Sprint(copied["id"])] = m.Module + out = append(out, copied) + } + + // **What only the mesh knows, where this module asked for it.** The graph is the control + // plane's; making a name resolve is the module's software. Emitted as ordinary files under + // this module's name, so they are applied, reported and removed exactly as anything else + // it declares. + // The resolver file is the node-resolver holder's where one is held here, and the uplink's + // holder steps back from it (novox/hq ADR 0247). + given, err := FactsFrom(stepsBack(m, r.Modules), r, with) + if err != nil { + return nil, err + } + for _, fact := range given { + fact["id"] = m.Module + "." + fmt.Sprint(fact["id"]) + owner[fmt.Sprint(fact["id"])] = m.Module + out = append(out, fact) + } + } + // The node's tool runtime, last (novox/hq ADR 0175, to-be 38 WP2.3): one process loading every + // bundle delivered above and holding the credential sealed above, so both exist before it starts + // — the order written here is the order the machine applies. + if r.runtimeHere() { + process, err := r.runtimeProcess(with) + if err != nil { + return nil, err + } + owner[fmt.Sprint(process["id"])] = RuntimeModule + out = append(out, process) + // And a runtime of its own for each module of its own account, after it (novox/hq ADR 0259 §8). + owns, err := r.ownRuntimes(with) + if err != nil { + return nil, err + } + for _, p := range owns { + id := fmt.Sprint(p["id"]) + owner[id] = strings.TrimSuffix(id, "."+OwnRuntimeID()) + out = append(out, p) + } + // What each module's bundles are given is read as the account the runtime runs as — not what a + // module of its own account is given, which its own account reads. + words := map[string]map[string]string{} + for _, m := range r.Modules { + if m.RunsAs != "" { + continue + } + w, err := bundleWords(m, with) + if err != nil { + return nil, err + } + words[m.Module] = w + } + // And a file a module's words name is one the runtime is restarted for when it changes. + if named := givenTo(out, owner, words, r.Account); len(named) > 0 { + restarts, _ := process["restart-on"].([]any) + seen := map[string]bool{} + for _, id := range restarts { + seen[fmt.Sprint(id)] = true + } + for _, id := range named { + if !seen[id] { + restarts = append(restarts, id) + seen[id] = true + } + } + process["restart-on"] = restarts + } + // **And what a module of its own account is given, its own account reads** (issue 723): the files and + // directories its words name that say no owner, given to the account it runs as, never the operator's. + for _, m := range r.Modules { + if m.RunsAs == "" { + continue + } + w, err := bundleWords(m, with) + if err != nil { + return nil, err + } + givenTo(out, owner, map[string]map[string]string{m.Module: w}, m.RunsAs) + } + } + if with.Adopted { + // First, before anything a module declares: what the mesh needs reachable, then its guard. + // The order a machine applies is the order written here. + ours := Openings(rules, with.Foundation, Published(out)) + ours = append(ours, GuardResources(r.guarded(out, owner, rules, with))...) + out = append(ours, out...) + } + return out, nil +} + +// guarded is what the mesh's guard refuses on an adopted node (novox/hq ADR 0103): derived, and +// for taken modules only. +// +// For each taken module, every machine port its containers publish that the filter would admit +// from the private network only — a published port is forwarded, not received, so a found +// firewall filtering only what it receives never sees it — together with the ports the module's +// manifest guards explicitly (the store's port, the broker's management port), wherever the +// machine put them. A port of a module assigned but not taken is not guarded: it may still be the +// predecessor's, serving the predecessor's other machines. The foundation's ports are admitted +// from everywhere and are never guarded. +func (r Resolution) guarded(out []map[string]any, owner map[string]string, rules []Rule, + with Rendering) []int { + meshOnly := map[int]bool{} + fromEverywhere := map[int]bool{} + for _, rule := range rules { + if rule.Protocol != "tcp" { + continue + } + switch rule.From { + case FromMesh: + meshOnly[rule.Port] = true + case FromEverywhere: + fromEverywhere[rule.Port] = true + } + } + for _, port := range with.Foundation { + delete(meshOnly, port) + fromEverywhere[port] = true + } + seen := map[int]bool{} + var ports []int + guard := func(at int) { + if !seen[at] { + seen[at] = true + ports = append(ports, at) + } + } + for _, m := range r.Modules { + if !with.Taken[m.Module] { + continue + } + var mine []map[string]any + for _, resource := range out { + if owner[fmt.Sprint(resource["id"])] == m.Module { + mine = append(mine, resource) + } + } + for outer := range Published(mine)["tcp"] { + if meshOnly[outer] { + guard(outer) + } + } + for _, want := range m.Guards { + at := with.machinePort(m.Module, want) + for _, resource := range mine { + if fmt.Sprint(resource["type"]) != "container" { + continue + } + listed, _ := resource["ports"].([]any) + for _, entry := range listed { + outer, inner, _, ok := mapping(fmt.Sprint(entry)) + if ok && inner == want { + at = outer + } + } + } + // Not what this node is told to open to everyone: a per-node exposure setting that + // widens a guarded port is the operator saying so, and declaring an opening for it + // and a guard dropping it would be one statement refusing the other. + if !fromEverywhere[at] { + guard(at) + } + } + } + sort.Ints(ports) + return ports +} + +// mapping reads a container's port mapping — `[address:]outer:inner[/protocol]`, the address +// possibly an IPv6 one in brackets — indexing from the end, so an address's own colons never +// shift the ports. Not ok for a short form or anything that is not a mapping. +func mapping(written string) (outer, inner int, address string, ok bool) { + written = strings.TrimSpace(written) + if cut := strings.LastIndex(written, "/"); cut >= 0 { + written = written[:cut] + } + parts := strings.Split(written, ":") + if len(parts) < 2 { + return 0, 0, "", false + } + inner, err := strconv.Atoi(parts[len(parts)-1]) + if err != nil { + return 0, 0, "", false + } + outer, err = strconv.Atoi(parts[len(parts)-2]) + if err != nil { + return 0, 0, "", false + } + return outer, inner, strings.Join(parts[:len(parts)-2], ":"), true +} + +// filtersHere is the module on this node that loads the machine's packet filter, or empty when none +// does. Named rather than counted: a refusal that says which module is one step from acted on. +func (r Resolution) filtersHere() string { + for _, m := range r.Modules { + if m.Filtering != nil { + return m.Module + } + } + return "" +} + +// Rules is the rule set this node's filter is derived from: every module's listens, what was +// computed for this machine, and each module's per-node exposure. The same answer whether the node +// is adopted or converged — the one loads it as a filter, the other declares it as openings. +func (r Resolution) Rules(with Rendering) ([]Rule, error) { + // A module whose settings cannot compose is left out of the declaration (novox/hq ADR 0163, + // rule 6), and out of the filter with it: nothing of it is declared, so nothing of it is let + // through. + left := r.LeftOut(with.Settings, with.Adopted) + exposure := map[string]map[int]string{} + for _, m := range r.Modules { + if _, isLeft := left[m.Module]; isLeft { + continue + } + e, err := Exposure(m, with.Settings[m.Module]) + if err != nil { + return nil, err + } + // And how far each endpoint reaches, which says the same thing to the filter and more + // besides (novox/hq ADR 0138). Folded in here rather than beside: the filter has one + // question — from where — and a reach answers it, so giving it two inputs would let them + // disagree. Reaches refuses a port that both name, so this cannot silently prefer one. + reaches, err := Reaches(m, with.Settings[m.Module]) + if err != nil { + return nil, err + } + // **Only for an endpoint the proxy does not serve.** A routed endpoint's port is how the + // proxy reaches it and nothing else (ADR 0045), so `public` there asks for a public name and + // says nothing about the port — opening it to the world as well would undo the arrangement + // the proxy exists for, and would silently reopen a port an operator had narrowed. + // + // Found by trying to express a real module: one whose routed name must be public and whose + // machine-side port must not be. Under one value for both, there was no way to say it. + routed := RoutedPorts(m) + for port, reach := range reaches { + if routed[port] { + continue + } + source, ok := FilterSource(reach) + if !ok { + return nil, fmt.Errorf("%s: %q is not a reach the filter can read", m.Module, reach) + } + if e == nil { + e = map[int]string{} + } + e[port] = source + } + if e != nil { + exposure[m.Module] = e + } + } + return r.Filtering(with.Generators, with.Ports, exposure) +} + +// Contribution is one module telling the answer to a requirement what it needs from it. +type Contribution struct { + // From is the module that said it, so the provider and a person reading the file can tell + // which route belongs to what. + From string `json:"from"` + // Node is the machine it said it from, empty when that is this one. + // + // A provision answered from anywhere in the mesh has consumers on other machines, and the + // provider has to know who they are — a database told to create a password and not who for + // cannot do anything with it. Contributions were node-local until this, which meant the one + // case that most needed them was the one they did not reach. + Node string `json:"node,omitempty"` + // At is where that machine is on the private network, empty when it is not on one or when it + // is this machine. + // + // The mesh knows it and a provider should not have to derive it. A reverse proxy is told + // *send traffic to this consumer* and has to open a connection — so without this every + // provider that reaches back to a consumer would have to know how the mesh names machines, + // which is a convention leaking into every module that implements a provision. + At string `json:"at,omitempty"` + // As is who this consumer is: what the provider should call the login it creates. + // + // **Said by the mesh rather than invented by the provisioner** (novox/hq 04-ISSUES/023). It + // used to be neither — the provisioner made a name, and the consumer, which has to present it + // to authenticate, had no way to learn it. One derivation reaches both ends, so they agree by + // construction. + // Omitted when there is none. A contribution that is not a credential grant — a module + // offering something to another on its own machine — has nobody to be identified to, and a + // field that is always present and usually empty teaches a reader to ignore it. + As string `json:"as,omitempty"` + // Secret is the file on this machine holding that consumer's credential, sealed to it. + // + // Named rather than carried, for the same reason the private network's key is: the mesh + // discarded the value and could not put it here if it wanted to. What is here is where to + // find it. + Secret string `json:"secret,omitempty"` + // Values are the module's own, with settings applied. What the keys mean is agreed by the + // requirement's name — everything providing `reverse-proxy` understands the same shape, which + // is what makes swapping one for another cost nothing. + Values map[string]any `json:"values"` + // Derived is what this provider's own definition said it derives for this consumer, already + // derived (novox/hq ADR 0202). + // + // **The provider is told, rather than recomputing it.** A served value may name the consumer's + // identity — a bucket named for who is asking, a database prefixed with it — and before this + // the rule lived twice: once in the provisioner's code, once transcribed into every consumer's + // definition. The mesh fills the provider's own statement here and delivers the same filled + // value to the consumer, so the two cannot disagree: there is no second computation to + // disagree with. + // + // Only the keys that are per-consumer. The rest of what the provider serves is the same for + // everyone and is in its own definition, where it already is. + Derived map[string]any `json:"derived,omitempty"` +} + +// provisionsAs is the account that reads what the mesh writes for this provider: the one secret +// per consumer it must open to set that consumer's password (novox/hq issue 225). +// +// **A root-owned 0600 file is one that process cannot read**, which is the same sentence already +// written above for a module's own secrets — and the grant secret is the other kind of secret +// the mesh writes for a module, so it is the same rule. +// +// Which account depends on where the module's code runs. A module whose code is a bundle is run +// by the node's tool runtime, as the node's account ([ADR 0198](0198)); one still in a container +// is whatever it declares as its secrets owner. Nothing names these paths, so the rule that +// claims a bundle's other files by the words that name them (givenTo) cannot reach them: the +// harness composes a grant secret's path from the contributions file, not from a word. +// +// Empty is root, which is what it was and what a module with no bundle and no declared owner +// still wants. +func (r Resolution) provisionsAs(m Manifest) string { + // A module of its own account reads them as that account (ADR 0259 §8, issue 723). + if m.RunsAs != "" { + return m.RunsAs + } + if len(m.Bundles) > 0 && r.Account != "" { + return r.Account + } + return m.SecretsOwner +} + +// grantPath is where one consumer's sealed credential lands on the providing machine. +// +// Suffixed, so the directory can also hold whatever the module writing it keeps there and so a +// node named like something else in that directory cannot collide with it. +// One file per consumer, and a consumer is a module on a machine (novox/hq 04-ISSUES/022). +// +// Named after both. Named after the machine alone, two modules on one node wrote to one path: the +// second overwrote the first, and the provisioner — reading a directory — saw one consumer where +// there were two. +// reflectsRenamed is a resource's restart-on list under the module's prefix, or nil when it has +// none. It reads both the shape JSON parsing produces ([]any) and the shape code composing +// resources natively produces ([]string): a reference that was skipped because its list arrived +// in the other shape would point at nothing — silently, with the service never restarting and +// every check passing, which is how a runtime kept serving without the registry trust its +// daemon file already carried. +func reflectsRenamed(module string, reflects any) []any { + var names []string + switch v := reflects.(type) { + case []any: + for _, id := range v { + names = append(names, fmt.Sprint(id)) + } + case []string: + names = v + } + if names == nil { + return nil + } + var renamed []any + for _, named := range names { + if strings.Contains(named, ".") { + renamed = append(renamed, named) + continue + } + renamed = append(renamed, module+"."+named) + } + return renamed +} + +func grantPath(directory, consumer, module string) string { + return strings.TrimRight(directory, "/") + "/" + consumer + "." + module + ".secret" +} + +// holderAs is a consumer's name at the provider with a local name after it, where it keeps several +// credentials for one provision (ADR 0094); the name alone otherwise. +func holderAs(as, local string) string { + if local == "" { + return as + } + return as + "_" + local +} + +// contributions collects what every module in this set contributes, by requirement. +// +// Ordered by contributing module, because the result becomes a file on a machine and a file whose +// lines move about is a file that looks changed when nothing changed. +func (r Resolution) contributions(settings SettingsBy, grants []Grant, + directories map[string]string) (map[string][]Contribution, error) { + out := map[string][]Contribution{} + modules := append([]Manifest{}, r.Modules...) + sort.Slice(modules, func(i, j int) bool { return modules[i].Module < modules[j].Module }) + + // What consumers on other machines asked for. Merged in with this machine's own, because from + // the provider's side they are the same thing — somebody wanting something — and a provider + // that had to read two lists would be a provider that reads one of them. + sorted := append([]Grant{}, grants...) + sort.Slice(sorted, func(i, j int) bool { + if sorted[i].Provision != sorted[j].Provision { + return sorted[i].Provision < sorted[j].Provision + } + if sorted[i].Consumer != sorted[j].Consumer { + return sorted[i].Consumer < sorted[j].Consumer + } + if sorted[i].From != sorted[j].From { + return sorted[i].From < sorted[j].From + } + return sorted[i].Local < sorted[j].Local + }) + // A consumer already carried by the grants loop, keyed (provision, module). When provider and + // consumer are co-located, `grantsFor` enumerates the same-node consumer too, so without this the + // module would be emitted a second time by the m.Contributes loop below — once full (with the + // grant's secret/as) and once partial — which is the duplicate seen in a co-located mesh.json. + granted := map[string]map[string]bool{} + for _, g := range sorted { + if g.From == "" { + // As above: nothing on that machine asks for this any more, so the provider is not + // told about it and withdraws the login on its next pass. + continue + } + as := holderAs(ConsumerIdentity(g.Consumer, IdentitySource(g.Slug, g.From)), g.Local) + derived, err := r.derivedFor(g.Provision, as, g.From, g.Local, settings) + if err != nil { + return nil, err + } + out[g.Provision] = append(out[g.Provision], Contribution{ + From: g.From, Node: g.Consumer, At: g.At, Values: g.Values, Derived: derived, + // One holder per local name: the identity the consumer is known by, and the local name + // after it where the module keeps several (ADR 0094). Not a login any backend checks — + // a secret is not a login — so the identity limit does not apply to the suffix. + As: as, + Secret: grantPath(directories[g.Provision], g.Consumer, holderAs(g.From, g.Local)), + }) + if granted[g.Provision] == nil { + granted[g.Provision] = map[string]bool{} + } + granted[g.Provision][g.From] = true + } + for _, m := range modules { + for _, to := range sortedKeys(m.Contributes) { + // The grants loop already emitted this module's contribution to this provision, with its + // minted secret — emitting the partial copy again would duplicate it. A contribution with + // no grant (a reverse-proxy route names a host, not a credential) is not in `granted`, so + // it still reaches the provider from here. + if granted[to][m.Module] { + continue + } + // Settings reach a contribution the same way they reach a file. A route's hostname is + // exactly the kind of thing that differs between one mesh and the next, and a module + // that could not have it set would have to be edited to be reused. + values, err := r.composed(m, to, m.Contributes[to], settings[m.Module], + m.Module+" contributing to "+to) + if err != nil { + return nil, err + } + out[to] = append(out[to], Contribution{From: m.Module, Values: values}) + } + // Several contributions to one requirement (ADR 0094's sibling for `contributes`): an + // object store's data API and its console are two different public names from one module, + // not one. Never in `granted` — a route names a host, not a credential — so every local + // name always reaches the provider from here. + for _, to := range sortedKeys(m.ContributesMany) { + for _, local := range sortedKeys(m.ContributesMany[to]) { + values, err := r.composed(m, to, m.ContributesMany[to][local], settings[m.Module], + m.Module+" contributing "+local+" to "+to) + if err != nil { + return nil, err + } + out[to] = append(out[to], Contribution{From: m.Module, Values: values}) + } + } + } + return out, nil +} + +// composed is one contribution as its provider receives it: settled with this node's settings, its +// endpoint's port filled in, and its names composed from the label. +// +// **One function, because two readers must agree.** The provider is told the names in its received +// file; the contributing module is told the same names in its own binding (novox/hq 04-ISSUES/122). +// Composing them twice, in two places, is how the proxy would come to serve one name while the +// module wrote another into its configuration. +func (r Resolution) composed(m Manifest, to string, raw map[string]any, layers []Layer, what string) ( + map[string]any, error) { + // Overridden, not merged: a setting changes a key the contribution declares and adds none. + // The provider reads the contribution as a contract, and a setting made for one of this + // module's files is no part of it (novox/hq 04-ISSUES/173). + values, err := overridden(raw, WithDefaults(m, layers), what) + if err != nil { + return nil, fmt.Errorf("%s: %w", what, err) + } + reaches, err := Reaches(m, layers) + if err != nil { + return nil, fmt.Errorf("%s: %w", what, err) + } + blocks, err := Endpoints(m, layers) + if err != nil { + return nil, fmt.Errorf("%s: %w", what, err) + } + portOfEndpoint(values, endpointPorts(m)) + composeName(values, r.PublicDomain, servingAt(r, to), reaches, endpointPorts(m), blocks) + return values, nil +} + +// ownNames is what a module is known by through what it contributes to one requirement — the names +// the mesh composed for it, and nothing else of the contribution. +// +// **The half a module could not learn** (novox/hq 04-ISSUES/122). A module contributes a label, the +// mesh joins it with this node's domains, and the provider serves the result — and the module itself +// was never told. Software that must know its own address (a login redirect, a canonical URL, an +// issuer) had it written into the manifest as a literal, which is a domain in a definition and wrong +// on every other machine. `${bound::name}` is the answer, from the same composition the +// provider receives. +// +// Several contributions to one requirement are keyed by their local name under `names`. +func (r Resolution) ownNames(m Manifest, to string, layers []Layer) (map[string]any, error) { + pick := func(values map[string]any) map[string]any { + names := map[string]any{} + for _, key := range []string{"name", "internal-name"} { + if v, ok := values[key].(string); ok && v != "" { + names[key] = v + } + } + return names + } + out := map[string]any{} + if raw, ok := m.Contributes[to]; ok { + values, err := r.composed(m, to, raw, layers, m.Module+" contributing to "+to) + if err != nil { + return nil, err + } + for k, v := range pick(values) { + out[k] = v + } + } + if locals := m.ContributesMany[to]; len(locals) > 0 { + many := map[string]any{} + for _, local := range sortedKeys(locals) { + values, err := r.composed(m, to, locals[local], layers, + m.Module+" contributing "+local+" to "+to) + if err != nil { + return nil, err + } + if names := pick(values); len(names) > 0 { + many[local] = names + } + } + if len(many) > 0 { + out["names"] = many + } + } + return out, nil +} + +// composeName joins a contribution's label with a node's public domain, and separately with its +// private one, in place (novox/hq ADR 0056). +// +// **The whole of what the mesh does with a route's name: join two given strings — twice.** A +// contribution carries a `label` — the subdomain its operator chose — and the node carries its +// public domain and its own private-network address; the granted names are `