From 0cdbcc60190b40b3d9be86ea287f7d8849b4381b Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 01:38:41 +0000 Subject: [PATCH 01/41] Say which mesh-sdk version a program was built with, from the build info (issue 1355) --- go/sdkversion/sdkversion.go | 57 +++++++++++++++++++++++++++++++ go/sdkversion/sdkversion_test.go | 58 ++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+) create mode 100644 go/sdkversion/sdkversion.go create mode 100644 go/sdkversion/sdkversion_test.go diff --git a/go/sdkversion/sdkversion.go b/go/sdkversion/sdkversion.go new file mode 100644 index 0000000..46a705a --- /dev/null +++ b/go/sdkversion/sdkversion.go @@ -0,0 +1,57 @@ +// Package sdkversion says which mesh-sdk version a program was built with (novox/hq ADR 0446 clause 4: a composition +// check names its mesh-sdk version, so the controller and the assigner can be compared; ADR 0453 clause 1). +// +// The version is read from the build info, never kept by hand: a constant nobody bumps would report two different +// SDKs as one. A program that imports the SDK reports the module's pinned (pseudo-)version, or its replacement's; +// the SDK's own binaries and tests report "(devel)" with the stamped vcs.revision. +package sdkversion + +import ( + "runtime/debug" +) + +// Module is the SDK's Go module path. +const Module = "git.novox.be/novox/mesh-sdk/go" + +// Version is the mesh-sdk version this binary was built with: "unknown" when the build carries no build info. +func Version() string { + bi, ok := debug.ReadBuildInfo() + if !ok { + return "unknown" + } + return FromBuildInfo(bi) +} + +// FromBuildInfo reads the SDK's version out of a build info. A dependency replaced by another module version is that +// version; replaced by a local path it is the path, never empty. When the SDK is the main module it is "(devel)", +// followed by a space and the vcs.revision when the build stamped one. Without the SDK in the build it is "unknown". +func FromBuildInfo(bi *debug.BuildInfo) string { + if bi == nil { + return "unknown" + } + if bi.Main.Path == Module { + for _, s := range bi.Settings { + if s.Key == "vcs.revision" && s.Value != "" { + return "(devel) " + s.Value + } + } + return "(devel)" + } + for _, d := range bi.Deps { + if d == nil || d.Path != Module { + continue + } + if r := d.Replace; r != nil { + if r.Version != "" { + return r.Version + } + if r.Path != "" { + return r.Path + } + } + if d.Version != "" { + return d.Version + } + } + return "unknown" +} diff --git a/go/sdkversion/sdkversion_test.go b/go/sdkversion/sdkversion_test.go new file mode 100644 index 0000000..7a4b04d --- /dev/null +++ b/go/sdkversion/sdkversion_test.go @@ -0,0 +1,58 @@ +package sdkversion + +import ( + "runtime/debug" + "testing" +) + +const sdkPath = "git.novox.be/novox/mesh-sdk/go" + +// The build infos are constructed in the shape runtime/debug.ReadBuildInfo gives a binary that imports the SDK +// (go.mod pins like mesh-controller's v0.1.14-0.20261019091011-aa68b67389c9). +func TestAPinnedDependencyIsItsPseudoVersion(t *testing.T) { + bi := &debug.BuildInfo{Main: debug.Module{Path: "github.com/novox/mesh-controller", Version: "(devel)"}, + Deps: []*debug.Module{{Path: "example.org/other", Version: "v1.0.0"}, {Path: sdkPath, Version: "v0.1.14-0.20261019091011-aa68b67389c9"}}} + if got := FromBuildInfo(bi); got != "v0.1.14-0.20261019091011-aa68b67389c9" { + t.Fatalf("got %q", got) + } +} + +func TestAReplacedDependencyIsTheReplacement(t *testing.T) { + bi := &debug.BuildInfo{Main: debug.Module{Path: "x"}, Deps: []*debug.Module{{Path: sdkPath, Version: "v0.1.14", + Replace: &debug.Module{Path: sdkPath, Version: "v0.2.0-0.20261020000000-abcdef123456"}}}} + if got := FromBuildInfo(bi); got != "v0.2.0-0.20261020000000-abcdef123456" { + t.Fatalf("got %q", got) + } + local := &debug.BuildInfo{Main: debug.Module{Path: "x"}, Deps: []*debug.Module{{Path: sdkPath, Version: "v0.1.14", + Replace: &debug.Module{Path: "../mesh-sdk/go"}}}} + if got := FromBuildInfo(local); got != "../mesh-sdk/go" { + t.Fatalf("a local replacement: got %q", got) + } +} + +func TestTheMainModuleIsDevelWithItsRevision(t *testing.T) { + bi := &debug.BuildInfo{Main: debug.Module{Path: sdkPath, Version: "(devel)"}, + Settings: []debug.BuildSetting{{Key: "vcs", Value: "git"}, {Key: "vcs.revision", Value: "5786ba190c69bcee983223dbbc149bfb18fe01ed"}}} + if got := FromBuildInfo(bi); got != "(devel) 5786ba190c69bcee983223dbbc149bfb18fe01ed" { + t.Fatalf("got %q", got) + } + bare := &debug.BuildInfo{Main: debug.Module{Path: sdkPath, Version: "(devel)"}} + if got := FromBuildInfo(bare); got != "(devel)" { + t.Fatalf("no stamp: got %q", got) + } +} + +func TestNoBuildInfoIsUnknown(t *testing.T) { + if got := FromBuildInfo(nil); got != "unknown" { + t.Fatalf("got %q", got) + } + if got := FromBuildInfo(&debug.BuildInfo{Main: debug.Module{Path: "x"}}); got != "unknown" { + t.Fatalf("an SDK-less binary: got %q", got) + } +} + +func TestVersionIsNeverEmpty(t *testing.T) { + if Version() == "" { + t.Fatal("empty") + } +} -- 2.54.0 From 605eca2c0a1deda1bd697c28c7ed98c668646bb5 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 01:40:35 +0000 Subject: [PATCH 02/41] Compute the source fingerprint's new form beside src1, and its key part (issue 1356) --- go/delivery/fingerprint2.go | 105 ++++++++++++++++++++ go/delivery/fingerprint2_test.go | 160 +++++++++++++++++++++++++++++++ 2 files changed, 265 insertions(+) create mode 100644 go/delivery/fingerprint2.go create mode 100644 go/delivery/fingerprint2_test.go diff --git a/go/delivery/fingerprint2.go b/go/delivery/fingerprint2.go new file mode 100644 index 0000000..23a991a --- /dev/null +++ b/go/delivery/fingerprint2.go @@ -0,0 +1,105 @@ +package delivery + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "sort" + "strings" +) + +// SourcePrefix2 opens every source fingerprint of the new form. +const SourcePrefix2 = "src2:" + +// Toolchain2 is what an artifact's toolchain enters the new fingerprint as (novox/hq ADR 0405 clause 7): the +// artifact's language, the installed toolchain's version and the build module's recipe version, in place of the +// toolchain image's digest. A toolchain update so changes the fingerprints of that language's modules only. +type Toolchain2 struct { + Language string + Version string + Recipe string +} + +// SourceInputs2 are what a module's source fingerprint of the new form is of (novox/hq ADR 0429 clause 5 as ADR 0437 +// clause 5 changed it; ADR 0453 clause 6). It differs from SourceInputs in exactly four things: a base enters by its +// module's fingerprint (src2:...), not its image digest; the builder enters by its own module's fingerprint, not its +// executable's digest; an artifact that resolves packages when it is built enters by the hash of its lock file; and +// an artifact's toolchain enters as Toolchain2. Module, Tree, Contexts and Unpinned are as in SourceInputs. +// +// Locks names the artifacts that resolve packages when built, each with the hash of its lock file; an entry with an +// empty hash is such an artifact with no lock, which has no fingerprint. An artifact that resolves nothing is not in +// Locks. +type SourceInputs2 struct { + Module string + Tree string + Bases []string + Builder string + Contexts map[string]string + Locks map[string]string + Toolchains map[string]Toolchain2 + Unpinned string +} + +// SourceFingerprint2 is the module's source fingerprint of the new form, `src2:` and the hex sha256 of its inputs' +// lines, or none and the reason when the source does not pin the build: Unpinned when set, "no tree" when Tree is +// empty, "no lock" when an artifact resolves packages and has no lock. It is a pure function of its inputs: the order +// of maps and a base given twice change nothing. The lines are "module ", "tree " and "builder ", then +// "base " (once each), "context ", "lock " and "toolchain +// ", sorted, all joined by a newline. +func SourceFingerprint2(in SourceInputs2) (fp, none string) { + if in.Unpinned != "" { + return "", in.Unpinned + } + if in.Tree == "" { + return "", "no tree" + } + for _, h := range in.Locks { + if h == "" { + return "", "no lock" + } + } + lines := []string{"module " + in.Module, "tree " + in.Tree, "builder " + in.Builder} + bases := map[string]bool{} + for _, b := range in.Bases { + bases[b] = true + } + for b := range bases { + lines = append(lines, "base "+b) + } + for a, t := range in.Contexts { + lines = append(lines, "context "+a+" "+t) + } + for a, h := range in.Locks { + lines = append(lines, "lock "+a+" "+h) + } + for a, t := range in.Toolchains { + lines = append(lines, "toolchain "+a+" "+t.Language+" "+t.Version+" "+t.Recipe) + } + sort.Strings(lines[3:]) + sum := sha256.Sum256([]byte(strings.Join(lines, "\n"))) + return SourcePrefix2 + hex.EncodeToString(sum[:]), "" +} + +// FingerprintKeyPart is a new-form fingerprint as a key part: its 64 hex digits without the prefix, since a bus key +// may not hold ':'. It refuses a value that is not `src2:` and 64 lowercase hex digits, a src1 fingerprint included, +// naming it. +func FingerprintKeyPart(fp string) (string, error) { + part, ok := strings.CutPrefix(fp, SourcePrefix2) + if !ok || !isHex64(part) { + return "", fmt.Errorf("delivery: %q is not a fingerprint of the new form, %s and 64 lowercase hex digits", fp, SourcePrefix2) + } + return part, nil +} + +// ParseFingerprintKeyPart is the inverse of FingerprintKeyPart. It refuses a part that is not 64 lowercase hex +// digits, a src1 fingerprint or a prefixed value included, naming it. +func ParseFingerprintKeyPart(part string) (string, error) { + if !isHex64(part) { + return "", fmt.Errorf("delivery: %q is not the key part of a fingerprint, 64 lowercase hex digits", part) + } + return SourcePrefix2 + part, nil +} + +func isHex64(s string) bool { + return len(s) == 64 && strings.Trim(s, "0123456789abcdef") == "" +} diff --git a/go/delivery/fingerprint2_test.go b/go/delivery/fingerprint2_test.go new file mode 100644 index 0000000..053b8dc --- /dev/null +++ b/go/delivery/fingerprint2_test.go @@ -0,0 +1,160 @@ +package delivery + +import ( + "strings" + "testing" +) + +// The new form's inputs (novox/hq ADR 0429 clause 5 as ADR 0437 clause 5 changed it, ADR 0405 clause 7). The base and +// builder values are module fingerprints in the new form; a lock is the caller's hash of a lock file, which this +// function only enters (its form is the builder's to settle; the values here are stand-in tokens); the toolchain is a +// language, the installed toolchain's version and the build module's recipe version. +func inputs2() SourceInputs2 { + return SourceInputs2{ + Module: "mesh-mail", + Tree: "t-module", + Bases: []string{SourcePrefix2 + strings.Repeat("a", 64), SourcePrefix2 + strings.Repeat("b", 64)}, + Builder: SourcePrefix2 + strings.Repeat("c", 64), + Contexts: map[string]string{"mail": "t-root", "tool": "t-root2"}, + Locks: map[string]string{"mail": "sha256:1111", "tool": "sha256:2222"}, + Toolchains: map[string]Toolchain2{ + "mail": {Language: "go", Version: "1.26.0", Recipe: "3"}, + "tool": {Language: "typescript", Version: "5.9.2", Recipe: "1"}, + }, + } +} + +func mustFP2(t *testing.T, in SourceInputs2) string { + t.Helper() + fp, none := SourceFingerprint2(in) + if fp == "" || none != "" { + t.Fatalf("fingerprint = %q, none = %q", fp, none) + } + return fp +} + +// 1356/AC1: a fixed input has the fingerprint the test fixes, with the new prefix. The hex was computed from the function and checked against an independent sha256 of the lines; +// it is pinned here, so a change of the lines is a change of every fingerprint, and shows. +func TestFingerprint2Vector(t *testing.T) { + const want = "src2:56b6f2f6ffc1767225e0b277ab78539521e9a9fbaadfe4ec158d2fddee1d63ef" + if got := mustFP2(t, inputs2()); got != want { + t.Errorf("fingerprint = %s, want %s", got, want) + } + if SourcePrefix2 != "src2:" || SourcePrefix == SourcePrefix2 { + t.Errorf("the new prefix is %q", SourcePrefix2) + } +} + +// 1356/AC2: each input on its own changes the fingerprint. +func TestFingerprint2Changes(t *testing.T) { + want := mustFP2(t, inputs2()) + for name, f := range map[string]func(*SourceInputs2){ + "module": func(in *SourceInputs2) { in.Module = "other" }, + "tree": func(in *SourceInputs2) { in.Tree = "t-other" }, + "a base": func(in *SourceInputs2) { in.Bases[0] = SourcePrefix2 + strings.Repeat("d", 64) }, + "a base added": func(in *SourceInputs2) { in.Bases = append(in.Bases, SourcePrefix2+strings.Repeat("e", 64)) }, + "the builder": func(in *SourceInputs2) { in.Builder = SourcePrefix2 + strings.Repeat("f", 64) }, + "a lock": func(in *SourceInputs2) { in.Locks["mail"] = "sha256:changed" }, + "a lock added": func(in *SourceInputs2) { in.Locks["extra"] = "sha256:3333" }, + "a toolchain": func(in *SourceInputs2) { in.Toolchains["mail"] = Toolchain2{"go", "1.26.1", "3"} }, + "a toolchain lang": func(in *SourceInputs2) { in.Toolchains["mail"] = Toolchain2{"typescript", "1.26.0", "3"} }, + "a recipe": func(in *SourceInputs2) { in.Toolchains["mail"] = Toolchain2{"go", "1.26.0", "4"} }, + "a context": func(in *SourceInputs2) { in.Contexts["mail"] = "t-changed" }, + "a context added": func(in *SourceInputs2) { in.Contexts["extra"] = "t-root" }, + "a toolchain added": func(in *SourceInputs2) { in.Toolchains["extra"] = Toolchain2{"go", "1.26.0", "3"} }, + } { + in := inputs2() + f(&in) + if got := mustFP2(t, in); got == want { + t.Errorf("a changed %s keeps the fingerprint", name) + } + } +} + +// 1356/AC3: a base given twice, and maps in any order, give the same fingerprint. +func TestFingerprint2OrderAndDuplicates(t *testing.T) { + want := mustFP2(t, inputs2()) + in := inputs2() + in.Bases = []string{in.Bases[1], in.Bases[0]} + if got := mustFP2(t, in); got != want { + t.Errorf("the order of bases changes it: %s", got) + } + in = inputs2() + in.Bases = append(in.Bases, in.Bases[0], in.Bases[1]) + if got := mustFP2(t, in); got != want { + t.Errorf("a base given twice changes it: %s", got) + } + for i := 0; i < 20; i++ { + in = inputs2() + in.Contexts, in.Locks, in.Toolchains = map[string]string{}, map[string]string{}, map[string]Toolchain2{} + for _, k := range []string{"tool", "mail"} { + in.Contexts[k] = inputs2().Contexts[k] + in.Locks[k] = inputs2().Locks[k] + in.Toolchains[k] = inputs2().Toolchains[k] + } + if got := mustFP2(t, in); got != want { + t.Fatalf("map order changes it: %s", got) + } + } +} + +// 1356/AC4: an artifact that resolves packages when built, with no lock, gives no fingerprint and the reason "no lock"; +// a module with no tree gets none with today's reason, and the reason the source is unpinned stands over both. +func TestFingerprint2None(t *testing.T) { + in := inputs2() + in.Locks["mail"] = "" + if fp, none := SourceFingerprint2(in); fp != "" || none != "no lock" { + t.Errorf("no lock: fp = %q, none = %q", fp, none) + } + in = inputs2() + in.Tree = "" + if fp, none := SourceFingerprint2(in); fp != "" || none != "no tree" { + t.Errorf("no tree: fp = %q, none = %q", fp, none) + } + in = inputs2() + in.Tree, in.Unpinned = "", "a context is a Go program's build source" + if _, none := SourceFingerprint2(in); none != in.Unpinned { + t.Errorf("the reason for unpinned stands over no tree: %q", none) + } +} + +// 1356/AC5: src1 is unchanged by the new form; its vectors in fingerprint_test.go run unchanged. This pins that the +// two forms do not collide for equal module and tree, and that src1 keeps its prefix. +func TestSource1StaysSource1(t *testing.T) { + if SourcePrefix != "src1:" { + t.Fatalf("src1's prefix is %q", SourcePrefix) + } + fp1, _ := SourceFingerprint(SourceInputs{Module: "m", Tree: "t"}) + fp2, _ := SourceFingerprint2(SourceInputs2{Module: "m", Tree: "t"}) + if fp1 == "" || fp2 == "" || strings.TrimPrefix(fp1, SourcePrefix) == strings.TrimPrefix(fp2, SourcePrefix2) { + t.Errorf("src1 %q and src2 %q", fp1, fp2) + } +} + +// 1356/AC6: the key part holds only bus key characters and parsing gives the fingerprint back; a src1 or malformed +// value is refused, naming it. +func TestFingerprintKeyPart(t *testing.T) { + fp := mustFP2(t, inputs2()) + part, err := FingerprintKeyPart(fp) + if err != nil { + t.Fatal(err) + } + if len(part) != 64 || strings.Trim(part, "0123456789abcdef") != "" || !nameOK.MatchString(part) { + t.Errorf("key part %q is not 64 lowercase hex characters a key allows", part) + } + back, err := ParseFingerprintKeyPart(part) + if err != nil || back != fp { + t.Errorf("parse(%q) = %q, %v; want %q", part, back, err, fp) + } + src1, _ := SourceFingerprint(SourceInputs{Module: "m", Tree: "t"}) + for _, bad := range []string{src1, "src2:", "src2:xyz", "", "src2:" + strings.Repeat("A", 64), "src2:" + strings.Repeat("a", 63)} { + if _, err := FingerprintKeyPart(bad); err == nil || !strings.Contains(err.Error(), "\""+bad+"\"") { + t.Errorf("FingerprintKeyPart(%q) = %v, want a refusal naming it", bad, err) + } + } + for _, bad := range []string{src1, "src2:" + strings.Repeat("a", 64), strings.Repeat("a", 63), strings.Repeat("A", 64), "", "zz"} { + if _, err := ParseFingerprintKeyPart(bad); err == nil || !strings.Contains(err.Error(), "\""+bad+"\"") { + t.Errorf("ParseFingerprintKeyPart(%q) = %v, want a refusal naming it", bad, err) + } + } +} -- 2.54.0 From 1bbf75ae193c50f095af6de3adb78b0847f0f023 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 01:43:17 +0000 Subject: [PATCH 03/41] Add the layout of the indexer's manifests bucket: the key by digest and its value (issue 1357) --- go/delivery/manifests.go | 85 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 go/delivery/manifests.go diff --git a/go/delivery/manifests.go b/go/delivery/manifests.go new file mode 100644 index 0000000..d0d0bcd --- /dev/null +++ b/go/delivery/manifests.go @@ -0,0 +1,85 @@ +package delivery + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "unicode/utf8" +) + +// ManifestDigest is the hex sha256 of a manifest's bytes exactly as read at the commit: the key of a manifests value +// and the digest a Module and a candidate's module carry (novox/hq ADR 0429 clause 16). +func ManifestDigest(raw []byte) string { + sum := sha256.Sum256(raw) + return hex.EncodeToString(sum[:]) +} + +// ManifestsKey is the key, in ManifestsBucket, of a manifest: its digest, 64 lowercase hex digits. The indexer +// writes it once, by a create that fails when the key exists. +func ManifestsKey(digest string) (string, error) { + if !isHex64(digest) { + return "", fmt.Errorf("delivery: %q is not a manifest digest, 64 lowercase hex digits", digest) + } + return digest, nil +} + +// ParseManifestsKey is the inverse of ManifestsKey. +func ParseManifestsKey(key string) (digest string, err error) { + if digest, err = ManifestsKey(key); err != nil { + return "", fmt.Errorf("delivery: %q is not a manifests key: %w", key, err) + } + return digest, nil +} + +// Manifest is the value of a manifests key: the module and its manifest, the raw text of its module.json as read at the +// commit, byte for byte (ManifestDigest of JSON is the key). It is the one value of delivery that is larger than about +// 500 bytes: the largest manifest of the catalogue is about 21 KB. It is a string, not nested JSON, because nesting +// would re-write the manifest's spacing and so change its digest. +type Manifest struct { + V int `json:"v"` + Module string `json:"module"` + JSON string `json:"manifest"` +} + +func (m Manifest) check() error { + if err := checkName("module", m.Module); err != nil { + return err + } + if m.JSON == "" { + return fmt.Errorf("delivery: the manifest of %s is empty", m.Module) + } + if !utf8.ValidString(m.JSON) { + return fmt.Errorf("delivery: the manifest of %s is not UTF-8 text, which a value cannot hold byte for byte", m.Module) + } + if !json.Valid([]byte(m.JSON)) { + return fmt.Errorf("delivery: the manifest of %s is not JSON", m.Module) + } + return nil +} + +// EncodeManifest is m as JSON at Version; it refuses a module that is not a name, and a manifest that is empty, not +// UTF-8 or not JSON. +func EncodeManifest(m Manifest) ([]byte, error) { + if err := m.check(); err != nil { + return nil, err + } + m.V = Version + return json.Marshal(m) +} + +// DecodeManifest reads a manifests value; it refuses an unknown version (ErrUnknownVersion) and what EncodeManifest +// refuses. +func DecodeManifest(data []byte) (Manifest, error) { + var m Manifest + if err := decode(data, &m); err != nil { + return Manifest{}, err + } + if err := m.check(); err != nil { + return Manifest{}, err + } + return m, nil +} + +// SameManifest says whether two manifests are the same value, so a writer writes only when they differ. +func SameManifest(a, b Manifest) bool { return a == b } -- 2.54.0 From 9b4c2ba5916e7dee39bd98d846d1752372b127aa Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 01:43:39 +0000 Subject: [PATCH 04/41] Give a module and a candidate the new-form fingerprint, the manifest digest and the fpset, in version 1 (issue 1357) --- go/delivery/delivery.go | 4 ++ go/delivery/source.go | 111 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 111 insertions(+), 4 deletions(-) diff --git a/go/delivery/delivery.go b/go/delivery/delivery.go index ff6625e..848fbfa 100644 --- a/go/delivery/delivery.go +++ b/go/delivery/delivery.go @@ -30,6 +30,10 @@ // **The source fingerprint** (issue 1203; ADR 0437 §5): one function of named inputs that the controller's builder and // the indexer both call, so they agree byte for byte (fingerprint.go). // +// **The release 4 layouts** (issues 1356 and 1357; ADR 0453 clauses 6 and 16): the new fingerprint form (fingerprint2.go), +// and the indexer's manifests by digest and the new fields of its modules and candidates (manifests.go, source.go). A +// manifests value is the one exception to the size above: a manifest is up to about 21 KB, whole, byte for byte. +// // Standard library only, and this module's own asks package. package delivery diff --git a/go/delivery/source.go b/go/delivery/source.go index 25bbe5e..31c1a9a 100644 --- a/go/delivery/source.go +++ b/go/delivery/source.go @@ -31,6 +31,10 @@ const ( ModulesBucket = IndexerModule + "_modules" // ModulesHistory is the revisions the modules bucket keeps per key. ModulesHistory = 20 + // ManifestsBucket is the indexer's bucket of manifests by digest (novox/hq ADR 0429 clause 16); see manifests.go. + ManifestsBucket = IndexerModule + "_manifests" + // ManifestsHistory is the revisions the manifests bucket keeps per key: a key is written once. + ManifestsHistory = 1 ) // ForgeStream is the forge bridge's stream as its manifest declares it; the broker derives the real names. @@ -245,19 +249,86 @@ func (s Source) check() error { // Candidate is the value of a candidates key: the group's pull requests and heads, the heads hash, the base (the // source revision it was previewed onto) and the modules each pull request touches. It carries no fingerprint -// until release 4 (ADR 0437 §5). +// until release 4 (ADR 0437 §5); from R4c the indexer writes the new form's fingerprint of each touched module in +// the pull request's Fps and, over all of them, FPSet (novox/hq ADR 0453 clauses 6 and 16). Both are absent in a +// value written before, and a reader of today's shape ignores them. type Candidate struct { V int `json:"v"` Group string `json:"group"` Base uint64 `json:"base"` HeadsHash string `json:"heads"` Pulls []CandidatePull `json:"pulls"` + FPSet string `json:"fpset,omitempty"` } -// CandidatePull is one pull request of a candidate and the modules it touches. +// CandidatePull is one pull request of a candidate and the modules it touches. Fps is, by module name, the new +// form's fingerprint of the module at the merge preview, or why it has none, and the digest of its manifest (the key +// of the indexer's manifests bucket); a module of Modules without an entry has none written yet. type CandidatePull struct { Head - Modules []string `json:"modules"` + Modules []string `json:"modules"` + Fps map[string]CandidateModule `json:"fps,omitempty"` +} + +// CandidateModule is one touched module's new-form fingerprint FP2 (`src2:` and 64 hex digits), or None2, the reason +// it has none; exactly one of them. Manifest is the digest of the module's manifest as read at the preview. +type CandidateModule struct { + FP2 string `json:"fp2,omitempty"` + None2 string `json:"none2,omitempty"` + Manifest string `json:"manifest,omitempty"` +} + +func (m CandidateModule) check() error { + switch { + case m.FP2 != "" && m.None2 != "": + return fmt.Errorf("delivery: a module has both fp2 and none2") + case m.FP2 == "" && m.None2 == "": + return fmt.Errorf("delivery: a module has neither fp2 nor none2") + } + return checkNewForm(m.FP2, m.Manifest) +} + +// checkNewForm refuses an fp2 that is not of the new form and a manifest digest that is not a digest, naming it. +func checkNewForm(fp2, manifest string) error { + if fp2 != "" { + if _, err := FingerprintKeyPart(fp2); err != nil { + return err + } + } + if manifest != "" { + if _, err := ManifestsKey(manifest); err != nil { + return err + } + } + return nil +} + +// FingerprintSet is the fpset of a candidate's pull requests: the first 12 hex digits of the sha256 over the lines +// `=` or `=none:` of every module with an entry in Fps, sorted and joined by a newline, +// so it is the same for every order and changes with any fingerprint or reason. A manifest digest does not enter: the +// fingerprint already covers the manifest. It is the empty string when no pull request carries a fingerprint. The +// checker keys its verdict by it (research 063 section 01). +func FingerprintSet(pulls []CandidatePull) string { + lines := map[string]bool{} + for _, p := range pulls { + for module, m := range p.Fps { + if m.FP2 != "" { + lines[module+"="+m.FP2] = true + } else { + lines[module+"=none:"+m.None2] = true + } + } + } + if len(lines) == 0 { + return "" + } + all := make([]string, 0, len(lines)) + for l := range lines { + all = append(all, l) + } + sort.Strings(all) + sum := sha256.Sum256([]byte(strings.Join(all, "\n"))) + return hex.EncodeToString(sum[:])[:12] } func (c Candidate) check() error { @@ -272,16 +343,46 @@ func (c Candidate) check() error { if c.HeadsHash != want { return fmt.Errorf("delivery: the candidate's heads hash %q is not the hash %q of its pull requests' heads", c.HeadsHash, want) } + seen := map[string]CandidateModule{} + for _, p := range c.Pulls { + touched := map[string]bool{} + for _, m := range p.Modules { + touched[m] = true + } + for module, m := range p.Fps { + if !touched[module] { + return fmt.Errorf("delivery: the candidate gives a fingerprint of the module %q, which pull request %s#%d does not touch", module, p.Repository, p.Number) + } + if err := m.check(); err != nil { + return fmt.Errorf("delivery: the candidate's module %q: %w", module, err) + } + if before, ok := seen[module]; ok && before != m { + return fmt.Errorf("delivery: two pull requests of the candidate give the module %q different fingerprints", module) + } + seen[module] = m + } + } + if wantSet := FingerprintSet(c.Pulls); c.FPSet != wantSet { + return fmt.Errorf("delivery: the candidate's fpset %q is not the hash %q of its fingerprints", c.FPSet, wantSet) + } return nil } // Module is the value of a modules key: the module's source fingerprint, or none and why (a build that resolves // packages when built gets none, said here), and where it was computed: Repository and Commit are the trunk merge, // Src the source revision then (0 before the gate writes source.main). Exactly one of FP and None is set. +// +// From R4c the indexer writes the new form beside it (novox/hq ADR 0453 clause 6): FP2, the fingerprint of the new +// form (`src2:` and 64 hex digits), or None2, why it has none; one of them whenever either is, never both; and +// Manifest, the digest of the module's manifest, the key of the manifests bucket. A value written before has none of +// the three, and a reader of today's shape ignores them. type Module struct { V int `json:"v"` FP string `json:"fp,omitempty"` None string `json:"none,omitempty"` + FP2 string `json:"fp2,omitempty"` + None2 string `json:"none2,omitempty"` + Manifest string `json:"manifest,omitempty"` Src uint64 `json:"src,omitempty"` Repository string `json:"repository"` Commit string `json:"commit"` @@ -295,8 +396,10 @@ func (m Module) check() error { return fmt.Errorf("delivery: a module has neither a fingerprint nor a reason for none") case m.FP != "" && !strings.HasPrefix(m.FP, SourcePrefix): return fmt.Errorf("delivery: the module's fingerprint %q does not start with %s", m.FP, SourcePrefix) + case m.FP2 != "" && m.None2 != "": + return fmt.Errorf("delivery: a module has both fp2 and none2") } - return nil + return checkNewForm(m.FP2, m.Manifest) } // Forge is a message of the forge bridge's stream, trimmed to repository, branch, pull request and head. A pull -- 2.54.0 From d4967f186e9f240ae667b1dc2b58a5e64dc5bfb3 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 01:44:22 +0000 Subject: [PATCH 05/41] Test the indexer's manifests, the new fields of modules and candidates, and the fpset (issue 1357) --- go/delivery/indexer2_test.go | 321 +++++++++++++++++++++++++++++++++++ 1 file changed, 321 insertions(+) create mode 100644 go/delivery/indexer2_test.go diff --git a/go/delivery/indexer2_test.go b/go/delivery/indexer2_test.go new file mode 100644 index 0000000..4aa1fe0 --- /dev/null +++ b/go/delivery/indexer2_test.go @@ -0,0 +1,321 @@ +package delivery + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "strings" + "testing" +) + +// The values below that today's SDK wrote were produced by EncodeCandidate(sampleCandidate()) and EncodeModule at +// mesh-sdk 5786ba19 (source_test.go's samples), and are pasted as the bytes it wrote. +const ( + oldCandidateJSON = `{"v":1,"group":"feat/x","base":12,"heads":"4b5491557853","pulls":[{"repository":"novox/mesh-sdk","number":36,"head":"aaaa","modules":["mesh-mail"]},{"repository":"novox/mesh-control","number":7,"head":"bbbb","modules":["mesh-controller"]}]}` + oldModuleJSON = `{"v":1,"fp":"src1:abc","src":5,"repository":"novox/mesh-sdk","commit":"cccc"}` + oldNoneJSON = `{"v":1,"none":"a build that resolves packages when built","repository":"novox/mesh-sdk","commit":"cccc"}` +) + +// The reader structs of today's shape (source.go at 5786ba19): what the gate and the view are built on until they move +// to this SDK. They decode with plain json.Unmarshal after the version check, as decode does. +type oldModule struct { + V int `json:"v"` + FP string `json:"fp,omitempty"` + None string `json:"none,omitempty"` + Src uint64 `json:"src,omitempty"` + Repository string `json:"repository"` + Commit string `json:"commit"` +} + +type oldCandidate struct { + V int `json:"v"` + Group string `json:"group"` + Base uint64 `json:"base"` + HeadsHash string `json:"heads"` + Pulls []struct { + Head + Modules []string `json:"modules"` + } `json:"pulls"` +} + +func fp2of(c byte) string { return SourcePrefix2 + strings.Repeat(string(c), 64) } + +func digestOf(c byte) string { return strings.Repeat(string(c), 64) } + +func newCandidate() Candidate { + c := sampleCandidate() + c.Pulls[0].Fps = map[string]CandidateModule{"mesh-mail": {FP2: fp2of('a'), Manifest: digestOf('1')}} + c.Pulls[1].Fps = map[string]CandidateModule{"mesh-controller": {None2: "no lock", Manifest: digestOf('2')}} + c.FPSet = FingerprintSet(c.Pulls) + return c +} + +// 1357/AC1: the digest is the hex sha256 of exactly the bytes; the key is that hex; a key that is not 64 lowercase hex +// is refused. +func TestManifestKeys(t *testing.T) { + raw := []byte("{\n \"module\": \"x\"\n}\n") + sum := sha256.Sum256(raw) + want := hex.EncodeToString(sum[:]) + if got := ManifestDigest(raw); got != want { + t.Errorf("ManifestDigest = %s, want %s", got, want) + } + key, err := ManifestsKey(want) + if err != nil || key != want { + t.Errorf("ManifestsKey = %q, %v", key, err) + } + back, err := ParseManifestsKey(key) + if err != nil || back != want { + t.Errorf("ParseManifestsKey = %q, %v", back, err) + } + for _, bad := range []string{"", "abc", strings.ToUpper(want), want + "0", want[:63], "sha256:" + want, strings.Repeat("g", 64)} { + if _, err := ManifestsKey(bad); err == nil || !strings.Contains(err.Error(), "\""+bad+"\"") { + t.Errorf("ManifestsKey(%q) = %v, want a refusal naming it", bad, err) + } + if _, err := ParseManifestsKey(bad); err == nil || !strings.Contains(err.Error(), "\""+bad+"\"") { + t.Errorf("ParseManifestsKey(%q) = %v, want a refusal naming it", bad, err) + } + } + if ManifestsBucket != "mesh-delivery-indexer_manifests" || ManifestsHistory != 1 { + t.Errorf("bucket %s history %d", ManifestsBucket, ManifestsHistory) + } +} + +// 1357/AC2 (the shape of it; the 21 KB case is in manifests_mailu_test.go): a manifests value encodes and decodes with +// its manifest byte for byte, whatever its spacing, escapes and characters, and its digest is the key. +func TestManifestRoundTripBytesForByte(t *testing.T) { + // odd spacing, a tab, an escaped newline and quote, an em dash and an HTML-looking character, a trailing newline: + // what a JSON re-encoder would change. + raw := []byte("{\n\t\"module\": \"x\",\n \"why\": \"a — b & \\n \\\" d é\",\n\n \"list\": [ \"a\",\"b\" ]\n}\n\n") + m := Manifest{Module: "x", JSON: string(raw)} + data, err := EncodeManifest(m) + if err != nil { + t.Fatal(err) + } + got, err := DecodeManifest(data) + if err != nil { + t.Fatal(err) + } + if got.JSON != string(raw) || ManifestDigest([]byte(got.JSON)) != ManifestDigest(raw) || got.Module != "x" || got.V != Version { + t.Errorf("the manifest changed on the way: %q", got.JSON) + } + for name, bad := range map[string]Manifest{ + "no module": {JSON: string(raw)}, + "a dotted module": {Module: "a.b", JSON: string(raw)}, + "no manifest": {Module: "x"}, + "not JSON": {Module: "x", JSON: "{nope"}, + "not UTF-8": {Module: "x", JSON: "{\"a\":\"\xff\"}"}, + } { + if _, err := EncodeManifest(bad); err == nil { + t.Errorf("a manifest with %s was encoded", name) + } + } + if _, err := DecodeManifest([]byte(`{"v":2,"module":"m","manifest":"{}"}`)); !errors.Is(err, ErrUnknownVersion) { + t.Errorf("version 2: %v", err) + } +} + +// 1357/AC3: a Module and a Candidate written by today's SDK decode and check under the new one, new fields empty. +func TestOldValuesDecodeUnderTheNewSDK(t *testing.T) { + c, err := DecodeCandidate([]byte(oldCandidateJSON)) + if err != nil { + t.Fatal(err) + } + if c.FPSet != "" || c.Pulls[0].Fps != nil || c.Pulls[1].Fps != nil || c.Group != "feat/x" { + t.Errorf("a candidate of today grew fields: %+v", c) + } + for _, in := range []string{oldModuleJSON, oldNoneJSON} { + m, err := DecodeModule([]byte(in)) + if err != nil { + t.Fatal(err) + } + if m.FP2 != "" || m.None2 != "" || m.Manifest != "" { + t.Errorf("a module of today grew fields: %+v", m) + } + } + // And the new SDK writes today's bytes for today's value: nothing new appears when nothing new is set. + data, err := EncodeCandidate(sampleCandidate()) + if err != nil || string(data) != oldCandidateJSON { + t.Errorf("EncodeCandidate of an old value = %s, %v", data, err) + } + data, err = EncodeModule(Module{FP: "src1:abc", Src: 5, Repository: "novox/mesh-sdk", Commit: "cccc"}) + if err != nil || string(data) != oldModuleJSON { + t.Errorf("EncodeModule of an old value = %s, %v", data, err) + } +} + +// 1357/AC4: a value with the new fields decodes, under a reader struct of today's shape, with no error. +func TestNewValuesDecodeUnderTheOldReader(t *testing.T) { + m := Module{FP: "src1:abc", FP2: fp2of('b'), Manifest: digestOf('3'), Src: 5, Repository: "novox/mesh-sdk", Commit: "cccc"} + data, err := EncodeModule(m) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), `"fp2"`) || !strings.Contains(string(data), `"manifest"`) { + t.Fatalf("the new fields are not in %s", data) + } + if err := checkVersion(data); err != nil { + t.Fatal(err) + } + var om oldModule + if err := json.Unmarshal(data, &om); err != nil || om.FP != "src1:abc" || om.Commit != "cccc" { + t.Errorf("the old reader of a module: %+v %v", om, err) + } + data, err = EncodeCandidate(newCandidate()) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), `"fpset"`) || !strings.Contains(string(data), `"fp2"`) { + t.Fatalf("the new fields are not in %s", data) + } + var oc oldCandidate + if err := json.Unmarshal(data, &oc); err != nil || oc.Group != "feat/x" || len(oc.Pulls) != 2 || oc.Pulls[0].Modules[0] != "mesh-mail" { + t.Errorf("the old reader of a candidate: %+v %v", oc, err) + } + // and the new reader reads it back whole + got, err := DecodeCandidate(data) + want := newCandidate() + want.V = Version + if err != nil || !SameCandidate(got, want) { + t.Errorf("round trip: %+v %v", got, err) + } + gm, err := DecodeModule(mustJSON(t, m)) + m.V = Version + if err != nil || !SameModule(gm, m) { + t.Errorf("module round trip: %+v %v", gm, err) + } +} + +func mustJSON(t *testing.T, m Module) []byte { + t.Helper() + data, err := EncodeModule(m) + if err != nil { + t.Fatal(err) + } + return data +} + +// 1357/AC5: a Module with an fp2 not of the new form's prefix, or with both fp2 and none2, is refused naming it; a +// Candidate whose fpset is not the hash of its fingerprints is refused. +func TestNewFieldsAreChecked(t *testing.T) { + wantErr := func(name string, err error, words ...string) { + t.Helper() + if err == nil { + t.Errorf("%s accepted", name) + return + } + for _, w := range words { + if !strings.Contains(err.Error(), w) { + t.Errorf("%s: error %q does not name %q", name, err, w) + } + } + } + base := Module{FP: "src1:abc", Repository: "a/b", Commit: "c"} + m := base + m.FP2 = "src1:" + strings.Repeat("a", 64) + _, err := EncodeModule(m) + wantErr("fp2 of the old form", err, "src1:"+strings.Repeat("a", 64)) + m.FP2 = "src2:xyz" + _, err = EncodeModule(m) + wantErr("fp2 malformed", err, "src2:xyz") + m = base + m.FP2, m.None2 = fp2of('a'), "no lock" + _, err = EncodeModule(m) + wantErr("fp2 and none2", err, "both", "fp2", "none2") + m = base + m.Manifest = "abc" + _, err = EncodeModule(m) + wantErr("a manifest digest malformed", err, "abc") + m = base + m.None2 = "no lock" + if _, err = EncodeModule(m); err != nil { + t.Errorf("none2 alone refused: %v", err) + } + // decode refuses what encode refuses + _, err = DecodeModule([]byte(`{"v":1,"fp":"src1:abc","fp2":"src2:zz","repository":"a/b","commit":"c"}`)) + wantErr("decoded fp2 malformed", err, "src2:zz") + + c := newCandidate() + c.FPSet = "000000000000" + _, err = EncodeCandidate(c) + wantErr("a stale fpset", err, "fpset", "000000000000") + c = newCandidate() + c.FPSet = "" + _, err = EncodeCandidate(c) + wantErr("fingerprints and no fpset", err, "fpset") + c = sampleCandidate() + c.FPSet = FingerprintSet(newCandidate().Pulls) + _, err = EncodeCandidate(c) + wantErr("an fpset and no fingerprints", err, "fpset") + c = newCandidate() + c.Pulls[0].Fps["mesh-mail"] = CandidateModule{FP2: "src1:abc"} + c.FPSet = FingerprintSet(c.Pulls) + _, err = EncodeCandidate(c) + wantErr("a candidate's fp2 of the old form", err, "src1:abc") + c = newCandidate() + c.Pulls[0].Fps["mesh-mail"] = CandidateModule{FP2: fp2of('a'), None2: "x"} + c.FPSet = FingerprintSet(c.Pulls) + _, err = EncodeCandidate(c) + wantErr("a candidate's module with both", err, "both", "mesh-mail") + c = newCandidate() + c.Pulls[0].Fps["other"] = CandidateModule{FP2: fp2of('a')} + c.FPSet = FingerprintSet(c.Pulls) + _, err = EncodeCandidate(c) + wantErr("a fingerprint of a module the pull does not touch", err, "other") +} + +// The fpset is the same for any order and changes with any fingerprint or reason (research 063 section 01: +// verdicts..check. is keyed by it). +func TestFingerprintSet(t *testing.T) { + c := newCandidate() + want := FingerprintSet(c.Pulls) + if len(want) != 12 || strings.Trim(want, "0123456789abcdef") != "" { + t.Errorf("fpset %q is not 12 lowercase hex digits", want) + } + c.Pulls[0], c.Pulls[1] = c.Pulls[1], c.Pulls[0] + if got := FingerprintSet(c.Pulls); got != want { + t.Errorf("the order of pull requests changes it: %s", got) + } + c = newCandidate() + c.Pulls[0].Fps["mesh-mail"] = CandidateModule{FP2: fp2of('b')} + if got := FingerprintSet(c.Pulls); got == want { + t.Error("a changed fingerprint keeps it") + } + c = newCandidate() + c.Pulls[1].Fps["mesh-controller"] = CandidateModule{None2: "no tree"} + if got := FingerprintSet(c.Pulls); got == want { + t.Error("a changed reason keeps it") + } + c = newCandidate() + c.Pulls[0].Fps["mesh-mail"] = CandidateModule{FP2: fp2of('a'), Manifest: digestOf('9')} + if got := FingerprintSet(c.Pulls); got != want { + t.Error("a manifest digest, which the fingerprint already covers, changes it") + } +} + +// 1357/AC6: both values keep version 1, and a value of another version is refused with ErrUnknownVersion. +func TestVersionsStayOne(t *testing.T) { + if Version != 1 { + t.Fatalf("Version = %d", Version) + } + for _, data := range [][]byte{mustJSON(t, Module{FP: "src1:a", FP2: fp2of('a'), Repository: "a/b", Commit: "c"})} { + if !strings.HasPrefix(string(data), `{"v":1,`) { + t.Errorf("module at %s", data) + } + } + data, _ := EncodeCandidate(newCandidate()) + if !strings.HasPrefix(string(data), `{"v":1,`) { + t.Errorf("candidate at %s", data) + } + for _, in := range []string{ + `{"v":2,"fp":"src1:a","fp2":"` + fp2of('a') + `","repository":"a/b","commit":"c"}`, + `{"fp2":"` + fp2of('a') + `","repository":"a/b","commit":"c"}`, + } { + if _, err := DecodeModule([]byte(in)); !errors.Is(err, ErrUnknownVersion) { + t.Errorf("DecodeModule(%s) = %v", in, err) + } + } + if _, err := DecodeCandidate([]byte(`{"v":2,"group":"g","fpset":"abc"}`)); !errors.Is(err, ErrUnknownVersion) { + t.Errorf("candidate version 2: %v", err) + } +} -- 2.54.0 From 21cccc6a8db1b725b5b914ca754c8553254b2a97 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 01:46:16 +0000 Subject: [PATCH 06/41] Lay out the builder's build keys, builds.., beside the controller's (issue 1358) --- go/delivery/builder.go | 172 +++++++++++++++++++++++++++++++++ go/delivery/builder_test.go | 187 ++++++++++++++++++++++++++++++++++++ 2 files changed, 359 insertions(+) create mode 100644 go/delivery/builder.go create mode 100644 go/delivery/builder_test.go diff --git a/go/delivery/builder.go b/go/delivery/builder.go new file mode 100644 index 0000000..205e5d5 --- /dev/null +++ b/go/delivery/builder.go @@ -0,0 +1,172 @@ +package delivery + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" +) + +// The builder's keys (novox/hq ADR 0429 clauses 5 and 16, ADR 0437 clause 5, ADR 0405 clauses 2 and 7, ADR 0453 +// clause 4): a build is `.` in the builder's bucket, one key per fingerprint, since equal +// fingerprints are equal content and the same source is never built twice. They are written only by the builder part +// from R4c and read by the checker, the assigner and the view, and from R4e by the controller. They are not the +// controller's build records (buildrecord.go: BuildsBucket, BuildKey), which stay as they are. +// +// **The rules its writer keeps.** A builder claims a fingerprint that has no build key by creating it queued or +// building, a create that fails when the key exists, and moves it by compare-and-set; a builder reclaims only a build +// whose `by` is offline in presence. A failed key is built again as a new revision of the same key (ADR 0429 clause +// 18). Only the builder's credential writes the bucket (clause 5); every other part reads it. +const ( + // BuilderModule is the module of the builder part. + BuilderModule = "mesh-delivery-builder" + // BuilderBuildsBucket is the builder's bucket of builds: `_`. + BuilderBuildsBucket = BuilderModule + "_builds" + // BuilderBuildsHistory is the revisions the bucket keeps per key. + BuilderBuildsHistory = 5 +) + +// The states of a build. +const ( + BuildQueued = "queued" + BuildBuilding = "building" + BuildDone = "done" + BuildFailed = "failed" +) + +// BuilderBuildKey is the key, in BuilderBuildsBucket, of a module's build at a fingerprint: `.`, the hex +// being the fingerprint's key part (FingerprintKeyPart), since a bus key may not hold ':'. A module holding a dot or a +// character a key refuses, and a fingerprint not of the new form, are refused, naming them. +func BuilderBuildKey(module, fingerprint string) (string, error) { + if err := checkName("module", module); err != nil { + return "", err + } + part, err := FingerprintKeyPart(fingerprint) + if err != nil { + return "", err + } + return module + "." + part, nil +} + +// ParseBuilderBuildKey is the inverse of BuilderBuildKey: the module and the fingerprint (`src2:` and the hex). +func ParseBuilderBuildKey(key string) (module, fingerprint string, err error) { + module, part, ok := strings.Cut(key, ".") + if !ok { + return "", "", fmt.Errorf("delivery: %q is not a builder build key, .", key) + } + if err := checkName("module", module); err != nil { + return "", "", fmt.Errorf("delivery: %q is not a builder build key: %w", key, err) + } + if fingerprint, err = ParseFingerprintKeyPart(part); err != nil { + return "", "", fmt.Errorf("delivery: %q is not a builder build key: %w", key, err) + } + return module, fingerprint, nil +} + +// BuiltArtifact is one artifact of a done build: its name in the manifest, its kind, its content digest and, for an +// image, the reference it was pushed to. +type BuiltArtifact struct { + Name string `json:"name"` + Kind string `json:"kind,omitempty"` + Digest string `json:"digest"` + Reference string `json:"reference,omitempty"` +} + +// ToolchainUsed is what a build used for one language (ADR 0405 clauses 2 and 7): the installed toolchain's version and +// the build module's recipe version. +type ToolchainUsed struct { + Version string `json:"version"` + Recipe string `json:"recipe"` +} + +// BuilderBuild is the value of a builder build key: its State (queued, building, done or failed), By the builder +// instance that holds it, Step what it is doing or did last, Manifest the digest of the module's manifest (the key of +// the indexer's manifests bucket), Artifacts when done, Toolchains by language when it built or is building (a done +// build names them, so two builders that differ are seen: ADR 0405 clause 2), and Error when failed. +type BuilderBuild struct { + V int `json:"v"` + State string `json:"state"` + By string `json:"by,omitempty"` + Step string `json:"step,omitempty"` + Manifest string `json:"manifest,omitempty"` + Artifacts []BuiltArtifact `json:"artifacts,omitempty"` + Toolchains map[string]ToolchainUsed `json:"toolchains,omitempty"` + Error string `json:"error,omitempty"` +} + +func (b BuilderBuild) check() error { + switch b.State { + case BuildQueued, BuildBuilding, BuildDone, BuildFailed: + default: + return fmt.Errorf("delivery: the build state %q is not queued, building, done or failed", b.State) + } + if b.Manifest != "" { + if _, err := ManifestsKey(b.Manifest); err != nil { + return err + } + } + for language, t := range b.Toolchains { + if t.Version == "" { + return fmt.Errorf("delivery: the toolchain of %q has no version", language) + } + if t.Recipe == "" { + return fmt.Errorf("delivery: the toolchain of %q has no recipe version", language) + } + } + for i, a := range b.Artifacts { + if a.Name == "" { + return fmt.Errorf("delivery: artifact %d of the build has no name", i) + } + if a.Digest == "" { + return fmt.Errorf("delivery: the artifact %q of the build has no digest", a.Name) + } + } + switch b.State { + case BuildQueued, BuildBuilding: + if len(b.Artifacts) > 0 || b.Error != "" { + return fmt.Errorf("delivery: a %s build carries no artifact and no error", b.State) + } + case BuildDone: + if len(b.Artifacts) == 0 { + return fmt.Errorf("delivery: a done build names no artifact") + } + if b.Manifest == "" { + return fmt.Errorf("delivery: a done build names no manifest digest") + } + if b.Error != "" { + return fmt.Errorf("delivery: a done build carries an error") + } + case BuildFailed: + if b.Error == "" { + return fmt.Errorf("delivery: a failed build says no error") + } + } + return nil +} + +// EncodeBuilderBuild is b as JSON at Version; it refuses a build its state does not allow (done without artifacts or a +// manifest digest, failed without an error, an artifact without a name or digest, a toolchain without a version or a +// recipe version). +func EncodeBuilderBuild(b BuilderBuild) ([]byte, error) { + if err := b.check(); err != nil { + return nil, err + } + b.V = Version + return json.Marshal(b) +} + +// DecodeBuilderBuild reads a builder build value; it refuses an unknown version (ErrUnknownVersion) and what +// EncodeBuilderBuild refuses. +func DecodeBuilderBuild(data []byte) (BuilderBuild, error) { + var b BuilderBuild + if err := decode(data, &b); err != nil { + return BuilderBuild{}, err + } + if err := b.check(); err != nil { + return BuilderBuild{}, err + } + return b, nil +} + +// SameBuilderBuild says whether two builds are the same value, so the builder writes only when they differ. +func SameBuilderBuild(a, b BuilderBuild) bool { return reflect.DeepEqual(a, b) } diff --git a/go/delivery/builder_test.go b/go/delivery/builder_test.go new file mode 100644 index 0000000..2731819 --- /dev/null +++ b/go/delivery/builder_test.go @@ -0,0 +1,187 @@ +package delivery + +import ( + "errors" + "reflect" + "strings" + "testing" +) + +func doneBuild() BuilderBuild { + return BuilderBuild{ + State: BuildDone, + By: "builder.novox.1", + Step: "pushed", + Manifest: digestOf('4'), + Artifacts: []BuiltArtifact{ + {Name: "code", Kind: "bundle", Digest: "sha256:" + digestOf('5')}, + {Name: "server", Kind: "image", Digest: "sha256:" + digestOf('6'), Reference: "registry.example/mail@sha256:" + digestOf('6')}, + }, + Toolchains: map[string]ToolchainUsed{ + "go": {Version: "1.26.0", Recipe: "3"}, + "typescript": {Version: "5.9.2", Recipe: "1"}, + }, + } +} + +func builderSamples() map[string]BuilderBuild { + failed := BuilderBuild{State: BuildFailed, By: "builder.novox.1", Step: "test", Error: "go test: 2 failed"} + return map[string]BuilderBuild{ + "queued": {State: BuildQueued, Manifest: digestOf('4')}, + "building": {State: BuildBuilding, By: "builder.novox.1", Step: "fetch", Manifest: digestOf('4')}, + "done": doneBuild(), + "failed": failed, + } +} + +// 1358/AC1: the key is ., parsing gives both back; a module with a dot, an empty one, and a fingerprint not +// of the new form are refused, naming it. +func TestBuilderBuildKey(t *testing.T) { + fp := fp2of('a') + key, err := BuilderBuildKey("mesh-mail", fp) + if err != nil || key != "mesh-mail."+strings.Repeat("a", 64) { + t.Fatalf("key = %q, %v", key, err) + } + module, back, err := ParseBuilderBuildKey(key) + if err != nil || module != "mesh-mail" || back != fp { + t.Errorf("parse = %q, %q, %v", module, back, err) + } + src1, _ := SourceFingerprint(SourceInputs{Module: "m", Tree: "t"}) + for module, fp := range map[string]string{"a.b": fp, "": fp, "mesh-mail": src1, "mail ": fp} { + if _, err := BuilderBuildKey(module, fp); err == nil { + t.Errorf("BuilderBuildKey(%q, %q) accepted", module, fp) + } + } + if _, err := BuilderBuildKey("a.b", fp); err == nil || !strings.Contains(err.Error(), "a.b") { + t.Errorf("a dotted module is not named: %v", err) + } + if _, err := BuilderBuildKey("mesh-mail", src1); err == nil || !strings.Contains(err.Error(), src1) { + t.Errorf("a src1 fingerprint is not named: %v", err) + } + for _, bad := range []string{"", "mesh-mail", "mesh-mail.", "mesh-mail.abc", "mesh-mail." + strings.Repeat("A", 64), ".x", "mesh-mail.build.7", "a.b." + strings.Repeat("a", 64)} { + if _, _, err := ParseBuilderBuildKey(bad); err == nil || !strings.Contains(err.Error(), "\""+bad+"\"") { + t.Errorf("ParseBuilderBuildKey(%q) = %v, want a refusal naming it", bad, err) + } + } +} + +// 1358/AC2: a build value in each state encodes and decodes unchanged. +func TestBuilderBuildRoundTrip(t *testing.T) { + for name, b := range builderSamples() { + data, err := EncodeBuilderBuild(b) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if !strings.HasPrefix(string(data), `{"v":1,`) { + t.Errorf("%s at %s", name, data) + } + got, err := DecodeBuilderBuild(data) + b.V = Version + if err != nil || !SameBuilderBuild(got, b) { + t.Errorf("%s: %+v %v", name, got, err) + } + } +} + +// 1358/AC3: done without artifacts or without a manifest digest, failed without an error, an artifact without a digest +// and an unknown state are refused. +func TestBuilderBuildChecked(t *testing.T) { + cases := map[string]struct { + edit func(*BuilderBuild) + words []string + }{ + "done without artifacts": {func(b *BuilderBuild) { b.Artifacts = nil }, []string{"artifact"}}, + "done without a manifest": {func(b *BuilderBuild) { b.Manifest = "" }, []string{"manifest"}}, + "a malformed manifest": {func(b *BuilderBuild) { b.Manifest = "abc" }, []string{"abc"}}, + "an artifact without a digest": {func(b *BuilderBuild) { b.Artifacts[1].Digest = "" }, []string{"digest", "server"}}, + "an artifact without a name": {func(b *BuilderBuild) { b.Artifacts[0].Name = "" }, []string{"name"}}, + "an unknown state": {func(b *BuilderBuild) { b.State = "paused" }, []string{"paused"}}, + "a toolchain without a version": {func(b *BuilderBuild) { b.Toolchains["go"] = ToolchainUsed{Recipe: "3"} }, []string{"go", "version"}}, + "a toolchain without a recipe": {func(b *BuilderBuild) { b.Toolchains["go"] = ToolchainUsed{Version: "1.26.0"} }, []string{"go", "recipe"}}, + } + for name, c := range cases { + b := doneBuild() + c.edit(&b) + _, err := EncodeBuilderBuild(b) + if err == nil { + t.Errorf("%s accepted", name) + continue + } + for _, w := range c.words { + if !strings.Contains(err.Error(), w) { + t.Errorf("%s: error %q does not name %q", name, err, w) + } + } + } + f := builderSamples()["failed"] + f.Error = "" + if _, err := EncodeBuilderBuild(f); err == nil || !strings.Contains(err.Error(), "error") { + t.Errorf("failed without an error: %v", err) + } + // a queued or building value may not already carry an outcome + q := builderSamples()["queued"] + q.Artifacts = doneBuild().Artifacts + if _, err := EncodeBuilderBuild(q); err == nil { + t.Error("a queued build with artifacts accepted") + } + // decode refuses what encode refuses + if _, err := DecodeBuilderBuild([]byte(`{"v":1,"state":"done","manifest":"` + digestOf('4') + `"}`)); err == nil { + t.Error("decoded a done build without artifacts") + } +} + +// 1358/AC4: a done value names the toolchain version per language and the recipe version; two values that differ only +// there are not the same. +func TestBuilderBuildNamesItsToolchains(t *testing.T) { + a, b, c := doneBuild(), doneBuild(), doneBuild() + b.Toolchains["go"] = ToolchainUsed{Version: "1.26.1", Recipe: "3"} + c.Toolchains["go"] = ToolchainUsed{Version: "1.26.0", Recipe: "4"} + if SameBuilderBuild(a, b) || SameBuilderBuild(a, c) || SameBuilderBuild(b, c) { + t.Error("values differing only in a toolchain's version or recipe are the same") + } + if !SameBuilderBuild(a, doneBuild()) { + t.Error("equal values differ") + } + data, err := EncodeBuilderBuild(a) + if err != nil { + t.Fatal(err) + } + got, err := DecodeBuilderBuild(data) + if err != nil || !reflect.DeepEqual(got.Toolchains, a.Toolchains) { + t.Errorf("toolchains = %+v, %v", got.Toolchains, err) + } +} + +// 1358/AC5: a value of another version, or none, is refused with ErrUnknownVersion. +func TestBuilderBuildVersion(t *testing.T) { + for _, in := range []string{`{"v":2,"state":"queued"}`, `{"state":"queued"}`} { + if _, err := DecodeBuilderBuild([]byte(in)); !errors.Is(err, ErrUnknownVersion) { + t.Errorf("DecodeBuilderBuild(%s) = %v", in, err) + } + } +} + +// 1358/AC6: the builder's bucket and module do not collide with the controller's build layout (buildrecord.go), which +// this change leaves as it was. +func TestBuilderLayoutDoesNotCollideWithTheControllers(t *testing.T) { + if BuilderBuildsBucket != "mesh-delivery-builder_builds" || BuilderBuildsHistory != 5 || BuilderModule != "mesh-delivery-builder" { + t.Errorf("bucket %s history %d module %s", BuilderBuildsBucket, BuilderBuildsHistory, BuilderModule) + } + if BuilderBuildsBucket == BuildsBucket || BuilderModule == "mesh-controller" { + t.Error("the builder's bucket is the controller's") + } + if BuildsBucket != "mesh-controller_builds" || BuildKey("m", "7") != "m.build.7" || NewestKey("m") != "m.newest" { + t.Error("the controller's layout changed") + } + // the two key forms are never read as each other + key, _ := BuilderBuildKey("m", fp2of('a')) + if key == BuildKey("m", "a") || key == NewestKey("m") { + t.Errorf("builder key %q equals a controller key", key) + } + if _, _, err := ParseBuilderBuildKey(BuildKey("m", digestOf('a'))); err == nil { + t.Error("a controller build key parsed as the builder's") + } + if _, _, err := ParseBuilderBuildKey(NewestKey("m")); err == nil { + t.Error("a controller newest key parsed as the builder's") + } +} -- 2.54.0 From 33b87b794cd35f4253d6f6f3e38e61ce520c562f Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 01:48:29 +0000 Subject: [PATCH 07/41] Lay out the checker's verdicts, keyed by group and heads, and the kept verdicts (issue 1359) --- go/delivery/checker.go | 295 ++++++++++++++++++++++++++++++++++++ go/delivery/checker_test.go | 222 +++++++++++++++++++++++++++ 2 files changed, 517 insertions(+) create mode 100644 go/delivery/checker.go create mode 100644 go/delivery/checker_test.go diff --git a/go/delivery/checker.go b/go/delivery/checker.go new file mode 100644 index 0000000..bdb57e9 --- /dev/null +++ b/go/delivery/checker.go @@ -0,0 +1,295 @@ +package delivery + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + "time" +) + +// The checker's keys (novox/hq ADR 0447 clauses 1 to 3, ADR 0444, ADR 0446 clause 5, ADR 0453 clause 4): the verdicts of +// its checks and the passing verdicts it keeps per package and level. The checker writes them from R4c, dark; the +// forge bridge writes the status `mesh/check` from a verdict (unrequired until R4f), the gate reads it from R4f, and +// the view and the watchman read them (a running check carries a pulse). +// +// **The verdict's key is the group and the heads, not the fingerprints** (the lead's limit for R4a). Research 063 +// section 01 drew `verdicts..check.`; ADR 0447 clause 1 judges a pull request's head merged onto its +// trunk's head, and a repository without modules has no fingerprint set, so the key is `.` as the +// heads hash of HeadsHash gives it. The fpset (FingerprintSet) is not part of it. +const ( + // CheckerModule is the module of the checker part. + CheckerModule = "mesh-delivery-checker" + // CheckerVerdictsBucket is the checker's bucket of verdicts, `_`. + CheckerVerdictsBucket = CheckerModule + "_verdicts" + // CheckerVerdictsHistory is the revisions the verdicts bucket keeps per key. + CheckerVerdictsHistory = 5 + // CheckerKeptBucket is the checker's bucket of kept verdicts. A key is written once, by a create that fails when + // the key exists, by the checker's credential alone, from the result the checker reads from the test run and + // never from anything the code under test writes (ADR 0444 clause 5); nothing changes it afterwards. + CheckerKeptBucket = CheckerModule + "_kept" + // CheckerKeptHistory is the revisions the kept bucket keeps per key. + CheckerKeptHistory = 1 +) + +// The test levels (ADR 0445 clause 2) and the states of a verdict. +const ( + LevelUnit = "unit" + LevelIntegration = "integration" + + VerdictRunning = "running" + VerdictPass = "pass" + VerdictFail = "fail" +) + +func checkLevel(level string) error { + if level != LevelUnit && level != LevelIntegration { + return fmt.Errorf("delivery: the level %q is not %s or %s", level, LevelUnit, LevelIntegration) + } + return nil +} + +func isHex12(s string) bool { return len(s) == 12 && strings.Trim(s, "0123456789abcdef") == "" } + +// CheckerVerdictKey is the key, in CheckerVerdictsBucket, of a group's check at its heads: `.`, the group escaped as CandidatesKey does and the heads hash as HeadsHash gives it (12 lowercase hex digits). +func CheckerVerdictKey(group, headsHash string) (string, error) { + g, err := groupKey(group) + if err != nil { + return "", err + } + if !isHex12(headsHash) { + return "", fmt.Errorf("delivery: %q is not a heads hash, 12 lowercase hex digits", headsHash) + } + return g + "." + headsHash, nil +} + +// ParseCheckerVerdictKey is the inverse of CheckerVerdictKey. +func ParseCheckerVerdictKey(key string) (group, headsHash string, err error) { + i := strings.LastIndex(key, ".") + if i < 0 { + return "", "", fmt.Errorf("delivery: %q is not a verdict key, .", key) + } + part, headsHash := key[:i], key[i+1:] + if !isHex12(headsHash) { + return "", "", fmt.Errorf("delivery: %q is not a verdict key: %q is not a heads hash, 12 lowercase hex digits", key, headsHash) + } + if group, err = parseGroupKey(part); err != nil { + return "", "", fmt.Errorf("delivery: %q is not a verdict key: %w", key, err) + } + if strings.Contains(part, ".") { + return "", "", fmt.Errorf("delivery: %q is not a verdict key: the group part holds a dot", key) + } + return group, headsHash, nil +} + +// LevelResult is a level's tests: how many ran in this check and how many were taken from kept verdicts (ADR 0444 +// clause 2). +type LevelResult struct { + Run int `json:"run"` + Kept int `json:"kept"` +} + +// Took is the time of each part of a check in integer milliseconds (ADR 0447 clause 3): waiting in the queue, cloning, +// the unit level, the integration level's setup and its running, and the composition. +type Took struct { + Queue int64 `json:"queue"` + Clone int64 `json:"clone"` + Unit int64 `json:"unit"` + IntegrationSetup int64 `json:"integration-setup"` + IntegrationRun int64 `json:"integration-run"` + Composition int64 `json:"composition"` +} + +// FailedFirst names what failed first: a test (its package and name) or a machine (and the words of the composition's +// refusal); one of them, with Words optional. +type FailedFirst struct { + Test string `json:"test,omitempty"` + Machine string `json:"machine,omitempty"` + Words string `json:"words,omitempty"` +} + +// Verdict is the value of a verdict key: the group and its heads hash, the trunk head of each repository it was judged +// against (repository to full sha), the State (running, pass or fail), what failed first, the tests run and kept per +// level, each part's time, the machines composed and the mesh-sdk version the composition ran with (ADR 0446 clause 5). +type Verdict struct { + V int `json:"v"` + Group string `json:"group"` + Heads string `json:"heads"` + Trunk map[string]string `json:"trunk,omitempty"` + State string `json:"state"` + Failed FailedFirst `json:"failed,omitzero"` + Levels map[string]LevelResult `json:"levels,omitempty"` + Took Took `json:"took,omitzero"` + Machines []string `json:"machines,omitempty"` + SDK string `json:"sdk,omitempty"` +} + +func (v Verdict) check() error { + if v.Group == "" { + return fmt.Errorf("delivery: a verdict names no group") + } + if !isHex12(v.Heads) { + return fmt.Errorf("delivery: the verdict's heads hash %q is not 12 lowercase hex digits", v.Heads) + } + for repository, head := range v.Trunk { + if head == "" { + return fmt.Errorf("delivery: the verdict's trunk head of %s is empty", repository) + } + } + for level, r := range v.Levels { + if err := checkLevel(level); err != nil { + return err + } + if r.Run < 0 || r.Kept < 0 { + return fmt.Errorf("delivery: the verdict's %s level counts a negative number of tests", level) + } + } + for part, ms := range map[string]int64{"queue": v.Took.Queue, "clone": v.Took.Clone, "unit": v.Took.Unit, + "integration-setup": v.Took.IntegrationSetup, "integration-run": v.Took.IntegrationRun, "composition": v.Took.Composition} { + if ms < 0 { + return fmt.Errorf("delivery: the verdict's time of the part %s is negative", part) + } + } + named := v.Failed.Test != "" || v.Failed.Machine != "" + switch v.State { + case VerdictRunning: + if v.Failed != (FailedFirst{}) { + return fmt.Errorf("delivery: a running verdict names no failure") + } + case VerdictPass: + if v.Failed != (FailedFirst{}) { + return fmt.Errorf("delivery: a pass names no failure") + } + for _, level := range []string{LevelUnit, LevelIntegration} { + if _, ok := v.Levels[level]; !ok { + return fmt.Errorf("delivery: a pass says nothing of the %s level", level) + } + } + case VerdictFail: + if !named { + return fmt.Errorf("delivery: a fail names neither a test nor a machine") + } + default: + return fmt.Errorf("delivery: the verdict state %q is not running, pass or fail", v.State) + } + return nil +} + +// EncodeVerdict is v as JSON at Version; it refuses a fail that names neither a test nor a machine, a pass without both +// levels, an unknown state or level, and a negative count or time. +func EncodeVerdict(v Verdict) ([]byte, error) { + if err := v.check(); err != nil { + return nil, err + } + v.V = Version + return json.Marshal(v) +} + +// DecodeVerdict reads a verdict; it refuses an unknown version (ErrUnknownVersion) and what EncodeVerdict refuses. +func DecodeVerdict(data []byte) (Verdict, error) { + var v Verdict + if err := decode(data, &v); err != nil { + return Verdict{}, err + } + if err := v.check(); err != nil { + return Verdict{}, err + } + return v, nil +} + +// SameVerdict says whether two verdicts are the same value, so the checker writes only when they differ. +func SameVerdict(a, b Verdict) bool { return reflect.DeepEqual(a, b) } + +// KeptKey is the key, in CheckerKeptBucket, of a kept verdict: `.`, the level unit or integration (ADR +// 0445 clause 2) and the digest, lowercase hex, of everything that package's tests read at that level as the +// language's build module names it (ADR 0444 clause 1, ADR 0445 clause 5). +func KeptKey(level, digest string) (string, error) { + if err := checkLevel(level); err != nil { + return "", err + } + if digest == "" || strings.Trim(digest, "0123456789abcdef") != "" { + return "", fmt.Errorf("delivery: %q is not a digest, lowercase hex", digest) + } + return level + "." + digest, nil +} + +// ParseKeptKey is the inverse of KeptKey. +func ParseKeptKey(key string) (level, digest string, err error) { + level, digest, ok := strings.Cut(key, ".") + if !ok { + return "", "", fmt.Errorf("delivery: %q is not a kept key, .", key) + } + if _, err = KeptKey(level, digest); err != nil { + return "", "", fmt.Errorf("delivery: %q is not a kept key: %w", key, err) + } + return level, digest, nil +} + +// Kept is the value of a kept key: a passing verdict of one package at one level (State is pass and nothing else: a +// failure is never kept, ADR 0444 clause 1), with the repository, the package, the number of tests, the time they took +// in milliseconds, the toolchain's version and when it was kept. +type Kept struct { + V int `json:"v"` + State string `json:"state"` + Repository string `json:"repository"` + Package string `json:"package"` + Level string `json:"level"` + Tests int `json:"tests"` + Took int64 `json:"took"` + Toolchain string `json:"toolchain"` + At time.Time `json:"at"` +} + +func (k Kept) check() error { + if k.State != VerdictPass { + return fmt.Errorf("delivery: a kept verdict is a pass, never %q: a failure is never kept", k.State) + } + if err := checkRepository(k.Repository); err != nil { + return err + } + if k.Package == "" { + return fmt.Errorf("delivery: a kept verdict names no package") + } + if err := checkLevel(k.Level); err != nil { + return err + } + if k.Tests < 0 || k.Took < 0 { + return fmt.Errorf("delivery: a kept verdict counts a negative number of tests or a negative time") + } + if k.Toolchain == "" { + return fmt.Errorf("delivery: a kept verdict names no toolchain version") + } + if k.At.IsZero() { + return fmt.Errorf("delivery: a kept verdict says no time") + } + return nil +} + +// EncodeKept is k as JSON at Version; it refuses a kept value that is not a pass. +func EncodeKept(k Kept) ([]byte, error) { + if err := k.check(); err != nil { + return nil, err + } + k.V = Version + return json.Marshal(k) +} + +// DecodeKept reads a kept value; it refuses an unknown version (ErrUnknownVersion) and what EncodeKept refuses. +func DecodeKept(data []byte) (Kept, error) { + var k Kept + if err := decode(data, &k); err != nil { + return Kept{}, err + } + if err := k.check(); err != nil { + return Kept{}, err + } + return k, nil +} + +// SameKept says whether two kept values are the same value. +func SameKept(a, b Kept) bool { + at := a.At.Equal(b.At) + a.At, b.At = time.Time{}, time.Time{} + return at && a == b +} diff --git a/go/delivery/checker_test.go b/go/delivery/checker_test.go new file mode 100644 index 0000000..2dd939f --- /dev/null +++ b/go/delivery/checker_test.go @@ -0,0 +1,222 @@ +package delivery + +import ( + "errors" + "strings" + "testing" + "time" +) + +func passVerdict() Verdict { + return Verdict{ + Group: "feat/x", + Heads: "4b5491557853", + Trunk: map[string]string{"novox/mesh-sdk": "aaaa", "novox/mesh-control": "bbbb"}, + State: VerdictPass, + Levels: map[string]LevelResult{LevelUnit: {Run: 12, Kept: 301}, LevelIntegration: {Run: 3, Kept: 40}}, + Took: Took{Queue: 1200, Clone: 800, Unit: 950, IntegrationSetup: 2100, IntegrationRun: 7400, Composition: 3300}, + Machines: []string{"ace", "g14", "novox", "shanks"}, + SDK: "v0.1.14-0.20261019091011-aa68b67389c9", + } +} + +func verdictSamples() map[string]Verdict { + running := Verdict{Group: "feat/x", Heads: "4b5491557853", Trunk: map[string]string{"novox/mesh-sdk": "aaaa"}, State: VerdictRunning} + failTest := passVerdict() + failTest.State, failTest.Failed = VerdictFail, FailedFirst{Test: "novox/mesh-sdk/go/delivery TestHeadsHash"} + failTest.Levels = map[string]LevelResult{LevelUnit: {Run: 1}} + failMachine := passVerdict() + failMachine.State, failMachine.Failed = VerdictFail, FailedFirst{Machine: "ace", Words: "path /etc/x is owned by two modules"} + return map[string]Verdict{"running": running, "pass": passVerdict(), "fail by a test": failTest, "fail by a machine": failMachine} +} + +func keptSample() Kept { + return Kept{State: VerdictPass, Repository: "novox/mesh-sdk", Package: "git.novox.be/novox/mesh-sdk/go/delivery", Level: LevelUnit, + Tests: 41, Took: 190, Toolchain: "1.26.0", At: time.Date(2026, 10, 20, 2, 0, 0, 0, time.UTC)} +} + +// 1359/AC1: the key is .; parsing gives back the group and the hash. +func TestCheckerVerdictKey(t *testing.T) { + heads := []Head{{Repository: "novox/mesh-sdk", Number: 36, Head: "aaaa"}} + hh, _ := HeadsHash(heads) + for _, group := range []string{"feat/x", "fix/1.2", "plain", "odd name=1"} { + key, err := CheckerVerdictKey(group, hh) + if err != nil { + t.Fatal(err) + } + if want := EscapeName(group) + "." + hh; key != want { + t.Errorf("key = %q, want %q", key, want) + } + g, h, err := ParseCheckerVerdictKey(key) + if err != nil || g != group || h != hh { + t.Errorf("parse(%q) = %q, %q, %v", key, g, h, err) + } + } + if k, _ := CheckerVerdictKey("feat/x", hh); !strings.HasPrefix(k, "feat/x.") { + t.Errorf("a group with a slash is not kept readable: %q", k) + } + if _, err := CheckerVerdictKey("", hh); err == nil { + t.Error("an empty group accepted") + } + for _, bad := range []string{"", "feat/x", "feat/x.", "feat/x.abc", "feat/x." + strings.ToUpper(hh), ".4b5491557853", "a.b.4b5491557853", "feat/x.4b5491557853a"} { + if _, _, err := ParseCheckerVerdictKey(bad); err == nil || !strings.Contains(err.Error(), "\""+bad+"\"") { + t.Errorf("ParseCheckerVerdictKey(%q) = %v, want a refusal naming it", bad, err) + } + } + if _, err := CheckerVerdictKey("feat/x", "abc"); err == nil || !strings.Contains(err.Error(), "abc") { + t.Errorf("a short heads hash: %v", err) + } +} + +// 1359/AC2: a verdict in each state round-trips, with times per part in milliseconds, the tests run and kept per level, +// the machines composed and the mesh-sdk version composed with. +func TestVerdictRoundTrip(t *testing.T) { + for name, v := range verdictSamples() { + data, err := EncodeVerdict(v) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if !strings.HasPrefix(string(data), `{"v":1,`) { + t.Errorf("%s at %s", name, data) + } + got, err := DecodeVerdict(data) + v.V = Version + if err != nil || !SameVerdict(got, v) { + t.Errorf("%s: %+v %v", name, got, err) + } + } + data, _ := EncodeVerdict(passVerdict()) + for _, field := range []string{`"queue":1200`, `"clone":800`, `"unit":950`, `"integration-setup":2100`, `"integration-run":7400`, `"composition":3300`, `"run":12`, `"kept":301`, `"machines":["ace","g14","novox","shanks"]`, `"sdk":"v0.1.14`} { + if !strings.Contains(string(data), field) { + t.Errorf("the value lacks %s: %s", field, data) + } + } +} + +// 1359/AC3: a fail that names neither a test nor a machine, and a pass whose levels are missing, are refused. +func TestVerdictChecked(t *testing.T) { + cases := map[string]struct { + edit func(*Verdict) + words []string + }{ + "fail naming nothing": {func(v *Verdict) { v.State, v.Failed = VerdictFail, FailedFirst{} }, []string{"test", "machine"}}, + "fail with words only": {func(v *Verdict) { v.State, v.Failed = VerdictFail, FailedFirst{Words: "x"} }, []string{"test", "machine"}}, + "pass without levels": {func(v *Verdict) { v.Levels = nil }, []string{"level"}}, + "pass without unit": {func(v *Verdict) { delete(v.Levels, LevelUnit) }, []string{"unit"}}, + "pass without integration": {func(v *Verdict) { delete(v.Levels, LevelIntegration) }, []string{"integration"}}, + "pass naming a failure": {func(v *Verdict) { v.Failed = FailedFirst{Test: "T"} }, []string{"pass"}}, + "an unknown state": {func(v *Verdict) { v.State = "maybe" }, []string{"maybe"}}, + "an unknown level": {func(v *Verdict) { v.Levels["e2e"] = LevelResult{} }, []string{"e2e"}}, + "a negative time": {func(v *Verdict) { v.Took.Clone = -1 }, []string{"clone"}}, + "a bad heads hash": {func(v *Verdict) { v.Heads = "xyz" }, []string{"xyz"}}, + "no group": {func(v *Verdict) { v.Group = "" }, []string{"group"}}, + "a trunk head left empty": {func(v *Verdict) { v.Trunk["novox/mesh-sdk"] = "" }, []string{"novox/mesh-sdk"}}, + "a negative count": {func(v *Verdict) { v.Levels[LevelUnit] = LevelResult{Run: -1} }, []string{"unit"}}, + } + for name, c := range cases { + v := passVerdict() + c.edit(&v) + _, err := EncodeVerdict(v) + if err == nil { + t.Errorf("%s accepted", name) + continue + } + for _, w := range c.words { + if !strings.Contains(err.Error(), w) { + t.Errorf("%s: error %q does not name %q", name, err, w) + } + } + } + if _, err := DecodeVerdict([]byte(`{"v":1,"group":"g","heads":"4b5491557853","state":"fail"}`)); err == nil { + t.Error("decoded a fail naming nothing") + } +} + +// 1359/AC4: the kept key is .; a level other than unit or integration, or a digest that is not hex, +// is refused, naming it. +func TestKeptKey(t *testing.T) { + digest := strings.Repeat("ab12", 16) + for _, level := range []string{LevelUnit, LevelIntegration} { + key, err := KeptKey(level, digest) + if err != nil || key != level+"."+digest { + t.Fatalf("key = %q, %v", key, err) + } + l, d, err := ParseKeptKey(key) + if err != nil || l != level || d != digest { + t.Errorf("parse = %q, %q, %v", l, d, err) + } + } + if _, err := KeptKey("e2e", digest); err == nil || !strings.Contains(err.Error(), "e2e") { + t.Errorf("a level e2e: %v", err) + } + if _, err := KeptKey("", digest); err == nil { + t.Error("an empty level accepted") + } + for _, d := range []string{"", "xyz", "ABCD", "ab 12", "sha256:abcd", "ab12.cd"} { + if _, err := KeptKey(LevelUnit, d); err == nil || !strings.Contains(err.Error(), "\""+d+"\"") { + t.Errorf("KeptKey(unit, %q) = %v, want a refusal naming it", d, err) + } + } + for _, bad := range []string{"", "unit", "unit.", "e2e." + digest, "unit.xyz", "unit." + digest + ".x", "." + digest} { + if _, _, err := ParseKeptKey(bad); err == nil || !strings.Contains(err.Error(), "\""+bad+"\"") { + t.Errorf("ParseKeptKey(%q) = %v, want a refusal naming it", bad, err) + } + } + if CheckerVerdictsBucket != "mesh-delivery-checker_verdicts" || CheckerVerdictsHistory != 5 || + CheckerKeptBucket != "mesh-delivery-checker_kept" || CheckerKeptHistory != 1 || CheckerModule != "mesh-delivery-checker" { + t.Errorf("buckets %s %d %s %d", CheckerVerdictsBucket, CheckerVerdictsHistory, CheckerKeptBucket, CheckerKeptHistory) + } +} + +// 1359/AC5: a kept value that is not a pass cannot be encoded; a failure is never kept (ADR 0444 clause 1). +func TestAFailureIsNeverKept(t *testing.T) { + data, err := EncodeKept(keptSample()) + if err != nil { + t.Fatal(err) + } + got, err := DecodeKept(data) + want := keptSample() + want.V = Version + if err != nil || !SameKept(got, want) { + t.Errorf("round trip: %+v %v", got, err) + } + for _, state := range []string{VerdictFail, VerdictRunning, "", "skipped"} { + k := keptSample() + k.State = state + if _, err := EncodeKept(k); err == nil || !strings.Contains(err.Error(), "pass") { + t.Errorf("a kept value in state %q: %v", state, err) + } + } + if _, err := DecodeKept([]byte(`{"v":1,"state":"fail","repository":"novox/mesh-sdk","package":"p","level":"unit","tests":1,"took":1,"toolchain":"1.26.0","at":"2026-10-20T02:00:00Z"}`)); err == nil { + t.Error("decoded a kept failure") + } + for name, edit := range map[string]func(*Kept){ + "a level e2e": func(k *Kept) { k.Level = "e2e" }, + "no package": func(k *Kept) { k.Package = "" }, + "a bad repository": func(k *Kept) { k.Repository = "nope" }, + "no toolchain": func(k *Kept) { k.Toolchain = "" }, + "no time": func(k *Kept) { k.At = time.Time{} }, + "negative tests": func(k *Kept) { k.Tests = -1 }, + "a negative took": func(k *Kept) { k.Took = -1 }, + } { + k := keptSample() + edit(&k) + if _, err := EncodeKept(k); err == nil { + t.Errorf("%s accepted", name) + } + } +} + +// 1359/AC6: a value of another version, or none, is refused with ErrUnknownVersion. +func TestCheckerVersions(t *testing.T) { + for name, dec := range map[string]func([]byte) error{ + "verdict": func(b []byte) error { _, err := DecodeVerdict(b); return err }, + "kept": func(b []byte) error { _, err := DecodeKept(b); return err }, + } { + for _, in := range []string{`{"v":2,"state":"pass"}`, `{"state":"pass"}`} { + if err := dec([]byte(in)); !errors.Is(err, ErrUnknownVersion) { + t.Errorf("%s decode of %s = %v", name, in, err) + } + } + } +} -- 2.54.0 From 3387abf50e5d2c760d8236d81d6e4fd69211396d Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 01:48:41 +0000 Subject: [PATCH 08/41] Name the builder's and the checker's layouts in the delivery package doc (issues 1358, 1359) --- go/delivery/delivery.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/go/delivery/delivery.go b/go/delivery/delivery.go index 848fbfa..d961956 100644 --- a/go/delivery/delivery.go +++ b/go/delivery/delivery.go @@ -30,9 +30,10 @@ // **The source fingerprint** (issue 1203; ADR 0437 §5): one function of named inputs that the controller's builder and // the indexer both call, so they agree byte for byte (fingerprint.go). // -// **The release 4 layouts** (issues 1356 and 1357; ADR 0453 clauses 6 and 16): the new fingerprint form (fingerprint2.go), -// and the indexer's manifests by digest and the new fields of its modules and candidates (manifests.go, source.go). A -// manifests value is the one exception to the size above: a manifest is up to about 21 KB, whole, byte for byte. +// **The release 4 layouts** (issues 1356 to 1359; ADR 0453 clauses 4, 6 and 16): the new fingerprint form (fingerprint2.go), +// the indexer's manifests by digest and the new fields of its modules and candidates (manifests.go, source.go), the +// builder's build keys (builder.go) and the checker's verdicts and kept verdicts (checker.go). A manifests value is the +// one exception to the size above: a manifest is up to about 21 KB, whole, byte for byte. // // Standard library only, and this module's own asks package. package delivery -- 2.54.0 From e5e3e62f28adea065b381f1ada8a3a30dc145943 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 06:01:48 +0000 Subject: [PATCH 09/41] Copy the node-engine's declaration parser into mesh-sdk, from mesh-host main (issue 1353) --- go/declaration/adoption_test.go | 132 + go/declaration/block_test.go | 52 + go/declaration/busy.go | 87 + go/declaration/cron.go | 182 ++ go/declaration/cron_test.go | 100 + go/declaration/declaration.go | 2607 +++++++++++++++++++ go/declaration/declaration_test.go | 595 +++++ go/declaration/emitted_check_test.go | 33 + go/declaration/health.go | 188 ++ go/declaration/health_test.go | 49 + go/declaration/identity_test.go | 61 + go/declaration/into_test.go | 35 + go/declaration/module_test.go | 60 + go/declaration/order_test.go | 59 + go/declaration/ownership_test.go | 130 + go/declaration/process_test.go | 163 ++ go/declaration/provisioner_test.go | 37 + go/declaration/replaces_test.go | 63 + go/declaration/root_test.go | 26 + go/declaration/schedule_declaration_test.go | 53 + go/declaration/stateless_test.go | 30 + go/declaration/trusted_test.go | 23 + go/declaration/userscope_test.go | 49 + 23 files changed, 4814 insertions(+) create mode 100644 go/declaration/adoption_test.go create mode 100644 go/declaration/block_test.go create mode 100644 go/declaration/busy.go create mode 100644 go/declaration/cron.go create mode 100644 go/declaration/cron_test.go create mode 100644 go/declaration/declaration.go create mode 100644 go/declaration/declaration_test.go create mode 100644 go/declaration/emitted_check_test.go create mode 100644 go/declaration/health.go create mode 100644 go/declaration/health_test.go create mode 100644 go/declaration/identity_test.go create mode 100644 go/declaration/into_test.go create mode 100644 go/declaration/module_test.go create mode 100644 go/declaration/order_test.go create mode 100644 go/declaration/ownership_test.go create mode 100644 go/declaration/process_test.go create mode 100644 go/declaration/provisioner_test.go create mode 100644 go/declaration/replaces_test.go create mode 100644 go/declaration/root_test.go create mode 100644 go/declaration/schedule_declaration_test.go create mode 100644 go/declaration/stateless_test.go create mode 100644 go/declaration/trusted_test.go create mode 100644 go/declaration/userscope_test.go diff --git a/go/declaration/adoption_test.go b/go/declaration/adoption_test.go new file mode 100644 index 0000000..e422516 --- /dev/null +++ b/go/declaration/adoption_test.go @@ -0,0 +1,132 @@ +package declaration + +import ( + "strings" + "testing" +) + +// Defends novox/hq ADR 0100: every declaration says whether the node is adopted and which of its +// modules are taken, and the host refuses one it cannot read that from unambiguously. + +const adoptedResources = `"resources":[ + {"id":"hello-web.page","type":"file","path":"/var/lib/hello-web/index.html","content":"a\n"}, + {"id":"hello-web.server","type":"container","name":"hello-web","image":"sha256:` + sixtyFour + `"}, + {"id":"hello-web.data","type":"directory","path":"/var/lib/hello-web"} + ]` + +const sixtyFour = "0000000000000000000000000000000000000000000000000000000000000000" + +func TestAnAdoptionIsReadWithTheDeclaration(t *testing.T) { + d, err := Parse([]byte(`{"adoption":{"taken":["postgres"],"untaken":{"hello-web":["hello-web.page","hello-web.server"]}}, + "declaration":1,` + adoptedResources + `}`)) + if err != nil { + t.Fatal(err) + } + if d.Adoption == nil { + t.Fatal("the adoption was dropped") + } + if len(d.Adoption.Taken) != 1 || d.Adoption.Taken[0] != "postgres" { + t.Errorf("taken read as %v", d.Adoption.Taken) + } + if module, ok := d.Adoption.UntakenModuleOf("hello-web.server"); !ok || module != "hello-web" { + t.Errorf("the container's untaken module read as %q, %v", module, ok) + } + if _, ok := d.Adoption.UntakenModuleOf("hello-web.data"); ok { + t.Error("a resource the adoption does not name was said to be untaken") + } +} + +func TestADeclarationWithNoAdoptionIsConverged(t *testing.T) { + d, err := Parse([]byte(`{"declaration":1,` + adoptedResources + `}`)) + if err != nil { + t.Fatal(err) + } + if d.Adoption != nil { + t.Errorf("a declaration saying nothing about adoption read as adopted: %+v", d.Adoption) + } + if _, ok := d.Adoption.UntakenModuleOf("hello-web.page"); ok { + t.Error("a converged node has an untaken module") + } +} + +func TestAnAdoptionNamingAnUnknownIDIsRefused(t *testing.T) { + refusal := refusalFor(t, `{"adoption":{"taken":[],"untaken":{"hello-web":["hello-web.missing"]}}, + "declaration":1,`+adoptedResources+`}`) + if !strings.Contains(strings.Join(refusal.Problems, "\n"), "hello-web.missing") { + t.Errorf("the unknown id was not named: %v", refusal.Problems) + } +} + +func TestAnAdoptionMayNameAResourceOfAnyKind(t *testing.T) { + // A directory, a service or an action can reach what was found as surely as a file can, so + // the controller lists every resource of an untaken module (novox/hq ADR 0103). + d, err := Parse([]byte(`{"adoption":{"taken":[],"untaken":{"hello-web":["hello-web.data"]}}, + "declaration":1,` + adoptedResources + `}`)) + if err != nil { + t.Fatalf("a directory of an untaken module was refused: %v", err) + } + if module, ok := d.Adoption.UntakenModuleOf("hello-web.data"); !ok || module != "hello-web" { + t.Errorf("the directory is not its module's: %q %v", module, ok) + } +} + +func TestAnIDUnderTwoModulesIsRefused(t *testing.T) { + refusal := refusalFor(t, `{"adoption":{"taken":[],"untaken":{"a":["hello-web.page"],"b":["hello-web.page"]}}, + "declaration":1,`+adoptedResources+`}`) + if !strings.Contains(strings.Join(refusal.Problems, "\n"), "both") { + t.Errorf("an id under two modules was accepted: %v", refusal.Problems) + } +} + +func TestAModuleBothTakenAndUntakenIsRefused(t *testing.T) { + refusal := refusalFor(t, `{"adoption":{"taken":["hello-web"],"untaken":{"hello-web":["hello-web.page"]}}, + "declaration":1,`+adoptedResources+`}`) + if !strings.Contains(strings.Join(refusal.Problems, "\n"), "both taken and untaken") { + t.Errorf("a module both taken and untaken was accepted: %v", refusal.Problems) + } +} + +func TestTheMeshsOwnResourcesAreNeverUntaken(t *testing.T) { + refusal := refusalFor(t, `{"adoption":{"taken":[],"untaken":{"x":["adoption.guard"]}}, + "declaration":1,"resources":[ + {"id":"adoption.guard","type":"file","path":"/etc/mesh/guard.nft","content":"x"}]}`) + if !strings.Contains(strings.Join(refusal.Problems, "\n"), "belongs to no module") { + t.Errorf("an adoption. id was accepted as untaken: %v", refusal.Problems) + } +} + +func TestAnAdoptionWithAnUnknownFieldIsRefused(t *testing.T) { + refusalFor(t, `{"adoption":{"taken":[],"held":["x"]},"declaration":1,`+adoptedResources+`}`) +} + +func TestACarriedBundleCannotSayTheNodeIsAdopted(t *testing.T) { + _, err := ParseTrusted([]byte(`{"adoption":{"taken":[]},"declaration":1,` + adoptedResources + `}`)) + if err == nil || !strings.Contains(err.Error(), "only the mesh can say") { + t.Fatalf("a bundle claiming adoption was not refused: %v", err) + } +} + +// novox/hq ADR 0105: a service may take over a found tunnel, said whole and on an adopted node. +func TestTakingOverATunnelIsSaidWholeAndForARunningService(t *testing.T) { + adoptedWith := func(service string) error { + _, err := Parse([]byte(`{"declaration":1,"adoption":{"taken":[]},"resources":[` + service + `]}`)) + return err + } + good := `{"id":"up","type":"service","unit":"wg-quick@mesh0","state":"running", + "takes-over":{"interface":"wg0","unit":"wg-quick@wg0","config":"/etc/wireguard/wg0.conf"}}` + if err := adoptedWith(good); err != nil { + t.Fatalf("a whole takeover on an adopted node was refused: %v", err) + } + for name, bad := range map[string]string{ + "its own unit": `{"id":"up","type":"service","unit":"wg-quick@wg0","state":"running", + "takes-over":{"interface":"wg0","unit":"wg-quick@wg0","config":"/etc/wireguard/wg0.conf"}}`, + "no config": `{"id":"up","type":"service","unit":"wg-quick@mesh0","state":"running", + "takes-over":{"interface":"wg0","unit":"wg-quick@wg0"}}`, + "a stopped service": `{"id":"up","type":"service","unit":"wg-quick@mesh0","state":"stopped", + "takes-over":{"interface":"wg0","unit":"wg-quick@wg0","config":"/etc/wireguard/wg0.conf"}}`, + } { + if err := adoptedWith(bad); err == nil { + t.Errorf("a takeover naming %s was accepted", name) + } + } +} diff --git a/go/declaration/block_test.go b/go/declaration/block_test.go new file mode 100644 index 0000000..1512a00 --- /dev/null +++ b/go/declaration/block_test.go @@ -0,0 +1,52 @@ +package declaration + +import ( + "strings" + "testing" +) + +// Defends novox/hq issue 128: a file written into a block carries only its lines, in content, +// never a line the host keeps as its own marker, and says where a new region goes only as a block. + +func TestAFileWrittenIntoABlockIsRefusedUnlessItIsOnlyItsLines(t *testing.T) { + for name, c := range map[string]struct{ resource, refusal string }{ + "a begin marker in content": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","content":"a\n# BEGIN mesh f\nb\n"}`, "# BEGIN mesh f"}, + "an end marker in content": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","content":"a\n# END mesh other\n"}`, "# END mesh other"}, + "a marker with a CR": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","content":"# BEGIN mesh x\r\n"}`, "marker"}, + "sealed": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","sealed":"abc"}`, "not sealed"}, + "bytes": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","bytes":"YQ=="}`, "not sealed, bytes"}, + "secrets": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","content":"${secret:s}","secrets":{"s":"abc"}}`, "secrets"}, + "create-once": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","content":"a","create-once":true}`, "create-once"}, + "at on a whole file": {`{"id":"f","type":"file","path":"/etc/hosts","content":"a","at":"start"}`, `at "start"`}, + "at on a JSON file": {`{"id":"f","type":"file","path":"/etc/x.json","into":"json","content":"{}","at":"end"}`, `at "end"`}, + "at somewhere else": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","content":"a","at":"middle"}`, `"start" or "end"`}, + "an id ending in a space": {`{"id":"f ","type":"file","path":"/etc/hosts","into":"block","content":"a"}`, "whitespace"}, + "an unknown format": {`{"id":"f","type":"file","path":"/etc/hosts","into":"lines","content":"a"}`, `"json" or "block"`}, + } { + _, err := Parse([]byte(`{"declaration":1,"resources":[` + c.resource + `]}`)) + if err == nil || !strings.Contains(err.Error(), c.refusal) { + t.Errorf("%s: want a refusal naming %q, got %v", name, c.refusal, err) + } + } +} + +func TestAFileWrittenIntoABlockIsRead(t *testing.T) { + d, err := Parse([]byte(`{"declaration":1,"resources":[ + {"id":"mesh-wireguard.fact-node-names","type":"file","path":"/etc/hosts","into":"block","content":"10.42.0.1 ace\n# BEGIN devtool x\n"}, + {"id":"dhcpcd.options","type":"file","path":"/etc/dhcpcd.conf","into":"block","content":"nohook resolv.conf\n","at":"start"}, + {"id":"empty","type":"file","path":"/etc/x","into":"block","content":"","at":"end"} + ]}`)) + if err != nil { + t.Fatal(err) + } + if f := d.Resources[0].(*File); f.Into != IntoBlock || f.At != "" { + t.Errorf("read as into %q at %q", f.Into, f.At) + } + if f := d.Resources[1].(*File); f.At != AtStart { + t.Errorf("at was read as %q", f.At) + } + if begin, end := BlockMarkers("mesh-wireguard.fact-node-names"); begin != "# BEGIN mesh mesh-wireguard.fact-node-names" || + end != "# END mesh mesh-wireguard.fact-node-names" { + t.Errorf("the markers are %q and %q", begin, end) + } +} diff --git a/go/declaration/busy.go b/go/declaration/busy.go new file mode 100644 index 0000000..3b73485 --- /dev/null +++ b/go/declaration/busy.go @@ -0,0 +1,87 @@ +package declaration + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + "time" +) + +// Busy is how a long-running container says it holds work that would be lost if it were stopped now +// (novox/hq ADR 0335 §1): a command run inside it, by its shell, as an exec health check is, that exits 0 +// while it is busy and non-zero when it is idle. Being asked is also the container's notice that it is +// about to be replaced: from the first ask it takes no new work, so its answer ends once the work in hand +// ends. +// +// **Not part of what the container is.** Busy is not in the container's spec: a changed command changes +// when the container is replaced, never whether — a container is not recreated for its busy command alone. +type Busy struct { + Command string `json:"command"` + // Timeout is how long one ask may take; empty is BusyTimeoutDefault. + Timeout string `json:"timeout,omitempty"` +} + +// The bounds of an ask (novox/hq ADR 0335). +const ( + BusyTimeoutDefault = 10 * time.Second + // BusyTimeoutCeiling is the longest one ask may take: an ask runs inside an apply, and an apply waits + // for nothing longer than a look. + BusyTimeoutCeiling = time.Minute + // BusyBound is how long a container that says busy is left running after the first apply that would + // have replaced it (ADR 0335 §3). The node-engine's, not the module's: no module holds its own + // replacement for ever. + BusyBound = 15 * time.Minute +) + +// UnmarshalJSON reads a busy strictly, as everything a declaration carries is read. +func (b *Busy) UnmarshalJSON(raw []byte) error { + type plain Busy + var p plain + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(&p); err != nil { + return fmt.Errorf("busy: %w", err) + } + *b = Busy(p) + return nil +} + +// Within is how long one ask may take. Validated on arrival, so an unreadable timeout is the default. +func (b *Busy) Within() time.Duration { + if d, err := time.ParseDuration(b.Timeout); err == nil && d > 0 { + return d + } + return BusyTimeoutDefault +} + +// problems holds a container's busy to its shape: a one-line command, a timeout within its bounds, and only +// on a container that stays up. +func (b *Busy) problems(where string, longRunning bool) []string { + if b == nil { + return nil + } + var problems []string + say := func(format string, args ...any) { + problems = append(problems, where+": "+fmt.Sprintf(format, args...)) + } + if !longRunning { + say("busy is asked of a container that stays up; a step or a scheduled run is never replaced mid-run") + } + if strings.TrimSpace(b.Command) == "" { + say("busy needs a command") + } + if strings.ContainsAny(b.Command, "\n\r") { + say("busy's command is one line") + } + if b.Timeout != "" { + d, err := time.ParseDuration(b.Timeout) + switch { + case err != nil: + say("busy's timeout is a duration, not %q", b.Timeout) + case d <= 0 || d > BusyTimeoutCeiling: + say("an ask of busy takes more than nothing and at most %s, not %s", BusyTimeoutCeiling, d) + } + } + return problems +} diff --git a/go/declaration/cron.go b/go/declaration/cron.go new file mode 100644 index 0000000..69b97df --- /dev/null +++ b/go/declaration/cron.go @@ -0,0 +1,182 @@ +package declaration + +// A minimal five-field cron, enough to say when a scheduled step is due (novox/hq ADR 0053). +// +// **Written rather than pulled in.** The host depends on nothing it does not have to +// (novox/hq ADR 0005), and a scheduled step needs exactly two questions answered — *is this minute +// a match* and *when is the next one* — over the ordinary five fields (minute, hour, day-of-month, +// month, day-of-week) with `*`, lists (`,`), ranges (`-`) and steps (`/`). That is small enough to +// keep in the vocabulary the host already owns, and a cron library would be a dependency carried +// for the parts of it nobody here uses. +// +// **The host both validates and evaluates.** The control plane refuses a malformed cron near its +// author (novox/hq ADR 0053), and the host refuses it again on arrival for the same near-versus-far +// reason every other field is checked here: a declaration the host does not fully understand is +// refused whole rather than half-applied. The evaluation half — `Next` — is what the scheduler +// fires on. + +import ( + "fmt" + "strconv" + "strings" + "time" +) + +// Cron is a parsed five-field expression, each field held as a bitset of the values it permits. +// +// The two day fields carry a flag for whether they were written as a bare `*`, because standard +// cron gives them a special rule: when day-of-month and day-of-week are *both* restricted, a day +// matches if *either* does; when one is `*`, only the other is consulted. Getting that wrong is the +// classic cron surprise, so the flag is kept rather than rediscovered. +type Cron struct { + minute uint64 + hour uint64 + dom uint64 + month uint64 + dow uint64 + domStar bool + dowStar bool +} + +// ParseCron reads a five-field cron expression, or says why it is not one. +func ParseCron(expr string) (*Cron, error) { + fields := strings.Fields(expr) + if len(fields) != 5 { + return nil, fmt.Errorf( + "a schedule is a five-field cron expression (minute hour day-of-month month "+ + "day-of-week), and %q has %d field(s)", expr, len(fields)) + } + + minute, _, err := parseCronField(fields[0], 0, 59) + if err != nil { + return nil, fmt.Errorf("schedule minute field: %w", err) + } + hour, _, err := parseCronField(fields[1], 0, 23) + if err != nil { + return nil, fmt.Errorf("schedule hour field: %w", err) + } + dom, domStar, err := parseCronField(fields[2], 1, 31) + if err != nil { + return nil, fmt.Errorf("schedule day-of-month field: %w", err) + } + month, _, err := parseCronField(fields[3], 1, 12) + if err != nil { + return nil, fmt.Errorf("schedule month field: %w", err) + } + // Day-of-week is 0-6 with Sunday at 0, and 7 is also accepted for Sunday — the convention every + // cron keeps, so a crontab copied from elsewhere is not refused for saying 7. + dow, dowStar, err := parseCronField(fields[4], 0, 7) + if err != nil { + return nil, fmt.Errorf("schedule day-of-week field: %w", err) + } + if dow&(1<<7) != 0 { + dow |= 1 << 0 + dow &^= 1 << 7 + } + + return &Cron{ + minute: minute, hour: hour, dom: dom, month: month, dow: dow, + domStar: domStar, dowStar: dowStar, + }, nil +} + +// parseCronField turns one field into the bitset of values it permits, and reports whether it was a +// bare `*` (which the day fields treat specially). +func parseCronField(spec string, min, max int) (uint64, bool, error) { + if spec == "" { + return 0, false, fmt.Errorf("is empty") + } + star := spec == "*" + + var bits uint64 + for _, part := range strings.Split(spec, ",") { + if part == "" { + return 0, false, fmt.Errorf("%q has an empty element between commas", spec) + } + + // A step may follow either `*` or a range: `*/15`, `0-30/5`. + step := 1 + rangePart := part + if slash := strings.IndexByte(part, '/'); slash >= 0 { + rangePart = part[:slash] + n, err := strconv.Atoi(part[slash+1:]) + if err != nil || n < 1 { + return 0, false, fmt.Errorf("step in %q is not a positive number", part) + } + step = n + } + + lo, hi := min, max + switch { + case rangePart == "*": + // Full range, already set. + case strings.IndexByte(rangePart, '-') >= 0: + dash := strings.IndexByte(rangePart, '-') + a, errA := strconv.Atoi(rangePart[:dash]) + b, errB := strconv.Atoi(rangePart[dash+1:]) + if errA != nil || errB != nil { + return 0, false, fmt.Errorf("range %q is not two numbers", rangePart) + } + lo, hi = a, b + default: + n, err := strconv.Atoi(rangePart) + if err != nil { + return 0, false, fmt.Errorf("%q is not a number", rangePart) + } + lo, hi = n, n + } + + if lo < min || hi > max || lo > hi { + return 0, false, fmt.Errorf( + "%q is outside the allowed range %d-%d", part, min, max) + } + for v := lo; v <= hi; v += step { + bits |= 1 << uint(v) + } + } + return bits, star, nil +} + +// Matches reports whether a scheduled step is due at this minute. +func (c *Cron) Matches(t time.Time) bool { + if c.minute&(1< 0 && unknown == "" { + problems = append(problems, where+": knows is given without unknown, and says nothing without it") + } + seen := map[string]bool{} + for _, k := range knows { + switch { + case k == "": + problems = append(problems, where+": knows holds an empty entry") + continue + case strings.HasPrefix(k, "/"): + problems = append(problems, fmt.Sprintf("%s: knows entry %q is absolute; it is relative to the path", where, k)) + continue + } + bad := false + for _, c := range strings.Split(strings.TrimSuffix(k, "/"), "/") { + if c == "" || c == "." || c == ".." { + bad = true + } + } + if bad { + problems = append(problems, fmt.Sprintf("%s: knows entry %q holds an empty, . or .. component", where, k)) + continue + } + if seen[k] { + problems = append(problems, fmt.Sprintf("%s: knows entry %q is repeated", where, k)) + } + seen[k] = true + } + return problems +} + +func (d *Directory) Identity() string { return d.ID } +func (d *Directory) Kind() Type { return TypeDirectory } +func (d *Directory) Target() string { return d.Path } + +func (d *Directory) validate(where string, _ bool) []string { + var problems []string + if d.Path == "" { + problems = append(problems, where+": a directory needs a path") + } + // A directory's mode may carry the setgid bit (2xxx), so what is made beneath it keeps its group (novox/hq ADR + // 0334 §1); never setuid or sticky, which mean nothing the mesh asks of a directory it owns. + mode := d.Mode + if len(mode) == 4 && mode[0] == '2' { + mode = "0" + mode[1:] + } + problems = append(problems, checkUnknown(where, "directory", d.Unknown, d.Knows)...) + return append(problems, checkMode(where, mode)...) +} + +// File is a file with literal content. The host renders nothing. +type File struct { + ID string `json:"id"` + Type Type `json:"type"` + // Module is the module this resource belongs to, as the controller says it (novox/hq ADR 0429 §4, issue 1132). Absent + // is an older controller, and the module is then guessed from the id (ModuleGuess). + Module string `json:"module,omitempty"` + Path string `json:"path"` + Content string `json:"content"` + Mode string `json:"mode,omitempty"` + + // CreateOnce says the content is a seed: written when the file is absent, and left alone — + // content, mode and owner — whenever it is present. + // + // **Two intentions had one vocabulary** (novox/hq issue 035, ADR 0087). "This file has this + // content, for ever" is what an ordinary file says, and the host holds the machine to it. A + // module that needs a file to exist before a program first starts — an access list the + // program then persists into, a bootstrap configuration it rewrites — needs the other thing, + // and with only the first available, every reconcile restored the seed behind the running + // program and erased what had grown in it, reporting success. What grows in a seeded file + // is somebody else's work the mesh asked for; the mesh removes nothing it did not create + // (ADR 0030), and it does not overwrite that either. + CreateOnce bool `json:"create-once,omitempty"` + + // Into says the file is shared with software the mesh did not install, and the content is + // the mesh's part of it: written into what is there, never over it (novox/hq ADR 0102). Only + // "json" is spoken — the content is a JSON object whose keys the host sets in the file's + // object, keeping every other key as it found it and recording what each of its keys held + // before, so undeclaring the file gives those back. + // + // "block" is the same idea for a file that is not structured (novox/hq issue 128): the + // content is the mesh's lines, and the host owns only the region between `# BEGIN mesh ` + // and `# END mesh `, keeping every line outside it byte for byte. The machine's hosts file + // is the case that needed it — on a workstation the distribution, a local development tool and + // the operator all write into it, and the mesh writing it whole took their lines away at the + // next change to the mesh's names, silently. Marked blocks are the shape the other tools in + // that file already use, and `#` is the comment character of every file this serves. + // + // Written into, in either format, the file's mode and owner are the machine's: a declared mode + // and owner apply only to a file the host creates, and a file that was there keeps its own. + Into string `json:"into,omitempty"` + + // At is where a file written into a block has its region added when the file does not hold + // one yet: "end", the default, or "start". A region already there stays where it is, whatever + // this says — moving it would move the lines around it, and those are the machine's. + // + // **Some files give a line its meaning by what stands above it.** dhcpcd's configuration scopes + // every line after `interface X` to that interface, and a real one ends with exactly that — an + // interface and its static address. A region added at the end would make the mesh's global + // options (`nohook resolv.conf`, `denyinterfaces mesh0`) options of one interface, and dhcpcd + // would read them without complaint. At the start, nothing stands above them. + At string `json:"at,omitempty"` + + // Sealed is content encrypted to this node's sealing key, for a file the mesh must deliver + // without being able to read. + // + // The one thing here the host cannot simply write. Everything else in a declaration is + // visible to whatever carried it — the broker relays the message, and the message is signed + // so it cannot be forged, but signing does not make it unreadable. A password travelling in + // `content` would be a password the broker sees, which is the transitive trust the design + // refuses everywhere else (novox/hq ADR 0004). + // + // Exclusive with Content: a file is one or the other, so that "was this secret" is answerable + // by looking rather than by knowing which field won. + Sealed string `json:"sealed,omitempty"` + + // VaultSealed is content the vault sealed to this node's sealing key, in the mesh's sealed box (novox/hq ADR + // 0206): a core module's own credential, which the controller carries in the declaration and cannot open + // (ADR 0349 §16, issue 755). The same promise as Sealed, from the vault rather than the controller; sent only to + // an engine that states VaultBoxContract. Exclusive with Content, Sealed and Bytes. + VaultSealed *seal.Box `json:"vault-sealed,omitempty"` + + // Trusted is the catalogue's word on whether the settings this file's content took are trusted (novox/hq + // issue 339). It is the controller's to act on, and a controller takes it out before it sends a declaration. + // Accepted here, and nothing is done with it, so that a controller that passes it on — an older one, or one + // rolled back to — never costs a node its whole declaration. Not part of the file's digest. + Trusted *bool `json:"trusted,omitempty"` + + // Secrets are sealed values put into Content where it says `${secret:name}`. + // + // **The one place a secret and a configuration meet, and it happens on the machine.** A + // program that wants its token inside a JSON document cannot be given a file that is entirely + // a token, and the mesh cannot compose the document itself — it discarded the value + // (novox/hq ADR 0024). So the module supplies the document with a hole in it, the mesh + // delivers the value sealed, and the host is the only thing that ever sees both. + // + // **Substitution is textual and the host learns no formats.** That is deliberate: a mechanism + // that understood JSON would be asked to understand YAML next, and then INI, which is how the + // arrangement this replaces became something nobody could hold in their head. The module knows + // its own format, because it wrote the rest of the file. + // + // The sharp edge, stated rather than discovered: a value containing a quote or a backslash + // will not be escaped for whatever syntax surrounds it. + Secrets map[string]string `json:"secrets,omitempty"` + + // Bytes is content that is not text, base64-encoded — a wallpaper, a font, an icon. + // + // A third way of saying what is in a file, and the three are exclusive. It would have been + // tempting to let Content carry base64 and add a flag, and then "what is in this file" would + // depend on a field somewhere else. + Bytes string `json:"bytes,omitempty"` + + // Owner is the user this belongs to, by name. Absent means root. + Owner string `json:"owner,omitempty"` + + // OwnerAtRuntime says who owns the file once it is placed (novox/hq ADR 0349 §5): "owned", the mesh's + // whole, written back whenever it differs; "seed", written once and then the program's; or "managed", + // only its named keys or its marked block the mesh's, the program keeping the rest. + // + // **Three words for what the forms above already do.** Owned is an ordinary file, seed is create-once + // and managed is a file written into, so the word changes no write: a file that says it is applied + // exactly as its old form is, and one that does not say is owned by its old form (Ownership). The word + // is what the provisioner's files say of themselves, which the old forms only implied. Seed is enough + // alone; managed still needs `into`, which says the format. A word that contradicts the form is refused. + OwnerAtRuntime string `json:"owner-at-runtime,omitempty"` +} + +// The words a file says who owns it at runtime with (novox/hq ADR 0349 §5). +const ( + RuntimeOwned = "owned" + RuntimeSeed = "seed" + RuntimeManaged = "managed" +) + +// Ownership is who owns the file at runtime: the word it says, or the one its old form implies — a file +// written into is managed, one created once is a seed, and any other is owned. +func (f *File) Ownership() string { + switch { + case f.OwnerAtRuntime != "": + return f.OwnerAtRuntime + case f.Into != "": + return RuntimeManaged + case f.CreateOnce: + return RuntimeSeed + default: + return RuntimeOwned + } +} + +// Seeded reports whether the file is written only when absent: created once, by create-once or by the +// word seed. +func (f *File) Seeded() bool { return f.CreateOnce || f.OwnerAtRuntime == RuntimeSeed } + +// Secret reports whether this file arrived sealed, which is what decides both that it must be +// opened before writing and that its contents must never appear in a report. +func (f *File) Secret() bool { return f.Sealed != "" || f.VaultSealed != nil } + +func (f *File) Identity() string { return f.ID } +func (f *File) Kind() Type { return TypeFile } +func (f *File) Target() string { return f.Path } + +// placeholder is what Content says where a sealed value belongs: ${secret:name}. +var placeholder = regexp.MustCompile(`\$\{secret:([a-z0-9][a-z0-9-]*)\}`) + +// SecretsUsed are the names Content asks for, in the order they first appear. +func (f *File) SecretsUsed() []string { + var used []string + seen := map[string]bool{} + for _, m := range placeholder.FindAllStringSubmatch(f.Content, -1) { + if !seen[m[1]] { + seen[m[1]] = true + used = append(used, m[1]) + } + } + return used +} + +func (f *File) validate(where string, _ bool) []string { + var problems []string + if f.Path == "" { + problems = append(problems, where+": a file needs a path") + } + switch f.Into { + case "": + case IntoJSON: + var object map[string]json.RawMessage + if err := json.Unmarshal([]byte(f.Content), &object); err != nil || object == nil { + problems = append(problems, where+ + ": a file written into JSON carries a JSON object of the keys it sets") + } + case IntoBlock: + // The markers are how the host finds its region again. A marker in the content would be + // a second region, or the end of this one, the next time the file is read — and the host + // would then rewrite, or on undeclare take out, lines that were never the mesh's. + for _, line := range strings.Split(f.Content, "\n") { + if strings.HasPrefix(line, BlockBegin) || strings.HasPrefix(line, BlockEnd) { + problems = append(problems, fmt.Sprintf( + "%s: a file written into a block carries the mesh's lines, and %q is a marker the "+ + "host keeps for itself", where, strings.TrimRight(line, "\r"))) + break + } + } + // The id is written into the markers, so it has to stay on one line — and whitespace at + // either end of it is whitespace the host would have to match exactly in a line some + // editor may trim. + if strings.ContainsAny(f.ID, "\r\n") { + problems = append(problems, where+ + ": a file written into a block names its region by its id, and this id spans lines") + } else if strings.TrimSpace(f.ID) != f.ID { + problems = append(problems, where+ + ": a file written into a block names its region by its id, and this id begins or ends in whitespace") + } + default: + problems = append(problems, fmt.Sprintf( + "%s: into %q; a file is written into \"json\" or \"block\", or omits it to be written whole", + where, f.Into)) + } + switch { + case f.At == "": + case f.Into != IntoBlock: + problems = append(problems, fmt.Sprintf( + "%s: at %q; only a file written into a block has a place its region is added", where, f.At)) + case f.At != AtStart && f.At != AtEnd: + problems = append(problems, fmt.Sprintf( + "%s: at %q; a block is added at \"start\" or \"end\", or omits it to be added at the end", + where, f.At)) + } + if f.Into == IntoJSON || f.Into == IntoBlock { + if f.Secret() || f.Bytes != "" || len(f.Secrets) > 0 || f.CreateOnce { + problems = append(problems, where+ + ": a file written into says only its part, in content — not sealed, bytes, "+ + "secrets or create-once") + } + } + problems = append(problems, f.checkOwnership(where)...) + var said []string + for name, value := range map[string]string{ + "content": f.Content, "sealed": f.Sealed, "bytes": f.Bytes, + } { + if value != "" { + said = append(said, name) + } + } + if f.VaultSealed != nil { + said = append(said, "vault-sealed") + if f.VaultSealed.V != seal.Version { + problems = append(problems, fmt.Sprintf("%s: vault-sealed is a sealed box of version %d, and this "+ + "node-engine opens version %d", where, f.VaultSealed.V, seal.Version)) + } + if len(f.Secrets) > 0 { + problems = append(problems, where+": a file the vault sealed whole has no holes for secrets") + } + } + if len(said) > 1 { + sort.Strings(said) + problems = append(problems, where+ + ": a file says what is in it exactly once, and this says it as "+ + strings.Join(said, " and ")+ + " — otherwise nobody can tell by looking which one landed on the machine") + } + // A file whose content names a secret must be given exactly the secrets it names. + // + // **Both directions, and both are refusals rather than warnings.** A placeholder with nothing + // to fill it would write `${secret:x}` into a configuration file, which the program reads as + // a value and fails on somewhere unrelated. A secret nobody uses means whoever wrote this + // believes a credential is in a file where it is not. + if len(f.Secrets) > 0 && f.Content == "" { + problems = append(problems, where+ + ": secrets were given and there is no content to put them in") + } + used := f.SecretsUsed() + for _, name := range used { + if f.Secrets[name] == "" { + problems = append(problems, fmt.Sprintf( + "%s: the content asks for the secret %q and none was given", where, name)) + } + } + for name := range f.Secrets { + if !slices.Contains(used, name) { + problems = append(problems, fmt.Sprintf( + "%s: the secret %q was given and the content never asks for it", where, name)) + } + } + return append(problems, checkMode(where, f.Mode)...) +} + +// checkOwnership refuses a word that contradicts the file's form (novox/hq ADR 0349 §5): owned is a whole +// file written back, so not created once nor written into; a seed is a whole file created once, so not +// written into; managed is a file written into, which says its format. Managed and created once together +// is already refused by the rule that a file written into is not created once. +func (f *File) checkOwnership(where string) []string { + switch f.OwnerAtRuntime { + case "": + case RuntimeOwned: + var also []string + if f.CreateOnce { + also = append(also, "create-once") + } + if f.Into != "" { + also = append(also, "into") + } + if len(also) > 0 { + return []string{fmt.Sprintf("%s: owned at runtime is the mesh's whole file, written back whenever it "+ + "differs, and this also says %s", where, strings.Join(also, " and "))} + } + case RuntimeSeed: + if f.Into != "" { + return []string{where + ": a seed is a whole file written once, and this says into; a file the " + + "program shares with the mesh is managed"} + } + case RuntimeManaged: + if f.Into == "" { + return []string{where + ": a managed file names the format its keys or block are written into, " + + "with into \"json\" or \"block\""} + } + default: + return []string{fmt.Sprintf("%s: owner-at-runtime %q; a placed file is %q, %q or %q, or omits it to be "+ + "owned by its form", where, f.OwnerAtRuntime, RuntimeOwned, RuntimeSeed, RuntimeManaged)} + } + return nil +} + +// User is a login on the machine. +// +// The thing that makes a shell, a chat client or a desktop expressible at all: each is a package +// plus configuration in somebody's home, and until this the mesh could only own /etc. +// +// It also makes "zsh is my login shell" **declared state** rather than an action. `chsh` is a +// command, the link may not carry one (novox/hq ADR 0005), and a shell that could only be set by +// hand would be a shell the mesh cannot manage — which is most of the reason to manage a machine +// at all. +type User struct { + ID string `json:"id"` + Type Type `json:"type"` + // Module is the module this resource belongs to, as the controller says it (novox/hq ADR 0429 §4, issue 1132). Absent + // is an older controller, and the module is then guessed from the id (ModuleGuess). + Module string `json:"module,omitempty"` + Name string `json:"name"` + + // Shell this user logs in with. Absent means the host asserts nothing and leaves whatever is + // there — the same rule Service.Boot follows, for the same reason: a field that always + // asserts cannot express "I do not care". + Shell string `json:"shell,omitempty"` + + // Groups this user must be in. Additive: the host puts the user in these and does not remove + // it from others, because a machine's own groups are not the mesh's to know about. + Groups []string `json:"groups,omitempty"` + + // Home directory. Absent means the system's default for a new user, and is not changed for + // one that exists — moving somebody's home is not something a declaration should do quietly. + Home string `json:"home,omitempty"` + + // Linger is whether the account's own service manager runs with nobody logged in (novox/hq ADR + // 0177). A user-scoped unit lives in that manager, and the manager runs only from the account's + // first login to its last logout — so a server's user unit, where nobody ever logs in, never + // runs without it, and a workstation's runs only while its person is there, which on a desktop + // is what is wanted. Absent asserts nothing, as Shell's does: true has the account linger, false + // has it not. On the account rather than on the unit, because it is the account's: two units of + // one account cannot disagree about it, and undeclaring one of them must not stop the other. + Linger *bool `json:"linger,omitempty"` + + // Root says whether this account may become root without a person (novox/hq ADR 0266). "never" + // is the agents' own account: the login an agent session runs as on a machine where it must not + // reach root by itself. Empty asserts nothing, as Shell's does. + // + // **A statement the engine judges, never one it acts on.** The apply gives an account it creates + // no password, no sudo rule and no group beyond those declared, as it always has, and takes none + // away from one it finds: a sudo rule or a group granted by hand is a person's to remove, and a + // declaration that silently stripped them would be the mesh deciding what a person's machine + // grants. What "never" adds is the look: on every look the engine reads whether the account can + // become root by itself — by its uid, a group that grants root, any sudo rule, or a secret the mesh + // placed that it can read — and says it unhealthy while it can (internal/accounts), so the + // controller can tell a machine where it holds from one where it does not. + Root string `json:"root,omitempty"` +} + +// RootNever is the one value Root takes besides empty: the account never becomes root without a person. +const RootNever = "never" + +// Network is a named network on this machine. +// +// **A name and nothing else.** Not a driver, a subnet or a gateway: each of those is something a +// module would have to know about the machine it lands on, and a module naming a subnet is a +// module that collides with whatever else chose the same one. The runtime picks; the mesh names +// (novox/hq ADR 0029). +type Network struct { + ID string `json:"id"` + Type Type `json:"type"` + // Module is the module this resource belongs to, as the controller says it (novox/hq ADR 0429 §4, issue 1132). Absent + // is an older controller, and the module is then guessed from the id (ModuleGuess). + Module string `json:"module,omitempty"` + Name string `json:"name"` +} + +func (n *Network) Identity() string { return n.ID } +func (n *Network) Kind() Type { return TypeNetwork } +func (n *Network) Target() string { return n.Name } + +func (n *Network) validate(where string, _ bool) []string { + var problems []string + if n.Name == "" { + problems = append(problems, where+": a network needs a name") + } + // The runtimes accept more than this, and the mesh does not: a name with a slash or a colon + // in it reads as a reference to something else entirely wherever it is later printed. + for _, r := range n.Name { + if (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') && (r < '0' || r > '9') && + r != '-' && r != '_' && r != '.' { + problems = append(problems, where+ + ": a network name is letters, digits, dashes, underscores and dots, and "+ + n.Name+" is not") + break + } + } + return problems +} + +// Modes an access may be granted at. Plain words, not the octal a directory's mode is: an access +// is not a thing the host chmods, it is a statement of how this module reaches what the operator +// owns. +const ( + AccessRead = "read" + AccessReadWrite = "read-write" +) + +// Access is a pre-existing, operator-owned path this module is granted use of but does not own. +// +// **The distinction 04-ISSUES/036 and 026 turn on.** A `directory` resource is the mesh's own — +// it creates it, sets its owner and mode, and removes it when empty ([ADR 0030](novox/hq)). A +// media library, a download spool is the operator's: it existed before the mesh, several modules +// read and write it at once, and the mesh must not create, chown, chmod or remove it. The host +// confirms it is there and mounts it; nothing else. Library data is a library's (novox/hq ADR 0328), +// not an access. +type Access struct { + ID string `json:"id"` + Type Type `json:"type"` + // Module is the module this resource belongs to, as the controller says it (novox/hq ADR 0429 §4, issue 1132). Absent + // is an older controller, and the module is then guessed from the id (ModuleGuess). + Module string `json:"module,omitempty"` + Path string `json:"path"` + // Mode is how this module reaches the path: read or read-write. Absent narrows to read. + Mode string `json:"mode,omitempty"` + + // Users and Taken are what a controller speaking ADR 0325 sends (the identities that use the folder, and + // that the operator's warrant was carried out): accepted, so its declaration is not refused whole, and not + // acted on since ADR 0328 superseded ADR 0325 — no ACL entry is kept. + Users []AccessUser `json:"users,omitempty"` + Taken bool `json:"taken,omitempty"` + + // Grant is what a controller older than ADR 0325 still sends (ADR 0312's group grant, superseded): accepted + // so its declaration is not refused whole, and never acted on — no account is put in a group for data (§5). + Grant json.RawMessage `json:"grant,omitempty"` +} + +// The uses an access's user has (ADR 0051 §3), and where it runs (ADR 0325 §4). +const ( + FromContainer = "container" + FromToolRunner = "tool-runner" +) + +// AccessUser is one identity that uses an accessed folder: its name and uid as the controller allocated them, its +// use, and where it runs. A tool runner's account may come with no uid (0 or absent): the engine reads it by name. +type AccessUser struct { + Identity string `json:"identity"` + UID int `json:"uid,omitempty"` + Use string `json:"use"` + From string `json:"from"` +} + +// UnmarshalJSON reads a user as strictly as a resource is read. +func (u *AccessUser) UnmarshalJSON(raw []byte) error { + type plain AccessUser + var p plain + if err := strictDecode(raw, &p); err != nil { + return fmt.Errorf("an access's user: %w", err) + } + *u = AccessUser(p) + return nil +} + +func (a *Access) Identity() string { return a.ID } +func (a *Access) Kind() Type { return TypeAccess } +func (a *Access) Target() string { return a.Path } + +func (a *Access) validate(where string, _ bool) []string { + var problems []string + if !strings.HasPrefix(a.Path, "/") { + problems = append(problems, where+": an access needs an absolute path, and "+ + a.Path+" is not one") + } + switch a.Mode { + case "", AccessRead, AccessReadWrite: + default: + problems = append(problems, fmt.Sprintf( + "%s: an access is %q or %q, not %q", where, AccessRead, AccessReadWrite, a.Mode)) + } + for _, u := range a.Users { + switch { + case u.Identity == "" || !accountName.MatchString(u.Identity): + problems = append(problems, fmt.Sprintf("%s: a user of an access is an account's name, and %q is not one", + where, u.Identity)) + case u.UID < 0 || (u.UID == 0 && u.From != FromToolRunner): + // Never root (novox/hq ADR 0325 §11): an entry for uid 0 would be a right root has anyway, and a + // number below it is no account. A tool runner's account alone may come without its number: the + // controller cannot know the operator's account's, and the engine reads it by name + // (apply.ResolveRunners), refusing one that is root's. + problems = append(problems, fmt.Sprintf("%s: the user %s has uid %d, and a module's identity is never "+ + "root or below it (novox/hq ADR 0325 §11)", where, u.Identity, u.UID)) + } + if u.Use != AccessRead && u.Use != AccessReadWrite { + problems = append(problems, fmt.Sprintf("%s: the user %s uses it %q, and a use is %q or %q", where, + u.Identity, u.Use, AccessRead, AccessReadWrite)) + } + if u.Use == AccessReadWrite && a.Mode != AccessReadWrite { + problems = append(problems, fmt.Sprintf("%s: the user %s writes an access declared %s", where, + u.Identity, orRead(a.Mode))) + } + if u.From != FromContainer && u.From != FromToolRunner { + problems = append(problems, fmt.Sprintf("%s: the user %s runs from %q, and that is %q or %q", where, + u.Identity, u.From, FromContainer, FromToolRunner)) + } + } + return problems +} + +// orRead is an access's mode as said: absent narrows to read. +func orRead(mode string) string { + if mode == "" { + return AccessRead + } + return mode +} + +func (u *User) Identity() string { return u.ID } +func (u *User) Kind() Type { return TypeUser } +func (u *User) Target() string { return u.Name } + +func (u *User) validate(where string, _ bool) []string { + var problems []string + if u.Name == "" { + problems = append(problems, where+": a user needs a name") + } + if u.Shell != "" && !strings.HasPrefix(u.Shell, "/") { + problems = append(problems, where+ + ": a login shell is an absolute path, and "+u.Shell+" is not one") + } + if u.Home != "" && !strings.HasPrefix(u.Home, "/") { + problems = append(problems, where+": a home directory is an absolute path") + } + if u.Root != "" && u.Root != RootNever { + problems = append(problems, fmt.Sprintf("%s: root is %q or absent, and %q is neither", where, RootNever, u.Root)) + } + return problems +} + +// Archive is a set of files, fetched by digest and unpacked. +// +// For the case inlining cannot serve: a theme, an icon set, a tree of configuration. Hundreds of +// files inlined would make every declaration enormous and rewrite all of it when one changed. +// +// **Pinned by digest, and the digest is checked before anything is unpacked.** The same discipline +// the bootstrap uses for images, and for the same reason — this is fetched over a network the +// mesh does not control, and a reference that can be made to point elsewhere is not a reference. +type Archive struct { + ID string `json:"id"` + Type Type `json:"type"` + // Module is the module this resource belongs to, as the controller says it (novox/hq ADR 0429 §4, issue 1132). Absent + // is an older controller, and the module is then guessed from the id (ModuleGuess). + Module string `json:"module,omitempty"` + // Source is where to fetch it from. + Source string `json:"source"` + // Digest is sha256 of the archive, as "sha256:". + Digest string `json:"digest"` + // Path is the directory it is unpacked into. + Path string `json:"path"` + // Owner is the user the unpacked files belong to. Absent means root. + Owner string `json:"owner,omitempty"` + // Unknown and Knows are as a directory's (issue 1024, ADR 0422). + Unknown string `json:"unknown,omitempty"` + Knows []string `json:"knows,omitempty"` +} + +func (a *Archive) Identity() string { return a.ID } +func (a *Archive) Kind() Type { return TypeArchive } +func (a *Archive) Target() string { return a.Path } + +func (a *Archive) validate(where string, _ bool) []string { + var problems []string + if a.Source == "" { + problems = append(problems, where+": an archive needs somewhere to fetch it from") + } + if a.Path == "" { + problems = append(problems, where+": an archive needs somewhere to unpack into") + } + if !strings.HasPrefix(a.Digest, "sha256:") || len(a.Digest) != len("sha256:")+64 { + // Refused rather than fetched and trusted. Everything else pinned in this vocabulary is + // pinned by digest, and an archive that was not would be the one way in. + problems = append(problems, where+ + ": an archive is pinned by digest, as sha256:<64 hex characters>") + } + problems = append(problems, checkUnknown(where, "archive", a.Unknown, a.Knows)...) + return problems +} + +// Process is the module's own code, run on the machine, in one of three modes. +// +// The difference from Service is who owns the unit: a Service puts an EXISTING unit into a state +// and deliberately does not install one, which is right for software that ships its own. This is +// the mesh's own code — a bundle it built — so there is no unit until the mesh writes it, and +// nothing else will. +// +// The difference from Container is the hosting, and a module should not have to choose: what this +// says is what to run, and the machine's own supervisor is how. Code that genuinely needs a +// container's isolation declares a container and says so. +// +// **Three modes, matching a container's**, because they are the same thing at different cadences: +// stays up, runs once, runs on a schedule. A module's scheduled task, its run-once migration, its +// health check and its tool host are all this — and each being its own resource kind would be four +// entries in a vocabulary where every entry widens what a compromised control plane can express. +type Process struct { + ID string `json:"id"` + Type Type `json:"type"` + // Module is the module this resource belongs to, as the controller says it (novox/hq ADR 0429 §4, issue 1132). Absent + // is an older controller, and the module is then guessed from the id (ModuleGuess). + Module string `json:"module,omitempty"` + // Name is what the unit is called, and what an operator will see in the process table. + Name string `json:"name"` + // Source is where to fetch the bundle from, and Digest is what it must hash to. The same + // discipline as an archive, for the same reason: this crosses a network the mesh does not + // control. + Source string `json:"source"` + Digest string `json:"digest"` + // Run is the command, relative to the unpacked bundle. The first element is the program. + // + // **Named by the module, never inferred.** Guessing an entrypoint from which files exist makes + // a daemon change what it runs when somebody adds a file. + Run []string `json:"run"` + // Env and EnvFile are what it runs with. A file rather than inline values is how a credential + // reaches a daemon without passing through the declaration. + Env map[string]string `json:"env,omitempty"` + EnvFile []string `json:"env-file,omitempty"` + // User is who it runs as. Absent means root, which is what the mesh's own modules need for + // the things they do to a machine. + User string `json:"user,omitempty"` + // RestartOn names resources whose change means this must be restarted — the same rule a + // service follows, and for the same reason: a running process does not re-read its + // configuration, so replacing a file and finding the process already up leaves the machine + // behaving the way it did before while every check passes. + RestartOn []string `json:"restart-on,omitempty"` + + // RunOnce marks code the host runs to completion rather than leaves running: a migration, a + // seed, a first-boot step. What follows it is gated on it finishing, because a step that did + // not make the machine ready must not be followed by the thing that needed it. + RunOnce bool `json:"run-once,omitempty"` + + // Schedule runs it on a cadence — a five-field cron expression (novox/hq ADR 0053). The + // recurring twin of RunOnce: the same code, run again rather than left running. + // + // Exclusive with RunOnce and with RestartOn, for the same reason a container's is: something + // that runs once does not run on a schedule, and something that is not running cannot be + // restarted when a file changes. + Schedule string `json:"schedule,omitempty"` + + // Replaces names resources this declaration no longer declares that this process takes the + // place of (novox/hq issue 213). Such a resource is not removed with the other orphans, before + // anything is applied: it is removed only once this process is applied and still running a + // moment later, and kept when it is not. So the thing being replaced answers until the thing + // replacing it does — the controller moving from its container to a process is the case: the + // container removed first left nothing answering the mesh's verbs for as long as fetching, + // unpacking and starting the process took, and for ever if the process did not start. + // + // For a process that stays up; a step or a scheduled run is not running a moment later by + // design, so there is nothing to hand over to. + Replaces []string `json:"replaces,omitempty"` + + // Witness is how the node-engine judges a new build of this process, and restores the build + // before it when the new one is not healthy in bound (novox/hq to-be 45 §8): "lease" — the + // controller this machine started holds the controller's lease; "ping" — this machine's runtime + // answers the services protocol's PING; "none". Absent is the default for the process's name: + // the mesh's two core processes are judged, nothing else is. For a process that stays up. + Witness string `json:"witness,omitempty"` + + // NotReversible says why this build may not be rolled back, when it may not: the build before it + // would run against what this one changes — a migration it runs that the older build cannot read + // (to-be 45 §8, rule 8). A build so declared that is not healthy in bound is left running and said + // as urgent; the build before it is never started against the newer data. + NotReversible string `json:"not-reversible,omitempty"` + + // Health is how this resource is ready (novox/hq ADR 0240 rule 2, Phase B): one kind and its + // timing, judged by the node-engine beside liveness. Absent: judged alive or not, and nothing more. + Health *Health `json:"health,omitempty"` +} + +func (d *Process) Identity() string { return d.ID } +func (d *Process) Kind() Type { return TypeProcess } +func (d *Process) Target() string { return d.Name } + +// ProcessNameProblem says what is wrong with a process name, or nothing. +// +// The name becomes a unit name, a file under the unit directory and a directory under the mesh's +// own — the one removing the process deletes, whole (novox/hq ADR 0118). So a name that is not one +// plain path element is refused: with a separator it writes somewhere nobody meant, and "." or +// ".." IS the mesh's directory or its parent — removing a process named ".." would delete every +// bundle the mesh has, and more. One with a leading dash is read by the service manager as an +// option, not a unit. Exported because the removal checks the recorded name again: a record is +// what the host wrote, and a host of an older version wrote it under looser rules. +func ProcessNameProblem(name string) string { + switch { + case name == "": + return "a process needs a name, which is what its unit is called" + case strings.ContainsAny(name, "/ \t"): + return "a process name becomes a unit name, so it cannot contain a path separator or a space" + case name == "." || name == "..": + return fmt.Sprintf("a process name becomes a directory under the mesh's own, and %q would be "+ + "that directory or its parent", name) + case strings.HasPrefix(name, "-"): + return "a process name cannot begin with a dash: the service manager would read it as an option" + } + return "" +} + +func (d *Process) validate(where string, _ bool) []string { + problems := d.Health.problems(where, false, !d.RunOnce && d.Schedule == "") + if problem := ProcessNameProblem(d.Name); problem != "" { + problems = append(problems, where+": "+problem) + } + if d.Source == "" { + problems = append(problems, where+": a process needs somewhere to fetch its bundle from") + } + if !strings.HasPrefix(d.Digest, "sha256:") || len(d.Digest) != len("sha256:")+64 { + // The same rule an archive follows, and for the same reason: this crosses a network the + // mesh does not control, and a reference that can be made to point elsewhere is not one. + problems = append(problems, where+ + ": a process bundle is pinned by digest, as sha256:<64 hex characters>") + } + if len(d.Run) == 0 { + problems = append(problems, where+": a process needs to say what to run") + } + switch d.Witness { + case "", "lease", "ping", "none": + default: + problems = append(problems, fmt.Sprintf("%s: witness %q is not one this host keeps: lease, ping or none", + where, d.Witness)) + } + if d.Witness != "" && d.Witness != "none" && (d.RunOnce || d.Schedule != "") { + problems = append(problems, where+": a witness judges a process that stays up; a step or a "+ + "scheduled run is not running between its runs") + } + if strings.ContainsAny(d.NotReversible, "\n\r") { + problems = append(problems, where+": not-reversible is one line") + } + for _, part := range d.Run { + if part == "" { + problems = append(problems, where+": a process command has an empty element") + break + } + } + // **A newline cannot be represented in a unit's environment, so it is refused rather than + // mangled.** Everything else a unit file reinterprets — a percent specifier, whitespace + // splitting assignments, a quote ending one early — can be escaped. A newline cannot: it ends + // the line, and what follows is read as a unit DIRECTIVE. A value carrying one could write + // ExecStart= and have the machine run something nobody declared. + // + // Refused here, near whoever wrote it, rather than at the far end of a declaration. + for key, value := range d.Env { + if strings.ContainsAny(value, "\n\r") { + problems = append(problems, fmt.Sprintf( + "%s: the value of %s contains a line break, which cannot be written into a unit's "+ + "environment — what followed it would be read as a unit directive", where, key)) + } + if key == "" { + problems = append(problems, where+": an environment value with no name") + } + } + + if d.Schedule != "" { + if d.RunOnce { + problems = append(problems, where+ + ": a process runs once or on a schedule, not both") + } + if len(d.RestartOn) > 0 { + problems = append(problems, where+ + ": a scheduled process is not running between its fires, so there is nothing to "+ + "restart when something it reads changes") + } + if _, err := ParseCron(d.Schedule); err != nil { + problems = append(problems, where+": "+err.Error()) + } + } + if len(d.Replaces) > 0 && (d.RunOnce || d.Schedule != "") { + problems = append(problems, where+ + ": only a process that stays up replaces something — a step or a scheduled run is not "+ + "running a moment later, so what it replaced would be removed with nothing in its place, "+ + "or never") + } + for _, id := range d.Replaces { + switch { + case strings.TrimSpace(id) == "": + problems = append(problems, where+": replaces names an empty id") + case id == d.ID: + problems = append(problems, where+": a process cannot replace itself") + } + } + return problems +} + +// Service is a unit the host puts into a state. It does not install the unit. +// +// Two states, and they are orthogonal rather than one scale. A unit can be enabled and stopped +// (it will come back at boot), or disabled and running (started by hand, gone after a reboot). +// Folding them into one field would make the second expressible only by accident. +type Service struct { + ID string `json:"id"` + Type Type `json:"type"` + // Module is the module this resource belongs to, as the controller says it (novox/hq ADR 0429 §4, issue 1132). Absent + // is an older controller, and the module is then guessed from the id (ModuleGuess). + Module string `json:"module,omitempty"` + Unit string `json:"unit"` + // State is "running" or "stopped" — or absent, and then **the unit's lifecycle is the + // machine's; the mesh only reflects its triggers** (novox/hq ADR 0117). The uplink modules + // declare the machine's own network manager this way: the mesh writes into its configuration + // and needs it to read that again, and nothing more. Stated, the host would start the manager + // on a machine that uses another one — two managers fighting over the same links — and, when + // the module was unassigned, stop it: the machine's network, the channel the mesh itself + // arrives on, gone at the moment of a routine change. So a service without a state is never + // started, stopped, enabled or disabled, is reloaded or restarted only when a trigger changed + // and it is already running, and undeclared is simply forgotten. It says nothing unless it + // names a trigger, and it may not say boot or takes-over, which are both lifecycle. + State string `json:"state,omitempty"` + // Boot is "enabled" or "disabled" — whether the unit starts at boot. Optional: absent means + // the host asserts nothing about it and leaves whatever is there. + // + // Without this the host could start a unit and not make it survive a reboot, which is a + // declaration that reports success and stops being true at the next power cut. + Boot string `json:"boot,omitempty"` + + // Scope is whose service manager the unit belongs to: "system" (absent means system), or + // "user" — the operator account's own manager (novox/hq ADR 0177). A workstation's per-user + // daemons — a window manager's reload watcher, an audio mask, a memory guard — are units in + // that manager, and until this they had no form the mesh could send. A user-scoped unit names + // its User; the host talks to that account's manager and never starts a system unit by the + // same name. + Scope string `json:"scope,omitempty"` + // User is the account whose manager a user-scoped unit lives in. Required with scope "user", + // refused otherwise; a module names it ${machine:account} and never the person. + User string `json:"user,omitempty"` + // RestartOn names resources whose change means this service must be restarted. + // + // Because a running service does not re-read its configuration. Replace the file, find the + // service already running, do nothing, and the machine keeps behaving the way it did before — + // while every check passes, because the file is right and the service is up. That is not + // hypothetical: it is how a third node joining a mesh left the first two carrying a network + // that no longer existed, and every part of it reported success. + // + // This is declared state rather than a command. The declaration says the running service must + // reflect these files; the host works out that it does not and acts. A *command* to restart + // would be an action, and the link may not carry one (novox/hq ADR 0005) — so this is not a + // way around that rule, it is the shape the rule leaves. + RestartOn []string `json:"restart-on,omitempty"` + + // ReloadOn names resources whose change means this service must be reloaded — for a service + // that re-reads its configuration when told to, where a restart would stop what it runs: the + // container runtime, whose restart stops every container on the machine (novox/hq ADR 0102). + // A change that is also in RestartOn restarts it, which covers a reload. + ReloadOn []string `json:"reload-on,omitempty"` + + // TakesOver names the found tunnel this service replaces (novox/hq ADR 0105): before this unit + // is started, the named unit is stopped and disabled — never flushed — and its configuration + // file is kept like any held file — until the take is proven by a peer's handshake, and then + // retired, its original staying kept (novox/hq ADR 0119). Only on an adopted node, and only + // said by the controller, which knows the found tunnel's key is this node's own: without that, + // starting this unit on the found one's port would drop every peer's packets. + TakesOver *TakeOver `json:"takes-over,omitempty"` + + // Health is how this resource is ready (novox/hq ADR 0240 rule 2, Phase B): one kind and its + // timing, judged by the node-engine beside liveness. Absent: judged alive or not, and nothing more. + Health *Health `json:"health,omitempty"` +} + +// TakeOver is a found tunnel a service replaces: its interface, the unit that raised it, and its +// configuration file. +type TakeOver struct { + Interface string `json:"interface"` + Unit string `json:"unit"` + Config string `json:"config"` +} + +// Stateless reports whether the unit's lifecycle is the machine's, and the mesh only reflects the +// service's triggers (novox/hq ADR 0117). +func (s *Service) Stateless() bool { return s.State == "" } + +func (s *Service) Identity() string { return s.ID } +func (s *Service) Kind() Type { return TypeService } +func (s *Service) Target() string { return s.Unit } + +// ScopeSystem and ScopeUser are the two managers a unit may belong to (novox/hq ADR 0177). +const ( + ScopeSystem = "system" + ScopeUser = "user" +) + +// UserScoped is whether this unit lives in an account's own service manager. +func (s *Service) UserScoped() bool { return s.Scope == ScopeUser } + +func (s *Service) validate(where string, _ bool) []string { + problems := s.Health.problems(where, false, s.State == "running") + if s.Unit == "" { + problems = append(problems, where+": a service needs a unit") + } + switch s.Scope { + case "", ScopeSystem: + if s.User != "" { + problems = append(problems, where+": a system unit names no user; only a user-scoped unit does") + } + case ScopeUser: + if s.User == "" { + problems = append(problems, where+": a user-scoped unit names the account whose manager it lives in") + } + default: + problems = append(problems, fmt.Sprintf( + "%s: scope %q; a unit is in the \"system\" manager or the operator account's \"user\" one", where, s.Scope)) + } + switch { + case s.State == "running" || s.State == "stopped": + case s.State != "": + problems = append(problems, fmt.Sprintf( + "%s: state %q; a service is \"running\" or \"stopped\", or omits state to leave the "+ + "unit's lifecycle to the machine", where, s.State)) + case s.Boot != "" || s.TakesOver != nil: + problems = append(problems, where+": a service that omits state leaves the unit's lifecycle "+ + "to the machine, and boot and takes-over are both its lifecycle") + case len(s.RestartOn) == 0 && len(s.ReloadOn) == 0: + problems = append(problems, where+": a service that omits state leaves the unit's lifecycle "+ + "to the machine, and names no restart-on or reload-on — it declares nothing") + } + if s.Boot != "" && s.Boot != "enabled" && s.Boot != "disabled" { + problems = append(problems, fmt.Sprintf( + "%s: boot %q; a service is \"enabled\" or \"disabled\" at boot, or omits it to "+ + "leave the machine's own setting alone", where, s.Boot)) + } + if t := s.TakesOver; t != nil { + switch { + case t.Unit == "" || t.Config == "" || t.Interface == "": + problems = append(problems, where+": takes-over names the found tunnel's interface, unit "+ + "and config, and this leaves one out") + case t.Unit == s.Unit: + problems = append(problems, fmt.Sprintf("%s: takes-over names %s, which is this service's own unit", + where, t.Unit)) + case s.State != "running" && s.State != "": + problems = append(problems, where+": a service that takes over a tunnel is running — stopping "+ + "the found one for a service that will not run would leave the peers with nothing") + } + } + return problems +} + +// What a file is written into (novox/hq ADR 0102): a JSON object whose keys the mesh sets, or a +// text file in which the mesh owns one marked block of lines (novox/hq issue 128). +const ( + IntoJSON = "json" + IntoBlock = "block" +) + +// The lines that delimit the mesh's region in a file written into a block, each followed by the +// resource's id. Exact lines, never patterns: another tool's `# BEGIN …` block in the same file is +// that tool's, and a marker that merely resembled the mesh's must not be taken for it. +const ( + BlockBegin = "# BEGIN mesh " + BlockEnd = "# END mesh " +) + +// Where a file written into a block has its region added, when it has none yet. +const ( + AtStart = "start" + AtEnd = "end" +) + +// BlockMarkers are the two lines, without their line ends, that delimit a resource's region. +func BlockMarkers(id string) (begin, end string) { + return BlockBegin + id, BlockEnd + id +} + +// Opening is a port reachable on an adopted node, from where, and on which path. +// +// **From** is everywhere or mesh — the private network, by its interface. **Path** is incoming, +// for something listening on the machine, or forwarded, for a published container port: the found +// firewall sees a published port after the runtime has translated it, so a forwarded opening names +// the container's own port in To as well as the machine's in Port. +type Opening struct { + ID string `json:"id"` + Type Type `json:"type"` + // Module is the module this resource belongs to, as the controller says it (novox/hq ADR 0429 §4, issue 1132). Absent + // is an older controller, and the module is then guessed from the id (ModuleGuess). + Module string `json:"module,omitempty"` + Port int `json:"port"` + Protocol string `json:"protocol"` + From string `json:"from"` + Path string `json:"path"` + To int `json:"to,omitempty"` +} + +// Where an opening admits from, and the path it is on. +const ( + FromEverywhere = "everywhere" + FromMesh = "mesh" + PathIncoming = "incoming" + PathForwarded = "forwarded" +) + +func (o *Opening) Identity() string { return o.ID } +func (o *Opening) Kind() Type { return TypeOpening } + +func (o *Opening) Target() string { + if o.Path == PathForwarded { + return fmt.Sprintf("%s/%d forwarded to %d from %s", o.Protocol, o.Port, o.To, o.From) + } + return fmt.Sprintf("%s/%d %s from %s", o.Protocol, o.Port, o.Path, o.From) +} + +func (o *Opening) validate(where string, _ bool) []string { + var problems []string + if o.Port < 1 || o.Port > 65535 { + problems = append(problems, fmt.Sprintf("%s: an opening's port is 1-65535, not %d", where, o.Port)) + } + if o.Protocol != "tcp" && o.Protocol != "udp" { + problems = append(problems, fmt.Sprintf("%s: an opening is tcp or udp, not %q", where, o.Protocol)) + } + if o.From != FromEverywhere && o.From != FromMesh { + problems = append(problems, fmt.Sprintf( + "%s: an opening is from %q or %q, not %q", where, FromEverywhere, FromMesh, o.From)) + } + switch o.Path { + case PathIncoming: + if o.To != 0 { + problems = append(problems, where+ + ": an incoming opening names no container port; only a forwarded one does") + } + case PathForwarded: + if o.To < 1 || o.To > 65535 { + problems = append(problems, where+ + ": a forwarded opening names the container's port it reaches, as to, 1-65535") + } + default: + problems = append(problems, fmt.Sprintf( + "%s: an opening's path is %q or %q, not %q", where, PathIncoming, PathForwarded, o.Path)) + } + if !strings.HasPrefix(o.ID, AdoptionPrefix) { + problems = append(problems, fmt.Sprintf( + "%s: an opening is the mesh's own, so its id starts %q", where, AdoptionPrefix)) + } + return problems +} + +// Package is a package that should be present. +// +// Present is the whole of what it asserts, never a version: version is the package manager's +// business and the mesh does not hold a second opinion about it. +type Package struct { + ID string `json:"id"` + Type Type `json:"type"` + // Module is the module this resource belongs to, as the controller says it (novox/hq ADR 0429 §4, issue 1132). Absent + // is an older controller, and the module is then guessed from the id (ModuleGuess). + Module string `json:"module,omitempty"` + Package string `json:"package"` + // Absent declares that the package is NOT installed (novox/hq ADR 0180): the host removes it + // when it is, and leaves a machine that never had it alone. For the one case a module replaces + // software the machine was found with and the operator has decided it does not come back — the + // firewall front end a converged machine's filter module retired. Nothing to undo when the + // declaration drops it: the host does not install what a declaration stopped saying is absent. + Absent bool `json:"absent,omitempty"` +} + +func (p *Package) Identity() string { return p.ID } +func (p *Package) Kind() Type { return TypePackage } +func (p *Package) Target() string { return p.Package } + +func (p *Package) validate(where string, _ bool) []string { + if p.Package == "" { + return []string{where + ": a package needs a package name"} + } + return nil +} + +// Container is a container that should be running, from an image pinned by digest. +type Container struct { + ID string `json:"id"` + Type Type `json:"type"` + // Module is the module this resource belongs to, as the controller says it (novox/hq ADR 0429 §4, issue 1132). Absent + // is an older controller, and the module is then guessed from the id (ModuleGuess). + Module string `json:"module,omitempty"` + Name string `json:"name"` + // Image is pinned by digest (novox/hq ADR 0006) — a tag moves and a digest does not. + Image string `json:"image"` + Env map[string]string `json:"env,omitempty"` + // EnvFile names files the runtime reads environment from, in order. + // + // **Because a secret may not travel in Env.** A declaration reaches a node over the broker, + // and `env` is plain text in it — so a password there is a password the broker sees, which is + // the transitive trust refused everywhere else (novox/hq ADR 0004). A sealed file reaches the + // machine unreadable, the host writes it, and the runtime reads it: the mesh never holds it + // and neither does anything between them. + // + // It is also simply how third-party software takes credentials. Nothing that ships in a + // container will read a path the mesh invented; every one of them reads its environment. + EnvFile []string `json:"env-file,omitempty"` + Ports []string `json:"ports,omitempty"` + Volumes []string `json:"volumes,omitempty"` + Args []string `json:"args,omitempty"` + // Names this container can reach, as `name:address`. + // + // **Because a container does not inherit the machine's names.** It gets its own `/etc/hosts` + // holding only its own hostname, so every internal name the mesh wrote for this machine is + // invisible to the thing the machine is running. That was hit for real: a database client on + // one node could not resolve another node, on a mesh where both names were correct and + // present on both machines. + // + // **A file rather than a resolver, which is the decision the mesh already made about names** + // and this extends rather than overturns: it works on every runtime, needs no package, and + // has no failure mode of its own. A resolver becomes necessary when names are wanted that are + // not one-per-node — service names, wildcards — and that is still not true. + // + // Set by the mesh, not by a module: which machines exist is a fact about the mesh, and a + // module that listed them would be a module that goes stale when one joins. + Hosts []string `json:"hosts,omitempty"` + + // Network is the container's network, passed to the runtime unchanged. + // + // Needed because the control plane must reach the store and the broker on the machine it was + // raised on, before there is any mesh to arrange that. The alternative was publishing ports + // and guessing an address that works from inside a container, which is the same thing with a + // worse failure mode. + Network string `json:"network,omitempty"` + + // Dns is the resolvers this container asks, passed to the runtime unchanged. + // + // **Because some software refuses to run behind the runtime's forwarding resolver.** A mail + // server's admin demands a DNSSEC-validating resolver, and the runtime's own (127.0.0.11) + // forwards to whatever the machine has — so a module that ships its own validating resolver + // must be able to point its other containers at it. Addresses, not names: the runtime's flag + // takes only addresses, which is also why IP below exists — the resolver has to be somewhere + // its siblings can name before any of them can resolve anything. + Dns []string `json:"dns,omitempty"` + + // Capabilities are the Linux capabilities this container is granted beyond the runtime's + // default set, by name (novox/hq ADR 0170): a holder's runtime that changes the machine's packet + // filter asks for NET_ADMIN. Exactly these, named in the spec so a change recreates the + // container; a privileged container stays undeclarable. + Capabilities []string `json:"capabilities,omitempty"` + + // Logging names where the runtime sends this container's output: "journald" sends it to the + // machine's journal, under the container's name, where what reads the machine's logs — its + // intrusion prevention first of all (novox/hq ADR 0179) — can read it the way it reads the + // machine's own services. Empty keeps the runtime's default, which is a file of the runtime's + // own that nothing but the runtime reads. Part of the spec: a container that logs elsewhere + // is a different container, and the runtime cannot change a running one's driver. + Logging string `json:"logging,omitempty"` + + // Networks are networks this container also joins once created, by name — a found network a + // per-machine setting keeps for a taken container (novox/hq ADR 0163, rule 4), so a + // neighbour that resolves it there keeps resolving it until the neighbour is taken too. + // Joined after creation, because a runtime starts a container on one network; part of the + // container's spec, so a network kept or let go recreates it. + Networks []string `json:"networks,omitempty"` + + // IP is this container's address on its network, passed to the runtime unchanged. + // + // Only meaningful on a user-defined network, and refused by the runtime elsewhere. Exists for + // exactly one shape: a container others must reach *before* name resolution works — a + // module's own DNS resolver being the case that forced it (see Dns). + IP string `json:"ip,omitempty"` + + // RestartOn names resources whose change means this container must be recreated — the same + // field a service has, for the same reason (novox/hq 04-ISSUES/009). A container reads a + // mounted file once at start; a changed file leaves the running process holding the old value, + // while every check passes because the file on disk is right. The host recreates the container + // when one of these resources changed this pass, even if the spec matches. + // + // What a running container reads at creation — its env-files, and a file mounted into it + // directly — is part of its spec by content since novox/hq 04-ISSUES/103, and needs no naming + // here. A directory mounted into it is NOT looked inside, not even for files the host wrote + // there: whether a service reads such a file once or watches it live is the service's, and + // RestartOn is how a module says "once, at start" — a config the host renders under the + // module's state directory, a step whose result it consumes. On a run-once step it means *run + // again*: a step that fetches a fact from a provider names the binding it reads, and is run + // again when the provider moved (novox/hq ADR 0099). + RestartOn []string `json:"restart-on,omitempty"` + + // RunOnce marks a container the host runs to completion rather than leaves running: a step, + // not a service (novox/hq ADR 0052). The host runs it, requires it to exit 0, and records that + // it did — and because the declaration is applied in order and a failed step halts the apply, + // whatever is declared after a run-once container starts only once the step has finished. It is + // how a module runs its own code at first boot — seed a store, migrate, health-gate — under its + // own account (ADR 0047), before the container that depends on it. The record that it ran is + // the digest of this declaration, so a re-apply does not re-run it unless the declaration + // changed. + RunOnce bool `json:"run-once,omitempty"` + + // Schedule marks a container the host runs on a recurring cadence — a five-field cron + // expression (novox/hq ADR 0053). It is the recurring twin of RunOnce: the same container, run + // to completion, but again and again on the clock rather than once. Installing it does not run + // it — the schedule is state that is present, like a running service, so the apply is current as + // soon as it is recorded and does NOT gate what follows. The host's scheduler fires the + // container when the cron is due, re-established from this declaration each apply because the + // declaration is the source of truth (ADR 0018). A run that exits non-zero is recorded and never + // fails the apply or flips the node's state; a run still going when the next is due is skipped + // rather than stacked. It is exclusive with RunOnce and with restart-on: a container runs once + // and gates, runs on a cadence, or stays up — never two of these. + Schedule string `json:"schedule,omitempty"` + + // WhileStopped names resources of the same module — containers — that must be held still for + // the duration of this step's run (novox/hq ADR 0189). The host stops each before the run and + // starts each again after it, **whatever the step did**: a step that failed must leave the + // service running, because the one real risk of this field is a window that never closes. + // + // **For the work a service cannot have done underneath it.** The artifact store's collector + // walks the storage and requires every writer stopped; a run-once step runs beside containers + // and a scheduled one is the same container again, so until this there was no way for a module + // to say it. The predecessor said it with a shell script, which is how the mesh inherited a + // store that has never collected anything. + // + // **Its own module's containers, and only on a schedule.** A module that could quiesce a + // neighbour could stop the mesh. And at apply time the host already has a window — the + // declaration is applied in order and a run-once step gates what follows — so a one-time + // offline job says *before*, not *instead of*; a recurring window is the case order cannot + // express, and the only one this serves. + WhileStopped []string `json:"while-stopped,omitempty"` + + // Health is how this resource is ready (novox/hq ADR 0240 rule 2, Phase B): one kind and its + // timing, judged by the node-engine beside liveness. Absent: judged alive or not, and nothing more. + Health *Health `json:"health,omitempty"` + + // Busy is how this container says it holds work a replacement would lose (novox/hq ADR 0335): asked + // before it is replaced, and while it says busy it is left running, within the bound. Absent: never + // busy, and replaced at once, as before. + Busy *Busy `json:"busy,omitempty"` + + // RunAs is who the container runs as, resolved by the controller from the module's identity and the + // image's declared interface (novox/hq ADR 0325 §2, §3): told by environment (the image reads PUID and + // PGID, say) or by the runtime's own user option. Part of the spec, so a change recreates it. Absent: the + // container runs as its image says. Its key is `identity`, the manifest's word; the field is not, because + // a resource's Identity is its id. + RunAs *ContainerIdentity `json:"identity,omitempty"` +} + +// The ways an image is told who to run as (novox/hq ADR 0325 §2). +const ( + IdentityByEnv = "env" + IdentityByRun = "run" +) + +// ContainerIdentity is who a container runs as, and how its image is told. +type ContainerIdentity struct { + // Mode is IdentityByEnv or IdentityByRun. + Mode string `json:"mode"` + // UserEnv and GroupEnv are the variables the image reads the user's and the group's number from, for + // IdentityByEnv: "PUID", "PGID". + UserEnv string `json:"user_env,omitempty"` + GroupEnv string `json:"group_env,omitempty"` + UID int `json:"uid"` + GID int `json:"gid"` +} + +// UnmarshalJSON reads an identity as strictly as a resource is read. +func (i *ContainerIdentity) UnmarshalJSON(raw []byte) error { + type plain ContainerIdentity + var p plain + if err := strictDecode(raw, &p); err != nil { + return fmt.Errorf("a container's identity: %w", err) + } + *i = ContainerIdentity(p) + return nil +} + +// envName is a variable's name as a runtime takes it. +var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + +func (i *ContainerIdentity) problems(where string, env map[string]string) []string { + if i == nil { + return nil + } + var problems []string + switch i.Mode { + case IdentityByEnv: + if !envName.MatchString(i.UserEnv) || (i.GroupEnv != "" && !envName.MatchString(i.GroupEnv)) { + problems = append(problems, fmt.Sprintf("%s: an identity told by environment names the variables "+ + "it is told in, and %q and %q are not both names", where, i.UserEnv, i.GroupEnv)) + } + for _, name := range []string{i.UserEnv, i.GroupEnv} { + if _, set := env[name]; set && name != "" { + problems = append(problems, fmt.Sprintf("%s: env sets %s, which the identity sets; one value "+ + "for one variable", where, name)) + } + } + case IdentityByRun: + if i.UserEnv != "" || i.GroupEnv != "" { + problems = append(problems, where+": an identity told by the runtime's user option names no variable") + } + default: + problems = append(problems, fmt.Sprintf("%s: an identity is told by %q or %q, not %q", where, + IdentityByEnv, IdentityByRun, i.Mode)) + } + // Never root (novox/hq ADR 0325 §11): refused here, so nothing of the declaration is made or started. + if i.UID <= 0 || i.GID <= 0 { + problems = append(problems, fmt.Sprintf("%s: the identity is uid %d, gid %d, and a module's identity is "+ + "never root, nor a number below it (novox/hq ADR 0325 §11)", where, i.UID, i.GID)) + } + return problems +} + +func (c *Container) Identity() string { return c.ID } +func (c *Container) Kind() Type { return TypeContainer } +func (c *Container) Target() string { return c.Name } + +func (c *Container) validate(where string, _ bool) []string { + problems := c.Health.problems(where, true, !c.RunOnce && c.Schedule == "") + problems = append(problems, c.Busy.problems(where, !c.RunOnce && c.Schedule == "")...) + if c.Name == "" { + problems = append(problems, where+": a container needs a name") + } + // A run-once step may name what it reads under restart-on. For a step the word means *run + // again*: what a container reads is part of its digest, so a step whose named resource changed + // is a different step and runs again (novox/hq ADR 0099). Not refused. + // A container runs once and gates, on a cadence, or stays up — never two of these + // (novox/hq ADR 0053). run-once and schedule are the two "runs to completion" lifecycles and + // contradict each other, and a scheduled step does not stay running to be brought back by + // restart-on either. Refused here on arrival, as the control plane refuses it near its author. + if c.RunOnce && c.Schedule != "" { + problems = append(problems, where+": a container is run-once or scheduled, not both; "+ + "run-once runs once and gates what follows, a schedule runs it again on a cadence") + } + if c.Schedule != "" && len(c.RestartOn) > 0 { + problems = append(problems, where+": a scheduled container cannot also declare restart-on; "+ + "it runs to completion on its cadence rather than staying running to be restarted") + } + if c.Schedule != "" { + if _, err := ParseCron(c.Schedule); err != nil { + problems = append(problems, where+": "+err.Error()) + } + } + // A maintenance window belongs to a recurring step (novox/hq ADR 0189). Refused on anything + // else here, where the field is; that it names containers of the same module, and not itself, + // is judged against the whole declaration (see whileStoppedNames). + if len(c.WhileStopped) > 0 && c.Schedule == "" { + problems = append(problems, where+": while-stopped needs a schedule; at apply the host "+ + "already has a window — the declaration is applied in order and a run-once step gates "+ + "what follows — so a one-time offline job is declared before what it works on") + } + for _, id := range c.WhileStopped { + if id == c.ID { + problems = append(problems, where+": while-stopped names "+strconv.Quote(id)+ + ", which is this step itself") + } + } + // The runtime's flags take addresses, and a name here would be handed to it verbatim and + // refused at create — after the old container was already removed. Refused on arrival instead. + for _, d := range c.Dns { + if net.ParseIP(d) == nil { + problems = append(problems, where+": dns "+strconv.Quote(d)+" is not an address; "+ + "the runtime's resolver flag takes only addresses") + } + } + if c.IP != "" { + if net.ParseIP(c.IP) == nil { + problems = append(problems, where+": ip "+strconv.Quote(c.IP)+" is not an address") + } + if c.Network == "" { + problems = append(problems, where+": an ip needs a network; the runtime refuses a "+ + "static address anywhere but a user-defined one") + } + } + for _, cap := range c.Capabilities { + if !capabilityName.MatchString(cap) { + problems = append(problems, where+": capabilities names "+strconv.Quote(cap)+", which is not a "+ + "capability's name (CAP_NET_ADMIN or NET_ADMIN)") + } + } + if c.Logging != "" && c.Logging != "journald" { + problems = append(problems, where+": logging is "+strconv.Quote(c.Logging)+", and the only place a "+ + "container's output can be sent besides the runtime's own file is \"journald\"") + } + for _, n := range c.Networks { + problems = append(problems, (&Network{Name: n}).validate(where+": networks", false)...) + if n == c.Network { + problems = append(problems, where+": networks names "+n+", which is already the container's network") + } + } + if len(c.Networks) > 0 && (c.RunOnce || c.Schedule != "") { + problems = append(problems, where+": networks is for a container that keeps running; a step "+ + "runs and exits, and joins nothing afterwards") + } + problems = append(problems, c.RunAs.problems(where+": container "+c.Name, c.Env)...) + return append(problems, checkImage(where, c.Image)...) +} + +// Action runs something the bundle declared, and the host never learns what it means. +type Action struct { + ID string `json:"id"` + Type Type `json:"type"` + // Module is the module this resource belongs to, as the controller says it (novox/hq ADR 0429 §4, issue 1132). Absent + // is an older controller, and the module is then guessed from the id (ModuleGuess). + Module string `json:"module,omitempty"` + Command []string `json:"command"` + // Verify is not optional and is not a courtesy. It is the read-back AND the idempotency + // check: the host does not know what a database is, so "is it already there" is a question + // only the declaration can ask (novox/hq ADR 0005). + Verify []string `json:"verify"` + // In names a container to run inside. Empty means the machine itself. + In string `json:"in,omitempty"` +} + +func (a *Action) Identity() string { return a.ID } +func (a *Action) Kind() Type { return TypeAction } + +func (a *Action) Target() string { + target := strings.Join(a.Command, " ") + if a.In != "" { + return "in " + a.In + ": " + target + } + return target +} + +func (a *Action) validate(where string, allowActions bool) []string { + // The bound the whole security argument rests on (novox/hq ADR 0005). + if !allowActions { + return []string{where + + ": an action arrived over the link, and the link may not carry one. The host " + + "applies declarations of known shape; a command to run is not one. A bundle may " + + "carry an action because it arrives with the binary — anyone able to put a " + + "hostile action there could have put it in the host itself"} + } + var problems []string + if len(a.Command) == 0 { + problems = append(problems, where+": an action needs a command") + } + if len(a.Verify) == 0 { + problems = append(problems, where+ + ": an action needs a verify. An action that runs and reports success without "+ + "reading anything back is the fault this host exists to prevent, and verify is "+ + "also how the host knows whether the action is already done") + } + return problems +} + +// newOf returns an empty resource of a kind, or nil if the kind is unknown. +// +// This is the whole vocabulary, in one place. A kind that is not here cannot be declared. +func newOf(t Type) Resource { + switch t { + case TypeDirectory: + return &Directory{} + case TypeFile: + return &File{} + case TypeService: + return &Service{} + case TypePackage: + return &Package{} + case TypeContainer: + return &Container{} + case TypeAction: + return &Action{} + case TypeNetwork: + return &Network{} + case TypeUser: + return &User{} + case TypeArchive: + return &Archive{} + case TypeAccess: + return &Access{} + case TypeProcess: + return &Process{} + case TypeOpening: + return &Opening{} + } + return nil +} + +// Vocabulary is every kind this host speaks. +func Vocabulary() []Type { + return []Type{ + TypeAccess, TypeAction, TypeArchive, TypeContainer, TypeDirectory, TypeFile, + TypeNetwork, TypeOpening, TypePackage, TypeProcess, TypeService, TypeUser, + } +} + +// Declaration is what a machine should be, in the order it should be made so. +type Declaration struct { + Version int + // For names the node this is meant for. A host with an identity refuses one addressed + // elsewhere; a host without one — the first node, applying the bundle it carries — has + // nothing to check against. + For string + // Resources, in the order they are applied. The host does not sort them: ordering is a + // decision, and deciding is not what the host does (novox/hq ADR 0005). + Resources []Resource + + // Adoption says this node is adopted, and which of its modules have been taken. Nil is a + // converged node — which is every node the mesh raised before adoption existed, and so the + // only form an older controller ever sends (novox/hq ADR 0100). + Adoption *Adoption + + // Sequence orders this declaration against every other the mesh has sent this node: each + // send is one higher than the last, assigned under the control plane's hold on the node + // (novox/hq 04-ISSUES/107). Zero is a declaration that carries no order — every one an older + // controller sent, and the bundle genesis applies — and a host makes no ordering claim about + // one of those. + // + // **The one property a declaration needs that its signature does not give it.** A signature + // says the mesh sent this; it cannot say the mesh sent it AFTER the one the host is holding. + // Before this, "older" was inferred from arrival within a batch and a 750ms window, and a + // backlog longer than the batch, or a slow broker, applied a declaration the mesh had already + // superseded. + Sequence int64 + + // Epoch is the controller's lease epoch this declaration was sent under (novox/hq to-be 45 §6): + // the revision at which the sending controller took the lease. A controller that lost its lease + // and goes on sending sends an older epoch than the holder's, and the node-engine refuses what + // is older than what it applied. Zero is a declaration from a controller without a lease — every + // one sent before the lease existed — and carries no claim. + // + // Inside what is signed, beside the sequence, so a message cannot be given a newer epoch than + // the controller gave it. + Epoch int64 + + // LeftOut names the modules of this machine's set the mesh left out of this declaration, + // because a setting stored for one cannot compose with its definition (novox/hq ADR 0163, + // rule 6). A machine is told everything or nothing about what it IS told; this is what it is + // not told, said. The host keeps what it holds for a left-out module and touches none of + // what it wrote for it — its resources are absent from the declaration, and absence would + // otherwise read as removal. + LeftOut []string + + // Generation is the assignment generation this declaration was composed from: the controller's + // counter, raised in the same transaction as every change to what is assigned where (novox/hq + // issue 234). The sequence orders arrival and cannot tell a later send that carries an older view + // of the assignments; this can. A node-engine refuses a declaration composed from a generation + // older than the highest it applied — unless it is a put-back — because applying it would + // undeclare what the mesh still assigns. Zero is a declaration from a controller that claims none, + // and is applied as before. + Generation int64 + + // PutBack says this declaration is a gate putting a machine back (novox/hq issue 234): it carries + // the generation of what it puts back, which may be older than what the machine applied, and is + // not refused for it. + PutBack bool + + // Identities is the identity of every module assigned to this machine (novox/hq ADR 0328 §4): an account + // and a group `mesh-` with the numbers the controller allocated once for the module, the same on + // every machine. The engine makes each before any resource is applied, and removes what it made of one no + // longer declared. Absent is a controller older than ADR 0325. + Identities []Identity + + // Groups is the group of every library this machine is reached by (novox/hq ADR 0328 §3): `mesh-` + // with the number the controller allocated once, mesh-wide. The engine makes each, and removes what it made of + // one no longer declared. Absent is a controller older than ADR 0328. + Groups []Group + + // Libraries is every library this machine is reached by (ADR 0328 §1, §6, §9): its root here, its group, whether + // this is its home, whether the operator's warrant to take it was carried out, the sub-paths its modules ask for, + // and the tool runner's account to put in its group (§11). Absent is a controller older than ADR 0328. + Libraries []Library + + // Filled is the paths of the files the provisioner fills on this machine (novox/hq ADR 0349 §4, §5): each a + // file this declaration declares, which says who owns it at runtime. The provisioner's client has the + // node-engine write only these. Absent is a controller that hands no file to the provisioner, and every + // file is written from the declaration as before. + Filled []string + + // Provisioner is the account the provisioner's client runs as on this machine, by name (novox/hq ADR 0349 §4): + // the only account whose write of a filled file the node-engine's local write socket accepts. Absent is a + // controller that names none, and every such write is refused. + Provisioner string + + // Builds is the build id of every module this declaration names, by module (novox/hq ADR 0429 §4, issue 1132). + Builds map[string]string + + // declared is what each resource says of its module, by id; nil on a declaration built by hand. + declared map[string]string +} + +// ModuleGuess is the module a resource id names when the declaration does not say: everything before its last dot. +// +// The mesh composes a module's resource ids as `.`, and **a module's name may contain a dot** +// (`novox.be` is one on this mesh) while a resource's own id never does, so the owner is everything before the *last* +// dot. False for what the mesh declares in its own right: the foundation's resources carry no dot, and the +// adoption's are named for the mesh. This is the guess an older controller's declaration is read by; a declaration +// that says `module` is read by ModuleOfID, which never guesses for a resource that says (issue 1132). +func ModuleGuess(id string) (string, bool) { + if strings.HasPrefix(id, AdoptionPrefix) { + return "", false + } + at := strings.LastIndex(id, ".") + if at <= 0 { + return "", false + } + return id[:at], true +} + +// ModuleOfID is the module a resource of this declaration belongs to: the module it says (novox/hq ADR 0429 §4, +// issue 1132), and otherwise the guess from its id (ModuleGuess), so a declaration from an older controller reads as +// it always did. False for what belongs to no module. +func (d *Declaration) ModuleOfID(id string) (string, bool) { + if d != nil { + if d.declared == nil { + for _, r := range d.Resources { + if r.Identity() == id && r.DeclaredModule() != "" { + return r.DeclaredModule(), true + } + } + } else if m, ok := d.declared[id]; ok { + return m, true + } + } + return ModuleGuess(id) +} + +// indexModules keeps what each resource says of its module, so ModuleOfID is a lookup. +func (d *Declaration) indexModules() { + d.declared = map[string]string{} + for _, r := range d.Resources { + if m := r.DeclaredModule(); m != "" { + d.declared[r.Identity()] = m + } + } +} + +// checkProvisioner refuses a provisioner client's account that is no account name, or is root: the account whose +// writes the node-engine accepts must be one that could not write the file without it. +func checkProvisioner(name string) []string { + switch { + case name == "": + return nil + case !accountName.MatchString(name): + return []string{fmt.Sprintf("provisioner_account %q is not an account name", name)} + case name == "root": + return []string{"provisioner_account is root; the provisioner's client runs as an account of its own"} + } + return nil +} + +// Fills reports whether the provisioner fills the file at path on this machine. +func (d *Declaration) Fills(path string) bool { return slices.Contains(d.Filled, path) } + +// checkFilled refuses a filled path that is not a declared file saying who owns it at runtime, or that is named +// twice or empty. Every file declared at the path must say it: the provisioner writes the path, and a file there +// the mesh writes some other way would be a second writer of it (ADR 0349 §5). +func checkFilled(filled []string, resources []Resource) []string { + var problems []string + files := map[string][]*File{} + for _, r := range resources { + if f, ok := r.(*File); ok { + files[f.Path] = append(files[f.Path], f) + } + } + seen := map[string]bool{} + for _, path := range filled { + switch { + case path == "": + problems = append(problems, "filled_files names a file with no path") + continue + case seen[path]: + problems = append(problems, fmt.Sprintf("filled_files names %q twice", path)) + continue + } + seen[path] = true + if len(files[path]) == 0 { + problems = append(problems, fmt.Sprintf("filled_files names %q, and this declaration declares no "+ + "file there", path)) + continue + } + for _, f := range files[path] { + if f.OwnerAtRuntime == "" { + problems = append(problems, fmt.Sprintf("resource %q: the provisioner fills %q, and a file it "+ + "fills says who owns it at runtime with owner-at-runtime", f.ID, path)) + } + } + } + return problems +} + +// Identity is one module's identity, as the controller allocated it (ADR 0328 §4): its user `mesh-` with +// uid, its own group of the same name and number, its primary group gid (its own, or its first library's group), +// and the groups of its other libraries as added groups. +type Identity struct { + Module string `json:"module"` + Name string `json:"name"` + UID int `json:"uid"` + GID int `json:"gid"` + Groups []int `json:"groups,omitempty"` + // Runner is the tool runner's account, put in the module's own group (novox/hq ADR 0334 §2) when its tools run + // on this machine. Absent for a module that declares no tools here, and from a controller older than ADR 0334. + Runner string `json:"runner,omitempty"` +} + +// Group is one library's group, as the controller allocated it (ADR 0328 §3). +type Group struct { + Name string `json:"name"` + GID int `json:"gid"` +} + +// GroupPrefix starts every library group's name, as IdentityPrefix starts every module's. +const GroupPrefix = "mesh-" + +// LibraryAccessPrefix starts the access a library's wait and its take act name: `library:`. +const LibraryAccessPrefix = "library:" + +// Library is one library as this machine sees it (ADR 0328). +type Library struct { + Library string `json:"library"` + // Root is the library's root on this machine: on its home the data itself, elsewhere the mount point the mounts + // module mounts it at. + Root string `json:"root"` + Group string `json:"group"` + GID int `json:"gid"` + // Home is that the library's data lives on this machine: only there does the engine keep it. + Home bool `json:"home"` + // Taken is the controller saying the operator's warrant to take it was carried out (§7). + Taken bool `json:"taken,omitempty"` + // Paths are the sub-paths its modules ask for, relative to the root. + Paths []string `json:"paths,omitempty"` + // Runner is the tool runner's account, put in the library's group (§11). Absent when no module's tools use it. + Runner string `json:"runner,omitempty"` +} + +// checkLibraries judges the groups and libraries on arrival: every library's group declared, named for it, never +// root's number; paths that stay below the root; a runner that is an account's name; and no two roots that nest +// (§13). +func checkLibraries(groups []Group, libs []Library) []string { + var problems []string + byName := map[string]int{} + numbers := map[int]string{} + for _, g := range groups { + where := fmt.Sprintf("group %q", g.Name) + if !strings.HasPrefix(g.Name, GroupPrefix) || !accountName.MatchString(g.Name) { + problems = append(problems, fmt.Sprintf("%s: a library's group is named %s", where, GroupPrefix)) + } + if g.GID <= 0 { + problems = append(problems, fmt.Sprintf("%s: gid %d, and a library's group is never root, nor a number "+ + "below it (novox/hq ADR 0328 §15); nothing of this declaration was made", where, g.GID)) + } + if _, twice := byName[g.Name]; twice { + problems = append(problems, where+": declared twice") + } + if other, used := numbers[g.GID]; used { + problems = append(problems, fmt.Sprintf("%s: gid %d is %s's too", where, g.GID, other)) + } + byName[g.Name], numbers[g.GID] = g.GID, g.Name + } + seen := map[string]bool{} + for _, l := range libs { + where := fmt.Sprintf("library %q", l.Library) + switch { + case l.Library == "" || strings.ContainsAny(l.Library, "/ \t\n"): + problems = append(problems, where+": a library is named by one word") + case seen[l.Library]: + problems = append(problems, where+": declared twice") + } + seen[l.Library] = true + if !filepath.IsAbs(l.Root) || filepath.Clean(l.Root) != l.Root || l.Root == "/" { + problems = append(problems, fmt.Sprintf("%s: its root %q is not an absolute, clean path below /", where, + l.Root)) + } + if gid, ok := byName[l.Group]; !ok || gid != l.GID { + problems = append(problems, fmt.Sprintf("%s: its group %s (gid %d) is not one this declaration makes", + where, l.Group, l.GID)) + } + if l.Group != GroupPrefix+l.Library { + problems = append(problems, fmt.Sprintf("%s: its group is %s%s, and %s is not", where, GroupPrefix, + l.Library, l.Group)) + } + for _, p := range l.Paths { + if p == "" || filepath.IsAbs(p) || filepath.Clean(p) != p || p == ".." || strings.HasPrefix(p, "../") { + problems = append(problems, fmt.Sprintf("%s: the path %q is not a clean path below its root", where, p)) + } + } + if l.Runner != "" && !accountName.MatchString(l.Runner) { + problems = append(problems, fmt.Sprintf("%s: its runner %q is not an account's name", where, l.Runner)) + } + } + for i, a := range libs { + for _, b := range libs[i+1:] { + if a.Root != "" && b.Root != "" && (a.Root == b.Root || strings.HasPrefix(a.Root, b.Root+"/") || + strings.HasPrefix(b.Root, a.Root+"/")) { + problems = append(problems, fmt.Sprintf("library %q: its root and library %q's nest, and libraries "+ + "never overlap (novox/hq ADR 0328 §13)", a.Library, b.Library)) + } + } + } + return problems +} + +// IdentityPrefix starts every module identity's name. +const IdentityPrefix = "mesh-" + +// checkIdentities judges the identities on arrival: each named for its module, never root, and no name or number +// twice — two modules with one account would write as each other. +func checkIdentities(ids []Identity, groups []Group) []string { + var problems []string + names, uids := map[string]bool{}, map[int]string{} + libraryGroups := map[int]bool{} + for _, g := range groups { + libraryGroups[g.GID] = true + } + for _, id := range ids { + where := fmt.Sprintf("identity %q", id.Name) + if id.Module == "" || id.Name != IdentityPrefix+id.Module || !accountName.MatchString(id.Name) { + problems = append(problems, fmt.Sprintf("%s: a module's identity is named %s, and this one is "+ + "for the module %q", where, IdentityPrefix, id.Module)) + } + if id.UID <= 0 || id.GID <= 0 { + problems = append(problems, fmt.Sprintf("%s: uid %d, gid %d, and a module's identity is never root, "+ + "nor a number below it (novox/hq ADR 0328 §15); nothing of this declaration was made", where, + id.UID, id.GID)) + } + if names[id.Name] { + problems = append(problems, where+": declared twice") + } + if other, used := uids[id.UID]; used { + problems = append(problems, fmt.Sprintf("%s: uid %d is %s's too", where, id.UID, other)) + } + // Its primary group is its own (gid == uid) or a library's this declaration makes (ADR 0328 §4); so is + // every added group. A module's own number is never a library group's. + if id.GID != id.UID && !libraryGroups[id.GID] { + problems = append(problems, fmt.Sprintf("%s: its primary group gid %d is neither its own (%d) nor a "+ + "library's this declaration makes", where, id.GID, id.UID)) + } + if libraryGroups[id.UID] { + problems = append(problems, fmt.Sprintf("%s: uid %d is a library group's number too", where, id.UID)) + } + for _, g := range id.Groups { + if !libraryGroups[g] { + problems = append(problems, fmt.Sprintf("%s: its added group gid %d is not a library's this "+ + "declaration makes", where, g)) + } + } + if id.Runner != "" && !accountName.MatchString(id.Runner) { + problems = append(problems, fmt.Sprintf("%s: its runner %q is not an account's name", where, id.Runner)) + } + names[id.Name], uids[id.UID] = true, id.Name + } + return problems +} + +// LeftOutModuleOf says which left-out module a recorded resource belongs to, if any: its id is the +// module's name, a dot, and the module's own id for it. A module's name may contain a dot, so the +// longest left-out name that prefixes the id wins; a false match keeps a thing an apply would +// otherwise remove, which is the conservative mistake. +func (d *Declaration) LeftOutModuleOf(id string) (string, bool) { + best := "" + for _, m := range d.LeftOut { + if strings.HasPrefix(id, m+".") && len(m) > len(best) { + best = m + } + } + return best, best != "" +} + +// Adoption is a node's mode, as the controller records it: the node is adopted, and these are +// the modules taken on it so far (novox/hq ADR 0100). +// +// **Authoritative, and only ever stated by the controller.** A host does not work out whether it +// is adopted; it is told, in every declaration, so a host restarted from the declaration it kept +// is in the same mode it was in before. +// +// Untaken names, per module assigned here and not yet taken, the ids of its resources. Any kind +// may be listed: a file, a directory, a service's unit or a container can already be on the +// machine, and an action run inside a held container reaches what was found (novox/hq ADR 0103); +// the host decides per kind what can be held. The host cannot split a resource id into its +// module, because module names may contain dots, so the controller says which ids belong to which +// module rather than leaving the host to guess. +type Adoption struct { + Taken []string `json:"taken"` + Untaken map[string][]string `json:"untaken,omitempty"` +} + +// capabilityName is what a Linux capability is called: upper case, underscores, an optional CAP_ +// prefix. The runtime accepts either spelling. +var capabilityName = regexp.MustCompile(`^(CAP_)?[A-Z][A-Z0-9_]*$`) + +// AdoptionPrefix is the id prefix of what the mesh itself declares because a node is adopted — +// its openings and its guard. Nothing under it belongs to a module, so none of it is ever held. +const AdoptionPrefix = "adoption." + +// UntakenModuleOf says which untaken module declares a resource, if any. +func (a *Adoption) UntakenModuleOf(id string) (string, bool) { + if a == nil { + return "", false + } + for module, ids := range a.Untaken { + if slices.Contains(ids, id) { + return module, true + } + } + return "", false +} + +// checkAdoption holds what an adoption says against the resources beside it. Every problem is a +// refusal: a host that misread which module is untaken would replace a predecessor's service the +// operator never took. +func checkAdoption(a *Adoption, resources []Resource, allowActions bool) []string { + if a == nil { + return nil + } + if allowActions { + // The bundle is carried with the binary and raises a foundation before any mesh exists. + // Whether a node is adopted is the controller's record, and a bundle that claimed it would + // be the host deciding its own mode (novox/hq ADR 0100). + return []string{"a carried bundle says the node is adopted, and only the mesh can say " + + "that: a node's mode is the controller's record, sent in every declaration"} + } + kinds := map[string]Type{} + for _, r := range resources { + kinds[r.Identity()] = r.Kind() + } + var problems []string + for _, module := range a.Taken { + if _, both := a.Untaken[module]; both { + problems = append(problems, fmt.Sprintf( + "adoption: the module %q is said to be both taken and untaken", module)) + } + } + owner := map[string]string{} + modules := make([]string, 0, len(a.Untaken)) + for module := range a.Untaken { + modules = append(modules, module) + } + sort.Strings(modules) + for _, module := range modules { + for _, id := range a.Untaken[module] { + if strings.HasPrefix(id, AdoptionPrefix) { + problems = append(problems, fmt.Sprintf( + "adoption: %q is the mesh's own and belongs to no module, so it cannot be untaken", id)) + continue + } + if first, twice := owner[id]; twice { + problems = append(problems, fmt.Sprintf( + "adoption: %q is said to belong to both %q and %q", id, first, module)) + continue + } + owner[id] = module + if _, declared := kinds[id]; !declared { + problems = append(problems, fmt.Sprintf( + "adoption: %q of the untaken module %q is not in this declaration", id, module)) + } + } + } + return problems +} + +// RefusalError refuses a whole declaration, naming every problem at once. +// +// Every problem rather than the first: a caller fixing one at a time learns the next only by +// running again, and a declaration is generated, so a person reading this is debugging the +// generator. +type RefusalError struct { + Problems []string +} + +func (e *RefusalError) Error() string { + return fmt.Sprintf( + "this declaration is refused, and none of it was applied:\n - %s\n\n"+ + "A host that applied the parts it understood would leave a machine that looks "+ + "configured and is not.", + strings.Join(e.Problems, "\n - ")) +} + +// Parse reads a declaration that arrived over the link, and refuses anything it does not fully +// understand — including any action, which the link may not carry (novox/hq ADR 0005). +func Parse(raw []byte) (*Declaration, error) { return parse(raw, false) } + +// ParseTrusted reads a declaration from a source already as privileged as the host itself: the +// bundle it carries, or a file handed to it by someone who is running it as root. +// +// Actions are permitted here and nowhere else. The asymmetry is deliberate and is the entire +// content of ADR 0005: refusing actions from the bundle buys nothing, because whoever built the +// bundle built the binary; refusing them from the link buys the bound on what a compromised +// control plane can express. +func ParseTrusted(raw []byte) (*Declaration, error) { return parse(raw, true) } + +// envelope is the declaration with its resources still unread. +// +// Two passes, because which fields are legal depends on the "type" inside each resource. The +// first pass takes the envelope and each resource's bytes; the second decodes each one into +// the struct for its kind, strictly. +type envelope struct { + Version int `json:"declaration"` + For string `json:"for,omitempty"` + Adoption *Adoption `json:"adoption,omitempty"` + // OwnsNothing is the control plane saying, explicitly, that this node's declaration is empty + // on purpose — it owns nothing the mesh put there (novox/hq issue 127). Without it an empty + // resources list is refused as a likely mistake; with it the node applies the empty + // declaration and drops what it last held. The two are distinguished because a truncated or + // mis-composed body arrives as empty too, and a host that could not tell them apart would let + // a bug quietly strip a machine. + OwnsNothing bool `json:"owns_nothing,omitempty"` + Resources []json.RawMessage `json:"resources"` + // Sequence is optional on the wire, so a controller that does not send one is still + // understood: absent reads as zero, which is "no ordering claimed" rather than "first". + Sequence int64 `json:"sequence,omitempty"` + // Epoch is optional on the wire as the sequence is: absent is a controller without a lease. + // **An older host refuses this key**, decoding strictly; a controller sends it only to a host + // whose reports carry a report sequence, which a host that reads it does. + Epoch int64 `json:"epoch,omitempty"` + // LeftOut is optional on the wire too, and absent when nothing was left out (ADR 0163). + LeftOut []string `json:"left_out,omitempty"` + // Generation and PutBack are optional on the wire too (novox/hq issue 234): absent is a controller + // that claims no generation. **An older host refuses these keys**, decoding strictly; a controller + // sends them only to a host whose reports say `reads_generation`. + Generation int64 `json:"generation,omitempty"` + PutBack bool `json:"put_back,omitempty"` + // Identities is optional on the wire too (novox/hq ADR 0325 §3, ADR 0328 §4). **An older host refuses this key**; a + // controller sends it only to a host whose health says link.IdentityContract. + Identities []Identity `json:"identities,omitempty"` + // Groups and Libraries are optional too (novox/hq ADR 0328). **An older host refuses these keys**; a controller + // sends them only to a host whose health says link.LibraryContract. + Groups []Group `json:"groups,omitempty"` + Libraries []Library `json:"libraries,omitempty"` + // Filled is optional too (novox/hq ADR 0349 §5), as is a file's `owner-at-runtime`. **An older host refuses + // these keys**; a controller sends them only to a host whose health says link.OwnershipContract. + Filled []string `json:"filled_files,omitempty"` + // Provisioner is optional too (novox/hq ADR 0349 §4). **An older host refuses this key**; a controller sends it + // only to a host whose health says link.ProvisionerContract. + Provisioner string `json:"provisioner_account,omitempty"` + // Builds is optional too (novox/hq ADR 0429 §4, issue 1132): the build id of each module the declaration names, and + // every resource's `module`. **An older host refuses these keys**; a controller sends them only to a host whose + // reports say `reads_modules`. + Builds map[string]string `json:"builds,omitempty"` +} + +func parse(raw []byte, allowActions bool) (*Declaration, error) { + var env envelope + if err := strictDecode(raw, &env); err != nil { + return nil, &RefusalError{Problems: []string{"not a declaration: " + err.Error()}} + } + + if env.Version != Version { + // Everything below assumes the vocabulary, so there is nothing further to say. + return nil, &RefusalError{Problems: []string{fmt.Sprintf( + "declaration version %d; this host speaks version %d. Refused whole rather than "+ + "partly, so a newer vocabulary is never half-applied by an older host", + env.Version, Version)}} + } + + d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption, Sequence: env.Sequence, + Epoch: env.Epoch, LeftOut: env.LeftOut, Generation: env.Generation, PutBack: env.PutBack, + Identities: env.Identities, Groups: env.Groups, Libraries: env.Libraries, Filled: env.Filled, + Provisioner: env.Provisioner, Builds: env.Builds} + problems := checkIdentities(env.Identities, env.Groups) + problems = append(problems, checkProvisioner(env.Provisioner)...) + problems = append(problems, checkLibraries(env.Groups, env.Libraries)...) + if env.Generation < 0 { + // Below zero would read as "none claimed" and pass every refusal of an older generation. + problems = append(problems, fmt.Sprintf("an assignment generation below zero (%d) is not one the "+ + "mesh assigns", env.Generation)) + } + if env.PutBack && allowActions { + // A put-back is a gate's, sent by the mesh; a carried bundle puts nothing back. + problems = append(problems, "a carried bundle says it is a put-back, and only the mesh's gate can say that") + } + if env.Sequence < 0 || env.Epoch < 0 { + // Below zero is no order any controller assigns, and read as "none claimed" it would let the + // declaration past every refusal of what is older. + problems = append(problems, fmt.Sprintf("an order below zero (epoch %d, sequence %d) is not "+ + "one the mesh assigns", env.Epoch, env.Sequence)) + } + if len(env.LeftOut) > 0 && allowActions { + // The bundle is carried with the binary and leaves nothing out: which module a setting + // stopped composing for is the mesh's record (ADR 0163). + problems = append(problems, "a carried bundle says modules were left out, and only the "+ + "mesh can say that") + } + for _, m := range env.LeftOut { + if strings.TrimSpace(m) == "" { + problems = append(problems, "left_out names a module with no name") + } + } + + if len(env.Resources) == 0 && !env.OwnsNothing { + problems = append(problems, "no resources. An empty declaration is a mistake, not a "+ + "machine with nothing on it — say so with an explicit empty list if that is meant") + } + + seen := map[string]int{} + for i, rawResource := range env.Resources { + // Peek, leniently. This pass only needs to know which struct to decode into; reading + // strictly here would report an unknown field before knowing which fields are known. + var head struct { + ID string `json:"id"` + Type Type `json:"type"` + } + _ = json.Unmarshal(rawResource, &head) + + where := fmt.Sprintf("resource %d", i) + if head.ID != "" { + where = fmt.Sprintf("resource %q", head.ID) + } + + if head.ID == "" { + problems = append(problems, where+": no id. Identity is what lets the host know "+ + "this is the same resource it applied last time") + } else if first, ok := seen[head.ID]; ok { + problems = append(problems, fmt.Sprintf( + "%s: id already used by resource %d. Two resources with one identity cannot "+ + "both be tracked", where, first)) + } else { + seen[head.ID] = i + } + + resource := newOf(head.Type) + if resource == nil { + problems = append(problems, fmt.Sprintf( + "%s: unknown type %q. This host understands %s", where, head.Type, vocabulary())) + continue + } + + // A field the kind does not have is refused, and the struct is what says so — there + // is no list of exclusions for anyone to keep current. + // + // Asked separately rather than taken from the decoder's error, because the decoder + // stops at the first unknown field and this record promises every problem at once. A + // caller fixing one field at a time learns the next only by running again. + if unknown := unknownFields(rawResource, resource); len(unknown) > 0 { + for _, field := range unknown { + problems = append(problems, fmt.Sprintf( + "%s: a %s does not use %q, and it is set. Refused rather than ignored", + where, head.Type, field)) + } + continue + } + if err := json.Unmarshal(rawResource, resource); err != nil { + problems = append(problems, fmt.Sprintf("%s: %s", where, err)) + continue + } + + problems = append(problems, resource.validate(where, allowActions)...) + d.Resources = append(d.Resources, resource) + } + problems = append(problems, checkWhileStopped(d.Resources)...) + problems = append(problems, checkAdoption(env.Adoption, d.Resources, allowActions)...) + problems = append(problems, checkReplaces(d.Resources)...) + problems = append(problems, checkFilled(env.Filled, d.Resources)...) + if env.Adoption == nil { + for _, r := range d.Resources { + if r.Kind() == TypeOpening { + // On a converged node the mesh's own filter admits what is declared, and the + // found firewall is retired; an opening there would be a rule in a firewall the + // mesh has disabled (novox/hq ADR 0100). + problems = append(problems, fmt.Sprintf( + "resource %q: an opening is for an adopted node, and this declaration does not "+ + "say the node is adopted", r.Identity())) + } + if svc, ok := r.(*Service); ok && svc.TakesOver != nil { + // A tunnel is taken over on an adopted node, where what is found is kept: on a + // converged one there is nothing found to take over, and stopping a unit the + // mesh did not declare would be the host deciding (novox/hq ADR 0105). + problems = append(problems, fmt.Sprintf( + "resource %q: taking over a tunnel is for an adopted node, and this declaration "+ + "does not say the node is adopted", r.Identity())) + } + } + } + + if len(problems) > 0 { + return nil, &RefusalError{Problems: problems} + } + d.indexModules() + return d, nil +} + +// checkWhileStopped judges a maintenance window against the whole declaration (novox/hq ADR 0189). +// +// A step may hold still only a container that is **here** — in this same declaration, which is to +// say on this machine and placed by the mesh. That is what makes it the module's own: a node's +// declaration carries one module's resources beside another's, so the id must also be a container +// and not a file or a directory, which there would be nothing to stop. +// +// Refused on arrival rather than discovered at the first fire. A window that names something the +// host cannot stop is a window that opens at 03:00 and reports nothing until somebody reads a log. +func checkWhileStopped(resources []Resource) []string { + containers := map[string]bool{} + for _, r := range resources { + if r.Kind() == TypeContainer { + containers[r.Identity()] = true + } + } + var problems []string + for _, r := range resources { + c, ok := r.(*Container) + if !ok { + continue + } + for _, id := range c.WhileStopped { + if containers[id] { + continue + } + problems = append(problems, fmt.Sprintf( + "resource %q: while-stopped names %q, and this declaration has no container by "+ + "that id. A step may hold still only a container placed on this machine "+ + "beside it", c.ID, id)) + } + } + return problems +} + +// accountName is what an identity's or an access user's name may be: the portable account name the user tools +// take, optionally ending in `$`, never starting with a dash — it is handed to them as an argument. +var accountName = regexp.MustCompile(`^[A-Za-z0-9_.][A-Za-z0-9_.-]*\$?$`) + +func strictDecode(raw []byte, into any) error { + // DisallowUnknownFields is the whole point rather than strictness for its own sake: a + // field the host does not know is a thing the control plane believes it asked for. + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(into); err != nil { + return err + } + // And **nothing after it**. A decoder reads one value and stops, so a file holding a + // declaration followed by anything at all — a truncated rewrite, two declarations + // concatenated, a stray line from whatever wrote the file — parses as the first value and the + // rest is never looked at. + // + // That is the same fault this host refuses everywhere else, in its quietest form: the machine + // applies something, reports success, and what it applied is not what the file says. Found + // when a test harness appended a line to a bundle by accident and every apply kept working. + if _, err := dec.Token(); err != io.EOF { + return fmt.Errorf( + "there is more in this file after the declaration ends. Refused whole: a file with " + + "something after it may be a truncated rewrite or two declarations run together, " + + "and applying the first would be applying something nobody wrote") + } + return nil +} + +// unknownFields names every JSON key the kind's struct has no field for. +// +// The struct's own tags are the list of what is legal, so adding a field to a kind is the +// whole of adding it — there is nowhere else that has to agree. +func unknownFields(raw []byte, into Resource) []string { + var got map[string]json.RawMessage + if err := json.Unmarshal(raw, &got); err != nil { + return nil // not an object; the decode below will say so properly + } + + known := map[string]bool{} + t := reflect.TypeOf(into).Elem() + for i := 0; i < t.NumField(); i++ { + name, _, _ := strings.Cut(t.Field(i).Tag.Get("json"), ",") + if name != "" && name != "-" { + known[name] = true + } + } + + var unknown []string + for field := range got { + if !known[field] { + unknown = append(unknown, field) + } + } + sort.Strings(unknown) + return unknown +} + +func checkMode(where, mode string) []string { + if mode == "" { + return nil + } + if len(mode) != 4 || mode[0] != '0' { + return []string{fmt.Sprintf( + "%s: mode %q; write it as four octal digits such as \"0644\", so it means the "+ + "same thing here as it does in the manifest it came from", where, mode)} + } + for _, c := range mode[1:] { + if c < '0' || c > '7' { + return []string{fmt.Sprintf("%s: mode %q is not octal", where, mode)} + } + } + return nil +} + +// checkImage insists on content, not on a name. +// +// A tag moves and a digest does not. The bundle's whole claim is that what it names is exact +// (novox/hq ADR 0006), and a bundle pinning `postgres:17` pins nothing — it names whatever +// that tag points at on the day the host happens to run. +// +// **Two forms say something exact, and only one of them needs a registry.** `name@sha256:…` is a +// manifest digest, which a registry assigns on push. A bare `sha256:…` is an image the machine +// already holds, addressed by the digest of its own configuration — equally immutable, equally +// unforgeable, and requiring nothing to have served it. +// +// That second form is what a first machine needs. The mesh's own control plane exists in no public +// registry and never will: it is built from source, and until this mesh has a registry of its own +// there is nowhere to push it to and therefore no manifest digest to name it by. Insisting on one +// would mean a registry has to exist before the thing that lets a mesh have a registry can start — +// which is not a pin, it is a dependency the rule accidentally created. A machine that built an +// image, or was handed one, can name it by what it is. +func checkImage(where, image string) []string { + if image == "" { + return []string{where + ": a container needs an image"} + } + // An image this machine holds, named by the digest of its own configuration. + if strings.HasPrefix(image, "sha256:") { + if len(image) != len("sha256:")+64 { + return []string{fmt.Sprintf( + "%s: image id %q is not a sha256 digest", where, image)} + } + return nil + } + name, digest, found := strings.Cut(image, "@") + if !found || name == "" { + return []string{fmt.Sprintf( + "%s: image %q is not pinned. Write it as name@sha256:… — or as sha256:… for an image "+ + "this machine already holds. A tag moves, and a bundle that pinned a tag would "+ + "not be pinned", where, image)} + } + if !strings.HasPrefix(digest, "sha256:") || len(digest) != len("sha256:")+64 { + return []string{fmt.Sprintf( + "%s: image digest %q is not a sha256 digest", where, digest)} + } + return nil +} + +func vocabulary() string { + kinds := Vocabulary() + names := make([]string, 0, len(kinds)) + for _, t := range kinds { + names = append(names, string(t)) + } + sort.Strings(names) + return strings.Join(names, ", ") +} + +// ParseFileTrusted reads a declaration from a file somebody handed this host. +// +// The same as ParseTrusted, and it allows whole-line `//` comments first. A pinned, hand-authored +// artefact that nobody can annotate is one nobody can review — the foundation bundle is mostly +// explanation of why each digest is what it is. +// +// **Only for a file, never for the link.** Over the link the format stays exactly JSON, because +// a wire format with a second thing to strip is a wire format with a second thing to disagree +// about. +// +// It exists because there were two readers for one file: the bundle stripped comments and `apply` +// did not, so the example bundle in this repository could be built into a binary and not applied +// from disk. The failure was `invalid character '/'`, which names the symptom and not the cause. +func ParseFileTrusted(raw []byte) (*Declaration, error) { + return ParseTrusted(stripComments(raw)) +} + +// stripComments removes whole lines beginning with `//`. +// +// Only whole lines: anything cleverer would need to know where strings begin and end, and a +// parser that half-understands its input is worse than one that does not try. A `//` inside a +// value — every image reference has one — is untouched. +func stripComments(raw []byte) []byte { + var kept []string + for _, line := range strings.Split(string(raw), "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "//") { + continue + } + kept = append(kept, line) + } + return []byte(strings.Join(kept, "\n")) +} + +// checkReplaces refuses a `replaces` naming something this same declaration still declares, or +// named by two processes (novox/hq issue 213). What is replaced is what the declaration no longer +// says — a resource still declared is applied, not handed over, and one handed to two replacements +// would go when the first of them came up, whatever became of the second. +func checkReplaces(resources []Resource) []string { + declared := map[string]bool{} + for _, r := range resources { + declared[r.Identity()] = true + } + var problems []string + by := map[string]string{} + for _, r := range resources { + p, ok := r.(*Process) + if !ok { + continue + } + for _, id := range p.Replaces { + if declared[id] { + problems = append(problems, fmt.Sprintf( + "resource %q: replaces %q, which this declaration still declares — a process "+ + "replaces what the declaration no longer says", p.ID, id)) + } + if other, taken := by[id]; taken && other != p.ID { + problems = append(problems, fmt.Sprintf( + "resource %q: replaces %q, which %q replaces too — one thing has one replacement", + p.ID, id, other)) + } + by[id] = p.ID + } + } + return problems +} diff --git a/go/declaration/declaration_test.go b/go/declaration/declaration_test.go new file mode 100644 index 0000000..36063d4 --- /dev/null +++ b/go/declaration/declaration_test.go @@ -0,0 +1,595 @@ +package declaration + +import ( + "errors" + "strings" + "testing" +) + +// Each test names the decision it defends (novox/hq ADR 0017). The decision here is ADR 0005, +// and the property it turns on is that unknown is REFUSED, never skipped. + +func valid() string { + return `{"declaration":1,"resources":[ + {"id":"etc","type":"directory","path":"/etc/mesh","mode":"0755"}, + {"id":"conf","type":"file","path":"/etc/mesh/host.conf","content":"a\n","mode":"0640"}, + {"id":"svc","type":"service","unit":"mesh-host.service","state":"running"} + ]}` +} + +func refusalFor(t *testing.T, raw string) *RefusalError { + t.Helper() + _, err := Parse([]byte(raw)) + if err == nil { + t.Fatal("expected a refusal") + } + var refusal *RefusalError + if !errors.As(err, &refusal) { + t.Fatalf("expected a RefusalError, got %T: %v", err, err) + } + return refusal +} + +func TestAValidDeclarationParsesInOrder(t *testing.T) { + d, err := Parse([]byte(valid())) + if err != nil { + t.Fatalf("unexpected refusal: %v", err) + } + // Order is stated, not derived. The host must not sort. + got := []string{d.Resources[0].Identity(), d.Resources[1].Identity(), d.Resources[2].Identity()} + want := []string{"etc", "conf", "svc"} + for i := range want { + if got[i] != want[i] { + t.Fatalf("resources reordered: %v, want %v", got, want) + } + } +} + +func TestAnUnknownTypeRefusesTheWholeDeclaration(t *testing.T) { + // The property everything else rests on. A host that skipped what it did not understand + // would apply most of a declaration and report success — a node that looks configured and + // is not, which is 04-ISSUES/003 with the declaration on the other side of the wire. + refusal := refusalFor(t, `{"declaration":1,"resources":[ + {"id":"ok","type":"directory","path":"/etc/mesh"}, + {"id":"what","type":"blockchain","path":"/etc/mesh"} + ]}`) + + joined := strings.Join(refusal.Problems, "\n") + if !strings.Contains(joined, "blockchain") { + t.Errorf("the unknown type was not named: %v", refusal.Problems) + } + // And it must say what IS understood, or the reader goes to the source to find out. + if !strings.Contains(joined, "directory") || !strings.Contains(joined, "service") { + t.Errorf("the refusal does not say what this host understands: %v", refusal.Problems) + } +} + +func TestAnUnknownFieldIsRefused(t *testing.T) { + // A field the host does not know is a thing the control plane believes it asked for. + refusal := refusalFor(t, `{"declaration":1,"resources":[ + {"id":"conf","type":"file","path":"/etc/x","content":"a","immutable":true} + ]}`) + if !strings.Contains(strings.Join(refusal.Problems, "\n"), "immutable") { + t.Errorf("the unknown field was not named: %v", refusal.Problems) + } +} + +func TestAFieldTheTypeDoesNotUseIsRefusedNotIgnored(t *testing.T) { + // The same fault in miniature: set and ignored means the control plane believes it asked + // for something the host will never do. + refusal := refusalFor(t, `{"declaration":1,"resources":[ + {"id":"svc","type":"service","unit":"a.service","state":"running","path":"/etc/x"} + ]}`) + joined := strings.Join(refusal.Problems, "\n") + if !strings.Contains(joined, "path") || !strings.Contains(joined, "Refused rather than ignored") { + t.Errorf("a field a service does not use was accepted: %v", refusal.Problems) + } +} + +func TestAnUnknownVersionIsRefusedWhole(t *testing.T) { + // An older host handed a newer vocabulary must not quietly do half of it. + refusal := refusalFor(t, `{"declaration":99,"resources":[ + {"id":"a","type":"directory","path":"/etc/mesh"} + ]}`) + joined := strings.Join(refusal.Problems, "\n") + if !strings.Contains(joined, "99") || !strings.Contains(joined, "version 1") { + t.Errorf("the version mismatch was not stated plainly: %v", refusal.Problems) + } + // Nothing else is reported, because everything else assumes a vocabulary this host does + // not have — a list of complaints derived from the wrong grammar is noise. + if len(refusal.Problems) != 1 { + t.Errorf("expected only the version problem, got: %v", refusal.Problems) + } +} + +func TestEveryProblemIsReportedAtOnce(t *testing.T) { + // A declaration is generated, so a person reading a refusal is debugging the generator. + // Fixing one problem at a time and re-running to find the next wastes their afternoon. + refusal := refusalFor(t, `{"declaration":1,"resources":[ + {"id":"","type":"directory","path":"/a"}, + {"id":"b","type":"file"}, + {"id":"c","type":"service","unit":"x.service","state":"dancing"} + ]}`) + if len(refusal.Problems) < 3 { + t.Errorf("expected every problem at once, got: %v", refusal.Problems) + } +} + +func TestIdentityIsRequiredAndUnique(t *testing.T) { + // Identity is what lets the store know this is the same resource it applied last time, + // which is what makes removal possible at all. + refusal := refusalFor(t, `{"declaration":1,"resources":[ + {"id":"same","type":"directory","path":"/a"}, + {"id":"same","type":"directory","path":"/b"} + ]}`) + if !strings.Contains(strings.Join(refusal.Problems, "\n"), "already used") { + t.Errorf("a duplicate identity was accepted: %v", refusal.Problems) + } +} + +func TestModeIsRefusedUnlessItMeansWhatItLooksLike(t *testing.T) { + // "644" and "0644" differ, and the one that looks right in a manifest is the four-digit + // form. Accepting both would make a mode mean two things. + for _, mode := range []string{"644", "0999", "rwxr-xr-x", "07777777"} { + refusal := refusalFor(t, `{"declaration":1,"resources":[ + {"id":"f","type":"file","path":"/a","mode":"`+mode+`"} + ]}`) + if !strings.Contains(strings.Join(refusal.Problems, "\n"), "mode") { + t.Errorf("mode %q was accepted: %v", mode, refusal.Problems) + } + } + if _, err := Parse([]byte(`{"declaration":1,"resources":[ + {"id":"f","type":"file","path":"/a","mode":"0644"} + ]}`)); err != nil { + t.Errorf("a well-formed mode was refused: %v", err) + } +} + +func TestARefusalSaysNothingWasApplied(t *testing.T) { + // The reader's first question is whether the machine was left half-changed. + refusal := refusalFor(t, `{"declaration":1,"resources":[{"id":"x","type":"nope"}]}`) + if !strings.Contains(refusal.Error(), "none of it was applied") { + t.Errorf("the refusal does not say the machine is untouched: %s", refusal.Error()) + } +} + +func TestAnEmptyDeclarationIsAMistake(t *testing.T) { + refusalFor(t, `{"declaration":1,"resources":[]}`) +} + +// --- the vocabulary the foundation bootstrap needs (novox/hq 07-the-foundation.md) --- + +func TestAnActionOverTheLinkIsRefused(t *testing.T) { + // novox/hq ADR 0005. The link may push declarations of known shape and never a command to + // run. This is the boundary the whole security argument rests on, so it is asserted + // directly rather than inferred from the type list. + raw := []byte(`{"declaration":1,"resources":[ + {"id":"schema","type":"action","command":["psql","-f","x.sql"],"verify":["psql","-c","select 1"]} + ]}`) + + if _, err := Parse(raw); err == nil { + t.Fatal("an action arriving over the link was accepted") + } else if !strings.Contains(err.Error(), "the link may not carry one") { + t.Errorf("refused for the wrong reason: %v", err) + } + + // And the same bytes from the bundle are fine — the asymmetry IS the decision. + if _, err := ParseTrusted(raw); err != nil { + t.Errorf("the bundle may carry an action, and this one was refused: %v", err) + } +} + +func TestAnActionWithoutVerifyIsRefused(t *testing.T) { + // An action that runs and reports success without reading anything back is the fault this + // host exists to prevent. Verify is also the idempotency check, so an action without one + // cannot be applied twice safely either. + _, err := ParseTrusted([]byte(`{"declaration":1,"resources":[ + {"id":"schema","type":"action","command":["psql","-f","x.sql"]} + ]}`)) + if err == nil { + t.Fatal("an action with no verify was accepted") + } + if !strings.Contains(err.Error(), "needs a verify") { + t.Errorf("refused for the wrong reason: %v", err) + } +} + +func TestAnImageMustBePinnedByDigest(t *testing.T) { + // novox/hq ADR 0006: reproducibility comes from pinning the identity of a thing. A bundle + // naming a tag pins nothing — it names whatever that tag points at on the day it runs. + for _, image := range []string{ + "postgres:17", + "postgres", + "postgres@sha256:short", + "@sha256:0000000000000000000000000000000000000000000000000000000000000000", + } { + _, err := ParseTrusted([]byte(`{"declaration":1,"resources":[ + {"id":"store","type":"container","name":"store","image":"` + image + `"} + ]}`)) + if err == nil { + t.Errorf("image %q was accepted and is not pinned", image) + } + } + + good := "postgres@sha256:" + strings.Repeat("a", 64) + if _, err := ParseTrusted([]byte(`{"declaration":1,"resources":[ + {"id":"store","type":"container","name":"store","image":"` + good + `"} + ]}`)); err != nil { + t.Errorf("a properly pinned image was refused: %v", err) + } +} + +// An image the machine already holds, addressed by the digest of its own configuration. +// +// **The form a first machine needs.** A manifest digest is assigned by a registry on push, so +// insisting on one means a registry has to exist before the thing that lets a mesh have a registry +// can start. The mesh's own control plane is built from source and lives in no public registry; a +// machine that built it, or was handed it, names it by what it is — a content address, not a name, +// and immutable in exactly the way the rule asks for. +func TestAnImageTheMachineHoldsIsNamedByItsOwnDigest(t *testing.T) { + held := "sha256:" + strings.Repeat("b", 64) + if _, err := ParseTrusted([]byte(`{"declaration":1,"resources":[ + {"id":"control","type":"container","name":"mesh-controller","image":"` + held + `"} + ]}`)); err != nil { + t.Errorf("an image named by its own digest was refused: %v", err) + } + + // Still a digest, though. A truncated one names several images, and which one ran would be + // whichever the runtime happened to match first. + for _, bad := range []string{"sha256:abc", "sha256:", "sha256:" + strings.Repeat("b", 63)} { + if _, err := ParseTrusted([]byte(`{"declaration":1,"resources":[ + {"id":"control","type":"container","name":"mesh-controller","image":"` + bad + `"} + ]}`)); err == nil { + t.Errorf("image id %q was accepted and is not a digest", bad) + } + } +} + +func TestAFieldTheNewTypesDoNotUseIsRefused(t *testing.T) { + // The field-set check must cover the types added last, not only the three it was written + // for. A package that carries a `content` is a control plane believing it asked for + // something that will never happen. + for _, body := range []string{ + `{"id":"p","type":"package","package":"docker","content":"x"}`, + `{"id":"p","type":"package","package":"docker","image":"x"}`, + `{"id":"c","type":"container","name":"n","image":"i@sha256:` + strings.Repeat("a", 64) + `","unit":"x.service"}`, + `{"id":"a","type":"action","command":["x"],"verify":["y"],"path":"/tmp/x"}`, + } { + _, err := ParseTrusted([]byte(`{"declaration":1,"resources":[` + body + `]}`)) + if err == nil { + t.Errorf("a resource carrying a field its type does not use was accepted: %s", body) + continue + } + if !strings.Contains(err.Error(), "Refused rather than ignored") { + t.Errorf("refused for the wrong reason: %v", err) + } + } +} + +func TestTheVocabularyIsTheTwelveShapesTheMeshNeeds(t *testing.T) { + // Six of them the bootstrap uses (novox/hq 07-the-foundation.md), and removing one is a + // failing test rather than a discovery during a first-node install. + // + // Two were added on 2026-08-30 and the count is asserted precisely because adding one is a + // decision. `user` and `archive` exist because most of what a person installs is not a + // service: a shell, a chat client, a desktop are a package plus configuration **in + // somebody's home**, and a mesh with no user can only own /etc. `archive` is for the case + // inlining cannot serve — a theme is hundreds of files, and inlining them would rewrite all + // of them whenever one changed. + speaks := map[Type]bool{} + for _, t := range Vocabulary() { + speaks[t] = true + } + for _, want := range []Type{ + TypeDirectory, TypeFile, TypeService, TypePackage, TypeContainer, TypeAction, + TypeUser, TypeArchive, TypeNetwork, TypeAccess, TypeProcess, TypeOpening, + } { + if !speaks[want] { + t.Errorf("the host no longer speaks %q", want) + } + if newOf(want) == nil { + t.Errorf("%q is in the vocabulary and cannot be constructed", want) + } + } + // `network` is the ninth, and novox/hq ADR 0029 is the decision that made it one: an action + // could create a network and nothing could remove it, because an action leaves no footprint + // the host can undo — so the network would outlive every module that was ever unassigned. + // + // `access` is the tenth, and novox/hq ADR 0051 is its decision: shared, pre-existing data is + // the operator's, and a module is granted use of it without owning it — a shape the host must + // tell apart from a directory precisely because it must NOT create, chown or remove it. + // + // `daemon` is the eleventh, and it exists because the mechanism was leaking into every module. + // Running code of one's own meant a `container` and therefore an image; running a script meant + // a `service` and a unit somebody else had to install. One intent — run this and keep it + // running — expressed two unrelated ways, with the hosting chosen before anything could be + // declared. A daemon says what to run; the machine's own supervisor is how, and the mesh owns + // the unit because it is the mesh's own code (novox/hq 03-DESIGN/01-to-be/18-building-a-module.md). + // + // It is a full-host shape rather than a portable one: it needs a process supervisor to install + // into. It does NOT need a container runtime, which is the point — only software that + // genuinely needs isolation asks for a container. + // + // `opening` is the twelfth, and novox/hq ADR 0100 is its decision: on an adopted node the + // firewall found there stays in force, and what the mesh needs reachable is converged through + // it as a state the host marks as the mesh's — never a command, which the link may not carry. + if len(speaks) != 12 { + t.Errorf("the vocabulary is %d shapes rather than 12; every addition widens what a compromised "+ + "control plane can express, so a change here is a decision: %s", + len(speaks), vocabulary()) + } +} + +func TestADeclarationFromDiskMayBeAnnotated(t *testing.T) { + // The bundle in this repository is mostly explanation of why each digest is what it is, and + // it could be built into a binary and not applied from disk — two readers for one file. The + // failure was `invalid character '/'`, which names the symptom and not the cause. + raw := []byte(`// why this exists +{ + "declaration": 1, + // and why this resource is here + "resources": [ + {"id": "f", "type": "file", "path": "/etc/x", "content": "hello"} + ] +}`) + d, err := ParseFileTrusted(raw) + if err != nil { + t.Fatalf("a file with comments was refused: %v", err) + } + if len(d.Resources) != 1 { + t.Fatalf("got %d resources", len(d.Resources)) + } + // And the wire format is untouched: over the link it is exactly JSON, because a format with + // a second thing to strip is a format with a second thing to disagree about. + if _, err := Parse(raw); err == nil { + t.Fatal("the link accepted a declaration with comments in it") + } +} + +func TestSomethingInsideAValueIsNotAComment(t *testing.T) { + // Every image reference has a `//` in it somewhere near. Only whole lines are dropped. + d, err := ParseFileTrusted([]byte(`{"declaration":1,"resources":[ + {"id":"f","type":"file","path":"/etc/x","content":"see https://example.invalid/ for why"}]}`)) + if err != nil { + t.Fatal(err) + } + file, ok := d.Resources[0].(*File) + if !ok { + t.Fatalf("got %T", d.Resources[0]) + } + if !strings.Contains(file.Content, "https://example.invalid/") { + t.Fatalf("a value was mangled: %q", file.Content) + } +} + +// A declaration followed by anything at all is refused whole. +// +// A JSON decoder reads one value and stops, so a file holding a declaration and then a stray line +// parses as the declaration and the rest is never looked at. The machine applies something, +// reports success, and what it applied is not what the file says. +// +// Not hypothetical: a test harness appended a line to the foundation bundle by accident, every +// apply kept working, and nothing said so for the entire time it was wrong. +func TestSomethingAfterTheDeclarationIsRefused(t *testing.T) { + good := `{"declaration":1,"resources":[{"id":"a","type":"file","path":"/tmp/a",` + + `"content":"x","mode":"0644"}]}` + if _, err := ParseFileTrusted([]byte(good)); err != nil { + t.Fatalf("an ordinary declaration was refused: %v", err) + } + for _, after := range []string{ + "MESHBUNDLE 2>&1; echo \"__exit=$?\"", + good, + "garbage", + } { + _, err := ParseFileTrusted([]byte(good + "\n" + after)) + if err == nil { + t.Fatalf("a file with %q after the declaration was accepted", after) + } + } + // Trailing whitespace is not "something after it", and refusing it would make every file + // written by an editor unusable. + if _, err := ParseFileTrusted([]byte(good + "\n\n \n")); err != nil { + t.Fatalf("a declaration with a trailing newline was refused: %v", err) + } +} + +// A name that would read as a reference to something else is refused before it reaches a runtime. +func TestANetworkNameIsRefusedIfItIsNotOne(t *testing.T) { + for _, name := range []string{"", "mail/private", "host:mail", "a b"} { + refusal := refusalFor(t, `{"declaration":1,"resources":[ + {"id":"private","type":"network","name":"`+name+`"} + ]}`) + if len(refusal.Problems) == 0 { + t.Errorf("a network named %q was accepted", name) + } + } +} + +// A placeholder with nothing to fill it is refused before anything is written. +// +// The alternative is a configuration file containing the literal `${secret:x}`, which the program +// reads as a value and fails on somewhere with no connection to this. +func TestAFileAskingForASecretItWasNotGivenIsRefused(t *testing.T) { + refusal := refusalFor(t, `{"declaration":1,"resources":[ + {"id":"c","type":"file","path":"/tmp/x","content":"token=${secret:absent}"} + ]}`) + if len(refusal.Problems) == 0 { + t.Fatal("a file naming a secret nobody gave it was accepted") + } +} + +// And a secret nobody asked for, because somebody believes it is in a file where it is not. +func TestASecretTheContentNeverUsesIsRefused(t *testing.T) { + refusal := refusalFor(t, `{"declaration":1,"resources":[ + {"id":"c","type":"file","path":"/tmp/x","content":"nothing here","secrets":{"spare":"S"}} + ]}`) + if len(refusal.Problems) == 0 { + t.Fatal("a secret the content never mentions was accepted") + } +} + +// The runtime's resolver and address flags take only addresses; a name would be refused at +// create, after the old container was already gone. Refused on arrival instead — and an address +// without a user-defined network is refused for the same reason. +func TestAContainersResolverAndAddressAreAddressesOrRefused(t *testing.T) { + refused := func(body string) []string { + _, err := Parse([]byte(`{"declaration":1,"resources":[` + body + `]}`)) + if err == nil { + return nil + } + return []string{err.Error()} + } + base := `"id":"c","type":"container","name":"x",` + + `"image":"a@sha256:0000000000000000000000000000000000000000000000000000000000000000"` + if p := refused(`{` + base + `,"network":"m","dns":["resolver.local"]}`); len(p) == 0 { + t.Error("a resolver named by name was accepted; the runtime takes only addresses") + } + if p := refused(`{` + base + `,"ip":"192.168.203.7"}`); len(p) == 0 { + t.Error("a static address with no network was accepted; the runtime refuses it") + } + if p := refused(`{` + base + `,"network":"m","dns":["192.168.203.254"],"ip":"192.168.203.7"}`); len(p) != 0 { + t.Errorf("a well-formed resolver and address were refused: %v", p) + } +} + +func TestAnExplicitlyEmptyDeclarationIsAccepted(t *testing.T) { + // A deliberately-empty declaration (novox/hq issue 127) says owns_nothing, and is applied so + // the node drops what it last held — distinct from an accidental empty body, which is refused. + if _, err := Parse([]byte(`{"declaration":1,"owns_nothing":true,"resources":[]}`)); err != nil { + t.Fatalf("an explicitly-empty declaration must be accepted: %v", err) + } + // Without the marker, an empty declaration is still refused as a likely mistake. + _, err := Parse([]byte(`{"declaration":1,"resources":[]}`)) + if err == nil || !strings.Contains(err.Error(), "no resources") { + t.Fatalf("an unmarked empty declaration must still be refused; got %v", err) + } +} + +// A container's kept networks are names, not its own network, and not for a step (novox/hq ADR +// 0163, rule 4); and the mesh may say which modules it left out, which a carried bundle may not. +func TestKeptNetworksAndLeftOutModulesAreReadStrictly(t *testing.T) { + pinnedImage := "postgres@sha256:" + strings.Repeat("a", 64) + d, err := Parse([]byte(`{"declaration":1,"left_out":["web"],"resources":[ + {"id":"app","type":"container","name":"app","image":"` + pinnedImage + `","networks":["predecessor_default"]} + ]}`)) + if err != nil { + t.Fatal(err) + } + if got := d.Resources[0].(*Container).Networks; len(got) != 1 || got[0] != "predecessor_default" { + t.Fatalf("the kept network was not read: %v", got) + } + if m, left := d.LeftOutModuleOf("web.server"); !left || m != "web" { + t.Fatalf("web.server is not web's: %q %v", m, left) + } + if _, left := d.LeftOutModuleOf("webapp.server"); left { + t.Fatal("webapp.server was taken for web's") + } + for name, raw := range map[string]string{ + "a bad network name": `{"declaration":1,"resources":[ + {"id":"app","type":"container","name":"app","image":"` + pinnedImage + `","networks":["a/b"]}]}`, + "its own network": `{"declaration":1,"resources":[ + {"id":"app","type":"container","name":"app","image":"` + pinnedImage + `","network":"own","networks":["own"]}]}`, + "a step": `{"declaration":1,"resources":[ + {"id":"app","type":"container","name":"app","image":"` + pinnedImage + `","run-once":true,"networks":["x"]}]}`, + "a nameless module": `{"declaration":1,"left_out":[""],"resources":[ + {"id":"app","type":"container","name":"app","image":"` + pinnedImage + `"}]}`, + } { + if _, err := Parse([]byte(raw)); err == nil { + t.Errorf("%s was accepted", name) + } + } + if _, err := ParseTrusted([]byte(`{"declaration":1,"left_out":["web"],"resources":[ + {"id":"app","type":"container","name":"app","image":"` + pinnedImage + `"}]}`)); err == nil || + !strings.Contains(err.Error(), "only the mesh can say that") { + t.Fatalf("a carried bundle leaving modules out was accepted: %v", err) + } +} + +// A container may ask for a capability by name, and nothing else (novox/hq ADR 0170). +func TestACapabilityIsNamedOrRefused(t *testing.T) { + image := "postgres@sha256:" + strings.Repeat("a", 64) + d, err := Parse([]byte(`{"declaration":1,"resources":[ + {"id":"fw","type":"container","name":"fw","image":"` + image + `","network":"host","capabilities":["NET_ADMIN","CAP_NET_RAW"]} + ]}`)) + if err != nil { + t.Fatal(err) + } + if got := d.Resources[0].(*Container).Capabilities; len(got) != 2 || got[0] != "NET_ADMIN" { + t.Fatalf("capabilities read as %v", got) + } + for _, bad := range []string{`"net_admin"`, `"ALL;rm -rf /"`, `"privileged"`} { + if _, err := Parse([]byte(`{"declaration":1,"resources":[ + {"id":"fw","type":"container","name":"fw","image":"` + image + `","capabilities":[` + bad + `]}]}`)); err == nil { + t.Errorf("%s was accepted as a capability", bad) + } + } +} + +// A container may send its output to the machine's journal, and nowhere else but the runtime's own +// file (novox/hq ADR 0179): what reads the machine's logs then reads the container's too. +func TestAContainerMayLogToTheJournalAndNowhereElse(t *testing.T) { + image := "postgres@sha256:" + strings.Repeat("a", 64) + d, err := Parse([]byte(`{"declaration":1,"resources":[ + {"id":"front","type":"container","name":"front","image":"` + image + `","logging":"journald"} + ]}`)) + if err != nil { + t.Fatal(err) + } + if got := d.Resources[0].(*Container).Logging; got != "journald" { + t.Fatalf("logging read as %q", got) + } + for _, bad := range []string{`"syslog"`, `"none"`, `"json-file"`} { + if _, err := Parse([]byte(`{"declaration":1,"resources":[ + {"id":"front","type":"container","name":"front","image":"` + image + `","logging":` + bad + `}]}`)); err == nil { + t.Errorf("%s was accepted as a place to log", bad) + } + } +} + +// Issue 1024, ADR 0422: a directory or archive says how it treats files it did not write. + +func TestADirectoryAndAnArchiveSayWhatTheyDoWithUnknownFiles(t *testing.T) { + d, err := Parse([]byte(`{"declaration":1,"resources":[ + {"id":"d","type":"directory","path":"/home/a/x","unknown":"move-out","knows":["keep","sub/"]}, + {"id":"r","type":"archive","source":"https://x/y","digest":"sha256:` + strings.Repeat("a", 64) + `","path":"/home/a/y","unknown":"module"} + ]}`)) + if err != nil { + t.Fatalf("refused: %v", err) + } + dir := d.Resources[0].(*Directory) + if dir.Unknown != UnknownMoveOut || len(dir.Knows) != 2 { + t.Errorf("directory: %+v", dir) + } + if ar := d.Resources[1].(*Archive); ar.Unknown != UnknownModule { + t.Errorf("archive: %+v", ar) + } + for _, v := range []string{UnknownMoveOut, UnknownModule, UnknownProgram} { + if _, err := Parse([]byte(`{"declaration":1,"resources":[{"id":"d","type":"directory","path":"/x","unknown":"` + v + `"}]}`)); err != nil { + t.Errorf("%s refused: %v", v, err) + } + } +} + +func TestBadUnknownAndKnowsFormsAreRefusedNamingTheResource(t *testing.T) { + digest := `"digest":"sha256:` + strings.Repeat("a", 64) + `"` + cases := map[string]string{ + "bad value": `{"id":"d","type":"directory","path":"/x","unknown":"delete"}`, + "empty entry": `{"id":"d","type":"directory","path":"/x","unknown":"move-out","knows":[""]}`, + "absolute": `{"id":"d","type":"directory","path":"/x","unknown":"move-out","knows":["/etc"]}`, + "empty comp": `{"id":"d","type":"directory","path":"/x","unknown":"move-out","knows":["a//b"]}`, + "dot": `{"id":"d","type":"directory","path":"/x","unknown":"move-out","knows":["./a"]}`, + "dotdot": `{"id":"d","type":"directory","path":"/x","unknown":"move-out","knows":["a/../b"]}`, + "repeat": `{"id":"d","type":"directory","path":"/x","unknown":"move-out","knows":["a","a"]}`, + "no unknown": `{"id":"d","type":"directory","path":"/x","knows":["a"]}`, + "archive bad": `{"id":"d","type":"archive","source":"s",` + digest + `,"path":"/x","unknown":"nope"}`, + "archive knws": `{"id":"d","type":"archive","source":"s",` + digest + `,"path":"/x","knows":["a"]}`, + } + for name, res := range cases { + t.Run(name, func(t *testing.T) { + refusal := refusalFor(t, `{"declaration":1,"resources":[`+res+`]}`) + if !strings.Contains(strings.Join(refusal.Problems, "\n"), `resource "d"`) { + t.Errorf("resource not named: %v", refusal.Problems) + } + }) + } +} diff --git a/go/declaration/emitted_check_test.go b/go/declaration/emitted_check_test.go new file mode 100644 index 0000000..948dca5 --- /dev/null +++ b/go/declaration/emitted_check_test.go @@ -0,0 +1,33 @@ +package declaration + +import ( + "os" + "testing" +) + +// The control plane's actual output, put through the host's actual parser. +// +// Skipped unless MESH_EMITTED names a file, so this is a check somebody runs deliberately rather +// than a dependency between two repositories. It exists because "the host needs no new +// vocabulary" is the load-bearing claim of every computed and contributed resource, and the only +// honest way to know it is to hand the host one and see. +func TestWhatTheControlPlaneEmittedIsSomethingThisHostAccepts(t *testing.T) { + path := os.Getenv("MESH_EMITTED") + if path == "" { + t.Skip("set MESH_EMITTED to a declaration the control plane produced") + } + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + d, err := Parse(raw) + if err != nil { + t.Fatalf("the host refuses what the control plane sends:\n%v", err) + } + if len(d.Resources) == 0 { + t.Fatal("parsed, and empty") + } + for _, r := range d.Resources { + t.Logf("accepted %s %s → %s", r.Kind(), r.Identity(), r.Target()) + } +} diff --git a/go/declaration/health.go b/go/declaration/health.go new file mode 100644 index 0000000..d6395bf --- /dev/null +++ b/go/declaration/health.go @@ -0,0 +1,188 @@ +package declaration + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + "time" +) + +// Health is how a long-running resource is ready, as the controller composed it from the module's +// `health` (novox/hq ADR 0240 rule 2, to-be 48 §2–§3, Phase B): one kind and its timing, the endpoint +// already the port this machine published it on. +// +// **The node-engine runs every kind and owns every verdict.** http and tcp it makes itself, from the +// machine to the port; unit it reads from the service manager it already reads; exec and runtime it hands +// to the container runtime as the container's own check, with this timing, and reads the state; tool it +// asks of its own node tools. Nothing else on the machine sets a container's check. +// +// Refused here as the controller refuses it near the author, in the same bounds: an engine that took a +// check it could not judge would say a module ready that nothing looked at. +type Health struct { + Kind string `json:"kind"` + // Endpoint is the module's name for what Port is: for the words a verdict is said in. + Endpoint string `json:"endpoint,omitempty"` + Port int `json:"port,omitempty"` + Path string `json:"path,omitempty"` + Status int `json:"status,omitempty"` + Body string `json:"body,omitempty"` + Scheme string `json:"scheme,omitempty"` + Command string `json:"command,omitempty"` + Tool string `json:"tool,omitempty"` + Interval string `json:"interval"` + Timeout string `json:"timeout"` + Looks int `json:"looks"` + Grace string `json:"grace"` + // Needs is the provision the check exercises (to-be 48 §6): said with every verdict, so the + // controller can hold what it finds under the provider's own condition. + Needs string `json:"needs,omitempty"` +} + +// UnmarshalJSON reads a health strictly, as everything a declaration carries is read: a field this host +// does not know is a part of the check the controller believes it asked for, and nothing would look at it. +func (h *Health) UnmarshalJSON(raw []byte) error { + type plain Health + var p plain + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(&p); err != nil { + return fmt.Errorf("health: %w", err) + } + *h = Health(p) + return nil +} + +// The kinds. +const ( + HealthRuntime = "runtime" + HealthHTTP = "http" + HealthTCP = "tcp" + HealthExec = "exec" + HealthUnit = "unit" + HealthTool = "tool" +) + +// The bounds (ADR 0240 rule 2) — the controller's, held again here. +const ( + HealthIntervalFloor = 10 * time.Second + HealthLooksFloor = 2 + HealthWithin = 5 * time.Minute +) + +// Every, Within and GraceOf are the timing, read. Validated on arrival, so a parse error here is +// impossible on a declaration that was accepted; it reads as zero. +func (h *Health) Every() time.Duration { d, _ := time.ParseDuration(h.Interval); return d } +func (h *Health) Within() time.Duration { d, _ := time.ParseDuration(h.Timeout); return d } +func (h *Health) GraceOf() time.Duration { d, _ := time.ParseDuration(h.Grace); return d } + +// RunByRuntime says the container runtime runs this check as the container's own: exec and runtime. +func (h *Health) RunByRuntime() bool { + return h != nil && (h.Kind == HealthExec || h.Kind == HealthRuntime) +} + +// Words is the check in a few words, as a verdict is said: "http /healthz on web". +func (h *Health) Words() string { + switch h.Kind { + case HealthHTTP: + return "http " + h.Path + " on " + orPort(h.Endpoint, h.Port) + case HealthTCP: + return "tcp on " + orPort(h.Endpoint, h.Port) + case HealthTool: + return "its tool " + h.Tool + case HealthRuntime: + return "its image's own check" + case HealthExec: + return "its command" + case HealthUnit: + return "its unit" + } + return h.Kind +} + +func orPort(endpoint string, port int) string { + if endpoint != "" { + return endpoint + } + return fmt.Sprint(port) +} + +// problems holds a resource's health to its kind and bounds. container says whether the resource is a +// container; longRunning whether it stays up. +func (h *Health) problems(where string, container, longRunning bool) []string { + if h == nil { + return nil + } + var problems []string + say := func(format string, args ...any) { + problems = append(problems, where+": "+fmt.Sprintf(format, args...)) + } + if !longRunning { + say("health is judged on what stays up; a step or a scheduled run is judged by its own outcome") + } + switch h.Kind { + case HealthHTTP, HealthTCP: + if h.Port < 1 || h.Port > 65535 { + say("a %s check needs the port it looks at", h.Kind) + } + case HealthExec: + if !container { + say("an exec check runs inside a container") + } + if strings.TrimSpace(h.Command) == "" { + say("an exec check needs a command") + } + case HealthRuntime: + if !container { + say("a runtime check is a container image's own") + } + case HealthUnit: + if container { + say("a unit check is a service's or a process's own") + } + case HealthTool: + if strings.TrimSpace(h.Tool) == "" { + say("a tool check names the tool") + } + default: + say("health of kind %q; it is runtime, http, tcp, exec, unit or tool", h.Kind) + } + if h.Kind == HealthHTTP { + if !strings.HasPrefix(h.Path, "/") { + say("an http check asks a path starting with /") + } + if h.Status != 0 && (h.Status < 100 || h.Status > 599) { + say("an http check expects status %d, which is not one", h.Status) + } + if h.Scheme != "" && h.Scheme != "http" && h.Scheme != "https" { + say("an http check is over http or https, not %q", h.Scheme) + } + } + if strings.ContainsAny(h.Command, "\n\r") { + say("an exec check's command is one line") + } + every, everyErr := time.ParseDuration(h.Interval) + within, withinErr := time.ParseDuration(h.Timeout) + grace, graceErr := time.ParseDuration(h.Grace) + switch { + case everyErr != nil || withinErr != nil || graceErr != nil: + say("health's interval, timeout and grace are durations") + default: + if every < HealthIntervalFloor { + say("a check looks no more often than every %s, not every %s", HealthIntervalFloor, every) + } + if within <= 0 || within >= every { + say("a look takes more than nothing and less than its interval") + } + if grace < 0 { + say("a grace is not negative") + } + if h.Looks >= HealthLooksFloor && grace+time.Duration(h.Looks)*every > HealthWithin { + say("a grace and the failing looks take at most %s", HealthWithin) + } + } + if h.Looks < HealthLooksFloor { + say("a check is unhealthy after at least %d failing looks, not %d", HealthLooksFloor, h.Looks) + } + return problems +} diff --git a/go/declaration/health_test.go b/go/declaration/health_test.go new file mode 100644 index 0000000..000d53a --- /dev/null +++ b/go/declaration/health_test.go @@ -0,0 +1,49 @@ +package declaration + +import ( + "strings" + "testing" +) + +// A resource's health is held to its kind and bounds on arrival, as the controller holds it near the +// author (novox/hq ADR 0240 rule 2): an engine that took a check it could not judge would say a module ready +// that nothing looked at. +func TestAHealthIsHeldToItsKindAndBounds(t *testing.T) { + pinned := "postgres@sha256:" + strings.Repeat("a", 64) + parse := func(resource string) error { + _, err := Parse([]byte(`{"declaration":1,"resources":[` + resource + `]}`)) + return err + } + container := func(health string, more string) string { + return `{"id":"m.db","type":"container","name":"db","image":"` + pinned + `"` + more + `,"health":` + health + `}` + } + timing := `"interval":"30s","timeout":"5s","looks":3,"grace":"1m0s"` + for _, ok := range []string{ + container(`{"kind":"http","endpoint":"web","port":31001,"path":"/",`+timing+`}`, ""), + container(`{"kind":"exec","command":"pg_isready",`+timing+`}`, ""), + container(`{"kind":"runtime",`+timing+`}`, ""), + `{"id":"m.d","type":"service","unit":"d.service","state":"running","health":{"kind":"unit",` + timing + `}}`, + } { + if err := parse(ok); err != nil { + t.Errorf("refused %s: %v", ok, err) + } + } + for _, c := range []struct{ resource, says string }{ + {container(`{"kind":"http","path":"/",`+timing+`}`, ""), "needs the port"}, + {container(`{"kind":"exec",`+timing+`}`, ""), "needs a command"}, + {container(`{"kind":"unit",`+timing+`}`, ""), "a service's or a process's own"}, + {container(`{"kind":"ping",`+timing+`}`, ""), `of kind "ping"`}, + {container(`{"kind":"tcp","port":1,"interval":"5s","timeout":"1s","looks":3,"grace":"0s"}`, ""), "no more often than every 10s"}, + {container(`{"kind":"tcp","port":1,"interval":"30s","timeout":"30s","looks":3,"grace":"0s"}`, ""), "less than its interval"}, + {container(`{"kind":"tcp","port":1,"interval":"30s","timeout":"5s","looks":1,"grace":"0s"}`, ""), "at least 2 failing looks"}, + {container(`{"kind":"tcp","port":1,"interval":"60s","timeout":"5s","looks":3,"grace":"3m"}`, ""), "at most 5m0s"}, + {container(`{"kind":"runtime",`+timing+`}`, `,"run-once":true`), "judged on what stays up"}, + {`{"id":"m.d","type":"service","unit":"d.service","state":"running","health":{"kind":"runtime",` + timing + `}}`, "a container image's own"}, + {container(`{"kind":"tcp","port":1,`+timing+`,"retries":2}`, ""), "retries"}, + } { + err := parse(c.resource) + if err == nil || !strings.Contains(err.Error(), c.says) { + t.Errorf("%s: want a refusal saying %q, got %v", c.resource, c.says, err) + } + } +} diff --git a/go/declaration/identity_test.go b/go/declaration/identity_test.go new file mode 100644 index 0000000..e97225f --- /dev/null +++ b/go/declaration/identity_test.go @@ -0,0 +1,61 @@ +package declaration + +import ( + "strings" + "testing" +) + +const anImage = "docker.io/library/x@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + +// The fields ADR 0325 adds, read as the controller sends them (novox/hq ADR 0325 §2, §3, §5, issue 567). +func TestIdentitiesUsersAndARunAsAreReadAsSent(t *testing.T) { + d, err := Parse([]byte(`{"declaration":1, + "identities":[{"module":"radarr","name":"mesh-radarr","uid":61001,"gid":61001}], + "resources":[ + {"id":"radarr.app","type":"container","name":"radarr","image":"` + anImage + `", + "identity":{"mode":"env","user_env":"PUID","group_env":"PGID","uid":61001,"gid":61001}}, + {"id":"plex.app","type":"container","name":"plex","image":"` + anImage + `","identity":{"mode":"run","uid":61002,"gid":61002}}, + {"id":"radarr.movies","type":"access","path":"/srv/media/movies","mode":"read-write","taken":true, + "users":[{"identity":"mesh-radarr","uid":61001,"use":"read-write","from":"container"}, + {"identity":"nox","uid":61003,"use":"read","from":"tool-runner"}]}, + {"id":"old.movies","type":"access","path":"/srv/media/movies","mode":"read-write","grant":{"account":"nox"}}]}`)) + if err != nil { + t.Fatal(err) + } + if len(d.Identities) != 1 || d.Identities[0].UID != 61001 { + t.Fatalf("identities: %+v", d.Identities) + } + if c := d.Resources[0].(*Container); c.RunAs == nil || c.RunAs.UserEnv != "PUID" { + t.Fatalf("a container's identity: %+v", c.RunAs) + } + if a := d.Resources[2].(*Access); !a.Taken || len(a.Users) != 2 || a.Users[1].From != FromToolRunner { + t.Fatalf("an access's users: %+v", a) + } +} + +// What the engine refuses on arrival: a root number, an identity not named for its module, a field it does not +// know, a user that writes a read access, a variable set twice. +func TestWhatADeclarationMayNotSayOfAnIdentity(t *testing.T) { + for want, raw := range map[string]string{ + "never root": `"identities":[{"module":"m","name":"mesh-m","uid":0,"gid":61001}],"resources":[]`, + "named mesh-": `"identities":[{"module":"m","name":"m","uid":61001,"gid":61001}],` + + `"resources":[]`, + "is mesh-a's too": `"identities":[{"module":"a","name":"mesh-a","uid":61001,"gid":61001},` + + `{"module":"b","name":"mesh-b","uid":61001,"gid":61002}],"resources":[]`, + "unknown field": `"identities":[{"module":"m","name":"mesh-m","uid":61001,"gid":61001,"home":"/"}],` + + `"resources":[]`, + "writes an access declared read": `"resources":[{"id":"m.a","type":"access","path":"/srv/x","mode":"read",` + + `"users":[{"identity":"mesh-m","uid":61001,"use":"read-write","from":"container"}]}]`, + "uid 0": `"resources":[{"id":"m.a","type":"access","path":"/srv/x","users":[{"identity":"root","uid":0,` + + `"use":"read","from":"container"}]}]`, + "one value": `"resources":[{"id":"m.c","type":"container","name":"c","image":"` + anImage + `",` + + `"env":{"PUID":"1000"},"identity":{"mode":"env","user_env":"PUID","uid":61001,"gid":61001}}]`, + `"env" or "run"`: `"resources":[{"id":"m.c","type":"container","name":"c","image":"` + anImage + `",` + + `"identity":{"mode":"sudo","uid":61001,"gid":61001}}]`, + } { + _, err := Parse([]byte(`{"declaration":1,` + raw + `}`)) + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("want a refusal saying %q, got %v", want, err) + } + } +} diff --git a/go/declaration/into_test.go b/go/declaration/into_test.go new file mode 100644 index 0000000..531e015 --- /dev/null +++ b/go/declaration/into_test.go @@ -0,0 +1,35 @@ +package declaration + +import ( + "strings" + "testing" +) + +// Defends novox/hq ADR 0102: a file written into carries only a JSON object of its keys, in a +// format the host speaks, and a service may name what it is reloaded on. + +func TestAFileWrittenIntoIsRefusedUnlessItIsAnObjectOfKeys(t *testing.T) { + for name, c := range map[string]struct{ resource, refusal string }{ + "another format": {`{"id":"f","type":"file","path":"/etc/x","into":"toml","content":"a = 1"}`, `into "toml"`}, + "not an object": {`{"id":"f","type":"file","path":"/etc/x","into":"json","content":"[1,2]"}`, "JSON object"}, + "with create-once": {`{"id":"f","type":"file","path":"/etc/x","into":"json","content":"{}","create-once":true}`, "create-once"}, + } { + _, err := Parse([]byte(`{"declaration":1,"resources":[` + c.resource + `]}`)) + if err == nil || !strings.Contains(err.Error(), c.refusal) { + t.Errorf("%s: want a refusal naming %q, got %v", name, c.refusal, err) + } + } + d, err := Parse([]byte(`{"declaration":1,"resources":[ + {"id":"f","type":"file","path":"/etc/x","into":"json","content":"{\"k\":1}"}, + {"id":"s","type":"service","unit":"docker.service","state":"running","reload-on":["f"]} + ]}`)) + if err != nil { + t.Fatal(err) + } + if f := d.Resources[0].(*File); f.Into != IntoJSON { + t.Errorf("into was read as %q", f.Into) + } + if s := d.Resources[1].(*Service); len(s.ReloadOn) != 1 || s.ReloadOn[0] != "f" { + t.Errorf("reload-on was read as %v", s.ReloadOn) + } +} diff --git a/go/declaration/module_test.go b/go/declaration/module_test.go new file mode 100644 index 0000000..cdd4b02 --- /dev/null +++ b/go/declaration/module_test.go @@ -0,0 +1,60 @@ +package declaration + +import "testing" + +// 1132/AC1: a resource says its module, and the declaration says each module's build; a resource that does +// not say gets today's guess, so an older controller still works. +func TestAResourceSaysItsModuleAndTheDeclarationItsBuilds(t *testing.T) { + d, err := Parse([]byte(`{"declaration":1, + "builds":{"novox.be":"b-9","web":"b-4"}, + "resources":[ + {"id":"novox.be.data","type":"directory","path":"/srv/a","module":"novox.be"}, + {"id":"web.conf","type":"file","path":"/etc/w","content":"x","module":"web"}, + {"id":"old.dir","type":"directory","path":"/srv/old"}, + {"id":"base","type":"directory","path":"/srv/base"}]}`)) + if err != nil { + t.Fatal(err) + } + for id, want := range map[string]string{"novox.be.data": "novox.be", "web.conf": "web", "old.dir": "old"} { + got, ok := d.ModuleOfID(id) + if !ok || got != want { + t.Errorf("%s belongs to %q (%v), want %q", id, got, ok, want) + } + } + if m, ok := d.ModuleOfID("base"); ok { + t.Errorf("a resource with no module and no dot belongs to %q, want none", m) + } + if d.Builds["web"] != "b-4" || d.Builds["novox.be"] != "b-9" || len(d.Builds) != 2 { + t.Errorf("builds read as %v", d.Builds) + } +} + +// 1132/AC1: the module the declaration gives wins over the guess, even when the id would guess another. +func TestTheDeclaredModuleWinsOverTheGuess(t *testing.T) { + d, err := Parse([]byte(`{"declaration":1,"resources":[ + {"id":"a.b.c","type":"directory","path":"/srv/a","module":"a.b"}, + {"id":"x.y","type":"directory","path":"/srv/x","module":"other"}]}`)) + if err != nil { + t.Fatal(err) + } + if m, _ := d.ModuleOfID("x.y"); m != "other" { + t.Errorf("x.y belongs to %q, want the declared %q", m, "other") + } +} + +// 1132/AC1: every kind accepts the field. +func TestEveryKindAcceptsAModule(t *testing.T) { + for _, typ := range []Type{TypeDirectory, TypeFile, TypeUser, TypeNetwork, TypeAccess, TypeArchive, TypeProcess, + TypeService, TypeOpening, TypePackage, TypeContainer, TypeAction} { + r := newOf(typ) + if r == nil { + t.Fatalf("no kind %s", typ) + } + if _, ok := r.(interface{ DeclaredModule() string }); !ok { + t.Errorf("%s does not say its module", typ) + } + if unknown := unknownFields([]byte(`{"id":"m.x","type":"`+string(typ)+`","module":"m"}`), r); len(unknown) != 0 { + t.Errorf("%s refuses %v", typ, unknown) + } + } +} diff --git a/go/declaration/order_test.go b/go/declaration/order_test.go new file mode 100644 index 0000000..4d2c93d --- /dev/null +++ b/go/declaration/order_test.go @@ -0,0 +1,59 @@ +package declaration + +import ( + "strings" + "testing" +) + +// A declaration says where it stands — the controller's lease epoch and its sequence — inside what is +// signed (novox/hq to-be 45 §6). Absent is no claim; below zero is no order the mesh assigns. + +func withOrder(order string) string { + return `{"declaration":1,` + order + `"resources":[{"id":"etc","type":"directory","path":"/etc/mesh","mode":"0755"}]}` +} + +func TestADeclarationCarriesItsEpochAndSequence(t *testing.T) { + d, err := Parse([]byte(withOrder(`"epoch":57,"sequence":3,`))) + if err != nil { + t.Fatalf("a declaration carrying its order was refused: %v", err) + } + if d.Epoch != 57 || d.Sequence != 3 { + t.Fatalf("read epoch %d, sequence %d", d.Epoch, d.Sequence) + } + // An older controller's: no order, and no claim. + d, err = Parse([]byte(withOrder(``))) + if err != nil || d.Epoch != 0 || d.Sequence != 0 { + t.Fatalf("a declaration with no order read as %+v, %v", d, err) + } +} + +func TestAnOrderBelowZeroIsRefused(t *testing.T) { + for _, order := range []string{`"epoch":-1,`, `"sequence":-4,`} { + refusal := refusalFor(t, withOrder(order)) + if !strings.Contains(refusal.Error(), "below zero") { + t.Errorf("%s: refused for something else: %v", order, refusal) + } + } +} + +// **The assignment generation it was composed from, and whether it is a put-back** (novox/hq issue 234): +// absent is no claim; below zero is none the mesh assigns; a carried bundle puts nothing back. +func TestADeclarationCarriesItsGenerationAndWhetherItIsAPutBack(t *testing.T) { + d, err := Parse([]byte(withOrder(`"sequence":3,"generation":41,"put_back":true,`))) + if err != nil { + t.Fatalf("a declaration carrying its generation was refused: %v", err) + } + if d.Generation != 41 || !d.PutBack { + t.Fatalf("read generation %d, put-back %v", d.Generation, d.PutBack) + } + if d, err = Parse([]byte(withOrder(``))); err != nil || d.Generation != 0 || d.PutBack { + t.Fatalf("a declaration with no generation read as %+v, %v", d, err) + } + if refusal := refusalFor(t, withOrder(`"generation":-2,`)); !strings.Contains(refusal.Error(), "below zero") { + t.Errorf("a generation below zero was refused for something else: %v", refusal) + } + if _, err := ParseTrusted([]byte(withOrder(`"generation":3,"put_back":true,`))); err == nil || + !strings.Contains(err.Error(), "put-back") { + t.Errorf("a carried bundle said it was a put-back and was not refused for it: %v", err) + } +} diff --git a/go/declaration/ownership_test.go b/go/declaration/ownership_test.go new file mode 100644 index 0000000..76b0674 --- /dev/null +++ b/go/declaration/ownership_test.go @@ -0,0 +1,130 @@ +package declaration + +import ( + "strings" + "testing" +) + +// Defends novox/hq ADR 0349 §5 (issue 667): a file the provisioner places says who owns it at runtime +// — owned, seed or managed — and the declaration names the files the provisioner fills. The words are +// read and judged here before any controller sends them. + +func TestAFileSaysWhoOwnsItAtRuntime(t *testing.T) { + d, err := Parse([]byte(`{"declaration":1,"resources":[ + {"id":"o","type":"file","path":"/etc/o","content":"a","owner-at-runtime":"owned"}, + {"id":"s","type":"file","path":"/etc/s","content":"a","owner-at-runtime":"seed"}, + {"id":"s2","type":"file","path":"/etc/s2","content":"a","owner-at-runtime":"seed","create-once":true}, + {"id":"mj","type":"file","path":"/etc/mj","into":"json","content":"{\"k\":1}","owner-at-runtime":"managed"}, + {"id":"mb","type":"file","path":"/etc/mb","into":"block","content":"a\n","owner-at-runtime":"managed"} + ]}`)) + if err != nil { + t.Fatal(err) + } + want := map[string]string{"o": RuntimeOwned, "s": RuntimeSeed, "s2": RuntimeSeed, "mj": RuntimeManaged, + "mb": RuntimeManaged} + for _, r := range d.Resources { + f := r.(*File) + if f.OwnerAtRuntime != want[f.ID] || f.Ownership() != want[f.ID] { + t.Errorf("%s: read as %q, ownership %q; want %q", f.ID, f.OwnerAtRuntime, f.Ownership(), want[f.ID]) + } + } +} + +// A file that does not say is owned by its old form, so the word means exactly what that form does. +func TestAFileThatDoesNotSayIsOwnedByItsOldForm(t *testing.T) { + d, err := Parse([]byte(`{"declaration":1,"resources":[ + {"id":"o","type":"file","path":"/etc/o","content":"a"}, + {"id":"s","type":"file","path":"/etc/s","content":"a","create-once":true}, + {"id":"mj","type":"file","path":"/etc/mj","into":"json","content":"{}"}, + {"id":"mb","type":"file","path":"/etc/mb","into":"block","content":"a\n"} + ]}`)) + if err != nil { + t.Fatal(err) + } + want := map[string]string{"o": RuntimeOwned, "s": RuntimeSeed, "mj": RuntimeManaged, "mb": RuntimeManaged} + for _, r := range d.Resources { + f := r.(*File) + if f.OwnerAtRuntime != "" || f.Ownership() != want[f.ID] { + t.Errorf("%s: said %q, ownership %q; want unsaid and %q", f.ID, f.OwnerAtRuntime, f.Ownership(), want[f.ID]) + } + if f.Seeded() != (want[f.ID] == RuntimeSeed) { + t.Errorf("%s: seeded %v", f.ID, f.Seeded()) + } + } +} + +func TestAnOwnershipThatContradictsTheFileIsRefused(t *testing.T) { + for name, c := range map[string]struct{ resource, refusal string }{ + "another word": {`{"id":"f","type":"file","path":"/etc/x","content":"a","owner-at-runtime":"mine"}`, + `owner-at-runtime "mine"`}, + "owned and create-once": {`{"id":"f","type":"file","path":"/etc/x","content":"a","owner-at-runtime":"owned","create-once":true}`, + "create-once"}, + "owned and into": {`{"id":"f","type":"file","path":"/etc/x","into":"json","content":"{}","owner-at-runtime":"owned"}`, + "into"}, + "seed and into": {`{"id":"f","type":"file","path":"/etc/x","into":"block","content":"a","owner-at-runtime":"seed"}`, + "into"}, + "managed and whole": {`{"id":"f","type":"file","path":"/etc/x","content":"a","owner-at-runtime":"managed"}`, + "into"}, + "managed and create-once": {`{"id":"f","type":"file","path":"/etc/x","into":"json","content":"{}","owner-at-runtime":"managed","create-once":true}`, + "create-once"}, + // The rule that was there before stays: a file written into is never sealed or created once. + "managed and sealed": {`{"id":"f","type":"file","path":"/etc/x","into":"json","content":"{}","sealed":"x","owner-at-runtime":"managed"}`, + "sealed"}, + } { + _, err := Parse([]byte(`{"declaration":1,"resources":[` + c.resource + `]}`)) + if err == nil || !strings.Contains(err.Error(), c.refusal) { + t.Errorf("%s: want a refusal naming %q, got %v", name, c.refusal, err) + } + } +} + +func TestTheDeclarationNamesTheFilesTheProvisionerFills(t *testing.T) { + d, err := Parse([]byte(`{"declaration":1,"filled_files":["/etc/a","/etc/b"],"resources":[ + {"id":"a","type":"file","path":"/etc/a","content":"a","owner-at-runtime":"owned"}, + {"id":"b","type":"file","path":"/etc/b","into":"block","content":"b\n","owner-at-runtime":"managed"}, + {"id":"c","type":"file","path":"/etc/c","content":"c"} + ]}`)) + if err != nil { + t.Fatal(err) + } + if len(d.Filled) != 2 || d.Filled[0] != "/etc/a" || d.Filled[1] != "/etc/b" { + t.Fatalf("filled files read as %v", d.Filled) + } + if !d.Fills("/etc/a") || !d.Fills("/etc/b") || d.Fills("/etc/c") { + t.Errorf("Fills: a %v, b %v, c %v", d.Fills("/etc/a"), d.Fills("/etc/b"), d.Fills("/etc/c")) + } + + // A declaration that names none fills none, as every declaration before this did. + d, err = Parse([]byte(`{"declaration":1,"resources":[{"id":"c","type":"file","path":"/etc/c","content":"c"}]}`)) + if err != nil { + t.Fatal(err) + } + if d.Filled != nil || d.Fills("/etc/c") { + t.Errorf("a declaration naming no filled files fills %v", d.Filled) + } +} + +func TestAFilledFileMustBeADeclaredFileThatSaysItsOwner(t *testing.T) { + for name, c := range map[string]struct{ filled, resources, refusal string }{ + "not declared": {`["/etc/none"]`, + `{"id":"a","type":"file","path":"/etc/a","content":"a","owner-at-runtime":"owned"}`, + `"/etc/none"`}, + "a directory, not a file": {`["/etc/d"]`, + `{"id":"d","type":"directory","path":"/etc/d"}`, + `"/etc/d"`}, + "says no owner": {`["/etc/a"]`, + `{"id":"a","type":"file","path":"/etc/a","content":"a"}`, + "owner-at-runtime"}, + "named twice": {`["/etc/a","/etc/a"]`, + `{"id":"a","type":"file","path":"/etc/a","content":"a","owner-at-runtime":"owned"}`, + "twice"}, + "no path": {`[""]`, + `{"id":"a","type":"file","path":"/etc/a","content":"a","owner-at-runtime":"owned"}`, + "no path"}, + } { + _, err := Parse([]byte(`{"declaration":1,"filled_files":` + c.filled + `,"resources":[` + c.resources + `]}`)) + if err == nil || !strings.Contains(err.Error(), c.refusal) { + t.Errorf("%s: want a refusal naming %q, got %v", name, c.refusal, err) + } + } +} diff --git a/go/declaration/process_test.go b/go/declaration/process_test.go new file mode 100644 index 0000000..9a7c7e9 --- /dev/null +++ b/go/declaration/process_test.go @@ -0,0 +1,163 @@ +package declaration + +import ( + "strings" + "testing" +) + +func aProcess() *Process { + return &Process{ + ID: "server", Type: TypeProcess, Name: "greeter", + Source: "https://store.invalid/greeter/daemon", + Digest: "sha256:" + strings.Repeat("a", 64), + Run: []string{"node", "index.js"}, + } +} + +// A process is part of the vocabulary, or a declaration carrying one is refused whole. +func TestAProcessIsSomethingTheHostSpeaks(t *testing.T) { + var found bool + for _, kind := range Vocabulary() { + if kind == TypeProcess { + found = true + } + } + if !found { + t.Fatal("a process cannot be declared, so a module that declares one is refused") + } + if newOf(TypeProcess) == nil { + t.Fatal("the decoder has no daemon, so one would be refused as an unknown kind") + } +} + +// **Pinned by digest, like everything else that crosses a network.** A bundle fetched by a +// reference somebody can repoint is not pinned, and it is the one thing on a machine that would +// then be running code nobody reviewed. +func TestAProcesssBundleMustBePinned(t *testing.T) { + for _, bad := range []string{"", "latest", "sha256:short", strings.Repeat("a", 64)} { + d := aProcess() + d.Digest = bad + if problems := d.validate("a process", false); len(problems) == 0 { + t.Fatalf("a process pinned by %q was accepted", bad) + } + } +} + +// What to run is named, never inferred. Guessing an entrypoint from which files are present makes +// a process change what it runs when somebody adds a file. +func TestAProcessMustSayWhatToRun(t *testing.T) { + d := aProcess() + d.Run = nil + if problems := d.validate("a process", false); len(problems) == 0 { + t.Fatal("a process with no command was accepted") + } +} + +// Its name becomes a unit name and a path, so a separator in it would write somewhere nobody meant. +func TestAProcesssNameCannotEscapeItsUnit(t *testing.T) { + // "." and ".." are one path element each, and the mesh's own bundle directory and its parent: + // removing a process named ".." would delete every bundle the mesh has, and more (novox/hq ADR + // 0118). A leading dash is an option to the service manager, not a unit. + for _, bad := range []string{"", "../escape", "two words", "a/b", ".", "..", "-", "--now"} { + d := aProcess() + d.Name = bad + if problems := d.validate("a process", false); len(problems) == 0 { + t.Fatalf("a process called %q was accepted", bad) + } + } +} + +// And a well-formed one is accepted, or the tests above prove only that everything is refused. +func TestAWellFormedDaemonIsAccepted(t *testing.T) { + if problems := aProcess().validate("a process", false); len(problems) != 0 { + t.Fatalf("a well-formed daemon was refused: %v", problems) + } +} + +// **The modes are exclusive, and saying so is the point of having one kind.** Something that runs +// once does not run on a schedule; something not running between fires cannot be restarted when a +// file changes. A container's modes carry the same rule, and this is the same rule because it is +// the same thing hosted differently. +func TestTheModesAreExclusive(t *testing.T) { + both := aProcess() + both.RunOnce = true + both.Schedule = "0 3 * * *" + if problems := both.validate("a process", false); len(problems) == 0 { + t.Fatal("a process that runs once and on a schedule was accepted") + } + + watching := aProcess() + watching.Schedule = "0 3 * * *" + watching.RestartOn = []string{"some-file"} + if problems := watching.validate("a process", false); len(problems) == 0 { + t.Fatal("a scheduled process was given something to restart on, and it is never running") + } +} + +// A cadence that is not a cadence is refused near its author, rather than by a machine at the far +// end of a declaration. +func TestAScheduleMustBeACadence(t *testing.T) { + for _, bad := range []string{"often", "0 3 * *", "99 3 * * *"} { + p := aProcess() + p.Schedule = bad + if problems := p.validate("a process", false); len(problems) == 0 { + t.Fatalf("a process scheduled %q was accepted", bad) + } + } +} + +// And each mode on its own is accepted, or the tests above prove only that everything is refused. +func TestEachModeOnItsOwnIsAccepted(t *testing.T) { + once := aProcess() + once.RunOnce = true + if problems := once.validate("a process", false); len(problems) != 0 { + t.Fatalf("a step was refused: %v", problems) + } + every := aProcess() + every.Schedule = "0 3 * * *" + if problems := every.validate("a process", false); len(problems) != 0 { + t.Fatalf("a scheduled process was refused: %v", problems) + } +} + +// **The one that cannot be escaped, only refused.** +// +// Everything else a unit file reinterprets can be escaped: a percent specifier doubled, whitespace +// quoted, a quote backslashed. A newline cannot — it ends the line, and what follows is read as a +// unit DIRECTIVE. A value carrying one could write ExecStart= and have the machine run something +// nobody declared. +// +// So it is refused here, near whoever wrote it, rather than rendered into a unit at the far end of +// a declaration. +func TestAnEnvironmentValueCannotCarryALineBreak(t *testing.T) { + for _, bad := range []string{ + "safe\nExecStart=/usr/bin/whatever", + "carriage\rreturn", + } { + p := aProcess() + p.Env = map[string]string{"X": bad} + problems := p.validate("a process", false) + if len(problems) == 0 { + t.Fatalf("a value containing %q was accepted", bad) + } + var said bool + for _, problem := range problems { + if strings.Contains(problem, "line break") { + said = true + } + } + if !said { + t.Fatalf("refused for some other reason, which would stop being true: %v", problems) + } + } +} + +// And ordinary awkward values are accepted, because escaping is what handles those — refusing them +// too would make a module unable to hold a generated password. +func TestOrdinaryAwkwardValuesAreAccepted(t *testing.T) { + p := aProcess() + p.Env = map[string]string{"PASSWORD": `a%H b"c\d`} + if problems := p.validate("a process", false); len(problems) != 0 { + t.Fatalf("a password containing the characters passwords contain was refused: %v", problems) + } +} diff --git a/go/declaration/provisioner_test.go b/go/declaration/provisioner_test.go new file mode 100644 index 0000000..233aa40 --- /dev/null +++ b/go/declaration/provisioner_test.go @@ -0,0 +1,37 @@ +package declaration + +import ( + "strings" + "testing" +) + +// Defends novox/hq ADR 0349 §4 (issue 668): the declaration names the account the provisioner's client runs as, +// the only one whose write of a filled file the node-engine accepts. Read here before any controller sends it. + +func TestTheDeclarationNamesTheProvisionersAccount(t *testing.T) { + d, err := Parse([]byte(`{"declaration":1,"provisioner_account":"mesh-node-provisioner","filled_files":["/etc/a"], + "resources":[{"id":"a","type":"file","path":"/etc/a","content":"a","owner-at-runtime":"owned"}]}`)) + if err != nil { + t.Fatal(err) + } + if d.Provisioner != "mesh-node-provisioner" { + t.Errorf("the provisioner's account read as %q", d.Provisioner) + } + d, err = Parse([]byte(`{"declaration":1,"resources":[{"id":"a","type":"file","path":"/etc/a","content":"a"}]}`)) + if err != nil { + t.Fatal(err) + } + if d.Provisioner != "" { + t.Errorf("a declaration naming no account names %q", d.Provisioner) + } +} + +func TestAProvisionersAccountThatIsNoAccountIsRefused(t *testing.T) { + for name, account := range map[string]string{"root": "root", "not a name": "a b", "a path": "../x"} { + _, err := Parse([]byte(`{"declaration":1,"provisioner_account":"` + account + `", + "resources":[{"id":"a","type":"file","path":"/etc/a","content":"a"}]}`)) + if err == nil || !strings.Contains(err.Error(), "provisioner_account") { + t.Errorf("%s: want a refusal naming provisioner_account, got %v", name, err) + } + } +} diff --git a/go/declaration/replaces_test.go b/go/declaration/replaces_test.go new file mode 100644 index 0000000..c470505 --- /dev/null +++ b/go/declaration/replaces_test.go @@ -0,0 +1,63 @@ +package declaration + +import ( + "strings" + "testing" +) + +// novox/hq issue 213: a process says what it takes the place of, so the host can keep the old one +// answering until the new one does. What it may name is narrow, and each refusal is said here. + +const aReplacingProcess = `{"id":"m.controller","type":"process","name":"m","source":"https://store.invalid/m", + "digest":"sha256:` + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + `","run":["./m","serve"]%s}` + +func replacing(extra string) string { + return `{"declaration":1,"resources":[` + strings.Replace(aReplacingProcess, "%s", extra, 1) + `]}` +} + +func TestAProcessMayNameWhatItReplaces(t *testing.T) { + d, err := Parse([]byte(replacing(`,"replaces":["m.server"]`))) + if err != nil { + t.Fatalf("a process replacing an undeclared container was refused: %v", err) + } + if p := d.Resources[0].(*Process); len(p.Replaces) != 1 || p.Replaces[0] != "m.server" { + t.Fatalf("replaces was not read: %+v", p) + } +} + +// What is replaced is what the declaration no longer says. A resource still declared is applied, +// and handing it over as well would remove something the same declaration asks to keep. +func TestAProcessMayNotReplaceSomethingStillDeclared(t *testing.T) { + raw := `{"declaration":1,"resources":[ + {"id":"m.server","type":"directory","path":"/tmp/x"}, + ` + strings.Replace(aReplacingProcess, "%s", `,"replaces":["m.server"]`, 1) + `]}` + if _, err := Parse([]byte(raw)); err == nil || !strings.Contains(err.Error(), "still declares") { + t.Fatalf("a process replacing a declared resource was accepted: %v", err) + } +} + +func TestOnlyAProcessThatStaysUpReplacesAnything(t *testing.T) { + for _, mode := range []string{`,"run-once":true`, `,"schedule":"0 3 * * *"`} { + if _, err := Parse([]byte(replacing(mode + `,"replaces":["m.server"]`))); err == nil { + t.Errorf("a process with %s was allowed to replace something", mode) + } + } +} + +func TestAProcessCannotReplaceItselfOrNothing(t *testing.T) { + for _, bad := range []string{`,"replaces":["m.controller"]`, `,"replaces":[""]`} { + if _, err := Parse([]byte(replacing(bad))); err == nil { + t.Errorf("replaces %s was accepted", bad) + } + } +} + +func TestOneThingHasOneReplacement(t *testing.T) { + second := strings.NewReplacer(`"id":"m.controller"`, `"id":"m.other"`, `"name":"m"`, `"name":"other"`). + Replace(strings.Replace(aReplacingProcess, "%s", `,"replaces":["m.server"]`, 1)) + raw := `{"declaration":1,"resources":[` + + strings.Replace(aReplacingProcess, "%s", `,"replaces":["m.server"]`, 1) + "," + second + `]}` + if _, err := Parse([]byte(raw)); err == nil { + t.Fatal("two processes replacing one resource were accepted") + } +} diff --git a/go/declaration/root_test.go b/go/declaration/root_test.go new file mode 100644 index 0000000..cd2908c --- /dev/null +++ b/go/declaration/root_test.go @@ -0,0 +1,26 @@ +package declaration + +import ( + "strings" + "testing" +) + +// A user's root (novox/hq ADR 0266): "never" or absent, and nothing else. +func TestAUsersRootIsNeverOrAbsent(t *testing.T) { + for _, c := range []struct { + root string + ok bool + }{{`,"root":"never"`, true}, {``, true}, {`,"root":"always"`, false}, {`,"root":"no"`, false}} { + d, err := Parse([]byte(`{"declaration":1,"resources":[{"id":"claude-code.agent","type":"user","name":"agent"` + + c.root + `}]}`)) + if c.ok != (err == nil) { + t.Errorf("%s: err %v", c.root, err) + } + if err != nil && !strings.Contains(err.Error(), `"never"`) { + t.Errorf("%s: the refusal does not name the allowed value: %v", c.root, err) + } + if c.ok && c.root != "" && d.Resources[0].(*User).Root != RootNever { + t.Errorf("%s: read as %+v", c.root, d.Resources[0]) + } + } +} diff --git a/go/declaration/schedule_declaration_test.go b/go/declaration/schedule_declaration_test.go new file mode 100644 index 0000000..d87d1fb --- /dev/null +++ b/go/declaration/schedule_declaration_test.go @@ -0,0 +1,53 @@ +package declaration + +import ( + "strings" + "testing" +) + +// The host refuses a schedule it does not fully understand, on arrival, for the same reason the +// control plane refuses it near its author (novox/hq ADR 0053): a declaration half-understood is +// refused whole rather than half-applied. + +func schedulePinned() string { return "registry.example/runtime@sha256:" + strings.Repeat("a", 64) } + +func TestAScheduledContainerCarriesTheCronField(t *testing.T) { + d, err := Parse([]byte(`{"declaration":1,"resources":[ + {"id":"sync","type":"container","name":"sync","image":"` + schedulePinned() + `","schedule":"0 3 * * *"} + ]}`)) + if err != nil { + t.Fatalf("a valid scheduled container was refused: %v", err) + } + c, ok := d.Resources[0].(*Container) + if !ok { + t.Fatalf("the scheduled resource is not a container: %T", d.Resources[0]) + } + if c.Schedule != "0 3 * * *" { + t.Errorf("the schedule did not survive parsing: %q", c.Schedule) + } +} + +func TestAMalformedScheduleIsRefused(t *testing.T) { + _, err := Parse([]byte(`{"declaration":1,"resources":[ + {"id":"sync","type":"container","name":"sync","image":"` + schedulePinned() + `","schedule":"every night"} + ]}`)) + if err == nil { + t.Fatal("a container with a malformed schedule was accepted") + } + if !strings.Contains(err.Error(), "cron") && !strings.Contains(err.Error(), "field") { + t.Errorf("refused for the wrong reason: %v", err) + } +} + +func TestAContainerCannotBeBothRunOnceAndScheduled(t *testing.T) { + // A container runs once and gates, or on a cadence, or stays up — never two (novox/hq ADR 0053). + _, err := Parse([]byte(`{"declaration":1,"resources":[ + {"id":"sync","type":"container","name":"sync","image":"` + schedulePinned() + `","run-once":true,"schedule":"0 3 * * *"} + ]}`)) + if err == nil { + t.Fatal("a container that was both run-once and scheduled was accepted") + } + if !strings.Contains(err.Error(), "run-once") && !strings.Contains(err.Error(), "scheduled") { + t.Errorf("refused for the wrong reason: %v", err) + } +} diff --git a/go/declaration/stateless_test.go b/go/declaration/stateless_test.go new file mode 100644 index 0000000..3338908 --- /dev/null +++ b/go/declaration/stateless_test.go @@ -0,0 +1,30 @@ +package declaration + +import ( + "strings" + "testing" +) + +// Defends novox/hq ADR 0117: a service may leave its unit's lifecycle to the machine, and then +// says nothing but its triggers. +func TestAServiceWithoutAStateSaysOnlyItsTriggers(t *testing.T) { + for name, c := range map[string]struct{ resource, refusal string }{ + "no trigger": {`{"id":"s","type":"service","unit":"NetworkManager.service"}`, "declares nothing"}, + "with boot": {`{"id":"s","type":"service","unit":"NetworkManager.service","boot":"enabled","reload-on":["f"]}`, "boot and takes-over"}, + "with takes-over": {`{"id":"s","type":"service","unit":"a.service","reload-on":["f"],"takes-over":{"interface":"wg0","unit":"b.service","config":"/etc/x"}}`, "boot and takes-over"}, + "an unknown state": {`{"id":"s","type":"service","unit":"a.service","state":"paused"}`, "omits state to leave the unit's lifecycle to the machine"}, + } { + _, err := Parse([]byte(`{"declaration":1,"resources":[{"id":"f","type":"file","path":"/etc/x","content":"x"},` + c.resource + `]}`)) + if err == nil || !strings.Contains(err.Error(), c.refusal) { + t.Errorf("%s: want a refusal naming %q, got %v", name, c.refusal, err) + } + } + d, err := Parse([]byte(`{"declaration":1,"resources":[{"id":"f","type":"file","path":"/etc/x","content":"x"}, + {"id":"s","type":"service","unit":"NetworkManager.service","restart-on":["f"]}]}`)) + if err != nil { + t.Fatal(err) + } + if s := d.Resources[1].(*Service); !s.Stateless() { + t.Error("a service without a state was not read as stateless") + } +} diff --git a/go/declaration/trusted_test.go b/go/declaration/trusted_test.go new file mode 100644 index 0000000..c533334 --- /dev/null +++ b/go/declaration/trusted_test.go @@ -0,0 +1,23 @@ +package declaration + +import "testing" + +// `trusted` is the catalogue's word on a file (novox/hq issue 339): a controller takes it out before it sends a +// declaration, and one that does not — an older one, or one rolled back to — must not cost the node its whole +// declaration. So a file accepts it and the engine does nothing with it; any other resource still refuses it. +func TestAFileAcceptsAndIgnoresTrusted(t *testing.T) { + for _, v := range []string{"true", "false"} { + d, err := Parse([]byte(`{"declaration":1,"resources":[{"id":"m.f","type":"file","path":"/etc/m.conf",` + + `"content":"x","trusted":` + v + `}]}`)) + if err != nil { + t.Fatalf("trusted %s: %v", v, err) + } + if f, ok := d.Resources[0].(*File); !ok || f.Content != "x" { + t.Fatalf("trusted %s: %+v", v, d.Resources[0]) + } + } + if _, err := Parse([]byte(`{"declaration":1,"resources":[{"id":"m.d","type":"directory","path":"/srv/m",` + + `"trusted":true}]}`)); err == nil { + t.Fatal("a directory took trusted") + } +} diff --git a/go/declaration/userscope_test.go b/go/declaration/userscope_test.go new file mode 100644 index 0000000..fdce635 --- /dev/null +++ b/go/declaration/userscope_test.go @@ -0,0 +1,49 @@ +package declaration + +import ( + "strings" + "testing" +) + +// novox/hq ADR 0177: a unit is in the system manager or an account's own; a user-scoped one names +// the account, a system one may not, and any other word is refused. +func TestAUserScopedUnitNamesItsAccountAndASystemOneMayNot(t *testing.T) { + cases := []struct{ scope, user, wants string }{ + {"user", "ops", ""}, + {"", "", ""}, + {"system", "", ""}, + {"user", "", "names the account"}, + {"", "ops", "names no user"}, + {"session", "ops", "scope \"session\""}, + } + for _, c := range cases { + s := &Service{ID: "m.u", Type: TypeService, Unit: "u.service", State: "running", Scope: c.scope, User: c.user} + problems := s.validate("m.u", false) + got := strings.Join(problems, "; ") + if c.wants == "" && len(problems) != 0 { + t.Fatalf("scope %q user %q refused: %s", c.scope, c.user, got) + } + if c.wants != "" && !strings.Contains(got, c.wants) { + t.Fatalf("scope %q user %q: wanted a refusal saying %q, got %q", c.scope, c.user, c.wants, got) + } + } +} + +// novox/hq ADR 0177: lingering is a field of the account, absent meaning nothing asserted. +func TestAnAccountMayBeDeclaredToLinger(t *testing.T) { + d, err := Parse([]byte(`{"declaration":1,"resources":[{"id":"m.login","type":"user","name":"ops","linger":true}]}`)) + if err != nil { + t.Fatal(err) + } + u, ok := d.Resources[0].(*User) + if !ok || u.Linger == nil || !*u.Linger { + t.Fatalf("linger not read: %+v", d.Resources[0]) + } + d, err = Parse([]byte(`{"declaration":1,"resources":[{"id":"m.login","type":"user","name":"ops"}]}`)) + if err != nil { + t.Fatal(err) + } + if u := d.Resources[0].(*User); u.Linger != nil { + t.Fatal("an account that said nothing about lingering asserts it") + } +} -- 2.54.0 From 1d6a0e892ec7108cf3433911bf1b1f8860032851 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 06:02:17 +0000 Subject: [PATCH 10/41] Name the source commit in the declaration package's doc (issue 1353) --- go/declaration/declaration.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/go/declaration/declaration.go b/go/declaration/declaration.go index 1fd3c5b..ea16b91 100644 --- a/go/declaration/declaration.go +++ b/go/declaration/declaration.go @@ -4,6 +4,9 @@ // refuses the whole declaration rather than being skipped — a host that applied most of what // it was sent and reported success is a node that looks configured and is not // (novox/hq ADR 0005). +// +// Copied from mesh-host internal/declaration at commit 619e14aa8e572c5056e3d2a2412c7f8b364bf840, files unchanged but +// for this note (novox/hq ADR 0453 clause 3, ADR 0446 clause 4). mesh-host keeps its own copy until release 4b. package declaration import ( -- 2.54.0 From be506bb502d0eb0240a18b014afdd53b98d50340 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 06:02:28 +0000 Subject: [PATCH 11/41] Copy the node-engine's validator into mesh-sdk, on the SDK's parser (issue 1353) --- go/validate/sdk_test.go | 31 ++++++++++++++++++++++ go/validate/validate.go | 42 ++++++++++++++++++++++++++++++ go/validate/validate_test.go | 50 ++++++++++++++++++++++++++++++++++++ 3 files changed, 123 insertions(+) create mode 100644 go/validate/sdk_test.go create mode 100644 go/validate/validate.go create mode 100644 go/validate/validate_test.go diff --git a/go/validate/sdk_test.go b/go/validate/sdk_test.go new file mode 100644 index 0000000..d946fff --- /dev/null +++ b/go/validate/sdk_test.go @@ -0,0 +1,31 @@ +package validate + +import ( + "errors" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/declaration" +) + +// The SDK's validator speaks the copied parser's vocabulary version (issue 1353 AC3). +func TestVersionIsTheParsersVersion(t *testing.T) { + if Version != declaration.Version { + t.Fatalf("validate.Version = %d, declaration.Version = %d", Version, declaration.Version) + } +} + +// A body of another declaration version is refused whole, by both the validator and the parser (AC3). The body is +// the shape mesh-host's validate_test.go uses for its good declaration, with the version changed. +func TestAnotherVersionIsRefusedWhole(t *testing.T) { + body := `{"declaration": 99, "resources": [{"id": "d", "type": "directory", "path": "/var/lib/x", "mode": "0755"}]}` + problems := Declaration([]byte(body)) + if len(problems) == 0 { + t.Fatal("a declaration of another version was taken") + } + _, err := declaration.Parse([]byte(body)) + var refused *declaration.RefusalError + if !errors.As(err, &refused) || strings.Join(refused.Problems, "\n") != strings.Join(problems, "\n") { + t.Fatalf("the parser and the validator disagree: %v / %v", err, problems) + } +} diff --git a/go/validate/validate.go b/go/validate/validate.go new file mode 100644 index 0000000..1291866 --- /dev/null +++ b/go/validate/validate.go @@ -0,0 +1,42 @@ +// Package validate is the node-engine's own judgement of a declaration, for whoever must know before +// a declaration is sent whether a machine would take it (novox/hq to-be 45 §4, D1; §9, the merge gate). +// +// **One validator.** The controller composed declarations the host then refused whole — a manifest +// the catalogue check passed (novox/hq issue 236), a consumer's identity too long for the machine's +// names (issue 263) — because the only validator was the one the host runs as it applies. A second, +// written in the controller from the host's rules, would drift from them the first time either +// changed. So the host's own parsing is exported here, unchanged: what the controller's self-check and +// the merge gate run is what every machine runs. +// +// Copied from mesh-host at commit 619e14aa8e572c5056e3d2a2412c7f8b364bf840 (validate/validate.go), unchanged but for +// the import path of the parser, which is mesh-sdk's go/declaration (novox/hq ADR 0453 clause 3, ADR 0446 clause 4). +// mesh-host keeps its own copy until release 4b. +// +// It judges a declaration as it arrives over the link: actions are refused, as the host refuses them +// from the link (novox/hq ADR 0005). Nothing here touches a machine. +package validate + +import ( + "errors" + + "git.novox.be/novox/mesh-sdk/go/declaration" +) + +// Declaration is every problem the node-engine would refuse a declaration for, each in its own words; +// nil when it would take it. The body is the declaration as the controller composes it — the bytes a +// signature is made over — not the signed envelope. +func Declaration(raw []byte) []string { + _, err := declaration.Parse(raw) + if err == nil { + return nil + } + var refused *declaration.RefusalError + if errors.As(err, &refused) { + return refused.Problems + } + return []string{err.Error()} +} + +// Version is the declaration vocabulary this validator speaks: a declaration of another version is +// refused whole by Declaration, as by the host. +const Version = declaration.Version diff --git a/go/validate/validate_test.go b/go/validate/validate_test.go new file mode 100644 index 0000000..51ade73 --- /dev/null +++ b/go/validate/validate_test.go @@ -0,0 +1,50 @@ +package validate + +import ( + "strings" + "testing" +) + +// **The validator a controller imports is the host's**: a declaration the host takes passes, and one +// it refuses is refused with the host's own words, every problem at once. +func TestTheValidatorIsTheHosts(t *testing.T) { + good := `{"declaration": 1, "resources": [{"id": "d", "type": "directory", "path": "/var/lib/x", "mode": "0755"}]}` + if problems := Declaration([]byte(good)); problems != nil { + t.Fatalf("a declaration the host takes was refused: %v", problems) + } + // And one carrying the lease epoch and sequence the controller sends it under (novox/hq to-be 45 + // §6): a controller validating with this package may send them. + ordered := `{"declaration": 1, "epoch": 57, "sequence": 3, "resources": [{"id": "d", "type": "directory", "path": "/var/lib/x", "mode": "0755"}]}` + if problems := Declaration([]byte(ordered)); problems != nil { + t.Fatalf("a declaration carrying its order was refused: %v", problems) + } + // And one carrying a file's owner-at-runtime and the files the provisioner fills (novox/hq ADR 0349 §5): the + // reader takes them before any controller sends them. + owned := `{"declaration": 1, "filled_files": ["/etc/x"], "resources": [{"id": "f", "type": "file", "path": "/etc/x", "content": "a", "owner-at-runtime": "owned"}]}` + if problems := Declaration([]byte(owned)); problems != nil { + t.Fatalf("a declaration saying who owns a file at runtime was refused: %v", problems) + } + // And one naming the provisioner client's account (novox/hq ADR 0349 §4). + named := `{"declaration": 1, "provisioner_account": "mesh-node-provisioner", "filled_files": ["/etc/x"], "resources": [{"id": "f", "type": "file", "path": "/etc/x", "content": "a", "owner-at-runtime": "owned"}]}` + if problems := Declaration([]byte(named)); problems != nil { + t.Fatalf("a declaration naming the provisioner client's account was refused: %v", problems) + } + for _, bad := range []string{ + `{"declaration": 1, "provisioner_account": "root", "resources": [{"id": "f", "type": "file", "path": "/etc/x", "content": "a"}]}`, + `{"declaration": 1, "filled_files": ["/etc/none"], "resources": [{"id": "f", "type": "file", "path": "/etc/x", "content": "a", "owner-at-runtime": "owned"}]}`, + `{"declaration": 1, "resources": []}`, + `{"declaration": 99, "resources": [{"id": "d", "type": "directory", "path": "/x"}]}`, + `{"declaration": 1, "resources": [{"id": "d", "type": "nothing-the-host-knows"}]}`, + `not json`, + } { + problems := Declaration([]byte(bad)) + if len(problems) == 0 { + t.Errorf("a declaration the host refuses was passed: %s", bad) + } + for _, p := range problems { + if strings.TrimSpace(p) == "" { + t.Errorf("a problem said nothing, for %s", bad) + } + } + } +} -- 2.54.0 From aa70feb9b52c3c2b3183eac8a6687566efc152b3 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 06:02:43 +0000 Subject: [PATCH 12/41] Test a 21 KB manifest round trip byte for byte, on mailu's manifest copied from mesh-catalog (issue 1357) --- go/delivery/manifests_mailu_test.go | 30 + go/delivery/testdata/manifest-mailu.json | 772 +++++++++++++++++++++++ 2 files changed, 802 insertions(+) create mode 100644 go/delivery/manifests_mailu_test.go create mode 100644 go/delivery/testdata/manifest-mailu.json diff --git a/go/delivery/manifests_mailu_test.go b/go/delivery/manifests_mailu_test.go new file mode 100644 index 0000000..a4c9f44 --- /dev/null +++ b/go/delivery/manifests_mailu_test.go @@ -0,0 +1,30 @@ +package delivery + +import ( + "os" + "testing" +) + +// 1357/AC2: a manifests value of 21 KB encodes and decodes with its manifest byte for byte. The manifest is a copy of +// mesh-catalog modules/mailu/module.json at origin/main c0880c4dc27194cbb99955579a97f3212c5be93e (20,713 bytes), the +// largest manifest of the catalogue. +func TestMailuManifestRoundTripBytesForByte(t *testing.T) { + raw, err := os.ReadFile("testdata/manifest-mailu.json") + if err != nil { + t.Fatal(err) + } + if len(raw) != 20713 { + t.Fatalf("the fixture is %d bytes, the catalogue's is 20713", len(raw)) + } + data, err := EncodeManifest(Manifest{Module: "mailu", JSON: string(raw)}) + if err != nil { + t.Fatal(err) + } + got, err := DecodeManifest(data) + if err != nil { + t.Fatal(err) + } + if got.JSON != string(raw) || ManifestDigest([]byte(got.JSON)) != ManifestDigest(raw) || got.Module != "mailu" { + t.Errorf("the manifest changed on the way: %d bytes", len(got.JSON)) + } +} diff --git a/go/delivery/testdata/manifest-mailu.json b/go/delivery/testdata/manifest-mailu.json new file mode 100644 index 0000000..568be14 --- /dev/null +++ b/go/delivery/testdata/manifest-mailu.json @@ -0,0 +1,772 @@ +{ + "module": "mailu", + "version": "1", + "upgrade": { + "policy": "record", + "why": "people's mail: any change to how its containers are declared recreates them together in one send, and the mail is down for everyone until they are up again — a person chooses the moment (hq ADR 0242, issue 295)" + }, + "unhealthy": { + "severity": "urgent", + "why": "while mailu is broken, mail is refused or stuck for everyone with a mailbox here (hq ADR 0355, issue 687)" + }, + "capabilities": [ + "container-runtime" + ], + "requires": [ + "postgres-database", + "route", + "secret" + ], + "contributes": { + "postgres-database": { + "name": "mailu" + }, + "route": { + "web": { + "label": "mail", + "endpoint": "web-tls", + "scheme": "https", + "insecure": true + }, + "acme": { + "label": "mail", + "path": "/.well-known/acme-challenge", + "endpoint": "web", + "priority": 100 + }, + "autoconfig": { + "label": "autoconfig", + "endpoint": "autoconfig" + }, + "autodiscover": { + "label": "autodiscover", + "endpoint": "autoconfig" + }, + "automx": { + "label": "automx", + "endpoint": "autoconfig" + } + } + }, + "binds": { + "postgres-database": "${dir:state}/database.json", + "route": "${dir:state}/route.json" + }, + "secrets": { + "postgres-database": "${dir:state}/database.secret", + "secret": { + "secret-key": "${dir:state}/secret-key.secret", + "admin": "${dir:state}/admin.secret", + "api-token": "${dir:state}/api-token.secret" + } + }, + "tools": [ + "mailu_list_users", + "mailu_create_user", + "mailu_change_password", + "mailu_delete_user", + "mailu_list_aliases", + "mailu_create_alias", + "mailu_delete_alias", + "mailu_list_domains", + "mailu_read_mail", + "mailu_search_mail", + "mailu_health" + ], + "emits": [ + "user.created", + "user.deleted", + "alias.created", + "alias.deleted" + ], + "listens": [ + { + "name": "smtp", + "port": 25, + "protocol": "tcp", + "from": "anywhere", + "why": "mail from other mail servers", + "fixed": true + }, + { + "name": "pop3", + "port": 110, + "protocol": "tcp", + "from": "anywhere", + "why": "POP3, kept at parity with the predecessor; pruning legacy protocols is its own deliberate change", + "fixed": true + }, + { + "name": "imap", + "port": 143, + "protocol": "tcp", + "from": "anywhere", + "why": "IMAP with STARTTLS, kept at parity", + "fixed": true + }, + { + "name": "smtps", + "port": 465, + "protocol": "tcp", + "from": "anywhere", + "why": "submission over TLS", + "fixed": true + }, + { + "name": "submission", + "port": 587, + "protocol": "tcp", + "from": "anywhere", + "why": "submission; also what the smtp provision serves consumers", + "fixed": true + }, + { + "name": "imaps", + "port": 993, + "protocol": "tcp", + "from": "anywhere", + "why": "IMAP over TLS", + "fixed": true + }, + { + "name": "pop3s", + "port": 995, + "protocol": "tcp", + "from": "anywhere", + "why": "POP3 over TLS, kept at parity", + "fixed": true + }, + { + "name": "web", + "port": 7080, + "protocol": "tcp", + "from": "mesh", + "why": "the web front over http; only the ACME HTTP-01 passthrough is routed here — everything else 301s to https and would loop a proxy" + }, + { + "name": "web-tls", + "port": 7443, + "protocol": "tcp", + "from": "mesh", + "why": "the web front over its own TLS (admin, webmail, API); its public name is a route grant reaching it here" + }, + { + "name": "autoconfig", + "port": 4243, + "protocol": "tcp", + "from": "mesh", + "why": "automx: mail client autoconfiguration; the autoconfig, autodiscover and automx names are route grants reaching it here" + }, + { + "name": "admin-api", + "port": 8080, + "protocol": "tcp", + "from": "machine", + "why": "the admin API, which this module's own code reaches on loopback from the node's runtime now that it runs outside the mailu network" + } + ], + "data": { + "own": [ + { + "id": "mail", + "path": "${dir:data-mail}", + "class": "valuable", + "why": "every mailbox" + }, + { + "id": "dkim", + "path": "${dir:data-dkim}", + "class": "valuable", + "why": "the domain's signing keys; a new one means a new DNS record" + }, + { + "id": "data", + "path": "${dir:data-data}", + "class": "valuable", + "why": "the server's own data" + }, + { + "id": "dav", + "path": "${dir:data-dav}", + "class": "valuable", + "why": "calendars and contacts" + }, + { + "id": "webmail", + "path": "${dir:data-webmail}", + "class": "valuable", + "why": "the webmail's own data: its users' settings and address books" + }, + { + "id": "queue", + "path": "${dir:data-mailqueue}", + "class": "valuable", + "why": "mail accepted and not yet delivered, kept nowhere else" + }, + { + "id": "filter", + "path": "${dir:data-filter}", + "class": "rebuildable", + "why": "the spam filter's learned statistics; it learns again" + }, + { + "id": "certs", + "path": "${dir:data-certs}", + "class": "rebuildable", + "why": "certificates, issued again" + }, + { + "id": "automx", + "path": "${dir:data-automx}", + "class": "rebuildable", + "why": "client autoconfiguration, written again" + }, + { + "id": "fetchmail", + "path": "${dir:data-fetchmail}", + "class": "rebuildable", + "why": "which remote messages were fetched; lost, some are fetched twice" + }, + { + "id": "clamav", + "path": "${dir:data-clamav}", + "class": "cache", + "why": "virus signatures, downloaded again" + }, + { + "id": "redis", + "path": "${dir:data-redis}", + "class": "cache", + "why": "the filter's working set" + } + ], + "consumers": { + "smtp": { + "class": "valuable", + "in": "mail", + "why": "a consumer's mailbox holds the only copy of its mail" + } + } + }, + "config": { + "domain": { + "type": "string", + "description": "The mail domain Mailu serves, such as example.org: its first mail domain, the postmaster's and the one consumers send as; the operator's, with no default (novox/hq ADR 0112).", + "trusted": true, + "required": true + }, + "proxy-address": { + "type": "string", + "description": "The address of the reverse proxy in front of Mailu, whose X-Real-IP header Mailu believes; the operator's, with no default.", + "trusted": true, + "required": true + }, + "sitename": { + "type": "string", + "description": "The name Mailu's web pages show for the mail service; the operator's, with no default.", + "trusted": true, + "required": true + }, + "website": { + "type": "string", + "description": "The address of the website Mailu's web pages link to, as a URL; the operator's, with no default.", + "trusted": true, + "required": true + }, + "url": { + "type": "string", + "description": "The address the module's tools reach Mailu's API at, as a URL; empty leaves them to MESH_MAILU_URL.", + "trusted": true, + "default": "" + }, + "container": { + "type": "string", + "description": "The name of the container holding Mailu's mailboxes, which the module's tools read mail in; empty is mailu-imap.", + "trusted": true, + "default": "" + } + }, + "resources": [ + { + "id": "mesh-state", + "type": "directory", + "mode": "0700", + "place": "mesh", + "unknown": "program" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "place": ".", + "unknown": "program" + }, + { + "id": "grants", + "type": "directory", + "mode": "0700", + "unknown": "program" + }, + { + "id": "data-automx", + "type": "directory", + "mode": "0700", + "unknown": "program" + }, + { + "id": "config-env", + "type": "file", + "path": "${dir:state}/mailu.env", + "mode": "0644", + "template": "files/mailu.env" + }, + { + "id": "secret-env", + "type": "file", + "path": "${dir:state}/secret.env", + "mode": "0600", + "template": "files/secret.env" + }, + { + "id": "database-env", + "type": "file", + "path": "${dir:state}/database.env", + "mode": "0600", + "template": "files/database.env" + }, + { + "id": "admin-env", + "type": "file", + "path": "${dir:state}/admin.env", + "mode": "0600", + "template": "files/admin.env" + }, + { + "id": "data-certs", + "type": "directory", + "mode": "0700", + "unknown": "program" + }, + { + "id": "data-data", + "type": "directory", + "mode": "0700", + "unknown": "program" + }, + { + "id": "data-dkim", + "type": "directory", + "mode": "0700", + "unknown": "program" + }, + { + "id": "data-mail", + "type": "directory", + "mode": "0700", + "unknown": "program" + }, + { + "id": "data-mailqueue", + "type": "directory", + "mode": "0755", + "unknown": "program" + }, + { + "id": "data-filter", + "type": "directory", + "mode": "0700", + "unknown": "program" + }, + { + "id": "data-clamav", + "type": "directory", + "mode": "0700", + "unknown": "program" + }, + { + "id": "data-redis", + "type": "directory", + "mode": "0700", + "unknown": "program" + }, + { + "id": "data-webmail", + "type": "directory", + "mode": "0700", + "unknown": "program" + }, + { + "id": "data-dav", + "type": "directory", + "mode": "0700", + "unknown": "program" + }, + { + "id": "data-fetchmail", + "type": "directory", + "mode": "0700", + "unknown": "program" + }, + { + "id": "data-overrides-nginx", + "type": "directory", + "mode": "0700", + "unknown": "program" + }, + { + "id": "data-overrides-dovecot", + "type": "directory", + "mode": "0700", + "unknown": "program" + }, + { + "id": "data-overrides-postfix", + "type": "directory", + "mode": "0700", + "unknown": "program" + }, + { + "id": "data-overrides-rspamd", + "type": "directory", + "mode": "0700", + "unknown": "program" + }, + { + "id": "data-overrides-roundcube", + "type": "directory", + "mode": "0700", + "unknown": "program" + }, + { + "id": "net", + "type": "network", + "name": "mailu" + }, + { + "id": "resolver", + "type": "container", + "name": "mailu-resolver", + "image": "ghcr.io/mailu/unbound@sha256:2c037bc5c988b5c0ab7e1b6e6ffa762e14ba1e45c96281ac87f8b9be8eee8b6e", + "health": { + "kind": "runtime" + }, + "network": "mailu", + "env-file": [ + "${dir:state}/mailu.env", + "${dir:state}/secret.env" + ], + "secrets-in-environment": "mailu-admin honours SECRET_KEY_FILE, DB_PW_FILE and API_TOKEN_FILE (configuration.py) but INITIAL_ADMIN_PW is env-only (start.py); the remaining containers' need for SECRET_KEY is unverified", + "ip": "192.168.203.254" + }, + { + "id": "redis", + "type": "container", + "name": "mailu-redis", + "image": "redis@sha256:4bed291aa5efb9f0d77b76ff7d4ab71eee410962965d052552db1fb80576431d", + "network": "mailu", + "volumes": [ + "${dir:data-redis}:/data" + ] + }, + { + "id": "admin", + "type": "container", + "name": "mailu-admin", + "image": "ghcr.io/mailu/admin@sha256:6dbfdadc4a9590dcb7652357b505200115b689b74008653bbf369e4599a3be5a", + "health": { + "kind": "runtime" + }, + "network": "mailu", + "ports": [ + "8080" + ], + "env-file": [ + "${dir:state}/mailu.env", + "${dir:state}/secret.env", + "${dir:state}/database.env", + "${dir:state}/admin.env" + ], + "volumes": [ + "${dir:data-data}:/data", + "${dir:data-dkim}:/dkim" + ], + "secrets-in-environment": "mailu-admin honours SECRET_KEY_FILE, DB_PW_FILE and API_TOKEN_FILE (configuration.py) but INITIAL_ADMIN_PW is env-only (start.py); the remaining containers' need for SECRET_KEY is unverified" + }, + { + "id": "imap", + "type": "container", + "name": "mailu-imap", + "image": "ghcr.io/mailu/dovecot@sha256:7f0ed5db996fbdc00adc5c5e38a08492e04f7eb4a9fbd66a03aa9a28ddf23993", + "health": { + "kind": "runtime" + }, + "network": "mailu", + "env-file": [ + "${dir:state}/mailu.env" + ], + "volumes": [ + "${dir:data-mail}:/mail", + "${dir:data-overrides-dovecot}:/overrides:ro" + ], + "dns": [ + "192.168.203.254" + ] + }, + { + "id": "smtp", + "type": "container", + "name": "mailu-smtp", + "image": "ghcr.io/mailu/postfix@sha256:e2e49f39e53b80eac9e7a2f18d9df11edeb4914fd62dbba89b3155e8e034f62e", + "health": { + "kind": "tool", + "tool": "mailu_health", + "interval": "60s", + "timeout": "30s", + "looks": 3, + "grace": "120s" + }, + "network": "mailu", + "env-file": [ + "${dir:state}/mailu.env" + ], + "volumes": [ + "${dir:data-mailqueue}:/queue", + "${dir:data-overrides-postfix}:/overrides:ro" + ], + "dns": [ + "192.168.203.254" + ] + }, + { + "id": "antispam", + "type": "container", + "name": "mailu-antispam", + "image": "ghcr.io/mailu/rspamd@sha256:ff3666d8a61f17d309c5c6f6bcf4d40470b82299ca706ac650301175bb1a079d", + "health": { + "kind": "runtime" + }, + "network": "mailu", + "env-file": [ + "${dir:state}/mailu.env" + ], + "volumes": [ + "${dir:data-filter}:/var/lib/rspamd", + "${dir:data-overrides-rspamd}:/etc/rspamd/override.d:ro" + ], + "dns": [ + "192.168.203.254" + ] + }, + { + "id": "antivirus", + "type": "container", + "name": "mailu-antivirus", + "image": "clamav/clamav-debian@sha256:b12ef8fefddbba7d88de59bea8a32622f365339154adf02d38fd089112e6745a", + "network": "mailu", + "volumes": [ + "${dir:data-clamav}:/var/lib/clamav" + ], + "dns": [ + "192.168.203.254" + ] + }, + { + "id": "webmail", + "type": "container", + "name": "mailu-webmail", + "image": "ghcr.io/mailu/webmail@sha256:bdbee44cdb05a4658f0e3b62cc448de55ca8f8aea172279fda594826144c04f6", + "health": { + "kind": "runtime" + }, + "network": "mailu", + "env-file": [ + "${dir:state}/mailu.env", + "${dir:state}/secret.env" + ], + "volumes": [ + "${dir:data-webmail}:/data", + "${dir:data-overrides-roundcube}:/overrides:ro" + ], + "secrets-in-environment": "mailu-admin honours SECRET_KEY_FILE, DB_PW_FILE and API_TOKEN_FILE (configuration.py) but INITIAL_ADMIN_PW is env-only (start.py); the remaining containers' need for SECRET_KEY is unverified", + "dns": [ + "192.168.203.254" + ] + }, + { + "id": "webdav", + "type": "container", + "name": "mailu-webdav", + "image": "ghcr.io/mailu/radicale@sha256:690ed6edf189dfef100a5a8b37c195ebf5d9241ac5f23f2f44b8b7b75726e3de", + "health": { + "kind": "runtime" + }, + "network": "mailu", + "env-file": [ + "${dir:state}/mailu.env", + "${dir:state}/secret.env" + ], + "volumes": [ + "${dir:data-dav}:/data" + ], + "secrets-in-environment": "mailu-admin honours SECRET_KEY_FILE, DB_PW_FILE and API_TOKEN_FILE (configuration.py) but INITIAL_ADMIN_PW is env-only (start.py); the remaining containers' need for SECRET_KEY is unverified", + "dns": [ + "192.168.203.254" + ] + }, + { + "id": "fetchmail", + "type": "container", + "name": "mailu-fetchmail", + "image": "ghcr.io/mailu/fetchmail@sha256:f881c8412d3bbe73d638469b48321558d6403a9d45bfa043c1e52c752103d42d", + "health": { + "kind": "runtime" + }, + "network": "mailu", + "env-file": [ + "${dir:state}/mailu.env", + "${dir:state}/secret.env" + ], + "volumes": [ + "${dir:data-fetchmail}:/data" + ], + "secrets-in-environment": "mailu-admin honours SECRET_KEY_FILE, DB_PW_FILE and API_TOKEN_FILE (configuration.py) but INITIAL_ADMIN_PW is env-only (start.py); the remaining containers' need for SECRET_KEY is unverified", + "dns": [ + "192.168.203.254" + ] + }, + { + "id": "front", + "type": "container", + "name": "mailu-front", + "image": "ghcr.io/mailu/nginx@sha256:36f98897cd1bc9d27628bbb4e04bdf60147af2ec7507d6da77f002c4f256896d", + "health": { + "kind": "exec", + "command": "curl -m3 -skfLo /dev/null http://127.0.0.1:10204/health && kill -0 $(cat /run/dovecot/master.pid) && python3 -c \"import smtplib; s = smtplib.SMTP('127.0.0.1', 25, timeout=5); s.ehlo('health'); s.quit()\" && openssl x509 -checkend 1209600 -noout -in /certs/letsencrypt/live/mailu/nginx-chain.pem", + "interval": "60s", + "timeout": "15s", + "looks": 3, + "grace": "120s" + }, + "network": "mailu", + "env-file": [ + "${dir:state}/mailu.env" + ], + "ports": [ + "25", + "110", + "143", + "465", + "587", + "993", + "995", + "7080:80", + "7443:443" + ], + "volumes": [ + "${dir:data-certs}:/certs", + "${dir:data-overrides-nginx}:/overrides:ro" + ], + "dns": [ + "192.168.203.254" + ], + "logging": "journald" + }, + { + "id": "runtime-config", + "type": "file", + "path": "${dir:mesh-state}/config.json", + "mode": "0600", + "template": "files/config.json", + "takes-config": ["domain", "proxy-address", "sitename", "website", "url", "container"], + "merge": "json" + }, + { + "id": "automx", + "type": "container", + "name": "mailu-automx", + "artifact": "automx", + "network": "mailu", + "env-file": [ + "${dir:state}/mailu.env" + ], + "ports": [ + "4243" + ], + "volumes": [ + "${dir:data-automx}:/data" + ] + } + ], + "build": { + "on": [ + { + "arg": "PYTHON_BASE", + "image": "python@sha256:25f3cfeaceca14921366af4d1240b56457ef46273bdb508c7b0e8f469f6fd228" + } + ], + "artifacts": [ + { + "name": "code", + "kind": "bundle", + "language": "typescript", + "entrypoints": [ + "index.js", + "tools/index.js", + "provisioner/index.js" + ], + "loads": [ + "index.js", + "tools/index.js", + "provisioner/index.js" + ], + "env": { + "MESH_MAILU_URL": "http://127.0.0.1:${port:8080}/api/v1", + "MESH_MAILU_API_KEY_FILE": "${dir:state}/api-token.secret", + "MESH_MAILU_IMAP_CONTAINER": "mailu-imap", + "MESH_MAILU_SMTP_CONTAINER": "mailu-smtp", + "MESH_MAILU_CONFIG_FILE": "${dir:mesh-state}/config.json", + "MESH_RECEIVES": "${dir:grants}/mesh.json" + } + }, + { + "name": "automx", + "kind": "image", + "from": "automx/Dockerfile" + }, + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + }, + "provides": [ + { + "name": "smtp", + "scope": "mesh", + "identity": { + "max": 64, + "in": "a mailbox's local part" + } + } + ], + "serves": { + "smtp": { + "port": 587, + "domain": "${config:domain}" + } + }, + "receives": { + "smtp": "${dir:grants}/mesh.json" + }, + "grants": { + "smtp": "${dir:grants}" + }, + "jails": [ + { + "name": "mailu-front", + "failregex": "^.*(?:imap|pop3|submission|managesieve)-login: .*\\(auth failed, \\d+ attempts(?: in \\d+ secs)?\\):.*rip=(?:,|$)\n ^.*client login failed: \"AUTH not supported\" while in http auth state, client: , server: \\S+:25,", + "jail": "backend = systemd\njournalmatch = CONTAINER_NAME=mailu-front\nport = smtp,submission,submissions,imap,imaps,pop3,pop3s\nmaxretry = 3\nfindtime = 1d\nbantime = 1d" + } + ] +} -- 2.54.0 From 5e4fcc6db77eaecde838e4c08aed7573c8102650 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 06:05:52 +0000 Subject: [PATCH 13/41] Copy mesh-controller's catalogue into mesh-sdk, part 1: the package and its testdata, from mesh-control 6c2a8d62 (issue 1354) --- go/catalogue/adoption.go | 242 ++ go/catalogue/artifacts.go | 145 + go/catalogue/backup_placement.go | 53 + go/catalogue/bound.go | 186 + go/catalogue/bound_into_files.go | 192 + go/catalogue/build.go | 651 ++++ go/catalogue/busy.go | 124 + go/catalogue/chosen.go | 100 + go/catalogue/collections.go | 635 ++++ go/catalogue/config.go | 504 +++ go/catalogue/config_move.go | 40 + go/catalogue/consumer_into_serves.go | 338 ++ go/catalogue/consumers.go | 110 + go/catalogue/core_credentials.go | 33 + go/catalogue/cron.go | 92 + go/catalogue/data.go | 791 ++++ go/catalogue/declaration.go | 2819 ++++++++++++++ go/catalogue/delivery_seat.go | 140 + go/catalogue/dir_into.go | 373 ++ go/catalogue/environment_into.go | 731 ++++ go/catalogue/events.go | 199 + go/catalogue/files.go | 147 + go/catalogue/filtering.go | 1147 ++++++ go/catalogue/graphical_session.go | 247 ++ go/catalogue/handed.go | 802 ++++ go/catalogue/health.go | 495 +++ go/catalogue/identity.go | 220 ++ go/catalogue/installation.go | 236 ++ go/catalogue/jails_into.go | 72 + go/catalogue/localnetwork.go | 154 + go/catalogue/logs.go | 105 + go/catalogue/machine_into_files.go | 177 + go/catalogue/manifest.go | 3263 +++++++++++++++++ go/catalogue/naming.go | 17 + go/catalogue/node_resolver.go | 119 + go/catalogue/optional_requirements.go | 282 ++ go/catalogue/placement.go | 616 ++++ go/catalogue/port_into.go | 168 + go/catalogue/predefined.go | 263 ++ go/catalogue/process_bus.go | 189 + go/catalogue/reads.go | 220 ++ go/catalogue/recreates.go | 332 ++ go/catalogue/resolve.go | 1459 ++++++++ go/catalogue/roster.go | 287 ++ go/catalogue/runtime.go | 479 +++ go/catalogue/seat_contributions.go | 291 ++ go/catalogue/seat_data.go | 518 +++ go/catalogue/seat_dependencies.go | 392 ++ go/catalogue/seat_into.go | 222 ++ go/catalogue/seat_pass.go | 85 + go/catalogue/seat_paths.go | 315 ++ go/catalogue/seats.go | 951 +++++ go/catalogue/seats_declared.go | 587 +++ go/catalogue/secrets_into_files.go | 238 ++ go/catalogue/served_own_names.go | 202 + go/catalogue/setting_defaults.go | 261 ++ go/catalogue/setting_into.go | 367 ++ go/catalogue/setting_takes.go | 337 ++ go/catalogue/settings.go | 655 ++++ go/catalogue/shares.go | 99 + go/catalogue/state.go | 169 + go/catalogue/streams.go | 632 ++++ go/catalogue/template.go | 454 +++ go/catalogue/terminal_keys.go | 188 + go/catalogue/testdata/bars/bar-battery | 9 + go/catalogue/testdata/bars/bar-power-draw | 6 + .../testdata/bars/battery.bottom-bar.toml | 119 + .../testdata/bars/battery.top-bar.toml | 37 + .../testdata/bars/both.bottom-bar.toml | 126 + go/catalogue/testdata/bars/both.top-bar.toml | 37 + go/catalogue/testdata/bars/bottom-bar.toml | 109 + .../testdata/bars/none.bottom-bar.toml | 109 + go/catalogue/testdata/bars/none.top-bar.toml | 37 + .../testdata/bars/power-meter.bottom-bar.toml | 116 + .../testdata/bars/power-meter.top-bar.toml | 37 + go/catalogue/testdata/bars/top-bar.toml | 37 + go/catalogue/testdata/environment/all.posix | 26 + go/catalogue/testdata/environment/all.systemd | 23 + go/catalogue/testdata/environment/zsh.posix | 9 + go/catalogue/testdata/environment/zsh.systemd | 6 + .../grafana-ace.json | 35 + .../grafana-unlayered.json | 35 + ...yrus-g14.contribution-node-hotkeys-trigger | 10 + .../asus-zephyrus-g14.shell-after-wake-normal | 7 + .../items/claude-code.shell-zsh-normal | 12 + .../l2-moved/items/mesh-cli.shell-zsh-normal | 3 + .../items/powerlevel10k.shell-zsh-normal | 3 + .../zsh-autosuggestions.shell-zsh-normal | 2 + .../zsh-syntax-highlighting.shell-zsh-last | 3 + .../testdata/l2-moved/templates/power | 4 + .../testdata/l2-moved/templates/triggerhappy | 4 + go/catalogue/testdata/l2-moved/templates/zsh | 60 + go/catalogue/testdata/l2-moved/want/power | 11 + .../testdata/l2-moved/want/triggerhappy | 14 + go/catalogue/testdata/l2-moved/want/zsh | 88 + go/catalogue/unconsumed_placeholder.go | 178 + go/catalogue/unhealthy.go | 55 + go/catalogue/unknown_field.go | 206 ++ go/catalogue/upgrade.go | 120 + go/catalogue/verbs.go | 764 ++++ go/catalogue/zones.go | 117 + 101 files changed, 29531 insertions(+) create mode 100644 go/catalogue/adoption.go create mode 100644 go/catalogue/artifacts.go create mode 100644 go/catalogue/backup_placement.go create mode 100644 go/catalogue/bound.go create mode 100644 go/catalogue/bound_into_files.go create mode 100644 go/catalogue/build.go create mode 100644 go/catalogue/busy.go create mode 100644 go/catalogue/chosen.go create mode 100644 go/catalogue/collections.go create mode 100644 go/catalogue/config.go create mode 100644 go/catalogue/config_move.go create mode 100644 go/catalogue/consumer_into_serves.go create mode 100644 go/catalogue/consumers.go create mode 100644 go/catalogue/core_credentials.go create mode 100644 go/catalogue/cron.go create mode 100644 go/catalogue/data.go create mode 100644 go/catalogue/declaration.go create mode 100644 go/catalogue/delivery_seat.go create mode 100644 go/catalogue/dir_into.go create mode 100644 go/catalogue/environment_into.go create mode 100644 go/catalogue/events.go create mode 100644 go/catalogue/files.go create mode 100644 go/catalogue/filtering.go create mode 100644 go/catalogue/graphical_session.go create mode 100644 go/catalogue/handed.go create mode 100644 go/catalogue/health.go create mode 100644 go/catalogue/identity.go create mode 100644 go/catalogue/installation.go create mode 100644 go/catalogue/jails_into.go create mode 100644 go/catalogue/localnetwork.go create mode 100644 go/catalogue/logs.go create mode 100644 go/catalogue/machine_into_files.go create mode 100644 go/catalogue/manifest.go create mode 100644 go/catalogue/naming.go create mode 100644 go/catalogue/node_resolver.go create mode 100644 go/catalogue/optional_requirements.go create mode 100644 go/catalogue/placement.go create mode 100644 go/catalogue/port_into.go create mode 100644 go/catalogue/predefined.go create mode 100644 go/catalogue/process_bus.go create mode 100644 go/catalogue/reads.go create mode 100644 go/catalogue/recreates.go create mode 100644 go/catalogue/resolve.go create mode 100644 go/catalogue/roster.go create mode 100644 go/catalogue/runtime.go create mode 100644 go/catalogue/seat_contributions.go create mode 100644 go/catalogue/seat_data.go create mode 100644 go/catalogue/seat_dependencies.go create mode 100644 go/catalogue/seat_into.go create mode 100644 go/catalogue/seat_pass.go create mode 100644 go/catalogue/seat_paths.go create mode 100644 go/catalogue/seats.go create mode 100644 go/catalogue/seats_declared.go create mode 100644 go/catalogue/secrets_into_files.go create mode 100644 go/catalogue/served_own_names.go create mode 100644 go/catalogue/setting_defaults.go create mode 100644 go/catalogue/setting_into.go create mode 100644 go/catalogue/setting_takes.go create mode 100644 go/catalogue/settings.go create mode 100644 go/catalogue/shares.go create mode 100644 go/catalogue/state.go create mode 100644 go/catalogue/streams.go create mode 100644 go/catalogue/template.go create mode 100644 go/catalogue/terminal_keys.go create mode 100644 go/catalogue/testdata/bars/bar-battery create mode 100644 go/catalogue/testdata/bars/bar-power-draw create mode 100644 go/catalogue/testdata/bars/battery.bottom-bar.toml create mode 100644 go/catalogue/testdata/bars/battery.top-bar.toml create mode 100644 go/catalogue/testdata/bars/both.bottom-bar.toml create mode 100644 go/catalogue/testdata/bars/both.top-bar.toml create mode 100644 go/catalogue/testdata/bars/bottom-bar.toml create mode 100644 go/catalogue/testdata/bars/none.bottom-bar.toml create mode 100644 go/catalogue/testdata/bars/none.top-bar.toml create mode 100644 go/catalogue/testdata/bars/power-meter.bottom-bar.toml create mode 100644 go/catalogue/testdata/bars/power-meter.top-bar.toml create mode 100644 go/catalogue/testdata/bars/top-bar.toml create mode 100644 go/catalogue/testdata/environment/all.posix create mode 100644 go/catalogue/testdata/environment/all.systemd create mode 100644 go/catalogue/testdata/environment/zsh.posix create mode 100644 go/catalogue/testdata/environment/zsh.systemd create mode 100644 go/catalogue/testdata/influxdb-served-by-setting/grafana-ace.json create mode 100644 go/catalogue/testdata/influxdb-served-by-setting/grafana-unlayered.json create mode 100644 go/catalogue/testdata/l2-moved/items/asus-zephyrus-g14.contribution-node-hotkeys-trigger create mode 100644 go/catalogue/testdata/l2-moved/items/asus-zephyrus-g14.shell-after-wake-normal create mode 100644 go/catalogue/testdata/l2-moved/items/claude-code.shell-zsh-normal create mode 100644 go/catalogue/testdata/l2-moved/items/mesh-cli.shell-zsh-normal create mode 100644 go/catalogue/testdata/l2-moved/items/powerlevel10k.shell-zsh-normal create mode 100644 go/catalogue/testdata/l2-moved/items/zsh-autosuggestions.shell-zsh-normal create mode 100644 go/catalogue/testdata/l2-moved/items/zsh-syntax-highlighting.shell-zsh-last create mode 100644 go/catalogue/testdata/l2-moved/templates/power create mode 100644 go/catalogue/testdata/l2-moved/templates/triggerhappy create mode 100644 go/catalogue/testdata/l2-moved/templates/zsh create mode 100644 go/catalogue/testdata/l2-moved/want/power create mode 100644 go/catalogue/testdata/l2-moved/want/triggerhappy create mode 100644 go/catalogue/testdata/l2-moved/want/zsh create mode 100644 go/catalogue/unconsumed_placeholder.go create mode 100644 go/catalogue/unhealthy.go create mode 100644 go/catalogue/unknown_field.go create mode 100644 go/catalogue/upgrade.go create mode 100644 go/catalogue/verbs.go create mode 100644 go/catalogue/zones.go diff --git a/go/catalogue/adoption.go b/go/catalogue/adoption.go new file mode 100644 index 0000000..72bc564 --- /dev/null +++ b/go/catalogue/adoption.go @@ -0,0 +1,242 @@ +package catalogue + +import ( + "fmt" + "sort" + "strconv" + "strings" +) + +// What an adopted node is declared in place of a filter (novox/hq ADR 0100). +// +// On an adopted node the firewall found on the machine stays in force: the mesh loads no table +// that drops by default or holds an accept. What the mesh needs reachable is declared as +// openings, which the host converges through the found firewall in its own terms; and the mesh +// guards its own foundation ports itself, in a table that only refuses. + +// AdoptionPrefix is the id prefix of what the mesh declares of its own on an adopted node. It is +// never a module's, so none of it is ever held as found. +const AdoptionPrefix = "adoption." + +// Where an opening admits from, on the wire. +const ( + OpeningFromEverywhere = "everywhere" + OpeningFromMesh = "mesh" +) + +// The two paths a packet reaches a port by: received by the machine, or forwarded to a +// container that publishes it. +const ( + PathIncoming = "incoming" + PathForwarded = "forwarded" +) + +// Guard resources: the refusal-only table, the unit that loads it, and that unit running. +const ( + GuardPath = "/etc/mesh/guard.nft" + GuardUnit = "mesh-guard.service" + // GuardUnitPath is where the unit is written. + GuardUnitPath = "/etc/systemd/system/" + GuardUnit +) + +// GuardID, GuardUnitID and GuardRunningID are the guard's resource identities. The installer +// raises the same three on an adopted genesis, so the first push finds them already there. +func GuardID() string { return AdoptionPrefix + "guard" } +func GuardUnitID() string { return AdoptionPrefix + "guard-unit" } +func GuardRunningID() string { return AdoptionPrefix + "guard-running" } + +// GuardPackageID is the tool that loads the guard, declared first: a node joining adopted has +// no filter module and may have no nft at all, and a table nothing can load guards nothing. +func GuardPackageID() string { return AdoptionPrefix + "guard-package" } + +// GuardPackage is the package that carries nft. +const GuardPackage = "nftables" + +// OpeningID is an opening's resource identity: its protocol, port and path say what it is. +func OpeningID(protocol string, port int, path string) string { + return fmt.Sprintf("%sopening-%s-%d-%s", AdoptionPrefix, protocol, port, path) +} + +// Openings are what the mesh needs reachable on an adopted node, from the same inputs as the +// filter it would load were the node converged, each from where that filter would admit it. +// +// `rules` is Filtering's answer — every module's listens, the hub's port, the per-node exposure — +// and `foundation` is the ports the mesh itself needs, from everywhere. A rule for this machine +// only opens nothing. `published` maps a machine port a container publishes to the container's +// port: a published port is forwarded, not received, so its opening names the forwarded path and +// the port the packet is forwarded to. +func Openings(rules []Rule, foundation []int, published map[string]map[int]int) []map[string]any { + type key struct { + protocol string + port int + } + from := map[key]string{} + var order []key + widen := func(k key, f string) { + was, seen := from[k] + if !seen { + order = append(order, k) + } + if !seen || was != OpeningFromEverywhere { + from[k] = f + } + } + for _, rule := range rules { + switch rule.From { + case FromEverywhere: + widen(key{rule.Protocol, rule.Port}, OpeningFromEverywhere) + case FromMesh: + widen(key{rule.Protocol, rule.Port}, OpeningFromMesh) + case FromLocalNetwork: + // The private network's part only (novox/hq ADR 0340). An adopted node's found firewall + // stays, and it is what admits the machine's local network; an opening has no word for a + // local network, and widening it to everywhere would open the port to the internet. + widen(key{rule.Protocol, rule.Port}, OpeningFromMesh) + } + } + for _, port := range foundation { + widen(key{"tcp", port}, OpeningFromEverywhere) + } + sort.Slice(order, func(a, b int) bool { + if order[a].port != order[b].port { + return order[a].port < order[b].port + } + return order[a].protocol < order[b].protocol + }) + out := make([]map[string]any, 0, len(order)) + for _, k := range order { + opening := map[string]any{"type": "opening", "port": k.port, "protocol": k.protocol, + "from": from[k]} + if to, forwarded := published[k.protocol][k.port]; forwarded { + opening["id"] = OpeningID(k.protocol, k.port, PathForwarded) + opening["path"] = PathForwarded + opening["to"] = to + } else { + opening["id"] = OpeningID(k.protocol, k.port, PathIncoming) + opening["path"] = PathIncoming + } + out = append(out, opening) + } + return out +} + +// Published is every port the given containers publish on the machine, by protocol and machine +// port, mapped to the container's own port. A mapping bound to loopback is left out: nothing off +// the machine reaches it, forwarded or not. +func Published(resources []map[string]any) map[string]map[int]int { + out := map[string]map[int]int{} + for _, r := range resources { + if fmt.Sprint(r["type"]) != "container" { + continue + } + listed, _ := r["ports"].([]any) + for _, entry := range listed { + written := strings.TrimSpace(fmt.Sprint(entry)) + protocol := "tcp" + if cut := strings.LastIndex(written, "/"); cut >= 0 { + protocol = written[cut+1:] + } + // Indexed from the end, so an IPv6 address's own colons never shift the ports. + outer, inner, address, ok := mapping(written) + if !ok { + continue + } + switch strings.Trim(address, "[]") { + case "127.0.0.1", "localhost", "::1": + continue + } + if out[protocol] == nil { + out[protocol] = map[int]int{} + } + out[protocol][outer] = inner + } + } + return out +} + +// AsGuard renders the mesh's refusal-only table for the given machine ports. +// +// It passes everything by default and holds nothing but a refusal, so it cannot close anything +// the machine serves; and it is the mesh's own table, so the found firewall reloading does not +// touch it. It refuses only packets addressed to this machine, and the ports except from the +// machine itself — its loopback and the container runtime's own networks — and from the private +// network, known by the interface a packet arrives on and never by its source address. At +// prerouting, ahead of the runtime's destination translation, so it matches the port the packet +// was sent to; in the inet family, so both address families. +// +// **The machine's own interfaces are named, where the derived filter names address ranges.** The +// filter accepts the container runtime's networks by CIDR; this excludes its bridges by name — lo, +// docker0, the br-* a compose network gets, and the mesh's own mesh0. A runtime whose bridge is +// named anything else (a podman or libvirt bridge, or a docker network created with a fixed name) +// would have its containers' traffic to a guarded port refused, which reads as the port being +// down. Names rather than addresses is deliberate: a source address can be claimed by whoever +// sends the packet, and this table exists to refuse what the found firewall never sees. Widening +// it means adding names here and in the installer's copy together, which the golden test holds to +// one text. +// +// The same text the installer raises on an adopted genesis; a test holds both to it. +func AsGuard(ports []int) string { + sorted := append([]int{}, ports...) + sort.Ints(sorted) + listed := make([]string, len(sorted)) + for i, p := range sorted { + listed[i] = strconv.Itoa(p) + } + var b strings.Builder + b.WriteString("table inet mesh_guard {}\n") + b.WriteString("delete table inet mesh_guard\n") + b.WriteString("table inet mesh_guard {\n") + b.WriteString("\tchain prerouting {\n") + b.WriteString("\t\ttype filter hook prerouting priority raw; policy accept;\n") + // Only packets addressed to this machine: traffic it routes for others — a predecessor's hub, + // say — is never the guard's business (novox/hq ADR 0103). + fmt.Fprintf(&b, "\t\tfib daddr type local iifname != \"lo\" iifname != \"docker0\" "+ + "iifname != \"br-*\" iifname != \"mesh0\" tcp dport { %s } drop\n", + strings.Join(listed, ", ")) + b.WriteString("\t}\n") + b.WriteString("}\n") + return b.String() +} + +// GuardUnitText is the unit that loads the guard. Stopping it deletes only its own table: never +// a flush, which would take the container runtime's rules and the found firewall with it. +func GuardUnitText() string { + return "[Unit]\n" + + "Description=The mesh's guard: refuses its own ports from outside (novox/hq ADR 0100)\n" + + // Early, before the network is up, and without the default dependencies that would + // order it after the network; stopped at shutdown like any unit. + "DefaultDependencies=no\n" + + "Wants=network-pre.target\n" + + "Before=network-pre.target shutdown.target\n" + + "Conflicts=shutdown.target\n" + + "\n" + + "[Service]\n" + + "Type=oneshot\n" + + "RemainAfterExit=yes\n" + + "ExecStart=nft -f " + GuardPath + "\n" + + "ExecReload=nft -f " + GuardPath + "\n" + + "ExecStop=nft delete table inet mesh_guard\n" + + "\n" + + "[Install]\n" + + "WantedBy=multi-user.target\n" +} + +// GuardResources are the guard as four resources of the existing kinds: the tool that loads it, +// the table, the unit, and the unit running — reloaded when the table changes, so the new table +// replaces the old in one `nft -f` through the unit's ExecReload with no moment unguarded, and +// restarted only when the unit itself changes. Nothing when there is nothing to guard: an empty +// set is not a table nft loads. +func GuardResources(ports []int) []map[string]any { + if len(ports) == 0 { + return nil + } + return []map[string]any{ + {"id": GuardPackageID(), "type": "package", "package": GuardPackage}, + {"id": GuardID(), "type": "file", "path": GuardPath, "content": AsGuard(ports), + "mode": "0644"}, + {"id": GuardUnitID(), "type": "file", "path": GuardUnitPath, "content": GuardUnitText(), + "mode": "0644"}, + {"id": GuardRunningID(), "type": "service", "unit": GuardUnit, "state": "running", + "boot": "enabled", "restart-on": []any{GuardUnitID()}, "reload-on": []any{GuardID()}}, + } +} diff --git a/go/catalogue/artifacts.go b/go/catalogue/artifacts.go new file mode 100644 index 0000000..e9a010d --- /dev/null +++ b/go/catalogue/artifacts.go @@ -0,0 +1,145 @@ +package catalogue + +import ( + "fmt" + "strings" +) + +// What the mesh built, named by what it is rather than by where it was pushed. +// +// **An image is recorded by its digest and its path; the registry's address is a route to it** +// (novox/hq 04-ISSUES/102). A build used to be recorded as `://@sha256:…` +// — the reference the builder pushed to, kept whole — and every declaration carried that literal. +// Move the registry's port, or the registry, and every fresh pull of a mesh image fails: a new +// node, a recreate after eviction. The digest is the identity; the address is the node's setting +// for the module that serves the artifact store, and it is read from there when a reference is +// composed, never written into a record. +// +// So a kept reference has a scheme of the mesh's own, named after the provision that answers it: +// +// artifact-store:///@sha256: an image +// artifact-store:////blobs/sha256: an archive +// +// No runtime knows the scheme. That is the point of it being one: a reference that leaks to a +// machine uncomposed is refused by the runtime as malformed, in front of whoever sent it, rather +// than pulled from a public registry that happens to have a repository by that name — which is +// what an address-less `/@sha256:…` would be. + +// ArtifactStoreScheme marks a reference to something in the mesh's artifact store, kept without +// the store's address. +const ArtifactStoreScheme = ArtifactStoreProvision + "://" + +// Recorded is a reference as the mesh records it: the artifact store's address, if the builder wrote +// one, taken off. +// +// For a reference the builder announced — one it pushed to the store — which is the only kind +// this is called on. An image the builder named `/@sha256:…` is kept as its path; an +// archive it named `http:///v2//blobs/` likewise. A reference with no address +// in it — an image id from a genesis build that had nowhere to publish, a package version — is +// what it was. +func Recorded(reference string) string { + if strings.HasPrefix(reference, ArtifactStoreScheme) { + return reference + } + if rest, isURL := strings.CutPrefix(reference, "http://"); isURL { + if _, path, ok := strings.Cut(rest, "/v2/"); ok && strings.Contains(path, "/blobs/") { + return ArtifactStoreScheme + path + } + return reference + } + host, path, ok := strings.Cut(reference, "/") + if !ok || !isRegistryHost(host) || !strings.Contains(path, "@sha256:") { + return reference + } + return ArtifactStoreScheme + path +} + +// isRegistryHost is the runtime's own rule for reading the first component of a reference as a +// registry rather than as a namespace: it has a dot or a port in it, or it is localhost. +func isRegistryHost(component string) bool { + return component == "localhost" || strings.ContainsAny(component, ".:") +} + +// InArtifactStore reports whether a reference is a kept one, and what it names there. +func InArtifactStore(reference string) (path string, kept bool) { + return strings.CutPrefix(reference, ArtifactStoreScheme) +} + +// Routed is a kept reference as a machine fetches it, through the artifact store at `address` +// (host:port). A reference that is not a kept one is what it was. +func Routed(reference, address string) string { + path, kept := InArtifactStore(reference) + if !kept { + return reference + } + if strings.Contains(path, "/blobs/") { + return "http://" + address + "/v2/" + path + } + return address + "/" + path +} + +// Rerouted is a reference the mesh recorded, whichever way it was recorded, as a machine fetches +// it now: a kept one composed with the store's address, and one recorded before references were +// kept without their address — the builder's own `/@sha256:…` — re-routed to where +// the store is now. Only for references that are the mesh's own: everything a build record +// holds is, by construction. +func Rerouted(reference, address string) string { + return Routed(Recorded(reference), address) +} + +// artifactsInto composes the artifact store's address into a resource's `image` and `source`. +// +// **Composed here, at the last moment before a machine, and stored nowhere.** A kept reference is +// routed through the store as this network reaches it now. A reference recorded with an address +// before references were kept without one is re-routed the same way — but only when the mesh +// built it (`with.Built` names every `/` it has), because a module may run an +// image from a public registry under its own name and that one is exactly where it says. +// +// A kept reference with no store to route it through is refused: sent as it is, the runtime would +// refuse the scheme on the machine, one push away from the reason. +// +// **What this does not reach: the images genesis pinned.** The installer builds the control plane +// and the builder before the mesh exists, pushes them itself and pins their manifests to +// `:/mesh-controller@…` and `:/mesh-builder@…` — single-segment +// repositories with no build record, so `with.Built` does not name them and they are left as +// written until each is rebuilt through the mesh, which records it by digest and path. Until then +// a registry that moves strands exactly those two on a recreate, and the control plane's is the +// one that cannot be repaired through the mesh. Rebuild both through `build` before moving the +// store (novox/hq 04-ISSUES/102, finding F4). +func artifactsInto(resource map[string]any, module string, with Rendering) error { + for _, key := range []string{"image", "source"} { + written, ok := resource[key].(string) + if !ok { + continue + } + if _, kept := InArtifactStore(written); kept { + if with.ArtifactStore == "" { + return fmt.Errorf( + "%s's %v names %s, which is in the mesh's artifact store, and this mesh has no "+ + "artifact store on its network to fetch it from — nothing assigned offers "+ + "%s, or the machine offering it is not on the private network", + module, resource["id"], written, ArtifactStoreProvision) + } + resource[key] = Routed(written, with.ArtifactStore) + continue + } + if with.ArtifactStore == "" { + continue + } + recorded := Recorded(written) + if recorded == written { + continue + } + path, _ := InArtifactStore(recorded) + repository := path + if at := strings.IndexAny(path, "@"); at >= 0 { + repository = path[:at] + } else if blobs := strings.Index(path, "/blobs/"); blobs >= 0 { + repository = path[:blobs] + } + if with.Built[repository] { + resource[key] = Routed(recorded, with.ArtifactStore) + } + } + return nil +} diff --git a/go/catalogue/backup_placement.go b/go/catalogue/backup_placement.go new file mode 100644 index 0000000..ddd3517 --- /dev/null +++ b/go/catalogue/backup_placement.go @@ -0,0 +1,53 @@ +package catalogue + +import ( + "fmt" + "regexp" +) + +// What the machine's modules back up is placement (design 57 L3, issue 794). The holder of node-backup names it as +// `${backup:lines}` (what to run and which paths to keep) and `${backup:data}` (every item with its class and +// protection): the text `${contribution:node-backup:backup}` and `${contribution:node-backup:data}` render, from the +// same function, derived from each placed module's declared data (novox/hq ADR 0233). A handed holder has the text in +// its plan's placement for its client to fill; one not handed has the controller fill it where it stands. + +// BackupNamespace is the placeholder kind of the machine's backup lines and data. +const BackupNamespace = "backup" + +// ofBackup is where a holder places them: ${backup:}. Loose inside the braces, so a misspelt key is found and +// refused rather than written out as text. +var ofBackup = regexp.MustCompile(`\$\{backup:([^}]*)\}`) + +// backupKinds is the contribution kind each key of the backup placement renders. +var backupKinds = map[string]string{"lines": BackupKindBackup, "data": BackupKindData} + +// backupPlacer is the machine's modules the backup placement is composed over, as the old contributions are. +type backupPlacer struct { + modules []Manifest + with Rendering + facts map[string]string + caps map[string]bool + unplaced *[]string +} + +// placed is the text of one key for a holder, or what is wrong with naming it. +func (b backupPlacer) placed(holder Manifest, key string, item bool) (string, string) { + kind, known := backupKinds[key] + switch { + case item: + return "", "an item is kept per item; the machine's backup lines are placed only in a file" + case !known: + return "", fmt.Sprintf("the backup placement has `lines` and `data`, not %q", key) + case !holder.ClaimsSeat(BackupSeat): + return "", fmt.Sprintf("%s does not claim %s; the machine's backup lines are placed by its holder alone "+ + "(novox/hq ADR 0212)", holder.Module, BackupSeat) + } + text, left, err := seatContributions(b.modules, holder, BackupSeat+":"+kind, b.with, b.facts, b.caps) + if b.unplaced != nil { + *b.unplaced = append(*b.unplaced, left...) + } + if err != nil { + return "", err.Error() + } + return text, "" +} diff --git a/go/catalogue/bound.go b/go/catalogue/bound.go new file mode 100644 index 0000000..12941a4 --- /dev/null +++ b/go/catalogue/bound.go @@ -0,0 +1,186 @@ +package catalogue + +import ( + "fmt" + "slices" + "strings" +) + +// A consumer of a provision that keeps its data stays bound where its data is (novox/hq ADR 0232). +// +// **What a resolution chooses is not where a consumer's data is.** Resolving answers "which provider +// would I pick now", from the seats' holders, the pins and what is assigned where, and every one of +// those can change under a consumer without anybody meaning to move it. For a resolver that is the +// point: any provider answers alike. For a database it is the consumer's whole state: on 2026-10-05 +// one change to how a seat's holder answers (issue 258) re-bound five database consumers on one +// machine to the store on another, each was given a fresh, empty database there, and nothing warned +// for twenty hours (issue 273). Nothing was lost only because the old provider kept everything. +// +// So the mesh records where each such consumer was last sent (the store's `binding` table), and a +// resolution that would answer it from anywhere else keeps the recorded provider instead and says so. +// **Only a pin moves it**, because a pin is a person: the one act that says "answer this machine's +// consumers from there", taken knowing the data must go first. + +// KeptBinding is one consumer this resolution would have moved, and did not. +type KeptBinding struct { + // Machine is where the consumer runs, and Consumer the module there that is bound. + Machine string + Consumer string + // Provision is what it is bound for. + Provision string + // Bound is the provider it was recorded at, and still is; Would is the one the resolution chose. + Bound Chosen + Would Chosen +} + +// String is the condition's sentence: the move, where the data is, and the act that confirms it. +func (k KeptBinding) String() string { + return fmt.Sprintf("would move %s's %s from %s to %s — its data is on %s; kept there. "+ + "`pin %s %s %s %s` to confirm a move (and move the data first)", + k.Consumer, k.Provision, k.Bound, k.Would, k.Bound, k.Machine, k.Provision, k.Would.Node, k.Would.Module) +} + +// keepBound holds every need for a provision that keeps its consumers' data at the provider its +// consumer was bound to, where the resolution chose another. +// +// A need with no record is a binding being made, and is left as resolved: it is recorded when it is +// first sent. A pin naming the provider the resolution chose is a person moving it, and is left too. +// Otherwise the recorded provider answers, if it still provides the provision — beside the consumer, +// or offered from elsewhere — and the move is returned as kept. **One that no longer does is refused, +// never answered by the provider chosen**: answering it there is exactly the silent move this exists +// to stop, and the consumer's data is still wherever it was. +func keepBound(needs []Needed, catalogue map[string]Manifest, node Node, world World, + keeps map[string]bool, here func(string) bool) ([]Needed, []KeptBinding, []string) { + var kept []KeptBinding + var problems []string + refused := map[[2]string]bool{} + out := make([]Needed, 0, len(needs)) + for _, n := range needs { + if n.ByRecord || !keeps[n.Name] { + out = append(out, n) + continue + } + n.KeepsData = true + bound, recorded := world.Bound[n.For][n.Name] + chose := Chosen{Node: n.From, Module: n.Module} + if !recorded || sameProvider(bound, chose) { + out = append(out, n) + continue + } + if pin, pinned := world.Pinned[n.Name]; pinned && pin.matches(Provider{Node: chose.Node, Module: chose.Module}) { + out = append(out, n) + continue + } + held, ok, why := boundNeed(n, bound, catalogue, node, world, here) + if !ok { + if key := [2]string{n.For, n.Name}; !refused[key] { + refused[key] = true + problems = append(problems, fmt.Sprintf( + "%s on %s is bound to %s for %q, which keeps its data, and %s — it would move to %s, "+ + "which holds none of it. Move its data and say so with `pin %s %s %s %s`, or give "+ + "%s back what it provided", + n.For, node.Name, bound, n.Name, why, chose, node.Name, n.Name, chose.Node, chose.Module, + bound.Node)) + } + continue + } + out = append(out, held) + kept = append(kept, KeptBinding{Machine: node.Name, Consumer: n.For, Provision: n.Name, Bound: bound, Would: chose}) + } + return out, kept, problems +} + +// sameProvider is whether a recorded provider is the one chosen. A record naming no module (none +// is written without one, but a person's hand might) matches any module on its node. +func sameProvider(bound, chose Chosen) bool { + return bound.Node == chose.Node && (bound.Module == "" || bound.Module == chose.Module) +} + +// boundNeed is the need answered by the recorded provider, or why it cannot be. +func boundNeed(n Needed, bound Chosen, catalogue map[string]Manifest, node Node, world World, + here func(string) bool) (Needed, bool, string) { + held := Needed{Name: n.Name, For: n.For, Local: n.Local, KeepsData: true} + if bound.Node == node.Name { + m, known := catalogue[bound.Module] + if !known || !here(bound.Module) || !providesAt(m, n.Name, ScopeMesh) { + return Needed{}, false, fmt.Sprintf("%s no longer runs on %s", bound.Module, node.Name) + } + at := node.At + if at == "" { + at = "127.0.0.1" + } + held.From, held.At, held.Module = node.Name, at, m.Module + held.Serves, held.SharedOwn, held.Identity = servedByOne(m, n.Name), sharedByOne(m, n.Name), m.IdentityBoundOf(n.Name) + held.CredentialForm, held.CredentialTool = m.CredentialFormOf(n.Name), m.CredentialToolOf(n.Name) + return held, true, "" + } + matching := bound.among(world.Offered[n.Name]) + if len(matching) != 1 { + return Needed{}, false, fmt.Sprintf("%s no longer provides it", bound) + } + p := matching[0] + if node.At == "" || p.At == "" { + return Needed{}, false, fmt.Sprintf("%s and %s are not both on the private network", node.Name, p.Node) + } + identity := DefaultIdentityBound + if pm, known := catalogue[p.Module]; known { + held.SharedOwn, _ = pm.SharedCredentialOf(n.Name) + held.CredentialForm, held.CredentialTool = pm.CredentialFormOf(n.Name), pm.CredentialToolOf(n.Name) + identity = pm.IdentityBoundOf(n.Name) + } + held.From, held.At, held.Module, held.Serves, held.Identity = p.Node, p.At, p.Module, p.Serves, identity + held.LeftOut = p.LeftOut + return held, true, "" +} + +// Unbound is one consumer that still asks for a provision and whose own resolution binds it to +// another provider than the one a pair credential on record was made with (novox/hq issue 274). +// +// **A provider is granted exactly the consumers bound to it.** The credential from the old provider +// stays on record — it is the key to a login that provider keeps, disabled, with the consumer's data, +// until a person deletes it with `cleanup delete` (ADR 0230), and a pin back must find it — but it is +// no longer granted, so the provider stops being asked for it and retires it. Said on every plan and +// push of the provider, so a credential the mesh keeps and does not use is never kept silently. +type Unbound struct { + // Provision is what was required, Provider the machine the credential on record is from. + Provision string `json:"provision"` + Provider string `json:"provider"` + // Consumer is the machine, Module the module on it that requires it, Local the credential's + // name inside it where it keeps several (ADR 0094). + Consumer string `json:"consumer"` + Module string `json:"module"` + Local string `json:"local,omitempty"` + // BoundTo is every provider the consumer's resolution binds this credential to now; empty when + // it binds it nowhere — the module asks for the provision under other local names. + BoundTo []string `json:"bound_to,omitempty"` +} + +func (u Unbound) String() string { + who := u.Module + if u.Local != "" { + who += " (as " + u.Local + ")" + } + now := "is bound to no provider under that name" + if len(u.BoundTo) > 0 { + now = "is bound to " + strings.Join(u.BoundTo, ", ") + } + return fmt.Sprintf("%s on %s %s for %s, not to %s — %s no longer grants it, so it retires that "+ + "login and keeps its data until `cleanup delete` (ADR 0230); the credential from %s stays on "+ + "record while that login does", who, u.Consumer, now, u.Provision, u.Provider, u.Provider, u.Provider) +} + +// BindsFrom is the providers this resolution binds a pair credential's need to — the provision, the +// module that requires it and the credential's local name — answered by a machine rather than by a +// record. None means this machine states no such binding. +func (r Resolution) BindsFrom(provision, module, local string) []string { + var from []string + for _, n := range r.Needs { + if n.ByRecord || n.Name != provision || n.For != module || n.Local != local { + continue + } + if !slices.Contains(from, n.From) { + from = append(from, n.From) + } + } + return from +} diff --git a/go/catalogue/bound_into_files.go b/go/catalogue/bound_into_files.go new file mode 100644 index 0000000..e04caad --- /dev/null +++ b/go/catalogue/bound_into_files.go @@ -0,0 +1,192 @@ +package catalogue + +import ( + "fmt" + "regexp" + "sort" + "strings" +) + +// The half of a connection that is not secret, put where the program reading it can find it. +// +// **The asymmetry this removes was backwards** (novox/hq 04-ISSUES/023). A sealed credential can +// be placed inside any configuration file a module writes: the module leaves a hole, the mesh +// delivers the value sealed beside it, and the host — the only thing that sees both — fills it in. +// The host and the port and the name to present are ordinary facts the mesh holds in the clear, +// and they were the ones stuck: readable only inside a JSON binding, which a program reading +// `KEY=value` cannot use. +// +// So the same shape, and simpler. These values are not secret, so **the control plane substitutes +// them itself** before the declaration is sent. Nothing new reaches the host, which learns no +// formats and gains no fields. +// +// **It stays name-agnostic** ([ADR 0027]). The mesh does not learn what a `postgres-database` is: +// `at`, `as`, `from` and `address` (the providing machine's private address, novox/hq ADR 0194) are +// facts about any provision at all, and everything else comes from what the provider said it +// serves — whose keys are agreed by the requirement's name, not by this file. + +// bound is where a module says a value from one of its bindings belongs: +// ${bound::}, and for an optional requirement the value it takes while absent, +// ${bound::|default:} (novox/hq ADR 0330 §4). A default is never empty: an +// absent requirement is never rendered as an empty string. +var bound = regexp.MustCompile(`\$\{bound:([a-z0-9][a-z0-9.-]*[a-z0-9]):([a-z0-9][a-z0-9_-]*)(?:\|default:([^}\n]+))?\}`) + +// knownFor is everything a module may name from one of its bindings. +// +// Three facts the mesh states about any provision, plus whatever the provider said it serves. A +// module may not reach a binding it does not have — the same boundary as a secret, for the same +// reason. +func knownFor(m Manifest, needs []Needed, node string) (map[string]map[string]string, error) { + out := map[string]map[string]string{} + for _, want := range m.Wants() { + for i := range needs { + n := needs[i] + if n.Name != want || n.For != m.Module { + continue + } + as := ConsumerIdentity(node, IdentitySource(m.Slug, m.Module)) + values := map[string]string{ + "at": n.At, + "from": n.From, + "as": as, + } + // What the provider derives for this consumer rather than for all of them + // (novox/hq ADR 0202). Filled here, the one place a provision and the module + // requiring it are both in hand. + served, err := ServedTo(n.Serves, as) + if err != nil { + return nil, fmt.Errorf("%s requires %s: %w", m.Module, want, err) + } + for key, value := range served { + // The provider's own vocabulary. Rendered plainly: a port is 5432, not 5432.000000, + // which is what a float would write and what a connection string would refuse. + values[key] = plainly(value) + } + out[want] = values + } + } + return out, nil +} + +// withOwnNames adds a module's own composed names to what it may name from one binding: +// `${bound::name}` and `:internal-name`, and for several contributions to one requirement +// `:name-` / `:internal-name-`. Set over anything the provider serves under those keys: +// what the module is called is the mesh's statement, not the provider's. +func withOwnNames(values map[string]string, own map[string]any) { + for _, key := range []string{"name", "internal-name"} { + if v, ok := own[key].(string); ok { + values[key] = v + } + } + many, _ := own["names"].(map[string]any) + for local, raw := range many { + names, _ := raw.(map[string]any) + for _, key := range []string{"name", "internal-name"} { + if v, ok := names[key].(string); ok { + values[key+"-"+local] = v + } + } + } +} + +// plainly renders a served value as a program would expect to read it. +func plainly(value any) string { + switch v := value.(type) { + case string: + return v + case float64: + if v == float64(int64(v)) { + return fmt.Sprintf("%d", int64(v)) + } + return strings.TrimRight(strings.TrimRight(fmt.Sprintf("%f", v), "0"), ".") + case bool: + return fmt.Sprintf("%t", v) + case nil: + return "" + default: + return fmt.Sprint(v) + } +} + +// boundInto replaces a file's ${bound:…} placeholders with what the mesh knows. +// +// A placeholder naming something the module does not require, or a key the provider does not +// serve, is refused. Left as it was, the literal `${bound:x:y}` would be written into a +// configuration file and read as a value — a connection to a host called `${bound:x:y}`, failing +// somewhere that names neither the module nor the mesh. +// +// **An optional requirement left absent takes the default its placeholder names** (novox/hq ADR +// 0330 §4), and one that names none is refused rather than rendered empty. The module check refuses +// that already; this is the same rule where the file is made, for a manifest registered before it. +func boundInto(resource map[string]any, known map[string]map[string]string, module string, absent map[string]bool) error { + if fmt.Sprint(resource["type"]) != "file" { + return nil + } + content, ok := resource["content"].(string) + if !ok { + return nil + } + var refused error + filled := bound.ReplaceAllStringFunc(content, func(token string) string { + if refused != nil { + return token + } + m := bound.FindStringSubmatch(token) + provision, key, fallback := m[1], m[2], m[3] + values, has := known[provision] + if !has { + if absent[provision] { + if fallback != "" { + return fallback + } + refused = fmt.Errorf( + "%s has a file that says ${bound:%s:%s}, and %q is optional and absent here: it names no "+ + "value for then (${bound:%s:%s|default:}), so it would be written empty "+ + "(novox/hq ADR 0330 §4)", module, provision, key, provision, provision, key) + return token + } + refused = fmt.Errorf( + "%s has a file that says ${bound:%s:%s}, and %s does not require %q. It may name %s", + module, provision, key, module, provision, orNothing(namesOfBindings(known))) + return token + } + value, said := values[key] + if !said { + refused = fmt.Errorf( + "%s asks its %s binding for %q, and what answers it says %s", + module, provision, key, orNothing(namesOfKeys(values))) + return token + } + return value + }) + if refused != nil { + return refused + } + resource["content"] = filled + return nil +} + +func namesOfBindings(known map[string]map[string]string) []string { + var names []string + for name := range known { + names = append(names, fmt.Sprintf("%q", name)) + } + sort.Strings(names) + return names +} + +func namesOfKeys(values map[string]string) []string { + var names []string + for key := range values { + names = append(names, fmt.Sprintf("%q", key)) + } + sort.Strings(names) + return names +} + +func orNothing(names []string) string { + if len(names) == 0 { + return "nothing" + } + return join(names) +} diff --git a/go/catalogue/build.go b/go/catalogue/build.go new file mode 100644 index 0000000..e36de7f --- /dev/null +++ b/go/catalogue/build.go @@ -0,0 +1,651 @@ +package catalogue + +import ( + "fmt" + "net" + "net/url" + "path" + "regexp" + "sort" + "strings" +) + +// Turning a manifest that names artifacts into one that names digests. +// +// **Two documents, deliberately.** The manifest in a repository says *this resource uses the +// archive called `config`*; the manifest the mesh holds says *this resource is sha256:…*. A digest +// is not knowable until something is built, so a repository carrying one would be a repository +// whose file is wrong the moment anybody edits anything — and the mesh would be pinning a value +// nobody could have checked. +// +// So the built manifest is **derived**, and the record of which commit it was derived from is what +// makes "is this current?" answerable without building (novox/hq ADR 0009). + +// Built is one artifact after it exists: where it is and what it hashes to. +type Built struct { + // Name is what the manifest called it. + Name string + // Kind is "image" or "archive". + Kind string + // Reference is what a machine uses to get it — an image reference for an image, a URL for an + // archive. Both already carry the digest for an image; an archive carries it separately. + Reference string + // Digest is "sha256:", for an archive. An image reference already ends in one. + Digest string + // Launchers are, for a bundle in an interpreted language, the executable the build wrote beside + // each entrypoint, by entrypoint (novox/hq ADR 0193): what the node's runtime starts to serve it. + Launchers map[string]string + // Texts are, for the archive a module packs its templates into (TemplatesArtifact), each file's text as the + // build read it back from the packed archive, by its path inside it (novox/hq ADR 0349 §2). Never carried on + // the wire or kept: Resolve reads them into the files' content, which is what the mesh holds. + Texts map[string]string `json:"-"` +} + +// Resolve fills a manifest's resources in from what was built. +// +// Every resource naming an artifact is rewritten to name the thing itself, and the `artifact` key +// is removed — because it is a build-time word and the host has never heard of it. A resource +// naming an artifact nothing produced is refused: it would otherwise reach a machine with an +// empty image or an unpinned archive, which is the shape of failure that looks like success. +func (m Manifest) Resolve(built []Built) (Manifest, error) { + if m.Build == nil && len(built) == 0 { + return m, nil + } + + by := map[string]Built{} + for _, b := range built { + by[b.Name] = b + } + // Declared and not produced is a build that did not do what the manifest asked, and saying so + // here beats a machine reporting it later. + var missing []string + if m.Build != nil { + for _, a := range m.Build.Artifacts { + if _, ok := by[a.Name]; !ok { + missing = append(missing, a.Name) + } + } + } + if len(missing) > 0 { + sort.Strings(missing) + return Manifest{}, fmt.Errorf( + "%s says it builds %s and the build did not produce %s — the build did not do what "+ + "the manifest asked, which is a different fault from a resource asking for the "+ + "wrong thing", + m.Module, strings.Join(missing, " and "), oneOrOther(len(missing))) + } + + // **Each template's text, as the build read it back from its templates archive** (novox/hq ADR 0349 §2): into + // the file's content, so what the mesh holds and composes is what the same text inline would be. + var handed *Handed + if m.Templated() { + name := TemplatesArtifact(m) + texts := by[name].Texts + if name == "" || texts == nil { + return Manifest{}, fmt.Errorf("%s names templates, and the build carried no text read back from the "+ + "archive it packs %s/ into (%q): the builder that built it does not read templates", + m.Module, m.TemplatesDir(), name) + } + dir := m.TemplatesDir() + filled, err := fillTemplates(m.Module, dir, m.Resources, func(template string) (string, bool) { + t, ok := texts[strings.TrimPrefix(template, dir+"/")] + return t, ok + }, fmt.Sprintf("the archive %q", name)) + if err != nil { + return Manifest{}, err + } + items, err := fillItemTemplates(m, m.Items, func(template string) (string, bool) { + t, ok := texts[strings.TrimPrefix(template, dir+"/")] + return t, ok + }, fmt.Sprintf("the archive %q", name)) + if err != nil { + return Manifest{}, err + } + // And, for a moved module, where its templates are and which one each file and item is filled from + // (handed.go): read while its resources and items still name their templates. + handed = handedBy(m, by[name]) + m.Resources = filled + m.Items = items + } + + out := m + out.Handed = handed + out.Build = nil + // Read with the build, like Build: the manifest the mesh holds carries each file's text (issue 731). + out.Templates = "" + out.Resources = nil + // What the build compiled, kept on the resolved manifest (novox/hq ADR 0175): a tools bundle is + // named by no resource of the module's own — the node's runtime loads it — so this is the only + // place the mesh would otherwise not have it. In artifact order, so two resolutions of one + // build compare equal. + out.Bundles = nil + if m.Build != nil { + run := runByAResource(m) + for _, a := range m.Build.Artifacts { + if a.Kind != ArtifactBundle { + continue + } + made := by[a.Name] + // What the runtime loads: what the artifact said, else every entrypoint of a module + // that declares tools, else nothing (the field's own rule; see Artifact.Loads). + // + // **Never, unasked, a bundle one of the module's own resources runs** (novox/hq issue 213). + // A process the host runs is the module's service, not its tools: the controller declares + // the verbs it answers as `tools` and serves them itself, and its bundle would otherwise + // have been launched a second time by the node's runtime, as an MCP child it is not. + loads := append([]string(nil), a.Loads...) + if a.Loads == nil && len(m.Tools) > 0 && !run[a.Name] { + loads = append([]string(nil), a.Entrypoints...) + // A bundle compiled to a binary has no entrypoints: the binary is what it is, and what + // the runtime starts to serve it (novox/hq ADR 0193). So a Go tools bundle is served + // as Go — the runtime execs it — exactly as a TypeScript one is through its launcher. + if bin := BinaryOf(a); bin != "" { + loads = []string{bin} + } + } + // **Kept, never routed** (ADR 0155): the builder publishes to the store at the address + // it reached it by, and a manifest carrying that address names an installation — + // registration refused node-tools for exactly this on 2026-10-02. The build record + // already keeps the store-relative form; the resolved manifest keeps the same, and + // composition routes it through the store a machine reaches (Routed). + out.Bundles = append(out.Bundles, Bundle{ + Name: a.Name, Source: Recorded(made.Reference), Digest: made.Digest, + Language: a.Language, Entrypoints: append([]string(nil), a.Entrypoints...), + Loads: loads, Env: copyWords(a.Env), Launchers: copyWords(made.Launchers), + Binary: BinaryOf(a), + }) + } + sort.Slice(out.Bundles, func(i, j int) bool { return out.Bundles[i].Name < out.Bundles[j].Name }) + } + for _, r := range m.Resources { + named, _ := r["artifact"].(string) + if named == "" { + out.Resources = append(out.Resources, r) + continue + } + artifact, ok := by[named] + if !ok { + return Manifest{}, fmt.Errorf( + "%s: %v uses the artifact %q, and this module builds no such thing", + m.Module, r["id"], named) + } + + filled := map[string]any{} + for k, v := range r { + filled[k] = v + } + delete(filled, "artifact") + switch artifact.Kind { + case ArtifactPackage: + // A package is not a resource on any machine; it is consumed by other builds. A + // resource that names one is a manifest error, named here rather than shipped. + return Manifest{}, fmt.Errorf( + "%s: %v uses %q, which is a package — a build input, not a resource a machine runs", + m.Module, r["id"], named) + case ArtifactImage, ArtifactUpstream: + filled["image"] = artifact.Reference + // An image is not unpacked anywhere, so it has no directory to be named for its + // version and `${version}` has nothing to mean. Refused rather than left as literal + // text in a path, which is how it would reach a machine and be created as a directory + // called `${version}`. + for key, value := range filled { + if text, isText := value.(string); isText && strings.Contains(text, versionRef) { + return Manifest{}, fmt.Errorf( + "%s: %v says %s in %q, and %q is an image — an image is not unpacked, so "+ + "it has no versioned place. %s is for an archive or a bundle", + m.Module, r["id"], versionRef, key, named, versionRef) + } + } + case ArtifactArchive, ArtifactBundle: + // The same on the wire: both are bytes fetched by digest and unpacked. They differ in + // how they were made — one packed as it stood, the other compiled first — and a + // machine has no reason to care which. + // Kept, not routed, for the reason the bundles above are (ADR 0155). + filled["source"] = Recorded(artifact.Reference) + filled["digest"] = artifact.Digest + // **And `${version}`, so a resource can name a place that is this build's alone** + // (novox/hq ADR 0141, 04-ISSUES/142). A component is unpacked into a directory named + // for its version so it can read its own version from its path — and until this, + // nothing could compose that path: an archive named a fixed one in the manifest and + // nothing interpolated the build into it, so nothing could ask for + // `…/versions//` and every machine took a hand-placed fallback. + // + // The version is the artifact's own digest, short. Not the commit: two builds of one + // commit are meant to be the same bytes (the toolchains are `-trimpath` for this), and + // a content-addressed version means an unchanged build resolves to the path it already + // had — so re-composing a declaration moves nothing, where a commit would move the + // path of an identical binary and recreate everything that reads it. + for key, value := range filled { + filled[key] = withVersion(value, versionOf(artifact.Digest)) + } + default: + return Manifest{}, fmt.Errorf("%s: %q is a %q, and an artifact is %q, %q, %q or %q", + m.Module, named, artifact.Kind, ArtifactImage, ArtifactArchive, ArtifactUpstream, + ArtifactBundle) + } + out.Resources = append(out.Resources, filled) + } + return out, nil +} + +// withVersion is a resource's value with `${version}` filled: in a string, and in each string of a map — +// a process's env, where the controller is told which build it is (novox/hq issue 352). Anything else is +// left as it is. +func withVersion(value any, version string) any { + switch v := value.(type) { + case string: + if strings.Contains(v, versionRef) { + return strings.ReplaceAll(v, versionRef, version) + } + case map[string]any: + out := make(map[string]any, len(v)) + for k, x := range v { + out[k] = withVersion(x, version) + } + return out + case map[string]string: + out := make(map[string]string, len(v)) + for k, x := range v { + out[k] = strings.ReplaceAll(x, versionRef, version) + } + return out + } + return value +} + +// checkBuild is the manifest's own account of what it builds. +func (b *Build) problems(module string) []string { + if b == nil { + return nil + } + var problems []string + seen := map[string]bool{} + for _, a := range b.Artifacts { + if a.Name == "" { + problems = append(problems, module+" builds an artifact with no name") + continue + } + if seen[a.Name] { + problems = append(problems, fmt.Sprintf( + "%s builds two artifacts called %q, and a resource naming it could mean either", + module, a.Name)) + } + seen[a.Name] = true + switch a.Kind { + case ArtifactImage, ArtifactArchive, ArtifactUpstream, ArtifactBundle, ArtifactPackage: + default: + problems = append(problems, fmt.Sprintf( + "%s: %q is a %q, and an artifact is %q, %q, %q or %q", + module, a.Name, a.Kind, ArtifactImage, ArtifactArchive, ArtifactUpstream, + ArtifactBundle+", "+ArtifactPackage)) + } + // **A bundle is built from the module itself, so it says a language instead.** Everything + // else names what it is built from: a Dockerfile, a directory, somebody else's reference. + // A bundle's source is the module's own directory by definition, and what it needs to say + // is which compiler — because the mesh chooses that, and cannot choose for a module that + // has not said. + if len(a.External) > 0 && (a.Kind != ArtifactBundle || a.Language != "typescript") { + problems = append(problems, fmt.Sprintf( + "%s: %q names packages it keeps external, and only a TypeScript bundle is bundled into "+ + "one file with some kept out (novox/hq ADR 0193)", module, a.Name)) + } + if len(a.Env) > 0 && a.Kind != ArtifactBundle { + problems = append(problems, fmt.Sprintf( + "%s: %q is a %q and says what it is given (env). Only a bundle the node's runtime "+ + "serves is given words (novox/hq ADR 0192); a container says its own environment", + module, a.Name, a.Kind)) + } + if a.Kind == ArtifactBundle || a.Kind == ArtifactPackage { + // **Except for a language that compiles to a binary, where it names which one** + // (novox/hq 04-ISSUES/142). A bundle in an interpreted language is the module's own + // directory compiled whole, and naming a source would be describing its own build. A + // repository written in a compiled language holds several commands — the host and its + // bootstrap live in one, and the mesh needs the host — and "the module's own directory" + // is then not a package at all. So the compiled case may say which package, and says + // the module root by saying nothing. + if a.From != "" && !compilesToABinary(a.Language) { + problems = append(problems, fmt.Sprintf( + "%s: %q is a bundle and names what it is built from (%q). A bundle is built "+ + "from the module's own directory; what it says is the language", + module, a.Name, a.From)) + } + if strings.TrimSpace(a.Language) == "" { + problems = append(problems, fmt.Sprintf( + "%s: %q is a bundle and says no language, so nothing can choose a compiler "+ + "for it", module, a.Name)) + } + problems = append(problems, bundleEnvProblems(module, a)...) + // What the runtime loads is among what was compiled (ADR 0175): a name here that is + // not an entrypoint is a file the bundle does not contain, and the runtime would + // fail to import it on every machine rather than here. + for _, load := range a.Loads { + found := false + for _, e := range a.Entrypoints { + found = found || e == load + } + // A bundle compiled to a binary is one executable: the runtime loads that or nothing + // (novox/hq ADR 0193). + if bin := BinaryOf(a); bin != "" { + found = load == bin + } + if !found { + problems = append(problems, fmt.Sprintf( + "%s: %q says the runtime loads %q, which is not among its entrypoints — "+ + "what is loaded is compiled, so it is named there too", module, a.Name, load)) + } + } + // **A system, for a language that compiles to a binary** (novox/hq ADR 0142). A binary + // is pinned to one operating system at link time so a host refuses to touch a machine + // it was not built for (novox/hq ADR 0005); an artifact that says nothing would be + // compiled for whatever the build machine happened to be, which reads as portable and + // is not. + if compiled := compilesToABinary(a.Language); compiled && strings.TrimSpace(a.System) == "" { + problems = append(problems, fmt.Sprintf( + "%s: %q is compiled to a binary and says no system, so it would be built for "+ + "whatever the build machine happens to be. Declare one artifact per "+ + "system: %s", module, a.Name, spokenSystems())) + } else if !compiled && strings.TrimSpace(a.System) != "" { + problems = append(problems, fmt.Sprintf( + "%s: %q names the system %q and is written in %q, which compiles to code that "+ + "runs anywhere — a system that decides nothing reads as though it did", + module, a.Name, a.System, a.Language)) + } else if compiled && !knownSystem(a.System) { + problems = append(problems, fmt.Sprintf( + "%s: %q is built for %q, and a system is %s", + module, a.Name, a.System, spokenSystems())) + } + } else { + if a.From == "" && a.URL == "" { + problems = append(problems, fmt.Sprintf( + "%s: %q says nothing about what it is built from", module, a.Name)) + } + if a.Language != "" { + problems = append(problems, fmt.Sprintf( + "%s: %q is a %q and names a language. Only a bundle is compiled by the mesh; "+ + "everything else brings its own recipe", module, a.Name, a.Kind)) + } + } + if a.Compiles != "" { + if a.Kind != ArtifactImage { + problems = append(problems, fmt.Sprintf( + "%s: %q is a %q and names a Go package it compiles. Only an image's recipe is "+ + "told which; a bundle names its package in from (novox/hq ADR 0267)", module, a.Name, a.Kind)) + } else if c := strings.TrimSpace(a.Compiles); strings.HasPrefix(c, "/") || c == ".." || + strings.HasPrefix(c, "../") || strings.Contains(c, "/../") || strings.HasSuffix(c, "/..") { + problems = append(problems, fmt.Sprintf( + "%s: %q compiles %q, which leaves the tree it is built in", module, a.Name, a.Compiles)) + } + } + // An upstream image is named, not read from the repository, so the path rule does not + // apply to it — and applying it anyway would refuse every reference with a registry host + // in it. + if a.Kind != ArtifactUpstream && + (strings.HasPrefix(a.From, "/") || strings.Contains(a.From, "..")) { + // A build reads its own repository and nothing else. A path leaving it would make + // what gets built depend on whatever happens to be on the machine building it. + problems = append(problems, fmt.Sprintf( + "%s: %q is built from %q, which is outside its own repository", + module, a.Name, a.From)) + } + if a.Kind == ArtifactUpstream && !strings.Contains(a.From, ":") { + // Without a tag or digest, what gets mirrored is whatever `latest` means today, and + // a module pinned to that is not pinned. + problems = append(problems, fmt.Sprintf( + "%s: %q mirrors %q, which names no tag or digest", module, a.Name, a.From)) + } + problems = append(problems, upstreamFileProblems(module, a)...) + } + return problems +} + +// reservedSuffixes are names no public vendor publishes under: an installation's own and documentation's. +var reservedSuffixes = []string{".internal", ".local", ".lan", ".home", ".corp", ".localhost", ".home.arpa", + ".test", ".invalid", ".example", ".intranet", ".private"} + +// notAPublicHost says why a host is not a public vendor's name, or "". +func notAPublicHost(host string) string { + h := strings.ToLower(strings.TrimSuffix(host, ".")) + if net.ParseIP(h) != nil { + return "an address rather than a name" + } + if !strings.Contains(h, ".") { + return "a name with no domain" + } + for _, s := range reservedSuffixes { + if strings.HasSuffix(h, s) { + return "a name under " + s + ", which is nobody's public domain" + } + } + return "" +} + +// sha256Hex is a pinned sha256 as a manifest writes it: 64 lower-case hexadecimal characters. +var sha256Hex = regexp.MustCompile(`^[0-9a-f]{64}$`) + +// upstreamFileProblems is what is wrong with an archive pinned by URL (novox/hq ADR 0316 §2, §6): it is an +// archive, fetched over https, pinned by a sha256 and a version, and it names no directory of the repository +// besides; a file name or a directory within is a plain path that stays where it is put. +func upstreamFileProblems(module string, a Artifact) []string { + pinned := a.URL != "" || a.SHA256 != "" || a.Version != "" || a.File != "" || a.Within != "" + if !pinned { + return nil + } + var problems []string + say := func(format string, args ...any) { + problems = append(problems, fmt.Sprintf("%s: %q ", module, a.Name)+fmt.Sprintf(format, args...)) + } + if a.Kind != ArtifactArchive { + say("is a %q and names an upstream file (url, sha256, version, file, within): only an archive is "+ + "mirrored from one (novox/hq ADR 0316)", a.Kind) + return problems + } + if a.URL == "" { + say("names a sha256, a version, a file or a directory within, and no url to fetch it from") + return problems + } + if a.From != "" { + say("is built from %q and fetched from %s: an archive is one or the other", a.From, a.URL) + } + if u, err := url.Parse(a.URL); err != nil || u.Scheme != "https" || u.Host == "" { + say("is fetched from %q, and an upstream file is fetched over https from a named host", a.URL) + } else if why := notAPublicHost(u.Hostname()); why != "" { + say("is fetched from %s, %s: an upstream file is somebody else's release, published under a public name", a.URL, why) + } + if !sha256Hex.MatchString(a.SHA256) { + say("is fetched from %s without a pinned sha256 (64 lower-case hexadecimal characters): what that "+ + "address serves changes under the mesh, and only the digest says it is what was reviewed", a.URL) + } + if strings.TrimSpace(a.Version) == "" { + say("pins %s and says no version, so a person reading the pin cannot tell which release it is", a.URL) + } + if a.File != "" && a.Within != "" { + say("names a file (%q) and a directory within (%q): a file is kept whole, a directory is taken out "+ + "of an archive", a.File, a.Within) + } + for field, p := range map[string]string{"file": a.File, "within": a.Within} { + if p == "" { + continue + } + clean := path.Clean(p) + if clean != p || strings.HasPrefix(p, "/") || clean == "." || clean == ".." || strings.HasPrefix(clean, "../") || + (field == "file" && strings.Contains(p, "/")) { + say("names %s %q, which is not a plain relative %s", field, p, map[string]string{ + "file": "file name", "within": "directory"}[field]) + } + } + return problems +} + +// oneOrOther keeps the message readable for one artifact and for several, because a message that +// says "neither" about one thing reads as a bug in the message. +func oneOrOther(n int) string { + if n == 1 { + return "it" + } + return "them" +} + +// Systems the mesh builds binaries for, which is the set a host may be pinned to (novox/hq ADR 0005). +// +// **A closed list, and the host's own, not the compiler's.** These are not the values a Go toolchain +// would call an operating system — the difference between two of them is a C library, not a kernel. +// They are what a machine reports itself to be and what a host is linked to refuse, so the list that +// matters is the one the host understands. +var systems = []string{"alpine", "android", "arch"} + +// knownSystem is whether the mesh builds for it. +func knownSystem(system string) bool { + want := strings.ToLower(strings.TrimSpace(system)) + for _, s := range systems { + if s == want { + return true + } + } + return false +} + +// spokenSystems is the list as a refusal says it, so a reader is one edit from right. +func spokenSystems() string { + return strings.Join(systems, ", ") +} + +// compilesToABinary is whether this language's bundle is a binary for one operating system rather +// than code that runs wherever its interpreter does. +// +// **Asked of the language, not of the artifact.** A module says what it is written in; what that +// implies is the mesh's to know, exactly as the compiler is (novox/hq ADR 0142). Asking the artifact +// would let two artifacts in one language disagree about whether they are portable. +func compilesToABinary(language string) bool { + switch strings.ToLower(strings.TrimSpace(language)) { + case "go": + return true + default: + return false + } +} + +// versionRef is how a resource names the version of the artifact it uses: ${version}. +// +// No artifact name in it, because the resource already says which artifact it is for — a second +// name would be a second thing to keep in step with the first. +const versionRef = "${version}" + +// versionOf is an artifact's version as a path names it: its digest, short. +// +// **Content-addressed on purpose.** The alternative is the commit a build came from, and two builds +// of one commit are meant to produce the same bytes — every toolchain here is `-trimpath` for that +// reason. A commit-named path would move for an identical binary, and everything reading that path +// would be recreated for a change that is not one. A digest-named path moves exactly when the bytes +// do. +// +// Twelve hex characters: enough that two of this mesh's builds will not collide, short enough to +// read in a path and in a journal line. The `sha256:` prefix goes, because a directory name carrying +// a colon is a directory name people quote wrong. +func versionOf(digest string) string { + hex := strings.TrimPrefix(strings.TrimSpace(digest), "sha256:") + if len(hex) > 12 { + return hex[:12] + } + return hex +} + +// bundleEnvWords are the words the runtime sets for itself; a bundle that named one would be +// telling the runtime what it is, which is the mesh's to say (novox/hq ADR 0192). +var bundleEnvWords = map[string]bool{ + RuntimeToolModules: true, RuntimeBrokerFile: true, RuntimeOperatorAccount: true, + RuntimeOperatorHome: true, RuntimeToolEnv: true, RuntimeAgentAccount: true, RuntimeAgentHome: true, +} + +// bundleEnvProblems says what is wrong with what a bundle says it is given (novox/hq ADR 0192): +// a value is a path or a constant written with the references a container's environment may use +// for a place or a port, and never a secret's content or another module's binding — a secret +// reaches a tool as a file whose path is named. +func bundleEnvProblems(module string, a Artifact) []string { + if len(a.Env) == 0 { + return nil + } + var problems []string + for _, word := range sortedKeys(a.Env) { + value := a.Env[word] + if bundleEnvWords[word] { + problems = append(problems, fmt.Sprintf( + "%s: %q gives itself %s, which the node's runtime sets for itself; a bundle is "+ + "given its own words beside the runtime's, never in place of them (novox/hq ADR 0192)", + module, a.Name, word)) + } + rest := ofPort.ReplaceAllString(dirRef.ReplaceAllString(value, ""), "") + if strings.Contains(rest, "${") { + problems = append(problems, fmt.Sprintf( + "%s: %q gives %s the value %q. A bundle's word is a path or a constant, written with "+ + "${dir:…} and ${port:…} only; a secret reaches a tool as a file the mesh places, "+ + "named by its path, never as its content (novox/hq ADR 0192)", + module, a.Name, word, value)) + } + } + return problems +} + +func copyWords(in map[string]string) map[string]string { + if len(in) == 0 { + return nil + } + out := make(map[string]string, len(in)) + for k, v := range in { + out[k] = v + } + return out +} + +// BinaryOf is what a bundle compiled to a binary is called once built: what the artifact names, else +// the package it is built from, else the artifact's own name (novox/hq 04-ISSUES/142). Empty for a +// language that does not compile to one. The builder writes the binary under this name, and the +// composer runs it by it, so both ask here. +func BinaryOf(a Artifact) string { + if !compilesToABinary(a.Language) { + return "" + } + if name := strings.TrimSpace(a.Binary); name != "" { + return name + } + if from := strings.Trim(a.From, "./"); from != "" { + return path.Base(from) + } + return a.Name +} + +// runByAResource is the artifacts one of a module's own resources names — a process that runs it, +// a step, an archive that unpacks it — by name. +func runByAResource(m Manifest) map[string]bool { + named := map[string]bool{} + for _, r := range m.Resources { + if a, ok := r["artifact"].(string); ok && a != "" { + named[a] = true + } + } + return named +} + +// undeliveredBundles says which of a module's bundles nothing would ever put on a machine (novox/hq +// 04-ISSUES/216). A bundle reaches a machine three ways: the node's runtime serves it (it says +// `loads`, or its module declares `tools`), a resource names it (a process, a step, an archive), or +// it is the runtime itself. One reached by none of them was built, recorded and pushed as success, +// and was simply absent — seven modules' tools went missing that way on 2026-10-03. Refused here, +// naming the field that would deliver it. +func undeliveredBundles(m Manifest) []string { + if m.Build == nil || m.Module == RuntimeModule { + return nil + } + named := runByAResource(m) + var problems []string + for _, a := range m.Build.Artifacts { + if a.Kind != ArtifactBundle || named[a.Name] || len(a.Loads) > 0 || len(m.Tools) > 0 { + continue + } + problems = append(problems, fmt.Sprintf( + "%s: the bundle %q would be built and never reach a machine: nothing loads it, runs it or "+ + "unpacks it. A tools bundle says `loads` (the entrypoints the node's runtime serves) or its "+ + "module lists its `tools`; a daemon or a step is a resource naming it (novox/hq 04-ISSUES/216)", + m.Module, a.Name)) + } + return problems +} diff --git a/go/catalogue/busy.go b/go/catalogue/busy.go new file mode 100644 index 0000000..18cbbf0 --- /dev/null +++ b/go/catalogue/busy.go @@ -0,0 +1,124 @@ +package catalogue + +import ( + "fmt" + "sort" + "strings" + "time" +) + +// A long-running container may say it is busy (novox/hq ADR 0335 §1). +// +// **Beside its health, on the container**: a command run inside it, by its shell, as an exec health check +// is, that exits 0 while the container holds work a replacement would lose and non-zero when it is idle; +// and how long one ask may take. Being asked is also its notice that it is about to be replaced: from the +// first ask it takes no new work. The node-engine asks before it replaces the container, leaves it running +// while it says busy, and replaces it on the first pass it says idle, or at 15 minutes — the engine's bound, +// never the module's. +// +// Sent only to a node-engine that reads it (link.BusyContract): an older, strict engine refuses a field it +// does not know, whole, and to it the container is replaced at once, as before. + +// BusyField is the resource field. +const BusyField = "busy" + +// The bounds of one ask (ADR 0335), the node-engine's own, held here near the author. +const ( + BusyTimeoutDefault = 10 * time.Second + BusyTimeoutCeiling = time.Minute +) + +// Busy is one container's busy, read. +type Busy struct { + Command string + Timeout time.Duration +} + +// ReadBusy reads a resource's `busy` field, its default applied. False when it carries none. +func ReadBusy(r map[string]any) (Busy, bool, []string) { + raw, present := r[BusyField] + if !present { + return Busy{}, false, nil + } + id := fmt.Sprint(r["id"]) + fields, ok := raw.(map[string]any) + if !ok { + return Busy{}, true, []string{fmt.Sprintf("%s: busy is %T; it is an object with a command", id, raw)} + } + var problems []string + keys := make([]string, 0, len(fields)) + for k := range fields { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + if k != "command" && k != "timeout" { + problems = append(problems, fmt.Sprintf("%s: its busy says %q; busy has a command and a timeout", id, k)) + } + } + b := Busy{Timeout: BusyTimeoutDefault} + if v, has := fields["command"]; has { + s, ok := v.(string) + if !ok { + problems = append(problems, fmt.Sprintf("%s: its busy's command is %T; it is text", id, v)) + } + b.Command = strings.TrimSpace(s) + } + if v, has := fields["timeout"]; has { + s, _ := v.(string) + d, err := time.ParseDuration(strings.TrimSpace(s)) + switch { + case err != nil: + problems = append(problems, fmt.Sprintf("%s: its busy's timeout is %v; it is a duration such as \"10s\"", id, v)) + case d <= 0 || d > BusyTimeoutCeiling: + problems = append(problems, fmt.Sprintf("%s: an ask of its busy takes more than nothing and at most %s, not %s", + id, BusyTimeoutCeiling, d)) + default: + b.Timeout = d + } + } + return b, true, problems +} + +// busyProblems is everything wrong with a manifest's `busy` fields, in the manifest's words: a command on +// one line, a timeout within its bounds, and only on a container that stays up. +func busyProblems(m Manifest) []string { + var problems []string + for _, r := range m.Resources { + b, has, read := ReadBusy(r) + if !has { + continue + } + where := m.Module + ": " + fmt.Sprint(r["id"]) + for _, p := range read { + problems = append(problems, m.Module+": "+p) + } + if fmt.Sprint(r["type"]) != "container" || !LongRunning(r) { + problems = append(problems, fmt.Sprintf("%s declares busy and is not a container that stays up: busy is "+ + "asked of a running container before it is replaced (ADR 0335)", where)) + } + if b.Command == "" { + problems = append(problems, where+"'s busy names no command: one run inside the container, exiting 0 "+ + "while it is busy") + } + if strings.ContainsAny(b.Command, "\n\r") { + problems = append(problems, where+"'s busy command is one line") + } + } + return problems +} + +// busyInto composes a container's `busy` into the node-engine's words, or takes it away: for an engine that +// reads it, the command and its timeout written out; for one that does not, the field is not sent, and the +// container is replaced at once, as before. +func busyInto(resource map[string]any, with Rendering) { + if _, has := resource[BusyField]; !has { + return + } + if !with.ReadsBusy { + delete(resource, BusyField) + return + } + b, _, _ := ReadBusy(resource) + resource[BusyField] = map[string]any{"command": b.Command, "timeout": b.Timeout.String()} +} diff --git a/go/catalogue/chosen.go b/go/catalogue/chosen.go new file mode 100644 index 0000000..1fb8233 --- /dev/null +++ b/go/catalogue/chosen.go @@ -0,0 +1,100 @@ +package catalogue + +import ( + "sort" +) + +// Chosen is the provider somebody named for a provision: the module, and the node it runs on. Both, +// always (novox/hq #258) — a provision comes from a module, and the same module on two machines is +// two answers, so neither half alone says which. Module is empty only on a record made before this +// was asked, and such a record is honoured exactly as long as it is unambiguous. +type Chosen struct { + Node string + Module string +} + +func (c Chosen) String() string { + if c.Module == "" { + return c.Node + } + return c.Node + "/" + c.Module +} + +// matches is whether this provider is the one chosen. +func (c Chosen) matches(p Provider) bool { + return p.Node == c.Node && (c.Module == "" || p.Module == c.Module) +} + +// among is every offered provider the choice names — one, when the choice is whole. +func (c Chosen) among(where []Provider) []Provider { + var out []Provider + for _, p := range where { + if c.matches(p) { + out = append(out, p) + } + } + return out +} + +// nameOf is how a refusal names a provider: the node and the module on it. +func nameOf(p Provider) string { + return Chosen{Node: p.Node, Module: p.Module}.String() +} + +// providerNames is every provider named, sorted, for a refusal to list. +func providerNames(where []Provider) []string { + out := make([]string, 0, len(where)) + for _, p := range where { + out = append(out, nameOf(p)) + } + sort.Strings(out) + return out +} + +// providersHere is which modules in this node's own set offer a provision, sorted. +func providersHere(catalogue map[string]Manifest, here func(string) bool, want string) []string { + var out []string + for name, m := range catalogue { + if !here(name) { + continue + } + for _, o := range m.Offers() { + if o == want { + out = append(out, name) + break + } + } + } + sort.Strings(out) + return out +} + +// servedByOne is what one provider beside the consumer says a consumer needs to know, or nothing. +// +// Serving is *whether* a need is created at all when the provider is on this same machine (novox/hq +// 04-ISSUES/038's sibling): a need never created is a binding the consumer never gets. The manifest +// alone answers that; the values are settled later, with the node's settings. +func servedByOne(m Manifest, want string) map[string]any { + if _, ok := m.Serves[want]; ok { + return ServedOn(m, want, nil) + } + return nil +} + +// sharedByOne is the own secret that provider names as its credential (ADR 0158), or "" when it +// gives each consumer its own. +func sharedByOne(m Manifest, want string) string { + if own, shared := m.SharedCredentialOf(want); shared { + return own + } + return "" +} + +func oneOf(list []string, s string) bool { + for _, x := range list { + if x == s { + return true + } + } + return false +} diff --git a/go/catalogue/collections.go b/go/catalogue/collections.go new file mode 100644 index 0000000..c2a6562 --- /dev/null +++ b/go/catalogue/collections.go @@ -0,0 +1,635 @@ +package catalogue + +import ( + "fmt" + "path" + "regexp" + "slices" + "sort" + "strconv" + "strings" +) + +// Collections (novox/hq ADR 0349 clause 13, design 57 L1, issue 684): a provision with many providers. +// +// **A collector requires a collection; a contributor provides an item from a template.** A module taking additions +// from others names a collection in a file of its own, `${collected:}`, and declares it under +// `collections`, with the comment its tool writes a line in. Each contributor ships its item as a template of its +// own repository and declares it under `items`: `{"collection": …, "template": "files/"}`. Its build packs the +// template with the module's others, and the mesh holds its text as it holds a file's (template.go). +// +// **The node's provisioner client fills both.** An item is filled on the node by the client of the node its +// contributor is placed on, from the same owners as a file, and recorded in mesh-provisioner_collect; a collector's +// file is filled by the same client from every item of its collection recorded for that node, so an item goes when +// its contributor leaves. So a module contributing an item is a moved module (handed.go), and the controller renders +// no item: the plan of a contributor names its items and their templates, and the plan of a collector names each +// collection it reads and its comment. Neither carries a contributor's text. +// +// **The three old namespaces read as collections.** `${shell::}`, `${environment:}` and +// `${contribution::}` name, in a handed collector's file, the collection of the same name — +// `shell:zsh:normal` — with the comment the controller wrote before each module's piece. An item is contributed to +// one by that name, by a module the old form would have let contribute it. Each file is filled one way, never +// both (ADR 0349 clause 17): a handed collector is filled from items alone and an unhanded one from the old forms +// alone, and a piece of the other form on the same machine is said as not placed, never dropped quietly. + +// Item is one item a module contributes to a collection: its template in the module's repository, or, in the +// manifest the mesh holds, the template's text, read back from the build's templates archive like a file's. +// +// **Attributes, a capability and a name** (design 57 L3, issue 777): an item may carry `attributes` (string keys and +// values; `order`, a whole number 0–99 defaulting to 50, orders the items of one placeholder) and `if-capability`, +// the capability a machine must report for the item to be placed on it. A module may give several items to one +// collection, each then with a `name` unique within the module and the collection. A collector takes only the items +// whose attributes match, `${collected::=,…}`, and all of them without the selector. +type Item struct { + Collection string `json:"collection"` + Name string `json:"name,omitempty"` + Template string `json:"template,omitempty"` + Content string `json:"content,omitempty"` + Attributes map[string]string `json:"attributes,omitempty"` + IfCapability string `json:"if-capability,omitempty"` +} + +// ItemKey is an item's key among its module's: the collection, and `/` for a named item. +func ItemKey(collection, name string) string { + if name == "" { + return collection + } + return collection + "/" + name +} + +func (it Item) key() string { return ItemKey(it.Collection, it.Name) } + +// order is the item's place among the items of one placeholder: its `order` attribute, or OrderDefault. +func (it Item) order() int { + if n, err := strconv.Atoi(it.Attributes[OrderAttribute]); err == nil && n >= 0 && n <= 99 { + return n + } + return OrderDefault +} + +// OrderAttribute is the attribute that orders items. +const OrderAttribute = "order" + +// matches is whether the item's attributes carry every pair of the selector. +func (it Item) matches(where map[string]string) bool { + for k, v := range where { + if got, has := it.Attributes[k]; !has || got != v { + return false + } + } + return true +} + +// itemsInOrder is the module's items by order, then name; the order the old form placed a module's pieces in. +func itemsInOrder(items []Item) []Item { + out := append([]Item(nil), items...) + sort.SliceStable(out, func(a, b int) bool { + if oa, ob := out[a].order(), out[b].order(); oa != ob { + return oa < ob + } + return out[a].Name < out[b].Name + }) + return out +} + +// Collection is one collection a module collects, and the comment its tool writes a line in: each contributor's item +// is introduced by ` `, and by nothing without one. +type Collection struct { + Name string `json:"name"` + Comment string `json:"comment,omitempty"` +} + +// CollectedNamespace is a collection's placeholder namespace, `${collected:}`. +const CollectedNamespace = "collected" + +// EnvironmentCollection is the one collection of the session environment (design 57 L3, issue 795): its items are +// typed by attributes, a variable (`kind=variable`, `name`) or a PATH entry (`kind=path`, `at=start|end`), and the +// template's text is the value or the entry. A collector names the rendering, `${collected:environment:posix}` or +// `${collected:environment:systemd}`, which is not a selector. +const EnvironmentCollection = "environment" + +// oldCollectionNamespaces are the namespaces read as collections of their own name. +var oldCollectionNamespaces = []string{"shell", "environment", "contribution"} + +var ( + // ofCollected is where a collector places a collection. Loose inside the braces, so a misspelt name is refused. + ofCollected = regexp.MustCompile(`\$\{collected:([^}]*)\}`) + // collectionName is words joined by colons, so an old namespace's placeholder names its collection as it reads. + collectionName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*(:[a-z0-9][a-z0-9-]*)*$`) + // itemName, attributeKey and attributeValue are the words an item's name and attributes are made of; none holds + // a colon, a comma or an equals sign, which tell a selector from a collection's name. + itemName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`) + attributeKey = regexp.MustCompile(`^[a-z][a-z0-9-]*$`) + attributeValue = regexp.MustCompile(`^[a-z0-9][a-z0-9_.-]*$`) +) + +// splitSelector is a placeholder's name without its selector, and the selector: `widgets:bar=top,place=left` is +// `widgets` and `bar=top,place=left`. A collection's name holds no `=`, so the selector begins at the last colon +// before the first one. +func splitSelector(name string) (collection, selector string) { + eq := strings.Index(name, "=") + if eq < 0 { + return name, "" + } + colon := strings.LastIndex(name[:eq], ":") + if colon < 0 { + return name, "" + } + return name[:colon], name[colon+1:] +} + +// selectorOf is a selector's pairs, or why it is none. +func selectorOf(selector string) (map[string]string, error) { + out := map[string]string{} + if selector == "" { + return out, nil + } + for _, pair := range strings.Split(selector, ",") { + k, v, ok := strings.Cut(pair, "=") + switch { + case !ok || !attributeKey.MatchString(k) || !attributeValue.MatchString(v): + return nil, fmt.Errorf("%q in %q is not =", pair, selector) + case out[k] != "": + return nil, fmt.Errorf("%q names %s twice", selector, k) + } + out[k] = v + } + return out, nil +} + +// collectionOfPlaceholder is the collection a placeholder's namespace and key name, and whether they name one. +func collectionOfPlaceholder(namespace, key string) (string, bool) { + key, _ = splitSelector(key) + switch { + case namespace == CollectedNamespace && strings.HasPrefix(key, EnvironmentCollection+":"): + // The environment is one collection, and the rest of the name picks the rendering (issue 795). + return EnvironmentCollection, true + case namespace == CollectedNamespace: + return key, true + case namespace == "environment": + return EnvironmentCollection, true + case slices.Contains(oldCollectionNamespaces, namespace): + return namespace + ":" + key, true + } + return "", false +} + +// oldCollection is an old namespace's collection by its name: its comment, or why no item may be contributed to it. +// ok is false for a name that is not an old namespace's. +// +// forItem is whether an item is to be contributed to it: an item may be contributed to a kind received as data, +// which a collector may place as a collection only where its shape says its pieces are text (design 57 L3). +func oldCollection(name string, forItem bool) (comment string, old bool, wrong error) { + namespace, key, _ := strings.Cut(name, ":") + if !slices.Contains(oldCollectionNamespaces, namespace) { + return "", false, nil + } + switch namespace { + case "shell": + target, slot, two := strings.Cut(key, ":") + if !two || !oneOf(contributionTargets(), target) || !oneOf(knownSlots, slot) { + return "", true, fmt.Errorf("%s is no slot: shell code is shell::, the shell one of %s, "+ + "the session's file one of %s or the power moment one of %s, and the slot one of %s", name, + strings.Join(knownShells, ", "), strings.Join(sessionFiles, ", "), strings.Join(powerMoments, ", "), + strings.Join(knownSlots, ", ")) + } + // shellCode's line before each module's code. + return "#", true, nil + case "environment": + if key == "" { + // The collection itself, which an item is contributed to; a placeholder names a rendering. + return "#", true, nil + } + if key != EnvironmentPOSIX && key != EnvironmentSystemd { + return "", true, fmt.Errorf("%s is no format: the environment is environment:%s or environment:%s", + name, EnvironmentPOSIX, EnvironmentSystemd) + } + return "#", true, nil + } + seat, kind, _ := strings.Cut(key, ":") + s, r, ok := receivable(seat, kind) + switch { + case !ok && s.Name == "": + return "", true, fmt.Errorf("%s names the seat %q, which the mesh does not define", name, seat) + case !ok: + return "", true, fmt.Errorf("%s: %s; %s", name, s.Name, kindsOf(s)) + case r.Shape != nil && (forItem || collectableAsText(r)): + return r.Comment, true, nil + case r.Shape != nil: + return "", true, fmt.Errorf("%s is received by %s as data in its own shape, which no item of text is "+ + "(novox/hq ADR 0255)", name, s.Name) + case r.Dirs: + return "", true, fmt.Errorf("%s names its contributors' directories, which the holder reads as paths "+ + "the controller fills; it is not yet a collection (design 57 L1)", name) + case s.Name != canonicalSeat(seat): + return "", true, fmt.Errorf("%s names the seat by another name; a collection names it %s", name, s.Name) + } + return r.Comment, true, nil +} + +// placerOfOld is the seat whose holder alone may collect an old namespace's collection (ADR 0203, 0204, 0208, +// 0211, 0212): the same seat that places it the old way. +func placerOfOld(name string) string { + namespace, key, _ := strings.Cut(name, ":") + switch namespace { + case "shell": + target, _, _ := strings.Cut(key, ":") + return placerOf(target) + case "environment": + return EnvironmentSeat + } + seat, _, _ := strings.Cut(key, ":") + return canonicalSeat(seat) +} + +// collectionProblems is what is wrong with a module's collections and items, from the manifest alone. +func collectionProblems(m Manifest) []string { + var problems []string + declared := map[string]bool{} + for i, c := range m.Collections { + switch { + case !collectionName.MatchString(c.Name): + problems = append(problems, fmt.Sprintf("%s's collection %d is %q, which is not a collection's name: "+ + "words of lower-case letters, digits and hyphens, joined by colons", m.Module, i+1, c.Name)) + case declared[c.Name]: + problems = append(problems, fmt.Sprintf("%s collects %s twice", m.Module, c.Name)) + } + if _, old, _ := oldCollection(c.Name, false); old { + problems = append(problems, fmt.Sprintf("%s collects %s, which is read from its placeholder: a module "+ + "places it as ${%s} where it holds the seat that places it, and declares nothing", m.Module, c.Name, + c.Name)) + } + if strings.ContainsAny(c.Comment, "\n\r") { + problems = append(problems, fmt.Sprintf("%s's collection %s has a comment holding a line break", + m.Module, c.Name)) + } + declared[c.Name] = true + } + // A collection declared and placed nowhere is a word read by nothing; judged where every file's text is here. + if !m.Templated() { + named := map[string]bool{} + for _, r := range m.Resources { + if text, ok := r["content"].(string); ok { + for _, f := range ofCollected.FindAllStringSubmatch(text, -1) { + base, _ := splitSelector(f[1]) + named[base] = true + } + } + } + for _, c := range m.Collections { + if !named[c.Name] && collectionName.MatchString(c.Name) { + problems = append(problems, fmt.Sprintf("%s collects %s and no file of its places it as "+ + "${collected:%s}", m.Module, c.Name, c.Name)) + } + } + } + contributed := map[string]bool{} + for i, it := range m.Items { + where := fmt.Sprintf("%s's item %d", m.Module, i+1) + if !collectionName.MatchString(it.Collection) { + problems = append(problems, fmt.Sprintf("%s is to %q, which is not a collection's name", where, + it.Collection)) + } else if _, old, wrong := oldCollection(it.Collection, true); old && wrong != nil { + problems = append(problems, fmt.Sprintf("%s: %v", where, wrong)) + } + switch { + case it.Name != "" && !itemName.MatchString(it.Name): + problems = append(problems, fmt.Sprintf("%s: the name %q is not a name: lower-case letters, digits and "+ + "hyphens", where, it.Name)) + case contributed[it.key()]: + problems = append(problems, fmt.Sprintf("%s contributes to %s twice; a module contributes one item to "+ + "a collection, or several each with a name of its own", m.Module, it.key())) + case it.Name == "" && slices.ContainsFunc(m.Items, func(o Item) bool { + return o.Collection == it.Collection && o.Name != "" + }): + problems = append(problems, fmt.Sprintf("%s contributes several items to %s and %s has no name; each "+ + "of several items has a name of its own", m.Module, it.Collection, where)) + } + contributed[it.key()] = true + problems = append(problems, itemAttributeProblems(m.Module, where, it)...) + if it.Collection == EnvironmentCollection { + problems = append(problems, environmentItemProblems(where, it)...) + } + switch { + case it.Template == "" && it.Content == "": + problems = append(problems, fmt.Sprintf("%s names no template: an item is a template of the module's "+ + "own, \"template\": \"%s/\" (novox/hq ADR 0349 §13)", where, m.TemplatesDir())) + case it.Template != "" && it.Content != "": + problems = append(problems, fmt.Sprintf("%s names its template and says its content too", where)) + case it.Template != "": + if wrong := itemTemplateProblem(m, it.Template); wrong != "" { + problems = append(problems, fmt.Sprintf("%s: %s", where, wrong)) + } + // **Filled by the provisioner's client alone**: the controller renders no item. + if !m.Moved { + problems = append(problems, fmt.Sprintf("%s: an item is filled by the provisioner's client of the "+ + "node its contributor is placed on, so a module contributing one is marked moved "+ + "(\"moved\": true; novox/hq ADR 0349 §13, §17)", where)) + } + } + if it.Content != "" { + problems = append(problems, itemTextProblems(m.Module, it)...) + } + } + return problems +} + +// itemAttributeProblems is what is wrong with an item's attributes and capability. The attributes of an item of a kind +// received as data are the fields its shape places by (and `order`), each with a value the shape takes. +func itemAttributeProblems(module, where string, it Item) []string { + var problems []string + for _, k := range sortedKeys(it.Attributes) { + v := it.Attributes[k] + switch { + case !attributeKey.MatchString(k): + problems = append(problems, fmt.Sprintf("%s: the attribute %q is not a name: lower-case letters, digits "+ + "and hyphens", where, k)) + continue + case !attributeFits(it.Collection, k, v): + problems = append(problems, fmt.Sprintf("%s: %s=%q is not a value: lower-case letters, digits, hyphens, "+ + "dots and underscores (an environment variable's name, letters, digits and underscores)", where, k, v)) + continue + } + if n, err := strconv.Atoi(v); k == OrderAttribute && (err != nil || n < 0 || n > 99) { + problems = append(problems, fmt.Sprintf("%s: order=%q is not a whole number from 0 to 99", where, v)) + } + } + if namespace, key, _ := strings.Cut(it.Collection, ":"); namespace == "contribution" { + seat, kind, _ := strings.Cut(key, ":") + if _, r, ok := receivable(seat, kind); ok && r.Shape != nil { + for _, k := range sortedKeys(it.Attributes) { + var place *PlaceField + for i := range r.Shape.Places { + if r.Shape.Places[i].Field == k { + place = &r.Shape.Places[i] + } + } + switch { + case k == OrderAttribute: + case place == nil: + problems = append(problems, fmt.Sprintf("%s: %s is placed by %s, and %q is none of them", where, + it.Collection, placeFields(r), k)) + case !oneOf(place.Values, it.Attributes[k]): + problems = append(problems, fmt.Sprintf("%s: %s=%s; %s is one of %s", where, k, it.Attributes[k], k, + strings.Join(place.Values, ", "))) + } + } + } + } + switch { + case it.IfCapability == "": + case !capabilityName.MatchString(it.IfCapability): + problems = append(problems, fmt.Sprintf("%s: if-capability %q is not a capability's name", where, it.IfCapability)) + case !oneOf(KnownCapabilities, it.IfCapability): + problems = append(problems, fmt.Sprintf("%s: if-capability %q is one no machine reports; the node-engine "+ + "detects %s", where, it.IfCapability, strings.Join(KnownCapabilities, ", "))) + } + return problems +} + +// AttributeFits is whether an item's attribute is of the shape its collection allows: words of lower case, except +// the environment's `name`, which is a variable's name (design 57 L3, issue 795). The same rule is mesh-catalog's +// provisioner client's, and the inventory's. +func attributeFits(collection, k, v string) bool { + if collection == EnvironmentCollection && k == "name" { + return variableName.MatchString(v) + } + return attributeKey.MatchString(k) && attributeValue.MatchString(v) +} + +// environmentItemProblems is what is wrong with an item of the environment collection: typed by `kind`, a variable +// with its `name`, a PATH entry `at` the start or the end, and no attribute besides `order`. +func environmentItemProblems(where string, it Item) []string { + var problems []string + for _, k := range sortedKeys(it.Attributes) { + if !slices.Contains([]string{"kind", "name", "at", OrderAttribute}, k) { + problems = append(problems, fmt.Sprintf("%s: %s is typed by kind, name and at, and %q is none of them", + where, it.Collection, k)) + } + } + switch kind := it.Attributes["kind"]; kind { + case "variable": + if it.Attributes["name"] == "" || it.Attributes["at"] != "" { + problems = append(problems, fmt.Sprintf("%s: a variable names itself with name= and has no at", where)) + } else if it.Attributes["name"] == "PATH" { + problems = append(problems, fmt.Sprintf("%s: PATH is not a variable; a PATH entry is kind=path", where)) + } + case "path": + if it.Attributes["name"] != "" || (it.Attributes["at"] != PathAtStart && it.Attributes["at"] != PathAtEnd) { + problems = append(problems, fmt.Sprintf("%s: a PATH entry is at=%s or at=%s and has no name", where, + PathAtStart, PathAtEnd)) + } + default: + problems = append(problems, fmt.Sprintf("%s: an item of %s is kind=variable or kind=path, not %q", where, + it.Collection, kind)) + } + return problems +} + +// itemTemplateProblem is what is wrong with an item's template path, or "". +func itemTemplateProblem(m Manifest, name string) string { + dir := m.TemplatesDir() + if strings.HasPrefix(name, "/") || path.Clean(name) != name || !strings.HasPrefix(name, dir+"/") || + strings.Contains(name, "\\") { + return fmt.Sprintf("the template %q is not a file under %s/ of its own repository: a template is %q, a "+ + "plain relative path, and the build packs only %s/", name, dir, dir+"/", dir) + } + return "" +} + +// itemTextProblems is what an item's text may not name: a secret or a bound value, since an item is kept in a +// bucket and no secret is (ADR 0349 clause 7), and a collection, since an item is not a collector. And every +// placeholder no pass fills, as in a file's content. +func itemTextProblems(module string, it Item) []string { + var problems []string + for _, f := range templatePlaceholder.FindAllStringSubmatch(it.Content, -1) { + namespace, key := f[1], f[2] + switch _, collection := collectionOfPlaceholder(namespace, key); { + case namespace == "secret" || namespace == "bound": + problems = append(problems, fmt.Sprintf("%s's item of %s names %s: an item is kept in a bucket, where "+ + "no secret is (novox/hq ADR 0349 §7)", module, it.Collection, f[0])) + case collection: + problems = append(problems, fmt.Sprintf("%s's item of %s names %s: an item is not a collector", + module, it.Collection, f[0])) + } + } + return append(problems, unconsumedPlaceholders(module, map[string]any{"id": "item:" + it.key(), + "type": "file", "content": it.Content})...) +} + +// isOld is whether a collection's name is an old namespace's. +func isOld(name string) bool { + _, old, _ := oldCollection(name, true) + return old +} + +// collectedPlaceholderProblems is what is wrong with one resource's ${collected:…}: placed only in a file's content, +// and naming a collection the module declares it collects. +func collectedPlaceholderProblems(m Manifest, r map[string]any) []string { + var problems []string + for _, field := range sortedKeys(r) { + v, ok := r[field].(string) + if !ok { + continue + } + for _, f := range ofCollected.FindAllStringSubmatch(v, -1) { + base, selector := splitSelector(f[1]) + if _, err := selectorOf(selector); err != nil { + problems = append(problems, fmt.Sprintf("%s's resource %v names %s: %v", m.Module, r["id"], f[0], err)) + continue + } + switch { + case field != "content": + problems = append(problems, fmt.Sprintf("%s's resource %v names %s in its %s; a collection is placed "+ + "only in a file's content", m.Module, r["id"], f[0], field)) + case isOld(base): + if _, _, wrong := oldCollection(base, false); wrong != nil { + problems = append(problems, fmt.Sprintf("%s's resource %v names %s: %v", m.Module, r["id"], f[0], wrong)) + } + case !slices.ContainsFunc(m.Collections, func(c Collection) bool { return c.Name == base }): + problems = append(problems, fmt.Sprintf("%s's resource %v names %s, and %s does not collect %q: "+ + "declare it under collections (novox/hq ADR 0349 §13)", m.Module, r["id"], f[0], m.Module, base)) + } + } + } + return problems +} + +// collectionComment is the comment of a collection a module's file names: declared, or the old namespace's. +func collectionComment(m Manifest, name string) (string, error) { + if comment, old, wrong := oldCollection(name, false); old { + return comment, wrong + } + for _, c := range m.Collections { + if c.Name == name { + return c.Comment, nil + } + } + return "", fmt.Errorf("%s names ${collected:%s} and does not collect it", m.Module, name) +} + +// itemsLeftOut says each item of a collection a file is filled with the old way that the controller could not render +// there (heldItemText): the collector is not handed to the provisioner on this machine (ADR 0349 clause 17). Said, +// never dropped quietly. +func itemsLeftOut(modules []Manifest, holder Manifest, collection string, facts map[string]string) []string { + return itemsLeftOutFor(modules, holder, collection, nil, facts, nil) +} + +// itemsLeftOutFor is itemsLeftOut for a placeholder that selects by attributes (where), on a machine with these +// capabilities: an item the selector does not take, or whose capability the machine lacks, is not left out but not +// wanted here. +func itemsLeftOutFor(modules []Manifest, holder Manifest, collection string, where map[string]string, + facts map[string]string, caps map[string]bool) []string { + var out []string + for _, m := range inModuleOrder(modules) { + for _, it := range itemsInOrder(m.Items) { + if it.Collection != collection || !it.matches(where) || (it.IfCapability != "" && !caps[it.IfCapability]) { + continue + } + if _, rendered := heldText(it, facts); rendered && renderedOldWay(collection) { + continue + } + out = append(out, fmt.Sprintf("%s's item of %s is not placed: %s is filled the old way on this "+ + "machine, from contributions, and an item only by the provisioner's client (design 57 L1)", + m.Module, it.key(), holder.Module)) + } + } + return out +} + +// oldFormsLeftOut says each piece of an old form a handed collector's collections would have held: the collector is +// filled from items alone, and the piece's module contributes none (ADR 0349 clause 17). +func oldFormsLeftOut(modules []Manifest, holder Manifest, collections []PlanCollection) []string { + var out []string + for _, c := range collections { + namespace, key, _ := strings.Cut(c.Name, ":") + for _, m := range inModuleOrder(modules) { + old := false + switch namespace { + case "shell": + target, slot, _ := strings.Cut(key, ":") + old = slices.ContainsFunc(m.Shell, func(s ShellCode) bool { return s.For == target && s.Slot == slot }) + case "environment": + old = m.Environment != nil && (len(m.Environment.Variables) > 0 || len(m.Environment.Path) > 0) + case "contribution": + seat, kind, _ := strings.Cut(key, ":") + old = slices.ContainsFunc(m.allContributions(), func(s SeatContribution) bool { + return canonicalSeat(s.Seat) == canonicalSeat(seat) && s.Kind == kind + }) + } + if old { + out = append(out, fmt.Sprintf("%s's %s is not placed: %s is filled by the provisioner's client on "+ + "this machine, from items, and %s contributes it the old way (design 57 L1)", m.Module, c.Name, + holder.Module, m.Module)) + } + } + } + return out +} + +// heldItemText is a moved contributor's items of an old namespace's collection as the controller renders them into a +// collector not handed on this machine (design 57 L2, issue 681): the text the mesh holds, read back from the +// contributor's templates archive, with the machine's facts filled, each ending its line. As a moved module's file +// keeps its text for a machine that cannot take it handed (handed.go), an item keeps its place in a collector filled +// the old way, so the file stays what its old form made it. Only `shell:` and `contribution:` collections, which the +// controller renders as text; false for a module with no such item, or an item whose text the mesh does not hold or +// which names anything but a machine fact this machine has, which is then said as not placed. +func heldItemText(m Manifest, collection string, facts map[string]string) (string, bool) { + items := heldItems(m, collection, nil, facts, nil) + var b strings.Builder + for _, h := range items { + b.WriteString(h.text) + } + return b.String(), len(items) > 0 +} + +// renderedOldWay is whether the controller renders a collection's items into a collector not handed. +func renderedOldWay(collection string) bool { + namespace, _, _ := strings.Cut(collection, ":") + return namespace == "shell" || namespace == "contribution" || namespace == EnvironmentCollection +} + +// heldItem is one item of a module rendered the old way. +type heldItem struct { + item Item + text string +} + +// heldItems is a module's items of the collection that the selector takes and the machine's capabilities allow, by +// order then name, each rendered as heldText does; the ones it cannot render are left out (itemsLeftOutFor says so). +func heldItems(m Manifest, collection string, where map[string]string, facts map[string]string, + caps map[string]bool) []heldItem { + if !renderedOldWay(collection) { + return nil + } + var out []heldItem + for _, it := range itemsInOrder(m.Items) { + if it.Collection != collection || !it.matches(where) || (it.IfCapability != "" && !caps[it.IfCapability]) { + continue + } + if text, ok := heldText(it, facts); ok { + out = append(out, heldItem{item: it, text: text}) + } + } + return out +} + +// heldText is an item's text with the machine's facts filled, ending its line; false for an item whose text the mesh +// does not hold or which names anything but a machine fact this machine has. +func heldText(it Item, facts map[string]string) (string, bool) { + if it.Content == "" { + return "", false + } + text := it.Content + for _, f := range templatePlaceholder.FindAllStringSubmatch(text, -1) { + value, has := facts[f[2]] + if f[1] != "machine" || !has { + return "", false + } + text = strings.ReplaceAll(text, f[0], value) + } + if !strings.HasSuffix(text, "\n") { + text += "\n" + } + return text, true +} diff --git a/go/catalogue/config.go b/go/catalogue/config.go new file mode 100644 index 0000000..0a54370 --- /dev/null +++ b/go/catalogue/config.go @@ -0,0 +1,504 @@ +package catalogue + +import ( + "encoding/json" + "fmt" + "regexp" + "slices" + "strings" +) + +// A module declares its config (novox/hq ADR 0348 §1, §4, §5, §6, §8, §11; design 57 package K1, issue 660). +// +// **One concept, two spellings for a while.** "Config" replaces "settings": the manifest says `config` for +// `settings`, `takes-config` for `takes-settings` and `${config:}` for `${setting:}`. The old names stay +// accepted as aliases until every module has moved (ADR 0348 §1), and a module uses one form or the other, never +// both. The config form is read into the very fields the settings form fills — Settings, a mergeable file's +// takes-settings and `${setting:}` — so composition has one path and `${config:}` fills exactly as +// `${setting:}` does: a module rewritten from settings to config composes byte-identically. It is written +// back in the form it was read in, so what the mesh holds is what its author wrote. +// +// **A moved module is held to the whole declaration.** Under `config` every value has a type (number, string or +// boolean; an object or a list only when trusted), a description including its unit, its validation (enum, +// minimum and maximum, pattern), whether it is trusted, optionally the predefined key it falls back to, and either +// a default or `required`; it lands in a file or is read at runtime (`runtime: true`); every mergeable file names +// the keys it takes; a `reload-on` or `restart-on` names a file the module declares. A module not yet moved keeps +// today's form and today's rules (SettingProblems, TakenSettingsProblems) until the catalogue's list of such +// modules is empty (ADR 0348 §11). +// +// **A required value nothing sets** is the operator's own: it composes as an operator's setting nobody gave does +// today, refused by name at composition (UnsetSettingError). Who reports it once mesh-config answers values is the +// provisioner's (ADR 0349), not this file's. + +// ConfigField is the manifest key a module declares its config values under. +const ConfigField = "config" + +// TakesConfigField is the key a mergeable file names the config values it takes by. +const TakesConfigField = "takes-config" + +// SettingsField is the old name of ConfigField, read as its alias. +const SettingsField = "settings" + +// configRef is how a file asks for a config value: ${config:}, the same keys as settingRef. +var configRef = regexp.MustCompile(`\$\{config:([a-z0-9][a-z0-9_.-]*)\}`) + +// takesConfigKey and takesSettingsKey are the two spellings as a key of a JSON object. A string value is never +// followed by a colon, and a quote inside a string is escaped, so these match only keys. +var ( + takesConfigKey = regexp.MustCompile(`"` + TakesConfigField + `"(\s*):`) + takesSettingsKey = regexp.MustCompile(`"` + TakesSettingsField + `"(\s*):`) +) + +// The types a config value may have (ADR 0348 §4). +const ( + ConfigNumber = "number" + ConfigString = "string" + ConfigBoolean = "boolean" + ConfigObject = "object" + ConfigList = "list" +) + +// accountFacts are what the mesh places for a module's account, never config values (ADR 0348 §11). +var accountFacts = map[string]bool{"puid": true, "pgid": true} + +// configForms is which of the two forms a manifest was written in, by the words of each it uses. +type configForms struct { + config, settings []string +} + +// DeclaresConfig is whether the manifest is written in the config form: `config`, `takes-config` or +// `${config:}` (ADR 0348 §1, §11). +func (m Manifest) DeclaresConfig() bool { return len(m.forms.config) > 0 } + +// toSettingsForm is raw with the config form's placeholders and takes-config read as the settings form's, and +// which words of each form it used. `config` itself is a top-level key, renamed where the keys are read +// (UnmarshalJSON). +func toSettingsForm(raw []byte) ([]byte, configForms) { + var f configForms + if configRef.Match(raw) { + f.config = append(f.config, "${config:…}") + } + if takesConfigKey.Match(raw) { + f.config = append(f.config, TakesConfigField) + } + if settingRef.Match(raw) { + f.settings = append(f.settings, "${setting:…}") + } + if takesSettingsKey.Match(raw) { + f.settings = append(f.settings, TakesSettingsField) + } + if len(f.config) == 0 { + return raw, f + } + raw = configRef.ReplaceAllFunc(raw, func(ref []byte) []byte { + return []byte("${setting:" + string(configRef.FindSubmatch(ref)[1]) + "}") + }) + raw = takesConfigKey.ReplaceAllFunc(raw, func(key []byte) []byte { + return []byte(`"` + TakesSettingsField + `"` + string(takesConfigKey.FindSubmatch(key)[1]) + ":") + }) + return raw, f +} + +// toConfigForm is a manifest written in the settings form's words, as JSON, in the config form's: what +// toSettingsForm and UnmarshalJSON read, written back. +func toConfigForm(raw []byte) ([]byte, error) { + var keys map[string]json.RawMessage + if err := json.Unmarshal(raw, &keys); err != nil { + return nil, err + } + if settings, has := keys[SettingsField]; has { + keys[ConfigField] = settings + delete(keys, SettingsField) + var err error + if raw, err = json.Marshal(keys); err != nil { + return nil, err + } + } + raw = settingRef.ReplaceAllFunc(raw, func(ref []byte) []byte { + return []byte("${config:" + string(settingRef.FindSubmatch(ref)[1]) + "}") + }) + raw = takesSettingsKey.ReplaceAllFunc(raw, func(key []byte) []byte { + return []byte(`"` + TakesConfigField + `"` + string(takesSettingsKey.FindSubmatch(key)[1]) + ":") + }) + return raw, nil +} + +// inConfigWords is a refusal said of a module in the config form in its words: `${config:}` and +// takes-config, where the rules shared with the settings form say `${setting:}` and takes-settings. +func inConfigWords(problem string) string { + problem = settingRef.ReplaceAllStringFunc(problem, func(ref string) string { + return "${config:" + settingRef.FindStringSubmatch(ref)[1] + "}" + }) + return strings.ReplaceAll(problem, TakesSettingsField, TakesConfigField) +} + +// MixedFormProblems is a manifest using both forms, said with the words of each (ADR 0348 §1). +func MixedFormProblems(m Manifest) []string { + if len(m.forms.config) == 0 || len(m.forms.settings) == 0 { + return nil + } + return []string{fmt.Sprintf("%s mixes the config form (%s) with the settings form (%s): a module uses one form "+ + "or the other, never both; write %s, %s and ${config:} throughout (novox/hq ADR 0348 §1)", + m.Module, strings.Join(m.forms.config, ", "), strings.Join(m.forms.settings, ", "), + ConfigField, TakesConfigField)} +} + +// configDeclaration is the config form's declaration as written: nothing of the settings form's, and nothing +// unsaid. +type configDeclaration struct { + Type string `json:"type,omitempty"` + Description string `json:"description,omitempty"` + Enum []any `json:"enum,omitempty"` + Minimum *float64 `json:"minimum,omitempty"` + Maximum *float64 `json:"maximum,omitempty"` + Pattern string `json:"pattern,omitempty"` + Trusted bool `json:"trusted,omitempty"` + Predefined string `json:"predefined,omitempty"` + Default any `json:"default,omitempty"` + Required bool `json:"required,omitempty"` + Runtime bool `json:"runtime,omitempty"` +} + +// MarshalJSON writes a settings declaration exactly as before, and a config declaration without the settings +// form's kind and why, and without a default it does not have. +func (d SettingDeclaration) MarshalJSON() ([]byte, error) { + type plain SettingDeclaration + if d.Kind != "" || d.Why != "" { + return json.Marshal(plain(d)) + } + return json.Marshal(configDeclaration{Type: d.Type, Description: d.Description, Enum: d.Enum, + Minimum: d.Minimum, Maximum: d.Maximum, Pattern: d.Pattern, Trusted: d.Trusted, Predefined: d.Predefined, + Default: d.Default, Required: d.Required, Runtime: d.Runtime}) +} + +// configShaped is whether a declaration says anything of the config form's. +func configShaped(d SettingDeclaration) bool { + return d.Type != "" || d.Description != "" || len(d.Enum) > 0 || d.Minimum != nil || d.Maximum != nil || + d.Pattern != "" || d.Trusted || d.Predefined != "" || d.Required || d.Runtime +} + +// TypedSettingProblems is a declaration under `settings` that says what only `config` says: a value with a type, +// a description and its validation is declared in the config form (ADR 0348 §1, §4). +func TypedSettingProblems(m Manifest) []string { + var problems []string + for _, key := range sortedSettingKeys(m.Settings) { + if configShaped(m.Settings[key]) { + problems = append(problems, fmt.Sprintf("%s's setting %q says its type, description or validation "+ + "under %s: a value declared with those is declared under config, with ${config:} and "+ + "takes-config, never under %s (novox/hq ADR 0348 §1, §4)", m.Module, key, SettingsField, SettingsField)) + } + } + return problems +} + +// ConfigProblems is every way a module in the config form declares its config wrongly, each said by name with +// what to do (ADR 0348 How it is checked, design 57 package K1). +func ConfigProblems(m Manifest) []string { + if !m.DeclaresConfig() { + return nil + } + var problems []string + placed := configPlaced(m) + fallsBack := fileFallsBack(m) + renamed := renamedTaken(m) + for _, key := range sortedSettingKeys(m.Settings) { + d := m.Settings[key] + say := func(format string, args ...any) { + problems = append(problems, fmt.Sprintf("%s's config value %q: ", m.Module, key)+fmt.Sprintf(format, args...)) + } + if !settingRef.MatchString("${setting:" + key + "}") { + say("not a usable key: lower-case letters, digits, dots, dashes and underscores") + continue + } + if meshWords[key] || accountFacts[key] { + say("%q is placement, which the mesh decides, never a config value (novox/hq ADR 0348 §11)", key) + continue + } + if d.Kind != "" || d.Why != "" { + say("kind and why are the settings form's; under config a value says its type and its description " + + "(novox/hq ADR 0348 §1, §4)") + } + switch d.Type { + case ConfigNumber, ConfigString, ConfigBoolean: + case ConfigObject, ConfigList: + if !d.Trusted { + say("is %s and not trusted: an object or a list decides more than one value can say, so it is "+ + "trusted, \"trusted\": true, or split into numbers, strings and booleans (novox/hq ADR 0348 §4)", d.Type) + } + case "": + say("no type: say \"type\": number, string or boolean, or object or list when trusted (novox/hq ADR 0348 §4)") + default: + say("type %q is none of number, string, boolean, object and list (novox/hq ADR 0348 §4)", d.Type) + } + if strings.TrimSpace(d.Description) == "" { + say("no description: say in a sentence what the value is, including its unit (novox/hq ADR 0348 §4)") + } + switch { + case d.Required && d.Default != nil: + say("both a default and required: a required value is the operator's own and has no default; give " + + "one or the other (novox/hq ADR 0348 §4)") + case !d.Required && d.Default == nil: + say("neither a default nor required: give a default, or say \"required\": true for a value that is " + + "the operator's own (novox/hq ADR 0348 §4)") + } + if as := renamed[key]; as != "" && d.Required { + say("is laid into its file as %q, and only when it is set, so the file and its code fall back when "+ + "it is not: it is never required; give it the file's default, the empty value of its type "+ + "(novox/hq issue 751)", as) + } + for _, wrong := range validationProblems(d) { + say("%s", wrong) + } + if word := operatorsWord(key); word != "" && d.Default != nil && !fallsBack[key] { + say("a key naming %q is the operator's value: declare it \"required\": true, with no default, "+ + "unless a mergeable file takes it and no placeholder asks for it, where the file falls back "+ + "(novox/hq ADR 0348 §4, ADR 0112, issue 735)", word) + } + if d.Predefined != "" { + p, listed := m.predefined[d.Predefined] + switch { + case m.predefined == nil: + say("falls back to the predefined key %q, and the list of predefined keys (%s at the root of "+ + "the catalogue's repository) is not present where it is judged: a value falls back only to a "+ + "key on that list; judge it in the catalogue, or leave \"predefined\" out (novox/hq ADR 0348 §7)", + d.Predefined, PredefinedFile) + case !listed: + say("falls back to %q, which is not on the list of predefined keys: name a key on it, or leave "+ + "\"predefined\" out (novox/hq ADR 0348 §7)", d.Predefined) + default: + if p.Type != d.Type { + say("falls back to the predefined key %q, which is a %s, and this value is a %s: a value falls "+ + "back only to a key of its own type (novox/hq ADR 0348 §4)", d.Predefined, p.Type, d.Type) + } + if p.Trusted && !d.Trusted { + say("falls back to the trusted predefined key %q and is not trusted itself: a value that falls "+ + "back to a trusted key is trusted, \"trusted\": true (novox/hq ADR 0348 §8)", d.Predefined) + } + } + } + // A template's text is not here yet. Resolve fills it from the build, WithTemplatesFrom from the tree; neither + // judges again itself, but WithTemplatesFrom parses its result again, and a resolved build is parsed again + // where it is registered, and this is judged then. + if !placed[key] && !d.Runtime && !m.Templated() { + say("placed nowhere: no file says ${config:%s} or takes it under %s, and it is not marked "+ + "\"runtime\": true for a value the module's code reads itself (novox/hq ADR 0348 §5)", key, TakesConfigField) + } + } + used := settingKeysUsedBy(m) + for _, key := range sortedKeys(used) { + if _, declared := m.Settings[key]; !declared { + problems = append(problems, fmt.Sprintf("%s says ${config:%s} and declares no such value under %s: "+ + "declare %q with its type, description and a default or required (novox/hq ADR 0348 §4, §5)", + m.Module, key, ConfigField, key)) + } + } + problems = append(problems, configTakesProblems(m)...) + problems = append(problems, reflectsProblems(m)...) + return problems +} + +// configPlaced is every value a file, a unit, a directory's owner, a contribution or a served fact asks for, and +// every value a mergeable file takes. +func configPlaced(m Manifest) map[string]bool { + placed := settingKeysUsedBy(m) + for _, r := range m.Resources { + for _, k := range takenKeyList(r) { + placed[k] = true + } + } + return placed +} + +// fileFallsBack is every value a mergeable file takes and no placeholder asks for. Unset, such a value is simply +// not laid over the file: the file's own content, or the module's code reading the file, falls back. So one named +// after the operator's value (a url, a user, an address) is not `required`, and its default is the file's: its +// value in the content, else the empty value of its type, never laid over the file (the operator's decision on +// issue 664, issue 735). A placeholder has no such fallback: its default would be filled in where the operator's +// value belongs (ADR 0112). +func fileFallsBack(m Manifest) map[string]bool { + asked := settingKeysUsedBy(m) + out := map[string]bool{} + for _, r := range m.Resources { + if how, _ := r["merge"].(string); how == "" { + continue + } + for _, k := range takenKeyList(r) { + if !asked[k] { + out[k] = true + } + } + } + return out +} + +// renamedTaken is every value a mergeable file takes under a name of its own, with that name (issue 751). +func renamedTaken(m Manifest) map[string]string { + out := map[string]string{} + for _, r := range m.Resources { + if how, _ := r["merge"].(string); how == "" { + continue + } + for _, e := range takenEntries(r[TakesSettingsField]) { + if e.renamed { + out[e.key] = e.as + } + } + } + return out +} + +// validationProblems is what is wrong with a declaration's validation, and its default against it. +func validationProblems(d SettingDeclaration) []string { + var problems []string + isNumber := d.Type == ConfigNumber + if len(d.Enum) > 0 && d.Type != ConfigNumber && d.Type != ConfigString { + problems = append(problems, fmt.Sprintf("an enum is for a number or a string, and this is a %s", d.Type)) + } + if (d.Minimum != nil || d.Maximum != nil) && !isNumber { + problems = append(problems, fmt.Sprintf("a minimum or maximum is for a number, and this is a %s", d.Type)) + } + if d.Minimum != nil && d.Maximum != nil && *d.Minimum > *d.Maximum { + problems = append(problems, fmt.Sprintf("its minimum %v is above its maximum %v", *d.Minimum, *d.Maximum)) + } + var pattern *regexp.Regexp + if d.Pattern != "" { + if d.Type != ConfigString { + problems = append(problems, fmt.Sprintf("a pattern is for a string, and this is a %s", d.Type)) + } + var err error + if pattern, err = regexp.Compile(`^(?:` + d.Pattern + `)$`); err != nil { + problems = append(problems, fmt.Sprintf("its pattern %q is not a regular expression: %v", d.Pattern, err)) + } + } + for _, v := range d.Enum { + if !ofType(v, d.Type) { + problems = append(problems, fmt.Sprintf("its enum holds %v, which is not a %s", v, d.Type)) + } + } + if d.Default == nil { + return problems + } + if !ofType(d.Default, d.Type) { + if d.Type != "" { + problems = append(problems, fmt.Sprintf("its default %v is not a %s", d.Default, d.Type)) + } + return problems + } + if len(d.Enum) > 0 && !slices.ContainsFunc(d.Enum, func(v any) bool { return v == d.Default }) { + problems = append(problems, fmt.Sprintf("its default %v is not one of %v", d.Default, d.Enum)) + } + if n, ok := d.Default.(float64); ok { + if d.Minimum != nil && n < *d.Minimum { + problems = append(problems, fmt.Sprintf("its default %v is below its minimum %v", n, *d.Minimum)) + } + if d.Maximum != nil && n > *d.Maximum { + problems = append(problems, fmt.Sprintf("its default %v is above its maximum %v", n, *d.Maximum)) + } + } + if s, ok := d.Default.(string); ok && pattern != nil && !pattern.MatchString(s) { + problems = append(problems, fmt.Sprintf("its default %q does not match its pattern %q", s, d.Pattern)) + } + return problems +} + +// ofType is whether a value read from JSON is of a config type. +func ofType(v any, kind string) bool { + switch v.(type) { + case float64: + return kind == ConfigNumber + case string: + return kind == ConfigString + case bool: + return kind == ConfigBoolean + case map[string]any: + return kind == ConfigObject + case []any: + return kind == ConfigList + } + return false +} + +// configTakesProblems is what ADR 0348 §11 adds for a module in the config form: an account fact in +// takes-config (the mesh's own words are refused for both forms by TakenSettingsProblems), and a mergeable file +// naming no keys where none of the module's does (where one does, TakenSettingsProblems names every one that +// does not). +func configTakesProblems(m Manifest) []string { + var problems, silent []string + naming := false + for _, r := range m.Resources { + if how, _ := r["merge"].(string); how == "" { + continue + } + if _, said := r[TakesSettingsField]; !said { + silent = append(silent, fmt.Sprint(r["id"])) + continue + } + naming = true + for _, k := range takenKeyList(r) { + _, declared := m.Settings[k] + switch { + case accountFacts[k]: + problems = append(problems, fmt.Sprintf("%s's %v names %q in %s, and %q is an account fact the "+ + "mesh places, never a config value (novox/hq ADR 0348 §11)", m.Module, r["id"], k, + TakesConfigField, k)) + case meshWords[k]: + // Refused for both forms by TakenSettingsProblems, in its words. + case !declared: + problems = append(problems, fmt.Sprintf("%s's %v names %q in %s and declares no such value under "+ + "%s: a value a file takes is declared like every other, with its type, description and a "+ + "default or required, or the config verb could never set it (novox/hq ADR 0348 §4, §11)", + m.Module, r["id"], k, TakesConfigField, ConfigField)) + } + } + } + if !naming { + for _, id := range silent { + problems = append(problems, fmt.Sprintf("%s's mergeable %s names no %s: a module that declares config "+ + "names the keys each mergeable file takes, an empty list for none (novox/hq ADR 0348 §11)", + m.Module, id, TakesConfigField)) + } + } + return problems +} + +// meshMadeIDs are the resources the mesh composes into a module's own set, which its `restart-on` may name +// without declaring them; meshMadePrefixes are those named after what they are for. +var ( + meshMadeIDs = []string{ComposedJailsID(), FilteringID(), CertificateID(), AuthorityID(), BusUsersID()} + meshMadePrefixes = []string{NeedID(""), SecretID(""), strings.TrimSuffix(GrantID("", ""), "-"), BoundID(""), ReceivedID(""), AccessID("/")} +) + +// reflectsProblems is a `reload-on` or `restart-on` naming a file the module does not declare (ADR 0348 §6): not +// one of its own resources, not one the mesh composes for it, and not another module's, which is named +// `.`. +func reflectsProblems(m Manifest) []string { + own := map[string]bool{} + for _, r := range m.Resources { + own[fmt.Sprint(r["id"])] = true + } + meshMade := func(id string) bool { + if slices.Contains(meshMadeIDs, id) { + return true + } + for _, p := range meshMadePrefixes { + if strings.HasPrefix(id, p) && len(id) > len(p) { + return true + } + } + return false + } + var problems []string + for _, r := range m.Resources { + for _, field := range []string{"reload-on", "restart-on"} { + for _, id := range stringsOf(r[field]) { + if strings.Contains(id, ".") || own[id] || meshMade(id) { + continue + } + problems = append(problems, fmt.Sprintf("%s's %v names %q in its %s, and %s declares no such "+ + "file: name a file of its own, or another module's as . (novox/hq ADR 0348 §6)", + m.Module, r["id"], id, field, m.Module)) + } + } + } + return problems +} diff --git a/go/catalogue/config_move.go b/go/catalogue/config_move.go new file mode 100644 index 0000000..e902b91 --- /dev/null +++ b/go/catalogue/config_move.go @@ -0,0 +1,40 @@ +package catalogue + +import "strings" + +// What the move of config reads of a module's declaration (novox/hq ADR 0348 §9, design 57 K5, issue 664): where a +// layer's key goes, how two layers' values become one, and whether a value is one its declaration takes. + +// TakenAs is each name a mergeable file takes a config value under that is not the value's config key, with that +// key (issue 751): a layer set before the module moved may still say the file's name, and the move sets it under +// the config key. +func TakenAs(m Manifest) map[string]string { + out := map[string]string{} + for key, as := range renamedTaken(m) { + out[as] = key + } + return out +} + +// AskedKeys is every value a placeholder of the module asks for: a file, a unit, a directory's owner, a +// contribution or a served fact. Such a value is taken whole from the last layer that sets it, where a mergeable +// file lays one object over another key by key. +func AskedKeys(m Manifest) map[string]bool { return settingKeysUsedBy(m) } + +// LaidOver is a value laid over another as a later layer is laid over an earlier: two objects key by key, anything +// else replaced whole. +func LaidOver(under, over any) any { return mergeValue(under, over) } + +// ValueProblems is what is wrong with a value for a declaration: its type, and its enum, bounds or pattern. +func ValueProblems(d SettingDeclaration, v any) []string { + if v == nil { + return []string{"no value"} + } + d.Default = v + d.Required = false + var problems []string + for _, p := range validationProblems(d) { + problems = append(problems, strings.ReplaceAll(p, "its default", "the value")) + } + return problems +} diff --git a/go/catalogue/consumer_into_serves.go b/go/catalogue/consumer_into_serves.go new file mode 100644 index 0000000..e5d4791 --- /dev/null +++ b/go/catalogue/consumer_into_serves.go @@ -0,0 +1,338 @@ +package catalogue + +import ( + "fmt" + "regexp" + "sort" + "strings" +) + +// What a provider derives for one consumer, said once in the provider's definition and delivered +// to both ends (novox/hq ADR 0202, issue 124). +// +// A `serves` block is otherwise literal: the same values for every consumer. Where the provider +// *names the resource* — a bucket, a database, a vhost — the name is derived from who is asking, +// and before this the mesh had no channel for it. The provider recomputed it in its own code and +// every consumer transcribed it into its own definition by hand, which is a copy of somebody +// else's rule kept in agreement by nobody. One of three transcriptions was wrong for months. +// +// **The mesh learns no protocol here; it spells its own name in an alphabet it already knows.** +// The only fact a served value may name is the identity the mesh itself minted for the consumer, +// in one of two alphabets: as it was minted, and as a DNS label. Everything a provider wants +// around it — a prefix, a suffix, a separator — it writes around the placeholder, because a +// served value is a string. + +// consumerFact is `${consumer:}` or `${consumer::}`. +var consumerFact = regexp.MustCompile(`\$\{consumer:([a-z][a-z0-9-]*)(?::([a-z][a-z0-9-]*))?\}`) + +// consumerFacts are what a served value may name about the consumer it is being derived for. +// One entry, deliberately: the identity is the one thing about a consumer the mesh itself chose, +// so it is the one thing the mesh can hand to a provider without either end guessing. +var consumerFacts = []string{"as"} + +// consumerAlphabets are the ways the mesh will write that identity. `dns` is the mesh's own +// identifier with its separator written `-` instead of `_` — the whole of the difference between +// the alphabet the mesh mints in and the one buckets, vhosts and hostnames accept. +var consumerAlphabets = []string{"dns"} + +// ServedTo fills a provider's served values for one consumer. +// +// `as` is the identity the mesh minted for that consumer — the same string it is told to present +// as a login. Values with no placeholder are returned exactly as they were, and a block with no +// placeholder at all is returned unchanged, so this costs nothing for the providers that derive +// nothing. +// +// Only strings carry placeholders. A number, a boolean or a nested object is a value the provider +// stated outright, and is left alone. +func ServedTo(serves map[string]any, as string) (map[string]any, error) { + if len(serves) == 0 { + return serves, nil + } + var out map[string]any + for _, key := range sortedAnyKeys(serves) { + text, ok := serves[key].(string) + if !ok || !strings.Contains(text, "${consumer:") { + continue + } + filled, err := consumerInto(text, as) + if err != nil { + return nil, fmt.Errorf("the value served as %q: %w", key, err) + } + if out == nil { + // Copied only once something actually changes: the caller's map is the manifest's, + // and a provider that derives nothing must not have it rewritten underneath it. + out = make(map[string]any, len(serves)) + for k, v := range serves { + out[k] = v + } + } + out[key] = filled + } + if out == nil { + return serves, nil + } + return out, nil +} + +// consumerInto replaces every `${consumer:…}` in one value. +// +// **A fact or an alphabet the mesh does not have is refused, not left standing.** Written through, +// the literal `${consumer:as}` would reach a configuration file and be read as a bucket name, +// failing somewhere that names neither the module nor the mesh — the same reasoning `${bound:…}` +// is refused by (boundInto). +func consumerInto(value, as string) (string, error) { + var failed error + out := consumerFact.ReplaceAllStringFunc(value, func(match string) string { + parts := consumerFact.FindStringSubmatch(match) + fact, alphabet := parts[1], parts[2] + if fact != "as" { + if failed == nil { + failed = fmt.Errorf( + "says %s, and the mesh states %s about a consumer", match, orNothing(consumerFacts)) + } + return match + } + switch alphabet { + case "": + return as + case "dns": + return asDNSLabel(as) + default: + if failed == nil { + failed = fmt.Errorf( + "says %s, and the mesh writes an identity as %s", match, orNothing(consumerAlphabets)) + } + return match + } + }) + if failed != nil { + return "", failed + } + return out, nil +} + +// asDNSLabel writes a minted identity as a DNS label. +// +// The mesh's identities are already lower-case letters, digits and `_` (ConsumerIdentity), and +// already inside the bound of the provision serving them: a provider that serves one as a DNS label +// bounds it at 63 or less (CheckServes, novox/hq ADR 0225), and one that serves it at all without +// saying is held to twenty (IdentityBoundOf). So +// this is the separator and nothing else — no lower-casing of what is already lower case, no +// truncation to a limit the identity is already inside, no padding of a name that is already long +// enough. Each of those would be the mesh guessing at a rule it has not been given. +func asDNSLabel(as string) string { + return strings.ReplaceAll(as, "_", "-") +} + +// CheckServes refuses a `serves` block that names a consumer fact or an alphabet the mesh does not +// have, when the definition is parsed rather than when a consumer is resolved. +// +// A provision nobody consumes yet still has its rule read: a definition that would be refused the +// first time somebody required it is a definition that is wrong now. +func CheckServes(m Manifest) []string { + var problems []string + for _, provision := range sortedServes(m.Serves) { + for _, key := range sortedAnyKeys(m.Serves[provision]) { + text, ok := m.Serves[provision][key].(string) + if !ok { + // A provider's own name is filled only in a value of its own (novox/hq ADR 0346): one + // nested in a list or an object would be served as the placeholder, unsaid. + problems = append(problems, nestedOwnNames(m, provision, key, m.Serves[provision][key])...) + continue + } + // A probe identity, because what is checked is the shape of the statement and not + // what any consumer is called. + if _, err := consumerInto(text, "mesh_node_module"); err != nil { + problems = append(problems, fmt.Sprintf( + "%s serves %s, and the value it serves as %q %s", m.Module, provision, key, err)) + } + // And the provider's own names it may serve (novox/hq ADR 0346). + problems = append(problems, checkServedOwnNames(m, provision, key, text)...) + // A label longer than DNS keeps is not truncated here (asDNSLabel), so the offer's own + // bound has to keep the identity inside one (ADR 0225). + if strings.Contains(text, "${consumer:as:dns}") { + if b := m.IdentityBoundOf(provision); !b.Bounded() || b.Max > dnsLabelLimit { + problems = append(problems, fmt.Sprintf( + "%s serves %s's consumers their identity as a DNS label in %q, and bounds "+ + "that identity at %s: a label keeps %d — state an `identity` of at most %d", + m.Module, provision, key, boundWords(b), dnsLabelLimit, dnsLabelLimit)) + } + } + } + } + return problems +} + +// dnsLabelLimit is the longest DNS label (RFC 1035 §2.3.4). +const dnsLabelLimit = 63 + +func boundWords(b IdentityBound) string { + if !b.Bounded() { + return "nothing" + } + return fmt.Sprintf("%d", b.Max) +} + +func sortedServes(serves map[string]map[string]any) []string { + out := make([]string, 0, len(serves)) + for k := range serves { + out = append(out, k) + } + sort.Strings(out) + return out +} + +func sortedAnyKeys(values map[string]any) []string { + out := make([]string, 0, len(values)) + for k := range values { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// derivedFor is what the provider on this machine derives for one consumer of one provision +// (novox/hq ADR 0202). +// +// Settled first, then derived: an operator may set a prefix on what the provider serves and the +// mesh still fills the consumer's half of it ([ADR 0174]). Only the keys that actually name the +// consumer are returned — the rest of a `serves` block is the same for every consumer and is +// already in the provider's own definition, so repeating it here would be a second copy to go +// stale. +// +// The first module in the resolved order that says it serves the provision answers, which is the +// choice servedOnThisMachine makes for the consumer's half. Nothing serving it on this machine is +// not an error: a contribution can reach a machine whose provider is a record or an adapter, and +// then there is nothing derived to tell. +func (r Resolution) derivedFor(provision, as, consumer, local string, settings SettingsBy) (map[string]any, error) { + for _, m := range r.Modules { + serves, said := m.Serves[provision] + if !said { + continue + } + var names map[string]any + for key, value := range serves { + if text, ok := value.(string); ok && strings.Contains(text, "${consumer:") { + if names == nil { + names = map[string]any{} + } + names[key] = value + } + } + if names == nil { + return nil, nil + } + // **A consumer that keeps several holders of this provision is refused** — this is issue + // 124's own failure one case to the side, and it would be just as quiet. + // + // Each holder gets its own login, `…_` (ADR 0094), and a provider derives from the + // login, so it would make one resource per holder. The consumer's side has no such + // dimension: one binding file per provision, one `${bound::}`, both + // derived from the un-suffixed identity. So the provider would create the holder's + // resource and the consumer would be configured against a name nothing made — it would + // authenticate successfully and be refused on every object, which reads like a credential + // fault and is not one. + // + // Lifting this means giving the consumer's side a local dimension. That is a decision, + // not an omission, and until it is taken the mesh says so rather than guessing. + if local != "" { + return nil, fmt.Errorf( + "%s keeps several holders of %s (this one is %q), and %s derives %s for each "+ + "consumer from the login the mesh minted. Each holder has its own login, and a "+ + "consumer is told one value per requirement — so the two ends would name "+ + "different things and nothing would compare them (novox/hq ADR 0202)", + consumer, local, provision, m.Module, orNothing(sortedAnyKeys(names))) + } + settled, err := Settle(names, WithDefaults(m, settings[m.Module])) + if err != nil { + return nil, fmt.Errorf("%s serving %s: %w", m.Module, provision, err) + } + derived, err := ServedTo(settled, as) + if err != nil { + return nil, fmt.Errorf("%s serving %s to %s: %w", m.Module, provision, as, err) + } + return derived, nil + } + return nil, nil +} + +// notTranscribed refuses a consumer's file that writes out the value its provider derives for it, +// instead of asking for it (novox/hq ADR 0202, issue 124). +// +// **What would have caught the one wrong instance.** The object store's three consumers each wrote +// their bucket into their own configuration by hand. One of them named a predecessor's bucket, and +// nothing compared it to what the provider would actually create: the module would have +// authenticated successfully and been refused on every object, which reads like a credential fault +// and is not one. It looked authoritative for months. +// +// The test is exact and costs one string search: a definition whose file already contains the +// value the mesh is about to derive for it has written down somebody else's rule. It cannot be a +// coincidence — a derived value carries the identity the mesh minted for this very consumer on +// this very machine, which nothing else would spell out — and it cannot be checked afterwards, +// because after substitution every consumer's file contains it legitimately. +// +// Only values that actually name the consumer are judged. A provider that serves a constant under +// the same key serves the same constant to everyone, and a consumer repeating it is redundant +// rather than wrong. +func notTranscribed(resource map[string]any, known map[string]map[string]string, module string) error { + if fmt.Sprint(resource["type"]) != "file" { + return nil + } + content, ok := resource["content"].(string) + if !ok || content == "" { + return nil + } + for _, provision := range sortedKnown(known) { + values := known[provision] + identity := values["as"] + if identity == "" { + continue + } + for _, key := range sortedStringKeys(values) { + if key == "as" { + // The login is not derived from itself, and a consumer that must present it in a + // connection string legitimately has it from `${bound:…}` — which is what it will + // be after substitution, so this would judge the substitution, not the module. + continue + } + value := values[key] + if value == "" || !namesTheConsumer(value, identity) { + continue + } + if !strings.Contains(content, value) { + continue + } + return fmt.Errorf( + "%s writes %q into %v, and that is exactly what %s derives for it — a definition "+ + "keeping its own copy of somebody else's naming rule is one that can disagree "+ + "with it, silently. Say ${bound:%s:%s} and be told", + module, value, resource["id"], provision, provision, key) + } + } + return nil +} + +// namesTheConsumer is whether a derived value was built from this consumer's identity — in the +// alphabet it was minted in, or as a DNS label. A value that does not contain it was not derived +// from it, whatever else it may be. +func namesTheConsumer(value, identity string) bool { + return strings.Contains(value, identity) || strings.Contains(value, asDNSLabel(identity)) +} + +func sortedKnown(known map[string]map[string]string) []string { + out := make([]string, 0, len(known)) + for k := range known { + out = append(out, k) + } + sort.Strings(out) + return out +} + +func sortedStringKeys(values map[string]string) []string { + out := make([]string, 0, len(values)) + for k := range values { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/go/catalogue/consumers.go b/go/catalogue/consumers.go new file mode 100644 index 0000000..46a64e0 --- /dev/null +++ b/go/catalogue/consumers.go @@ -0,0 +1,110 @@ +package catalogue + +import ( + "fmt" + "regexp" + "slices" +) + +// A provider's consumers' credentials, in a file of the provider's own (novox/hq ADR 0373, issue 679). +// +// A provider in the tool runner may not ask the vault (ADR 0349 clause 11, ADR 0370 clause 2), so its template names +// `${consumers:}` and its machine's provisioner client fills it: one entry per consumer bound to the +// provider for that provision, each carrying the credential the vault keeps for the pair. Only the client fills it, +// so the placeholder is a provider's alone, for a provision it provides, in a file only the provider reads (mode 0600 +// or 0400, owned at runtime, so a change on disk is written back); and a module not handed to the client on a machine +// is refused there, never written the placeholder's text, nor an empty list the provider would read as no consumers. + +// ConsumersNamespace is a provider's consumers' credentials' placeholder namespace, `${consumers:}`. +const ConsumersNamespace = "consumers" + +// ofConsumers is where a provider places its consumers' credentials. Loose inside the braces, so a misspelt name is +// refused rather than passed over. +var ofConsumers = regexp.MustCompile(`\$\{consumers:([^}]*)\}`) + +// consumersPlaceholderProblems is what is wrong with one resource's ${consumers:…}: placed only in a file's content, +// for a provision the module provides, in a file only the provider reads and the client writes back. +func consumersPlaceholderProblems(m Manifest, r map[string]any) []string { + var problems []string + for _, field := range sortedKeys(r) { + v, ok := r[field].(string) + if !ok { + continue + } + for _, f := range ofConsumers.FindAllStringSubmatch(v, -1) { + switch { + case field != "content" || fmt.Sprint(r["type"]) != "file": + problems = append(problems, fmt.Sprintf("%s's resource %v names %s in its %s; a provider's consumers "+ + "are placed only in a file's content (novox/hq ADR 0373)", m.Module, r["id"], f[0], field)) + continue + case !slices.ContainsFunc(m.Provides, func(o Offer) bool { return o.Name == f[1] }): + problems = append(problems, fmt.Sprintf("%s's resource %v names %s, and %s does not provide %q: only a "+ + "provider's template holds its consumers' credentials (novox/hq ADR 0373)", m.Module, r["id"], f[0], + m.Module, f[1])) + } + if mode := fmt.Sprint(r["mode"]); mode != "0600" && mode != "0400" { + problems = append(problems, fmt.Sprintf("%s's resource %v holds its consumers' credentials (%s) with "+ + "mode %q: the provider alone reads it, 0600 or 0400 (novox/hq ADR 0373)", m.Module, r["id"], f[0], + r["mode"])) + } + if owner := ownershipOf(r); owner != RuntimeOwned { + problems = append(problems, fmt.Sprintf("%s's resource %v holds its consumers' credentials (%s) and is "+ + "%s at runtime: it is owned, written back whenever it differs (novox/hq ADR 0373)", m.Module, + r["id"], f[0], owner)) + } + } + } + return problems +} + +// consumersHandedOnly says what a file naming ${consumers:…} is where it would be written the old way: the provisioner's +// client alone fills it, so on a machine that does not hand the module it is **left out** of the declaration, never +// written as that text or as an empty list the provider would read as no consumers, and never the reason the module is +// refused: a provider not handed there keeps working as it did, from its grant files (novox/hq ADR 0373, issue 780). +// A moved module judged from its repository's tree (WithTemplatesFrom: `module check`), on a machine +// that would hand it, has no templates archive to hand and is judged as it composes; there the placeholder is judged +// and taken out of the text, since what is composed so is never sent (judged answers whether the file is so judged). +func consumersHandedOnly(m Manifest, r map[string]any, with Rendering) (judged, leftOut bool) { + content, _ := r["content"].(string) + switch { + case ofConsumers.FindString(content) == "": + return false, false + case m.Moved && m.Handed == nil && with.Hands && with.Provisioner != "": + return true, false + } + return false, true +} + +// consumersLeftOutLine is the plan's one line for a file left out because the module is not handed. +func consumersLeftOutLine(m Manifest, r map[string]any) string { + return fmt.Sprintf("%s's %v names ${consumers:…}, which the provisioner's client alone fills: left out, handed only "+ + "(novox/hq ADR 0373)", m.Module, r["id"]) +} + +// consumersNotHanded is consumersHandedOnly as a refusal, for a caller that has no way to leave the file out. +func consumersNotHanded(m Manifest, r map[string]any, with Rendering) (judged bool, err error) { + judged, left := consumersHandedOnly(m, r, with) + if left { + return false, fmt.Errorf("%s's resource %v names %s, which the provisioner's client alone fills, and %s is not "+ + "handed to it on this machine: it would be written as that text, or empty, and the provider would read no "+ + "consumers (novox/hq ADR 0373)", m.Module, r["id"], ofConsumers.FindString(fmt.Sprint(r["content"])), m.Module) + } + return judged, nil +} + +// consumersOfTemplate is what is wrong with a handed file's template naming ${consumers:…}, judged on the file's +// resource as if its text were inline. +func consumersOfTemplate(m Manifest, id, text string) []string { + for _, r := range m.Resources { + if fmt.Sprint(r["id"]) != id { + continue + } + withText := map[string]any{} + for k, v := range r { + withText[k] = v + } + withText["content"] = text + return consumersPlaceholderProblems(m, withText) + } + return nil +} diff --git a/go/catalogue/core_credentials.go b/go/catalogue/core_credentials.go new file mode 100644 index 0000000..f11b70c --- /dev/null +++ b/go/catalogue/core_credentials.go @@ -0,0 +1,33 @@ +package catalogue + +import ( + "encoding/json" + "slices" +) + +// The core's own credentials (novox/hq ADR 0349 §16, issue 755). The bus, the node-engine, the controller and +// mesh-provisioner need their files before any provisioner client runs, so their files stay placement and their +// credentials travel in the declaration as values the vault sealed to the machine they run on, which the controller +// carries and cannot open; the node-engine opens them there (mesh-host's VaultBoxContract). + +// CoreCredentialModules are the modules whose own secrets are the core's credentials: the bus, mesh-provisioner and +// its client. The node-engine and the controller hold no own secret of a module's. **Fixed here, in the controller's +// code, as the clause says** — and the vault keeps the same list (mesh-vault's CoreModules), answering the controller +// for these alone. +var CoreCredentialModules = []string{"mesh-provisioner", "mesh-provisioner-client", "nats"} + +// CarriesVaultSealed says whether module's own secrets are the core's credentials. +func CarriesVaultSealed(module string) bool { return slices.Contains(CoreCredentialModules, module) } + +// VaultSealedField is the field of a file the vault sealed to its machine (mesh-host's File.VaultSealed). +const VaultSealedField = "vault-sealed" + +// ForeseenVaultBox stands for a core credential's box in a composition asked ahead of the send, when none is kept +// yet: the send asks the vault for it. Never sent. +var ForeseenVaultBox = json.RawMessage(`{"v":1,"eph":"","iv":"","tag":"","ct":"","foreseen":"asked of the vault by the next send"}`) + +// vaultSealedOf is the vault's box for one of module's own secrets, if the rendering was given one. +func (with Rendering) vaultSealedOf(module, name string) (json.RawMessage, bool) { + box, ok := with.VaultSealed[module][name] + return box, ok && len(box) > 0 +} diff --git a/go/catalogue/cron.go b/go/catalogue/cron.go new file mode 100644 index 0000000..fbb6d29 --- /dev/null +++ b/go/catalogue/cron.go @@ -0,0 +1,92 @@ +package catalogue + +// A five-field cron validator, enough to refuse a malformed schedule near its author +// (novox/hq ADR 0053). +// +// **Validation only, and written rather than pulled in.** The control plane's part in a scheduled +// step is small and exact: carry the field to the host unchanged, and refuse a `schedule` that is +// not a well-formed cron here rather than on the machine — the same near-versus-far discipline the +// manifest keeps for an action a module may not declare and a run-once that is not a boolean. What +// *when to run it again* means is the host's to evaluate; the control plane only checks the string +// is one it could. A cron library would be a dependency carried for evaluation nobody does here. +// +// The ordinary five fields — minute, hour, day-of-month, month, day-of-week — with `*`, lists +// (`,`), ranges (`-`) and steps (`/`). No names (jan, mon): a numeric cron is what the host +// evaluates, and refusing a name here is refusing it near whoever wrote it. + +import ( + "fmt" + "strconv" + "strings" +) + +// validateCron reports why a string is not a five-field cron expression, or nil if it is one. +func validateCron(expr string) error { + fields := strings.Fields(expr) + if len(fields) != 5 { + return fmt.Errorf( + "a schedule is a five-field cron expression (minute hour day-of-month month "+ + "day-of-week), and this has %d field(s)", len(fields)) + } + bounds := []struct { + name string + min, max int + }{ + {"minute", 0, 59}, + {"hour", 0, 23}, + {"day-of-month", 1, 31}, + {"month", 1, 12}, + {"day-of-week", 0, 7}, // 0 and 7 are both Sunday, the convention every cron keeps + } + for i, b := range bounds { + if err := validateCronField(fields[i], b.min, b.max); err != nil { + return fmt.Errorf("%s field: %w", b.name, err) + } + } + return nil +} + +// validateCronField checks one field's elements are within range and well-formed. +func validateCronField(spec string, min, max int) error { + if spec == "" { + return fmt.Errorf("is empty") + } + for _, part := range strings.Split(spec, ",") { + if part == "" { + return fmt.Errorf("%q has an empty element between commas", spec) + } + + rangePart := part + if slash := strings.IndexByte(part, '/'); slash >= 0 { + rangePart = part[:slash] + n, err := strconv.Atoi(part[slash+1:]) + if err != nil || n < 1 { + return fmt.Errorf("step in %q is not a positive number", part) + } + } + + switch { + case rangePart == "*": + // Any value; nothing to bound. + case strings.IndexByte(rangePart, '-') >= 0: + dash := strings.IndexByte(rangePart, '-') + lo, errLo := strconv.Atoi(rangePart[:dash]) + hi, errHi := strconv.Atoi(rangePart[dash+1:]) + if errLo != nil || errHi != nil { + return fmt.Errorf("range %q is not two numbers", rangePart) + } + if lo < min || hi > max || lo > hi { + return fmt.Errorf("range %q is outside the allowed %d-%d", part, min, max) + } + default: + v, err := strconv.Atoi(rangePart) + if err != nil { + return fmt.Errorf("%q is not a number", rangePart) + } + if v < min || v > max { + return fmt.Errorf("%q is outside the allowed range %d-%d", part, min, max) + } + } + } + return nil +} diff --git a/go/catalogue/data.go b/go/catalogue/data.go new file mode 100644 index 0000000..5a43203 --- /dev/null +++ b/go/catalogue/data.go @@ -0,0 +1,791 @@ +package catalogue + +import ( + "bytes" + "encoding/json" + "fmt" + "regexp" + "sort" + "strconv" + "strings" + "time" +) + +// A module declares the data it holds, and the mesh protects and watches it from that declaration +// (novox/hq ADR 0233). +// +// **One section, `data`, for every kind of data a module keeps:** its own, by directory — a store's +// files, a mail spool, an application's uploads — or by an operator's path it was given; what it keeps +// for its consumers, by the provision they reach it through; and what of its own lives with a +// provider, by the provision it requires. Each entry has a class — how precious it is — and, for its +// own data, how it is protected; everything the mesh does is derived from those, never written per +// module: +// +// - a binding to a consumer's data does not move (ADR 0232) where the provision's class keeps data; +// - the backup holder's lines are composed from it — a module no longer writes them by hand; +// - what an unassignment leaves behind of an irreplaceable or valuable item is retired, listed by +// `cleanup list` and deleted only by `cleanup delete` (ADR 0230); +// - the self-check measures every item and says when one shrinks, disappears, stops being written, +// goes without its backup, sits on a degraded array, or is replaced by an empty copy of itself — +// urgent for what is irreplaceable, a warning for what is valuable. + +// The classes: how precious the data is. A fixed vocabulary, ranked by the operator (2026-10-06): a +// class is what the protections are derived from, so a new one is a decision, not a manifest's choice. +const ( + // ClassIrreplaceable is what must never be lost: the operator names it (the media library, the + // photo sites' storage). Protected by a backup or by a declared redundancy — one is required — + // retired rather than removed, and every alert about it is urgent. + ClassIrreplaceable = "irreplaceable" + // ClassValuable is anybody's work that would be painful to lose: in the nightly backup by default, + // retired rather than removed, and every alert about it a warning. + ClassValuable = "valuable" + // ClassRebuildable can be made again from something kept elsewhere — a clone, an index, a + // download, a night's dump — at a cost in time, not in data. In the nightly backup by default; + // forgotten when its module goes, and never alerted on. + ClassRebuildable = "rebuildable" + // ClassCache may be emptied at any moment with nothing lost but speed: never backed up, never + // measured, never alerted on. + ClassCache = "cache" + // ClassNone is a provision that keeps nothing of its consumers' (consumers only): a resolver, a + // certificate authority, an artifact store. + ClassNone = "none" +) + +// classRank orders the classes by how precious they are, so the stricter of two is chosen. +var classRank = map[string]int{ClassNone: 0, ClassCache: 1, ClassRebuildable: 2, ClassValuable: 3, ClassIrreplaceable: 4} + +// StricterClass is the more precious of two classes. +func StricterClass(a, b string) string { + if classRank[b] > classRank[a] { + return b + } + return a +} + +// Retires is whether a class's data is retired, not forgotten, when its machine no longer declares it. +func Retires(class string) bool { return class == ClassIrreplaceable || class == ClassValuable } + +// Watched is whether a class's data raises conditions: irreplaceable and valuable. +func Watched(class string) bool { return Retires(class) } + +// DefaultBackupWithin is how old the last good backup of an item may be before the self-check says so +// (novox/hq ADR 0214: a machine with data and no good backup in 48 hours). +const DefaultBackupWithin = 48 * time.Hour + +// Data is a manifest's `data` section. +type Data struct { + // Own is the data this module keeps itself. + Own []DataItem `json:"own,omitempty"` + // Consumers is what it keeps for its consumers, per provision it grants. + Consumers map[string]ConsumerData `json:"consumers,omitempty"` + // KeptBy is what of its own lives with the provider of a provision it requires — its rows, its + // objects — and how precious that is. The provider's protections follow the stricter of its own + // class for its consumers and this. + KeptBy map[string]KeptData `json:"kept-by,omitempty"` +} + +// DataItem is one piece of a module's own data. +type DataItem struct { + // ID names it within the module: what `data`, `cleanup` and a condition call it. + ID string `json:"id"` + // Path is one of the module's directories, `${dir:}`, or an operator's path it was given, + // `${access:}`, or a path beneath either. Never a machine path (novox/hq ADR 0112). + Path string `json:"path"` + Class string `json:"class"` + // Backup is how it is copied: "copy" (the holder reads it as it stands), "none", or a dump — a + // command that writes a consistent copy into another item, which is copied. Unsaid, anything but a + // cache is copied, unless it says it is protected by redundancy instead. + Backup *DataBackup `json:"backup,omitempty"` + // Redundancy says it is protected by the redundancy of the storage it lives on rather than by a + // copy, and why that is enough: the media library on an array there is no room to copy. The + // backup holder then watches that array, and a degraded one is said. + Redundancy string `json:"redundancy,omitempty"` + // Within is how old its last good backup may be; unsaid, DefaultBackupWithin. + Within string `json:"within,omitempty"` + // Measure is how the backup holder measures it: `walk` (the default — every file, at most daily and + // bounded, for small items), `dataset` (a ZFS dataset's own counters, hourly, nothing walked) or + // `shallow` (its top-level entries only, no size). A large item never says walk. + Measure string `json:"measure,omitempty"` + // Active is how long it may go unwritten before that is a fault — for data something is + // expected to write all the time. Unsaid, a quiet item is not a fault. + Active string `json:"active,omitempty"` + // Why is a line for the reviewer: why this class. + Why string `json:"why,omitempty"` +} + +// ConsumerData is what a provider keeps for the consumers of one provision. +type ConsumerData struct { + Class string `json:"class"` + // In is where it lives: one of the module's own items (whose protection covers it), or a + // provision this module requires (whose provider keeps it, as its consumer). Unsaid only for none + // and cache. + In string `json:"in,omitempty"` + Why string `json:"why,omitempty"` +} + +// KeptData is how precious what a module keeps with a provider is. +type KeptData struct { + Class string `json:"class"` + Why string `json:"why,omitempty"` +} + +// DataBackup is how an item is copied. +type DataBackup struct { + Copy bool + None bool + // Dump is the command writing a consistent copy into the item Into. + Dump string + Into string +} + +// UnmarshalJSON reads "copy", "none" or {"dump": "...", "into": ""}. +func (b *DataBackup) UnmarshalJSON(raw []byte) error { + var word string + if err := json.Unmarshal(raw, &word); err == nil { + switch word { + case "copy": + *b = DataBackup{Copy: true} + case "none": + *b = DataBackup{None: true} + default: + return fmt.Errorf("a data item's backup is \"copy\", \"none\" or {dump, into}, not %q", word) + } + return nil + } + var full struct { + Dump string `json:"dump"` + Into string `json:"into"` + } + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(&full); err != nil { + return fmt.Errorf("a data item's backup is \"copy\", \"none\" or {dump, into}: %w", typedUnknown(err)) + } + *b = DataBackup{Dump: full.Dump, Into: full.Into} + return nil +} + +// MarshalJSON writes it back in the form it was written. +func (b DataBackup) MarshalJSON() ([]byte, error) { + switch { + case b.Copy: + return json.Marshal("copy") + case b.None: + return json.Marshal("none") + } + return json.Marshal(struct { + Dump string `json:"dump"` + Into string `json:"into"` + }{b.Dump, b.Into}) +} + +// IsDump is whether the item is copied by a dump. +func (b *DataBackup) IsDump() bool { return b != nil && b.Dump != "" } + +// BackedUp is whether the holder keeps restore points of this item: anything but a cache by default — +// the nightly backup is the standard plan — unless it says "none", or says it is protected by +// redundancy and says nothing of a backup. +func (it DataItem) BackedUp() bool { + switch { + case it.Backup == nil: + return it.Class != ClassCache && it.Redundancy == "" + case it.Backup.None: + return false + } + return true +} + +// Protection is how the item is protected, in one word: "backup", "redundancy", both joined by "+", +// or "none". +func (it DataItem) Protection() string { + var by []string + if it.BackedUp() { + by = append(by, "backup") + } + if it.Redundancy != "" { + by = append(by, "redundancy") + } + if len(by) == 0 { + return "none" + } + return strings.Join(by, "+") +} + +// The ways an item is measured. +const ( + MeasureWalk = "walk" + MeasureDataset = "dataset" + MeasureShallow = "shallow" +) + +// MeasuredBy is how the item is measured, with the default applied. +func (it DataItem) MeasuredBy() string { + if it.Measure == "" { + return MeasureWalk + } + return it.Measure +} + +// OwnedByModule is whether the item is in one of the module's own directories — the mesh's to retire — +// rather than an operator's path it was given, which the mesh never retires and never deletes. +func (it DataItem) OwnedByModule() bool { return strings.HasPrefix(it.Path, "${dir:") } + +// BackupWithin is the item's bound on its last good backup. +func (it DataItem) BackupWithin() time.Duration { + if d, err := ParseDataDuration(it.Within); err == nil && d > 0 { + return d + } + return DefaultBackupWithin +} + +// ActiveWithin is how long the item may go unwritten; zero when quiet is not a fault. +func (it DataItem) ActiveWithin() time.Duration { + d, _ := ParseDataDuration(it.Active) + return d +} + +// ParseDataDuration reads "48h", "90m" or "7d"; empty is zero. +func ParseDataDuration(s string) (time.Duration, error) { + s = strings.TrimSpace(s) + if s == "" { + return 0, nil + } + if days, ok := strings.CutSuffix(s, "d"); ok { + n, err := strconv.Atoi(days) + if err != nil || n <= 0 { + return 0, fmt.Errorf("%q is not a number of days", s) + } + return time.Duration(n) * 24 * time.Hour, nil + } + d, err := time.ParseDuration(s) + if err != nil || d <= 0 { + return 0, fmt.Errorf("%q is not a duration (48h, 90m, 7d)", s) + } + return d, nil +} + +// DataItems is the module's own data, in the order declared. +func (m Manifest) DataItems() []DataItem { + if m.Data == nil { + return nil + } + return m.Data.Own +} + +// DataItem is one own item by id. +func (m Manifest) DataItem(id string) (DataItem, bool) { + for _, it := range m.DataItems() { + if it.ID == id { + return it, true + } + } + return DataItem{}, false +} + +// ConsumerDataOf is what this module says it keeps for a provision's consumers, and whether it says. +func (m Manifest) ConsumerDataOf(provision string) (ConsumerData, bool) { + if m.Data == nil { + return ConsumerData{}, false + } + c, ok := m.Data.Consumers[provision] + return c, ok +} + +// KeptByOf is how precious what this module keeps with a provision's provider is, and whether it says. +func (m Manifest) KeptByOf(provision string) (KeptData, bool) { + if m.Data == nil { + return KeptData{}, false + } + k, ok := m.Data.KeptBy[provision] + return k, ok +} + +// keepsByClass is whether a class keeps something a binding must not move away from. +func keepsByClass(class string) bool { + return class == ClassIrreplaceable || class == ClassValuable || class == ClassRebuildable +} + +// dataID is what an item's id may be: it is a token in a condition's key and a word on a line. +var dataID = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`) + +// dataPathRef is an item's path: one of the module's directories or accesses, and optionally a path +// beneath it. +var dataPathRef = regexp.MustCompile(`^\$\{(dir|access):([a-z0-9][a-z0-9-]*)\}(/[^\s$]*)?$`) + +// dataProblems is what is wrong with the `data` section itself, from the manifest alone: judged at +// registration as every other per-manifest problem is. Whether a module declares what it should is +// the catalogue check's (DataProblems), because a module already running was written before it. +func (m Manifest) dataProblems() []string { + if m.Data == nil { + return nil + } + var problems []string + say := func(format string, args ...any) { + problems = append(problems, fmt.Sprintf("%s's data: ", m.Module)+fmt.Sprintf(format, args...)) + } + dirs := map[string]bool{} + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) == "directory" { + dirs[fmt.Sprint(r["id"])] = true + } + } + accesses := map[string]bool{} + for _, a := range m.Accesses { + if a.ID != "" { + accesses[a.ID] = true + } + } + ids := map[string]DataItem{} + paths := map[string]string{} + for i, it := range m.Data.Own { + label := it.ID + if label == "" { + label = fmt.Sprintf("item %d", i+1) + } + if !dataID.MatchString(it.ID) { + say("%s has no usable id: lower-case letters, digits and dashes", label) + } else if _, twice := ids[it.ID]; twice { + say("%s is declared twice", it.ID) + } + ids[it.ID] = it + ref := dataPathRef.FindStringSubmatch(it.Path) + switch { + case ref == nil: + say("%s's path %q is not one of the module's directories or accesses: ${dir:} or ${access:}, "+ + "or a path beneath one (a definition names no machine path, novox/hq ADR 0112)", label, it.Path) + case ref[1] == "dir" && !dirs[ref[2]]: + say("%s's path names ${dir:%s}, and %s declares no directory %q", label, ref[2], m.Module, ref[2]) + case ref[1] == "access" && !accesses[ref[2]]: + say("%s's path names ${access:%s}, and %s declares no access %q", label, ref[2], m.Module, ref[2]) + case strings.Contains(it.Path, ".."): + say("%s's path %q climbs out of its directory", label, it.Path) + } + if other, twice := paths[it.Path]; twice && it.Path != "" { + say("%s and %s name the same path %s", other, label, it.Path) + } + paths[it.Path] = label + switch it.Class { + case ClassIrreplaceable, ClassValuable, ClassRebuildable, ClassCache: + case ClassNone: + say("%s is class none, which only a provision keeping nothing of its consumers' is; own data "+ + "that is disposable is cache", label) + default: + say("%s's class %q is not one the mesh protects by: irreplaceable, valuable, rebuildable or cache", + label, it.Class) + } + if it.Class == ClassIrreplaceable && !it.BackedUp() && strings.TrimSpace(it.Redundancy) == "" { + say("%s is irreplaceable and is neither backed up nor said to be protected by redundancy; the only "+ + "copy of something is protected one way or the other (novox/hq ADR 0233) — copy it, dump it into "+ + "another item, or say `redundancy` with why that is enough", label) + } + if it.Class == ClassCache && it.Backup != nil && !it.Backup.None { + say("%s is a cache and asks to be backed up; a cache is disposable, or it is not a cache", label) + } + if it.Class == ClassCache && it.Redundancy != "" { + say("%s is a cache and says it is protected by redundancy; a cache is not protected", label) + } + switch it.Measure { + case "", MeasureWalk, MeasureDataset, MeasureShallow: + default: + say("%s's measure %q is walk, dataset or shallow", label, it.Measure) + } + if _, err := ParseDataDuration(it.Within); err != nil { + say("%s's within: %v", label, err) + } + if _, err := ParseDataDuration(it.Active); err != nil { + say("%s's active: %v", label, err) + } + if it.Within != "" && !it.BackedUp() { + say("%s states within, and is not backed up", label) + } + if it.Active != "" && !Watched(it.Class) { + say("%s is %s and expects writes; only irreplaceable and valuable data is watched", label, it.Class) + } + } + // Dumps go into an item of the same module, declared, and not into themselves. + for _, it := range m.Data.Own { + if it.Backup == nil || it.Backup.Copy || it.Backup.None { + continue + } + switch into, ok := ids[it.Backup.Into]; { + case strings.TrimSpace(it.Backup.Dump) == "": + say("%s's backup names no dump command", it.ID) + case it.Backup.Into == "": + say("%s's dump says no item it writes into", it.ID) + case !ok: + say("%s's dump writes into %q, which is not one of the module's data items", it.ID, it.Backup.Into) + case into.ID == it.ID: + say("%s's dump writes into itself; a dump is copied from where it lands", it.ID) + case into.Class == ClassCache: + say("%s's dump writes into %s, a cache; what is copied must not be disposable", it.ID, into.ID) + } + if it.Backup.Dump != "" { + declared := map[string]string{} + for d := range dirs { + declared[d] = "" + } + if _, err := dirFill(it.Backup.Dump, declared, m.Module); err != nil { + say("%s's dump: %v", it.ID, err) + } + } + } + provided := map[string]bool{} + for _, o := range m.Provides { + provided[o.Name] = true + } + wants := map[string]bool{} + for _, w := range m.Wants() { + wants[w] = true + } + for _, provision := range sortedKeys(m.Data.Consumers) { + c := m.Data.Consumers[provision] + if !provided[provision] { + say("says what it keeps for the consumers of %q, which it does not provide", provision) + } + switch c.Class { + case ClassIrreplaceable, ClassValuable, ClassRebuildable, ClassCache, ClassNone: + default: + say("its consumers' %s is class %q; one of irreplaceable, valuable, rebuildable, cache or none", provision, c.Class) + } + switch { + case c.Class == ClassNone && c.In != "": + say("its consumers' %s keeps nothing and says where it is kept (%s)", provision, c.In) + case keepsByClass(c.Class) && c.In == "": + say("its consumers' %s is %s and says nowhere it lives: `in` names one of the module's items, or a "+ + "provision it requires", provision, c.Class) + case c.In != "": + if own, ok := ids[c.In]; ok { + if c.Class == ClassIrreplaceable && !own.BackedUp() && own.Redundancy == "" { + say("its consumers' %s is irreplaceable and lives in %s, which is not protected", provision, c.In) + } + if classRank[own.Class] < classRank[c.Class] { + say("its consumers' %s is %s and lives in %s, which is only %s", provision, c.Class, c.In, own.Class) + } + } else if !wants[c.In] { + say("its consumers' %s lives in %q, which is neither one of its data items nor a provision it "+ + "requires", provision, c.In) + } + } + } + for _, provision := range sortedKeys(m.Data.KeptBy) { + k := m.Data.KeptBy[provision] + if !wants[provision] { + say("says it keeps data with the provider of %q, which it does not require", provision) + } + switch k.Class { + case ClassIrreplaceable, ClassValuable, ClassRebuildable, ClassCache: + default: + say("what it keeps with %s is class %q; one of irreplaceable, valuable, rebuildable or cache", provision, k.Class) + } + } + return problems +} + +// KeepsConsumerData is whether this module, providing a provision, keeps what each consumer writes +// there (novox/hq ADR 0232, ADR 0233): whether a consumer bound to it is bound to its data. +// +// **What the data section says, it gets**: irreplaceable, valuable or rebuildable keeps; cache and none +// do not — a cache lost in a move costs speed, not data. Before the section, an offer said it with +// `keeps-consumer-data`, which is still read; unsaid in both, a provider that grants each consumer a +// credential of its own keeps that consumer's data (ADR 0232 §1). The catalogue check refuses the +// unsaid case for a provider that grants (DataProblems), so the inference is what an older definition +// on the shelf gets, never a new one. +func (m Manifest) KeepsConsumerData(provision string) bool { + if c, ok := m.ConsumerDataOf(provision); ok { + return keepsByClass(c.Class) + } + for _, o := range m.Provides { + if o.Name == provision && o.KeepsConsumerData != nil { + return *o.KeepsConsumerData + } + } + _, grants := m.Grants[provision] + return grants +} + +// NeedsBackupHolder is whether a module's data needs the machine's backup holder to be there: it keeps +// something irreplaceable — backed up by the holder, or protected by an array the holder watches. A +// module keeping only valuable or rebuildable data is backed up where a holder is, and refused nowhere +// for want of one: the standard plan, not a requirement. +func (m Manifest) NeedsBackupHolder() bool { + for _, it := range m.DataItems() { + if it.Class == ClassIrreplaceable { + return true + } + } + return false +} + +// --- derived: the backup holder's lines --------------------------------------------------------- + +// The node-backup seat's kinds (novox/hq to-be 43, ADR 0233): `backup` is what to run and which paths +// to keep, `data` every item with its class and protection, for the holder to measure and watch. +const ( + BackupKindBackup = "backup" + BackupKindData = "data" +) + +// derivedContributions is what a module's data section gives the backup holder: its backup lines and +// its items. Every item is listed, so a valuable one on a machine is measured whether or not it is +// copied; a cache is listed and not measured. +func (m Manifest) derivedContributions() []SeatContribution { + items := m.DataItems() + if len(items) == 0 { + return nil + } + var lines []string + kept := map[string]bool{} + keep := func(path string) { + if !kept[path] { + kept[path] = true + lines = append(lines, "path "+path) + } + } + for _, it := range items { + if !it.BackedUp() { + continue + } + if it.Backup.IsDump() { + lines = append(lines, "run "+strings.TrimSpace(it.Backup.Dump)) + if into, ok := m.DataItem(it.Backup.Into); ok { + keep(into.Path) + } + continue + } + keep(it.Path) + } + var described []string + for _, it := range items { + covered := "-" + switch { + case it.Backup.IsDump(): + if into, ok := m.DataItem(it.Backup.Into); ok { + covered = into.Path + } + case it.BackedUp(): + covered = it.Path + } + described = append(described, fmt.Sprintf("item %s %s %s %s %s %s", it.ID, it.Class, it.Path, covered, + it.Protection(), it.MeasuredBy())) + } + var out []SeatContribution + if len(lines) > 0 { + out = append(out, SeatContribution{Seat: BackupSeat, Kind: BackupKindBackup, Content: strings.Join(lines, "\n") + "\n"}) + } + return append(out, SeatContribution{Seat: BackupSeat, Kind: BackupKindData, Content: strings.Join(described, "\n") + "\n"}) +} + +// allContributions is every contribution a module makes to a seat's holder: what it wrote, and what +// its data section derives. **One list**: a module that declares its data has its backup lines +// composed from it, and a backup line it still writes by hand is not placed — the catalogue check +// refuses it — so the holder never copies two lists that disagree. +func (m Manifest) allContributions() []SeatContribution { + if m.Data == nil { + return m.Contributions + } + derived := m.derivedContributions() + out := make([]SeatContribution, 0, len(m.Contributions)+len(derived)) + for _, c := range m.Contributions { + if s, known := SeatNamed(c.Seat); known && s.Name == BackupSeat { + continue + } + out = append(out, c) + } + return append(out, derived...) +} + +// --- the catalogue check ------------------------------------------------------------------------ + +// DataProblems is what the catalogue check refuses about the data a module declares (novox/hq ADR +// 0233), judged over the manifests given together: +// +// - a provider that grants a provision says what it keeps for that provision's consumers; +// - nobody says it on the offer any more (`keeps-consumer-data`): the data section is the one place; +// - nobody writes a backup line by hand: it is derived from the data section; +// - a directory a container writes, mounted whole, is a data item of some class; +// - consumer data said to live in a required provision lives where that provision's provider keeps +// its consumers' data, as preciously, when the provider is given; +// - data a consumer keeps with a provider as irreplaceable is protected there, when the provider is +// given. +// +// **The check's, not registration's**, as ADR 0214's backup rule was: a module already on the shelf +// was written before the rule, and refusing it there would refuse the very providers whose data the +// rule protects. Each module meets it where it is written. +func DataProblems(shelf Shelf) []string { + var problems []string + for _, name := range shelfOrder(shelf) { + m := shelf[name] + for _, provision := range sortedKeys(m.Grants) { + if _, said := m.ConsumerDataOf(provision); !said { + problems = append(problems, fmt.Sprintf( + "%s grants %s and does not say what it keeps for its consumers: data.consumers.%s with a "+ + "class — irreplaceable, valuable, rebuildable, cache, or none (novox/hq ADR 0233)", name, provision, provision)) + } + } + for _, o := range m.Provides { + if o.KeepsConsumerData != nil { + problems = append(problems, fmt.Sprintf( + "%s says keeps-consumer-data on its offer of %s; it is said once, in data.consumers.%s, with a "+ + "class (novox/hq ADR 0233)", name, o.Name, o.Name)) + } + } + for i, c := range m.Contributions { + if s, known := SeatNamed(c.Seat); known && s.Name == BackupSeat { + problems = append(problems, fmt.Sprintf( + "%s's contribution %d is a backup line written by hand; backups are derived from the data "+ + "section — declare the data, with its class and how it is protected (novox/hq ADR 0233)", name, i+1)) + } + } + for _, dir := range writtenDirectories(m) { + if !coversDirectory(m, dir) { + problems = append(problems, fmt.Sprintf( + "%s mounts its directory %q into a container to be written, and declares no data in it: "+ + "data.own with a class — cache if it is disposable (novox/hq ADR 0233)", name, dir)) + } + } + if m.Data == nil { + continue + } + for _, provision := range sortedKeys(m.Data.Consumers) { + c := m.Data.Consumers[provision] + if c.In == "" { + continue + } + if _, own := m.DataItem(c.In); own { + continue + } + for _, pname := range shelfOrder(shelf) { + p := shelf[pname] + pc, said := p.ConsumerDataOf(c.In) + if !said || !providesName(p, c.In) { + continue + } + if classRank[pc.Class] < classRank[c.Class] { + problems = append(problems, fmt.Sprintf( + "%s keeps its consumers' %s, %s, in %s — and %s keeps its consumers' %s as %s, "+ + "so it is not protected as %s", name, provision, c.Class, c.In, pname, c.In, pc.Class, c.Class)) + } + } + } + for _, provision := range sortedKeys(m.Data.KeptBy) { + k := m.Data.KeptBy[provision] + if k.Class != ClassIrreplaceable { + continue + } + for _, pname := range shelfOrder(shelf) { + p := shelf[pname] + pc, said := p.ConsumerDataOf(provision) + if !said || !providesName(p, provision) { + continue + } + if !keepsByClass(pc.Class) { + problems = append(problems, fmt.Sprintf( + "%s keeps irreplaceable data with %s, and %s keeps its consumers' %s as %s — nothing of it is "+ + "protected there", name, provision, pname, provision, pc.Class)) + continue + } + if in, own := p.DataItem(pc.In); own && !in.BackedUp() && in.Redundancy == "" { + problems = append(problems, fmt.Sprintf( + "%s keeps irreplaceable data with %s, and %s keeps it in %s, which is neither backed up nor "+ + "on declared redundancy", name, provision, pname, pc.In)) + } + } + } + } + return problems +} + +func providesName(m Manifest, provision string) bool { + for _, o := range m.Provides { + if o.Name == provision { + return true + } + } + return false +} + +// writtenDirectories are the module's directories a container mounts whole and may write: a volume +// `${dir:}:` without `:ro`, or a directory stated by an absolute path mounted the same +// way. A file beneath a directory, or a read-only mount, is configuration the mesh wrote. +func writtenDirectories(m Manifest) []string { + absolute := map[string]string{} + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) != "directory" { + continue + } + if p, ok := r["path"].(string); ok && strings.HasPrefix(p, "/") { + absolute[strings.TrimRight(p, "/")] = fmt.Sprint(r["id"]) + } + } + seen := map[string]bool{} + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) != "container" { + continue + } + vols, _ := r["volumes"].([]any) + for _, v := range vols { + s, _ := v.(string) + mount := volumeMount.FindStringSubmatch(s) + if mount == nil || volumeReadOnly(mount[3]) { + continue + } + src := strings.TrimRight(mount[1], "/") + if ref := dirPlain.FindStringSubmatch(src); ref != nil { + seen[ref[1]] = true + } else if id, ok := absolute[src]; ok { + seen[id] = true + } + } + } + out := make([]string, 0, len(seen)) + for id := range seen { + out = append(out, id) + } + sort.Strings(out) + return out +} + +// volumeMount is a container's volume, `:[:]`, its source possibly a +// `${dir:}` — whose own colon is not the separator. +var volumeMount = regexp.MustCompile(`^(\$\{(?:dir|access):[a-z0-9][a-z0-9-]*\}[^:]*|[^:]+):([^:]+)(?::(.*))?$`) + +// volumeReadOnly is whether a volume's options mount it read-only. +func volumeReadOnly(options string) bool { + for _, o := range strings.Split(options, ",") { + if strings.TrimSpace(o) == "ro" { + return true + } + } + return false +} + +// dirPlain is a whole directory, `${dir:}` and nothing after it. +var dirPlain = regexp.MustCompile(`^\$\{dir:([a-z0-9][a-z0-9-]*)\}$`) + +// coversDirectory is whether a data item names the directory, a path within it, or a directory it +// is placed beneath. +func coversDirectory(m Manifest, dir string) bool { + parents := map[string]string{} + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) != "directory" { + continue + } + if p, ok := r["path"].(string); ok { + if ref := dirRef.FindStringSubmatch(p); ref != nil && strings.HasPrefix(p, "${dir:") { + parents[fmt.Sprint(r["id"])] = ref[1] + } + } + } + for _, it := range m.DataItems() { + ref := dataPathRef.FindStringSubmatch(it.Path) + if ref == nil || ref[1] != "dir" { + continue + } + for d, hops := dir, 0; d != "" && hops < 4; d, hops = parents[d], hops+1 { + if ref[2] == d { + return true + } + } + } + return false +} diff --git a/go/catalogue/declaration.go b/go/catalogue/declaration.go new file mode 100644 index 0000000..a2d7597 --- /dev/null +++ b/go/catalogue/declaration.go @@ -0,0 +1,2819 @@ +package catalogue + +// Turning a resolution into the declaration a node is sent. +// +// Separate from resolving because they answer different questions. Resolving asks *what should +// this machine run*; this asks *what does that look like as resources*, and the second is where +// settings are applied, generators are called, contributions are collected and credentials are +// placed. Both lived in one file until it was doing four jobs at once — which is the shape the +// system this replaces failed in, one import at a time. + +import ( + "encoding/json" + "fmt" + "sort" + "strconv" + "strings" + "time" +) + +// SettingsBy is the layers that apply to each module, keyed by module name. +type SettingsBy map[string][]Layer + +// Generator works out a module's resources for one node, where they cannot be written in advance. +type Generator interface { + // Resources for this node. Absent means the node is not part of whatever this generates, + // which is an ordinary answer rather than a failure — a machine assigned the module before it + // has an address on the network is in exactly that state. + Resources(node string) ([]map[string]any, bool, error) +} + +// OpensPorts is a generator that also says what its resources accept connections on. +// +// **Separate from Generator, because most generators have nothing to say here** and requiring an +// empty method of each would be a cost paid everywhere for one caller. +// +// It exists because a static field cannot express this. A hub accepts connections from every node +// at other sites; a machine that is not a hub dials out and needs nothing open — and they are the +// same module. `listens` in a manifest is one answer for every machine that runs it, so the +// machine that most needs filtering, the one facing the public internet, was the one whose rule +// set would have closed its own overlay. +type OpensPorts interface { + // Listens is what this node accepts on because of what was computed for it. Nothing is the + // ordinary answer: most machines running a computed module open no port at all. + Listens(node string) ([]Listening, error) +} + +// Grant is one consumer's credential, on the machine that must create it. +type Grant struct { + // Provision is what was required. + Provision string + // Consumer is the node that will use it. + Consumer string + // From is the module on that machine which asked. + // + // **Part of who this credential is for, not a label** (novox/hq 04-ISSUES/022). Together with + // Consumer it names one consumer; the node alone does not, because a machine routinely runs + // several modules wanting the same thing. It is also what the provider names the role or the + // bucket or the client after, so withdrawing one consumer does not take another's away. + From string + // Values are what that module contributed — the name it wants, and anything else the + // provision's own vocabulary defines. + Values map[string]any + // Local is the name the credential goes by inside the consumer where it keeps several for one + // provision (ADR 0094); empty for the ordinary one. The provider sees it as a holder of its own. + Local string + // Slug is the consumer module's identity slug, if it declared one — carried on the grant so the + // provider side derives the same login the consumer does, even across nodes where the consumer's + // manifest is not in view (novox/hq ADR 0049). Empty means "use the module name". + Slug string + // At is where the consuming machine is on the private network, empty if it is not on one. + // + // Passed in with the grant because it is a fact about another machine, and resolution answers + // questions about one. A provider that must reach back to its consumer — a reverse proxy is + // the whole reason this exists — otherwise has to know how the mesh names machines. + At string + // Sealed is the credential, closed to the providing node. + Sealed string +} + +// Rendering is everything needed to turn a resolution into the declaration a node is sent. +type Rendering struct { + // Certificate is what the mesh issued for this machine's internal name, and the mesh's own + // certificate. Both public — the key they belong to never left the machine. + Certificate string + Authority string + + // Needed is each module's own secrets, sealed to this node, keyed by module and then by the + // name the module gave it. + Needed map[string]map[string]string + // Foreseen is each own secret not made yet that the next send WILL make, keyed like Needed: a + // composition asked ahead of the send (the self-check's D1) composes it with ForeseenSealed in + // its place and lists it in Composed.Foreseen, rather than failing for a value only a send + // makes. Nil on every composition that is sent, and on every other question, which then refuse + // a secret not made with a *NotMadeError. + Foreseen map[string]map[string]bool + // VaultSealed is, for a module on the core's list (CoreCredentialModules), each own secret as the vault sealed it + // to this machine (the mesh's sealed box, as the vault answered it), keyed like Needed (novox/hq ADR 0349 §16, + // issue 755). Given only for a machine whose engine opens the box (VaultBoxContract): its file then carries the + // box and never the controller's own sealed value. Nil everywhere else, and the composition is as it was. + VaultSealed map[string]map[string]json.RawMessage + // NotPlaced is each core module left out of this machine's declaration because the vault gave nothing for its + // credential and no box was kept, with why (issue 755): the machine keeps what it holds of it, and a module not + // yet there is not placed. Nil everywhere else. + NotPlaced map[string]string + + // Mesh is every node's address on the private network, which is what a rule saying "from the + // mesh" resolves to. Passed in for the same reason grants are: who else is on the network is + // a fact about the mesh, and resolution answers questions about one machine. + Mesh []string + // Suffix is what a machine's internal name ends in, as the control plane composed Names — + // `internal` unless the operator chose another — so a fact writing those names does not + // compose it a second time. + Suffix string + + // BusUsers is the mesh's composed user list, for the module holding `mesh-broker`. Empty on + // every other node, and on this one until the controller has composed it. + // + // **Only the users, never the server's own settings**: those are the module's, in its image and + // its mounts (Manifest.BusUsers). + BusUsers string + // BusMembership is this machine's membership for the bus the mesh is moving to, sealed to it + // (design 28, task 5.2). Empty for a machine not being moved. Written as a file the host reads + // after the declaration has applied, so the bus it names is standing before the machine leaves + // the one it is on. + BusMembership string + + // MeshRange is the private network's CIDR (the range node addresses are allocated from), for a + // module that must name the whole mesh rather than one machine — an intrusion filter that must + // never ban a tunnel peer, say. A per-mesh value the module cannot know, so it is carried here + // and offered as ${machine:mesh-range}, the same way one machine's address is. + MeshRange string + + // Accounts is each machine's operator account, by the same internal name Names uses (novox/hq + // to-be 29). What an ssh Host block's `User` line is composed from; empty for a machine no + // operator account is known on. + Accounts map[string]string + + // Kept is every operator-sealed secret in the mesh, for a module that `keeps` them. Nil when + // nothing on this node keeps them, or the mesh has no operator key. + Kept *KeptExport + + // OutwardLinks is the links this machine reported as facing outside it, which the filter is + // written around (novox/hq ADR 0140). Empty means the machine has not said, and the mesh + // composes no filter for it rather than writing a rule around a link with no name. + OutwardLinks []string + + // LocalNetworks is this machine's local networks (novox/hq ADR 0340): the networks it reported on a + // link facing outside it that lie wholly in a private range (JudgeLocalNetworks), which a port + // reached from `local-network` is admitted from beside the private network. Empty when it reported + // none, and the rule set then says so rather than widening the port or dropping it. + LocalNetworks []string + + // TunnelInterface is the interface the mesh's private network runs on, named here rather than + // imported because the overlay package rests on this one. Traffic arriving on it is the mesh's, + // not this machine's own guest, so the filter admits it only by a rule. + TunnelInterface string + + // Foundation is the ports the mesh itself needs reachable on every machine, which no module + // declares because the foundation is not a module (novox/hq 04-ISSUES/051 and 052). The broker + // is the one that matters: a machine dials it to enrol, and a firewall derived only from + // modules closes it. + Foundation []int + + // Names is every machine's internal name and its address, for containers to be given. + // + // **A container does not inherit the machine's names**, so every internal name the mesh wrote + // is invisible to what the machine runs. Given here rather than looked up on the machine, + // because which machines exist is a fact about the mesh. + Names map[string]string + + // Machines is only the machines, by the same internal name — the subset of Names that is a + // node of this mesh rather than a name it was told to serve. Both matter and they are not the + // same set: a container's hosts wants every name, so a routed name resolves to the proxy that + // serves it, while a resolver told the mesh's suffix is authoritative for it answers from what + // it is given and forwards nothing — so a routed name written there is a name nobody asks for, + // standing beside the machines and looking as real as they do. + Machines map[string]string + + // Zones is every zone a module in the mesh answers itself, where it is answered (novox/hq ADR + // 0199): the mesh's resolver forwards each one there. + Zones []ZoneAt + + // Holders is, for each replicated mesh seat, every machine holding it, by internal name and + // private address (novox/hq ADR 0223) — the same shape as Machines. What a machine's resolver + // file lists: every holder of the mesh's resolver, this machine first if it is one. + Holders map[string]map[string]string + + Settings SettingsBy + Generators map[string]Generator + // Grants are the credentials this node must create, for the provisions it offers. Passed in + // rather than resolved, because who consumes a node is a fact about the rest of the mesh and + // resolution answers questions about one machine. + Grants []Grant + // Withheld is every consumer left out of Grants because its identity overflows the provision's + // bound (novox/hq ADR 0225). Composed into nothing; carried so the machine's declaration can say + // whom it does not serve, and why, beside what it does. + Withheld []Overflow + // Unbound is every consumer whose pair credential from this node is on record and whose own + // resolution binds it elsewhere (novox/hq issue 274): never granted, carried to be said. + Unbound []Unbound + + // Ports is where this machine puts what each module needs reachable, by module and by the + // port the software itself uses (novox/hq ADR 0038). + // + // **The one place the number now lives.** A module used to write it three times — for the rule + // set, for what a consumer is told, and for what the runtime publishes — and nothing checked + // that the three agreed. They are all derived from this. + Ports map[string]map[int]int + + // Adopted says the node is adopted (novox/hq ADR 0100): the firewall found on it stays in + // force, so no module that loads a filter is declared there, and what the mesh needs + // reachable is declared as openings, with its own ports guarded by a table that only refuses. + Adopted bool + + // Given is the machine ports this node was given for its modules' ports, by module and by the + // port the software uses (novox/hq ADR 0100) — the foundation's ports, as genesis chose them. + // They win over anything the mesh would assign and over a manifest's own long-form mapping. + Given map[string]map[int]int + + // Taken is the modules taken on this adopted node (novox/hq ADR 0100). The guard is derived + // from these only (ADR 0103): a port of a module assigned but not taken may still be the + // predecessor's. + Taken map[string]bool + + // Seats is where this machine put each mesh-scoped seat's holder, by seat and by the port the + // holder's software uses (novox/hq 04-ISSUES/102) — read from the node's settings and + // assignments for whichever module claims the seat, whether or not it is in this node's set. + // What ${seat:…} answers with; see seat_into.go for why the answer may be absent. + Seats map[string]map[int]int + + // ArtifactStore is the mesh's artifact store as this network reaches it (host:port) — the + // node holding it and the port that node put it on — or empty when the mesh has none on its + // network yet. Composed into every image and archive the mesh built, at this moment and never + // stored (novox/hq 04-ISSUES/102). + ArtifactStore string + + // SeatReach is where this machine reaches the holder of each mesh-scoped seat it may be asked + // about (host:port), by seat: what ${seat::reach} answers with (novox/hq ADR 0222). Absent + // when no holder is reachable yet; see seat_into.go for which seats are answered. + SeatReach map[string]string + + // Built is every `/` the mesh has built. What tells a reference recorded + // with an address — before references were kept without one — from an image a module runs + // straight from a public registry. + Built map[string]bool + + // DataRoot is where this node keeps the directories the mesh places for its modules + // (novox/hq ADR 0112, to-be 27): a directory resource that states no path resolves to + // //, and ${dir:} names that place from the module's own files, + // mounts and environment. A node setting fixed at installation; empty means the default, + // /var/lib — see dir_into.go. + DataRoot string + + // ReadsHealth says this machine's node-engine reads a resource's `health` (novox/hq ADR 0240 Phase B: + // its statement's contract is link.ReadinessContract or later). An older engine parses strictly and + // refuses a field it does not know, whole — so to it the field is not sent, and what it runs is + // judged by liveness alone. + ReadsHealth bool + + // JudgesRoot says this machine's node-engine judges a user's declared `root` (novox/hq ADR 0266: its + // statement's contract is link.RootContract or later). To an older, strict engine the field is not + // sent, and the account it names is not judged — which the self-check says, as not judged. + JudgesRoot bool + + // ReadsBusy says this machine's node-engine reads a container's `busy` (novox/hq ADR 0335: its statement's + // contract is link.BusyContract or later). To an older, strict engine the field is not sent, and the + // container is replaced at once, as before. + ReadsBusy bool + + // Hands says this machine's node-engine writes a filled file on the provisioner client's word alone (its + // statement's contract is link.HandedContract or later, design 57 P7), and Provisioner is the account the + // provisioner's client runs as here, empty where none runs (novox/hq ADR 0349 §4). With both, a moved module's + // templated files are handed to the client (handed.go); without either, it composes as before. + Hands bool + Provisioner string +} + +// RootField is a user resource's field saying the account must never become root without a person +// (novox/hq ADR 0266). +const RootField = "root" + +// rootInto composes a user's `root` for the node-engine: taken away when it asserts nothing (empty — a +// machine where agents run as the operator) or when the engine is older than the field and parses +// strictly; kept as "never" otherwise. +func rootInto(resource map[string]any, with Rendering) { + if resource["type"] != "user" { + return + } + value, has := resource[RootField] + if !has { + return + } + if s, _ := value.(string); s == "" || !with.JudgesRoot { + delete(resource, RootField) + } +} + +// machinePort is where a module's port lives on this machine, or the port itself when the mesh has +// not been asked. Unassigned is not an error here: a module with no `listens` never needed one, +// and a caller composing a declaration without a store still gets something coherent. +func (r Rendering) machinePort(module string, wanted int) int { + if at, given := r.Given[module][wanted]; given { + return at + } + if at, known := r.Ports[module][wanted]; known { + return at + } + return wanted +} + +// Declaration is everything the resolved modules put on the node, with settings applied. +// +// Resource identities are prefixed with the module they came from. Two modules may reasonably +// both call something "config", and without this the second would silently replace the first — +// the node applying one of them and reporting success. +func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) { + composed, err := r.Compose(with) + if err != nil { + return nil, err + } + return composed.Resources, nil +} + +// Composed is a declaration's resources and which module each came from. +// +// Owner is kept beside the resources because a resource id cannot be split back into its module: +// a module's name may itself contain a dot. What the mesh adds of its own — an opening, the guard — +// has no owner. +// +// Received is what each module on the machine is given for each requirement it receives — the same +// contributions its received file is written from, kept beside it so the mesh can also issue them +// on the bus in the module's membership (novox/hq ADR 0167). By module, then requirement. +type Composed struct { + Resources []map[string]any + Owner map[string]string + Received map[string]map[string][]Contribution + // LeftOut is every module of this machine's set that was left out of its declaration, and + // why (novox/hq ADR 0163, rule 6): a setting stored for it that its definition can no longer + // compose. Its held things are kept and its containers untouched — the machine is told so — + // and it is told everything else. + LeftOut map[string]string + // Foreseen is every own secret composed with ForeseenSealed in its place (Rendering.Foreseen), + // as `module/name`, sorted: what the next send will make. Never set on a declaration that is + // sent — a placeholder in a sealed file is a credential the process cannot read. + Foreseen []string + // Unplaced is every contribution the holder's template could not render into something, each + // naming its module and why (novox/hq ADR 0255): that piece is left out and the machine is told + // everything else. Push and plan say it; the merge gate refuses a change that adds one. + Unplaced []string + // HandedOnly is each file left out because only the provisioner's client fills it and the module is not handed on + // this machine (a provider's consumers file, ADR 0373), said once in the plan. Not an unplaced contribution, which + // the merge gate refuses: a module not handed composes as it did. + HandedOnly []string + // Filled is the path of every file handed to the provisioner, in the order composed: the declaration's + // filled files (novox/hq ADR 0349 §4, §5). Plans is each handed module's plan on this machine, by module. + // Both empty where nothing is handed, and the declaration is then what it was. + Filled []string + Plans map[string]Plan +} + +// ForeseenSealed is what stands for an own secret a send will make, in a composition asked ahead of +// the send. Not a sealed value: nothing composed with it may be sent. +const ForeseenSealed = "foreseen: made by the next send" + +// NotMadeError is a module's own secret the composition was given no value for (novox/hq issue 275): +// typed, so that a caller can tell "a send would make this, and none has yet" from a composition that +// fails for any other reason without reading the words. +type NotMadeError struct { + Module, Name string +} + +func (e *NotMadeError) Error() string { + return fmt.Sprintf("%s needs a secret called %q and none was made for it", e.Module, e.Name) +} + +// LeftOut is which of this machine's modules a declaration composed with these settings leaves +// out, and why (novox/hq ADR 0163, rule 6): each whose stored settings its definition can no longer +// compose. Empty when every module composes. The same judgement SetSettings makes before storing. +func (r Resolution) LeftOut(settings SettingsBy, adopted bool) map[string]string { + out := map[string]string{} + for module, why := range r.LeftOutBecause(settings, adopted) { + out[module] = why.Error() + } + return out +} + +// LeftOutBecause is LeftOut with each reason as the error it was, so a reader can tell a setting nobody +// gave (an *UnsetSettingError) from any other cause and say it as the operator's to give (novox/hq issue +// 380). An UnreadManifestError for a stored manifest this controller cannot read whole. +func (r Resolution) LeftOutBecause(settings SettingsBy, adopted bool) map[string]error { + out := map[string]error{} + for _, m := range r.Modules { + if why := UnknownFieldReason(m); why != "" { + out[m.Module] = &UnreadManifestError{Module: m.Module, said: why} + continue + } + if err := JudgeSettings(m, settings[m.Module], adopted); err != nil { + out[m.Module] = err + } + } + return out +} + +// UnreadManifestError is a module left out because its stored manifest has a key this controller does not know +// (novox/hq ADR 0262): said once for the whole mesh, by the catalogue's own condition, not per machine. +type UnreadManifestError struct { + Module string + said string +} + +func (e *UnreadManifestError) Error() string { return e.said } + +// Compose is Declaration with the owner of every resource said. +func (r Resolution) Compose(with Rendering) (Composed, error) { + owner := map[string]string{} + received := map[string]map[string][]Contribution{} + leftOut := map[string]string{} + var foreseen, unplaced, handedOnly []string + handed := &handedFiles{plans: map[string]Plan{}} + resources, err := r.compose(with, owner, received, leftOut, &foreseen, &unplaced, &handedOnly, handed) + if err != nil { + return Composed{}, err + } + sort.Strings(foreseen) + if with.BusMembership != "" { + // The machine's own, not any module's: how it reaches the mesh from now on. Sealed like a + // secret and placed where the host looks for exactly this (design 28, task 5.2). + resources = append(resources, map[string]any{ + "id": BusMembershipID(), "type": "file", "path": BusMembershipPath, + "sealed": with.BusMembership, "mode": "0600", + }) + } + sort.Strings(unplaced) + sort.Strings(handedOnly) + out := Composed{Resources: resources, Owner: owner, Received: received, LeftOut: leftOut, + Foreseen: foreseen, Unplaced: unplaced, HandedOnly: handedOnly, Filled: handed.paths} + if plans := handed.finished(r.Needs); len(plans) > 0 { + out.Plans = plans + } + return out, nil +} + +// BusMembershipID names the resource carrying a machine's membership for the new bus, and +// BusMembershipPath is where the host reads it — the same constant on both sides. +func BusMembershipID() string { return "bus-membership" } + +const BusMembershipPath = "/var/lib/mesh/membership-next.json" + +func (r Resolution) compose(with Rendering, owner map[string]string, + received map[string]map[string][]Contribution, leftOut map[string]string, + foreseen, unplaced, handedOnly *[]string, handed *handedFiles) ([]map[string]any, error) { + // **A setting is judged where it is stored, and an impossible one costs a module, not a + // machine** (novox/hq ADR 0163, rule 6). A definition that moved under a stored setting makes + // this module uncomposable; it is left out of the declaration — its held things kept, its + // containers untouched, the machine told so by name — and the machine is told everything else. + // Before placing, because a placement is a setting too. + left := r.LeftOut(with.Settings, with.Adopted) + kept := make([]Manifest, 0, len(r.Modules)) + var stillBackedUp []Manifest + for module, why := range with.NotPlaced { + if _, already := left[module]; !already { + left[module] = why + } + } + for _, m := range r.Modules { + if why, isLeft := left[m.Module]; isLeft { + if leftOut != nil { + leftOut[m.Module] = why + } + // **Its data is still copied** (novox/hq ADR 0262): a module left out runs nothing new, and + // the data it already holds on the machine is the reason to keep copying it. Only its data, + // as the backup holder's lines are derived from it, and the directories they name. + stillBackedUp = append(stillBackedUp, backupView(m)) + continue + } + kept = append(kept, m) + } + r.Modules = kept + + // Every manifest is placed first (novox/hq ADR 0112): the maps naming where its bindings, + // credentials and contributions land are resolved against this node's directories, so every + // reader below — the binding files, the sealed secrets, the grant paths a contribution + // names — sees a concrete place and none learns the vocabulary. + // Into a fresh slice, never the caller's: one resolution may compose for many nodes, and a + // slice element written in place would carry the first node's places into the second's. + placed := make([]Manifest, len(r.Modules)) + for i, m := range r.Modules { + var err error + if placed[i], err = placedManifest(m, with); err != nil { + return nil, err + } + } + r.Modules = placed + + // Where each provision's credentials land, so a contribution can name the file rather than + // carry a value the mesh does not have. + directories := map[string]string{} + for _, m := range r.Modules { + for provision, where := range m.Grants { + directories[provision] = where + } + } + given, err := r.contributions(with.Settings, with.Grants, directories) + if err != nil { + return nil, err + } + + // A workload on THIS machine contributes the port it declared, and the machine may have + // published it somewhere else (novox/hq ADR 0066). The mirror of the redirect below: 038 fixed + // what a consumer is TOLD about a provider, and this is what a workload TELLS the provider about + // itself — gitea declaring 3000, published as 20000:3000, and the co-located proxy dialling 3000, + // where nothing listens, for every request. + // + // **The contributing module's assignment, never the provider's.** The port belongs to the + // workload; using the map of whatever answers the requirement would move a route to wherever the + // proxy happens to be published. A contribution carried here from another machine is left exactly + // as it is — its port is that machine's to assign, and this map knows nothing about it. + for provision := range given { + for i := range given[provision] { + said := &given[provision][i] + if said.Node != "" && said.Node != r.Node { + continue + } + said.Values = atMachinePort(said.Values, said.From, with.Ports) + } + } + + // A provider answered on this same machine never passed through the walk that works out what a + // provider on ANOTHER machine serves (novox/hq 04-ISSUES/038, ADR 0066). That walk does two + // things — it derives the served facts with the provider node's port assignments, and it settles + // them with that node's settings layers — and the same-node paths (resolve.go's servedHere, and + // here() below) did neither, because both run while resolving, before either is known. + // + // The port was the first half to be noticed (038). The second half is worse and quieter: a served + // value the OPERATOR supplied — a certificate authority's root, which is the one kind of value a + // manifest cannot carry, because it differs on every mesh — arrived as the manifest's empty + // default. A proxy then wrote an empty CA bundle, fell back to the system trust store, could not + // verify the internal authority, and stopped issuing with nothing saying why. + // + // So it is re-derived here, in full, exactly as the cross-node path derives it. **After the + // closure is known**, which also removes an order dependence: the resolver built these needs + // mid-walk from whichever modules had been chosen by then, so what a co-located binding carried + // depended on the order somebody happened to assign things in. + for i := range r.Needs { + if r.Needs[i].ByRecord || r.Needs[i].From != r.Node { + continue + } + serves, answered, err := r.servedOnThisMachine(r.Needs[i].Name, with) + if err != nil { + return nil, err + } + if answered { + r.Needs[i].Serves = serves + } + } + + // Once, from every module's listens -- not per module. A module receiving only its own ports + // would write a rule set that closed every other module on the machine. Each module's per-node + // exposure settings override its listens' source first (novox/hq ADR 0046). + rules, err := r.Rules(with) + if err != nil { + return nil, err + } + // **A machine that has not said which links face outside is sent no filter** (novox/hq ADR + // 0140). The whole chain is written around those links: with none, the rule that lets this + // machine's own guests keep working would name an empty set, which nftables refuses, and a rule + // set that does not load is a machine filtering nothing while its unit reports success. Refused + // here, where a person reads it, rather than on the machine — and the machine keeps the filter + // it already has. + if filters := r.filtersHere(); filters != "" && len(with.OutwardLinks) == 0 { + return nil, fmt.Errorf( + "%s cannot be sent a filter: it has not reported which of its links face outside, and "+ + "every rule in the chain is written around them. It reports that on each apply; "+ + "`node show %s` says whether it has. Until then %s is not sent, and the machine "+ + "keeps the filter it has", r.Node, r.Node, filters) + } + filtering := AsNftablesWithLocal(rules, with.Mesh, r.PublicDomain != "", with.Foundation, + with.OutwardLinks, with.TunnelInterface, with.LocalNetworks) + + // **A variable two modules set is refused whether or not anything places it** (novox/hq ADR + // 0203 §5): the account has one environment, and a machine whose holder arrives later should not + // be the moment two modules are found to disagree about it. + if err := variablesSetOnce(r.Modules); err != nil { + return nil, err + } + + generated, err := r.generatedHere(with) + if err != nil { + return nil, err + } + + var out []map[string]any + for _, m := range r.Modules { + if with.Adopted && m.Filtering != nil { + // Nothing of a module that loads a filter, on an adopted node: its table would drop + // by default and hold accepts, and the found firewall stays in force. Every resource, + // not only the rule set — its service must not run, and a node returned to adopted + // stops it by the ordinary removal of what is no longer declared. + continue + } + resources := m.Resources + + // What the mesh computes for this module goes FIRST, before the module's own resources. + // + // **Order is stated, not derived — the host does not sort** (novox/hq ADR 0005), so + // whatever the mesh writes down is the order a machine applies. A module's service or + // container routinely depends on one of these files; nothing here ever depends on a + // module's resources, because none of it is computed from them. + // + // Appended, this was wrong in a way that only showed on the first apply and then healed: + // the service started before its certificate or its rule set existed, failed, and the next + // reconcile fixed it. A fault that repairs itself on the second attempt is worse than one + // that does not, because what gets remembered is that it works. + var first []map[string]any + if f := m.Filtering; f != nil { + first = append(first, map[string]any{ + "id": FilteringID(), "type": "file", "path": f.Into, + "content": filtering, "mode": "0600", + }) + } + // The node's fail2ban jails, composed from every module it runs (novox/hq to-be 31), written + // where the intrusion-prevention holder owns them. Like the rule set above: gathered from all + // modules, written by the one that holds the role. + if j := m.Jailing; j != nil { + first = append(first, jailsInto(r.Modules, j)...) + } + if c := m.Certificate; c != nil { + if with.Certificate == "" { + // Asked for and not issued. Refused rather than skipped: a module that serves TLS + // with no certificate does not start, and the reason is somewhere else entirely. + return nil, fmt.Errorf( + "%s wants a certificate for this machine and none was issued", m.Module) + } + first = append(first, map[string]any{ + "id": CertificateID(), "type": "file", "path": c.Into, + // Public. It travels in the open like any other file, because it is a statement + // about a key rather than the key. + "content": with.Certificate, "mode": "0644", + }) + if c.Authority != "" { + first = append(first, map[string]any{ + "id": AuthorityID(), "type": "file", "path": c.Authority, + "content": with.Authority, "mode": "0644", + }) + } + } + if m.BusUsers != "" { + // **The claim authorises it, not the field.** This file holds every user's password + // hash, so a module that could ask for it could read every credential on the bus. + // Checked from this manifest alone, which is the cheapest check there is: whether some + // other module also claims the seat is resolution's business elsewhere, and one holder + // mesh-wide is already guaranteed. + if !m.ClaimsSeat("mesh-broker") { + return nil, fmt.Errorf( + "%s asks for the mesh's user list and does not claim mesh-broker. That file "+ + "holds every user's password hash, so the seat is what authorises it", + m.Module) + } + if with.BusUsers == "" { + // Asked for and not composed. Refused rather than skipped, for the reason a + // certificate is: a bus with no user list refuses every connection in the mesh, and + // an empty file would look like a configuration problem on the machine. + return nil, fmt.Errorf( + "%s holds mesh-broker and the mesh composed no user list, so the bus would "+ + "refuse every connection", m.Module) + } + first = append(first, map[string]any{ + "id": BusUsersID(), "type": "file", "path": m.BusUsers, + "content": with.BusUsers, + // Readable by the server and nothing else. Hashes rather than passwords, so this is + // not a set of working credentials — but a list of every user in the mesh is worth + // keeping to the one process that needs it. + "mode": "0600", + }) + } + // **A secret family's members, as given** (novox/hq ADR 0283): one file each at the family's path, and + // nothing for a member not given — the mesh never makes one, and the module says it waits for it. + for _, name := range sortedKeys(with.Needed[m.Module]) { + own, family, ok := m.OwnSecrets.Lookup(name) + if !ok || family == "" || with.Needed[m.Module][name] == "" { + continue + } + // A per-secret owner (novox/hq ADR 0436 §1) stands in for the module's secrets-owner, when + // the manifest named one for this secret (or the family it belongs to). + owner := m.SecretsOwner + if own.Owner != "" { + owner = own.Owner + } + first = append(first, ownedBy(owner, map[string]any{ + "id": NeedID(name), "type": "file", "path": own.Path, "sealed": with.Needed[m.Module][name], + })) + } + // **A handed module's own secrets are the vault's** (design 57 V4, issue 647): each file holding one is + // declared placed and without a value, filled by the machine's provisioner client from the vault; the + // controller makes and seals none of them here. + handsSecrets := with.Handing(m) + for _, name := range sortedKeys(m.OwnSecrets.Plain()) { + if handsSecrets { + owner := m.SecretsOwner + if m.Module == RuntimeModule && r.Account != "" { + owner = r.Account + } + if o := m.OwnSecrets[name].Owner; o != "" { + owner = o + } + first = append(first, ownedBy(owner, map[string]any{ + "id": NeedID(name), "type": "file", "path": m.OwnSecrets[name].Path, HandedSecretField: name, + })) + continue + } + // **A core credential is the vault's box** (ADR 0349 §16, issue 755): carried as the vault sealed it to this + // machine, which the controller cannot open; the controller's own sealed value is not sent with it. + if box, ok := with.vaultSealedOf(m.Module, name); ok { + if string(box) == string(ForeseenVaultBox) { + *foreseen = append(*foreseen, m.Module+"/"+name) + } + owner := m.SecretsOwner + if m.Module == RuntimeModule && r.Account != "" { + owner = r.Account + } + if o := m.OwnSecrets[name].Owner; o != "" { + owner = o + } + first = append(first, ownedBy(owner, map[string]any{ + "id": NeedID(name), "type": "file", "path": m.OwnSecrets[name].Path, VaultSealedField: box, + })) + continue + } + sealed := with.Needed[m.Module][name] + if sealed == "" && with.Foreseen[m.Module][name] { + // Not made, and the next send makes it: composed with a stand-in so that whatever + // else this composition would refuse is still found (novox/hq issue 275). + sealed = ForeseenSealed + *foreseen = append(*foreseen, m.Module+"/"+name) + } + if sealed == "" && m.OwnSecrets[name].IssuedBy == IssuedOutside { + // **Not given, and the mesh never makes it** (novox/hq issue 401, ADR 0228): no file, as for a + // family's member (ADR 0283), so the module reads its absence as not given. + continue + } + if sealed == "" { + // Declared and not made. Refused rather than skipped: a module whose own + // credential is silently absent starts, fails to authenticate, and the reason is + // three layers away from the machine reporting it. + return nil, &NotMadeError{Module: m.Module, Name: name} + } + // The runtime's credential belongs to the account the runtime runs as (novox/hq ADR 0175, + // to-be 38 WP3): its process is composed `user: ` where the node has one, and a + // root-owned 0600 file is one that process cannot read. Composed here rather than said in + // the manifest, because a manifest cannot say ${machine:account} safely — a node with no + // account has nothing to resolve it to, and then the runtime runs as root and the file + // stays root's. + owner := m.SecretsOwner + if m.Module == RuntimeModule && r.Account != "" { + owner = r.Account + } + if o := m.OwnSecrets[name].Owner; o != "" { + owner = o + } + first = append(first, ownedBy(owner, map[string]any{ + "id": NeedID(name), "type": "file", "path": m.OwnSecrets[name].Path, "sealed": sealed, + })) + } + // This module's tools bundles, where the machine runs the node's tool runtime (novox/hq + // ADR 0175, to-be 38 WP2). Mesh-computed like everything above it, and before the module's + // own resources for the same reason: the runtime's process names the files inside these + // and is restarted when one changes, so they are on the machine before it is. + if r.runtimeHere() { + first = append(first, bundleArchives(m)...) + } + // Operator-owned paths this module is granted use of (novox/hq ADR 0051). Written before + // the module's own resources, and so before the container that mounts them: the host must + // find each present — refusing clearly if the operator has not provided it — before it + // starts anything that depends on it. The mesh creates, chowns and reconciles none of it; + // an `access` resource says only *this path must exist, and this module reaches it*. + // Where each is on THIS machine is the assignment's (novox/hq issue 153): placed by id + // where the operator said, the definition's default otherwise, refused where neither. + accesses, accessPaths, err := accessesFor(m, with.Settings[m.Module]) + if err != nil { + return nil, err + } + for _, a := range accesses { + first = append(first, map[string]any{ + "id": AccessID(a.Path), "type": "access", "path": a.Path, "mode": a.Mode, + }) + } + for _, to := range m.SecretRequirements() { + for _, file := range m.SecretFiles(to) { + var found *Needed + for i, n := range r.Needs { + // **This module's need, not the provision's** (novox/hq 04-ISSUES/022). Matching + // on the name alone, every consumer of a provision took whichever credential + // happened to be last in the list — so on a node with two of them, one module + // would be given the other's password and fail to authenticate with a valid + // credential belonging to somebody else. And this file's local name, where the + // module keeps several (ADR 0094). + if n.Name == to && n.For == m.Module && n.Local == file.Local { + found = &r.Needs[i] + } + } + // **A credential the vault keeps for this holder alone** (issue 757): where the module is handed, its + // file is declared placed and without a value, and the machine's client fills it from the vault. + if found != nil && found.IssuedToHolder && with.Handing(m) { + first = append(first, ownedBy(m.SecretsOwner, map[string]any{ + "id": SecretID(SecretLocal(to, file.Local)), "type": "file", "path": file.Path, + HandedProvisionField: PlanProvision{Name: SecretLocal(to, file.Local), Provision: to, + Local: file.Local, Provider: found.From, ProviderModule: found.Module}, + })) + continue + } + if found != nil && found.ByRecord && found.Sealed == "" && !found.Manager { + // Answered by a record whose key has not been supplied since this consumer was + // put on it. **Refused, not skipped.** The mesh discarded the plaintext when the + // key was accepted and cannot seal another, so a machine that resolved cleanly + // would receive no file at all and fail at whatever tried to read it — which is + // the outcome ADR 0024 exists to avoid, arrived at politely. + // + // The manager holder is the one exception (novox/hq ADR 0050): an empty refresh token + // is a licence whose manager has not adopted one yet, a real waiting state rather than + // a lost key. It falls through to the skip below — its bound facts (carrying the + // manager's public key) are still delivered, which is what adoption needs to seal the + // first refresh token. + return nil, fmt.Errorf( + "%s on this machine uses the licence %q and no key has been sealed to it. "+ + "The mesh cannot make one; supply it again with `licence key %s`", + m.Module, found.From, found.From) + } + if found == nil || found.Sealed == "" { + // Answered on this machine, or answered by a node the mesh could not seal to. + // Nothing to write either way, and writing an empty credential file would be + // worse than none: something would read it and fail authenticating. + continue + } + first = append(first, ownedBy(m.SecretsOwner, map[string]any{ + "id": SecretID(SecretLocal(to, file.Local)), "type": "file", "path": file.Path, + "sealed": found.Sealed, + })) + } + } + for _, to := range sortedKeys(m.Grants) { + for _, g := range with.Grants { + if g.Provision != to { + continue + } + if g.From == "" { + // Nothing on that machine asks for this any more. Skipped here rather than + // where grants are gathered, so the rule holds whoever gathers them. + // + // **This is how a credential is withdrawn.** The provisioner removes what + // nobody asks for, and it can only do that if the mesh stops asking — a + // consumer that was unassigned would otherwise keep a working login for ever, + // and nothing would say so. + continue + } + first = append(first, ownedBy(r.provisionsAs(m), map[string]any{ + // One file per holder — the consumer's module with its local name after it + // where it keeps several (ADR 0094); the lab found two files with one id. + "id": GrantID(to, g.Consumer+"."+holderAs(g.From, g.Local)), + "type": "file", + "path": grantPath(m.Grants[to], g.Consumer, holderAs(g.From, g.Local)), + "sealed": g.Sealed, + })) + } + } + for _, to := range sortedKeys(m.Binds) { + var found *Needed + for i, n := range r.Needs { + if n.Name == to && n.For == m.Module { + found = &r.Needs[i] + } + } + if found == nil { + // Answered on this same machine rather than from elsewhere in the mesh. + // + // **Still written.** It used to be skipped, reasoning that "it is on this node" + // is a fact nobody needs — and that is true of the *location* and false of + // everything beside it. A binding also carries what the provider said a consumer + // must know, which is the port; a consumer cannot invent that, and got an absent + // file with no explanation. A build machine sharing a node with the registry it + // pushes to sat in a loop saying it could not read its own binding. + here, err := here(r, to, with) + if err != nil { + return nil, err + } + if here == nil { + // Nothing in this node's set offers it either, so there is genuinely nothing + // to say. Resolution has already refused anything unanswerable, so this is a + // requirement met by the module itself. + continue + } + found = here + } + own, err := r.ownNames(m, to, with.Settings[m.Module]) + if err != nil { + return nil, err + } + as := ConsumerIdentity(r.Node, IdentitySource(m.Slug, m.Module)) + // What the provider derives for THIS consumer, filled here where the consumer is + // known (novox/hq ADR 0202). The same fill knownFor does below, so the binding file + // and the module's `${bound:…}` substitutions cannot say different things. + told := *found + told.Serves, err = ServedTo(told.Serves, as) + if err != nil { + return nil, fmt.Errorf("%s is told about %s: %w", m.Module, to, err) + } + file, err := boundFile(told, m.Binds[to], as, own) + if err != nil { + return nil, err + } + first = append(first, file) + } + for _, to := range sortedKeys(m.Receives) { + file, err := receivedFile(to, m.Receives[to], given[to]) + if err != nil { + return nil, err + } + first = append(first, file) + if received[m.Module] == nil { + received[m.Module] = map[string][]Contribution{} + } + // Empty rather than absent when nobody contributed, for the reason the file is + // written empty: "nothing asked" and "never told" want different responses. + received[m.Module][to] = append([]Contribution{}, given[to]...) + } + if m.Keeps != "" && with.Kept != nil { + file, err := keptFile(m.Keeps, with.Kept) + if err != nil { + return nil, err + } + // Read by the module as its secrets' owner when it names one: the vault, as its own account (issue 723). + first = append(first, ownedBy(m.SecretsOwner, file)) + } + if m.Computed != "" { + mine, part := generated[m.Module] + if !part { + // Assigned, and not yet part of what this generates. Nothing to put on the + // machine, which is different from an error: a node given the network module + // before it has an address is in exactly that state, briefly. + continue + } + resources = mine + } + + // Now, and not before: a module whose resources are computed replaces them wholesale, and + // merging earlier would throw away the files it still needs. + // + // **Except the module's own accounts, which go before even those** (novox/hq issue 213). What + // the mesh computes may belong to one: a module whose code runs as an account it declares has + // its secrets written owned by that account, and a file given to a user the machine does not + // have yet fails — so on the first apply the secrets were refused, the process started without + // them, and the second apply healed it, which is the fault the paragraph above describes. + // An account depends on nothing the mesh computes. + accounts, rest := accountsFirst(resources) + resources = append(append(accounts, first...), rest...) + // **A file delivered only while its optional requirement is bound is left out while it is + // absent** (novox/hq ADR 0330 §5), and the field is dropped from the rest: the node-engine + // parses strictly and knows no such field. + absent := r.AbsentFor(m.Module) + resources = withoutAbsent(resources, absent) + + // No container is given the mesh's names (novox/hq ADR 0148). It used to be: every + // container got the whole roster as `--add-host` entries at creation, and a name that + // moved afterwards was wrong inside it for as long as it ran (issues 109, 135) — and once + // the roster was made part of a container's identity so that could be caught, one name + // moving anywhere replaced every container in the mesh (issue 151). A container resolves a + // mesh name through its machine's resolver at the moment it asks, which the runtime is + // told once per machine, as a file, by the resolver's own module. The names a module + // declares for itself are its own and stay exactly as written: they are part of what the + // module is, and the mesh does not know what they mean. + // What this module may name from inside one of its own files. Gathered once per module + // rather than per file, because it is a fact about the module. + sealed, err := sealedFor(m, r.Needs, with) + if err != nil { + return nil, err + } + // And what its bindings say, for the half of a connection that is not secret. + known, err := knownFor(m, r.Needs, r.Node) + if err != nil { + return nil, err + } + // A requirement answered on this same machine is not in r.Needs — its binding file is + // written from `here` (above) — and so `${bound:…}` could not name it, though the file + // beside it said the same facts. Filled from the same answer, so the two cannot disagree. + for _, want := range m.Wants() { + if _, has := known[want]; has { + continue + } + answered, err := here(r, want, with) + if err != nil { + return nil, err + } + if answered == nil { + continue + } + local := *answered + local.For = m.Module + here, err := knownFor(m, []Needed{local}, r.Node) + if err != nil { + return nil, err + } + for provision, values := range here { + known[provision] = values + } + } + // And the providing machine's private address, beside its name (novox/hq ADR 0194). A name is + // what nearly every consumer wants; the one that cannot use it is a machine's resolver + // configuration, which must reach the resolver before it can resolve anything — the resolver's + // own name included. Absent when the machine has no address yet, so a file naming it is refused + // rather than written with a blank where an address belongs. + for _, values := range known { + if address := with.Machines[values["at"]]; address != "" { + values["address"] = address + } + } + // And what the module is called through each requirement it contributes to (novox/hq + // 04-ISSUES/122) — the same composition its binding file carries. + for provision, values := range known { + own, err := r.ownNames(m, provision, with.Settings[m.Module]) + if err != nil { + return nil, err + } + withOwnNames(values, own) + } + // And where this node places the directories the module declared without a path + // (novox/hq ADR 0112) — resolved once per module, named by ${dir:…} from any resource. + // — and, on an adopted machine, where the assignment says they already are, with the + // owner the data already has (novox/hq issue 153). Malformed placements are refused here. + placed, err := Places(m, with.Settings[m.Module]) + if err != nil { + return nil, err + } + dirs := dirsFor(m, with) + // And the machine underneath, which no binding of its own can tell it. + thisMachine := machineFacts(r, with.Names, with.MeshRange) + + // **A definition that already holds the answer transcribed it** (novox/hq ADR 0202). + // Judged over what the module itself declares, and before anything is substituted: the + // mesh's own generated files — the binding, the contributions — legitimately carry the + // derived value, and after substitution so does every consumer's file, so this is the one + // moment the two can be told apart. + for _, own := range m.Resources { + if err := notTranscribed(own, known, m.Module); err != nil { + return nil, err + } + } + // **And every placeholder no pass fills where it stands is refused** (novox/hq issue 231): + // judged here, over the definition as written and before any pass, by the function the + // catalogue check runs — so a manifest registered by an older binary is judged too, and + // what a setting or a binding later puts into a file is its software's text, never swept. + for _, own := range m.Resources { + if problems := unconsumedPlaceholders(m.Module, own); len(problems) > 0 { + return nil, fmt.Errorf("%s", problems[0]) + } + } + + // Which of this module's files carry a secret, for the rule that a container may not read + // one of them as its environment without saying so (ADR 0086, issue 041). + secretFiles := secretFilesOf(resources) + + // Which of this module's resources its preparation runs before, if it prepares anything. + prepareBefore := preparationTarget(m) + + // Which found networks this machine's setting keeps for each of its containers (novox/hq + // ADR 0163, rule 4); judged above, so an invalid one is not here. + keptNetworks, _ := KeptNetworks(m, with.Settings[m.Module], with.Adopted) + + // **A moved module's templated files are the provisioner's** (novox/hq ADR 0349 §4, §6, design 57 P6): on a + // machine that can take them, each is declared placed and without its text, and no pass below fills it. + handing, err := handed.begin(r.Node, m, dirs, accessPaths, thisMachine, with, r.Capabilities, r.Needs, + backupPlacer{append(append([]Manifest(nil), r.Modules...), stillBackedUp...), with, thisMachine, r.Capabilities, + unplaced}) + if err != nil { + return nil, err + } + // **What the mesh calls the module through each provision it requires** (issue 792): the names its binding + // file carries, composed by the same call, handed to the provider through the plan's entry. + if _, ok := handed.plans[m.Module]; ok && handing != nil { + for _, to := range requirementsContributedTo(m) { + own, err := r.ownNames(m, to, with.Settings[m.Module]) + if err != nil { + return nil, err + } + handed.name(m.Module, to, own) + } + } + // A handed collector is filled from items alone: each old form its collections would have held is said + // (design 57 L1). + if plan, ok := handed.plans[m.Module]; ok && handing != nil && len(plan.Collections) > 0 { + *unplaced = append(*unplaced, oldFormsLeftOut(append(append([]Manifest(nil), r.Modules...), + stillBackedUp...), m, plan.Collections)...) + } + for _, unsettled := range resources { + if pr, ok := unsettled[HandedProvisionField].(PlanProvision); ok { + if handing == nil { + return nil, fmt.Errorf("%s: its credential of %s is handed to the provisioner and the module is "+ + "not: nothing would fill it", m.Module, pr.Provision) + } + marked := map[string]any{} + for k, v := range unsettled { + marked[k] = v + } + delete(marked, HandedProvisionField) + marked[HandedSecretField] = pr.Name + file, sec, err := handedSecretFile(marked, m, dirs, accessPaths, thisMachine, placed) + if err != nil { + return nil, err + } + file["id"] = m.Module + "." + fmt.Sprint(unsettled["id"]) + owner[fmt.Sprint(file["id"])] = m.Module + pr.Path = sec.Path + if err := handed.addProvisionFile(m.Module, pr); err != nil { + return nil, err + } + out = append(out, file) + continue + } + if unsettled[HandedSecretField] != nil { + if handing == nil { + return nil, fmt.Errorf("%s: its own secret %v is handed to the provisioner and the module is "+ + "not: nothing would fill it", m.Module, unsettled[HandedSecretField]) + } + file, sec, err := handedSecretFile(unsettled, m, dirs, accessPaths, thisMachine, placed) + if err != nil { + return nil, err + } + file["id"] = m.Module + "." + fmt.Sprint(unsettled["id"]) + owner[fmt.Sprint(file["id"])] = m.Module + if err := handed.addSecretFile(m.Module, sec); err != nil { + return nil, err + } + out = append(out, file) + continue + } + if handing != nil && fmt.Sprint(unsettled["type"]) == "file" { + if text, _ := unsettled["content"].(string); len(secretsUsed(text)) > 0 && + !handing[fmt.Sprint(unsettled["id"])] { + return nil, fmt.Errorf("%s: its file %v names a secret in its text, and a handed module's "+ + "secrets reach only the files its provisioner client fills: keep it in a template "+ + "(design 57 V4)", m.Module, unsettled["id"]) + } + } + if handing[fmt.Sprint(unsettled["id"])] { + file, line, err := handedFile(unsettled, m, dirs, accessPaths, thisMachine, placed, with) + if err != nil { + return nil, err + } + file["id"] = m.Module + "." + fmt.Sprint(unsettled["id"]) + owner[fmt.Sprint(file["id"])] = m.Module + if err := handed.add(m.Module, line); err != nil { + return nil, err + } + out = append(out, file) + continue + } + if fmt.Sprint(unsettled["type"]) == "file" { + if _, left := consumersHandedOnly(m, unsettled, with); left { + *handedOnly = append(*handedOnly, consumersLeftOutLine(m, unsettled)) + continue + } + } + resource, err := ApplySettings(unsettled, with.Settings[m.Module]) + if err != nil { + return nil, err + } + copied := map[string]any{} + for k, v := range resource { + copied[k] = v + } + if networks, keeps := keptNetworks[fmt.Sprint(copied["id"])]; keeps { + // The container also joins the found network the setting names, so a neighbour + // that resolves it there keeps resolving it. Passed to the host as its own field, + // which it joins after the container is made. + joins := make([]any, 0, len(networks)) + for _, n := range networks { + joins = append(joins, n) + } + copied["networks"] = joins + } + if err := refuseSecretsInEnvironment(copied, secretFiles, m.Module); err != nil { + return nil, err + } + // Said in the catalogue, not on the machine: the host parses strictly and knows no + // such field, and the reason is for a reader of the manifest. + delete(copied, SecretsInEnvironment) + delete(copied, NamesOnPurpose) + delete(copied, TrustedField) + // A process's bus stays controller-side (novox/hq ADR 0436 §1): never composed onto the + // node-engine's process resource, whose wire declaration is mesh-host's own, vendored, with a + // strict decoder that refuses an unknown key. + delete(copied, ProcessBusField) + // **An operator's value, from the assignment** (novox/hq ADR 0112, ADR 0155): what a + // definition may not carry because it is true of one installation only. Filled from + // the same layers a mergeable file takes, and refused when no layer set it. + if err := unsetInConfigWords(settingInto(copied, WithDefaults(m, with.Settings[m.Module]), m.Module), m); err != nil { + return nil, err + } + // **Placed before anything reads a path.** A pathless directory receives the path + // this node resolves for it, and every ${dir:…} — in paths, mounts, content and + // environment — becomes that path, so what follows sees only concrete places + // (novox/hq ADR 0112). The host receives paths exactly as it always has. + if err := dirInto(copied, dirs, m.Module); err != nil { + return nil, err + } + // The operator's data the same way: ${access:…} becomes where this node keeps it, + // and a placed directory takes the owner the assignment said (issue 153). + if err := accessInto(copied, accessPaths, m.Module); err != nil { + return nil, err + } + ownerInto(copied, placed) + // **After settings, and that is the whole reason it is here.** A module's file + // content is where a setting lands, so a placeholder may only exist once the setting + // has been put in — filling secrets first would look at content that is not yet what + // the machine receives. + if err := intoFile(copied, sealed, outsideNotGiven(m, with.Needed[m.Module]), m.Module); err != nil { + return nil, err + } + // **Substituted here, not on the machine.** A bound value is not secret — the mesh + // holds it in the clear — so there is nothing for the host to be the only witness of, + // and sending it already filled in means the host learns no new field. + if err := boundInto(copied, known, m.Module, absent); err != nil { + return nil, err + } + // And what the module could not have written: the machine it turned out to be + // assigned to. Beside the bound values because it is the same kind of fact — the + // mesh's own, held in the clear — and because a module that must name itself to + // something else has no binding to learn it from (novox/hq ADR 0066). + // Which port this machine gave it, for a module that binds directly rather than + // through a runtime that can remap (ADR 0038). Applied before the machine's facts so + // a refusal names the port rather than whatever came after it. + if err := portInto(copied, m.Module, m.Listens, with); err != nil { + return nil, err + } + // And where this machine put the foundation's servers, for the one module that + // reaches them by seat rather than by binding (novox/hq 04-ISSUES/102). + if err := seatInto(copied, m.Module, with); err != nil { + return nil, err + } + if err := machineInto(copied, thisMachine, m.Module); err != nil { + return nil, err + } + if err := pinned(copied, m.Module); err != nil { + return nil, err + } + // And the other half of the same question: an image nobody has published yet is named + // by an artifact rather than by a placeholder digest, and is just as unrunnable. + if err := built(copied, m.Module); err != nil { + return nil, err + } + // What the mesh built is kept by digest and path; the store's address is this + // network's now, composed here and never recorded (novox/hq 04-ISSUES/102). + if err := artifactsInto(copied, m.Module, with); err != nil { + return nil, err + } + publishedOn(copied, m.Module, with) + // How it is ready, in the node-engine's words: its endpoint as the port this machine + // published it on — or not sent at all to an engine older than the field (ADR 0240). + healthInto(copied, m, with) + // And whether it is busy (novox/hq ADR 0335): sent only to an engine that asks it. + busyInto(copied, with) + // And a user's `root` (novox/hq ADR 0266): sent only when it asserts something, to an engine + // that judges it. + rootInto(copied, with) + // The account's environment and every module's shell code, where this module holds the + // seat that places them (novox/hq ADR 0203, ADR 0204). Gathered from every module on + // the node, as the jails are, and **last of every placeholder pass**: shell code is a + // shell's own syntax, full of `${…}` no pass above should ever be shown. + contributing := append(append([]Manifest(nil), r.Modules...), stillBackedUp...) + if err := contributionsInto(copied, m, contributing, thisMachine, with, r.Capabilities, unplaced); err != nil { + return nil, err + } + copied["id"] = m.Module + "." + fmt.Sprint(resource["id"]) + // A service saying what it reflects names resources within its own module, so those + // are prefixed too or they would point at nothing. + // + // **Unless it already names one.** A module may reflect a file another module put on + // the machine — the case this exists for is a resolver restarting when the mesh + // rewrites the names, which are computed by the mesh and belong to its module, not to + // the daemon's. Written `.`, and a dot is what marks it as already + // answered: prefixing it again would point at nothing, silently, and the daemon would + // serve the old names for ever while everything reported success. + if renamed := reflectsRenamed(m.Module, resource["restart-on"]); renamed != nil { + copied["restart-on"] = renamed + } + // And what it is reloaded on, by the same rule (novox/hq ADR 0102): an id left + // unprefixed matches nothing, and the service is never reloaded. + if renamed := reflectsRenamed(m.Module, resource["reload-on"]); renamed != nil { + copied["reload-on"] = renamed + } + // And which of its module's containers a scheduled step holds still (novox/hq ADR 0189). + // **The loudest of the three when it is missed.** An unprefixed `restart-on` matches + // nothing and a service quietly never restarts; an unprefixed `while-stopped` names a + // container the declaration does not contain, and the host refuses the whole + // declaration — so the machine takes nothing at all, for every push, until this is + // right. That is what it did on the control node (2026-10-04). + if renamed := reflectsRenamed(m.Module, resource[WhileStopped]); renamed != nil { + copied[WhileStopped] = renamed + } + // And what a process replaces (novox/hq issue 213): a resource of this module's that it + // no longer declares, named as the host recorded it, or the host hands nothing over and + // removes it first. + if renamed := reflectsRenamed(m.Module, resource["replaces"]); renamed != nil { + copied["replaces"] = renamed + } + // **What reads one of this module's own secrets is restarted when it changes** (novox/hq + // issue 203, issue 206). A credential is re-issued by the mesh, and a container that + // mounted the old file keeps the old one open: the build machine ran for an hour on a + // credential the mesh had replaced, because its manifest restarted it on its + // environment file and nobody had thought to name the credential too. Composed here so + // no manifest has to say it, for a container or a daemon that names the secret's path. + if reads := secretsReadBy(copied, m, with.Needed[m.Module]); len(reads) > 0 { + copied["restart-on"] = withRestartOn(copied["restart-on"], reads) + } + // **A version prepares its state before it runs** (novox/hq ADR 0135). Derived from the + // module's own resource rather than declared beside it: what prepares the state is the + // module's own code, so what it is given has to be what that code is given — and a + // second resource written by hand is a second copy to drift from the first. Placed + // immediately before it, because a run-once step stops everything the declaration + // places after it (ADR 0052), which is how a version whose preparation failed does not + // serve. + if prepareBefore != "" && fmt.Sprint(resource["id"]) == prepareBefore { + step := prepared(copied) + owner[fmt.Sprint(step["id"])] = m.Module + out = append(out, step) + } + owner[fmt.Sprint(copied["id"])] = m.Module + out = append(out, copied) + } + + // **What only the mesh knows, where this module asked for it.** The graph is the control + // plane's; making a name resolve is the module's software. Emitted as ordinary files under + // this module's name, so they are applied, reported and removed exactly as anything else + // it declares. + // The resolver file is the node-resolver holder's where one is held here, and the uplink's + // holder steps back from it (novox/hq ADR 0247). + given, err := FactsFrom(stepsBack(m, r.Modules), r, with) + if err != nil { + return nil, err + } + for _, fact := range given { + fact["id"] = m.Module + "." + fmt.Sprint(fact["id"]) + owner[fmt.Sprint(fact["id"])] = m.Module + out = append(out, fact) + } + } + // The node's tool runtime, last (novox/hq ADR 0175, to-be 38 WP2.3): one process loading every + // bundle delivered above and holding the credential sealed above, so both exist before it starts + // — the order written here is the order the machine applies. + if r.runtimeHere() { + process, err := r.runtimeProcess(with) + if err != nil { + return nil, err + } + owner[fmt.Sprint(process["id"])] = RuntimeModule + out = append(out, process) + // And a runtime of its own for each module of its own account, after it (novox/hq ADR 0259 §8). + owns, err := r.ownRuntimes(with) + if err != nil { + return nil, err + } + for _, p := range owns { + id := fmt.Sprint(p["id"]) + owner[id] = strings.TrimSuffix(id, "."+OwnRuntimeID()) + out = append(out, p) + } + // What each module's bundles are given is read as the account the runtime runs as — not what a + // module of its own account is given, which its own account reads. + words := map[string]map[string]string{} + for _, m := range r.Modules { + if m.RunsAs != "" { + continue + } + w, err := bundleWords(m, with) + if err != nil { + return nil, err + } + words[m.Module] = w + } + // And a file a module's words name is one the runtime is restarted for when it changes. + if named := givenTo(out, owner, words, r.Account); len(named) > 0 { + restarts, _ := process["restart-on"].([]any) + seen := map[string]bool{} + for _, id := range restarts { + seen[fmt.Sprint(id)] = true + } + for _, id := range named { + if !seen[id] { + restarts = append(restarts, id) + seen[id] = true + } + } + process["restart-on"] = restarts + } + // **And what a module of its own account is given, its own account reads** (issue 723): the files and + // directories its words name that say no owner, given to the account it runs as, never the operator's. + for _, m := range r.Modules { + if m.RunsAs == "" { + continue + } + w, err := bundleWords(m, with) + if err != nil { + return nil, err + } + givenTo(out, owner, map[string]map[string]string{m.Module: w}, m.RunsAs) + } + } + if with.Adopted { + // First, before anything a module declares: what the mesh needs reachable, then its guard. + // The order a machine applies is the order written here. + ours := Openings(rules, with.Foundation, Published(out)) + ours = append(ours, GuardResources(r.guarded(out, owner, rules, with))...) + out = append(ours, out...) + } + return out, nil +} + +// guarded is what the mesh's guard refuses on an adopted node (novox/hq ADR 0103): derived, and +// for taken modules only. +// +// For each taken module, every machine port its containers publish that the filter would admit +// from the private network only — a published port is forwarded, not received, so a found +// firewall filtering only what it receives never sees it — together with the ports the module's +// manifest guards explicitly (the store's port, the broker's management port), wherever the +// machine put them. A port of a module assigned but not taken is not guarded: it may still be the +// predecessor's, serving the predecessor's other machines. The foundation's ports are admitted +// from everywhere and are never guarded. +func (r Resolution) guarded(out []map[string]any, owner map[string]string, rules []Rule, + with Rendering) []int { + meshOnly := map[int]bool{} + fromEverywhere := map[int]bool{} + for _, rule := range rules { + if rule.Protocol != "tcp" { + continue + } + switch rule.From { + case FromMesh: + meshOnly[rule.Port] = true + case FromEverywhere: + fromEverywhere[rule.Port] = true + } + } + for _, port := range with.Foundation { + delete(meshOnly, port) + fromEverywhere[port] = true + } + seen := map[int]bool{} + var ports []int + guard := func(at int) { + if !seen[at] { + seen[at] = true + ports = append(ports, at) + } + } + for _, m := range r.Modules { + if !with.Taken[m.Module] { + continue + } + var mine []map[string]any + for _, resource := range out { + if owner[fmt.Sprint(resource["id"])] == m.Module { + mine = append(mine, resource) + } + } + for outer := range Published(mine)["tcp"] { + if meshOnly[outer] { + guard(outer) + } + } + for _, want := range m.Guards { + at := with.machinePort(m.Module, want) + for _, resource := range mine { + if fmt.Sprint(resource["type"]) != "container" { + continue + } + listed, _ := resource["ports"].([]any) + for _, entry := range listed { + outer, inner, _, ok := mapping(fmt.Sprint(entry)) + if ok && inner == want { + at = outer + } + } + } + // Not what this node is told to open to everyone: a per-node exposure setting that + // widens a guarded port is the operator saying so, and declaring an opening for it + // and a guard dropping it would be one statement refusing the other. + if !fromEverywhere[at] { + guard(at) + } + } + } + sort.Ints(ports) + return ports +} + +// mapping reads a container's port mapping — `[address:]outer:inner[/protocol]`, the address +// possibly an IPv6 one in brackets — indexing from the end, so an address's own colons never +// shift the ports. Not ok for a short form or anything that is not a mapping. +func mapping(written string) (outer, inner int, address string, ok bool) { + written = strings.TrimSpace(written) + if cut := strings.LastIndex(written, "/"); cut >= 0 { + written = written[:cut] + } + parts := strings.Split(written, ":") + if len(parts) < 2 { + return 0, 0, "", false + } + inner, err := strconv.Atoi(parts[len(parts)-1]) + if err != nil { + return 0, 0, "", false + } + outer, err = strconv.Atoi(parts[len(parts)-2]) + if err != nil { + return 0, 0, "", false + } + return outer, inner, strings.Join(parts[:len(parts)-2], ":"), true +} + +// filtersHere is the module on this node that loads the machine's packet filter, or empty when none +// does. Named rather than counted: a refusal that says which module is one step from acted on. +func (r Resolution) filtersHere() string { + for _, m := range r.Modules { + if m.Filtering != nil { + return m.Module + } + } + return "" +} + +// Rules is the rule set this node's filter is derived from: every module's listens, what was +// computed for this machine, and each module's per-node exposure. The same answer whether the node +// is adopted or converged — the one loads it as a filter, the other declares it as openings. +func (r Resolution) Rules(with Rendering) ([]Rule, error) { + // A module whose settings cannot compose is left out of the declaration (novox/hq ADR 0163, + // rule 6), and out of the filter with it: nothing of it is declared, so nothing of it is let + // through. + left := r.LeftOut(with.Settings, with.Adopted) + exposure := map[string]map[int]string{} + for _, m := range r.Modules { + if _, isLeft := left[m.Module]; isLeft { + continue + } + e, err := Exposure(m, with.Settings[m.Module]) + if err != nil { + return nil, err + } + // And how far each endpoint reaches, which says the same thing to the filter and more + // besides (novox/hq ADR 0138). Folded in here rather than beside: the filter has one + // question — from where — and a reach answers it, so giving it two inputs would let them + // disagree. Reaches refuses a port that both name, so this cannot silently prefer one. + reaches, err := Reaches(m, with.Settings[m.Module]) + if err != nil { + return nil, err + } + // **Only for an endpoint the proxy does not serve.** A routed endpoint's port is how the + // proxy reaches it and nothing else (ADR 0045), so `public` there asks for a public name and + // says nothing about the port — opening it to the world as well would undo the arrangement + // the proxy exists for, and would silently reopen a port an operator had narrowed. + // + // Found by trying to express a real module: one whose routed name must be public and whose + // machine-side port must not be. Under one value for both, there was no way to say it. + routed := RoutedPorts(m) + for port, reach := range reaches { + if routed[port] { + continue + } + source, ok := FilterSource(reach) + if !ok { + return nil, fmt.Errorf("%s: %q is not a reach the filter can read", m.Module, reach) + } + if e == nil { + e = map[int]string{} + } + e[port] = source + } + if e != nil { + exposure[m.Module] = e + } + } + return r.Filtering(with.Generators, with.Ports, exposure) +} + +// Contribution is one module telling the answer to a requirement what it needs from it. +type Contribution struct { + // From is the module that said it, so the provider and a person reading the file can tell + // which route belongs to what. + From string `json:"from"` + // Node is the machine it said it from, empty when that is this one. + // + // A provision answered from anywhere in the mesh has consumers on other machines, and the + // provider has to know who they are — a database told to create a password and not who for + // cannot do anything with it. Contributions were node-local until this, which meant the one + // case that most needed them was the one they did not reach. + Node string `json:"node,omitempty"` + // At is where that machine is on the private network, empty when it is not on one or when it + // is this machine. + // + // The mesh knows it and a provider should not have to derive it. A reverse proxy is told + // *send traffic to this consumer* and has to open a connection — so without this every + // provider that reaches back to a consumer would have to know how the mesh names machines, + // which is a convention leaking into every module that implements a provision. + At string `json:"at,omitempty"` + // As is who this consumer is: what the provider should call the login it creates. + // + // **Said by the mesh rather than invented by the provisioner** (novox/hq 04-ISSUES/023). It + // used to be neither — the provisioner made a name, and the consumer, which has to present it + // to authenticate, had no way to learn it. One derivation reaches both ends, so they agree by + // construction. + // Omitted when there is none. A contribution that is not a credential grant — a module + // offering something to another on its own machine — has nobody to be identified to, and a + // field that is always present and usually empty teaches a reader to ignore it. + As string `json:"as,omitempty"` + // Secret is the file on this machine holding that consumer's credential, sealed to it. + // + // Named rather than carried, for the same reason the private network's key is: the mesh + // discarded the value and could not put it here if it wanted to. What is here is where to + // find it. + Secret string `json:"secret,omitempty"` + // Values are the module's own, with settings applied. What the keys mean is agreed by the + // requirement's name — everything providing `reverse-proxy` understands the same shape, which + // is what makes swapping one for another cost nothing. + Values map[string]any `json:"values"` + // Derived is what this provider's own definition said it derives for this consumer, already + // derived (novox/hq ADR 0202). + // + // **The provider is told, rather than recomputing it.** A served value may name the consumer's + // identity — a bucket named for who is asking, a database prefixed with it — and before this + // the rule lived twice: once in the provisioner's code, once transcribed into every consumer's + // definition. The mesh fills the provider's own statement here and delivers the same filled + // value to the consumer, so the two cannot disagree: there is no second computation to + // disagree with. + // + // Only the keys that are per-consumer. The rest of what the provider serves is the same for + // everyone and is in its own definition, where it already is. + Derived map[string]any `json:"derived,omitempty"` +} + +// provisionsAs is the account that reads what the mesh writes for this provider: the one secret +// per consumer it must open to set that consumer's password (novox/hq issue 225). +// +// **A root-owned 0600 file is one that process cannot read**, which is the same sentence already +// written above for a module's own secrets — and the grant secret is the other kind of secret +// the mesh writes for a module, so it is the same rule. +// +// Which account depends on where the module's code runs. A module whose code is a bundle is run +// by the node's tool runtime, as the node's account ([ADR 0198](0198)); one still in a container +// is whatever it declares as its secrets owner. Nothing names these paths, so the rule that +// claims a bundle's other files by the words that name them (givenTo) cannot reach them: the +// harness composes a grant secret's path from the contributions file, not from a word. +// +// Empty is root, which is what it was and what a module with no bundle and no declared owner +// still wants. +func (r Resolution) provisionsAs(m Manifest) string { + // A module of its own account reads them as that account (ADR 0259 §8, issue 723). + if m.RunsAs != "" { + return m.RunsAs + } + if len(m.Bundles) > 0 && r.Account != "" { + return r.Account + } + return m.SecretsOwner +} + +// grantPath is where one consumer's sealed credential lands on the providing machine. +// +// Suffixed, so the directory can also hold whatever the module writing it keeps there and so a +// node named like something else in that directory cannot collide with it. +// One file per consumer, and a consumer is a module on a machine (novox/hq 04-ISSUES/022). +// +// Named after both. Named after the machine alone, two modules on one node wrote to one path: the +// second overwrote the first, and the provisioner — reading a directory — saw one consumer where +// there were two. +// reflectsRenamed is a resource's restart-on list under the module's prefix, or nil when it has +// none. It reads both the shape JSON parsing produces ([]any) and the shape code composing +// resources natively produces ([]string): a reference that was skipped because its list arrived +// in the other shape would point at nothing — silently, with the service never restarting and +// every check passing, which is how a runtime kept serving without the registry trust its +// daemon file already carried. +func reflectsRenamed(module string, reflects any) []any { + var names []string + switch v := reflects.(type) { + case []any: + for _, id := range v { + names = append(names, fmt.Sprint(id)) + } + case []string: + names = v + } + if names == nil { + return nil + } + var renamed []any + for _, named := range names { + if strings.Contains(named, ".") { + renamed = append(renamed, named) + continue + } + renamed = append(renamed, module+"."+named) + } + return renamed +} + +func grantPath(directory, consumer, module string) string { + return strings.TrimRight(directory, "/") + "/" + consumer + "." + module + ".secret" +} + +// holderAs is a consumer's name at the provider with a local name after it, where it keeps several +// credentials for one provision (ADR 0094); the name alone otherwise. +func holderAs(as, local string) string { + if local == "" { + return as + } + return as + "_" + local +} + +// contributions collects what every module in this set contributes, by requirement. +// +// Ordered by contributing module, because the result becomes a file on a machine and a file whose +// lines move about is a file that looks changed when nothing changed. +func (r Resolution) contributions(settings SettingsBy, grants []Grant, + directories map[string]string) (map[string][]Contribution, error) { + out := map[string][]Contribution{} + modules := append([]Manifest{}, r.Modules...) + sort.Slice(modules, func(i, j int) bool { return modules[i].Module < modules[j].Module }) + + // What consumers on other machines asked for. Merged in with this machine's own, because from + // the provider's side they are the same thing — somebody wanting something — and a provider + // that had to read two lists would be a provider that reads one of them. + sorted := append([]Grant{}, grants...) + sort.Slice(sorted, func(i, j int) bool { + if sorted[i].Provision != sorted[j].Provision { + return sorted[i].Provision < sorted[j].Provision + } + if sorted[i].Consumer != sorted[j].Consumer { + return sorted[i].Consumer < sorted[j].Consumer + } + if sorted[i].From != sorted[j].From { + return sorted[i].From < sorted[j].From + } + return sorted[i].Local < sorted[j].Local + }) + // A consumer already carried by the grants loop, keyed (provision, module). When provider and + // consumer are co-located, `grantsFor` enumerates the same-node consumer too, so without this the + // module would be emitted a second time by the m.Contributes loop below — once full (with the + // grant's secret/as) and once partial — which is the duplicate seen in a co-located mesh.json. + granted := map[string]map[string]bool{} + for _, g := range sorted { + if g.From == "" { + // As above: nothing on that machine asks for this any more, so the provider is not + // told about it and withdraws the login on its next pass. + continue + } + as := holderAs(ConsumerIdentity(g.Consumer, IdentitySource(g.Slug, g.From)), g.Local) + derived, err := r.derivedFor(g.Provision, as, g.From, g.Local, settings) + if err != nil { + return nil, err + } + out[g.Provision] = append(out[g.Provision], Contribution{ + From: g.From, Node: g.Consumer, At: g.At, Values: g.Values, Derived: derived, + // One holder per local name: the identity the consumer is known by, and the local name + // after it where the module keeps several (ADR 0094). Not a login any backend checks — + // a secret is not a login — so the identity limit does not apply to the suffix. + As: as, + Secret: grantPath(directories[g.Provision], g.Consumer, holderAs(g.From, g.Local)), + }) + if granted[g.Provision] == nil { + granted[g.Provision] = map[string]bool{} + } + granted[g.Provision][g.From] = true + } + for _, m := range modules { + for _, to := range sortedKeys(m.Contributes) { + // The grants loop already emitted this module's contribution to this provision, with its + // minted secret — emitting the partial copy again would duplicate it. A contribution with + // no grant (a reverse-proxy route names a host, not a credential) is not in `granted`, so + // it still reaches the provider from here. + if granted[to][m.Module] { + continue + } + // Settings reach a contribution the same way they reach a file. A route's hostname is + // exactly the kind of thing that differs between one mesh and the next, and a module + // that could not have it set would have to be edited to be reused. + values, err := r.composed(m, to, m.Contributes[to], settings[m.Module], + m.Module+" contributing to "+to) + if err != nil { + return nil, err + } + out[to] = append(out[to], Contribution{From: m.Module, Values: values}) + } + // Several contributions to one requirement (ADR 0094's sibling for `contributes`): an + // object store's data API and its console are two different public names from one module, + // not one. Never in `granted` — a route names a host, not a credential — so every local + // name always reaches the provider from here. + for _, to := range sortedKeys(m.ContributesMany) { + for _, local := range sortedKeys(m.ContributesMany[to]) { + values, err := r.composed(m, to, m.ContributesMany[to][local], settings[m.Module], + m.Module+" contributing "+local+" to "+to) + if err != nil { + return nil, err + } + out[to] = append(out[to], Contribution{From: m.Module, Values: values}) + } + } + } + return out, nil +} + +// composed is one contribution as its provider receives it: settled with this node's settings, its +// endpoint's port filled in, and its names composed from the label. +// +// **One function, because two readers must agree.** The provider is told the names in its received +// file; the contributing module is told the same names in its own binding (novox/hq 04-ISSUES/122). +// Composing them twice, in two places, is how the proxy would come to serve one name while the +// module wrote another into its configuration. +func (r Resolution) composed(m Manifest, to string, raw map[string]any, layers []Layer, what string) ( + map[string]any, error) { + // Overridden, not merged: a setting changes a key the contribution declares and adds none. + // The provider reads the contribution as a contract, and a setting made for one of this + // module's files is no part of it (novox/hq 04-ISSUES/173). + values, err := overridden(raw, WithDefaults(m, layers), what) + if err != nil { + return nil, fmt.Errorf("%s: %w", what, err) + } + reaches, err := Reaches(m, layers) + if err != nil { + return nil, fmt.Errorf("%s: %w", what, err) + } + blocks, err := Endpoints(m, layers) + if err != nil { + return nil, fmt.Errorf("%s: %w", what, err) + } + portOfEndpoint(values, endpointPorts(m)) + composeName(values, r.PublicDomain, servingAt(r, to), reaches, endpointPorts(m), blocks) + return values, nil +} + +// ownNames is what a module is known by through what it contributes to one requirement — the names +// the mesh composed for it, and nothing else of the contribution. +// +// **The half a module could not learn** (novox/hq 04-ISSUES/122). A module contributes a label, the +// mesh joins it with this node's domains, and the provider serves the result — and the module itself +// was never told. Software that must know its own address (a login redirect, a canonical URL, an +// issuer) had it written into the manifest as a literal, which is a domain in a definition and wrong +// on every other machine. `${bound::name}` is the answer, from the same composition the +// provider receives. +// +// Several contributions to one requirement are keyed by their local name under `names`. +func (r Resolution) ownNames(m Manifest, to string, layers []Layer) (map[string]any, error) { + pick := func(values map[string]any) map[string]any { + names := map[string]any{} + for _, key := range []string{"name", "internal-name"} { + if v, ok := values[key].(string); ok && v != "" { + names[key] = v + } + } + return names + } + out := map[string]any{} + if raw, ok := m.Contributes[to]; ok { + values, err := r.composed(m, to, raw, layers, m.Module+" contributing to "+to) + if err != nil { + return nil, err + } + for k, v := range pick(values) { + out[k] = v + } + } + if locals := m.ContributesMany[to]; len(locals) > 0 { + many := map[string]any{} + for _, local := range sortedKeys(locals) { + values, err := r.composed(m, to, locals[local], layers, + m.Module+" contributing "+local+" to "+to) + if err != nil { + return nil, err + } + if names := pick(values); len(names) > 0 { + many[local] = names + } + } + if len(many) > 0 { + out["names"] = many + } + } + return out, nil +} + +// composeName joins a contribution's label with a node's public domain, and separately with its +// private one, in place (novox/hq ADR 0056). +// +// **The whole of what the mesh does with a route's name: join two given strings — twice.** A +// contribution carries a `label` — the subdomain its operator chose — and the node carries its +// public domain and its own private-network address; the granted names are `