diff --git a/go/catalogue/file_references_test.go b/go/catalogue/file_references_test.go new file mode 100644 index 0000000..62f726d --- /dev/null +++ b/go/catalogue/file_references_test.go @@ -0,0 +1,10 @@ +package catalogue + +import "testing" + +// 1694/AC5: the capability a node-engine reports for files by digest (novox/hq ADR 0486 §5) is a known one. +func TestFileReferencesIsAKnownCapability(t *testing.T) { + if !oneOf(KnownCapabilities, "file-references") { + t.Fatal("file-references is not a known capability") + } +} diff --git a/go/catalogue/files.go b/go/catalogue/files.go index f1356ed..a1231a3 100644 --- a/go/catalogue/files.go +++ b/go/catalogue/files.go @@ -74,6 +74,22 @@ func filesVerbs() []Verb { map[string]string{"path": "an absolute path", "owner": "an account's name or number", "group": "a group's name or number (optional)", "recursive": "\"true\": also everything under a directory"}, []string{"path", "owner"}, []string{"group", "recursive"}, "chown", "chgrp"), + Verb{Name: "upload", Description: "Send one file of this node into a bucket of the object store, over a short signed link " + + "and never over the bus; answers the key (the file's SHA-256 when none is given), the size and the SHA-256. " + + "At most 100 MiB. A place a secret is kept, a known credential file, or a link to one is refused by name, " + + "before anything is sent. Read " + filesAsAccount + ".", + Input: schema(map[string]string{"path": "an absolute path to a file", "bucket": "the bucket the file goes into", + "key": "the object's key (optional; the file's SHA-256 when not given)"}, []string{"path", "bucket"})}, + // download is a plain Verb, not through write(): write() types its optional arguments as true/false + // switches, which a hash and a reason are not (issue 1719). + Verb{Name: "download", Description: "Write one object of a bucket to a file of this node, over a short signed link and " + + "never over the bus; the old file goes to the node's trash. At most 100 MiB. The bytes must hash to sha256, or to " + + "the key when the key is a SHA-256; otherwise no file is written. Refuses a path a module owns " + + "unless override names a reason; done as the node's operator account, through sudo -n where its rights do not reach.", + Input: schema(map[string]string{"bucket": "the bucket", "key": "the object's key", "path": "an absolute path", + "sha256": "the SHA-256 the bytes must have (optional; the key when that is a SHA-256)", + "reason": "why the file is changed, for the audit trail", + "override": overrideWord}, []string{"bucket", "key", "path", "reason"})}, Verb{Name: "trash", Description: "What the node's trash holds: each deleted or replaced item with its id, " + "original path and when it went.", Input: schema(map[string]string{}, nil), Replaces: []string{"ls ~/.local/share/Trash"}}, diff --git a/go/catalogue/files_test.go b/go/catalogue/files_test.go index 63f6923..6cd1742 100644 --- a/go/catalogue/files_test.go +++ b/go/catalogue/files_test.go @@ -2,6 +2,7 @@ package catalogue import ( "encoding/json" + "sort" "strings" "testing" ) @@ -22,7 +23,8 @@ func TestNodeFilesIsANodeSeatWithEveryVerbOfADR0394(t *testing.T) { if v.Optional { t.Errorf("%s.%s is optional", FilesSeat, v.Name) } - if len(v.Replaces) == 0 { + // upload and download (ADR 0468 §9, issue 1718) move bytes through the object store: no command is theirs. + if len(v.Replaces) == 0 && v.Name != "upload" && v.Name != "download" { t.Errorf("%s.%s says nothing it replaces", FilesSeat, v.Name) } for _, c := range v.Replaces { @@ -30,7 +32,7 @@ func TestNodeFilesIsANodeSeatWithEveryVerbOfADR0394(t *testing.T) { } } for _, name := range []string{"stat", "readlink", "list", "read", "search", "account", "write", "copy", "move", - "delete", "mkdir", "chmod", "chown", "trash", "restore"} { + "delete", "mkdir", "chmod", "chown", "upload", "download", "trash", "restore"} { if !have[name] { t.Errorf("%s serves no verb %s", FilesSeat, name) } @@ -109,3 +111,93 @@ func TestNodeFilesRestoreTakesIdAndReason(t *testing.T) { } t.Error("no restore verb") } + +// 1718/AC1: node-files names upload and download, in mesh-files' order, before trash. +func TestNodeFilesNamesUploadAndDownloadBeforeTrash(t *testing.T) { + s, _ := SeatNamed(FilesSeat) + var names []string + for _, v := range s.Serves { + names = append(names, v.Name) + } + joined := strings.Join(names, ",") + if !strings.Contains(joined, "chown,upload,download,trash") { + t.Errorf("1718/AC1: node-files' verbs are %s, want upload and download between chown and trash", joined) + } +} + +// 1718/AC2: a holder serving every verb including upload and download is accepted; one naming a verb outside the +// contract is still refused, and one leaving upload out is refused. +func TestAHolderOfNodeFilesServesUploadAndDownloadAndNothingElse(t *testing.T) { + s, _ := SeatNamed(FilesSeat) + var verbs []string + for _, v := range s.Serves { + verbs = append(verbs, v.Name) + } + hold := func(serves []string) error { + return CanHold(Manifest{Module: "mesh-files", Version: "1", + Claims: []Claim{{Name: FilesSeat, Scope: ScopeNode, Serves: serves}}}, s) + } + if err := hold(verbs); err != nil { + t.Errorf("1718/AC2: a holder serving every verb was refused: %v", err) + } + if err := hold(append(append([]string{}, verbs...), "sync")); err == nil || !strings.Contains(err.Error(), "does not promise") { + t.Errorf("1718/AC2: a verb outside the contract was accepted: %v", err) + } + var without []string + for _, v := range verbs { + if v != "upload" { + without = append(without, v) + } + } + if err := hold(without); err == nil || !strings.Contains(err.Error(), "upload") { + t.Errorf("1718/AC2: a holder leaving upload out was accepted: %v", err) + } +} + +// 1718/AC3: upload requires path and bucket; download requires bucket, key, path and reason; sha256, key (upload) +// and override are plain strings, not true/false switches. +func TestNodeFilesUploadAndDownloadInputs(t *testing.T) { + s, _ := SeatNamed(FilesSeat) + want := map[string]struct { + required string + strs []string + }{ + "upload": {"path,bucket", []string{"path", "bucket", "key"}}, + "download": {"bucket,key,path,reason", []string{"bucket", "key", "path", "sha256", "reason", "override"}}, + } + seen := 0 + for _, v := range s.Serves { + w, ok := want[v.Name] + if !ok { + continue + } + seen++ + req, _ := v.Input["required"].([]string) + got := append([]string{}, req...) + sortStrings(got) + exp := strings.Split(w.required, ",") + sortStrings(exp) + if strings.Join(got, ",") != strings.Join(exp, ",") { + t.Errorf("1718/AC3: %s requires %v, want %s", v.Name, req, w.required) + } + props, _ := v.Input["properties"].(map[string]any) + if len(props) != len(w.strs) { + t.Errorf("1718/AC3: %s has %d inputs, want %v", v.Name, len(props), w.strs) + } + for _, name := range w.strs { + p, has := props[name].(map[string]any) + if !has { + t.Errorf("1718/AC3: %s has no input %s", v.Name, name) + continue + } + if _, enum := p["enum"]; enum || p["type"] != "string" { + t.Errorf("1718/AC3: %s input %s is %v, want a plain string", v.Name, name, p) + } + } + } + if seen != 2 { + t.Errorf("1718/AC3: found %d of upload and download", seen) + } +} + +func sortStrings(s []string) { sort.Strings(s) } diff --git a/go/catalogue/groups_test.go b/go/catalogue/groups_test.go new file mode 100644 index 0000000..b12276d --- /dev/null +++ b/go/catalogue/groups_test.go @@ -0,0 +1,149 @@ +package catalogue + +import ( + "reflect" + "strings" + "testing" +) + +// A container's supplementary groups may be an operator's value (novox/hq ADR 0480 clause 7, ADR 0348 §11, +// issue 1636): `groups: ["${config:media-gid}"]`, each element a whole placeholder or a whole number, filled at +// composition with the number the config value holds. + +func aContainerWithGroups(groups ...any) map[string]any { + return map[string]any{"id": "server", "type": "container", "name": "n8n", "image": "x@sha256:" + strings.Repeat("a", 64), + "groups": groups} +} + +// 1636/AC3: a manifest container with groups: ["${setting:media-gid}"] resolves to the number the setting says. +func TestAContainersGroupIsFilledFromTheSetting(t *testing.T) { + r := aContainerWithGroups("${setting:media-gid}", float64(2001)) + layers := []Layer{{From: "ace", Values: map[string]any{"media-gid": float64(2000)}}} + if err := settingInto(r, layers, "n8n"); err != nil { + t.Fatal(err) + } + if want := []any{2000, 2001}; !reflect.DeepEqual(r["groups"], want) { + t.Fatalf("composed as %#v, want %#v", r["groups"], want) + } + // A value the operator wrote as text is read as the number it spells. + r = aContainerWithGroups("${setting:media-gid}") + if err := settingInto(r, []Layer{{From: "ace", Values: map[string]any{"media-gid": "2000"}}}, "n8n"); err != nil { + t.Fatal(err) + } + if want := []any{2000}; !reflect.DeepEqual(r["groups"], want) { + t.Fatalf("composed as %#v, want %#v", r["groups"], want) + } +} + +// 1636/AC3: an unset value refuses the module by name, and the key and the remedy are named; nothing is dropped. +func TestAnUnsetGroupRefusesTheModuleByName(t *testing.T) { + r := aContainerWithGroups("${setting:media-gid}") + err := settingInto(r, nil, "n8n") + if err == nil || !strings.Contains(err.Error(), "n8n") || !strings.Contains(err.Error(), "media-gid") { + t.Fatalf("want a refusal naming n8n and media-gid, got %v", err) + } + if got, _ := r["groups"].([]any); len(got) != 1 || got[0] != "${setting:media-gid}" { + t.Fatalf("the resource was changed: %#v", r["groups"]) + } +} + +// 1636/AC2: a value that is not a whole number above zero cannot be a group. +func TestAGroupThatIsNotAWholeNumberAboveZeroIsRefused(t *testing.T) { + for _, bad := range []any{"media", "-1", "0", float64(0), float64(2000.5), "2000:2000", "", true} { + r := aContainerWithGroups("${setting:media-gid}") + err := settingInto(r, []Layer{{From: "ace", Values: map[string]any{"media-gid": bad}}}, "n8n") + if err == nil || !strings.Contains(err.Error(), "media-gid") { + t.Errorf("value %#v: want a refusal naming the key, got %v", bad, err) + } + } +} + +// A manifest in the config form registers with the group a config value fills, and the value is placed (a container's +// groups count as a place for it); a placeholder in any other shape is refused where the manifest is read. +func TestAManifestWhoseContainerTakesItsGroupFromConfigRegisters(t *testing.T) { + m := map[string]any{ + "module": "n8n", "version": "1", + "config": map[string]any{"media-gid": map[string]any{"type": "number", "minimum": 1, "required": true, + "description": "the group that owns the media library on this machine"}}, + "resources": []any{aContainerWithGroups("${config:media-gid}")}, + } + if _, err := parseRaw(t, m); err != nil { + t.Fatalf("refused: %v", err) + } + for _, shape := range []string{"x${config:media-gid}", "${config:media-gid}:1", "${config:other}"} { + m["resources"] = []any{aContainerWithGroups(shape)} + if _, err := parseRaw(t, m); err == nil { + t.Errorf("groups %q registered", shape) + } + } +} + +// 1636/AC3 composed for a machine, the way a push writes it: n8n's group is the number the machine's config value +// says, written as a number, and the module is left out by name when the machine says none. +func TestAComposedMachineGetsTheGroupTheConfigValueNames(t *testing.T) { + manifest, err := parseRaw(t, map[string]any{ + "module": "n8n", "version": "1", + "config": map[string]any{"media-gid": map[string]any{"type": "number", "minimum": 1, "required": true, + "description": "the group that owns the media library on this machine"}}, + "resources": []any{aContainerWithGroups("${config:media-gid}")}, + }) + if err != nil { + t.Fatal(err) + } + r := anAdoptedAnchor() + r.Modules = append(r.Modules, manifest) + with := anchorRendering(false) + with.Settings["n8n"] = []Layer{{From: "anchor", Values: map[string]any{"media-gid": float64(2000)}}} + composed, err := r.Compose(with) + if err != nil { + t.Fatal(err) + } + if why, left := composed.LeftOut["n8n"]; left { + t.Fatalf("left out: %s", why) + } + if got := byID(composed.Resources)["n8n.server"]["groups"]; !reflect.DeepEqual(got, []any{2000}) { + t.Fatalf("composed groups %#v", got) + } + + without := anchorRendering(false) + composed, err = r.Compose(without) + if err == nil { + why, left := composed.LeftOut["n8n"] + if !left || !strings.Contains(why, "media-gid") { + t.Fatalf("a machine that says no group kept n8n or left it out without the key: %q (%v)", why, left) + } + } else if !strings.Contains(err.Error(), "media-gid") { + t.Fatalf("refused without the key: %v", err) + } +} + +// The node-engine reads a container's groups only when it reports the capability (novox/hq ADR 0480 clause 7, issue 1638): +// a module with a container that names groups is hosted on a machine that reports container-groups, and left out of one +// that does not, the capability named, so the field is never sent to an engine that would refuse it. +func TestAModuleWhoseContainerNamesGroupsIsHeldOnAMachineThatDoesNotReadThem(t *testing.T) { + n8n := Manifest{Module: "n8n", Resources: []map[string]any{aContainerWithGroups("${config:media-gid}")}} + plain := Manifest{Module: "plain", Resources: []map[string]any{ + {"id": "server", "type": "container", "name": "p", "image": "x@sha256:" + strings.Repeat("b", 64)}}} + for _, m := range []Manifest{n8n, plain} { + if got := impliedCapabilities(m); (m.Module == "n8n") != reflect.DeepEqual(got, []string{ContainerGroups}) { + t.Errorf("%s implies %v", m.Module, got) + } + } + shelf := shelf(n8n, plain) + reads := Node{Name: "ace", Capabilities: map[string]bool{"container-runtime": true, ContainerGroups: true}} + got, err := Resolve(shelf, []string{"n8n", "plain"}, reads, World{}) + if err != nil || len(got.Unhostable) != 0 || len(got.Modules) != 2 { + t.Fatalf("a machine that reads groups: %v, held %+v, hosted %v", err, got.Unhostable, names(got)) + } + old := Node{Name: "g14", Capabilities: map[string]bool{"container-runtime": true}} + got, err = Resolve(shelf, []string{"n8n", "plain"}, old, World{}) + if err != nil { + t.Fatal(err) + } + if len(got.Unhostable) != 1 || got.Unhostable[0].Module != "n8n" || !reflect.DeepEqual(got.Unhostable[0].Missing, []string{ContainerGroups}) { + t.Fatalf("n8n was not held, naming the capability: %+v", got.Unhostable) + } + if h := names(got); len(h) != 1 || h[0] != "plain" { + t.Errorf("the module without groups is not the one hosted: %v", h) + } +} diff --git a/go/catalogue/manifest.go b/go/catalogue/manifest.go index 0464654..870ea4d 100644 --- a/go/catalogue/manifest.go +++ b/go/catalogue/manifest.go @@ -2214,6 +2214,7 @@ func parseManifest(raw []byte, predefined PredefinedList) (Manifest, error) { problems = append(problems, replacesProblems(m)...) problems = append(problems, UnitSettingProblems(m)...) problems = append(problems, OwnerSettingProblems(m)...) + problems = append(problems, GroupSettingProblems(m)...) problems = append(problems, endpointNameProblems(m)...) problems = append(problems, RouteProblems(m)...) for _, port := range m.Guards { diff --git a/go/catalogue/resolve.go b/go/catalogue/resolve.go index 6e2e08f..006646e 100644 --- a/go/catalogue/resolve.go +++ b/go/catalogue/resolve.go @@ -904,7 +904,7 @@ func isModule(catalogue map[string]Manifest, want string) bool { // machine can host the module as far as its capabilities go. func missingCapabilities(m Manifest, node Node) []string { var missing []string - for _, c := range m.Capabilities { + for _, c := range append(append([]string(nil), m.Capabilities...), impliedCapabilities(m)...) { if !node.Capabilities[c] { missing = append(missing, c) } @@ -1457,3 +1457,20 @@ func packageNames(m map[string][]string) []string { sort.Strings(names) return names } + +// ContainerGroups is the capability a node-engine reports when it reads a container's `groups` and passes each as a +// supplementary group (novox/hq ADR 0480 clause 7, issue 1638). An older engine refuses the key (strict decoding). +const ContainerGroups = "container-groups" + +// impliedCapabilities are the capabilities a module needs for what its resources say, beyond those it declares: a +// container that names groups needs an engine that reads them, so the module is left out of a machine whose report +// lacks the capability, naming it, and the field is sent to no engine that would refuse it. None for a module whose +// containers name no groups. +func impliedCapabilities(m Manifest) []string { + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) == "container" && len(groupsOf(r)) > 0 { + return []string{ContainerGroups} + } + } + return nil +} diff --git a/go/catalogue/seat_data.go b/go/catalogue/seat_data.go index bea4fc7..90a1fb7 100644 --- a/go/catalogue/seat_data.go +++ b/go/catalogue/seat_data.go @@ -66,7 +66,8 @@ var capabilityName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`) // names an `if-capability` may give (novox/hq ADR 0255). A name nothing detects would leave its // contribution out on every machine, silently, so it is refused. A detector added there is added here. var KnownCapabilities = []string{ - "battery", "container-runtime", "firewall", "graphical-session", "overlay", "package-manager", + "battery", "container-groups", "container-runtime", "file-references", "firewall", "graphical-session", "overlay", + "package-manager", "power-meter", "privileged", "seat", "service-manager", "uplink-dhcpcd", "uplink-networkmanager", "uplink-systemd-networkd", "user-namespaces", "virtualisation", } diff --git a/go/catalogue/setting_into.go b/go/catalogue/setting_into.go index 3c38bb7..e16e28d 100644 --- a/go/catalogue/setting_into.go +++ b/go/catalogue/setting_into.go @@ -85,6 +85,9 @@ func settingInto(resource map[string]any, layers []Layer, module string) error { if fmt.Sprint(resource["type"]) == "directory" { return settingIntoOwner(resource, layers, module) } + if fmt.Sprint(resource["type"]) == "container" { + return settingIntoGroups(resource, layers, module) + } if fmt.Sprint(resource["type"]) != "file" { return nil } @@ -343,6 +346,12 @@ func settingKeysAskedOutsideOwners(m Manifest) map[string]bool { if unit, ok := r["unit"].(string); ok { note(unit) } + case "container": + for _, g := range groupsOf(r) { + if s, ok := g.(string); ok { + note(s) + } + } } } // An item's text, filled from its template (WithTemplatesFrom), places a key as a file's does (issue 1658). @@ -369,3 +378,100 @@ func settingKeysAskedOutsideOwners(m Manifest) map[string]bool { } return used } + +// A container's supplementary groups may be an operator's value (novox/hq ADR 0480 clause 7, ADR 0348 §11, issue 1636): +// n8n joins the media library's group, whose number is the machine's, never the definition's (ADR 0112 §1). Each +// element of `groups` is a whole placeholder or a whole number; a placeholder is filled with the number the +// config value holds, and the list reaches the node-engine as numbers. + +// groupSide is one element of a container's groups: a whole placeholder or a whole number. +var groupSide = regexp.MustCompile(`^(?:\$\{setting:[a-z0-9][a-z0-9_.-]*\}|[0-9]+)$`) + +// groupsOf is a container resource's groups as the manifest wrote them (text or numbers), none when it names none. +func groupsOf(r map[string]any) []any { + groups, _ := r["groups"].([]any) + return groups +} + +// GroupSettingProblems refuses, where a manifest is read, a container whose groups hold anything but whole +// placeholders and whole numbers, or a placeholder whose key the module neither declares nor asks for elsewhere. +func GroupSettingProblems(m Manifest) []string { + var problems []string + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) != "container" { + continue + } + for _, g := range groupsOf(r) { + s, isText := g.(string) + if !isText { + continue // a number: the declaration judges its value + } + if !groupSide.MatchString(s) { + problems = append(problems, fmt.Sprintf("%s: the container %v's group %q is not a whole "+ + "${config:key} or a whole number: a supplementary group is a number, and nothing else is "+ + "written into it (novox/hq ADR 0480 clause 7)", m.Module, r["id"], s)) + continue + } + if m.Templated() { + continue + } + for _, key := range settingsUsed(s) { + if _, declared := m.Settings[key]; !declared && !m.DeclaresConfig() { + continue // the settings form judges its own keys + } + if _, declared := m.Settings[key]; !declared { + problems = append(problems, fmt.Sprintf("%s: the container %v's group says ${config:%s}, and the module "+ + "declares no such value (novox/hq ADR 0348 §4)", m.Module, r["id"], key)) + } + } + } + } + return problems +} + +// settingIntoGroups fills ${setting:…} in a container's groups, refusing — with the key and why, the resource left as +// it was — a key nothing sets and a value that is not a whole number above zero. A number is kept; the list that +// results is numbers alone. +func settingIntoGroups(resource map[string]any, layers []Layer, module string) error { + groups := groupsOf(resource) + if len(groups) == 0 { + return nil + } + filled := make([]any, 0, len(groups)) + for _, g := range groups { + s, isText := g.(string) + if !isText { + if f, isNumber := g.(float64); isNumber && f == float64(int(f)) { + g = int(f) // a number the manifest wrote, as the numbers a placeholder gives + } + filled = append(filled, g) + continue + } + keys := settingsUsed(s) + if len(keys) != 1 || settingRef.ReplaceAllString(s, "") != "" { + filled = append(filled, g) // not a whole placeholder: the manifest check refuses it + continue + } + key := keys[0] + value, set := settingValue(layers, key) + if !set { + return &UnsetSettingError{Module: module, Setting: key, + where: fmt.Sprintf("a container %v whose groups", resource["id"]), setToday: orNoSettings(layers), + said: fmt.Sprintf( + "%s has a container %v whose groups say ${setting:%s}, and nothing sets %q for it — an operator's "+ + "value is the assignment's, never the definition's (novox/hq ADR 0112): "+ + "`settings set %s ` with {%q: …}%s", + module, resource["id"], key, key, module, key, orNoSettings(layers))} + } + v := plainly(value) + id, whole := wholeID(v) + if !whole || id == 0 { + return fmt.Errorf("%s: the setting %q is %q, which cannot be a group of the container %v: a supplementary "+ + "group is a whole number above zero, never a name or other text (novox/hq ADR 0480 clause 7, ADR 0328 §15)", + module, key, v, resource["id"]) + } + filled = append(filled, int(id)) + } + resource["groups"] = filled + return nil +} diff --git a/go/catalogue/toolchain_manifests_test.go b/go/catalogue/toolchain_manifests_test.go new file mode 100644 index 0000000..a56fd2b --- /dev/null +++ b/go/catalogue/toolchain_manifests_test.go @@ -0,0 +1,88 @@ +package catalogue + +import ( + "strings" + "testing" +) + +// Defends novox/hq ADR 0477 §4 (issue 1573): the go and nodejs modules own the toolchain packages and provide +// them with the machine's reach, so a node without one refuses the module that needs it at resolution, naming +// the provision, and does not find out at its first build. The manifests are the catalogue's, as they are. + +// builderNode is a machine that can run a build module: the package manager and user namespaces it asks of +// the machine, and the forge account (mesh-scope, provided by the forge) stood in for. +func builderNode() Node { + return Node{Name: "builder", Site: "house", + Capabilities: map[string]bool{"package-manager": true, "user-namespaces": true}} +} + +func toolchainShelf(t *testing.T) map[string]Manifest { + t.Helper() + m := map[string]Manifest{ + "forge": {Module: "forge", Version: "1", Provides: []Offer{{Name: "forge-account"}, {Name: "npm-package-registry"}}}, + } + for _, name := range []string{"go", "nodejs", "mesh-build-go", "mesh-build-node", "dev-tools"} { + m[name] = catalogueManifest(t, name) + } + return m +} + +func TestAModuleNeedingAToolchainIsRefusedOnANodeWithoutIt(t *testing.T) { + for _, c := range []struct{ module, provision string }{ + {"mesh-build-go", "go-toolchain"}, + {"mesh-build-node", "node-toolchain"}, + {"dev-tools", "go-toolchain"}, + } { + _, err := Resolve(toolchainShelf(t), []string{"forge", c.module}, builderNode(), World{}) + if err == nil { + t.Fatalf("%s composed on a node with no toolchain module", c.module) + } + for _, want := range []string{`"` + c.provision + `" is wanted by ` + c.module, + "usable only on the machine that provides it"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("%s: the refusal does not say %q:\n%v", c.module, want, err) + } + } + } +} + +func TestAModuleNeedingAToolchainComposesOnANodeWithIt(t *testing.T) { + for _, c := range []struct{ module, owner string }{ + {"mesh-build-go", "go"}, + {"mesh-build-node", "nodejs"}, + {"dev-tools", "go"}, + } { + assigned := []string{"forge", c.owner, c.module} + if c.module == "dev-tools" { + assigned = append(assigned, "nodejs") + } + got, err := Resolve(toolchainShelf(t), assigned, builderNode(), World{}) + if err != nil { + t.Fatalf("%s with %s did not compose: %v", c.module, c.owner, err) + } + in := map[string]bool{} + for _, n := range names(got) { + in[n] = true + } + if !in[c.module] || !in[c.owner] { + t.Errorf("%s with %s composed %v", c.module, c.owner, names(got)) + } + } +} + +// Only the owners declare the packages: a build module that declared go again would compose without the +// owner's provision being met by anything, which is the gap the provision closes. +func TestOnlyTheToolchainModulesDeclareTheToolchainPackages(t *testing.T) { + owner := map[string]string{"go": "go", "nodejs": "nodejs", "npm": "nodejs"} + for name, m := range toolchainShelf(t) { + for _, r := range m.Resources { + if r["type"] != "package" { + continue + } + pkg, _ := r["package"].(string) + if o, ok := owner[pkg]; ok && o != name { + t.Errorf("%s declares the package %s, which only %s may", name, pkg, o) + } + } + } +} diff --git a/go/catalogue/unconsumed_placeholder.go b/go/catalogue/unconsumed_placeholder.go index 39df9ba..b017d5a 100644 --- a/go/catalogue/unconsumed_placeholder.go +++ b/go/catalogue/unconsumed_placeholder.go @@ -56,7 +56,7 @@ type filler struct { // seatInto, machineInto and contributionsInto. A pass that comes to read another field adds it here, // or the sweep refuses the placeholder it would have filled. var fillers = []filler{ - {namespace: "setting", patterns: []*regexp.Regexp{settingRef}, fields: map[string][]string{"file": {"content"}, "service": {"unit"}, "directory": {"owner"}}}, + {namespace: "setting", patterns: []*regexp.Regexp{settingRef}, fields: map[string][]string{"file": {"content"}, "service": {"unit"}, "directory": {"owner"}, "container": {"groups"}}}, {namespace: "dir", patterns: []*regexp.Regexp{dirRef}, fields: map[string][]string{"*": {"path", "content", "volumes", "env", "env-file"}}}, {namespace: "access", patterns: []*regexp.Regexp{accessRef}, fields: map[string][]string{"*": {"path", "content", "volumes", "env", "env-file"}}}, {namespace: "secret", patterns: []*regexp.Regexp{placeholder}, fields: map[string][]string{"file": {"content"}}, diff --git a/go/cli/refusal_test.go b/go/cli/refusal_test.go index 2adbc28..d05ec87 100644 --- a/go/cli/refusal_test.go +++ b/go/cli/refusal_test.go @@ -106,7 +106,7 @@ func TestAFailedWriteWaitsNoLongerThanTheDeadline(t *testing.T) { default: } }) - ctx, cancel := context.WithTimeout(context.Background(), time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) defer cancel() start := time.Now() _, err = Ask(ctx, socket, aLineTheSocketCannotHold()) @@ -117,7 +117,7 @@ func TestAFailedWriteWaitsNoLongerThanTheDeadline(t *testing.T) { if !errors.Is(err, os.ErrDeadlineExceeded) { t.Fatalf("the error is not the deadline: %v", err) } - if waited > 3*time.Second { - t.Fatalf("waited %s with a deadline of 1s", waited) + if waited > time.Second { + t.Fatalf("waited %s with a deadline of 50ms", waited) } } diff --git a/go/declaration/declaration.go b/go/declaration/declaration.go index ea16b91..cec208d 100644 --- a/go/declaration/declaration.go +++ b/go/declaration/declaration.go @@ -324,6 +324,15 @@ type File struct { // depend on a field somewhere else. Bytes string `json:"bytes,omitempty"` + // Digest and Size say the content is not carried but fetched: a file over FileReferenceThreshold travels as + // the sha256 digest of its content, "sha256:<64 hex>", and its size in bytes, in place of Content or Bytes + // (novox/hq ADR 0486 §1). The node-engine fetches the object of that name from FileBucketFor(node), checks what it + // received against Digest and Size, and only then writes the file. Exclusive with Content, Sealed, Bytes, + // VaultSealed and Secrets; sent only to an engine that reports FileReferencesCapability, as an older engine + // refuses a field it does not know, whole. + Digest string `json:"digest,omitempty"` + Size int64 `json:"size,omitempty"` + // Owner is the user this belongs to, by name. Absent means root. Owner string `json:"owner,omitempty"` @@ -465,6 +474,10 @@ func (f *File) validate(where string, _ bool) []string { problems = append(problems, where+": a file the vault sealed whole has no holes for secrets") } } + problems = append(problems, f.checkReference(where)...) + if f.Digest != "" || f.Size != 0 { + said = append(said, "digest") + } if len(said) > 1 { sort.Strings(said) problems = append(problems, where+ @@ -1454,6 +1467,14 @@ type Container struct { // container runs as its image says. Its key is `identity`, the manifest's word; the field is not, because // a resource's Identity is its id. RunAs *ContainerIdentity `json:"identity,omitempty"` + + // Groups are the supplementary groups the container's process joins, as numbers, each passed to the runtime + // as a group to add (novox/hq ADR 0480 clause 7, issue 1636): n8n writes the media library through the + // library's group, which a stock image knows nothing of. Each is above zero and below the kernel's "no group" + // (ADR 0328 §15: never root). Part of the spec, so a change recreates the container. Absent: none are added. + // An older node-engine refuses the key (strict decoding), so the controller sends it only to a node that + // reports it reads it. + Groups []int64 `json:"groups,omitempty"` } // The ways an image is told who to run as (novox/hq ADR 0325 §2). @@ -1521,6 +1542,30 @@ func (i *ContainerIdentity) problems(where string, env map[string]string) []stri return problems } +// groupProblems refuses a supplementary group that is root's or below it, the kernel's "no group" number, or a +// number named twice (novox/hq ADR 0480 clause 7, ADR 0328 §15). +func (c *Container) groupProblems(where string) []string { + var problems []string + seen := map[int64]bool{} + for _, g := range c.Groups { + switch { + case g == 0: + problems = append(problems, fmt.Sprintf("%s: container %s adds group 0, and the mesh never makes or keeps anything "+ + "as root: a supplementary group is never root's (novox/hq ADR 0328 §15)", where, c.Name)) + case g < 0: + problems = append(problems, fmt.Sprintf("%s: container %s adds group %d, below zero: a group is a whole number "+ + "above zero", where, c.Name, g)) + case g >= 1<<32-1: + problems = append(problems, fmt.Sprintf("%s: container %s adds group %d, which is the kernel's \"no group\" "+ + "number or beyond it", where, c.Name, g)) + case seen[g]: + problems = append(problems, fmt.Sprintf("%s: container %s adds group %d twice", where, c.Name, g)) + } + seen[g] = true + } + return problems +} + func (c *Container) Identity() string { return c.ID } func (c *Container) Kind() Type { return TypeContainer } func (c *Container) Target() string { return c.Name } @@ -1603,6 +1648,7 @@ func (c *Container) validate(where string, _ bool) []string { "runs and exits, and joins nothing afterwards") } problems = append(problems, c.RunAs.problems(where+": container "+c.Name, c.Env)...) + problems = append(problems, c.groupProblems(where)...) return append(problems, checkImage(where, c.Image)...) } diff --git a/go/declaration/filereference.go b/go/declaration/filereference.go new file mode 100644 index 0000000..665b0fe --- /dev/null +++ b/go/declaration/filereference.go @@ -0,0 +1,150 @@ +package declaration + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "fmt" + "regexp" +) + +// The three names of a file travelling by digest (novox/hq ADR 0486 §1, §2, §5), shared by the node-engine that +// reads them and the controller that writes them. +const ( + // FileReferenceThreshold is the size above which a file travels by digest: a file of this many bytes or fewer + // stays in the declaration as it is. + FileReferenceThreshold = 64 * 1024 + // FileBucketPrefix starts the name of a node's object bucket, FileBucketFor(node), which holds the contents of the + // files of that node's declaration, each under the hex of its sha256 digest (ObjectName). One bucket per node, + // so a grant names it and a literal `*` is never needed (ADR 0486 §2, changed by ADR 0488 §1 and §2). The controller alone writes it; the + // node's engine reads an object by name. + FileBucketPrefix = "declaration-files-" + // FileStreamPrefix starts the name of the stream behind such a bucket, which the bus's grants name. + FileStreamPrefix = "OBJ_" + // FileReferencesCapability is what a node-engine reports when it reads a file given as a digest and size. + FileReferencesCapability = "file-references" +) + +var nodeToken = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) + +// FileBucketFor is the name of the object bucket of a node's files, "declaration-files-" (novox/hq ADR 0488 +// §4, which replaces the single bucket constant of issue 1694). The node name must be a name the bus accepts in a +// bucket and in a subject token: letters, digits, '_' and '-', so no '.', '*', '>' or space. +func FileBucketFor(node string) (string, error) { + if !nodeToken.MatchString(node) { + return "", fmt.Errorf("node %q; a node's bucket is named by letters, digits, '_' and '-' only", node) + } + return FileBucketPrefix + node, nil +} + +// FileStreamFor is the name of the stream behind a node's bucket, "OBJ_" and the bucket's name. +func FileStreamFor(node string) (string, error) { + b, err := FileBucketFor(node) + if err != nil { + return "", err + } + return FileStreamPrefix + b, nil +} + +var digestForm = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) + +// DigestOf is the digest a reference names for a content: "sha256:<64 lower-case hex>". +func DigestOf(content []byte) string { + sum := sha256.Sum256(content) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +// ObjectName is the name of the object holding the content a digest names: the hex without its "sha256:" prefix. +func ObjectName(digest string) (string, error) { + if !digestForm.MatchString(digest) { + return "", fmt.Errorf("digest %q is not sha256:<64 lower-case hex characters>", digest) + } + return digest[len("sha256:"):], nil +} + +// IsReference reports whether the file gives its content as a digest and size instead of carrying it. +func (f *File) IsReference() bool { return f.Digest != "" } + +// checkReference judges the digest and size, and what a reference may not be given with. +func (f *File) checkReference(where string) []string { + if f.Digest == "" && f.Size == 0 { + return nil + } + var problems []string + if !digestForm.MatchString(f.Digest) { + problems = append(problems, fmt.Sprintf("%s: digest %q; a file by reference names its content as "+ + "sha256:<64 lower-case hex characters>", where, f.Digest)) + } + if f.Size <= 0 { + problems = append(problems, fmt.Sprintf("%s: size %d; a file by reference states its size in bytes, "+ + "more than zero", where, f.Size)) + } + if f.Content != "" || f.Bytes != "" || f.Sealed != "" || f.VaultSealed != nil || len(f.Secrets) > 0 { + problems = append(problems, where+": a file by reference carries its content in the object its digest "+ + "names, so it says no content, bytes, sealed, vault-sealed or secrets") + } + if f.Into != "" { + problems = append(problems, where+": a file by reference is written whole; a file written into says its "+ + "part in content") + } + return problems +} + +// Body is the bytes the file's content stands for: its content, or its bytes decoded. +func (f *File) Body() ([]byte, error) { + if f.Bytes == "" { + return []byte(f.Content), nil + } + b, err := base64.StdEncoding.DecodeString(f.Bytes) + if err != nil { + return nil, fmt.Errorf("file %s: bytes are not base64: %w", f.ID, err) + } + return b, nil +} + +// TravelsByDigest reports whether the controller may put this file's content in the bucket: it is plain content +// or bytes (not sealed, not filled with secrets, not written into another file) of more than FileReferenceThreshold +// bytes. +func (f *File) TravelsByDigest() bool { + if f.IsReference() || f.Sealed != "" || f.VaultSealed != nil || len(f.Secrets) > 0 || f.Into != "" { + return false + } + if f.Bytes != "" { + // DecodedLen counts padding, so the exact length is taken off the decoded bytes. + body, err := f.Body() + return err == nil && len(body) > FileReferenceThreshold + } + return len(f.Content) > FileReferenceThreshold +} + +// AsReference is the file as it is sent to an engine that reports FileReferencesCapability: a copy with the digest +// and size in place of the content, and the content the controller puts into the bucket under ObjectName(digest). +// A file that does not travel by digest is refused, not passed through, so a caller cannot send a small file empty. +func (f *File) AsReference() (File, []byte, error) { + if !f.TravelsByDigest() { + return File{}, nil, fmt.Errorf("file %s does not travel by digest", f.ID) + } + body, err := f.Body() + if err != nil { + return File{}, nil, err + } + ref := *f + ref.Content, ref.Bytes = "", "" + ref.Digest, ref.Size = DigestOf(body), int64(len(body)) + return ref, body, nil +} + +// Verify checks what was fetched against the digest and size the file names; the node-engine writes the file only +// after it returns nil. A mismatch is the apply's failure, never a partial apply (ADR 0486 §1). +func (f *File) Verify(body []byte) error { + if !f.IsReference() { + return fmt.Errorf("file %s is not given by reference", f.ID) + } + if int64(len(body)) != f.Size { + return fmt.Errorf("file %s: the object is %d bytes and the declaration says %d", f.ID, len(body), f.Size) + } + if got := DigestOf(body); got != f.Digest { + return fmt.Errorf("file %s: the object hashes to %s and the declaration names %s", f.ID, got, f.Digest) + } + return nil +} diff --git a/go/declaration/filereference_test.go b/go/declaration/filereference_test.go new file mode 100644 index 0000000..be34ce1 --- /dev/null +++ b/go/declaration/filereference_test.go @@ -0,0 +1,158 @@ +package declaration + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "reflect" + "strings" + "testing" +) + +func refFile(extra string) string { + return `{"declaration":1,"resources":[{"id":"m.f","type":"file","path":"/etc/m","digest":"sha256:` + + strings.Repeat("a", 64) + `","size":70000` + extra + `}]}` +} + +// 1694/AC1: a reference round trips, and an invalid one is refused naming the field. +func TestAFileReferenceRoundTripsAndABadOneNamesItsField(t *testing.T) { + d, err := Parse([]byte(refFile(""))) + if err != nil { + t.Fatal(err) + } + f := d.Resources[0].(*File) + if !f.IsReference() || f.Size != 70000 || f.Digest != "sha256:"+strings.Repeat("a", 64) { + t.Fatalf("read: %+v", f) + } + raw, _ := json.Marshal(d.Resources[0]) + d2, err := Parse([]byte(`{"declaration":1,"resources":[` + string(raw) + `]}`)) + if err != nil || !reflect.DeepEqual(d2.Resources[0], d.Resources[0]) { + t.Fatalf("round trip: %v %s", err, raw) + } + for want, raw := range map[string]string{ + "digest": strings.Replace(refFile(""), strings.Repeat("a", 64), "abc", 1), + "size": strings.Replace(refFile(""), `"size":70000`, `"size":0`, 1), + "lower-case": strings.Replace(refFile(""), strings.Repeat("a", 64), strings.Repeat("A", 64), 1), + "content": refFile(`,"content":"x"`), + "bytes": refFile(`,"bytes":"eA=="`), + "whole": refFile(`,"into":"block"`), + } { + _, err := Parse([]byte(raw)) + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("want a refusal naming %q, got %v", want, err) + } + } + // a size alone is not a reference + if _, err := Parse([]byte(`{"declaration":1,"resources":[{"id":"m.f","type":"file","path":"/e","size":5}]}`)); err == nil { + t.Error("a size without a digest was accepted") + } +} + +// 1694/AC2: a declaration as today's SDK writes it reads unchanged and its bytes carry no new key. +func TestAnInlineFileIsReadAndWrittenAsBefore(t *testing.T) { + const in = `{"id":"m.f","type":"file","path":"/e","content":"hello"}` + d, err := Parse([]byte(`{"declaration":1,"resources":[` + in + `]}`)) + if err != nil { + t.Fatal(err) + } + out, _ := json.Marshal(d.Resources[0]) + if string(out) != in { + t.Fatalf("signed bytes changed: %s", out) + } + if d.Resources[0].(*File).IsReference() { + t.Fatal("inline read as a reference") + } +} + +// 1694/AC3: the controller's helper makes a reference of a big file and the engine's check refuses a changed byte. +func TestABigFileBecomesAReferenceAndAChangedByteFailsTheCheck(t *testing.T) { + big := strings.Repeat("x", FileReferenceThreshold+1) + f := &File{ID: "m.f", Type: TypeFile, Path: "/e", Content: big} + if !f.TravelsByDigest() { + t.Fatal("a file over the threshold stays inline") + } + ref, body, err := f.AsReference() + if err != nil || ref.Content != "" || ref.Size != int64(len(big)) || string(body) != big { + t.Fatalf("%v %+v", err, ref) + } + if p := ref.validate("f", false); len(p) > 0 { + t.Fatalf("the reference is refused: %v", p) + } + if name, err := ObjectName(ref.Digest); err != nil || len(name) != 64 { + t.Fatalf("object name %q %v", name, err) + } + if err := ref.Verify(body); err != nil { + t.Fatal(err) + } + bad := bytes.Clone(body) + bad[7] ^= 1 + if err := ref.Verify(bad); err == nil || !strings.Contains(err.Error(), "hashes to") { + t.Fatalf("a changed byte: %v", err) + } + if err := ref.Verify(body[1:]); err == nil || !strings.Contains(err.Error(), "bytes") { + t.Fatalf("a short object: %v", err) + } +} + +// 1694/AC4: what stays inline, and bytes files travel decoded. +func TestWhichFilesTravelByDigest(t *testing.T) { + small := strings.Repeat("x", FileReferenceThreshold) + big := strings.Repeat("x", FileReferenceThreshold+1) + for name, f := range map[string]File{ + "at the threshold": {Content: small}, + "sealed": {Sealed: "s"}, + "secrets": {Content: big, Secrets: map[string]string{"a": "b"}}, + "into": {Content: big, Into: IntoBlock}, + } { + if f.TravelsByDigest() { + t.Errorf("%s travels by digest", name) + } + if _, _, err := f.AsReference(); err == nil { + t.Errorf("%s was made a reference", name) + } + } + exact := File{ID: "m.e", Bytes: base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{7}, FileReferenceThreshold))} + if exact.TravelsByDigest() { + t.Error("a bytes file of exactly the threshold travels by digest") + } + if _, _, err := exact.AsReference(); err == nil { + t.Error("a bytes file of exactly the threshold was made a reference") + } + raw := bytes.Repeat([]byte{0, 1, 2}, FileReferenceThreshold) + f := File{ID: "m.b", Bytes: base64.StdEncoding.EncodeToString(raw)} + ref, body, err := f.AsReference() + if err != nil || !bytes.Equal(body, raw) || ref.Size != int64(len(raw)) || ref.Digest != DigestOf(raw) { + t.Fatalf("bytes file: %v", err) + } +} + +func TestTheSharedNames(t *testing.T) { + if FileBucketPrefix != "declaration-files-" || FileReferencesCapability != "file-references" || FileReferenceThreshold != 65536 { + t.Fatal("a name changed") + } + if _, err := ObjectName("sha256:zz"); err == nil { + t.Fatal("bad digest named an object") + } +} + +// 1694/AC6: each node has a bucket of its own, so a grant names it and never needs a `*` (ADR 0486 §2). +func TestEachNodeHasABucketOfItsOwn(t *testing.T) { + for _, n := range []string{"ace", "g14", "novox", "shanks"} { + b, err := FileBucketFor(n) + if err != nil || b != "declaration-files-"+n { + t.Errorf("bucket for %q: %q %v", n, b, err) + } + s, err := FileStreamFor(n) + if err != nil || s != "OBJ_declaration-files-"+n { + t.Errorf("stream for %q: %q %v", n, s, err) + } + } + for _, bad := range []string{"", "a.b", "a*", "*", ">", "a b", "a>", "a/b", "ä"} { + if _, err := FileBucketFor(bad); err == nil { + t.Errorf("bucket for %q accepted", bad) + } + if _, err := FileStreamFor(bad); err == nil { + t.Errorf("stream for %q accepted", bad) + } + } +} diff --git a/go/declaration/groups_test.go b/go/declaration/groups_test.go new file mode 100644 index 0000000..193d8f5 --- /dev/null +++ b/go/declaration/groups_test.go @@ -0,0 +1,55 @@ +package declaration + +import ( + "encoding/json" + "strings" + "testing" +) + +// A container may be given supplementary groups (novox/hq ADR 0480 clause 7, issue 1636): numbers, each above zero. + +// 1636/AC1: a declaration with groups decodes, is written back with them, and a declaration without them has none. +func TestAContainerWithGroupsDecodesAndIsWrittenBack(t *testing.T) { + d, err := Parse([]byte(`{"declaration":1,"resources":[ + {"id":"n8n.server","type":"container","name":"n8n","image":"` + anImage + `","groups":[2000,2001]}, + {"id":"n8n.plain","type":"container","name":"plain","image":"` + anImage + `"}]}`)) + if err != nil { + t.Fatal(err) + } + c := d.Resources[0].(*Container) + if len(c.Groups) != 2 || c.Groups[0] != 2000 || c.Groups[1] != 2001 { + t.Fatalf("groups read as %v", c.Groups) + } + raw, err := json.Marshal(c) + if err != nil || !strings.Contains(string(raw), `"groups":[2000,2001]`) { + t.Fatalf("written as %s (%v)", raw, err) + } + plain, _ := json.Marshal(d.Resources[1].(*Container)) + if strings.Contains(string(plain), "groups") { + t.Fatalf("a container without groups is written with the key: %s", plain) + } + if g := d.Resources[1].(*Container).Groups; g != nil { + t.Fatalf("a container without groups reads %v", g) + } +} + +// 1636/AC2: zero, a number below it, one the kernel keeps for "no group" and a number twice are refused, naming why. +func TestAContainersGroupsAreWholeNumbersAboveZeroAndNotRepeated(t *testing.T) { + for want, groups := range map[string]string{ + "never root": `[0]`, + "below zero": `[-5]`, + "no group": `[4294967295]`, + "twice": `[2000,2000]`, + } { + _, err := Parse([]byte(`{"declaration":1,"resources":[{"id":"m.c","type":"container","name":"c","image":"` + + anImage + `","groups":` + groups + `}]}`)) + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("groups %s: want a refusal saying %q, got %v", groups, want, err) + } + } + // Text where a number stands is refused by the decoder itself. + if _, err := Parse([]byte(`{"declaration":1,"resources":[{"id":"m.c","type":"container","name":"c","image":"` + + anImage + `","groups":["media"]}]}`)); err == nil { + t.Error("a group named by text was accepted") + } +} diff --git a/go/delivery/buildrecord.go b/go/delivery/buildrecord.go index d3f9fb9..1bccc22 100644 --- a/go/delivery/buildrecord.go +++ b/go/delivery/buildrecord.go @@ -25,7 +25,7 @@ type BuildRecord struct { } // Artifact is one thing a build made: mesh-controller internal/chain/records.go Artifact, json keys name, kind and -// reference, nothing else (novox/hq ADR 0478 clause 7). +// reference, nothing else (novox/hq ADR 0478 clause 8). type Artifact struct { Name string `json:"name"` Kind string `json:"kind"` diff --git a/go/delivery/source.go b/go/delivery/source.go index e82ce1d..8d85b64 100644 --- a/go/delivery/source.go +++ b/go/delivery/source.go @@ -390,7 +390,7 @@ type Module struct { Repository string `json:"repository"` Commit string `json:"commit"` // Path is the module's directory in its repository, trimmed of slashes, and RootPath (".") when its manifest is the - // repository's root (novox/hq ADR 0478 clause 8, issue 1615): the build tools need it. Empty means not known: a value + // repository's root (novox/hq ADR 0478 clause 9, issue 1615): the build tools need it. Empty means not known: a value // written before the indexer learned paths has none. Path string `json:"path,omitempty"` // Toolchains are the toolchains FP2 was computed with, by language (go, typescript): the version installed and the diff --git a/go/provider/ended_test.go b/go/provider/ended_test.go new file mode 100644 index 0000000..c4e25cb --- /dev/null +++ b/go/provider/ended_test.go @@ -0,0 +1,62 @@ +package provider + +import ( + "context" + "encoding/json" + "slices" + "testing" +) + +// Issue 1098: an assignment with `ended` set, or with a `state`, is not a standing assignment: the helper reads it +// as a removed key, so the entry of its consumer is removed and the adapter hears it is gone. The fields the +// controller adds change nothing. +func TestAnEndedAssignmentIsReadAsARemovedKey(t *testing.T) { + bus, m := newStandIn(), &maker{} + v := &vault{values: map[string]string{"pair/novox/postgres-database/ace/letta": "s1"}} + assign(t, bus, "ace", "letta", bind("postgres-database", "", "novox", "postgres")) + p := postgres(bus, v, m) + if _, err := p.Pass(context.Background()); err != nil { + t.Fatal(err) + } + if got := bus.keys(BucketOf("postgres")); !slices.Equal(got, []string{"ace.letta.postgres-database"}) { + t.Fatalf("entries %v", got) + } + for _, mark := range []map[string]any{{"ended": "2026-10-18T01:00:00Z"}, {"state": "pending"}} { + raw := bus.get(AssignmentsBucket, "ace.letta") + var a map[string]any + if err := json.Unmarshal(raw, &a); err != nil { + t.Fatal(err) + } + for k, v := range mark { + a[k] = v + } + raw, _ = json.Marshal(a) + _ = bus.Put(context.Background(), AssignmentsBucket, "ace.letta", raw) + out, err := p.Pass(context.Background()) + if err != nil { + t.Fatal(err) + } + if got := bus.keys(BucketOf("postgres")); len(got) != 0 || len(out) != 1 || out[0].Status != Removed { + t.Fatalf("%v: entries %v, outcomes %+v", mark, got, out) + } + assign(t, bus, "ace", "letta", bind("postgres-database", "", "novox", "postgres")) + if _, err := p.Pass(context.Background()); err != nil { + t.Fatal(err) + } + } +} + +func TestStandingIsFalseOnlyForAnEndedOrPendingValue(t *testing.T) { + for raw, want := range map[string]bool{ + `{"node":"a","module":"b"}`: true, + `{"node":"a","module":"b","ended":"","state":"","x":{"y":1}}`: true, + `{"node":"a","module":"b","ended":null}`: true, + `{"node":"a","module":"b","build":"b1","commit":"c","rollout":"r","since":"t","artifact":{"name":"n"}}`: true, + `{"node":"a","module":"b","ended":"2026-10-18T01:00:00Z"}`: false, + `{"node":"a","module":"b","state":"pending"}`: false, + } { + if got := Standing([]byte(raw)); got != want { + t.Errorf("%s: %v, want %v", raw, got, want) + } + } +} diff --git a/go/provider/provider.go b/go/provider/provider.go index 33bf61f..4adee9e 100644 --- a/go/provider/provider.go +++ b/go/provider/provider.go @@ -335,6 +335,9 @@ func (p *Provider) pass(ctx context.Context, assignments, entries map[string][]b var out []Outcome unknown := map[string]bool{} // consumers whose assignment was skipped: their entries are not judged for _, key := range sortedKeys(assignments) { + if !Standing(assignments[key]) { + continue // ended or pending: read as a removed key (issue 1098) + } var a planned if err := json.Unmarshal(assignments[key], &a); err != nil || a.Node+"."+a.Module != key { o := Outcome{Key: key, Status: Failed, Why: fmt.Sprintf("the assignment %s is not the controller's, so "+ @@ -744,3 +747,23 @@ func (b stateBus) Delete(_ context.Context, bucket, key string) error { } return s.Delete(key) } + +// Standing is whether an assignment's value is a standing assignment: false when `ended` is set (the module was +// taken off its node, novox/hq ADR 0408 clause 4) or `state` is not empty (a pending one). Such a value is read +// exactly as a removed key. Every other field, known or not, changes nothing (issue 1098). +func Standing(value []byte) bool { + var v struct { + Ended json.RawMessage `json:"ended"` + State json.RawMessage `json:"state"` + } + if json.Unmarshal(value, &v) != nil { + return true // not decided here: the parse that follows refuses it + } + return !fieldSet(v.Ended) && !fieldSet(v.State) +} + +// fieldSet is whether a field carries something: not absent, null or empty. +func fieldSet(f json.RawMessage) bool { + s := strings.TrimSpace(string(f)) + return s != "" && s != "null" && s != `""` +} diff --git a/go/state_counter_test.go b/go/state_counter_test.go index 764a1db..fb89617 100644 --- a/go/state_counter_test.go +++ b/go/state_counter_test.go @@ -138,11 +138,11 @@ func text(t *testing.T, a map[string]any) (string, bool) { return content[0].(map[string]any)["text"].(string), isError } -// Two writers incrementing one counter a thousand times each end at two thousand: every number handed out +// Two writers incrementing one counter two hundred times each end at four hundred: every number handed out // once, none lost, each writer's numbers rising — read, then written only at the revision read. func TestTwoWritersIncrementingOneCounterLoseNoUpdate(t *testing.T) { r := &casRuntime{keys: map[string]StateEntry{}, writes: map[string]int{}} - const each = 1000 + const each = 200 // enough to collide every run (about a third of writes are lost), cheap on a slow builder got := counting(t, r, func() (any, error) { var wg sync.WaitGroup numbers := make([][]uint64, 2) @@ -179,7 +179,7 @@ func TestTwoWritersIncrementingOneCounterLoseNoUpdate(t *testing.T) { } return len(seen), nil }) - if out, isError := text(t, got); isError || out != "2000" { + if out, isError := text(t, got); isError || out != fmt.Sprint(2*each) { t.Fatalf("two writers counted %s (error: %v)", out, isError) } var final uint64 @@ -190,7 +190,7 @@ func TestTwoWritersIncrementingOneCounterLoseNoUpdate(t *testing.T) { if r.lost == 0 { t.Fatal("the two writers never collided: the test proved nothing about contention") } - t.Logf("2000 increments, %d writes lost to the other writer and tried again", r.lost) + t.Logf("%d increments, %d writes lost to the other writer and tried again", 2*each, r.lost) if r.writes["create"] != 1 || r.writes["update"] != 2*each-1 || r.writes["put"] != 0 { t.Fatalf("written by %v: one create, then updates at the revision read, never a put", r.writes) } diff --git a/go/testdata/beside/CAPTURED b/go/testdata/beside/CAPTURED index dc56ca5..9f1bb1c 100644 --- a/go/testdata/beside/CAPTURED +++ b/go/testdata/beside/CAPTURED @@ -7,10 +7,10 @@ The copy carries the catalogue's private details: domains, first names in module private addresses. That is acceptable while mesh-sdk is private; if it ever goes public, this copy is a second place to clean besides the catalogue itself. -Only what go/catalogue's tests read is captured (R4b, issue 1379). Refreshed 2026-10-20 from the mains of both +Only what go/catalogue's tests read is captured (R4b, issue 1379). Refreshed 2026-10-21 from the mains of both repositories; before that it was mesh-controller's copy, which still held two modules that had left the catalogue. -mesh-catalog bb0fe93bbdddd869ad808210352461efc506c3d3 modules/*/module.json, modules/*/files/**, modules/*/templates/**, predefined.json +mesh-catalog b5dbfbb784507e211372a7c9640d91f7d6dadd84 modules/*/module.json, modules/*/files/**, modules/*/templates/**, predefined.json mesh-host cdcba9ca73755854d21e4e657d7e46c32b9b5265 internal/apply/placement_guard.go To move them, from this repository's go/ directory, with the two repositories checked out beside mesh-sdk: diff --git a/go/testdata/beside/mesh-catalog/modules/asus-zephyrus-g14/files/bin/zephyrus-backlight b/go/testdata/beside/mesh-catalog/modules/asus-zephyrus-g14/files/bin/zephyrus-backlight old mode 100644 new mode 100755 diff --git a/go/testdata/beside/mesh-catalog/modules/asus-zephyrus-g14/files/bin/zephyrus-display b/go/testdata/beside/mesh-catalog/modules/asus-zephyrus-g14/files/bin/zephyrus-display old mode 100644 new mode 100755 diff --git a/go/testdata/beside/mesh-catalog/modules/asus-zephyrus-g14/files/bin/zephyrus-kbd-notify b/go/testdata/beside/mesh-catalog/modules/asus-zephyrus-g14/files/bin/zephyrus-kbd-notify old mode 100644 new mode 100755 diff --git a/go/testdata/beside/mesh-catalog/modules/asus-zephyrus-g14/files/bin/zephyrus-media b/go/testdata/beside/mesh-catalog/modules/asus-zephyrus-g14/files/bin/zephyrus-media old mode 100644 new mode 100755 diff --git a/go/testdata/beside/mesh-catalog/modules/asus-zephyrus-g14/files/bin/zephyrus-notify b/go/testdata/beside/mesh-catalog/modules/asus-zephyrus-g14/files/bin/zephyrus-notify old mode 100644 new mode 100755 diff --git a/go/testdata/beside/mesh-catalog/modules/asus-zephyrus-g14/files/bin/zephyrus-session b/go/testdata/beside/mesh-catalog/modules/asus-zephyrus-g14/files/bin/zephyrus-session old mode 100644 new mode 100755 diff --git a/go/testdata/beside/mesh-catalog/modules/asus-zephyrus-g14/files/bin/zephyrus-touchpad b/go/testdata/beside/mesh-catalog/modules/asus-zephyrus-g14/files/bin/zephyrus-touchpad old mode 100644 new mode 100755 diff --git a/go/testdata/beside/mesh-catalog/modules/betterbird/files/betterbird b/go/testdata/beside/mesh-catalog/modules/betterbird/files/betterbird old mode 100644 new mode 100755 diff --git a/go/testdata/beside/mesh-catalog/modules/brother/files/settings.json b/go/testdata/beside/mesh-catalog/modules/brother/files/settings.json new file mode 100644 index 0000000..80aed32 --- /dev/null +++ b/go/testdata/beside/mesh-catalog/modules/brother/files/settings.json @@ -0,0 +1,9 @@ +{ + "scanner": "", + "scan-folder": "", + "printer": "", + "printer-certificate": "", + "resolution": 300, + "color-mode": "color", + "crop": true +} diff --git a/go/testdata/beside/mesh-catalog/modules/brother/files/store.json b/go/testdata/beside/mesh-catalog/modules/brother/files/store.json new file mode 100644 index 0000000..8c2075d --- /dev/null +++ b/go/testdata/beside/mesh-catalog/modules/brother/files/store.json @@ -0,0 +1,9 @@ +{ + "scheme": "${bound:s3-bucket:scheme}", + "at": "${bound:s3-bucket:at}", + "port": "${bound:s3-bucket:port}", + "bucket": "${bound:s3-bucket:bucket}", + "region": "${bound:s3-bucket:region}", + "key": "${bound:s3-bucket:as}", + "secret": "${secret:s3-bucket}" +} diff --git a/go/testdata/beside/mesh-catalog/modules/brother/module.json.captured b/go/testdata/beside/mesh-catalog/modules/brother/module.json.captured new file mode 100644 index 0000000..57559d5 --- /dev/null +++ b/go/testdata/beside/mesh-catalog/modules/brother/module.json.captured @@ -0,0 +1,280 @@ +{ + "module": "brother", + "version": "1", + "requires": [ + "s3-bucket" + ], + "binds": { + "s3-bucket": "${dir:state}/store-binding.json" + }, + "secrets": { + "s3-bucket": "${dir:state}/store.secret" + }, + "config": { + "scanner": { + "type": "string", + "description": "The network scanner's eSCL address, such as http://192.0.2.10/eSCL: the scanner's own address and the path its eSCL service answers on; the operator's, with no default.", + "pattern": "^https?://.+", + "trusted": true, + "required": true + }, + "scan-folder": { + "type": "string", + "description": "The absolute path of the folder the scans are filed in as PDF files; the module makes it when it files the first scan, and never touches a file in it that it did not write. The operator's, with no default.", + "pattern": "^/.+$", + "trusted": true, + "required": true + }, + "printer": { + "type": "string", + "description": "The printer's host, such as 192.0.2.10: where its admin page (HTTPS) and its IPP service answer. The device is the scanner's own; the operator's, with no default.", + "pattern": "^[A-Za-z0-9.:-]+$", + "trusted": true, + "required": true + }, + "printer-certificate": { + "type": "string", + "description": "The SHA-256 fingerprint of the certificate the printer's admin page serves, as 32 upper-case hex pairs joined by colons; several, separated by commas, when the printer serves more than one (it serves an ECDSA and an RSA certificate for one name). The module trusts those certificates alone and refuses any other; the operator's, with no default.", + "pattern": "^([0-9A-F]{2}:){31}[0-9A-F]{2}(, *([0-9A-F]{2}:){31}[0-9A-F]{2})*$", + "trusted": true, + "required": true + }, + "resolution": { + "type": "number", + "description": "The resolution asked of the scanner, in dpi: 100, 200, 300 or 600. A scanner may give another; the scan is filed at the resolution it gave.", + "enum": [ + 100, + 200, + 300, + 600 + ], + "trusted": false, + "default": 300 + }, + "color-mode": { + "type": "string", + "description": "Whether the scan is in colour or in grays: color or gray.", + "enum": [ + "color", + "gray" + ], + "trusted": false, + "default": "color" + }, + "crop": { + "type": "boolean", + "description": "Whether the document is cut out of the platen's white lid; false files the whole platen. A feeder scan is never cropped.", + "trusted": false, + "default": true + } + }, + "own-secrets": { + "admin-password": { + "path": "${dir:state}/admin-password.secret", + "issued-by": "outside", + "prompt": "the admin password of the Brother printer", + "owner": "brother-device" + }, + "broker-device": { + "path": "${dir:state}/broker-device.secret", + "owner": "brother-device" + } + }, + "streams": [ + { + "name": "device-asks", + "events": [ + "device.asked" + ], + "max-age-days": 1, + "max-mib": 8, + "retention": "work-queue", + "max-deliveries": 3, + "duplicates": 24 + } + ], + "state": [ + { + "name": "device-answers", + "ttl-seconds": 600 + } + ], + "seats": [ + { + "name": "scanner", + "scope": "mesh", + "serves": [ + "scan", + "status", + "list", + "document" + ] + }, + { + "name": "printer", + "scope": "mesh", + "serves": [ + "status", + "settings", + "print", + "upload" + ] + } + ], + "claims": [ + { + "name": "scanner", + "scope": "mesh", + "serves": [ + "scan", + "status", + "list", + "document" + ] + }, + { + "name": "printer", + "scope": "mesh", + "serves": [ + "status", + "settings", + "print", + "upload" + ] + } + ], + "emits": [ + "scan.added" + ], + "tools": [ + "scanner.scan", + "scanner.status", + "scanner.list", + "scanner.document", + "printer.status", + "printer.settings", + "printer.print", + "printer.upload", + "brother_health" + ], + "resources": [ + { + "id": "account", + "type": "user", + "name": "brother-device", + "shell": "/usr/bin/nologin" + }, + { + "id": "state", + "type": "directory", + "mode": "0711", + "owner": "${machine:account}", + "place": ".", + "unknown": "program" + }, + { + "id": "settings", + "type": "file", + "path": "${dir:state}/settings.json", + "owner": "${machine:account}", + "mode": "0644", + "takes-config": [ + "scanner", + "scan-folder", + "printer", + "printer-certificate", + "resolution", + "color-mode", + "crop" + ], + "merge": "json", + "template": "files/settings.json", + "trusted": true + }, + { + "id": "store", + "type": "file", + "path": "${dir:state}/store.json", + "owner": "brother-device", + "mode": "0600", + "template": "files/store.json" + }, + { + "id": "device", + "type": "process", + "name": "brother-device", + "artifact": "tools", + "run": [ + "./brother", + "device" + ], + "user": "brother-device", + "env": { + "BROTHER_SETTINGS": "${dir:state}/settings.json", + "BROTHER_ADMIN_PASSWORD": "${dir:state}/admin-password.secret", + "BROTHER_BROKER": "${dir:state}/broker-device.secret", + "BROTHER_STORE": "${dir:state}/store.json" + }, + "restart-on": [ + "settings", + "store" + ], + "bus": { + "queues": [ + "device-asks" + ], + "state": [ + "device-answers" + ], + "emits": [ + "scan.added" + ] + }, + "health": { + "kind": "unit" + } + }, + { + "id": "watch", + "type": "process", + "name": "brother-watch", + "artifact": "tools", + "run": [ + "./brother", + "watch" + ], + "user": "brother-device", + "health": { + "kind": "tool", + "tool": "brother_health", + "interval": "60s", + "timeout": "10s", + "looks": 2, + "grace": "60s" + } + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/brother", + "binary": "brother", + "loads": [ + "brother" + ], + "env": { + "BROTHER_SETTINGS": "${dir:state}/settings.json" + } + }, + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + } +} diff --git a/go/testdata/beside/mesh-catalog/modules/ca-trust/files/anchor b/go/testdata/beside/mesh-catalog/modules/ca-trust/files/anchor old mode 100644 new mode 100755 diff --git a/go/testdata/beside/mesh-catalog/modules/claude-code/files/claude-agent b/go/testdata/beside/mesh-catalog/modules/claude-code/files/claude-agent old mode 100644 new mode 100755 diff --git a/go/testdata/beside/mesh-catalog/modules/clipmenu/files/bin/xsel b/go/testdata/beside/mesh-catalog/modules/clipmenu/files/bin/xsel old mode 100644 new mode 100755 diff --git a/go/testdata/beside/mesh-catalog/modules/dev-tools/module.json.captured b/go/testdata/beside/mesh-catalog/modules/dev-tools/module.json.captured index 8ee6fd1..094f833 100644 --- a/go/testdata/beside/mesh-catalog/modules/dev-tools/module.json.captured +++ b/go/testdata/beside/mesh-catalog/modules/dev-tools/module.json.captured @@ -4,12 +4,11 @@ "capabilities": [ "package-manager" ], + "requires": [ + "go-toolchain", + "node-toolchain" + ], "resources": [ - { - "id": "go", - "type": "package", - "package": "go" - }, { "id": "c-compiler", "type": "package", @@ -20,16 +19,6 @@ "type": "package", "package": "make" }, - { - "id": "nodejs", - "type": "package", - "package": "nodejs" - }, - { - "id": "npm", - "type": "package", - "package": "npm" - }, { "id": "python", "type": "package", diff --git a/go/testdata/beside/mesh-catalog/modules/feh/files/fehbg b/go/testdata/beside/mesh-catalog/modules/feh/files/fehbg old mode 100644 new mode 100755 diff --git a/go/testdata/beside/mesh-catalog/modules/gitea/module.json.captured b/go/testdata/beside/mesh-catalog/modules/gitea/module.json.captured index 2de9730..118224c 100644 --- a/go/testdata/beside/mesh-catalog/modules/gitea/module.json.captured +++ b/go/testdata/beside/mesh-catalog/modules/gitea/module.json.captured @@ -540,6 +540,8 @@ ], "reads": [ "mesh-controller.assignments", - "ssh-client.keys" + "ssh-client.keys", + "mesh-build-go.keys", + "mesh-build-node.keys" ] } diff --git a/go/testdata/beside/mesh-catalog/modules/go/module.json.captured b/go/testdata/beside/mesh-catalog/modules/go/module.json.captured new file mode 100644 index 0000000..48a6bf5 --- /dev/null +++ b/go/testdata/beside/mesh-catalog/modules/go/module.json.captured @@ -0,0 +1,20 @@ +{ + "module": "go", + "version": "1", + "capabilities": [ + "package-manager" + ], + "provides": [ + { + "name": "go-toolchain", + "reach": "machine" + } + ], + "resources": [ + { + "id": "go", + "type": "package", + "package": "go" + } + ] +} diff --git a/go/testdata/beside/mesh-catalog/modules/i3/files/wms/i3 b/go/testdata/beside/mesh-catalog/modules/i3/files/wms/i3 old mode 100644 new mode 100755 diff --git a/go/testdata/beside/mesh-catalog/modules/i3status-rust/files/bin/i3bar-watchdog b/go/testdata/beside/mesh-catalog/modules/i3status-rust/files/bin/i3bar-watchdog old mode 100644 new mode 100755 diff --git a/go/testdata/beside/mesh-catalog/modules/i3status-rust/files/bin/i3status-updates b/go/testdata/beside/mesh-catalog/modules/i3status-rust/files/bin/i3status-updates old mode 100644 new mode 100755 diff --git a/go/testdata/beside/mesh-catalog/modules/i3status-rust/files/bin/i3status-value b/go/testdata/beside/mesh-catalog/modules/i3status-rust/files/bin/i3status-value old mode 100644 new mode 100755 diff --git a/go/testdata/beside/mesh-catalog/modules/lab/module.json.captured b/go/testdata/beside/mesh-catalog/modules/lab/module.json.captured index e23963d..ef82b87 100644 --- a/go/testdata/beside/mesh-catalog/modules/lab/module.json.captured +++ b/go/testdata/beside/mesh-catalog/modules/lab/module.json.captured @@ -6,7 +6,9 @@ "virtualisation" ], "requires": [ - "npm-package-registry" + "npm-package-registry", + "go-toolchain", + "node-toolchain" ], "contributes": { "npm-package-registry": { @@ -84,16 +86,6 @@ "type": "package", "package": "iproute2" }, - { - "id": "npm", - "type": "package", - "package": "npm" - }, - { - "id": "go", - "type": "package", - "package": "go" - }, { "id": "incus", "type": "package", diff --git a/go/testdata/beside/mesh-catalog/modules/lemurs/files/xserver b/go/testdata/beside/mesh-catalog/modules/lemurs/files/xserver old mode 100644 new mode 100755 diff --git a/go/testdata/beside/mesh-catalog/modules/mesh-build-go/files/forge b/go/testdata/beside/mesh-catalog/modules/mesh-build-go/files/forge new file mode 100644 index 0000000..6bbf4e2 --- /dev/null +++ b/go/testdata/beside/mesh-catalog/modules/mesh-build-go/files/forge @@ -0,0 +1,7 @@ +Host ${bound:forge-account:ssh_alias} + HostName ${bound:forge-account:ssh_host} + Port ${bound:forge-account:ssh_port} + HostKeyAlias ${bound:forge-account:ssh_alias} + User git + IdentityFile ~/.ssh/mesh_ed25519 + IdentitiesOnly yes diff --git a/go/testdata/beside/mesh-catalog/modules/mesh-build-go/files/gitconfig b/go/testdata/beside/mesh-catalog/modules/mesh-build-go/files/gitconfig new file mode 100644 index 0000000..0525230 --- /dev/null +++ b/go/testdata/beside/mesh-catalog/modules/mesh-build-go/files/gitconfig @@ -0,0 +1,2 @@ +[url "ssh://${bound:forge-account:ssh_alias}/"] + insteadOf = ${bound:forge-account:https_base} diff --git a/go/testdata/beside/mesh-catalog/modules/mesh-build-go/files/known_hosts b/go/testdata/beside/mesh-catalog/modules/mesh-build-go/files/known_hosts new file mode 100644 index 0000000..e0df6d8 --- /dev/null +++ b/go/testdata/beside/mesh-catalog/modules/mesh-build-go/files/known_hosts @@ -0,0 +1 @@ +${bound:forge-account:ssh_alias} ${bound:forge-account:host_key} diff --git a/go/testdata/beside/mesh-catalog/modules/mesh-build-go/module.json.captured b/go/testdata/beside/mesh-catalog/modules/mesh-build-go/module.json.captured new file mode 100644 index 0000000..9f0fd7f --- /dev/null +++ b/go/testdata/beside/mesh-catalog/modules/mesh-build-go/module.json.captured @@ -0,0 +1,148 @@ +{ + "module": "mesh-build-go", + "version": "1", + "moved": true, + "runs-as": "mesh-build-go", + "capabilities": [ + "package-manager", + "user-namespaces" + ], + "state": [ + "toolchains", + "keys", + "steps" + ], + "tools": [ + "build_go_fetch", + "build_go_test", + "build_go_digest", + "build_go_build", + "build_go_step", + "build_go_steps", + "build_go_toolchains", + "build_go_status", + "build_go_health" + ], + "streams": [ + { + "name": "toolchains-go", + "events": ["toolchain.**"] + } + ], + "requires": [ + "forge-account", + "go-toolchain" + ], + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-build-go", + "runtime-health": { + "kind": "tool", + "tool": "build_go_health", + "interval": "60s", + "timeout": "10s", + "looks": 2, + "grace": "60s" + }, + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-build-go", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-build-go" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-build-go", + "place": ".", + "unknown": "program" + }, + { + "id": "work", + "type": "directory", + "path": "/var/lib/mesh-build-go-work", + "mode": "0700", + "owner": "mesh-build-go", + "unknown": "program" + }, + { + "id": "cache", + "type": "directory", + "path": "/var/lib/mesh-build-go-cache", + "mode": "0700", + "owner": "mesh-build-go", + "unknown": "program" + }, + { + "id": "ssh-dir", + "type": "directory", + "path": "${dir:state}/.ssh", + "mode": "0700", + "owner": "mesh-build-go", + "unknown": "program" + }, + { + "id": "ssh-config", + "type": "file", + "path": "${dir:ssh-dir}/config", + "owner": "mesh-build-go", + "mode": "0600", + "template": "files/forge" + }, + { + "id": "known-hosts", + "type": "file", + "path": "${dir:ssh-dir}/known_hosts", + "owner": "mesh-build-go", + "mode": "0644", + "template": "files/known_hosts" + }, + { + "id": "git-config", + "type": "file", + "path": "${dir:state}/.config/git/config", + "owner": "mesh-build-go", + "mode": "0644", + "template": "files/gitconfig" + }, + { + "id": "git", + "type": "package", + "package": "git" + }, + { + "id": "openssh", + "type": "package", + "package": "openssh" + }, + { + "id": "postgresql", + "type": "package", + "package": "postgresql" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-build-go", + "binary": "mesh-build-go", + "loads": [ + "mesh-build-go" + ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + } +} diff --git a/go/testdata/beside/mesh-catalog/modules/mesh-build-node/files/forge b/go/testdata/beside/mesh-catalog/modules/mesh-build-node/files/forge new file mode 100644 index 0000000..6bbf4e2 --- /dev/null +++ b/go/testdata/beside/mesh-catalog/modules/mesh-build-node/files/forge @@ -0,0 +1,7 @@ +Host ${bound:forge-account:ssh_alias} + HostName ${bound:forge-account:ssh_host} + Port ${bound:forge-account:ssh_port} + HostKeyAlias ${bound:forge-account:ssh_alias} + User git + IdentityFile ~/.ssh/mesh_ed25519 + IdentitiesOnly yes diff --git a/go/testdata/beside/mesh-catalog/modules/mesh-build-node/files/gitconfig b/go/testdata/beside/mesh-catalog/modules/mesh-build-node/files/gitconfig new file mode 100644 index 0000000..0525230 --- /dev/null +++ b/go/testdata/beside/mesh-catalog/modules/mesh-build-node/files/gitconfig @@ -0,0 +1,2 @@ +[url "ssh://${bound:forge-account:ssh_alias}/"] + insteadOf = ${bound:forge-account:https_base} diff --git a/go/testdata/beside/mesh-catalog/modules/mesh-build-node/files/known_hosts b/go/testdata/beside/mesh-catalog/modules/mesh-build-node/files/known_hosts new file mode 100644 index 0000000..e0df6d8 --- /dev/null +++ b/go/testdata/beside/mesh-catalog/modules/mesh-build-node/files/known_hosts @@ -0,0 +1 @@ +${bound:forge-account:ssh_alias} ${bound:forge-account:host_key} diff --git a/go/testdata/beside/mesh-catalog/modules/mesh-build-node/module.json.captured b/go/testdata/beside/mesh-catalog/modules/mesh-build-node/module.json.captured new file mode 100644 index 0000000..b92212a --- /dev/null +++ b/go/testdata/beside/mesh-catalog/modules/mesh-build-node/module.json.captured @@ -0,0 +1,169 @@ +{ + "module": "mesh-build-node", + "version": "1", + "moved": true, + "runs-as": "mesh-build-node", + "capabilities": [ + "package-manager", + "user-namespaces" + ], + "state": [ + "toolchains", + "keys", + "steps" + ], + "tools": [ + "build_node_fetch", + "build_node_test", + "build_node_build", + "build_node_step", + "build_node_steps", + "build_node_toolchains", + "build_node_status", + "build_node_health" + ], + "streams": [ + { + "name": "toolchains-node", + "events": [ + "toolchain.**" + ] + } + ], + "requires": [ + "forge-account", + "npm-package-registry", + "node-toolchain" + ], + "contributes": { + "npm-package-registry": { + "access": "read" + } + }, + "binds": { + "npm-package-registry": "${dir:state}/package-registry.json" + }, + "secrets": { + "npm-package-registry": "${dir:state}/package-registry.secret" + }, + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-build-node", + "runtime-health": { + "kind": "tool", + "tool": "build_node_health", + "interval": "60s", + "timeout": "10s", + "looks": 2, + "grace": "60s" + }, + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-build-node", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-build-node" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-build-node", + "place": ".", + "unknown": "program" + }, + { + "id": "work", + "type": "directory", + "path": "/var/lib/mesh-build-node-work", + "mode": "0700", + "owner": "mesh-build-node", + "unknown": "program" + }, + { + "id": "cache", + "type": "directory", + "path": "/var/lib/mesh-build-node-cache", + "mode": "0700", + "owner": "mesh-build-node", + "unknown": "program" + }, + { + "id": "ssh-dir", + "type": "directory", + "path": "${dir:state}/.ssh", + "mode": "0700", + "owner": "mesh-build-node", + "unknown": "program" + }, + { + "id": "ssh-config", + "type": "file", + "path": "${dir:ssh-dir}/config", + "owner": "mesh-build-node", + "mode": "0600", + "template": "files/forge" + }, + { + "id": "known-hosts", + "type": "file", + "path": "${dir:ssh-dir}/known_hosts", + "owner": "mesh-build-node", + "mode": "0644", + "template": "files/known_hosts" + }, + { + "id": "git-config", + "type": "file", + "path": "${dir:state}/.config/git/config", + "owner": "mesh-build-node", + "mode": "0644", + "template": "files/gitconfig" + }, + { + "id": "toolchain-dir", + "type": "archive", + "path": "${dir:state}/toolchain", + "owner": "mesh-build-node", + "artifact": "toolchain", + "unknown": "program" + }, + { + "id": "git", + "type": "package", + "package": "git" + }, + { + "id": "openssh", + "type": "package", + "package": "openssh" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-build-node", + "binary": "mesh-build-node", + "loads": [ + "mesh-build-node" + ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" + }, + { + "name": "toolchain", + "kind": "archive", + "from": "toolchain" + } + ] + } +} diff --git a/go/testdata/beside/mesh-catalog/modules/mesh-delivery/module.json.captured b/go/testdata/beside/mesh-catalog/modules/mesh-delivery/module.json.captured deleted file mode 100644 index 1a40896..0000000 --- a/go/testdata/beside/mesh-catalog/modules/mesh-delivery/module.json.captured +++ /dev/null @@ -1,95 +0,0 @@ -{ - "module": "mesh-delivery", - "version": "1", - "slug": "deliver", - "claims": [ - { - "name": "mesh-delivery", - "scope": "mesh", - "serves": [ - "deliveries", - "times", - "show", - "groups", - "what-if", - "checks", - "table", - "stalled", - "recheck", - "release", - "stop", - "close", - "retire-history" - ] - } - ], - "consumes": [ - "gitea.pull.updated", - "gitea.pull.merged", - "gitea.pull.closed", - "gitea.forge.status", - "gitea.forge.pull-request.edited", - "gitea.forge.pull-request.closed", - "mesh-controller.checked", - "mesh-controller.plan-moved" - ], - "emits": [ - "transition", - "group", - "merged" - ], - "invokes": [ - "seat:mesh-controller.delivery-plan", - "seat:mesh-controller.delivery-check", - "seat:mesh-controller.deliver", - "seat:mesh-controller.delivery-backups-again", - "seat:mesh-controller.delivery-stop", - "seat:mesh-controller.delivery-walks", - "seat:mesh-controller.forget-checks", - "gitea.gitea_note_append", - "gitea.gitea_delivery_view", - "gitea.gitea_commit_status", - "gitea.gitea_commit_statuses", - "gitea.gitea_contains", - "gitea.gitea_open_pulls", - "gitea.gitea_merge_pull_request", - "gitea.gitea_add_comment", - "seat:operator-channel.notify" - ], - "uses": [ - "operator-channel" - ], - "state": [ - "deliveries", - "groups", - "asked", - "pulls" - ], - "tools": [ - "delivery_status" - ], - "resources": [ - { - "id": "state", - "type": "directory", - "mode": "0700", - "place": ".", - "unknown": "program" - } - ], - "build": { - "artifacts": [ - { - "name": "tools", - "kind": "bundle", - "language": "go", - "system": "arch", - "from": "cmd/mesh-delivery", - "binary": "mesh-delivery", - "loads": [ - "mesh-delivery" - ] - } - ] - } -} diff --git a/go/testdata/beside/mesh-catalog/modules/mesh-issues/module.json.captured b/go/testdata/beside/mesh-catalog/modules/mesh-issues/module.json.captured index 592e81c..dcfc71c 100644 --- a/go/testdata/beside/mesh-catalog/modules/mesh-issues/module.json.captured +++ b/go/testdata/beside/mesh-catalog/modules/mesh-issues/module.json.captured @@ -57,7 +57,6 @@ "consumes": [ "mesh-issues.opened", "mesh-issues.moved", - "mesh-delivery.transition", "mesh-delivery-gate.transition" ], "invokes": [ diff --git a/go/testdata/beside/mesh-catalog/modules/mesh-mail-reader/files/callers.json b/go/testdata/beside/mesh-catalog/modules/mesh-mail-reader/files/callers.json new file mode 100644 index 0000000..84dd800 --- /dev/null +++ b/go/testdata/beside/mesh-catalog/modules/mesh-mail-reader/files/callers.json @@ -0,0 +1,3 @@ +{ + "callers": ["mesh-mail-web"] +} diff --git a/go/testdata/beside/mesh-catalog/modules/mesh-mail-reader/files/database-url b/go/testdata/beside/mesh-catalog/modules/mesh-mail-reader/files/database-url new file mode 100644 index 0000000..dc59da9 --- /dev/null +++ b/go/testdata/beside/mesh-catalog/modules/mesh-mail-reader/files/database-url @@ -0,0 +1 @@ +postgresql://${bound:postgres-database:as}:${secret:postgres-database}@${bound:postgres-database:at}:${bound:postgres-database:port}/${bound:postgres-database:as} diff --git a/go/testdata/beside/mesh-catalog/modules/mesh-mail-reader/module.json.captured b/go/testdata/beside/mesh-catalog/modules/mesh-mail-reader/module.json.captured new file mode 100644 index 0000000..b83e8bb --- /dev/null +++ b/go/testdata/beside/mesh-catalog/modules/mesh-mail-reader/module.json.captured @@ -0,0 +1,141 @@ +{ + "module": "mesh-mail-reader", + "version": "1", + "runs-as": "mesh-mail-reader", + "claims": [ + { + "name": "mail-reading", + "scope": "mesh", + "serves": ["entries", "groups", "correct", "lessons", "undo", "status"] + } + ], + "consumes": [ + "mesh-mail.mail.*.message.added" + ], + "invokes": [ + "seat:mail-store.read", + "seat:mail-store.accounts", + "claude-code.claude_code_judge", + "claude-code.claude_code_judged", + "messenger.messenger_check", + "seat:operator-channel.tell" + ], + "requires": ["postgres-database"], + "contributes": { + "postgres-database": { + "name": "mail-reader" + } + }, + "binds": { + "postgres-database": "${dir:state}/database.json" + }, + "secrets": { + "postgres-database": "${dir:state}/database.secret" + }, + "config": { + "callers": { + "type": "list", + "description": "Who may call one of mail-reading's six verbs (entries, groups, correct, lessons, undo, status): a module's name (on every machine), ., or person. (novox/hq ADR 0459 §2). mesh-mail-web alone by default.", + "trusted": true, + "default": ["mesh-mail-web"] + } + }, + "tools": [ + "mail-reading.entries", + "mail-reading.groups", + "mail-reading.correct", + "mail-reading.lessons", + "mail-reading.undo", + "mail-reading.status", + "mail_reader_health" + ], + "runtime-health": { + "kind": "tool", + "tool": "mail_reader_health", + "interval": "60s", + "timeout": "10s", + "looks": 2, + "grace": "60s" + }, + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-mail-reader", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-mail-reader", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-mail-reader" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-mail-reader", + "place": ".", + "unknown": "program" + }, + { + "id": "callers", + "trusted": true, + "type": "file", + "path": "${dir:state}/callers.json", + "mode": "0600", + "owner": "mesh-mail-reader", + "merge": "json", + "template": "files/callers.json", + "takes-config": ["callers"] + }, + { + "id": "database-url", + "type": "file", + "path": "${dir:state}/database.url", + "mode": "0600", + "owner": "mesh-mail-reader", + "template": "files/database-url" + }, + { + "id": "prepare", + "type": "process", + "name": "mesh-mail-reader-prepare", + "artifact": "tools", + "run": [ + "./mesh-mail-reader", + "prepare" + ], + "run-once": true, + "env": { + "DATABASE_URL_FILE": "${dir:state}/database.url" + }, + "restart-on": [ + "database-url" + ] + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-mail-reader", + "binary": "mesh-mail-reader", + "loads": [ + "mesh-mail-reader" + ], + "env": { + "DATABASE_URL_FILE": "${dir:state}/database.url", + "MESH_MAIL_READER_CALLERS": "${dir:state}/callers.json" + } + }, + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + } +} diff --git a/go/testdata/beside/mesh-catalog/modules/mesh-mail-web/module.json.captured b/go/testdata/beside/mesh-catalog/modules/mesh-mail-web/module.json.captured index fd92839..c6c0d3d 100644 --- a/go/testdata/beside/mesh-catalog/modules/mesh-mail-web/module.json.captured +++ b/go/testdata/beside/mesh-catalog/modules/mesh-mail-web/module.json.captured @@ -15,7 +15,13 @@ "seat:mail-store.search", "seat:mail-store.read", "seat:mail-store.send", - "seat:mail-store.send-status" + "seat:mail-store.send-status", + "seat:mail-reading.entries", + "seat:mail-reading.groups", + "seat:mail-reading.status", + "seat:mail-reading.correct", + "seat:mail-reading.lessons", + "seat:mail-reading.undo" ], "reads": ["mesh-mail.accounts"], "requires": ["route", "oidc-client"], diff --git a/go/testdata/beside/mesh-catalog/modules/mesh-mail/module.json.captured b/go/testdata/beside/mesh-catalog/modules/mesh-mail/module.json.captured index 1eba0cf..735fbc5 100644 --- a/go/testdata/beside/mesh-catalog/modules/mesh-mail/module.json.captured +++ b/go/testdata/beside/mesh-catalog/modules/mesh-mail/module.json.captured @@ -18,6 +18,11 @@ "scope": "node", "needs": ["node-notifier"], "serves": ["status"] + }, + { + "name": "mail-reading", + "scope": "mesh", + "serves": ["entries", "groups", "correct", "lessons", "undo", "status"] } ], "claims": [ diff --git a/go/testdata/beside/mesh-catalog/modules/messenger/files/settings.json b/go/testdata/beside/mesh-catalog/modules/messenger/files/settings.json index ce76334..6d22854 100644 --- a/go/testdata/beside/mesh-catalog/modules/messenger/files/settings.json +++ b/go/testdata/beside/mesh-catalog/modules/messenger/files/settings.json @@ -1,4 +1,5 @@ { "time-zone": "", - "only-what-needs-me": [] + "only-what-needs-me": [], + "messages-per-hour": 100 } diff --git a/go/testdata/beside/mesh-catalog/modules/messenger/module.json.captured b/go/testdata/beside/mesh-catalog/modules/messenger/module.json.captured index 8501bd4..6045195 100644 --- a/go/testdata/beside/mesh-catalog/modules/messenger/module.json.captured +++ b/go/testdata/beside/mesh-catalog/modules/messenger/module.json.captured @@ -132,6 +132,14 @@ "description": "The kinds of channel set to say only what needs the operator: a walk that delivered with nothing held and nothing open is not told there (ADR 0308 §4); empty is none.", "trusted": true, "default": [] + }, + "messages-per-hour": { + "type": "number", + "description": "messages per channel per hour before the rest are held as one; 0 turns the pause off. The tell verb has its own cap of 10 and is not counted here.", + "minimum": 0, + "maximum": 1000, + "trusted": false, + "default": 100 } }, "resources": [ @@ -155,7 +163,7 @@ "type": "file", "path": "${dir:state}/settings.json", "mode": "0600", - "takes-config": ["time-zone", "desktop-machines", "only-what-needs-me"], + "takes-config": ["time-zone", "desktop-machines", "only-what-needs-me", "messages-per-hour"], "merge": "json", "template": "files/settings.json", "owner": "messenger" diff --git a/go/testdata/beside/mesh-catalog/modules/nodejs/module.json.captured b/go/testdata/beside/mesh-catalog/modules/nodejs/module.json.captured new file mode 100644 index 0000000..c357344 --- /dev/null +++ b/go/testdata/beside/mesh-catalog/modules/nodejs/module.json.captured @@ -0,0 +1,25 @@ +{ + "module": "nodejs", + "version": "1", + "capabilities": [ + "package-manager" + ], + "provides": [ + { + "name": "node-toolchain", + "reach": "machine" + } + ], + "resources": [ + { + "id": "nodejs", + "type": "package", + "package": "nodejs" + }, + { + "id": "npm", + "type": "package", + "package": "npm" + } + ] +} diff --git a/go/testdata/beside/mesh-catalog/modules/pacman/files/mesh-pacman-weekly-upgrade.service b/go/testdata/beside/mesh-catalog/modules/pacman/files/mesh-pacman-weekly-upgrade.service new file mode 100644 index 0000000..69741dd --- /dev/null +++ b/go/testdata/beside/mesh-catalog/modules/pacman/files/mesh-pacman-weekly-upgrade.service @@ -0,0 +1,12 @@ +# Generated by the mesh. Do not edit — module pacman writes this file and replaces it at every push. +[Unit] +Description=Weekly full upgrade (the mesh's pacman module, ADR 0477) +Wants=network-online.target +After=network-online.target + +[Service] +# The upgrade pacman_upgrade runs (pacman -Syu --noconfirm), as a oneshot whose result is the module's health +# (pacman_upgrade_health). The mesh restarts no machine for an upgrade: a new kernel waits for the next restart. +Type=oneshot +Nice=10 +ExecStart=/usr/bin/pacman -Syu --noconfirm diff --git a/go/testdata/beside/mesh-catalog/modules/pacman/files/mesh-pacman-weekly-upgrade.timer b/go/testdata/beside/mesh-catalog/modules/pacman/files/mesh-pacman-weekly-upgrade.timer new file mode 100644 index 0000000..a8a9b4d --- /dev/null +++ b/go/testdata/beside/mesh-catalog/modules/pacman/files/mesh-pacman-weekly-upgrade.timer @@ -0,0 +1,11 @@ +# Generated by the mesh. Do not edit — module pacman writes this file and replaces it at every push. +[Unit] +Description=Weekly full upgrade, Saturday 04:00 local time on every node (the mesh's pacman module, ADR 0477) + +[Timer] +# The same moment on every node: no random delay. A node that was off or asleep then upgrades at its next start. +OnCalendar=Sat *-*-* 04:00:00 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/go/testdata/beside/mesh-catalog/modules/pacman/module.json.captured b/go/testdata/beside/mesh-catalog/modules/pacman/module.json.captured index 746c8c5..ca1a81a 100644 --- a/go/testdata/beside/mesh-catalog/modules/pacman/module.json.captured +++ b/go/testdata/beside/mesh-catalog/modules/pacman/module.json.captured @@ -19,6 +19,7 @@ "pacman_files", "pacman_updates", "pacman_upgrade", + "pacman_upgrade_health", "pacman_orphans", "pacman_remove_orphans", "pacman_cache", @@ -63,14 +64,53 @@ "type": "service", "unit": "reflector.timer", "state": "running", - "boot": "enabled" + "boot": "enabled", + "health": { + "kind": "unit" + } }, { "id": "cache-cleaning", "type": "service", "unit": "paccache.timer", "state": "running", - "boot": "enabled" + "boot": "enabled", + "health": { + "kind": "unit" + } + }, + { + "id": "upgrade-service", + "type": "file", + "path": "/etc/systemd/system/mesh-pacman-weekly-upgrade.service", + "mode": "0644", + "template": "files/mesh-pacman-weekly-upgrade.service" + }, + { + "id": "upgrade-timer", + "type": "file", + "path": "/etc/systemd/system/mesh-pacman-weekly-upgrade.timer", + "mode": "0644", + "template": "files/mesh-pacman-weekly-upgrade.timer" + }, + { + "id": "weekly-upgrade", + "type": "service", + "unit": "mesh-pacman-weekly-upgrade.timer", + "state": "running", + "boot": "enabled", + "restart-on": [ + "upgrade-service", + "upgrade-timer" + ], + "health": { + "kind": "tool", + "tool": "pacman_upgrade_health", + "interval": "60s", + "timeout": "20s", + "looks": 2, + "grace": "60s" + } } ], "build": { diff --git a/go/testdata/beside/mesh-catalog/modules/power/files/bin/power-moment b/go/testdata/beside/mesh-catalog/modules/power/files/bin/power-moment old mode 100644 new mode 100755 diff --git a/go/testdata/beside/mesh-catalog/modules/rofi/files/bin/dmenu b/go/testdata/beside/mesh-catalog/modules/rofi/files/bin/dmenu old mode 100644 new mode 100755 diff --git a/go/testdata/beside/mesh-catalog/modules/rofi/files/bin/rofi-launch b/go/testdata/beside/mesh-catalog/modules/rofi/files/bin/rofi-launch old mode 100644 new mode 100755 diff --git a/go/testdata/beside/mesh-catalog/modules/rofi/files/bin/rofi-powermenu b/go/testdata/beside/mesh-catalog/modules/rofi/files/bin/rofi-powermenu old mode 100644 new mode 100755 diff --git a/go/testdata/beside/mesh-catalog/modules/screen-lock/files/bin/screen-lock b/go/testdata/beside/mesh-catalog/modules/screen-lock/files/bin/screen-lock old mode 100644 new mode 100755 diff --git a/go/testdata/beside/mesh-catalog/modules/xorg/files/xinitrc b/go/testdata/beside/mesh-catalog/modules/xorg/files/xinitrc old mode 100644 new mode 100755