From d8d6d196a44e78a4a1a1fb352981e32c829ac0c6 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 14:30:26 +0200 Subject: [PATCH 1/6] The controller holds a module whose container names groups on a node that does not report container-groups, naming the capability (issue 1638, ADR 0480 clause 7) --- cmd/mesh-controller/container_groups_test.go | 65 +++++++++++ go.mod | 2 +- go.sum | 4 +- testdata/beside/CAPTURED | 2 +- .../novox/mesh-sdk/go/catalogue/manifest.go | 1 + .../novox/mesh-sdk/go/catalogue/resolve.go | 19 +++- .../novox/mesh-sdk/go/catalogue/seat_data.go | 2 +- .../mesh-sdk/go/catalogue/setting_into.go | 106 ++++++++++++++++++ .../go/catalogue/unconsumed_placeholder.go | 2 +- .../mesh-sdk/go/declaration/declaration.go | 33 ++++++ .../novox/mesh-sdk/go/delivery/buildrecord.go | 2 +- .../novox/mesh-sdk/go/delivery/source.go | 18 ++- vendor/modules.txt | 2 +- 13 files changed, 247 insertions(+), 11 deletions(-) create mode 100644 cmd/mesh-controller/container_groups_test.go diff --git a/cmd/mesh-controller/container_groups_test.go b/cmd/mesh-controller/container_groups_test.go new file mode 100644 index 00000000..6fd9f9ba --- /dev/null +++ b/cmd/mesh-controller/container_groups_test.go @@ -0,0 +1,65 @@ +package main + +import ( + "io" + "os" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// Issue 1638 (novox/hq ADR 0480 clause 7), taken in with mesh-sdk's container `groups`: the controller sends a container's +// groups only to a node-engine that reports the capability container-groups, and holds the module on any other, naming it. + +const n8nWithAGroup = `{"module":"n8n","version":"1", + "config":{"media-gid":{"type":"number","minimum":1,"required":true,"description":"the group that owns the media library"}}, + "resources":[{"id":"server","type":"container","name":"n8n", + "image":"n8nio/n8n@sha256:4846eb2f4b874ab04cde7fc1e249d2ddaec66e9aea64439beb2972cfea88e3c0", + "groups":["${config:media-gid}"]}]}` + +// printedBy runs reportUnhostable and returns what it printed. +func printedBy(t *testing.T, node string, plan catalogue.Resolution) string { + t.Helper() + stdout := os.Stdout + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + os.Stdout = w + reportUnhostable(node, plan) + w.Close() + os.Stdout = stdout + out, _ := io.ReadAll(r) + return string(out) +} + +// 1638/AC1: a node that reports the capability gets the module; one that does not has it held, the capability named. +func TestAModuleWithGroupsIsHeldOnANodeThatDoesNotReportTheCapability(t *testing.T) { + m, err := catalogue.ParseManifest([]byte(n8nWithAGroup)) + if err != nil { + t.Fatal(err) + } + shelf := catalogue.Shelf{"n8n": m} + reads := catalogue.Node{Name: "ace", Capabilities: map[string]bool{"container-runtime": true, "container-groups": true}} + old := catalogue.Node{Name: "g14", Capabilities: map[string]bool{"container-runtime": true}} + + got, err := catalogue.Resolve(shelf, []string{"n8n"}, reads, catalogue.World{}) + if err != nil || len(got.Unhostable) != 0 || len(got.Modules) != 1 { + t.Fatalf("a node that reads groups: %v, held %+v", err, got.Unhostable) + } + if out := printedBy(t, "ace", got); out != "" { + t.Errorf("a node that reads groups was told: %q", out) + } + + got, err = catalogue.Resolve(shelf, []string{"n8n"}, old, catalogue.World{}) + if err != nil { + t.Fatal(err) + } + if len(got.Modules) != 0 || len(got.Unhostable) != 1 { + t.Fatalf("n8n was sent to a node that does not report container-groups: hosted %d, held %+v", len(got.Modules), got.Unhostable) + } + if out := printedBy(t, "g14", got); !strings.Contains(out, "n8n") || !strings.Contains(out, "container-groups") { + t.Errorf("the hold does not name the module and the capability: %q", out) + } +} diff --git a/go.mod b/go.mod index 1bb6c910..e15f5dd1 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-controller go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021093645-465beb4ea5e3 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122745-e0e1a91496de github.com/jackc/pgx/v5 v5.10.0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 diff --git a/go.sum b/go.sum index f7884acb..69cc7def 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5 h1:E5stfIeCN6uYgA9TCv+4K8JgaURg3zQppvUgJLaDMP4= git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5/go.mod h1:4wyoCr6IIIIarEUuROThWDj7DAvJvIidqMjWcpSnOwE= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021093645-465beb4ea5e3 h1:OKxU64kHN1yQ/GOa6LwoUwVmiQ74d83WMSZJRFPVm9c= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021093645-465beb4ea5e3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122745-e0e1a91496de h1:0iw6Xg/DzsvjIOGq2NE/YeLULuVG0Dv6eUKKMQS+4ys= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122745-e0e1a91496de/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/testdata/beside/CAPTURED b/testdata/beside/CAPTURED index 711e1cdd..5126bb49 100644 --- a/testdata/beside/CAPTURED +++ b/testdata/beside/CAPTURED @@ -9,7 +9,7 @@ a second place to clean besides the catalogue itself. mesh-catalog e0bc919d4945bb5f13f971f74643a0b714cf45da modules/*/module.json, modules/nats/Dockerfile mesh-host 9d9e3a46c4f5943cfffb413d962ce48b75e2fd39 examples/foundation-first-node.lock -mesh-sdk 465beb4ea5e37eb36f6ba7fc60274de2d5a0d060 conformance/events/module-event.json +mesh-sdk e0e1a91496de28becd8387ac7acad4cdb85fdda2 conformance/events/module-event.json To move them, from this repository's root, with the two repositories checked out beside it: diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/manifest.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/manifest.go index 04646544..870ea4d0 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/manifest.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/manifest.go @@ -2214,6 +2214,7 @@ func parseManifest(raw []byte, predefined PredefinedList) (Manifest, error) { problems = append(problems, replacesProblems(m)...) problems = append(problems, UnitSettingProblems(m)...) problems = append(problems, OwnerSettingProblems(m)...) + problems = append(problems, GroupSettingProblems(m)...) problems = append(problems, endpointNameProblems(m)...) problems = append(problems, RouteProblems(m)...) for _, port := range m.Guards { diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/resolve.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/resolve.go index 6e2e08f2..006646ed 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/resolve.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/resolve.go @@ -904,7 +904,7 @@ func isModule(catalogue map[string]Manifest, want string) bool { // machine can host the module as far as its capabilities go. func missingCapabilities(m Manifest, node Node) []string { var missing []string - for _, c := range m.Capabilities { + for _, c := range append(append([]string(nil), m.Capabilities...), impliedCapabilities(m)...) { if !node.Capabilities[c] { missing = append(missing, c) } @@ -1457,3 +1457,20 @@ func packageNames(m map[string][]string) []string { sort.Strings(names) return names } + +// ContainerGroups is the capability a node-engine reports when it reads a container's `groups` and passes each as a +// supplementary group (novox/hq ADR 0480 clause 7, issue 1638). An older engine refuses the key (strict decoding). +const ContainerGroups = "container-groups" + +// impliedCapabilities are the capabilities a module needs for what its resources say, beyond those it declares: a +// container that names groups needs an engine that reads them, so the module is left out of a machine whose report +// lacks the capability, naming it, and the field is sent to no engine that would refuse it. None for a module whose +// containers name no groups. +func impliedCapabilities(m Manifest) []string { + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) == "container" && len(groupsOf(r)) > 0 { + return []string{ContainerGroups} + } + } + return nil +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/seat_data.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/seat_data.go index bea4fc72..e623c8a9 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/seat_data.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/seat_data.go @@ -66,7 +66,7 @@ var capabilityName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`) // names an `if-capability` may give (novox/hq ADR 0255). A name nothing detects would leave its // contribution out on every machine, silently, so it is refused. A detector added there is added here. var KnownCapabilities = []string{ - "battery", "container-runtime", "firewall", "graphical-session", "overlay", "package-manager", + "battery", "container-groups", "container-runtime", "firewall", "graphical-session", "overlay", "package-manager", "power-meter", "privileged", "seat", "service-manager", "uplink-dhcpcd", "uplink-networkmanager", "uplink-systemd-networkd", "user-namespaces", "virtualisation", } diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/setting_into.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/setting_into.go index fbb966fd..73bd2b08 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/setting_into.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/setting_into.go @@ -85,6 +85,9 @@ func settingInto(resource map[string]any, layers []Layer, module string) error { if fmt.Sprint(resource["type"]) == "directory" { return settingIntoOwner(resource, layers, module) } + if fmt.Sprint(resource["type"]) == "container" { + return settingIntoGroups(resource, layers, module) + } if fmt.Sprint(resource["type"]) != "file" { return nil } @@ -343,6 +346,12 @@ func settingKeysAskedOutsideOwners(m Manifest) map[string]bool { if unit, ok := r["unit"].(string); ok { note(unit) } + case "container": + for _, g := range groupsOf(r) { + if s, ok := g.(string); ok { + note(s) + } + } } } inValues := func(values map[string]any) { @@ -365,3 +374,100 @@ func settingKeysAskedOutsideOwners(m Manifest) map[string]bool { } return used } + +// A container's supplementary groups may be an operator's value (novox/hq ADR 0480 clause 7, ADR 0348 §11, issue 1636): +// n8n joins the media library's group, whose number is the machine's, never the definition's (ADR 0112 §1). Each +// element of `groups` is a whole placeholder or a whole number; a placeholder is filled with the number the +// config value holds, and the list reaches the node-engine as numbers. + +// groupSide is one element of a container's groups: a whole placeholder or a whole number. +var groupSide = regexp.MustCompile(`^(?:\$\{setting:[a-z0-9][a-z0-9_.-]*\}|[0-9]+)$`) + +// groupsOf is a container resource's groups as the manifest wrote them (text or numbers), none when it names none. +func groupsOf(r map[string]any) []any { + groups, _ := r["groups"].([]any) + return groups +} + +// GroupSettingProblems refuses, where a manifest is read, a container whose groups hold anything but whole +// placeholders and whole numbers, or a placeholder whose key the module neither declares nor asks for elsewhere. +func GroupSettingProblems(m Manifest) []string { + var problems []string + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) != "container" { + continue + } + for _, g := range groupsOf(r) { + s, isText := g.(string) + if !isText { + continue // a number: the declaration judges its value + } + if !groupSide.MatchString(s) { + problems = append(problems, fmt.Sprintf("%s: the container %v's group %q is not a whole "+ + "${config:key} or a whole number: a supplementary group is a number, and nothing else is "+ + "written into it (novox/hq ADR 0480 clause 7)", m.Module, r["id"], s)) + continue + } + if m.Templated() { + continue + } + for _, key := range settingsUsed(s) { + if _, declared := m.Settings[key]; !declared && !m.DeclaresConfig() { + continue // the settings form judges its own keys + } + if _, declared := m.Settings[key]; !declared { + problems = append(problems, fmt.Sprintf("%s: the container %v's group says ${config:%s}, and the module "+ + "declares no such value (novox/hq ADR 0348 §4)", m.Module, r["id"], key)) + } + } + } + } + return problems +} + +// settingIntoGroups fills ${setting:…} in a container's groups, refusing — with the key and why, the resource left as +// it was — a key nothing sets and a value that is not a whole number above zero. A number is kept; the list that +// results is numbers alone. +func settingIntoGroups(resource map[string]any, layers []Layer, module string) error { + groups := groupsOf(resource) + if len(groups) == 0 { + return nil + } + filled := make([]any, 0, len(groups)) + for _, g := range groups { + s, isText := g.(string) + if !isText { + if f, isNumber := g.(float64); isNumber && f == float64(int(f)) { + g = int(f) // a number the manifest wrote, as the numbers a placeholder gives + } + filled = append(filled, g) + continue + } + keys := settingsUsed(s) + if len(keys) != 1 || settingRef.ReplaceAllString(s, "") != "" { + filled = append(filled, g) // not a whole placeholder: the manifest check refuses it + continue + } + key := keys[0] + value, set := settingValue(layers, key) + if !set { + return &UnsetSettingError{Module: module, Setting: key, + where: fmt.Sprintf("a container %v whose groups", resource["id"]), setToday: orNoSettings(layers), + said: fmt.Sprintf( + "%s has a container %v whose groups say ${setting:%s}, and nothing sets %q for it — an operator's "+ + "value is the assignment's, never the definition's (novox/hq ADR 0112): "+ + "`settings set %s ` with {%q: …}%s", + module, resource["id"], key, key, module, key, orNoSettings(layers))} + } + v := plainly(value) + id, whole := wholeID(v) + if !whole || id == 0 { + return fmt.Errorf("%s: the setting %q is %q, which cannot be a group of the container %v: a supplementary "+ + "group is a whole number above zero, never a name or other text (novox/hq ADR 0480 clause 7, ADR 0328 §15)", + module, key, v, resource["id"]) + } + filled = append(filled, int(id)) + } + resource["groups"] = filled + return nil +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/unconsumed_placeholder.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/unconsumed_placeholder.go index 39df9ba5..b017d5a9 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/unconsumed_placeholder.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/unconsumed_placeholder.go @@ -56,7 +56,7 @@ type filler struct { // seatInto, machineInto and contributionsInto. A pass that comes to read another field adds it here, // or the sweep refuses the placeholder it would have filled. var fillers = []filler{ - {namespace: "setting", patterns: []*regexp.Regexp{settingRef}, fields: map[string][]string{"file": {"content"}, "service": {"unit"}, "directory": {"owner"}}}, + {namespace: "setting", patterns: []*regexp.Regexp{settingRef}, fields: map[string][]string{"file": {"content"}, "service": {"unit"}, "directory": {"owner"}, "container": {"groups"}}}, {namespace: "dir", patterns: []*regexp.Regexp{dirRef}, fields: map[string][]string{"*": {"path", "content", "volumes", "env", "env-file"}}}, {namespace: "access", patterns: []*regexp.Regexp{accessRef}, fields: map[string][]string{"*": {"path", "content", "volumes", "env", "env-file"}}}, {namespace: "secret", patterns: []*regexp.Regexp{placeholder}, fields: map[string][]string{"file": {"content"}}, diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/declaration/declaration.go b/vendor/git.novox.be/novox/mesh-sdk/go/declaration/declaration.go index ea16b919..b2fcccf1 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/declaration/declaration.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/declaration/declaration.go @@ -1454,6 +1454,14 @@ type Container struct { // container runs as its image says. Its key is `identity`, the manifest's word; the field is not, because // a resource's Identity is its id. RunAs *ContainerIdentity `json:"identity,omitempty"` + + // Groups are the supplementary groups the container's process joins, as numbers, each passed to the runtime + // as a group to add (novox/hq ADR 0480 clause 7, issue 1636): n8n writes the media library through the + // library's group, which a stock image knows nothing of. Each is above zero and below the kernel's "no group" + // (ADR 0328 §15: never root). Part of the spec, so a change recreates the container. Absent: none are added. + // An older node-engine refuses the key (strict decoding), so the controller sends it only to a node that + // reports it reads it. + Groups []int64 `json:"groups,omitempty"` } // The ways an image is told who to run as (novox/hq ADR 0325 §2). @@ -1521,6 +1529,30 @@ func (i *ContainerIdentity) problems(where string, env map[string]string) []stri return problems } +// groupProblems refuses a supplementary group that is root's or below it, the kernel's "no group" number, or a +// number named twice (novox/hq ADR 0480 clause 7, ADR 0328 §15). +func (c *Container) groupProblems(where string) []string { + var problems []string + seen := map[int64]bool{} + for _, g := range c.Groups { + switch { + case g == 0: + problems = append(problems, fmt.Sprintf("%s: container %s adds group 0, and the mesh never makes or keeps anything "+ + "as root: a supplementary group is never root's (novox/hq ADR 0328 §15)", where, c.Name)) + case g < 0: + problems = append(problems, fmt.Sprintf("%s: container %s adds group %d, below zero: a group is a whole number "+ + "above zero", where, c.Name, g)) + case g >= 1<<32-1: + problems = append(problems, fmt.Sprintf("%s: container %s adds group %d, which is the kernel's \"no group\" "+ + "number or beyond it", where, c.Name, g)) + case seen[g]: + problems = append(problems, fmt.Sprintf("%s: container %s adds group %d twice", where, c.Name, g)) + } + seen[g] = true + } + return problems +} + func (c *Container) Identity() string { return c.ID } func (c *Container) Kind() Type { return TypeContainer } func (c *Container) Target() string { return c.Name } @@ -1603,6 +1635,7 @@ func (c *Container) validate(where string, _ bool) []string { "runs and exits, and joins nothing afterwards") } problems = append(problems, c.RunAs.problems(where+": container "+c.Name, c.Env)...) + problems = append(problems, c.groupProblems(where)...) return append(problems, checkImage(where, c.Image)...) } diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/delivery/buildrecord.go b/vendor/git.novox.be/novox/mesh-sdk/go/delivery/buildrecord.go index d3f9fb9c..1bccc222 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/delivery/buildrecord.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/delivery/buildrecord.go @@ -25,7 +25,7 @@ type BuildRecord struct { } // Artifact is one thing a build made: mesh-controller internal/chain/records.go Artifact, json keys name, kind and -// reference, nothing else (novox/hq ADR 0478 clause 7). +// reference, nothing else (novox/hq ADR 0478 clause 8). type Artifact struct { Name string `json:"name"` Kind string `json:"kind"` diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/delivery/source.go b/vendor/git.novox.be/novox/mesh-sdk/go/delivery/source.go index a21bf9d1..8d85b64a 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/delivery/source.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/delivery/source.go @@ -389,11 +389,20 @@ type Module struct { Src uint64 `json:"src,omitempty"` Repository string `json:"repository"` Commit string `json:"commit"` - // Path is the module's directory in its repository, trimmed of slashes, empty when its manifest is the repository's - // root (novox/hq ADR 0478 clause 8): the build tools need it. A value written before has none. + // Path is the module's directory in its repository, trimmed of slashes, and RootPath (".") when its manifest is the + // repository's root (novox/hq ADR 0478 clause 9, issue 1615): the build tools need it. Empty means not known: a value + // written before the indexer learned paths has none. Path string `json:"path,omitempty"` + // Toolchains are the toolchains FP2 was computed with, by language (go, typescript): the version installed and the + // build module's recipe version the build nodes agreed on then (novox/hq ADR 0477 clause 2, ADR 0478 clause 3). A builder + // builds the fingerprint only with these. Empty when FP2 is empty or the module has no toolchain; a value written + // before has none (issue 1615). + Toolchains map[string]ToolchainUsed `json:"toolchains,omitempty"` } +// RootPath is Module.Path of a module whose manifest is its repository's root. +const RootPath = "." + // KeptInputs2 are SourceInputs2 without the module's name (the key has it) and without toolchains (those come from // the build modules' toolchain keys when the fingerprint is computed): what a Module keeps of a merge. type KeptInputs2 struct { @@ -421,6 +430,11 @@ func (m Module) check() error { case m.FP2 != "" && m.None2 != "": return fmt.Errorf("delivery: a module has both fp2 and none2") } + for language, t := range m.Toolchains { + if t.Version == "" || t.Recipe == "" { + return fmt.Errorf("delivery: the module's toolchain of %q needs its version and its recipe version", language) + } + } if m.Inputs2 != nil && m.Manifest == "" { return fmt.Errorf("delivery: a module keeps new-form inputs but has no manifest digest") } diff --git a/vendor/modules.txt b/vendor/modules.txt index c26aec0c..8c7106d9 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,4 +1,4 @@ -# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021093645-465beb4ea5e3 +# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122745-e0e1a91496de ## explicit; go 1.26.0 git.novox.be/novox/mesh-sdk/go/asks git.novox.be/novox/mesh-sdk/go/catalogue From c24139a0fd9fc1d4aa3f263fc1e8145a1023439f Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 14:33:09 +0200 Subject: [PATCH 2/6] Take a build's outcome on the event loop without waiting for the plans, and leave it to the build records with a look asked, so a holder sending for minutes no longer stops the controller listening (issue 1708) --- .../outcome_off_the_loop_test.go | 106 ++++++++++++++++++ cmd/mesh-controller/release_plan.go | 79 ++++++++++++- internal/inventory/hold.go | 5 +- 3 files changed, 183 insertions(+), 7 deletions(-) create mode 100644 cmd/mesh-controller/outcome_off_the_loop_test.go diff --git a/cmd/mesh-controller/outcome_off_the_loop_test.go b/cmd/mesh-controller/outcome_off_the_loop_test.go new file mode 100644 index 00000000..f8bdad1b --- /dev/null +++ b/cmd/mesh-controller/outcome_off_the_loop_test.go @@ -0,0 +1,106 @@ +package main + +import ( + "context" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// heldElsewhere is the plans held by another act for longer than any test runs: waiting for them ends only with the +// context, and a look that does not wait is told at once that they are busy. +func heldElsewhere(ctx context.Context, wait bool) (func(), error) { + if !wait { + return nil, inventory.ErrPlansBusy + } + <-ctx.Done() + return nil, inventory.ErrPlansBusy +} + +// 1708/AC1: given the plans held by another act, when the event loop takes a build's outcome, then it returns at +// once, without waiting the plans' two minutes (on 2026-10-21 at 14:20:45 nodered's outcome held the loop until +// 14:22:45, and the operator was told the controller stopped listening). +func TestAnOutcomeDoesNotWaitForThePlansAnotherActHolds(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + defer cancel() + start := time.Now() + release, err := holdPlansForOutcome(ctx, heldElsewhere) + if took := time.Since(start); took > time.Second { + t.Fatalf("taking an outcome waited %s for plans another act holds; the event loop takes nothing meanwhile", took) + } + if err == nil || release != nil { + t.Fatalf("plans another act holds were said to be held: release %v, err %v", release != nil, err) + } +} + +// 1708/AC4: given the plans free, when the event loop takes a build's outcome, then it holds them and marks the plan +// from the outcome directly, as before. +func TestAnOutcomeHoldsThePlansWhenTheyAreFree(t *testing.T) { + let := false + release, err := holdPlansForOutcome(t.Context(), func(context.Context, bool) (func(), error) { + return func() { let = true }, nil + }) + if err != nil || release == nil { + t.Fatalf("free plans were not held: %v", err) + } + release() + if !let { + t.Fatal("the hold given was not the one taken") + } +} + +// withLooker installs a looker that is never run, and answers it, so a test can read whether a look was asked. +func withLooker(t *testing.T) *looker { + t.Helper() + l := newLooker(func(context.Context) bool { return false }) + lookerMu.Lock() + before := theLooker + theLooker = l + lookerMu.Unlock() + t.Cleanup(func() { + lookerMu.Lock() + theLooker = before + lookerMu.Unlock() + }) + return l +} + +// 1708/AC2: given an outcome left to the build records because the plans were held, when it is left, then the look +// at the plans is asked for, so the records settle it once the holder lets go rather than at the next event. +func TestAnOutcomeLeftToTheRecordsAsksForTheLook(t *testing.T) { + l := withLooker(t) + leaveToRecords("nodered", "build-1708-ac2", "", inventory.ErrPlansBusy) + select { + case <-l.signal: + default: + t.Fatal("an outcome left to the build records asked for no look: the plan waits for whatever event comes next") + } +} + +// 1708/AC3: given an outcome left to the records that said what the build did (ADR 0350 §4), when the look settles +// the plan from the build records by that build's id, then the module keeps those words, once. +func TestTheRecordsSettleAnOutcomeWithWhatTheBuildDid(t *testing.T) { + withLooker(t) + asked := time.Date(2026, 10, 21, 12, 15, 0, 0, time.UTC) + plan := func() inventory.Plan { + return inventory.Plan{ID: "plan-1708", Tiers: [][]string{{"nodered"}}, + Modules: map[string]*inventory.PlanModule{"nodered": {State: "asked", AskedAt: &asked, Build: "build-1708-ac3"}}} + } + byID := map[string]inventory.Build{"build-1708-ac3": {ID: "build-1708-ac3", Module: "nodered", Commit: "4efac366", + At: asked.Add(5 * time.Minute)}} + leaveToRecords("nodered", "build-1708-ac3", "reused from build-1708-check", inventory.ErrPlansBusy) + + p := plan() + if !settleFromRecords(&p, p.Tiers[0], nil, byID) || p.Modules["nodered"].State != "built" { + t.Fatalf("the records did not settle the outcome: %+v", p.Modules["nodered"]) + } + if got := p.Modules["nodered"].Made; got != "reused from build-1708-check" { + t.Fatalf("settled from the records, the module says %q of what its build did, not the outcome's words", got) + } + again := plan() + settleFromRecords(&again, again.Tiers[0], nil, byID) + if got := again.Modules["nodered"].Made; got != "" { + t.Fatalf("an outcome's words were kept for a second settle: %q", got) + } +} diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 5dea2cea..0ddb310b 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -625,16 +625,81 @@ func planBuilt(ctx context.Context, open *stores, module, commit, failed string, planBuiltSaying(ctx, open, module, commit, failed, asked, id, "") } +// holdPlansForOutcome is how an outcome taken on the event loop holds the plans: **without waiting for them** +// (novox/hq issue 1708). The loop takes one message at a time, and the plans' holder may keep them for minutes while +// it sends machines their declarations; an outcome that waited its turn (up to inventory.HoldWaitFor, two minutes) +// held every report, build and merge behind it for as long, and the watchdog said the controller stopped listening +// (2026-10-21 14:20:45 to 14:22:45). Busy, the outcome is left to the build records, where it already is, and the +// look settles the plan from them (leaveToRecords). +func holdPlansForOutcome(ctx context.Context, hold func(context.Context, bool) (func(), error)) (func(), error) { + return hold(ctx, false) +} + +// leaveToRecords says an outcome was not taken into the plans, which settle it from the build records (novox/hq issue +// 214). When another act holds the plans it asks for the look, which waits for them off the event loop and settles +// the plan from the records once they are let go (issue 1708); what the build did is kept for that settle by the +// build's id, so the walk's module says it as it would have (ADR 0350 §4). +func leaveToRecords(module, id, made string, err error) { + fmt.Printf("plans: %s's outcome is left to the build records: %v\n", module, err) + if !errors.Is(err, inventory.ErrPlansBusy) { + return + } + keepMadeFor(id, made, time.Now()) + askToLook() +} + +// madeLeft is what each build left to the records did, by the build's id, until a settle takes it: kept in the +// process, so a controller replaced before the settle loses only these words, never the outcome. +var ( + madeLeftMu sync.Mutex + madeLeft = map[string]madeKept{} +) + +type madeKept struct { + made string + at time.Time +} + +// madeLeftFor is how long words nobody settled are kept: a plan that ended or moved on never asks for them. +const madeLeftFor = time.Hour + +// keepMadeFor keeps what a build left to the records did for the settle that takes it, forgetting words older than +// madeLeftFor. +func keepMadeFor(id, made string, now time.Time) { + if id == "" || made == "" { + return + } + madeLeftMu.Lock() + defer madeLeftMu.Unlock() + for k, v := range madeLeft { + if now.Sub(v.at) > madeLeftFor { + delete(madeLeft, k) + } + } + madeLeft[id] = madeKept{made: made, at: now} +} + +// takeMadeOf is what the build of this id did, if its outcome was left to the records with words; taken once. +func takeMadeOf(id string) string { + madeLeftMu.Lock() + defer madeLeftMu.Unlock() + k, ok := madeLeft[id] + if !ok { + return "" + } + delete(madeLeft, id) + return k.made +} + // planBuiltSaying is planBuilt keeping what the build did — "reused from " or "built: " (novox/hq // ADR 0350 §4) — on the module of the walk it answers. func planBuiltSaying(ctx context.Context, open *stores, module, commit, failed string, asked time.Time, id, made string) { inv := open.inventory - // One controller works the plans at a time (novox/hq issue 213); an outcome waits its turn rather - // than write over what the holder is about to save. Not taken, it is still in the build records, - // which the holder settles the plan from (issue 214). - release, err := inv.HoldPlans(ctx, true) + // One controller works the plans at a time (novox/hq issue 213). Not taken, the outcome is still in the build + // records, which the holder settles the plan from (issue 214). + release, err := holdPlansForOutcome(ctx, inv.HoldPlans) if err != nil { - fmt.Printf("plans: %s's outcome is left to the build records: %v\n", module, err) + leaveToRecords(module, id, made, err) return } defer release() @@ -2531,6 +2596,10 @@ func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]i s.State = "built" s.BuiltAt = &at s.Commit = outcome.Commit + // What the build did, when its outcome was left to the records with words (issue 1708). + if made := takeMadeOf(outcome.ID); made != "" { + s.Made = made + } } else { s.State = "failed" s.Why = outcome.Failed diff --git a/internal/inventory/hold.go b/internal/inventory/hold.go index 30f20ac3..272a40c7 100644 --- a/internal/inventory/hold.go +++ b/internal/inventory/hold.go @@ -100,8 +100,9 @@ var ErrPlansBusy = errors.New("another controller is working the plans") // returned function and by the session ending, so a controller that dies holding it holds nothing. // // wait false gives ErrPlansBusy at once when another holds them — the timer's way: the holder is -// moving the plans already. wait true looks again every HoldPoll for up to HoldWaitFor — an -// outcome's or a merge's way, which must be written. +// moving the plans already; and a build outcome's on the event loop, which the build records keep +// for the look to settle (novox/hq issue 1708). wait true looks again every HoldPoll for up to +// HoldWaitFor — a merge's way, which must be written. func (i *Inventory) HoldPlans(ctx context.Context, wait bool) (func(), error) { deadline := time.Now().Add(HoldWaitFor) for { From 61c8b56107fa103adfe1a5a4e0de739c59bf0f0c Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 14:39:17 +0200 Subject: [PATCH 3/6] Leave moving the plans after a build's outcome to the look in the serving controller, so a release's retried send no longer runs on the event loop (issue 1708) --- .../outcome_off_the_loop_test.go | 36 +++++++++++++++++++ cmd/mesh-controller/plan_looks.go | 12 +++++++ cmd/mesh-controller/release_plan.go | 14 +++++++- 3 files changed, 61 insertions(+), 1 deletion(-) diff --git a/cmd/mesh-controller/outcome_off_the_loop_test.go b/cmd/mesh-controller/outcome_off_the_loop_test.go index f8bdad1b..940745d9 100644 --- a/cmd/mesh-controller/outcome_off_the_loop_test.go +++ b/cmd/mesh-controller/outcome_off_the_loop_test.go @@ -104,3 +104,39 @@ func TestTheRecordsSettleAnOutcomeWithWhatTheBuildDid(t *testing.T) { t.Fatalf("an outcome's words were kept for a second settle: %q", got) } } + +// 1708/AC5: given a serving controller (its look at the plans running), when an outcome was taken into the plans, +// then moving them (a release's sends, which may retry for minutes) is left to the look, off the event loop +// (2026-10-21 14:28:40 to 14:32:10: the loop sent okioki's release to g14 after mesh-delivery-builder's outcome). +func TestAnOutcomeLeavesMovingThePlansToTheLook(t *testing.T) { + l := withLooker(t) + moved := false + advanceAfterOutcome(func() { moved = true }) + if moved { + t.Fatal("the plans were moved on the event loop after an outcome, where a send that retries holds every message") + } + select { + case <-l.signal: + default: + t.Fatal("the look was not asked to move the plans after an outcome") + } +} + +// 1708/AC5: given no look running (a command, which has no event loop), when an outcome was taken, then the plans are +// moved at once, as before. +func TestAnOutcomeMovesThePlansWhereNoLookRuns(t *testing.T) { + lookerMu.Lock() + before := theLooker + theLooker = nil + lookerMu.Unlock() + t.Cleanup(func() { + lookerMu.Lock() + theLooker = before + lookerMu.Unlock() + }) + moved := false + advanceAfterOutcome(func() { moved = true }) + if !moved { + t.Fatal("with no look running, an outcome left the plans unmoved") + } +} diff --git a/cmd/mesh-controller/plan_looks.go b/cmd/mesh-controller/plan_looks.go index ded34bcb..f0098525 100644 --- a/cmd/mesh-controller/plan_looks.go +++ b/cmd/mesh-controller/plan_looks.go @@ -136,6 +136,18 @@ func askToLook() { } } +// askedToLook asks the serving controller's look at the plans, and says whether there was one to ask. +func askedToLook() bool { + lookerMu.RLock() + l := theLooker + lookerMu.RUnlock() + if l == nil { + return false + } + l.ask() + return true +} + // startPlanLooks starts the look at the open plans, and asks for the first: the controller took the lease, and a // plan left by a replaced controller resumes from there (ADR 0406 §6). It replaces the rollout's 30 second ticker. // It answers a channel that is closed once the look has returned after the context ended, for a test that restores what diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 0ddb310b..7112e812 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -761,7 +761,19 @@ func planBuiltSaying(ctx context.Context, open *stores, module, commit, failed s fmt.Printf("%s: %s; the tiers after it are not asked\n", p.ID, p.Note) } } - advanceHeld(ctx, open) + advanceAfterOutcome(func() { advanceHeld(ctx, open) }) +} + +// advanceAfterOutcome moves the plans after an outcome was taken into them: **in the serving controller, by asking +// the look** (novox/hq issue 1708). The outcome is taken on the event loop, and moving the plans may send machines +// their declarations, a release's send retried for minutes among them; done here it held every report, build and +// merge behind it (2026-10-21 14:28:40 to 14:32:10). The look runs off the loop, holding the plans itself. Where no +// look runs (a command, which has no event loop) the plans are moved at once, as before. +func advanceAfterOutcome(advance func()) { + if askedToLook() { + return + } + advance() } // startedBeside is the id of a started walk open beside this one that it is related to — a merge's walk past From 0e0c775bfa5e20e8d1dce0c664c17db8305a26f6 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 15:03:23 +0200 Subject: [PATCH 4/6] Keep a refused registration left to the build records, in the store and the process, so the look fails the plan rather than settling it built; and forget a deleted module and say what was never sent when the records settle (issue 1708) --- .../outcome_off_the_loop_test.go | 60 +++++++- cmd/mesh-controller/release_plan.go | 129 +++++++++++++++++- .../0107-a-refused-registration-is-kept.sql | 10 ++ internal/inventory/refused.go | 33 +++++ internal/inventory/refused_test.go | 28 ++++ testdata/store-tests | 1 + 6 files changed, 257 insertions(+), 4 deletions(-) create mode 100644 internal/inventory/migrations/0107-a-refused-registration-is-kept.sql create mode 100644 internal/inventory/refused.go create mode 100644 internal/inventory/refused_test.go diff --git a/cmd/mesh-controller/outcome_off_the_loop_test.go b/cmd/mesh-controller/outcome_off_the_loop_test.go index 940745d9..6b52e126 100644 --- a/cmd/mesh-controller/outcome_off_the_loop_test.go +++ b/cmd/mesh-controller/outcome_off_the_loop_test.go @@ -2,6 +2,7 @@ package main import ( "context" + "strings" "testing" "time" @@ -70,7 +71,7 @@ func withLooker(t *testing.T) *looker { // at the plans is asked for, so the records settle it once the holder lets go rather than at the next event. func TestAnOutcomeLeftToTheRecordsAsksForTheLook(t *testing.T) { l := withLooker(t) - leaveToRecords("nodered", "build-1708-ac2", "", inventory.ErrPlansBusy) + leaveToRecords("nodered", "build-1708-ac2", "", "", inventory.ErrPlansBusy) select { case <-l.signal: default: @@ -89,7 +90,7 @@ func TestTheRecordsSettleAnOutcomeWithWhatTheBuildDid(t *testing.T) { } byID := map[string]inventory.Build{"build-1708-ac3": {ID: "build-1708-ac3", Module: "nodered", Commit: "4efac366", At: asked.Add(5 * time.Minute)}} - leaveToRecords("nodered", "build-1708-ac3", "reused from build-1708-check", inventory.ErrPlansBusy) + leaveToRecords("nodered", "build-1708-ac3", "reused from build-1708-check", "", inventory.ErrPlansBusy) p := plan() if !settleFromRecords(&p, p.Tiers[0], nil, byID) || p.Modules["nodered"].State != "built" { @@ -140,3 +141,58 @@ func TestAnOutcomeMovesThePlansWhereNoLookRuns(t *testing.T) { t.Fatal("with no look running, an outcome left the plans unmoved") } } + +// 1708/AC6: given a registration the mesh refused (its build recorded as worked, the outcome failed), left to the +// records because the plans were held, when the look settles the plan from the records, then the module and the plan +// are failed, never built: settled as built, the tier would be sent the version from before (reviewer-1708@shanks). +func TestARefusedRegistrationLeftToTheRecordsFailsThePlan(t *testing.T) { + withLooker(t) + asked := time.Date(2026, 10, 21, 12, 15, 0, 0, time.UTC) + p := inventory.Plan{ID: "plan-1708-ac6", State: inventory.PlanBuilding, Tiers: [][]string{{"nodered"}}, + Modules: map[string]*inventory.PlanModule{"nodered": {State: "asked", AskedAt: &asked, Build: "build-1708-ac6"}}} + byID := map[string]inventory.Build{"build-1708-ac6": {ID: "build-1708-ac6", Module: "nodered", Commit: "4efac366", + At: asked.Add(5 * time.Minute)}} + leaveToRecords("nodered", "build-1708-ac6", "", "the mesh does not register it: a port is taken", inventory.ErrPlansBusy) + + overlayRefusals(nil, byID, refusalsKeptFor([]string{"build-1708-ac6"})) + settleFromRecords(&p, p.Tiers[0], nil, byID) + if s := p.Modules["nodered"]; s.State != "failed" || p.State != inventory.PlanFailed { + t.Fatalf("a refused registration left to the records settled as %q, the plan %q", s.State, p.State) + } + if !strings.Contains(p.Modules["nodered"].Why, "a port is taken") { + t.Fatalf("the refusal's words were lost: %q", p.Modules["nodered"].Why) + } +} + +// 1708/AC6: a refusal kept in the store (a controller replaced before the settle) fails the record as one kept in +// the process does. +func TestARefusalFromTheStoreFailsTheRecord(t *testing.T) { + recorded := map[string][]inventory.Build{"nodered": {{ID: "b-store", Module: "nodered"}, {ID: "b-other", Module: "nodered"}}} + byID := map[string]inventory.Build{"b-store": {ID: "b-store", Module: "nodered"}} + overlayRefusals(recorded, byID, map[string]string{"b-store": "not registered"}) + if recorded["nodered"][0].Worked() || byID["b-store"].Worked() { + t.Fatal("a build whose registration was refused still reads as worked") + } + if !recorded["nodered"][1].Worked() { + t.Fatal("a build nobody refused was failed") + } +} + +// 1708/AC7: given the records settle a module as deleted at its source, or fail the plan, when the look settles, then +// it does what taking the outcome directly does: the deleted module is named to be forgotten, and the plan's failure +// is said (reviewer-1708@shanks). +func TestSettlingFromTheRecordsDoesWhatTakingTheOutcomeDoes(t *testing.T) { + p := inventory.Plan{State: inventory.PlanFailed, Modules: map[string]*inventory.PlanModule{ + "gone": {State: planDeleted}, "x": {State: "failed"}, "y": {State: "built"}}} + before := map[string]string{"gone": "asked", "x": "asked", "y": "built"} + deleted, failed := settledEffects(before, &p, inventory.PlanBuilding) + if len(deleted) != 1 || deleted[0] != "gone" { + t.Fatalf("the module the records found deleted is not forgotten: %v", deleted) + } + if !failed { + t.Fatal("the plan the records failed is not said as failed") + } + if d, f := settledEffects(map[string]string{"gone": planDeleted}, &p, inventory.PlanFailed); len(d) != 0 || f { + t.Fatalf("what was deleted or failed before the settle is acted on again: %v %v", d, f) + } +} diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 7112e812..7331626c 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -639,15 +639,52 @@ func holdPlansForOutcome(ctx context.Context, hold func(context.Context, bool) ( // 214). When another act holds the plans it asks for the look, which waits for them off the event loop and settles // the plan from the records once they are let go (issue 1708); what the build did is kept for that settle by the // build's id, so the walk's module says it as it would have (ADR 0350 §4). -func leaveToRecords(module, id, made string, err error) { +func leaveToRecords(module, id, made, failed string, err error) { fmt.Printf("plans: %s's outcome is left to the build records: %v\n", module, err) if !errors.Is(err, inventory.ErrPlansBusy) { return } keepMadeFor(id, made, time.Now()) + // A refused outcome (failed, its build recorded as worked) is kept as refused, or the records settle it built. + keepRefusalFor(id, failed, time.Now()) askToLook() } +// refusalsKeptFor is the refusals kept in this process for these builds (keepRefusalFor), by id. +func refusalsKeptFor(ids []string) map[string]string { + out := map[string]string{} + refusedLeftMu.Lock() + defer refusedLeftMu.Unlock() + for _, id := range ids { + if k, ok := refusedLeft[id]; ok { + out[id] = k.made + } + } + return out +} + +// refusedLeft is why each outcome left to the records was refused, by the build's id: kept in the process beside the +// store's record (RecordRefused), so a store that did not take it still fails the plan while this controller serves. +var ( + refusedLeftMu sync.Mutex + refusedLeft = map[string]madeKept{} +) + +// keepRefusalFor keeps why an outcome left to the records was refused, forgetting refusals older than madeLeftFor. +func keepRefusalFor(id, why string, now time.Time) { + if id == "" || why == "" { + return + } + refusedLeftMu.Lock() + defer refusedLeftMu.Unlock() + for k, v := range refusedLeft { + if now.Sub(v.at) > madeLeftFor { + delete(refusedLeft, k) + } + } + refusedLeft[id] = madeKept{made: why, at: now} +} + // madeLeft is what each build left to the records did, by the build's id, until a settle takes it: kept in the // process, so a controller replaced before the settle loses only these words, never the outcome. var ( @@ -699,7 +736,13 @@ func planBuiltSaying(ctx context.Context, open *stores, module, commit, failed s // records, which the holder settles the plan from (issue 214). release, err := holdPlansForOutcome(ctx, inv.HoldPlans) if err != nil { - leaveToRecords(module, id, made, err) + if failed != "" && id != "" && errors.Is(err, inventory.ErrPlansBusy) { + // In the store too, so a controller replaced before the look settles still fails the plan (issue 1708). + if err := inv.RecordRefused(ctx, id, module, failed); err != nil { + fmt.Printf("plans: %s's refusal could not be kept in the store, only in this controller: %v\n", module, err) + } + } + leaveToRecords(module, id, made, failed, err) return } defer release() @@ -998,7 +1041,17 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, if err != nil { return false, err } + before, was := statesOf(p), p.State if settleFromRecords(p, tier, recorded, byID) { + // As taking the outcome directly does (planBuiltSaying): a module deleted at its source is forgotten, and a + // failed plan says what it built and never sent (issue 1708). + deleted, failed := settledEffects(before, p, was) + for _, m := range deleted { + forgetDeleted(ctx, inv, m, p) + } + if failed { + sayUnsent(p, rollsOut) + } return true, nil } // Asked: wait for every build. @@ -2556,9 +2609,81 @@ func recordsOfAsked(ctx context.Context, inv *inventory.Inventory, p *inventory. } } } + refused, err := refusalsOf(ctx, inv, recorded, byID) + if err != nil { + return nil, nil, err + } + overlayRefusals(recorded, byID, refused) return recorded, byID, nil } +// overlayRefusals marks failed every build record whose registration the mesh refused (novox/hq issue 1708): the record +// keeps the builder's words, which worked, and settled from it as it is the refused build would be built and sent. +func overlayRefusals(recorded map[string][]inventory.Build, byID map[string]inventory.Build, refused map[string]string) { + if len(refused) == 0 { + return + } + fail := func(b *inventory.Build) { + if why, ok := refused[b.ID]; ok && b.Worked() { + b.Failed = why + } + } + for m := range recorded { + for i := range recorded[m] { + fail(&recorded[m][i]) + } + } + for id, b := range byID { + fail(&b) + byID[id] = b + } +} + +// refusalsOf is every refusal kept for the builds these records name, in the store and in this process. +func refusalsOf(ctx context.Context, inv *inventory.Inventory, recorded map[string][]inventory.Build, + byID map[string]inventory.Build) (map[string]string, error) { + var ids []string + for _, bs := range recorded { + for _, b := range bs { + ids = append(ids, b.ID) + } + } + for id := range byID { + ids = append(ids, id) + } + refused, err := inv.RefusedOf(ctx, ids) + if err != nil { + return nil, fmt.Errorf("the refused registrations cannot be read: %w", err) + } + for id, why := range refusalsKeptFor(ids) { + refused[id] = why + } + return refused, nil +} + +// settledEffects is what settling from the records did that needs more than the plan: the modules it found deleted +// at their source, and whether it failed the plan. +func settledEffects(before map[string]string, p *inventory.Plan, was string) (deleted []string, failed bool) { + for m, s := range p.Modules { + if s != nil && s.State == planDeleted && before[m] != planDeleted { + deleted = append(deleted, m) + } + } + sort.Strings(deleted) + return deleted, p.State == inventory.PlanFailed && was != inventory.PlanFailed +} + +// statesOf is each module's state in a plan, for settledEffects. +func statesOf(p *inventory.Plan) map[string]string { + out := map[string]string{} + for m, s := range p.Modules { + if s != nil { + out[m] = s.State + } + } + return out +} + // settleFromRecords marks every module of the tier still `asked` built — or failed — from a build // recorded after it was asked, and says whether it changed anything (novox/hq 04-ISSUES/214). // Newest first, as Builds answers: the first record after the ask is the outcome of that ask. diff --git a/internal/inventory/migrations/0107-a-refused-registration-is-kept.sql b/internal/inventory/migrations/0107-a-refused-registration-is-kept.sql new file mode 100644 index 00000000..f0b606a3 --- /dev/null +++ b/internal/inventory/migrations/0107-a-refused-registration-is-kept.sql @@ -0,0 +1,10 @@ +-- A build's outcome whose registration the mesh refused, by the build's id (novox/hq issue 1708). The build record says +-- what the builder said, which worked; an outcome the event loop could not take into the plans (another act held them) +-- is settled from the records by the look, and without this it would read the refused build as built. Kept in the +-- store so a controller replaced before the settle still fails the plan. +create table build_refused ( + id text primary key, + module text not null, + why text not null, + at timestamptz not null default now() +); diff --git a/internal/inventory/refused.go b/internal/inventory/refused.go new file mode 100644 index 00000000..f07809fb --- /dev/null +++ b/internal/inventory/refused.go @@ -0,0 +1,33 @@ +package inventory + +import "context" + +// RecordRefused keeps that the mesh refused to register a build's outcome, and why (novox/hq issue 1708): the build +// record says the builder's words, which worked, and the look that settles a plan from the records reads this to fail +// it instead. The first refusal of an id stands. +func (i *Inventory) RecordRefused(ctx context.Context, id, module, why string) error { + _, err := i.store.Pool().Exec(ctx, + `insert into build_refused (id, module, why) values ($1, $2, $3) on conflict (id) do nothing`, id, module, why) + return err +} + +// RefusedOf is why each of these builds' registration was refused, by id; a build not refused is absent. +func (i *Inventory) RefusedOf(ctx context.Context, ids []string) (map[string]string, error) { + out := map[string]string{} + if len(ids) == 0 { + return out, nil + } + rows, err := i.store.Pool().Query(ctx, `select id, why from build_refused where id = any($1)`, ids) + if err != nil { + return nil, err + } + defer rows.Close() + for rows.Next() { + var id, why string + if err := rows.Scan(&id, &why); err != nil { + return nil, err + } + out[id] = why + } + return out, rows.Err() +} diff --git a/internal/inventory/refused_test.go b/internal/inventory/refused_test.go new file mode 100644 index 00000000..f38f7ea3 --- /dev/null +++ b/internal/inventory/refused_test.go @@ -0,0 +1,28 @@ +//go:build integration + +package inventory + +import "testing" + +// novox/hq issue 1708 (1708/AC6): a refused registration is kept by its build's id, the first refusal standing, and +// read back for the builds asked about only. +func TestARefusedRegistrationIsKeptByItsBuild(t *testing.T) { + inv := ForTest(t) + ctx := t.Context() + if err := inv.RecordRefused(ctx, "build-1708-a", "nodered", "a port is taken"); err != nil { + t.Fatal(err) + } + if err := inv.RecordRefused(ctx, "build-1708-a", "nodered", "said again"); err != nil { + t.Fatalf("a refusal said twice was refused: %v", err) + } + got, err := inv.RefusedOf(ctx, []string{"build-1708-a", "build-1708-b"}) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got["build-1708-a"] != "a port is taken" { + t.Fatalf("the refusals read back are %v", got) + } + if none, err := inv.RefusedOf(ctx, nil); err != nil || len(none) != 0 { + t.Fatalf("no builds asked about answered %v %v", none, err) + } +} diff --git a/testdata/store-tests b/testdata/store-tests index a16b7f30..93092c0f 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -167,3 +167,4 @@ ./cmd/mesh-controller TestAMergeOnTheApproveIntoANewRepositoryAsksTheFirstBuildAndItsOutcomeRegistersTheModule ./cmd/mesh-controller TestTheApprovedFirstBuildPassesTheGuardOfAVerbAndIsQueued ./cmd/mesh-controller TestAPlanGoesOnWithoutAMachineAwayWhenItsTierIsSent +./internal/inventory TestARefusedRegistrationIsKeptByItsBuild From 6ef78580cefccd16c882bf82a344c60570734444 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 15:04:37 +0200 Subject: [PATCH 5/6] Take the mesh-sdk that holds the container-groups capability rebased on main (issue 1638) --- go.mod | 2 +- go.sum | 4 +- testdata/beside/CAPTURED | 2 +- .../novox/mesh-sdk/go/catalogue/seat_data.go | 3 +- .../mesh-sdk/go/declaration/declaration.go | 13 ++ .../mesh-sdk/go/declaration/filereference.go | 125 ++++++++++++++++++ vendor/modules.txt | 2 +- 7 files changed, 145 insertions(+), 6 deletions(-) create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/declaration/filereference.go diff --git a/go.mod b/go.mod index e15f5dd1..2a80f13d 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-controller go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122745-e0e1a91496de + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021130332-b5d8eb03cea3 github.com/jackc/pgx/v5 v5.10.0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 diff --git a/go.sum b/go.sum index 69cc7def..4def8890 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5 h1:E5stfIeCN6uYgA9TCv+4K8JgaURg3zQppvUgJLaDMP4= git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5/go.mod h1:4wyoCr6IIIIarEUuROThWDj7DAvJvIidqMjWcpSnOwE= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122745-e0e1a91496de h1:0iw6Xg/DzsvjIOGq2NE/YeLULuVG0Dv6eUKKMQS+4ys= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122745-e0e1a91496de/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021130332-b5d8eb03cea3 h1:1LLMeQbvmmVPMQjqo6fFd/WtIx4e6v8TGn55dyB4wps= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021130332-b5d8eb03cea3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/testdata/beside/CAPTURED b/testdata/beside/CAPTURED index 5126bb49..e6c0f90b 100644 --- a/testdata/beside/CAPTURED +++ b/testdata/beside/CAPTURED @@ -9,7 +9,7 @@ a second place to clean besides the catalogue itself. mesh-catalog e0bc919d4945bb5f13f971f74643a0b714cf45da modules/*/module.json, modules/nats/Dockerfile mesh-host 9d9e3a46c4f5943cfffb413d962ce48b75e2fd39 examples/foundation-first-node.lock -mesh-sdk e0e1a91496de28becd8387ac7acad4cdb85fdda2 conformance/events/module-event.json +mesh-sdk b5d8eb03cea33fd78e9962a0f226f46c25d4143f conformance/events/module-event.json To move them, from this repository's root, with the two repositories checked out beside it: diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/seat_data.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/seat_data.go index e623c8a9..90a1fb7c 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/seat_data.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/seat_data.go @@ -66,7 +66,8 @@ var capabilityName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`) // names an `if-capability` may give (novox/hq ADR 0255). A name nothing detects would leave its // contribution out on every machine, silently, so it is refused. A detector added there is added here. var KnownCapabilities = []string{ - "battery", "container-groups", "container-runtime", "firewall", "graphical-session", "overlay", "package-manager", + "battery", "container-groups", "container-runtime", "file-references", "firewall", "graphical-session", "overlay", + "package-manager", "power-meter", "privileged", "seat", "service-manager", "uplink-dhcpcd", "uplink-networkmanager", "uplink-systemd-networkd", "user-namespaces", "virtualisation", } diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/declaration/declaration.go b/vendor/git.novox.be/novox/mesh-sdk/go/declaration/declaration.go index b2fcccf1..60e5521c 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/declaration/declaration.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/declaration/declaration.go @@ -324,6 +324,15 @@ type File struct { // depend on a field somewhere else. Bytes string `json:"bytes,omitempty"` + // Digest and Size say the content is not carried but fetched: a file over FileReferenceThreshold travels as + // the sha256 digest of its content, "sha256:<64 hex>", and its size in bytes, in place of Content or Bytes + // (novox/hq ADR 0486 §1). The node-engine fetches the object of that name from FileBucket, checks what it + // received against Digest and Size, and only then writes the file. Exclusive with Content, Sealed, Bytes, + // VaultSealed and Secrets; sent only to an engine that reports FileReferencesCapability, as an older engine + // refuses a field it does not know, whole. + Digest string `json:"digest,omitempty"` + Size int64 `json:"size,omitempty"` + // Owner is the user this belongs to, by name. Absent means root. Owner string `json:"owner,omitempty"` @@ -465,6 +474,10 @@ func (f *File) validate(where string, _ bool) []string { problems = append(problems, where+": a file the vault sealed whole has no holes for secrets") } } + problems = append(problems, f.checkReference(where)...) + if f.Digest != "" || f.Size != 0 { + said = append(said, "digest") + } if len(said) > 1 { sort.Strings(said) problems = append(problems, where+ diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/declaration/filereference.go b/vendor/git.novox.be/novox/mesh-sdk/go/declaration/filereference.go new file mode 100644 index 00000000..166022a0 --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/declaration/filereference.go @@ -0,0 +1,125 @@ +package declaration + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "fmt" + "regexp" +) + +// The three names of a file travelling by digest (novox/hq ADR 0486 §1, §2, §5), shared by the node-engine that +// reads them and the controller that writes them. +const ( + // FileReferenceThreshold is the size above which a file travels by digest: a file of this many bytes or fewer + // stays in the declaration as it is. + FileReferenceThreshold = 64 * 1024 + // FileBucket is the object bucket of the bus that holds the contents, each under the hex of its sha256 digest + // (ObjectName). The controller alone writes it; a node-engine reads an object by name. + FileBucket = "declaration-files" + // FileReferencesCapability is what a node-engine reports when it reads a file given as a digest and size. + FileReferencesCapability = "file-references" +) + +var digestForm = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) + +// DigestOf is the digest a reference names for a content: "sha256:<64 lower-case hex>". +func DigestOf(content []byte) string { + sum := sha256.Sum256(content) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +// ObjectName is the name of the object holding the content a digest names: the hex without its "sha256:" prefix. +func ObjectName(digest string) (string, error) { + if !digestForm.MatchString(digest) { + return "", fmt.Errorf("digest %q is not sha256:<64 lower-case hex characters>", digest) + } + return digest[len("sha256:"):], nil +} + +// IsReference reports whether the file gives its content as a digest and size instead of carrying it. +func (f *File) IsReference() bool { return f.Digest != "" } + +// checkReference judges the digest and size, and what a reference may not be given with. +func (f *File) checkReference(where string) []string { + if f.Digest == "" && f.Size == 0 { + return nil + } + var problems []string + if !digestForm.MatchString(f.Digest) { + problems = append(problems, fmt.Sprintf("%s: digest %q; a file by reference names its content as "+ + "sha256:<64 lower-case hex characters>", where, f.Digest)) + } + if f.Size <= 0 { + problems = append(problems, fmt.Sprintf("%s: size %d; a file by reference states its size in bytes, "+ + "more than zero", where, f.Size)) + } + if f.Content != "" || f.Bytes != "" || f.Sealed != "" || f.VaultSealed != nil || len(f.Secrets) > 0 { + problems = append(problems, where+": a file by reference carries its content in the object its digest "+ + "names, so it says no content, bytes, sealed, vault-sealed or secrets") + } + if f.Into != "" { + problems = append(problems, where+": a file by reference is written whole; a file written into says its "+ + "part in content") + } + return problems +} + +// Body is the bytes the file's content stands for: its content, or its bytes decoded. +func (f *File) Body() ([]byte, error) { + if f.Bytes == "" { + return []byte(f.Content), nil + } + b, err := base64.StdEncoding.DecodeString(f.Bytes) + if err != nil { + return nil, fmt.Errorf("file %s: bytes are not base64: %w", f.ID, err) + } + return b, nil +} + +// TravelsByDigest reports whether the controller may put this file's content in the bucket: it is plain content +// or bytes (not sealed, not filled with secrets, not written into another file) of more than FileReferenceThreshold +// bytes. +func (f *File) TravelsByDigest() bool { + if f.IsReference() || f.Sealed != "" || f.VaultSealed != nil || len(f.Secrets) > 0 || f.Into != "" { + return false + } + if f.Bytes != "" { + // DecodedLen counts padding, so the exact length is taken off the decoded bytes. + body, err := f.Body() + return err == nil && len(body) > FileReferenceThreshold + } + return len(f.Content) > FileReferenceThreshold +} + +// AsReference is the file as it is sent to an engine that reports FileReferencesCapability: a copy with the digest +// and size in place of the content, and the content the controller puts into the bucket under ObjectName(digest). +// A file that does not travel by digest is refused, not passed through, so a caller cannot send a small file empty. +func (f *File) AsReference() (File, []byte, error) { + if !f.TravelsByDigest() { + return File{}, nil, fmt.Errorf("file %s does not travel by digest", f.ID) + } + body, err := f.Body() + if err != nil { + return File{}, nil, err + } + ref := *f + ref.Content, ref.Bytes = "", "" + ref.Digest, ref.Size = DigestOf(body), int64(len(body)) + return ref, body, nil +} + +// Verify checks what was fetched against the digest and size the file names; the node-engine writes the file only +// after it returns nil. A mismatch is the apply's failure, never a partial apply (ADR 0486 §1). +func (f *File) Verify(body []byte) error { + if !f.IsReference() { + return fmt.Errorf("file %s is not given by reference", f.ID) + } + if int64(len(body)) != f.Size { + return fmt.Errorf("file %s: the object is %d bytes and the declaration says %d", f.ID, len(body), f.Size) + } + if got := DigestOf(body); got != f.Digest { + return fmt.Errorf("file %s: the object hashes to %s and the declaration names %s", f.ID, got, f.Digest) + } + return nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 8c7106d9..9af5fcc8 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,4 +1,4 @@ -# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122745-e0e1a91496de +# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021130332-b5d8eb03cea3 ## explicit; go 1.26.0 git.novox.be/novox/mesh-sdk/go/asks git.novox.be/novox/mesh-sdk/go/catalogue From 2aa291f3b30b7544cba852dd60e17a831f78a574 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 15:16:47 +0200 Subject: [PATCH 6/6] Pin mesh-sdk at its merge of the container-groups capability, on main (issue 1638) --- go.mod | 2 +- go.sum | 4 ++-- testdata/beside/CAPTURED | 2 +- vendor/modules.txt | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 2a80f13d..2c0bb303 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-controller go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021130332-b5d8eb03cea3 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021130958-85b5a2687e6a github.com/jackc/pgx/v5 v5.10.0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 diff --git a/go.sum b/go.sum index 4def8890..6b818e32 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5 h1:E5stfIeCN6uYgA9TCv+4K8JgaURg3zQppvUgJLaDMP4= git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5/go.mod h1:4wyoCr6IIIIarEUuROThWDj7DAvJvIidqMjWcpSnOwE= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021130332-b5d8eb03cea3 h1:1LLMeQbvmmVPMQjqo6fFd/WtIx4e6v8TGn55dyB4wps= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021130332-b5d8eb03cea3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021130958-85b5a2687e6a h1:CVTQBD4vxbGQ+TW9MWAut/ABQN6nvAZ1VTqojNpvGUE= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021130958-85b5a2687e6a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/testdata/beside/CAPTURED b/testdata/beside/CAPTURED index e6c0f90b..7d76163f 100644 --- a/testdata/beside/CAPTURED +++ b/testdata/beside/CAPTURED @@ -9,7 +9,7 @@ a second place to clean besides the catalogue itself. mesh-catalog e0bc919d4945bb5f13f971f74643a0b714cf45da modules/*/module.json, modules/nats/Dockerfile mesh-host 9d9e3a46c4f5943cfffb413d962ce48b75e2fd39 examples/foundation-first-node.lock -mesh-sdk b5d8eb03cea33fd78e9962a0f226f46c25d4143f conformance/events/module-event.json +mesh-sdk 85b5a2687e6a8d0854dd8e53ed68f98e0e9d3c1b conformance/events/module-event.json To move them, from this repository's root, with the two repositories checked out beside it: diff --git a/vendor/modules.txt b/vendor/modules.txt index 9af5fcc8..127b9a13 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,4 +1,4 @@ -# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021130332-b5d8eb03cea3 +# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021130958-85b5a2687e6a ## explicit; go 1.26.0 git.novox.be/novox/mesh-sdk/go/asks git.novox.be/novox/mesh-sdk/go/catalogue