Compare commits

...
Author SHA1 Message Date
jschoubben 82ac613b4e Merge pull request 'Leave the engine's acts out of the bus tools' removals, so the bus's user list is composed again (issue 774)' (#563) from fix/774-nats-tools-no-engine-acts into main
Reviewed-on: #563
2026-10-30 11:38:25 +00:00
jschoubben 787563358b Leave the engine's acts out of the bus tools' removals, so ace's tool runner, which carries the bundle on its own account, is no second writer and the grants part composes the bus's user list again (issue 774)
mesh/check checking 787563358b67 on 82ac613b4ed0
mesh/delivery merged: source.main moved to novox/mesh-catalog@82ac613b4ed0
2026-10-30 12:34:22 +01:00
mesh-tools 8f212e559a Merge pull request 'Design 47's last milestones: the keeper, self-check, healers, hand-acts, provisioner, vault, artifact sweep, disk readings, credential forms (issues 774, 1952, 1953, 1954, 1955)' (#560) from feat/774-l3-reland into main 2026-10-30 11:01:48 +00:00
mesh-tools 1aa22efb0d Merge pull request 'mesh-delivery-gate: an answer stands only for the ask it answers, so the approved release merges (issue 1730)' (#562) from fix/gate-answer-stands-for-its-ask into main 2026-10-30 11:01:15 +00:00
jschoubben d6a069ae49 mesh-delivery-gate: an answer stands only for the ask it answers, and a merge never handed says why
mesh/check being carried to d6a069ae4959 on 1aa22efb0d78 | from 12c2574c9d54
mesh/delivery merged: source.main moved to novox/mesh-catalog@1aa22efb0d78
On 2026-10-29 the group feat/774-vault-mints-l2 was approved and never handed: hq#495 kept a Decline of an earlier ask beside the Approve of the next, readyAsks skipped it and handTurn returned nil for every member, with no log line. The Approve and a new ask now clear an earlier Decline, a run's answer names its ask (answered_ask), an approved merge that waits for nothing and is not handed logs why once and is a stalled line, and delivery_turns lists no unwritten turn while the check gates no merge.
2026-10-30 11:35:35 +01:00
jschoubben f3a57dcc56 mesh-self-check: the whole-run integration test answers the setting probes with the probes it asserts, since the default list is D13 and D-fill
mesh/check checking f3a57dcc56fb on 1aa22efb0d78 | novox/mesh-catalog#560 in group feat/774-l3-reland
mesh/delivery merged: source.main moved to novox/mesh-catalog@8f212e559a3e
mesh/delivery-group group feat/774-l3-reland merged: every member is merged
2026-10-30 10:21:28 +01:00
jschoubben 35ff67bda0 Design 47's last milestones: the keeper, self-check, healers, hand-acts, provisioner, vault, artifact sweep, disk readings, credential forms (issues 774, 1952, 1953, 1954, 1955)
mesh/delivery-group group feat/774-l3-reland rejected: composed together: the checker's verdict at the members' heads (b48c90fea31f): integration: mesh-self-ch…
mesh/delivery closed: a newer head of the same pull request
mesh/check failed: integration: mesh-self-check/cmd/mesh-self-check.TestTheSelfCheckWholeRunsOnTheBus: the run answered {Run:doctor-1793351737-1 At:20…
One commit with exactly the tree of the reviewed head 35b4509c0a, on main 12c2574c9d, so two compiled programs added and removed on the branch leave no trace in history (issue 1577).
2026-10-30 02:21:51 +01:00
331 changed files with 24362 additions and 1792 deletions
+5
View File
@@ -0,0 +1,5 @@
module checks/credentialform
go 1.26.0
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0
+2
View File
@@ -0,0 +1,2 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+70
View File
@@ -0,0 +1,70 @@
// Package credentialform refuses an offer that does not say the form of its credential: an own secret, an issuer, or
// `form` pair or none (the lead, 2026-10-29). It reads every modules/*/module.json through mesh-sdk's
// CredentialFormProblems.
package credentialform
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"sort"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// Problems is every offer of the manifests under root/modules that says no credential form, one line each.
func Problems(root string) ([]string, error) {
files, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
if err != nil {
return nil, err
}
sort.Strings(files)
var out []string
reads := map[string]bool{}
var binders []string
for _, f := range files {
raw, err := os.ReadFile(f)
if err != nil {
return nil, err
}
var m catalogue.Manifest
if err := json.Unmarshal(raw, &m); err != nil {
return nil, fmt.Errorf("%s: %w", f, err)
}
out = append(out, catalogue.CredentialFormProblems(m)...)
var kept struct {
Reads []string `json:"reads"`
State []json.RawMessage `json:"state"`
}
_ = json.Unmarshal(raw, &kept) // reads may hold objects in another module; only the provisioner's are read
if m.Module == "mesh-provisioner" {
for _, r := range kept.Reads {
reads[r] = true
}
}
if len(m.Provides) > 0 && keepsBound(kept.State) {
binders = append(binders, m.Module)
}
}
for _, module := range binders {
if !reads[module+".bound"] {
out = append(out, fmt.Sprintf("%s provides and keeps a `bound` state, and mesh-provisioner's manifest does not read %s.bound", module, module))
}
}
return out, nil
}
// keepsBound says whether a manifest's `state` names `bound`, as a plain name or an object with a name.
func keepsBound(state []json.RawMessage) bool {
for _, s := range state {
var name string
var obj struct {
Name string `json:"name"`
}
if json.Unmarshal(s, &name) == nil && name == "bound" || json.Unmarshal(s, &obj) == nil && obj.Name == "bound" {
return true
}
}
return false
}
+49
View File
@@ -0,0 +1,49 @@
package credentialform
import (
"os"
"path/filepath"
"strings"
"testing"
)
func aCatalogue(t *testing.T, manifests map[string]string) string {
t.Helper()
root := t.TempDir()
for module, body := range manifests {
dir := filepath.Join(root, "modules", module)
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(body), 0o644); err != nil {
t.Fatal(err)
}
}
return root
}
// An offer that says no form is refused and named; one that says pair, none or an own secret passes; and the real
// catalogue is clean.
func TestEveryOfferSaysTheFormOfItsCredential(t *testing.T) {
root := aCatalogue(t, map[string]string{
"silent": `{"module":"silent","provides":[{"name":"thing","scope":"mesh"}]}`,
"binder": `{"module":"binder","provides":[{"name":"d","credential":{"form":"pair"}}],"state":["bound"]}`,
"mesh-provisioner": `{"module":"mesh-provisioner","reads":["other.bound"]}`,
"says": `{"module":"says","provides":[{"name":"a","credential":{"form":"pair"}},{"name":"b","credential":{"form":"none"}},{"name":"c","credential":{"own":"jwt"}}]}`,
})
got, err := Problems(root)
if err != nil {
t.Fatal(err)
}
if len(got) != 2 || !strings.Contains(got[0], `silent provides "thing"`) ||
!strings.Contains(got[1], "binder") || !strings.Contains(got[1], "mesh-provisioner") {
t.Fatalf("refused %v, wanted the silent offer and the binder the provisioner does not read", got)
}
real, err := Problems(filepath.Join("..", ".."))
if err != nil {
t.Fatal(err)
}
if len(real) != 0 {
t.Fatalf("the catalogue has offers with no credential form:\n%s", strings.Join(real, "\n"))
}
}
+3
View File
@@ -0,0 +1,3 @@
# What these tests read outside their package, as paths from the repository's root (issue 1936).
# every manifest, for what each offer says of its credential's form.
modules/*/module.json
+8
View File
@@ -0,0 +1,8 @@
# The calls of the controller's hand-act log a manifest of this catalogue may still invoke, each `<module> <verb> <why>`:
# the fallback of a recorder (mesh-sdk handacts.Recorder) while the seat mesh-hand-acts has no holder that answers, until
# the controller's lease is written to a holder that is no controller. The check of this directory (handacts) refuses any
# other, and an entry whose module no longer invokes it: the list only shrinks, and it is empty before the lease write
# (novox/hq ADR 0565 §11; issue 1953, issue 774).
mesh-config hand-act fallback until the lease write: its sets, clears and held values record through mesh-hand-acts first
mesh-config warranted fallback until the lease write: its warrants record through mesh-hand-acts first
mesh-healers drill fallback until the lease write: a drill records through mesh-hand-acts first
+3
View File
@@ -0,0 +1,3 @@
module git.novox.be/novox/mesh-catalog/checks/handacts
go 1.22
+109
View File
@@ -0,0 +1,109 @@
// Command handacts refuses a manifest that records into the controller's hand-act log (novox/hq ADR 0565 §11): every
// recorder records through the seat mesh-hand-acts from its delivery, so no manifest invokes
// `seat:mesh-controller.hand-act`, `.warranted` or `.drill`. The recorders' fallbacks to the controller's log while the
// seat has no holder are named in checks/handacts/fallbacks, each module and verb with why; that list only shrinks, and
// an entry whose module no longer invokes the verb is refused too.
//
// go run . <catalog root>
//
// Exits 1 on a finding.
package main
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
)
// Verbs are the controller's hand-act verbs, as a manifest invokes them, and Allowed the list of named fallbacks.
var Verbs = []string{"hand-act", "warranted", "drill"}
const (
invokePrefix = "seat:mesh-controller."
Allowed = "checks/handacts/fallbacks"
)
// Findings is every refusal in the catalog at root, sorted.
func Findings(root string) ([]string, error) {
allowed := map[string]bool{}
raw, err := os.ReadFile(filepath.Join(root, Allowed))
if err != nil && !os.IsNotExist(err) {
return nil, err
}
for _, line := range strings.Split(string(raw), "\n") {
if f := strings.Fields(line); len(f) >= 3 && !strings.HasPrefix(f[0], "#") {
allowed[f[0]+" "+f[1]] = true
} else if len(f) > 0 && !strings.HasPrefix(f[0], "#") {
return nil, fmt.Errorf("%s: %q is not `<module> <verb> <why>`", Allowed, line)
}
}
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
if err != nil {
return nil, err
}
var out []string
invoked := map[string]bool{}
for _, path := range manifests {
raw, err := os.ReadFile(path)
if err != nil {
return nil, err
}
var m struct {
Module string `json:"module"`
Invokes []string `json:"invokes"`
}
if json.Unmarshal(raw, &m) != nil || m.Module == "mesh-controller" {
continue
}
for _, inv := range m.Invokes {
verb, ok := strings.CutPrefix(strings.TrimSpace(inv), invokePrefix)
if !ok || !contains(Verbs, verb) {
continue
}
invoked[m.Module+" "+verb] = true
if !allowed[m.Module+" "+verb] {
out = append(out, fmt.Sprintf("%s invokes %s%s: record through seat:mesh-hand-acts.%s (novox/hq ADR 0565 §11), "+
"or name the fallback in %s", m.Module, invokePrefix, verb, verb, Allowed))
}
}
}
for entry := range allowed {
if !invoked[entry] {
out = append(out, fmt.Sprintf("%s names %q, which is no longer invoked: take it off the list, which only shrinks",
Allowed, entry))
}
}
sort.Strings(out)
return out, nil
}
func contains(list []string, s string) bool {
for _, x := range list {
if x == s {
return true
}
}
return false
}
func main() {
if len(os.Args) != 2 {
fmt.Fprintln(os.Stderr, "usage: handacts <catalog root>")
os.Exit(2)
}
found, err := Findings(os.Args[1])
if err != nil {
fmt.Println(err)
os.Exit(1)
}
for _, f := range found {
fmt.Println(f)
}
if len(found) > 0 {
os.Exit(1)
}
fmt.Println("no manifest records into the controller's hand-act log but the fallbacks named")
}
+59
View File
@@ -0,0 +1,59 @@
package main
import (
"os"
"path/filepath"
"strings"
"testing"
)
// No manifest invokes the controller's hand-act verbs but a named fallback; a named fallback no longer invoked is
// refused (novox/hq ADR 0565 §11's catalog test).
func TestFindings(t *testing.T) {
for _, row := range []struct {
name string
files map[string]string
want string // empty: clean
}{
{"clean", map[string]string{"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-hand-acts.hand-act"]}`}, ""},
{"an unnamed call", map[string]string{"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-controller.warranted"]}`},
"a invokes seat:mesh-controller.warranted"},
{"a named fallback, and the controller's own", map[string]string{
"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-controller.drill"]}`,
"modules/mesh-controller/module.json": `{"module":"mesh-controller","invokes":["seat:mesh-controller.hand-act"]}`,
Allowed: "# why\na drill fallback until the lease write\n"}, ""},
{"a fallback no longer invoked", map[string]string{Allowed: "gone hand-act fallback\n"}, `names "gone hand-act"`},
{"another verb of the controller", map[string]string{"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-controller.conditions"]}`}, ""},
} {
t.Run(row.name, func(t *testing.T) {
root := t.TempDir()
for path, text := range row.files {
p := filepath.Join(root, path)
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(p, []byte(text), 0o644); err != nil {
t.Fatal(err)
}
}
got, err := Findings(root)
if err != nil {
t.Fatal(err)
}
if all := strings.Join(got, "\n"); (row.want == "") != (all == "") || !strings.Contains(all, row.want) {
t.Fatalf("findings = %q, want one containing %q", all, row.want)
}
})
}
}
// The run of this check over the real tree.
func TestTheCatalogRecordsOnlyThroughTheSeat(t *testing.T) {
got, err := Findings(filepath.Join("..", ".."))
if err != nil {
t.Fatal(err)
}
for _, f := range got {
t.Error(f)
}
}
+3
View File
@@ -0,0 +1,3 @@
# What these tests read outside their package, as paths from the repository's root (issue 1936).
# every manifest, for what it invokes.
modules/*/module.json
+1 -1
View File
@@ -2,4 +2,4 @@ module checks/health
go 1.26.0
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0
+2 -2
View File
@@ -1,2 +1,2 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+4
View File
@@ -0,0 +1,4 @@
# The modules of this catalogue that still read the controller's assignments (mesh-controller.assignments), each with
# why. The check of this directory (noplacement) refuses any other module that reads them, and an entry here whose module no longer
# does: the list only shrinks, and it is empty before the controller's lease is written to a holder that is no
# controller (novox/hq ADR 0549 §3, §6; issue 774).
+4
View File
@@ -1,3 +1,7 @@
module git.novox.be/novox/mesh-catalog/checks/noplacement
go 1.22
require git.novox.be/novox/mesh-catalog/checks/trunk v0.0.0
replace git.novox.be/novox/mesh-catalog/checks/trunk => ../trunk
+73 -2
View File
@@ -8,7 +8,13 @@
// 2. a `particulars` file at the root (the mesh's node names come from the checker's run, §16);
//
// 3. a manifest naming a state, stream, event or bucket of the indexer's placement (`mesh-delivery-indexer.placement…`
// or `mesh-delivery-indexer_placement…`) in any string value, such as `reads`.
// or `mesh-delivery-indexer_placement…`) in any string value, such as `reads`;
//
// 4. a manifest, other than mesh-controller's, that reads the controller's assignments (`mesh-controller.assignments`):
// every reader takes placement from the seat mesh-placement (novox/hq ADR 0549 §3, ADR 0567), and the lease write is
// refused while one still reads them (ADR 0549 §6; issue 774). The modules that still read them for what the
// placement does not carry are named in checks/noplacement/assignments-readers, each with why; that list only
// shrinks: an entry whose module no longer reads them is refused too.
//
// go run . <catalogue root>
//
@@ -55,9 +61,74 @@ func Findings(root string) ([]string, error) {
out = append(out, fmt.Sprintf("%s: names %q, which the indexer no longer keeps; placement is the seat mesh-placement's (novox/hq ADR 0567 §12)", module, s))
}
}
readers, err := assignmentsReaders(root)
if err != nil {
return nil, err
}
out = append(out, readers...)
sort.Strings(out)
return out, nil
}
// AssignmentsRead is the controller's assignments as a manifest reads them, and Allowed the file, relative to the
// catalogue's root, that names the modules still reading them: `<module> <why>` per line, `#` a comment.
const (
AssignmentsRead = "mesh-controller.assignments"
Allowed = "checks/noplacement/assignments-readers"
)
// assignmentsReaders is every finding on the reads of the controller's assignments: a module other than the controller
// reading them that Allowed does not name, and a module Allowed names that no longer reads them (issue 774).
func assignmentsReaders(root string) ([]string, error) {
allowed := map[string]bool{}
raw, err := os.ReadFile(filepath.Join(root, Allowed))
if err != nil && !os.IsNotExist(err) {
return nil, err
}
for _, line := range strings.Split(string(raw), "\n") {
if f := strings.Fields(line); len(f) > 0 && !strings.HasPrefix(f[0], "#") {
allowed[f[0]] = true
}
}
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
if err != nil {
return nil, err
}
var found []string
reads := map[string]bool{}
for _, path := range manifests {
raw, err := os.ReadFile(path)
if err != nil {
return nil, err
}
var m struct {
Module string `json:"module"`
Reads []string `json:"reads"`
}
if json.Unmarshal(raw, &m) != nil || m.Module == "mesh-controller" {
continue // a manifest that does not read is the manifest checks' to refuse
}
for _, r := range m.Reads {
if strings.TrimSpace(r) != AssignmentsRead {
continue
}
reads[m.Module] = true
if !allowed[m.Module] {
found = append(found, fmt.Sprintf("%s reads %s: read where modules run from mesh-delivery-assigner.placement, "+
"the seat mesh-placement's (novox/hq ADR 0549 §3, ADR 0567); the lease write is refused while a module reads "+
"the assignments (ADR 0549 §6)", m.Module, AssignmentsRead))
}
}
}
for module := range allowed {
if !reads[module] {
found = append(found, fmt.Sprintf("%s names %s, which no longer reads %s: take it off the list, which only shrinks",
Allowed, module, AssignmentsRead))
}
}
return found, nil
}
// named collects the strings of a decoded manifest that start with one of the Prefixes.
func named(v any, into []string) []string {
switch v := v.(type) {
@@ -95,5 +166,5 @@ func main() {
if len(found) > 0 {
os.Exit(1)
}
fmt.Println("no placement directory, and no manifest reads the indexer's placement")
fmt.Println("no placement directory, no manifest reads the indexer's placement, and only the modules listed read the controller's assignments")
}
+34 -1
View File
@@ -5,6 +5,8 @@ import (
"path/filepath"
"strings"
"testing"
"git.novox.be/novox/mesh-catalog/checks/trunk"
)
func TestFindingsRefuseWhatThePlacementPathLeft(t *testing.T) {
@@ -13,12 +15,18 @@ func TestFindingsRefuseWhatThePlacementPathLeft(t *testing.T) {
files map[string]string
want string // empty: clean
}{
{"clean", map[string]string{"modules/a/module.json": `{"module":"a","reads":["mesh-controller.assignments"]}`}, ""},
{"clean", map[string]string{"modules/a/module.json": `{"module":"a","reads":["mesh-delivery-assigner.placement"]}`}, ""},
{"a placement directory", map[string]string{"placement/ace.json": `{}`}, "placement: the repository holds no placement"},
{"a particulars file", map[string]string{"particulars": "ace\n"}, "particulars: the repository lists no node names"},
{"a read of the indexer's placement", map[string]string{"modules/a/module.json": `{"module":"a","reads":["mesh-delivery-indexer.placement"]}`}, `a: names "mesh-delivery-indexer.placement"`},
{"a read of its candidates", map[string]string{"modules/b/module.json": `{"module":"b","reads":["mesh-delivery-indexer.placement-candidates"]}`}, "placement-candidates"},
{"its event", map[string]string{"modules/c/module.json": `{"module":"c","on":[{"event":"mesh-delivery-indexer.placement-changed"}]}`}, "placement-changed"},
// Issue 774 (ADR 0549 §3, §6): a reader of the controller's assignments is refused unless the list names it, the
// controller's own manifest is not judged, and a name on the list whose module no longer reads them is refused.
{"an unlisted reader of the assignments", map[string]string{"modules/r/module.json": `{"module":"r","reads":["mesh-controller.assignments"]}`}, "r reads mesh-controller.assignments"},
{"a listed reader and the controller", map[string]string{"modules/l/module.json": `{"module":"l","reads":["mesh-controller.assignments"]}`,
"modules/mesh-controller/module.json": `{"module":"mesh-controller","reads":["mesh-controller.assignments"]}`, Allowed: "# why\nl its plan\n"}, ""},
{"a listed module that reads them no more", map[string]string{Allowed: "gone no longer reads\n"}, "names gone, which no longer reads"},
{"the seat's own state is allowed", map[string]string{"modules/d/module.json": `{"module":"d","reads":["mesh-placement.placement"]}`}, ""},
} {
t.Run(row.name, func(t *testing.T) {
@@ -52,4 +60,29 @@ func TestTheCatalogueHoldsNoPlacementOfTheRepositoryPath(t *testing.T) {
for _, f := range got {
t.Error(f)
}
// Issue 774 (ADR 0549 §3, §6; the comparison of novox/hq ADR 0529 clause 2): the list of modules still reading the
// controller's assignments gains none the trunk's copy at the merge base does not hold. Its first copy gains nothing.
path, ok := trunk.Copy(t, Allowed)
if !ok {
return
}
names := func(path string) map[string]bool {
raw, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
out := map[string]bool{}
for _, line := range strings.Split(string(raw), "\n") {
if f := strings.Fields(line); len(f) > 0 && !strings.HasPrefix(f[0], "#") {
out[f[0]] = true
}
}
return out
}
base := names(path)
for module := range names(filepath.Join("..", "..", Allowed)) {
if !base[module] {
t.Errorf("%s gains %s, which the trunk's copy does not hold: the list only shrinks", Allowed, module)
}
}
}
@@ -0,0 +1,129 @@
package main
// The artifact store's door and the bus, as the holder reaches them through this module's runtime.
import (
"context"
"encoding/json"
"fmt"
"os"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"artifactstoretools/internal/registry"
)
// door is registry.Store at the store's address as this machine reaches it.
type door struct{ s registry.Store }
func (d *door) Address() string { return d.s.Address }
func (d *door) Hold(ctx context.Context, r string) (bool, error) { return d.s.Hold(ctx, r) }
func (d *door) Holds(ctx context.Context, r string) error { return d.s.Holds(ctx, r) }
func (d *door) IsIndex(ctx context.Context, r string) (bool, error) {
return d.s.IsIndex(ctx, r)
}
func (d *door) LetGo(ctx context.Context, r string) error { return d.s.LetGo(ctx, r) }
func (d *door) SpareKept(ctx context.Context, keptImages, touching []string) error {
spare, err := d.s.SpareKeptIndexes(ctx, keptImages, touching)
if err == nil {
d.s.Spare = spare
}
return err
}
// DoorAt reads the store's address (host:port) from the file the manifest fills from the seat mesh-artifact-store's
// reach, each time, so a store that moved is reached where it is now.
func DoorAt(file string) func() (Door, error) {
return func() (Door, error) {
if file == "" {
return nil, fmt.Errorf("this bundle was given no file naming the artifact store's address")
}
raw, err := os.ReadFile(file)
if err != nil {
return nil, fmt.Errorf("the artifact store's address: %w", err)
}
address := strings.TrimSpace(string(raw))
if address == "" {
return nil, fmt.Errorf("the artifact store's address is empty in %s", file)
}
return &door{s: registry.Store{Address: address}}, nil
}
}
// runtimeBus is the holder's bus through the runtime: its reads, its findings and its seat's event.
type runtimeBus struct{}
func (runtimeBus) Keys(state string) ([]string, error) { return stdio.State(state).Keys() }
func (runtimeBus) Get(state, key string) ([]byte, bool, error) {
e, err := stdio.State(state).Get(key)
if err != nil || e == nil {
return nil, false, err
}
return e.Value, true, nil
}
func (runtimeBus) History(state, key string) ([]stdio.StateRevision, error) {
return stdio.State(state).History(key)
}
func (runtimeBus) LogSince(read string, at time.Time) ([]stdio.LogEntry, error) {
var out []stdio.LogEntry
page, err := stdio.Log(read).Since(at, "", 0)
for err == nil {
out = append(out, page.Entries...)
if page.Next == 0 {
return out, nil
}
page, err = stdio.Log(read).Read(page.Next, "", 0)
}
return nil, err
}
func (runtimeBus) Put(state, key string, value any) error {
_, err := stdio.State(state).Put(key, value)
return err
}
func (runtimeBus) Delete(state, key string) error { return stdio.State(state).Delete(key) }
func (runtimeBus) Publish(subject string, body any) error {
return stdio.SeatPublish(subject, body, "")
}
// follow watches the builder's done keys and the wants: a build's end sweeps (once the lease names no controller) and
// observes again; a want changed observes again. Event-driven: nothing reads them on a timer.
func follow(h *Holder) {
<-stdio.Served()
watch := func(state string, handle func(stdio.StateChange) error) {
for {
if err := stdio.State(state).Watch("", handle); err == nil {
return
} else {
h.logf("watching %s: %v; asked again in 10s", state, err)
}
time.Sleep(10 * time.Second)
}
}
observe := func() {
if err := h.Observe(context.Background()); err != nil {
h.logf("image-missing could not be observed: %v", err)
}
}
watch(BuildsRead, func(c stdio.StateChange) error {
if c.Current || c.Op != stdio.OpPut {
return nil
}
var b struct {
State string `json:"state"`
}
if json.Unmarshal(c.Value, &b) == nil && b.State == "done" {
go func() { h.AfterBuild(c.Key); observe() }()
}
return nil
})
watch(WantsRead, func(c stdio.StateChange) error {
if !c.Current {
go observe()
}
return nil
})
observe()
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,260 @@
//go:build integration
package main
// The seat mesh-artifact-sweep's holder whole against a real bus (novox/hq ADR 0571 clause 2, issue 1955): the builder's done
// keys, the wants and the controller's lease in a real nats-server's key-value buckets, a build's end heard by a watch
// of the builds as the runtime hands it over, each deletion recorded by a request to the seat mesh-hand-acts on the bus,
// artifacts-gone said on the seat's subject, and S25's finding written to the holder's own bucket. Only the store's
// door is in memory: its HTTP is the copied registry package's, tested there.
import (
"context"
"encoding/json"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/handacts"
"git.novox.be/novox/mesh-sdk/go/testbus"
)
// realBus is the holder's Bus over a real nats-server: each read state a bucket named as the runtime names it.
type realBus struct {
ctx context.Context
nc *nats.Conn
js jetstream.JetStream
// log is the builder's log as the runtime's log verbs would read it (mesh-tools adds them; the sdk's own read of a
// log is tested in mesh-sdk's process package).
log []stdio.LogEntry
}
func (b realBus) LogSince(_ string, at time.Time) ([]stdio.LogEntry, error) {
var out []stdio.LogEntry
for _, e := range b.log {
if !e.At.Before(at) {
out = append(out, e)
}
}
return out, nil
}
func bucketOf(state string) string { return strings.Replace(state, ".", "_", 1) }
func (b realBus) kv(state string) (jetstream.KeyValue, error) {
return b.js.KeyValue(b.ctx, bucketOf(state))
}
func (b realBus) Keys(state string) ([]string, error) {
kv, err := b.kv(state)
if err != nil {
return nil, err
}
keys, err := kv.Keys(b.ctx)
if err == jetstream.ErrNoKeysFound {
return nil, nil
}
return keys, err
}
func (b realBus) Get(state, key string) ([]byte, bool, error) {
kv, err := b.kv(state)
if err != nil {
return nil, false, err
}
e, err := kv.Get(b.ctx, key)
if err == jetstream.ErrKeyNotFound {
return nil, false, nil
}
if err != nil {
return nil, false, err
}
return e.Value(), true, nil
}
func (b realBus) History(state, key string) ([]stdio.StateRevision, error) {
kv, err := b.kv(state)
if err != nil {
return nil, err
}
h, err := kv.History(b.ctx, key)
if err != nil {
return nil, err
}
var out []stdio.StateRevision
for _, e := range h {
out = append(out, stdio.StateRevision{Key: key, Value: e.Value(), Revision: e.Revision(), At: e.Created()})
}
return out, nil
}
func (b realBus) Put(state, key string, value any) error {
kv, err := b.kv(state)
if err != nil {
return err
}
raw, _ := json.Marshal(value)
_, err = kv.Put(b.ctx, key, raw)
return err
}
func (b realBus) Delete(state, key string) error {
kv, err := b.kv(state)
if err != nil {
return err
}
return kv.Delete(b.ctx, key)
}
func (b realBus) Publish(subject string, body any) error {
raw, _ := json.Marshal(body)
return b.nc.Publish(subject, raw)
}
func TestTheSeatSweepsAtABuildsEndOnARealBusOnceTheLeaseIsAway(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
nc, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
defer nc.Close()
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
for _, s := range []string{BuildsRead, WantsRead, HavesRead, LeaseRead, ManifestsRead, "artifact-store-tools." + FindingsState} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucketOf(s), History: 5}); err != nil {
t.Fatal(err)
}
}
bus := realBus{ctx: ctx, nc: nc, js: js}
// The mesh as aMesh lays it, copied onto the real buckets: letta's seven builds, a want, the store's listing.
mem, memb, door, _ := aMesh(t, &buscompose.LeaseHolder{Instance: "c1", Host: "novox", Epoch: 3})
bus.log = memb.log
for _, s := range []string{BuildsRead, WantsRead, LeaseRead, ManifestsRead} {
keys, _ := memb.Keys(s)
for _, k := range keys[:min(len(keys), 6)] { // the seventh build ends later, on the bus
raw, _, _ := memb.Get(s, k)
if err := bus.Put(s, k, json.RawMessage(raw)); err != nil {
t.Fatal(err)
}
}
}
time.Sleep(20 * time.Millisecond) // each build's end is its key's time on the bus: the seventh the newest
// The seat mesh-hand-acts, answering every act it records; artifacts-gone heard on the seat's subject.
var mu sync.Mutex
var acts, gone []string
if _, err := nc.Subscribe("mesh.seat."+handacts.Seat+".tool.hand-act", func(m *nats.Msg) {
mu.Lock()
acts = append(acts, string(m.Data))
mu.Unlock()
_ = m.Respond([]byte(`{"ok":true}`))
}); err != nil {
t.Fatal(err)
}
if _, err := nc.Subscribe(GoneSubject, func(m *nats.Msg) {
mu.Lock()
gone = append(gone, string(m.Data))
mu.Unlock()
}); err != nil {
t.Fatal(err)
}
h := &Holder{Bus: findingsAt{realBus: bus}, Door: mem.Door, Listed: mem.Listed, Now: time.Now, Logf: t.Logf}
h.Acts = handacts.Recorder{LeaseAway: h.LeaseAway, Ask: func(key string, body any) (json.RawMessage, error) {
raw, _ := json.Marshal(body)
m, err := nc.Request("mesh.seat."+strings.TrimPrefix(strings.Replace(key, ".", ".tool.", 1), "seat:"), raw, 5*time.Second)
if err != nil {
return nil, err
}
return m.Data, nil
}}
// A build's end heard from the builds bucket, as follow hands it over.
kv, _ := bus.kv(BuildsRead)
w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
if err != nil {
t.Fatal(err)
}
ended := func() {
keys, _ := memb.Keys(BuildsRead)
raw, _, _ := memb.Get(BuildsRead, keys[6])
if err := bus.Put(BuildsRead, keys[6], json.RawMessage(raw)); err != nil {
t.Fatal(err)
}
select {
case e := <-w.Updates():
h.AfterBuild(e.Key())
case <-ctx.Done():
t.Fatal("the build's end was not heard")
}
}
// While the lease names the controller, a build's end lets nothing go.
ended()
if len(door.asked) != 0 {
t.Fatalf("before the lease write the store was asked %v", door.asked)
}
// The lease handed away: the next build's end holds the kept archives and lets go of letta's 2nd, which the 7th moved
// past the five newest and no want names (the want keeps the 1st), recording each deletion.
if err := bus.Put(LeaseRead, buscompose.ControllerLeaseKey, buscompose.LeaseHolder{HandedTo: "mesh-grants", Why: "the lease write"}); err != nil {
t.Fatal(err)
}
ended()
deadline := time.Now().Add(5 * time.Second)
for {
mu.Lock()
n := len(gone)
mu.Unlock()
if n > 0 || time.Now().After(deadline) {
break
}
time.Sleep(20 * time.Millisecond)
}
mu.Lock()
defer mu.Unlock()
if len(acts) != 3 || !strings.Contains(acts[0], archiveRef("letta", 2)) || !strings.Contains(acts[1], imageRef("letta", 2)) ||
!strings.Contains(acts[2], mirror(1)) {
t.Fatalf("the hand acts recorded are %v; the store was asked %v", acts, door.asked)
}
if len(gone) != 1 || !strings.Contains(gone[0], `"fingerprint":"`+fp(2)+`"`) {
t.Fatalf("artifacts-gone said %v", gone)
}
// S25 through the real findings bucket: letta's wanted image missing from the store is a finding there.
view, _ := mem.Listed(ctx)
repo, digest, _ := splitReference(imageRef("letta", 1))
delete(view.L.Repos[repo].Revisions, digest)
if err := h.Observe(ctx); err != nil {
t.Fatal(err)
}
if keys, _ := bus.Keys("artifact-store-tools." + FindingsState); len(keys) != 1 || keys[0] != "module.letta.ace.image-missing" {
t.Fatalf("the findings are %v", keys)
}
}
// findingsAt reads and writes the holder's own findings in its bucket, artifact-store-tools_findings.
type findingsAt struct{ realBus }
func (f findingsAt) own(state string) string {
if state == FindingsState {
return Module + "." + state
}
return state
}
func (f findingsAt) Keys(state string) ([]string, error) { return f.realBus.Keys(f.own(state)) }
func (f findingsAt) Get(state, key string) ([]byte, bool, error) {
return f.realBus.Get(f.own(state), key)
}
func (f findingsAt) Put(state, key string, value any) error {
return f.realBus.Put(f.own(state), key, value)
}
func (f findingsAt) Delete(state, key string) error { return f.realBus.Delete(f.own(state), key) }
@@ -0,0 +1,351 @@
package main
import (
"context"
"encoding/json"
"fmt"
"sort"
"strings"
"sync"
"testing"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/handacts"
"git.novox.be/novox/mesh-sdk/go/health"
"artifactstoretools/internal/registry"
)
// memBus is the holder's bus in memory: states by name, each key's history, and what was published.
type memBus struct {
mu sync.Mutex
states map[string]map[string][]byte
history map[string][]stdio.StateRevision
log []stdio.LogEntry
published []string
}
func (b *memBus) LogSince(read string, at time.Time) ([]stdio.LogEntry, error) {
var out []stdio.LogEntry
for _, e := range b.log {
if read == delivery.BuildLogRead && !e.At.Before(at) {
out = append(out, e)
}
}
return out, nil
}
func (b *memBus) Keys(state string) ([]string, error) {
b.mu.Lock()
defer b.mu.Unlock()
var out []string
for k := range b.states[state] {
out = append(out, k)
}
sort.Strings(out)
return out, nil
}
func (b *memBus) Get(state, key string) ([]byte, bool, error) {
b.mu.Lock()
defer b.mu.Unlock()
v, ok := b.states[state][key]
return v, ok, nil
}
func (b *memBus) History(state, key string) ([]stdio.StateRevision, error) {
return b.history[state+" "+key], nil
}
func (b *memBus) Put(state, key string, value any) error {
raw, _ := json.Marshal(value)
b.mu.Lock()
defer b.mu.Unlock()
if b.states[state] == nil {
b.states[state] = map[string][]byte{}
}
b.states[state][key] = raw
return nil
}
func (b *memBus) Delete(state, key string) error {
b.mu.Lock()
defer b.mu.Unlock()
delete(b.states[state], key)
return nil
}
func (b *memBus) Publish(subject string, body any) error {
raw, _ := json.Marshal(body)
b.mu.Lock()
defer b.mu.Unlock()
b.published = append(b.published, subject+" "+string(raw))
return nil
}
// memDoor is the store's door in memory: what it holds, which references are indexes, and what was asked.
type memDoor struct {
holds map[string]bool
indexes map[string]bool
asked []string
}
func (d *memDoor) Address() string { return "10.0.0.5:5000" }
func (d *memDoor) Hold(_ context.Context, r string) (bool, error) {
d.asked = append(d.asked, "hold "+r)
if !d.holds[r] {
return false, registry.Gone
}
return false, nil
}
func (d *memDoor) Holds(_ context.Context, r string) error {
if !d.holds[r] {
return registry.Gone
}
return nil
}
func (d *memDoor) IsIndex(_ context.Context, r string) (bool, error) { return d.indexes[r], nil }
func (d *memDoor) LetGo(_ context.Context, r string) error {
d.asked = append(d.asked, "let go "+r)
delete(d.holds, r)
return nil
}
func (d *memDoor) SpareKept(context.Context, []string, []string) error {
d.asked = append(d.asked, "spare")
return nil
}
// mirror is a copy of an upstream base in the store, the n-th.
func mirror(n int) string {
return "artifact-store://upstream/docker.io/library/alpine@sha256:" + strings.Repeat(fmt.Sprintf("%x", n%16), 64)
}
// pinnedImage is the image letta's manifest names by itself, at the store's address.
const pinnedImage = "10.0.0.5:5000/letta/pinned@sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
func digestOf(ref string) string { _, d, _ := splitReference(ref); return d }
func fp(n int) string { return "src2:" + strings.Repeat(fmt.Sprintf("%x", n%16), 64) }
func imageRef(module string, n int) string {
return "artifact-store://" + module + "/image@sha256:" + strings.Repeat(fmt.Sprintf("%x", n%16), 64)
}
func archiveRef(module string, n int) string {
return "artifact-store://" + module + "/files/blobs/sha256:" + strings.Repeat(fmt.Sprintf("%x", (n+8)%16), 64)
}
// A mesh with seven done builds of letta, the n-th ending n hours after a day began, each with an image and an archive;
// a want names the first on ace; the store lists them all and one image no build names; the lease as given.
func aMesh(t *testing.T, lease *buscompose.LeaseHolder) (*Holder, *memBus, *memDoor, *[]string) {
t.Helper()
day := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
b := &memBus{states: map[string]map[string][]byte{}, history: map[string][]stdio.StateRevision{}}
door := &memDoor{holds: map[string]bool{}, indexes: map[string]bool{}}
listing := &Listing{Blobs: map[string]int64{}, Repos: map[string]*Repo{}}
for n := 1; n <= 7; n++ {
key, _ := delivery.BuilderBuildKey("letta", fp(n))
raw, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: strings.Repeat("d", 64), Artifacts: []delivery.BuiltArtifact{
{Name: "image", Kind: "image", Digest: digestOf(imageRef("letta", n)), Reference: imageRef("letta", n)},
{Name: "files", Kind: "archive", Digest: digestOf(archiveRef("letta", n)), Reference: archiveRef("letta", n)},
{Name: "pkg", Kind: "package", Digest: "sha256:" + strings.Repeat("f", 64), Reference: "letta@1.0." + fmt.Sprint(n)}}})
if err != nil {
t.Fatal(err)
}
_ = b.Put(BuildsRead, key, json.RawMessage(raw))
b.history[BuildsRead+" "+key] = []stdio.StateRevision{{Key: key, Value: raw, At: day.Add(time.Duration(n) * time.Hour)}}
for _, r := range []string{imageRef("letta", n), archiveRef("letta", n)} {
door.holds[r] = true
repo, digest, kind := splitReference(r)
if kind == "archive" {
listing.Blobs[digest] = 10
} else {
listing.repo(repo).Revisions[digest] = true
}
}
}
listing.repo("letta/image").Revisions["sha256:"+strings.Repeat("e", 64)] = true // no build names it
listing.repo("letta/pinned").Revisions["sha256:"+strings.Repeat("c", 64)] = true
// Three copies of a base: letta's 2nd stood on the first, its 7th on the second, and no build on the third.
for n := 1; n <= 3; n++ {
repo, digest, _ := splitReference(mirror(n))
listing.repo(repo).Revisions[digest] = true
door.holds[mirror(n)] = true
}
for n := 1; n <= 7; n++ {
e := delivery.BuildLogEntry{Event: delivery.BuildEndedEvent, Module: "letta", FP: fp(n), Repository: "novox/x", Commit: "c",
Node: "ace", At: day.Add(time.Duration(n) * time.Hour), Outcome: "done"}
switch n {
case 2:
e.Against = []string{mirror(1)}
case 7:
e.Against = []string{mirror(2)}
}
raw, _ := delivery.EncodeBuildLogEntry(e)
b.log = append(b.log, stdio.LogEntry{Sequence: uint64(n), Key: "letta", Value: raw, At: e.At})
}
manifest, _ := delivery.EncodeManifest(delivery.Manifest{Module: "letta",
JSON: `{"module":"letta","resources":[{"id":"app","type":"container","image":"` + pinnedImage + `"},` +
`{"id":"built","type":"container","image":"${artifact:image}"}]}`})
mk, _ := delivery.ManifestsKey(strings.Repeat("d", 64))
_ = b.Put(ManifestsRead, mk, json.RawMessage(manifest))
want, _ := json.Marshal(delivery.Want{V: delivery.Version, FP: fp(1), Self: true})
_ = b.Put(WantsRead, "want.ace.letta", json.RawMessage(want))
if lease != nil {
_ = b.Put(LeaseRead, buscompose.ControllerLeaseKey, lease)
}
var acts []string
h := &Holder{Bus: b, Door: func() (Door, error) { return door, nil },
Listed: func(context.Context) (*View, error) { return &View{L: listing}, nil },
Now: func() time.Time { return day.Add(48 * time.Hour) },
Acts: handacts.Recorder{Ask: func(key string, body any) (json.RawMessage, error) {
raw, _ := json.Marshal(body)
acts = append(acts, key+" "+string(raw))
return json.RawMessage(`{}`), nil
}}}
h.Acts.LeaseAway = h.LeaseAway
return h, b, door, &acts
}
// ADR 0571 clause 2: the keep set from the store's listing, the wants and the five newest builds; nothing let go of while
// the lease names a controller; once it names another holder, a build's end lets go of the eligible within the bound,
// leaves an index for a confirmed collect, records each deletion and says artifacts-gone.
func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsOnlyOnceTheLeaseIsAway(t *testing.T) {
controller := &buscompose.LeaseHolder{Instance: "c1", Host: "novox", Epoch: 3}
handed := &buscompose.LeaseHolder{HandedTo: "mesh-grants", HandedBy: "person.jochen", Why: "the lease write"}
h, _, _, _ := aMesh(t, controller)
r, err := h.Artifacts("")
if err != nil {
t.Fatal(err)
}
states := map[string]string{}
for _, x := range r.References {
states[x.Reference] = x.State
}
// letta's 3rd to 7th are the five newest; its 1st a want names on ace, however old; its 2nd is eligible; the package
// and the manifest no build names are no reference of the seat's.
for n, want := range map[int]string{1: "kept", 2: "eligible", 3: "kept", 7: "kept"} {
if states[imageRef("letta", n)] != want || states[archiveRef("letta", n)] != want {
t.Errorf("letta's build %d is %s and %s, not %s", n, states[imageRef("letta", n)], states[archiveRef("letta", n)], want)
}
}
// And the copies: the one letta's 2nd stood on is eligible with it, the 7th's kept; the third no build names.
if states[mirror(1)] != "eligible" || states[mirror(2)] != "kept" || states[mirror(3)] != "" {
t.Errorf("the copies are %s, %s and %q", states[mirror(1)], states[mirror(2)], states[mirror(3)])
}
if m, err := h.MirrorsNow(); err != nil || len(m.Copies) != 3 || m.Copies[2].State != "unrecorded" || m.Held != "" {
t.Errorf("mirrors answered %+v: %v", m, err)
}
if len(r.References) != 16 || r.Counts["kept"] != 13 || r.Counts["eligible"] != 3 || r.Store != "10.0.0.5:5000" {
t.Errorf("the references are %d, counted %v, at %q", len(r.References), r.Counts, r.Store)
}
for _, c := range []struct {
name string
lease *buscompose.LeaseHolder
index bool
most int
letGo []string
left int
skipped int
unlog int
}{
{name: "the lease names a controller", lease: controller},
{name: "no lease key", lease: nil},
{name: "handed away", lease: handed, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), imageRef("letta", 2), mirror(1)}},
{name: "an index waits", lease: handed, index: true, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), mirror(1)}, skipped: 1},
{name: "bounded", lease: handed, most: 1, letGo: []string{archiveRef("letta", 2)}, left: 2},
// A kept build the log no longer holds: what it stood on is unknown, so no copy goes.
{name: "a kept build's bases unknown", lease: handed, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), imageRef("letta", 2)}, unlog: 3},
} {
t.Run(c.name, func(t *testing.T) {
h, b, door, acts := aMesh(t, c.lease)
door.indexes[imageRef("letta", 2)] = c.index
if c.unlog > 0 {
b.log = append(b.log[:c.unlog-1:c.unlog-1], b.log[c.unlog:]...)
}
if c.most == 0 {
h.AfterBuild("letta.x")
if _, err := h.Collect("tidy", true, 0); err == nil {
t.Error("a confirmed collect was let through while the controller sweeps")
}
if len(door.asked) != 0 || len(*acts) != 0 || len(b.published) != 0 {
t.Fatalf("before the lease write the store was asked %v, acts %v, said %v", door.asked, *acts, b.published)
}
return
}
a, err := h.Sweep(context.Background(), Sweeping{Most: c.most, By: "after the build letta.x"})
if err != nil {
t.Fatal(err)
}
if fmt.Sprint(a.LetGo) != fmt.Sprint(c.letGo) || a.Left != c.left || a.Skipped != c.skipped || a.Held != 6 || a.Missing != 0 {
t.Fatalf("let go of %v (left %d, skipped %d, held %d, missing %d), want %v", a.LetGo, a.Left, a.Skipped, a.Held, a.Missing, c.letGo)
}
// Every kept archive held before the first deletion.
first := -1
for i, x := range door.asked {
if strings.HasPrefix(x, "let go ") && first < 0 {
first = i
}
}
if first != 6 {
t.Errorf("the store was asked %v: the six kept archives are held first", door.asked)
}
if len(*acts) != len(c.letGo) || !strings.HasPrefix((*acts)[0], "seat:mesh-hand-acts.hand-act ") ||
!strings.Contains((*acts)[0], c.letGo[0]) {
t.Errorf("the hand acts recorded are %v", *acts)
}
if len(b.published) != 1 || !strings.HasPrefix(b.published[0], GoneSubject+" ") ||
!strings.Contains(b.published[0], `"fingerprint":"`+fp(2)+`"`) {
t.Errorf("said %v", b.published)
}
})
}
}
// S25 (image-missing): a want whose build names an image the store does not hold is a finding in the seat's own state,
// written once and deleted once the store holds it again; images names what a machine's wants and haves name.
func TestAWantedImageTheStoreLacksIsAFindingAndImagesNamesWantsAndHaves(t *testing.T) {
h, b, _, _ := aMesh(t, nil)
view, _ := h.Listed(context.Background())
repo, digest, _ := splitReference(imageRef("letta", 1))
delete(view.L.Repos[repo].Revisions, digest)
h.Listed = func(context.Context) (*View, error) { return view, nil }
if err := h.Observe(context.Background()); err != nil {
t.Fatal(err)
}
key := health.FindingKey(health.ScopeModule, "letta.ace", ImageMissing)
raw, found, _ := b.Get(FindingsState, key)
f, err := health.DecodeFinding(raw)
if !found || err != nil || f.Subject.Machine != "ace" || f.Source != "S25" || !strings.Contains(f.Evidence[0].Said, imageRef("letta", 1)) {
t.Fatalf("the finding is %+v (%v, %v)", f, found, err)
}
view.L.Repos[repo].Revisions[digest] = true
if err := h.Observe(context.Background()); err != nil {
t.Fatal(err)
}
if _, found, _ := b.Get(FindingsState, key); found {
t.Fatal("the finding stayed once the store held the image again")
}
// An image the manifest pins by itself, under the store's address, is judged as the controller judged it.
delete(view.L.Repos["letta/pinned"].Revisions, "sha256:"+strings.Repeat("c", 64))
if err := h.Observe(context.Background()); err != nil {
t.Fatal(err)
}
if raw, found, _ := b.Get(FindingsState, key); !found || !strings.Contains(string(raw), "letta/pinned@sha256:") {
t.Fatalf("the pinnedImage image missing is %s", raw)
}
have, _ := json.Marshal(delivery.Have{V: delivery.Version, FP: fp(7), Phase: "running"})
_ = b.Put(HavesRead, "ace.letta", json.RawMessage(have))
images, err := h.ImagesOf("ace", "10.0.0.5:5000")
if err != nil || len(images.Images) != 3 || !images.SentKnown || images.Images[0].Image != "10.0.0.5:5000/letta/image@sha256:"+strings.Repeat("1", 64) ||
fmt.Sprint(images.Images[0].In) != "[declared]" || fmt.Sprint(images.Images[1].In) != "[sent]" ||
images.Images[2].Image != pinnedImage || fmt.Sprint(images.Images[2].In) != "[declared sent]" {
t.Fatalf("the images of ace are %+v: %v", images, err)
}
}
@@ -1,30 +1,38 @@
// The artifact-store-tools module's Go bundle (novox/hq ADR 0251 §1–3, to-be 51): a process the node's
// runtime launches and speaks MCP over stdio to. It says what the artifact store holds — its
// repositories, its size, and what the controller's records say of each manifest — read from the
// store's own files through the store's own container, and writing nothing. A collection is asked of
// the controller, which decides and records what it lets go of (ADR 0189).
// The artifact-store-tools module's Go bundle (novox/hq ADR 0251 §1–3, ADR 0571 clause 2, to-be 51): a process the
// node's runtime launches and speaks MCP over stdio to. It says what the artifact store holds — its repositories, its
// size, and what the mesh keeps of each manifest — read from the store's own files through the store's own container,
// and it holds the seat mesh-artifact-sweep: the keep set, the sweep at each build's end once the controller's lease names
// no controller, and S25's image-missing.
//
// **A module beside the store, not the store's own.** A module that provides the artifact store cannot
// also build an artifact: building publishes to the store, so the store would be needed to create
// itself, and the controller refuses that manifest. So these tools are a second module, assigned to the
// machine that holds the store. stdout is the protocol; what this bundle says, it says on stderr.
// **A module beside the store, not the store's own.** A module that provides the artifact store cannot also build an
// artifact: building publishes to the store, so the store would be needed to create itself. So these tools are a
// second module, assigned to the machine that holds the store. stdout is the protocol; what this bundle says, it says
// on stderr.
package main
import (
"fmt"
"os"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/handacts"
)
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
func main() {
s := &Store{
Container: os.Getenv("MESH_ARTIFACT_STORE_CONTAINER"),
Run: ExecRunner,
UID: os.Getuid(),
}
h := &Holder{Bus: runtimeBus{}, Door: DoorAt(os.Getenv("MESH_ARTIFACT_STORE_ADDRESS_FILE")), Listed: s.View,
Now: time.Now, Logf: logf}
h.Acts = handacts.Recorder{Ask: stdio.Ask, LeaseAway: h.LeaseAway}
go follow(h)
// An empty name serves as the module the runtime names (MESH_SERVED_MODULE): artifact-store-tools.
if err := stdio.Serve("", Tools(s, Controller{Ask: stdio.Ask}, &Catalogue{Ask: stdio.Ask})); err != nil {
if err := stdio.Serve("", append(Tools(s, h, &Catalogue{Ask: stdio.Ask}), SeatTools(h, s)...)); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
@@ -0,0 +1,122 @@
package main
// The seat mesh-artifact-sweep's verbs (novox/hq ADR 0571 clause 2; catalogue.ArtifactSweepSeat), each a tool named
// `artifact-store.<verb>` as the runtime serves a seat's verb, answering as the controller's verb answered: {output,
// ok, answer}.
import (
"fmt"
"strings"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// verbAnswer is the controller's verbs' answer shape, which the builder and docker read (answerOf).
type verbAnswer struct {
Output string `json:"output"`
OK bool `json:"ok"`
Answer any `json:"answer"`
}
// SeatTools are the seat's verbs over h; s reads the store's listing for mirrors.
func SeatTools(h *Holder, s *Store) []stdio.Tool {
seat := catalogue.ArtifactSweepSeat + "."
return []stdio.Tool{
{Name: seat + "artifacts", Description: "Every artifact the mesh made that the store holds: kept and why, eligible, " +
"or with collected those it no longer holds.",
Run: func(args map[string]any) (any, error) {
r, err := h.Artifacts(text(args, "repository"))
if err != nil {
return nil, err
}
if text(args, "collected") != "true" {
refs := r.References[:0:0]
for _, x := range r.References {
if x.State != "collected" {
refs = append(refs, x)
}
}
r.References = refs
}
return verbAnswer{Output: fmt.Sprintf("%d kept, %d eligible, %d collected\n", r.Counts["kept"],
r.Counts["eligible"], r.Counts["collected"]), OK: true, Answer: r}, nil
}},
{Name: seat + "held", Description: "Whether the store holds each reference given: held, gone or unknown with why.",
Run: func(args map[string]any) (any, error) {
a, err := h.Held(listOf(args["references"]))
if err != nil {
return nil, err
}
return verbAnswer{Output: fmt.Sprintf("the artifact store at %s holds %d of the %d reference(s) asked\n",
a.Store, len(a.Held), len(a.Held)+len(a.Gone)+len(a.Unknown)), OK: true, Answer: a}, nil
}},
{Name: seat + "collect", Description: "The sweep now: a dry run without confirm; with confirm and why, every kept " +
"archive held, then each eligible artifact let go of, each recorded in the hand-act log.",
Run: func(args map[string]any) (any, error) {
most, err := count(args, "most", MostPerSweep, 1, MostAtAll)
if err != nil {
return nil, err
}
a, err := h.Collect(text(args, "why"), text(args, "confirm") == "true", most)
if err != nil {
return nil, err
}
return verbAnswer{Output: fmt.Sprintf("let go of %d, would let go of %d, %d left\n", len(a.LetGo),
len(a.WouldLetGo), a.Left), OK: true, Answer: a}, nil
}},
{Name: seat + "mirrors", Description: "The bases the mesh copied into its store, each kept (a kept build stood on " +
"it), eligible (only builds no longer kept stood on it) or unrecorded (no build the builder's log holds stood " +
"on it: never removed).",
Run: func(args map[string]any) (any, error) {
if text(args, "record") != "" {
return nil, fmt.Errorf("mirrors records nothing by hand on this seat: a copy is recorded by the builds the " +
"builder's log says stood on it (novox/hq ADR 0571 clause 1)")
}
m, err := h.MirrorsNow()
if err != nil {
return nil, err
}
return verbAnswer{Output: fmt.Sprintf("%d copies\n", len(m.Copies)), OK: true, Answer: m}, nil
}},
{Name: seat + "images", Description: "Every container image one machine's wants and haves name.",
Run: func(args map[string]any) (any, error) {
node := text(args, "node")
if node == "" {
return nil, fmt.Errorf("images names the machine")
}
address := ""
if d, err := h.Door(); err == nil {
address = d.Address()
}
a, err := h.ImagesOf(node, address)
if err != nil {
return nil, err
}
return verbAnswer{Output: fmt.Sprintf("%s: %d images\n", node, len(a.Images)), OK: true, Answer: a}, nil
}},
}
}
// listOf is a list argument: a list, or one text separated by commas.
func listOf(v any) []string {
var out []string
add := func(s string) {
for _, p := range strings.Split(s, ",") {
if p = strings.TrimSpace(p); p != "" && !contains(out, p) {
out = append(out, p)
}
}
}
switch x := v.(type) {
case string:
add(x)
case []any:
for _, e := range x {
if s, ok := e.(string); ok {
add(s)
}
}
}
return out
}
@@ -1,6 +1,8 @@
package main
import (
"git.novox.be/novox/mesh-sdk/go/catalogue"
"context"
"encoding/json"
"fmt"
@@ -411,6 +413,11 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
var m struct {
Tools []string `json:"tools"`
Invokes []string `json:"invokes"`
Reads []string `json:"reads"`
Claims []struct {
Name string `json:"name"`
Serves []string `json:"serves"`
} `json:"claims"`
}
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
@@ -428,10 +435,30 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
if !reflect.DeepEqual(served, listed) {
t.Fatalf("served %v, manifest %v", served, listed)
}
want := []string{"seat:mesh-controller.artifacts", "seat:mesh-controller.collect", CatalogueModules, CatalogueModule}
// ADR 0571 clause 2: it holds the seat mesh-artifact-sweep and serves its verbs, and calls no verb of the controller's;
// the lease is the one controller state it reads (clause 10's exemption).
want := []string{"seat:mesh-hand-acts.hand-act", CatalogueModules, CatalogueModule}
if !reflect.DeepEqual(m.Invokes, want) {
t.Errorf("invokes %v", m.Invokes)
}
var verbs []string
for _, tl := range SeatTools(&Holder{}, &Store{}) {
verbs = append(verbs, strings.TrimPrefix(tl.Name, catalogue.ArtifactSweepSeat+"."))
}
seat, _ := catalogue.SeatNamed(catalogue.ArtifactSweepSeat)
var contract []string
for _, v := range seat.Serves {
contract = append(contract, v.Name)
}
if len(m.Claims) != 1 || m.Claims[0].Name != catalogue.ArtifactSweepSeat || !reflect.DeepEqual(m.Claims[0].Serves, verbs) ||
!reflect.DeepEqual(verbs, contract) {
t.Errorf("claims %+v, serves %v, the seat's verbs %v", m.Claims, verbs, contract)
}
for _, r := range m.Reads {
if strings.HasPrefix(r, "mesh-controller.") && r != LeaseRead {
t.Errorf("reads the controller's %s", r)
}
}
}
func TestASocketRefusalIsAskedAgainThroughSudo(t *testing.T) {
@@ -11,9 +11,16 @@ import (
stdio "git.novox.be/novox/mesh-sdk/go"
)
// Recorded is what says which references the mesh keeps and lets go of them: the seat's holder (Holder, novox/hq ADR
// 0571 clause 2), or before it the controller (Controller), each in the controller's answer's shape.
type Recorded interface {
Artifacts(repository string) (*Records, error)
Collect(why string, confirm bool, most int) (*CollectAnswer, error)
}
// Tools are the artifact store's tools (novox/hq ADR 0251 §1). The first three only read; artifact_store_collect
// only reads too, unless it is a real run, and then it asks the controller, which decides and deletes.
func Tools(s *Store, c Controller, cat *Catalogue) []stdio.Tool {
// only reads too, unless it is a real run, and then it asks the seat's sweep, which decides and deletes.
func Tools(s *Store, c Recorded, cat *Catalogue) []stdio.Tool {
ctx := context.Background()
repoArg := map[string]any{"type": "string", "description": "one repository, as the store names it (<module>/<artifact>)"}
return []stdio.Tool{
@@ -0,0 +1 @@
${seat:mesh-artifact-store:reach}
+19 -2
View File
@@ -1,5 +1,22 @@
module artifactstoretools
go 1.22
go 1.26.0
require git.novox.be/novox/mesh-sdk/go v0.1.7
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0
github.com/nats-io/nats.go v1.54.0
)
require (
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
github.com/google/go-tpm v0.9.8 // indirect
github.com/klauspost/compress v1.20.0 // indirect
github.com/minio/highwayhash v1.0.4 // indirect
github.com/nats-io/jwt/v2 v2.8.1 // indirect
github.com/nats-io/nats-server/v2 v2.11.17 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/time v0.15.0 // indirect
)
+27 -2
View File
@@ -1,2 +1,27 @@
git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w=
git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
@@ -0,0 +1,57 @@
package registry
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"net/http"
)
// blobBound is the most GetBlob reads: a templates archive is a few kilobytes; anything near this is no archive a
// take-in reads text from.
const blobBound = 64 << 20
// GetBlob is the archive a recorded reference names (`artifact-store://<repository>/blobs/sha256:<hex>`), read whole
// and checked against the digest it names: bytes that hash to another digest are refused, never read (R4e: the
// controller reads a templates archive by the reference of a builder's done key). Gone when the store does not hold
// it; ErrNotOurs for a reference outside the store or one that is not an archive.
func (s Store) GetBlob(ctx context.Context, reference string) ([]byte, error) {
repository, digest, archive, err := s.archive(reference)
if err != nil {
return nil, err
}
if !archive {
return nil, fmt.Errorf("%w: %s is not an archive", ErrNotOurs, reference)
}
request, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url(repository, "blobs", digest), nil)
if err != nil {
return nil, err
}
response, err := s.client().Do(request)
if err != nil {
return nil, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
}
defer response.Body.Close()
switch response.StatusCode {
case http.StatusOK:
case http.StatusNotFound:
return nil, Gone
default:
return nil, fmt.Errorf("the artifact store answered %s for %s/blobs/%s", response.Status, repository, digest)
}
body, err := io.ReadAll(io.LimitReader(response.Body, blobBound+1))
if err != nil {
return nil, fmt.Errorf("reading %s/blobs/%s: %w", repository, digest, err)
}
if len(body) > blobBound {
return nil, fmt.Errorf("%s/blobs/%s is larger than %d bytes, which no archive read here is", repository, digest, blobBound)
}
sum := sha256.Sum256(body)
if got := "sha256:" + hex.EncodeToString(sum[:]); got != digest {
return nil, fmt.Errorf("the artifact store answered %s/blobs/%s with bytes of the digest %s: refused", repository,
digest, got)
}
return body, nil
}
@@ -0,0 +1,44 @@
package registry
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
// A take-in reads a templates archive by its reference, and only bytes of the digest the reference names.
func TestGetBlobReadsOnlyTheBytesOfItsDigest(t *testing.T) {
good := []byte("the archive")
sum := sha256.Sum256(good)
digest := "sha256:" + hex.EncodeToString(sum[:])
liar := "sha256:" + strings.Repeat("0", 64)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case strings.HasSuffix(r.URL.Path, "/x/files/blobs/"+digest), strings.HasSuffix(r.URL.Path, "/x/files/blobs/"+liar):
_, _ = w.Write(good)
default:
w.WriteHeader(http.StatusNotFound)
}
}))
defer server.Close()
s := Store{Address: strings.TrimPrefix(server.URL, "http://")}
ctx := context.Background()
body, err := s.GetBlob(ctx, "artifact-store://x/files/blobs/"+digest)
if err != nil || string(body) != string(good) {
t.Fatalf("the archive: %q %v", body, err)
}
if _, err := s.GetBlob(ctx, "artifact-store://x/files/blobs/"+liar); err == nil || !strings.Contains(err.Error(), "refused") {
t.Fatalf("bytes of another digest are read: %v", err)
}
if _, err := s.GetBlob(ctx, "artifact-store://x/other/blobs/"+digest); !errors.Is(err, Gone) {
t.Fatalf("a blob the store does not hold: %v", err)
}
if _, err := s.GetBlob(ctx, "registry.example/x@"+digest); !errors.Is(err, ErrNotOurs) {
t.Fatalf("a reference outside the store: %v", err)
}
}
@@ -0,0 +1,345 @@
package registry
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"net/http"
"strconv"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// Every archive the store keeps is held by a manifest (novox/hq issue 253, ADR 0189).
//
// **The store's collector marks only from manifests.** The mesh's store is a stock registry, and
// its nightly `registry garbage-collect` walks every manifest in every repository, marks the blobs
// those manifests name, and deletes every blob it did not mark. An image is a manifest, so what
// the mesh keeps of an image survives. An archive was not: the builder put it in the store as a
// bare blob — upload, then `PUT ?digest=` — and nothing in the store names it. To the collector a
// bare blob is unreferenced, so the first real collection would have deleted every archive the
// mesh holds, kept or not, and every machine pinning a bundle would have found it gone. The
// collector runs `--dry-run` until this is true.
//
// **So each archive gets a holder**: the smallest OCI image manifest that names it — the empty
// config, one layer, nothing else — put in the archive's own repository, by digest, untagged. The
// collector marks it and so keeps the archive; the sweep lets go of an archive by deleting its
// holder first, which is what lets the bytes go at the next collection.
//
// **Nothing a machine reads changes.** The recorded reference stays
// `artifact-store://<module>/<artifact>/blobs/sha256:…`, and machines fetch the blob exactly as
// before. The holder is the store's bookkeeping, not a second way to reach anything.
//
// **Deterministic, so it never needs recording.** The holder is composed from the archive's digest
// and size alone, in a fixed field order with no timestamps or annotations, so the sweep can
// compute which manifest holds any archive from the reference it already has plus one HEAD for the
// size. No schema change, no second record that could disagree with the store.
const (
// mediaManifest is the type a holder is put and asked for as.
mediaManifest = "application/vnd.oci.image.manifest.v1+json"
// mediaEmpty is the OCI empty descriptor's type: a config that says nothing, for a manifest
// whose only purpose is to name its layer.
mediaEmpty = "application/vnd.oci.empty.v1+json"
// emptyDigest is the digest of `{}`, the empty config's content, fixed by the OCI spec.
emptyDigest = "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"
// mediaArchive is the layer type an archive is held as. Every archive the builder publishes is
// `pack`'s gzipped tar, so this is the true type and not a placeholder — and it is a constant,
// not read from anywhere, because the holder must be recomputable from the reference alone.
mediaArchive = "application/vnd.oci.image.layer.v1.tar+gzip"
)
// emptyConfig is the content emptyDigest names.
var emptyConfig = []byte("{}")
// manifestAccept is what a manifest is asked for as. A registry answers a manifest HEAD only in a
// type the caller named, and answers 404 to a bare one for a manifest it holds perfectly well
// (measured 2026-09-28; internal/builder/registry.go says how that was found).
var manifestAccept = []string{
mediaManifest,
"application/vnd.docker.distribution.manifest.v2+json",
}
type descriptor struct {
MediaType string `json:"mediaType"`
Digest string `json:"digest"`
Size int64 `json:"size"`
}
type holderManifest struct {
SchemaVersion int `json:"schemaVersion"`
MediaType string `json:"mediaType"`
Config descriptor `json:"config"`
Layers []descriptor `json:"layers"`
}
// Holder is the manifest that holds an archive in the store, and its digest.
//
// A pure function of the archive's digest and size: the same two in give the same bytes out,
// always, because `encoding/json` writes a struct's fields in their declared order and there is
// nothing here that varies by when or where it was composed.
func Holder(digest string, size int64) (body []byte, holder string) {
body, err := json.Marshal(holderManifest{
SchemaVersion: 2,
MediaType: mediaManifest,
Config: descriptor{MediaType: mediaEmpty, Digest: emptyDigest, Size: int64(len(emptyConfig))},
Layers: []descriptor{{MediaType: mediaArchive, Digest: digest, Size: size}},
})
if err != nil {
// Marshalling a struct of strings and integers cannot fail.
panic(err)
}
sum := sha256.Sum256(body)
return body, "sha256:" + hex.EncodeToString(sum[:])
}
// Hold makes sure the store holds this archive by a manifest, and says whether it had to write one.
//
// Takes a reference as the mesh records it. An image is its own manifest and needs no holder, so
// it answers false and nothing is asked. Idempotent: a holder already there is left alone, which
// is what lets the sweep run it over every kept archive on every build and so backfill the bare
// blobs published before holders existed (novox/hq issue 253).
//
// Gone when the store does not hold the archive at all: there is nothing to hold, and that is a
// fact the caller reports rather than one this invents a remedy for.
func (s Store) Hold(ctx context.Context, reference string) (bool, error) {
repository, digest, archive, err := s.archive(reference)
if err != nil || !archive {
return false, err
}
size, err := s.blobSize(ctx, repository, digest)
if err != nil {
return false, err
}
return s.HoldBlob(ctx, repository, digest, size)
}
// Held is whether the store holds this archive by its manifest. Asks and changes nothing — the
// question an operator needs answered with "none unheld" before the collector is let loose.
//
// An image answers true: it is its own manifest. An archive the store does not have answers Gone.
func (s Store) Held(ctx context.Context, reference string) (bool, error) {
repository, digest, archive, err := s.archive(reference)
if err != nil {
return false, err
}
if !archive {
return true, nil
}
size, err := s.blobSize(ctx, repository, digest)
if err != nil {
return false, err
}
_, holder := Holder(digest, size)
return s.has(ctx, s.url(repository, "manifests", holder), manifestAccept...)
}
// HoldBlob puts the holder for a blob of this digest and size into its repository, unless it is
// there already. Answers whether it wrote one.
//
// The builder calls this with the size it has just uploaded; the sweep, through Hold, with the size
// the store reports. Both arrive at the same holder, which is the point of composing it.
func (s Store) HoldBlob(ctx context.Context, repository, digest string, size int64) (bool, error) {
if s.Address == "" {
return false, fmt.Errorf("this mesh has no artifact store on its network to hold %s/%s in", repository, digest)
}
body, holder := Holder(digest, size)
there, err := s.has(ctx, s.url(repository, "manifests", holder), manifestAccept...)
if err != nil {
return false, err
}
if there {
return false, nil
}
// The config must be in the repository before a manifest naming it is accepted: a registry
// refuses a manifest whose blobs it cannot find there, which is the property that makes a
// holder mean something.
if err := s.putBlob(ctx, repository, emptyDigest, emptyConfig); err != nil {
return false, err
}
// **By digest, never by tag.** A tag would be one more name to move and one more thing the
// collector's `--delete-untagged` would read as meaningful; the mesh names nothing by tag that
// it pins by digest, and an untagged manifest is kept by plain collection.
request, err := http.NewRequestWithContext(ctx, http.MethodPut,
s.url(repository, "manifests", holder), bytes.NewReader(body))
if err != nil {
return false, err
}
request.Header.Set("Content-Type", mediaManifest)
response, err := s.client().Do(request)
if err != nil {
return false, err
}
defer response.Body.Close()
if response.StatusCode != http.StatusCreated {
said, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
return false, fmt.Errorf("the artifact store refused to hold %s/%s: %s %s",
repository, digest, response.Status, strings.TrimSpace(string(said)))
}
return true, nil
}
// letGoOfHolder deletes the manifest holding an archive, before the archive's own link goes.
//
// **Holder first.** Deleting the blob link alone leaves a manifest still naming the blob, and the
// collector would keep its bytes for ever on the strength of it — the sweep would record the
// archive collected while the disk said otherwise. Deleting the holder first and failing before
// the link goes leaves an unheld archive that the next sweep still offers, which is safe.
//
// A store that no longer has the blob answers Gone: without its size the holder cannot be named,
// and without the blob there is nothing left for a holder to keep. A store that never had a holder
// for it — an archive published before holders, never backfilled — answers 404 to the delete, and
// that is the outcome wanted.
func (s Store) letGoOfHolder(ctx context.Context, repository, digest string) error {
size, err := s.blobSize(ctx, repository, digest)
if err != nil {
return err
}
_, holder := Holder(digest, size)
err = s.remove(ctx, s.url(repository, "manifests", holder), repository+"/manifests/"+holder)
if err == Gone {
return nil
}
return err
}
// archive reads a recorded reference into its repository and digest, and whether it is an archive
// at all. Refuses as ErrNotOurs anything the mesh did not put in its own store.
func (s Store) archive(reference string) (repository, digest string, archive bool, err error) {
path, kept := catalogue.InArtifactStore(reference)
if !kept {
return "", "", false, fmt.Errorf("%w: %s", ErrNotOurs, reference)
}
if s.Address == "" {
return "", "", false, fmt.Errorf("this mesh has no artifact store on its network to ask about %s", reference)
}
repository, kind, digest, err := split(path)
if err != nil {
return "", "", false, err
}
return repository, digest, kind == "blobs", nil
}
// blobSize is how large the store says a blob is; Gone when it does not have it.
func (s Store) blobSize(ctx context.Context, repository, digest string) (int64, error) {
request, err := http.NewRequestWithContext(ctx, http.MethodHead, s.url(repository, "blobs", digest), nil)
if err != nil {
return 0, err
}
response, err := s.client().Do(request)
if err != nil {
return 0, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
}
defer response.Body.Close()
switch response.StatusCode {
case http.StatusOK:
case http.StatusNotFound:
return 0, Gone
default:
return 0, fmt.Errorf("the artifact store answered %s for %s/blobs/%s", response.Status, repository, digest)
}
// Read from the header rather than ContentLength: a HEAD's ContentLength is what the response
// says it would have sent, which Go reports faithfully, but a proxy in between is free to drop
// it, and the header is what the registry itself wrote.
if length := response.Header.Get("Content-Length"); length != "" {
if n, err := strconv.ParseInt(length, 10, 64); err == nil && n >= 0 {
return n, nil
}
}
if response.ContentLength >= 0 {
return response.ContentLength, nil
}
return 0, fmt.Errorf("the artifact store holds %s/blobs/%s and will not say how large it is", repository, digest)
}
// putBlob uploads a small blob unless the repository already has it: ask where, then put it there
// naming the digest — the registry's own two steps, the same the builder takes for an archive.
func (s Store) putBlob(ctx context.Context, repository, digest string, body []byte) error {
if there, err := s.has(ctx, s.url(repository, "blobs", digest)); err != nil {
return err
} else if there {
return nil
}
start, err := http.NewRequestWithContext(ctx, http.MethodPost,
"http://"+s.Address+"/v2/"+repository+"/blobs/uploads/", nil)
if err != nil {
return err
}
begun, err := s.client().Do(start)
if err != nil {
return fmt.Errorf("cannot start an upload to %s: %w", repository, err)
}
begun.Body.Close()
if begun.StatusCode != http.StatusAccepted {
return fmt.Errorf("the artifact store answered %s when asked where to put a blob in %s", begun.Status, repository)
}
where := begun.Header.Get("Location")
if where == "" {
return fmt.Errorf("the artifact store accepted an upload to %s and said nowhere to put it", repository)
}
if strings.HasPrefix(where, "/") {
where = "http://" + s.Address + where
}
separator := "?"
if strings.Contains(where, "?") {
separator = "&"
}
put, err := http.NewRequestWithContext(ctx, http.MethodPut, where+separator+"digest="+digest, bytes.NewReader(body))
if err != nil {
return err
}
put.Header.Set("Content-Type", "application/octet-stream")
done, err := s.client().Do(put)
if err != nil {
return err
}
defer done.Body.Close()
if done.StatusCode != http.StatusCreated {
said, _ := io.ReadAll(io.LimitReader(done.Body, 4096))
return fmt.Errorf("the artifact store refused a blob in %s: %s %s", repository, done.Status, strings.TrimSpace(string(said)))
}
return nil
}
// has is whether the store answers 200 for a HEAD at that URL.
func (s Store) has(ctx context.Context, url string, accept ...string) (bool, error) {
request, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
if err != nil {
return false, err
}
for _, media := range accept {
request.Header.Add("Accept", media)
}
response, err := s.client().Do(request)
if err != nil {
return false, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
}
defer response.Body.Close()
switch response.StatusCode {
case http.StatusOK:
return true, nil
case http.StatusNotFound:
return false, nil
default:
return false, fmt.Errorf("the artifact store answered %s for %s", response.Status, url)
}
}
func (s Store) url(repository, kind, digest string) string {
return "http://" + s.Address + "/v2/" + repository + "/" + kind + "/" + digest
}
func (s Store) client() *http.Client {
if s.HTTP != nil {
return s.HTTP
}
return &http.Client{Timeout: 30 * time.Second}
}
@@ -0,0 +1,268 @@
package registry
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strconv"
"strings"
"sync"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// Every kept archive is held by a manifest (novox/hq issue 253, ADR 0189).
//
// Against an in-memory registry that keeps blobs and manifests per repository and refuses what a
// registry refuses — a blob whose digest does not match, a manifest whose digest does not match
// or whose blobs the repository does not have, a manifest asked for without an Accept naming its
// type. What is asserted is this side's decisions; the live test below asserts the registry's.
type memRegistry struct {
mu sync.Mutex
blobs map[string][]byte // repository + "@" + digest
manifests map[string][]byte // repository + "@" + digest
writes []string // every PUT and DELETE, as "METHOD path"
}
func digestOf(body []byte) string {
sum := sha256.Sum256(body)
return "sha256:" + hex.EncodeToString(sum[:])
}
func (m *memRegistry) serve(t *testing.T) Store {
t.Helper()
m.blobs = map[string][]byte{}
m.manifests = map[string][]byte{}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
m.mu.Lock()
defer m.mu.Unlock()
path := strings.TrimPrefix(r.URL.Path, "/v2/")
if r.Method == http.MethodPut || r.Method == http.MethodDelete {
m.writes = append(m.writes, r.Method+" "+r.URL.Path)
}
switch {
case r.Method == http.MethodPost && strings.HasSuffix(path, "/blobs/uploads/"):
repository := strings.TrimSuffix(path, "/blobs/uploads/")
w.Header().Set("Location", "/upload/"+repository+"?state=x")
w.WriteHeader(http.StatusAccepted)
case r.Method == http.MethodPut && strings.HasPrefix(r.URL.Path, "/upload/"):
repository := strings.TrimPrefix(r.URL.Path, "/upload/")
body, _ := io.ReadAll(r.Body)
digest := r.URL.Query().Get("digest")
if digest != digestOf(body) {
w.WriteHeader(http.StatusBadRequest)
return
}
m.blobs[repository+"@"+digest] = body
w.WriteHeader(http.StatusCreated)
case strings.Contains(path, "/blobs/"):
repository, digest, _ := strings.Cut(path, "/blobs/")
key := repository + "@" + digest
body, ok := m.blobs[key]
if !ok {
w.WriteHeader(http.StatusNotFound)
return
}
switch r.Method {
case http.MethodHead:
w.Header().Set("Content-Length", strconv.Itoa(len(body)))
w.WriteHeader(http.StatusOK)
case http.MethodDelete:
delete(m.blobs, key)
w.WriteHeader(http.StatusAccepted)
default:
w.WriteHeader(http.StatusMethodNotAllowed)
}
case strings.Contains(path, "/manifests/"):
repository, digest, _ := strings.Cut(path, "/manifests/")
key := repository + "@" + digest
switch r.Method {
case http.MethodHead:
if _, ok := m.manifests[key]; !ok || !strings.Contains(r.Header.Get("Accept"), mediaManifest) {
w.WriteHeader(http.StatusNotFound)
return
}
w.WriteHeader(http.StatusOK)
case http.MethodPut:
body, _ := io.ReadAll(r.Body)
if digest != digestOf(body) {
w.WriteHeader(http.StatusBadRequest)
return
}
var named holderManifest
if err := json.Unmarshal(body, &named); err != nil {
w.WriteHeader(http.StatusBadRequest)
return
}
for _, d := range append([]descriptor{named.Config}, named.Layers...) {
if _, ok := m.blobs[repository+"@"+d.Digest]; !ok {
w.WriteHeader(http.StatusBadRequest)
fmt.Fprintf(w, "MANIFEST_BLOB_UNKNOWN %s", d.Digest)
return
}
}
m.manifests[key] = body
w.WriteHeader(http.StatusCreated)
case http.MethodDelete:
if _, ok := m.manifests[key]; !ok {
w.WriteHeader(http.StatusNotFound)
return
}
delete(m.manifests, key)
w.WriteHeader(http.StatusAccepted)
}
default:
w.WriteHeader(http.StatusNotFound)
}
}))
t.Cleanup(server.Close)
return Store{Address: strings.TrimPrefix(server.URL, "http://")}
}
// bare puts an archive in the store the way the builder did before holders: a blob, nothing more.
func (m *memRegistry) bare(repository string, body []byte) string {
m.mu.Lock()
defer m.mu.Unlock()
digest := digestOf(body)
m.blobs[repository+"@"+digest] = body
return catalogue.ArtifactStoreScheme + repository + "/blobs/" + digest
}
func TestTheHolderIsComposedFromTheDigestAndSizeAlone(t *testing.T) {
// The sweep must arrive at the very manifest the builder wrote, with nothing recorded between
// them. Same inputs, same bytes — and a different size is a different holder, so a holder can
// never be mistaken for one of a different blob.
digest := "sha256:" + strings.Repeat("a", 64)
one, first := Holder(digest, 42)
two, second := Holder(digest, 42)
if !bytes.Equal(one, two) || first != second {
t.Fatalf("the same archive composed two holders:\n%s\n%s", one, two)
}
if _, other := Holder(digest, 43); other == first {
t.Fatal("a different size composed the same holder")
}
want := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json",` +
`"config":{"mediaType":"application/vnd.oci.empty.v1+json",` +
`"digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","size":2},` +
`"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"` + digest + `","size":42}]}`
if string(one) != want {
t.Fatalf("the holder is\n%s\nwant\n%s", one, want)
}
if digestOf(emptyConfig) != emptyDigest {
t.Fatalf("the empty config's digest is %s, not %s", digestOf(emptyConfig), emptyDigest)
}
}
func TestHoldBackfillsABareArchiveAndIsIdempotent(t *testing.T) {
// The archives published before this have no holder. Hold, run over every kept archive on
// every sweep, writes one the first time and nothing after.
m := &memRegistry{}
store := m.serve(t)
ctx := context.Background()
body := []byte("a theme")
reference := m.bare("shell/config", body)
if held, err := store.Held(ctx, reference); err != nil || held {
t.Fatalf("a bare blob reads as held=%v (%v)", held, err)
}
wrote, err := store.Hold(ctx, reference)
if err != nil {
t.Fatal(err)
}
if !wrote {
t.Fatal("holding a bare archive wrote nothing")
}
_, holder := Holder(digestOf(body), int64(len(body)))
if _, ok := m.manifests["shell/config@"+holder]; !ok {
t.Fatalf("the store holds manifests %v; want %s", m.manifests, holder)
}
if held, err := store.Held(ctx, reference); err != nil || !held {
t.Fatalf("after holding, held=%v (%v)", held, err)
}
writes := len(m.writes)
wrote, err = store.Hold(ctx, reference)
if err != nil {
t.Fatal(err)
}
if wrote || len(m.writes) != writes {
t.Fatalf("holding again wrote %v", m.writes[writes:])
}
}
func TestAnImageNeedsNoHolderAndAMissingArchiveIsGone(t *testing.T) {
m := &memRegistry{}
store := m.serve(t)
ctx := context.Background()
// An image is its own manifest: nothing is asked.
wrote, err := store.Hold(ctx, catalogue.ArtifactStoreScheme+"web/app@sha256:"+strings.Repeat("b", 64))
if err != nil || wrote || len(m.writes) != 0 {
t.Fatalf("holding an image wrote=%v err=%v writes=%v", wrote, err, m.writes)
}
// An archive the store does not have is a fact to report, not something to invent a holder for.
_, err = store.Hold(ctx, catalogue.ArtifactStoreScheme+"web/config/blobs/sha256:"+strings.Repeat("c", 64))
if !errors.Is(err, Gone) {
t.Fatalf("holding a missing archive answered %v, want Gone", err)
}
// And a reference that is not the mesh's is refused as such.
if _, err := store.Hold(ctx, "docker.io/library/registry@sha256:abc"); !errors.Is(err, ErrNotOurs) {
t.Fatalf("holding a vendor's image answered %v, want ErrNotOurs", err)
}
}
func TestLettingGoOfAnArchiveDeletesItsHolderFirst(t *testing.T) {
// A holder left behind would keep the bytes through every collection while the record said
// collected; the link deleted first and the holder failing after would be that exactly.
m := &memRegistry{}
store := m.serve(t)
ctx := context.Background()
body := []byte("an old theme")
reference := m.bare("shell/config", body)
if _, err := store.Hold(ctx, reference); err != nil {
t.Fatal(err)
}
m.writes = nil
if err := store.LetGo(ctx, reference); err != nil {
t.Fatal(err)
}
_, holder := Holder(digestOf(body), int64(len(body)))
want := []string{
"DELETE /v2/shell/config/manifests/" + holder,
"DELETE /v2/shell/config/blobs/" + digestOf(body),
}
if strings.Join(m.writes, "\n") != strings.Join(want, "\n") {
t.Fatalf("the store was asked\n%s\nwant\n%s", strings.Join(m.writes, "\n"), strings.Join(want, "\n"))
}
if len(m.manifests) != 0 {
t.Fatalf("a holder survived: %v", m.manifests)
}
// Asked again, the archive is already gone, which is the outcome wanted.
if err := store.LetGo(ctx, reference); !errors.Is(err, Gone) {
t.Fatalf("letting go twice answered %v, want Gone", err)
}
}
func TestLettingGoOfAnUnheldArchiveStillDeletesIt(t *testing.T) {
// An archive published before holders and let go of before any sweep held it: the holder's
// delete answers 404, which is the outcome wanted, and the blob still goes.
m := &memRegistry{}
store := m.serve(t)
reference := m.bare("shell/config", []byte("never held"))
if err := store.LetGo(context.Background(), reference); err != nil {
t.Fatal(err)
}
if len(m.blobs) != 0 {
t.Fatalf("the blob survived: %v", m.blobs)
}
}
@@ -0,0 +1,176 @@
package registry
import (
"context"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"slices"
"strings"
"sync"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// An index is let go of with the platform manifests it names, except those a kept index of the same
// repository names too (novox/hq ADR 0257). Each platform manifest is a manifest of the repository in
// its own right, and the store's collector keeps every manifest a repository holds — so an index let go
// of alone frees nothing of the images it names.
func digestN(n int) string { return fmt.Sprintf("sha256:%064x", n) }
// indexStore holds indexes and images in one repository, answers GETs with their documents, and records
// every delete.
type indexStore struct {
mu sync.Mutex
indexes map[string][]string // digest → the platform manifests it names
images map[string]bool
deleted []string
}
func (s *indexStore) serve(t *testing.T) Store {
t.Helper()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
s.mu.Lock()
defer s.mu.Unlock()
digest := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:]
switch r.Method {
case http.MethodGet:
if children, ok := s.indexes[digest]; ok {
var named []string
for _, c := range children {
named = append(named, `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"`+c+`","size":1}`)
}
w.Header().Set("Content-Type", "application/vnd.oci.image.index.v1+json")
_, _ = w.Write([]byte(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[` +
strings.Join(named, ",") + `]}`))
return
}
if s.images[digest] {
w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
_, _ = w.Write([]byte(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","layers":[]}`))
return
}
w.WriteHeader(http.StatusNotFound)
case http.MethodHead:
if _, ok := s.indexes[digest]; ok || s.images[digest] {
w.WriteHeader(http.StatusOK)
return
}
w.WriteHeader(http.StatusNotFound)
case http.MethodDelete:
if _, ok := s.indexes[digest]; !ok && !s.images[digest] {
w.WriteHeader(http.StatusNotFound)
return
}
s.deleted = append(s.deleted, digest)
delete(s.indexes, digest)
delete(s.images, digest)
w.WriteHeader(http.StatusAccepted)
default:
w.WriteHeader(http.StatusBadRequest)
}
}))
t.Cleanup(server.Close)
return Store{Address: strings.TrimPrefix(server.URL, "http://")}
}
func TestAnIndexGoesWithItsPlatformsAndAKeptIndexKeepsItsOwn(t *testing.T) {
// Two indexes of one image's repository: the old one names platforms 11 and 12, the kept one 12 and 13.
s := &indexStore{
indexes: map[string][]string{digestN(1): {digestN(11), digestN(12)}, digestN(2): {digestN(12), digestN(13)}},
images: map[string]bool{digestN(11): true, digestN(12): true, digestN(13): true},
}
store := s.serve(t)
repository := "upstream/docker.io/library/golang"
old := catalogue.ArtifactStoreScheme + repository + "@" + digestN(1)
kept := catalogue.ArtifactStoreScheme + repository + "@" + digestN(2)
spare, err := store.SpareKeptIndexes(context.Background(), []string{kept}, []string{old})
if err != nil {
t.Fatal(err)
}
store.Spare = spare
if err := store.LetGo(context.Background(), old); err != nil {
t.Fatal(err)
}
// Its own platform first, the index last; the platform the kept index names is left in place.
if !slices.Equal(s.deleted, []string{digestN(11), digestN(1)}) {
t.Fatalf("deleted %v; want the old index's own platform, then the index", s.deleted)
}
if !s.images[digestN(12)] || !s.images[digestN(13)] {
t.Fatal("a platform a kept index names was let go of")
}
}
func TestWithoutSparingAnIndexGoesAloneAndAnImageNamesNoPlatforms(t *testing.T) {
s := &indexStore{indexes: map[string][]string{digestN(1): {digestN(11)}}, images: map[string]bool{digestN(11): true, digestN(5): true}}
store := s.serve(t)
// No Spare: as before ADR 0257, the index alone.
if err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/base@"+digestN(1)); err != nil {
t.Fatal(err)
}
if !slices.Equal(s.deleted, []string{digestN(1)}) {
t.Fatalf("without a Spare, deleted %v", s.deleted)
}
// An image is one delete, with a Spare or without.
s.deleted = nil
spare, err := store.SpareKeptIndexes(context.Background(), nil, []string{catalogue.ArtifactStoreScheme + "web/app@" + digestN(5)})
if err != nil {
t.Fatal(err)
}
store.Spare = spare
if err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/app@"+digestN(5)); err != nil {
t.Fatal(err)
}
if !slices.Equal(s.deleted, []string{digestN(5)}) {
t.Fatalf("an image was let go of as %v", s.deleted)
}
// An index the store no longer holds is Gone, and nothing is deleted on its account.
s.deleted = nil
err = store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/base@"+digestN(9))
if !errors.Is(err, Gone) || len(s.deleted) != 0 {
t.Fatalf("an index the store does not hold: %v, deleted %v", err, s.deleted)
}
}
func TestASpareThatCannotBeReadKeepsTheIndex(t *testing.T) {
s := &indexStore{indexes: map[string][]string{digestN(1): {digestN(11)}}, images: map[string]bool{digestN(11): true}}
store := s.serve(t)
store.Spare = func(context.Context, string) (map[string]bool, error) { return nil, fmt.Errorf("the store went away") }
err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/base@"+digestN(1))
if err == nil || !strings.Contains(err.Error(), "was kept") {
t.Fatalf("a spare that could not be read: %v", err)
}
if len(s.deleted) != 0 {
t.Fatalf("deleted %v without knowing what a kept index names", s.deleted)
}
}
// The spare list is read whole before the sweep: a kept index that cannot be read is an error before
// anything is deleted, and a repository not read before keeps its index.
func TestASpareListIsReadWholeBeforeAnythingGoes(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
}))
t.Cleanup(server.Close)
store := Store{Address: strings.TrimPrefix(server.URL, "http://")}
kept := catalogue.ArtifactStoreScheme + "web/base@" + digestN(2)
old := catalogue.ArtifactStoreScheme + "web/base@" + digestN(1)
if _, err := store.SpareKeptIndexes(context.Background(), []string{kept}, []string{old}); err == nil {
t.Fatal("a kept index the store would not answer for was taken as naming nothing")
}
s := &indexStore{indexes: map[string][]string{digestN(1): {digestN(11)}}, images: map[string]bool{digestN(11): true}}
good := s.serve(t)
spare, err := good.SpareKeptIndexes(context.Background(), nil, []string{catalogue.ArtifactStoreScheme + "other/repo@" + digestN(3)})
if err != nil {
t.Fatal(err)
}
good.Spare = spare
if err := good.LetGo(context.Background(), old); err == nil || len(s.deleted) != 0 {
t.Fatalf("an index in a repository not read before the sweep was let go of: %v, deleted %v", err, s.deleted)
}
}
@@ -0,0 +1,325 @@
// Package registry speaks to the mesh's artifact store over its own door.
//
// Only what the mesh needs that nothing else does: letting go of something it put there
// (novox/hq ADR 0189, issue 108), and holding every archive it keeps by a manifest so the store's
// own collector does not take it (novox/hq issue 253). Pushing is the builder's, through the container runtime; reading
// is every machine's, through its runtime. This is the one operation that belongs to the thing
// holding the records, because it is the only one that is a decision rather than a transfer.
//
// Copied from mesh-controller internal/artifacts at e04846e9 (novox/hq ADR 0571 clause 2): the seat mesh-artifact-sweep lets
// go of what it no longer keeps exactly as the controller's sweep did; the controller's copy goes with its lease.
package registry
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// Store is the artifact store at an address, as this machine reaches it.
type Store struct {
// Address is `host:port` — the store as the caller reaches it now, composed and never
// recorded (novox/hq 04-ISSUES/102).
Address string
// HTTP is the client used; nil is a client with a modest timeout.
HTTP *http.Client
// Spare says, for one repository, which manifests a kept index there names: the platform
// manifests an index being let go of must leave in place (novox/hq ADR 0257). Nil spares every
// platform manifest — an index is then let go of alone, as it was before.
Spare func(ctx context.Context, repository string) (map[string]bool, error)
}
// Gone is the answer when the store does not hold it: the outcome wanted, already true.
var Gone = errors.New("the store does not hold it")
// ErrNotOurs is a reference this sweep will not address: not the mesh's own, or naming nothing
// the store holds by digest.
//
// **A fact about the record, not about the store** (novox/hq issue 226). The two deserve opposite
// responses — skip one and go on, abandon the sweep for the other — and collapsing them into "an
// error" is how a cautious loop became one that did nothing while reporting the right number.
var ErrNotOurs = errors.New("not a reference into the mesh's artifact store")
// LetGo asks the store to drop one artifact the mesh recorded making.
//
// Takes a reference as the mesh records it — `artifact-store://<module>/<artifact>@sha256:…` for
// an image, `…/blobs/sha256:…` for an archive — because that is the identity every record uses,
// and composes the address here at the moment of use.
//
// An archive is let go of in two deletes, its holder manifest and then the blob's link (novox/hq
// issue 253); an image in one.
//
// Returns Gone when the store answers that it does not have it. That is not a failure: the sweep
// wants the artifact absent, and it is. It is distinguished from success only so a caller can say
// which of the two happened.
func (s Store) LetGo(ctx context.Context, reference string) error {
// **Strict, and deliberately** (novox/hq issue 226). Only a reference the mesh keeps in its
// own vocabulary is addressed here. `Recorded` would read `docker.io/library/registry@sha256:…`
// as the mesh's too — it cannot tell one registry host from another — so normalising belongs
// where the provenance is known, which is the sweep reading its own build records, not here
// where the only job is to refuse anything that is not plainly ours.
path, kept := catalogue.InArtifactStore(reference)
if !kept {
// Nothing the mesh put in its own store. Refused rather than attempted: composing a
// delete for a reference of unknown shape is how a sweep reaches something that is not
// the mesh's. Distinguished from a store that refuses, so a sweep skips this and goes on.
return fmt.Errorf("%w: %s", ErrNotOurs, reference)
}
if s.Address == "" {
return fmt.Errorf("this mesh has no artifact store on its network to ask about %s", reference)
}
repository, kind, digest, err := split(path)
if err != nil {
return err
}
if kind == "blobs" {
// **An archive's holder goes before the archive** (novox/hq issue 253): a manifest left
// naming the blob would keep its bytes through every collection while the record said
// collected. Gone here means the store has no such blob, so there is nothing to let go.
if err := s.letGoOfHolder(ctx, repository, digest); err != nil {
return err
}
}
if kind == "manifests" && s.Spare != nil {
// **An index's platform manifests go before the index** (novox/hq ADR 0257). Each was put
// under its own digest, so each is a manifest of the repository in its own right, and the
// store's collector keeps every manifest a repository holds: an index let go of alone frees
// no byte of the images it names. Before, not after, so a sweep cut short leaves the index
// to be asked about again, and with it the list of what is still to go.
if err := s.letGoOfPlatforms(ctx, repository, digest, reference); err != nil {
return err
}
}
return s.remove(ctx, s.url(repository, kind, digest), reference)
}
// letGoOfPlatforms deletes the manifests an index names, except those a kept index of the same
// repository also names. A manifest that is not an index names none.
func (s Store) letGoOfPlatforms(ctx context.Context, repository, digest, reference string) error {
children, err := s.Platforms(ctx, repository, digest)
if err != nil || len(children) == 0 {
// Gone: there is no index to read, and the delete that follows says so.
if errors.Is(err, Gone) {
return nil
}
return err
}
spared, err := s.Spare(ctx, repository)
if err != nil {
return fmt.Errorf("cannot say which platform manifests of %s a kept index names, so %s was kept: %w",
repository, reference, err)
}
for _, child := range children {
if spared[child] {
continue
}
if err := s.remove(ctx, s.url(repository, "manifests", child), reference+" (its "+child+")"); err != nil && !errors.Is(err, Gone) {
return err
}
}
return nil
}
// anyManifest is every manifest the store may hold, indexes included, so it answers with the
// document as it is.
var anyManifest = []string{
"application/vnd.oci.image.index.v1+json",
"application/vnd.docker.distribution.manifest.list.v2+json",
"application/vnd.oci.image.manifest.v1+json",
"application/vnd.docker.distribution.manifest.v2+json",
}
// Platforms is the manifests an index names, by digest; none for a manifest that is not an index.
// Gone when the store does not hold it.
func (s Store) Platforms(ctx context.Context, repository, digest string) ([]string, error) {
request, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url(repository, "manifests", digest), nil)
if err != nil {
return nil, err
}
request.Header.Set("Accept", strings.Join(anyManifest, ", "))
response, err := s.client().Do(request)
if err != nil {
return nil, err
}
defer response.Body.Close()
switch response.StatusCode {
case http.StatusOK:
case http.StatusNotFound:
return nil, Gone
default:
return nil, fmt.Errorf("the artifact store answered %s when asked for %s@%s", response.Status, repository, digest)
}
var document struct {
Manifests []struct {
Digest string `json:"digest"`
} `json:"manifests"`
}
if err := json.NewDecoder(io.LimitReader(response.Body, 4<<20)).Decode(&document); err != nil {
return nil, fmt.Errorf("%s@%s is not a manifest: %w", repository, digest, err)
}
var out []string
for _, m := range document.Manifests {
if strings.HasPrefix(m.Digest, "sha256:") {
out = append(out, m.Digest)
}
}
return out, nil
}
// SpareKeptIndexes is a Spare for a sweep, read whole before it starts: for every repository an
// image reference of `touching` is in, each kept manifest there and every platform manifest a kept
// index there names. Any read that fails is an error, and the sweep lets nothing go. A repository not
// read here is answered with an error, so an index in it is kept. A kept index the store no longer
// holds names nothing.
func (s Store) SpareKeptIndexes(ctx context.Context, kept, touching []string) (func(context.Context, string) (map[string]bool, error), error) {
imageIn := func(reference string) (repository, digest string, ok bool) {
path, ours := catalogue.InArtifactStore(reference)
if !ours {
return "", "", false
}
repository, kind, digest, err := split(path)
if err != nil || kind != "manifests" {
return "", "", false
}
return repository, digest, true
}
read := map[string]map[string]bool{}
for _, reference := range touching {
if repository, _, ok := imageIn(reference); ok {
read[repository] = map[string]bool{}
}
}
for _, reference := range kept {
repository, digest, ok := imageIn(reference)
spared, touched := read[repository]
if !ok || !touched {
continue
}
spared[digest] = true
children, err := s.Platforms(ctx, repository, digest)
if errors.Is(err, Gone) {
continue
}
if err != nil {
return nil, fmt.Errorf("reading the kept %s@%s: %w", repository, digest, err)
}
for _, c := range children {
spared[c] = true
}
}
return func(_ context.Context, repository string) (map[string]bool, error) {
spared, done := read[repository]
if !done {
return nil, fmt.Errorf("%s was not read before the sweep began", repository)
}
return spared, nil
}, nil
}
// remove asks the store to delete what is at url. Gone when it has no such thing.
func (s Store) remove(ctx context.Context, url, what string) error {
request, err := http.NewRequestWithContext(ctx, http.MethodDelete, url, nil)
if err != nil {
return err
}
response, err := s.client().Do(request)
if err != nil {
return err
}
defer response.Body.Close()
switch response.StatusCode {
case http.StatusAccepted, http.StatusOK, http.StatusNoContent:
return nil
case http.StatusNotFound:
return Gone
case http.StatusMethodNotAllowed:
// The registry was started without deletion enabled. Said plainly, because the remedy is
// a setting on the store's module and not anything about this artifact.
return fmt.Errorf(
"the artifact store refuses deletion: its server was started without it enabled "+
"(REGISTRY_STORAGE_DELETE_ENABLED), so nothing can be collected until the store "+
"module is applied again (novox/hq ADR 0189). Asking about %s", what)
default:
return fmt.Errorf("the artifact store answered %s for %s", response.Status, what)
}
}
// split reads a recorded path into the repository, which endpoint names the thing, and the digest.
//
// Two shapes, which are the two the mesh records: `<repository>@sha256:<hex>` is a manifest, and
// `<repository>/blobs/sha256:<hex>` is a blob.
func split(path string) (repository, kind, digest string, err error) {
if before, after, ok := strings.Cut(path, "@sha256:"); ok {
return before, "manifests", "sha256:" + after, nil
}
if before, after, ok := strings.Cut(path, "/blobs/sha256:"); ok {
return before, "blobs", "sha256:" + after, nil
}
return "", "", "", fmt.Errorf("%w: %q names nothing the store holds by digest", ErrNotOurs, path)
}
// IsIndex is whether a recorded reference is an index the store holds: one that names manifests. A
// blob, or a manifest that names none, is not. Gone when the store does not hold it.
func (s Store) IsIndex(ctx context.Context, reference string) (bool, error) {
path, ours := catalogue.InArtifactStore(reference)
if !ours {
return false, fmt.Errorf("%w: %s", ErrNotOurs, reference)
}
repository, kind, digest, err := split(path)
if err != nil || kind != "manifests" {
return false, err
}
children, err := s.Platforms(ctx, repository, digest)
return len(children) > 0, err
}
// HoldsManifest is whether the store holds the manifest a recorded image reference names.
func (s Store) HoldsManifest(ctx context.Context, reference string) (bool, error) {
path, ours := catalogue.InArtifactStore(reference)
if !ours {
return false, fmt.Errorf("%w: %s", ErrNotOurs, reference)
}
repository, kind, digest, err := split(path)
if err != nil {
return false, err
}
if kind != "manifests" {
return false, fmt.Errorf("%w: %s names a blob, not a manifest", ErrNotOurs, reference)
}
return s.has(ctx, s.url(repository, kind, digest), anyManifest...)
}
// Holds is whether the store holds what a recorded reference names: the blob of an archive, the manifest of an image.
// Gone when it does not; asks and changes nothing (ADR 0526 clause 3: a registration by hand is refused at bytes that
// are not there).
func (s Store) Holds(ctx context.Context, reference string) error {
path, ours := catalogue.InArtifactStore(reference)
if !ours {
return fmt.Errorf("%w: %s", ErrNotOurs, reference)
}
repository, kind, digest, err := split(path)
if err != nil {
return err
}
if s.Address == "" {
return fmt.Errorf("this mesh has no artifact store on its network to ask about %s", reference)
}
if kind == "blobs" {
_, err := s.blobSize(ctx, repository, digest)
return err
}
held, err := s.has(ctx, s.url(repository, kind, digest), anyManifest...)
if err != nil {
return err
}
if !held {
return Gone
}
return nil
}
@@ -0,0 +1,134 @@
package registry
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// Asking the store to let go of what the mesh no longer keeps (novox/hq ADR 0189, issue 108).
//
// A fake store records what it was asked to delete, so what is asserted is the mesh's decision
// and the shape of the request — not the registry's behaviour, which is the registry's to test.
func fakeStore(t *testing.T, answer int) (Store, *[]string) {
t.Helper()
var asked []string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodHead && strings.Contains(r.URL.Path, "/blobs/") {
// An archive's size, asked so its holder can be named (novox/hq issue 253). A store
// that does not have the thing does not have its blob either.
if answer == http.StatusNotFound {
w.WriteHeader(http.StatusNotFound)
return
}
w.Header().Set("Content-Length", "7")
w.WriteHeader(http.StatusOK)
return
}
if r.Method != http.MethodDelete {
t.Errorf("the store was asked %s %s; collecting is a delete", r.Method, r.URL.Path)
}
asked = append(asked, r.URL.Path)
w.WriteHeader(answer)
}))
t.Cleanup(server.Close)
return Store{Address: strings.TrimPrefix(server.URL, "http://")}, &asked
}
func TestAnImageAndAnArchiveAreAskedForAtTheirOwnEndpoints(t *testing.T) {
// The two shapes the mesh records: a manifest by digest, and a blob by digest. They are
// different endpoints, and asking at the wrong one answers 404 — which this would then
// record as collected, leaving the bytes on disk for ever while the record says otherwise.
store, asked := fakeStore(t, http.StatusAccepted)
ctx := context.Background()
image := catalogue.ArtifactStoreScheme + "web/app@sha256:abc123"
archive := catalogue.ArtifactStoreScheme + "web/config/blobs/sha256:def456"
if err := store.LetGo(ctx, image); err != nil {
t.Fatal(err)
}
if err := store.LetGo(ctx, archive); err != nil {
t.Fatal(err)
}
// The archive's holder goes first, then the archive (novox/hq issue 253).
_, holder := Holder("sha256:def456", 7)
want := []string{
"/v2/web/app/manifests/sha256:abc123",
"/v2/web/config/manifests/" + holder,
"/v2/web/config/blobs/sha256:def456",
}
if strings.Join(*asked, " ") != strings.Join(want, " ") {
t.Fatalf("the store was asked %v; want %v", *asked, want)
}
}
func TestAStoreThatDoesNotHaveItAnswersGone(t *testing.T) {
// The outcome wanted, already true. Told apart from success only so the sweep can say which
// happened; both are recorded, because retrying for ever is the thing to avoid.
store, _ := fakeStore(t, http.StatusNotFound)
err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/app@sha256:abc123")
if !errors.Is(err, Gone) {
t.Fatalf("a store that does not hold it answered %v, want Gone", err)
}
}
func TestAStoreWithDeletionOffSaysSoAndNamesTheRemedy(t *testing.T) {
// The registry answers 405 when it was started without deletion enabled. The remedy is a
// setting on the store's module, and saying "405" would send somebody to the wrong place.
store, _ := fakeStore(t, http.StatusMethodNotAllowed)
err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/app@sha256:abc123")
if err == nil {
t.Fatal("a store that refuses deletion was read as success")
}
if !strings.Contains(err.Error(), "REGISTRY_STORAGE_DELETE_ENABLED") {
t.Fatalf("the refusal does not name the remedy: %v", err)
}
}
func TestAReferenceThatIsNotTheMeshsOwnIsNeverAsked(t *testing.T) {
// The whole safety of the sweep is that it names only what the mesh recorded putting there.
// A reference of another shape — a vendor's image, a package version — is refused rather
// than composed into a delete somewhere that is not the mesh's store.
store, asked := fakeStore(t, http.StatusAccepted)
for _, reference := range []string{
"docker.io/library/registry@sha256:abc123",
"registry@sha256:abc123",
"1.4.2",
} {
if err := store.LetGo(context.Background(), reference); err == nil {
t.Errorf("%s was asked about; it is not a reference into the mesh's store", reference)
}
}
if len(*asked) != 0 {
t.Fatalf("the store was asked about %v", *asked)
}
}
// A reference this sweep will not address says so as ErrNotOurs, which is a fact about the
// record and not about the store (novox/hq issue 226).
//
// The sweep skips one and abandons itself for the other, so they cannot be the same error. The
// first live run met a reference recorded with the store's old address, read the refusal as "the
// store refuses everything", and collected none of the 1681 it had found.
func TestAReferenceThisSweepWillNotAddressIsToldApartFromAStoreRefusing(t *testing.T) {
store, asked := fakeStore(t, http.StatusAccepted)
for _, reference := range []string{
"docker.io/library/registry@sha256:abc123",
"127.0.0.1:5100/mesh-tools/build@sha256:abc123",
"1.4.2",
} {
err := store.LetGo(context.Background(), reference)
if !errors.Is(err, ErrNotOurs) {
t.Errorf("%s answered %v; a sweep must be able to skip it and go on", reference, err)
}
}
if len(*asked) != 0 {
t.Fatalf("the store was asked about %v", *asked)
}
}
+46 -11
View File
@@ -1,26 +1,52 @@
{
"module": "artifact-store-tools",
"version": "1",
"claims": [
{
"name": "mesh-artifact-sweep",
"scope": "mesh",
"serves": [
"artifacts",
"held",
"collect",
"mirrors",
"images"
]
}
],
"requires": [
"artifact-store"
],
"invokes": [
"seat:mesh-controller.artifacts",
"seat:mesh-controller.collect",
"seat:mesh-hand-acts.hand-act",
"mesh-catalog.catalog_modules",
"mesh-catalog.catalog_module"
],
"reads": [
"mesh-delivery-assigner.wants",
"mesh-delivery-builder.builds",
"mesh-delivery-builder.build-log",
"mesh-delivery-indexer.manifests",
"node-engine.have",
"mesh-controller.lease"
],
"state": [
"findings"
],
"tools": [
"artifact_store_repositories",
"artifact_store_usage",
"artifact_store_references",
"artifact_store_collect"
],
"config": {
"older-store-names": {
"type": "string",
"description": "The names the artifact store had before, as host names or host:port separated by spaces, or none when it had no other. artifact_store_references reads every definition under these as well as under the store's current name, so an image a definition pins under one is said to be named by a definition (novox/hq issue 323).",
"pattern": "none|[A-Za-z0-9.:-]+( [A-Za-z0-9.:-]+)*",
"default": "none"
}
},
"config": {
"older-store-names": {
"type": "string",
"description": "The names the artifact store had before, as host names or host:port separated by spaces, or none when it had no other. artifact_store_references reads every definition under these as well as under the store's current name, so an image a definition pins under one is said to be named by a definition (novox/hq issue 323).",
"pattern": "none|[A-Za-z0-9.:-]+( [A-Za-z0-9.:-]+)*",
"default": "none"
}
},
"resources": [
{
"id": "state",
@@ -36,6 +62,14 @@
"mode": "0600",
"trusted": false,
"template": "files/store-names"
},
{
"id": "store-address",
"type": "file",
"path": "${dir:state}/store-address",
"mode": "0600",
"trusted": false,
"template": "files/store-address"
}
],
"build": {
@@ -52,7 +86,8 @@
],
"env": {
"MESH_ARTIFACT_STORE_CONTAINER": "mesh-registry",
"MESH_ARTIFACT_STORE_NAMES_FILE": "${dir:state}/store-names"
"MESH_ARTIFACT_STORE_NAMES_FILE": "${dir:state}/store-names",
"MESH_ARTIFACT_STORE_ADDRESS_FILE": "${dir:state}/store-address"
}
},
{
+4 -1
View File
@@ -8,7 +8,10 @@
"provides": [
{
"name": "rootless-image-build",
"reach": "machine"
"reach": "machine",
"credential": {
"form": "none"
}
}
],
"resources": [
+1
View File
@@ -0,0 +1 @@
disk-load
+248
View File
@@ -0,0 +1,248 @@
package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/health"
"os"
"os/exec"
"sort"
"strings"
"sync"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// Store is the state `readings` as the keeper uses it.
type Store interface {
Get(key string) (json.RawMessage, error)
Put(key string, value any) error
Keys() ([]string, error)
}
// busStore is the state through the runtime.
type busStore struct{}
func (busStore) Get(key string) (json.RawMessage, error) {
e, err := stdio.State(StateReadings).Get(key)
if err != nil || e == nil {
return nil, err
}
return e.Value, nil
}
func (busStore) Put(key string, v any) error {
_, err := stdio.State(StateReadings).Put(key, v)
return err
}
func (busStore) Keys() ([]string, error) { return stdio.State(StateReadings).Keys() }
// Keeper measures the items of its node and keeps the readings.
type Keeper struct {
Node string
// Items is every data item the modules placed on this node declare, composed from the bus (frombus.DataItems).
Items func(ctx context.Context) (map[string][]catalogue.PlacedDataItem, error)
Measurer Measurer
Store Store
Say func(string, ...any)
mu sync.Mutex // one round at a time
}
// Round measures every item that is not a cache, one at a time, and writes the readings that changed. An item
// measured by a walk is walked only when walks is true or its last walk is older than WalkEvery: the hourly round
// reads counters, and a walk runs at most once a day (ADR 0233 §22).
func (k *Keeper) Round(ctx context.Context, walks bool) (written, kept int, err error) {
k.mu.Lock()
defer k.mu.Unlock()
items, err := k.Items(ctx)
if err != nil {
return 0, 0, err
}
modules := make([]string, 0, len(items))
for m := range items {
modules = append(modules, m)
}
sort.Strings(modules)
var failed []string
for _, module := range modules {
for _, it := range items[module] {
if it.Class == ClassCache {
continue
}
w, err := k.measureOne(ctx, it, walks)
if err != nil {
failed = append(failed, fmt.Sprintf("%s.%s: %v", module, it.ID, err))
continue
}
if w {
written++
} else {
kept++
}
}
}
retired, err := k.retireGone(items)
written += retired
if err != nil {
failed = append(failed, err.Error())
}
if len(failed) > 0 {
return written, kept, errors.New("readings not kept: " + strings.Join(failed, "; "))
}
return written, kept, nil
}
// retireGone marks retired every reading of this node whose item the node no longer declares — its module was unassigned
// there, or is no longer pulled in — with when and why, and keeps it: the record of what the module left behind, until a
// person deletes it (ADR 0233). It is cleared when the item is declared again (measureOne).
func (k *Keeper) retireGone(declared map[string][]catalogue.PlacedDataItem) (int, error) {
keys, err := k.Store.Keys()
if err != nil {
return 0, err
}
declaredNow := map[string]bool{}
for module, its := range declared {
for _, it := range its {
declaredNow[health.ReadingKey(k.Node, module, it.ID)] = true
}
}
now := k.Measurer.Now().UTC()
n := 0
for _, key := range keys {
if !strings.HasPrefix(key, k.Node+".") || declaredNow[key] {
continue
}
raw, err := k.Store.Get(key)
if err != nil || len(raw) == 0 {
continue
}
r, err := health.DecodeReading(raw)
if err != nil || r.Node != k.Node || !r.RetiredAt.IsZero() {
continue
}
r.RetiredAt = now
r.RetiredWhy = fmt.Sprintf("no longer declared on %s: its module was unassigned there, or is no longer pulled in", k.Node)
if err := k.Store.Put(key, r); err != nil {
return n, err
}
n++
}
return n, nil
}
func (k *Keeper) measureOne(ctx context.Context, it catalogue.PlacedDataItem, walks bool) (bool, error) {
key := health.ReadingKey(k.Node, it.Module, it.ID)
var prev *health.Reading
raw, err := k.Store.Get(key)
if err != nil {
return false, err
}
if len(raw) > 0 {
if p, err := health.DecodeReading(raw); err == nil {
prev = &p
}
}
now := k.Measurer.Now()
if prev != nil && prev.RetiredAt.IsZero() && it.Measure == catalogue.MeasureWalk && !walks && prev.MeasuredAt != nil && now.Sub(*prev.MeasuredAt) < WalkEvery {
return false, nil
}
m := k.Measurer.Measure(ctx, it)
next, changed := health.NextReading(prev, k.Node, it, m, now)
if prev != nil && !prev.RetiredAt.IsZero() {
// Declared again: it is no longer retired.
next.RetiredAt, next.RetiredWhy, changed = time.Time{}, "", true
}
if !changed {
return false, nil
}
return true, k.Store.Put(key, next)
}
// MeasureNow measures one item now, walk or not, and writes its reading when it changed.
func (k *Keeper) MeasureNow(ctx context.Context, module, item string) (health.Reading, bool, error) {
k.mu.Lock()
defer k.mu.Unlock()
items, err := k.Items(ctx)
if err != nil {
return health.Reading{}, false, err
}
for _, it := range items[module] {
if it.ID != item {
continue
}
if it.Class == ClassCache {
return health.Reading{}, false, fmt.Errorf("%s.%s is a cache, which is listed and never measured", module, item)
}
w, err := k.measureOne(ctx, it, true)
if err != nil {
return health.Reading{}, false, err
}
raw, err := k.Store.Get(health.ReadingKey(k.Node, module, item))
if err != nil {
return health.Reading{}, w, err
}
r, err := health.DecodeReading(raw)
return r, w, err
}
return health.Reading{}, false, fmt.Errorf("%s declares no data item %q on this machine", module, item)
}
// Loop runs a round after a few minutes and then every `every`, with the walks of the night's round when it is the
// hour after the night; ctx ends it.
func (k *Keeper) Loop(ctx context.Context, every time.Duration, first time.Duration) {
select {
case <-ctx.Done():
return
case <-time.After(first):
}
for {
w, kept, err := k.Round(ctx, false)
if err != nil {
k.Say("measuring the data declared here: %v", err)
} else {
k.Say("measured the data declared here: %d readings changed, %d unchanged", w, kept)
}
select {
case <-ctx.Done():
return
case <-time.After(every):
}
}
}
// RootRunner runs a command as root, through sudo without a prompt when this process is not root. A walk is bounded
// by WalkFor itself; the runner's own bound is a little longer.
func RootRunner(ctx context.Context, name string, args ...string) (string, error) {
ctx, cancel := context.WithTimeout(ctx, WalkFor+time.Minute)
defer cancel()
program, argv := name, args
if os.Getuid() != 0 {
program, argv = "sudo", append([]string{"-n", name}, args...)
}
var stdout, stderr bytes.Buffer
cmd := exec.CommandContext(ctx, program, argv...)
cmd.Stdout, cmd.Stderr = &stdout, &stderr
if err := cmd.Run(); err != nil {
said := strings.TrimSpace(stderr.String())
if said == "" {
said = strings.TrimSpace(stdout.String())
}
if program == "sudo" && strings.HasPrefix(said, "sudo:") {
return "", fmt.Errorf("%s needs root and the runtime's account may not run it without a prompt: %s", name, said)
}
lines := strings.Split(said, "\n")
if len(lines) > 3 {
lines = lines[len(lines)-3:]
}
if said == "" {
return "", fmt.Errorf("%s: %w", name, err)
}
return "", errors.New(strings.Join(lines, " / "))
}
return stdout.String(), nil
}
+43 -2
View File
@@ -1,20 +1,61 @@
// The disk-load module's Go bundle (novox/hq ADR 0188, ADR 0193, issue 312): a process the node's
// runtime launches and speaks MCP over stdio to. It reads the machine's storage load from /proc and
// /sys — iowait, pressure stall, each disk's busy share, throughput, IOPS, wait and queue depth, the
// filesystems' space and the processes doing the I/O — and changes nothing.
// filesystems' space and the processes doing the I/O — and changes nothing of the machine.
//
// It also measures the data items the modules of its node declare (ADR 0233) and keeps each reading and the item's
// seven-day peak as its own state `readings` (ADR 0571 clause 3), composed from the bus (no file).
package main
import (
"context"
"fmt"
"os"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
"git.novox.be/novox/mesh-sdk/go/compose/fromruntime"
)
func logf(format string, args ...any) { fmt.Fprintf(os.Stderr, "[disk-load] "+format+"\n", args...) }
func main() {
if err := stdio.Keeps(announce.State{Name: StateReadings}); err != nil {
logf("%v", err)
os.Exit(1)
}
node := os.Getenv("MESH_NODE")
k := &Keeper{Node: node, Store: busStore{}, Say: logf, Measurer: Measurer{Run: RootRunner, Now: time.Now},
Items: func(ctx context.Context) (map[string][]catalogue.PlacedDataItem, error) {
return placedItems(ctx, node)
}}
if node != "" {
every := time.Hour
if d, err := time.ParseDuration(os.Getenv("MESH_DISK_MEASURE_EVERY")); err == nil && d >= time.Minute {
every = d
}
go k.Loop(context.Background(), every, 3*time.Minute)
}
tools := append(Tools(Here()), ReadingTools(k)...)
// An empty name serves as the module the runtime names (MESH_SERVED_MODULE): disk-load.
if err := stdio.Serve("", Tools(Here())); err != nil {
if err := stdio.Serve("", tools); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
// placedItems is every data item the modules placed on node declare, composed from the bus as the node composes itself.
func placedItems(ctx context.Context, node string) (map[string][]catalogue.PlacedDataItem, error) {
items, unplaced, err := frombus.DataItems(ctx, fromruntime.Reader(), frombus.Mesh{}, frombus.Scope{Node: node})
if err != nil {
return nil, err
}
if len(unplaced) > 0 {
logf("data items left out as not placeable: %s", strings.Join(unplaced, "; "))
}
return items, nil
}
+215
View File
@@ -0,0 +1,215 @@
package main
// How an item is measured (novox/hq ADR 0233 §22 to §25), moved here from the backup holder, which only reads the
// readings now (ADR 0571 clause 3). Never an unbounded walk of something large.
import (
"context"
"fmt"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/health"
"regexp"
"strconv"
"strings"
"time"
)
// The bounds on a walk: a walk is for small items, and one that meets a large one stops and says so.
var (
WalkFor = 10 * time.Minute
WalkFiles = 2_000_000
WalkEvery = 23 * time.Hour
)
// walkCommand sums the files under a path and finds its newest change, in one walk, bounded by time and by files.
// One line out: "<bytes> <epoch seconds> <files>", then the status of find.
func walkCommand(path string) string {
return fmt.Sprintf("timeout %d find '%s' -xdev -printf '%%y %%s %%T@\\n' 2>/dev/null | head -n %d | "+
"awk 'BEGIN{s=0;m=0;n=0} {n++; if ($1==\"f\") s+=$2; if ($3>m) m=$3} END {printf \"%%d %%.0f %%d\\n\", s, m, n}'; "+
"echo \"${PIPESTATUS[0]:-0}\"", int(WalkFor.Seconds()), path, WalkFiles)
}
// topCommand is the newest change among a path and its top-level entries, and how many there are.
func topCommand(path string) string {
return "timeout 60 find '" + path + "' -maxdepth 1 -printf '%T@\\n' 2>/dev/null | " +
"awk 'BEGIN{m=0;n=0} {n++; if ($1>m) m=$1} END {printf \"%.0f %d\\n\", m, n-1}'"
}
func epochTime(s string) *time.Time {
epoch, err := strconv.ParseInt(s, 10, 64)
if err != nil || epoch <= 0 {
return nil
}
t := time.Unix(epoch, 0).UTC()
return &t
}
// Measurer measures items through a runner that has the root the walks need.
type Measurer struct {
Run Runner
Now func() time.Time
}
func (b Measurer) exists(ctx context.Context, path string) bool {
_, err := b.Run(ctx, "test", "-e", path)
return err == nil
}
// Measure is one item, measured now, by its own method.
func (b Measurer) Measure(ctx context.Context, it catalogue.PlacedDataItem) health.Measured {
at := b.Now().UTC()
m := health.Measured{MeasuredAt: &at}
if !b.exists(ctx, it.Path) {
m.Error = it.Path + " does not exist"
zero := int64(0)
m.SizeBytes, m.Precision = &zero, "exact"
return m
}
m.Redundancy = b.redundancyOf(ctx, it.Path)
switch it.Measure {
case catalogue.MeasureDataset, catalogue.MeasureShallow:
out, err := b.Run(ctx, "bash", "-c", topCommand(it.Path))
if f := strings.Fields(out); err == nil && len(f) == 2 {
m.LastWrite = epochTime(f[0])
} else if err != nil {
m.Error = err.Error()
}
m.Precision = "no size: the newest change among its top-level entries only"
if it.Measure == catalogue.MeasureShallow {
return m
}
out, err = b.Run(ctx, "zfs", "list", "-Hp", "-o", "name,used", it.Path)
f := strings.Fields(out)
if err != nil || len(f) != 2 {
m.Precision = "no size: it is on no ZFS dataset to read one from; the newest change among its top-level entries only"
return m
}
used, err := strconv.ParseInt(f[1], 10, 64)
if err != nil {
return m
}
m.SizeBytes = &used
m.Precision = "dataset " + f[0] + ": its whole size, from the filesystem's counters; the newest change among the top-level entries"
return m
}
out, err := b.Run(ctx, "bash", "-c", walkCommand(it.Path))
if err != nil {
m.Error = err.Error()
return m
}
lines := strings.Split(strings.TrimSpace(out), "\n")
f := strings.Fields(lines[0])
if len(f) != 3 || len(lines) < 2 {
m.Error = "the measurement said " + strings.TrimSpace(out)
return m
}
size, err1 := strconv.ParseInt(f[0], 10, 64)
files, err2 := strconv.Atoi(f[2])
if err1 != nil || err2 != nil {
m.Error = "the measurement said " + strings.TrimSpace(out)
return m
}
m.SizeBytes, m.LastWrite = &size, epochTime(f[1])
m.Precision = "exact"
if status := strings.TrimSpace(lines[len(lines)-1]); status == "124" || files >= WalkFiles {
m.Precision = fmt.Sprintf("partial: measured partially — stopped after %s or %d files; the size is a lower bound, "+
"and this item wants measure: dataset or shallow", WalkFor, WalkFiles)
}
return m
}
// redundancyOf is the redundant storage a path is on: a ZFS pool's health, an md array's members, a btrfs
// filesystem's error counters. Nil for storage with no redundancy it knows how to read.
func (b Measurer) redundancyOf(ctx context.Context, path string) *health.Redundancy {
out, err := b.Run(ctx, "findmnt", "-no", "SOURCE,FSTYPE", "--target", path)
if err != nil {
return nil
}
f := strings.Fields(out)
if len(f) < 2 {
return nil
}
source, fstype := f[0], f[1]
yes, no := true, false
switch {
case fstype == "zfs":
pool, _, _ := strings.Cut(source, "/")
r := &health.Redundancy{Kind: "zfs", Where: pool}
health, err := b.Run(ctx, "zpool", "list", "-H", "-o", "health", pool)
if err != nil {
r.Said = "zpool list: " + err.Error()
return r
}
status, err := b.Run(ctx, "zpool", "status", "-x", pool)
if err != nil {
r.Said = "zpool status: " + err.Error()
return r
}
health, status = strings.TrimSpace(health), strings.TrimSpace(status)
r.Said = "health " + health + "; " + firstLineOf(status)
if health == "ONLINE" && strings.Contains(status, "is healthy") {
r.Healthy = &yes
} else {
r.Healthy = &no
r.Said = "health " + health + "; " + oneLineOf(status)
}
return r
case strings.HasPrefix(source, "/dev/md"):
device := strings.TrimPrefix(source, "/dev/")
r := &health.Redundancy{Kind: "md", Where: device}
mdstat, err := b.Run(ctx, "cat", "/proc/mdstat")
if err != nil {
r.Said = err.Error()
return r
}
healthy, said, found := mdHealth(mdstat, device)
if !found {
r.Said = device + " is not in /proc/mdstat"
return r
}
r.Said = said
if healthy {
r.Healthy = &yes
} else {
r.Healthy = &no
}
return r
case fstype == "btrfs":
r := &health.Redundancy{Kind: "btrfs", Where: source}
stats, err := b.Run(ctx, "btrfs", "device", "stats", "--check", path)
if err != nil {
r.Healthy, r.Said = &no, "device errors: "+oneLineOf(err.Error())
return r
}
r.Healthy, r.Said = &yes, firstLineOf(stats)
return r
}
return nil
}
var mdMembers = regexp.MustCompile(`\[([U_]+)\]`)
// mdHealth reads one array's block of /proc/mdstat: healthy when every member is up and it is not recovering.
func mdHealth(mdstat, device string) (bool, string, bool) {
lines := strings.Split(mdstat, "\n")
for i, l := range lines {
if !strings.HasPrefix(l, device+" ") {
continue
}
block := l
for j := i + 1; j < len(lines) && strings.HasPrefix(lines[j], " "); j++ {
block += " " + strings.TrimSpace(lines[j])
}
members := mdMembers.FindStringSubmatch(block)
healthy := members != nil && !strings.Contains(members[1], "_") && !strings.Contains(block, "recovery")
return healthy, oneLineOf(block), true
}
return false, "", false
}
func firstLineOf(s string) string {
line, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
return line
}
func oneLineOf(s string) string { return strings.Join(strings.Fields(s), " ") }
@@ -0,0 +1,12 @@
package main
// The readings layout, their key and the peak are mesh-sdk's (health.Reading, health.NextReading; novox/hq ADR 0571
// clause 3): disk-load measures each placed data item of its node and keeps the readings as its own state.
import "git.novox.be/novox/mesh-sdk/go/health"
// StateReadings is the state disk-load keeps.
const StateReadings = health.ReadingsState
// ClassCache is the class of an item that is listed and never measured.
const ClassCache = "cache"
@@ -0,0 +1,144 @@
package main
import (
"context"
"encoding/json"
"os"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/health"
)
type memStore struct{ m map[string]json.RawMessage }
func (s *memStore) Get(k string) (json.RawMessage, error) { return s.m[k], nil }
func (s *memStore) Put(k string, v any) error {
raw, _ := json.Marshal(v)
s.m[k] = raw
return nil
}
func (s *memStore) Keys() ([]string, error) {
var out []string
for k := range s.m {
out = append(out, k)
}
return out, nil
}
// A round measures what the data file declares but caches, keeps the readings, writes nothing the second time, and a
// walk is not repeated within a day unless asked.
func TestARoundMeasuresTheDeclaredItemsAndWritesWhatChanged(t *testing.T) {
text := "# pg\nitem data valuable /srv/pg /srv/pg backup walk\nitem tmp cache /srv/tmp - none\n# bad-line-module\nitem logs rebuildable /srv/logs - none shallow\n"
placed, err := catalogue.ParseDataItems(text)
if err != nil {
t.Fatal(err)
}
walks := 0
run := func(_ context.Context, name string, args ...string) (string, error) {
switch {
case name == "test":
return "", nil
case name == "findmnt":
return "", os.ErrNotExist
case name == "bash" && strings.Contains(args[1], "-maxdepth 1"):
return "1760000000 2\n", nil
case name == "bash":
walks++
return "4096 1760000000 3\n0\n", nil
}
return "", os.ErrNotExist
}
now := time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC)
store := &memStore{m: map[string]json.RawMessage{}}
k := &Keeper{Node: "ace", Items: func(context.Context) (map[string][]catalogue.PlacedDataItem, error) { return placed, nil }, Store: store, Say: func(string, ...any) {},
Measurer: Measurer{Run: run, Now: func() time.Time { return now }}}
w, kept, err := k.Round(context.Background(), false)
if err != nil || w != 2 || kept != 0 || len(store.m) != 2 {
t.Fatalf("first round: %d written %d kept %v, keys %d", w, kept, err, len(store.m))
}
now = now.Add(time.Hour)
if w, kept, _ := k.Round(context.Background(), false); w != 0 || kept != 2 || walks != 1 {
t.Fatalf("second round: %d written, %d kept, %d walks; wanted none written and the walk not repeated", w, kept, walks)
}
if _, _, err := k.MeasureNow(context.Background(), "pg", "data"); err != nil || walks != 2 {
t.Fatalf("measure now: %v, %d walks", err, walks)
}
// An item no longer declared is kept, retired with when and why; declared again, it is not retired any more.
delete(placed, "pg")
if _, _, err := k.Round(context.Background(), false); err != nil {
t.Fatal(err)
}
gone, _ := health.DecodeReading(store.m["ace.pg.data"])
if gone.RetiredAt.IsZero() || !strings.Contains(gone.RetiredWhy, "no longer declared on ace") || gone.FirstSeen.IsZero() {
t.Fatalf("a reading of an item no longer declared: %+v", gone)
}
placed["pg"] = []catalogue.PlacedDataItem{{Module: "pg", ID: "data", Class: "valuable", Path: "/srv/pg", Measure: catalogue.MeasureWalk}}
if _, _, err := k.Round(context.Background(), false); err != nil {
t.Fatal(err)
}
if back, _ := health.DecodeReading(store.m["ace.pg.data"]); !back.RetiredAt.IsZero() {
t.Fatalf("a reading declared again is still retired: %+v", back)
}
if _, _, err := k.MeasureNow(context.Background(), "pg", "tmp"); err == nil {
t.Fatal("a cache was measured")
}
}
// The redundant storage under an item is read: a ZFS pool healthy, then degraded; an md array with a member missing; and
// plain storage, which is no redundancy at all. A dataset is measured from the filesystem's counters and a shallow item
// from its top-level entries only; a walk that meets more than its bound stops and says so.
func TestTheStorageUnderAnItemAndTheLargeMethodsAreRead(t *testing.T) {
poolHealth, statusX, fs := "ONLINE\n", "pool 'storage' is healthy\n", "storage/media zfs\n"
run := func(_ context.Context, name string, args ...string) (string, error) {
line := name + " " + strings.Join(args, " ")
switch {
case name == "test":
return "", nil
case name == "findmnt":
return fs, nil
case name == "zpool" && args[0] == "list":
return poolHealth, nil
case name == "zpool":
return statusX, nil
case name == "cat":
return "Personalities : [raid1]\nmd127 : active raid1 sdb1[1] sda1[0]\n 976630464 blocks super 1.2 [2/1] [U_]\n\nunused devices: <none>\n", nil
case name == "zfs" && args[len(args)-1] == "/storage/media/movies":
return "storage/media\t97067690722560\n", nil
case name == "zfs":
return "", os.ErrNotExist
case name == "bash" && strings.Contains(line, "-maxdepth 1"):
return "1759744800 12673\n", nil
case name == "bash":
return "123 1759700000 2000000\n141\n", nil
}
return "", nil
}
m := Measurer{Run: run, Now: time.Now}
ctx := context.Background()
got := m.Measure(ctx, catalogue.PlacedDataItem{Module: "media", ID: "movies", Path: "/storage/media/movies", Measure: catalogue.MeasureDataset})
if got.SizeBytes == nil || *got.SizeBytes != 97067690722560 || !strings.HasPrefix(got.Precision, "dataset storage/media") ||
got.LastWrite == nil || got.Redundancy == nil || got.Redundancy.Kind != "zfs" || got.Redundancy.Healthy == nil || !*got.Redundancy.Healthy {
t.Fatalf("the library from its dataset: %+v", got)
}
poolHealth, statusX = "DEGRADED\n", " pool: storage\n state: DEGRADED\nstatus: One or more devices has been removed\n"
if r := m.redundancyOf(ctx, "/storage/media/movies"); r.Healthy == nil || *r.Healthy || !strings.Contains(r.Said, "DEGRADED") {
t.Fatalf("a degraded pool: %+v", r)
}
fs = "/dev/md127 ext4\n"
if r := m.redundancyOf(ctx, "/srv/x"); r == nil || r.Kind != "md" || r.Healthy == nil || *r.Healthy {
t.Fatalf("an array missing a member: %+v", r)
}
fs = "/dev/nvme0n1p3 ext4\n"
if r := m.redundancyOf(ctx, "/var/lib/x"); r != nil {
t.Fatalf("plain storage read as redundant: %+v", r)
}
if s := m.Measure(ctx, catalogue.PlacedDataItem{Path: "/mnt/plex/config", Measure: catalogue.MeasureShallow}); s.SizeBytes != nil || !strings.HasPrefix(s.Precision, "no size") {
t.Fatalf("a shallow item: %+v", s)
}
if w := m.Measure(ctx, catalogue.PlacedDataItem{Path: "/srv/big", Measure: catalogue.MeasureWalk}); !strings.HasPrefix(w.Precision, "partial") {
t.Fatalf("a walk stopped at its bound: %+v", w)
}
}
@@ -0,0 +1,93 @@
package main
import (
"context"
"fmt"
"sort"
"strings"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/health"
)
// ReadingTools are the tools over the state `readings`: every key, one key, how full the state is, and one item
// measured now (a module's tools are its command line).
func ReadingTools(k *Keeper) []stdio.Tool {
return []stdio.Tool{
{
Name: "disk_readings",
Description: "The data readings disk-load keeps for this machine's data items (ADR 0233, ADR 0571 clause 3): every key " +
"`<node>.<module>.<item>` with its size, newest write, measured-at, precision and seven-day peak, or one key with " +
"all it holds when `key` is given; and how full the state is: keys kept, their bytes, and the cap the bus puts on " +
"a state (none declared). Filter by `module`. (r)",
Input: map[string]any{
"key": map[string]any{"type": "string", "description": "one key, <node>.<module>.<item>, with everything it holds"},
"module": map[string]any{"type": "string", "description": "only this module's items"},
},
Run: func(args map[string]any) (any, error) {
if key := text(args, "key"); key != "" {
raw, err := k.Store.Get(key)
if err != nil {
return nil, err
}
if len(raw) == 0 {
return nil, fmt.Errorf("no reading is kept at %q", key)
}
r, err := health.DecodeReading(raw)
return r, err
}
keys, err := k.Store.Keys()
if err != nil {
return nil, err
}
sort.Strings(keys)
module := text(args, "module")
type line struct {
Key string `json:"key"`
SizeBytes *int64 `json:"size_bytes,omitempty"`
PeakBytes *int64 `json:"peak_bytes,omitempty"`
Precision string `json:"precision,omitempty"`
Error string `json:"error,omitempty"`
Reading health.Reading `json:"-"`
}
var lines []line
total := 0
for _, key := range keys {
raw, err := k.Store.Get(key)
if err != nil {
return nil, err
}
total += len(raw)
r, err := health.DecodeReading(raw)
if err != nil || (module != "" && r.Module != module) {
continue
}
lines = append(lines, line{Key: key, SizeBytes: r.SizeBytes, PeakBytes: r.Peak, Precision: r.Precision, Error: r.Error})
}
return map[string]any{"readings": lines, "fill": map[string]any{"keys": len(keys), "bytes": total,
"cap": "none declared: the bus's own limit on a state applies"}}, nil
},
},
{
Name: "disk_measure",
Description: "Measure one data item of this machine now, whatever its method (a walk too), and write its reading when it " +
"changed. Reads the disks as root: a walk is bounded by time and files and says when it was partial.",
Input: map[string]any{
"module": map[string]any{"type": "string", "description": "the module that declares the item"},
"item": map[string]any{"type": "string", "description": "the item's id"},
},
Run: func(args map[string]any) (any, error) {
module, item := text(args, "module"), text(args, "item")
if module == "" || item == "" {
return nil, fmt.Errorf("module and item are required")
}
r, written, err := k.MeasureNow(context.Background(), module, item)
if err != nil {
return nil, err
}
return map[string]any{"written": written, "reading": r, "key": health.ReadingKey(k.Node, module, item),
"note": strings.TrimSpace(map[bool]string{true: "the reading changed and was written", false: "the reading had not changed"}[written])}, nil
},
},
}
}
@@ -3,6 +3,7 @@ package main
import (
"encoding/json"
"os"
"regexp"
"sort"
"strings"
"testing"
@@ -35,9 +36,11 @@ func TestTheManifestNamesExactlyTheToolsTheBundleServesAndNoMachine(t *testing.T
t.Fatal(err)
}
var served []string
for _, tl := range Tools(newFake(t).machine()) {
k := &Keeper{Store: &memStore{m: map[string]json.RawMessage{}}}
for _, tl := range append(Tools(newFake(t).machine()), ReadingTools(k)...) {
served = append(served, tl.Name)
if !strings.HasSuffix(tl.Description, "(r)") {
// disk_measure writes the module's own state of readings and nothing else of the machine.
if !strings.HasSuffix(tl.Description, "(r)") && tl.Name != "disk_measure" {
t.Errorf("%s only reads and does not say so", tl.Name)
}
}
@@ -50,7 +53,7 @@ func TestTheManifestNamesExactlyTheToolsTheBundleServesAndNoMachine(t *testing.T
t.Errorf("the module only reads, and declares %d resources", len(m.Resources))
}
for _, banned := range []string{"g14", "novox", "ace", "shanks", "jochen", "/home/"} {
if strings.Contains(string(raw), banned) {
if regexp.MustCompile(`(^|[^a-z0-9-])` + regexp.QuoteMeta(banned) + `($|[^a-z0-9-])`).Match(raw) {
t.Errorf("the manifest says %q", banned)
}
}
+2 -2
View File
@@ -1,5 +1,5 @@
module diskload
go 1.22
go 1.26.0
require git.novox.be/novox/mesh-sdk/go v0.1.6
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0
+14 -2
View File
@@ -1,2 +1,14 @@
git.novox.be/novox/mesh-sdk/go v0.1.6 h1:9qzdYONYbJdWcu6sxQcq9v1LI0JxcfkiKYkMUzJSkVQ=
git.novox.be/novox/mesh-sdk/go v0.1.6/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
+15 -2
View File
@@ -7,7 +7,9 @@
"disk_filesystems",
"disk_folders",
"disk_devices",
"cpu_top"
"cpu_top",
"disk_readings",
"disk_measure"
],
"build": {
"artifacts": [
@@ -23,5 +25,16 @@
]
}
]
}
},
"state": [
"readings"
],
"reads": [
"mesh-config.resolved",
"mesh-delivery-assigner.placement",
"mesh-delivery-builder.builds",
"mesh-delivery-indexer.manifests",
"mesh-delivery-indexer.modules",
"node-engine.have"
]
}
+4 -1
View File
@@ -4,7 +4,10 @@
"provides": [
{
"name": "artifact-store",
"scope": "mesh"
"scope": "mesh",
"credential": {
"form": "none"
}
}
],
"claims": [
+2 -2
View File
@@ -1,5 +1,5 @@
module dnsmasq
go 1.25.0
go 1.26.0
require git.novox.be/novox/mesh-sdk/go v0.1.7
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0
+2 -2
View File
@@ -1,2 +1,2 @@
git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w=
git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+4 -1
View File
@@ -9,7 +9,10 @@
{
"name": "wildcard-resolution",
"scope": "mesh",
"identity": false
"identity": false,
"credential": {
"form": "none"
}
}
],
"requires": [
@@ -9,7 +9,7 @@ package main
//
// - every image a container on this machine uses, in any state;
// - every image this machine's declaration names — what the mesh would send it now and what it was
// last sent — as the controller's `images` verb answers. **No answer, no removal**: what the
// last sent — as the mesh-artifact-sweep seat's `images` verb answers. **No answer, no removal**: what the
// declaration names is the one thing that cannot be guessed;
// - every image younger than the floor (seven days, the weekly prune's week);
// - in each line of images holding one of those, the newest image besides them: the previous
@@ -43,7 +43,7 @@ type PruneImagesAsk struct {
Limit int
}
// declaredImages is the controller's `images` answer (novox/hq ADR 0251 §5).
// declaredImages is the mesh-artifact-sweep seat's `images` answer (novox/hq ADR 0251 §5).
type declaredImages struct {
Node string `json:"node"`
Images []struct {
@@ -129,7 +129,7 @@ func digestOf(ref string) string {
return ""
}
// askDeclared asks the controller what this machine's declarations name. An error means unknown.
// askDeclared asks the mesh-artifact-sweep seat what this machine's declarations name. An error means unknown.
func (c *Client) askDeclared() (*declaredImages, error) {
if c.Node == "" {
return nil, errors.New("the runtime did not say which machine this is (MESH_NODE is empty)")
@@ -137,20 +137,20 @@ func (c *Client) askDeclared() (*declaredImages, error) {
if c.Ask == nil {
return nil, errors.New("this bundle cannot ask the mesh")
}
raw, err := c.Ask("seat:mesh-controller.images", map[string]any{"node": c.Node})
raw, err := c.Ask("seat:mesh-artifact-sweep.images", map[string]any{"node": c.Node})
if err != nil {
return nil, fmt.Errorf("the controller's images verb did not answer: %w", err)
return nil, fmt.Errorf("the mesh-artifact-sweep seat's images verb did not answer: %w", err)
}
answer, output, ok := answerOf(raw)
if !ok {
return nil, fmt.Errorf("the controller refused images: %s", firstLine(output))
return nil, fmt.Errorf("the mesh-artifact-sweep seat refused images: %s", firstLine(output))
}
var d declaredImages
if err := json.Unmarshal(answer, &d); err != nil {
return nil, fmt.Errorf("the controller's images answer is not readable: %v", err)
return nil, fmt.Errorf("the mesh-artifact-sweep seat's images answer is not readable: %v", err)
}
if d.Node != "" && d.Node != c.Node {
return nil, fmt.Errorf("the controller answered for %q, not for this machine %q", d.Node, c.Node)
return nil, fmt.Errorf("the mesh-artifact-sweep seat answered for %q, not for this machine %q", d.Node, c.Node)
}
return &d, nil
}
@@ -43,7 +43,7 @@ func imagesMachine() *fake {
func declaring(t *testing.T, answer string) Asker {
return func(key string, body any) (json.RawMessage, error) {
if key != "seat:mesh-controller.images" {
if key != "seat:mesh-artifact-sweep.images" {
t.Errorf("asked %s", key)
}
if b, _ := body.(map[string]any); b["node"] != "laptop" {
+2 -2
View File
@@ -1,5 +1,5 @@
module docker
go 1.22
go 1.26.0
require git.novox.be/novox/mesh-sdk/go v0.1.6
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0
+2 -2
View File
@@ -1,2 +1,2 @@
git.novox.be/novox/mesh-sdk/go v0.1.6 h1:9qzdYONYbJdWcu6sxQcq9v1LI0JxcfkiKYkMUzJSkVQ=
git.novox.be/novox/mesh-sdk/go v0.1.6/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+1 -1
View File
@@ -7,7 +7,7 @@
"privileged"
],
"invokes": [
"seat:mesh-controller.images"
"seat:mesh-artifact-sweep.images"
],
"claims": [
{
@@ -192,17 +192,14 @@ func (r *teamRig) pass() []provider.Outcome {
return out
}
// bind makes module on node bind gitea's forge-account in the assignments mirror.
// bind makes module on node bind gitea's forge-account in the seat mesh-provisioner's plans.
func (r *teamRig) bind(node, module string) {
r.bus.set(provider.AssignmentsBucket, node+"."+module, []byte(fmt.Sprintf(
`{"node":%q,"module":%q,"provisioning":{"provisions":[{"provision":"forge-account","provider":"novox","provider-module":"gitea","as":"mesh_%s"}]}}`,
node, module, strings.ReplaceAll(node+"_"+module, "-", "_"))))
r.bus.bindPlan(node, module, "mesh_"+strings.ReplaceAll(node+"_"+module, "-", "_"))
}
// unbind is the operator's yes to retiring module's forge-account on node (ADR 0570 clause 5).
func (r *teamRig) unbind(node, module string) {
r.bus.mu.Lock()
delete(r.bus.buckets[provider.AssignmentsBucket], node+"."+module)
r.bus.mu.Unlock()
r.bus.retire(node, module)
}
// key reports a key in module's state keys, of the kind and for the account given.
@@ -30,6 +30,7 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/provider"
"git.novox.be/novox/mesh-sdk/go/provisions"
)
const bundleVar = "MESH_FORGE_ACCOUNT_TEST_BUNDLE"
@@ -80,7 +81,7 @@ func startBus(t *testing.T) (*runtime, context.CancelFunc) {
rt := &runtime{t: t, js: js, kvs: map[string]jetstream.KeyValue{}, ctx: ctx, wait: map[string]chan json.RawMessage{}}
// The controller's job: a bucket per state the module declares (bound) or reads (the others).
for state, bucket := range map[string]string{"bound": "gitea_bound", "ssh-client.keys": "ssh-client_keys",
"mesh-controller.assignments": "mesh-controller_assignments"} {
"mesh-provisioner.plans": "mesh-provisioner_plans", "mesh-provisioner.provisions": "mesh-provisioner_provisions"} {
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucket, History: 1})
if err != nil {
t.Fatal(err)
@@ -296,9 +297,11 @@ func TestTheProviderWholeAnswersAMachineThatReportedItsKey(t *testing.T) {
}
rt, cancel := startBus(t)
t.Cleanup(cancel)
rt.put("mesh-controller.assignments", "g14.ssh-client", map[string]any{"node": "g14", "module": "ssh-client",
"provisioning": map[string]any{"provisions": []map[string]any{{"provision": "forge-account", "provider": "novox",
rt.put("mesh-provisioner.plans", "g14.ssh-client", map[string]any{"v": 1, "node": "g14", "module": "ssh-client",
"plan": map[string]any{"provisions": []map[string]any{{"provision": "forge-account", "provider": "novox",
"provider-module": "gitea", "as": "mesh_g14_ssh_client"}}}})
rt.put("mesh-provisioner.provisions", "g14.ssh-client.forge-account",
json.RawMessage(provisionKey("g14", "ssh-client", provisions.Asked, nil)))
rt.put("ssh-client.keys", "g14.nox", map[string]string{"node": "g14", "account": "nox", "kind": "machine",
"public": pubA + " nox@g14 (mesh)", "fingerprint": "SHA256:x", "at": "2026-10-16T10:00:00Z"})
rt.start([]string{"MESH_NODE=novox", "MESH_MODULE=gitea", "MESH_SERVED_MODULE=gitea",
@@ -25,7 +25,7 @@ func TestConfigFromEnvNamesWhatIsMissing(t *testing.T) {
}
// Issue 955: the runtime grants a module only the states its manifest `reads`, and the provider watches the machines'
// keys and, through the SDK's provider, the controller's assignments (where it learns its consumers). Run as the
// keys and, through the SDK's provider, the seat mesh-provisioner's plans and provisions (where it learns its consumers, ADR 0570). Run as the
// bus test does, with every bucket made, a missing grant passes unseen; the manifest is read here instead.
func TestTheManifestReadsEveryStateTheProviderWatches(t *testing.T) {
raw, err := os.ReadFile("../../module.json")
@@ -38,7 +38,7 @@ func TestTheManifestReadsEveryStateTheProviderWatches(t *testing.T) {
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
for _, state := range []string{"mesh-controller.assignments", "ssh-client.keys"} {
for _, state := range []string{"mesh-provisioner.plans", "mesh-provisioner.provisions", "ssh-client.keys"} {
if !slices.Contains(m.Reads, state) {
t.Errorf("gitea's reads %v lack %q, so the runtime refuses its watch", m.Reads, state)
}
@@ -184,7 +184,9 @@ func (s *Service) Make(ctx context.Context, c provider.Consumer) error {
s.mu.Unlock()
if !loaded {
// After a restart the first pass can come before the keys arrive: nothing is removed or deleted on that.
return fmt.Errorf("the keys of %s are not read yet, so nothing is made, removed or deleted", source)
// Not ready, not failed: the entry waits and keeps serving what it served (provider.ErrNotReady).
return fmt.Errorf("the keys of %s are not read yet, so nothing is made, removed or deleted: %w", source,
provider.ErrNotReady)
}
have := s.keysFrom(source, node, kind)
if len(have) == 0 {
@@ -17,6 +17,7 @@ import (
"time"
"git.novox.be/novox/mesh-sdk/go/provider"
"git.novox.be/novox/mesh-sdk/go/provisions"
"gitea/internal/forgehttp"
)
@@ -78,6 +79,22 @@ func (b *fakeBus) entry(t *testing.T, key string) provider.Entry {
}
return e
}
// serves is whether gitea's entry under key serves its consumer: there, in state (provided when none is given) and with
// values (ADR 0570 clause 3: an entry the provider could not make is kept, failed, serving nothing; one it is not
// ready to make waits and keeps serving).
func (b *fakeBus) serves(key string, state ...string) bool {
want := provisions.Provided
if len(state) > 0 {
want = state[0]
}
b.mu.Lock()
raw, ok := b.buckets[provider.BucketOf("gitea")][key]
b.mu.Unlock()
var e provider.Entry
return ok && json.Unmarshal(raw, &e) == nil && e.State == want && len(e.Values) > 0
}
func (b *fakeBus) has(key string) bool {
b.mu.Lock()
defer b.mu.Unlock()
@@ -85,11 +102,41 @@ func (b *fakeBus) has(key string) bool {
return ok
}
// bindClient makes ssh-client on node bind gitea's forge-account in the assignments mirror.
// bindClient makes ssh-client on node bind gitea's forge-account in the seat mesh-provisioner's plans.
func (b *fakeBus) bindClient(node string) {
b.set(provider.AssignmentsBucket, node+".ssh-client", []byte(fmt.Sprintf(
`{"node":%q,"module":"ssh-client","provisioning":{"provisions":[{"provision":"forge-account","provider":"novox","provider-module":"gitea","as":"mesh_%s_ssh_client"}]}}`,
node, node)))
b.bindPlan(node, "ssh-client", "mesh_"+node+"_ssh_client")
}
// bindPlan makes module on node bind gitea's forge-account as the seat mesh-provisioner keeps it: its plan, and the
// provision key asked of gitea on novox (novox/hq ADR 0570 clauses 1 to 3).
func (b *fakeBus) bindPlan(node, module, as string) {
b.set(provisions.PlansBucket, node+"."+module, []byte(fmt.Sprintf(
`{"v":1,"node":%q,"module":%q,"plan":{"provisions":[{"provision":"forge-account","provider":"novox","provider-module":"gitea","as":%q}]}}`,
node, module, as)))
b.set(provisions.ProvisionsBucket, node+"."+module+".forge-account", provisionKey(node, module, provisions.Asked, nil))
}
// retire is the operator's yes to the retiring of module's forge-account on node: its plan gone, its key retiring
// with the warrant (ADR 0570 clause 5), the only way the provider lets an entry go.
func (b *fakeBus) retire(node, module string) {
b.mu.Lock()
delete(b.buckets[provisions.PlansBucket], node+"."+module)
b.mu.Unlock()
b.set(provisions.ProvisionsBucket, node+"."+module+".forge-account", provisionKey(node, module, provisions.Retiring,
&provisions.Warrant{Ask: "ask-1", Option: "yes", By: "operator", At: time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC)}))
}
func provisionKey(node, module, state string, w *provisions.Warrant) []byte {
pr := provisions.Provision{Provision: "forge-account", State: state, Warrant: w,
Consumer: provisions.End{Node: node, Module: module}, Provider: provisions.End{Node: "novox", Module: "gitea"}}
if state == provisions.Retiring {
pr.Why = "no want needs it"
}
raw, err := provisions.Encode(pr)
if err != nil {
panic(err)
}
return raw
}
// fakeForge is gitea's API as far as the provider uses it.
@@ -343,9 +390,7 @@ func TestAConsumerThatNoLongerBindsLosesItsMeshKeysAndKeepsItsAccount(t *testing
r.forge.mu.Lock()
r.forge.held["nox-agent"] = append(r.forge.held["nox-agent"], map[string]any{"id": int64(99), "title": "my own", "key": pubB})
r.forge.mu.Unlock()
r.bus.mu.Lock()
delete(r.bus.buckets[provider.AssignmentsBucket], "g14.ssh-client")
r.bus.mu.Unlock()
r.bus.retire("g14", "ssh-client")
r.pass()
if r.bus.has("g14.ssh-client.forge-account") {
t.Fatal("the entry stayed")
@@ -460,7 +505,7 @@ func TestADeletedKeyEntryRemovesTheMeshKeyAndTheBoundEntryButNotTheOperatorsKey(
if got := r.forge.titles("nox-agent"); len(got) != 1 || got[0] != "my own" {
t.Fatalf("keys = %v", got)
}
if r.bus.has("g14.ssh-client.forge-account") {
if r.bus.serves("g14.ssh-client.forge-account") {
t.Fatal("the entry still serves a machine whose key is gone")
}
}
@@ -482,8 +527,8 @@ func TestAPassBeforeTheKeysAreLoadedRemovesAndDeletesNothing(t *testing.T) {
if got := strings.Join(r.forge.titles("nox-agent"), ","); got != "mesh:g14.nox,my own" {
t.Fatalf("keys = %s", got)
}
if !r.bus.has("g14.ssh-client.forge-account") {
t.Fatal("the entry was deleted before the keys were read")
if !r.bus.serves("g14.ssh-client.forge-account", provisions.Waits) {
t.Fatal("the entry stopped serving before the keys were read; it waits and keeps serving")
}
if len(out) != 1 || !strings.Contains(out[0].Why, "not read yet") {
t.Fatalf("the wait should be said: %+v", out)
@@ -493,7 +538,7 @@ func TestAPassBeforeTheKeysAreLoadedRemovesAndDeletesNothing(t *testing.T) {
r.svc.loaded = true
r.svc.mu.Unlock()
r.pass()
if got := strings.Join(r.forge.titles("nox-agent"), ","); got != "my own" || r.bus.has("g14.ssh-client.forge-account") {
if got := strings.Join(r.forge.titles("nox-agent"), ","); got != "my own" || r.bus.serves("g14.ssh-client.forge-account") {
t.Fatalf("keys = %s", got)
}
}
@@ -515,7 +560,7 @@ func TestKeepRunningStartsTheProviderAgainAfterARefusedWatch(t *testing.T) {
r.svc.KeepRunning(ctx, func(ctx context.Context) error {
calls++
if calls <= 2 {
return errors.New(`watching mesh-controller_assignments: gitea reads no state called "mesh-controller.assignments"`)
return errors.New(`watching mesh-provisioner_plans: gitea reads no state called "mesh-provisioner.plans"`)
}
close(serving)
<-ctx.Done()
@@ -535,7 +580,7 @@ func TestKeepRunningStartsTheProviderAgainAfterARefusedWatch(t *testing.T) {
if len(pauses) != 2 || pauses[0] != 5*time.Second || pauses[1] != 10*time.Second {
t.Fatalf("the backoff is %v", pauses)
}
if len(said) == 0 || !strings.Contains(said[0], "mesh-controller.assignments") {
if len(said) == 0 || !strings.Contains(said[0], "mesh-provisioner.plans") {
t.Fatalf("a refusal is said loudly: %v", said)
}
loud := 0
@@ -142,7 +142,7 @@ func TestABuildModulesStateThatNeverAnswersHoldsBackOnlyItsOwnEntries(t *testing
}
// 1482/AC4 (the provider's half): the manifest reads the states of both build modules beside ssh-client's and the
// controller's assignments; the provider watches exactly the modules the code names as builders.
// seat mesh-provisioner's plans; the provider watches exactly the modules the code names as builders.
func TestTheManifestReadsTheBuildModulesKeys(t *testing.T) {
raw, err := os.ReadFile("../../module.json")
if err != nil {
@@ -154,7 +154,7 @@ func TestTheManifestReadsTheBuildModulesKeys(t *testing.T) {
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
for _, state := range []string{"mesh-controller.assignments", "ssh-client.keys", "mesh-build-go.keys", "mesh-build-node.keys", "mesh-build-image.keys"} {
for _, state := range []string{"mesh-provisioner.plans", "mesh-provisioner.provisions", "ssh-client.keys", "mesh-build-go.keys", "mesh-build-node.keys", "mesh-build-image.keys"} {
if !slices.Contains(m.Reads, state) {
t.Errorf("gitea's reads %v lack %q", m.Reads, state)
}
+1 -1
View File
@@ -3,7 +3,7 @@ module gitea
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
golang.org/x/crypto v0.57.0
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
+5 -1
View File
@@ -400,6 +400,7 @@
"provides": [
{
"name": "npm-package-registry",
"credential": {"form": "pair"},
"scope": "mesh",
"identity": {
"max": 40,
@@ -408,6 +409,7 @@
},
{
"name": "git",
"credential": {"form": "none"},
"scope": "mesh"
},
{
@@ -420,6 +422,7 @@
},
{
"name": "forge-account",
"credential": {"form": "none"},
"scope": "mesh"
}
],
@@ -564,7 +567,8 @@
"announced"
],
"reads": [
"mesh-controller.assignments",
"mesh-provisioner.plans",
"mesh-provisioner.provisions",
"ssh-client.keys",
"mesh-build-go.keys",
"mesh-build-node.keys",
+4 -1
View File
@@ -7,7 +7,10 @@
"provides": [
{
"name": "go-toolchain",
"reach": "machine"
"reach": "machine",
"credential": {
"form": "none"
}
}
],
"resources": [
+3
View File
@@ -7,6 +7,9 @@
"scope": "mesh",
"identity": {
"in": "an InfluxDB v1 authorization"
},
"credential": {
"form": "pair"
}
}
],
@@ -15,6 +15,7 @@ import (
"git.novox.be/novox/mesh-sdk/go/provider"
"git.novox.be/novox/mesh-sdk/go/provider/harness"
"git.novox.be/novox/mesh-sdk/go/provisions"
)
// grantGives writes a grant file (MESH_RECEIVES) naming consumers the plans do not move (issue 937): each one's
@@ -38,11 +39,33 @@ func grantGives(t *testing.T, given ...map[string]any) string {
return at
}
// noOpPlan puts one assignment the bus can read that binds nothing of keycloak's, so the moved path has plans it
// noOpPlan puts one plan the bus can read that binds nothing of keycloak's, so the moved path has plans it
// could read (m.asked answers "read", issue 241) even when nothing is moved.
func noOpPlan(bus *bucketBus, node, module string) {
raw, _ := json.Marshal(map[string]any{"node": node, "module": module})
_ = bus.Put(ctx, provider.AssignmentsBucket, node+"."+module, raw)
seatPlan(bus, node, module, map[string]any{})
}
// seatPlan puts a module's plan as the seat mesh-provisioner keeps it, and for each of its provisions served by
// keycloak on novox the key the seat composes, asked (novox/hq ADR 0570 clauses 1 to 3).
func seatPlan(bus *bucketBus, node, module string, plan map[string]any) {
raw, _ := json.Marshal(map[string]any{"v": 1, "node": node, "module": module, "plan": plan})
_ = bus.Put(ctx, provisions.PlansBucket, node+"."+module, raw)
list, _ := plan["provisions"].([]any)
for _, e := range list {
entry := e.(map[string]any)
name, _ := entry["provision"].(string)
local, _ := entry["local"].(string)
key, err := provisions.Key(node, module, name, local)
if err != nil {
panic(err)
}
asked, err := provisions.Encode(provisions.Provision{Provision: name, Local: local, State: provisions.Asked,
Consumer: provisions.End{Node: node, Module: module}, Provider: provisions.End{Node: "novox", Module: "keycloak"}})
if err != nil {
panic(err)
}
_ = bus.Put(ctx, provisions.ProvisionsBucket, key, asked)
}
}
// waitUntil polls cond until it is true or the timeout passes.
@@ -62,6 +62,14 @@ func TestLiveRetirement(t *testing.T) {
if code := authorize(); code == 200 {
t.Fatal("a retired client is still served")
}
// Its secret is removed (ADR 0570 clause 5): the mesh's no longer authenticates.
if found, err := o.KC.FindClient(ctx, o.Realm, p.As); err != nil || found == nil {
t.Fatal(found, err)
} else if id, _ := found["id"].(string); true {
if secret, err := o.KC.ClientSecretByID(ctx, o.Realm, id); err != nil || secret == p.Password {
t.Fatal("a retired client keeps the mesh's secret", err)
}
}
// Retired, so a person may delete it.
if err := o.Delete(ctx, p.As); err != nil {
t.Fatal(err)
@@ -155,10 +155,9 @@ func TestBothPathsLeaveTheSameClientInKeycloak(t *testing.T) {
t.Fatal(err)
}
bus := &bucketBus{buckets: map[string]map[string][]byte{}}
plan, _ := json.Marshal(map[string]any{"node": "home", "module": "grafana", "provisioning": map[string]any{
seatPlan(bus, "home", "grafana", map[string]any{
"provisions": []any{map[string]any{"provision": ProvisionName, "provider": "novox",
"provider-module": "keycloak", "contributes": grafanaContributes, "names": grafanaNames}}}})
_ = bus.Put(ctx, provider.AssignmentsBucket, "home.grafana", plan)
"provider-module": "keycloak", "contributes": grafanaContributes, "names": grafanaNames}}})
pr := provider.New(provider.Options{Node: "novox", Module: "keycloak", Bus: bus,
Vault: provider.ConsumersFile{Paths: []string{file}},
Offers: map[string]provider.Offer{ProvisionName: {Credential: provider.PerConsumer,
+13 -3
View File
@@ -24,15 +24,18 @@ package main
import (
"context"
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/provider/harness"
"net/url"
"regexp"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/provider/harness"
)
// Mark is the attribute marking a client as the mesh's own work.
@@ -307,8 +310,10 @@ func (o OidcClients) Holds(ctx context.Context, p harness.Provision) (bool, erro
return secret == p.Password, nil
}
// Retire disables a consumer's client — only one the mesh made — and marks it with when and why.
// Answers what happened: "retired", "absent" or "not ours".
// Retire, on the operator's yes (novox/hq ADR 0570 clause 5), removes the client's secret — replaced by one nobody
// is given, so the mesh's credential no longer authenticates — and disables and marks the client with when and why.
// The client and its settings are kept; asked for again, Ensure sets the mesh's secret and enables it. Only a client
// the mesh made. Answers what happened: "retired", "absent" or "not ours".
func (o OidcClients) Retire(ctx context.Context, as, why string, at time.Time) (string, error) {
found, err := o.KC.FindClient(ctx, o.Realm, as)
if err != nil {
@@ -332,6 +337,11 @@ func (o OidcClients) Retire(ctx context.Context, as, why string, at time.Time) (
attrs[MarkRetiredWhy] = why
merged["attributes"] = attrs
merged["enabled"] = false
unknown := make([]byte, 32)
if _, err := rand.Read(unknown); err != nil {
return "", err
}
merged["secret"] = hex.EncodeToString(unknown) // never kept, never said
id, _ := found["id"].(string)
return "retired", o.KC.UpdateClient(ctx, o.Realm, id, merged)
}
@@ -186,7 +186,7 @@ func TestAClientTheMeshDidNotMakeIsRefusedAndLeftAlone(t *testing.T) {
}
}
// Retired is disabled and marked, everything else kept; asked for again it is enabled as it was; only
// Retired is its secret removed, disabled and marked, everything else kept (ADR 0570 clause 5); asked for again it is enabled as it was; only
// a deletion removes it, and never while enabled (novox/hq ADR 0230).
func TestARetiredClientIsDisabledKeptAndEnabledAgainAsItWas(t *testing.T) {
f, o := oidcWorld(t)
@@ -204,7 +204,7 @@ func TestARetiredClientIsDisabledKeptAndEnabledAgainAsItWas(t *testing.T) {
t.Fatal(done, err)
}
c := f.clients[id]
if c["enabled"] != false || c["secret"] != "s" || attributes(c)[MarkRetired] != "2026-10-06T12:00:00Z" ||
if c["enabled"] != false || c["secret"] == "s" || c["secret"] == "" || attributes(c)[MarkRetired] != "2026-10-06T12:00:00Z" ||
attributes(c)[MarkRetiredWhy] != "the mesh stopped asking for it" || c["description2"] == nil {
t.Fatalf("%v", c)
}
@@ -219,7 +219,7 @@ func TestARetiredClientIsDisabledKeptAndEnabledAgainAsItWas(t *testing.T) {
t.Fatal(err)
}
c = f.clients[id]
if c["enabled"] != true || attributes(c)[MarkRetired] != nil || attributes(c)[MarkRetiredWhy] != nil || c["description2"] == nil {
if c["enabled"] != true || c["secret"] != "s" || attributes(c)[MarkRetired] != nil || attributes(c)[MarkRetiredWhy] != nil || c["description2"] == nil {
t.Fatalf("%v", c)
}
mustHold(t, o, p, true)
@@ -62,7 +62,8 @@ func (a provisioner) Create(ctx context.Context, p harness.Provision) error {
return nil
}
// Retire disables the consumer's client and marks it, never deletes it (novox/hq ADR 0230).
// Retire removes the consumer's client secret and disables and marks the client, never deletes it (novox/hq ADR 0570
// clause 5, ADR 0230).
func (a provisioner) Retire(ctx context.Context, as string, _ map[string]any, why string, at time.Time) error {
if err := a.refused(); err != nil {
return err
@@ -75,7 +76,7 @@ func (a provisioner) Retire(ctx context.Context, as string, _ map[string]any, wh
case "not ours":
a.log("[provisioner:oidc-client] %s: a client of that id exists that the mesh did not make — left alone", as)
case "retired":
a.log("[provisioner:oidc-client] retired client %s in realm %s: disabled, kept, marked to delete", as, a.clients.Realm)
a.log("[provisioner:oidc-client] retired client %s in realm %s: its secret removed, disabled, kept, marked to delete", as, a.clients.Realm)
a.announce("client.retired", map[string]any{"realm": a.clients.Realm, "clientId": as})
}
return nil
+1 -1
View File
@@ -2,4 +2,4 @@ module keycloak
go 1.26.0
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0
+2 -2
View File
@@ -1,2 +1,2 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2 h1:Cdl5N56QPfmFBoLHrDfvvyc5Q+PUnTjf3fgYD7imcco=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+3 -1
View File
@@ -9,6 +9,7 @@
"provides": [
{
"name": "oidc-client",
"credential": {"form": "pair"},
"scope": "mesh",
"identity": {
"max": 255,
@@ -240,6 +241,7 @@
"bound"
],
"reads": [
"mesh-controller.assignments"
"mesh-provisioner.plans",
"mesh-provisioner.provisions"
]
}
+11 -1
View File
@@ -665,7 +665,14 @@
"path": "${dir:mesh-state}/config.json",
"mode": "0600",
"template": "files/config.json",
"takes-config": ["domain", "proxy-address", "sitename", "website", "url", "container"],
"takes-config": [
"domain",
"proxy-address",
"sitename",
"website",
"url",
"container"
],
"merge": "json"
},
{
@@ -758,6 +765,9 @@
"identity": {
"max": 64,
"in": "a mailbox's local part"
},
"credential": {
"form": "pair"
}
}
],
+1
View File
@@ -0,0 +1 @@
mesh-build-image
+1 -1
View File
@@ -3,7 +3,7 @@ module meshbuildimage
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 h1:Tt16HJk8JZRce1i3JzzDSNdCWOrv7dw8FQ/bckNQ5Ig=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
+1
View File
@@ -0,0 +1 @@
mesh-build-node
+1 -1
View File
@@ -3,7 +3,7 @@ module meshbuildnode
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0
github.com/distribution/distribution/v3 v3.1.2
github.com/nats-io/nats.go v1.54.0
)
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 h1:Tt16HJk8JZRce1i3JzzDSNdCWOrv7dw8FQ/bckNQ5Ig=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
+66 -7
View File
@@ -45,13 +45,72 @@ export interface Built {
branches?: string[];
}
/** The events a build outcome reaches this module on: the build seat's, as it happens, and the control plane's replay. */
export const BUILD_EVENTS = [
// The build seat node-build-agent and the retired mesh-build-machine went with build-agent (novox/hq ADR 0524 §5,
// ADR 0525 §10): no build seat says `built` since R4g, so what the mesh holds is asked when this module starts.
// What the mesh already held, when this module asks what it missed (novox/hq ADR 0134).
"mesh-controller.built-before",
] as const;
/**
* Where a build outcome comes from now: the builder's log (novox/hq ADR 0571 clauses 1 and 9). The module asks the
* builder's tool for the builds that ended since a time, and again whenever the builder's state of done keys changes.
*/
export const BUILDER = "mesh-delivery-builder";
export const BUILDS_SINCE_TOOL = "builder_builds_since";
export const BUILDS_STATE = "mesh-delivery-builder.builds";
export const MANIFESTS_STATE = "mesh-delivery-indexer.manifests";
/** One entry of the builder's log, an ended build (mesh-sdk delivery.BuildLogEntry). */
export interface LogEntry {
event: string;
module: string;
fp: string;
repository: string;
directory: string;
commit: string;
ref?: string;
node?: string;
at: string;
/** done, failed or queued. */
outcome?: string;
/** The digest of the manifest built, read from the indexer's state of manifests. */
manifest?: string;
against?: string[];
made?: Made[];
}
/** The answer of the builder's tool: every ended build since a time, oldest first, and whether the log held more. */
export interface BuildsSince {
since: string;
builds: LogEntry[];
more?: boolean;
}
/**
* What a tool answers, whether bare or wrapped as the runtime wraps a tool's reply ({output} or {content:[{text}]}).
*/
export function answerOf<T>(raw: unknown): T {
const r = raw as { output?: unknown; content?: { text?: string }[] } | null;
if (r && typeof r.output === "string") return JSON.parse(r.output) as T;
if (r && Array.isArray(r.content) && typeof r.content[0]?.text === "string") return JSON.parse(r.content[0].text) as T;
if (typeof raw === "string") return JSON.parse(raw) as T;
return raw as T;
}
/**
* A log entry as the build outcome the catalogue places: a build that is done is placed with its manifest, what it
* made and what it was built against; one that failed is passed over (placeable says so); one still queued has not
* ended and is not taken in. `replay` is set by the caller for history.
*/
export function builtFrom(e: LogEntry, manifest: unknown, replay: boolean): Built | undefined {
if (e.outcome === "queued") return undefined;
return {
module: e.module,
commit: e.commit,
repository: e.repository,
path: e.directory,
ref: e.ref ?? "",
manifest,
made: e.made ?? [],
against: e.against ?? [],
replay,
...(e.outcome === "failed" ? { failed: "the builder's log says the build failed" } : {}),
};
}
/**
* Whether a build outcome is a module-version to place in the graph, and why not when it is not.
+77 -15
View File
@@ -9,9 +9,15 @@
// and appear the next time round, once whatever they were waiting for is registered. A chain and a
// diamond need no special handling, and nothing holds a plan.
import { on, emit } from "@novox/mesh-sdk/events";
import { emit } from "@novox/mesh-sdk/events";
import { broker } from "@novox/mesh-sdk/messaging";
import { invokeTool } from "@novox/mesh-sdk/tools";
import { state } from "@novox/mesh-sdk/state";
import { Graph } from "./store.js";
import { BUILD_EVENTS, placeable, type Built } from "./heard.js";
import {
answerOf, builtFrom, placeable, BUILDER, BUILDS_SINCE_TOOL, BUILDS_STATE, MANIFESTS_STATE,
type Built, type BuildsSince, type LogEntry,
} from "./heard.js";
const graph = Graph.fromEnv();
@@ -77,18 +83,74 @@ const placeTheBuild = async (event: { body: unknown }): Promise<void> => {
}
};
// As it happens — from the build seat that builds now, and the retired one while it still answers — and
// what the mesh already held when this module asked what it missed (novox/hq issue 712).
for (const heard of BUILD_EVENTS) await on(heard, placeTheBuild);
// **From the builder's log** (novox/hq ADR 0571 clauses 1 and 9). The builder keeps every build's end in a log of its
// own and answers the builds that ended since a time by its tool. This module asks at its start for what it missed (all
// the log holds, taken in as history: registered, announced to nobody), and again whenever the builder's state of done
// keys changes, for what ended since the last build it took in. Registering a build already held changes nothing, and a
// build is taken in once per run of this module.
const taken = new Set<string>();
let since = new Date(0).toISOString();
let caughtUp = false;
let asking: Promise<void> | undefined;
let askAgain = false;
// **And ask for what was built before this catalogue existed** (novox/hq 04-ISSUES/050).
//
// The queue above is durable, so nothing is missed once this is running. What it cannot have is
// what was announced before it first ran — and on a fresh mesh that is never arbitrary: the shared
// base, the store this runs on, and this module itself are each necessarily built BEFORE a
// catalogue exists to hear about them. The graph's foundation is the part it never sees.
//
// Asked on every start, not only the first. A catalogue cannot tell whether it has a gap, and the
// answer is idempotent: registering a build already held changes nothing and announces nothing.
// Asked AFTER subscribing, so a build arriving during the replay is not lost between the two.
const manifestOf = async (digest: string | undefined): Promise<unknown> => {
if (!digest) return {};
const held = await state<{ manifest?: string }>(MANIFESTS_STATE).get(digest);
if (!held?.value.manifest) return {};
return JSON.parse(held.value.manifest);
};
const takeIn = async (e: LogEntry, replay: boolean): Promise<void> => {
const key = `${e.module}/${e.fp}`;
if (taken.has(key)) return;
const built = builtFrom(e, await manifestOf(e.manifest), replay);
if (!built) return;
await placeTheBuild({ body: built });
taken.add(key);
};
const catchUp = async (): Promise<void> => {
for (;;) {
const answer = answerOf<BuildsSince>(
await invokeTool(broker(), BUILDER, BUILDS_SINCE_TOOL, { since }),
);
for (const e of answer.builds ?? []) {
await takeIn(e, !caughtUp);
if (e.at > since) since = e.at;
}
if (!answer.more) break;
}
caughtUp = true;
};
/** One ask at a time; a change heard while asking is asked for again once, so none is missed. */
const ask = (): Promise<void> => {
if (asking) {
askAgain = true;
return asking;
}
asking = (async () => {
try {
do {
askAgain = false;
await catchUp();
} while (askAgain);
} catch (err) {
console.error("mesh-catalog: the builder's builds since", since, "could not be taken in:", err);
} finally {
asking = undefined;
}
})();
return asking;
};
// Asked AFTER watching begins, so a build ending while the first ask runs asks again rather than being missed: the watch
// hands every current key first (the start's ask), then every change.
await state(BUILDS_STATE).watch(async (change) => {
if (!change.current) await ask();
});
await ask();
// **And say that this catalogue has caught up** (novox/hq 04-ISSUES/050).
await emit("catching-up", {});
+9 -4
View File
@@ -25,9 +25,7 @@
"secrets": {
"postgres-database": "${dir:state}/database.secret"
},
"consumes": [
"mesh-controller.built-before"
],
"consumes": [],
"emits": [
"registered",
"upgraded",
@@ -92,5 +90,12 @@
"from": "files"
}
]
}
},
"invokes": [
"mesh-delivery-builder.builder_builds_since"
],
"reads": [
"mesh-delivery-builder.builds",
"mesh-delivery-indexer.manifests"
]
}
+19 -8
View File
@@ -2,7 +2,7 @@ import { test } from "node:test";
import assert from "node:assert/strict";
import { readFile } from "node:fs/promises";
import { BUILD_EVENTS, placeable } from "../heard.js";
import { answerOf, builtFrom, placeable, BUILDER, BUILDS_SINCE_TOOL, BUILDS_STATE, MANIFESTS_STATE } from "../heard.js";
// A walk's build as the build seat says it (mesh-controller's link.BuildResult), here mailu's of novox/hq issue 712.
const walkBuild = {
@@ -20,14 +20,25 @@ const walkBuild = {
made: [{ name: "code", kind: "bundle", reference: "artifact-store://mailu/code/blobs/sha256:d041" }],
};
test("the catalogue hears no retired build seat, and what the mesh held (issue 712, issue 1887 E2)", async () => {
for (const retired of ["node-build-agent.built", "mesh-build-machine.built"]) {
assert.ok(!(BUILD_EVENTS as readonly string[]).includes(retired), `${retired} is still heard`);
}
assert.ok(BUILD_EVENTS.includes("mesh-controller.built-before"));
// Subscribing is not enough: the mesh grants and routes only what the manifest consumes.
test("the catalogue takes builds from the builder's log, and hears no controller event (ADR 0571 clause 9)", async () => {
const manifest = JSON.parse(await readFile(new URL("../module.json", import.meta.url), "utf8"));
for (const heard of BUILD_EVENTS) assert.ok(manifest.consumes.includes(heard), `${heard} is not consumed`);
assert.deepEqual(manifest.consumes, []);
assert.ok(manifest.invokes.includes(`${BUILDER}.${BUILDS_SINCE_TOOL}`), "the builder's tool is not invoked");
for (const read of [BUILDS_STATE, MANIFESTS_STATE]) assert.ok(manifest.reads.includes(read), `${read} is not read`);
// A log entry is placed as a build: done with its manifest, made and against; failed passed over; queued not taken in.
const entry = { event: "build.ended", module: "mailu", fp: "fp1", repository: walkBuild.repository, directory: "modules/mailu",
commit: walkBuild.commit, ref: "main", at: "2026-10-29T20:00:00Z", outcome: "done", manifest: "ab", against: ["img@sha256:1"],
made: walkBuild.made };
const done = builtFrom(entry, walkBuild.manifest, false);
assert.deepEqual(placeable(done!), { placed: "yes", module: "mailu", commit: walkBuild.commit });
assert.deepEqual([done!.path, done!.against, done!.replay], ["modules/mailu", ["img@sha256:1"], false]);
assert.equal(placeable(builtFrom({ ...entry, outcome: "failed" }, {}, true)!).placed, "no");
assert.equal(builtFrom({ ...entry, outcome: "queued" }, {}, true), undefined);
// The tool's answer is read bare or as the runtime wraps it.
const bare = { since: "x", builds: [entry] };
for (const raw of [bare, { output: JSON.stringify(bare) }, { content: [{ text: JSON.stringify(bare) }] }]) {
assert.equal(answerOf<typeof bare>(raw).builds[0].module, "mailu");
}
});
test("a walk's build is placed, under its module and commit", () => {
@@ -0,0 +1,221 @@
package main
// **Built dark, compared, then switched by the operator** (novox/hq ADR 0564 clause 9; issue 1952 C5, C8): `compare`
// lists the open keys of the keeper and of the controller, each difference with its reason; the kinds that end with the
// controller, the kinds only the keeper raises and the keys held while their module is delivered are listed apart and
// not counted. `take-over` raises an approve ask under the keeper's own name (ADR 0259), refused while compare shows a
// difference, and on the warrant writes the take-over key naming the keeper. `hand-back` writes it back to the
// controller until the lease write, and is refused after it.
import (
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/health"
)
// Difference is one key the two sides do not agree on, or one listed apart.
type Difference struct {
Key string `json:"key"`
Kind string `json:"kind"`
Reason string `json:"reason"`
}
// Comparison is compare's answer.
type Comparison struct {
Differences []Difference `json:"differences"`
// Apart are the keys not counted: of kinds that end with the controller, raised only by the keeper, or held.
Apart []Difference `json:"apart"`
Same int `json:"same"`
At time.Time `json:"at"`
}
// Empty says nothing differs.
func (c Comparison) Empty() bool { return len(c.Differences) == 0 }
// Compare is the keeper's open conditions against the controller's.
func (k *Keeper) Compare() (Comparison, error) {
mine, err := k.Open()
if err != nil {
return Comparison{}, err
}
k.mu.Lock()
k.init()
theirs := map[string]health.Condition{}
for key, c := range k.ctrl {
theirs[key] = c
}
heldNow := map[string]held{}
for key, h := range k.held {
heldNow[key] = h
}
k.mu.Unlock()
out := Comparison{Differences: []Difference{}, Apart: []Difference{}, At: k.Now().UTC()}
open := map[string]health.Condition{}
for _, c := range mine {
open[c.Key] = c
}
keys := map[string]bool{}
for key := range open {
keys[key] = true
}
for key := range theirs {
keys[key] = true
}
for key := range heldNow {
keys[key] = true
}
sorted := make([]string, 0, len(keys))
for key := range keys {
sorted = append(sorted, key)
}
sort.Strings(sorted)
for _, key := range sorted {
c, inMine := open[key]
t, inTheirs := theirs[key]
kind := c.Kind
if !inMine {
kind = t.Kind
}
home, homed := k.homeOf(kind)
if h, ok := heldNow[key]; ok {
out.Apart = append(out.Apart, Difference{Key: key, Kind: kind,
Reason: fmt.Sprintf("held while %s is delivered on %s (version %s)", h.Module, h.Node, short(h.Version))})
continue
}
switch {
case homed && (home.Carried || home.Replaced):
why := "ends with the controller: carried from its bucket until the lease write"
if home.Replaced {
why = "ends with the controller: its fault is observed otherwise"
}
if home.Why != "" {
why += " (" + home.Why + ")"
}
out.Apart = append(out.Apart, Difference{Key: key, Kind: kind, Reason: why})
case inMine && !inTheirs && (!homed || !home.Controller):
out.Apart = append(out.Apart, Difference{Key: key, Kind: kind, Reason: "keeper only: the controller never raises this kind"})
case inMine && !inTheirs:
out.Differences = append(out.Differences, Difference{Key: key, Kind: kind,
Reason: "open at the keeper, not at the controller"})
case !inMine && !homed:
out.Differences = append(out.Differences, Difference{Key: key, Kind: kind,
Reason: fmt.Sprintf("open at the controller; no home is decided for the kind %s (ADR 0564's gap)", kind)})
case !inMine:
out.Differences = append(out.Differences, Difference{Key: key, Kind: kind,
Reason: fmt.Sprintf("open at the controller; %s has not written its finding", home.Observer)})
case c.Severity != t.Severity:
out.Differences = append(out.Differences, Difference{Key: key, Kind: kind,
Reason: fmt.Sprintf("%s at the keeper, %s at the controller", c.Severity, t.Severity)})
default:
out.Same++
}
}
return out, nil
}
// Asks is the operator channel as the keeper asks it: an approve ask under its own name, and where one stands.
type Asks interface {
Ask(a asks.Ask) error
}
// TakeOver raises the approve ask of a switch to the keeper, refused while compare differs or after the lease write.
func (k *Keeper) TakeOver(a Asks, by, why string) (map[string]any, error) {
if strings.TrimSpace(why) == "" {
return nil, errors.New("a take-over says why")
}
if k.LeaseAway() {
return nil, errors.New("the controller's lease is handed away: the keeper is the one source already")
}
cmp, err := k.Compare()
if err != nil {
return nil, err
}
if !cmp.Empty() {
return nil, fmt.Errorf("refused: compare shows %d difference(s), the first %s (%s); the take-over waits for none",
len(cmp.Differences), cmp.Differences[0].Key, cmp.Differences[0].Reason)
}
now := k.Now().UTC()
id := fmt.Sprintf("take-over-%d", now.UnixNano())
act := asks.Act{"verb": Seat + ".take-over", "module": Module, "ask": id}
bind, err := asks.ActDigest(act)
if err != nil {
return nil, err
}
ask := asks.Ask{ID: id, Headline: "Switch the mesh's alarms to the new keeper?", Who: asks.Operator,
Explanation: fmt.Sprintf("The new condition keeper holds the same %d open alarms as the controller. Your yes makes "+
"the alarm messages come from the keeper, with the same words. It can be handed back until the controller "+
"gives up its lease.", cmp.Same),
Options: []asks.Option{{ID: "yes", Label: "Yes, switch", Does: "Alarms come from the keeper.", Level: asks.Approve, Binds: bind},
{ID: "no", Label: "No, leave it", Does: "Alarms keep coming from the controller.", Level: asks.Acknowledge}},
Expires: now.Add(asks.ApproveLasts - time.Minute), OnExpiry: "Alarms keep coming from the controller.",
About: Module + "-take-over", Details: "Asked by " + by + ": " + why}
if err := a.Ask(ask); err != nil {
return nil, err
}
return map[string]any{"asked": id, "same": cmp.Same, "apart": len(cmp.Apart)}, nil
}
// Warranted is the act on the operator's answer: a yes, compare still empty, writes the take-over key naming the keeper.
func (k *Keeper) Warranted(o asks.Option, w asks.Warrant) error {
if o.ID != "yes" {
k.Logf("the take-over %s was answered %s: alarms stay with the controller", w.Ask, o.ID)
return nil
}
cmp, err := k.Compare()
if err != nil {
return err
}
if !cmp.Empty() {
return fmt.Errorf("the take-over %s was approved, and compare now shows %d difference(s): not switched", w.Ask,
len(cmp.Differences))
}
return k.writeTakeover(health.Takeover{Follow: health.FollowKeeper, At: k.Now().UTC(), By: w.Says(), Ask: w.Ask})
}
// HandBack switches the readers back to the controller, until the lease write.
func (k *Keeper) HandBack(by, why string) (map[string]any, error) {
if strings.TrimSpace(why) == "" {
return nil, errors.New("a hand-back says why")
}
if k.LeaseAway() {
return nil, errors.New("refused: the controller's lease is handed away, so there is no controller to hand back to")
}
t := health.Takeover{Follow: health.FollowController, At: k.Now().UTC(), By: by, Why: why}
if err := k.writeTakeover(t); err != nil {
return nil, err
}
return map[string]any{"follow": t.Follow, "at": t.At}, nil
}
// writeTakeover writes the take-over key, and keeps it in history as the record of the switch.
func (k *Keeper) writeTakeover(t health.Takeover) error {
raw, err := health.EncodeTakeover(t)
if err != nil {
return err
}
_, rev, found, err := k.Store.Get(TakeoverState, health.TakeoverKey)
if err != nil {
return err
}
if found {
_, err = k.Store.Update(TakeoverState, health.TakeoverKey, json.RawMessage(raw), rev)
} else {
_, err = k.Store.Create(TakeoverState, health.TakeoverKey, json.RawMessage(raw))
}
if errors.Is(err, stdio.ErrChanged) {
return fmt.Errorf("the take-over key moved under this write: read it again and ask again")
}
if err != nil {
return err
}
k.Takeover(t)
k.Logf("the readers of conditions follow the %s from now (%s)", t.Follow, t.By)
return nil
}
@@ -0,0 +1,340 @@
package main
// **A module being delivered raises no condition until its delivery ends** (novox/hq ADR 0564 clause 13 and Limits 12;
// issue 1952 C7, C8). The keeper reads the assigner's wants and the nodes' haves, and judges a module's delivery on a
// node with mesh-sdk's delivery.MotionOf, as the view does. While it is delivered there, a finding about it there, or
// about what it serves, and a condition carried from the controller's bucket on those subjects, is kept in the state
// `held` and raises nothing. When the delivery ends, healthy at the want or failed, what still stands is raised once,
// saying it happened during the delivery; what went is dropped with a log line. The hold lasts at most the delivery's
// bound, a key of the state `bounds` whose time to live is the bound: its expiry raises what is held, saying the delivery
// did not end. A finding about another module or about the machine itself is never held. A condition already open is
// not held: it is no news.
import (
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
)
// The modules whose delivery holds what they serve beyond their own findings (Limits 12).
const (
controllerModule = "mesh-controller"
runtimeModule = "node-tools"
)
// controllerServes are the sources of what the controller serves while it is delivered: its listening, its lease, its
// writes and its facts (ADR 0505 §2 (a)), matched to mesh-controller whatever its key names.
var controllerServes = map[string]bool{"S4": true, "S7": true, "S12": true, "S13": true, "S14": true, "S25": true}
// about is a node and a module a finding or a condition is about, for the hold.
type about struct{ Node, Module string }
// aboutOf are the modules on nodes a condition's subject is about (Limits 12): a module on a machine, a provider on its
// node, a seat of the same name as its module on a machine, the controller's own work wherever mesh-controller is
// delivered, and every module's tools on a machine whose tool runner is delivered. Never the machine itself.
func (k *Keeper) aboutOf(kind, source string, s health.Subject) []about {
var out []about
parts := strings.Split(s.ID, ".")
switch s.Scope {
case health.ScopeModule:
if len(parts) >= 2 {
out = append(out, about{Node: parts[len(parts)-1], Module: strings.Join(parts[:len(parts)-1], ".")})
}
case health.ScopeProvider:
if len(parts) >= 2 {
out = append(out, about{Node: parts[1], Module: parts[0]})
}
case health.ScopeSeat:
// The module the placement says holds the seat on that node, or the seat's own name where none is read.
if len(parts) >= 2 {
node, seat := parts[len(parts)-1], strings.Join(parts[:len(parts)-1], ".")
module := k.holders[node+"."+seat]
if module == "" {
module = seat
}
out = append(out, about{Node: node, Module: module})
}
case health.ScopeMachine:
// The machine itself (presence, network, resolver, disk) is never held; its tools are, while its tool runner is
// delivered.
if source == "S11" || kind == "tools-silent" {
out = append(out, about{Node: s.Machine, Module: runtimeModule})
if s.Machine == "" && len(parts) > 0 {
out[len(out)-1].Node = parts[0]
}
}
return out
}
if controllerServes[source] || (s.Scope == health.ScopeCore && strings.HasPrefix(s.ID, "controller")) {
for hk := range k.wants {
if node, module := splitNodeModule(hk); module == controllerModule {
out = append(out, about{Node: node, Module: module})
}
}
}
if s.Machine != "" && s.Scope != health.ScopeMachine {
out = append(out, about{Node: s.Machine, Module: runtimeModule})
}
return out
}
func splitNodeModule(hk string) (string, string) {
node, module, _ := strings.Cut(hk, ".")
return node, module
}
// motion is the delivery of module on node, from its want and have.
func (k *Keeper) motion(node, module string) (delivery.Motion, delivery.Want) {
w, wanted := k.wants[node+"."+module]
h, has := k.haves[node+"."+module]
return delivery.MotionOf(w, wanted, h, has, "", false), w
}
// holdFor says whether key is held now, and as what. Called with mu held.
func (k *Keeper) holdFor(key string, f *health.Finding, carried *health.Condition, open bool) (held, bool) {
if open || (f == nil && carried == nil) {
return held{}, false
}
var kind, source string
var subject health.Subject
if f != nil {
kind, source, subject = f.Kind, f.Source, f.Subject
} else {
kind, source, subject = carried.Kind, carried.Source, carried.Subject
}
if _, raisedOut := k.outOfHold[key]; raisedOut {
return held{}, false
}
for _, a := range k.aboutOf(kind, source, subject) {
m, w := k.motion(a.Node, a.Module)
if !m.Delivering || m.Ended() || k.passed[a.Node+"."+a.Module] == w.FP {
continue
}
since := k.Now().UTC()
if was, ok := k.held[key]; ok {
since = was.Since
}
return held{Key: key, Node: a.Node, Module: a.Module, Since: since, Version: w.FP}, true
}
return held{}, false
}
// keepHeld keeps a held key in the state `held`, and starts its delivery's bound once.
func (k *Keeper) keepHeld(h held) error {
k.mu.Lock()
_, was := k.held[h.Key]
k.held[h.Key] = h
bound := k.bound(h.Module)
k.mu.Unlock()
if was {
return nil
}
if _, err := k.Store.Create(HeldState, h.Key, h); err != nil && !errors.Is(err, stdio.ErrChanged) {
return fmt.Errorf("keeping %s held: %w", h.Key, err)
}
b := boundKey(h.Node, h.Module)
err := k.Store.CreateTTL(BoundsState, b, map[string]any{"node": h.Node, "module": h.Module, "version": h.Version,
"bound-seconds": int(bound / time.Second)}, bound)
if err != nil && !errors.Is(err, stdio.ErrChanged) {
return fmt.Errorf("starting the bound of %s's delivery on %s: %w", h.Module, h.Node, err)
}
k.Logf("held %s while %s is delivered on %s (version %s), at most %s", h.Key, h.Module, h.Node, h.Version, bound)
return nil
}
// bound is a module's delivery bound: its manifest names none yet, so the default (clause 13 (d)). Each key of the state
// `bounds` is written with it as its own time to live, and its expiry is the bound passing.
func (k *Keeper) bound(string) time.Duration {
if k.Bound > 0 {
return k.Bound
}
return delivery.DefaultDeliveryBound
}
func boundKey(node, module string) string { return node + "." + module }
// dropHeld lets a held key go: raised, or gone with why said.
func (k *Keeper) dropHeld(key, why string) {
k.mu.Lock()
h, ok := k.held[key]
delete(k.held, key)
k.mu.Unlock()
if !ok {
return
}
if err := k.Store.Delete(HeldState, key); err != nil {
k.Logf("the held %s could not be removed from held: %v", key, err)
}
if why != "" {
k.Logf("dropped %s, held while %s was delivered on %s: %s", key, h.Module, h.Node, why)
}
}
// Want takes one change of the assigner's wants; Have one of the nodes' haves. Each may end a delivery, which raises or
// drops what it held.
func (k *Keeper) Want(key string, value []byte, deleted bool) error {
node, module, err := delivery.ParseWantKey(key)
if err != nil {
return nil // not a want of a module on a node (a held fingerprint's key)
}
k.mu.Lock()
k.init()
if deleted {
delete(k.wants, node+"."+module)
} else if w, err := delivery.DecodeWant(value); err == nil {
k.wants[node+"."+module] = w
} else {
k.Logf("the want %s is not readable: %v", key, err)
}
k.mu.Unlock()
return k.judge(node, module, "")
}
// Have takes one change of a node's have.
func (k *Keeper) Have(key string, value []byte, deleted bool) error {
node, module, isNode, err := delivery.ParseHaveKey(key)
if err != nil || isNode {
return nil
}
k.mu.Lock()
k.init()
if deleted {
delete(k.haves, node+"."+module)
} else if h, err := delivery.DecodeHave(value); err == nil {
k.haves[node+"."+module] = h
} else {
k.Logf("the have %s is not readable: %v", key, err)
}
k.mu.Unlock()
return k.judge(node, module, "")
}
// Placement takes one change of the placement: which module holds which seat on a node.
func (k *Keeper) Placement(key string, value []byte, deleted bool) {
node, err := delivery.ParsePlacementKey(key)
if err != nil {
return
}
k.mu.Lock()
defer k.mu.Unlock()
k.init()
for hk := range k.holders {
if strings.HasPrefix(hk, node+".") {
delete(k.holders, hk)
}
}
if deleted {
return
}
p, err := delivery.DecodePlacement(value)
if err != nil {
k.Logf("the placement of %s is not readable, so its seats' holders are not known: %v", node, err)
return
}
for _, h := range p.Holds {
k.holders[node+"."+h.Claim] = h.Module
}
}
// BoundPassed is a delivery's bound key expiring: what it holds is raised, saying the delivery did not end.
func (k *Keeper) BoundPassed(key string) error {
node, module := splitNodeModule(key)
k.mu.Lock()
k.init()
_, w := k.motion(node, module)
holding := false
for _, h := range k.held {
holding = holding || (h.Node == node && h.Module == module)
}
if !holding {
k.mu.Unlock()
return nil
}
k.passed[key] = w.FP
k.mu.Unlock()
return k.judge(node, module, fmt.Sprintf("Its delivery of version %s on %s did not end within %s.", short(w.FP), node,
k.bound(module)))
}
// judge raises or drops what node and module hold when their delivery ended, or with line past its bound.
func (k *Keeper) judge(node, module, line string) error {
k.mu.Lock()
m, w := k.motion(node, module)
var keys []string
for key, h := range k.held {
if h.Node == node && h.Module == module {
keys = append(keys, key)
}
}
if line == "" && !m.Ended() && m.Delivering {
k.mu.Unlock()
return nil
}
if line == "" {
line = fmt.Sprintf("It happened during the delivery of version %s of %s on %s.", short(w.FP), module, node)
if m.Failed {
line = fmt.Sprintf("It happened as the delivery of version %s of %s on %s failed.", short(w.FP), module, node)
}
delete(k.passed, node+"."+module)
}
for _, key := range keys {
k.outOfHold[key] = line
}
k.mu.Unlock()
sort.Strings(keys)
var problems []string
for _, key := range keys {
k.dropHeld(key, "")
if err := k.Settle(key); err != nil {
problems = append(problems, err.Error())
}
k.mu.Lock()
delete(k.outOfHold, key)
k.mu.Unlock()
}
if len(problems) > 0 {
return errors.New(strings.Join(problems, "; "))
}
return nil
}
func short(fp string) string {
if _, rest, ok := strings.Cut(fp, ":"); ok {
fp = rest
}
if len(fp) > 12 {
return fp[:12]
}
return fp
}
// Held is every key held now, for the read tool and compare.
func (k *Keeper) Held() []held {
k.mu.Lock()
defer k.mu.Unlock()
out := make([]held, 0, len(k.held))
for _, h := range k.held {
out = append(out, h)
}
sort.Slice(out, func(i, j int) bool { return out[i].Key < out[j].Key })
return out
}
// restoreHeld takes the state `held` as read at the start.
func (k *Keeper) restoreHeld(key string, value []byte) {
var h held
if json.Unmarshal(value, &h) != nil {
return
}
k.mu.Lock()
k.init()
k.held[key] = h
k.mu.Unlock()
}
@@ -0,0 +1,129 @@
package main
import (
"encoding/json"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
)
// The fingerprints and composition of mesh-controller on novox, copied from its have (mesh-delivery-view.node_haves,
// 2026-10-29): the delivery replayed moves it from the running fingerprint to the next, through the node-engine's phases
// (delivery.go Phase*). No tool reads a have's or a want's history yet, so the sequence is the node-engine's documented
// order, not a capture; the values are captured.
const (
ctrlRunning = "src2:fcaa7d7ccb35ea2beed5b9e4b87f4c24c99452244d49f4786d4022e5c543f39c"
ctrlComposed = "sha256:e9a4de75c9ca70a32cef3137cb0e6692b0f9558b699013caaee399d88dd1bac0"
ctrlNext = "src2:0d1e2f3a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f9001a2b3c4d5e6"
ctrlNextComp = "sha256:1f2e3d4c5b6a79881726354453627180918a7b6c5d4e3f2a1b0c9d8e7f605142"
)
// ADR 0564 clause 13 and Limits 12 (issue 1952 C7, C8): a delivery of mesh-controller on novox replayed want by have.
// Its own findings and the carried controller-deaf raise nothing during it and none after when they went; one still
// standing at the end is raised once with its during-the-delivery line; one past the bound with its did-not-end line; a
// failed delivery raises what stands at once; another module's and the machine's own are raised at once; compare lists a
// held key as held.
func TestADeliveryHoldsItsOwnFindingsUntilItEnds(t *testing.T) {
want := func(fp, composed string) []byte {
raw, _ := json.Marshal(delivery.Want{V: delivery.Version, FP: fp, Composed: composed})
return raw
}
have := func(fp, composed string, phase delivery.Phase, h delivery.Health) []byte {
raw, _ := json.Marshal(delivery.Have{V: delivery.Version, FP: fp, Composed: composed, Phase: phase, Health: h})
return raw
}
deaf, _ := json.Marshal(health.Condition{Key: "core.controller.deaf", Kind: "controller-deaf", Source: "S4",
Subject: health.Subject{Scope: health.ScopeCore, ID: "controller"}, Severity: health.Urgent, Summary: "deaf", Resolver: "self"})
for _, c := range []struct {
name string
// end is the have the delivery ends with, or nil for a delivery past its bound.
end []byte
standing bool
said string
line string
}{
{name: "gone by its end", end: have(ctrlNext, ctrlNextComp, delivery.PhaseRunning, delivery.HealthOK)},
{name: "still standing at its end", end: have(ctrlNext, ctrlNextComp, delivery.PhaseRunning, delivery.HealthOK),
standing: true, said: "raised module.mesh-controller.novox.unhealthy", line: "during the delivery of version 0d1e2f3a4b5c"},
{name: "past its bound", standing: true, said: "raised module.mesh-controller.novox.unhealthy", line: "did not end within"},
{name: "failed", end: have(ctrlNext, ctrlNextComp, delivery.PhaseFailed, delivery.HealthFailed), standing: true,
said: "raised module.mesh-controller.novox.unhealthy", line: "as the delivery of version 0d1e2f3a4b5c"},
} {
r := newRig()
_ = r.k.Want("want.novox.mesh-controller", want(ctrlRunning, ctrlComposed), false)
_ = r.k.Have("novox.mesh-controller", have(ctrlRunning, ctrlComposed, delivery.PhaseRunning, delivery.HealthOK), false)
// The delivery starts: the assigner wants the next fingerprint; the node applies it.
_ = r.k.Want("want.novox.mesh-controller", want(ctrlNext, ctrlNextComp), false)
_ = r.k.Have("novox.mesh-controller", have(ctrlNext, ctrlNextComp, delivery.PhaseApplying, delivery.HealthUnknown), false)
own, ownRaw := finding(t, watchman, health.ScopeModule, "mesh-controller.novox", "unhealthy", "health", health.Warning, r.now)
other, otherRaw := finding(t, watchman, health.ScopeModule, "postgres.novox", "unhealthy", "health", health.Warning, r.now)
machine, machineRaw := finding(t, watchman, health.ScopeMachine, "novox", "silent", "S1", health.Urgent, r.now)
for _, f := range []struct {
key string
raw []byte
}{{own, ownRaw}, {other, otherRaw}, {machine, machineRaw}} {
if err := r.k.Finding(watchman, f.key, f.raw, false); err != nil {
t.Fatal(err)
}
}
if err := r.k.Controller("core.controller.deaf", deaf, false); err != nil {
t.Fatal(err)
}
// The delivery's bound is its own key's time to live, the module's bound (the default where it names none).
if r.st.ttls["bounds/novox.mesh-controller"] != delivery.DefaultDeliveryBound {
t.Errorf("%s: the bound key lives %s", c.name, r.st.ttls["bounds/novox.mesh-controller"])
}
if got := r.changes(); got != "raised "+other+", raised "+machine {
t.Errorf("%s: during the delivery said %q: another module's and the machine's at once, nothing of its own", c.name, got)
}
// C8: compare lists the held keys as held, never as a difference.
cmp, _ := r.k.Compare()
held := 0
for _, d := range cmp.Apart {
if strings.HasPrefix(d.Reason, "held while mesh-controller is delivered on novox") {
held++
}
}
if held != 2 {
t.Errorf("%s: compare lists %d held, %+v", c.name, held, cmp)
}
// The carried condition about the controller goes within the delivery; its own finding goes unless it stands.
_ = r.k.Controller("core.controller.deaf", nil, true)
if !c.standing {
_ = r.k.Finding(watchman, own, nil, true)
}
if c.end != nil {
_ = r.k.Have("novox.mesh-controller", c.end, false)
} else {
r.now = r.now.Add(delivery.DefaultDeliveryBound)
_ = r.k.BoundPassed("novox.mesh-controller")
}
if got := r.changes(); got != c.said {
t.Errorf("%s: at its end said %q, want %q", c.name, got, c.said)
}
if co, open := r.open(own); c.standing != open || (open && !strings.Contains(co.Explanation, c.line)) {
t.Errorf("%s: its own condition is %v: %q", c.name, open, co.Explanation)
}
if _, open := r.open("core.controller.deaf"); open || len(r.k.Held()) != 0 {
t.Errorf("%s: the carried deaf condition was raised, or something is still held: %+v", c.name, r.k.Held())
}
}
// A seat's finding is about the module the placement says holds the seat on that node (the lead, 2026-10-29): held
// while the assigner is delivered on ace, though the seat is named mesh-placement.
r := newRig()
placed, err := delivery.EncodePlacement(delivery.Placement{Node: "ace", Modules: []delivery.PlacedModule{{Module: "mesh-delivery-assigner"}},
Holds: []delivery.Holding{{Claim: "mesh-placement", Scope: "mesh", Module: "mesh-delivery-assigner"}}})
if err != nil {
t.Fatal(err)
}
r.k.Placement("placement.ace", placed, false)
_ = r.k.Want("want.ace.mesh-delivery-assigner", want(ctrlNext, ctrlNextComp), false)
_ = r.k.Have("ace.mesh-delivery-assigner", have(ctrlRunning, ctrlComposed, delivery.PhaseRunning, delivery.HealthOK), false)
seat, seatRaw := finding(t, selfCheck, health.ScopeSeat, "mesh-placement.ace", "holder-silent", "D3", health.Warning, r.now)
_ = r.k.Finding(selfCheck, seat, seatRaw, false)
if got := r.changes(); got != "" || len(r.k.Held()) != 1 || r.k.Held()[0].Module != "mesh-delivery-assigner" {
t.Errorf("a seat's finding while its holder is delivered: said %q, held %+v", got, r.k.Held())
}
}
@@ -0,0 +1,160 @@
package main
// **Where each condition kind lives after the take-over** (novox/hq ADR 0564 clauses 4, 6, 7 and 9, the Context's probe
// table; issue 1952 C3, C5): data, so a kind's home is a row and never code. `compare` reads it to tell a difference from
// a kind that ends with the controller or that the controller never raises, and the keeper reads it to carry, until the
// lease write, the controller's conditions of the kinds that end with it.
//
// The rows are ADR 0564's and the lead's table of 2026-10-29, by the controller's source of each kind. A kind with no
// row is named by `compare` as "no home decided".
// Home is where one condition kind lives.
type Home struct {
// Source is the controller's row or probe that raises it today, for whoever reads the table.
Source string
// Controller says the controller raises this kind today: a kind it never raises is the keeper's alone and is no
// difference in compare.
Controller bool
// Observer is the module whose findings raise it after the take-over; empty for a kind that ends.
Observer string
// Carried says it ends with the controller's lease and is carried from the controller's bucket until then
// (clause 9: S4, S7, S12, S13, S14, S25, D5, D9, H-controller).
Carried bool
// Replaced says the kind itself ends and another observes its fault (D1, D10, the stuck probe): listed apart in
// compare and not carried.
Replaced bool
// Why is why a kind is carried when the record's table gave it another home: said by compare beside it.
Why string
}
// Dropped is the reason of the probes the operator dropped from this release (D2, D3, D4, D12, D-heard, D-root, D11,
// and D13's data held twice): their kinds end with the controller and are carried until the lease write.
const Dropped = "dropped 2026-10-30 by the operator; the owning module declares its own check later (research 074)"
// The kinds' homes, by kind: ADR 0564's rows, and the 63 kinds the controller's code raises that the record left without
// one, as the lead settled them under the operator's yes on 2026-10-29 (.work/47-release/kinds-section.html, with the
// lead's three changes: lease-split and image-missing do not end, and the data rows read their readings from disk-load).
var Homes = map[string]Home{
// The watchman's watchdogs (clause 4).
"silent": {Source: "S1", Controller: true, Observer: watchman},
"max-deliveries": {Source: "S9", Controller: true, Observer: watchman},
"slow-consumer": {Source: "S9", Controller: true, Observer: watchman},
"refused": {Source: "S9", Controller: true, Observer: watchman},
"connection-closed": {Source: "S9", Controller: true, Observer: watchman},
"bus-faults-unread": {Source: "S9", Controller: true, Observer: watchman},
"tools-silent": {Source: "S11", Controller: true, Observer: watchman},
"apply-failing": {Source: "S21", Controller: true, Observer: watchman},
"module-unhealthy": {Source: "health", Controller: true, Observer: watchman},
"stuck": {Source: "DM", Controller: true, Observer: watchman},
"provider-failing": {Source: "S8", Controller: true, Observer: watchman},
"provider-silent": {Source: "S8", Controller: true, Observer: watchman},
"self-check-silent": {Source: "S10", Controller: true, Observer: watchman},
// From the provision keys in waits (ADR 0570 §7), a setting's wait from the node's own health report.
"needs-operator": {Source: "health", Controller: true, Observer: watchman},
// From the vault's record of each secret given (ADR 0570 §7).
"secret-given": {Source: "desk", Controller: true, Observer: watchman},
// The watchman reading each node's own health report key (a4's, mesh-host).
"machine-network": {Source: "node", Controller: true, Observer: watchman},
"network-rewritten": {Source: "node", Controller: true, Observer: watchman},
"network-unreachable": {Source: "node", Controller: true, Observer: watchman},
"machine-units": {Source: "node", Controller: true, Observer: watchman},
"relogin-needed": {Source: "health", Controller: true, Observer: watchman},
"directory-used-as-found": {Source: "health", Controller: true, Observer: watchman},
// The self-check's probes (clause 6, the Context's table).
"lease-split": {Source: "D5", Controller: true, Observer: selfCheck}, // a seat held once answers from two holders
"consumer-wrong": {Source: "D6", Controller: true, Observer: selfCheck},
"consumer-lost": {Source: "D6", Controller: true, Observer: selfCheck}, // and the watchman's S9
"consumer-behind": {Source: "D6", Controller: true, Observer: selfCheck},
"stream-wrong": {Source: "D7", Controller: true, Observer: selfCheck},
"own-address-banned": {Source: "D8", Controller: true, Observer: selfCheck},
"data-unmeasured": {Source: "D13", Controller: true, Observer: selfCheck},
"data-shrank": {Source: "D13", Controller: true, Observer: selfCheck}, // readings from disk-load (L7)
"empty-replacement": {Source: "D13", Controller: true, Observer: selfCheck},
"data-quiet": {Source: "D13", Controller: true, Observer: selfCheck},
"backup-stale": {Source: "D13", Controller: true, Observer: selfCheck},
"data-missing": {Source: "D13", Controller: true, Observer: selfCheck},
"array-degraded": {Source: "D13", Controller: true, Observer: selfCheck},
"protection-missing": {Source: "D13", Controller: true, Observer: selfCheck},
"bus-reconnects": {Source: "D15", Controller: true, Observer: selfCheck},
"bucket-or-log-filling": {Source: "fill", Controller: true, Observer: selfCheck},
"bucket-or-log-unread": {Source: "fill", Controller: true, Observer: selfCheck},
"watchdogs-silent": {Source: "DW", Controller: true, Observer: selfCheck},
"core-unhealthy": {Source: "H-bus", Controller: true, Observer: selfCheck},
"agent-can-become-root": {Source: "DA", Controller: true, Observer: selfCheck},
"probe-failed": {Source: "doctor", Controller: true, Observer: selfCheck},
// The healers' own (clause 7, Limits 2).
"healer-wanted": {Source: "S15", Controller: true, Observer: healers},
"healers-braked": {Source: "healers", Controller: true, Observer: healers},
// The assigner's, as its own findings (the bus step, ADR 0565 §9).
"bus-maintenance": {Source: "bus", Controller: true, Observer: assigner},
"bus-upgrade-failed": {Source: "DB", Controller: true, Observer: assigner},
// The provisioner's (ADR 0570 §5): its own ask replaces retire-waiting; a declined key stays its finding.
"retire-waiting": {Source: "retire", Controller: true, Observer: provisioner},
"retire-rejected": {Source: "retire", Controller: true, Observer: provisioner},
// The router's own: no channel delivered a question.
"asks-undelivered": {Source: "asks", Controller: true, Observer: messenger},
// The seat mesh-artifact-sweep (L7): an image a want names is missing from the store, artifact-store-tools' finding.
"image-missing": {Source: "S25", Controller: true, Observer: artifactStore},
// The kinds that end with the controller and are carried until its lease write (clauses 4, 9).
"controller-deaf": {Source: "S4", Controller: true, Carried: true},
"lease-lost": {Source: "S12", Controller: true, Carried: true},
"call-hung": {Source: "S7", Controller: true, Carried: true},
"stale-writer": {Source: "S13", Controller: true, Carried: true},
"facts-stale": {Source: "S14", Controller: true, Carried: true},
"status-slow": {Source: "D9", Controller: true, Carried: true},
"vault-silent": {Source: "send", Controller: true, Carried: true},
"declaration-files-not-put": {Source: "send", Controller: true, Carried: true},
"config-not-followed": {Source: "config", Controller: true, Carried: true},
"assignment-not-made": {Source: "assign", Controller: true, Carried: true},
"builds-not-laid-out": {Source: "start", Controller: true, Carried: true},
"sent-builds-not-copied": {Source: "start", Controller: true, Carried: true},
"fingerprint-not-composed": {Source: "done keys", Controller: true, Carried: true},
"take-in-not-consumed": {Source: "take-in", Controller: true, Carried: true},
"take-in-stalled": {Source: "take-in", Controller: true, Carried: true},
"new-module-waits": {Source: "register", Controller: true, Carried: true},
"left-out": {Source: "send", Controller: true, Carried: true},
"declaration-refused": {Source: "D1", Controller: true, Carried: true},
"unknown-field": {Source: "read", Controller: true, Carried: true},
"bus-objects-unasserted": {Source: "send", Controller: true, Carried: true},
// The probes the operator dropped from this release (D2 resolvers, D3 holders, D4 archives, D12 bindings, D-heard
// withheld hearings, D-root, D11 retired data, and D13's data held twice): their kinds are carried until the lease
// write.
"resolver-wrong": {Source: "D2", Controller: true, Carried: true, Why: Dropped},
"holder-silent": {Source: "D3", Controller: true, Carried: true, Why: Dropped},
"archives-missing": {Source: "D4", Controller: true, Carried: true, Why: Dropped},
"archives-unheld": {Source: "D4", Controller: true, Carried: true, Why: Dropped},
"binding-kept": {Source: "D12", Controller: true, Carried: true, Why: Dropped},
"binding-moved": {Source: "D12", Controller: true, Carried: true, Why: Dropped},
"binding-moving": {Source: "D12", Controller: true, Carried: true, Why: Dropped},
"grant-composes-nothing": {Source: "withheld", Controller: true, Carried: true, Why: Dropped},
"root-not-free": {Source: "D-root", Controller: true, Carried: true, Why: Dropped},
"cleanup-waiting": {Source: "D11", Controller: true, Carried: true, Why: Dropped},
"data-held-twice": {Source: "D13", Controller: true, Carried: true, Why: Dropped},
// The lead, 2026-10-29: no delivery plans since R6 (wants replace them), and a stalled delivery is the watchman's
// stuck finding seen directly.
"plan-failed": {Source: "S23", Controller: true, Carried: true},
"stalled": {Source: "D14", Controller: true, Carried: true},
// The kinds that end, their faults observed otherwise (the Context's table).
"core-behind": {Source: "D10", Controller: true, Replaced: true},
}
// The observers, as the table names them.
const (
watchman = "mesh-delivery-watchman"
selfCheck = "mesh-self-check"
healers = "mesh-healers"
assigner = "mesh-delivery-assigner"
provisioner = "mesh-provisioner"
messenger = "messenger"
artifactStore = "artifact-store-tools"
)
// homeOf is the home of a condition's kind, and whether the table has one.
func (k *Keeper) homeOf(kind string) (Home, bool) {
homes := k.Homes
if homes == nil {
homes = Homes
}
h, ok := homes[kind]
return h, ok
}
@@ -0,0 +1,668 @@
package main
// The condition keeper (novox/hq ADR 0564 clauses 2, 3, 7, 8, 9 and 13; issue 1952 C2, C3, C7): the one writer of the
// mesh's open conditions. It watches every observer's `findings`, the healers' `attempts`, the controller's lease and,
// until the lease write, the controller's `conditions`, and the assigner's wants and the nodes' haves for the hold. From
// all of them it derives, per key, the condition that should be open, and writes the difference by compare-and-set,
// saying each transition as the controller says it (go/health) and keeping it in `history`. Nothing clears a condition
// but its finding going (ADR 0231 §14).
import (
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"sync"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
)
// The module, its seat, and the states it keeps.
const (
Module = health.KeeperModule
Seat = "mesh-conditions"
ConditionsState = health.ConditionsState
TakeoverState = health.TakeoverState
HistoryState = "history"
HeldState = "held"
BoundsState = "bounds"
// AskedState keeps the keeper's asks of the operator (go/asker).
AskedState = "asked"
)
// HistoryKeptFor is how long a transition is kept (clause 3).
const HistoryKeptFor = 90 * 24 * time.Hour
// Observers are the modules whose `findings` the keeper reads (clause 2; ADR 0565 §8 for the assigner's; the lead's
// table of 2026-10-29 for the provisioner's and the router's; ADR 0571 clause 2 for the seat artifact-store's holder,
// which observes S25, image-missing).
var Observers = []string{health.WatchmanModule, health.SelfCheckModule, health.HealersModule, delivery.AssignerModule,
"mesh-provisioner", "messenger", artifactStore}
// The other states it reads, as `<module>.<name>`.
const (
AttemptsRead = health.HealersModule + "." + health.AttemptsState
ControllerConditions = "mesh-controller.conditions"
ControllerLease = "mesh-controller.lease"
ControllerLeaseKey = "holder"
WantsRead = delivery.AssignerModule + ".wants"
// PlacementRead is the mesh-placement seat's placement, read for which module holds a seat on a node (hold.go; the
// lead, 2026-10-29, for ADR 0564 clause 2's inputs).
PlacementRead = delivery.PlacementRead
HavesRead = "node-engine.have"
ConditionWrites = 8
)
// Store is the bus as the keeper uses it: its own states, written by compare-and-set, and a key with its own time to
// live for a hold's bound.
type Store interface {
Get(state, key string) (value []byte, revision uint64, found bool, err error)
Create(state, key string, value any) (uint64, error)
Update(state, key string, value any, revision uint64) (uint64, error)
Delete(state, key string) error
Keys(state string) ([]string, error)
CreateTTL(state, key string, value any, ttl time.Duration) error
}
// held is a finding or a carried condition kept while its module is delivered (clause 13 (b)).
type held struct {
Key string `json:"key"`
Node string `json:"node"`
Module string `json:"module"`
Since time.Time `json:"since"`
Version string `json:"version"`
Wanted json.RawMessage `json:"-"`
}
// clearedMemo is what a cleared condition leaves for ReopenWithin: the same fault again is a reopening.
type clearedMemo struct {
at time.Time
count int
began time.Time
gaps []health.Gap
silenced *health.Silence
tried []health.Tried
}
// Keeper is the condition keeper's state and its writes.
type Keeper struct {
Store Store
// Say publishes one event on the seat; History keeps one transition.
Say func(e health.Event) error
Now func() time.Time
Logf func(format string, a ...any)
// Homes says, by kind, where each controller kind lives after the take-over (homes.go); Homes when nil.
Homes map[string]Home
// Bound is a delivery's hold bound; delivery.DefaultDeliveryBound when zero (hold.go).
Bound time.Duration
mu sync.Mutex
findings map[string]map[string]health.Finding
blind map[string]string
attempts map[string]health.Attempt
ctrl map[string]health.Condition
leaseOut bool
takeover health.Takeover
wants map[string]delivery.Want
haves map[string]delivery.Have
// holders are, by `<node>.<seat>`, the module the placement says holds the seat there.
holders map[string]string
stuck map[string]string
held map[string]held
// passed are the deliveries whose bound passed, by node and module, at the fingerprint wanted then; outOfHold the
// line a key raised out of a hold says (hold.go).
passed map[string]string
outOfHold map[string]string
cleared map[string]clearedMemo
seq uint64
}
func (k *Keeper) init() {
if k.findings == nil {
k.findings, k.blind, k.attempts, k.ctrl = map[string]map[string]health.Finding{}, map[string]string{},
map[string]health.Attempt{}, map[string]health.Condition{}
k.wants, k.haves, k.stuck, k.held, k.cleared = map[string]delivery.Want{}, map[string]delivery.Have{},
map[string]string{}, map[string]held{}, map[string]clearedMemo{}
k.takeover = health.Takeover{V: health.TakeoverVersion, Follow: health.FollowController}
k.passed, k.outOfHold, k.holders = map[string]string{}, map[string]string{}, map[string]string{}
}
if k.Now == nil {
k.Now = time.Now
}
if k.Logf == nil {
k.Logf = func(string, ...any) {}
}
}
// Finding takes one change of an observer's finding: a key put raises or changes its condition, a key deleted clears
// it (clause 2). A value that is not a finding is the observer's fault, said and left as it was.
func (k *Keeper) Finding(observer, key string, value []byte, deleted bool) error {
k.mu.Lock()
k.init()
if k.findings[observer] == nil {
k.findings[observer] = map[string]health.Finding{}
}
if deleted {
delete(k.findings[observer], key)
} else {
f, err := health.DecodeFinding(value)
if err != nil {
k.mu.Unlock()
k.Logf("%s's finding %s is not one the keeper can raise, so it is left as it stood: %v", observer, key, err)
return nil
}
if f.Key != key {
k.mu.Unlock()
k.Logf("%s's finding under %s names the key %s, so it is left as it stood", observer, key, f.Key)
return nil
}
k.findings[observer][key] = f
}
k.mu.Unlock()
return k.Settle(key)
}
// Blind says an observer's findings cannot be read: its keys stay as they were, and nothing is cleared for not seeing
// (ADR 0227 §4). Empty why says it is read again.
func (k *Keeper) Blind(observer, why string) {
k.mu.Lock()
defer k.mu.Unlock()
k.init()
if why == "" {
delete(k.blind, observer)
} else {
k.blind[observer] = why
}
}
// Attempt takes one change of the healers' `attempts`: the condition it names gets its tried, resolver and escalation
// (clause 7).
func (k *Keeper) Attempt(key string, value []byte, deleted bool) error {
k.mu.Lock()
k.init()
var condition string
if old, ok := k.attempts[key]; ok {
condition = old.Condition
}
if deleted {
delete(k.attempts, key)
} else {
var a health.Attempt
if err := json.Unmarshal(value, &a); err != nil || a.Condition == "" {
k.mu.Unlock()
k.Logf("the healers' attempt %s is not readable, so it is left out: %v", key, err)
return nil
}
k.attempts[key], condition = a, a.Condition
}
k.mu.Unlock()
if condition == "" {
return nil
}
return k.Settle(condition)
}
// Controller takes one change of the controller's `conditions` bucket (clauses 8, 9): read until the lease write.
func (k *Keeper) Controller(key string, value []byte, deleted bool) error {
k.mu.Lock()
k.init()
if deleted {
delete(k.ctrl, key)
} else {
var c health.Condition
if err := json.Unmarshal(value, &c); err != nil {
k.mu.Unlock()
k.Logf("the controller's condition %s is not readable, so nothing is carried from it: %v", key, err)
return nil
}
k.ctrl[key] = c
}
k.mu.Unlock()
return k.Settle(key)
}
// Lease takes the controller's lease key as read (clause 8, Limits 8): once it is handed to a holder that is no
// controller, nothing more is carried from the controller's bucket, and the kinds that ended with it clear.
func (k *Keeper) Lease(away bool) error {
k.mu.Lock()
k.init()
was := k.leaseOut
k.leaseOut = away
keys := make([]string, 0, len(k.ctrl))
for key := range k.ctrl {
keys = append(keys, key)
}
k.mu.Unlock()
if was == away {
return nil
}
return k.settleAll(keys)
}
// LeaseAway says whether the controller's lease is handed away.
func (k *Keeper) LeaseAway() bool {
k.mu.Lock()
defer k.mu.Unlock()
return k.leaseOut
}
// Takeover takes the keeper's own take-over key as read.
func (k *Keeper) Takeover(t health.Takeover) {
k.mu.Lock()
defer k.mu.Unlock()
k.init()
k.takeover = t
}
// Settle derives the condition key should be and writes the difference (clause 2), retried while another write moves
// it under this one.
func (k *Keeper) Settle(key string) error {
var err error
for i := 0; i < ConditionWrites; i++ {
if err = k.settleOnce(key); !errors.Is(err, stdio.ErrChanged) {
return err
}
}
return fmt.Errorf("the condition %s kept moving under this write, %d tries: %w", key, ConditionWrites, err)
}
func (k *Keeper) settleAll(keys []string) error {
sort.Strings(keys)
var problems []string
for _, key := range keys {
if err := k.Settle(key); err != nil {
problems = append(problems, err.Error())
}
}
if len(problems) > 0 {
return errors.New(strings.Join(problems, "; "))
}
return nil
}
// wanted is what the inputs say key should be: a finding of an observer, or a condition carried from the controller's
// bucket of a kind that ends with the controller, until the lease write (clause 9). Nil when nothing says it.
func (k *Keeper) wanted(key string) (*health.Finding, *health.Condition) {
observers := make([]string, 0, len(k.findings))
for o := range k.findings {
observers = append(observers, o)
}
sort.Strings(observers)
for _, o := range observers {
if f, ok := k.findings[o][key]; ok {
return &f, nil
}
}
if c, ok := k.ctrl[key]; ok && !k.leaseOut && k.endsWithController(c) {
return nil, &c
}
return nil, nil
}
func (k *Keeper) endsWithController(c health.Condition) bool {
h, ok := k.homeOf(c.Kind)
return ok && h.Carried
}
// settleOnce is one try of Settle.
func (k *Keeper) settleOnce(key string) error {
now := k.Now().UTC()
raw, rev, open, err := k.Store.Get(ConditionsState, key)
if err != nil {
return fmt.Errorf("reading the condition %s: %w", key, err)
}
var was health.Condition
if open {
if err := json.Unmarshal(raw, &was); err != nil {
return fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err)
}
}
k.mu.Lock()
k.init()
f, carried := k.wanted(key)
hold, holding := k.holdFor(key, f, carried, open)
k.mu.Unlock()
if f == nil && carried == nil {
k.dropHeld(key, "its finding went while its module was delivered")
if !open {
return nil
}
return k.clear(key, was, rev, now, "its finding went")
}
if holding {
return k.keepHeld(hold)
}
k.dropHeld(key, "")
next := k.derive(key, was, open, f, carried, now)
events := transitions(was, next, open)
if len(events) == 0 && open && sameBody(was, next) {
return nil
}
if open {
_, err = k.Store.Update(ConditionsState, key, next, rev)
} else {
_, err = k.Store.Create(ConditionsState, key, next)
}
if err != nil {
return err
}
k.mu.Lock()
delete(k.cleared, key)
k.mu.Unlock()
for _, e := range events {
e.Condition, e.At, e.Event = next, now, health.EventFor(e.Change)
e.Show = k.show(next, carried != nil)
k.tell(e)
}
return nil
}
func (k *Keeper) show(c health.Condition, carried bool) string {
if carried {
return health.ShowController(c.Key)
}
return health.ShowKeeper(c.Key)
}
// derive is the condition key should be now, from what was open and what the inputs say.
func (k *Keeper) derive(key string, was health.Condition, open bool, f *health.Finding, carried *health.Condition, now time.Time) health.Condition {
var c health.Condition
switch {
case carried != nil:
// Raised and cleared as the controller's bucket says, with its show and actions (Limits 9).
c = *carried
c.Epoch = 0
c.LooksUnread = ""
case open:
c = was
k.fromFinding(&c, *f, now)
default:
c = health.Condition{Key: key, Raised: now, Count: 1, Resolver: health.ResolverSelf}
k.fromFinding(&c, *f, now)
k.mu.Lock()
if before, ok := k.cleared[key]; ok && now.Sub(before.at) <= health.ReopenWithin {
c.Count = before.count + 1
c.First = before.began
c.Gaps = append(append([]health.Gap(nil), before.gaps...), health.Gap{Cleared: before.at, Reopened: now})
if over := len(c.Gaps) - health.KeptGaps; over > 0 {
c.First, c.Gaps = c.Gaps[over-1].Reopened, c.Gaps[over:]
}
if before.silenced != nil && now.Before(before.silenced.Until) {
c.Silenced = before.silenced
}
c.Tried = before.tried
}
k.mu.Unlock()
}
if carried == nil {
k.overlay(&c, now)
}
if !open {
k.mu.Lock()
if line := k.outOfHold[key]; line != "" {
c.Explanation = strings.TrimSpace(c.Explanation + " " + line)
}
k.mu.Unlock()
}
if c.Silenced != nil && !now.Before(c.Silenced.Until) {
c.Silenced = nil
}
return c
}
// fromFinding is what the condition says of its finding: its words, subject, severity and source, and its evidence
// newest first (clause 2). A healer's escalation is not undone by the finding saying a warning again.
func (k *Keeper) fromFinding(c *health.Condition, f health.Finding, now time.Time) {
escalated := c.Escalated()
c.Kind, c.Subject, c.Summary, c.Source = f.Kind, f.Subject, f.Summary, f.Source
c.Headline, c.Explanation, c.Resolved, c.Needs, c.Actions = f.Headline, f.Explanation, f.Resolved, f.Needs, f.Actions
if !(escalated && f.Severity != health.Urgent) {
c.Severity = f.Severity
}
if !escalated {
c.Resolver = f.Resolver
if c.Resolver == "" {
c.Resolver = health.ResolverSelf
}
}
if len(f.Evidence) > 0 {
c.Evidence = append([]health.Evidence(nil), f.Evidence...)
c.LastObserved = f.Evidence[0].At
} else if c.Evidence == nil {
c.Evidence = []health.Evidence{{At: f.At, Said: f.Summary}}
c.LastObserved = f.At
}
c.Observations = len(c.Evidence)
if c.Actions == nil {
c.Actions = []health.Action{}
}
_ = now
}
// overlay writes the healers' attempts into the condition (clause 7) and, until the lease write, what the controller's
// bucket holds for the same key (clause 8): its tried, its resolver and escalation, and its silence where the keeper has
// none of its own.
func (k *Keeper) overlay(c *health.Condition, now time.Time) {
k.mu.Lock()
var mine []health.Attempt
for _, a := range k.attempts {
if a.Condition == c.Key {
mine = append(mine, a)
}
}
ctrl, carry := k.ctrl[c.Key]
carry = carry && !k.leaseOut
k.mu.Unlock()
sort.Slice(mine, func(i, j int) bool { return mine[i].Begun.Before(mine[j].Begun) })
if carry && len(ctrl.Tried) > 0 {
c.Tried = append([]health.Tried(nil), ctrl.Tried...)
if ctrl.Escalated() {
c.Resolver, c.Severity = health.ResolverOperator, health.Urgent
} else if ctrl.Resolver != "" && ctrl.Resolver != health.ResolverSelf {
c.Resolver = ctrl.Resolver
}
}
for _, a := range mine {
t := a.Tried()
if !containsTried(c.Tried, t) {
c.Tried = append(c.Tried, t)
}
switch a.Outcome {
case health.OutcomeEscalated:
c.Resolver, c.Severity = health.ResolverOperator, health.Urgent
default:
if c.Resolver != health.ResolverOperator {
c.Resolver = health.ResolverHealer(a.Healer)
}
}
}
if len(c.Tried) > health.KeptTried {
c.Tried = c.Tried[len(c.Tried)-health.KeptTried:]
}
if carry && c.Silenced == nil && ctrl.Silenced != nil && now.Before(ctrl.Silenced.Until) {
c.Silenced = ctrl.Silenced
}
}
func containsTried(list []health.Tried, t health.Tried) bool {
for _, x := range list {
if x.At.Equal(t.At) && x.By == t.By && x.What == t.What {
return true
}
}
return false
}
// transitions are the events between what was open and what is now, in the controller's order: raised or reopened;
// else severity, resolver, silence and words changes. An observation that changes none of these says nothing.
func transitions(was, now health.Condition, open bool) []health.Event {
if !open {
change := health.ChangeRaised
if now.Count > 1 {
change = health.ChangeReopened
}
return []health.Event{{Change: change}}
}
var out []health.Event
if was.Severity != now.Severity {
out = append(out, health.Event{Change: health.ChangeSeverity, Was: string(was.Severity)})
}
if was.Resolver != now.Resolver {
out = append(out, health.Event{Change: health.ChangeResolver, Was: was.Resolver})
}
switch {
case was.Silenced == nil && now.Silenced != nil:
out = append(out, health.Event{Change: health.ChangeSilenced, Why: now.Silenced.Why})
case was.Silenced != nil && now.Silenced == nil:
out = append(out, health.Event{Change: health.ChangeUnsilenced})
}
if was.Needs != now.Needs || !sameActions(was.Actions, now.Actions) {
out = append(out, health.Event{Change: health.ChangeWords, Was: was.Needs})
}
return out
}
func sameActions(a, b []health.Action) bool {
x, _ := json.Marshal(a)
y, _ := json.Marshal(b)
return string(x) == string(y) || (len(a) == 0 && len(b) == 0)
}
func sameBody(a, b health.Condition) bool {
x, _ := json.Marshal(a)
y, _ := json.Marshal(b)
return string(x) == string(y)
}
// clear removes an open condition whose finding went, and says it (clause 2).
func (k *Keeper) clear(key string, was health.Condition, rev uint64, now time.Time, why string) error {
// A delete at the revision read: a write that moved it meanwhile is read again.
if _, err := k.Store.Update(ConditionsState, key, json.RawMessage("null"), rev); err != nil {
return err
}
if err := k.Store.Delete(ConditionsState, key); err != nil {
return err
}
k.mu.Lock()
k.cleared[key] = clearedMemo{at: now, count: was.Count, began: was.Began(), gaps: was.Gaps, silenced: was.Silenced, tried: was.Tried}
k.mu.Unlock()
cleared := now
k.tell(health.Event{Condition: was, Event: health.EventCleared, At: now, Change: health.ChangeCleared, Why: why,
Cleared: &cleared, Show: k.show(was, false)})
return nil
}
// tell keeps the transition in history and says it on the seat. A transition not said is said in the log: the condition
// and its history still hold it.
func (k *Keeper) tell(e health.Event) {
k.mu.Lock()
k.seq++
hk := fmt.Sprintf("%020d-%06d", e.At.UnixNano(), k.seq)
k.mu.Unlock()
if _, err := k.Store.Create(HistoryState, hk, e); err != nil {
k.Logf("the %s of %s could not be kept in history: %v", e.Change, e.Key, err)
}
if k.Say == nil {
return
}
if err := k.Say(e); err != nil {
k.Logf("the %s of %s could NOT be said on the seat: %v", e.Change, e.Key, err)
}
}
// Open is every open condition, ordered as `conditions` says them.
func (k *Keeper) Open() ([]health.Condition, error) {
keys, err := k.Store.Keys(ConditionsState)
if err != nil {
return nil, err
}
var out []health.Condition
for _, key := range keys {
raw, _, found, err := k.Store.Get(ConditionsState, key)
if err != nil {
return nil, err
}
if !found || string(raw) == "null" {
continue
}
var c health.Condition
if err := json.Unmarshal(raw, &c); err != nil {
return nil, fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err)
}
out = append(out, c)
}
health.Order(out)
return out, nil
}
// Resync settles every key any input names or any open condition holds (clause 12): what differs is raised or cleared.
func (k *Keeper) Resync() (int, error) {
open, err := k.Open()
if err != nil {
return 0, err
}
seen := map[string]bool{}
for _, c := range open {
seen[c.Key] = true
}
k.mu.Lock()
k.init()
for o, fs := range k.findings {
if k.blind[o] != "" {
continue
}
for key := range fs {
seen[key] = true
}
}
for key := range k.ctrl {
seen[key] = true
}
k.mu.Unlock()
keys := make([]string, 0, len(seen))
for key := range seen {
keys = append(keys, key)
}
return len(keys), k.settleAll(keys)
}
// Silence stops a condition's messages for a while, a hand act with why (clause 3, Limits 1): kept as its transition in
// history, naming who and why.
func (k *Keeper) Silence(key string, d time.Duration, by, why string) (health.Condition, error) {
if strings.TrimSpace(why) == "" {
return health.Condition{}, errors.New("a silence says why")
}
if d <= 0 || d > health.MaxSilence {
return health.Condition{}, fmt.Errorf("a silence lasts more than nothing and at most %s, not %s", health.MaxSilence, d)
}
for i := 0; i < ConditionWrites; i++ {
raw, rev, found, err := k.Store.Get(ConditionsState, key)
if err != nil {
return health.Condition{}, err
}
if !found || string(raw) == "null" {
return health.Condition{}, fmt.Errorf("no condition %s is open", key)
}
var c health.Condition
if err := json.Unmarshal(raw, &c); err != nil {
return health.Condition{}, err
}
now := k.Now().UTC()
c.Silenced = &health.Silence{Until: now.Add(d), By: by, Why: why, Since: now}
if _, err := k.Store.Update(ConditionsState, key, c, rev); errors.Is(err, stdio.ErrChanged) {
continue
} else if err != nil {
return health.Condition{}, err
}
k.tell(health.Event{Condition: c, Event: health.EventChanged, At: now, Change: health.ChangeSilenced,
Why: why + " (" + by + ")", Show: health.ShowKeeper(key)})
return c, nil
}
return health.Condition{}, fmt.Errorf("the condition %s kept moving under the silence", key)
}
@@ -0,0 +1,341 @@
package main
import (
"encoding/json"
"fmt"
"sort"
"strings"
"sync"
"testing"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
)
// fakeStore is the bus's states in memory, with revisions, as the runtime answers them.
type fakeStore struct {
mu sync.Mutex
vals map[string][]byte
revs map[string]uint64
next uint64
// ttls are the times to live the keys were created with.
ttls map[string]time.Duration
}
func newFakeStore() *fakeStore {
return &fakeStore{vals: map[string][]byte{}, revs: map[string]uint64{}}
}
func (f *fakeStore) Get(state, key string) ([]byte, uint64, bool, error) {
f.mu.Lock()
defer f.mu.Unlock()
v, ok := f.vals[state+"/"+key]
return v, f.revs[state+"/"+key], ok, nil
}
func (f *fakeStore) put(state, key string, value any) uint64 {
raw, _ := json.Marshal(value)
f.next++
f.vals[state+"/"+key], f.revs[state+"/"+key] = raw, f.next
return f.next
}
func (f *fakeStore) Create(state, key string, value any) (uint64, error) {
f.mu.Lock()
defer f.mu.Unlock()
if _, ok := f.vals[state+"/"+key]; ok {
return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged)
}
return f.put(state, key, value), nil
}
func (f *fakeStore) Update(state, key string, value any, rev uint64) (uint64, error) {
f.mu.Lock()
defer f.mu.Unlock()
if f.revs[state+"/"+key] != rev {
return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged)
}
return f.put(state, key, value), nil
}
func (f *fakeStore) Delete(state, key string) error {
f.mu.Lock()
defer f.mu.Unlock()
delete(f.vals, state+"/"+key)
delete(f.revs, state+"/"+key)
return nil
}
func (f *fakeStore) Keys(state string) ([]string, error) {
f.mu.Lock()
defer f.mu.Unlock()
var out []string
for k := range f.vals {
if s, key, _ := strings.Cut(k, "/"); s == state {
out = append(out, key)
}
}
sort.Strings(out)
return out, nil
}
func (f *fakeStore) CreateTTL(state, key string, value any, ttl time.Duration) error {
f.mu.Lock()
if f.ttls == nil {
f.ttls = map[string]time.Duration{}
}
f.ttls[state+"/"+key] = ttl
f.mu.Unlock()
_, err := f.Create(state, key, value)
return err
}
// rig is a keeper over a fake store, its clock and what it said.
type rig struct {
k *Keeper
st *fakeStore
now time.Time
said []health.Event
}
func newRig() *rig {
r := &rig{st: newFakeStore(), now: time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC)}
r.k = &Keeper{Store: r.st, Now: func() time.Time { return r.now }, Say: func(e health.Event) error {
r.said = append(r.said, e)
return nil
}}
return r
}
func (r *rig) changes() string {
var out []string
for _, e := range r.said {
out = append(out, e.Change+" "+e.Key)
}
r.said = nil
return strings.Join(out, ", ")
}
func (r *rig) open(key string) (health.Condition, bool) {
raw, _, ok, _ := r.st.Get(ConditionsState, key)
var c health.Condition
if ok {
_ = json.Unmarshal(raw, &c)
}
return c, ok
}
// finding is an observer's finding as mesh-sdk's layout writes it (health.EncodeFinding).
func finding(t *testing.T, observer, scope, id, kind, source string, sev health.Severity, at time.Time) (string, []byte) {
t.Helper()
f := health.Finding{Key: health.Key(scope, id, kind), Kind: kind, Subject: health.Subject{Scope: scope, ID: id},
Severity: sev, Summary: kind + " of " + id, Headline: kind, Explanation: "It is " + kind + ".", Source: source,
Observer: observer, At: at, Evidence: []health.Evidence{{At: at, Said: kind}}}
raw, err := health.EncodeFinding(f)
if err != nil {
t.Fatal(err)
}
return f.Key, raw
}
// ADR 0564 clauses 2, 3, 7, 8 (issue 1952 C2, C3): a finding raises, changes and clears its condition; a blind observer
// clears nothing; the healers' attempts and the controller's escalation reach the condition; a reopening within ten
// minutes is the same fault; a silence is a transition in history.
func TestTheKeeperRaisesChangesAndClearsFromFindings(t *testing.T) {
r := newRig()
key, raw := finding(t, watchman, health.ScopeMachine, "g14", "silent", "S1", health.Warning, r.now)
steps := []struct {
name string
do func() error
said string
check func(c health.Condition, open bool) string
}{
{name: "a finding written raises", do: func() error { return r.k.Finding(watchman, key, raw, false) },
said: "raised machine.g14.silent", check: func(c health.Condition, open bool) string {
if !open || c.Severity != health.Warning || c.Resolver != health.ResolverSelf || c.Epoch != 0 || c.Count != 1 {
return fmt.Sprintf("%+v", c)
}
return ""
}},
{name: "the same finding again says nothing", do: func() error { return r.k.Finding(watchman, key, raw, false) }},
{name: "an observer gone blind clears nothing", do: func() error { r.k.Blind(watchman, "refused"); _, err := r.k.Resync(); return err },
check: func(c health.Condition, open bool) string {
if !open {
return "cleared by blindness"
}
return ""
}},
{name: "a healer's attempt reaches tried and the resolver", do: func() error {
a, _ := json.Marshal(health.Attempt{Healer: "H3", Condition: key, Act: "asserted", Outcome: health.OutcomeActed, Begun: r.now})
return r.k.Attempt("h3-1", a, false)
}, said: "resolver machine.g14.silent", check: func(c health.Condition, _ bool) string {
if c.Resolver != "healer:H3" || len(c.Tried) != 1 || c.Tried[0].By != "healer H3" {
return fmt.Sprintf("%+v", c)
}
return ""
}},
{name: "an escalation makes it urgent, the operator's", do: func() error {
a, _ := json.Marshal(health.Attempt{Healer: "H3", Condition: key, Act: "handed over", Outcome: health.OutcomeEscalated,
Begun: r.now.Add(time.Minute)})
return r.k.Attempt("h3-2", a, false)
}, said: "severity machine.g14.silent, resolver machine.g14.silent", check: func(c health.Condition, _ bool) string {
if c.Severity != health.Urgent || c.Resolver != health.ResolverOperator || len(c.Tried) != 2 {
return fmt.Sprintf("%+v", c)
}
return ""
}},
{name: "a silence is a transition with who and why", do: func() error {
_, err := r.k.Silence(key, time.Hour, "jochen", "known, a cable")
return err
}, said: "silenced machine.g14.silent"},
{name: "the finding deleted clears", do: func() error { return r.k.Finding(watchman, key, nil, true) },
said: "cleared machine.g14.silent", check: func(_ health.Condition, open bool) string {
if open {
return "still open"
}
return ""
}},
{name: "raised again within ten minutes is a reopening", do: func() error {
r.now = r.now.Add(5 * time.Minute)
return r.k.Finding(watchman, key, raw, false)
}, said: "reopened machine.g14.silent", check: func(c health.Condition, _ bool) string {
if c.Count != 2 || len(c.Gaps) != 1 {
return fmt.Sprintf("%+v", c)
}
return ""
}},
// Until the lease write the controller's escalation for the same key reaches the condition (clause 8).
{name: "the controller's escalation is carried", do: func() error {
ctrl, _ := json.Marshal(health.Condition{Key: key, Kind: "silent", Severity: health.Urgent, Resolver: health.ResolverOperator,
Tried: []health.Tried{{At: r.now, What: "x", Outcome: "escalated", By: "healer H1"}}})
_ = r.k.Attempt("h3-1", nil, true)
_ = r.k.Attempt("h3-2", nil, true)
return r.k.Controller(key, ctrl, false)
}, check: func(c health.Condition, _ bool) string {
if c.Resolver != health.ResolverOperator || c.Severity != health.Urgent {
return fmt.Sprintf("%+v", c)
}
return ""
}},
}
for _, s := range steps {
if err := s.do(); err != nil {
t.Fatalf("%s: %v", s.name, err)
}
if got := r.changes(); s.said != "" && got != s.said || s.said == "" && got != "" && s.name != "the controller's escalation is carried" {
t.Errorf("%s: said %q, want %q", s.name, got, s.said)
}
if s.check != nil {
c, open := r.open(key)
if why := s.check(c, open); why != "" {
t.Errorf("%s: %s", s.name, why)
}
}
}
// Every transition is kept in history, and each event carries the keeper's show.
keys, _ := r.st.Keys(HistoryState)
if len(keys) < 7 {
t.Errorf("history keeps %d transitions", len(keys))
}
raw0, _, _, _ := r.st.Get(HistoryState, keys[0])
if !strings.Contains(string(raw0), `"show":"mesh-conditions.conditions key=machine.g14.silent"`) {
t.Errorf("the first transition is %s", raw0)
}
// A carried kind (it ends with the controller) is raised and cleared as the controller's bucket says, until the
// lease is handed away, with the controller's show (clause 9, Limits 9).
deaf, _ := json.Marshal(health.Condition{Key: "core.controller.deaf", Kind: "controller-deaf", Source: "S4",
Subject: health.Subject{Scope: health.ScopeCore, ID: "controller"}, Severity: health.Urgent, Summary: "deaf", Resolver: "self", Epoch: 9})
if err := r.k.Controller("core.controller.deaf", deaf, false); err != nil {
t.Fatal(err)
}
if c, open := r.open("core.controller.deaf"); !open || c.Epoch != 0 || r.said[0].Show != "mesh-controller.conditions key=core.controller.deaf" {
t.Errorf("a carried kind is %+v, said %+v", c, r.said)
}
r.said = nil
if err := r.k.Lease(true); err != nil {
t.Fatal(err)
}
if _, open := r.open("core.controller.deaf"); open || r.changes() != "cleared core.controller.deaf" {
t.Error("a carried kind outlived the lease write")
}
}
// ADR 0564 clause 9 (issue 1952 C5, C8; the lead 2026-10-29): compare counts only what both sides can raise. A key held
// while its module is delivered, a kind that ends with the controller and a kind the controller never raises are listed
// apart; a kind with no home decided is a difference naming the gap; take-over is refused while compare differs, and
// hand-back after the lease write.
func TestCompareCountsOnlyWhatBothSidesRaise(t *testing.T) {
r := newRig()
put := func(key, kind string) []byte {
raw, _ := json.Marshal(health.Condition{Key: key, Kind: kind, Severity: health.Warning, Source: "x"})
return raw
}
key, raw := finding(t, watchman, health.ScopeMachine, "g14", "silent", "S1", health.Warning, r.now)
_ = r.k.Finding(watchman, key, raw, false)
_ = r.k.Controller(key, put(key, "silent"), false)
only, onlyRaw := finding(t, delivery.AssignerModule, health.ScopeSeat, "mesh-placement.ace", "states-unbound", "say", health.Warning, r.now)
_ = r.k.Finding(delivery.AssignerModule, only, onlyRaw, false)
_ = r.k.Controller("core.controller.deaf", put("core.controller.deaf", "controller-deaf"), false)
_ = r.k.Controller("delivery.x.mystery", put("delivery.x.mystery", "mystery"), false)
_ = r.k.Controller("bus.x.consumer-wrong", put("bus.x.consumer-wrong", "consumer-wrong"), false)
_ = r.k.Controller("seat.vault.ace.holder-silent", put("seat.vault.ace.holder-silent", "holder-silent"), false)
cmp, err := r.k.Compare()
if err != nil {
t.Fatal(err)
}
reasons := map[string]string{}
for _, d := range append(append([]Difference(nil), cmp.Differences...), cmp.Apart...) {
reasons[d.Key] = d.Reason
}
for key, says := range map[string]string{
only: "keeper only",
"core.controller.deaf": "carried",
"delivery.x.mystery": "no home is decided for the kind mystery",
"bus.x.consumer-wrong": "mesh-self-check has not written its finding",
// A kind of a probe the operator dropped (2026-10-30) is carried, never missing.
"seat.vault.ace.holder-silent": "carried from its bucket until the lease write (dropped 2026-10-30 by the operator",
} {
if !strings.Contains(reasons[key], says) {
t.Errorf("%s: %q, want it to say %q", key, reasons[key], says)
}
}
if cmp.Same != 1 || len(cmp.Differences) != 2 {
t.Errorf("compare: %+v", cmp)
}
// The lead's table of 2026-10-29: each kind has its home, lease-split and image-missing do not end, and the 22
// that end are carried; with the 11 kinds of the probes the operator dropped on 2026-10-30, 33.
carried := 0
for _, h := range Homes {
if h.Carried {
carried++
}
}
for kind, observer := range map[string]string{"needs-operator": watchman, "machine-network": watchman, "lease-split": selfCheck,
"data-shrank": selfCheck, "bus-upgrade-failed": assigner, "retire-waiting": provisioner, "asks-undelivered": messenger,
"image-missing": artifactStore} {
if h, ok := Homes[kind]; !ok || h.Observer != observer || h.Carried {
t.Errorf("%s's home is %+v", kind, h)
}
}
for _, kind := range []string{"resolver-wrong", "holder-silent", "archives-missing", "archives-unheld", "binding-kept",
"binding-moved", "binding-moving", "grant-composes-nothing", "root-not-free", "cleanup-waiting", "data-held-twice"} {
if h := Homes[kind]; !h.Carried || h.Observer != "" || h.Why != Dropped {
t.Errorf("the dropped probe's kind %s is %+v", kind, h)
}
}
if len(Homes) != 84 || carried != 33 { // 19 of ADR 0564, the 63 of the table, plan-failed and stalled
t.Errorf("the table holds %d kinds, %d carried", len(Homes), carried)
}
if _, err := r.k.TakeOver(nil, "jochen", "now"); err == nil || !strings.Contains(err.Error(), "2 difference") {
t.Errorf("take-over while compare differs: %v", err)
}
if _, err := r.k.HandBack("jochen", "back"); err != nil {
t.Errorf("hand-back before the lease write: %v", err)
}
_ = r.k.Lease(true)
if _, err := r.k.HandBack("jochen", "back"); err == nil {
t.Error("hand-back after the lease write was taken")
}
}
@@ -0,0 +1,164 @@
// mesh-conditions: the condition keeper, holder of the seat mesh-conditions (novox/hq ADR 0564 clauses 1-3, 9, 13). A
// Go bundle served by a runtime of its own, on its own bus account (`runs-as`), so that its account alone writes its
// states: `conditions` (one key per open condition), `history` (every transition, ninety days), `takeover` (the key the
// readers follow), `held` and `bounds` (what a delivery holds, and for how long). It watches every observer's
// `findings`, the healers' `attempts`, the assigner's wants, the nodes' haves and the controller's lease and, until the
// lease write, its `conditions`; it says every transition on its seat with the controller's body. stdout is the MCP
// channel; what this module says, it says on stderr.
package main
import (
"fmt"
"os"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/asker"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/health"
)
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
// runtimeStore is the bus, through the runtime that launched this bundle.
type runtimeStore struct{}
func (runtimeStore) Get(state, key string) ([]byte, uint64, bool, error) {
e, err := stdio.State(state).Get(key)
if err != nil || e == nil {
return nil, 0, false, err
}
return e.Value, e.Revision, true, nil
}
func (runtimeStore) Create(state, key string, value any) (uint64, error) {
return stdio.State(state).Create(key, value)
}
func (runtimeStore) Update(state, key string, value any, revision uint64) (uint64, error) {
return stdio.State(state).Update(key, value, revision)
}
func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) }
func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() }
// CreateTTL creates the key with its own time to live (`bounds` is key-ttl); its expiry is handed over as OpExpire.
func (runtimeStore) CreateTTL(state, key string, value any, ttl time.Duration) error {
_, err := stdio.State(state).CreateTTL(key, value, ttl)
return err
}
// sayOnSeat says a transition on the seat, as the controller says it on its own (ADR 0564 clause 3).
func sayOnSeat(e health.Event) error {
return stdio.SeatPublish("mesh.seat."+Seat+".event."+e.Event, e, "")
}
// NewKeeper is the keeper over the runtime.
func NewKeeper() *Keeper {
return &Keeper{Store: runtimeStore{}, Say: sayOnSeat, Now: time.Now, Logf: logf}
}
// watch keeps a watch of one state, said blind while it cannot be read: a state not on the bus yet (an observer not
// delivered) is read again when it is.
func watch(k *Keeper, state, observer string, handle func(stdio.StateChange) error) {
go func() {
for wait := 5 * time.Second; ; wait = min(wait*2, 5*time.Minute) {
err := stdio.State(state).Watch("", handle)
if err == nil {
if observer != "" {
k.Blind(observer, "")
}
return
}
if observer != "" {
k.Blind(observer, err.Error())
}
logf("%s cannot be watched (%v); watched again in %s", state, err, wait)
time.Sleep(wait)
}
}()
}
// Keep starts every watch the keeper reads by (clauses 2, 8, 9, 13).
func Keep(k *Keeper) {
for _, s := range []string{HeldState} {
keys, _ := stdio.State(s).Keys()
for _, key := range keys {
if e, err := stdio.State(s).Get(key); err == nil && e != nil {
k.restoreHeld(key, e.Value)
}
}
}
if e, err := stdio.State(TakeoverState).Get(health.TakeoverKey); err == nil {
var raw []byte
if e != nil {
raw = e.Value
}
if t, err := health.DecodeTakeover(raw, e != nil); err == nil {
k.Takeover(t)
}
}
said := func(err error) error {
if err != nil {
logf("%v", err)
}
return nil
}
watch(k, ControllerLease, "", func(c stdio.StateChange) error {
if c.Key != ControllerLeaseKey {
return nil
}
away, why := buscompose.LeaseHandedAway(c.Value, c.Op == "put", nil)
logf("the controller's lease: %s", why)
return said(k.Lease(away))
})
watch(k, WantsRead, "", func(c stdio.StateChange) error { return said(k.Want(c.Key, c.Value, c.Op != "put")) })
watch(k, HavesRead, "", func(c stdio.StateChange) error { return said(k.Have(c.Key, c.Value, c.Op != "put")) })
watch(k, AttemptsRead, health.HealersModule+" attempts", func(c stdio.StateChange) error {
return said(k.Attempt(c.Key, c.Value, c.Op != "put"))
})
watch(k, ControllerConditions, "", func(c stdio.StateChange) error {
if k.LeaseAway() {
return nil
}
return said(k.Controller(c.Key, c.Value, c.Op != "put"))
})
for _, o := range Observers {
watch(k, o+"."+health.FindingsState, o, func(c stdio.StateChange) error {
return said(k.Finding(o, c.Key, c.Value, c.Op != "put"))
})
}
watch(k, BoundsState, "", func(c stdio.StateChange) error {
if c.Op == stdio.OpExpire {
return said(k.BoundPassed(c.Key))
}
return nil
})
watch(k, PlacementRead, "", func(c stdio.StateChange) error {
k.Placement(c.Key, c.Value, c.Op != stdio.OpPut)
return nil
})
}
func main() {
k := NewKeeper()
client := asker.New(AskedState)
client.Module = Module
go func() {
<-stdio.Served()
Keep(k)
if err := client.Listen(k.Warranted, func(h asker.Handled, w asks.Warrant, err error) {
if err != nil {
logf("the answer to %s (%s): %v", w.Ask, h, err)
}
}); err != nil {
logf("the operator's answers cannot be heard: %v", err)
}
}()
if err := stdio.Serve("", Tools(k, client)); err != nil {
logf("%v", err)
os.Exit(1)
}
}
@@ -0,0 +1,69 @@
package main
import (
"os"
"path/filepath"
"slices"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/health"
)
// ADR 0564 clauses 1, 2, 3 and 12 (issue 1952 C1): the manifest says what the code does. It claims the seat
// mesh-conditions once for the mesh with the verbs mesh-sdk defines for it; reads every observer's findings, the healers'
// attempts, the controller's conditions and lease, the wants and the haves, and never the healers' would; keeps its
// states, history ninety days and each delivery's bound as long as the bound; and every state has a tool that reads it,
// every act a tool that starts it.
func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) {
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
if err != nil {
t.Fatal(err)
}
m, err := catalogue.ParseManifest(raw)
if err != nil {
t.Fatalf("the manifest is refused: %v", err)
}
seat, _ := catalogue.SeatNamed(Seat)
if m.Module != Module || m.RunsAs != Module || len(m.Claims) != 1 || m.Claims[0].Name != Seat ||
!slices.Equal(Verbs, catalogue.VerbNames(seat.Serves)) {
t.Errorf("module %s runs as %s and claims %+v; the seat serves %v", m.Module, m.RunsAs, m.Claims, catalogue.VerbNames(seat.Serves))
}
reads := []string{ControllerConditions, ControllerLease, WantsRead, HavesRead, AttemptsRead, PlacementRead}
for _, o := range Observers {
reads = append(reads, o+"."+health.FindingsState)
}
slices.Sort(reads)
got := slices.Clone(m.Reads)
slices.Sort(got)
if !slices.Equal(got, reads) || slices.Contains(m.Reads, health.HealersModule+"."+health.WouldState) {
t.Errorf("reads %v, want %v and never the healers' would", got, reads)
}
ttl, keyTTL := map[string]int{}, map[string]bool{}
for _, s := range m.State {
ttl[s.Name], keyTTL[s.Name] = s.TTLSeconds, s.KeyTTL
}
// Each bound is its own key's time to live (the runtime's per-key TTL), never the bucket's age.
if ttl[HistoryState] != int(HistoryKeptFor.Seconds()) || ttl[BoundsState] != 0 || !keyTTL[BoundsState] {
t.Errorf("history is kept %d s, a bound's bucket ages %d s, its keys have their own time to live: %v",
ttl[HistoryState], ttl[BoundsState], keyTTL[BoundsState])
}
// Clause 12: each state with the tool that reads it, each act with the tool that starts it.
readBy := map[string]string{ConditionsState: Seat + ".conditions", HistoryState: HistoryTool, TakeoverState: TakeoverTool,
HeldState: HeldTool, BoundsState: HeldTool, AskedState: Seat + ".take-over", "a resync": Seat + ".resync",
"what it reads": InputsTool}
tools := map[string]bool{}
for _, tool := range Tools(&Keeper{}, nil) {
tools[tool.Name] = true
}
for name := range ttl {
if readBy[name] == "" {
t.Errorf("the state %s has no tool that reads it", name)
}
}
for what, tool := range readBy {
if !tools[tool] {
t.Errorf("%s is read or started by %s, which is not served", what, tool)
}
}
}
@@ -0,0 +1,2 @@
# What these tests read outside their package, as paths from the repository's root (issue 1936).
modules/mesh-conditions/module.json
@@ -0,0 +1,252 @@
package main
// The seat's verbs and the keeper's own tools (novox/hq ADR 0564 clauses 3, 9 and 12; issue 1952 C4): `conditions`
// answers as the controller's verb answers, so a caller changes only the seat it asks; and a read of every state the
// keeper keeps and of what it reads, so a person sees all it holds and a recovery is a call.
import (
"encoding/json"
"fmt"
"slices"
"strconv"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/health"
)
// Verbs are the seat's verbs, as the manifest claims them.
var Verbs = []string{"conditions", "silence", "compare", "take-over", "hand-back", "resync"}
// The keeper's own read tools.
const (
HistoryTool = "conditions_history"
HeldTool = "conditions_held"
TakeoverTool = "conditions_takeover"
InputsTool = "conditions_inputs"
)
// briefEvidence is how many observations a condition carries in a list: its newest (the controller's, issue 314).
const briefEvidence = 1
// inBrief is a condition in a list: its newest evidence, with how many it keeps and how to read it whole.
type inBrief struct {
health.Condition
EvidenceKept int `json:"evidence-kept,omitempty"`
More string `json:"more,omitempty"`
}
func obj(props map[string]any, required ...string) map[string]any {
out := map[string]any{"type": "object", "properties": props}
if len(required) > 0 {
out["required"] = required
}
return out
}
func str(description string) map[string]any {
return map[string]any{"type": "string", "description": description}
}
// Tools are what the runtime serves.
func Tools(k *Keeper, a Asks) []stdio.Tool {
return []stdio.Tool{
{Name: Seat + ".conditions", Description: "What is wrong with the mesh now: every open condition, urgent first, then " +
"oldest, raised from the observers' findings and cleared when the finding goes, never by hand. Given a key, that " +
"one whole; with history, every transition lately.",
Input: obj(map[string]any{"key": str("a condition's key"), "scope": str("only this scope"),
"severity": str("urgent or warning"), "machine": str("only those about this machine"),
"history": str("\"true\": every transition lately"), "days": str("with history: days back (7, at most 90)")}),
Run: func(args map[string]any) (any, error) { return k.ConditionsVerb(args) }},
{Name: Seat + ".silence", Description: "Stop one condition's messages for a while, a hand act with why, kept as its " +
"transition in history; it stays open.",
Input: obj(map[string]any{"key": str("the condition's key"), "for": str("30m, 4h, 2d; at most 7d"),
"why": str("why"), "cause": str("the cause in a word")}, "key", "for", "why"),
Run: func(args map[string]any) (any, error) {
d, err := duration(strArg(args, "for"))
if err != nil {
return nil, err
}
return k.Silence(strArg(args, "key"), d, stdio.CallerOf(args), strArg(args, "why"))
}},
{Name: Seat + ".compare", Description: "The keeper's open conditions against the controller's, each difference with " +
"its reason; what ends with the controller, what only the keeper raises and what is held listed apart.",
Run: func(map[string]any) (any, error) { return k.Compare() }},
{Name: Seat + ".take-over", Description: "Ask the operator's Approve to switch the readers of conditions to the " +
"keeper; refused while compare differs.",
Input: obj(map[string]any{"why": str("why now")}, "why"),
Run: func(args map[string]any) (any, error) {
return k.TakeOver(a, stdio.CallerOf(args), strArg(args, "why"))
}},
{Name: Seat + ".hand-back", Description: "Switch the readers of conditions back to the controller, until its lease " +
"is handed away.",
Input: obj(map[string]any{"why": str("why")}, "why"),
Run: func(args map[string]any) (any, error) { return k.HandBack(stdio.CallerOf(args), strArg(args, "why")) }},
{Name: Seat + ".resync", Description: "Read every finding again and raise and clear what differs.",
Run: func(map[string]any) (any, error) {
n, err := k.Resync()
return map[string]any{"settled": n}, err
}},
{Name: HistoryTool, Description: "The keeper's history: every transition it said, newest first, at most limit.",
Input: obj(map[string]any{"limit": str("how many (default 50)"), "key": str("only this condition")}),
Run: func(args map[string]any) (any, error) {
return k.History(strArg(args, "key"), intArg(args, "limit", 50))
}},
{Name: HeldTool, Description: "What the keeper holds while its module is delivered (ADR 0564 clause 13): each key " +
"with its module, node, version and since when, and the deliveries' bounds.",
Run: func(map[string]any) (any, error) {
bounds, err := k.Store.Keys(BoundsState)
return map[string]any{"held": k.Held(), "bounds": bounds}, err
}},
{Name: TakeoverTool, Description: "The take-over key: which source the router, the grants part and mesh-watcher hear.",
Run: func(map[string]any) (any, error) {
raw, _, found, err := k.Store.Get(TakeoverState, health.TakeoverKey)
if err != nil {
return nil, err
}
return health.DecodeTakeover(raw, found)
}},
{Name: InputsTool, Description: "What the keeper reads, as it stands: each observer's findings counted, an observer it " +
"cannot read and why, the healers' attempts, the controller's conditions and whether its lease is handed away.",
Run: func(map[string]any) (any, error) { return k.Inputs(), nil }},
}
}
// ConditionsVerb is `conditions`: the list, one whole, or the history.
func (k *Keeper) ConditionsVerb(args map[string]any) (any, error) {
key := strArg(args, "key")
if strArg(args, "history") == "true" {
days := intArg(args, "days", 7)
if days < 1 || days > 90 {
return nil, fmt.Errorf("days is from 1 to 90, not %d", days)
}
return k.HistorySince(key, k.Now().Add(-time.Duration(days)*24*time.Hour), days)
}
if key != "" {
raw, _, found, err := k.Store.Get(ConditionsState, key)
if err != nil {
return nil, err
}
if !found || string(raw) == "null" {
return nil, fmt.Errorf("no condition %s is open", key)
}
var c health.Condition
return c, json.Unmarshal(raw, &c)
}
scope, severity, machine := strArg(args, "scope"), strArg(args, "severity"), strArg(args, "machine")
if severity != "" && severity != string(health.Urgent) && severity != string(health.Warning) {
return nil, fmt.Errorf("a severity is urgent or warning, not %q", severity)
}
open, err := k.Open()
if err != nil {
return nil, err
}
list := []inBrief{}
counted := map[string]int{}
for _, c := range open {
if (scope != "" && c.Subject.Scope != scope) || (severity != "" && string(c.Severity) != severity) ||
(machine != "" && !concerns(c, machine)) {
continue
}
b := inBrief{Condition: c}
if len(c.Evidence) > briefEvidence {
b.EvidenceKept, b.Evidence, b.More = len(c.Evidence), c.Evidence[:briefEvidence:briefEvidence], health.ShowKeeper(c.Key)
}
list = append(list, b)
counted[c.Source+" "+c.Kind]++
}
return map[string]any{"conditions": list, "open": len(open), "counted": counted,
"note": "urgent first, then oldest first; a condition clears when observation says so, never by hand; each with " +
"its newest evidence — `conditions key=<key>` gives one whole"}, nil
}
// concerns says whether a condition is about a machine: it names it, or its key does.
func concerns(c health.Condition, machine string) bool {
return c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) ||
slices.Contains(strings.Split(c.Key, "."), machine)
}
// History is the newest transitions, at most limit, of one key or all.
func (k *Keeper) History(key string, limit int) ([]health.Event, error) {
all, err := k.HistorySince(key, time.Time{}, 0)
if err != nil {
return nil, err
}
events := all["history"].([]health.Event)
slices.Reverse(events)
if len(events) > limit {
events = events[:limit]
}
return events, nil
}
// HistorySince is every transition since a moment, oldest first, as the controller's verb answers it.
func (k *Keeper) HistorySince(key string, since time.Time, days int) (map[string]any, error) {
keys, err := k.Store.Keys(HistoryState)
if err != nil {
return nil, err
}
events := []health.Event{}
for _, hk := range keys {
raw, _, found, err := k.Store.Get(HistoryState, hk)
if err != nil || !found {
continue
}
var e health.Event
if json.Unmarshal(raw, &e) != nil || e.At.Before(since) || (key != "" && e.Key != key) {
continue
}
events = append(events, e)
}
return map[string]any{"history": events, "days": days}, nil
}
// Inputs is what the keeper reads, as it stands.
func (k *Keeper) Inputs() map[string]any {
k.mu.Lock()
defer k.mu.Unlock()
k.init()
findings := map[string]int{}
for o, fs := range k.findings {
findings[o] = len(fs)
}
return map[string]any{"findings": findings, "blind": k.blind, "attempts": len(k.attempts),
"controller-conditions": len(k.ctrl), "lease-handed-away": k.leaseOut, "follow": k.takeover.Follow,
"wants": len(k.wants), "haves": len(k.haves)}
}
func strArg(a map[string]any, name string) string {
switch v := a[name].(type) {
case string:
return strings.TrimSpace(v)
case bool:
return strconv.FormatBool(v)
case float64:
return strconv.FormatFloat(v, 'f', -1, 64)
}
return ""
}
func intArg(a map[string]any, name string, def int) int {
if n, err := strconv.Atoi(strArg(a, name)); err == nil {
return n
}
return def
}
// duration reads 30m, 4h or 2d.
func duration(s string) (time.Duration, error) {
if n, ok := strings.CutSuffix(s, "d"); ok {
days, err := strconv.Atoi(n)
if err != nil {
return 0, fmt.Errorf("%q is not a duration: 30m, 4h or 2d", s)
}
return time.Duration(days) * 24 * time.Hour, nil
}
d, err := time.ParseDuration(s)
if err != nil {
return 0, fmt.Errorf("%q is not a duration: 30m, 4h or 2d", s)
}
return d, nil
}
@@ -0,0 +1,335 @@
//go:build integration
package main
// The condition keeper whole against a real bus (novox/hq ADR 0564 clauses 2 and 3, issue 1952 C6): the built binary,
// played over its real stdin and stdout as the node's tool runtime plays it (tools/call; mesh/state.* answered from a
// real nats-server's key-value buckets, watches handed over as the runtime hands them, mesh/seat.publish onto its
// subjects). An observer's finding written to its bucket raises a condition the keeper keeps in its bucket and says as
// condition-raised with the controller's body; the seat's verb lists it; the finding deleted clears it, said as
// condition-cleared.
import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/testbus"
)
type wire struct {
ID json.RawMessage `json:"id,omitempty"`
Method string `json:"method,omitempty"`
Params json.RawMessage `json:"params,omitempty"`
Result json.RawMessage `json:"result,omitempty"`
Error *struct {
Message string `json:"message"`
} `json:"error,omitempty"`
}
// player is the node's tool runtime, as far as the keeper needs it, over a real bus.
type player struct {
ctx context.Context
nc *nats.Conn
js jetstream.JetStream
mu sync.Mutex
in io.Writer
next int
pending map[string]chan wire
}
func (p *player) send(m map[string]any) {
raw, _ := json.Marshal(m)
p.mu.Lock()
defer p.mu.Unlock()
_, _ = p.in.Write(append(raw, '\n'))
}
func (p *player) answer(id json.RawMessage, result any, err error) {
if err != nil {
p.send(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}})
return
}
p.send(map[string]any{"id": id, "result": result})
}
func (p *player) serve(out io.Reader) {
scanner := bufio.NewScanner(out)
scanner.Buffer(make([]byte, 1<<20), 1<<24)
for scanner.Scan() {
var m wire
if json.Unmarshal(scanner.Bytes(), &m) != nil {
continue
}
if m.Method == "" {
var key string
_ = json.Unmarshal(m.ID, &key)
p.mu.Lock()
ch := p.pending[key]
delete(p.pending, key)
p.mu.Unlock()
if ch != nil {
ch <- m
}
continue
}
go p.handle(m)
}
}
// kv is a state's bucket: the keeper's own by its local name, another's by `<module>.<name>`.
func (p *player) kv(state string) (jetstream.KeyValue, error) {
if owner, name, read := strings.Cut(state, "."); read {
return p.js.KeyValue(p.ctx, owner+"_"+name)
}
return p.js.KeyValue(p.ctx, Module+"_"+state)
}
func (p *player) handle(m wire) {
var a struct {
State string `json:"state"`
Key string `json:"key"`
Value json.RawMessage `json:"value"`
Revision uint64 `json:"revision"`
Subject string `json:"subject"`
Body json.RawMessage `json:"body"`
}
_ = json.Unmarshal(m.Params, &a)
kv, kvErr := p.kv(a.State)
switch m.Method {
case "mesh/state.get":
if kvErr != nil {
p.answer(m.ID, nil, kvErr)
return
}
e, err := kv.Get(p.ctx, a.Key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
p.answer(m.ID, nil, nil)
return
}
if err != nil {
p.answer(m.ID, nil, err)
return
}
p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil)
case "mesh/state.create", "mesh/state.update", "mesh/state.put":
if kvErr != nil {
p.answer(m.ID, nil, kvErr)
return
}
var rev uint64
var err error
switch m.Method {
case "mesh/state.create":
rev, err = kv.Create(p.ctx, a.Key, a.Value)
case "mesh/state.update":
rev, err = kv.Update(p.ctx, a.Key, a.Value, a.Revision)
default:
rev, err = kv.Put(p.ctx, a.Key, a.Value)
}
if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
p.send(map[string]any{"id": m.ID, "error": map[string]any{"code": -32010, "message": "changed: " + err.Error()}})
return
}
p.answer(m.ID, map[string]any{"revision": rev}, err)
case "mesh/state.keys":
if kvErr != nil {
p.answer(m.ID, nil, kvErr)
return
}
keys, err := kv.Keys(p.ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
keys, err = []string{}, nil
}
p.answer(m.ID, keys, err)
case "mesh/state.delete":
err := kvErr
if err == nil {
err = kv.Delete(p.ctx, a.Key)
}
p.answer(m.ID, map[string]any{}, err)
case "mesh/state.watch":
if kvErr != nil {
p.answer(m.ID, nil, kvErr)
return
}
w, err := kv.WatchAll(p.ctx)
if err != nil {
p.answer(m.ID, nil, err)
return
}
var watchID string
_ = json.Unmarshal(m.ID, &watchID)
current := true
for e := range w.Updates() {
if e == nil {
current = false
p.answer(m.ID, map[string]any{}, nil)
continue
}
op := "put"
if e.Operation() != jetstream.KeyValuePut {
op = "delete"
}
p.mu.Lock()
p.next++
id := fmt.Sprintf("w-%d", p.next)
p.mu.Unlock()
p.send(map[string]any{"id": id, "method": "mesh/state", "params": map[string]any{"watch": watchID,
"change": map[string]any{"state": a.State, "key": e.Key(), "op": op, "value": json.RawMessage(nullIfEmpty(e.Value())),
"revision": e.Revision(), "current": current}}})
}
case "mesh/seat.publish":
p.answer(m.ID, map[string]any{"sequence": 1}, p.nc.Publish(a.Subject, a.Body))
case "mesh/subscribe":
p.answer(m.ID, map[string]any{}, nil)
default:
p.answer(m.ID, nil, errors.New("nats: no responders available for request"))
}
}
func nullIfEmpty(b []byte) []byte {
if len(b) == 0 {
return []byte("null")
}
return b
}
func (p *player) call(name string, args map[string]any) (json.RawMessage, error) {
p.mu.Lock()
p.next++
id := fmt.Sprintf("c-%d", p.next)
ch := make(chan wire, 1)
p.pending[id] = ch
p.mu.Unlock()
p.send(map[string]any{"id": id, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args,
"_meta": map[string]any{"mesh/caller": "person.jochen"}}})
select {
case m := <-ch:
if m.Error != nil {
return nil, errors.New(m.Error.Message)
}
var r struct {
Content []struct {
Text string `json:"text"`
} `json:"content"`
IsError bool `json:"isError"`
}
if err := json.Unmarshal(m.Result, &r); err != nil || len(r.Content) != 1 {
return nil, fmt.Errorf("%s answered %s", name, m.Result)
}
if r.IsError {
return nil, errors.New(r.Content[0].Text)
}
return json.RawMessage(r.Content[0].Text), nil
case <-time.After(60 * time.Second):
return nil, fmt.Errorf("%s did not answer in 60s", name)
}
}
// 1952/C6: an observer's finding raises a condition on the bus, said with the controller's body, and its deletion clears
// it.
func TestTheKeeperWholeRaisesAndClearsOnTheBus(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
bin := filepath.Join(t.TempDir(), Module)
build := exec.Command("go", "build", "-o", bin, ".")
build.Stderr = os.Stderr
if err := build.Run(); err != nil {
t.Fatalf("the keeper does not build: %v", err)
}
nc, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
buckets := []string{"mesh-controller_conditions", "mesh-controller_lease", "mesh-delivery-assigner_wants", "node-engine_have", "mesh-delivery-assigner_placement",
"mesh-healers_attempts"}
for _, o := range Observers {
buckets = append(buckets, o+"_findings")
}
for _, s := range []string{ConditionsState, HistoryState, TakeoverState, HeldState, BoundsState, AskedState} {
buckets = append(buckets, Module+"_"+s)
}
for _, b := range buckets {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil {
t.Fatal(err)
}
}
events, err := nc.SubscribeSync("mesh.seat." + Seat + ".event.>")
if err != nil {
t.Fatal(err)
}
cmd := exec.CommandContext(ctx, bin)
cmd.Env = append(os.Environ(), "MESH_NODE=ace", "MESH_MODULE="+Module)
stdin, _ := cmd.StdinPipe()
stdout, _ := cmd.StdoutPipe()
cmd.Stderr = os.Stderr
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() })
p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}}
go p.serve(stdout)
// The first call serves; the watches start after it.
if _, err := p.call(InputsTool, map[string]any{}); err != nil {
t.Fatal(err)
}
time.Sleep(time.Second)
at := time.Now().UTC()
f := health.Finding{Key: "machine.g14.silent", Kind: "silent", Subject: health.Subject{Scope: health.ScopeMachine, ID: "g14",
Machine: "g14"}, Severity: health.Warning, Summary: "g14 is silent", Headline: "g14 is offline",
Explanation: "g14 said nothing for three minutes.", Resolved: "g14 is back", Source: "S1",
Observer: health.WatchmanModule, At: at, Evidence: []health.Evidence{{At: at, Said: "no presence for 3m"}}}
raw, err := health.EncodeFinding(f)
if err != nil {
t.Fatal(err)
}
findings, _ := js.KeyValue(ctx, health.WatchmanModule+"_findings")
if _, err := findings.Put(ctx, f.Key, raw); err != nil {
t.Fatal(err)
}
msg, err := events.NextMsg(10 * time.Second)
if err != nil {
t.Fatalf("no condition-raised was said: %v", err)
}
var e health.Event
if err := json.Unmarshal(msg.Data, &e); err != nil || msg.Subject != "mesh.seat.mesh-conditions.event.condition-raised" ||
e.Key != f.Key || e.Change != health.ChangeRaised || e.Show != health.ShowKeeper(f.Key) || e.Headline != f.Headline {
t.Fatalf("said %s on %s", msg.Data, msg.Subject)
}
listed, err := p.call(Seat+".conditions", map[string]any{})
if err != nil || !strings.Contains(string(listed), `"key":"machine.g14.silent"`) {
t.Fatalf("conditions answered %s, %v", listed, err)
}
if err := findings.Delete(ctx, f.Key); err != nil {
t.Fatal(err)
}
msg, err = events.NextMsg(10 * time.Second)
if err != nil || msg.Subject != "mesh.seat.mesh-conditions.event.condition-cleared" {
t.Fatalf("no condition-cleared was said: %v", err)
}
conditions, _ := js.KeyValue(ctx, Module+"_"+ConditionsState)
if _, err := conditions.Get(ctx, f.Key); !errors.Is(err, jetstream.ErrKeyNotFound) {
t.Errorf("the cleared condition is still kept: %v", err)
}
}
+22
View File
@@ -0,0 +1,22 @@
module mesh-conditions
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0
github.com/nats-io/nats.go v1.54.0
)
require (
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
github.com/google/go-tpm v0.9.8 // indirect
github.com/klauspost/compress v1.20.0 // indirect
github.com/minio/highwayhash v1.0.4 // indirect
github.com/nats-io/jwt/v2 v2.8.1 // indirect
github.com/nats-io/nats-server/v2 v2.11.17 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/time v0.15.0 // indirect
)
+27
View File
@@ -0,0 +1,27 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=

Some files were not shown because too many files have changed in this diff Show More