Compare commits

..
576 changed files with 2598 additions and 31939 deletions
-5
View File
@@ -1,5 +0,0 @@
module checks/credentialform
go 1.26.0
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915
-2
View File
@@ -1,2 +0,0 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
-98
View File
@@ -1,98 +0,0 @@
// Package credentialform refuses what the catalogue's tree holds about credentials and routes that no manifest judge
// sees: a module that provides and keeps a `bound` state the provisioner does not read, and a route received as a file.
// It judges no offer's credential form: that rule (an own secret, an issuer, or `form` pair or none; the lead,
// 2026-10-29) is part of the catalogue's rules, which the checker's judge runs on every change (CatalogueProblems,
// novox/hq ADR 0578 §2); this package called it itself until 2026-11.
//
// It refuses a route received as a file too (issue 774, research 072 row 34, the operator on 2026-10-31: "there is only
// 1 path, the other is simply deprecated/invalid"): a route is a provision, served from the seat mesh-provisioner's
// plans alone, and a manifest's `receives.route` is the deprecated path. A module still moving off it is listed in
// stillReceivingRoutes with why; every refusal names them all, and a listed module that no longer receives one is
// refused too, so the list only shrinks.
package credentialform
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// Problems is every module of the manifests under root/modules that keeps a `bound` state the provisioner does not read,
// and every route received as a file, one line each.
func Problems(root string) ([]string, error) {
files, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
if err != nil {
return nil, err
}
sort.Strings(files)
var out []string
var receiving []string
reads := map[string]bool{}
var binders []string
for _, f := range files {
raw, err := os.ReadFile(f)
if err != nil {
return nil, err
}
m, err := catalogue.Decode(raw)
if err != nil {
return nil, fmt.Errorf("%s: %w", f, err)
}
if _, received := m.Receives["route"]; received {
receiving = append(receiving, m.Module)
} else if _, listed := stillReceivingRoutes[m.Module]; listed {
out = append(out, fmt.Sprintf("%s no longer receives route; take it off stillReceivingRoutes", m.Module))
}
var kept struct {
Reads []string `json:"reads"`
State []json.RawMessage `json:"state"`
}
_ = json.Unmarshal(raw, &kept) // reads may hold objects in another module; only the provisioner's are read
if m.Module == "mesh-provisioner" {
for _, r := range kept.Reads {
reads[r] = true
}
}
if len(m.Provides) > 0 && keepsBound(kept.State) {
binders = append(binders, m.Module)
}
}
for _, module := range binders {
if !reads[module+".bound"] {
out = append(out, fmt.Sprintf("%s provides and keeps a `bound` state, and mesh-provisioner's manifest does not read %s.bound", module, module))
}
}
for _, module := range receiving {
if _, listed := stillReceivingRoutes[module]; !listed {
out = append(out, fmt.Sprintf("%s receives route as a file: a route is a provision, served from the seat "+
"mesh-provisioner's plans alone (issue 774); still on the deprecated path: %s", module,
strings.Join(receiving, ", ")))
}
}
return out, nil
}
// stillReceivingRoutes is every module still receiving its routes as a file, and why it has not moved yet.
var stillReceivingRoutes = map[string]string{
"route-adapter": "migration scaffolding writing into a predecessor's proxy (novox/hq ADR 0104), deleted when that " +
"migration ends; it reads the received file its TypeScript adapter turns into the predecessor's route files",
}
// keepsBound says whether a manifest's `state` names `bound`, as a plain name or an object with a name.
func keepsBound(state []json.RawMessage) bool {
for _, s := range state {
var name string
var obj struct {
Name string `json:"name"`
}
if json.Unmarshal(s, &name) == nil && name == "bound" || json.Unmarshal(s, &obj) == nil && obj.Name == "bound" {
return true
}
}
return false
}
-54
View File
@@ -1,54 +0,0 @@
package credentialform
import (
"os"
"path/filepath"
"strings"
"testing"
)
func aCatalogue(t *testing.T, manifests map[string]string) string {
t.Helper()
root := t.TempDir()
for module, body := range manifests {
dir := filepath.Join(root, "modules", module)
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(body), 0o644); err != nil {
t.Fatal(err)
}
}
return root
}
// A binder the provisioner does not read is refused and named; an offer that says no form is the judge's to refuse, not
// this check's; a route received as a file is refused but for the module still moving off it; and the real catalogue is
// clean.
func TestNoBinderIsUnreadAndNoRouteIsReceivedAsAFile(t *testing.T) {
root := aCatalogue(t, map[string]string{
"silent": `{"module":"silent","provides":[{"name":"thing","scope":"mesh"}]}`,
"binder": `{"module":"binder","provides":[{"name":"d","credential":{"form":"pair"}}],"state":["bound"]}`,
"mesh-provisioner": `{"module":"mesh-provisioner","reads":["other.bound"]}`,
"says": `{"module":"says","provides":[{"name":"a","credential":{"form":"pair"}},{"name":"b","credential":{"form":"none"}},{"name":"c","credential":{"own":"jwt"}}]}`,
// A route received as a file (issue 774, research 072 row 34): refused, naming every module still on it; the one
// listed as still moving passes.
"old-proxy": `{"module":"old-proxy","receives":{"route":"/var/lib/old-proxy/mesh.json"}}`,
"route-adapter": `{"module":"route-adapter","receives":{"route":"/var/lib/route-adapter/mesh.json"}}`,
})
got, err := Problems(root)
if err != nil {
t.Fatal(err)
}
if len(got) != 2 || !strings.Contains(got[0], "binder") || !strings.Contains(got[0], "mesh-provisioner") ||
!strings.HasPrefix(got[1], "old-proxy receives route") || !strings.Contains(got[1], "old-proxy, route-adapter") {
t.Fatalf("refused %v, wanted the binder the provisioner does not read and the received route", got)
}
real, err := Problems(filepath.Join("..", ".."))
if err != nil {
t.Fatal(err)
}
if len(real) != 0 {
t.Fatalf("the catalogue has binders or routes it must not:\n%s", strings.Join(real, "\n"))
}
}
-3
View File
@@ -1,3 +0,0 @@
# What these tests read outside their package, as paths from the repository's root (issue 1936).
# every manifest, for its bound state and its received routes.
modules/*/module.json
-5
View File
@@ -1,5 +0,0 @@
# The calls of the controller's hand-act log a manifest of this catalogue may still invoke, each `<module> <verb> <why>`:
# the fallback of a recorder (mesh-sdk handacts.Recorder) while the seat mesh-hand-acts has no holder that answers, until
# the controller's lease is written to a holder that is no controller. The check of this directory (handacts) refuses any
# other, and an entry whose module no longer invokes it: the list only shrinks, and it is empty before the lease write
# (novox/hq ADR 0565 §11; issue 1953, issue 774). It is empty: no manifest invokes the controller's log (issue 1953, 1955).
-3
View File
@@ -1,3 +0,0 @@
module git.novox.be/novox/mesh-catalog/checks/handacts
go 1.22
-109
View File
@@ -1,109 +0,0 @@
// Command handacts refuses a manifest that records into the controller's hand-act log (novox/hq ADR 0565 §11): every
// recorder records through the seat mesh-hand-acts from its delivery, so no manifest invokes
// `seat:mesh-controller.hand-act`, `.warranted` or `.drill`. The recorders' fallbacks to the controller's log while the
// seat has no holder are named in checks/handacts/fallbacks, each module and verb with why; that list only shrinks, and
// an entry whose module no longer invokes the verb is refused too.
//
// go run . <catalog root>
//
// Exits 1 on a finding.
package main
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
)
// Verbs are the controller's hand-act verbs, as a manifest invokes them, and Allowed the list of named fallbacks.
var Verbs = []string{"hand-act", "warranted", "drill"}
const (
invokePrefix = "seat:mesh-controller."
Allowed = "checks/handacts/fallbacks"
)
// Findings is every refusal in the catalog at root, sorted.
func Findings(root string) ([]string, error) {
allowed := map[string]bool{}
raw, err := os.ReadFile(filepath.Join(root, Allowed))
if err != nil && !os.IsNotExist(err) {
return nil, err
}
for _, line := range strings.Split(string(raw), "\n") {
if f := strings.Fields(line); len(f) >= 3 && !strings.HasPrefix(f[0], "#") {
allowed[f[0]+" "+f[1]] = true
} else if len(f) > 0 && !strings.HasPrefix(f[0], "#") {
return nil, fmt.Errorf("%s: %q is not `<module> <verb> <why>`", Allowed, line)
}
}
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
if err != nil {
return nil, err
}
var out []string
invoked := map[string]bool{}
for _, path := range manifests {
raw, err := os.ReadFile(path)
if err != nil {
return nil, err
}
var m struct {
Module string `json:"module"`
Invokes []string `json:"invokes"`
}
if json.Unmarshal(raw, &m) != nil || m.Module == "mesh-controller" {
continue
}
for _, inv := range m.Invokes {
verb, ok := strings.CutPrefix(strings.TrimSpace(inv), invokePrefix)
if !ok || !contains(Verbs, verb) {
continue
}
invoked[m.Module+" "+verb] = true
if !allowed[m.Module+" "+verb] {
out = append(out, fmt.Sprintf("%s invokes %s%s: record through seat:mesh-hand-acts.%s (novox/hq ADR 0565 §11), "+
"or name the fallback in %s", m.Module, invokePrefix, verb, verb, Allowed))
}
}
}
for entry := range allowed {
if !invoked[entry] {
out = append(out, fmt.Sprintf("%s names %q, which is no longer invoked: take it off the list, which only shrinks",
Allowed, entry))
}
}
sort.Strings(out)
return out, nil
}
func contains(list []string, s string) bool {
for _, x := range list {
if x == s {
return true
}
}
return false
}
func main() {
if len(os.Args) != 2 {
fmt.Fprintln(os.Stderr, "usage: handacts <catalog root>")
os.Exit(2)
}
found, err := Findings(os.Args[1])
if err != nil {
fmt.Println(err)
os.Exit(1)
}
for _, f := range found {
fmt.Println(f)
}
if len(found) > 0 {
os.Exit(1)
}
fmt.Println("no manifest records into the controller's hand-act log but the fallbacks named")
}
-59
View File
@@ -1,59 +0,0 @@
package main
import (
"os"
"path/filepath"
"strings"
"testing"
)
// No manifest invokes the controller's hand-act verbs but a named fallback; a named fallback no longer invoked is
// refused (novox/hq ADR 0565 §11's catalog test).
func TestFindings(t *testing.T) {
for _, row := range []struct {
name string
files map[string]string
want string // empty: clean
}{
{"clean", map[string]string{"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-hand-acts.hand-act"]}`}, ""},
{"an unnamed call", map[string]string{"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-controller.warranted"]}`},
"a invokes seat:mesh-controller.warranted"},
{"a named fallback, and the controller's own", map[string]string{
"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-controller.drill"]}`,
"modules/mesh-controller/module.json": `{"module":"mesh-controller","invokes":["seat:mesh-controller.hand-act"]}`,
Allowed: "# why\na drill fallback until the lease write\n"}, ""},
{"a fallback no longer invoked", map[string]string{Allowed: "gone hand-act fallback\n"}, `names "gone hand-act"`},
{"another verb of the controller", map[string]string{"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-controller.conditions"]}`}, ""},
} {
t.Run(row.name, func(t *testing.T) {
root := t.TempDir()
for path, text := range row.files {
p := filepath.Join(root, path)
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(p, []byte(text), 0o644); err != nil {
t.Fatal(err)
}
}
got, err := Findings(root)
if err != nil {
t.Fatal(err)
}
if all := strings.Join(got, "\n"); (row.want == "") != (all == "") || !strings.Contains(all, row.want) {
t.Fatalf("findings = %q, want one containing %q", all, row.want)
}
})
}
}
// The run of this check over the real tree.
func TestTheCatalogRecordsOnlyThroughTheSeat(t *testing.T) {
got, err := Findings(filepath.Join("..", ".."))
if err != nil {
t.Fatal(err)
}
for _, f := range got {
t.Error(f)
}
}
-3
View File
@@ -1,3 +0,0 @@
# What these tests read outside their package, as paths from the repository's root (issue 1936).
# every manifest, for what it invokes.
modules/*/module.json
+1 -1
View File
@@ -2,4 +2,4 @@ module checks/health
go 1.26.0
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468
+2 -2
View File
@@ -1,2 +1,2 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+3 -8
View File
@@ -1,8 +1,7 @@
// Package health is the catalogue's count of long-running resources that do not say how they are ready (novox/hq ADR 0240 rule
// 8), kept in health-undeclared and judged by a test over the real tree (ADR 0529 clauses 2 and 4): it replaces the step of
// merge-check.sh that read it out of the controller's `module check`. Every manifest is decoded with mesh-sdk's catalogue
// package, the one the controller counts with, and judged by nothing here (novox/hq ADR 0578 §3): a resource's `health` is
// in the manifest itself, not in a template's text.
// merge-check.sh that read it out of the controller's `module check`. Every manifest is read through mesh-sdk's catalogue
// package, the one the controller counts with.
package health
import (
@@ -28,11 +27,7 @@ func Count(root string) (int, error) {
}
n := 0
for _, path := range manifests {
raw, err := os.ReadFile(path)
if err != nil {
return 0, err
}
m, err := catalogue.Decode(raw)
m, err := catalogue.ReadTreeManifest(root, filepath.Dir(path))
if err != nil {
return 0, fmt.Errorf("%s: %w", path, err)
}
+5 -1
View File
@@ -1,6 +1,10 @@
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
# every manifest, decoded; and the root's kept count of resources that do not say how they are ready.
# every manifest through mesh-sdk's ReadTreeManifest: the root's predefined keys and each templated module's
# templates (files/ by default, or the directory its manifest names); and the root's kept count of resources that do not say how they are ready.
modules/*/module.json
predefined.json
modules/*/files
modules/*/templates
health-undeclared
-5
View File
@@ -1,5 +0,0 @@
module checks/manifests
go 1.26.0
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915
-2
View File
@@ -1,2 +0,0 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
-48
View File
@@ -1,48 +0,0 @@
// Package manifests reads every module's manifest as the catalogue's tree holds it and refuses what no judge could
// read: a module.json that does not decode, and two manifests that name one module.
//
// It judges no manifest (novox/hq ADR 0578 §2): the checker judges every change's manifests with the indexer's judge, the
// one judge, so the field, seat, bus and catalogue rules are said once, there. Until 2026-11 this check parsed each
// manifest with mesh-sdk's strict parser and ran the rules between manifests (issue 774, power's bar item of
// mesh-catalog ca8d6f33); that judgement moved to the checker, and what stays is the two refusals that are about the
// tree, not a manifest's rules.
package manifests
import (
"fmt"
"os"
"path/filepath"
"sort"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// Problems is what is wrong with the tree's manifests under root/modules, one line each: a module.json whose bytes are
// no manifest (catalogue.Decode: not JSON, or a field of the wrong shape), then a module named by two manifests.
func Problems(root string) ([]string, error) {
files, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
if err != nil {
return nil, err
}
sort.Strings(files)
var out []string
named := map[string]bool{}
for _, f := range files {
raw, err := os.ReadFile(f)
if err != nil {
return nil, err
}
rel, _ := filepath.Rel(root, f)
m, err := catalogue.Decode(raw)
if err != nil {
out = append(out, fmt.Sprintf("%s: %v", rel, err))
continue
}
if named[m.Module] {
out = append(out, fmt.Sprintf("%s: %s was already given; two manifests name one module", rel, m.Module))
continue
}
named[m.Module] = true
}
return out, nil
}
-50
View File
@@ -1,50 +0,0 @@
package manifests
import (
"os"
"path/filepath"
"strings"
"testing"
)
// The tree's manifests are ones a judge can read, and no module is named twice. A row is a catalogue of its own: a
// manifest that is no JSON and two manifests of one module are refused, naming the file; a manifest with a field this
// check does not know is not (it judges nothing: the checker does, ADR 0578 §2). The catalogue itself passes.
func TestEveryManifestDecodesAndNamesOneModule(t *testing.T) {
for _, row := range []struct {
name string
files map[string]string
refused string
}{
{name: "a manifest the judge can read", files: map[string]string{"a": `{"module":"a","version":"1"}`}},
{name: "a field no part knows is the judge's to refuse", files: map[string]string{"a": `{"module":"a","version":"1","unheard":true}`}},
{name: "bytes that are no manifest", files: map[string]string{"a": `{"module":`}, refused: "modules/a/module.json"},
{name: "two manifests of one module", files: map[string]string{"a": `{"module":"x","version":"1"}`, "b": `{"module":"x","version":"1"}`},
refused: "two manifests name one module"},
} {
root := t.TempDir()
for dir, manifest := range row.files {
if err := os.MkdirAll(filepath.Join(root, "modules", dir), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(root, "modules", dir, "module.json"), []byte(manifest), 0o644); err != nil {
t.Fatal(err)
}
}
got, err := Problems(root)
if err != nil {
t.Fatal(err)
}
joined := strings.Join(got, "\n")
if (row.refused == "") != (len(got) == 0) || !strings.Contains(joined, row.refused) {
t.Errorf("%s: refused %q, want %q", row.name, joined, row.refused)
}
}
real, err := Problems(filepath.Join("..", ".."))
if err != nil {
t.Fatal(err)
}
if len(real) != 0 {
t.Fatalf("the catalogue holds manifests no judge can read:\n%s", strings.Join(real, "\n"))
}
}
-3
View File
@@ -1,3 +0,0 @@
# What these tests read outside their package, as paths from the repository's root (issue 1936).
# every module's directory (its manifest), decoded.
modules
-4
View File
@@ -1,4 +0,0 @@
# The modules of this catalogue that still read the controller's assignments (mesh-controller.assignments), each with
# why. The check of this directory (noplacement) refuses any other module that reads them, and an entry here whose module no longer
# does: the list only shrinks, and it is empty before the controller's lease is written to a holder that is no
# controller (novox/hq ADR 0549 §3, §6; issue 774).
-4
View File
@@ -1,7 +1,3 @@
module git.novox.be/novox/mesh-catalog/checks/noplacement
go 1.22
require git.novox.be/novox/mesh-catalog/checks/trunk v0.0.0
replace git.novox.be/novox/mesh-catalog/checks/trunk => ../trunk
+2 -73
View File
@@ -8,13 +8,7 @@
// 2. a `particulars` file at the root (the mesh's node names come from the checker's run, §16);
//
// 3. a manifest naming a state, stream, event or bucket of the indexer's placement (`mesh-delivery-indexer.placement…`
// or `mesh-delivery-indexer_placement…`) in any string value, such as `reads`;
//
// 4. a manifest, other than mesh-controller's, that reads the controller's assignments (`mesh-controller.assignments`):
// every reader takes placement from the seat mesh-placement (novox/hq ADR 0549 §3, ADR 0567), and the lease write is
// refused while one still reads them (ADR 0549 §6; issue 774). The modules that still read them for what the
// placement does not carry are named in checks/noplacement/assignments-readers, each with why; that list only
// shrinks: an entry whose module no longer reads them is refused too.
// or `mesh-delivery-indexer_placement…`) in any string value, such as `reads`.
//
// go run . <catalogue root>
//
@@ -61,74 +55,9 @@ func Findings(root string) ([]string, error) {
out = append(out, fmt.Sprintf("%s: names %q, which the indexer no longer keeps; placement is the seat mesh-placement's (novox/hq ADR 0567 §12)", module, s))
}
}
readers, err := assignmentsReaders(root)
if err != nil {
return nil, err
}
out = append(out, readers...)
sort.Strings(out)
return out, nil
}
// AssignmentsRead is the controller's assignments as a manifest reads them, and Allowed the file, relative to the
// catalogue's root, that names the modules still reading them: `<module> <why>` per line, `#` a comment.
const (
AssignmentsRead = "mesh-controller.assignments"
Allowed = "checks/noplacement/assignments-readers"
)
// assignmentsReaders is every finding on the reads of the controller's assignments: a module other than the controller
// reading them that Allowed does not name, and a module Allowed names that no longer reads them (issue 774).
func assignmentsReaders(root string) ([]string, error) {
allowed := map[string]bool{}
raw, err := os.ReadFile(filepath.Join(root, Allowed))
if err != nil && !os.IsNotExist(err) {
return nil, err
}
for _, line := range strings.Split(string(raw), "\n") {
if f := strings.Fields(line); len(f) > 0 && !strings.HasPrefix(f[0], "#") {
allowed[f[0]] = true
}
}
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
if err != nil {
return nil, err
}
var found []string
reads := map[string]bool{}
for _, path := range manifests {
raw, err := os.ReadFile(path)
if err != nil {
return nil, err
}
var m struct {
Module string `json:"module"`
Reads []string `json:"reads"`
}
if json.Unmarshal(raw, &m) != nil || m.Module == "mesh-controller" {
continue // a manifest that does not read is the manifest checks' to refuse
}
for _, r := range m.Reads {
if strings.TrimSpace(r) != AssignmentsRead {
continue
}
reads[m.Module] = true
if !allowed[m.Module] {
found = append(found, fmt.Sprintf("%s reads %s: read where modules run from mesh-delivery-assigner.placement, "+
"the seat mesh-placement's (novox/hq ADR 0549 §3, ADR 0567); the lease write is refused while a module reads "+
"the assignments (ADR 0549 §6)", m.Module, AssignmentsRead))
}
}
}
for module := range allowed {
if !reads[module] {
found = append(found, fmt.Sprintf("%s names %s, which no longer reads %s: take it off the list, which only shrinks",
Allowed, module, AssignmentsRead))
}
}
return found, nil
}
// named collects the strings of a decoded manifest that start with one of the Prefixes.
func named(v any, into []string) []string {
switch v := v.(type) {
@@ -166,5 +95,5 @@ func main() {
if len(found) > 0 {
os.Exit(1)
}
fmt.Println("no placement directory, no manifest reads the indexer's placement, and only the modules listed read the controller's assignments")
fmt.Println("no placement directory, and no manifest reads the indexer's placement")
}
+1 -34
View File
@@ -5,8 +5,6 @@ import (
"path/filepath"
"strings"
"testing"
"git.novox.be/novox/mesh-catalog/checks/trunk"
)
func TestFindingsRefuseWhatThePlacementPathLeft(t *testing.T) {
@@ -15,18 +13,12 @@ func TestFindingsRefuseWhatThePlacementPathLeft(t *testing.T) {
files map[string]string
want string // empty: clean
}{
{"clean", map[string]string{"modules/a/module.json": `{"module":"a","reads":["mesh-delivery-assigner.placement"]}`}, ""},
{"clean", map[string]string{"modules/a/module.json": `{"module":"a","reads":["mesh-controller.assignments"]}`}, ""},
{"a placement directory", map[string]string{"placement/ace.json": `{}`}, "placement: the repository holds no placement"},
{"a particulars file", map[string]string{"particulars": "ace\n"}, "particulars: the repository lists no node names"},
{"a read of the indexer's placement", map[string]string{"modules/a/module.json": `{"module":"a","reads":["mesh-delivery-indexer.placement"]}`}, `a: names "mesh-delivery-indexer.placement"`},
{"a read of its candidates", map[string]string{"modules/b/module.json": `{"module":"b","reads":["mesh-delivery-indexer.placement-candidates"]}`}, "placement-candidates"},
{"its event", map[string]string{"modules/c/module.json": `{"module":"c","on":[{"event":"mesh-delivery-indexer.placement-changed"}]}`}, "placement-changed"},
// Issue 774 (ADR 0549 §3, §6): a reader of the controller's assignments is refused unless the list names it, the
// controller's own manifest is not judged, and a name on the list whose module no longer reads them is refused.
{"an unlisted reader of the assignments", map[string]string{"modules/r/module.json": `{"module":"r","reads":["mesh-controller.assignments"]}`}, "r reads mesh-controller.assignments"},
{"a listed reader and the controller", map[string]string{"modules/l/module.json": `{"module":"l","reads":["mesh-controller.assignments"]}`,
"modules/mesh-controller/module.json": `{"module":"mesh-controller","reads":["mesh-controller.assignments"]}`, Allowed: "# why\nl its plan\n"}, ""},
{"a listed module that reads them no more", map[string]string{Allowed: "gone no longer reads\n"}, "names gone, which no longer reads"},
{"the seat's own state is allowed", map[string]string{"modules/d/module.json": `{"module":"d","reads":["mesh-placement.placement"]}`}, ""},
} {
t.Run(row.name, func(t *testing.T) {
@@ -60,29 +52,4 @@ func TestTheCatalogueHoldsNoPlacementOfTheRepositoryPath(t *testing.T) {
for _, f := range got {
t.Error(f)
}
// Issue 774 (ADR 0549 §3, §6; the comparison of novox/hq ADR 0529 clause 2): the list of modules still reading the
// controller's assignments gains none the trunk's copy at the merge base does not hold. Its first copy gains nothing.
path, ok := trunk.Copy(t, Allowed)
if !ok {
return
}
names := func(path string) map[string]bool {
raw, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
out := map[string]bool{}
for _, line := range strings.Split(string(raw), "\n") {
if f := strings.Fields(line); len(f) > 0 && !strings.HasPrefix(f[0], "#") {
out[f[0]] = true
}
}
return out
}
base := names(path)
for module := range names(filepath.Join("..", "..", Allowed)) {
if !base[module] {
t.Errorf("%s gains %s, which the trunk's copy does not hold: the list only shrinks", Allowed, module)
}
}
}
-3
View File
@@ -1,3 +0,0 @@
module checks/planhands
go 1.26.0
-48
View File
@@ -1,48 +0,0 @@
// Package planhands refuses a module's line that takes a plan existing for its module handed to the provisioner (issue
// 774). Whether a plan hands its module is mesh-sdk's one rule, catalogue.Plan.Hands: since the seat mesh-provisioner keys a
// provision its provider's helper answers for a module not handed, a plan naming provisions alone exists for modules
// whose files and own secrets stay their node's (on 2026-10-30 the vault read every such plan as handed and answered
// twenty modules' own secrets to nobody, and the provisioner client refused each as a plan it could not act on). A line
// that tests whether a plan exists either asks Hands on the same line, or says "not handing": it reads only who binds
// whom, or that there is no plan at all.
package planhands
import (
"fmt"
"io/fs"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
)
// existence is a test of whether an assignment's or a plan's `provisioning` is there: nil, or an empty raw value.
var existence = regexp.MustCompile(`Provisioning\s*[!=]=\s*nil|len\([^)]*Provisioning\)\s*[!=]=\s*0`)
// Problems is each such line under root's modules/ that neither asks Hands nor says "not handing", as
// `path:line: text`, sorted. Test files are left out: a test may build a plan by hand.
func Problems(root string) ([]string, error) {
var out []string
err := filepath.WalkDir(filepath.Join(root, "modules"), func(p string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() || !strings.HasSuffix(p, ".go") || strings.HasSuffix(p, "_test.go") {
return nil
}
raw, err := os.ReadFile(p)
if err != nil {
return err
}
rel, _ := filepath.Rel(root, p)
for i, line := range strings.Split(string(raw), "\n") {
if existence.MatchString(line) && !strings.Contains(line, ".Hands()") && !strings.Contains(line, "not handing") {
out = append(out, fmt.Sprintf("%s:%d: %s", rel, i+1, strings.TrimSpace(line)))
}
}
return nil
})
sort.Strings(out)
return out, err
}
-45
View File
@@ -1,45 +0,0 @@
package planhands
import (
"os"
"path/filepath"
"strings"
"testing"
)
// A line that takes a plan existing for its module handed is refused and named; one that asks Hands, one marked "not
// handing" and any test file pass; and the real catalogue is clean.
func TestNoModuleTakesAPlanExistingForItsModuleHanded(t *testing.T) {
root := t.TempDir()
for f, body := range map[string]string{
"modules/vault/own.go": "if a.Provisioning != nil {\n",
"modules/client/plan.go": "if len(a.Provisioning) == 0 {\n",
"modules/vault/rule.go": "return a.Provisioning != nil && a.Provisioning.Hands(), nil\n",
"modules/client/binds.go": "if a.Provisioning != nil { // binds, not handing\n",
"modules/vault/own_test.go": "if a.Provisioning == nil {\n",
"modules/client/notgo/plan.md": "if a.Provisioning == nil {\n",
} {
p := filepath.Join(root, f)
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
t.Fatal(err)
}
}
got, err := Problems(root)
if err != nil {
t.Fatal(err)
}
if len(got) != 2 || !strings.HasPrefix(got[0], "modules/client/plan.go:1:") || !strings.HasPrefix(got[1], "modules/vault/own.go:1:") {
t.Fatalf("refused %v, wanted the client's and the vault's existence tests alone", got)
}
real, err := Problems(filepath.Join("..", ".."))
if err != nil {
t.Fatal(err)
}
if len(real) > 0 {
t.Errorf("a module takes a plan existing for its module handed; ask catalogue.Plan.Hands, or mark a line that reads "+
"only who binds whom \"not handing\":\n%s", strings.Join(real, "\n"))
}
}
-3
View File
@@ -1,3 +0,0 @@
# What these tests read outside their package, as paths from the repository's root (issue 1936).
# every module whole (a directory is read whole, so a module's Go source at any depth enters the digest), for a line that takes a plan existing for its module handed.
modules
-5
View File
@@ -1,5 +0,0 @@
module checks/readable
go 1.26.0
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915
-2
View File
@@ -1,2 +0,0 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
-333
View File
@@ -1,333 +0,0 @@
// Command readable refuses a catalogue where a module reads another module's state that the owner does not announce
// readable (novox/hq ADR 0495 clause 3, issue 1790): the controller grants a read only from the owner's announcement, so
// such a read is refused on the live mesh, said only in the controller's journal. The sweep of 2026-10-30 (issue 774)
// found three: gitea's `bound` (read by mesh-provisioner, so no forge account was bound), disk-load's `readings` (read by
// restic, nfs-server and mounts) and restic's `good` (read by the self-check). The merge gate's check of reads against
// the owner's manifest went with clause 5, and announcetest holds a module to its own list, not to its readers.
//
// An owner announces from its code. A Go module announces when its code imports mesh-sdk's go/announce: every
// announce.State literal there is read, its Name a string, a constant of the package, or a constant of mesh-sdk this
// check knows (Known). A TypeScript module announces when its code says the event `states` or calls the SDK's keeps:
// every `{ name: "...", readable: true }` there is read. An owner that announces nothing from code is read as before
// on the mesh, and is not judged; neither is an owner outside the catalogue (another repository's).
//
// go run . <catalog root>
//
// Exits 1 on a finding.
package main
import (
"encoding/json"
"fmt"
"go/ast"
"go/parser"
"go/token"
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"git.novox.be/novox/mesh-sdk/go/health"
)
// Known are the mesh-sdk constants an announcement may name a state by, by the selector the code writes.
var Known = map[string]string{
"health.GoodState": health.GoodState,
"health.ReadingsState": health.ReadingsState,
}
const announcePkg = `"git.novox.be/novox/mesh-sdk/go/announce"`
// Announced is what one owner's code announces: each state, readable or not.
type Announced map[string]bool
// Findings is every refusal in the catalogue at root, sorted.
func Findings(root string) ([]string, error) {
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
if err != nil {
return nil, err
}
dirs := map[string]string{}
reads := map[string][]string{}
for _, path := range manifests {
raw, err := os.ReadFile(path)
if err != nil {
return nil, err
}
var m struct {
Module string `json:"module"`
Reads []json.RawMessage `json:"reads"`
}
if err := json.Unmarshal(raw, &m); err != nil {
return nil, fmt.Errorf("%s: %w", path, err)
}
dirs[m.Module] = filepath.Dir(path)
for _, r := range m.Reads {
var s string
if json.Unmarshal(r, &s) == nil {
reads[m.Module] = append(reads[m.Module], s)
}
}
}
var out []string
owners := map[string]Announced{}
for reader, rs := range reads {
for _, r := range rs {
owner, state, ok := strings.Cut(r, ".")
dir, inCatalogue := dirs[owner]
if !ok || !inCatalogue || owner == reader {
continue
}
a, seen := owners[owner]
if !seen {
var problems []string
a, problems, err = AnnouncedBy(dir)
if err != nil {
return nil, err
}
owners[owner] = a
for _, p := range problems {
out = append(out, owner+": "+p)
}
}
if a == nil {
continue // announces nothing from code: read as before
}
readable, keeps := a[state]
switch {
case !keeps:
out = append(out, fmt.Sprintf("%s reads %s.%s, which %s's code does not announce: the read is refused on the mesh", reader, owner, state, owner))
case !readable:
out = append(out, fmt.Sprintf("%s reads %s.%s, which %s's code announces not readable: the read is refused on the mesh; announce it readable", reader, owner, state, owner))
}
}
}
sort.Strings(out)
return dedupe(out), nil
}
func dedupe(in []string) []string {
var out []string
for i, s := range in {
if i == 0 || s != in[i-1] {
out = append(out, s)
}
}
return out
}
var (
tsAnnounces = regexp.MustCompile(`emit\(\s*"states"|\bkeeps\(`)
tsState = regexp.MustCompile(`\{\s*name:\s*"([a-z0-9][a-z0-9-]*)"\s*(,\s*readable:\s*(true|false)\s*)?\}`)
)
// AnnouncedBy is what the module in dir announces from its code, nil when it announces nothing; problems are
// announcements whose state's name cannot be read.
func AnnouncedBy(dir string) (Announced, []string, error) {
var out Announced
var problems []string
pkgs := map[string][]*ast.File{}
fset := token.NewFileSet()
err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
if n := d.Name(); path != dir && (n == "node_modules" || n == "dist" || n == "test" || n == "testdata" || strings.HasPrefix(n, ".")) {
return filepath.SkipDir
}
return nil
}
name := d.Name()
switch {
case strings.HasSuffix(name, ".go") && !strings.HasSuffix(name, "_test.go"):
f, err := parser.ParseFile(fset, path, nil, 0)
if err != nil {
return err
}
pkgs[filepath.Dir(path)] = append(pkgs[filepath.Dir(path)], f)
case strings.HasSuffix(name, ".ts") && !strings.HasSuffix(name, ".d.ts") && !strings.HasSuffix(name, ".test.ts"):
raw, err := os.ReadFile(path)
if err != nil {
return err
}
if !tsAnnounces.Match(raw) && !strings.Contains(string(raw), "KEPT") {
return nil
}
for _, m := range tsState.FindAllStringSubmatch(string(raw), -1) {
if out == nil {
out = Announced{}
}
out[m[1]] = out[m[1]] || m[3] == "true"
}
}
return nil
})
if err != nil {
return nil, nil, err
}
for _, files := range pkgs {
imports := false
consts := map[string]ast.Expr{}
for _, f := range files {
for _, im := range f.Imports {
if im.Path.Value == announcePkg && (im.Name == nil || im.Name.Name == "announce") {
imports = true
}
}
for _, decl := range f.Decls {
g, ok := decl.(*ast.GenDecl)
if !ok || g.Tok != token.CONST {
continue
}
for _, spec := range g.Specs {
v := spec.(*ast.ValueSpec)
for i, n := range v.Names {
if i < len(v.Values) {
consts[n.Name] = v.Values[i]
}
}
}
}
}
if !imports {
continue
}
if out == nil {
out = Announced{}
}
for _, f := range files {
ast.Inspect(f, func(n ast.Node) bool {
lit, ok := n.(*ast.CompositeLit)
if !ok {
return true
}
states := []*ast.CompositeLit{}
switch {
case isAnnounceState(lit.Type):
states = append(states, lit)
case isStateSlice(lit.Type):
for _, e := range lit.Elts {
if c, ok := e.(*ast.CompositeLit); ok && (c.Type == nil || isAnnounceState(c.Type)) {
states = append(states, c)
}
}
default:
return true
}
for _, s := range states {
name, readable, problem := stateOf(s, consts)
if problem != "" {
problems = append(problems, fmt.Sprintf("%s: %s", fset.Position(s.Pos()), problem))
continue
}
out[name] = out[name] || readable
}
return false
})
}
}
return out, problems, nil
}
func isAnnounceState(e ast.Expr) bool {
s, ok := e.(*ast.SelectorExpr)
if !ok {
return false
}
x, ok := s.X.(*ast.Ident)
return ok && x.Name == "announce" && s.Sel.Name == "State"
}
func isStateSlice(e ast.Expr) bool {
a, ok := e.(*ast.ArrayType)
return ok && isAnnounceState(a.Elt)
}
// stateOf is one announce.State literal's name and whether it is readable (Readable true, or Readers named).
func stateOf(lit *ast.CompositeLit, consts map[string]ast.Expr) (string, bool, string) {
var name string
var readable bool
named := false
for _, e := range lit.Elts {
kv, ok := e.(*ast.KeyValueExpr)
if !ok {
return "", false, "an announce.State without field names cannot be read; name its fields"
}
key, _ := kv.Key.(*ast.Ident)
if key == nil {
continue
}
switch key.Name {
case "Name":
v, ok := resolve(kv.Value, consts, 0)
if !ok {
return "", false, fmt.Sprintf("the state's name %s cannot be read: name it by a string, a constant of the package, or a mesh-sdk constant in checks/readable's Known", exprString(kv.Value))
}
name, named = v, true
case "Readable":
if id, ok := kv.Value.(*ast.Ident); ok {
readable = readable || id.Name == "true"
}
case "Readers":
readable = true
}
}
if !named {
return "", false, "an announce.State without a Name"
}
return name, readable, ""
}
func resolve(e ast.Expr, consts map[string]ast.Expr, depth int) (string, bool) {
if depth > 8 {
return "", false
}
switch v := e.(type) {
case *ast.BasicLit:
if v.Kind == token.STRING {
s, err := strconv.Unquote(v.Value)
return s, err == nil
}
case *ast.Ident:
if c, ok := consts[v.Name]; ok {
return resolve(c, consts, depth+1)
}
case *ast.SelectorExpr:
s, ok := Known[exprString(v)]
return s, ok
}
return "", false
}
func exprString(e ast.Expr) string {
switch v := e.(type) {
case *ast.Ident:
return v.Name
case *ast.SelectorExpr:
return exprString(v.X) + "." + v.Sel.Name
case *ast.BasicLit:
return v.Value
}
return fmt.Sprintf("%T", e)
}
func main() {
root := "."
if len(os.Args) > 1 {
root = os.Args[1]
}
found, err := Findings(root)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(2)
}
for _, f := range found {
fmt.Println(f)
}
if len(found) > 0 {
os.Exit(1)
}
}
-90
View File
@@ -1,90 +0,0 @@
package main
import (
"os"
"path/filepath"
"strings"
"testing"
)
// A read of another module's state is refused here when the owner's code announces that state not readable, or not at
// all (the sweep of 2026-10-30, issue 774); an owner that announces nothing from code, or lives in another repository,
// is not judged.
func TestFindings(t *testing.T) {
const goOwner = "package main\n\nimport \"git.novox.be/novox/mesh-sdk/go/announce\"\n\n"
for _, row := range []struct {
name string
files map[string]string
want string // empty: clean
}{
{"go, readable by a constant", map[string]string{
"modules/r/module.json": `{"module":"r","reads":["o.s"]}`,
"modules/o/module.json": `{"module":"o"}`,
"modules/o/cmd/o/main.go": goOwner + "const S = \"s\"\n\nvar _ = announce.State{Name: S, Readable: true}\n",
"modules/o/cmd/o/main_test.go": goOwner + "var _ = announce.State{Name: \"s\"}\n"}, ""},
{"go, not readable", map[string]string{
"modules/r/module.json": `{"module":"r","reads":["o.s"]}`,
"modules/o/module.json": `{"module":"o"}`,
"modules/o/cmd/o/main.go": goOwner + "var Kept = []announce.State{{Name: \"t\", Readable: true}, {Name: \"s\"}}\n"},
"r reads o.s, which o's code announces not readable"},
{"go, a known mesh-sdk name and readers", map[string]string{
"modules/r/module.json": `{"module":"r","reads":["o.good","o.s"]}`,
"modules/o/module.json": `{"module":"o"}`,
"modules/o/cmd/o/main.go": goOwner + "var _ = []announce.State{{Name: health.GoodState, Readable: true}, {Name: \"s\", Readers: []string{\"r\"}}}\n"}, ""},
{"go, a name that cannot be read", map[string]string{
"modules/r/module.json": `{"module":"r","reads":["o.s"]}`,
"modules/o/module.json": `{"module":"o"}`,
"modules/o/cmd/o/main.go": goOwner + "var _ = announce.State{Name: other.S, Readable: true}\n"},
"the state's name other.S cannot be read"},
{"go, a state not announced", map[string]string{
"modules/r/module.json": `{"module":"r","reads":["o.s"]}`,
"modules/o/module.json": `{"module":"o"}`,
"modules/o/cmd/o/main.go": goOwner + "var _ = announce.State{Name: \"t\"}\n"},
"r reads o.s, which o's code does not announce"},
{"typescript, not readable", map[string]string{
"modules/r/module.json": `{"module":"r","reads":["o.bound"]}`,
"modules/o/module.json": `{"module":"o"}`,
"modules/o/kept.ts": `export const KEPT = [{ name: "announced" }, { name: "bound" }];`},
"r reads o.bound, which o's code announces not readable"},
{"typescript, readable", map[string]string{
"modules/r/module.json": `{"module":"r","reads":["o.bound"]}`,
"modules/o/module.json": `{"module":"o"}`,
"modules/o/index.ts": `await keeps([{ name: "bound", readable: true }]);`}, ""},
{"an owner that announces nothing, and one outside the catalogue", map[string]string{
"modules/r/module.json": `{"module":"r","reads":["o.s","mesh-controller.lease"]}`,
"modules/o/module.json": `{"module":"o"}`,
"modules/o/cmd/o/main.go": "package main\n\nvar _ = 1\n"}, ""},
} {
t.Run(row.name, func(t *testing.T) {
root := t.TempDir()
for path, text := range row.files {
p := filepath.Join(root, path)
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(p, []byte(text), 0o644); err != nil {
t.Fatal(err)
}
}
got, err := Findings(root)
if err != nil {
t.Fatal(err)
}
joined := strings.Join(got, "\n")
if row.want == "" && len(got) != 0 || row.want != "" && !strings.Contains(joined, row.want) {
t.Errorf("findings %q, want %q", joined, row.want)
}
})
}
}
// The catalogue itself: every state a module reads of another module that announces from code is announced readable.
func TestTheCatalogueReadsOnlyWhatItsOwnersAnnounceReadable(t *testing.T) {
got, err := Findings(filepath.Join("..", ".."))
if err != nil {
t.Fatal(err)
}
for _, f := range got {
t.Error(f)
}
}
-6
View File
@@ -1,6 +0,0 @@
# What these tests read outside their package, as paths from the repository's root (issue 1936).
# every manifest, for what it reads; and every module's Go and TypeScript code, for the states it announces.
modules/*/module.json
modules/*/cmd
modules/*/*.go
modules/*/*.ts
-3
View File
@@ -1,3 +0,0 @@
module git.novox.be/novox/mesh-catalog/checks/sdkjudge
go 1.22
-189
View File
@@ -1,189 +0,0 @@
// Package sdkjudge refuses mesh-sdk a manifest judge (novox/hq ADR 0578 §4): a package every module compiles in carries
// the contract only, so it holds no function that gathers a manifest's problems and no table of what a manifest may
// declare. The judge is a package of mesh-delivery that only the indexer and the checker use, and a reader decodes a
// manifest and takes the indexer's verdict (ADR 0578 §3).
//
// It reads mesh-sdk's source as the checker places it beside this check (MESH_CHECK_BESIDE, the repository mesh-sdk: at its
// trunk, or at the head of the group's pull request when the group holds one), so a change adding either to mesh-sdk in a
// delivery group with a catalogue change is refused here. A change to mesh-sdk alone is checked by mesh-sdk's own check,
// which does not run this package: the catalogue's check sees the SDK only beside a catalogue change.
//
// What it refuses, in the non-test Go files of mesh-sdk's go/ tree:
// - a function or method named for problems (a name ending in Problems or Problem) that takes a Manifest or a Shelf, and
// a function named ParseManifest, ParseManifestWith, ParseManifestForInputs, ValidateManifest... or ValidateShelf...: the
// judge's entry points;
// - in the package catalogue, a package-level variable named for what is allowed, permitted, removable, reserved or
// forbidden whose value is a map or a list: an allow-list table.
//
// The composer's refusals of composition itself (what a machine can be composed with) are not a judgement of a manifest
// (ADR 0578 §4, clause 5; issue 1585 tracks moving them): they are named in composition, each with why, and the list only
// shrinks, since a name that no longer exists is refused too.
package sdkjudge
import (
"fmt"
"go/ast"
"go/parser"
"go/token"
"io/fs"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
)
// composition is every function or table the check lets stay in mesh-sdk, as "path-from-go/:name", and why. It names no
// manifest judgement: each is the composer's refusal of composition itself (ADR 0578 clause 5).
var composition = map[string]string{
"catalogue/consumers.go:consumersPlaceholderProblems": "composition: rendering a consumer's template",
"catalogue/collections.go:collectedPlaceholderProblems": "composition: placeholders a contribution renders",
"catalogue/environment_into.go:placeholderProblems": "composition: placeholders a contribution renders",
"catalogue/seat_contributions.go:seatPlaceholderProblems": "composition: placeholders a contribution renders",
"catalogue/collections.go:itemTemplateProblem": "composition: filling an item's template",
"catalogue/setting_takes.go:takenSettingsProblems": "composition: whether stored settings still compose",
"catalogue/template.go:HeldTemplateProblem": "composition: the take-in refusal of a resolved build",
}
// Problems is what mesh-sdk's Go tree under root (its go/ directory) holds of a manifest judge, one line each, but for
// what composition lets stay (the map the caller gives; the test of the real tree gives the package's), and the
// entries of composition it no longer holds.
func Problems(root string, composition map[string]string) ([]string, error) {
var out []string
seen := map[string]bool{}
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
if n := d.Name(); n == "testdata" || n == "vendor" || strings.HasPrefix(n, ".") {
return filepath.SkipDir
}
return nil
}
if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
return nil
}
rel, _ := filepath.Rel(root, path)
file, err := parser.ParseFile(token.NewFileSet(), path, nil, parser.SkipObjectResolution)
if err != nil {
return fmt.Errorf("%s: %w", rel, err)
}
inCatalogue := filepath.Base(filepath.Dir(path)) == "catalogue"
for _, d := range file.Decls {
switch d := d.(type) {
case *ast.FuncDecl:
if why := judgeFunc(d); why != "" {
out = append(out, note(composition, seen, rel, d.Name.Name, why))
}
case *ast.GenDecl:
if !inCatalogue || d.Tok != token.VAR {
continue
}
for _, spec := range d.Specs {
v, _ := spec.(*ast.ValueSpec)
if v == nil {
continue
}
for i, name := range v.Names {
if i < len(v.Values) && allowName.MatchString(name.Name) && isTable(v.Values[i]) {
out = append(out, note(composition, seen, rel, name.Name,
"a table of what a manifest may declare (an allow-list)"))
}
}
}
}
}
return nil
})
if err != nil {
return nil, err
}
var kept []string
for _, line := range out {
if line != "" {
kept = append(kept, line)
}
}
for key, why := range composition {
if !seen[key] {
kept = append(kept, fmt.Sprintf("composition lists %s (%s), which mesh-sdk no longer holds: take it off the list", key, why))
}
}
sort.Strings(kept)
return kept, nil
}
// note is the line refusing name in rel for why, or "" when composition lets it stay.
func note(composition map[string]string, seen map[string]bool, rel, name, why string) string {
key := rel + ":" + name
if _, ok := composition[key]; ok {
seen[key] = true
return ""
}
return fmt.Sprintf("%s: %s is %s; mesh-sdk carries the contract only, the judge is the indexer's and the checker's (ADR 0578 §4)", rel, name, why)
}
// allowName is what names a table of what is allowed.
var allowName = regexp.MustCompile(`(?i)allow|permit|removable|reserved|forbid|denied`)
// isTable says whether an expression is a map or a list literal of more than one entry.
func isTable(e ast.Expr) bool {
c, ok := e.(*ast.CompositeLit)
if !ok || len(c.Elts) < 2 {
return false
}
switch c.Type.(type) {
case *ast.MapType, *ast.ArrayType:
return true
}
return false
}
// judgeFunc says why a function is a manifest judge, or "".
func judgeFunc(f *ast.FuncDecl) string {
name := f.Name.Name
if judgeEntry.MatchString(name) {
return "a function that judges a manifest"
}
if !strings.HasSuffix(name, "Problems") && !strings.HasSuffix(name, "Problem") {
return ""
}
if takes(f.Recv) || takes(f.Type.Params) {
return "a function that gathers a manifest's problems"
}
return ""
}
// judgeEntry is the names of the judge's entry points: the strict parsers and the validators of a manifest or a shelf.
var judgeEntry = regexp.MustCompile(`^(ParseManifest(With|ForInputs)?|Validate(Manifest|Shelf)\w*)$`)
// takes says whether a list of parameters names a Manifest or a Shelf anywhere in a type.
func takes(fields *ast.FieldList) bool {
if fields == nil {
return false
}
found := false
for _, f := range fields.List {
ast.Inspect(f.Type, func(n ast.Node) bool {
if id, ok := n.(*ast.Ident); ok && (id.Name == "Manifest" || id.Name == "Shelf") {
found = true
}
return !found
})
}
return found
}
// BesideDir is where the checker places mesh-sdk's clone, as a directory under MESH_CHECK_BESIDE; its go/ tree is judged.
func BesideDir() (string, error) {
beside := os.Getenv("MESH_CHECK_BESIDE")
if beside == "" {
return "", fmt.Errorf("NOT CHECKED: MESH_CHECK_BESIDE is not set, so mesh-sdk's source is not beside this check; the checker places the SDK there (besideRules)")
}
dir := filepath.Join(beside, "mesh-sdk", "go")
if _, err := os.Stat(dir); err != nil {
return "", fmt.Errorf("NOT CHECKED: mesh-sdk is not beside this check in MESH_CHECK_BESIDE=%s: %w", beside, err)
}
return dir, nil
}
-66
View File
@@ -1,66 +0,0 @@
package sdkjudge
import (
"os"
"path/filepath"
"strings"
"testing"
)
// mesh-sdk carries the contract only (ADR 0578 §4). A row is a source tree of its own: a function gathering a manifest's
// problems, a manifest parser that judges and a table of what a manifest may declare are refused, named; a Problems
// function over something that is no manifest, a test file, a table outside the catalogue package and a function that
// merely decodes are not. Then the SDK beside this check is judged.
func TestMeshSDKCarriesNoManifestJudge(t *testing.T) {
for _, row := range []struct {
name, dir, file, src, refused string
}{
{name: "a function gathering a manifest's problems", dir: "catalogue", refused: "StreamProblems is a function that gathers a manifest's problems",
src: "package catalogue\ntype Manifest struct{}\nfunc StreamProblems(m Manifest) []string { return nil }\n"},
{name: "a method over a manifest", dir: "catalogue", refused: "healthProblems is a function that gathers",
src: "package catalogue\ntype Manifest struct{}\nfunc (m Manifest) healthProblems() []string { return nil }\n"},
{name: "a function over the shelf", dir: "catalogue", refused: "CatalogueProblems is a function that gathers",
src: "package catalogue\ntype Shelf map[string]int\nfunc CatalogueProblems(s Shelf) []string { return nil }\n"},
{name: "a parser that judges", dir: "catalogue", refused: "ParseManifestWith is a function that judges a manifest",
src: "package catalogue\nfunc ParseManifestWith(raw []byte) error { return nil }\n"},
{name: "a table of what may be removed", dir: "catalogue", refused: "removable is a table of what a manifest may declare",
src: "package catalogue\nvar removable = map[string]bool{\"A\": true, \"B\": true}\n"},
{name: "a key parser", dir: "delivery", src: "package delivery\nfunc ParseManifestsKey(k string) error { return nil }\n"},
{name: "a decoder", dir: "catalogue", src: "package catalogue\ntype Manifest struct{}\nfunc Decode(raw []byte) (Manifest, error) { return Manifest{}, nil }\n"},
{name: "problems of something else", dir: "asks", src: "package asks\ntype Condition struct{}\nfunc DecodeProblems(c Condition) []string { return nil }\n"},
{name: "a table beside the catalogue", dir: "asks", src: "package asks\nvar allowed = []string{\"a\", \"b\"}\n"},
{name: "a test file", dir: "catalogue", file: "x_test.go",
src: "package catalogue\ntype Manifest struct{}\nfunc StreamProblems(m Manifest) []string { return nil }\n"},
} {
root := t.TempDir()
dir := filepath.Join(root, row.dir)
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
if row.file == "" {
row.file = "x.go"
}
if err := os.WriteFile(filepath.Join(dir, row.file), []byte(row.src), 0o644); err != nil {
t.Fatal(err)
}
got, err := Problems(root, nil)
if err != nil {
t.Fatal(err)
}
joined := strings.Join(got, "\n")
if (row.refused == "") != (len(got) == 0) || !strings.Contains(joined, row.refused) {
t.Errorf("%s: refused %q, want %q", row.name, joined, row.refused)
}
}
dir, err := BesideDir()
if err != nil {
t.Fatal(err)
}
got, err := Problems(dir, composition)
if err != nil {
t.Fatal(err)
}
if len(got) != 0 {
t.Fatalf("mesh-sdk holds a manifest judge:\n%s", strings.Join(got, "\n"))
}
}
-4
View File
@@ -1,4 +0,0 @@
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
# None: the tests read mesh-sdk's source, the clone beside this repository (MESH_CHECK_BESIDE), which is no path of it.
+2 -11
View File
@@ -1,14 +1,5 @@
module adwaita
go 1.26.0
go 1.22
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915
require (
github.com/klauspost/compress v1.20.0 // indirect
github.com/nats-io/nats.go v1.54.0 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
)
require git.novox.be/novox/mesh-sdk/go v0.1.7
+2 -14
View File
@@ -1,14 +1,2 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w=
git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
@@ -1,129 +0,0 @@
package main
// The artifact store's door and the bus, as the holder reaches them through this module's runtime.
import (
"context"
"encoding/json"
"fmt"
"os"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"artifactstoretools/internal/registry"
)
// door is registry.Store at the store's address as this machine reaches it.
type door struct{ s registry.Store }
func (d *door) Address() string { return d.s.Address }
func (d *door) Hold(ctx context.Context, r string) (bool, error) { return d.s.Hold(ctx, r) }
func (d *door) Holds(ctx context.Context, r string) error { return d.s.Holds(ctx, r) }
func (d *door) IsIndex(ctx context.Context, r string) (bool, error) {
return d.s.IsIndex(ctx, r)
}
func (d *door) LetGo(ctx context.Context, r string) error { return d.s.LetGo(ctx, r) }
func (d *door) SpareKept(ctx context.Context, keptImages, touching []string) error {
spare, err := d.s.SpareKeptIndexes(ctx, keptImages, touching)
if err == nil {
d.s.Spare = spare
}
return err
}
// DoorAt reads the store's address (host:port) from the file the manifest fills from the seat mesh-artifact-store's
// reach, each time, so a store that moved is reached where it is now.
func DoorAt(file string) func() (Door, error) {
return func() (Door, error) {
if file == "" {
return nil, fmt.Errorf("this bundle was given no file naming the artifact store's address")
}
raw, err := os.ReadFile(file)
if err != nil {
return nil, fmt.Errorf("the artifact store's address: %w", err)
}
address := strings.TrimSpace(string(raw))
if address == "" {
return nil, fmt.Errorf("the artifact store's address is empty in %s", file)
}
return &door{s: registry.Store{Address: address}}, nil
}
}
// runtimeBus is the holder's bus through the runtime: its reads, its findings and its seat's event.
type runtimeBus struct{}
func (runtimeBus) Keys(state string) ([]string, error) { return stdio.State(state).Keys() }
func (runtimeBus) Get(state, key string) ([]byte, bool, error) {
e, err := stdio.State(state).Get(key)
if err != nil || e == nil {
return nil, false, err
}
return e.Value, true, nil
}
func (runtimeBus) History(state, key string) ([]stdio.StateRevision, error) {
return stdio.State(state).History(key)
}
func (runtimeBus) LogSince(read string, at time.Time) ([]stdio.LogEntry, error) {
var out []stdio.LogEntry
page, err := stdio.Log(read).Since(at, "", 0)
for err == nil {
out = append(out, page.Entries...)
if page.Next == 0 {
return out, nil
}
page, err = stdio.Log(read).Read(page.Next, "", 0)
}
return nil, err
}
func (runtimeBus) Put(state, key string, value any) error {
_, err := stdio.State(state).Put(key, value)
return err
}
func (runtimeBus) Delete(state, key string) error { return stdio.State(state).Delete(key) }
func (runtimeBus) Publish(subject string, body any) error {
return stdio.SeatPublish(subject, body, "")
}
// follow watches the builder's done keys and the wants: a build's end sweeps (once the lease names no controller) and
// observes again; a want changed observes again. Event-driven: nothing reads them on a timer.
func follow(h *Holder) {
<-stdio.Served()
watch := func(state string, handle func(stdio.StateChange) error) {
for {
if err := stdio.State(state).Watch("", handle); err == nil {
return
} else {
h.logf("watching %s: %v; asked again in 10s", state, err)
}
time.Sleep(10 * time.Second)
}
}
observe := func() {
if err := h.Observe(context.Background()); err != nil {
h.logf("image-missing could not be observed: %v", err)
}
}
watch(BuildsRead, func(c stdio.StateChange) error {
if c.Current || c.Op != stdio.OpPut {
return nil
}
var b struct {
State string `json:"state"`
}
if json.Unmarshal(c.Value, &b) == nil && b.State == "done" {
go func() { h.AfterBuild(c.Key); observe() }()
}
return nil
})
watch(WantsRead, func(c stdio.StateChange) error {
if !c.Current {
go observe()
}
return nil
})
observe()
}
File diff suppressed because it is too large Load Diff
@@ -1,266 +0,0 @@
//go:build integration
package main
// The seat mesh-artifact-sweep's holder whole against a real bus (novox/hq ADR 0571 clause 2, issue 1955): the builder's done
// keys, the wants and the controller's lease in a real nats-server's key-value buckets, a build's end heard by a watch
// of the builds as the runtime hands it over, each deletion recorded by a request to the seat mesh-hand-acts on the bus,
// artifacts-gone said on the seat's subject, and S25's finding written to the holder's own bucket. Only the store's
// door is in memory: its HTTP is the copied registry package's, tested there.
import (
"context"
"encoding/json"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/handacts"
"git.novox.be/novox/mesh-sdk/go/testbus"
)
// realBus is the holder's Bus over a real nats-server: each read state a bucket named as the runtime names it.
type realBus struct {
ctx context.Context
nc *nats.Conn
js jetstream.JetStream
// log is the builder's log as the runtime's log verbs would read it (mesh-tools adds them; the sdk's own read of a
// log is tested in mesh-sdk's process package).
log []stdio.LogEntry
}
func (b realBus) LogSince(_ string, at time.Time) ([]stdio.LogEntry, error) {
var out []stdio.LogEntry
for _, e := range b.log {
if !e.At.Before(at) {
out = append(out, e)
}
}
return out, nil
}
func bucketOf(state string) string { return strings.Replace(state, ".", "_", 1) }
func (b realBus) kv(state string) (jetstream.KeyValue, error) {
return b.js.KeyValue(b.ctx, bucketOf(state))
}
func (b realBus) Keys(state string) ([]string, error) {
kv, err := b.kv(state)
if err != nil {
return nil, err
}
keys, err := kv.Keys(b.ctx)
if err == jetstream.ErrNoKeysFound {
return nil, nil
}
return keys, err
}
func (b realBus) Get(state, key string) ([]byte, bool, error) {
kv, err := b.kv(state)
if err != nil {
return nil, false, err
}
e, err := kv.Get(b.ctx, key)
if err == jetstream.ErrKeyNotFound {
return nil, false, nil
}
if err != nil {
return nil, false, err
}
return e.Value(), true, nil
}
func (b realBus) History(state, key string) ([]stdio.StateRevision, error) {
kv, err := b.kv(state)
if err != nil {
return nil, err
}
h, err := kv.History(b.ctx, key)
if err != nil {
return nil, err
}
var out []stdio.StateRevision
for _, e := range h {
out = append(out, stdio.StateRevision{Key: key, Value: e.Value(), Revision: e.Revision(), At: e.Created()})
}
return out, nil
}
func (b realBus) Put(state, key string, value any) error {
kv, err := b.kv(state)
if err != nil {
return err
}
raw, _ := json.Marshal(value)
_, err = kv.Put(b.ctx, key, raw)
return err
}
func (b realBus) Delete(state, key string) error {
kv, err := b.kv(state)
if err != nil {
return err
}
return kv.Delete(b.ctx, key)
}
func (b realBus) Publish(subject string, body any) error {
raw, _ := json.Marshal(body)
return b.nc.Publish(subject, raw)
}
func TestTheSeatSweepsAtABuildsEndOnARealBusOnceTheLeaseIsAway(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
nc, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
defer nc.Close()
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
for _, s := range []string{BuildsRead, WantsRead, HavesRead, LeaseRead, ManifestsRead, "artifact-store-tools." + FindingsState} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucketOf(s), History: 5}); err != nil {
t.Fatal(err)
}
}
bus := realBus{ctx: ctx, nc: nc, js: js}
// The mesh as aMesh lays it, copied onto the real buckets: letta's seven builds, a want, the store's listing.
mem, memb, door, _ := aMesh(t, &buscompose.LeaseHolder{Instance: "c1", Host: "novox", Epoch: 3})
// The builder's log as written today: aMesh dates it from 2026-10-01 for its fixed clock, and this holder runs on the
// real clock, so read as dated the entries fall out of the log's 30 days and every copy is kept as unrecorded.
shift := time.Since(memb.log[len(memb.log)-1].At)
for _, e := range memb.log {
e.At = e.At.Add(shift)
bus.log = append(bus.log, e)
}
for _, s := range []string{BuildsRead, WantsRead, LeaseRead, ManifestsRead} {
keys, _ := memb.Keys(s)
for _, k := range keys[:min(len(keys), 6)] { // the seventh build ends later, on the bus
raw, _, _ := memb.Get(s, k)
if err := bus.Put(s, k, json.RawMessage(raw)); err != nil {
t.Fatal(err)
}
}
}
time.Sleep(20 * time.Millisecond) // each build's end is its key's time on the bus: the seventh the newest
// The seat mesh-hand-acts, answering every act it records; artifacts-gone heard on the seat's subject.
var mu sync.Mutex
var acts, gone []string
if _, err := nc.Subscribe("mesh.seat."+handacts.Seat+".tool.hand-act", func(m *nats.Msg) {
mu.Lock()
acts = append(acts, string(m.Data))
mu.Unlock()
_ = m.Respond([]byte(`{"ok":true}`))
}); err != nil {
t.Fatal(err)
}
if _, err := nc.Subscribe(GoneSubject, func(m *nats.Msg) {
mu.Lock()
gone = append(gone, string(m.Data))
mu.Unlock()
}); err != nil {
t.Fatal(err)
}
h := &Holder{Bus: findingsAt{realBus: bus}, Door: mem.Door, Listed: mem.Listed, Now: time.Now, Logf: t.Logf}
h.Acts = handacts.Recorder{LeaseAway: h.LeaseAway, Ask: func(key string, body any) (json.RawMessage, error) {
raw, _ := json.Marshal(body)
m, err := nc.Request("mesh.seat."+strings.TrimPrefix(strings.Replace(key, ".", ".tool.", 1), "seat:"), raw, 5*time.Second)
if err != nil {
return nil, err
}
return m.Data, nil
}}
// A build's end heard from the builds bucket, as follow hands it over.
kv, _ := bus.kv(BuildsRead)
w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
if err != nil {
t.Fatal(err)
}
ended := func() {
keys, _ := memb.Keys(BuildsRead)
raw, _, _ := memb.Get(BuildsRead, keys[6])
if err := bus.Put(BuildsRead, keys[6], json.RawMessage(raw)); err != nil {
t.Fatal(err)
}
select {
case e := <-w.Updates():
h.AfterBuild(e.Key())
case <-ctx.Done():
t.Fatal("the build's end was not heard")
}
}
// While the lease names the controller, a build's end lets nothing go.
ended()
if len(door.asked) != 0 {
t.Fatalf("before the lease write the store was asked %v", door.asked)
}
// The lease handed away: the next build's end holds the kept archives and lets go of letta's 2nd, which the 7th moved
// past the five newest and no want names (the want keeps the 1st), recording each deletion.
if err := bus.Put(LeaseRead, buscompose.ControllerLeaseKey, buscompose.LeaseHolder{HandedTo: "mesh-grants", Why: "the lease write"}); err != nil {
t.Fatal(err)
}
ended()
deadline := time.Now().Add(5 * time.Second)
for {
mu.Lock()
n := len(gone)
mu.Unlock()
if n > 0 || time.Now().After(deadline) {
break
}
time.Sleep(20 * time.Millisecond)
}
mu.Lock()
defer mu.Unlock()
if len(acts) != 3 || !strings.Contains(acts[0], archiveRef("letta", 2)) || !strings.Contains(acts[1], imageRef("letta", 2)) ||
!strings.Contains(acts[2], mirror(1)) {
t.Fatalf("the hand acts recorded are %v; the store was asked %v", acts, door.asked)
}
if len(gone) != 1 || !strings.Contains(gone[0], `"fingerprint":"`+fp(2)+`"`) {
t.Fatalf("artifacts-gone said %v", gone)
}
// S25 through the real findings bucket: letta's wanted image missing from the store is a finding there.
view, _ := mem.Listed(ctx)
repo, digest, _ := splitReference(imageRef("letta", 1))
delete(view.L.Repos[repo].Revisions, digest)
if err := h.Observe(ctx); err != nil {
t.Fatal(err)
}
if keys, _ := bus.Keys("artifact-store-tools." + FindingsState); len(keys) != 1 || keys[0] != "module.letta.ace.image-missing" {
t.Fatalf("the findings are %v", keys)
}
}
// findingsAt reads and writes the holder's own findings in its bucket, artifact-store-tools_findings.
type findingsAt struct{ realBus }
func (f findingsAt) own(state string) string {
if state == FindingsState {
return Module + "." + state
}
return state
}
func (f findingsAt) Keys(state string) ([]string, error) { return f.realBus.Keys(f.own(state)) }
func (f findingsAt) Get(state, key string) ([]byte, bool, error) {
return f.realBus.Get(f.own(state), key)
}
func (f findingsAt) Put(state, key string, value any) error {
return f.realBus.Put(f.own(state), key, value)
}
func (f findingsAt) Delete(state, key string) error { return f.realBus.Delete(f.own(state), key) }
@@ -1,351 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"sort"
"strings"
"sync"
"testing"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/handacts"
"git.novox.be/novox/mesh-sdk/go/health"
"artifactstoretools/internal/registry"
)
// memBus is the holder's bus in memory: states by name, each key's history, and what was published.
type memBus struct {
mu sync.Mutex
states map[string]map[string][]byte
history map[string][]stdio.StateRevision
log []stdio.LogEntry
published []string
}
func (b *memBus) LogSince(read string, at time.Time) ([]stdio.LogEntry, error) {
var out []stdio.LogEntry
for _, e := range b.log {
if read == delivery.BuildLogRead && !e.At.Before(at) {
out = append(out, e)
}
}
return out, nil
}
func (b *memBus) Keys(state string) ([]string, error) {
b.mu.Lock()
defer b.mu.Unlock()
var out []string
for k := range b.states[state] {
out = append(out, k)
}
sort.Strings(out)
return out, nil
}
func (b *memBus) Get(state, key string) ([]byte, bool, error) {
b.mu.Lock()
defer b.mu.Unlock()
v, ok := b.states[state][key]
return v, ok, nil
}
func (b *memBus) History(state, key string) ([]stdio.StateRevision, error) {
return b.history[state+" "+key], nil
}
func (b *memBus) Put(state, key string, value any) error {
raw, _ := json.Marshal(value)
b.mu.Lock()
defer b.mu.Unlock()
if b.states[state] == nil {
b.states[state] = map[string][]byte{}
}
b.states[state][key] = raw
return nil
}
func (b *memBus) Delete(state, key string) error {
b.mu.Lock()
defer b.mu.Unlock()
delete(b.states[state], key)
return nil
}
func (b *memBus) Publish(subject string, body any) error {
raw, _ := json.Marshal(body)
b.mu.Lock()
defer b.mu.Unlock()
b.published = append(b.published, subject+" "+string(raw))
return nil
}
// memDoor is the store's door in memory: what it holds, which references are indexes, and what was asked.
type memDoor struct {
holds map[string]bool
indexes map[string]bool
asked []string
}
func (d *memDoor) Address() string { return "10.0.0.5:5000" }
func (d *memDoor) Hold(_ context.Context, r string) (bool, error) {
d.asked = append(d.asked, "hold "+r)
if !d.holds[r] {
return false, registry.Gone
}
return false, nil
}
func (d *memDoor) Holds(_ context.Context, r string) error {
if !d.holds[r] {
return registry.Gone
}
return nil
}
func (d *memDoor) IsIndex(_ context.Context, r string) (bool, error) { return d.indexes[r], nil }
func (d *memDoor) LetGo(_ context.Context, r string) error {
d.asked = append(d.asked, "let go "+r)
delete(d.holds, r)
return nil
}
func (d *memDoor) SpareKept(context.Context, []string, []string) error {
d.asked = append(d.asked, "spare")
return nil
}
// mirror is a copy of an upstream base in the store, the n-th.
func mirror(n int) string {
return "artifact-store://upstream/docker.io/library/alpine@sha256:" + strings.Repeat(fmt.Sprintf("%x", n%16), 64)
}
// pinnedImage is the image letta's manifest names by itself, at the store's address.
const pinnedImage = "10.0.0.5:5000/letta/pinned@sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
func digestOf(ref string) string { _, d, _ := splitReference(ref); return d }
func fp(n int) string { return "src2:" + strings.Repeat(fmt.Sprintf("%x", n%16), 64) }
func imageRef(module string, n int) string {
return "artifact-store://" + module + "/image@sha256:" + strings.Repeat(fmt.Sprintf("%x", n%16), 64)
}
func archiveRef(module string, n int) string {
return "artifact-store://" + module + "/files/blobs/sha256:" + strings.Repeat(fmt.Sprintf("%x", (n+8)%16), 64)
}
// A mesh with seven done builds of letta, the n-th ending n hours after a day began, each with an image and an archive;
// a want names the first on ace; the store lists them all and one image no build names; the lease as given.
func aMesh(t *testing.T, lease *buscompose.LeaseHolder) (*Holder, *memBus, *memDoor, *[]string) {
t.Helper()
day := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
b := &memBus{states: map[string]map[string][]byte{}, history: map[string][]stdio.StateRevision{}}
door := &memDoor{holds: map[string]bool{}, indexes: map[string]bool{}}
listing := &Listing{Blobs: map[string]int64{}, Repos: map[string]*Repo{}}
for n := 1; n <= 7; n++ {
key, _ := delivery.BuilderBuildKey("letta", fp(n))
raw, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: strings.Repeat("d", 64), Artifacts: []delivery.BuiltArtifact{
{Name: "image", Kind: "image", Digest: digestOf(imageRef("letta", n)), Reference: imageRef("letta", n)},
{Name: "files", Kind: "archive", Digest: digestOf(archiveRef("letta", n)), Reference: archiveRef("letta", n)},
{Name: "pkg", Kind: "package", Digest: "sha256:" + strings.Repeat("f", 64), Reference: "letta@1.0." + fmt.Sprint(n)}}})
if err != nil {
t.Fatal(err)
}
_ = b.Put(BuildsRead, key, json.RawMessage(raw))
b.history[BuildsRead+" "+key] = []stdio.StateRevision{{Key: key, Value: raw, At: day.Add(time.Duration(n) * time.Hour)}}
for _, r := range []string{imageRef("letta", n), archiveRef("letta", n)} {
door.holds[r] = true
repo, digest, kind := splitReference(r)
if kind == "archive" {
listing.Blobs[digest] = 10
} else {
listing.repo(repo).Revisions[digest] = true
}
}
}
listing.repo("letta/image").Revisions["sha256:"+strings.Repeat("e", 64)] = true // no build names it
listing.repo("letta/pinned").Revisions["sha256:"+strings.Repeat("c", 64)] = true
// Three copies of a base: letta's 2nd stood on the first, its 7th on the second, and no build on the third.
for n := 1; n <= 3; n++ {
repo, digest, _ := splitReference(mirror(n))
listing.repo(repo).Revisions[digest] = true
door.holds[mirror(n)] = true
}
for n := 1; n <= 7; n++ {
e := delivery.BuildLogEntry{Event: delivery.BuildEndedEvent, Module: "letta", FP: fp(n), Repository: "novox/x", Commit: "c",
Node: "ace", At: day.Add(time.Duration(n) * time.Hour), Outcome: "done"}
switch n {
case 2:
e.Against = []string{mirror(1)}
case 7:
e.Against = []string{mirror(2)}
}
raw, _ := delivery.EncodeBuildLogEntry(e)
b.log = append(b.log, stdio.LogEntry{Sequence: uint64(n), Key: "letta", Value: raw, At: e.At})
}
manifest, _ := delivery.EncodeManifest(delivery.Manifest{Module: "letta",
JSON: `{"module":"letta","resources":[{"id":"app","type":"container","image":"` + pinnedImage + `"},` +
`{"id":"built","type":"container","image":"${artifact:image}"}]}`})
mk, _ := delivery.ManifestsKey(strings.Repeat("d", 64))
_ = b.Put(ManifestsRead, mk, json.RawMessage(manifest))
want, _ := json.Marshal(delivery.Want{V: delivery.Version, FP: fp(1), Self: true})
_ = b.Put(WantsRead, "want.ace.letta", json.RawMessage(want))
if lease != nil {
_ = b.Put(LeaseRead, buscompose.ControllerLeaseKey, lease)
}
var acts []string
h := &Holder{Bus: b, Door: func() (Door, error) { return door, nil },
Listed: func(context.Context) (*View, error) { return &View{L: listing}, nil },
Now: func() time.Time { return day.Add(48 * time.Hour) },
Acts: handacts.Recorder{Ask: func(key string, body any) (json.RawMessage, error) {
raw, _ := json.Marshal(body)
acts = append(acts, key+" "+string(raw))
return json.RawMessage(`{}`), nil
}}}
h.Acts.LeaseAway = h.LeaseAway
return h, b, door, &acts
}
// ADR 0571 clause 2: the keep set from the store's listing, the wants and the five newest builds; nothing let go of while
// the lease names a controller; once it names another holder, a build's end lets go of the eligible within the bound,
// leaves an index for a confirmed collect, records each deletion and says artifacts-gone.
func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsOnlyOnceTheLeaseIsAway(t *testing.T) {
controller := &buscompose.LeaseHolder{Instance: "c1", Host: "novox", Epoch: 3}
handed := &buscompose.LeaseHolder{HandedTo: "mesh-grants", HandedBy: "person.jochen", Why: "the lease write"}
h, _, _, _ := aMesh(t, controller)
r, err := h.Artifacts("")
if err != nil {
t.Fatal(err)
}
states := map[string]string{}
for _, x := range r.References {
states[x.Reference] = x.State
}
// letta's 3rd to 7th are the five newest; its 1st a want names on ace, however old; its 2nd is eligible; the package
// and the manifest no build names are no reference of the seat's.
for n, want := range map[int]string{1: "kept", 2: "eligible", 3: "kept", 7: "kept"} {
if states[imageRef("letta", n)] != want || states[archiveRef("letta", n)] != want {
t.Errorf("letta's build %d is %s and %s, not %s", n, states[imageRef("letta", n)], states[archiveRef("letta", n)], want)
}
}
// And the copies: the one letta's 2nd stood on is eligible with it, the 7th's kept; the third no build names.
if states[mirror(1)] != "eligible" || states[mirror(2)] != "kept" || states[mirror(3)] != "" {
t.Errorf("the copies are %s, %s and %q", states[mirror(1)], states[mirror(2)], states[mirror(3)])
}
if m, err := h.MirrorsNow(); err != nil || len(m.Copies) != 3 || m.Copies[2].State != "unrecorded" || m.Held != "" {
t.Errorf("mirrors answered %+v: %v", m, err)
}
if len(r.References) != 16 || r.Counts["kept"] != 13 || r.Counts["eligible"] != 3 || r.Store != "10.0.0.5:5000" {
t.Errorf("the references are %d, counted %v, at %q", len(r.References), r.Counts, r.Store)
}
for _, c := range []struct {
name string
lease *buscompose.LeaseHolder
index bool
most int
letGo []string
left int
skipped int
unlog int
}{
{name: "the lease names a controller", lease: controller},
{name: "no lease key", lease: nil},
{name: "handed away", lease: handed, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), imageRef("letta", 2), mirror(1)}},
{name: "an index waits", lease: handed, index: true, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), mirror(1)}, skipped: 1},
{name: "bounded", lease: handed, most: 1, letGo: []string{archiveRef("letta", 2)}, left: 2},
// A kept build the log no longer holds: what it stood on is unknown, so no copy goes.
{name: "a kept build's bases unknown", lease: handed, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), imageRef("letta", 2)}, unlog: 3},
} {
t.Run(c.name, func(t *testing.T) {
h, b, door, acts := aMesh(t, c.lease)
door.indexes[imageRef("letta", 2)] = c.index
if c.unlog > 0 {
b.log = append(b.log[:c.unlog-1:c.unlog-1], b.log[c.unlog:]...)
}
if c.most == 0 {
h.AfterBuild("letta.x")
if _, err := h.Collect("tidy", true, 0); err == nil {
t.Error("a confirmed collect was let through while the controller sweeps")
}
if len(door.asked) != 0 || len(*acts) != 0 || len(b.published) != 0 {
t.Fatalf("before the lease write the store was asked %v, acts %v, said %v", door.asked, *acts, b.published)
}
return
}
a, err := h.Sweep(context.Background(), Sweeping{Most: c.most, By: "after the build letta.x"})
if err != nil {
t.Fatal(err)
}
if fmt.Sprint(a.LetGo) != fmt.Sprint(c.letGo) || a.Left != c.left || a.Skipped != c.skipped || a.Held != 6 || a.Missing != 0 {
t.Fatalf("let go of %v (left %d, skipped %d, held %d, missing %d), want %v", a.LetGo, a.Left, a.Skipped, a.Held, a.Missing, c.letGo)
}
// Every kept archive held before the first deletion.
first := -1
for i, x := range door.asked {
if strings.HasPrefix(x, "let go ") && first < 0 {
first = i
}
}
if first != 6 {
t.Errorf("the store was asked %v: the six kept archives are held first", door.asked)
}
if len(*acts) != len(c.letGo) || !strings.HasPrefix((*acts)[0], "seat:mesh-hand-acts.hand-act ") ||
!strings.Contains((*acts)[0], c.letGo[0]) {
t.Errorf("the hand acts recorded are %v", *acts)
}
if len(b.published) != 1 || !strings.HasPrefix(b.published[0], GoneSubject+" ") ||
!strings.Contains(b.published[0], `"fingerprint":"`+fp(2)+`"`) {
t.Errorf("said %v", b.published)
}
})
}
}
// S25 (image-missing): a want whose build names an image the store does not hold is a finding in the seat's own state,
// written once and deleted once the store holds it again; images names what a machine's wants and haves name.
func TestAWantedImageTheStoreLacksIsAFindingAndImagesNamesWantsAndHaves(t *testing.T) {
h, b, _, _ := aMesh(t, nil)
view, _ := h.Listed(context.Background())
repo, digest, _ := splitReference(imageRef("letta", 1))
delete(view.L.Repos[repo].Revisions, digest)
h.Listed = func(context.Context) (*View, error) { return view, nil }
if err := h.Observe(context.Background()); err != nil {
t.Fatal(err)
}
key := health.FindingKey(health.ScopeModule, "letta.ace", ImageMissing)
raw, found, _ := b.Get(FindingsState, key)
f, err := health.DecodeFinding(raw)
if !found || err != nil || f.Subject.Machine != "ace" || f.Source != "S25" || !strings.Contains(f.Evidence[0].Said, imageRef("letta", 1)) {
t.Fatalf("the finding is %+v (%v, %v)", f, found, err)
}
view.L.Repos[repo].Revisions[digest] = true
if err := h.Observe(context.Background()); err != nil {
t.Fatal(err)
}
if _, found, _ := b.Get(FindingsState, key); found {
t.Fatal("the finding stayed once the store held the image again")
}
// An image the manifest pins by itself, under the store's address, is judged as the controller judged it.
delete(view.L.Repos["letta/pinned"].Revisions, "sha256:"+strings.Repeat("c", 64))
if err := h.Observe(context.Background()); err != nil {
t.Fatal(err)
}
if raw, found, _ := b.Get(FindingsState, key); !found || !strings.Contains(string(raw), "letta/pinned@sha256:") {
t.Fatalf("the pinnedImage image missing is %s", raw)
}
have, _ := json.Marshal(delivery.Have{V: delivery.Version, FP: fp(7), Phase: "running"})
_ = b.Put(HavesRead, "ace.letta", json.RawMessage(have))
images, err := h.ImagesOf("ace", "10.0.0.5:5000")
if err != nil || len(images.Images) != 3 || !images.SentKnown || images.Images[0].Image != "10.0.0.5:5000/letta/image@sha256:"+strings.Repeat("1", 64) ||
fmt.Sprint(images.Images[0].In) != "[declared]" || fmt.Sprint(images.Images[1].In) != "[sent]" ||
images.Images[2].Image != pinnedImage || fmt.Sprint(images.Images[2].In) != "[declared sent]" {
t.Fatalf("the images of ace are %+v: %v", images, err)
}
}
@@ -1,38 +1,30 @@
// The artifact-store-tools module's Go bundle (novox/hq ADR 0251 §1–3, ADR 0571 clause 2, to-be 51): a process the
// node's runtime launches and speaks MCP over stdio to. It says what the artifact store holds — its repositories, its
// size, and what the mesh keeps of each manifest — read from the store's own files through the store's own container,
// and it holds the seat mesh-artifact-sweep: the keep set, the sweep at each build's end once the controller's lease names
// no controller, and S25's image-missing.
// The artifact-store-tools module's Go bundle (novox/hq ADR 0251 §1–3, to-be 51): a process the node's
// runtime launches and speaks MCP over stdio to. It says what the artifact store holds — its
// repositories, its size, and what the controller's records say of each manifest — read from the
// store's own files through the store's own container, and writing nothing. A collection is asked of
// the controller, which decides and records what it lets go of (ADR 0189).
//
// **A module beside the store, not the store's own.** A module that provides the artifact store cannot also build an
// artifact: building publishes to the store, so the store would be needed to create itself. So these tools are a
// second module, assigned to the machine that holds the store. stdout is the protocol; what this bundle says, it says
// on stderr.
// **A module beside the store, not the store's own.** A module that provides the artifact store cannot
// also build an artifact: building publishes to the store, so the store would be needed to create
// itself, and the controller refuses that manifest. So these tools are a second module, assigned to the
// machine that holds the store. stdout is the protocol; what this bundle says, it says on stderr.
package main
import (
"fmt"
"os"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/handacts"
)
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
func main() {
s := &Store{
Container: os.Getenv("MESH_ARTIFACT_STORE_CONTAINER"),
Run: ExecRunner,
UID: os.Getuid(),
}
h := &Holder{Bus: runtimeBus{}, Door: DoorAt(os.Getenv("MESH_ARTIFACT_STORE_ADDRESS_FILE")), Listed: s.View,
Now: time.Now, Logf: logf}
h.Acts = handacts.Recorder{Ask: stdio.Ask, LeaseAway: h.LeaseAway}
go follow(h)
// An empty name serves as the module the runtime names (MESH_SERVED_MODULE): artifact-store-tools.
if err := stdio.Serve("", append(Tools(s, h, &Catalogue{Ask: stdio.Ask}), SeatTools(h, s)...)); err != nil {
if err := stdio.Serve("", Tools(s, Controller{Ask: stdio.Ask}, &Catalogue{Ask: stdio.Ask})); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
@@ -1,122 +0,0 @@
package main
// The seat mesh-artifact-sweep's verbs (novox/hq ADR 0571 clause 2; catalogue.ArtifactSweepSeat), each a tool named
// `artifact-store.<verb>` as the runtime serves a seat's verb, answering as the controller's verb answered: {output,
// ok, answer}.
import (
"fmt"
"strings"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// verbAnswer is the controller's verbs' answer shape, which the builder and docker read (answerOf).
type verbAnswer struct {
Output string `json:"output"`
OK bool `json:"ok"`
Answer any `json:"answer"`
}
// SeatTools are the seat's verbs over h; s reads the store's listing for mirrors.
func SeatTools(h *Holder, s *Store) []stdio.Tool {
seat := catalogue.ArtifactSweepSeat + "."
return []stdio.Tool{
{Name: seat + "artifacts", Description: "Every artifact the mesh made that the store holds: kept and why, eligible, " +
"or with collected those it no longer holds.",
Run: func(args map[string]any) (any, error) {
r, err := h.Artifacts(text(args, "repository"))
if err != nil {
return nil, err
}
if text(args, "collected") != "true" {
refs := r.References[:0:0]
for _, x := range r.References {
if x.State != "collected" {
refs = append(refs, x)
}
}
r.References = refs
}
return verbAnswer{Output: fmt.Sprintf("%d kept, %d eligible, %d collected\n", r.Counts["kept"],
r.Counts["eligible"], r.Counts["collected"]), OK: true, Answer: r}, nil
}},
{Name: seat + "held", Description: "Whether the store holds each reference given: held, gone or unknown with why.",
Run: func(args map[string]any) (any, error) {
a, err := h.Held(listOf(args["references"]))
if err != nil {
return nil, err
}
return verbAnswer{Output: fmt.Sprintf("the artifact store at %s holds %d of the %d reference(s) asked\n",
a.Store, len(a.Held), len(a.Held)+len(a.Gone)+len(a.Unknown)), OK: true, Answer: a}, nil
}},
{Name: seat + "collect", Description: "The sweep now: a dry run without confirm; with confirm and why, every kept " +
"archive held, then each eligible artifact let go of, each recorded in the hand-act log.",
Run: func(args map[string]any) (any, error) {
most, err := count(args, "most", MostPerSweep, 1, MostAtAll)
if err != nil {
return nil, err
}
a, err := h.Collect(text(args, "why"), text(args, "confirm") == "true", most)
if err != nil {
return nil, err
}
return verbAnswer{Output: fmt.Sprintf("let go of %d, would let go of %d, %d left\n", len(a.LetGo),
len(a.WouldLetGo), a.Left), OK: true, Answer: a}, nil
}},
{Name: seat + "mirrors", Description: "The bases the mesh copied into its store, each kept (a kept build stood on " +
"it), eligible (only builds no longer kept stood on it) or unrecorded (no build the builder's log holds stood " +
"on it: never removed).",
Run: func(args map[string]any) (any, error) {
if text(args, "record") != "" {
return nil, fmt.Errorf("mirrors records nothing by hand on this seat: a copy is recorded by the builds the " +
"builder's log says stood on it (novox/hq ADR 0571 clause 1)")
}
m, err := h.MirrorsNow()
if err != nil {
return nil, err
}
return verbAnswer{Output: fmt.Sprintf("%d copies\n", len(m.Copies)), OK: true, Answer: m}, nil
}},
{Name: seat + "images", Description: "Every container image one machine's wants and haves name.",
Run: func(args map[string]any) (any, error) {
node := text(args, "node")
if node == "" {
return nil, fmt.Errorf("images names the machine")
}
address := ""
if d, err := h.Door(); err == nil {
address = d.Address()
}
a, err := h.ImagesOf(node, address)
if err != nil {
return nil, err
}
return verbAnswer{Output: fmt.Sprintf("%s: %d images\n", node, len(a.Images)), OK: true, Answer: a}, nil
}},
}
}
// listOf is a list argument: a list, or one text separated by commas.
func listOf(v any) []string {
var out []string
add := func(s string) {
for _, p := range strings.Split(s, ",") {
if p = strings.TrimSpace(p); p != "" && !contains(out, p) {
out = append(out, p)
}
}
}
switch x := v.(type) {
case string:
add(x)
case []any:
for _, e := range x {
if s, ok := e.(string); ok {
add(s)
}
}
}
return out
}
@@ -1,8 +1,6 @@
package main
import (
"git.novox.be/novox/mesh-sdk/go/catalogue"
"context"
"encoding/json"
"fmt"
@@ -413,11 +411,6 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
var m struct {
Tools []string `json:"tools"`
Invokes []string `json:"invokes"`
Reads []string `json:"reads"`
Claims []struct {
Name string `json:"name"`
Serves []string `json:"serves"`
} `json:"claims"`
}
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
@@ -435,30 +428,10 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
if !reflect.DeepEqual(served, listed) {
t.Fatalf("served %v, manifest %v", served, listed)
}
// ADR 0571 clause 2: it holds the seat mesh-artifact-sweep and serves its verbs, and calls no verb of the controller's;
// the lease is the one controller state it reads (clause 10's exemption).
want := []string{"seat:mesh-hand-acts.hand-act", CatalogueModules, CatalogueModule}
want := []string{"seat:mesh-controller.artifacts", "seat:mesh-controller.collect", CatalogueModules, CatalogueModule}
if !reflect.DeepEqual(m.Invokes, want) {
t.Errorf("invokes %v", m.Invokes)
}
var verbs []string
for _, tl := range SeatTools(&Holder{}, &Store{}) {
verbs = append(verbs, strings.TrimPrefix(tl.Name, catalogue.ArtifactSweepSeat+"."))
}
seat, _ := catalogue.SeatNamed(catalogue.ArtifactSweepSeat)
var contract []string
for _, v := range seat.Serves {
contract = append(contract, v.Name)
}
if len(m.Claims) != 1 || m.Claims[0].Name != catalogue.ArtifactSweepSeat || !reflect.DeepEqual(m.Claims[0].Serves, verbs) ||
!reflect.DeepEqual(verbs, contract) {
t.Errorf("claims %+v, serves %v, the seat's verbs %v", m.Claims, verbs, contract)
}
for _, r := range m.Reads {
if strings.HasPrefix(r, "mesh-controller.") && r != LeaseRead {
t.Errorf("reads the controller's %s", r)
}
}
}
func TestASocketRefusalIsAskedAgainThroughSudo(t *testing.T) {
@@ -11,16 +11,9 @@ import (
stdio "git.novox.be/novox/mesh-sdk/go"
)
// Recorded is what says which references the mesh keeps and lets go of them: the seat's holder (Holder, novox/hq ADR
// 0571 clause 2), or before it the controller (Controller), each in the controller's answer's shape.
type Recorded interface {
Artifacts(repository string) (*Records, error)
Collect(why string, confirm bool, most int) (*CollectAnswer, error)
}
// Tools are the artifact store's tools (novox/hq ADR 0251 §1). The first three only read; artifact_store_collect
// only reads too, unless it is a real run, and then it asks the seat's sweep, which decides and deletes.
func Tools(s *Store, c Recorded, cat *Catalogue) []stdio.Tool {
// only reads too, unless it is a real run, and then it asks the controller, which decides and deletes.
func Tools(s *Store, c Controller, cat *Catalogue) []stdio.Tool {
ctx := context.Background()
repoArg := map[string]any{"type": "string", "description": "one repository, as the store names it (<module>/<artifact>)"}
return []stdio.Tool{
@@ -1 +0,0 @@
${seat:mesh-artifact-store:reach}
+2 -19
View File
@@ -1,22 +1,5 @@
module artifactstoretools
go 1.26.0
go 1.22
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915
github.com/nats-io/nats.go v1.54.0
)
require (
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
github.com/google/go-tpm v0.9.8 // indirect
github.com/klauspost/compress v1.20.0 // indirect
github.com/minio/highwayhash v1.0.4 // indirect
github.com/nats-io/jwt/v2 v2.8.1 // indirect
github.com/nats-io/nats-server/v2 v2.11.17 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/time v0.15.0 // indirect
)
require git.novox.be/novox/mesh-sdk/go v0.1.7
+2 -27
View File
@@ -1,27 +1,2 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w=
git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
@@ -1,57 +0,0 @@
package registry
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"net/http"
)
// blobBound is the most GetBlob reads: a templates archive is a few kilobytes; anything near this is no archive a
// take-in reads text from.
const blobBound = 64 << 20
// GetBlob is the archive a recorded reference names (`artifact-store://<repository>/blobs/sha256:<hex>`), read whole
// and checked against the digest it names: bytes that hash to another digest are refused, never read (R4e: the
// controller reads a templates archive by the reference of a builder's done key). Gone when the store does not hold
// it; ErrNotOurs for a reference outside the store or one that is not an archive.
func (s Store) GetBlob(ctx context.Context, reference string) ([]byte, error) {
repository, digest, archive, err := s.archive(reference)
if err != nil {
return nil, err
}
if !archive {
return nil, fmt.Errorf("%w: %s is not an archive", ErrNotOurs, reference)
}
request, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url(repository, "blobs", digest), nil)
if err != nil {
return nil, err
}
response, err := s.client().Do(request)
if err != nil {
return nil, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
}
defer response.Body.Close()
switch response.StatusCode {
case http.StatusOK:
case http.StatusNotFound:
return nil, Gone
default:
return nil, fmt.Errorf("the artifact store answered %s for %s/blobs/%s", response.Status, repository, digest)
}
body, err := io.ReadAll(io.LimitReader(response.Body, blobBound+1))
if err != nil {
return nil, fmt.Errorf("reading %s/blobs/%s: %w", repository, digest, err)
}
if len(body) > blobBound {
return nil, fmt.Errorf("%s/blobs/%s is larger than %d bytes, which no archive read here is", repository, digest, blobBound)
}
sum := sha256.Sum256(body)
if got := "sha256:" + hex.EncodeToString(sum[:]); got != digest {
return nil, fmt.Errorf("the artifact store answered %s/blobs/%s with bytes of the digest %s: refused", repository,
digest, got)
}
return body, nil
}
@@ -1,44 +0,0 @@
package registry
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
// A take-in reads a templates archive by its reference, and only bytes of the digest the reference names.
func TestGetBlobReadsOnlyTheBytesOfItsDigest(t *testing.T) {
good := []byte("the archive")
sum := sha256.Sum256(good)
digest := "sha256:" + hex.EncodeToString(sum[:])
liar := "sha256:" + strings.Repeat("0", 64)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case strings.HasSuffix(r.URL.Path, "/x/files/blobs/"+digest), strings.HasSuffix(r.URL.Path, "/x/files/blobs/"+liar):
_, _ = w.Write(good)
default:
w.WriteHeader(http.StatusNotFound)
}
}))
defer server.Close()
s := Store{Address: strings.TrimPrefix(server.URL, "http://")}
ctx := context.Background()
body, err := s.GetBlob(ctx, "artifact-store://x/files/blobs/"+digest)
if err != nil || string(body) != string(good) {
t.Fatalf("the archive: %q %v", body, err)
}
if _, err := s.GetBlob(ctx, "artifact-store://x/files/blobs/"+liar); err == nil || !strings.Contains(err.Error(), "refused") {
t.Fatalf("bytes of another digest are read: %v", err)
}
if _, err := s.GetBlob(ctx, "artifact-store://x/other/blobs/"+digest); !errors.Is(err, Gone) {
t.Fatalf("a blob the store does not hold: %v", err)
}
if _, err := s.GetBlob(ctx, "registry.example/x@"+digest); !errors.Is(err, ErrNotOurs) {
t.Fatalf("a reference outside the store: %v", err)
}
}
@@ -1,345 +0,0 @@
package registry
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"net/http"
"strconv"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// Every archive the store keeps is held by a manifest (novox/hq issue 253, ADR 0189).
//
// **The store's collector marks only from manifests.** The mesh's store is a stock registry, and
// its nightly `registry garbage-collect` walks every manifest in every repository, marks the blobs
// those manifests name, and deletes every blob it did not mark. An image is a manifest, so what
// the mesh keeps of an image survives. An archive was not: the builder put it in the store as a
// bare blob — upload, then `PUT ?digest=` — and nothing in the store names it. To the collector a
// bare blob is unreferenced, so the first real collection would have deleted every archive the
// mesh holds, kept or not, and every machine pinning a bundle would have found it gone. The
// collector runs `--dry-run` until this is true.
//
// **So each archive gets a holder**: the smallest OCI image manifest that names it — the empty
// config, one layer, nothing else — put in the archive's own repository, by digest, untagged. The
// collector marks it and so keeps the archive; the sweep lets go of an archive by deleting its
// holder first, which is what lets the bytes go at the next collection.
//
// **Nothing a machine reads changes.** The recorded reference stays
// `artifact-store://<module>/<artifact>/blobs/sha256:…`, and machines fetch the blob exactly as
// before. The holder is the store's bookkeeping, not a second way to reach anything.
//
// **Deterministic, so it never needs recording.** The holder is composed from the archive's digest
// and size alone, in a fixed field order with no timestamps or annotations, so the sweep can
// compute which manifest holds any archive from the reference it already has plus one HEAD for the
// size. No schema change, no second record that could disagree with the store.
const (
// mediaManifest is the type a holder is put and asked for as.
mediaManifest = "application/vnd.oci.image.manifest.v1+json"
// mediaEmpty is the OCI empty descriptor's type: a config that says nothing, for a manifest
// whose only purpose is to name its layer.
mediaEmpty = "application/vnd.oci.empty.v1+json"
// emptyDigest is the digest of `{}`, the empty config's content, fixed by the OCI spec.
emptyDigest = "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"
// mediaArchive is the layer type an archive is held as. Every archive the builder publishes is
// `pack`'s gzipped tar, so this is the true type and not a placeholder — and it is a constant,
// not read from anywhere, because the holder must be recomputable from the reference alone.
mediaArchive = "application/vnd.oci.image.layer.v1.tar+gzip"
)
// emptyConfig is the content emptyDigest names.
var emptyConfig = []byte("{}")
// manifestAccept is what a manifest is asked for as. A registry answers a manifest HEAD only in a
// type the caller named, and answers 404 to a bare one for a manifest it holds perfectly well
// (measured 2026-09-28; internal/builder/registry.go says how that was found).
var manifestAccept = []string{
mediaManifest,
"application/vnd.docker.distribution.manifest.v2+json",
}
type descriptor struct {
MediaType string `json:"mediaType"`
Digest string `json:"digest"`
Size int64 `json:"size"`
}
type holderManifest struct {
SchemaVersion int `json:"schemaVersion"`
MediaType string `json:"mediaType"`
Config descriptor `json:"config"`
Layers []descriptor `json:"layers"`
}
// Holder is the manifest that holds an archive in the store, and its digest.
//
// A pure function of the archive's digest and size: the same two in give the same bytes out,
// always, because `encoding/json` writes a struct's fields in their declared order and there is
// nothing here that varies by when or where it was composed.
func Holder(digest string, size int64) (body []byte, holder string) {
body, err := json.Marshal(holderManifest{
SchemaVersion: 2,
MediaType: mediaManifest,
Config: descriptor{MediaType: mediaEmpty, Digest: emptyDigest, Size: int64(len(emptyConfig))},
Layers: []descriptor{{MediaType: mediaArchive, Digest: digest, Size: size}},
})
if err != nil {
// Marshalling a struct of strings and integers cannot fail.
panic(err)
}
sum := sha256.Sum256(body)
return body, "sha256:" + hex.EncodeToString(sum[:])
}
// Hold makes sure the store holds this archive by a manifest, and says whether it had to write one.
//
// Takes a reference as the mesh records it. An image is its own manifest and needs no holder, so
// it answers false and nothing is asked. Idempotent: a holder already there is left alone, which
// is what lets the sweep run it over every kept archive on every build and so backfill the bare
// blobs published before holders existed (novox/hq issue 253).
//
// Gone when the store does not hold the archive at all: there is nothing to hold, and that is a
// fact the caller reports rather than one this invents a remedy for.
func (s Store) Hold(ctx context.Context, reference string) (bool, error) {
repository, digest, archive, err := s.archive(reference)
if err != nil || !archive {
return false, err
}
size, err := s.blobSize(ctx, repository, digest)
if err != nil {
return false, err
}
return s.HoldBlob(ctx, repository, digest, size)
}
// Held is whether the store holds this archive by its manifest. Asks and changes nothing — the
// question an operator needs answered with "none unheld" before the collector is let loose.
//
// An image answers true: it is its own manifest. An archive the store does not have answers Gone.
func (s Store) Held(ctx context.Context, reference string) (bool, error) {
repository, digest, archive, err := s.archive(reference)
if err != nil {
return false, err
}
if !archive {
return true, nil
}
size, err := s.blobSize(ctx, repository, digest)
if err != nil {
return false, err
}
_, holder := Holder(digest, size)
return s.has(ctx, s.url(repository, "manifests", holder), manifestAccept...)
}
// HoldBlob puts the holder for a blob of this digest and size into its repository, unless it is
// there already. Answers whether it wrote one.
//
// The builder calls this with the size it has just uploaded; the sweep, through Hold, with the size
// the store reports. Both arrive at the same holder, which is the point of composing it.
func (s Store) HoldBlob(ctx context.Context, repository, digest string, size int64) (bool, error) {
if s.Address == "" {
return false, fmt.Errorf("this mesh has no artifact store on its network to hold %s/%s in", repository, digest)
}
body, holder := Holder(digest, size)
there, err := s.has(ctx, s.url(repository, "manifests", holder), manifestAccept...)
if err != nil {
return false, err
}
if there {
return false, nil
}
// The config must be in the repository before a manifest naming it is accepted: a registry
// refuses a manifest whose blobs it cannot find there, which is the property that makes a
// holder mean something.
if err := s.putBlob(ctx, repository, emptyDigest, emptyConfig); err != nil {
return false, err
}
// **By digest, never by tag.** A tag would be one more name to move and one more thing the
// collector's `--delete-untagged` would read as meaningful; the mesh names nothing by tag that
// it pins by digest, and an untagged manifest is kept by plain collection.
request, err := http.NewRequestWithContext(ctx, http.MethodPut,
s.url(repository, "manifests", holder), bytes.NewReader(body))
if err != nil {
return false, err
}
request.Header.Set("Content-Type", mediaManifest)
response, err := s.client().Do(request)
if err != nil {
return false, err
}
defer response.Body.Close()
if response.StatusCode != http.StatusCreated {
said, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
return false, fmt.Errorf("the artifact store refused to hold %s/%s: %s %s",
repository, digest, response.Status, strings.TrimSpace(string(said)))
}
return true, nil
}
// letGoOfHolder deletes the manifest holding an archive, before the archive's own link goes.
//
// **Holder first.** Deleting the blob link alone leaves a manifest still naming the blob, and the
// collector would keep its bytes for ever on the strength of it — the sweep would record the
// archive collected while the disk said otherwise. Deleting the holder first and failing before
// the link goes leaves an unheld archive that the next sweep still offers, which is safe.
//
// A store that no longer has the blob answers Gone: without its size the holder cannot be named,
// and without the blob there is nothing left for a holder to keep. A store that never had a holder
// for it — an archive published before holders, never backfilled — answers 404 to the delete, and
// that is the outcome wanted.
func (s Store) letGoOfHolder(ctx context.Context, repository, digest string) error {
size, err := s.blobSize(ctx, repository, digest)
if err != nil {
return err
}
_, holder := Holder(digest, size)
err = s.remove(ctx, s.url(repository, "manifests", holder), repository+"/manifests/"+holder)
if err == Gone {
return nil
}
return err
}
// archive reads a recorded reference into its repository and digest, and whether it is an archive
// at all. Refuses as ErrNotOurs anything the mesh did not put in its own store.
func (s Store) archive(reference string) (repository, digest string, archive bool, err error) {
path, kept := catalogue.InArtifactStore(reference)
if !kept {
return "", "", false, fmt.Errorf("%w: %s", ErrNotOurs, reference)
}
if s.Address == "" {
return "", "", false, fmt.Errorf("this mesh has no artifact store on its network to ask about %s", reference)
}
repository, kind, digest, err := split(path)
if err != nil {
return "", "", false, err
}
return repository, digest, kind == "blobs", nil
}
// blobSize is how large the store says a blob is; Gone when it does not have it.
func (s Store) blobSize(ctx context.Context, repository, digest string) (int64, error) {
request, err := http.NewRequestWithContext(ctx, http.MethodHead, s.url(repository, "blobs", digest), nil)
if err != nil {
return 0, err
}
response, err := s.client().Do(request)
if err != nil {
return 0, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
}
defer response.Body.Close()
switch response.StatusCode {
case http.StatusOK:
case http.StatusNotFound:
return 0, Gone
default:
return 0, fmt.Errorf("the artifact store answered %s for %s/blobs/%s", response.Status, repository, digest)
}
// Read from the header rather than ContentLength: a HEAD's ContentLength is what the response
// says it would have sent, which Go reports faithfully, but a proxy in between is free to drop
// it, and the header is what the registry itself wrote.
if length := response.Header.Get("Content-Length"); length != "" {
if n, err := strconv.ParseInt(length, 10, 64); err == nil && n >= 0 {
return n, nil
}
}
if response.ContentLength >= 0 {
return response.ContentLength, nil
}
return 0, fmt.Errorf("the artifact store holds %s/blobs/%s and will not say how large it is", repository, digest)
}
// putBlob uploads a small blob unless the repository already has it: ask where, then put it there
// naming the digest — the registry's own two steps, the same the builder takes for an archive.
func (s Store) putBlob(ctx context.Context, repository, digest string, body []byte) error {
if there, err := s.has(ctx, s.url(repository, "blobs", digest)); err != nil {
return err
} else if there {
return nil
}
start, err := http.NewRequestWithContext(ctx, http.MethodPost,
"http://"+s.Address+"/v2/"+repository+"/blobs/uploads/", nil)
if err != nil {
return err
}
begun, err := s.client().Do(start)
if err != nil {
return fmt.Errorf("cannot start an upload to %s: %w", repository, err)
}
begun.Body.Close()
if begun.StatusCode != http.StatusAccepted {
return fmt.Errorf("the artifact store answered %s when asked where to put a blob in %s", begun.Status, repository)
}
where := begun.Header.Get("Location")
if where == "" {
return fmt.Errorf("the artifact store accepted an upload to %s and said nowhere to put it", repository)
}
if strings.HasPrefix(where, "/") {
where = "http://" + s.Address + where
}
separator := "?"
if strings.Contains(where, "?") {
separator = "&"
}
put, err := http.NewRequestWithContext(ctx, http.MethodPut, where+separator+"digest="+digest, bytes.NewReader(body))
if err != nil {
return err
}
put.Header.Set("Content-Type", "application/octet-stream")
done, err := s.client().Do(put)
if err != nil {
return err
}
defer done.Body.Close()
if done.StatusCode != http.StatusCreated {
said, _ := io.ReadAll(io.LimitReader(done.Body, 4096))
return fmt.Errorf("the artifact store refused a blob in %s: %s %s", repository, done.Status, strings.TrimSpace(string(said)))
}
return nil
}
// has is whether the store answers 200 for a HEAD at that URL.
func (s Store) has(ctx context.Context, url string, accept ...string) (bool, error) {
request, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
if err != nil {
return false, err
}
for _, media := range accept {
request.Header.Add("Accept", media)
}
response, err := s.client().Do(request)
if err != nil {
return false, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
}
defer response.Body.Close()
switch response.StatusCode {
case http.StatusOK:
return true, nil
case http.StatusNotFound:
return false, nil
default:
return false, fmt.Errorf("the artifact store answered %s for %s", response.Status, url)
}
}
func (s Store) url(repository, kind, digest string) string {
return "http://" + s.Address + "/v2/" + repository + "/" + kind + "/" + digest
}
func (s Store) client() *http.Client {
if s.HTTP != nil {
return s.HTTP
}
return &http.Client{Timeout: 30 * time.Second}
}
@@ -1,268 +0,0 @@
package registry
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strconv"
"strings"
"sync"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// Every kept archive is held by a manifest (novox/hq issue 253, ADR 0189).
//
// Against an in-memory registry that keeps blobs and manifests per repository and refuses what a
// registry refuses — a blob whose digest does not match, a manifest whose digest does not match
// or whose blobs the repository does not have, a manifest asked for without an Accept naming its
// type. What is asserted is this side's decisions; the live test below asserts the registry's.
type memRegistry struct {
mu sync.Mutex
blobs map[string][]byte // repository + "@" + digest
manifests map[string][]byte // repository + "@" + digest
writes []string // every PUT and DELETE, as "METHOD path"
}
func digestOf(body []byte) string {
sum := sha256.Sum256(body)
return "sha256:" + hex.EncodeToString(sum[:])
}
func (m *memRegistry) serve(t *testing.T) Store {
t.Helper()
m.blobs = map[string][]byte{}
m.manifests = map[string][]byte{}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
m.mu.Lock()
defer m.mu.Unlock()
path := strings.TrimPrefix(r.URL.Path, "/v2/")
if r.Method == http.MethodPut || r.Method == http.MethodDelete {
m.writes = append(m.writes, r.Method+" "+r.URL.Path)
}
switch {
case r.Method == http.MethodPost && strings.HasSuffix(path, "/blobs/uploads/"):
repository := strings.TrimSuffix(path, "/blobs/uploads/")
w.Header().Set("Location", "/upload/"+repository+"?state=x")
w.WriteHeader(http.StatusAccepted)
case r.Method == http.MethodPut && strings.HasPrefix(r.URL.Path, "/upload/"):
repository := strings.TrimPrefix(r.URL.Path, "/upload/")
body, _ := io.ReadAll(r.Body)
digest := r.URL.Query().Get("digest")
if digest != digestOf(body) {
w.WriteHeader(http.StatusBadRequest)
return
}
m.blobs[repository+"@"+digest] = body
w.WriteHeader(http.StatusCreated)
case strings.Contains(path, "/blobs/"):
repository, digest, _ := strings.Cut(path, "/blobs/")
key := repository + "@" + digest
body, ok := m.blobs[key]
if !ok {
w.WriteHeader(http.StatusNotFound)
return
}
switch r.Method {
case http.MethodHead:
w.Header().Set("Content-Length", strconv.Itoa(len(body)))
w.WriteHeader(http.StatusOK)
case http.MethodDelete:
delete(m.blobs, key)
w.WriteHeader(http.StatusAccepted)
default:
w.WriteHeader(http.StatusMethodNotAllowed)
}
case strings.Contains(path, "/manifests/"):
repository, digest, _ := strings.Cut(path, "/manifests/")
key := repository + "@" + digest
switch r.Method {
case http.MethodHead:
if _, ok := m.manifests[key]; !ok || !strings.Contains(r.Header.Get("Accept"), mediaManifest) {
w.WriteHeader(http.StatusNotFound)
return
}
w.WriteHeader(http.StatusOK)
case http.MethodPut:
body, _ := io.ReadAll(r.Body)
if digest != digestOf(body) {
w.WriteHeader(http.StatusBadRequest)
return
}
var named holderManifest
if err := json.Unmarshal(body, &named); err != nil {
w.WriteHeader(http.StatusBadRequest)
return
}
for _, d := range append([]descriptor{named.Config}, named.Layers...) {
if _, ok := m.blobs[repository+"@"+d.Digest]; !ok {
w.WriteHeader(http.StatusBadRequest)
fmt.Fprintf(w, "MANIFEST_BLOB_UNKNOWN %s", d.Digest)
return
}
}
m.manifests[key] = body
w.WriteHeader(http.StatusCreated)
case http.MethodDelete:
if _, ok := m.manifests[key]; !ok {
w.WriteHeader(http.StatusNotFound)
return
}
delete(m.manifests, key)
w.WriteHeader(http.StatusAccepted)
}
default:
w.WriteHeader(http.StatusNotFound)
}
}))
t.Cleanup(server.Close)
return Store{Address: strings.TrimPrefix(server.URL, "http://")}
}
// bare puts an archive in the store the way the builder did before holders: a blob, nothing more.
func (m *memRegistry) bare(repository string, body []byte) string {
m.mu.Lock()
defer m.mu.Unlock()
digest := digestOf(body)
m.blobs[repository+"@"+digest] = body
return catalogue.ArtifactStoreScheme + repository + "/blobs/" + digest
}
func TestTheHolderIsComposedFromTheDigestAndSizeAlone(t *testing.T) {
// The sweep must arrive at the very manifest the builder wrote, with nothing recorded between
// them. Same inputs, same bytes — and a different size is a different holder, so a holder can
// never be mistaken for one of a different blob.
digest := "sha256:" + strings.Repeat("a", 64)
one, first := Holder(digest, 42)
two, second := Holder(digest, 42)
if !bytes.Equal(one, two) || first != second {
t.Fatalf("the same archive composed two holders:\n%s\n%s", one, two)
}
if _, other := Holder(digest, 43); other == first {
t.Fatal("a different size composed the same holder")
}
want := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json",` +
`"config":{"mediaType":"application/vnd.oci.empty.v1+json",` +
`"digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","size":2},` +
`"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"` + digest + `","size":42}]}`
if string(one) != want {
t.Fatalf("the holder is\n%s\nwant\n%s", one, want)
}
if digestOf(emptyConfig) != emptyDigest {
t.Fatalf("the empty config's digest is %s, not %s", digestOf(emptyConfig), emptyDigest)
}
}
func TestHoldBackfillsABareArchiveAndIsIdempotent(t *testing.T) {
// The archives published before this have no holder. Hold, run over every kept archive on
// every sweep, writes one the first time and nothing after.
m := &memRegistry{}
store := m.serve(t)
ctx := context.Background()
body := []byte("a theme")
reference := m.bare("shell/config", body)
if held, err := store.Held(ctx, reference); err != nil || held {
t.Fatalf("a bare blob reads as held=%v (%v)", held, err)
}
wrote, err := store.Hold(ctx, reference)
if err != nil {
t.Fatal(err)
}
if !wrote {
t.Fatal("holding a bare archive wrote nothing")
}
_, holder := Holder(digestOf(body), int64(len(body)))
if _, ok := m.manifests["shell/config@"+holder]; !ok {
t.Fatalf("the store holds manifests %v; want %s", m.manifests, holder)
}
if held, err := store.Held(ctx, reference); err != nil || !held {
t.Fatalf("after holding, held=%v (%v)", held, err)
}
writes := len(m.writes)
wrote, err = store.Hold(ctx, reference)
if err != nil {
t.Fatal(err)
}
if wrote || len(m.writes) != writes {
t.Fatalf("holding again wrote %v", m.writes[writes:])
}
}
func TestAnImageNeedsNoHolderAndAMissingArchiveIsGone(t *testing.T) {
m := &memRegistry{}
store := m.serve(t)
ctx := context.Background()
// An image is its own manifest: nothing is asked.
wrote, err := store.Hold(ctx, catalogue.ArtifactStoreScheme+"web/app@sha256:"+strings.Repeat("b", 64))
if err != nil || wrote || len(m.writes) != 0 {
t.Fatalf("holding an image wrote=%v err=%v writes=%v", wrote, err, m.writes)
}
// An archive the store does not have is a fact to report, not something to invent a holder for.
_, err = store.Hold(ctx, catalogue.ArtifactStoreScheme+"web/config/blobs/sha256:"+strings.Repeat("c", 64))
if !errors.Is(err, Gone) {
t.Fatalf("holding a missing archive answered %v, want Gone", err)
}
// And a reference that is not the mesh's is refused as such.
if _, err := store.Hold(ctx, "docker.io/library/registry@sha256:abc"); !errors.Is(err, ErrNotOurs) {
t.Fatalf("holding a vendor's image answered %v, want ErrNotOurs", err)
}
}
func TestLettingGoOfAnArchiveDeletesItsHolderFirst(t *testing.T) {
// A holder left behind would keep the bytes through every collection while the record said
// collected; the link deleted first and the holder failing after would be that exactly.
m := &memRegistry{}
store := m.serve(t)
ctx := context.Background()
body := []byte("an old theme")
reference := m.bare("shell/config", body)
if _, err := store.Hold(ctx, reference); err != nil {
t.Fatal(err)
}
m.writes = nil
if err := store.LetGo(ctx, reference); err != nil {
t.Fatal(err)
}
_, holder := Holder(digestOf(body), int64(len(body)))
want := []string{
"DELETE /v2/shell/config/manifests/" + holder,
"DELETE /v2/shell/config/blobs/" + digestOf(body),
}
if strings.Join(m.writes, "\n") != strings.Join(want, "\n") {
t.Fatalf("the store was asked\n%s\nwant\n%s", strings.Join(m.writes, "\n"), strings.Join(want, "\n"))
}
if len(m.manifests) != 0 {
t.Fatalf("a holder survived: %v", m.manifests)
}
// Asked again, the archive is already gone, which is the outcome wanted.
if err := store.LetGo(ctx, reference); !errors.Is(err, Gone) {
t.Fatalf("letting go twice answered %v, want Gone", err)
}
}
func TestLettingGoOfAnUnheldArchiveStillDeletesIt(t *testing.T) {
// An archive published before holders and let go of before any sweep held it: the holder's
// delete answers 404, which is the outcome wanted, and the blob still goes.
m := &memRegistry{}
store := m.serve(t)
reference := m.bare("shell/config", []byte("never held"))
if err := store.LetGo(context.Background(), reference); err != nil {
t.Fatal(err)
}
if len(m.blobs) != 0 {
t.Fatalf("the blob survived: %v", m.blobs)
}
}
@@ -1,176 +0,0 @@
package registry
import (
"context"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"slices"
"strings"
"sync"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// An index is let go of with the platform manifests it names, except those a kept index of the same
// repository names too (novox/hq ADR 0257). Each platform manifest is a manifest of the repository in
// its own right, and the store's collector keeps every manifest a repository holds — so an index let go
// of alone frees nothing of the images it names.
func digestN(n int) string { return fmt.Sprintf("sha256:%064x", n) }
// indexStore holds indexes and images in one repository, answers GETs with their documents, and records
// every delete.
type indexStore struct {
mu sync.Mutex
indexes map[string][]string // digest → the platform manifests it names
images map[string]bool
deleted []string
}
func (s *indexStore) serve(t *testing.T) Store {
t.Helper()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
s.mu.Lock()
defer s.mu.Unlock()
digest := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:]
switch r.Method {
case http.MethodGet:
if children, ok := s.indexes[digest]; ok {
var named []string
for _, c := range children {
named = append(named, `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"`+c+`","size":1}`)
}
w.Header().Set("Content-Type", "application/vnd.oci.image.index.v1+json")
_, _ = w.Write([]byte(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[` +
strings.Join(named, ",") + `]}`))
return
}
if s.images[digest] {
w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
_, _ = w.Write([]byte(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","layers":[]}`))
return
}
w.WriteHeader(http.StatusNotFound)
case http.MethodHead:
if _, ok := s.indexes[digest]; ok || s.images[digest] {
w.WriteHeader(http.StatusOK)
return
}
w.WriteHeader(http.StatusNotFound)
case http.MethodDelete:
if _, ok := s.indexes[digest]; !ok && !s.images[digest] {
w.WriteHeader(http.StatusNotFound)
return
}
s.deleted = append(s.deleted, digest)
delete(s.indexes, digest)
delete(s.images, digest)
w.WriteHeader(http.StatusAccepted)
default:
w.WriteHeader(http.StatusBadRequest)
}
}))
t.Cleanup(server.Close)
return Store{Address: strings.TrimPrefix(server.URL, "http://")}
}
func TestAnIndexGoesWithItsPlatformsAndAKeptIndexKeepsItsOwn(t *testing.T) {
// Two indexes of one image's repository: the old one names platforms 11 and 12, the kept one 12 and 13.
s := &indexStore{
indexes: map[string][]string{digestN(1): {digestN(11), digestN(12)}, digestN(2): {digestN(12), digestN(13)}},
images: map[string]bool{digestN(11): true, digestN(12): true, digestN(13): true},
}
store := s.serve(t)
repository := "upstream/docker.io/library/golang"
old := catalogue.ArtifactStoreScheme + repository + "@" + digestN(1)
kept := catalogue.ArtifactStoreScheme + repository + "@" + digestN(2)
spare, err := store.SpareKeptIndexes(context.Background(), []string{kept}, []string{old})
if err != nil {
t.Fatal(err)
}
store.Spare = spare
if err := store.LetGo(context.Background(), old); err != nil {
t.Fatal(err)
}
// Its own platform first, the index last; the platform the kept index names is left in place.
if !slices.Equal(s.deleted, []string{digestN(11), digestN(1)}) {
t.Fatalf("deleted %v; want the old index's own platform, then the index", s.deleted)
}
if !s.images[digestN(12)] || !s.images[digestN(13)] {
t.Fatal("a platform a kept index names was let go of")
}
}
func TestWithoutSparingAnIndexGoesAloneAndAnImageNamesNoPlatforms(t *testing.T) {
s := &indexStore{indexes: map[string][]string{digestN(1): {digestN(11)}}, images: map[string]bool{digestN(11): true, digestN(5): true}}
store := s.serve(t)
// No Spare: as before ADR 0257, the index alone.
if err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/base@"+digestN(1)); err != nil {
t.Fatal(err)
}
if !slices.Equal(s.deleted, []string{digestN(1)}) {
t.Fatalf("without a Spare, deleted %v", s.deleted)
}
// An image is one delete, with a Spare or without.
s.deleted = nil
spare, err := store.SpareKeptIndexes(context.Background(), nil, []string{catalogue.ArtifactStoreScheme + "web/app@" + digestN(5)})
if err != nil {
t.Fatal(err)
}
store.Spare = spare
if err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/app@"+digestN(5)); err != nil {
t.Fatal(err)
}
if !slices.Equal(s.deleted, []string{digestN(5)}) {
t.Fatalf("an image was let go of as %v", s.deleted)
}
// An index the store no longer holds is Gone, and nothing is deleted on its account.
s.deleted = nil
err = store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/base@"+digestN(9))
if !errors.Is(err, Gone) || len(s.deleted) != 0 {
t.Fatalf("an index the store does not hold: %v, deleted %v", err, s.deleted)
}
}
func TestASpareThatCannotBeReadKeepsTheIndex(t *testing.T) {
s := &indexStore{indexes: map[string][]string{digestN(1): {digestN(11)}}, images: map[string]bool{digestN(11): true}}
store := s.serve(t)
store.Spare = func(context.Context, string) (map[string]bool, error) { return nil, fmt.Errorf("the store went away") }
err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/base@"+digestN(1))
if err == nil || !strings.Contains(err.Error(), "was kept") {
t.Fatalf("a spare that could not be read: %v", err)
}
if len(s.deleted) != 0 {
t.Fatalf("deleted %v without knowing what a kept index names", s.deleted)
}
}
// The spare list is read whole before the sweep: a kept index that cannot be read is an error before
// anything is deleted, and a repository not read before keeps its index.
func TestASpareListIsReadWholeBeforeAnythingGoes(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
}))
t.Cleanup(server.Close)
store := Store{Address: strings.TrimPrefix(server.URL, "http://")}
kept := catalogue.ArtifactStoreScheme + "web/base@" + digestN(2)
old := catalogue.ArtifactStoreScheme + "web/base@" + digestN(1)
if _, err := store.SpareKeptIndexes(context.Background(), []string{kept}, []string{old}); err == nil {
t.Fatal("a kept index the store would not answer for was taken as naming nothing")
}
s := &indexStore{indexes: map[string][]string{digestN(1): {digestN(11)}}, images: map[string]bool{digestN(11): true}}
good := s.serve(t)
spare, err := good.SpareKeptIndexes(context.Background(), nil, []string{catalogue.ArtifactStoreScheme + "other/repo@" + digestN(3)})
if err != nil {
t.Fatal(err)
}
good.Spare = spare
if err := good.LetGo(context.Background(), old); err == nil || len(s.deleted) != 0 {
t.Fatalf("an index in a repository not read before the sweep was let go of: %v, deleted %v", err, s.deleted)
}
}
@@ -1,325 +0,0 @@
// Package registry speaks to the mesh's artifact store over its own door.
//
// Only what the mesh needs that nothing else does: letting go of something it put there
// (novox/hq ADR 0189, issue 108), and holding every archive it keeps by a manifest so the store's
// own collector does not take it (novox/hq issue 253). Pushing is the builder's, through the container runtime; reading
// is every machine's, through its runtime. This is the one operation that belongs to the thing
// holding the records, because it is the only one that is a decision rather than a transfer.
//
// Copied from mesh-controller internal/artifacts at e04846e9 (novox/hq ADR 0571 clause 2): the seat mesh-artifact-sweep lets
// go of what it no longer keeps exactly as the controller's sweep did; the controller's copy goes with its lease.
package registry
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// Store is the artifact store at an address, as this machine reaches it.
type Store struct {
// Address is `host:port` — the store as the caller reaches it now, composed and never
// recorded (novox/hq 04-ISSUES/102).
Address string
// HTTP is the client used; nil is a client with a modest timeout.
HTTP *http.Client
// Spare says, for one repository, which manifests a kept index there names: the platform
// manifests an index being let go of must leave in place (novox/hq ADR 0257). Nil spares every
// platform manifest — an index is then let go of alone, as it was before.
Spare func(ctx context.Context, repository string) (map[string]bool, error)
}
// Gone is the answer when the store does not hold it: the outcome wanted, already true.
var Gone = errors.New("the store does not hold it")
// ErrNotOurs is a reference this sweep will not address: not the mesh's own, or naming nothing
// the store holds by digest.
//
// **A fact about the record, not about the store** (novox/hq issue 226). The two deserve opposite
// responses — skip one and go on, abandon the sweep for the other — and collapsing them into "an
// error" is how a cautious loop became one that did nothing while reporting the right number.
var ErrNotOurs = errors.New("not a reference into the mesh's artifact store")
// LetGo asks the store to drop one artifact the mesh recorded making.
//
// Takes a reference as the mesh records it — `artifact-store://<module>/<artifact>@sha256:…` for
// an image, `…/blobs/sha256:…` for an archive — because that is the identity every record uses,
// and composes the address here at the moment of use.
//
// An archive is let go of in two deletes, its holder manifest and then the blob's link (novox/hq
// issue 253); an image in one.
//
// Returns Gone when the store answers that it does not have it. That is not a failure: the sweep
// wants the artifact absent, and it is. It is distinguished from success only so a caller can say
// which of the two happened.
func (s Store) LetGo(ctx context.Context, reference string) error {
// **Strict, and deliberately** (novox/hq issue 226). Only a reference the mesh keeps in its
// own vocabulary is addressed here. `Recorded` would read `docker.io/library/registry@sha256:…`
// as the mesh's too — it cannot tell one registry host from another — so normalising belongs
// where the provenance is known, which is the sweep reading its own build records, not here
// where the only job is to refuse anything that is not plainly ours.
path, kept := catalogue.InArtifactStore(reference)
if !kept {
// Nothing the mesh put in its own store. Refused rather than attempted: composing a
// delete for a reference of unknown shape is how a sweep reaches something that is not
// the mesh's. Distinguished from a store that refuses, so a sweep skips this and goes on.
return fmt.Errorf("%w: %s", ErrNotOurs, reference)
}
if s.Address == "" {
return fmt.Errorf("this mesh has no artifact store on its network to ask about %s", reference)
}
repository, kind, digest, err := split(path)
if err != nil {
return err
}
if kind == "blobs" {
// **An archive's holder goes before the archive** (novox/hq issue 253): a manifest left
// naming the blob would keep its bytes through every collection while the record said
// collected. Gone here means the store has no such blob, so there is nothing to let go.
if err := s.letGoOfHolder(ctx, repository, digest); err != nil {
return err
}
}
if kind == "manifests" && s.Spare != nil {
// **An index's platform manifests go before the index** (novox/hq ADR 0257). Each was put
// under its own digest, so each is a manifest of the repository in its own right, and the
// store's collector keeps every manifest a repository holds: an index let go of alone frees
// no byte of the images it names. Before, not after, so a sweep cut short leaves the index
// to be asked about again, and with it the list of what is still to go.
if err := s.letGoOfPlatforms(ctx, repository, digest, reference); err != nil {
return err
}
}
return s.remove(ctx, s.url(repository, kind, digest), reference)
}
// letGoOfPlatforms deletes the manifests an index names, except those a kept index of the same
// repository also names. A manifest that is not an index names none.
func (s Store) letGoOfPlatforms(ctx context.Context, repository, digest, reference string) error {
children, err := s.Platforms(ctx, repository, digest)
if err != nil || len(children) == 0 {
// Gone: there is no index to read, and the delete that follows says so.
if errors.Is(err, Gone) {
return nil
}
return err
}
spared, err := s.Spare(ctx, repository)
if err != nil {
return fmt.Errorf("cannot say which platform manifests of %s a kept index names, so %s was kept: %w",
repository, reference, err)
}
for _, child := range children {
if spared[child] {
continue
}
if err := s.remove(ctx, s.url(repository, "manifests", child), reference+" (its "+child+")"); err != nil && !errors.Is(err, Gone) {
return err
}
}
return nil
}
// anyManifest is every manifest the store may hold, indexes included, so it answers with the
// document as it is.
var anyManifest = []string{
"application/vnd.oci.image.index.v1+json",
"application/vnd.docker.distribution.manifest.list.v2+json",
"application/vnd.oci.image.manifest.v1+json",
"application/vnd.docker.distribution.manifest.v2+json",
}
// Platforms is the manifests an index names, by digest; none for a manifest that is not an index.
// Gone when the store does not hold it.
func (s Store) Platforms(ctx context.Context, repository, digest string) ([]string, error) {
request, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url(repository, "manifests", digest), nil)
if err != nil {
return nil, err
}
request.Header.Set("Accept", strings.Join(anyManifest, ", "))
response, err := s.client().Do(request)
if err != nil {
return nil, err
}
defer response.Body.Close()
switch response.StatusCode {
case http.StatusOK:
case http.StatusNotFound:
return nil, Gone
default:
return nil, fmt.Errorf("the artifact store answered %s when asked for %s@%s", response.Status, repository, digest)
}
var document struct {
Manifests []struct {
Digest string `json:"digest"`
} `json:"manifests"`
}
if err := json.NewDecoder(io.LimitReader(response.Body, 4<<20)).Decode(&document); err != nil {
return nil, fmt.Errorf("%s@%s is not a manifest: %w", repository, digest, err)
}
var out []string
for _, m := range document.Manifests {
if strings.HasPrefix(m.Digest, "sha256:") {
out = append(out, m.Digest)
}
}
return out, nil
}
// SpareKeptIndexes is a Spare for a sweep, read whole before it starts: for every repository an
// image reference of `touching` is in, each kept manifest there and every platform manifest a kept
// index there names. Any read that fails is an error, and the sweep lets nothing go. A repository not
// read here is answered with an error, so an index in it is kept. A kept index the store no longer
// holds names nothing.
func (s Store) SpareKeptIndexes(ctx context.Context, kept, touching []string) (func(context.Context, string) (map[string]bool, error), error) {
imageIn := func(reference string) (repository, digest string, ok bool) {
path, ours := catalogue.InArtifactStore(reference)
if !ours {
return "", "", false
}
repository, kind, digest, err := split(path)
if err != nil || kind != "manifests" {
return "", "", false
}
return repository, digest, true
}
read := map[string]map[string]bool{}
for _, reference := range touching {
if repository, _, ok := imageIn(reference); ok {
read[repository] = map[string]bool{}
}
}
for _, reference := range kept {
repository, digest, ok := imageIn(reference)
spared, touched := read[repository]
if !ok || !touched {
continue
}
spared[digest] = true
children, err := s.Platforms(ctx, repository, digest)
if errors.Is(err, Gone) {
continue
}
if err != nil {
return nil, fmt.Errorf("reading the kept %s@%s: %w", repository, digest, err)
}
for _, c := range children {
spared[c] = true
}
}
return func(_ context.Context, repository string) (map[string]bool, error) {
spared, done := read[repository]
if !done {
return nil, fmt.Errorf("%s was not read before the sweep began", repository)
}
return spared, nil
}, nil
}
// remove asks the store to delete what is at url. Gone when it has no such thing.
func (s Store) remove(ctx context.Context, url, what string) error {
request, err := http.NewRequestWithContext(ctx, http.MethodDelete, url, nil)
if err != nil {
return err
}
response, err := s.client().Do(request)
if err != nil {
return err
}
defer response.Body.Close()
switch response.StatusCode {
case http.StatusAccepted, http.StatusOK, http.StatusNoContent:
return nil
case http.StatusNotFound:
return Gone
case http.StatusMethodNotAllowed:
// The registry was started without deletion enabled. Said plainly, because the remedy is
// a setting on the store's module and not anything about this artifact.
return fmt.Errorf(
"the artifact store refuses deletion: its server was started without it enabled "+
"(REGISTRY_STORAGE_DELETE_ENABLED), so nothing can be collected until the store "+
"module is applied again (novox/hq ADR 0189). Asking about %s", what)
default:
return fmt.Errorf("the artifact store answered %s for %s", response.Status, what)
}
}
// split reads a recorded path into the repository, which endpoint names the thing, and the digest.
//
// Two shapes, which are the two the mesh records: `<repository>@sha256:<hex>` is a manifest, and
// `<repository>/blobs/sha256:<hex>` is a blob.
func split(path string) (repository, kind, digest string, err error) {
if before, after, ok := strings.Cut(path, "@sha256:"); ok {
return before, "manifests", "sha256:" + after, nil
}
if before, after, ok := strings.Cut(path, "/blobs/sha256:"); ok {
return before, "blobs", "sha256:" + after, nil
}
return "", "", "", fmt.Errorf("%w: %q names nothing the store holds by digest", ErrNotOurs, path)
}
// IsIndex is whether a recorded reference is an index the store holds: one that names manifests. A
// blob, or a manifest that names none, is not. Gone when the store does not hold it.
func (s Store) IsIndex(ctx context.Context, reference string) (bool, error) {
path, ours := catalogue.InArtifactStore(reference)
if !ours {
return false, fmt.Errorf("%w: %s", ErrNotOurs, reference)
}
repository, kind, digest, err := split(path)
if err != nil || kind != "manifests" {
return false, err
}
children, err := s.Platforms(ctx, repository, digest)
return len(children) > 0, err
}
// HoldsManifest is whether the store holds the manifest a recorded image reference names.
func (s Store) HoldsManifest(ctx context.Context, reference string) (bool, error) {
path, ours := catalogue.InArtifactStore(reference)
if !ours {
return false, fmt.Errorf("%w: %s", ErrNotOurs, reference)
}
repository, kind, digest, err := split(path)
if err != nil {
return false, err
}
if kind != "manifests" {
return false, fmt.Errorf("%w: %s names a blob, not a manifest", ErrNotOurs, reference)
}
return s.has(ctx, s.url(repository, kind, digest), anyManifest...)
}
// Holds is whether the store holds what a recorded reference names: the blob of an archive, the manifest of an image.
// Gone when it does not; asks and changes nothing (ADR 0526 clause 3: a registration by hand is refused at bytes that
// are not there).
func (s Store) Holds(ctx context.Context, reference string) error {
path, ours := catalogue.InArtifactStore(reference)
if !ours {
return fmt.Errorf("%w: %s", ErrNotOurs, reference)
}
repository, kind, digest, err := split(path)
if err != nil {
return err
}
if s.Address == "" {
return fmt.Errorf("this mesh has no artifact store on its network to ask about %s", reference)
}
if kind == "blobs" {
_, err := s.blobSize(ctx, repository, digest)
return err
}
held, err := s.has(ctx, s.url(repository, kind, digest), anyManifest...)
if err != nil {
return err
}
if !held {
return Gone
}
return nil
}
@@ -1,134 +0,0 @@
package registry
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// Asking the store to let go of what the mesh no longer keeps (novox/hq ADR 0189, issue 108).
//
// A fake store records what it was asked to delete, so what is asserted is the mesh's decision
// and the shape of the request — not the registry's behaviour, which is the registry's to test.
func fakeStore(t *testing.T, answer int) (Store, *[]string) {
t.Helper()
var asked []string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodHead && strings.Contains(r.URL.Path, "/blobs/") {
// An archive's size, asked so its holder can be named (novox/hq issue 253). A store
// that does not have the thing does not have its blob either.
if answer == http.StatusNotFound {
w.WriteHeader(http.StatusNotFound)
return
}
w.Header().Set("Content-Length", "7")
w.WriteHeader(http.StatusOK)
return
}
if r.Method != http.MethodDelete {
t.Errorf("the store was asked %s %s; collecting is a delete", r.Method, r.URL.Path)
}
asked = append(asked, r.URL.Path)
w.WriteHeader(answer)
}))
t.Cleanup(server.Close)
return Store{Address: strings.TrimPrefix(server.URL, "http://")}, &asked
}
func TestAnImageAndAnArchiveAreAskedForAtTheirOwnEndpoints(t *testing.T) {
// The two shapes the mesh records: a manifest by digest, and a blob by digest. They are
// different endpoints, and asking at the wrong one answers 404 — which this would then
// record as collected, leaving the bytes on disk for ever while the record says otherwise.
store, asked := fakeStore(t, http.StatusAccepted)
ctx := context.Background()
image := catalogue.ArtifactStoreScheme + "web/app@sha256:abc123"
archive := catalogue.ArtifactStoreScheme + "web/config/blobs/sha256:def456"
if err := store.LetGo(ctx, image); err != nil {
t.Fatal(err)
}
if err := store.LetGo(ctx, archive); err != nil {
t.Fatal(err)
}
// The archive's holder goes first, then the archive (novox/hq issue 253).
_, holder := Holder("sha256:def456", 7)
want := []string{
"/v2/web/app/manifests/sha256:abc123",
"/v2/web/config/manifests/" + holder,
"/v2/web/config/blobs/sha256:def456",
}
if strings.Join(*asked, " ") != strings.Join(want, " ") {
t.Fatalf("the store was asked %v; want %v", *asked, want)
}
}
func TestAStoreThatDoesNotHaveItAnswersGone(t *testing.T) {
// The outcome wanted, already true. Told apart from success only so the sweep can say which
// happened; both are recorded, because retrying for ever is the thing to avoid.
store, _ := fakeStore(t, http.StatusNotFound)
err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/app@sha256:abc123")
if !errors.Is(err, Gone) {
t.Fatalf("a store that does not hold it answered %v, want Gone", err)
}
}
func TestAStoreWithDeletionOffSaysSoAndNamesTheRemedy(t *testing.T) {
// The registry answers 405 when it was started without deletion enabled. The remedy is a
// setting on the store's module, and saying "405" would send somebody to the wrong place.
store, _ := fakeStore(t, http.StatusMethodNotAllowed)
err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/app@sha256:abc123")
if err == nil {
t.Fatal("a store that refuses deletion was read as success")
}
if !strings.Contains(err.Error(), "REGISTRY_STORAGE_DELETE_ENABLED") {
t.Fatalf("the refusal does not name the remedy: %v", err)
}
}
func TestAReferenceThatIsNotTheMeshsOwnIsNeverAsked(t *testing.T) {
// The whole safety of the sweep is that it names only what the mesh recorded putting there.
// A reference of another shape — a vendor's image, a package version — is refused rather
// than composed into a delete somewhere that is not the mesh's store.
store, asked := fakeStore(t, http.StatusAccepted)
for _, reference := range []string{
"docker.io/library/registry@sha256:abc123",
"registry@sha256:abc123",
"1.4.2",
} {
if err := store.LetGo(context.Background(), reference); err == nil {
t.Errorf("%s was asked about; it is not a reference into the mesh's store", reference)
}
}
if len(*asked) != 0 {
t.Fatalf("the store was asked about %v", *asked)
}
}
// A reference this sweep will not address says so as ErrNotOurs, which is a fact about the
// record and not about the store (novox/hq issue 226).
//
// The sweep skips one and abandons itself for the other, so they cannot be the same error. The
// first live run met a reference recorded with the store's old address, read the refusal as "the
// store refuses everything", and collected none of the 1681 it had found.
func TestAReferenceThisSweepWillNotAddressIsToldApartFromAStoreRefusing(t *testing.T) {
store, asked := fakeStore(t, http.StatusAccepted)
for _, reference := range []string{
"docker.io/library/registry@sha256:abc123",
"127.0.0.1:5100/mesh-tools/build@sha256:abc123",
"1.4.2",
} {
err := store.LetGo(context.Background(), reference)
if !errors.Is(err, ErrNotOurs) {
t.Errorf("%s answered %v; a sweep must be able to skip it and go on", reference, err)
}
}
if len(*asked) != 0 {
t.Fatalf("the store was asked about %v", *asked)
}
}
+11 -46
View File
@@ -1,52 +1,26 @@
{
"module": "artifact-store-tools",
"version": "1",
"claims": [
{
"name": "mesh-artifact-sweep",
"scope": "mesh",
"serves": [
"artifacts",
"held",
"collect",
"mirrors",
"images"
]
}
],
"requires": [
"artifact-store"
],
"invokes": [
"seat:mesh-hand-acts.hand-act",
"seat:mesh-controller.artifacts",
"seat:mesh-controller.collect",
"mesh-catalog.catalog_modules",
"mesh-catalog.catalog_module"
],
"reads": [
"mesh-delivery-assigner.wants",
"mesh-delivery-builder.builds",
"mesh-delivery-builder.build-log",
"mesh-delivery-indexer.manifests",
"node-engine.have",
"mesh-controller.lease"
],
"state": [
"findings"
],
"tools": [
"artifact_store_repositories",
"artifact_store_usage",
"artifact_store_references",
"artifact_store_collect"
],
"config": {
"older-store-names": {
"type": "string",
"description": "The names the artifact store had before, as host names or host:port separated by spaces, or none when it had no other. artifact_store_references reads every definition under these as well as under the store's current name, so an image a definition pins under one is said to be named by a definition (novox/hq issue 323).",
"pattern": "none|[A-Za-z0-9.:-]+( [A-Za-z0-9.:-]+)*",
"default": "none"
}
},
"config": {
"older-store-names": {
"type": "string",
"description": "The names the artifact store had before, as host names or host:port separated by spaces, or none when it had no other. artifact_store_references reads every definition under these as well as under the store's current name, so an image a definition pins under one is said to be named by a definition (novox/hq issue 323).",
"pattern": "none|[A-Za-z0-9.:-]+( [A-Za-z0-9.:-]+)*",
"default": "none"
}
},
"resources": [
{
"id": "state",
@@ -62,14 +36,6 @@
"mode": "0600",
"trusted": false,
"template": "files/store-names"
},
{
"id": "store-address",
"type": "file",
"path": "${dir:state}/store-address",
"mode": "0600",
"trusted": false,
"template": "files/store-address"
}
],
"build": {
@@ -86,8 +52,7 @@
],
"env": {
"MESH_ARTIFACT_STORE_CONTAINER": "mesh-registry",
"MESH_ARTIFACT_STORE_NAMES_FILE": "${dir:state}/store-names",
"MESH_ARTIFACT_STORE_ADDRESS_FILE": "${dir:state}/store-address"
"MESH_ARTIFACT_STORE_NAMES_FILE": "${dir:state}/store-names"
}
},
{
+2 -11
View File
@@ -1,14 +1,5 @@
module asuszephyrusg14
go 1.26.0
go 1.22
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915
require (
github.com/klauspost/compress v1.20.0 // indirect
github.com/nats-io/nats.go v1.54.0 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
)
require git.novox.be/novox/mesh-sdk/go v0.1.6
+2 -14
View File
@@ -1,14 +1,2 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
git.novox.be/novox/mesh-sdk/go v0.1.6 h1:9qzdYONYbJdWcu6sxQcq9v1LI0JxcfkiKYkMUzJSkVQ=
git.novox.be/novox/mesh-sdk/go v0.1.6/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
+2 -11
View File
@@ -1,14 +1,5 @@
module avahi
go 1.26.0
go 1.22
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915
require (
github.com/klauspost/compress v1.20.0 // indirect
github.com/nats-io/nats.go v1.54.0 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
)
require git.novox.be/novox/mesh-sdk/go v0.1.6
+2 -14
View File
@@ -1,14 +1,2 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
git.novox.be/novox/mesh-sdk/go v0.1.6 h1:9qzdYONYbJdWcu6sxQcq9v1LI0JxcfkiKYkMUzJSkVQ=
git.novox.be/novox/mesh-sdk/go v0.1.6/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
+2 -11
View File
@@ -1,14 +1,5 @@
module blueman
go 1.26.0
go 1.22
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915
require (
github.com/klauspost/compress v1.20.0 // indirect
github.com/nats-io/nats.go v1.54.0 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
)
require git.novox.be/novox/mesh-sdk/go v0.1.7
+2 -14
View File
@@ -1,14 +1,2 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w=
git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
+2 -11
View File
@@ -1,14 +1,5 @@
module bluetooth
go 1.26.0
go 1.22
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915
require (
github.com/klauspost/compress v1.20.0 // indirect
github.com/nats-io/nats.go v1.54.0 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
)
require git.novox.be/novox/mesh-sdk/go v0.1.6
+2 -14
View File
@@ -1,14 +1,2 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
git.novox.be/novox/mesh-sdk/go v0.1.6 h1:9qzdYONYbJdWcu6sxQcq9v1LI0JxcfkiKYkMUzJSkVQ=
git.novox.be/novox/mesh-sdk/go v0.1.6/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
+1 -1
View File
@@ -3,7 +3,7 @@ module brother
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+4 -2
View File
@@ -1,5 +1,7 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef h1:xy3wuGWJpqMAXYdXolocYG6w9xnWqBltVUffoaziIKo=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f h1:7J/9jOfSfjfYpf8MOj96HwQRViQUkBNMb71RHP2mjn8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
+1 -4
View File
@@ -8,10 +8,7 @@
"provides": [
{
"name": "rootless-image-build",
"reach": "machine",
"credential": {
"form": "none"
}
"reach": "machine"
}
],
"resources": [
@@ -7,7 +7,6 @@ import (
"encoding/json"
"os"
"path/filepath"
"slices"
"strings"
"testing"
)
@@ -261,25 +260,16 @@ func TestAHomeItemIsImportedAndTheStaleOnesAreNamed(t *testing.T) {
}
}
// The manifest lists exactly the tools the bundle serves: a tool missing from it is never announced. It invokes the
// hand-act seat's warranted and no verb of the controller, and the warrant is asked of that seat (ADR 0565 clause 11).
// The manifest lists exactly the tools the bundle serves: a tool missing from it is never announced.
func TestTheManifestListsEveryToolServed(t *testing.T) {
raw, err := os.ReadFile("../../module.json")
if err != nil {
t.Fatal(err)
}
var m struct {
Tools []string `json:"tools"`
Invokes []string `json:"invokes"`
Tools []string `json:"tools"`
}
_ = json.Unmarshal(raw, &m)
if !slices.Equal(m.Invokes, []string{"seat:mesh-hand-acts.warranted"}) {
t.Errorf("invokes = %v, want the hand-act seat's warranted alone", m.Invokes)
}
if src, _ := os.ReadFile("proposals.go"); !strings.Contains(string(src), `"seat:mesh-hand-acts.warranted"`) ||
strings.Contains(string(src), "seat:mesh-controller.warranted") {
t.Error("recordWarrant does not ask the hand-act seat alone")
}
listed := map[string]bool{}
for _, n := range m.Tools {
listed[n] = true
+1 -55
View File
@@ -24,8 +24,6 @@ import (
"os"
"path/filepath"
"strings"
"git.novox.be/novox/mesh-sdk/go/asks"
)
// ForgeProvision is the provision whose credential this is, and the file this module's manifest has it delivered to.
@@ -34,59 +32,7 @@ const (
forgeSecret = "gitea-read.secret"
)
func (p Paths) forgeSecret() string {
if p.Forge != "" {
return p.Forge
}
return filepath.Join(p.State, forgeSecret)
}
// ForgeHealth is what this module says to the operator of its forge credential (the sweep of 2026-10-30, issue 774:
// the file was root's, 0600, and the runtime, the operator's account, could not read it; said only in the journal,
// some 870 times a machine). raised is the reason last raised, "" when none.
type ForgeHealth struct {
Node string
raised string
looked bool
}
// forgeConditionKey is the condition's key: its last part is its kind.
func (f *ForgeHealth) key() string { return "claude-code.machine." + f.Node + ".forge-credential" }
// Look is the condition to say after one placing of the credential, nil when nothing changed: raised on a failure, again
// when its reason changes, cleared when it places again; on the first look a success clears what a run before raised.
func (f *ForgeHealth) Look(err error) *asks.Condition {
first := !f.looked
f.looked = true
subject := asks.Scope{Scope: "machine", ID: f.Node, Machine: f.Node}
if err == nil {
if f.raised == "" && !first {
return nil
}
f.raised = ""
return &asks.Condition{Key: f.key(), Subject: subject, State: asks.StateCleared}
}
why := err.Error()
if why == f.raised {
return nil
}
f.raised = why
needs := "read claude-code's journal on " + f.Node + " and the owner of its forge credential file"
return &asks.Condition{Key: f.key(), Severity: asks.SeverityWarning, State: asks.StateRaised, Subject: subject,
Headline: "Agents on " + f.Node + " cannot use the forge credential",
Summary: why, Needs: needs,
Explanation: asks.NeedsYou + needs + ". " + sentence(why) + " Until it is placed, an agent there cannot " +
"clone or fetch the mesh's private repositories.",
Resolved: "The forge's read credential is placed again on " + f.Node + "."}
}
func sentence(s string) string {
s = strings.TrimSpace(s)
if s == "" || strings.HasSuffix(s, ".") {
return s
}
return strings.ToUpper(s[:1]) + s[1:] + "."
}
func (p Paths) forgeSecret() string { return filepath.Join(p.State, forgeSecret) }
// forgeCredentials is where the agent's git reads the credential: under the agent's home, in a directory of the
// mesh's own.
@@ -2,7 +2,6 @@ package main
import (
"encoding/json"
"errors"
"net/http"
"net/http/cgi"
"net/http/httptest"
@@ -11,8 +10,6 @@ import (
"path/filepath"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/asks"
)
const agentToken = "0123456789abcdef0123456789abcdef01234567"
@@ -263,72 +260,3 @@ func TestForgeCredentialLinesAreReadStrictly(t *testing.T) {
}
}
}
// The forge credential's file is named by a word of the bundle, so the runtime's account owns it and reads it (the
// sweep of 2026-10-30: root's 0600 file, permission denied ~870 times a machine), and a failure to place it is said to
// the operator as a condition, raised once per reason, cleared when it places again, and cleared at a first good look.
func TestTheForgeCredentialIsTheRuntimesAndItsFailureIsAConditionOfTheModule(t *testing.T) {
raw, err := os.ReadFile("../../module.json")
if err != nil {
t.Fatal(err)
}
var m struct {
Secrets map[string]string `json:"secrets"`
Build struct {
Artifacts []struct {
Kind string `json:"kind"`
Env map[string]string `json:"env"`
} `json:"artifacts"`
} `json:"build"`
}
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
named := false
for _, a := range m.Build.Artifacts {
if a.Kind == "bundle" && a.Env["MESH_CLAUDE_CODE_FORGE"] == m.Secrets[ForgeProvision] && m.Secrets[ForgeProvision] != "" {
named = true
}
}
if !named {
t.Errorf("no word of the bundle names the forge credential's file %q: the runtime cannot read it", m.Secrets[ForgeProvision])
}
p, _ := PathsFrom(envMap{"MESH_CLAUDE_CODE_STATE": "/s", "MESH_CLAUDE_CODE_FORGE": "/s/forge"}.get)
if p.forgeSecret() != "/s/forge" {
t.Errorf("the word is not read: %s", p.forgeSecret())
}
f := &ForgeHealth{Node: "anchor"}
denied := errors.New("the forge's read credential cannot be read: open /s/gitea-read.secret: permission denied")
for i, row := range []struct {
err error
want string // "" nothing said, else the state said
}{
{nil, "cleared"}, // a first good look clears what a run before raised
{nil, ""},
{denied, "raised"},
{denied, ""},
{errors.New("the forge credential could not be written: disk full"), "raised"},
{nil, "cleared"},
{nil, ""},
} {
c := f.Look(row.err)
got := ""
if c != nil {
got = c.State
if _, err := asks.EncodeCondition(*c); err != nil {
t.Errorf("look %d: the condition is refused: %v", i, err)
}
if c.Key != "claude-code.machine.anchor.forge-credential" {
t.Errorf("look %d: key %s", i, c.Key)
}
}
if got != row.want {
t.Errorf("look %d: said %q, want %q", i, got, row.want)
}
}
}
type envMap map[string]string
func (e envMap) get(k string) string { return e[k] }
+1 -24
View File
@@ -22,7 +22,6 @@ import (
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/asks"
)
func say(format string, args ...any) {
@@ -178,19 +177,6 @@ func (s memoryStateOf) Delete(key string) error { return s.s.Delete(key) }
// memoryCarrier carries this node's memory folders (ADR 0343); nil until the long-running half starts it.
var memoryCarrier *MemoryCarrier
// sayCondition says a condition on the operator-channel seat under this module's name (novox/hq ADR 0437 §9).
func sayCondition(c asks.Condition) error {
subject, err := asks.ConditionSubject("claude-code")
if err != nil {
return err
}
raw, err := asks.EncodeCondition(c)
if err != nil {
return err
}
return stdio.SeatPublish(subject, json.RawMessage(raw), "")
}
// notifyOperator tells the operator through the operator channel's notify verb (ADR 0253).
func notifyOperator(args map[string]any) error {
_, err := ask("seat:operator-channel.notify", args)
@@ -613,21 +599,12 @@ func run(p Paths, view *ServerView, configView *ConfigView) {
// The forge's read credential (forge.go, ADR 0365 §4): placed as held, and put back when git or a person changed
// it; the sessions' environment names it only while one is held.
forgeHeld := ForgeCredentialHeld(p)
forgeHealth := &ForgeHealth{Node: p.Node}
placeForge := func() {
line, err := PlaceForgeCredential(p)
if err != nil {
if line, err := PlaceForgeCredential(p); err != nil {
say("%v", err)
} else if line != "" {
say("%s", line)
}
// Said to the operator as a condition, raised once per reason and cleared when it places again.
if c := forgeHealth.Look(err); c != nil {
if serr := sayCondition(*c); serr != nil {
say("the forge credential's condition could not be said (%s): %v", c.State, serr)
forgeHealth = &ForgeHealth{Node: p.Node} // said again at the next look
}
}
if held := ForgeCredentialHeld(p); held != forgeHeld {
forgeHeld = held
if _, err := RenderNow(p, writeManaged); err != nil {
+2 -5
View File
@@ -54,17 +54,14 @@ func Fingerprint(s string) string {
// AgentHome and OperatorHome are the runtime's words as given, read to judge whose home the home scope writes
// (novox/hq issue 340).
type Paths struct {
State, Facts, Settings, Home, Node string
// Forge is the file the forge's read credential arrives in (MESH_CLAUDE_CODE_FORGE): named by a word of the
// bundle, so the runtime's account owns it and can read it (catalogue givenTo); empty is gitea-read.secret in State.
Forge string
State, Facts, Settings, Home, Node string
Agent, Operator, AgentHome, OperatorHome string
}
// PathsFrom reads them, or answers false outside a launch.
func PathsFrom(env func(string) string) (Paths, bool) {
p := Paths{State: env("MESH_CLAUDE_CODE_STATE"), Facts: env("MESH_CLAUDE_CODE_FACTS"),
Settings: env("MESH_CLAUDE_CODE_SETTINGS"), Forge: env("MESH_CLAUDE_CODE_FORGE"), Home: env("MESH_AGENT_HOME"), Node: env("MESH_NODE"),
Settings: env("MESH_CLAUDE_CODE_SETTINGS"), Home: env("MESH_AGENT_HOME"), Node: env("MESH_NODE"),
Agent: env("MESH_AGENT_ACCOUNT"), Operator: env("MESH_OPERATOR_ACCOUNT"),
AgentHome: env("MESH_AGENT_HOME"), OperatorHome: env("MESH_OPERATOR_HOME")}
if (p.Home == "") != (p.Agent == "") {
@@ -696,7 +696,7 @@ type ProposalStore interface {
Get(key string) (json.RawMessage, bool, error)
}
// Recorder records a warrant in the hand-act log of the seat mesh-hand-acts: the controller reads the warrant from the
// Recorder records a warrant in the controller's hand-act log: the controller reads the warrant from the
// router's own record, never from this module.
type Recorder func(ask string) error
@@ -708,9 +708,9 @@ func (sdkAskBus) Publish(subject string, body any, id string) error {
func (sdkAskBus) Record(id string) (*stdio.StateEntry, error) { return stdio.SeatRecord("", id) }
func (sdkAskBus) Kinds() ([]stdio.KindHeld, error) { return stdio.SeatKinds() }
// recordWarrant asks the seat mesh-hand-acts (ADR 0565 clause 11) to record a warrant this module acted on.
// recordWarrant asks the controller to record a warrant this module acted on.
func recordWarrant(ask string) error {
_, err := stdio.Ask("seat:mesh-hand-acts.warranted", map[string]any{"asker": Module, "ask": ask})
_, err := stdio.Ask("seat:mesh-controller.warranted", map[string]any{"asker": Module, "ask": ask})
return err
}
+2 -11
View File
@@ -1,16 +1,7 @@
module claude-code
go 1.26.0
go 1.25.0
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101085317-790d1c82d027
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44
require gopkg.in/yaml.v3 v3.0.1
require (
github.com/klauspost/compress v1.20.0 // indirect
github.com/nats-io/nats.go v1.54.0 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
)
+8 -14
View File
@@ -1,17 +1,11 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101085317-790d1c82d027 h1:a33dsavoaqllh40eVICbhxi6Kero1EGSjt1V+2fa2Hk=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101085317-790d1c82d027/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
git.novox.be/novox/mesh-sdk/go v0.1.13 h1:Su4JYpZ+zhNovkGA2DgxiZdcuHpjw2tPJzc/7PljhXg=
git.novox.be/novox/mesh-sdk/go v0.1.13/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013141520-9588871d6629 h1:7TYFV1Q6Mcz+lsLP/sds87tysET4rKCn3rRIQV8gNmA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013141520-9588871d6629/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013145622-f4745447ba6d h1:HEJyqorKktrkJPeeJdwEo0kdrJjH3fm/YVS9MGftCqY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013145622-f4745447ba6d/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44 h1:r5xIo5qWOXQdCSMxvqJy8kV4U+P/QWXFK6971Ha6wU8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+1 -5
View File
@@ -22,9 +22,6 @@
"uses": [
"operator-channel"
],
"invokes": [
"seat:mesh-hand-acts.warranted"
],
"state": [
"servers",
"holdings",
@@ -239,8 +236,7 @@
"env": {
"MESH_CLAUDE_CODE_STATE": "${dir:state}",
"MESH_CLAUDE_CODE_FACTS": "${dir:state}/facts.json",
"MESH_CLAUDE_CODE_SETTINGS": "${dir:state}/settings.json",
"MESH_CLAUDE_CODE_FORGE": "${dir:state}/gitea-read.secret"
"MESH_CLAUDE_CODE_SETTINGS": "${dir:state}/settings.json"
}
},
{
+4 -10
View File
@@ -1,9 +1,9 @@
module claude-licence-manager
go 1.26.0
go 1.25.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261012082113-9747f3e1b1ac
github.com/jackc/pgx/v5 v5.11.0
)
@@ -11,12 +11,6 @@ require (
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/klauspost/compress v1.20.0 // indirect
github.com/nats-io/nats.go v1.54.0 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sync v0.23.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/text v0.42.0 // indirect
golang.org/x/sync v0.17.0 // indirect
golang.org/x/text v0.29.0 // indirect
)
+6 -18
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261012082113-9747f3e1b1ac h1:jnZ/n+fFMo7VQPNNXK1gjStHIJPw3SPbzPxsf/PpZqw=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261012082113-9747f3e1b1ac/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -11,14 +11,6 @@ github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg=
github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -26,14 +18,10 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+2 -11
View File
@@ -1,14 +1,5 @@
module clipmenu
go 1.26.0
go 1.22
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915
require (
github.com/klauspost/compress v1.20.0 // indirect
github.com/nats-io/nats.go v1.54.0 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
)
require git.novox.be/novox/mesh-sdk/go v0.1.7
+2 -14
View File
@@ -1,14 +1,2 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w=
git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
+2 -11
View File
@@ -1,14 +1,5 @@
module cups
go 1.26.0
go 1.22
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915
require (
github.com/klauspost/compress v1.20.0 // indirect
github.com/nats-io/nats.go v1.54.0 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
)
require git.novox.be/novox/mesh-sdk/go v0.1.6
+2 -14
View File
@@ -1,14 +1,2 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
git.novox.be/novox/mesh-sdk/go v0.1.6 h1:9qzdYONYbJdWcu6sxQcq9v1LI0JxcfkiKYkMUzJSkVQ=
git.novox.be/novox/mesh-sdk/go v0.1.6/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
+2 -11
View File
@@ -1,17 +1,8 @@
module dbus
go 1.26.0
go 1.22
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915
git.novox.be/novox/mesh-sdk/go v0.1.7
github.com/godbus/dbus/v5 v5.1.0
)
require (
github.com/klauspost/compress v1.20.0 // indirect
github.com/nats-io/nats.go v1.54.0 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
)
+2 -14
View File
@@ -1,16 +1,4 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w=
git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
+2 -11
View File
@@ -1,14 +1,5 @@
module deskchannel
go 1.26.0
go 1.22
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915
require (
github.com/klauspost/compress v1.20.0 // indirect
github.com/nats-io/nats.go v1.54.0 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
)
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44
+8 -14
View File
@@ -1,14 +1,8 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
git.novox.be/novox/mesh-sdk/go v0.1.10 h1:EHhnPRrkw7/CSQAqRsxYnTvlTZ3dI02IPO6WSmVX80o=
git.novox.be/novox/mesh-sdk/go v0.1.10/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013141520-9588871d6629 h1:7TYFV1Q6Mcz+lsLP/sds87tysET4rKCn3rRIQV8gNmA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013141520-9588871d6629/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013145622-f4745447ba6d h1:HEJyqorKktrkJPeeJdwEo0kdrJjH3fm/YVS9MGftCqY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013145622-f4745447ba6d/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44 h1:r5xIo5qWOXQdCSMxvqJy8kV4U+P/QWXFK6971Ha6wU8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
-2
View File
@@ -1,2 +0,0 @@
/disk-load
/cmd/disk-load/disk-load
-123
View File
@@ -1,123 +0,0 @@
package main
// Where disk-load learns the data items it measures (novox/hq ADR 0577 clause 2): the node-engine's state data
// (delivery.EngineDataBucket), one key `<node>.<module>` per module placed on this node that declares data items, as the
// engine composed them. disk-load watches its own node's keys and composes nothing: it reads no resolved values, no
// manifests and no placement, so a manifest its mesh-sdk cannot judge never silences it, and a round reads one watched
// map instead of several hundred keys over the tunnel (issue 1955).
import (
"context"
"errors"
"fmt"
"sync"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// Watched is this node's data items as the engine's state data holds them, by module, kept from a watch of its keys.
type Watched struct {
Node string
mu sync.Mutex
byKey map[string]string // key to its module, for a delete that carries no value
byModule map[string][]catalogue.PlacedDataItem
ready bool
}
// ErrNotWatchedYet is a round asked before the watch handed over the engine's current keys: no item is known yet, and
// none is taken for gone.
var ErrNotWatchedYet = errors.New("the node-engine's state " + delivery.EngineDataBucket + " is not watched yet: " +
"no data item of this machine is known")
// Take keeps one change of a key and answers the module it is of and whether its items changed.
func (w *Watched) Take(ch stdio.StateChange) (string, bool, error) {
w.mu.Lock()
defer w.mu.Unlock()
if w.byKey == nil {
w.byKey, w.byModule = map[string]string{}, map[string][]catalogue.PlacedDataItem{}
}
if ch.Op != stdio.OpPut {
module, known := w.byKey[ch.Key]
if !known {
return "", false, nil
}
delete(w.byKey, ch.Key)
delete(w.byModule, module)
return module, true, nil
}
d, err := delivery.DecodeEngineData(ch.Value)
if err != nil {
return "", false, fmt.Errorf("%s: %w", ch.Key, err)
}
if d.Node != w.Node {
return "", false, fmt.Errorf("%s names the machine %q, and this is %s", ch.Key, d.Node, w.Node)
}
for _, it := range d.Items {
if it.Module != d.Module {
return "", false, fmt.Errorf("%s holds an item of %q, and is %s's", ch.Key, it.Module, d.Module)
}
}
changed := !sameItems(w.byModule[d.Module], d.Items)
w.byKey[ch.Key] = d.Module
w.byModule[d.Module] = d.Items
return d.Module, changed, nil
}
// Ready says the watch handed over every current key: from now on an item missing is an item gone.
func (w *Watched) Ready() {
w.mu.Lock()
w.ready = true
w.mu.Unlock()
}
// All is every module's items, as the Keeper reads them; refused until the watch is ready.
func (w *Watched) All(context.Context) (map[string][]catalogue.PlacedDataItem, error) {
w.mu.Lock()
defer w.mu.Unlock()
if !w.ready {
return nil, ErrNotWatchedYet
}
out := make(map[string][]catalogue.PlacedDataItem, len(w.byModule))
for m, its := range w.byModule {
out[m] = append([]catalogue.PlacedDataItem(nil), its...)
}
return out, nil
}
func sameItems(a, b []catalogue.PlacedDataItem) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}
// Follow watches this node's keys of the engine's state data, and asks the keeper to measure a module whose items changed
// after the watch began; it returns once the current keys were handed over (stdio.KeptState.Follow).
func (w *Watched) Follow(k *Keeper) error {
pattern, err := delivery.EngineDataNodeKeys(w.Node)
if err != nil {
return err
}
stdio.State(delivery.EngineDataRead).Follow(pattern, func(ch stdio.StateChange) error {
module, changed, err := w.Take(ch)
if err != nil || !changed || ch.Current {
return err
}
k.Due(module)
return nil
})
w.Ready()
// The handed-over set is complete now: a module whose key went while the watch was down is retired once.
k.RetireUnkeyed(context.Background())
// A round refused while the watch caught up runs now, not at the next hour (the delivery's readings within minutes).
go k.RoundIfRefused(context.Background())
return nil
}
@@ -1,190 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"os"
"strings"
"testing"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
)
// ADR 0577 clauses 2, 3 (and the disk-load check of clause 5): the items measured come from the node-engine's keys of
// this machine as the watch hands them over, and none is known (none taken for gone) before the watch caught up; a key
// that changes after the watch began has its module measured again, and one deleted is measured no more; an item whose
// measurement fails is a failed item of the round, which measures the rest, and the round's key says so.
func TestTheItemsComeFromTheEnginesKeysAndARoundSaysHowFarItGot(t *testing.T) {
key := func(module string) string { k, _ := delivery.EngineDataKey("ace", module); return k }
value := func(module string, items ...catalogue.PlacedDataItem) json.RawMessage {
raw, err := delivery.EncodeEngineData(delivery.EngineData{Node: "ace", Module: module, Digest: "sha256:" + module, Items: items})
if err != nil {
t.Fatal(err)
}
return raw
}
pg := catalogue.PlacedDataItem{Module: "pg", ID: "data", Class: "valuable", Path: "/srv/pg", Measure: catalogue.MeasureShallow}
gone := catalogue.PlacedDataItem{Module: "pg", ID: "wal", Class: "valuable", Path: "/srv/missing", Measure: catalogue.MeasureShallow}
web := catalogue.PlacedDataItem{Module: "web", ID: "uploads", Class: "valuable", Path: "/srv/web", Measure: catalogue.MeasureShallow}
measured := map[string]int{}
run := func(_ context.Context, name string, args ...string) (string, error) {
switch {
case name == "test" && args[1] == "/srv/missing":
return "", errors.New("no such file")
case name == "test":
return "", nil
case name == "bash" && strings.Contains(args[1], "-maxdepth 1"):
measured[strings.Split(args[1], "'")[1]]++
return "1760000000 2\n", nil
}
return "", os.ErrNotExist
}
now := time.Date(2026, 11, 1, 0, 30, 0, 0, time.UTC)
readings, rounds := &memStore{m: map[string]json.RawMessage{}}, &memStore{m: map[string]json.RawMessage{}}
w := &Watched{Node: "ace"}
var said []string
k := &Keeper{Node: "ace", Items: w.All, Store: readings, Rounds: rounds,
Say: func(f string, a ...any) { said = append(said, f) },
Measurer: Measurer{Run: run, Now: func() time.Time { return now }}}
round := func() Round {
var r Round
raw, _ := rounds.Get("ace")
if err := json.Unmarshal(raw, &r); err != nil {
t.Fatalf("the round's key: %v", err)
}
return r
}
// Before the watch caught up nothing is known, and a round says so in its key rather than retiring every reading.
if _, _, err := k.Round(context.Background(), false); !errors.Is(err, ErrNotWatchedYet) {
t.Fatalf("a round before the watch: %v", err)
}
if r := round(); r.Ended == nil || !strings.Contains(r.Error, "not watched yet") {
t.Fatalf("the round before the watch is kept as %+v", r)
}
refusedAt := round().Started
for _, step := range []struct {
name string
change stdio.StateChange
module string
changed bool
err bool
}{
{"a current key", stdio.StateChange{Key: key("pg"), Op: stdio.OpPut, Value: value("pg", pg, gone), Current: true}, "pg", true, false},
{"another current key", stdio.StateChange{Key: key("web"), Op: stdio.OpPut, Value: value("web", web), Current: true}, "web", true, false},
{"the same items again", stdio.StateChange{Key: key("web"), Op: stdio.OpPut, Value: value("web", web)}, "web", false, false},
{"another machine's items", stdio.StateChange{Key: key("x"), Op: stdio.OpPut, Value: json.RawMessage(
`{"v":1,"node":"g14","module":"x","digest":"d","items":[]}`)}, "", false, true},
{"a delete of a key never seen", stdio.StateChange{Key: key("y"), Op: stdio.OpDelete}, "", false, false},
} {
module, changed, err := w.Take(step.change)
if module != step.module || changed != step.changed || (err != nil) != step.err {
t.Errorf("%s: taken as %q, changed %v, %v", step.name, module, changed, err)
}
}
w.Ready()
// The round refused before the watch caught up runs once it has (Watched.Follow), not at the next hour: it measures
// every item it was handed; the missing one fails, and the round goes on and says it.
now = now.Add(time.Minute)
if !k.RoundIfRefused(context.Background()) || k.RoundIfRefused(context.Background()) {
t.Fatal("the round refused before the watch was not run once when the watch caught up")
}
r := round()
if !r.Started.After(refusedAt) {
t.Fatalf("the round's key still says the refused round: %+v", r)
}
if r.Ended == nil || r.Items != 3 || r.Measured != 3 || len(r.Failed) != 1 || r.Failed[0].Item != "pg.wal" ||
!strings.Contains(r.Failed[0].Why, "does not exist") || measured["/srv/pg"] != 1 || measured["/srv/web"] != 1 {
t.Fatalf("the round's key says %+v; measured %v", r, measured)
}
if !strings.Contains(strings.Join(said, "\n"), "the round ended") || !strings.Contains(strings.Join(said, "\n"), "starts") {
t.Errorf("the journal said %v", said)
}
// A key that changes after the watch began has its module measured again, at once, by Work.
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go k.Work(ctx)
moved := web
moved.Path = "/srv/web2"
if module, changed, err := w.Take(stdio.StateChange{Key: key("web"), Op: stdio.OpPut, Value: value("web", moved)}); err != nil || !changed {
t.Fatalf("a changed key: %s %v %v", module, changed, err)
}
k.Due("web")
waitFor(t, func() bool {
raw := readingOf(readings, "ace.web.uploads")
return raw != nil && raw.Path == "/srv/web2"
})
// A key deleted: its module's items are measured no more, and its reading is retired.
if module, changed, _ := w.Take(stdio.StateChange{Key: key("web"), Op: stdio.OpDelete}); module != "web" || !changed {
t.Fatalf("a deleted key was taken as %q, %v", module, changed)
}
k.Due("web")
waitFor(t, func() bool { r := readingOf(readings, "ace.web.uploads"); return r != nil && !r.RetiredAt.IsZero() })
before := measured["/srv/web2"]
if _, _, err := k.Round(context.Background(), false); err != nil {
t.Fatal(err)
}
if measured["/srv/web2"] != before || round().Items != 2 {
t.Errorf("an item whose key was deleted was measured again: %d times, round %+v", measured["/srv/web2"]-before, round())
}
// A restart: oldie's key was deleted while the watch was down, so no delete arrives. Once the watch handed over its
// keys, oldie's reading is retired and pg's kept; an empty set retires nothing (the engine has not written yet).
_ = readings.Put("ace.oldie.x", health.Reading{V: health.ReadingVersion, Node: "ace",
PlacedDataItem: catalogue.PlacedDataItem{Module: "oldie", ID: "x", Path: "/srv/oldie"}})
for _, restart := range []struct {
name string
keys []stdio.StateChange
retired bool
}{
{"no key yet", nil, false},
{"pg's key alone", []stdio.StateChange{{Key: key("pg"), Op: stdio.OpPut, Value: value("pg", pg, gone), Current: true}}, true},
} {
w2 := &Watched{Node: "ace"}
said = nil
k2 := &Keeper{Node: "ace", Items: w2.All, Store: readings, Say: k.Say, Measurer: k.Measurer}
for _, ch := range restart.keys {
if _, _, err := w2.Take(ch); err != nil {
t.Fatal(err)
}
}
w2.Ready()
k2.RetireUnkeyed(context.Background())
oldie, pgData := readingOf(readings, "ace.oldie.x"), readingOf(readings, "ace.pg.data")
if oldie == nil || pgData == nil || oldie.RetiredAt.IsZero() == restart.retired || !pgData.RetiredAt.IsZero() {
t.Errorf("%s: oldie %+v, pg %+v; oldie retired: want %v, pg never", restart.name, oldie, pgData, restart.retired)
}
if !restart.retired && !strings.Contains(strings.Join(said, "\n"), "holds no key") {
t.Errorf("%s: the skip was not said: %v", restart.name, said)
}
}
}
func readingOf(s *memStore, key string) *health.Reading {
raw, _ := s.Get(key)
if raw == nil {
return nil
}
r, err := health.DecodeReading(raw)
if err != nil {
return nil
}
return &r
}
func waitFor(t *testing.T, done func() bool) {
t.Helper()
for end := time.Now().Add(5 * time.Second); time.Now().Before(end); time.Sleep(5 * time.Millisecond) {
if done() {
return
}
}
t.Fatal("not done within 5 s")
}
-465
View File
@@ -1,465 +0,0 @@
package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
"os"
"os/exec"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// Store is the state `readings` as the keeper uses it.
type Store interface {
Get(key string) (json.RawMessage, error)
Put(key string, value any) error
Keys() ([]string, error)
}
// busStore is one of disk-load's states through the runtime: readings, or rounds.
type busStore struct{ state string }
func (b busStore) Get(key string) (json.RawMessage, error) {
e, err := stdio.State(b.state).Get(key)
if err != nil || e == nil {
return nil, err
}
return e.Value, nil
}
func (b busStore) Put(key string, v any) error {
_, err := stdio.State(b.state).Put(key, v)
return err
}
func (b busStore) Keys() ([]string, error) { return stdio.State(b.state).Keys() }
// Keeper measures the items of its node and keeps the readings.
type Keeper struct {
Node string
// Items is every data item the modules placed on this node declare, as the node-engine composed them and its state
// data holds them (Watched.All, novox/hq ADR 0577 clause 2).
Items func(ctx context.Context) (map[string][]catalogue.PlacedDataItem, error)
Measurer Measurer
Store Store
// Rounds is the state rounds, where each round says how far it got (ADR 0577 clause 3); nil keeps none.
Rounds Store
Say func(string, ...any)
mu sync.Mutex // one round at a time
// pending are the modules whose items changed since they were last measured (Due), measured by Work.
pmu sync.Mutex
pending map[string]bool
wake chan struct{}
// unwatched says a round was refused because the watch had not caught up (ErrNotWatchedYet): one runs as soon as
// it has (Watched.Follow), not an hour later.
unwatched atomic.Bool
}
// RoundIfRefused runs one round now when the last was refused for a watch not caught up, and says whether it did.
func (k *Keeper) RoundIfRefused(ctx context.Context) bool {
if !k.unwatched.CompareAndSwap(true, false) {
return false
}
_, _, _ = k.Round(ctx, false)
return true
}
// SayEvery is how many items a round measures between two lines saying how far it got.
const SayEvery = 50
// RoundVersion is the version of a round's layout.
const RoundVersion = 1
// Round is the value of a node's key of the state rounds: when its last round started and ended, how many items it
// had to measure, how many it measured, wrote and kept, and each item that failed and why. Ended is nil while it runs;
// Error says why a round measured nothing at all.
type Round struct {
V int `json:"v"`
Node string `json:"node"`
Walks bool `json:"walks,omitempty"`
Started time.Time `json:"started"`
Ended *time.Time `json:"ended,omitempty"`
Items int `json:"items"`
Measured int `json:"measured"`
Written int `json:"written"`
Kept int `json:"kept"`
Failed []FailedItem `json:"failed,omitempty"`
Error string `json:"error,omitempty"`
}
// FailedItem is one item a round could not measure or keep, `<module>.<item>`, and why.
type FailedItem struct {
Item string `json:"item"`
Why string `json:"why"`
}
// putRound writes the round to this node's key; a round that cannot be written is said, and measures on.
func (k *Keeper) putRound(r Round) {
if k.Rounds == nil {
return
}
if err := k.Rounds.Put(k.Node, r); err != nil {
k.Say("the round's outcome cannot be kept in the state %s: %v", StateRounds, err)
}
}
// Round measures every item that is not a cache, one at a time, and writes the readings that changed. An item
// measured by a walk is walked only when walks is true or its last walk is older than WalkEvery: the hourly round
// reads counters, and a walk runs at most once a day (ADR 0233 §22).
//
// A round says how far it got (novox/hq ADR 0577 clause 3): it writes its outcome to this node's key of the state rounds
// when it starts, every SayEvery items and when it ends, and says the same in the journal. An item that fails is a
// failed item of the round, never its end.
func (k *Keeper) Round(ctx context.Context, walks bool) (written, kept int, err error) {
k.mu.Lock()
defer k.mu.Unlock()
r := Round{V: RoundVersion, Node: k.Node, Walks: walks, Started: k.Measurer.Now().UTC()}
end := func() {
at := k.Measurer.Now().UTC()
r.Ended = &at
k.putRound(r)
}
k.putRound(r)
k.Say("a round of measuring the data declared here starts (walks: %v)", walks)
items, err := k.Items(ctx)
if err != nil {
if errors.Is(err, ErrNotWatchedYet) {
k.unwatched.Store(true)
}
r.Error = err.Error()
end()
k.Say("the round measured nothing: %v", err)
return 0, 0, err
}
modules := make([]string, 0, len(items))
for m := range items {
modules = append(modules, m)
for _, it := range items[m] {
if it.Class != ClassCache {
r.Items++
}
}
}
sort.Strings(modules)
var failed []string
for _, module := range modules {
for _, it := range items[module] {
if it.Class == ClassCache {
continue
}
w, said, err := k.measureOne(ctx, it, walks)
r.Measured++
switch {
case err != nil:
failed = append(failed, fmt.Sprintf("%s.%s: %v", module, it.ID, err))
r.Failed = append(r.Failed, FailedItem{Item: module + "." + it.ID, Why: err.Error()})
case said != "":
// Measured, and the measurement failed: its reading says so, and the round counts it failed.
r.Failed = append(r.Failed, FailedItem{Item: module + "." + it.ID, Why: said})
}
if err == nil && w {
written++
} else if err == nil {
kept++
}
r.Written, r.Kept = written, kept
if r.Measured%SayEvery == 0 {
k.Say("the round has measured %d of %d items: %d written, %d unchanged, %d failed", r.Measured, r.Items,
written, kept, len(r.Failed))
k.putRound(r)
}
}
}
// A round retires nothing: a reading is retired only when its module's key says so (MeasureModule), never because
// an item is missing from what the watch holds now, which may be empty or partial.
end()
k.Say("the round ended: %d of %d items measured, %d readings written, %d unchanged, %d failed", r.Measured, r.Items,
written, kept, len(r.Failed))
if len(failed) > 0 {
return written, kept, errors.New("readings not kept: " + strings.Join(failed, "; "))
}
return written, kept, nil
}
// Due asks Work to measure a module's items: its key of the engine's state data changed (novox/hq ADR 0577 clause 2).
func (k *Keeper) Due(module string) {
k.pmu.Lock()
if k.pending == nil {
k.pending, k.wake = map[string]bool{}, make(chan struct{}, 1)
}
k.pending[module] = true
wake := k.wake
k.pmu.Unlock()
select {
case wake <- struct{}{}:
default:
}
}
// Work measures each module Due names, one at a time, until ctx ends.
func (k *Keeper) Work(ctx context.Context) {
k.pmu.Lock()
if k.pending == nil {
k.pending, k.wake = map[string]bool{}, make(chan struct{}, 1)
}
wake := k.wake
k.pmu.Unlock()
for {
select {
case <-ctx.Done():
return
case <-wake:
}
for {
k.pmu.Lock()
var module string
for m := range k.pending {
module = m
break
}
delete(k.pending, module)
k.pmu.Unlock()
if module == "" {
break
}
k.MeasureModule(ctx, module)
}
}
}
// MeasureModule measures one module's items now, as its key of the engine's state data changed, and retires the
// readings of a module that no longer declares any here. A walk is not repeated within a day for an item that did not
// change (measureOne).
func (k *Keeper) MeasureModule(ctx context.Context, module string) {
k.mu.Lock()
defer k.mu.Unlock()
items, err := k.Items(ctx)
if err != nil {
k.Say("%s's data items changed and are not measured: %v", module, err)
return
}
n, err := k.retireGone(module, items[module])
if err != nil {
k.Say("%s's readings of items it no longer declares here are not retired: %v", module, err)
} else if n > 0 {
k.Say("%s no longer declares %d item(s) here: their readings are retired, measured no more", module, n)
}
if len(items[module]) == 0 {
return
}
written, kept := 0, 0
var failed []string
for _, it := range items[module] {
if it.Class == ClassCache {
continue
}
w, said, err := k.measureOne(ctx, it, false)
switch {
case err != nil:
failed = append(failed, it.ID+": "+err.Error())
case said != "":
failed = append(failed, it.ID+": "+said)
case w:
written++
default:
kept++
}
}
k.Say("%s's data items changed and were measured: %d readings written, %d unchanged, %d failed%s", module, written,
kept, len(failed), map[bool]string{true: " (" + strings.Join(failed, "; ") + ")", false: ""}[len(failed) > 0])
}
// retireGone marks retired every reading of this node of module whose item its key of the engine's state data no longer
// names — the key was deleted (the module was unassigned there, or is no longer pulled in) or names fewer items — with
// when and why, and keeps it: the record of what the module left behind, until a person deletes it (ADR 0233). It is
// asked only on a change of that module's key (novox/hq ADR 0577 clause 2), never from a round. It is cleared when the
// item is declared again (measureOne).
func (k *Keeper) retireGone(module string, declared []catalogue.PlacedDataItem) (int, error) {
return k.retire(func(r health.Reading) bool { return r.Module == module }, declared)
}
// RetireUnkeyed retires, once the watch handed over every current key, the readings of this node whose module has no
// key in that set: a key deleted while the watch was not up (disk-load restarted, the machine rebooted or slept) never
// reaches disk-load as a delete. An empty set is skipped and said: the engine has not written its keys yet, and an
// empty set would retire every reading.
func (k *Keeper) RetireUnkeyed(ctx context.Context) {
k.mu.Lock()
defer k.mu.Unlock()
items, err := k.Items(ctx)
if err != nil {
k.Say("the readings of modules with no key are not looked for: %v", err)
return
}
if len(items) == 0 {
k.Say("the node-engine's state %s holds no key of this machine yet: no reading is retired for it", delivery.EngineDataBucket)
return
}
var declared []catalogue.PlacedDataItem
for _, its := range items {
declared = append(declared, its...)
}
n, err := k.retire(func(r health.Reading) bool { _, keyed := items[r.Module]; return !keyed }, declared)
switch {
case err != nil:
k.Say("the readings of modules with no key are not all retired: %v", err)
case n > 0:
k.Say("%d reading(s) of modules with no key of this machine retired, measured no more", n)
}
}
// retire marks retired every reading of this node that of picks and declared does not name.
func (k *Keeper) retire(of func(health.Reading) bool, declared []catalogue.PlacedDataItem) (int, error) {
keys, err := k.Store.Keys()
if err != nil {
return 0, err
}
declaredNow := map[string]bool{}
for _, it := range declared {
declaredNow[health.ReadingKey(k.Node, it.Module, it.ID)] = true
}
now := k.Measurer.Now().UTC()
n := 0
for _, key := range keys {
if !strings.HasPrefix(key, k.Node+".") || declaredNow[key] {
continue
}
raw, err := k.Store.Get(key)
if err != nil || len(raw) == 0 {
continue
}
r, err := health.DecodeReading(raw)
if err != nil || r.Node != k.Node || !of(r) || !r.RetiredAt.IsZero() {
continue
}
r.RetiredAt = now
r.RetiredWhy = fmt.Sprintf("no longer declared on %s: its module was unassigned there, or is no longer pulled in", k.Node)
if err := k.Store.Put(key, r); err != nil {
return n, err
}
n++
}
return n, nil
}
// measureOne measures one item, unless it is a walk done within WalkEvery of an unchanged item, and writes its reading
// when it changed. It answers whether it wrote the reading, what the measurement said failed ("" when it did not, or
// when the item was not measured now), and an error when the reading could not be read or kept.
func (k *Keeper) measureOne(ctx context.Context, it catalogue.PlacedDataItem, walks bool) (bool, string, error) {
key := health.ReadingKey(k.Node, it.Module, it.ID)
var prev *health.Reading
raw, err := k.Store.Get(key)
if err != nil {
return false, "", err
}
if len(raw) > 0 {
if p, err := health.DecodeReading(raw); err == nil {
prev = &p
}
}
now := k.Measurer.Now()
if prev != nil && prev.RetiredAt.IsZero() && prev.PlacedDataItem == it && it.Measure == catalogue.MeasureWalk && !walks &&
prev.MeasuredAt != nil && now.Sub(*prev.MeasuredAt) < WalkEvery {
return false, "", nil
}
m := k.Measurer.Measure(ctx, it)
next, changed := health.NextReading(prev, k.Node, it, m, now)
if prev != nil && !prev.RetiredAt.IsZero() {
// Declared again: it is no longer retired.
next.RetiredAt, next.RetiredWhy, changed = time.Time{}, "", true
}
if !changed {
return false, m.Error, nil
}
return true, m.Error, k.Store.Put(key, next)
}
// MeasureNow measures one item now, walk or not, and writes its reading when it changed.
func (k *Keeper) MeasureNow(ctx context.Context, module, item string) (health.Reading, bool, error) {
k.mu.Lock()
defer k.mu.Unlock()
items, err := k.Items(ctx)
if err != nil {
return health.Reading{}, false, err
}
for _, it := range items[module] {
if it.ID != item {
continue
}
if it.Class == ClassCache {
return health.Reading{}, false, fmt.Errorf("%s.%s is a cache, which is listed and never measured", module, item)
}
w, _, err := k.measureOne(ctx, it, true)
if err != nil {
return health.Reading{}, false, err
}
raw, err := k.Store.Get(health.ReadingKey(k.Node, module, item))
if err != nil {
return health.Reading{}, w, err
}
r, err := health.DecodeReading(raw)
return r, w, err
}
return health.Reading{}, false, fmt.Errorf("%s declares no data item %q on this machine", module, item)
}
// Loop runs a round after a few minutes and then every `every`, with the walks of the night's round when it is the
// hour after the night; ctx ends it.
func (k *Keeper) Loop(ctx context.Context, every time.Duration, first time.Duration) {
select {
case <-ctx.Done():
return
case <-time.After(first):
}
for {
_, _, _ = k.Round(ctx, false) // it says its outcome itself, in the journal and its key of the state rounds
select {
case <-ctx.Done():
return
case <-time.After(every):
}
}
}
// RootRunner runs a command as root, through sudo without a prompt when this process is not root. A walk is bounded
// by WalkFor itself; the runner's own bound is a little longer.
func RootRunner(ctx context.Context, name string, args ...string) (string, error) {
ctx, cancel := context.WithTimeout(ctx, WalkFor+time.Minute)
defer cancel()
program, argv := name, args
if os.Getuid() != 0 {
program, argv = "sudo", append([]string{"-n", name}, args...)
}
var stdout, stderr bytes.Buffer
cmd := exec.CommandContext(ctx, program, argv...)
cmd.Stdout, cmd.Stderr = &stdout, &stderr
if err := cmd.Run(); err != nil {
said := strings.TrimSpace(stderr.String())
if said == "" {
said = strings.TrimSpace(stdout.String())
}
if program == "sudo" && strings.HasPrefix(said, "sudo:") {
return "", fmt.Errorf("%s needs root and the runtime's account may not run it without a prompt: %s", name, said)
}
lines := strings.Split(said, "\n")
if len(lines) > 3 {
lines = lines[len(lines)-3:]
}
if said == "" {
return "", fmt.Errorf("%s: %w", name, err)
}
return "", errors.New(strings.Join(lines, " / "))
}
return stdout.String(), nil
}
+2 -42
View File
@@ -1,59 +1,19 @@
// The disk-load module's Go bundle (novox/hq ADR 0188, ADR 0193, issue 312): a process the node's
// runtime launches and speaks MCP over stdio to. It reads the machine's storage load from /proc and
// /sys — iowait, pressure stall, each disk's busy share, throughput, IOPS, wait and queue depth, the
// filesystems' space and the processes doing the I/O — and changes nothing of the machine.
//
// It also measures the data items the modules of its node declare (ADR 0233) and keeps each reading and the item's
// seven-day peak as its own state `readings` (ADR 0571 clause 3). It learns the items from the node-engine's state data,
// watching its own node's keys, and composes nothing (ADR 0577 clause 2); each round says how far it got in its state
// `rounds` (clause 3).
// filesystems' space and the processes doing the I/O — and changes nothing.
package main
import (
"context"
"fmt"
"os"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/announce"
)
func logf(format string, args ...any) { fmt.Fprintf(os.Stderr, "[disk-load] "+format+"\n", args...) }
// Kept is every state disk-load keeps, as it announces them: the readings, readable by restic, nfs-server, mounts and
// the self-check (their manifests' `reads`; checks/readable holds it), and the rounds, its own tools' alone.
var Kept = []announce.State{{Name: StateReadings, Readable: true}, {Name: StateRounds}}
func main() {
if err := stdio.Keeps(Kept...); err != nil {
logf("%v", err)
os.Exit(1)
}
node := os.Getenv("MESH_NODE")
watched := &Watched{Node: node}
k := &Keeper{Node: node, Store: busStore{StateReadings}, Rounds: busStore{StateRounds}, Say: logf,
Measurer: Measurer{Run: RootRunner, Now: time.Now}, Items: watched.All}
if node != "" {
every := time.Hour
if d, err := time.ParseDuration(os.Getenv("MESH_DISK_MEASURE_EVERY")); err == nil && d >= time.Minute {
every = d
}
ctx := context.Background()
go k.Work(ctx)
go func() {
<-stdio.Served() // the runtime's channel first: a watch is asked through it
if err := watched.Follow(k); err != nil {
logf("this machine's keys of the node-engine's data items cannot be watched, and nothing is measured: %v", err)
return
}
logf("watching this machine's keys of the node-engine's data items")
}()
go k.Loop(ctx, every, 3*time.Minute)
}
tools := append(Tools(Here()), ReadingTools(k)...)
// An empty name serves as the module the runtime names (MESH_SERVED_MODULE): disk-load.
if err := stdio.Serve("", tools); err != nil {
if err := stdio.Serve("", Tools(Here())); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
-215
View File
@@ -1,215 +0,0 @@
package main
// How an item is measured (novox/hq ADR 0233 §22 to §25), moved here from the backup holder, which only reads the
// readings now (ADR 0571 clause 3). Never an unbounded walk of something large.
import (
"context"
"fmt"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/health"
"regexp"
"strconv"
"strings"
"time"
)
// The bounds on a walk: a walk is for small items, and one that meets a large one stops and says so.
var (
WalkFor = 10 * time.Minute
WalkFiles = 2_000_000
WalkEvery = 23 * time.Hour
)
// walkCommand sums the files under a path and finds its newest change, in one walk, bounded by time and by files.
// One line out: "<bytes> <epoch seconds> <files>", then the status of find.
func walkCommand(path string) string {
return fmt.Sprintf("timeout %d find '%s' -xdev -printf '%%y %%s %%T@\\n' 2>/dev/null | head -n %d | "+
"awk 'BEGIN{s=0;m=0;n=0} {n++; if ($1==\"f\") s+=$2; if ($3>m) m=$3} END {printf \"%%d %%.0f %%d\\n\", s, m, n}'; "+
"echo \"${PIPESTATUS[0]:-0}\"", int(WalkFor.Seconds()), path, WalkFiles)
}
// topCommand is the newest change among a path and its top-level entries, and how many there are.
func topCommand(path string) string {
return "timeout 60 find '" + path + "' -maxdepth 1 -printf '%T@\\n' 2>/dev/null | " +
"awk 'BEGIN{m=0;n=0} {n++; if ($1>m) m=$1} END {printf \"%.0f %d\\n\", m, n-1}'"
}
func epochTime(s string) *time.Time {
epoch, err := strconv.ParseInt(s, 10, 64)
if err != nil || epoch <= 0 {
return nil
}
t := time.Unix(epoch, 0).UTC()
return &t
}
// Measurer measures items through a runner that has the root the walks need.
type Measurer struct {
Run Runner
Now func() time.Time
}
func (b Measurer) exists(ctx context.Context, path string) bool {
_, err := b.Run(ctx, "test", "-e", path)
return err == nil
}
// Measure is one item, measured now, by its own method.
func (b Measurer) Measure(ctx context.Context, it catalogue.PlacedDataItem) health.Measured {
at := b.Now().UTC()
m := health.Measured{MeasuredAt: &at}
if !b.exists(ctx, it.Path) {
m.Error = it.Path + " does not exist"
zero := int64(0)
m.SizeBytes, m.Precision = &zero, "exact"
return m
}
m.Redundancy = b.redundancyOf(ctx, it.Path)
switch it.Measure {
case catalogue.MeasureDataset, catalogue.MeasureShallow:
out, err := b.Run(ctx, "bash", "-c", topCommand(it.Path))
if f := strings.Fields(out); err == nil && len(f) == 2 {
m.LastWrite = epochTime(f[0])
} else if err != nil {
m.Error = err.Error()
}
m.Precision = "no size: the newest change among its top-level entries only"
if it.Measure == catalogue.MeasureShallow {
return m
}
out, err = b.Run(ctx, "zfs", "list", "-Hp", "-o", "name,used", it.Path)
f := strings.Fields(out)
if err != nil || len(f) != 2 {
m.Precision = "no size: it is on no ZFS dataset to read one from; the newest change among its top-level entries only"
return m
}
used, err := strconv.ParseInt(f[1], 10, 64)
if err != nil {
return m
}
m.SizeBytes = &used
m.Precision = "dataset " + f[0] + ": its whole size, from the filesystem's counters; the newest change among the top-level entries"
return m
}
out, err := b.Run(ctx, "bash", "-c", walkCommand(it.Path))
if err != nil {
m.Error = err.Error()
return m
}
lines := strings.Split(strings.TrimSpace(out), "\n")
f := strings.Fields(lines[0])
if len(f) != 3 || len(lines) < 2 {
m.Error = "the measurement said " + strings.TrimSpace(out)
return m
}
size, err1 := strconv.ParseInt(f[0], 10, 64)
files, err2 := strconv.Atoi(f[2])
if err1 != nil || err2 != nil {
m.Error = "the measurement said " + strings.TrimSpace(out)
return m
}
m.SizeBytes, m.LastWrite = &size, epochTime(f[1])
m.Precision = "exact"
if status := strings.TrimSpace(lines[len(lines)-1]); status == "124" || files >= WalkFiles {
m.Precision = fmt.Sprintf("partial: measured partially — stopped after %s or %d files; the size is a lower bound, "+
"and this item wants measure: dataset or shallow", WalkFor, WalkFiles)
}
return m
}
// redundancyOf is the redundant storage a path is on: a ZFS pool's health, an md array's members, a btrfs
// filesystem's error counters. Nil for storage with no redundancy it knows how to read.
func (b Measurer) redundancyOf(ctx context.Context, path string) *health.Redundancy {
out, err := b.Run(ctx, "findmnt", "-no", "SOURCE,FSTYPE", "--target", path)
if err != nil {
return nil
}
f := strings.Fields(out)
if len(f) < 2 {
return nil
}
source, fstype := f[0], f[1]
yes, no := true, false
switch {
case fstype == "zfs":
pool, _, _ := strings.Cut(source, "/")
r := &health.Redundancy{Kind: "zfs", Where: pool}
health, err := b.Run(ctx, "zpool", "list", "-H", "-o", "health", pool)
if err != nil {
r.Said = "zpool list: " + err.Error()
return r
}
status, err := b.Run(ctx, "zpool", "status", "-x", pool)
if err != nil {
r.Said = "zpool status: " + err.Error()
return r
}
health, status = strings.TrimSpace(health), strings.TrimSpace(status)
r.Said = "health " + health + "; " + firstLineOf(status)
if health == "ONLINE" && strings.Contains(status, "is healthy") {
r.Healthy = &yes
} else {
r.Healthy = &no
r.Said = "health " + health + "; " + oneLineOf(status)
}
return r
case strings.HasPrefix(source, "/dev/md"):
device := strings.TrimPrefix(source, "/dev/")
r := &health.Redundancy{Kind: "md", Where: device}
mdstat, err := b.Run(ctx, "cat", "/proc/mdstat")
if err != nil {
r.Said = err.Error()
return r
}
healthy, said, found := mdHealth(mdstat, device)
if !found {
r.Said = device + " is not in /proc/mdstat"
return r
}
r.Said = said
if healthy {
r.Healthy = &yes
} else {
r.Healthy = &no
}
return r
case fstype == "btrfs":
r := &health.Redundancy{Kind: "btrfs", Where: source}
stats, err := b.Run(ctx, "btrfs", "device", "stats", "--check", path)
if err != nil {
r.Healthy, r.Said = &no, "device errors: "+oneLineOf(err.Error())
return r
}
r.Healthy, r.Said = &yes, firstLineOf(stats)
return r
}
return nil
}
var mdMembers = regexp.MustCompile(`\[([U_]+)\]`)
// mdHealth reads one array's block of /proc/mdstat: healthy when every member is up and it is not recovering.
func mdHealth(mdstat, device string) (bool, string, bool) {
lines := strings.Split(mdstat, "\n")
for i, l := range lines {
if !strings.HasPrefix(l, device+" ") {
continue
}
block := l
for j := i + 1; j < len(lines) && strings.HasPrefix(lines[j], " "); j++ {
block += " " + strings.TrimSpace(lines[j])
}
members := mdMembers.FindStringSubmatch(block)
healthy := members != nil && !strings.Contains(members[1], "_") && !strings.Contains(block, "recovery")
return healthy, oneLineOf(block), true
}
return false, "", false
}
func firstLineOf(s string) string {
line, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
return line
}
func oneLineOf(s string) string { return strings.Join(strings.Fields(s), " ") }
@@ -1,16 +0,0 @@
package main
// The readings layout, their key and the peak are mesh-sdk's (health.Reading, health.NextReading; novox/hq ADR 0571
// clause 3): disk-load measures each placed data item of its node and keeps the readings as its own state.
import "git.novox.be/novox/mesh-sdk/go/health"
// StateReadings is the state disk-load keeps.
const StateReadings = health.ReadingsState
// StateRounds is its state of each node's last round, key `<node>` (novox/hq ADR 0577 clause 3): apart from the readings,
// so a reader of the readings never meets a round.
const StateRounds = "rounds"
// ClassCache is the class of an item that is listed and never measured.
const ClassCache = "cache"
@@ -1,161 +0,0 @@
package main
import (
"context"
"encoding/json"
"os"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/health"
)
type memStore struct {
mu sync.Mutex
m map[string]json.RawMessage
}
func (s *memStore) Get(k string) (json.RawMessage, error) {
s.mu.Lock()
defer s.mu.Unlock()
return s.m[k], nil
}
func (s *memStore) Put(k string, v any) error {
raw, _ := json.Marshal(v)
s.mu.Lock()
defer s.mu.Unlock()
s.m[k] = raw
return nil
}
func (s *memStore) Keys() ([]string, error) {
s.mu.Lock()
defer s.mu.Unlock()
var out []string
for k := range s.m {
out = append(out, k)
}
return out, nil
}
// A round measures what the data file declares but caches, keeps the readings, writes nothing the second time, and a
// walk is not repeated within a day unless asked.
func TestARoundMeasuresTheDeclaredItemsAndWritesWhatChanged(t *testing.T) {
text := "# pg\nitem data valuable /srv/pg /srv/pg backup walk\nitem tmp cache /srv/tmp - none\n# bad-line-module\nitem logs rebuildable /srv/logs - none shallow\n"
placed, err := catalogue.ParseDataItems(text)
if err != nil {
t.Fatal(err)
}
walks := 0
run := func(_ context.Context, name string, args ...string) (string, error) {
switch {
case name == "test":
return "", nil
case name == "findmnt":
return "", os.ErrNotExist
case name == "bash" && strings.Contains(args[1], "-maxdepth 1"):
return "1760000000 2\n", nil
case name == "bash":
walks++
return "4096 1760000000 3\n0\n", nil
}
return "", os.ErrNotExist
}
now := time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC)
store := &memStore{m: map[string]json.RawMessage{}}
k := &Keeper{Node: "ace", Items: func(context.Context) (map[string][]catalogue.PlacedDataItem, error) { return placed, nil }, Store: store, Say: func(string, ...any) {},
Measurer: Measurer{Run: run, Now: func() time.Time { return now }}}
w, kept, err := k.Round(context.Background(), false)
if err != nil || w != 2 || kept != 0 || len(store.m) != 2 {
t.Fatalf("first round: %d written %d kept %v, keys %d", w, kept, err, len(store.m))
}
now = now.Add(time.Hour)
if w, kept, _ := k.Round(context.Background(), false); w != 0 || kept != 2 || walks != 1 {
t.Fatalf("second round: %d written, %d kept, %d walks; wanted none written and the walk not repeated", w, kept, walks)
}
if _, _, err := k.MeasureNow(context.Background(), "pg", "data"); err != nil || walks != 2 {
t.Fatalf("measure now: %v, %d walks", err, walks)
}
// A round retires nothing, even when no item is declared (the watched set may be empty or partial); an item its
// module's key no longer names is kept, retired with when and why; declared again, it is not retired any more.
delete(placed, "pg")
if _, _, err := k.Round(context.Background(), false); err != nil {
t.Fatal(err)
}
if kept, _ := health.DecodeReading(store.m["ace.pg.data"]); !kept.RetiredAt.IsZero() {
t.Fatalf("a round retired a reading: %+v", kept)
}
k.MeasureModule(context.Background(), "pg")
gone, _ := health.DecodeReading(store.m["ace.pg.data"])
if gone.RetiredAt.IsZero() || !strings.Contains(gone.RetiredWhy, "no longer declared on ace") || gone.FirstSeen.IsZero() {
t.Fatalf("a reading of an item no longer declared: %+v", gone)
}
placed["pg"] = []catalogue.PlacedDataItem{{Module: "pg", ID: "data", Class: "valuable", Path: "/srv/pg", Measure: catalogue.MeasureWalk}}
if _, _, err := k.Round(context.Background(), false); err != nil {
t.Fatal(err)
}
if back, _ := health.DecodeReading(store.m["ace.pg.data"]); !back.RetiredAt.IsZero() {
t.Fatalf("a reading declared again is still retired: %+v", back)
}
if _, _, err := k.MeasureNow(context.Background(), "pg", "tmp"); err == nil {
t.Fatal("a cache was measured")
}
}
// The redundant storage under an item is read: a ZFS pool healthy, then degraded; an md array with a member missing; and
// plain storage, which is no redundancy at all. A dataset is measured from the filesystem's counters and a shallow item
// from its top-level entries only; a walk that meets more than its bound stops and says so.
func TestTheStorageUnderAnItemAndTheLargeMethodsAreRead(t *testing.T) {
poolHealth, statusX, fs := "ONLINE\n", "pool 'storage' is healthy\n", "storage/media zfs\n"
run := func(_ context.Context, name string, args ...string) (string, error) {
line := name + " " + strings.Join(args, " ")
switch {
case name == "test":
return "", nil
case name == "findmnt":
return fs, nil
case name == "zpool" && args[0] == "list":
return poolHealth, nil
case name == "zpool":
return statusX, nil
case name == "cat":
return "Personalities : [raid1]\nmd127 : active raid1 sdb1[1] sda1[0]\n 976630464 blocks super 1.2 [2/1] [U_]\n\nunused devices: <none>\n", nil
case name == "zfs" && args[len(args)-1] == "/storage/media/movies":
return "storage/media\t97067690722560\n", nil
case name == "zfs":
return "", os.ErrNotExist
case name == "bash" && strings.Contains(line, "-maxdepth 1"):
return "1759744800 12673\n", nil
case name == "bash":
return "123 1759700000 2000000\n141\n", nil
}
return "", nil
}
m := Measurer{Run: run, Now: time.Now}
ctx := context.Background()
got := m.Measure(ctx, catalogue.PlacedDataItem{Module: "media", ID: "movies", Path: "/storage/media/movies", Measure: catalogue.MeasureDataset})
if got.SizeBytes == nil || *got.SizeBytes != 97067690722560 || !strings.HasPrefix(got.Precision, "dataset storage/media") ||
got.LastWrite == nil || got.Redundancy == nil || got.Redundancy.Kind != "zfs" || got.Redundancy.Healthy == nil || !*got.Redundancy.Healthy {
t.Fatalf("the library from its dataset: %+v", got)
}
poolHealth, statusX = "DEGRADED\n", " pool: storage\n state: DEGRADED\nstatus: One or more devices has been removed\n"
if r := m.redundancyOf(ctx, "/storage/media/movies"); r.Healthy == nil || *r.Healthy || !strings.Contains(r.Said, "DEGRADED") {
t.Fatalf("a degraded pool: %+v", r)
}
fs = "/dev/md127 ext4\n"
if r := m.redundancyOf(ctx, "/srv/x"); r == nil || r.Kind != "md" || r.Healthy == nil || *r.Healthy {
t.Fatalf("an array missing a member: %+v", r)
}
fs = "/dev/nvme0n1p3 ext4\n"
if r := m.redundancyOf(ctx, "/var/lib/x"); r != nil {
t.Fatalf("plain storage read as redundant: %+v", r)
}
if s := m.Measure(ctx, catalogue.PlacedDataItem{Path: "/mnt/plex/config", Measure: catalogue.MeasureShallow}); s.SizeBytes != nil || !strings.HasPrefix(s.Precision, "no size") {
t.Fatalf("a shallow item: %+v", s)
}
if w := m.Measure(ctx, catalogue.PlacedDataItem{Path: "/srv/big", Measure: catalogue.MeasureWalk}); !strings.HasPrefix(w.Precision, "partial") {
t.Fatalf("a walk stopped at its bound: %+v", w)
}
}
@@ -1,126 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"sort"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/health"
)
// ReadingTools are the tools over the state `readings`: every key, one key, how full the state is, and one item
// measured now (a module's tools are its command line).
func ReadingTools(k *Keeper) []stdio.Tool {
return []stdio.Tool{
{
Name: "disk_readings",
Description: "The data readings disk-load keeps for this machine's data items (ADR 0233, ADR 0571 clause 3): every key " +
"`<node>.<module>.<item>` with its size, newest write, measured-at, precision and seven-day peak, or one key with " +
"all it holds when `key` is given; and how full the state is: keys kept, their bytes, and the cap the bus puts on " +
"a state (none declared). Filter by `module`. (r)",
Input: map[string]any{
"key": map[string]any{"type": "string", "description": "one key, <node>.<module>.<item>, with everything it holds"},
"module": map[string]any{"type": "string", "description": "only this module's items"},
},
Run: func(args map[string]any) (any, error) {
if key := text(args, "key"); key != "" {
raw, err := k.Store.Get(key)
if err != nil {
return nil, err
}
if len(raw) == 0 {
return nil, fmt.Errorf("no reading is kept at %q", key)
}
r, err := health.DecodeReading(raw)
return r, err
}
keys, err := k.Store.Keys()
if err != nil {
return nil, err
}
sort.Strings(keys)
module := text(args, "module")
type line struct {
Key string `json:"key"`
SizeBytes *int64 `json:"size_bytes,omitempty"`
PeakBytes *int64 `json:"peak_bytes,omitempty"`
Precision string `json:"precision,omitempty"`
Error string `json:"error,omitempty"`
Reading health.Reading `json:"-"`
}
var lines []line
total := 0
for _, key := range keys {
raw, err := k.Store.Get(key)
if err != nil {
return nil, err
}
total += len(raw)
r, err := health.DecodeReading(raw)
if err != nil || (module != "" && r.Module != module) {
continue
}
lines = append(lines, line{Key: key, SizeBytes: r.SizeBytes, PeakBytes: r.Peak, Precision: r.Precision, Error: r.Error})
}
return map[string]any{"readings": lines, "fill": map[string]any{"keys": len(keys), "bytes": total,
"cap": "none declared: the bus's own limit on a state applies"}}, nil
},
},
{
Name: "disk_round",
Description: "How far this machine's last round of measuring its data items got (novox/hq ADR 0577 clause 3), from " +
"disk-load's state `rounds`, this machine's key `<node>`: when it started and ended (no end while it runs), how " +
"many items it had, measured, wrote and kept unchanged, each item that failed and why, and how old the round is. " +
"A round that never ends, or fails every item, shows here. (r)",
Input: map[string]any{},
Run: func(args map[string]any) (any, error) {
node := k.Node
if k.Rounds == nil {
return nil, fmt.Errorf("this disk-load keeps no rounds")
}
raw, err := k.Rounds.Get(node)
if err != nil {
return nil, err
}
if len(raw) == 0 {
return nil, fmt.Errorf("no round of %s is kept yet: its first round starts a few minutes after disk-load starts", node)
}
var r Round
if err := json.Unmarshal(raw, &r); err != nil {
return nil, fmt.Errorf("the round of %s is not readable: %w", node, err)
}
state := "ended"
if r.Ended == nil {
state = "running"
}
return map[string]any{"round": r, "state": state,
"age": k.Measurer.Now().UTC().Sub(r.Started).Round(time.Second).String()}, nil
},
},
{
Name: "disk_measure",
Description: "Measure one data item of this machine now, whatever its method (a walk too), and write its reading when it " +
"changed. Reads the disks as root: a walk is bounded by time and files and says when it was partial.",
Input: map[string]any{
"module": map[string]any{"type": "string", "description": "the module that declares the item"},
"item": map[string]any{"type": "string", "description": "the item's id"},
},
Run: func(args map[string]any) (any, error) {
module, item := text(args, "module"), text(args, "item")
if module == "" || item == "" {
return nil, fmt.Errorf("module and item are required")
}
r, written, err := k.MeasureNow(context.Background(), module, item)
if err != nil {
return nil, err
}
return map[string]any{"written": written, "reading": r, "key": health.ReadingKey(k.Node, module, item),
"note": strings.TrimSpace(map[bool]string{true: "the reading changed and was written", false: "the reading had not changed"}[written])}, nil
},
},
}
}
@@ -3,7 +3,6 @@ package main
import (
"encoding/json"
"os"
"regexp"
"sort"
"strings"
"testing"
@@ -36,11 +35,9 @@ func TestTheManifestNamesExactlyTheToolsTheBundleServesAndNoMachine(t *testing.T
t.Fatal(err)
}
var served []string
k := &Keeper{Store: &memStore{m: map[string]json.RawMessage{}}}
for _, tl := range append(Tools(newFake(t).machine()), ReadingTools(k)...) {
for _, tl := range Tools(newFake(t).machine()) {
served = append(served, tl.Name)
// disk_measure writes the module's own state of readings and nothing else of the machine.
if !strings.HasSuffix(tl.Description, "(r)") && tl.Name != "disk_measure" {
if !strings.HasSuffix(tl.Description, "(r)") {
t.Errorf("%s only reads and does not say so", tl.Name)
}
}
@@ -53,7 +50,7 @@ func TestTheManifestNamesExactlyTheToolsTheBundleServesAndNoMachine(t *testing.T
t.Errorf("the module only reads, and declares %d resources", len(m.Resources))
}
for _, banned := range []string{"g14", "novox", "ace", "shanks", "jochen", "/home/"} {
if regexp.MustCompile(`(^|[^a-z0-9-])` + regexp.QuoteMeta(banned) + `($|[^a-z0-9-])`).Match(raw) {
if strings.Contains(string(raw), banned) {
t.Errorf("the manifest says %q", banned)
}
}
@@ -1,267 +0,0 @@
//go:build integration
package main
// disk-load whole against a real bus (novox/hq ADR 0577 clause 5, the group's integration test): the built binary, played
// over its real stdin and stdout as the node's tool runtime plays it (mesh/state.* answered from a real nats-server's
// key-value buckets, a watch handed over from the bus's own watch, mesh/publish onto its subjects). A key of the
// node-engine's state data written after disk-load started has its module's item read within a minute, and disk-load
// composes nothing to learn it.
import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/testbus"
)
type wire struct {
ID json.RawMessage `json:"id,omitempty"`
Method string `json:"method,omitempty"`
Params json.RawMessage `json:"params,omitempty"`
Result json.RawMessage `json:"result,omitempty"`
}
// player is the node's tool runtime, as far as disk-load needs it, over a real bus.
type player struct {
ctx context.Context
nc *nats.Conn
js jetstream.JetStream
mu sync.Mutex
in io.Writer
next int
asked []string // every state call, `<method> <state> <key>`
}
func (p *player) send(m map[string]any) {
raw, _ := json.Marshal(m)
p.mu.Lock()
defer p.mu.Unlock()
_, _ = p.in.Write(append(raw, '\n'))
}
func (p *player) answer(id json.RawMessage, result any, err error) {
if err != nil {
p.send(map[string]any{"jsonrpc": "2.0", "id": id, "error": map[string]any{"code": -32000, "message": err.Error()}})
return
}
p.send(map[string]any{"jsonrpc": "2.0", "id": id, "result": result})
}
// bucket is a state's bucket: another module's `<owner>.<name>`, or disk-load's own by its local name.
func bucket(state string) string {
if owner, name, other := strings.Cut(state, "."); other {
return owner + "_" + name
}
return "disk-load_" + state
}
func (p *player) serve(out io.Reader) {
scanner := bufio.NewScanner(out)
scanner.Buffer(make([]byte, 1<<20), 1<<24)
for scanner.Scan() {
var m wire
if json.Unmarshal(scanner.Bytes(), &m) != nil || m.Method == "" {
continue // an answer to a change handed over
}
go p.handle(m)
}
}
func (p *player) handle(m wire) {
var a struct {
State string `json:"state"`
Key string `json:"key"`
Value json.RawMessage `json:"value"`
Body json.RawMessage `json:"body"`
}
_ = json.Unmarshal(m.Params, &a)
p.mu.Lock()
p.asked = append(p.asked, m.Method+" "+a.State+" "+a.Key)
p.mu.Unlock()
if m.Method == "mesh/publish" {
var e struct {
Key string `json:"key"`
Body json.RawMessage `json:"body"`
}
_ = json.Unmarshal(m.Params, &e)
p.answer(m.ID, map[string]any{}, p.nc.Publish("mesh.mod.disk-load.event."+e.Key, e.Body))
return
}
if !strings.HasPrefix(m.Method, "mesh/state.") {
p.answer(m.ID, nil, errors.New("nats: no responders available for request"))
return
}
kv, err := p.js.KeyValue(p.ctx, bucket(a.State))
if err != nil {
p.answer(m.ID, nil, err)
return
}
switch m.Method {
case "mesh/state.get":
e, err := kv.Get(p.ctx, a.Key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
p.answer(m.ID, nil, nil)
return
}
if err != nil {
p.answer(m.ID, nil, err)
return
}
p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil)
case "mesh/state.put":
rev, err := kv.Put(p.ctx, a.Key, a.Value)
p.answer(m.ID, map[string]any{"revision": rev}, err)
case "mesh/state.keys":
keys, err := kv.Keys(p.ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
keys, err = []string{}, nil
}
p.answer(m.ID, keys, err)
case "mesh/state.watch":
p.watch(m.ID, a.State, kv, a.Key)
default:
p.answer(m.ID, nil, errors.New("not played here"))
}
}
// watch hands over the bus's watch of the pattern: every current value, then the answer to the watch, then each change.
func (p *player) watch(id json.RawMessage, state string, kv jetstream.KeyValue, pattern string) {
var watchID string
_ = json.Unmarshal(id, &watchID)
if pattern == "" {
pattern = ">"
}
w, err := kv.Watch(p.ctx, strings.ReplaceAll(pattern, "**", ">"))
if err != nil {
p.answer(id, nil, err)
return
}
current := true
for e := range w.Updates() {
if e == nil {
current = false
p.answer(id, map[string]any{}, nil)
continue
}
op := "put"
if e.Operation() != jetstream.KeyValuePut {
op = "delete"
}
p.mu.Lock()
p.next++
n := p.next
p.mu.Unlock()
p.send(map[string]any{"jsonrpc": "2.0", "id": fmt.Sprintf("rt-%d", n), "method": "mesh/state", "params": map[string]any{
"watch": watchID, "change": map[string]any{"state": state, "key": e.Key(), "op": op,
"value": json.RawMessage(e.Value()), "revision": e.Revision(), "current": current}}})
}
}
// 0577/AC5 (the group's integration test): a key of the node-engine's state data written after disk-load started is
// read within a minute, its item's reading is on the bus, and disk-load read no composition input to learn it.
func TestDiskLoadWholeMeasuresAnItemWithinAMinuteOfItsKey(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
bin := filepath.Join(t.TempDir(), "disk-load")
build := exec.Command("go", "build", "-o", bin, ".")
build.Stderr = os.Stderr
if err := build.Run(); err != nil {
t.Fatalf("disk-load does not build: %v", err)
}
nc, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
for _, b := range []string{delivery.EngineDataBucket, health.ReadingsBucket, "disk-load_" + StateRounds} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil {
t.Fatal(err)
}
}
cmd := exec.CommandContext(ctx, bin)
cmd.Env = append(os.Environ(), "MESH_NODE=ace", "MESH_MODULE=disk-load")
stdin, _ := cmd.StdinPipe()
stdout, _ := cmd.StdoutPipe()
cmd.Stderr = os.Stderr
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() })
p := &player{ctx: ctx, nc: nc, js: js, in: stdin}
go p.serve(stdout)
// Once disk-load watches its machine's keys, the engine writes one.
watching := func() bool {
p.mu.Lock()
defer p.mu.Unlock()
for _, a := range p.asked {
if a == "mesh/state.watch "+delivery.EngineDataRead+" ace.*" {
return true
}
}
return false
}
for end := time.Now().Add(20 * time.Second); !watching(); time.Sleep(50 * time.Millisecond) {
if time.Now().After(end) {
t.Fatalf("disk-load did not watch its machine's keys of %s: asked %v", delivery.EngineDataBucket, p.asked)
}
}
data, _ := js.KeyValue(ctx, delivery.EngineDataBucket)
dir := t.TempDir()
item := catalogue.PlacedDataItem{Module: "pg", ID: "data", Class: "valuable", Path: dir, Measure: catalogue.MeasureShallow}
raw, err := delivery.EncodeEngineData(delivery.EngineData{Node: "ace", Module: "pg", Digest: "sha256:pg", Items: []catalogue.PlacedDataItem{item}})
if err != nil {
t.Fatal(err)
}
key, _ := delivery.EngineDataKey("ace", "pg")
written := time.Now()
if _, err := data.Put(ctx, key, raw); err != nil {
t.Fatal(err)
}
readings, _ := js.KeyValue(ctx, health.ReadingsBucket)
for {
e, err := readings.Get(ctx, health.ReadingKey("ace", "pg", "data"))
if err == nil {
r, err := health.DecodeReading(e.Value())
if err != nil || r.Path != dir || r.MeasuredAt == nil {
t.Fatalf("the reading is %s (%v)", e.Value(), err)
}
break
}
if time.Since(written) > time.Minute {
t.Fatalf("no reading within a minute of the key's write: %v", err)
}
time.Sleep(200 * time.Millisecond)
}
// It composed nothing: every state it asked is the engine's data or its own.
p.mu.Lock()
defer p.mu.Unlock()
for _, a := range p.asked {
f := strings.Fields(a)
if len(f) > 1 && strings.HasPrefix(f[0], "mesh/state.") && f[1] != delivery.EngineDataRead && f[1] != StateReadings && f[1] != StateRounds {
t.Errorf("disk-load read %s, which is no state of its own nor the engine's data", a)
}
}
}
+2 -19
View File
@@ -1,22 +1,5 @@
module diskload
go 1.26.0
go 1.22
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915
github.com/nats-io/nats.go v1.54.0
)
require (
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
github.com/google/go-tpm v0.9.8 // indirect
github.com/klauspost/compress v1.20.0 // indirect
github.com/minio/highwayhash v1.0.4 // indirect
github.com/nats-io/jwt/v2 v2.8.1 // indirect
github.com/nats-io/nats-server/v2 v2.11.17 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/time v0.15.0 // indirect
)
require git.novox.be/novox/mesh-sdk/go v0.1.6
+2 -27
View File
@@ -1,27 +1,2 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
git.novox.be/novox/mesh-sdk/go v0.1.6 h1:9qzdYONYbJdWcu6sxQcq9v1LI0JxcfkiKYkMUzJSkVQ=
git.novox.be/novox/mesh-sdk/go v0.1.6/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=

Some files were not shown because too many files have changed in this diff Show More