Compare commits
336
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
125929dbc4 | ||
|
|
6be8f984f7 | ||
|
|
fdc3331465 | ||
|
|
3498e54bdb | ||
|
|
2b340d7ddb | ||
|
|
b3261ccc03 | ||
|
|
da0573e124 | ||
|
|
2b8b47d862 | ||
|
|
e97167c58f | ||
|
|
0d6d7acaae | ||
|
|
dc0349a220 | ||
|
|
ccb7eaff13 | ||
|
|
e958de14d2 | ||
|
|
00d6fe81bf | ||
|
|
f28d3bf286 | ||
|
|
f735392cb9 | ||
|
|
d320a19a8c | ||
|
|
d8e43b48c4 | ||
|
|
64c893e2c2 | ||
|
|
bccaf75556 | ||
|
|
8f5611a2d1 | ||
|
|
1443adc4f0 | ||
|
|
dce7d41671 | ||
|
|
6f89e622df | ||
|
|
3b01711e1f | ||
|
|
1b485eb2aa | ||
|
|
1bdca045c2 | ||
|
|
70fd02e8da | ||
|
|
3079b7f115 | ||
|
|
dee54a1a13 | ||
|
|
0554f10c0a | ||
|
|
019184cf5d | ||
|
|
69ceac7fcf | ||
|
|
9d09aeddb1 | ||
|
|
4438045b8c | ||
|
|
9c8d64e28c | ||
|
|
31f6c635f5 | ||
|
|
348f9631fe | ||
|
|
eb50bf1f48 | ||
|
|
b88c6dac5b | ||
|
|
37a16a4e98 | ||
|
|
4915e67520 | ||
|
|
8acad77d33 | ||
|
|
37a1f2be53 | ||
|
|
46505d42d8 | ||
|
|
1c4739698a | ||
|
|
a198a959fe | ||
|
|
6da45bc090 | ||
|
|
85d96d327f | ||
|
|
d47d231ec3 | ||
|
|
5601fdb51d | ||
|
|
d2f326dff4 | ||
|
|
8eb4465590 | ||
|
|
aef8b9c7cb | ||
|
|
e40ace34a6 | ||
|
|
7dd9e11239 | ||
|
|
aa5e2a0dd6 | ||
|
|
4a2555253c | ||
|
|
34d5c0d308 | ||
|
|
695ac0356d | ||
|
|
af280050d6 | ||
|
|
60a187cfa8 | ||
|
|
39a8d8d7ee | ||
|
|
d9d1119a72 | ||
|
|
e9ba05bd9a | ||
|
|
7b08f25d4a | ||
|
|
10d53dd02b | ||
|
|
89d0eb13af | ||
|
|
69303e677a | ||
|
|
84ab1fe80d | ||
|
|
6061c775d5 | ||
|
|
8f5667cca5 | ||
|
|
b5495d0cc8 | ||
|
|
c5bec10bb5 | ||
|
|
aaca1c6ca9 | ||
|
|
5450707aaf | ||
|
|
79c2fc7fe1 | ||
|
|
cb3db6c902 | ||
|
|
6146c8a76d | ||
|
|
4492bc3013 | ||
|
|
929b1ce13e | ||
|
|
55a88cdba4 | ||
|
|
29f6039fb1 | ||
|
|
57a18a4783 | ||
|
|
b0a275c126 | ||
|
|
d7ffc9cca0 | ||
|
|
38ae52e999 | ||
|
|
11acf04976 | ||
|
|
63322ef06f | ||
|
|
4fa457b9bb | ||
|
|
b665156b0f | ||
|
|
0b2260d9fb | ||
|
|
f16efbef4f | ||
|
|
ede8994799 | ||
|
|
d31b41bb7e | ||
|
|
b4584b0313 | ||
|
|
c6c046283d | ||
|
|
d683a78d9c | ||
|
|
ed7c6c4fb6 | ||
|
|
cd66db54bf | ||
|
|
9f01b9e79a | ||
|
|
e34fcbeda3 | ||
|
|
32699e35b8 | ||
|
|
45688f196e | ||
|
|
85fd46b4cc | ||
|
|
f8212dc203 | ||
|
|
d7214bb576 | ||
|
|
11048b4a2d | ||
|
|
99512eade4 | ||
|
|
ee54bae09f | ||
|
|
75986b2960 | ||
|
|
3daedac1bb | ||
|
|
313c1997a4 | ||
|
|
e6806d5698 | ||
|
|
d0f384610c | ||
|
|
8d4b2cb1c6 | ||
|
|
4f433bdaf8 | ||
|
|
91cd016f65 | ||
|
|
cdddeee4ea | ||
|
|
3e019c5d7e | ||
|
|
fa4ccbc0a9 | ||
|
|
76aab05289 | ||
|
|
4b2d5aeaab | ||
|
|
68dd69f4ef | ||
|
|
3b61a05532 | ||
|
|
6be2e91da8 | ||
|
|
905c34a537 | ||
|
|
cc1a942235 | ||
|
|
93bfdd9e2f | ||
|
|
1f68f80760 | ||
|
|
ff4d042be5 | ||
|
|
4f24cb0751 | ||
|
|
0d86833123 | ||
|
|
715bc088c4 | ||
|
|
ab5f767372 | ||
|
|
40b1307839 | ||
|
|
a3a11842bf | ||
|
|
beba03613c | ||
|
|
d8e8c258a6 | ||
|
|
c73641c117 | ||
|
|
47b3ae7528 | ||
|
|
e5e3d7dce6 | ||
|
|
5adb8e1fea | ||
|
|
2bf608fc5f | ||
|
|
b0bfdeb0d1 | ||
|
|
48624da9e3 | ||
|
|
25e5d00aba | ||
|
|
a4e3a9cbe0 | ||
|
|
dd120b1c90 | ||
|
|
6b929c9057 | ||
|
|
f49b6de79b | ||
|
|
4d05e0d290 | ||
|
|
972ecd105d | ||
|
|
7bf0e991d8 | ||
|
|
2bb964b000 | ||
|
|
692f18ef3b | ||
|
|
6128a570a2 | ||
|
|
c5dc10fa8c | ||
|
|
55eecc2d34 | ||
|
|
7fe569ef40 | ||
|
|
3e3294cd47 | ||
|
|
40e1e087e5 | ||
|
|
eeeb2056ae | ||
|
|
e1a182bf09 | ||
|
|
553cf50da7 | ||
|
|
4e318c8591 | ||
|
|
c9b98477b7 | ||
|
|
bd9764ef9a | ||
|
|
bb85684b2a | ||
|
|
17ba95bca6 | ||
|
|
9ad22ad1e1 | ||
|
|
4aa77a4237 | ||
|
|
4d95d8fc1e | ||
|
|
d6c3fc1e40 | ||
|
|
285099a940 | ||
|
|
80a7cee6f3 | ||
|
|
e0d7644bb6 | ||
|
|
376a8b0fb5 | ||
|
|
6a39f43404 | ||
|
|
c973b31820 | ||
|
|
7f626147be | ||
|
|
080aed6855 | ||
|
|
4f67143a78 | ||
|
|
1572ab9b25 | ||
|
|
b0b7dfc5eb | ||
|
|
bf4ce7ef7b | ||
|
|
052fb473d0 | ||
|
|
bff850f798 | ||
|
|
7a7db0166a | ||
|
|
237a534ca4 | ||
|
|
dd458cb155 | ||
|
|
6c09be4a70 | ||
|
|
448f1ef481 | ||
|
|
093967af9b | ||
|
|
be749c9513 | ||
|
|
4026392a6e | ||
|
|
d632071317 | ||
|
|
5fe7e3e6ba | ||
|
|
be24f5ac38 | ||
|
|
f2eb548204 | ||
|
|
e6de426303 | ||
|
|
5fab8f118e | ||
|
|
d5237091a6 | ||
|
|
1a35e5cf4d | ||
|
|
e2fad6a50a | ||
|
|
04ab9bc603 | ||
|
|
333ab9de22 | ||
|
|
e84b9d13c4 | ||
|
|
9c36d4edd4 | ||
|
|
4c137795a8 | ||
|
|
281f586621 | ||
|
|
843da56d09 | ||
|
|
2d34ff468b | ||
|
|
230230ce0e | ||
|
|
34ce2895dd | ||
|
|
0aa319154e | ||
|
|
90ff598207 | ||
|
|
1d1d5ec443 | ||
|
|
3112692def | ||
|
|
4915ff5d29 | ||
|
|
dd4c53eefb | ||
|
|
32dc74788e | ||
|
|
1fb32928ae | ||
|
|
0b2d393b11 | ||
|
|
134baf0e3b | ||
|
|
7c78fd2e0c | ||
|
|
9239177efd | ||
|
|
a1d98c2438 | ||
|
|
f469eb387a | ||
|
|
d221c63379 | ||
|
|
68e7a2ddbf | ||
|
|
ee75d4327c | ||
|
|
640455999e | ||
|
|
45084af849 | ||
|
|
7f4cc9e2ec | ||
|
|
06963203ff | ||
|
|
0697b2994e | ||
|
|
308d052ff6 | ||
|
|
1885f176f1 | ||
|
|
6bdf958cef | ||
|
|
4797790959 | ||
|
|
7aad491321 | ||
|
|
6118c790e6 | ||
|
|
6f781abb57 | ||
|
|
350aca05fb | ||
|
|
b83a1abe5f | ||
|
|
9c57f7afb3 | ||
|
|
2d2eb5dffb | ||
|
|
c52afdf88b | ||
|
|
019614aedc | ||
|
|
fe7c5ccb64 | ||
|
|
fa6dc3a48b | ||
|
|
7c5df1d7aa | ||
|
|
7f8701637f | ||
|
|
067f2f4d61 | ||
|
|
0ef81d85b3 | ||
|
|
04379702e5 | ||
|
|
9a3ead8fcd | ||
|
|
fda81c129e | ||
|
|
ecc7621274 | ||
|
|
3ed3266dde | ||
|
|
9e9877e6ec | ||
|
|
0f0ec66e9a | ||
|
|
344fc6f6bf | ||
|
|
70396caeab | ||
|
|
4aa24ba013 | ||
|
|
067117b8b8 | ||
|
|
e7b3fd470b | ||
|
|
f8e4c1b0ae | ||
|
|
a73b02ea30 | ||
|
|
f83e6e4713 | ||
|
|
f723d4372f | ||
|
|
b44e600e87 | ||
|
|
ab610ffed4 | ||
|
|
c4a6452f21 | ||
|
|
e009088150 | ||
|
|
40c4fabff6 | ||
|
|
0bfc72d230 | ||
|
|
32c4c52261 | ||
|
|
d2de18fd6b | ||
|
|
63de1f7e0d | ||
|
|
9523050a98 | ||
|
|
326897d877 | ||
|
|
a3e9f2cc0f | ||
|
|
5cdf8295f3 | ||
|
|
0e6c358623 | ||
|
|
fe7d4504bf | ||
|
|
db104bdb08 | ||
|
|
c84dfb5e0b | ||
|
|
67e4d4b062 | ||
|
|
f41a5a8df1 | ||
|
|
09824b4839 | ||
|
|
272eff8b39 | ||
|
|
9249ca7b57 | ||
|
|
049bd176c3 | ||
|
|
98e08c0dd0 | ||
|
|
2c5e9c9a98 | ||
|
|
e4382d016e | ||
|
|
2877b87f29 | ||
|
|
a8c1c275dd | ||
|
|
204a02240c | ||
|
|
432de55186 | ||
|
|
2f52f91b5c | ||
|
|
c1a32b4385 | ||
|
|
4497f26257 | ||
|
|
fc5b849d85 | ||
|
|
ce3bedd584 | ||
|
|
90f7f87c2a | ||
|
|
68b5febee7 | ||
|
|
3483f603ae | ||
|
|
5ccce4967a | ||
|
|
b3fcf408f2 | ||
|
|
3d1a450406 | ||
|
|
7375d613b4 | ||
|
|
0471b2e105 | ||
|
|
2f8cb338ff | ||
|
|
a6c2f7b31a | ||
|
|
2b495c4818 | ||
|
|
37613ec6a9 | ||
|
|
cff7588797 | ||
|
|
f06335cf7e | ||
|
|
c414d27c42 | ||
|
|
fc300e3d72 | ||
|
|
2002b08505 | ||
|
|
6de5c6680e | ||
|
|
4a6395feae | ||
|
|
1db0181920 | ||
|
|
8c07d2ef67 | ||
|
|
a673fc52a7 | ||
|
|
b0dc7e08de | ||
|
|
aa4a4253ad | ||
|
|
cd7582a0f2 | ||
|
|
1d3e2b8aca | ||
|
|
462bcfe451 | ||
|
|
432326264f | ||
|
|
3e08301b27 |
@@ -2,10 +2,13 @@ node_modules/
|
||||
dist/
|
||||
|
||||
# Go tool bundles built in place (go build in a module's cmd/<name>-tools) are build output.
|
||||
modules/pacman/pacman-tools
|
||||
modules/slack/cmd/slack-tools/slack-tools
|
||||
modules/jetbrains-toolbox/cmd/toolbox-tools/toolbox-tools
|
||||
modules/messenger/cmd/messenger/messenger
|
||||
modules/mesh-watcher/cmd/mesh-watcher/mesh-watcher
|
||||
modules/mesh-conditions/mesh-conditions
|
||||
modules/minio/object-store
|
||||
modules/claude-code/cmd/claude-code/claude-code
|
||||
modules/dnsmasq/cmd/dnsmasq-tools/dnsmasq-tools
|
||||
# ...and the same built from the module's root (go build ./cmd/<name>), which names it after the module.
|
||||
@@ -33,6 +36,7 @@ modules/mesh-build-image/cmd/mesh-build-image/mesh-build-image
|
||||
|
||||
# binaries a local go build leaves in a module root
|
||||
/checks/paths/paths
|
||||
/checks/environment/environment
|
||||
/modules/mesh-build-go/mesh-build-go
|
||||
/modules/mesh-delivery-gate/mesh-delivery-gate
|
||||
modules/mesh-config/cmd/mesh-config/mesh-config
|
||||
@@ -41,3 +45,5 @@ modules/mesh-config/cmd/mesh-config/mesh-config
|
||||
modules/forticlient/cmd/forticlient-tools/forticlient-tools
|
||||
modules/networkmanager/uplink-tools
|
||||
checks/words/words
|
||||
modules/mesh-cli/mesh-cli
|
||||
modules/mesh-cli/cmd/mesh-cli/mesh-cli
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
// Package controllerdeps is the catalogue's dependence check on the controller (novox/hq ADR 0590 §5, issue 1126): the
|
||||
// controller is not placed, so no module of this tree reads, requires, invokes, consumes, waits on, uses or otherwise
|
||||
// names it. Nothing is kept: not the lease, not the entries once kept to its write (ADR 0571 §13, superseded with ADR
|
||||
// 0582). A later decision that places a controller again must first lift this check.
|
||||
//
|
||||
// It refuses, over the tree at root:
|
||||
//
|
||||
// - a directory modules/mesh-controller: the controller is no module of the catalogue;
|
||||
// - a manifest string naming the controller's seat, a state, verb or event of it, or one of its buckets
|
||||
// (`mesh-controller.<name>`, `seat:mesh-controller.<verb>`, `mesh-controller_<bucket>`), anywhere in the manifest;
|
||||
// - a manifest string that is the bare module name `mesh-controller` outside the module's own `config`, where a value
|
||||
// is the module's data (an issue owner naming the controller's repository is no dependency), not an address;
|
||||
// - a go.mod of a module or of a check requiring the controller's repository (its module path, github.com/novox/mesh-controller,
|
||||
// or the forge's git.novox.be/novox/mesh-controller): the repository is archived once its last module is delivered
|
||||
// elsewhere (novox/hq ADR 0593 §11), so nothing here may be built from it;
|
||||
// - a line of any module's file naming one of the controller's buckets (`mesh-controller_`): its buckets are removed
|
||||
// after the controller's end (ADR 0590 §6), so a reader of one reads nothing. A test (`_test.go`, `.test.ts`) reads
|
||||
// no live bucket, its names are its own data, and a `testdata` directory is a copy of text from elsewhere (a design,
|
||||
// a backlog): both are skipped.
|
||||
//
|
||||
// The rule is said once, here, for the catalogue: decoding stays the manifests check's (a manifest that is no JSON is
|
||||
// an error here, never judged).
|
||||
package controllerdeps
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Module is the controller's module name, and Bucket the prefix every bucket of its carries.
|
||||
const (
|
||||
Module = "mesh-controller"
|
||||
Bucket = Module + "_"
|
||||
)
|
||||
|
||||
// address is a string naming the controller's seat, one of its states, verbs or events, or one of its buckets.
|
||||
var address = regexp.MustCompile(`(^|[^A-Za-z0-9_-])(seat:)?mesh-controller([._][A-Za-z0-9])`)
|
||||
|
||||
// repository is a go.mod line naming the controller's repository by either of its module paths.
|
||||
var repository = regexp.MustCompile(`(github\.com|git\.novox\.be)/novox/mesh-controller(/|\s|$)`)
|
||||
|
||||
// Refusals is each reason the check gives over the tree at root, sorted; none when it passes.
|
||||
func Refusals(root string) ([]string, error) {
|
||||
var out []string
|
||||
if _, err := os.Lstat(filepath.Join(root, "modules", Module)); err == nil {
|
||||
out = append(out, Module+": the controller is no module of the catalogue: it is not placed and nothing may depend on it (novox/hq ADR 0590 §5)")
|
||||
}
|
||||
files, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, f := range files {
|
||||
raw, err := os.ReadFile(f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var doc any
|
||||
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", f, err)
|
||||
}
|
||||
module := filepath.Base(filepath.Dir(f))
|
||||
for _, s := range names(doc, false, nil) {
|
||||
out = append(out, fmt.Sprintf("%s: names %q: the controller is not placed, so no manifest reads, requires, invokes, "+
|
||||
"consumes, waits on or uses it; name the seat that holds the job now (novox/hq ADR 0590 §5)", module, s))
|
||||
}
|
||||
}
|
||||
mods, err := filepath.Glob(filepath.Join(root, "*", "*", "go.mod"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(mods)
|
||||
for _, f := range mods {
|
||||
if rel, _ := filepath.Rel(root, f); !strings.HasPrefix(rel, "modules"+string(filepath.Separator)) && !strings.HasPrefix(rel, "checks"+string(filepath.Separator)) {
|
||||
continue
|
||||
}
|
||||
raw, err := os.ReadFile(f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for n, l := range strings.Split(string(raw), "\n") {
|
||||
if repository.MatchString(l) {
|
||||
rel, _ := filepath.Rel(root, f)
|
||||
out = append(out, fmt.Sprintf("%s:%d: requires the controller's repository: it is archived once its last module is built elsewhere, so nothing here is built from it (novox/hq ADR 0593 §11)", filepath.ToSlash(rel), n+1))
|
||||
}
|
||||
}
|
||||
}
|
||||
buckets, err := bucketLines(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, buckets...)
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// names collects the strings of a decoded manifest that name the controller; inConfig is under the module's `config`.
|
||||
func names(v any, inConfig bool, into []string) []string {
|
||||
switch v := v.(type) {
|
||||
case string:
|
||||
if address.MatchString(v) || (!inConfig && strings.TrimSpace(v) == Module) {
|
||||
into = append(into, v)
|
||||
}
|
||||
case []any:
|
||||
for _, e := range v {
|
||||
into = names(e, inConfig, into)
|
||||
}
|
||||
case map[string]any:
|
||||
keys := make([]string, 0, len(v))
|
||||
for k := range v {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
into = names(v[k], inConfig || k == "config", into)
|
||||
}
|
||||
}
|
||||
return into
|
||||
}
|
||||
|
||||
// bucketLines is every line of a module's file, outside tests and testdata, that names one of the controller's buckets.
|
||||
func bucketLines(root string) ([]string, error) {
|
||||
var out []string
|
||||
modules := filepath.Join(root, "modules")
|
||||
err := filepath.WalkDir(modules, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) && path == modules {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
if n := d.Name(); n == "testdata" || n == "node_modules" || n == ".git" {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !d.Type().IsRegular() || strings.HasSuffix(d.Name(), "_test.go") || strings.HasSuffix(d.Name(), ".test.ts") {
|
||||
return nil
|
||||
}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
sc := bufio.NewScanner(f)
|
||||
sc.Buffer(make([]byte, 0, 64*1024), 16*1024*1024)
|
||||
rel, _ := filepath.Rel(root, path)
|
||||
for n := 1; sc.Scan(); n++ {
|
||||
if strings.Contains(sc.Text(), Bucket) {
|
||||
out = append(out, fmt.Sprintf("%s:%d: names the controller's bucket (%s…): its buckets are removed after its "+
|
||||
"end (novox/hq ADR 0590 §6); read the seat that holds the job now", filepath.ToSlash(rel), n, Bucket))
|
||||
}
|
||||
}
|
||||
return sc.Err()
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package controllerdeps
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// ADR 0590 §5 (issue 1126 B10): no manifest of the catalogue at this head reads, invokes, consumes, waits on, uses or
|
||||
// otherwise names the controller's seat, states or buckets, and no module's code names a bucket of the controller's.
|
||||
func TestNothingInTheCatalogueNamesTheController(t *testing.T) {
|
||||
got, err := Refusals("../..")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range got {
|
||||
t.Error(r)
|
||||
}
|
||||
}
|
||||
|
||||
// The check refuses every name of the controller, the lease and the entries once kept to its write among them, on a
|
||||
// tree of its own; a repository named mesh-controller (an issue's owner) and a word in prose are no dependency.
|
||||
func TestRefusalsJudgeATree(t *testing.T) {
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
files map[string]string
|
||||
want string // empty: passes
|
||||
}{
|
||||
{"the lease is refused like any read", map[string]string{"modules/a/module.json": `{"module":"a","reads":["mesh-controller.lease"]}`}, `a: names "mesh-controller.lease"`},
|
||||
{"a verb once kept to the write", map[string]string{"modules/b/module.json": `{"module":"b","invokes":["seat:mesh-controller.collect"]}`}, `b: names "seat:mesh-controller.collect"`},
|
||||
{"an event consumed", map[string]string{"modules/c/module.json": `{"module":"c","consumes":[{"event":"mesh-controller.artifacts-gone"}]}`}, "mesh-controller.artifacts-gone"},
|
||||
{"a requirement", map[string]string{"modules/d/module.json": `{"module":"d","requires":[{"name":"x","from":"mesh-controller"}]}`}, `d: names "mesh-controller"`},
|
||||
{"the controller's own module", map[string]string{"modules/mesh-controller/module.json": `{"module":"mesh-controller"}`}, "mesh-controller: the controller is no module of the catalogue"},
|
||||
{"a bucket in code", map[string]string{"modules/e/cmd/e/main.go": "package main\nconst b = \"mesh-controller_assignments\"\n"}, "modules/e/cmd/e/main.go:2: names the controller's bucket"},
|
||||
{"a requirement of the controller's repository", map[string]string{"modules/h/go.mod": "module h\n\nrequire github.com/novox/mesh-controller v0.0.0\n"}, "modules/h/go.mod:3: requires the controller's repository"},
|
||||
{"its forge path in a check", map[string]string{"checks/i/go.mod": "module i\n\nrequire (\n\tgit.novox.be/novox/mesh-controller/modules/route-proxy v0.1.0\n)\n"}, "checks/i/go.mod:4"},
|
||||
{"another repository of the forge passes", map[string]string{"modules/j/go.mod": "module j\n\nrequire git.novox.be/novox/mesh-sdk/go v0.1.0\n"}, ""},
|
||||
{"a bucket in a test is its data", map[string]string{"modules/e/cmd/e/main_test.go": "package main\n// mesh-controller_lease\n"}, ""},
|
||||
{"a fixture copied from elsewhere", map[string]string{"modules/e/cmd/e/testdata/x.md": "mesh-controller_rollouts\n"}, ""},
|
||||
{"an owner and prose pass", map[string]string{"modules/f/module.json": `{"module":"f","config":{"owners":{"default":["mesh-controller","mesh-controller modules/route-proxy"]}},"why":"the mesh-controller module is gone"}`}, ""},
|
||||
{"the seat's successors pass", map[string]string{"modules/g/module.json": `{"module":"g","reads":["mesh-placement.placement"],"invokes":["seat:mesh-hand-acts.hand-act"]}`}, ""},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
for path, text := range row.files {
|
||||
p := filepath.Join(root, path)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(text), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, err := Refusals(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if joined := strings.Join(got, "\n"); (row.want == "") != (joined == "") || !strings.Contains(joined, row.want) {
|
||||
t.Errorf("refusals %q, want %q", joined, row.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
module checks/controllerdeps
|
||||
|
||||
go 1.22
|
||||
@@ -0,0 +1,3 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (issue 1936).
|
||||
# every module's directory: its manifest, for a name of the controller, and its files, for one of the controller's buckets.
|
||||
modules
|
||||
@@ -2,4 +2,6 @@ module checks/credentialform
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f
|
||||
|
||||
require git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639 // indirect
|
||||
|
||||
@@ -1,2 +1,4 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639 h1:Zt7CfWt2S2W/fVgWNyW2u2+xwz051LXvGxVTNIyfhdM=
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639/go.mod h1:0dD1XEJBJ0G5UYDQxSZI84nuujb6m3tOpJl4C8rQVvg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f h1:jDsfDMd48MXY9Zershc3eRxnhmx2sSpUfQkLq5jAIds=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f/go.mod h1:08OomDXdwVN8/87nyDtv5R3rH/6LoP8SFFXz30bTYCM=
|
||||
|
||||
+13
-2
@@ -111,13 +111,24 @@ func forgeRequirements(dir string) ([]requirement, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("its go.mod cannot be read: %s", lastLine(out))
|
||||
}
|
||||
var mod struct{ Require []requirement }
|
||||
var mod struct {
|
||||
Require []requirement
|
||||
Replace []struct{ Old, New requirement }
|
||||
}
|
||||
if err := json.Unmarshal(out, &mod); err != nil {
|
||||
return nil, fmt.Errorf("its go.mod cannot be read: %w", err)
|
||||
}
|
||||
// A requirement the go.mod replaces to a directory is a package of the module's own repository (novox/hq ADR 0593 §1): it
|
||||
// is in the tree under check, so it is fetched from nowhere.
|
||||
local := map[string]bool{}
|
||||
for _, r := range mod.Replace {
|
||||
if r.New.Version == "" {
|
||||
local[r.Old.Path] = true
|
||||
}
|
||||
}
|
||||
var on []requirement
|
||||
for _, r := range mod.Require {
|
||||
if strings.HasPrefix(r.Path, forge) {
|
||||
if strings.HasPrefix(r.Path, forge) && !local[r.Path] {
|
||||
on = append(on, r)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -175,3 +175,28 @@ func TestTheRefOfAVersion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0593 §1: a module requires a sibling package of its own repository and replaces it to the directory beside it; that
|
||||
// package is no fetch from the forge (it is in the tree under check), so it is never looked for in a clone beside the check,
|
||||
// and the module builds with the SDK taken from there.
|
||||
func TestASiblingPackageReplacedToItsDirectoryIsNotFetched(t *testing.T) {
|
||||
offline(t)
|
||||
beside, commit := sdkBeside(t)
|
||||
version := "v0.1.14-0.20261012082113-" + commit[:12]
|
||||
root := t.TempDir()
|
||||
sibling := "git.novox.be/novox/mesh-catalog/modules/builder/runner"
|
||||
write(t, filepath.Join(root, "builder", "runner", "go.mod"), "module "+sibling+"\n\ngo 1.22\n\nrequire git.novox.be/novox/mesh-sdk/go "+version+"\n")
|
||||
write(t, filepath.Join(root, "builder", "runner", "run.go"), "package runner\n\nimport \"git.novox.be/novox/mesh-sdk/go/sdk\"\n\nfunc Run() string { return sdk.Hello() }\n")
|
||||
dir := filepath.Join(root, "user")
|
||||
write(t, filepath.Join(dir, "go.mod"), "module git.novox.be/novox/mesh-catalog/modules/user\n\ngo 1.22\n\nrequire (\n\t"+sibling+" v0.0.0\n"+
|
||||
"\tgit.novox.be/novox/mesh-sdk/go "+version+"\n)\n\nreplace "+sibling+" => ../builder/runner\n")
|
||||
write(t, filepath.Join(dir, "main.go"), "package main\n\nimport \""+sibling+"\"\n\nfunc main() { println(runner.Run()) }\n")
|
||||
modfile, err := resolve(dir, beside, t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("resolve: %v", err)
|
||||
}
|
||||
if modfile == "" {
|
||||
t.Fatal("resolve said the module fetches its dependencies, with nothing to fetch the SDK from")
|
||||
}
|
||||
builds(t, dir, modfile)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/environment
|
||||
|
||||
go 1.22
|
||||
@@ -0,0 +1,252 @@
|
||||
// Command environment refuses a module's code reading a MESH_* word of its environment that nothing gives it: not its
|
||||
// manifest, and not the node-engine or the tool runner for every process (the lead's ruling on A3n, issue 1966).
|
||||
//
|
||||
// A MESH_* word is the mesh's contract with a module's process: the manifest sets it (an `env` of a build artifact or a
|
||||
// resource, an `environment`'s `variables`, a `NAME=` line of a file the module ships for an env-file), or the mesh gives
|
||||
// it to every process (Given). A word read and set by nothing is a contract nobody keeps: the program runs on its
|
||||
// default or on nothing, and says so to nobody (MESH_REGISTRY_REACH answered every published reference without its
|
||||
// registry; MESH_CONFIG_CALLERS held every window shut). Found here, it is refused: the code reads what the mesh gives
|
||||
// (a manifest word, a config value through its tools, a seat's name), or the manifest sets it.
|
||||
//
|
||||
// A read is `os.Getenv` or `os.LookupEnv` of a literal or of a constant of the same directory (Go), and `env.MESH_*` or
|
||||
// `env["MESH_*"]` (TypeScript, `process.env` included); tests, test directories, node_modules and dist are not read. The
|
||||
// modules still reading such a word are listed in the file `unset` beside this check, `<module> <WORD>` a line, until
|
||||
// each is fixed: a line no longer found is refused, so the list only shrinks.
|
||||
//
|
||||
// go run . <catalogue root>
|
||||
//
|
||||
// Exits 1 on a finding.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Given is every MESH_* word the mesh gives a module's processes whatever their manifest says, and who gives it.
|
||||
var Given = map[string]string{
|
||||
// The tool runner, to every bundle and to a module's own runtime (mesh-tools node-tools runtime.BundleEnv).
|
||||
"MESH_MODULE": "the tool runner",
|
||||
"MESH_SERVED_MODULE": "the tool runner",
|
||||
"MESH_NODE": "the tool runner",
|
||||
// The node-engine, to the tool runner it runs as the operator's account; its bundles inherit them (mesh-host
|
||||
// compose/resolution runtime.go).
|
||||
"MESH_OPERATOR_ACCOUNT": "the node-engine",
|
||||
"MESH_OPERATOR_HOME": "the node-engine",
|
||||
// The node-engine, to every module that yields to the node's builds (ADR 0553 §3; catalogue.BuildCPUShareWord).
|
||||
"MESH_BUILD_CPU_SHARE": "the node-engine",
|
||||
}
|
||||
|
||||
const listFile = "unset"
|
||||
|
||||
var (
|
||||
goRead = regexp.MustCompile(`os\.(?:Getenv|LookupEnv)\(\s*(?:"(MESH_[A-Z0-9_]+)"|([A-Za-z_][A-Za-z0-9_]*))\s*\)`)
|
||||
goConst = regexp.MustCompile(`\b([A-Za-z_][A-Za-z0-9_]*)\s*(?:string\s*)?=\s*"(MESH_[A-Z0-9_]+)"`)
|
||||
tsRead = regexp.MustCompile(`\benv(?:\.|\[\s*["'])(MESH_[A-Z0-9_]+)`)
|
||||
envLine = regexp.MustCompile(`(?m)^\s*(?:export\s+)?(MESH_[A-Z0-9_]+)=`)
|
||||
)
|
||||
|
||||
// skipped are the directories whose code no process of the module runs.
|
||||
var skipped = map[string]bool{"node_modules": true, "dist": true, "test": true, "testdata": true, ".git": true}
|
||||
|
||||
// Check answers, per module, each MESH_* word its code reads that its manifest does not set and the mesh does not
|
||||
// give, with the files reading it.
|
||||
func Check(root string) (map[string]map[string][]string, error) {
|
||||
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(manifests)
|
||||
out := map[string]map[string][]string{}
|
||||
for _, path := range manifests {
|
||||
dir := filepath.Dir(path)
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var manifest any
|
||||
if err := json.Unmarshal(raw, &manifest); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
set := map[string]bool{}
|
||||
setBy(manifest, set)
|
||||
reads := map[string]map[string]bool{}
|
||||
consts := map[string]map[string]string{} // directory -> constant -> word
|
||||
var goFiles []string
|
||||
err = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
if p != dir && skipped[d.Name()] {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
name := d.Name()
|
||||
text, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel, _ := filepath.Rel(dir, p)
|
||||
rel = filepath.ToSlash(rel)
|
||||
switch {
|
||||
case strings.HasSuffix(name, "_test.go"), strings.HasSuffix(name, ".test.ts"), strings.HasSuffix(name, ".d.ts"):
|
||||
case strings.HasSuffix(name, ".go"):
|
||||
goFiles = append(goFiles, p)
|
||||
c := consts[filepath.Dir(p)]
|
||||
if c == nil {
|
||||
c = map[string]string{}
|
||||
consts[filepath.Dir(p)] = c
|
||||
}
|
||||
for _, m := range goConst.FindAllStringSubmatch(string(text), -1) {
|
||||
c[m[1]] = m[2]
|
||||
}
|
||||
case strings.HasSuffix(name, ".ts"):
|
||||
for _, m := range tsRead.FindAllStringSubmatch(string(text), -1) {
|
||||
read(reads, m[1], rel)
|
||||
}
|
||||
case name != "module.json":
|
||||
// A file the module ships: a NAME= line is a word an env-file gives.
|
||||
for _, m := range envLine.FindAllStringSubmatch(string(text), -1) {
|
||||
set[m[1]] = true
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, p := range goFiles {
|
||||
text, _ := os.ReadFile(p)
|
||||
rel, _ := filepath.Rel(dir, p)
|
||||
for _, m := range goRead.FindAllStringSubmatch(string(text), -1) {
|
||||
word := m[1]
|
||||
if word == "" {
|
||||
word = consts[filepath.Dir(p)][m[2]]
|
||||
}
|
||||
if word != "" {
|
||||
read(reads, word, filepath.ToSlash(rel))
|
||||
}
|
||||
}
|
||||
}
|
||||
module := filepath.Base(dir)
|
||||
for word, files := range reads {
|
||||
if set[word] || Given[word] != "" {
|
||||
continue
|
||||
}
|
||||
if out[module] == nil {
|
||||
out[module] = map[string][]string{}
|
||||
}
|
||||
for f := range files {
|
||||
out[module][word] = append(out[module][word], f)
|
||||
}
|
||||
sort.Strings(out[module][word])
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func read(reads map[string]map[string]bool, word, file string) {
|
||||
if reads[word] == nil {
|
||||
reads[word] = map[string]bool{}
|
||||
}
|
||||
reads[word][file] = true
|
||||
}
|
||||
|
||||
// setBy collects the words a manifest sets: every key of an `env` object and of an `environment`'s `variables`.
|
||||
func setBy(v any, set map[string]bool) {
|
||||
switch v := v.(type) {
|
||||
case map[string]any:
|
||||
for k, e := range v {
|
||||
if m, ok := e.(map[string]any); ok && (k == "env" || k == "variables") {
|
||||
for word := range m {
|
||||
set[word] = true
|
||||
}
|
||||
}
|
||||
setBy(e, set)
|
||||
}
|
||||
case []any:
|
||||
for _, e := range v {
|
||||
setBy(e, set)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Listed is the list's `<module> <WORD>` lines; a line starting with # and a blank line are not entries.
|
||||
func Listed(path string) (map[string]bool, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
out := map[string]bool{}
|
||||
s := bufio.NewScanner(f)
|
||||
for s.Scan() {
|
||||
line := strings.Join(strings.Fields(s.Text()), " ")
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
out[line] = true
|
||||
}
|
||||
return out, s.Err()
|
||||
}
|
||||
|
||||
// Judge answers the reads not on the list (refused) and the list's lines no read stands for (stale).
|
||||
func Judge(found map[string]map[string][]string, listed map[string]bool) (refused, stale []string) {
|
||||
seen := map[string]bool{}
|
||||
for module, words := range found {
|
||||
for word := range words {
|
||||
line := module + " " + word
|
||||
seen[line] = true
|
||||
if !listed[line] {
|
||||
refused = append(refused, line)
|
||||
}
|
||||
}
|
||||
}
|
||||
for line := range listed {
|
||||
if !seen[line] {
|
||||
stale = append(stale, line)
|
||||
}
|
||||
}
|
||||
sort.Strings(refused)
|
||||
sort.Strings(stale)
|
||||
return refused, stale
|
||||
}
|
||||
|
||||
func main() {
|
||||
root := "."
|
||||
if len(os.Args) > 1 {
|
||||
root = os.Args[1]
|
||||
}
|
||||
found, err := Check(root)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
listed, err := Listed(listFile)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
refused, stale := Judge(found, listed)
|
||||
for _, line := range refused {
|
||||
module, word, _ := strings.Cut(line, " ")
|
||||
fmt.Printf("%s reads %s (%s), which its manifest does not set and the mesh gives no process: set it in the "+
|
||||
"manifest, or read what the mesh gives\n", module, word, strings.Join(found[module][word], ", "))
|
||||
}
|
||||
for _, line := range stale {
|
||||
fmt.Printf("checks/environment/%s names %q, which no code reads unset any more: take it off the list\n", listFile, line)
|
||||
}
|
||||
if len(refused) > 0 || len(stale) > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Printf("environment: every MESH_* word a module reads is set by its manifest or given by the mesh (%d still listed)\n", len(listed))
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// firstCount is how many lines the list held when it was first written: it only shrinks.
|
||||
const firstCount = 63
|
||||
|
||||
func module(t *testing.T, root, name, manifest string, files map[string]string) {
|
||||
t.Helper()
|
||||
dir := filepath.Join(root, "modules", name)
|
||||
for path, text := range map[string]string{"module.json": manifest} {
|
||||
files[path] = text
|
||||
}
|
||||
for path, text := range files {
|
||||
full := filepath.Join(dir, filepath.FromSlash(path))
|
||||
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(full, []byte(text), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A word read and set by nothing is found, in Go (a literal, a constant of the same directory) and in TypeScript; one
|
||||
// the manifest sets (an env, an environment's variables, an env-file line it ships) or the mesh gives every process is
|
||||
// not; tests, node_modules and dist are not read.
|
||||
func TestAWordReadThatNothingSetsIsFound(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
module(t, root, "reader", `{"module":"reader","build":{"artifacts":[{"name":"tools","env":{"MESH_SET":"x"}}]},`+
|
||||
`"resources":[{"id":"p","type":"process","environment":{"variables":{"MESH_VARIABLE":"y"}}}]}`, map[string]string{
|
||||
"cmd/a/main.go": `package main; const reach = "MESH_REACH"; var _ = os.Getenv(reach); var _ = os.Getenv("MESH_SET")`,
|
||||
"cmd/a/other.go": `package main; var _, _ = os.LookupEnv("MESH_LITERAL"); var _ = os.Getenv("MESH_MODULE")`,
|
||||
"cmd/a/write.go": `package main; var _ = []string{"MESH_WRITTEN_FOR_A_CHILD=1"}`,
|
||||
"cmd/a/a_test.go": `package main; var _ = os.Getenv("MESH_IN_A_TEST")`,
|
||||
"client.ts": `const a = process.env.MESH_TS ?? "d"; const b = env["MESH_BRACKET"]; const c = env.MESH_VARIABLE;`,
|
||||
"files/env": "MESH_FROM_FILE=${config:x}\n",
|
||||
"index.ts": `const f = env.MESH_FROM_FILE;`,
|
||||
"test/x.test.ts": `env.MESH_IN_A_TEST_DIR`,
|
||||
"node_modules/x.ts": `env.MESH_VENDORED`,
|
||||
})
|
||||
module(t, root, "quiet", `{"module":"quiet"}`, map[string]string{"main.go": `package main; var _ = os.Getenv("MESH_NODE")`})
|
||||
got, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string]map[string][]string{"reader": {
|
||||
"MESH_REACH": {"cmd/a/main.go"},
|
||||
"MESH_LITERAL": {"cmd/a/other.go"},
|
||||
"MESH_TS": {"client.ts"},
|
||||
"MESH_BRACKET": {"client.ts"},
|
||||
}}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("found %v, want %v", got, want)
|
||||
}
|
||||
refused, stale := Judge(got, map[string]bool{"reader MESH_REACH": true, "reader MESH_GONE": true})
|
||||
if !reflect.DeepEqual(refused, []string{"reader MESH_BRACKET", "reader MESH_LITERAL", "reader MESH_TS"}) ||
|
||||
!reflect.DeepEqual(stale, []string{"reader MESH_GONE"}) {
|
||||
t.Errorf("refused %v, stale %v", refused, stale)
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue itself: every word a module reads is set or given, but the list's, and the list only shrinks.
|
||||
func TestTheCatalogueReadsOnlyWordsTheMeshSets(t *testing.T) {
|
||||
found, err := Check(filepath.Join("..", ".."))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
listed, err := Listed(listFile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
refused, stale := Judge(found, listed)
|
||||
for _, line := range refused {
|
||||
t.Errorf("%s: read, set by nothing, and not on checks/environment/unset", line)
|
||||
}
|
||||
for _, line := range stale {
|
||||
t.Errorf("checks/environment/unset names %q, which no code reads unset any more", line)
|
||||
}
|
||||
if len(listed) > firstCount {
|
||||
t.Errorf("checks/environment/unset holds %d lines, more than its first %d: it only shrinks", len(listed), firstCount)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
|
||||
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
|
||||
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
|
||||
# Check("../.."): every manifest and the code and files of its module.
|
||||
modules
|
||||
@@ -0,0 +1,67 @@
|
||||
# The MESH_* words a module's code still reads that its manifest does not set and the mesh gives no process,
|
||||
# `<module> <WORD>` a line, as checks/environment first found them (issue 1966). Each is a contract nobody keeps: fix
|
||||
# it (the manifest sets it, or the code reads what the mesh gives) and take its line off. A line no code reads unset any
|
||||
# more is refused, so this list only shrinks; a new read set by nothing is refused, never added here.
|
||||
baserow MESH_BASEROW_EMAIL
|
||||
baserow MESH_BASEROW_HOST
|
||||
baserow MESH_BASEROW_PASSWORD
|
||||
claude-code MESH_AGENT_ACCOUNT
|
||||
claude-code MESH_AGENT_HOME
|
||||
claude-licence-manager MESH_ANTHROPIC_TOKEN_ENDPOINT
|
||||
claude-licence-manager MESH_ANTHROPIC_USAGE_ENDPOINT
|
||||
confluence MESH_CONFLUENCE_EMAIL
|
||||
confluence MESH_CONFLUENCE_TOKEN
|
||||
confluence MESH_CONFLUENCE_URL
|
||||
disk-load MESH_DISK_MEASURE_EVERY
|
||||
gitlab MESH_GITLAB_TOKEN
|
||||
gitlab MESH_GITLAB_URL
|
||||
grafana MESH_GRAFANA_PASSWORD
|
||||
grafana MESH_GRAFANA_TOKEN
|
||||
grafana MESH_GRAFANA_USER
|
||||
home-assistant MESH_HOMEASSISTANT_TOKEN
|
||||
home-assistant MESH_HOMEASSISTANT_WAIT_SECONDS
|
||||
home-assistant MESH_HOMEASSISTANT_WATCH
|
||||
icecast MESH_ICECAST_ADMIN_PASSWORD
|
||||
icecast MESH_ICECAST_ADMIN_USER
|
||||
influxdb MESH_INFLUXDB_ORG
|
||||
influxdb MESH_INFLUXDB_TOKEN
|
||||
jira MESH_JIRA_EMAIL
|
||||
jira MESH_JIRA_TOKEN
|
||||
jira MESH_JIRA_URL
|
||||
letta MESH_LETTA_PASSWORD
|
||||
mailu MESH_MAILU_API_KEY
|
||||
mailu MESH_MAILU_DOMAIN
|
||||
mesh-cli MESH_CONSOLE
|
||||
mesh-cli MESH_VERB
|
||||
mesh-files MESH_OWNED_PATHS
|
||||
mesh-files MESH_SECRET_PLACES
|
||||
mesh-reads MESH_SECRET_PLACES
|
||||
minio MESH_MINIO_ROOT_PASSWORD
|
||||
mongodb MESH_MONGODB_AUTHSOURCE
|
||||
mongodb MESH_MONGODB_HOST
|
||||
mongodb MESH_MONGODB_PASSWORD
|
||||
mongodb MESH_MONGODB_PORT
|
||||
mongodb MESH_MONGODB_USER
|
||||
mosquitto MESH_MQTT_ADMIN_USER
|
||||
mosquitto MESH_MQTT_BROKER_GID
|
||||
mosquitto MESH_MQTT_BROKER_UID
|
||||
mosquitto MESH_MQTT_HOST
|
||||
mosquitto MESH_MQTT_PASSWORD
|
||||
mosquitto MESH_MQTT_PORT
|
||||
mssql MESH_MSSQL_HOST
|
||||
mssql MESH_MSSQL_PASSWORD
|
||||
mssql MESH_MSSQL_PORT
|
||||
mssql MESH_MSSQL_READER_PASSWORD
|
||||
mssql MESH_MSSQL_USER
|
||||
nextcloud MESH_NEXTCLOUD_ADMIN_PASSWORD
|
||||
nextcloud MESH_NEXTCLOUD_CONTAINER
|
||||
nodered MESH_NODERED_TOKEN
|
||||
nodered MESH_NODERED_WAIT_SECONDS
|
||||
redis MESH_REDIS_HOST
|
||||
redis MESH_REDIS_PASSWORD
|
||||
redis MESH_REDIS_PORT
|
||||
redis MESH_REDIS_USERNAME
|
||||
restic MESH_BACKUP_HOUR
|
||||
unifi MESH_UNIFI_PASSWORD
|
||||
unifi MESH_UNIFI_SITE
|
||||
unifi MESH_UNIFI_USERNAME
|
||||
@@ -1,4 +1,4 @@
|
||||
// Package failurekit refuses a build module whose tests do not run mesh-sdk's conformance kit (issue 1901, its thorough
|
||||
// Package failurekit refuses a build module whose tests do not run the build runner's conformance kit (issue 1901, its thorough
|
||||
// fix): a module that claims a node-build-* seat and serves a test step must hold a test that calls conformance.Run, so
|
||||
// its parse of the test runner's output is held to the failure contract (a failing test named with its cause, never a
|
||||
// passing line in its place). A build module with no test step (mesh-build-image builds images and runs no tests) is
|
||||
@@ -71,7 +71,7 @@ func Judge(root string) ([]Judged, error) {
|
||||
}
|
||||
if !runs {
|
||||
refused = append(refused, fmt.Sprintf("it claims %s and serves the test step %s, and no test of it calls conformance.Run "+
|
||||
"(mesh-sdk go/buildstep/conformance): run the kit with the module's own parse path (issue 1901)", seat, test))
|
||||
"(mesh-build-go buildstep/conformance): run the kit with the module's own parse path (issue 1901)", seat, test))
|
||||
}
|
||||
}
|
||||
// Every build module keeps steps, so every one runs the left-running case, test step or not (issue 1890).
|
||||
@@ -81,7 +81,7 @@ func Judge(root string) ([]Judged, error) {
|
||||
}
|
||||
if !sweeps {
|
||||
refused = append(refused, fmt.Sprintf("it claims %s, and no test of it calls conformance.LeftRunning "+
|
||||
"(mesh-sdk go/buildstep/conformance): run the kit's left-running case with the module's own start sweep (issue 1890)", seat))
|
||||
"(mesh-build-go buildstep/conformance): run the kit's left-running case with the module's own start sweep (issue 1890)", seat))
|
||||
}
|
||||
j.Refused = strings.Join(refused, "; ")
|
||||
out = append(out, j)
|
||||
@@ -121,7 +121,7 @@ func runsTheKit(dir, call string) (bool, error) {
|
||||
return err
|
||||
}
|
||||
found = strings.Contains(string(raw), call) &&
|
||||
strings.Contains(string(raw), `"git.novox.be/novox/mesh-sdk/go/buildstep/conformance"`)
|
||||
strings.Contains(string(raw), `"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep/conformance"`)
|
||||
return nil
|
||||
})
|
||||
return found, err
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
// cases is refused, and a module that claims no node-build-* seat is not judged.
|
||||
func TestABuildModuleWithATestStepMustRunTheKit(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
const imp = "package main\n\nimport \"git.novox.be/novox/mesh-sdk/go/buildstep/conformance\"\n\n"
|
||||
const imp = "package main\n\nimport \"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep/conformance\"\n\n"
|
||||
const sweep = "func TestSweep(t *testing.T) { conformance.LeftRunning(t, nil) }\n"
|
||||
kit := imp + "func TestKit(t *testing.T) { conformance.Run(t, conformance.GoTest, nil) }\n" + sweep
|
||||
for path, body := range map[string]string{
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
// Package gitignored refuses an ignore pattern that hides a module's source (catalogue review F1, and its kind): the
|
||||
// bare pattern `mesh-build-node` in a module's .gitignore, meant for the binary `go build` leaves, also matched the
|
||||
// directory cmd/mesh-build-node, so git ignored every new file there and publish.go was never committed. A path under
|
||||
// a module's cmd/ or internal/ is source: no .gitignore, the module's or the catalogue's, may ignore one, tracked or
|
||||
// new. A binary is ignored by an anchored pattern (`/mesh-build-node`, `/cmd/mesh-build-node/mesh-build-node`).
|
||||
package gitignored
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Probe is the file name asked of each source directory: a Go file a change might add there.
|
||||
const Probe = "added_by_a_change.go"
|
||||
|
||||
// Hidden is one source path an ignore pattern hides, and the pattern (its file, line and text, as git names it).
|
||||
type Hidden struct {
|
||||
Path, Pattern string
|
||||
}
|
||||
|
||||
// Sources are the source directories of root's modules, relative to root: every directory under modules/*/cmd and
|
||||
// modules/*/internal (node_modules left out).
|
||||
func Sources(root string) ([]string, error) {
|
||||
mods, err := filepath.Glob(filepath.Join(root, "modules", "*"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
for _, m := range mods {
|
||||
for _, top := range []string{"cmd", "internal"} {
|
||||
base := filepath.Join(m, top)
|
||||
if st, err := os.Stat(base); err != nil || !st.IsDir() {
|
||||
continue
|
||||
}
|
||||
err := filepath.WalkDir(base, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if d.Name() == "node_modules" {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
rel, _ := filepath.Rel(root, p)
|
||||
out = append(out, filepath.ToSlash(rel))
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// isSource is whether a file is source a module's build reads: Go and TypeScript files and a Go module's own files.
|
||||
// What a build or a test script leaves beside them (a binary, compiled JavaScript) may be ignored.
|
||||
func isSource(name string) bool {
|
||||
switch filepath.Ext(name) {
|
||||
case ".go", ".ts", ".tsx":
|
||||
return true
|
||||
}
|
||||
return name == "go.mod" || name == "go.sum"
|
||||
}
|
||||
|
||||
// Find asks git which of the source paths root's ignore files hide: each source directory, a Go file added in it, and
|
||||
// every source file there (isSource). It reads the .gitignore files of root's tree through a repository of its own, so it judges a git
|
||||
// checkout and a plain folder alike, and no one's global ignores count.
|
||||
func Find(root string, dirs []string) ([]Hidden, error) {
|
||||
gitDir, err := os.MkdirTemp("", "gitignored-")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer os.RemoveAll(gitDir)
|
||||
if out, err := exec.Command("git", "init", "-q", "--bare", gitDir).CombinedOutput(); err != nil {
|
||||
return nil, fmt.Errorf("git init: %v: %s", err, out)
|
||||
}
|
||||
var ask bytes.Buffer
|
||||
for _, d := range dirs {
|
||||
ask.WriteString(d + "/\x00" + d + "/" + Probe + "\x00")
|
||||
entries, err := os.ReadDir(filepath.Join(root, d))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() && isSource(e.Name()) {
|
||||
ask.WriteString(d + "/" + e.Name() + "\x00")
|
||||
}
|
||||
}
|
||||
}
|
||||
abs, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cmd := exec.Command("git", "--git-dir="+gitDir, "--work-tree="+abs, "-c", "core.excludesFile=/dev/null",
|
||||
"check-ignore", "--no-index", "-v", "-z", "--stdin")
|
||||
cmd.Dir = abs
|
||||
cmd.Stdin = &ask
|
||||
out, err := cmd.Output()
|
||||
if ee, ok := err.(*exec.ExitError); ok && ee.ExitCode() == 1 {
|
||||
return nil, nil // nothing is ignored
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("git check-ignore: %w", err)
|
||||
}
|
||||
// -v -z answers source, line, pattern and path, each ended by NUL.
|
||||
f := strings.Split(strings.TrimRight(string(out), "\x00"), "\x00")
|
||||
var hidden []Hidden
|
||||
for i := 0; i+3 < len(f); i += 4 {
|
||||
if f[i+2] == "" || strings.HasPrefix(f[i+2], "!") {
|
||||
continue // a negation keeps the path
|
||||
}
|
||||
hidden = append(hidden, Hidden{Path: f[i+3], Pattern: fmt.Sprintf("%s:%s: %s", f[i], f[i+1], f[i+2])})
|
||||
}
|
||||
return hidden, nil
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package gitignored
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The kind of catalogue review F1: a bare pattern hides the command directory it shares a name with; an anchored one
|
||||
// hides the binary alone; a glob that ignores source is refused too; a negation keeps a path.
|
||||
func TestFindNamesEachPatternThatHidesSource(t *testing.T) {
|
||||
for name, c := range map[string]struct {
|
||||
ignore string
|
||||
hidden []string
|
||||
}{
|
||||
"a bare name hides cmd/<name>": {"nfs-server\n", []string{"modules/m/cmd/nfs-server/", "modules/m/cmd/nfs-server/" + Probe, "modules/m/cmd/nfs-server/main.go"}},
|
||||
"anchored to the binary": {"/nfs-server\n/cmd/nfs-server/nfs-server\n", nil},
|
||||
"a glob ignoring Go source": {"*.go\n!main.go\n", []string{"modules/m/cmd/" + Probe, "modules/m/cmd/nfs-server/" + Probe,
|
||||
"modules/m/internal/" + Probe, "modules/m/internal/x/" + Probe, "modules/m/internal/x/x.go"}},
|
||||
"a pattern over internal/": {"internal/\n", []string{"modules/m/internal/", "modules/m/internal/" + Probe, "modules/m/internal/x/",
|
||||
"modules/m/internal/x/" + Probe, "modules/m/internal/x/x.go"}},
|
||||
"nothing but generated JavaScript": {"*.js\n", nil},
|
||||
} {
|
||||
root := t.TempDir()
|
||||
for f, b := range map[string]string{
|
||||
"modules/m/.gitignore": c.ignore,
|
||||
"modules/m/cmd/nfs-server/main.go": "package main\n",
|
||||
"modules/m/cmd/nfs-server/run.js": "",
|
||||
"modules/m/internal/x/x.go": "package x\n",
|
||||
"modules/m/node_modules/y/index.go": "",
|
||||
"modules/other/cmd/other/other.go": "package main\n",
|
||||
} {
|
||||
p := filepath.Join(root, f)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(b), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
dirs, err := Sources(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := Find(root, dirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var paths []string
|
||||
for _, h := range got {
|
||||
paths = append(paths, h.Path)
|
||||
if !strings.HasPrefix(h.Pattern, "modules/m/.gitignore:") {
|
||||
t.Errorf("%s: %s is named hidden by %q, want the module's .gitignore", name, h.Path, h.Pattern)
|
||||
}
|
||||
}
|
||||
if strings.Join(paths, ",") != strings.Join(c.hidden, ",") {
|
||||
t.Errorf("%s: hidden %v, want %v", name, paths, c.hidden)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue's own tree: no ignore pattern hides a path under a module's cmd/ or internal/.
|
||||
func TestNoIgnorePatternHidesAModulesSource(t *testing.T) {
|
||||
root := filepath.Join("..", "..")
|
||||
dirs, err := Sources(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(dirs) == 0 {
|
||||
t.Fatal("no module source directory found: the check reads nothing")
|
||||
}
|
||||
hidden, err := Find(root, dirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, h := range hidden {
|
||||
t.Errorf("%s is hidden by %s: a module's source is never ignored; anchor the pattern to the binary (/<name>, /cmd/<name>/<name>)", h.Path, h.Pattern)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/gitignored
|
||||
|
||||
go 1.22
|
||||
@@ -0,0 +1,3 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/gofmt
|
||||
|
||||
go 1.22
|
||||
@@ -0,0 +1,58 @@
|
||||
// Package gofmt refuses a Go file of the catalogue that gofmt would change (the second of a kind in one milestone:
|
||||
// buildstep's generated.go and claude-code's config_test.go reached a join unformatted). A check here holds every
|
||||
// module to it, where a module's own tests hold only themselves.
|
||||
package gofmt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"go/format"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Files are the Go files under root, relative to it: every .go file but those in .git, node_modules and a testdata
|
||||
// directory (a test's fixture may be anything).
|
||||
func Files(root string) ([]string, error) {
|
||||
var out []string
|
||||
err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
switch d.Name() {
|
||||
case ".git", "node_modules", "testdata":
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if strings.HasSuffix(d.Name(), ".go") && d.Type().IsRegular() {
|
||||
rel, _ := filepath.Rel(root, p)
|
||||
out = append(out, filepath.ToSlash(rel))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
sort.Strings(out)
|
||||
return out, err
|
||||
}
|
||||
|
||||
// Unformatted are the files gofmt would change, and why for one that does not parse.
|
||||
func Unformatted(root string, files []string) (map[string]string, error) {
|
||||
out := map[string]string{}
|
||||
for _, f := range files {
|
||||
src, err := os.ReadFile(filepath.Join(root, f))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
got, err := format.Source(src)
|
||||
switch {
|
||||
case err != nil:
|
||||
out[f] = "it does not parse: " + err.Error()
|
||||
case !bytes.Equal(got, src):
|
||||
out[f] = "gofmt changes it: run gofmt -w " + f
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package gofmt
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestUnformattedNamesWhatGofmtChanges(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
for f, src := range map[string]string{
|
||||
"modules/a/a.go": "package a\n",
|
||||
"modules/b/b.go": "package b\nimport (\n\t\"os\"\n\t\"fmt\"\n)\nvar _, _ = fmt.Sprint, os.Exit\n",
|
||||
"modules/c/testdata/c.go": "package c",
|
||||
"modules/d/node_modules/d.go": "package d",
|
||||
} {
|
||||
p := filepath.Join(root, f)
|
||||
_ = os.MkdirAll(filepath.Dir(p), 0o755)
|
||||
if err := os.WriteFile(p, []byte(src), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
files, err := Files(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := Unformatted(root, files)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(files) != 2 || len(got) != 1 || got["modules/b/b.go"] == "" {
|
||||
t.Errorf("files %v, unformatted %v; want a.go and b.go read, b.go named", files, got)
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue's own Go files are as gofmt writes them.
|
||||
func TestTheCataloguesGoIsFormatted(t *testing.T) {
|
||||
root := filepath.Join("..", "..")
|
||||
files, err := Files(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(files) == 0 {
|
||||
t.Fatal("no Go file found: the check reads nothing")
|
||||
}
|
||||
got, err := Unformatted(root, files)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for f, why := range got {
|
||||
t.Errorf("%s: %s", f, why)
|
||||
}
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
# The calls of the controller's hand-act log a manifest of this catalogue may still invoke, each `<module> <verb> <why>`:
|
||||
# the fallback of a recorder (mesh-sdk handacts.Recorder) while the seat mesh-hand-acts has no holder that answers, until
|
||||
# the controller's lease is written to a holder that is no controller. The check of this directory (handacts) refuses any
|
||||
# other, and an entry whose module no longer invokes it: the list only shrinks, and it is empty before the lease write
|
||||
# (novox/hq ADR 0565 §11; issue 1953, issue 774).
|
||||
mesh-config hand-act fallback until the lease write: its sets, clears and held values record through mesh-hand-acts first
|
||||
mesh-config warranted fallback until the lease write: its warrants record through mesh-hand-acts first
|
||||
mesh-healers drill fallback until the lease write: a drill records through mesh-hand-acts first
|
||||
@@ -1,3 +0,0 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/handacts
|
||||
|
||||
go 1.22
|
||||
@@ -1,109 +0,0 @@
|
||||
// Command handacts refuses a manifest that records into the controller's hand-act log (novox/hq ADR 0565 §11): every
|
||||
// recorder records through the seat mesh-hand-acts from its delivery, so no manifest invokes
|
||||
// `seat:mesh-controller.hand-act`, `.warranted` or `.drill`. The recorders' fallbacks to the controller's log while the
|
||||
// seat has no holder are named in checks/handacts/fallbacks, each module and verb with why; that list only shrinks, and
|
||||
// an entry whose module no longer invokes the verb is refused too.
|
||||
//
|
||||
// go run . <catalog root>
|
||||
//
|
||||
// Exits 1 on a finding.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Verbs are the controller's hand-act verbs, as a manifest invokes them, and Allowed the list of named fallbacks.
|
||||
var Verbs = []string{"hand-act", "warranted", "drill"}
|
||||
|
||||
const (
|
||||
invokePrefix = "seat:mesh-controller."
|
||||
Allowed = "checks/handacts/fallbacks"
|
||||
)
|
||||
|
||||
// Findings is every refusal in the catalog at root, sorted.
|
||||
func Findings(root string) ([]string, error) {
|
||||
allowed := map[string]bool{}
|
||||
raw, err := os.ReadFile(filepath.Join(root, Allowed))
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return nil, err
|
||||
}
|
||||
for _, line := range strings.Split(string(raw), "\n") {
|
||||
if f := strings.Fields(line); len(f) >= 3 && !strings.HasPrefix(f[0], "#") {
|
||||
allowed[f[0]+" "+f[1]] = true
|
||||
} else if len(f) > 0 && !strings.HasPrefix(f[0], "#") {
|
||||
return nil, fmt.Errorf("%s: %q is not `<module> <verb> <why>`", Allowed, line)
|
||||
}
|
||||
}
|
||||
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
invoked := map[string]bool{}
|
||||
for _, path := range manifests {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Invokes []string `json:"invokes"`
|
||||
}
|
||||
if json.Unmarshal(raw, &m) != nil || m.Module == "mesh-controller" {
|
||||
continue
|
||||
}
|
||||
for _, inv := range m.Invokes {
|
||||
verb, ok := strings.CutPrefix(strings.TrimSpace(inv), invokePrefix)
|
||||
if !ok || !contains(Verbs, verb) {
|
||||
continue
|
||||
}
|
||||
invoked[m.Module+" "+verb] = true
|
||||
if !allowed[m.Module+" "+verb] {
|
||||
out = append(out, fmt.Sprintf("%s invokes %s%s: record through seat:mesh-hand-acts.%s (novox/hq ADR 0565 §11), "+
|
||||
"or name the fallback in %s", m.Module, invokePrefix, verb, verb, Allowed))
|
||||
}
|
||||
}
|
||||
}
|
||||
for entry := range allowed {
|
||||
if !invoked[entry] {
|
||||
out = append(out, fmt.Sprintf("%s names %q, which is no longer invoked: take it off the list, which only shrinks",
|
||||
Allowed, entry))
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func contains(list []string, s string) bool {
|
||||
for _, x := range list {
|
||||
if x == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) != 2 {
|
||||
fmt.Fprintln(os.Stderr, "usage: handacts <catalog root>")
|
||||
os.Exit(2)
|
||||
}
|
||||
found, err := Findings(os.Args[1])
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
for _, f := range found {
|
||||
fmt.Println(f)
|
||||
}
|
||||
if len(found) > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("no manifest records into the controller's hand-act log but the fallbacks named")
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// No manifest invokes the controller's hand-act verbs but a named fallback; a named fallback no longer invoked is
|
||||
// refused (novox/hq ADR 0565 §11's catalog test).
|
||||
func TestFindings(t *testing.T) {
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
files map[string]string
|
||||
want string // empty: clean
|
||||
}{
|
||||
{"clean", map[string]string{"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-hand-acts.hand-act"]}`}, ""},
|
||||
{"an unnamed call", map[string]string{"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-controller.warranted"]}`},
|
||||
"a invokes seat:mesh-controller.warranted"},
|
||||
{"a named fallback, and the controller's own", map[string]string{
|
||||
"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-controller.drill"]}`,
|
||||
"modules/mesh-controller/module.json": `{"module":"mesh-controller","invokes":["seat:mesh-controller.hand-act"]}`,
|
||||
Allowed: "# why\na drill fallback until the lease write\n"}, ""},
|
||||
{"a fallback no longer invoked", map[string]string{Allowed: "gone hand-act fallback\n"}, `names "gone hand-act"`},
|
||||
{"another verb of the controller", map[string]string{"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-controller.conditions"]}`}, ""},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
for path, text := range row.files {
|
||||
p := filepath.Join(root, path)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(text), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, err := Findings(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if all := strings.Join(got, "\n"); (row.want == "") != (all == "") || !strings.Contains(all, row.want) {
|
||||
t.Fatalf("findings = %q, want one containing %q", all, row.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The run of this check over the real tree.
|
||||
func TestTheCatalogRecordsOnlyThroughTheSeat(t *testing.T) {
|
||||
got, err := Findings(filepath.Join("..", ".."))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range got {
|
||||
t.Error(f)
|
||||
}
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (issue 1936).
|
||||
# every manifest, for what it invokes.
|
||||
modules/*/module.json
|
||||
@@ -2,4 +2,6 @@ module checks/health
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f
|
||||
|
||||
require git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639 // indirect
|
||||
|
||||
@@ -1,2 +1,4 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639 h1:Zt7CfWt2S2W/fVgWNyW2u2+xwz051LXvGxVTNIyfhdM=
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639/go.mod h1:0dD1XEJBJ0G5UYDQxSZI84nuujb6m3tOpJl4C8rQVvg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f h1:jDsfDMd48MXY9Zershc3eRxnhmx2sSpUfQkLq5jAIds=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f/go.mod h1:08OomDXdwVN8/87nyDtv5R3rH/6LoP8SFFXz30bTYCM=
|
||||
|
||||
@@ -2,4 +2,6 @@ module checks/manifests
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f
|
||||
|
||||
require git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639 // indirect
|
||||
|
||||
@@ -1,2 +1,4 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639 h1:Zt7CfWt2S2W/fVgWNyW2u2+xwz051LXvGxVTNIyfhdM=
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639/go.mod h1:0dD1XEJBJ0G5UYDQxSZI84nuujb6m3tOpJl4C8rQVvg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f h1:jDsfDMd48MXY9Zershc3eRxnhmx2sSpUfQkLq5jAIds=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f/go.mod h1:08OomDXdwVN8/87nyDtv5R3rH/6LoP8SFFXz30bTYCM=
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// Package manifests reads every module's manifest as the catalogue's tree holds it and refuses what no judge could
|
||||
// read: a module.json that does not decode, and two manifests that name one module.
|
||||
// read: a module.json that does not decode, a module.json giving one field twice, and two manifests that name one module.
|
||||
//
|
||||
// It judges no manifest (novox/hq ADR 0578 §2): the checker judges every change's manifests with the indexer's judge, the
|
||||
// one judge, so the field, seat, bus and catalogue rules are said once, there. Until 2026-11 this check parsed each
|
||||
@@ -9,6 +9,8 @@
|
||||
package manifests
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -18,7 +20,9 @@ import (
|
||||
)
|
||||
|
||||
// Problems is what is wrong with the tree's manifests under root/modules, one line each: a module.json whose bytes are
|
||||
// no manifest (catalogue.Decode: not JSON, or a field of the wrong shape), then a module named by two manifests.
|
||||
// no manifest (catalogue.Decode: not JSON, or a field of the wrong shape), one giving a field twice (a decoder keeps the
|
||||
// last and drops the first without a word: a merge of two branches that each added the field writes it so), then a module
|
||||
// named by two manifests.
|
||||
func Problems(root string) ([]string, error) {
|
||||
files, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
@@ -38,6 +42,10 @@ func Problems(root string) ([]string, error) {
|
||||
out = append(out, fmt.Sprintf("%s: %v", rel, err))
|
||||
continue
|
||||
}
|
||||
if twice := givenTwice(raw); twice != "" {
|
||||
out = append(out, fmt.Sprintf("%s: the field %s is given twice; a judge reads only the last", rel, twice))
|
||||
continue
|
||||
}
|
||||
if named[m.Module] {
|
||||
out = append(out, fmt.Sprintf("%s: %s was already given; two manifests name one module", rel, m.Module))
|
||||
continue
|
||||
@@ -46,3 +54,47 @@ func Problems(root string) ([]string, error) {
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// givenTwice is the first field an object of raw gives twice, as its path (`invokes`, `build.artifacts[0].env`), or empty.
|
||||
func givenTwice(raw []byte) string {
|
||||
d := json.NewDecoder(bytes.NewReader(raw))
|
||||
var walk func(path string) string
|
||||
walk = func(path string) string {
|
||||
t, err := d.Token()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
switch t {
|
||||
case json.Delim('{'):
|
||||
seen := map[string]bool{}
|
||||
for d.More() {
|
||||
k, err := d.Token()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
key, _ := k.(string)
|
||||
at := key
|
||||
if path != "" {
|
||||
at = path + "." + key
|
||||
}
|
||||
if seen[key] {
|
||||
return at
|
||||
}
|
||||
seen[key] = true
|
||||
if twice := walk(at); twice != "" {
|
||||
return twice
|
||||
}
|
||||
}
|
||||
_, _ = d.Token()
|
||||
case json.Delim('['):
|
||||
for i := 0; d.More(); i++ {
|
||||
if twice := walk(fmt.Sprintf("%s[%d]", path, i)); twice != "" {
|
||||
return twice
|
||||
}
|
||||
}
|
||||
_, _ = d.Token()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
return walk("")
|
||||
}
|
||||
|
||||
@@ -5,10 +5,12 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// The tree's manifests are ones a judge can read, and no module is named twice. A row is a catalogue of its own: a
|
||||
// manifest that is no JSON and two manifests of one module are refused, naming the file; a manifest with a field this
|
||||
// manifest that is no JSON, one giving a field twice and two manifests of one module are refused, naming the file; a manifest with a field this
|
||||
// check does not know is not (it judges nothing: the checker does, ADR 0578 §2). The catalogue itself passes.
|
||||
func TestEveryManifestDecodesAndNamesOneModule(t *testing.T) {
|
||||
for _, row := range []struct {
|
||||
@@ -19,6 +21,10 @@ func TestEveryManifestDecodesAndNamesOneModule(t *testing.T) {
|
||||
{name: "a manifest the judge can read", files: map[string]string{"a": `{"module":"a","version":"1"}`}},
|
||||
{name: "a field no part knows is the judge's to refuse", files: map[string]string{"a": `{"module":"a","version":"1","unheard":true}`}},
|
||||
{name: "bytes that are no manifest", files: map[string]string{"a": `{"module":`}, refused: "modules/a/module.json"},
|
||||
{name: "a field given twice", files: map[string]string{"a": `{"module":"a","version":"1","invokes":["x.y"],"build":{"artifacts":[{}]},"invokes":["z.w"]}`},
|
||||
refused: "modules/a/module.json: the field invokes is given twice"},
|
||||
{name: "a field given twice deeper", files: map[string]string{"a": `{"module":"a","version":"1","build":{"artifacts":[{"name":"t","name":"u"}]}}`},
|
||||
refused: "the field build.artifacts[0].name is given twice"},
|
||||
{name: "two manifests of one module", files: map[string]string{"a": `{"module":"x","version":"1"}`, "b": `{"module":"x","version":"1"}`},
|
||||
refused: "two manifests name one module"},
|
||||
} {
|
||||
@@ -48,3 +54,45 @@ func TestEveryManifestDecodesAndNamesOneModule(t *testing.T) {
|
||||
t.Fatalf("the catalogue holds manifests no judge can read:\n%s", strings.Join(real, "\n"))
|
||||
}
|
||||
}
|
||||
|
||||
// No manifest of this catalogue claims the seat mesh-catalog: the indexer of the delivery holds it, and the module that
|
||||
// held it before has dropped its claim (ADR 0604 §6). A row is a tree of its own: a manifest claiming it is named.
|
||||
func TestNoManifestClaimsTheSeatMeshCatalog(t *testing.T) {
|
||||
claimants := func(root string) []string {
|
||||
files, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out []string
|
||||
for _, f := range files {
|
||||
raw, err := os.ReadFile(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.Decode(raw)
|
||||
if err != nil {
|
||||
continue // Problems names it
|
||||
}
|
||||
for _, c := range m.Claims {
|
||||
if c.Name == "mesh-catalog" {
|
||||
out = append(out, m.Module)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
row := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(row, "modules", "a"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(row, "modules", "a", "module.json"),
|
||||
[]byte(`{"module":"a","version":"1","claims":[{"name":"mesh-catalog","scope":"mesh"}]}`), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := claimants(row); len(got) != 1 || got[0] != "a" {
|
||||
t.Fatalf("a manifest claiming the seat is named, got %v", got)
|
||||
}
|
||||
if got := claimants(filepath.Join("..", "..")); len(got) != 0 {
|
||||
t.Errorf("the catalogue's manifests %v claim the seat mesh-catalog: the indexer of the delivery holds it", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +0,0 @@
|
||||
# The modules of this catalogue that still read the controller's assignments (mesh-controller.assignments), each with
|
||||
# why. The check of this directory (noplacement) refuses any other module that reads them, and an entry here whose module no longer
|
||||
# does: the list only shrinks, and it is empty before the controller's lease is written to a holder that is no
|
||||
# controller (novox/hq ADR 0549 §3, §6; issue 774).
|
||||
@@ -1,7 +1,3 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/noplacement
|
||||
|
||||
go 1.22
|
||||
|
||||
require git.novox.be/novox/mesh-catalog/checks/trunk v0.0.0
|
||||
|
||||
replace git.novox.be/novox/mesh-catalog/checks/trunk => ../trunk
|
||||
|
||||
@@ -10,11 +10,8 @@
|
||||
// 3. a manifest naming a state, stream, event or bucket of the indexer's placement (`mesh-delivery-indexer.placement…`
|
||||
// or `mesh-delivery-indexer_placement…`) in any string value, such as `reads`;
|
||||
//
|
||||
// 4. a manifest, other than mesh-controller's, that reads the controller's assignments (`mesh-controller.assignments`):
|
||||
// every reader takes placement from the seat mesh-placement (novox/hq ADR 0549 §3, ADR 0567), and the lease write is
|
||||
// refused while one still reads them (ADR 0549 §6; issue 774). The modules that still read them for what the
|
||||
// placement does not carry are named in checks/noplacement/assignments-readers, each with why; that list only
|
||||
// shrinks: an entry whose module no longer reads them is refused too.
|
||||
// The controller's assignments, which item 4 once listed readers of, are refused with every other name of the
|
||||
// controller by checks/controllerdeps (novox/hq ADR 0590 §5).
|
||||
//
|
||||
// go run . <catalogue root>
|
||||
//
|
||||
@@ -61,74 +58,10 @@ func Findings(root string) ([]string, error) {
|
||||
out = append(out, fmt.Sprintf("%s: names %q, which the indexer no longer keeps; placement is the seat mesh-placement's (novox/hq ADR 0567 §12)", module, s))
|
||||
}
|
||||
}
|
||||
readers, err := assignmentsReaders(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, readers...)
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// AssignmentsRead is the controller's assignments as a manifest reads them, and Allowed the file, relative to the
|
||||
// catalogue's root, that names the modules still reading them: `<module> <why>` per line, `#` a comment.
|
||||
const (
|
||||
AssignmentsRead = "mesh-controller.assignments"
|
||||
Allowed = "checks/noplacement/assignments-readers"
|
||||
)
|
||||
|
||||
// assignmentsReaders is every finding on the reads of the controller's assignments: a module other than the controller
|
||||
// reading them that Allowed does not name, and a module Allowed names that no longer reads them (issue 774).
|
||||
func assignmentsReaders(root string) ([]string, error) {
|
||||
allowed := map[string]bool{}
|
||||
raw, err := os.ReadFile(filepath.Join(root, Allowed))
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return nil, err
|
||||
}
|
||||
for _, line := range strings.Split(string(raw), "\n") {
|
||||
if f := strings.Fields(line); len(f) > 0 && !strings.HasPrefix(f[0], "#") {
|
||||
allowed[f[0]] = true
|
||||
}
|
||||
}
|
||||
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var found []string
|
||||
reads := map[string]bool{}
|
||||
for _, path := range manifests {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Reads []string `json:"reads"`
|
||||
}
|
||||
if json.Unmarshal(raw, &m) != nil || m.Module == "mesh-controller" {
|
||||
continue // a manifest that does not read is the manifest checks' to refuse
|
||||
}
|
||||
for _, r := range m.Reads {
|
||||
if strings.TrimSpace(r) != AssignmentsRead {
|
||||
continue
|
||||
}
|
||||
reads[m.Module] = true
|
||||
if !allowed[m.Module] {
|
||||
found = append(found, fmt.Sprintf("%s reads %s: read where modules run from mesh-delivery-assigner.placement, "+
|
||||
"the seat mesh-placement's (novox/hq ADR 0549 §3, ADR 0567); the lease write is refused while a module reads "+
|
||||
"the assignments (ADR 0549 §6)", m.Module, AssignmentsRead))
|
||||
}
|
||||
}
|
||||
}
|
||||
for module := range allowed {
|
||||
if !reads[module] {
|
||||
found = append(found, fmt.Sprintf("%s names %s, which no longer reads %s: take it off the list, which only shrinks",
|
||||
Allowed, module, AssignmentsRead))
|
||||
}
|
||||
}
|
||||
return found, nil
|
||||
}
|
||||
|
||||
// named collects the strings of a decoded manifest that start with one of the Prefixes.
|
||||
func named(v any, into []string) []string {
|
||||
switch v := v.(type) {
|
||||
@@ -166,5 +99,5 @@ func main() {
|
||||
if len(found) > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("no placement directory, no manifest reads the indexer's placement, and only the modules listed read the controller's assignments")
|
||||
fmt.Println("no placement directory, and no manifest reads the indexer's placement")
|
||||
}
|
||||
|
||||
@@ -5,8 +5,6 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-catalog/checks/trunk"
|
||||
)
|
||||
|
||||
func TestFindingsRefuseWhatThePlacementPathLeft(t *testing.T) {
|
||||
@@ -21,12 +19,6 @@ func TestFindingsRefuseWhatThePlacementPathLeft(t *testing.T) {
|
||||
{"a read of the indexer's placement", map[string]string{"modules/a/module.json": `{"module":"a","reads":["mesh-delivery-indexer.placement"]}`}, `a: names "mesh-delivery-indexer.placement"`},
|
||||
{"a read of its candidates", map[string]string{"modules/b/module.json": `{"module":"b","reads":["mesh-delivery-indexer.placement-candidates"]}`}, "placement-candidates"},
|
||||
{"its event", map[string]string{"modules/c/module.json": `{"module":"c","on":[{"event":"mesh-delivery-indexer.placement-changed"}]}`}, "placement-changed"},
|
||||
// Issue 774 (ADR 0549 §3, §6): a reader of the controller's assignments is refused unless the list names it, the
|
||||
// controller's own manifest is not judged, and a name on the list whose module no longer reads them is refused.
|
||||
{"an unlisted reader of the assignments", map[string]string{"modules/r/module.json": `{"module":"r","reads":["mesh-controller.assignments"]}`}, "r reads mesh-controller.assignments"},
|
||||
{"a listed reader and the controller", map[string]string{"modules/l/module.json": `{"module":"l","reads":["mesh-controller.assignments"]}`,
|
||||
"modules/mesh-controller/module.json": `{"module":"mesh-controller","reads":["mesh-controller.assignments"]}`, Allowed: "# why\nl its plan\n"}, ""},
|
||||
{"a listed module that reads them no more", map[string]string{Allowed: "gone no longer reads\n"}, "names gone, which no longer reads"},
|
||||
{"the seat's own state is allowed", map[string]string{"modules/d/module.json": `{"module":"d","reads":["mesh-placement.placement"]}`}, ""},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
@@ -60,29 +52,4 @@ func TestTheCatalogueHoldsNoPlacementOfTheRepositoryPath(t *testing.T) {
|
||||
for _, f := range got {
|
||||
t.Error(f)
|
||||
}
|
||||
// Issue 774 (ADR 0549 §3, §6; the comparison of novox/hq ADR 0529 clause 2): the list of modules still reading the
|
||||
// controller's assignments gains none the trunk's copy at the merge base does not hold. Its first copy gains nothing.
|
||||
path, ok := trunk.Copy(t, Allowed)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
names := func(path string) map[string]bool {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := map[string]bool{}
|
||||
for _, line := range strings.Split(string(raw), "\n") {
|
||||
if f := strings.Fields(line); len(f) > 0 && !strings.HasPrefix(f[0], "#") {
|
||||
out[f[0]] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
base := names(path)
|
||||
for module := range names(filepath.Join("..", "..", Allowed)) {
|
||||
if !base[module] {
|
||||
t.Errorf("%s gains %s, which the trunk's copy does not hold: the list only shrinks", Allowed, module)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,4 +2,6 @@ module checks/readable
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f
|
||||
|
||||
require git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639 // indirect
|
||||
|
||||
@@ -1,2 +1,4 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639 h1:Zt7CfWt2S2W/fVgWNyW2u2+xwz051LXvGxVTNIyfhdM=
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639/go.mod h1:0dD1XEJBJ0G5UYDQxSZI84nuujb6m3tOpJl4C8rQVvg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f h1:jDsfDMd48MXY9Zershc3eRxnhmx2sSpUfQkLq5jAIds=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f/go.mod h1:08OomDXdwVN8/87nyDtv5R3rH/6LoP8SFFXz30bTYCM=
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/removedpackages
|
||||
|
||||
go 1.22
|
||||
@@ -0,0 +1,166 @@
|
||||
// Command removedpackages refuses a Go source of the catalogue that imports what mesh-sdk gave up in the release that
|
||||
// narrowed it (novox/hq ADR 0593 §1): a package that left for its owner, and the build runner's names that left
|
||||
// mesh-sdk's buildstep for mesh-build-go's package.
|
||||
//
|
||||
// A module compiles against the mesh-sdk version it pins, so a source that still names a removed package builds until the
|
||||
// pin moves, and then breaks in a delivery of its own; and a later change could bring the import back with the old pin.
|
||||
// This check refuses it at once, from the source alone:
|
||||
//
|
||||
// - an import of a package in Removed (or one under it);
|
||||
// - in a file importing mesh-sdk's buildstep (under any name), a selector naming anything that is not in
|
||||
// BuildstepContract: what is left of that package is what the delivery parts read, and the runner is
|
||||
// mesh-build-go's package (git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep).
|
||||
//
|
||||
// It reads the non-test and test Go files under modules/ and checks/ (testdata, vendor and node_modules left out).
|
||||
//
|
||||
// go run . <catalogue root>
|
||||
//
|
||||
// Exits 1 on a finding. The lists are the release's: a package that goes later is added here with the change.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const sdk = "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
// Removed are the packages of mesh-sdk that left it, each with where its logic is now.
|
||||
var Removed = map[string]string{
|
||||
"buildcpu": "the build runner's package, git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep/buildcpu",
|
||||
"buildstep/conformance": "the build runner's package, git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep/conformance",
|
||||
"delivery/buildrun": "the builder's package (mesh-delivery-builder), imported by its repository's modules",
|
||||
"compose": "the node-engine, which alone composes (ADR 0593 §6)",
|
||||
"compose/frombus": "the node-engine, which alone composes (ADR 0593 §6)",
|
||||
"compose/fromruntime": "the node-engine, which alone composes (ADR 0593 §6)",
|
||||
"compose/judge": "the assigner's own package",
|
||||
"takein": "the node-engine, which alone composes (ADR 0593 §6)",
|
||||
"validate": "the node-engine, which alone composes (ADR 0593 §6)",
|
||||
"facts": "nothing: the controller they served is ended (ADR 0590)",
|
||||
"example": "nothing: no module imports it (ADR 0593 §12)",
|
||||
"packageguard": "nothing: a test helper of mesh-sdk's own",
|
||||
}
|
||||
|
||||
// BuildstepContract are the names mesh-sdk's buildstep keeps: the failure contract, the fetch request, the stale-step sweep,
|
||||
// the step id and the module tree shape the delivery parts read.
|
||||
var BuildstepContract = map[string]bool{
|
||||
"AskWhat": true, "Boot": true, "By": true, "CapacitySetting": true, "Clip": true, "ConsoleFailure": true, "CutLine": true,
|
||||
"EndOne": true, "EndStale": true, "EndStaleAtStart": true, "EndStaleRound": true, "Failure": true, "FetchRequest": true,
|
||||
"FetchRequestProperties": true, "Hidden": true, "Launchers": true, "LauncherSuffix": true, "LineBytes": true,
|
||||
"ModuleTree": true, "NewID": true, "OutputBytes": true, "Owner": true, "PackagesArg": true, "ParseFetchRequest": true,
|
||||
"PassingLine": true, "RestartReason": true, "Scan": true, "ScanOf": true, "StaleAttempts": true, "StaleRetry": true,
|
||||
"StaleServed": true,
|
||||
}
|
||||
|
||||
// Findings is every refusal in the Go sources under root, sorted.
|
||||
func Findings(root string) ([]string, error) {
|
||||
var out []string
|
||||
for _, top := range []string{"modules", "checks"} {
|
||||
err := filepath.WalkDir(filepath.Join(root, top), func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
switch d.Name() {
|
||||
case "testdata", "vendor", "node_modules", ".mesh-build":
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !strings.HasSuffix(path, ".go") {
|
||||
return nil
|
||||
}
|
||||
rel, _ := filepath.Rel(root, path)
|
||||
found, err := fileFindings(path, filepath.ToSlash(rel))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out = append(out, found...)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func fileFindings(path, rel string) ([]string, error) {
|
||||
file, err := parser.ParseFile(token.NewFileSet(), path, nil, parser.SkipObjectResolution)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", rel, err)
|
||||
}
|
||||
var out []string
|
||||
buildstep := ""
|
||||
for _, imp := range file.Imports {
|
||||
p, err := strconv.Unquote(imp.Path.Value)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
sub, ok := strings.CutPrefix(p, sdk+"/")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for pkg, to := range Removed {
|
||||
if sub == pkg || strings.HasPrefix(sub, pkg+"/") {
|
||||
out = append(out, fmt.Sprintf("%s: imports %s, which left mesh-sdk; it is %s (novox/hq ADR 0593 §1)", rel, p, to))
|
||||
break
|
||||
}
|
||||
}
|
||||
if sub == "buildstep" {
|
||||
buildstep = "buildstep"
|
||||
if imp.Name != nil {
|
||||
buildstep = imp.Name.Name
|
||||
}
|
||||
}
|
||||
}
|
||||
if buildstep == "" || buildstep == "_" || buildstep == "." {
|
||||
return out, nil
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
ast.Inspect(file, func(n ast.Node) bool {
|
||||
sel, ok := n.(*ast.SelectorExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
id, ok := sel.X.(*ast.Ident)
|
||||
if !ok || id.Name != buildstep || !sel.Sel.IsExported() || BuildstepContract[sel.Sel.Name] || seen[sel.Sel.Name] {
|
||||
return true
|
||||
}
|
||||
seen[sel.Sel.Name] = true
|
||||
out = append(out, fmt.Sprintf("%s: uses %s.%s, which is the build runner's, no longer mesh-sdk's buildstep; import git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep (novox/hq ADR 0593 §1)", rel, buildstep, sel.Sel.Name))
|
||||
return true
|
||||
})
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) != 2 {
|
||||
fmt.Fprintln(os.Stderr, "usage: removedpackages <catalogue root>")
|
||||
os.Exit(2)
|
||||
}
|
||||
found, err := Findings(os.Args[1])
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
for _, f := range found {
|
||||
fmt.Println(f)
|
||||
}
|
||||
if len(found) > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("no source imports a package mesh-sdk gave up")
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// ADR 0593 §1: a source that imports a package mesh-sdk gave up, or names the build runner through mesh-sdk's buildstep, is
|
||||
// refused naming where it went; the contract that stays, the runner's own package and a testdata copy are not.
|
||||
func TestFindingsRefuseWhatMeshSDKGaveUp(t *testing.T) {
|
||||
const head = "package x\n\nimport (\n"
|
||||
for _, row := range []struct {
|
||||
name, file, src string
|
||||
want string // empty: clean
|
||||
}{
|
||||
{"clean", "modules/a/x.go", head + "\t\"git.novox.be/novox/mesh-sdk/go/delivery\"\n)\n\nvar _ = delivery.StepsState\n", ""},
|
||||
{"a removed package", "modules/a/x.go", head + "\t\"git.novox.be/novox/mesh-sdk/go/compose\"\n)\n\nvar _ = compose.Resolve\n", "imports git.novox.be/novox/mesh-sdk/go/compose, which left mesh-sdk; it is the node-engine"},
|
||||
{"under a removed package", "modules/a/x_test.go", head + "\t\"git.novox.be/novox/mesh-sdk/go/compose/frombus\"\n)\n", "the node-engine"},
|
||||
{"buildcpu", "checks/c/x.go", head + "\t\"git.novox.be/novox/mesh-sdk/go/buildcpu\"\n)\n", "buildstep/buildcpu"},
|
||||
{"the runner through mesh-sdk's buildstep", "modules/a/x.go", head + "\t\"git.novox.be/novox/mesh-sdk/go/buildstep\"\n)\n\nvar _ = buildstep.RunAsk\n", "uses buildstep.RunAsk, which is the build runner's"},
|
||||
{"the runner through an alias", "modules/a/x.go", head + "\tsdkstep \"git.novox.be/novox/mesh-sdk/go/buildstep\"\n)\n\nvar _ = sdkstep.Puller{}\n", "uses sdkstep.Puller"},
|
||||
{"the contract of buildstep", "modules/a/x.go", head + "\t\"git.novox.be/novox/mesh-sdk/go/buildstep\"\n)\n\nvar _ = buildstep.NewID\nvar _ buildstep.Failure\n", ""},
|
||||
{"the runner's own package", "modules/a/x.go", head + "\t\"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep\"\n)\n\nvar _ = buildstep.RunAsk\n", ""},
|
||||
{"a copy in testdata", "modules/a/testdata/x.go", head + "\t\"git.novox.be/novox/mesh-sdk/go/compose\"\n)\n", ""},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
p := filepath.Join(root, row.file)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(row.src), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := Findings(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if all := strings.Join(got, "\n"); (row.want == "") != (all == "") || !strings.Contains(all, row.want) {
|
||||
t.Fatalf("findings = %q, want one containing %q", all, row.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The script's run of this check over the real tree (novox/hq ADR 0529 clause 2).
|
||||
func TestTheCatalogueImportsNothingMeshSDKGaveUp(t *testing.T) {
|
||||
got, err := Findings(filepath.Join("..", ".."))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range got {
|
||||
t.Error(f)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
|
||||
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
|
||||
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
|
||||
# every Go source of the modules and of the checks, whose imports are read.
|
||||
modules
|
||||
checks
|
||||
@@ -36,13 +36,7 @@ import (
|
||||
// composition is every function or table the check lets stay in mesh-sdk, as "path-from-go/:name", and why. It names no
|
||||
// manifest judgement: each is the composer's refusal of composition itself (ADR 0578 clause 5).
|
||||
var composition = map[string]string{
|
||||
"catalogue/consumers.go:consumersPlaceholderProblems": "composition: rendering a consumer's template",
|
||||
"catalogue/collections.go:collectedPlaceholderProblems": "composition: placeholders a contribution renders",
|
||||
"catalogue/environment_into.go:placeholderProblems": "composition: placeholders a contribution renders",
|
||||
"catalogue/seat_contributions.go:seatPlaceholderProblems": "composition: placeholders a contribution renders",
|
||||
"catalogue/collections.go:itemTemplateProblem": "composition: filling an item's template",
|
||||
"catalogue/setting_takes.go:takenSettingsProblems": "composition: whether stored settings still compose",
|
||||
"catalogue/template.go:HeldTemplateProblem": "composition: the take-in refusal of a resolved build",
|
||||
"catalogue/collections.go:itemTemplateProblem": "composition: filling an item's template",
|
||||
}
|
||||
|
||||
// Problems is what mesh-sdk's Go tree under root (its go/ directory) holds of a manifest judge, one line each, but for
|
||||
|
||||
+37
-1
@@ -4,7 +4,9 @@
|
||||
// own code that is not a test (a tool's description, the notes and errors it answers with) and every
|
||||
// description in its manifest. Comments are not read: an agent never sees them. And no daemon's line
|
||||
// (novox/hq ADR 0272, lines.go): the copy in retired-lines, the lines a person would otherwise be told to
|
||||
// type of the node-engine's or the controller's binary, where the operator's command is `nox`.
|
||||
// type of the node-engine's or the controller's binary, where the operator's command is `nox`. And no address of the
|
||||
// controller (novox/hq ADR 0590 §5): it is not placed, so a text naming one of its verbs, states or buckets sends its
|
||||
// reader nowhere.
|
||||
//
|
||||
// A word is matched whole and in any case, a space in it matching any run of white space, and its plural
|
||||
// (`s` or `es` on its last part) is matched as the word. A vendor word is
|
||||
@@ -89,11 +91,45 @@ func run(root string) int {
|
||||
len(findings))
|
||||
return 1
|
||||
}
|
||||
// And no address of the controller (novox/hq ADR 0590 §5): it is not placed, so a text that sends an agent or the
|
||||
// operator to one of its verbs, states or buckets sends them nowhere.
|
||||
if findings := checkAddresses(texts); len(findings) > 0 {
|
||||
for _, f := range findings {
|
||||
fmt.Println(f)
|
||||
}
|
||||
fmt.Printf("words: %d address(es) of the controller in what a module shows — name the seat that holds the job now (novox/hq ADR 0590)\n",
|
||||
len(findings))
|
||||
return 1
|
||||
}
|
||||
fmt.Printf("words: %d retired words and %d daemons' lines, none in the %d strings of %d files\n", len(words), len(lines),
|
||||
len(texts), files)
|
||||
return 0
|
||||
}
|
||||
|
||||
// controllerAddress is an address of the controller: its seat's verb or state (`mesh-controller.<name>`, with or without
|
||||
// `seat:`) or one of its buckets (`mesh-controller_<bucket>`). The module's name in prose is no address.
|
||||
var controllerAddress = regexp.MustCompile(`(^|[^A-Za-z0-9_-])(seat:)?mesh-controller[._][A-Za-z0-9<]`)
|
||||
|
||||
// checkAddresses is every text that names an address of the controller (novox/hq ADR 0590 §5).
|
||||
func checkAddresses(texts []text) []string {
|
||||
var out []string
|
||||
for _, t := range texts {
|
||||
if !strings.Contains(t.s, "mesh-controller") {
|
||||
continue
|
||||
}
|
||||
if m := controllerAddress.FindStringIndex(t.s); m != nil {
|
||||
line := t.line
|
||||
if line > 0 {
|
||||
line += strings.Count(t.s[:m[0]], "\n")
|
||||
}
|
||||
out = append(out, fmt.Sprintf("%s:%d: names an address of the controller, which is not placed — in %q", t.file, line,
|
||||
excerpt(t.s, m[0], m[1])))
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// collect reads every text a module can show under root: the strings of its own code that is not a test, and every string of
|
||||
// its manifests.
|
||||
func collect(root string) ([]text, int, error) {
|
||||
|
||||
@@ -95,3 +95,25 @@ func TestTheCatalogueHoldsNoRetiredWordAndNoDaemonsLine(t *testing.T) {
|
||||
t.Fatalf("the check of the real catalogue exits %d (see what it printed above)", code)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0590 §5 (issue 1126 B10): no text a module shows names an address of the controller, which is not placed; prose
|
||||
// naming it passes.
|
||||
func TestCheckAddressesRefusesTheControllersAddresses(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
s string
|
||||
want bool
|
||||
}{
|
||||
{"push this machine (mesh-controller.push)", true},
|
||||
{"ask seat:mesh-controller.nodes", true},
|
||||
{"the bucket mesh-controller_assignments", true},
|
||||
{"`mesh-controller.<verb>`", true},
|
||||
{"the mesh-controller module is gone.", false},
|
||||
{"located in mesh-controller#246", false},
|
||||
{"ask mesh-placement.placement", false},
|
||||
} {
|
||||
got := checkAddresses([]text{{file: "m/a.go", line: 1, s: c.s}})
|
||||
if (len(got) == 1) != c.want {
|
||||
t.Errorf("checkAddresses(%q) = %q, want a finding: %v", c.s, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
58
|
||||
57
|
||||
The long-running resources of this catalogue that do not say how they are ready (`health`, novox/hq ADR 0240
|
||||
rule 8), as `module check` counts them. It may only go down: checks/health fails a change that raises it, and
|
||||
one that lowers it writes the new number on the first line. From 2026-11-18, or once it is 0, a long-running
|
||||
|
||||
@@ -2,10 +2,10 @@ module adwaita
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f
|
||||
|
||||
require (
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/klauspost/compress v1.20.1 // indirect
|
||||
github.com/nats-io/nats.go v1.54.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639 h1:Zt7CfWt2S2W/fVgWNyW2u2+xwz051LXvGxVTNIyfhdM=
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639/go.mod h1:0dD1XEJBJ0G5UYDQxSZI84nuujb6m3tOpJl4C8rQVvg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f h1:jDsfDMd48MXY9Zershc3eRxnhmx2sSpUfQkLq5jAIds=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f/go.mod h1:08OomDXdwVN8/87nyDtv5R3rH/6LoP8SFFXz30bTYCM=
|
||||
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
|
||||
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
|
||||
@@ -61,7 +61,7 @@ record names leaves its platform manifests unrecorded with it.
|
||||
**What the definitions pin** (hq issue 323). A definition may pin an image of the store the mesh did not
|
||||
make: pushed before the mesh built it, and pinned under the store's current name or a name it had before.
|
||||
No record of the controller names it, yet a machine runs it, so `artifact_store_references` reads every
|
||||
definition the catalogue holds (`mesh-catalog.catalog_modules`, then `catalog_module` for each, eight at
|
||||
definition the catalogue holds (the seat mesh-catalog's verbs `modules`, then `module` for each, eight at
|
||||
a time and within twenty seconds) and finds every `<name>/<repository>@sha256:…` in any of its strings,
|
||||
under each of the store's names: `artifact-store://`, the store's address as the controller answers it,
|
||||
and the names it had before. Those are the mesh's setting `older-store-names` (space-separated, `none` by
|
||||
@@ -115,3 +115,13 @@ Any command other than the two read-only scripts fails the test.
|
||||
Each directory or archive this module declares says how it treats a file it did not write (ADR 0422, ADR 0416).
|
||||
|
||||
- `${dir:state}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
|
||||
|
||||
### The sweep through the registry seats
|
||||
|
||||
The sweep writes nothing to the store's door (CAT-AS1, ADR 0617 §2): it asks the registry seats the store's holder
|
||||
serves. Each kept bundle or archive is held on `oci-artifact-registry` and each kept copy of an upstream base on
|
||||
`container-image-registry` (`hold`, with why it is kept) before anything goes; each eligible one is released, then
|
||||
removed for `retention` on its seat (`release`, `remove`). It reads a reference in both forms, `artifact-store://<path>`
|
||||
and `<seat>/<module>/<artifact>@<digest>`, as one artifact, and says artifacts-gone in the form the build named it. A
|
||||
call a seat refuses stops the sweep and writes the finding `sweep-refused` in the module's findings; the next sweep no
|
||||
seat refuses clears it (`sweepseats.go`).
|
||||
|
||||
@@ -23,10 +23,11 @@ import (
|
||||
// manifest's.
|
||||
const NamesFileEnv = "MESH_ARTIFACT_STORE_NAMES_FILE"
|
||||
|
||||
// CatalogueModules and CatalogueModule are the catalog's tools these definitions are read through.
|
||||
// CatalogueModules and CatalogueModule are the verbs of the seat mesh-catalog these definitions are read through
|
||||
// (ADR 0604 §3); the answers are catalogue.CatalogModules and catalogue.CatalogOne.
|
||||
const (
|
||||
CatalogueModules = "mesh-catalog.catalog_modules"
|
||||
CatalogueModule = "mesh-catalog.catalog_module"
|
||||
CatalogueModules = "seat:mesh-catalog.modules"
|
||||
CatalogueModule = "seat:mesh-catalog.module"
|
||||
)
|
||||
|
||||
// DefinitionsTimeout bounds reading every definition, below the runtime's thirty-second call limit; what
|
||||
@@ -279,11 +280,8 @@ func (c *Catalogue) definition(m catalogueEntry, names StoreNames, nameKey strin
|
||||
return pins, ""
|
||||
}
|
||||
}
|
||||
args := map[string]any{"module": m.Module}
|
||||
if m.Commit != "" {
|
||||
args["commit"] = m.Commit
|
||||
}
|
||||
raw, err := c.Ask(CatalogueModule, args)
|
||||
// The verb takes no commit: it answers the trunk's module, and the answer names the commit it is at.
|
||||
raw, err := c.Ask(CatalogueModule, map[string]any{"module": m.Module})
|
||||
if err != nil {
|
||||
return nil, "could not be asked: " + err.Error()
|
||||
}
|
||||
@@ -293,6 +291,7 @@ func (c *Catalogue) definition(m catalogueEntry, names StoreNames, nameKey strin
|
||||
}
|
||||
var one struct {
|
||||
Module *struct {
|
||||
Commit string `json:"commit"`
|
||||
Manifest json.RawMessage `json:"manifest"`
|
||||
} `json:"module"`
|
||||
Error string `json:"error"`
|
||||
@@ -301,9 +300,11 @@ func (c *Catalogue) definition(m catalogueEntry, names StoreNames, nameKey strin
|
||||
return nil, "not readable: " + firstOf(one.Error, errText(err), "no manifest in the answer")
|
||||
}
|
||||
pins := PinsIn(one.Module.Manifest, names)
|
||||
if m.Commit != "" {
|
||||
// Kept by the commit the answer names, which may be newer than the list's: a definition and its commit never
|
||||
// change together.
|
||||
if at := firstOf(one.Module.Commit, m.Commit); at != "" {
|
||||
c.mu.Lock()
|
||||
c.kept[key] = pins
|
||||
c.kept[m.Module+"@"+at+"|"+nameKey] = pins
|
||||
c.mu.Unlock()
|
||||
}
|
||||
return pins, ""
|
||||
|
||||
@@ -153,7 +153,7 @@ func TestOlderNamesNotReadAreSaidFirst(t *testing.T) {
|
||||
func references(t *testing.T, a *asked) func(map[string]any) (any, error) {
|
||||
f := world()
|
||||
s := &Store{Container: "mesh-registry", Run: f.run(t)}
|
||||
for _, tl := range Tools(s, Controller{Ask: a.ask}, &Catalogue{Ask: a.ask}) {
|
||||
for _, tl := range Tools(s, recordsAsked{a}, &Catalogue{Ask: a.ask}) {
|
||||
if tl.Name == "artifact_store_references" {
|
||||
return tl.Run
|
||||
}
|
||||
@@ -211,7 +211,7 @@ func TestReferencesReadTheDefinitionsUnderTheStoresOlderNames(t *testing.T) {
|
||||
}
|
||||
}
|
||||
if before != 2 || after != 2 {
|
||||
t.Errorf("asked catalog_module %d times, then %d: each definition at its commit is read once", before, after)
|
||||
t.Errorf("asked the seat's module %d times, then %d: each definition at its commit is read once", before, after)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -8,30 +8,25 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"artifactstoretools/internal/registry"
|
||||
)
|
||||
|
||||
// door is registry.Store at the store's address as this machine reaches it.
|
||||
type door struct{ s registry.Store }
|
||||
|
||||
func (d *door) Address() string { return d.s.Address }
|
||||
func (d *door) Hold(ctx context.Context, r string) (bool, error) { return d.s.Hold(ctx, r) }
|
||||
func (d *door) Holds(ctx context.Context, r string) error { return d.s.Holds(ctx, r) }
|
||||
// It is read only: what the sweep holds and removes it asks the registry seats (sweepseats.go).
|
||||
func (d *door) Address() string { return d.s.Address }
|
||||
func (d *door) Holds(ctx context.Context, r string) error { return d.s.Holds(ctx, r) }
|
||||
func (d *door) IsIndex(ctx context.Context, r string) (bool, error) {
|
||||
return d.s.IsIndex(ctx, r)
|
||||
}
|
||||
func (d *door) LetGo(ctx context.Context, r string) error { return d.s.LetGo(ctx, r) }
|
||||
func (d *door) SpareKept(ctx context.Context, keptImages, touching []string) error {
|
||||
spare, err := d.s.SpareKeptIndexes(ctx, keptImages, touching)
|
||||
if err == nil {
|
||||
d.s.Spare = spare
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// DoorAt reads the store's address (host:port) from the file the manifest fills from the seat mesh-artifact-store's
|
||||
// reach, each time, so a store that moved is reached where it is now.
|
||||
@@ -53,10 +48,22 @@ func DoorAt(file string) func() (Door, error) {
|
||||
}
|
||||
|
||||
// runtimeBus is the holder's bus through the runtime: its reads, its findings and its seat's event.
|
||||
type runtimeBus struct{}
|
||||
type runtimeBus struct{ views *views.Reader }
|
||||
|
||||
func (runtimeBus) Keys(state string) ([]string, error) { return stdio.State(state).Keys() }
|
||||
func (runtimeBus) Get(state, key string) ([]byte, bool, error) {
|
||||
// followed says whether the state is the indexer's or the assigner's, which the holder asks once and follows by events
|
||||
// (ADR 0615) rather than reads.
|
||||
func (b runtimeBus) followed(state string) bool { return b.views != nil && b.views.Follows(state) }
|
||||
|
||||
func (b runtimeBus) Keys(state string) ([]string, error) {
|
||||
if b.followed(state) {
|
||||
return b.views.Keys(state)
|
||||
}
|
||||
return stdio.State(state).Keys()
|
||||
}
|
||||
func (b runtimeBus) Get(state, key string) ([]byte, bool, error) {
|
||||
if b.followed(state) {
|
||||
return b.views.Get(state, key)
|
||||
}
|
||||
e, err := stdio.State(state).Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, false, err
|
||||
@@ -88,9 +95,9 @@ func (runtimeBus) Publish(subject string, body any) error {
|
||||
return stdio.SeatPublish(subject, body, "")
|
||||
}
|
||||
|
||||
// follow watches the builder's done keys and the wants: a build's end sweeps (once the lease names no controller) and
|
||||
// follow watches the builder's done keys and the wants: a build's end sweeps and
|
||||
// observes again; a want changed observes again. Event-driven: nothing reads them on a timer.
|
||||
func follow(h *Holder) {
|
||||
func follow(h *Holder, v *views.Reader) {
|
||||
<-stdio.Served()
|
||||
watch := func(state string, handle func(stdio.StateChange) error) {
|
||||
for {
|
||||
@@ -119,11 +126,22 @@ func follow(h *Holder) {
|
||||
}
|
||||
return nil
|
||||
})
|
||||
watch(WantsRead, func(c stdio.StateChange) error {
|
||||
if !c.Current {
|
||||
go observe()
|
||||
// The wants are the assigner's: the first view then its events (ADR 0615). What Watch hands over before it returns
|
||||
// is the first view, not a change.
|
||||
var current atomic.Bool
|
||||
current.Store(true)
|
||||
for {
|
||||
err := v.Watch(WantsRead, func(string, []byte, bool) {
|
||||
if !current.Load() {
|
||||
go observe()
|
||||
}
|
||||
})
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
return nil
|
||||
})
|
||||
h.logf("watching %s: %v; asked again in 10s", WantsRead, err)
|
||||
time.Sleep(10 * time.Second)
|
||||
}
|
||||
current.Store(false)
|
||||
observe()
|
||||
}
|
||||
|
||||
@@ -9,8 +9,8 @@ package main
|
||||
// At each build's end (a done key written) it sweeps as the controller's take-in did: every kept archive held first
|
||||
// (ADR 0251 §8), then what is eligible let go of, oldest first, within the same bound per sweep, an index left for a
|
||||
// `collect` a person confirms (ADR 0251 §7), each deletion recorded in the hand-act log (seat mesh-hand-acts) and said
|
||||
// as artifacts-gone. **Nothing is let go until the controller's lease names a holder that is no controller**: until
|
||||
// then the controller's take-in sweeps, and two sweeps never run at once. It also observes S25, image-missing: a want's
|
||||
// as artifacts-gone. **No lease turns it** (issue 1126): the controller ended by its unassignment, so its take-in sweeps
|
||||
// nothing and this seat is the one sweep. It also observes S25, image-missing: a want's
|
||||
// image the store does not hold, a finding in its own state `findings` the condition keeper reads.
|
||||
|
||||
import (
|
||||
@@ -24,12 +24,13 @@ import (
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"artifactstoretools/internal/registry"
|
||||
)
|
||||
|
||||
@@ -40,7 +41,6 @@ const (
|
||||
HavesRead = "node-engine.have"
|
||||
// ManifestsRead is the indexer's manifests: the images a manifest pins by itself.
|
||||
ManifestsRead = delivery.IndexerModule + ".manifests"
|
||||
LeaseRead = buscompose.ControllerLease
|
||||
// FindingsState is the holder's own findings (health.FindingsState): image-missing.
|
||||
FindingsState = health.FindingsState
|
||||
// Module is this module, the observer its findings name.
|
||||
@@ -58,7 +58,7 @@ const (
|
||||
ImageMissing = "image-missing"
|
||||
)
|
||||
|
||||
// Bus is what the holder reads and writes on the bus: its read states (wants, builds, haves, the lease), its own
|
||||
// Bus is what the holder reads and writes on the bus: its read states (wants, builds, haves), its own
|
||||
// findings, and its seat's event. The runtime in main; a test's in memory.
|
||||
type Bus interface {
|
||||
Keys(state string) ([]string, error)
|
||||
@@ -71,15 +71,12 @@ type Bus interface {
|
||||
Publish(subject string, body any) error
|
||||
}
|
||||
|
||||
// Door is the artifact store's own door: registry.Store, behind an interface so a test needs no store.
|
||||
// Door is the artifact store's own door, read only: registry.Store, behind an interface so a test needs no store. What
|
||||
// the sweep holds and removes it asks the registry seats (Seats, sweepseats.go).
|
||||
type Door interface {
|
||||
Address() string
|
||||
Hold(ctx context.Context, reference string) (bool, error)
|
||||
Holds(ctx context.Context, reference string) error
|
||||
IsIndex(ctx context.Context, reference string) (bool, error)
|
||||
LetGo(ctx context.Context, reference string) error
|
||||
// SpareKept sets the platform manifests a kept index names aside before an index is let go of (ADR 0257).
|
||||
SpareKept(ctx context.Context, keptImages, touching []string) error
|
||||
}
|
||||
|
||||
// Holder is the seat's holder.
|
||||
@@ -89,8 +86,10 @@ type Holder struct {
|
||||
// Listed is the store's own listing now (Store.View); nil reads none, and then nothing is said gone or collected.
|
||||
Listed func(ctx context.Context) (*View, error)
|
||||
Acts handacts.Recorder
|
||||
Now func() time.Time
|
||||
Logf func(string, ...any)
|
||||
// Seats asks a registry seat's verb, `seat:<seat>.<verb>` (hold, remove): the runtime's stdio.Ask in main.
|
||||
Seats func(key string, body any) (json.RawMessage, error)
|
||||
Now func() time.Time
|
||||
Logf func(string, ...any)
|
||||
|
||||
sweep sync.Mutex // one sweep at a time
|
||||
mu sync.Mutex
|
||||
@@ -106,13 +105,6 @@ type Build struct {
|
||||
Manifest string
|
||||
}
|
||||
|
||||
// LeaseAway says the controller's lease names a holder that is no controller: then the seat sweeps.
|
||||
func (h *Holder) LeaseAway() bool {
|
||||
raw, found, err := h.Bus.Get(LeaseRead, buscompose.ControllerLeaseKey)
|
||||
away, _ := buscompose.LeaseHandedAway(raw, found, err)
|
||||
return away
|
||||
}
|
||||
|
||||
// Builds is every done build the builder's keys hold, each with its end.
|
||||
func (h *Holder) Builds() ([]Build, error) {
|
||||
keys, err := h.Bus.Keys(BuildsRead)
|
||||
@@ -226,7 +218,7 @@ func KeepSet(builds []Build, wants []Want) map[string]*Ref {
|
||||
out := map[string]*Ref{}
|
||||
for _, b := range builds {
|
||||
for _, a := range b.Artifacts {
|
||||
reference := catalogue.Recorded(a.Reference)
|
||||
reference := canonical(catalogue.Recorded(a.Reference))
|
||||
if _, ours := catalogue.InArtifactStore(reference); !ours || a.Kind == "package" {
|
||||
continue
|
||||
}
|
||||
@@ -495,10 +487,6 @@ func (h *Holder) Collect(why string, confirm bool, most int) (*CollectAnswer, er
|
||||
if most > MostAtAll {
|
||||
return nil, fmt.Errorf("collect lets go of at most %d at once, not %d", MostAtAll, most)
|
||||
}
|
||||
if confirm && !h.LeaseAway() {
|
||||
return nil, errors.New("the controller holds its lease, and its take-in sweeps the store until it hands it away: " +
|
||||
"this seat lets nothing go before then (novox/hq ADR 0571 clause 2); ask the controller's collect")
|
||||
}
|
||||
return h.Sweep(context.Background(), Sweeping{Confirmed: confirm, DryRun: !confirm, Most: most, Why: why, By: "collect"})
|
||||
}
|
||||
|
||||
@@ -524,13 +512,13 @@ func (h *Holder) Sweep(ctx context.Context, s Sweeping) (*CollectAnswer, error)
|
||||
return nil, err
|
||||
}
|
||||
a := &CollectAnswer{DryRun: s.DryRun, WouldLetGo: []string{}, LetGo: []string{}}
|
||||
var keptArchives, keptImages, eligible []string
|
||||
var keptArchives, eligible []string
|
||||
for _, r := range j.sorted() {
|
||||
switch {
|
||||
case r.Kept && r.Kind == "archive":
|
||||
case r.Kept && (r.Kind == "archive" || isCopy(r.Reference)):
|
||||
// What retention would remove unless held: a bundle or an archive, and a copied base (ADR 0617 §2).
|
||||
keptArchives = append(keptArchives, r.Reference)
|
||||
case r.Kept:
|
||||
keptImages = append(keptImages, r.Reference)
|
||||
default:
|
||||
if held, known := inStore(j.view, r.Reference); known && !held {
|
||||
continue // collected already
|
||||
@@ -556,33 +544,25 @@ func (h *Holder) Sweep(ctx context.Context, s Sweeping) (*CollectAnswer, error)
|
||||
a.Store = door.Address()
|
||||
within, stop := context.WithTimeout(ctx, SweepBudget)
|
||||
defer stop()
|
||||
// Hold before letting go (ADR 0251 §8): a kept archive that cannot be held stops the sweep before any delete.
|
||||
// Hold before letting go (ADR 0251 §8, ADR 0617 §2): each kept archive and copied base is held on its seat; one that
|
||||
// cannot be held stops the sweep before any removal and writes the sweep's finding.
|
||||
var missing []string
|
||||
for _, r := range keptArchives {
|
||||
wrote, err := door.Hold(within, r)
|
||||
switch {
|
||||
case err == nil:
|
||||
if wrote {
|
||||
a.HoldersWritten++
|
||||
}
|
||||
case errors.Is(err, registry.Gone):
|
||||
if held, known := inStore(j.view, r); known && !held {
|
||||
missing = append(missing, r)
|
||||
case errors.Is(err, registry.ErrNotOurs):
|
||||
default:
|
||||
continue
|
||||
}
|
||||
if _, err := h.holdOnSeat(r, keptWhy(j.refs[r])); err != nil {
|
||||
a.Stopped = fmt.Sprintf("not every kept archive could be held, so nothing was let go: holding %s: %v", r, err)
|
||||
a.Left = len(eligible)
|
||||
h.sweepFinding("the hold of "+r, err)
|
||||
return a, nil
|
||||
}
|
||||
a.HoldersWritten++
|
||||
}
|
||||
a.Held, a.Missing = len(keptArchives)-len(missing), len(missing)
|
||||
h.sayGone(j, missing, "a kept archive the store does not hold")
|
||||
if s.Confirmed {
|
||||
if err := door.SpareKept(within, keptImages, eligible); err != nil {
|
||||
a.Stopped = fmt.Sprintf("which platform manifests a kept index names could not be read, so nothing was let go: %v", err)
|
||||
a.Left = len(eligible)
|
||||
return a, nil
|
||||
}
|
||||
}
|
||||
refused := false
|
||||
for i, r := range eligible {
|
||||
if i >= s.Most || within.Err() != nil {
|
||||
a.Left = len(eligible) - i
|
||||
@@ -606,24 +586,19 @@ func (h *Holder) Sweep(ctx context.Context, s Sweeping) (*CollectAnswer, error)
|
||||
continue
|
||||
}
|
||||
}
|
||||
err := door.LetGo(within, r)
|
||||
switch {
|
||||
case err == nil || errors.Is(err, registry.Gone):
|
||||
case errors.Is(err, registry.ErrNotOurs):
|
||||
a.Skipped++
|
||||
continue
|
||||
default:
|
||||
a.Stopped = fmt.Sprintf("the store refused to let go of %s, so nothing more was asked: %v", r, err)
|
||||
a.Left = len(eligible) - i
|
||||
}
|
||||
if a.Stopped != "" {
|
||||
break
|
||||
}
|
||||
// Recorded in the hand-act log, each one (ADR 0565 §10; the operator's "option A").
|
||||
// Removed through its seat (ADR 0617 §2), recorded in the hand-act log, each one (ADR 0565 §10; the operator's
|
||||
// "option A").
|
||||
why := s.Why
|
||||
if why == "" {
|
||||
why = "eligible: no want names it and it is not among its module's " + fmt.Sprint(KeptBuilds) + " newest builds"
|
||||
}
|
||||
if err := h.removeOnSeat(r, why); err != nil {
|
||||
a.Stopped = fmt.Sprintf("a registry seat refused to remove %s, so nothing more was asked: %v", r, err)
|
||||
a.Left = len(eligible) - i
|
||||
h.sweepFinding("the removal of "+r, err)
|
||||
refused = true
|
||||
break
|
||||
}
|
||||
if _, err := h.Acts.Record(handacts.Request{What: "let go of " + r + " (" + s.By + ")", Why: why,
|
||||
Cause: "artifact-store-sweep", Verb: "collect"}); err != nil {
|
||||
h.logf("%s was let go of and could not be recorded in the hand-act log: %v", r, err)
|
||||
@@ -631,6 +606,9 @@ func (h *Holder) Sweep(ctx context.Context, s Sweeping) (*CollectAnswer, error)
|
||||
a.LetGo = append(a.LetGo, r)
|
||||
}
|
||||
h.sayGone(j, a.LetGo, "let go of")
|
||||
if !refused {
|
||||
h.sweepClean()
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
@@ -650,7 +628,7 @@ func (h *Holder) sayGone(j *judged, refs []string, how string) {
|
||||
byKey[b.Key] = g
|
||||
keys = append(keys, b.Key)
|
||||
}
|
||||
g.Gone = append(g.Gone, r)
|
||||
g.Gone = append(g.Gone, namedBy(b, r))
|
||||
}
|
||||
}
|
||||
sort.Strings(keys)
|
||||
@@ -665,11 +643,19 @@ func (h *Holder) sayGone(j *judged, refs []string, how string) {
|
||||
}
|
||||
}
|
||||
|
||||
// AfterBuild is the sweep at a build's end: only once the lease names no controller.
|
||||
func (h *Holder) AfterBuild(key string) {
|
||||
if !h.LeaseAway() {
|
||||
return
|
||||
// namedBy is the reference b names r by, in the form its key holds (the older one or the seat's): the builder matches
|
||||
// artifacts-gone against its own key.
|
||||
func namedBy(b Build, r string) string {
|
||||
for _, a := range b.Artifacts {
|
||||
if rec := catalogue.Recorded(a.Reference); canonical(rec) == r {
|
||||
return rec
|
||||
}
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// AfterBuild is the sweep at a build's end.
|
||||
func (h *Holder) AfterBuild(key string) {
|
||||
a, err := h.Sweep(context.Background(), Sweeping{Most: MostPerSweep, By: "after the build " + key})
|
||||
if err != nil {
|
||||
h.logf("the sweep after %s: %v", key, err)
|
||||
@@ -701,7 +687,7 @@ func (h *Holder) Held(refs []string) (delivery.HeldAnswer, error) {
|
||||
defer stop()
|
||||
a := delivery.HeldAnswer{Store: door.Address(), Held: []string{}, Gone: []string{}}
|
||||
for _, r := range refs {
|
||||
switch err := door.Holds(asking, r); {
|
||||
switch err := door.Holds(asking, canonical(r)); {
|
||||
case err == nil:
|
||||
a.Held = append(a.Held, r)
|
||||
case errors.Is(err, registry.Gone):
|
||||
@@ -915,7 +901,16 @@ func (h *Holder) Observe(ctx context.Context) error {
|
||||
return fmt.Errorf("the findings: %w", err)
|
||||
}
|
||||
for _, k := range keys {
|
||||
if k == sweepFindingKey {
|
||||
continue // the sweep's own finding, which the sweep clears
|
||||
}
|
||||
if _, still := now[k]; !still {
|
||||
// A finding of a first view unanswered is the reader's (views.Reader), cleared by its answer, not by a sweep.
|
||||
if raw, found, err := h.Bus.Get(FindingsState, k); err == nil && found {
|
||||
if f, err := health.DecodeFinding(raw); err == nil && f.Kind == views.UnansweredKind {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if err := h.Bus.Delete(FindingsState, k); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
package main
|
||||
|
||||
// The seat mesh-artifact-sweep's holder whole against a real bus (novox/hq ADR 0571 clause 2, issue 1955): the builder's done
|
||||
// keys, the wants and the controller's lease in a real nats-server's key-value buckets, a build's end heard by a watch
|
||||
// keys and the wants in a real nats-server's key-value buckets, no lease key anywhere (issue 1126), a build's end heard by a watch
|
||||
// of the builds as the runtime hands it over, each deletion recorded by a request to the seat mesh-hand-acts on the bus,
|
||||
// artifacts-gone said on the seat's subject, and S25's finding written to the holder's own bucket. Only the store's
|
||||
// door is in memory: its HTTP is the copied registry package's, tested there.
|
||||
@@ -20,7 +20,6 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
)
|
||||
@@ -117,7 +116,7 @@ func (b realBus) Publish(subject string, body any) error {
|
||||
return b.nc.Publish(subject, raw)
|
||||
}
|
||||
|
||||
func TestTheSeatSweepsAtABuildsEndOnARealBusOnceTheLeaseIsAway(t *testing.T) {
|
||||
func TestTheSeatSweepsAtABuildsEndOnARealBusWithNoLease(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
nc, err := nats.Connect(testbus.URL(t))
|
||||
@@ -129,7 +128,7 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusOnceTheLeaseIsAway(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, s := range []string{BuildsRead, WantsRead, HavesRead, LeaseRead, ManifestsRead, "artifact-store-tools." + FindingsState} {
|
||||
for _, s := range []string{BuildsRead, WantsRead, HavesRead, ManifestsRead, "artifact-store-tools." + FindingsState} {
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucketOf(s), History: 5}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -137,9 +136,15 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusOnceTheLeaseIsAway(t *testing.T) {
|
||||
bus := realBus{ctx: ctx, nc: nc, js: js}
|
||||
|
||||
// The mesh as aMesh lays it, copied onto the real buckets: letta's seven builds, a want, the store's listing.
|
||||
mem, memb, door, _ := aMesh(t, &buscompose.LeaseHolder{Instance: "c1", Host: "novox", Epoch: 3})
|
||||
bus.log = memb.log
|
||||
for _, s := range []string{BuildsRead, WantsRead, LeaseRead, ManifestsRead} {
|
||||
mem, memb, door, _ := aMesh(t)
|
||||
// The builder's log as written today: aMesh dates it from 2026-10-01 for its fixed clock, and this holder runs on the
|
||||
// real clock, so read as dated the entries fall out of the log's 30 days and every copy is kept as unrecorded.
|
||||
shift := time.Since(memb.log[len(memb.log)-1].At)
|
||||
for _, e := range memb.log {
|
||||
e.At = e.At.Add(shift)
|
||||
bus.log = append(bus.log, e)
|
||||
}
|
||||
for _, s := range []string{BuildsRead, WantsRead, ManifestsRead} {
|
||||
keys, _ := memb.Keys(s)
|
||||
for _, k := range keys[:min(len(keys), 6)] { // the seventh build ends later, on the bus
|
||||
raw, _, _ := memb.Get(s, k)
|
||||
@@ -168,15 +173,32 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusOnceTheLeaseIsAway(t *testing.T) {
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h := &Holder{Bus: findingsAt{realBus: bus}, Door: mem.Door, Listed: mem.Listed, Now: time.Now, Logf: t.Logf}
|
||||
h.Acts = handacts.Recorder{LeaseAway: h.LeaseAway, Ask: func(key string, body any) (json.RawMessage, error) {
|
||||
// The registry seats' holder on the bus (CAT-AS1): the sweep holds and removes through their verbs.
|
||||
for _, seat := range []string{"oci-artifact-registry", "container-image-registry"} {
|
||||
if _, err := nc.Subscribe("mesh.seat."+seat+".tool.*", func(m *nats.Msg) {
|
||||
var body map[string]any
|
||||
_ = json.Unmarshal(m.Data, &body)
|
||||
mu.Lock()
|
||||
out, err := door.seat("seat:"+seat+"."+m.Subject[strings.LastIndex(m.Subject, ".")+1:], body)
|
||||
mu.Unlock()
|
||||
if err != nil {
|
||||
out, _ = json.Marshal(map[string]string{"error": err.Error()})
|
||||
}
|
||||
_ = m.Respond(out)
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
ask := func(key string, body any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(body)
|
||||
m, err := nc.Request("mesh.seat."+strings.TrimPrefix(strings.Replace(key, ".", ".tool.", 1), "seat:"), raw, 5*time.Second)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return m.Data, nil
|
||||
}}
|
||||
}
|
||||
h := &Holder{Bus: findingsAt{realBus: bus}, Door: mem.Door, Listed: mem.Listed, Seats: ask, Now: time.Now, Logf: t.Logf}
|
||||
h.Acts = handacts.Recorder{Ask: ask}
|
||||
|
||||
// A build's end heard from the builds bucket, as follow hands it over.
|
||||
kv, _ := bus.kv(BuildsRead)
|
||||
@@ -198,18 +220,10 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusOnceTheLeaseIsAway(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// While the lease names the controller, a build's end lets nothing go.
|
||||
// No lease key: the build's end holds the kept archives and lets go of letta's 2nd, which the 7th moved past the five
|
||||
// newest and no want names (the want keeps the 1st), recording each deletion.
|
||||
ended()
|
||||
if len(door.asked) != 0 {
|
||||
t.Fatalf("before the lease write the store was asked %v", door.asked)
|
||||
}
|
||||
// The lease handed away: the next build's end holds the kept archives and lets go of letta's 2nd, which the 7th moved
|
||||
// past the five newest and no want names (the want keeps the 1st), recording each deletion.
|
||||
if err := bus.Put(LeaseRead, buscompose.ControllerLeaseKey, buscompose.LeaseHolder{HandedTo: "mesh-grants", Why: "the lease write"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ended()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
deadline := time.Now().Add(30 * time.Second)
|
||||
for {
|
||||
mu.Lock()
|
||||
n := len(gone)
|
||||
|
||||
@@ -3,7 +3,9 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -11,11 +13,12 @@ import (
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
"artifactstoretools/internal/registry"
|
||||
)
|
||||
|
||||
@@ -86,21 +89,56 @@ func (b *memBus) Publish(subject string, body any) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// memDoor is the store's door in memory: what it holds, which references are indexes, and what was asked.
|
||||
// memDoor is the store's door in memory, read only, and the registry seats' holder: what it holds, which references are
|
||||
// indexes, what the seats were asked (asked: "hold <ref>", "let go <ref>"; calls: each seat call as its key and name), the
|
||||
// holds asked (held), and the calls it refuses (refuse: by "<verb> <ref>").
|
||||
type memDoor struct {
|
||||
holds map[string]bool
|
||||
indexes map[string]bool
|
||||
asked []string
|
||||
calls []string
|
||||
held map[string]bool
|
||||
refuse map[string]error
|
||||
}
|
||||
|
||||
// seat is the registry seats' holder, as Holder.Seats calls it.
|
||||
func (d *memDoor) seat(key string, body any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(body)
|
||||
var in struct{ Name, Digest, Reason string }
|
||||
_ = json.Unmarshal(raw, &in)
|
||||
seat, verb, _ := strings.Cut(strings.TrimPrefix(key, "seat:"), ".")
|
||||
ref := catalogue.ArtifactStoreScheme + in.Name + "@" + in.Digest
|
||||
if seat == "oci-artifact-registry" {
|
||||
ref = catalogue.ArtifactStoreScheme + in.Name + "/blobs/" + in.Digest
|
||||
}
|
||||
d.calls = append(d.calls, key+" "+in.Name+"@"+in.Digest+" "+in.Reason)
|
||||
if err := d.refuse[verb+" "+ref]; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if d.held == nil {
|
||||
d.held = map[string]bool{}
|
||||
}
|
||||
switch verb {
|
||||
case "hold":
|
||||
d.asked = append(d.asked, "hold "+ref)
|
||||
d.held[ref] = true
|
||||
return json.Marshal(map[string]any{"held": true})
|
||||
case "release":
|
||||
released := d.held[ref] // what is not held answers released false, as the seat's definition says
|
||||
delete(d.held, ref)
|
||||
return json.Marshal(map[string]any{"released": released})
|
||||
case "remove":
|
||||
if d.held[ref] {
|
||||
return nil, fmt.Errorf("remove: refused: %s@%s is held; release it first", in.Name, in.Digest)
|
||||
}
|
||||
d.asked = append(d.asked, "let go "+ref)
|
||||
delete(d.holds, ref)
|
||||
return json.Marshal(map[string]any{"removed": true})
|
||||
}
|
||||
return nil, fmt.Errorf("no verb %s", key)
|
||||
}
|
||||
|
||||
func (d *memDoor) Address() string { return "10.0.0.5:5000" }
|
||||
func (d *memDoor) Hold(_ context.Context, r string) (bool, error) {
|
||||
d.asked = append(d.asked, "hold "+r)
|
||||
if !d.holds[r] {
|
||||
return false, registry.Gone
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
func (d *memDoor) Holds(_ context.Context, r string) error {
|
||||
if !d.holds[r] {
|
||||
return registry.Gone
|
||||
@@ -108,15 +146,6 @@ func (d *memDoor) Holds(_ context.Context, r string) error {
|
||||
return nil
|
||||
}
|
||||
func (d *memDoor) IsIndex(_ context.Context, r string) (bool, error) { return d.indexes[r], nil }
|
||||
func (d *memDoor) LetGo(_ context.Context, r string) error {
|
||||
d.asked = append(d.asked, "let go "+r)
|
||||
delete(d.holds, r)
|
||||
return nil
|
||||
}
|
||||
func (d *memDoor) SpareKept(context.Context, []string, []string) error {
|
||||
d.asked = append(d.asked, "spare")
|
||||
return nil
|
||||
}
|
||||
|
||||
// mirror is a copy of an upstream base in the store, the n-th.
|
||||
func mirror(n int) string {
|
||||
@@ -139,8 +168,8 @@ func archiveRef(module string, n int) string {
|
||||
}
|
||||
|
||||
// A mesh with seven done builds of letta, the n-th ending n hours after a day began, each with an image and an archive;
|
||||
// a want names the first on ace; the store lists them all and one image no build names; the lease as given.
|
||||
func aMesh(t *testing.T, lease *buscompose.LeaseHolder) (*Holder, *memBus, *memDoor, *[]string) {
|
||||
// a want names the first on ace; the store lists them all and one image no build names; no lease key (issue 1126).
|
||||
func aMesh(t *testing.T) (*Holder, *memBus, *memDoor, *[]string) {
|
||||
t.Helper()
|
||||
day := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
|
||||
b := &memBus{states: map[string]map[string][]byte{}, history: map[string][]stdio.StateRevision{}}
|
||||
@@ -194,11 +223,8 @@ func aMesh(t *testing.T, lease *buscompose.LeaseHolder) (*Holder, *memBus, *memD
|
||||
_ = b.Put(ManifestsRead, mk, json.RawMessage(manifest))
|
||||
want, _ := json.Marshal(delivery.Want{V: delivery.Version, FP: fp(1), Self: true})
|
||||
_ = b.Put(WantsRead, "want.ace.letta", json.RawMessage(want))
|
||||
if lease != nil {
|
||||
_ = b.Put(LeaseRead, buscompose.ControllerLeaseKey, lease)
|
||||
}
|
||||
var acts []string
|
||||
h := &Holder{Bus: b, Door: func() (Door, error) { return door, nil },
|
||||
h := &Holder{Bus: b, Door: func() (Door, error) { return door, nil }, Seats: door.seat,
|
||||
Listed: func(context.Context) (*View, error) { return &View{L: listing}, nil },
|
||||
Now: func() time.Time { return day.Add(48 * time.Hour) },
|
||||
Acts: handacts.Recorder{Ask: func(key string, body any) (json.RawMessage, error) {
|
||||
@@ -206,18 +232,14 @@ func aMesh(t *testing.T, lease *buscompose.LeaseHolder) (*Holder, *memBus, *memD
|
||||
acts = append(acts, key+" "+string(raw))
|
||||
return json.RawMessage(`{}`), nil
|
||||
}}}
|
||||
h.Acts.LeaseAway = h.LeaseAway
|
||||
return h, b, door, &acts
|
||||
}
|
||||
|
||||
// ADR 0571 clause 2: the keep set from the store's listing, the wants and the five newest builds; nothing let go of while
|
||||
// the lease names a controller; once it names another holder, a build's end lets go of the eligible within the bound,
|
||||
// leaves an index for a confirmed collect, records each deletion and says artifacts-gone.
|
||||
func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsOnlyOnceTheLeaseIsAway(t *testing.T) {
|
||||
controller := &buscompose.LeaseHolder{Instance: "c1", Host: "novox", Epoch: 3}
|
||||
handed := &buscompose.LeaseHolder{HandedTo: "mesh-grants", HandedBy: "person.jochen", Why: "the lease write"}
|
||||
|
||||
h, _, _, _ := aMesh(t, controller)
|
||||
// ADR 0571 clause 2, 1126/A4/AC2: the keep set from the store's listing, the wants and the five newest builds; with no
|
||||
// lease key, a build's end and a confirmed collect let go of the eligible within the bound, leave an index for a
|
||||
// confirmed collect, record each deletion as a hand act and say artifacts-gone.
|
||||
func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsWithNoLease(t *testing.T) {
|
||||
h, _, _, _ := aMesh(t)
|
||||
r, err := h.Artifacts("")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -244,56 +266,121 @@ func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsOnlyOnceTheLeaseIsAway(t
|
||||
t.Errorf("the references are %d, counted %v, at %q", len(r.References), r.Counts, r.Store)
|
||||
}
|
||||
|
||||
all := []string{archiveRef("letta", 2), imageRef("letta", 2), mirror(1)}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
lease *buscompose.LeaseHolder
|
||||
how string // "build" (a build's end), "collect" (a confirmed collect), or a sweep asked with most
|
||||
index bool
|
||||
most int
|
||||
letGo []string
|
||||
left int
|
||||
skipped int
|
||||
unlog int
|
||||
// held is how many kept archives and copies are held (7 when 0); seatForm names letta's 2nd and 3rd builds in the
|
||||
// seat's form; refuse is the seat call refused, "<verb> <ref>".
|
||||
held int
|
||||
seatForm bool
|
||||
refuse string
|
||||
}{
|
||||
{name: "the lease names a controller", lease: controller},
|
||||
{name: "no lease key", lease: nil},
|
||||
{name: "handed away", lease: handed, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), imageRef("letta", 2), mirror(1)}},
|
||||
{name: "an index waits", lease: handed, index: true, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), mirror(1)}, skipped: 1},
|
||||
{name: "bounded", lease: handed, most: 1, letGo: []string{archiveRef("letta", 2)}, left: 2},
|
||||
{name: "a build's end", how: "build", letGo: all},
|
||||
// ADR 0605 §9: a build that names its artifacts in the seat's form is the same artifact: removed and held under the
|
||||
// seat's name and digest, and said gone in its own form.
|
||||
{name: "the seat's form", most: MostPerSweep, letGo: all, seatForm: true},
|
||||
// A seat that refuses a removal stops the sweep; the finding sweep-refused is written, and cleared by a clean run.
|
||||
{name: "a refused removal", most: MostPerSweep, letGo: []string{archiveRef("letta", 2)}, left: 2, refuse: "remove " + imageRef("letta", 2)},
|
||||
{name: "a confirmed collect", how: "collect", letGo: all},
|
||||
{name: "a sweep", most: MostPerSweep, letGo: all},
|
||||
{name: "an index waits", index: true, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), mirror(1)}, skipped: 1},
|
||||
{name: "bounded", most: 1, letGo: []string{archiveRef("letta", 2)}, left: 2},
|
||||
// A kept build the log no longer holds: what it stood on is unknown, so no copy goes.
|
||||
{name: "a kept build's bases unknown", lease: handed, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), imageRef("letta", 2)}, unlog: 3},
|
||||
{name: "a kept build's bases unknown", most: MostPerSweep, letGo: []string{archiveRef("letta", 2), imageRef("letta", 2)}, unlog: 3, held: 8},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
h, b, door, acts := aMesh(t, c.lease)
|
||||
h, b, door, acts := aMesh(t)
|
||||
door.indexes[imageRef("letta", 2)] = c.index
|
||||
if c.held == 0 {
|
||||
c.held = 7
|
||||
}
|
||||
if c.seatForm {
|
||||
for _, n := range []int{2, 3} {
|
||||
key, _ := delivery.BuilderBuildKey("letta", fp(n))
|
||||
raw, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: strings.Repeat("d", 64), Artifacts: []delivery.BuiltArtifact{
|
||||
{Name: "image", Kind: "image", Digest: digestOf(imageRef("letta", n)), Reference: catalogue.SeatReference("container-image-registry", "letta", "image", digestOf(imageRef("letta", n)))},
|
||||
{Name: "files", Kind: "archive", Digest: digestOf(archiveRef("letta", n)), Reference: catalogue.SeatReference("oci-artifact-registry", "letta", "files", digestOf(archiveRef("letta", n)))}}})
|
||||
_ = b.Put(BuildsRead, key, json.RawMessage(raw))
|
||||
}
|
||||
}
|
||||
if c.refuse != "" {
|
||||
door.refuse = map[string]error{c.refuse: errors.New("the store is read only")}
|
||||
}
|
||||
if c.unlog > 0 {
|
||||
b.log = append(b.log[:c.unlog-1:c.unlog-1], b.log[c.unlog:]...)
|
||||
}
|
||||
if c.most == 0 {
|
||||
switch c.how {
|
||||
case "build":
|
||||
h.AfterBuild("letta.x")
|
||||
if _, err := h.Collect("tidy", true, 0); err == nil {
|
||||
t.Error("a confirmed collect was let through while the controller sweeps")
|
||||
var gone []string
|
||||
for _, x := range door.asked {
|
||||
if r, ok := strings.CutPrefix(x, "let go "); ok {
|
||||
gone = append(gone, r)
|
||||
}
|
||||
}
|
||||
if len(door.asked) != 0 || len(*acts) != 0 || len(b.published) != 0 {
|
||||
t.Fatalf("before the lease write the store was asked %v, acts %v, said %v", door.asked, *acts, b.published)
|
||||
if fmt.Sprint(gone) != fmt.Sprint(c.letGo) {
|
||||
t.Fatalf("a build's end let go of %v, want %v", gone, c.letGo)
|
||||
}
|
||||
default:
|
||||
var a *CollectAnswer
|
||||
var err error
|
||||
if c.how == "collect" {
|
||||
a, err = h.Collect("tidy", true, 0)
|
||||
} else {
|
||||
a, err = h.Sweep(context.Background(), Sweeping{Most: c.most, By: "after the build letta.x"})
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fmt.Sprint(a.LetGo) != fmt.Sprint(c.letGo) || a.Left != c.left || a.Skipped != c.skipped || a.Held != c.held || a.Missing != 0 {
|
||||
t.Fatalf("let go of %v (left %d, skipped %d, held %d, missing %d), want %v", a.LetGo, a.Left, a.Skipped, a.Held, a.Missing, c.letGo)
|
||||
}
|
||||
}
|
||||
// Every kept archive held before the first deletion.
|
||||
held := 0
|
||||
for _, x := range door.asked {
|
||||
if strings.HasPrefix(x, "let go ") {
|
||||
break
|
||||
}
|
||||
if strings.HasPrefix(x, "hold ") {
|
||||
held++
|
||||
}
|
||||
}
|
||||
if held != c.held {
|
||||
t.Errorf("the seats were asked %v: the %d kept archives and copies are held first", door.asked, c.held)
|
||||
}
|
||||
// Each removal asks the seat that holds it, by its name and digest, for retention, after a release.
|
||||
for _, call := range door.calls {
|
||||
if strings.Contains(call, ".remove ") && !strings.HasSuffix(call, " "+Retention) {
|
||||
t.Errorf("a removal asked %q, not for retention", call)
|
||||
}
|
||||
}
|
||||
if c.letGo[0] == archiveRef("letta", 2) && !slices.Contains(door.calls, "seat:oci-artifact-registry.remove letta/files@"+digestOf(archiveRef("letta", 2))+" "+Retention) {
|
||||
t.Errorf("the seats were called %v", door.calls)
|
||||
}
|
||||
_, finding, _ := b.Get(FindingsState, sweepFindingKey)
|
||||
if finding != (c.refuse != "") {
|
||||
t.Errorf("the sweep's finding is written: %v, want %v", finding, c.refuse != "")
|
||||
}
|
||||
if c.refuse != "" {
|
||||
door.refuse = nil
|
||||
if _, err := h.Sweep(context.Background(), Sweeping{Most: MostPerSweep, By: "again"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, still, _ := b.Get(FindingsState, sweepFindingKey); still {
|
||||
t.Error("a clean sweep left the sweep's finding")
|
||||
}
|
||||
return
|
||||
}
|
||||
a, err := h.Sweep(context.Background(), Sweeping{Most: c.most, By: "after the build letta.x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fmt.Sprint(a.LetGo) != fmt.Sprint(c.letGo) || a.Left != c.left || a.Skipped != c.skipped || a.Held != 6 || a.Missing != 0 {
|
||||
t.Fatalf("let go of %v (left %d, skipped %d, held %d, missing %d), want %v", a.LetGo, a.Left, a.Skipped, a.Held, a.Missing, c.letGo)
|
||||
}
|
||||
// Every kept archive held before the first deletion.
|
||||
first := -1
|
||||
for i, x := range door.asked {
|
||||
if strings.HasPrefix(x, "let go ") && first < 0 {
|
||||
first = i
|
||||
}
|
||||
}
|
||||
if first != 6 {
|
||||
t.Errorf("the store was asked %v: the six kept archives are held first", door.asked)
|
||||
if c.seatForm && !strings.Contains(b.published[0], `"container-image-registry/letta/image@`+digestOf(imageRef("letta", 2))+`"`) {
|
||||
t.Errorf("artifacts-gone does not name the build's own form: %v", b.published)
|
||||
}
|
||||
if len(*acts) != len(c.letGo) || !strings.HasPrefix((*acts)[0], "seat:mesh-hand-acts.hand-act ") ||
|
||||
!strings.Contains((*acts)[0], c.letGo[0]) {
|
||||
@@ -310,7 +397,7 @@ func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsOnlyOnceTheLeaseIsAway(t
|
||||
// S25 (image-missing): a want whose build names an image the store does not hold is a finding in the seat's own state,
|
||||
// written once and deleted once the store holds it again; images names what a machine's wants and haves name.
|
||||
func TestAWantedImageTheStoreLacksIsAFindingAndImagesNamesWantsAndHaves(t *testing.T) {
|
||||
h, b, _, _ := aMesh(t, nil)
|
||||
h, b, _, _ := aMesh(t)
|
||||
view, _ := h.Listed(context.Background())
|
||||
repo, digest, _ := splitReference(imageRef("letta", 1))
|
||||
delete(view.L.Repos[repo].Revisions, digest)
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
// The artifact-store-tools module's Go bundle (novox/hq ADR 0251 §1–3, ADR 0571 clause 2, to-be 51): a process the
|
||||
// node's runtime launches and speaks MCP over stdio to. It says what the artifact store holds — its repositories, its
|
||||
// size, and what the mesh keeps of each manifest — read from the store's own files through the store's own container,
|
||||
// and it holds the seat mesh-artifact-sweep: the keep set, the sweep at each build's end once the controller's lease names
|
||||
// no controller, and S25's image-missing.
|
||||
// and it holds the seat mesh-artifact-sweep: the keep set, the sweep at each build's end, and S25's image-missing.
|
||||
//
|
||||
// **A module beside the store, not the store's own.** A module that provides the artifact store cannot also build an
|
||||
// artifact: building publishes to the store, so the store would be needed to create itself. So these tools are a
|
||||
@@ -11,12 +10,16 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
|
||||
@@ -27,13 +30,23 @@ func main() {
|
||||
Run: ExecRunner,
|
||||
UID: os.Getuid(),
|
||||
}
|
||||
h := &Holder{Bus: runtimeBus{}, Door: DoorAt(os.Getenv("MESH_ARTIFACT_STORE_ADDRESS_FILE")), Listed: s.View,
|
||||
Now: time.Now, Logf: logf}
|
||||
h.Acts = handacts.Recorder{Ask: stdio.Ask, LeaseAway: h.LeaseAway}
|
||||
go follow(h)
|
||||
v := &views.Reader{Module: "artifact-store-tools", Logf: logf, Findings: views.OwnFindings{}, Acts: handacts.Recorder{Ask: stdio.Ask}}
|
||||
v.Follow(delivery.AssignerModule, "wants", WantsTool)
|
||||
v.Follow(delivery.IndexerModule, "manifests", ManifestsTool)
|
||||
h := &Holder{Bus: runtimeBus{views: v}, Door: DoorAt(os.Getenv("MESH_ARTIFACT_STORE_ADDRESS_FILE")), Listed: s.View,
|
||||
Seats: stdio.Ask, Now: time.Now, Logf: logf}
|
||||
h.Acts = handacts.Recorder{Ask: stdio.Ask}
|
||||
v.Start(context.Background())
|
||||
go follow(h, v)
|
||||
// An empty name serves as the module the runtime names (MESH_SERVED_MODULE): artifact-store-tools.
|
||||
if err := stdio.Serve("", append(Tools(s, h, &Catalogue{Ask: stdio.Ask}), SeatTools(h, s)...)); err != nil {
|
||||
if err := stdio.Serve("", append(append(Tools(s, h, &Catalogue{Ask: stdio.Ask}), SeatTools(h, s)...), v.Tools("artifact_store")...)); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// The owners' read tools of the states the holder follows, which the manifest invokes (ADR 0615).
|
||||
const (
|
||||
WantsTool = delivery.AssignerModule + ".assigner_read_wants"
|
||||
ManifestsTool = delivery.IndexerModule + ".indexer_read_manifests"
|
||||
)
|
||||
|
||||
@@ -2,22 +2,16 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// What the records say, asked of the controller and never copied (novox/hq ADR 0251 §2). The
|
||||
// controller decides what the mesh keeps; this bundle only sets that beside what the store holds.
|
||||
// What the records say, in the shape the seat's holder answers them (Holder; novox/hq ADR 0251 §2, ADR 0571 clause 2):
|
||||
// the holder decides what the mesh keeps; the tools only set that beside what the store holds.
|
||||
|
||||
// Asking is how a tool on the mesh is asked: stdio.Ask, or a test's.
|
||||
type Asking func(key string, body any) (json.RawMessage, error)
|
||||
|
||||
// ControllerSeat is the seat whose verbs the controller serves.
|
||||
const ControllerSeat = "mesh-controller"
|
||||
|
||||
// Record is one reference the mesh recorded making, as the controller's `artifacts` answers it.
|
||||
// Record is one reference the mesh recorded making, as the seat's `artifacts` answers it.
|
||||
type Record struct {
|
||||
Reference string `json:"reference"`
|
||||
Repository string `json:"repository,omitempty"`
|
||||
@@ -27,7 +21,7 @@ type Record struct {
|
||||
Why []string `json:"why,omitempty"`
|
||||
}
|
||||
|
||||
// Records is the controller's `artifacts` answer.
|
||||
// Records is the seat's `artifacts` answer.
|
||||
type Records struct {
|
||||
Store string `json:"store"`
|
||||
KeptBuilds int `json:"kept_builds"`
|
||||
@@ -39,7 +33,7 @@ type Records struct {
|
||||
Note string `json:"-"`
|
||||
}
|
||||
|
||||
// CollectAnswer is the controller's `collect` answer.
|
||||
// CollectAnswer is the seat's `collect` answer.
|
||||
type CollectAnswer struct {
|
||||
DryRun bool `json:"dry_run"`
|
||||
Store string `json:"store"`
|
||||
@@ -53,15 +47,12 @@ type CollectAnswer struct {
|
||||
Skipped int `json:"skipped"`
|
||||
Left int `json:"left"`
|
||||
Stopped string `json:"stopped"`
|
||||
// Rest is whatever else the controller said, passed on as it said it.
|
||||
// Rest is whatever else the sweep said, passed on as it said it.
|
||||
Rest map[string]json.RawMessage `json:"-"`
|
||||
}
|
||||
|
||||
// Controller asks the controller's seat through the runtime.
|
||||
type Controller struct{ Ask Asking }
|
||||
|
||||
// answerOf reads a verb's answer whatever wraps it: the controller's {output, ok, answer}, a text the
|
||||
// runtime handed over, or the protocol's content list.
|
||||
// answerOf reads a verb's answer whatever wraps it: {output, ok, answer}, a text the runtime handed over, or the
|
||||
// protocol's content list.
|
||||
func answerOf(raw json.RawMessage) (json.RawMessage, string, bool) {
|
||||
var s string
|
||||
if json.Unmarshal(raw, &s) == nil {
|
||||
@@ -95,89 +86,11 @@ func answerOf(raw json.RawMessage) (json.RawMessage, string, bool) {
|
||||
return raw, string(raw), true
|
||||
}
|
||||
|
||||
func (c Controller) verb(name string, args map[string]any) (json.RawMessage, error) {
|
||||
if c.Ask == nil {
|
||||
return nil, errors.New("this bundle has no way to ask the controller")
|
||||
}
|
||||
raw, err := c.Ask("seat:"+ControllerSeat+"."+name, args)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the controller's %s could not be asked: %w", name, err)
|
||||
}
|
||||
answer, output, ok := answerOf(raw)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("the controller refused %s: %s", name, lastLine(output))
|
||||
}
|
||||
if len(answer) == 0 || string(answer) == "null" {
|
||||
return nil, fmt.Errorf("the controller's %s answered no data: %s", name, lastLine(output))
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
func lastLine(s string) string {
|
||||
lines := strings.Split(strings.TrimSpace(s), "\n")
|
||||
return strings.TrimSpace(lines[len(lines)-1])
|
||||
}
|
||||
|
||||
// Artifacts asks what the records say, with the references already let go of when the answer can carry
|
||||
// them; when it cannot, without, and says so.
|
||||
func (c Controller) Artifacts(repository string) (*Records, error) {
|
||||
args := map[string]any{"collected": "true"}
|
||||
if repository != "" {
|
||||
args["repository"] = repository
|
||||
}
|
||||
raw, err := c.verb("artifacts", args)
|
||||
collected := true
|
||||
note := ""
|
||||
if err != nil {
|
||||
delete(args, "collected")
|
||||
var again error
|
||||
raw, again = c.verb("artifacts", args)
|
||||
if again != nil {
|
||||
return nil, again
|
||||
}
|
||||
collected = false
|
||||
note = "the references the mesh already let go of could not be asked for (" + err.Error() +
|
||||
"), so a manifest the mesh let go of that the store still holds is counted as unrecorded"
|
||||
}
|
||||
var r Records
|
||||
if err := json.Unmarshal(raw, &r); err != nil {
|
||||
return nil, fmt.Errorf("the controller's artifacts answer is not readable: %v", err)
|
||||
}
|
||||
for i := range r.References {
|
||||
fill(&r.References[i])
|
||||
}
|
||||
r.CollectedAsked, r.Note = collected, note
|
||||
return &r, nil
|
||||
}
|
||||
|
||||
// Collect asks the controller's sweep: a dry run unless confirm, which needs why.
|
||||
func (c Controller) Collect(why string, confirm bool, most int) (*CollectAnswer, error) {
|
||||
args := map[string]any{}
|
||||
if confirm {
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return nil, errors.New("a real collection needs why: nothing was asked")
|
||||
}
|
||||
args["why"] = why
|
||||
args["confirm"] = "true"
|
||||
}
|
||||
if most > 0 {
|
||||
args["most"] = strconv.Itoa(most)
|
||||
}
|
||||
raw, err := c.verb("collect", args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var a CollectAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("the controller's collect answer is not readable: %v", err)
|
||||
}
|
||||
_ = json.Unmarshal(raw, &a.Rest)
|
||||
if !confirm && !a.DryRun && len(a.LetGo) > 0 {
|
||||
return nil, fmt.Errorf("the controller let go of %d artifacts when only a dry run was asked", len(a.LetGo))
|
||||
}
|
||||
return &a, nil
|
||||
}
|
||||
|
||||
// fill reads the repository, digest and kind out of a reference when the answer left them out.
|
||||
func fill(r *Record) {
|
||||
repo, digest, kind := splitReference(r.Reference)
|
||||
|
||||
@@ -2,13 +2,16 @@ package main
|
||||
|
||||
import (
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -280,7 +283,7 @@ func TestOnlyTheirBytesLeaveOutWhatAnotherStateMarks(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// asked records every ask, and answers as the controller would.
|
||||
// asked records every ask, and answers as the seat and the catalogue would.
|
||||
type asked struct {
|
||||
mu sync.Mutex
|
||||
calls []string
|
||||
@@ -293,6 +296,51 @@ type asked struct {
|
||||
noCatalogue bool
|
||||
}
|
||||
|
||||
// recordsAsked is the tools' Recorded in a test: each call asked of asked, as `artifacts` or `collect`, with the
|
||||
// arguments a real run gives the seat's sweep.
|
||||
type recordsAsked struct{ a *asked }
|
||||
|
||||
func (r recordsAsked) Artifacts(repository string) (*Records, error) {
|
||||
args := map[string]any{"collected": "true"}
|
||||
if repository != "" {
|
||||
args["repository"] = repository
|
||||
}
|
||||
raw, err := r.a.ask("artifacts", args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var recs Records
|
||||
answer, _, _ := answerOf(raw)
|
||||
if err := json.Unmarshal(answer, &recs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range recs.References {
|
||||
fill(&recs.References[i])
|
||||
}
|
||||
recs.CollectedAsked = true
|
||||
return &recs, nil
|
||||
}
|
||||
|
||||
func (r recordsAsked) Collect(why string, confirm bool, most int) (*CollectAnswer, error) {
|
||||
args := map[string]any{}
|
||||
if confirm {
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return nil, fmt.Errorf("a real collection needs why: nothing was asked")
|
||||
}
|
||||
args["why"], args["confirm"] = why, "true"
|
||||
}
|
||||
if most > 0 {
|
||||
args["most"] = strconv.Itoa(most)
|
||||
}
|
||||
raw, err := r.a.ask("collect", args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var c CollectAnswer
|
||||
answer, _, _ := answerOf(raw)
|
||||
return &c, json.Unmarshal(answer, &c)
|
||||
}
|
||||
|
||||
func (a *asked) ask(key string, body any) (json.RawMessage, error) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
@@ -304,7 +352,7 @@ func (a *asked) ask(key string, body any) (json.RawMessage, error) {
|
||||
}
|
||||
var answer any
|
||||
switch key {
|
||||
case "seat:mesh-controller.artifacts":
|
||||
case "artifacts":
|
||||
answer = records()
|
||||
case CatalogueModules:
|
||||
if a.noCatalogue {
|
||||
@@ -320,8 +368,8 @@ func (a *asked) ask(key string, body any) (json.RawMessage, error) {
|
||||
if a.failing[name] {
|
||||
return nil, fmt.Errorf("no answer")
|
||||
}
|
||||
if args["commit"] != "c-"+name {
|
||||
return nil, fmt.Errorf("asked %s at %v", name, args["commit"])
|
||||
if _, has := args["commit"]; has {
|
||||
return nil, fmt.Errorf("asked %s at %v: the seat's verb takes no commit", name, args["commit"])
|
||||
}
|
||||
manifest, has := a.definitions[name]
|
||||
if !has {
|
||||
@@ -329,9 +377,10 @@ func (a *asked) ask(key string, body any) (json.RawMessage, error) {
|
||||
break
|
||||
}
|
||||
// The catalogue's tool answers through the runtime as the protocol's content list.
|
||||
raw, _ := json.Marshal(map[string]any{"module": map[string]any{"module": name, "manifest": json.RawMessage(manifest)}})
|
||||
raw, _ := json.Marshal(catalogue.CatalogOne{Module: catalogue.CatalogModule{
|
||||
CatalogEntry: catalogue.CatalogEntry{Module: name, Commit: "c-" + name}, Manifest: json.RawMessage(manifest)}})
|
||||
return json.Marshal(map[string]any{"content": []map[string]any{{"type": "text", "text": string(raw)}}})
|
||||
case "seat:mesh-controller.collect":
|
||||
case "collect":
|
||||
if args["confirm"] == "true" {
|
||||
answer = CollectAnswer{LetGo: []string{"artifact-store://app/server@" + d("m-old")}}
|
||||
} else {
|
||||
@@ -347,7 +396,7 @@ func (a *asked) ask(key string, body any) (json.RawMessage, error) {
|
||||
func tool(t *testing.T, a *asked, name string) func(map[string]any) (any, error) {
|
||||
f := world()
|
||||
s := &Store{Container: "mesh-registry", Run: f.run(t)}
|
||||
for _, tl := range Tools(s, Controller{Ask: a.ask}, &Catalogue{Ask: a.ask}) {
|
||||
for _, tl := range Tools(s, recordsAsked{a}, &Catalogue{Ask: a.ask}) {
|
||||
if tl.Name == name {
|
||||
return tl.Run
|
||||
}
|
||||
@@ -390,31 +439,17 @@ func TestARealRunWithoutWhyIsRefusedAndAsksNothing(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReferencesFallBackWhenCollectedCannotBeCarried(t *testing.T) {
|
||||
a := &asked{fail: map[string]bool{"collected": true}}
|
||||
out, err := tool(t, a, "artifact_store_references")(map[string]any{"repository": "app/server"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := out.(map[string]any)
|
||||
if m["note"] == nil || m["manifests"] == nil {
|
||||
t.Errorf("answer %v", m)
|
||||
}
|
||||
if _, err := tool(t, a, "artifact_store_references")(map[string]any{"repository": "nope"}); err == nil {
|
||||
t.Error("an unknown repository was answered")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Tools []string `json:"tools"`
|
||||
Invokes []string `json:"invokes"`
|
||||
Reads []string `json:"reads"`
|
||||
Claims []struct {
|
||||
Tools []string `json:"tools"`
|
||||
Invokes []string `json:"invokes"`
|
||||
Reads []string `json:"reads"`
|
||||
Consumes []string `json:"consumes"`
|
||||
Claims []struct {
|
||||
Name string `json:"name"`
|
||||
Serves []string `json:"serves"`
|
||||
} `json:"claims"`
|
||||
@@ -423,7 +458,7 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var served []string
|
||||
for _, tl := range Tools(&Store{}, Controller{}, &Catalogue{}) {
|
||||
for _, tl := range append(Tools(&Store{}, recordsAsked{&asked{}}, &Catalogue{}), (&views.Reader{}).Tools("artifact_store")...) {
|
||||
if !strings.HasPrefix(tl.Name, "artifact_store_") || tl.Description == "" || tl.Run == nil {
|
||||
t.Errorf("tool %q", tl.Name)
|
||||
}
|
||||
@@ -436,11 +471,19 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
|
||||
t.Fatalf("served %v, manifest %v", served, listed)
|
||||
}
|
||||
// ADR 0571 clause 2: it holds the seat mesh-artifact-sweep and serves its verbs, and calls no verb of the controller's;
|
||||
// the lease is the one controller state it reads (clause 10's exemption).
|
||||
want := []string{"seat:mesh-hand-acts.hand-act", CatalogueModules, CatalogueModule}
|
||||
// it reads no controller state, the lease neither (issue 1126).
|
||||
// CAT-AS1: what it holds and removes it asks the two registry seats the artifact store's holder serves.
|
||||
want := []string{"seat:mesh-hand-acts.hand-act", CatalogueModules, CatalogueModule, WantsTool, ManifestsTool,
|
||||
"seat:oci-artifact-registry.hold", "seat:oci-artifact-registry.release", "seat:oci-artifact-registry.remove",
|
||||
"seat:container-image-registry.hold", "seat:container-image-registry.release", "seat:container-image-registry.remove"}
|
||||
if !reflect.DeepEqual(m.Invokes, want) {
|
||||
t.Errorf("invokes %v", m.Invokes)
|
||||
}
|
||||
// ADR 0615: the assigner's wants and the indexer's manifests are asked once and followed by their events, not read.
|
||||
if !reflect.DeepEqual(m.Consumes, []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.views.manifests"}) ||
|
||||
slices.Contains(m.Reads, WantsRead) || slices.Contains(m.Reads, ManifestsRead) {
|
||||
t.Errorf("consumes %v, reads %v", m.Consumes, m.Reads)
|
||||
}
|
||||
var verbs []string
|
||||
for _, tl := range SeatTools(&Holder{}, &Store{}) {
|
||||
verbs = append(verbs, strings.TrimPrefix(tl.Name, catalogue.ArtifactSweepSeat+"."))
|
||||
@@ -455,7 +498,7 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
|
||||
t.Errorf("claims %+v, serves %v, the seat's verbs %v", m.Claims, verbs, contract)
|
||||
}
|
||||
for _, r := range m.Reads {
|
||||
if strings.HasPrefix(r, "mesh-controller.") && r != LeaseRead {
|
||||
if strings.HasPrefix(r, "mesh-controller.") {
|
||||
t.Errorf("reads the controller's %s", r)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
package main
|
||||
|
||||
// **The sweep asks the registry seats** (CAT-AS1; novox/hq ADR 0617 §2, design 51): the artifact store's write side is
|
||||
// its holder's, so the sweep no longer deletes or holds over the store's door. What it keeps from retention it asks the
|
||||
// seat's `hold` for (a kept bundle or archive on oci-artifact-registry, a kept copy of an upstream base on
|
||||
// container-image-registry), and what is eligible it asks the seat's `remove` for, with the sweep's reason; both verbs
|
||||
// are in the manifest's `invokes`, and only this module is on their caller lists. A reference is read in both forms
|
||||
// (ADR 0605 §9): the older `artifact-store://<path>` and the seat's `<seat>/<module>/<artifact>@<digest>`; the keep set
|
||||
// keys each by the older form, so the two meet as one artifact. A call a seat refuses stops the sweep and writes the
|
||||
// sweep's finding, sweep-refused; the next sweep with no refusal clears it.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
ociseat "git.novox.be/novox/mesh-contracts/go/seats/ociartifactregistry"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
|
||||
reg "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry"
|
||||
)
|
||||
|
||||
// SweepRefused is the sweep's finding: a seat refused a hold or a remove the sweep asked.
|
||||
const SweepRefused = "sweep-refused"
|
||||
|
||||
// sweepFindingKey is the one key of the sweep's finding in the module's findings.
|
||||
var sweepFindingKey = health.FindingKey(health.ScopeModule, Module, SweepRefused)
|
||||
|
||||
// canonical is a kept reference in the older form, `artifact-store://<path>`, whichever form it came in; another is as
|
||||
// it was.
|
||||
func canonical(reference string) string {
|
||||
path, kept := catalogue.InArtifactStore(reference)
|
||||
if !kept {
|
||||
return reference
|
||||
}
|
||||
return catalogue.ArtifactStoreScheme + path
|
||||
}
|
||||
|
||||
// seatOf is the seat, name and digest a kept reference is held under: a bundle or an archive on oci-artifact-registry,
|
||||
// an image or a copy of an upstream base on container-image-registry.
|
||||
func seatOf(reference string) (seat, name, digest string, err error) {
|
||||
path, kept := catalogue.InArtifactStore(reference)
|
||||
if !kept {
|
||||
return "", "", "", fmt.Errorf("%s is no reference into the mesh's registries", reference)
|
||||
}
|
||||
if name, digest, ok := strings.Cut(path, "/blobs/"); ok {
|
||||
return ociseat.Seat, name, digest, nil
|
||||
}
|
||||
if name, digest, ok := strings.Cut(path, "@"); ok {
|
||||
return reg.Seat, name, digest, nil
|
||||
}
|
||||
return "", "", "", fmt.Errorf("%s names no digest", reference)
|
||||
}
|
||||
|
||||
// isCopy is whether a kept reference is a copy of an upstream base.
|
||||
func isCopy(reference string) bool {
|
||||
path, _ := catalogue.InArtifactStore(reference)
|
||||
return strings.HasPrefix(path, "upstream/")
|
||||
}
|
||||
|
||||
// askSeat asks verb of the seat reference is held on, with body built from its name and digest.
|
||||
func (h *Holder) askSeat(reference, verb string, body func(name, digest string) any, answer any) error {
|
||||
seat, name, digest, err := seatOf(reference)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if h.Seats == nil {
|
||||
return fmt.Errorf("this process reaches no registry seat")
|
||||
}
|
||||
raw, err := h.Seats("seat:"+seat+"."+verb, body(name, digest))
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s's %s of %s@%s: %w", seat, verb, name, digest, err)
|
||||
}
|
||||
if answer != nil && len(raw) > 0 {
|
||||
if err := json.Unmarshal(raw, answer); err != nil {
|
||||
return fmt.Errorf("%s's answer to %s: %w", seat, verb, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// holdOnSeat asks the seat to keep reference from retention, for why.
|
||||
func (h *Holder) holdOnSeat(reference, why string) (bool, error) {
|
||||
var a reg.HoldAnswer
|
||||
err := h.askSeat(reference, reg.VerbHold, func(name, digest string) any {
|
||||
return reg.HoldInput{Name: name, Digest: digest, Reason: why}
|
||||
}, &a)
|
||||
if err == nil && !a.Held {
|
||||
err = fmt.Errorf("%s: the seat answered it is not held", reference)
|
||||
}
|
||||
return a.Held, err
|
||||
}
|
||||
|
||||
// Retention is the reason of a removal by digest, the sweep's alone (design 51 *Who may call which verb*): the holder
|
||||
// removes an untagged thing only for it. The seat's definition names it.
|
||||
const Retention = reg.RemoveInputReasonRetention
|
||||
|
||||
// removeOnSeat lets reference go: a hold an earlier sweep asked while it was kept is released first (the holder
|
||||
// refuses to remove a held thing; one not held answers released false), then it is removed for retention. why goes with
|
||||
// the release; the hand act records it.
|
||||
func (h *Holder) removeOnSeat(reference, why string) error {
|
||||
if err := h.askSeat(reference, reg.VerbRelease, func(name, digest string) any {
|
||||
return reg.ReleaseInput{Name: name, Digest: digest, Reason: why}
|
||||
}, ®.ReleaseAnswer{}); err != nil {
|
||||
return err
|
||||
}
|
||||
return h.askSeat(reference, reg.VerbRemove, func(name, digest string) any {
|
||||
return reg.RemoveInput{Name: name, Digest: digest, Reason: Retention}
|
||||
}, ®.RemoveAnswer{})
|
||||
}
|
||||
|
||||
// sweepFinding writes the sweep's finding: what the seat refused, and why.
|
||||
func (h *Holder) sweepFinding(what string, err error) {
|
||||
at := h.now().UTC()
|
||||
f := health.Finding{V: health.FindingVersion, Key: sweepFindingKey, Kind: SweepRefused,
|
||||
Subject: health.Subject{Scope: health.ScopeModule, ID: Module},
|
||||
Severity: health.Warning, Resolver: health.ResolverOperator,
|
||||
Summary: fmt.Sprintf("the artifact sweep stopped: a registry seat refused %s (the evidence names why)", what),
|
||||
Headline: "The artifact sweep stopped",
|
||||
Explanation: "The sweep keeps what the mesh still needs and removes the rest through the registry seats. A seat " +
|
||||
"refused one of its asks, so it stopped before anything more was removed; what it kept stays kept.",
|
||||
Needs: "read the seat holder's journal for the refusal; the next sweep asks again.",
|
||||
Resolved: "Resolved: the artifact sweep ran through without a refusal",
|
||||
Evidence: []health.Evidence{{At: at, Said: fmt.Sprintf("%s: %v", what, err)}},
|
||||
Source: "CAT-AS1", Observer: Module, At: at}
|
||||
if werr := h.Bus.Put(FindingsState, sweepFindingKey, f); werr != nil {
|
||||
h.logf("the sweep's finding (%s: %v) could not be written: %v", what, err, werr)
|
||||
}
|
||||
}
|
||||
|
||||
// sweepClean clears the sweep's finding after a sweep no seat refused.
|
||||
func (h *Holder) sweepClean() {
|
||||
if _, found, err := h.Bus.Get(FindingsState, sweepFindingKey); err == nil && found {
|
||||
if err := h.Bus.Delete(FindingsState, sweepFindingKey); err != nil {
|
||||
h.logf("the sweep's finding could not be cleared: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// keptWhy is why a kept reference is held, as the seat's hold takes it.
|
||||
func keptWhy(r *Ref) string {
|
||||
if r == nil || len(r.Why) == 0 {
|
||||
return "kept by the artifact sweep"
|
||||
}
|
||||
return "kept: " + strings.Join(r.Why, "; ")
|
||||
}
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
)
|
||||
|
||||
// Recorded is what says which references the mesh keeps and lets go of them: the seat's holder (Holder, novox/hq ADR
|
||||
// 0571 clause 2), or before it the controller (Controller), each in the controller's answer's shape.
|
||||
// 0571 clause 2), or a test's stand-in.
|
||||
type Recorded interface {
|
||||
Artifacts(repository string) (*Records, error)
|
||||
Collect(why string, confirm bool, most int) (*CollectAnswer, error)
|
||||
|
||||
@@ -3,20 +3,24 @@ module artifactstoretools
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
require git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639 // indirect
|
||||
|
||||
require (
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry v0.0.0-20261104174344-ae5266e77639
|
||||
git.novox.be/novox/mesh-contracts/go/seats/ociartifactregistry v0.0.0-20261104174344-ae5266e77639
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/klauspost/compress v1.20.1 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.2 // indirect
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
golang.org/x/time v0.16.0 // indirect
|
||||
)
|
||||
|
||||
@@ -1,15 +1,21 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639 h1:Zt7CfWt2S2W/fVgWNyW2u2+xwz051LXvGxVTNIyfhdM=
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639/go.mod h1:0dD1XEJBJ0G5UYDQxSZI84nuujb6m3tOpJl4C8rQVvg=
|
||||
git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry v0.0.0-20261104174344-ae5266e77639 h1:5HeZzk6tKzcSwVivD3IcIvQ/2WIvysZvNvNKyq/oIBA=
|
||||
git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry v0.0.0-20261104174344-ae5266e77639/go.mod h1:pPxwhTMb3hIk7UUrBicJnGQCqfTEoBGJvNFspEK2U18=
|
||||
git.novox.be/novox/mesh-contracts/go/seats/ociartifactregistry v0.0.0-20261104174344-ae5266e77639 h1:+bkN8op2yrAlBYrOB+hYnIoIZkbjXsNmc/4//uMnL4E=
|
||||
git.novox.be/novox/mesh-contracts/go/seats/ociartifactregistry v0.0.0-20261104174344-ae5266e77639/go.mod h1:ooQa2CiP8JOvhP0bdQvwurV5wskRE7Cm68ecURBbCwU=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f h1:jDsfDMd48MXY9Zershc3eRxnhmx2sSpUfQkLq5jAIds=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f/go.mod h1:08OomDXdwVN8/87nyDtv5R3rH/6LoP8SFFXz30bTYCM=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
|
||||
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc=
|
||||
github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
@@ -23,5 +29,5 @@ golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZ
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
|
||||
golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
package registry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// blobBound is the most GetBlob reads: a templates archive is a few kilobytes; anything near this is no archive a
|
||||
// take-in reads text from.
|
||||
const blobBound = 64 << 20
|
||||
|
||||
// GetBlob is the archive a recorded reference names (`artifact-store://<repository>/blobs/sha256:<hex>`), read whole
|
||||
// and checked against the digest it names: bytes that hash to another digest are refused, never read (R4e: the
|
||||
// controller reads a templates archive by the reference of a builder's done key). Gone when the store does not hold
|
||||
// it; ErrNotOurs for a reference outside the store or one that is not an archive.
|
||||
func (s Store) GetBlob(ctx context.Context, reference string) ([]byte, error) {
|
||||
repository, digest, archive, err := s.archive(reference)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !archive {
|
||||
return nil, fmt.Errorf("%w: %s is not an archive", ErrNotOurs, reference)
|
||||
}
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url(repository, "blobs", digest), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
response, err := s.client().Do(request)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
switch response.StatusCode {
|
||||
case http.StatusOK:
|
||||
case http.StatusNotFound:
|
||||
return nil, Gone
|
||||
default:
|
||||
return nil, fmt.Errorf("the artifact store answered %s for %s/blobs/%s", response.Status, repository, digest)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(response.Body, blobBound+1))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading %s/blobs/%s: %w", repository, digest, err)
|
||||
}
|
||||
if len(body) > blobBound {
|
||||
return nil, fmt.Errorf("%s/blobs/%s is larger than %d bytes, which no archive read here is", repository, digest, blobBound)
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
if got := "sha256:" + hex.EncodeToString(sum[:]); got != digest {
|
||||
return nil, fmt.Errorf("the artifact store answered %s/blobs/%s with bytes of the digest %s: refused", repository,
|
||||
digest, got)
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
package registry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A take-in reads a templates archive by its reference, and only bytes of the digest the reference names.
|
||||
func TestGetBlobReadsOnlyTheBytesOfItsDigest(t *testing.T) {
|
||||
good := []byte("the archive")
|
||||
sum := sha256.Sum256(good)
|
||||
digest := "sha256:" + hex.EncodeToString(sum[:])
|
||||
liar := "sha256:" + strings.Repeat("0", 64)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case strings.HasSuffix(r.URL.Path, "/x/files/blobs/"+digest), strings.HasSuffix(r.URL.Path, "/x/files/blobs/"+liar):
|
||||
_, _ = w.Write(good)
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
s := Store{Address: strings.TrimPrefix(server.URL, "http://")}
|
||||
ctx := context.Background()
|
||||
body, err := s.GetBlob(ctx, "artifact-store://x/files/blobs/"+digest)
|
||||
if err != nil || string(body) != string(good) {
|
||||
t.Fatalf("the archive: %q %v", body, err)
|
||||
}
|
||||
if _, err := s.GetBlob(ctx, "artifact-store://x/files/blobs/"+liar); err == nil || !strings.Contains(err.Error(), "refused") {
|
||||
t.Fatalf("bytes of another digest are read: %v", err)
|
||||
}
|
||||
if _, err := s.GetBlob(ctx, "artifact-store://x/other/blobs/"+digest); !errors.Is(err, Gone) {
|
||||
t.Fatalf("a blob the store does not hold: %v", err)
|
||||
}
|
||||
if _, err := s.GetBlob(ctx, "registry.example/x@"+digest); !errors.Is(err, ErrNotOurs) {
|
||||
t.Fatalf("a reference outside the store: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -1,345 +0,0 @@
|
||||
package registry
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// Every archive the store keeps is held by a manifest (novox/hq issue 253, ADR 0189).
|
||||
//
|
||||
// **The store's collector marks only from manifests.** The mesh's store is a stock registry, and
|
||||
// its nightly `registry garbage-collect` walks every manifest in every repository, marks the blobs
|
||||
// those manifests name, and deletes every blob it did not mark. An image is a manifest, so what
|
||||
// the mesh keeps of an image survives. An archive was not: the builder put it in the store as a
|
||||
// bare blob — upload, then `PUT ?digest=` — and nothing in the store names it. To the collector a
|
||||
// bare blob is unreferenced, so the first real collection would have deleted every archive the
|
||||
// mesh holds, kept or not, and every machine pinning a bundle would have found it gone. The
|
||||
// collector runs `--dry-run` until this is true.
|
||||
//
|
||||
// **So each archive gets a holder**: the smallest OCI image manifest that names it — the empty
|
||||
// config, one layer, nothing else — put in the archive's own repository, by digest, untagged. The
|
||||
// collector marks it and so keeps the archive; the sweep lets go of an archive by deleting its
|
||||
// holder first, which is what lets the bytes go at the next collection.
|
||||
//
|
||||
// **Nothing a machine reads changes.** The recorded reference stays
|
||||
// `artifact-store://<module>/<artifact>/blobs/sha256:…`, and machines fetch the blob exactly as
|
||||
// before. The holder is the store's bookkeeping, not a second way to reach anything.
|
||||
//
|
||||
// **Deterministic, so it never needs recording.** The holder is composed from the archive's digest
|
||||
// and size alone, in a fixed field order with no timestamps or annotations, so the sweep can
|
||||
// compute which manifest holds any archive from the reference it already has plus one HEAD for the
|
||||
// size. No schema change, no second record that could disagree with the store.
|
||||
|
||||
const (
|
||||
// mediaManifest is the type a holder is put and asked for as.
|
||||
mediaManifest = "application/vnd.oci.image.manifest.v1+json"
|
||||
// mediaEmpty is the OCI empty descriptor's type: a config that says nothing, for a manifest
|
||||
// whose only purpose is to name its layer.
|
||||
mediaEmpty = "application/vnd.oci.empty.v1+json"
|
||||
// emptyDigest is the digest of `{}`, the empty config's content, fixed by the OCI spec.
|
||||
emptyDigest = "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"
|
||||
// mediaArchive is the layer type an archive is held as. Every archive the builder publishes is
|
||||
// `pack`'s gzipped tar, so this is the true type and not a placeholder — and it is a constant,
|
||||
// not read from anywhere, because the holder must be recomputable from the reference alone.
|
||||
mediaArchive = "application/vnd.oci.image.layer.v1.tar+gzip"
|
||||
)
|
||||
|
||||
// emptyConfig is the content emptyDigest names.
|
||||
var emptyConfig = []byte("{}")
|
||||
|
||||
// manifestAccept is what a manifest is asked for as. A registry answers a manifest HEAD only in a
|
||||
// type the caller named, and answers 404 to a bare one for a manifest it holds perfectly well
|
||||
// (measured 2026-09-28; internal/builder/registry.go says how that was found).
|
||||
var manifestAccept = []string{
|
||||
mediaManifest,
|
||||
"application/vnd.docker.distribution.manifest.v2+json",
|
||||
}
|
||||
|
||||
type descriptor struct {
|
||||
MediaType string `json:"mediaType"`
|
||||
Digest string `json:"digest"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
type holderManifest struct {
|
||||
SchemaVersion int `json:"schemaVersion"`
|
||||
MediaType string `json:"mediaType"`
|
||||
Config descriptor `json:"config"`
|
||||
Layers []descriptor `json:"layers"`
|
||||
}
|
||||
|
||||
// Holder is the manifest that holds an archive in the store, and its digest.
|
||||
//
|
||||
// A pure function of the archive's digest and size: the same two in give the same bytes out,
|
||||
// always, because `encoding/json` writes a struct's fields in their declared order and there is
|
||||
// nothing here that varies by when or where it was composed.
|
||||
func Holder(digest string, size int64) (body []byte, holder string) {
|
||||
body, err := json.Marshal(holderManifest{
|
||||
SchemaVersion: 2,
|
||||
MediaType: mediaManifest,
|
||||
Config: descriptor{MediaType: mediaEmpty, Digest: emptyDigest, Size: int64(len(emptyConfig))},
|
||||
Layers: []descriptor{{MediaType: mediaArchive, Digest: digest, Size: size}},
|
||||
})
|
||||
if err != nil {
|
||||
// Marshalling a struct of strings and integers cannot fail.
|
||||
panic(err)
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
return body, "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// Hold makes sure the store holds this archive by a manifest, and says whether it had to write one.
|
||||
//
|
||||
// Takes a reference as the mesh records it. An image is its own manifest and needs no holder, so
|
||||
// it answers false and nothing is asked. Idempotent: a holder already there is left alone, which
|
||||
// is what lets the sweep run it over every kept archive on every build and so backfill the bare
|
||||
// blobs published before holders existed (novox/hq issue 253).
|
||||
//
|
||||
// Gone when the store does not hold the archive at all: there is nothing to hold, and that is a
|
||||
// fact the caller reports rather than one this invents a remedy for.
|
||||
func (s Store) Hold(ctx context.Context, reference string) (bool, error) {
|
||||
repository, digest, archive, err := s.archive(reference)
|
||||
if err != nil || !archive {
|
||||
return false, err
|
||||
}
|
||||
size, err := s.blobSize(ctx, repository, digest)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return s.HoldBlob(ctx, repository, digest, size)
|
||||
}
|
||||
|
||||
// Held is whether the store holds this archive by its manifest. Asks and changes nothing — the
|
||||
// question an operator needs answered with "none unheld" before the collector is let loose.
|
||||
//
|
||||
// An image answers true: it is its own manifest. An archive the store does not have answers Gone.
|
||||
func (s Store) Held(ctx context.Context, reference string) (bool, error) {
|
||||
repository, digest, archive, err := s.archive(reference)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !archive {
|
||||
return true, nil
|
||||
}
|
||||
size, err := s.blobSize(ctx, repository, digest)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
_, holder := Holder(digest, size)
|
||||
return s.has(ctx, s.url(repository, "manifests", holder), manifestAccept...)
|
||||
}
|
||||
|
||||
// HoldBlob puts the holder for a blob of this digest and size into its repository, unless it is
|
||||
// there already. Answers whether it wrote one.
|
||||
//
|
||||
// The builder calls this with the size it has just uploaded; the sweep, through Hold, with the size
|
||||
// the store reports. Both arrive at the same holder, which is the point of composing it.
|
||||
func (s Store) HoldBlob(ctx context.Context, repository, digest string, size int64) (bool, error) {
|
||||
if s.Address == "" {
|
||||
return false, fmt.Errorf("this mesh has no artifact store on its network to hold %s/%s in", repository, digest)
|
||||
}
|
||||
body, holder := Holder(digest, size)
|
||||
there, err := s.has(ctx, s.url(repository, "manifests", holder), manifestAccept...)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if there {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// The config must be in the repository before a manifest naming it is accepted: a registry
|
||||
// refuses a manifest whose blobs it cannot find there, which is the property that makes a
|
||||
// holder mean something.
|
||||
if err := s.putBlob(ctx, repository, emptyDigest, emptyConfig); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
// **By digest, never by tag.** A tag would be one more name to move and one more thing the
|
||||
// collector's `--delete-untagged` would read as meaningful; the mesh names nothing by tag that
|
||||
// it pins by digest, and an untagged manifest is kept by plain collection.
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodPut,
|
||||
s.url(repository, "manifests", holder), bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
request.Header.Set("Content-Type", mediaManifest)
|
||||
response, err := s.client().Do(request)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusCreated {
|
||||
said, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
|
||||
return false, fmt.Errorf("the artifact store refused to hold %s/%s: %s %s",
|
||||
repository, digest, response.Status, strings.TrimSpace(string(said)))
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// letGoOfHolder deletes the manifest holding an archive, before the archive's own link goes.
|
||||
//
|
||||
// **Holder first.** Deleting the blob link alone leaves a manifest still naming the blob, and the
|
||||
// collector would keep its bytes for ever on the strength of it — the sweep would record the
|
||||
// archive collected while the disk said otherwise. Deleting the holder first and failing before
|
||||
// the link goes leaves an unheld archive that the next sweep still offers, which is safe.
|
||||
//
|
||||
// A store that no longer has the blob answers Gone: without its size the holder cannot be named,
|
||||
// and without the blob there is nothing left for a holder to keep. A store that never had a holder
|
||||
// for it — an archive published before holders, never backfilled — answers 404 to the delete, and
|
||||
// that is the outcome wanted.
|
||||
func (s Store) letGoOfHolder(ctx context.Context, repository, digest string) error {
|
||||
size, err := s.blobSize(ctx, repository, digest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, holder := Holder(digest, size)
|
||||
err = s.remove(ctx, s.url(repository, "manifests", holder), repository+"/manifests/"+holder)
|
||||
if err == Gone {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// archive reads a recorded reference into its repository and digest, and whether it is an archive
|
||||
// at all. Refuses as ErrNotOurs anything the mesh did not put in its own store.
|
||||
func (s Store) archive(reference string) (repository, digest string, archive bool, err error) {
|
||||
path, kept := catalogue.InArtifactStore(reference)
|
||||
if !kept {
|
||||
return "", "", false, fmt.Errorf("%w: %s", ErrNotOurs, reference)
|
||||
}
|
||||
if s.Address == "" {
|
||||
return "", "", false, fmt.Errorf("this mesh has no artifact store on its network to ask about %s", reference)
|
||||
}
|
||||
repository, kind, digest, err := split(path)
|
||||
if err != nil {
|
||||
return "", "", false, err
|
||||
}
|
||||
return repository, digest, kind == "blobs", nil
|
||||
}
|
||||
|
||||
// blobSize is how large the store says a blob is; Gone when it does not have it.
|
||||
func (s Store) blobSize(ctx context.Context, repository, digest string) (int64, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodHead, s.url(repository, "blobs", digest), nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
response, err := s.client().Do(request)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
switch response.StatusCode {
|
||||
case http.StatusOK:
|
||||
case http.StatusNotFound:
|
||||
return 0, Gone
|
||||
default:
|
||||
return 0, fmt.Errorf("the artifact store answered %s for %s/blobs/%s", response.Status, repository, digest)
|
||||
}
|
||||
// Read from the header rather than ContentLength: a HEAD's ContentLength is what the response
|
||||
// says it would have sent, which Go reports faithfully, but a proxy in between is free to drop
|
||||
// it, and the header is what the registry itself wrote.
|
||||
if length := response.Header.Get("Content-Length"); length != "" {
|
||||
if n, err := strconv.ParseInt(length, 10, 64); err == nil && n >= 0 {
|
||||
return n, nil
|
||||
}
|
||||
}
|
||||
if response.ContentLength >= 0 {
|
||||
return response.ContentLength, nil
|
||||
}
|
||||
return 0, fmt.Errorf("the artifact store holds %s/blobs/%s and will not say how large it is", repository, digest)
|
||||
}
|
||||
|
||||
// putBlob uploads a small blob unless the repository already has it: ask where, then put it there
|
||||
// naming the digest — the registry's own two steps, the same the builder takes for an archive.
|
||||
func (s Store) putBlob(ctx context.Context, repository, digest string, body []byte) error {
|
||||
if there, err := s.has(ctx, s.url(repository, "blobs", digest)); err != nil {
|
||||
return err
|
||||
} else if there {
|
||||
return nil
|
||||
}
|
||||
start, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
"http://"+s.Address+"/v2/"+repository+"/blobs/uploads/", nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
begun, err := s.client().Do(start)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot start an upload to %s: %w", repository, err)
|
||||
}
|
||||
begun.Body.Close()
|
||||
if begun.StatusCode != http.StatusAccepted {
|
||||
return fmt.Errorf("the artifact store answered %s when asked where to put a blob in %s", begun.Status, repository)
|
||||
}
|
||||
where := begun.Header.Get("Location")
|
||||
if where == "" {
|
||||
return fmt.Errorf("the artifact store accepted an upload to %s and said nowhere to put it", repository)
|
||||
}
|
||||
if strings.HasPrefix(where, "/") {
|
||||
where = "http://" + s.Address + where
|
||||
}
|
||||
separator := "?"
|
||||
if strings.Contains(where, "?") {
|
||||
separator = "&"
|
||||
}
|
||||
put, err := http.NewRequestWithContext(ctx, http.MethodPut, where+separator+"digest="+digest, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
put.Header.Set("Content-Type", "application/octet-stream")
|
||||
done, err := s.client().Do(put)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer done.Body.Close()
|
||||
if done.StatusCode != http.StatusCreated {
|
||||
said, _ := io.ReadAll(io.LimitReader(done.Body, 4096))
|
||||
return fmt.Errorf("the artifact store refused a blob in %s: %s %s", repository, done.Status, strings.TrimSpace(string(said)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// has is whether the store answers 200 for a HEAD at that URL.
|
||||
func (s Store) has(ctx context.Context, url string, accept ...string) (bool, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, media := range accept {
|
||||
request.Header.Add("Accept", media)
|
||||
}
|
||||
response, err := s.client().Do(request)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
switch response.StatusCode {
|
||||
case http.StatusOK:
|
||||
return true, nil
|
||||
case http.StatusNotFound:
|
||||
return false, nil
|
||||
default:
|
||||
return false, fmt.Errorf("the artifact store answered %s for %s", response.Status, url)
|
||||
}
|
||||
}
|
||||
|
||||
func (s Store) url(repository, kind, digest string) string {
|
||||
return "http://" + s.Address + "/v2/" + repository + "/" + kind + "/" + digest
|
||||
}
|
||||
|
||||
func (s Store) client() *http.Client {
|
||||
if s.HTTP != nil {
|
||||
return s.HTTP
|
||||
}
|
||||
return &http.Client{Timeout: 30 * time.Second}
|
||||
}
|
||||
@@ -1,268 +0,0 @@
|
||||
package registry
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// Every kept archive is held by a manifest (novox/hq issue 253, ADR 0189).
|
||||
//
|
||||
// Against an in-memory registry that keeps blobs and manifests per repository and refuses what a
|
||||
// registry refuses — a blob whose digest does not match, a manifest whose digest does not match
|
||||
// or whose blobs the repository does not have, a manifest asked for without an Accept naming its
|
||||
// type. What is asserted is this side's decisions; the live test below asserts the registry's.
|
||||
|
||||
type memRegistry struct {
|
||||
mu sync.Mutex
|
||||
blobs map[string][]byte // repository + "@" + digest
|
||||
manifests map[string][]byte // repository + "@" + digest
|
||||
writes []string // every PUT and DELETE, as "METHOD path"
|
||||
}
|
||||
|
||||
func digestOf(body []byte) string {
|
||||
sum := sha256.Sum256(body)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func (m *memRegistry) serve(t *testing.T) Store {
|
||||
t.Helper()
|
||||
m.blobs = map[string][]byte{}
|
||||
m.manifests = map[string][]byte{}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
path := strings.TrimPrefix(r.URL.Path, "/v2/")
|
||||
if r.Method == http.MethodPut || r.Method == http.MethodDelete {
|
||||
m.writes = append(m.writes, r.Method+" "+r.URL.Path)
|
||||
}
|
||||
switch {
|
||||
case r.Method == http.MethodPost && strings.HasSuffix(path, "/blobs/uploads/"):
|
||||
repository := strings.TrimSuffix(path, "/blobs/uploads/")
|
||||
w.Header().Set("Location", "/upload/"+repository+"?state=x")
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
case r.Method == http.MethodPut && strings.HasPrefix(r.URL.Path, "/upload/"):
|
||||
repository := strings.TrimPrefix(r.URL.Path, "/upload/")
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
digest := r.URL.Query().Get("digest")
|
||||
if digest != digestOf(body) {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
m.blobs[repository+"@"+digest] = body
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
case strings.Contains(path, "/blobs/"):
|
||||
repository, digest, _ := strings.Cut(path, "/blobs/")
|
||||
key := repository + "@" + digest
|
||||
body, ok := m.blobs[key]
|
||||
if !ok {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
switch r.Method {
|
||||
case http.MethodHead:
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(body)))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case http.MethodDelete:
|
||||
delete(m.blobs, key)
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
default:
|
||||
w.WriteHeader(http.StatusMethodNotAllowed)
|
||||
}
|
||||
case strings.Contains(path, "/manifests/"):
|
||||
repository, digest, _ := strings.Cut(path, "/manifests/")
|
||||
key := repository + "@" + digest
|
||||
switch r.Method {
|
||||
case http.MethodHead:
|
||||
if _, ok := m.manifests[key]; !ok || !strings.Contains(r.Header.Get("Accept"), mediaManifest) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case http.MethodPut:
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
if digest != digestOf(body) {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
var named holderManifest
|
||||
if err := json.Unmarshal(body, &named); err != nil {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
for _, d := range append([]descriptor{named.Config}, named.Layers...) {
|
||||
if _, ok := m.blobs[repository+"@"+d.Digest]; !ok {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
fmt.Fprintf(w, "MANIFEST_BLOB_UNKNOWN %s", d.Digest)
|
||||
return
|
||||
}
|
||||
}
|
||||
m.manifests[key] = body
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
case http.MethodDelete:
|
||||
if _, ok := m.manifests[key]; !ok {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
delete(m.manifests, key)
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
}
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
return Store{Address: strings.TrimPrefix(server.URL, "http://")}
|
||||
}
|
||||
|
||||
// bare puts an archive in the store the way the builder did before holders: a blob, nothing more.
|
||||
func (m *memRegistry) bare(repository string, body []byte) string {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
digest := digestOf(body)
|
||||
m.blobs[repository+"@"+digest] = body
|
||||
return catalogue.ArtifactStoreScheme + repository + "/blobs/" + digest
|
||||
}
|
||||
|
||||
func TestTheHolderIsComposedFromTheDigestAndSizeAlone(t *testing.T) {
|
||||
// The sweep must arrive at the very manifest the builder wrote, with nothing recorded between
|
||||
// them. Same inputs, same bytes — and a different size is a different holder, so a holder can
|
||||
// never be mistaken for one of a different blob.
|
||||
digest := "sha256:" + strings.Repeat("a", 64)
|
||||
one, first := Holder(digest, 42)
|
||||
two, second := Holder(digest, 42)
|
||||
if !bytes.Equal(one, two) || first != second {
|
||||
t.Fatalf("the same archive composed two holders:\n%s\n%s", one, two)
|
||||
}
|
||||
if _, other := Holder(digest, 43); other == first {
|
||||
t.Fatal("a different size composed the same holder")
|
||||
}
|
||||
want := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json",` +
|
||||
`"config":{"mediaType":"application/vnd.oci.empty.v1+json",` +
|
||||
`"digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","size":2},` +
|
||||
`"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"` + digest + `","size":42}]}`
|
||||
if string(one) != want {
|
||||
t.Fatalf("the holder is\n%s\nwant\n%s", one, want)
|
||||
}
|
||||
if digestOf(emptyConfig) != emptyDigest {
|
||||
t.Fatalf("the empty config's digest is %s, not %s", digestOf(emptyConfig), emptyDigest)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHoldBackfillsABareArchiveAndIsIdempotent(t *testing.T) {
|
||||
// The archives published before this have no holder. Hold, run over every kept archive on
|
||||
// every sweep, writes one the first time and nothing after.
|
||||
m := &memRegistry{}
|
||||
store := m.serve(t)
|
||||
ctx := context.Background()
|
||||
body := []byte("a theme")
|
||||
reference := m.bare("shell/config", body)
|
||||
|
||||
if held, err := store.Held(ctx, reference); err != nil || held {
|
||||
t.Fatalf("a bare blob reads as held=%v (%v)", held, err)
|
||||
}
|
||||
wrote, err := store.Hold(ctx, reference)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !wrote {
|
||||
t.Fatal("holding a bare archive wrote nothing")
|
||||
}
|
||||
_, holder := Holder(digestOf(body), int64(len(body)))
|
||||
if _, ok := m.manifests["shell/config@"+holder]; !ok {
|
||||
t.Fatalf("the store holds manifests %v; want %s", m.manifests, holder)
|
||||
}
|
||||
if held, err := store.Held(ctx, reference); err != nil || !held {
|
||||
t.Fatalf("after holding, held=%v (%v)", held, err)
|
||||
}
|
||||
|
||||
writes := len(m.writes)
|
||||
wrote, err = store.Hold(ctx, reference)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if wrote || len(m.writes) != writes {
|
||||
t.Fatalf("holding again wrote %v", m.writes[writes:])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnImageNeedsNoHolderAndAMissingArchiveIsGone(t *testing.T) {
|
||||
m := &memRegistry{}
|
||||
store := m.serve(t)
|
||||
ctx := context.Background()
|
||||
|
||||
// An image is its own manifest: nothing is asked.
|
||||
wrote, err := store.Hold(ctx, catalogue.ArtifactStoreScheme+"web/app@sha256:"+strings.Repeat("b", 64))
|
||||
if err != nil || wrote || len(m.writes) != 0 {
|
||||
t.Fatalf("holding an image wrote=%v err=%v writes=%v", wrote, err, m.writes)
|
||||
}
|
||||
// An archive the store does not have is a fact to report, not something to invent a holder for.
|
||||
_, err = store.Hold(ctx, catalogue.ArtifactStoreScheme+"web/config/blobs/sha256:"+strings.Repeat("c", 64))
|
||||
if !errors.Is(err, Gone) {
|
||||
t.Fatalf("holding a missing archive answered %v, want Gone", err)
|
||||
}
|
||||
// And a reference that is not the mesh's is refused as such.
|
||||
if _, err := store.Hold(ctx, "docker.io/library/registry@sha256:abc"); !errors.Is(err, ErrNotOurs) {
|
||||
t.Fatalf("holding a vendor's image answered %v, want ErrNotOurs", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLettingGoOfAnArchiveDeletesItsHolderFirst(t *testing.T) {
|
||||
// A holder left behind would keep the bytes through every collection while the record said
|
||||
// collected; the link deleted first and the holder failing after would be that exactly.
|
||||
m := &memRegistry{}
|
||||
store := m.serve(t)
|
||||
ctx := context.Background()
|
||||
body := []byte("an old theme")
|
||||
reference := m.bare("shell/config", body)
|
||||
if _, err := store.Hold(ctx, reference); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.writes = nil
|
||||
|
||||
if err := store.LetGo(ctx, reference); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, holder := Holder(digestOf(body), int64(len(body)))
|
||||
want := []string{
|
||||
"DELETE /v2/shell/config/manifests/" + holder,
|
||||
"DELETE /v2/shell/config/blobs/" + digestOf(body),
|
||||
}
|
||||
if strings.Join(m.writes, "\n") != strings.Join(want, "\n") {
|
||||
t.Fatalf("the store was asked\n%s\nwant\n%s", strings.Join(m.writes, "\n"), strings.Join(want, "\n"))
|
||||
}
|
||||
if len(m.manifests) != 0 {
|
||||
t.Fatalf("a holder survived: %v", m.manifests)
|
||||
}
|
||||
// Asked again, the archive is already gone, which is the outcome wanted.
|
||||
if err := store.LetGo(ctx, reference); !errors.Is(err, Gone) {
|
||||
t.Fatalf("letting go twice answered %v, want Gone", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLettingGoOfAnUnheldArchiveStillDeletesIt(t *testing.T) {
|
||||
// An archive published before holders and let go of before any sweep held it: the holder's
|
||||
// delete answers 404, which is the outcome wanted, and the blob still goes.
|
||||
m := &memRegistry{}
|
||||
store := m.serve(t)
|
||||
reference := m.bare("shell/config", []byte("never held"))
|
||||
if err := store.LetGo(context.Background(), reference); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(m.blobs) != 0 {
|
||||
t.Fatalf("the blob survived: %v", m.blobs)
|
||||
}
|
||||
}
|
||||
@@ -1,176 +0,0 @@
|
||||
package registry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// An index is let go of with the platform manifests it names, except those a kept index of the same
|
||||
// repository names too (novox/hq ADR 0257). Each platform manifest is a manifest of the repository in
|
||||
// its own right, and the store's collector keeps every manifest a repository holds — so an index let go
|
||||
// of alone frees nothing of the images it names.
|
||||
|
||||
func digestN(n int) string { return fmt.Sprintf("sha256:%064x", n) }
|
||||
|
||||
// indexStore holds indexes and images in one repository, answers GETs with their documents, and records
|
||||
// every delete.
|
||||
type indexStore struct {
|
||||
mu sync.Mutex
|
||||
indexes map[string][]string // digest → the platform manifests it names
|
||||
images map[string]bool
|
||||
deleted []string
|
||||
}
|
||||
|
||||
func (s *indexStore) serve(t *testing.T) Store {
|
||||
t.Helper()
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
digest := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:]
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
if children, ok := s.indexes[digest]; ok {
|
||||
var named []string
|
||||
for _, c := range children {
|
||||
named = append(named, `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"`+c+`","size":1}`)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/vnd.oci.image.index.v1+json")
|
||||
_, _ = w.Write([]byte(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[` +
|
||||
strings.Join(named, ",") + `]}`))
|
||||
return
|
||||
}
|
||||
if s.images[digest] {
|
||||
w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
|
||||
_, _ = w.Write([]byte(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","layers":[]}`))
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
case http.MethodHead:
|
||||
if _, ok := s.indexes[digest]; ok || s.images[digest] {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
case http.MethodDelete:
|
||||
if _, ok := s.indexes[digest]; !ok && !s.images[digest] {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
s.deleted = append(s.deleted, digest)
|
||||
delete(s.indexes, digest)
|
||||
delete(s.images, digest)
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
default:
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
return Store{Address: strings.TrimPrefix(server.URL, "http://")}
|
||||
}
|
||||
|
||||
func TestAnIndexGoesWithItsPlatformsAndAKeptIndexKeepsItsOwn(t *testing.T) {
|
||||
// Two indexes of one image's repository: the old one names platforms 11 and 12, the kept one 12 and 13.
|
||||
s := &indexStore{
|
||||
indexes: map[string][]string{digestN(1): {digestN(11), digestN(12)}, digestN(2): {digestN(12), digestN(13)}},
|
||||
images: map[string]bool{digestN(11): true, digestN(12): true, digestN(13): true},
|
||||
}
|
||||
store := s.serve(t)
|
||||
repository := "upstream/docker.io/library/golang"
|
||||
old := catalogue.ArtifactStoreScheme + repository + "@" + digestN(1)
|
||||
kept := catalogue.ArtifactStoreScheme + repository + "@" + digestN(2)
|
||||
spare, err := store.SpareKeptIndexes(context.Background(), []string{kept}, []string{old})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store.Spare = spare
|
||||
|
||||
if err := store.LetGo(context.Background(), old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Its own platform first, the index last; the platform the kept index names is left in place.
|
||||
if !slices.Equal(s.deleted, []string{digestN(11), digestN(1)}) {
|
||||
t.Fatalf("deleted %v; want the old index's own platform, then the index", s.deleted)
|
||||
}
|
||||
if !s.images[digestN(12)] || !s.images[digestN(13)] {
|
||||
t.Fatal("a platform a kept index names was let go of")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithoutSparingAnIndexGoesAloneAndAnImageNamesNoPlatforms(t *testing.T) {
|
||||
s := &indexStore{indexes: map[string][]string{digestN(1): {digestN(11)}}, images: map[string]bool{digestN(11): true, digestN(5): true}}
|
||||
store := s.serve(t)
|
||||
// No Spare: as before ADR 0257, the index alone.
|
||||
if err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/base@"+digestN(1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Equal(s.deleted, []string{digestN(1)}) {
|
||||
t.Fatalf("without a Spare, deleted %v", s.deleted)
|
||||
}
|
||||
// An image is one delete, with a Spare or without.
|
||||
s.deleted = nil
|
||||
spare, err := store.SpareKeptIndexes(context.Background(), nil, []string{catalogue.ArtifactStoreScheme + "web/app@" + digestN(5)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store.Spare = spare
|
||||
if err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/app@"+digestN(5)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Equal(s.deleted, []string{digestN(5)}) {
|
||||
t.Fatalf("an image was let go of as %v", s.deleted)
|
||||
}
|
||||
// An index the store no longer holds is Gone, and nothing is deleted on its account.
|
||||
s.deleted = nil
|
||||
err = store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/base@"+digestN(9))
|
||||
if !errors.Is(err, Gone) || len(s.deleted) != 0 {
|
||||
t.Fatalf("an index the store does not hold: %v, deleted %v", err, s.deleted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASpareThatCannotBeReadKeepsTheIndex(t *testing.T) {
|
||||
s := &indexStore{indexes: map[string][]string{digestN(1): {digestN(11)}}, images: map[string]bool{digestN(11): true}}
|
||||
store := s.serve(t)
|
||||
store.Spare = func(context.Context, string) (map[string]bool, error) { return nil, fmt.Errorf("the store went away") }
|
||||
err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/base@"+digestN(1))
|
||||
if err == nil || !strings.Contains(err.Error(), "was kept") {
|
||||
t.Fatalf("a spare that could not be read: %v", err)
|
||||
}
|
||||
if len(s.deleted) != 0 {
|
||||
t.Fatalf("deleted %v without knowing what a kept index names", s.deleted)
|
||||
}
|
||||
}
|
||||
|
||||
// The spare list is read whole before the sweep: a kept index that cannot be read is an error before
|
||||
// anything is deleted, and a repository not read before keeps its index.
|
||||
func TestASpareListIsReadWholeBeforeAnythingGoes(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
store := Store{Address: strings.TrimPrefix(server.URL, "http://")}
|
||||
kept := catalogue.ArtifactStoreScheme + "web/base@" + digestN(2)
|
||||
old := catalogue.ArtifactStoreScheme + "web/base@" + digestN(1)
|
||||
if _, err := store.SpareKeptIndexes(context.Background(), []string{kept}, []string{old}); err == nil {
|
||||
t.Fatal("a kept index the store would not answer for was taken as naming nothing")
|
||||
}
|
||||
|
||||
s := &indexStore{indexes: map[string][]string{digestN(1): {digestN(11)}}, images: map[string]bool{digestN(11): true}}
|
||||
good := s.serve(t)
|
||||
spare, err := good.SpareKeptIndexes(context.Background(), nil, []string{catalogue.ArtifactStoreScheme + "other/repo@" + digestN(3)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
good.Spare = spare
|
||||
if err := good.LetGo(context.Background(), old); err == nil || len(s.deleted) != 0 {
|
||||
t.Fatalf("an index in a repository not read before the sweep was let go of: %v, deleted %v", err, s.deleted)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package registry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
// blobSize is how large the store says a blob is; Gone when it does not have it.
|
||||
func (s Store) blobSize(ctx context.Context, repository, digest string) (int64, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodHead, s.url(repository, "blobs", digest), nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
response, err := s.client().Do(request)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
switch response.StatusCode {
|
||||
case http.StatusOK:
|
||||
case http.StatusNotFound:
|
||||
return 0, Gone
|
||||
default:
|
||||
return 0, fmt.Errorf("the artifact store answered %s for %s/blobs/%s", response.Status, repository, digest)
|
||||
}
|
||||
// Read from the header rather than ContentLength: a HEAD's ContentLength is what the response
|
||||
// says it would have sent, which Go reports faithfully, but a proxy in between is free to drop
|
||||
// it, and the header is what the registry itself wrote.
|
||||
if length := response.Header.Get("Content-Length"); length != "" {
|
||||
if n, err := strconv.ParseInt(length, 10, 64); err == nil && n >= 0 {
|
||||
return n, nil
|
||||
}
|
||||
}
|
||||
if response.ContentLength >= 0 {
|
||||
return response.ContentLength, nil
|
||||
}
|
||||
return 0, fmt.Errorf("the artifact store holds %s/blobs/%s and will not say how large it is", repository, digest)
|
||||
}
|
||||
|
||||
// has is whether the store answers 200 for a HEAD at that URL.
|
||||
func (s Store) has(ctx context.Context, url string, accept ...string) (bool, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, media := range accept {
|
||||
request.Header.Add("Accept", media)
|
||||
}
|
||||
response, err := s.client().Do(request)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
switch response.StatusCode {
|
||||
case http.StatusOK:
|
||||
return true, nil
|
||||
case http.StatusNotFound:
|
||||
return false, nil
|
||||
default:
|
||||
return false, fmt.Errorf("the artifact store answered %s for %s", response.Status, url)
|
||||
}
|
||||
}
|
||||
|
||||
func (s Store) url(repository, kind, digest string) string {
|
||||
return "http://" + s.Address + "/v2/" + repository + "/" + kind + "/" + digest
|
||||
}
|
||||
|
||||
func (s Store) client() *http.Client {
|
||||
if s.HTTP != nil {
|
||||
return s.HTTP
|
||||
}
|
||||
return &http.Client{Timeout: 30 * time.Second}
|
||||
}
|
||||
@@ -1,13 +1,6 @@
|
||||
// Package registry speaks to the mesh's artifact store over its own door.
|
||||
//
|
||||
// Only what the mesh needs that nothing else does: letting go of something it put there
|
||||
// (novox/hq ADR 0189, issue 108), and holding every archive it keeps by a manifest so the store's
|
||||
// own collector does not take it (novox/hq issue 253). Pushing is the builder's, through the container runtime; reading
|
||||
// is every machine's, through its runtime. This is the one operation that belongs to the thing
|
||||
// holding the records, because it is the only one that is a decision rather than a transfer.
|
||||
//
|
||||
// Copied from mesh-controller internal/artifacts at e04846e9 (novox/hq ADR 0571 clause 2): the seat mesh-artifact-sweep lets
|
||||
// go of what it no longer keeps exactly as the controller's sweep did; the controller's copy goes with its lease.
|
||||
// Package registry reads the mesh's artifact store over its own door: whether it holds what a reference names, and
|
||||
// whether a manifest is an index. It writes nothing: what the artifact sweep holds and removes it asks the registry
|
||||
// seats the store's holder serves (CAT-AS1, novox/hq ADR 0617 §2).
|
||||
package registry
|
||||
|
||||
import (
|
||||
@@ -29,10 +22,6 @@ type Store struct {
|
||||
Address string
|
||||
// HTTP is the client used; nil is a client with a modest timeout.
|
||||
HTTP *http.Client
|
||||
// Spare says, for one repository, which manifests a kept index there names: the platform
|
||||
// manifests an index being let go of must leave in place (novox/hq ADR 0257). Nil spares every
|
||||
// platform manifest — an index is then let go of alone, as it was before.
|
||||
Spare func(ctx context.Context, repository string) (map[string]bool, error)
|
||||
}
|
||||
|
||||
// Gone is the answer when the store does not hold it: the outcome wanted, already true.
|
||||
@@ -46,86 +35,6 @@ var Gone = errors.New("the store does not hold it")
|
||||
// error" is how a cautious loop became one that did nothing while reporting the right number.
|
||||
var ErrNotOurs = errors.New("not a reference into the mesh's artifact store")
|
||||
|
||||
// LetGo asks the store to drop one artifact the mesh recorded making.
|
||||
//
|
||||
// Takes a reference as the mesh records it — `artifact-store://<module>/<artifact>@sha256:…` for
|
||||
// an image, `…/blobs/sha256:…` for an archive — because that is the identity every record uses,
|
||||
// and composes the address here at the moment of use.
|
||||
//
|
||||
// An archive is let go of in two deletes, its holder manifest and then the blob's link (novox/hq
|
||||
// issue 253); an image in one.
|
||||
//
|
||||
// Returns Gone when the store answers that it does not have it. That is not a failure: the sweep
|
||||
// wants the artifact absent, and it is. It is distinguished from success only so a caller can say
|
||||
// which of the two happened.
|
||||
func (s Store) LetGo(ctx context.Context, reference string) error {
|
||||
// **Strict, and deliberately** (novox/hq issue 226). Only a reference the mesh keeps in its
|
||||
// own vocabulary is addressed here. `Recorded` would read `docker.io/library/registry@sha256:…`
|
||||
// as the mesh's too — it cannot tell one registry host from another — so normalising belongs
|
||||
// where the provenance is known, which is the sweep reading its own build records, not here
|
||||
// where the only job is to refuse anything that is not plainly ours.
|
||||
path, kept := catalogue.InArtifactStore(reference)
|
||||
if !kept {
|
||||
// Nothing the mesh put in its own store. Refused rather than attempted: composing a
|
||||
// delete for a reference of unknown shape is how a sweep reaches something that is not
|
||||
// the mesh's. Distinguished from a store that refuses, so a sweep skips this and goes on.
|
||||
return fmt.Errorf("%w: %s", ErrNotOurs, reference)
|
||||
}
|
||||
if s.Address == "" {
|
||||
return fmt.Errorf("this mesh has no artifact store on its network to ask about %s", reference)
|
||||
}
|
||||
repository, kind, digest, err := split(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if kind == "blobs" {
|
||||
// **An archive's holder goes before the archive** (novox/hq issue 253): a manifest left
|
||||
// naming the blob would keep its bytes through every collection while the record said
|
||||
// collected. Gone here means the store has no such blob, so there is nothing to let go.
|
||||
if err := s.letGoOfHolder(ctx, repository, digest); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if kind == "manifests" && s.Spare != nil {
|
||||
// **An index's platform manifests go before the index** (novox/hq ADR 0257). Each was put
|
||||
// under its own digest, so each is a manifest of the repository in its own right, and the
|
||||
// store's collector keeps every manifest a repository holds: an index let go of alone frees
|
||||
// no byte of the images it names. Before, not after, so a sweep cut short leaves the index
|
||||
// to be asked about again, and with it the list of what is still to go.
|
||||
if err := s.letGoOfPlatforms(ctx, repository, digest, reference); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return s.remove(ctx, s.url(repository, kind, digest), reference)
|
||||
}
|
||||
|
||||
// letGoOfPlatforms deletes the manifests an index names, except those a kept index of the same
|
||||
// repository also names. A manifest that is not an index names none.
|
||||
func (s Store) letGoOfPlatforms(ctx context.Context, repository, digest, reference string) error {
|
||||
children, err := s.Platforms(ctx, repository, digest)
|
||||
if err != nil || len(children) == 0 {
|
||||
// Gone: there is no index to read, and the delete that follows says so.
|
||||
if errors.Is(err, Gone) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
spared, err := s.Spare(ctx, repository)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot say which platform manifests of %s a kept index names, so %s was kept: %w",
|
||||
repository, reference, err)
|
||||
}
|
||||
for _, child := range children {
|
||||
if spared[child] {
|
||||
continue
|
||||
}
|
||||
if err := s.remove(ctx, s.url(repository, "manifests", child), reference+" (its "+child+")"); err != nil && !errors.Is(err, Gone) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// anyManifest is every manifest the store may hold, indexes included, so it answers with the
|
||||
// document as it is.
|
||||
var anyManifest = []string{
|
||||
@@ -172,84 +81,6 @@ func (s Store) Platforms(ctx context.Context, repository, digest string) ([]stri
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// SpareKeptIndexes is a Spare for a sweep, read whole before it starts: for every repository an
|
||||
// image reference of `touching` is in, each kept manifest there and every platform manifest a kept
|
||||
// index there names. Any read that fails is an error, and the sweep lets nothing go. A repository not
|
||||
// read here is answered with an error, so an index in it is kept. A kept index the store no longer
|
||||
// holds names nothing.
|
||||
func (s Store) SpareKeptIndexes(ctx context.Context, kept, touching []string) (func(context.Context, string) (map[string]bool, error), error) {
|
||||
imageIn := func(reference string) (repository, digest string, ok bool) {
|
||||
path, ours := catalogue.InArtifactStore(reference)
|
||||
if !ours {
|
||||
return "", "", false
|
||||
}
|
||||
repository, kind, digest, err := split(path)
|
||||
if err != nil || kind != "manifests" {
|
||||
return "", "", false
|
||||
}
|
||||
return repository, digest, true
|
||||
}
|
||||
read := map[string]map[string]bool{}
|
||||
for _, reference := range touching {
|
||||
if repository, _, ok := imageIn(reference); ok {
|
||||
read[repository] = map[string]bool{}
|
||||
}
|
||||
}
|
||||
for _, reference := range kept {
|
||||
repository, digest, ok := imageIn(reference)
|
||||
spared, touched := read[repository]
|
||||
if !ok || !touched {
|
||||
continue
|
||||
}
|
||||
spared[digest] = true
|
||||
children, err := s.Platforms(ctx, repository, digest)
|
||||
if errors.Is(err, Gone) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading the kept %s@%s: %w", repository, digest, err)
|
||||
}
|
||||
for _, c := range children {
|
||||
spared[c] = true
|
||||
}
|
||||
}
|
||||
return func(_ context.Context, repository string) (map[string]bool, error) {
|
||||
spared, done := read[repository]
|
||||
if !done {
|
||||
return nil, fmt.Errorf("%s was not read before the sweep began", repository)
|
||||
}
|
||||
return spared, nil
|
||||
}, nil
|
||||
}
|
||||
|
||||
// remove asks the store to delete what is at url. Gone when it has no such thing.
|
||||
func (s Store) remove(ctx context.Context, url, what string) error {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodDelete, url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response, err := s.client().Do(request)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
switch response.StatusCode {
|
||||
case http.StatusAccepted, http.StatusOK, http.StatusNoContent:
|
||||
return nil
|
||||
case http.StatusNotFound:
|
||||
return Gone
|
||||
case http.StatusMethodNotAllowed:
|
||||
// The registry was started without deletion enabled. Said plainly, because the remedy is
|
||||
// a setting on the store's module and not anything about this artifact.
|
||||
return fmt.Errorf(
|
||||
"the artifact store refuses deletion: its server was started without it enabled "+
|
||||
"(REGISTRY_STORAGE_DELETE_ENABLED), so nothing can be collected until the store "+
|
||||
"module is applied again (novox/hq ADR 0189). Asking about %s", what)
|
||||
default:
|
||||
return fmt.Errorf("the artifact store answered %s for %s", response.Status, what)
|
||||
}
|
||||
}
|
||||
|
||||
// split reads a recorded path into the repository, which endpoint names the thing, and the digest.
|
||||
//
|
||||
// Two shapes, which are the two the mesh records: `<repository>@sha256:<hex>` is a manifest, and
|
||||
@@ -279,22 +110,6 @@ func (s Store) IsIndex(ctx context.Context, reference string) (bool, error) {
|
||||
return len(children) > 0, err
|
||||
}
|
||||
|
||||
// HoldsManifest is whether the store holds the manifest a recorded image reference names.
|
||||
func (s Store) HoldsManifest(ctx context.Context, reference string) (bool, error) {
|
||||
path, ours := catalogue.InArtifactStore(reference)
|
||||
if !ours {
|
||||
return false, fmt.Errorf("%w: %s", ErrNotOurs, reference)
|
||||
}
|
||||
repository, kind, digest, err := split(path)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if kind != "manifests" {
|
||||
return false, fmt.Errorf("%w: %s names a blob, not a manifest", ErrNotOurs, reference)
|
||||
}
|
||||
return s.has(ctx, s.url(repository, kind, digest), anyManifest...)
|
||||
}
|
||||
|
||||
// Holds is whether the store holds what a recorded reference names: the blob of an archive, the manifest of an image.
|
||||
// Gone when it does not; asks and changes nothing (ADR 0526 clause 3: a registration by hand is refused at bytes that
|
||||
// are not there).
|
||||
|
||||
@@ -11,124 +11,66 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// Asking the store to let go of what the mesh no longer keeps (novox/hq ADR 0189, issue 108).
|
||||
//
|
||||
// A fake store records what it was asked to delete, so what is asserted is the mesh's decision
|
||||
// and the shape of the request — not the registry's behaviour, which is the registry's to test.
|
||||
|
||||
func fakeStore(t *testing.T, answer int) (Store, *[]string) {
|
||||
t.Helper()
|
||||
// The store's door is read only (CAT-AS1: what the sweep holds and removes it asks the registry seats): Holds says
|
||||
// whether the store holds what a reference names, in either reference form, IsIndex whether it is an index; a reference
|
||||
// that is not the mesh's own is never asked. A fake store answers by path and records what it was asked.
|
||||
func TestTheDoorReadsWhatTheStoreHoldsAndWritesNothing(t *testing.T) {
|
||||
const index = `{"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[{"digest":"sha256:c1"}]}`
|
||||
const image = `{"mediaType":"application/vnd.oci.image.manifest.v1+json","layers":[]}`
|
||||
var asked []string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodHead && strings.Contains(r.URL.Path, "/blobs/") {
|
||||
// An archive's size, asked so its holder can be named (novox/hq issue 253). A store
|
||||
// that does not have the thing does not have its blob either.
|
||||
if answer == http.StatusNotFound {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
asked = append(asked, r.Method+" "+r.URL.Path)
|
||||
switch r.URL.Path {
|
||||
case "/v2/web/app/manifests/sha256:abc", "/v2/web/files/blobs/sha256:def":
|
||||
if r.Method == http.MethodGet {
|
||||
_, _ = w.Write([]byte(image))
|
||||
} else {
|
||||
w.Header().Set("Content-Length", "7")
|
||||
}
|
||||
w.Header().Set("Content-Length", "7")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
case "/v2/web/app/manifests/sha256:idx":
|
||||
if r.Method == http.MethodGet {
|
||||
_, _ = w.Write([]byte(index))
|
||||
}
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
if r.Method != http.MethodDelete {
|
||||
t.Errorf("the store was asked %s %s; collecting is a delete", r.Method, r.URL.Path)
|
||||
}
|
||||
asked = append(asked, r.URL.Path)
|
||||
w.WriteHeader(answer)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
return Store{Address: strings.TrimPrefix(server.URL, "http://")}, &asked
|
||||
}
|
||||
|
||||
func TestAnImageAndAnArchiveAreAskedForAtTheirOwnEndpoints(t *testing.T) {
|
||||
// The two shapes the mesh records: a manifest by digest, and a blob by digest. They are
|
||||
// different endpoints, and asking at the wrong one answers 404 — which this would then
|
||||
// record as collected, leaving the bytes on disk for ever while the record says otherwise.
|
||||
store, asked := fakeStore(t, http.StatusAccepted)
|
||||
defer srv.Close()
|
||||
store := Store{Address: strings.TrimPrefix(srv.URL, "http://")}
|
||||
ctx := context.Background()
|
||||
|
||||
image := catalogue.ArtifactStoreScheme + "web/app@sha256:abc123"
|
||||
archive := catalogue.ArtifactStoreScheme + "web/config/blobs/sha256:def456"
|
||||
if err := store.LetGo(ctx, image); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.LetGo(ctx, archive); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The archive's holder goes first, then the archive (novox/hq issue 253).
|
||||
_, holder := Holder("sha256:def456", 7)
|
||||
want := []string{
|
||||
"/v2/web/app/manifests/sha256:abc123",
|
||||
"/v2/web/config/manifests/" + holder,
|
||||
"/v2/web/config/blobs/sha256:def456",
|
||||
}
|
||||
if strings.Join(*asked, " ") != strings.Join(want, " ") {
|
||||
t.Fatalf("the store was asked %v; want %v", *asked, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStoreThatDoesNotHaveItAnswersGone(t *testing.T) {
|
||||
// The outcome wanted, already true. Told apart from success only so the sweep can say which
|
||||
// happened; both are recorded, because retrying for ever is the thing to avoid.
|
||||
store, _ := fakeStore(t, http.StatusNotFound)
|
||||
err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/app@sha256:abc123")
|
||||
if !errors.Is(err, Gone) {
|
||||
t.Fatalf("a store that does not hold it answered %v, want Gone", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStoreWithDeletionOffSaysSoAndNamesTheRemedy(t *testing.T) {
|
||||
// The registry answers 405 when it was started without deletion enabled. The remedy is a
|
||||
// setting on the store's module, and saying "405" would send somebody to the wrong place.
|
||||
store, _ := fakeStore(t, http.StatusMethodNotAllowed)
|
||||
err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/app@sha256:abc123")
|
||||
if err == nil {
|
||||
t.Fatal("a store that refuses deletion was read as success")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "REGISTRY_STORAGE_DELETE_ENABLED") {
|
||||
t.Fatalf("the refusal does not name the remedy: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReferenceThatIsNotTheMeshsOwnIsNeverAsked(t *testing.T) {
|
||||
// The whole safety of the sweep is that it names only what the mesh recorded putting there.
|
||||
// A reference of another shape — a vendor's image, a package version — is refused rather
|
||||
// than composed into a delete somewhere that is not the mesh's store.
|
||||
store, asked := fakeStore(t, http.StatusAccepted)
|
||||
for _, reference := range []string{
|
||||
"docker.io/library/registry@sha256:abc123",
|
||||
"registry@sha256:abc123",
|
||||
"1.4.2",
|
||||
for name, c := range map[string]struct {
|
||||
reference string
|
||||
holds error // nil, Gone or ErrNotOurs
|
||||
index bool
|
||||
}{
|
||||
"an image": {reference: catalogue.ArtifactStoreScheme + "web/app@sha256:abc"},
|
||||
"an image in the seat's form": {reference: catalogue.SeatReference("container-image-registry", "web", "app", "sha256:abc")},
|
||||
"an archive": {reference: catalogue.ArtifactStoreScheme + "web/files/blobs/sha256:def"},
|
||||
"an archive in the seat's form": {reference: catalogue.SeatReference("oci-artifact-registry", "web", "files", "sha256:def")},
|
||||
"an index": {reference: catalogue.ArtifactStoreScheme + "web/app@sha256:idx", index: true},
|
||||
"an image the store lacks": {reference: catalogue.ArtifactStoreScheme + "web/app@sha256:999", holds: Gone},
|
||||
"an archive the store lacks": {reference: catalogue.ArtifactStoreScheme + "web/files/blobs/sha256:999", holds: Gone},
|
||||
"another registry's image": {reference: "docker.io/library/alpine@sha256:abc", holds: ErrNotOurs},
|
||||
"a reference naming no digest": {reference: catalogue.ArtifactStoreScheme + "web/app:latest", holds: ErrNotOurs},
|
||||
} {
|
||||
if err := store.LetGo(context.Background(), reference); err == nil {
|
||||
t.Errorf("%s was asked about; it is not a reference into the mesh's store", reference)
|
||||
}
|
||||
}
|
||||
if len(*asked) != 0 {
|
||||
t.Fatalf("the store was asked about %v", *asked)
|
||||
}
|
||||
}
|
||||
|
||||
// A reference this sweep will not address says so as ErrNotOurs, which is a fact about the
|
||||
// record and not about the store (novox/hq issue 226).
|
||||
//
|
||||
// The sweep skips one and abandons itself for the other, so they cannot be the same error. The
|
||||
// first live run met a reference recorded with the store's old address, read the refusal as "the
|
||||
// store refuses everything", and collected none of the 1681 it had found.
|
||||
func TestAReferenceThisSweepWillNotAddressIsToldApartFromAStoreRefusing(t *testing.T) {
|
||||
store, asked := fakeStore(t, http.StatusAccepted)
|
||||
for _, reference := range []string{
|
||||
"docker.io/library/registry@sha256:abc123",
|
||||
"127.0.0.1:5100/mesh-tools/build@sha256:abc123",
|
||||
"1.4.2",
|
||||
} {
|
||||
err := store.LetGo(context.Background(), reference)
|
||||
if !errors.Is(err, ErrNotOurs) {
|
||||
t.Errorf("%s answered %v; a sweep must be able to skip it and go on", reference, err)
|
||||
}
|
||||
}
|
||||
if len(*asked) != 0 {
|
||||
t.Fatalf("the store was asked about %v", *asked)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
asked = nil
|
||||
if err := store.Holds(ctx, c.reference); !errors.Is(err, c.holds) && (err != nil || c.holds != nil) {
|
||||
t.Errorf("Holds = %v, want %v", err, c.holds)
|
||||
}
|
||||
if c.holds == nil {
|
||||
if got, err := store.IsIndex(ctx, c.reference); err != nil || got != c.index {
|
||||
t.Errorf("IsIndex = %v, %v, want %v", got, err, c.index)
|
||||
}
|
||||
}
|
||||
for _, a := range asked {
|
||||
if !strings.HasPrefix(a, "HEAD ") && !strings.HasPrefix(a, "GET ") {
|
||||
t.Errorf("the door asked %s: it writes nothing", a)
|
||||
}
|
||||
}
|
||||
if errors.Is(c.holds, ErrNotOurs) && len(asked) != 0 {
|
||||
t.Errorf("a reference that is not the mesh's own was asked: %v", asked)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,16 +19,25 @@
|
||||
],
|
||||
"invokes": [
|
||||
"seat:mesh-hand-acts.hand-act",
|
||||
"mesh-catalog.catalog_modules",
|
||||
"mesh-catalog.catalog_module"
|
||||
"seat:mesh-catalog.modules",
|
||||
"seat:mesh-catalog.module",
|
||||
"mesh-delivery-assigner.assigner_read_wants",
|
||||
"mesh-delivery-indexer.indexer_read_manifests",
|
||||
"seat:oci-artifact-registry.hold",
|
||||
"seat:oci-artifact-registry.release",
|
||||
"seat:oci-artifact-registry.remove",
|
||||
"seat:container-image-registry.hold",
|
||||
"seat:container-image-registry.release",
|
||||
"seat:container-image-registry.remove"
|
||||
],
|
||||
"reads": [
|
||||
"mesh-delivery-assigner.wants",
|
||||
"mesh-delivery-builder.builds",
|
||||
"mesh-delivery-builder.build-log",
|
||||
"mesh-delivery-indexer.manifests",
|
||||
"node-engine.have",
|
||||
"mesh-controller.lease"
|
||||
"node-engine.have"
|
||||
],
|
||||
"consumes": [
|
||||
"mesh-delivery-assigner.views.wants",
|
||||
"mesh-delivery-indexer.views.manifests"
|
||||
],
|
||||
"state": [
|
||||
"findings"
|
||||
@@ -37,7 +46,9 @@
|
||||
"artifact_store_repositories",
|
||||
"artifact_store_usage",
|
||||
"artifact_store_references",
|
||||
"artifact_store_collect"
|
||||
"artifact_store_collect",
|
||||
"artifact_store_first_views",
|
||||
"artifact_store_first_view_again"
|
||||
],
|
||||
"config": {
|
||||
"older-store-names": {
|
||||
|
||||
@@ -171,7 +171,7 @@ func TestTheSwitcherDoesNotActOnAnotherModel(t *testing.T) {
|
||||
go func() { sw.Run(context.Background()); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("the switcher ran on another model")
|
||||
}
|
||||
if r := sw.Report(); r.Running || r.Disabled == "" || len(f.calls) != 0 {
|
||||
|
||||
@@ -2,10 +2,10 @@ module asuszephyrusg14
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f
|
||||
|
||||
require (
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/klauspost/compress v1.20.1 // indirect
|
||||
github.com/nats-io/nats.go v1.54.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639 h1:Zt7CfWt2S2W/fVgWNyW2u2+xwz051LXvGxVTNIyfhdM=
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639/go.mod h1:0dD1XEJBJ0G5UYDQxSZI84nuujb6m3tOpJl4C8rQVvg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f h1:jDsfDMd48MXY9Zershc3eRxnhmx2sSpUfQkLq5jAIds=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f/go.mod h1:08OomDXdwVN8/87nyDtv5R3rH/6LoP8SFFXz30bTYCM=
|
||||
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
|
||||
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
|
||||
@@ -2,10 +2,10 @@ module avahi
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f
|
||||
|
||||
require (
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/klauspost/compress v1.20.1 // indirect
|
||||
github.com/nats-io/nats.go v1.54.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639 h1:Zt7CfWt2S2W/fVgWNyW2u2+xwz051LXvGxVTNIyfhdM=
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639/go.mod h1:0dD1XEJBJ0G5UYDQxSZI84nuujb6m3tOpJl4C8rQVvg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f h1:jDsfDMd48MXY9Zershc3eRxnhmx2sSpUfQkLq5jAIds=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f/go.mod h1:08OomDXdwVN8/87nyDtv5R3rH/6LoP8SFFXz30bTYCM=
|
||||
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
|
||||
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
|
||||
@@ -2,10 +2,10 @@ module blueman
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f
|
||||
|
||||
require (
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/klauspost/compress v1.20.1 // indirect
|
||||
github.com/nats-io/nats.go v1.54.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639 h1:Zt7CfWt2S2W/fVgWNyW2u2+xwz051LXvGxVTNIyfhdM=
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639/go.mod h1:0dD1XEJBJ0G5UYDQxSZI84nuujb6m3tOpJl4C8rQVvg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f h1:jDsfDMd48MXY9Zershc3eRxnhmx2sSpUfQkLq5jAIds=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f/go.mod h1:08OomDXdwVN8/87nyDtv5R3rH/6LoP8SFFXz30bTYCM=
|
||||
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
|
||||
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
|
||||
@@ -2,10 +2,10 @@ module bluetooth
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f
|
||||
|
||||
require (
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/klauspost/compress v1.20.1 // indirect
|
||||
github.com/nats-io/nats.go v1.54.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639 h1:Zt7CfWt2S2W/fVgWNyW2u2+xwz051LXvGxVTNIyfhdM=
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639/go.mod h1:0dD1XEJBJ0G5UYDQxSZI84nuujb6m3tOpJl4C8rQVvg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f h1:jDsfDMd48MXY9Zershc3eRxnhmx2sSpUfQkLq5jAIds=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f/go.mod h1:08OomDXdwVN8/87nyDtv5R3rH/6LoP8SFFXz30bTYCM=
|
||||
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
|
||||
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
|
||||
@@ -181,7 +181,7 @@ func standInCapture(release <-chan struct{}, fail error) func(ctx context.Contex
|
||||
// waitFor polls until the condition holds.
|
||||
func waitFor(t *testing.T, what string, cond func() bool) {
|
||||
t.Helper()
|
||||
for i := 0; i < 400; i++ {
|
||||
for i := 0; i < 3000; i++ {
|
||||
if cond() {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -181,7 +181,7 @@ func TestAFinishedGlassDocumentIsNotFiledAgainByItsExpiry(t *testing.T) {
|
||||
if _, err := r.call(t, "scanner.scan", docArgs(id, true)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(400 * time.Millisecond)
|
||||
waitFor(t, "the expired document to be closed and filed", func() bool { e, _ := os.ReadDir(folder); return len(e) >= 1 })
|
||||
if e, _ := os.ReadDir(folder); len(e) != 1 {
|
||||
t.Errorf("%d files, want 1", len(e))
|
||||
}
|
||||
|
||||
@@ -311,8 +311,13 @@ func TestASlowUploadAndPrintAnswersStartedNeverATimeout(t *testing.T) {
|
||||
}
|
||||
got, _ = r.call(t, "printer.print", map[string]any{"path": "/home/ops/.ssh/id_ed25519", "node": "g14"})
|
||||
id2, _ := got.(map[string]any)["ask"].(string)
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if _, err := r.call(t, "printer.print", map[string]any{"ask": id2}); err == nil || !strings.Contains(err.Error(), "refused by name") {
|
||||
t.Errorf("the failed upload's outcome: %v", err)
|
||||
var outcome error
|
||||
for end := time.Now().Add(30 * time.Second); time.Now().Before(end); time.Sleep(10 * time.Millisecond) {
|
||||
if _, outcome = r.call(t, "printer.print", map[string]any{"ask": id2}); outcome != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
if outcome == nil || !strings.Contains(outcome.Error(), "refused by name") {
|
||||
t.Errorf("the failed upload's outcome: %v", outcome)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,20 +3,21 @@ module brother
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639 // indirect
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/klauspost/compress v1.20.1 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.2 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
golang.org/x/time v0.16.0 // indirect
|
||||
)
|
||||
|
||||
+12
-10
@@ -1,15 +1,17 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639 h1:Zt7CfWt2S2W/fVgWNyW2u2+xwz051LXvGxVTNIyfhdM=
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639/go.mod h1:0dD1XEJBJ0G5UYDQxSZI84nuujb6m3tOpJl4C8rQVvg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f h1:jDsfDMd48MXY9Zershc3eRxnhmx2sSpUfQkLq5jAIds=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f/go.mod h1:08OomDXdwVN8/87nyDtv5R3rH/6LoP8SFFXz30bTYCM=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
|
||||
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc=
|
||||
github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
@@ -23,5 +25,5 @@ golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZ
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
|
||||
golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
|
||||
|
||||
@@ -120,8 +120,10 @@ nothing else of the home. A project is one entry of the projects directory and a
|
||||
The agent kept reaching for `ssh <machine> journalctl` while the service manager's `journal` verb existed. So:
|
||||
|
||||
- **The "instead of" table.** Every seat verb and module tool may say which shell commands it replaces
|
||||
(`replaces`, in the seat's definition or the module's manifest). The module asks the controller — `tools`,
|
||||
`modules`, `nodes`, `node` — at start, every ten minutes and on `claude_code_render`, keeps the answer in its
|
||||
(`replaces`, in the seat's definition or the module's manifest). The module reads the bus — the seats' verbs from the
|
||||
seats' contract, the placed modules' manifests from `mesh-delivery-indexer.modules` and `.manifests`, the
|
||||
machines and where each module runs from `mesh-delivery-assigner.placement` (issue 1126: the controller that
|
||||
answered this is not placed) — at start, every ten minutes and on `claude_code_render`, keeps it in its
|
||||
state (`mesh-tools.json`), and renders it into `CLAUDE.md` as one row per seat or module: `<node>/<seat>.` and
|
||||
each verb with the commands it replaces. Generated, never written by hand: a verb that gains `replaces` is in
|
||||
the next render. Ordered by what the guard here refused most, then the machines' seats, the modules, the
|
||||
|
||||
@@ -505,7 +505,7 @@ func TestTheHelperReadsOnlyARegularFile(t *testing.T) {
|
||||
if a.Err == "" || len(a.Data) != 0 {
|
||||
t.Fatalf("%s was read: %q", path, a.Data)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatalf("reading %s held the helper", path)
|
||||
}
|
||||
}
|
||||
@@ -567,7 +567,7 @@ func TestAHelperThatDoesNotAnswerFailsTheRequestInTime(t *testing.T) {
|
||||
if err == nil || !errors.Is(err, errHelperSlow) {
|
||||
t.Fatalf("a silent helper answered: %v", err)
|
||||
}
|
||||
if time.Since(start) > 2*time.Second {
|
||||
if time.Since(start) > 10*time.Second {
|
||||
t.Fatalf("the request waited %s", time.Since(start))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
@@ -260,21 +261,37 @@ func TestAHomeItemIsImportedAndTheStaleOnesAreNamed(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The manifest lists exactly the tools the bundle serves: a tool missing from it is never announced.
|
||||
// The manifest lists exactly the tools the bundle serves: a tool missing from it is never announced. It invokes the
|
||||
// hand-act seat's warranted and no verb of the controller, and the warrant is asked of that seat (ADR 0565 clause 11).
|
||||
func TestTheManifestListsEveryToolServed(t *testing.T) {
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Tools []string `json:"tools"`
|
||||
Tools []string `json:"tools"`
|
||||
Invokes []string `json:"invokes"`
|
||||
Reads []string `json:"reads"`
|
||||
Consumes []string `json:"consumes"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &m)
|
||||
// The hand-act seat's warranted and hand-act, and the owners' read tools of the three followed states; no verb of
|
||||
// the controller, and no read of the indexer's or the assigner's state (ADR 0615 §1): their views are consumed.
|
||||
if !slices.Equal(m.Invokes, []string{"seat:mesh-hand-acts.warranted", "seat:mesh-hand-acts.hand-act", ModulesTool, ManifestsTool, PlacementTool}) ||
|
||||
!slices.Equal(m.Reads, []string{"claude-licence-manager.bindings"}) ||
|
||||
!slices.Equal(m.Consumes, []string{"mesh-delivery-indexer.views.modules", "mesh-delivery-indexer.views.manifests", "mesh-delivery-assigner.views.placement"}) {
|
||||
t.Errorf("invokes = %v, reads = %v, consumes = %v", m.Invokes, m.Reads, m.Consumes)
|
||||
}
|
||||
if src, _ := os.ReadFile("proposals.go"); !strings.Contains(string(src), `"seat:mesh-hand-acts.warranted"`) ||
|
||||
strings.Contains(string(src), "seat:mesh-controller.warranted") {
|
||||
t.Error("recordWarrant does not ask the hand-act seat alone")
|
||||
}
|
||||
listed := map[string]bool{}
|
||||
for _, n := range m.Tools {
|
||||
listed[n] = true
|
||||
}
|
||||
p, _ := node(t, "laptop")
|
||||
meshViews = newMeshViews()
|
||||
served := tools(p, nil, NewServerView(p), memConfig{}, NewConfigView(p))
|
||||
for _, tool := range served {
|
||||
if !listed[tool.Name] {
|
||||
|
||||
@@ -25,7 +25,8 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
occ "git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient"
|
||||
oc "git.novox.be/novox/mesh-contracts/go/seats/operatorchannel"
|
||||
)
|
||||
|
||||
// ForgeProvision is the provision whose credential this is, and the file this module's manifest has it delivered to.
|
||||
@@ -55,16 +56,16 @@ func (f *ForgeHealth) key() string { return "claude-code.machine." + f.Node + ".
|
||||
|
||||
// Look is the condition to say after one placing of the credential, nil when nothing changed: raised on a failure, again
|
||||
// when its reason changes, cleared when it places again; on the first look a success clears what a run before raised.
|
||||
func (f *ForgeHealth) Look(err error) *asks.Condition {
|
||||
func (f *ForgeHealth) Look(err error) *oc.Condition {
|
||||
first := !f.looked
|
||||
f.looked = true
|
||||
subject := asks.Scope{Scope: "machine", ID: f.Node, Machine: f.Node}
|
||||
subject := oc.ConditionSubject{Scope: "machine", ID: f.Node, Machine: f.Node}
|
||||
if err == nil {
|
||||
if f.raised == "" && !first {
|
||||
return nil
|
||||
}
|
||||
f.raised = ""
|
||||
return &asks.Condition{Key: f.key(), Subject: subject, State: asks.StateCleared}
|
||||
return &oc.Condition{Key: f.key(), Subject: subject, State: occ.StateCleared}
|
||||
}
|
||||
why := err.Error()
|
||||
if why == f.raised {
|
||||
@@ -72,10 +73,10 @@ func (f *ForgeHealth) Look(err error) *asks.Condition {
|
||||
}
|
||||
f.raised = why
|
||||
needs := "read claude-code's journal on " + f.Node + " and the owner of its forge credential file"
|
||||
return &asks.Condition{Key: f.key(), Severity: asks.SeverityWarning, State: asks.StateRaised, Subject: subject,
|
||||
return &oc.Condition{Key: f.key(), Severity: occ.SeverityWarning, State: occ.StateRaised, Subject: subject,
|
||||
Headline: "Agents on " + f.Node + " cannot use the forge credential",
|
||||
Summary: why, Needs: needs,
|
||||
Explanation: asks.NeedsYou + needs + ". " + sentence(why) + " Until it is placed, an agent there cannot " +
|
||||
Explanation: occ.NeedsYou + needs + ". " + sentence(why) + " Until it is placed, an agent there cannot " +
|
||||
"clone or fetch the mesh's private repositories.",
|
||||
Resolved: "The forge's read credential is placed again on " + f.Node + "."}
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
occ "git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient"
|
||||
)
|
||||
|
||||
const agentToken = "0123456789abcdef0123456789abcdef01234567"
|
||||
@@ -316,7 +316,7 @@ func TestTheForgeCredentialIsTheRuntimesAndItsFailureIsAConditionOfTheModule(t *
|
||||
got := ""
|
||||
if c != nil {
|
||||
got = c.State
|
||||
if _, err := asks.EncodeCondition(*c); err != nil {
|
||||
if _, err := occ.EncodeCondition(*c); err != nil {
|
||||
t.Errorf("look %d: the condition is refused: %v", i, err)
|
||||
}
|
||||
if c.Key != "claude-code.machine.anchor.forge-credential" {
|
||||
|
||||
@@ -8,22 +8,26 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats"
|
||||
)
|
||||
|
||||
// meshForTheGuard is a mesh of two machines and the verbs that replace what an agent runs over ssh, as the
|
||||
// controller's records say them.
|
||||
// seats' contract and the modules' manifests say them.
|
||||
func meshForTheGuard() GuardData {
|
||||
tools := json.RawMessage(`{"seats":[
|
||||
{"seat":"mesh-controller","scope":"mesh","tools":[{"name":"node","description":"What one machine reported it can do.","replaces":["hostnamectl","uptime"]}]},
|
||||
{"seat":"node-service-manager","scope":"node","tools":[
|
||||
{"name":"status","description":"One unit as the service manager sees it now: its states.","replaces":["systemctl status","systemctl is-active"]},
|
||||
{"name":"restart","description":"Restart one unit.","replaces":["systemctl restart"]},
|
||||
{"name":"journal","description":"The last lines of one unit's journal.","replaces":["journalctl"]}]},
|
||||
{"seat":"node-hostname","scope":"node","tools":[
|
||||
{"name":"add","description":"Add one address and its names to the operator's lines of /etc/hosts.","replaces":["edit /etc/hosts","HOSTALIASES"]}]}]}`)
|
||||
modules := json.RawMessage(`[{"module":"docker","on":["anchor"],"replaces":{"docker_logs":["docker logs"],"docker_list":["docker ps"]}},
|
||||
{"module":"systemd","on":["anchor","laptop"]}]`)
|
||||
return GuardData{Node: "laptop", Replacements: ReplacementsOf(tools, modules), Machines: []Machine{
|
||||
defined := []seats.Seat{
|
||||
{Name: "mesh-placement", Scope: "mesh", Serves: []seats.Verb{
|
||||
{Name: "placement", Description: "Read placement: one node's key.", Replaces: []string{"hostnamectl", "uptime"}}}},
|
||||
{Name: "node-service-manager", Scope: "node", Serves: []seats.Verb{
|
||||
{Name: "status", Description: "One unit as the service manager sees it now: its states.", Replaces: []string{"systemctl status", "systemctl is-active"}},
|
||||
{Name: "restart", Description: "Restart one unit.", Replaces: []string{"systemctl restart"}},
|
||||
{Name: "journal", Description: "The last lines of one unit's journal.", Replaces: []string{"journalctl"}}}},
|
||||
{Name: "node-hostname", Scope: "node", Serves: []seats.Verb{
|
||||
{Name: "add", Description: "Add one address and its names to the operator's lines of /etc/hosts.", Replaces: []string{"edit /etc/hosts", "HOSTALIASES"}}}},
|
||||
}
|
||||
modules := []ModuleTools{{Module: "docker", On: []string{"anchor"}, Replaces: map[string][]string{"docker_logs": {"docker logs"}, "docker_list": {"docker ps"}}},
|
||||
{Module: "systemd", On: []string{"anchor", "laptop"}}}
|
||||
return GuardData{Node: "laptop", Replacements: ReplacementsOf(defined, modules), Machines: []Machine{
|
||||
{Name: "anchor", Domains: []string{"anchor" + InternalSuffix, "anchor.example"}, Addresses: []string{"192.0.2.10", "10.10.0.1"}},
|
||||
{Name: "laptop", Domains: []string{"laptop" + InternalSuffix}, Addresses: []string{"10.10.0.4"}},
|
||||
}}
|
||||
@@ -69,21 +73,21 @@ func bash(command string) HookInput {
|
||||
func TestSSHToAMeshMachineIsRefusedNamingTheTool(t *testing.T) {
|
||||
g := guard()
|
||||
for command, want := range map[string]string{
|
||||
"ssh anchor journalctl -u mesh-controller -n 50": "anchor/node-service-manager.journal",
|
||||
"ssh anchor journalctl -u mesh-config -n 50": "anchor/node-service-manager.journal",
|
||||
"ssh anchor 'sudo journalctl -fu sshd'": "anchor/node-service-manager.journal",
|
||||
"ssh -p 22 root@anchor.example systemctl status nats": "anchor/node-service-manager.status",
|
||||
"ssh anchor.internal systemctl restart nats": "anchor/node-service-manager.restart",
|
||||
"ssh -o StrictHostKeyChecking=no 10.10.0.1 docker ps -a": "anchor/docker.docker_list",
|
||||
"ssh anchor docker logs --tail 100 nats": "anchor/docker.docker_logs",
|
||||
"ssh laptop docker logs x": "anchor/docker.docker_logs",
|
||||
"ssh a uptime": "mesh-controller.node",
|
||||
"ssh a uptime": "mesh-placement.placement",
|
||||
"cd /tmp && ssh anchor journalctl": "anchor/node-service-manager.journal",
|
||||
"sudo -u root ssh anchor journalctl": "anchor/node-service-manager.journal",
|
||||
"timeout 10 ssh anchor journalctl": "anchor/node-service-manager.journal",
|
||||
"env LANG=C ssh anchor journalctl": "anchor/node-service-manager.journal",
|
||||
"bash -c 'ssh anchor journalctl -u x'": "anchor/node-service-manager.journal",
|
||||
`echo "$(ssh anchor journalctl -n 5)" | tail`: "anchor/node-service-manager.journal",
|
||||
"for n in anchor laptop; do ssh $n uptime; done; ssh anchor uptime": "mesh-controller.node",
|
||||
"for n in anchor laptop; do ssh $n uptime; done; ssh anchor uptime": "mesh-placement.placement",
|
||||
"ssh -J anchor 203.0.113.9 journalctl": "anchor/node-service-manager.journal",
|
||||
"ssh via-anchor journalctl": "anchor/node-service-manager.journal",
|
||||
"ssh nas.lan journalctl": "anchor/node-service-manager.journal",
|
||||
|
||||
@@ -1,43 +1,86 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
// askingAController answers the controller's verbs as the seat does: `tools` in-process, the rest as the
|
||||
// command's output and its JSON.
|
||||
func askingAController(t *testing.T) Ask {
|
||||
// aBus is the states claude-code reads, as a test holds them: by read, key and value.
|
||||
type aBus map[string]map[string][]byte
|
||||
|
||||
func (b aBus) Keys(read string) ([]string, error) {
|
||||
if b[read] == nil {
|
||||
return nil, errors.New("no such state")
|
||||
}
|
||||
var keys []string
|
||||
for k := range b[read] {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
func (b aBus) Get(read, key string) ([]byte, bool, error) {
|
||||
v, ok := b[read][key]
|
||||
return v, ok, nil
|
||||
}
|
||||
|
||||
// someSeats are two seats as the seats' contract says them: one held once for the mesh, one on every machine.
|
||||
func someSeats() []seats.Seat {
|
||||
return []seats.Seat{
|
||||
{Name: "mesh-placement", Scope: "mesh", Serves: []seats.Verb{
|
||||
{Name: "placement", Description: "Read placement: one node's key.", Replaces: []string{"hostnamectl", "uptime"}},
|
||||
{Name: "propose", Description: "Propose a change."}}},
|
||||
{Name: "node-service-manager", Scope: "node", Serves: []seats.Verb{
|
||||
{Name: "status", Description: "One unit as the service manager sees it now: its states, whether it starts at boot.", Replaces: []string{"systemctl status", "systemctl is-active"}},
|
||||
{Name: "journal", Description: "The last lines of one unit's journal.", Replaces: []string{"journalctl"}}}},
|
||||
}
|
||||
}
|
||||
|
||||
// aMesh is two machines, anchor (with a public domain) and laptop, docker and zsh placed on anchor, docker's manifest
|
||||
// saying what its tool replaces, as the seat mesh-placement and the indexer keep them on the bus.
|
||||
func aMesh(t *testing.T) aBus {
|
||||
t.Helper()
|
||||
wrap := func(output string, answer any) json.RawMessage {
|
||||
raw, _ := json.Marshal(map[string]any{"ok": true, "output": output, "answer": answer})
|
||||
return raw
|
||||
}
|
||||
return func(address string, args any) (json.RawMessage, error) {
|
||||
switch address {
|
||||
case "seat:mesh-controller.tools":
|
||||
return json.RawMessage(`{"seats":[
|
||||
{"seat":"mesh-controller","scope":"mesh","tools":[{"name":"node","description":"What one machine reported it can do, what it is assigned, and why.","replaces":["hostnamectl","uptime"]},{"name":"nodes","description":"Every machine."}]},
|
||||
{"seat":"node-service-manager","scope":"node","tools":[
|
||||
{"name":"status","description":"One unit as the service manager sees it now: its states, whether it starts at boot.","replaces":["systemctl status","systemctl is-active"]},
|
||||
{"name":"journal","description":"The last lines of one unit's journal.","replaces":["journalctl"]}]}]}`), nil
|
||||
case "seat:mesh-controller.modules":
|
||||
return wrap("[...]", []any{map[string]any{"module": "docker", "on": []string{"anchor"},
|
||||
"replaces": map[string][]string{"docker_logs": {"docker logs"}}}, map[string]any{"module": "zsh", "on": []string{"anchor"}}}), nil
|
||||
case "seat:mesh-controller.nodes":
|
||||
return wrap("", []any{map[string]any{"name": "anchor"}, map[string]any{"name": "laptop"}}), nil
|
||||
case "seat:mesh-controller.node":
|
||||
node, _ := args.(map[string]any)["node"].(string)
|
||||
if node == "anchor" {
|
||||
return wrap("anchor\n mode converged\n public domain Anchor.Example\n\n what it runs\n", nil), nil
|
||||
}
|
||||
return wrap("laptop\n mode converged\n", nil), nil
|
||||
b := aBus{PlacementRead: {}, ModulesRead: {}, ManifestsRead: {}}
|
||||
for _, p := range []delivery.Placement{
|
||||
{V: 1, Node: "anchor", Decided: delivery.NodeDecided{PublicDomain: "Anchor.Example"},
|
||||
Modules: []delivery.PlacedModule{{Module: "docker"}, {Module: "zsh"}}},
|
||||
{V: 1, Node: "laptop"},
|
||||
} {
|
||||
values, err := delivery.PlacementValues(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for k, v := range values {
|
||||
b[PlacementRead][k] = v
|
||||
}
|
||||
return nil, errors.New("no such verb")
|
||||
}
|
||||
for module, manifest := range map[string]string{
|
||||
"docker": `{"module":"docker","replaces":{"docker_logs":["docker logs"]}}`,
|
||||
"zsh": `{"module":"zsh"}`,
|
||||
} {
|
||||
digest := fmt.Sprintf("%x", sha256.Sum256([]byte(manifest)))
|
||||
mkey, _ := delivery.ManifestsKey(digest)
|
||||
raw, err := json.Marshal(delivery.Manifest{V: 1, Module: module, JSON: manifest})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b[ManifestsRead][mkey] = raw
|
||||
key, _ := delivery.ModulesKey(module)
|
||||
b[ModulesRead][key], _ = json.Marshal(delivery.Module{V: 1, None: "nothing is built of it", Manifest: digest, Repository: "novox/mesh-catalog", Commit: "c"})
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func resolving(name string) []string {
|
||||
@@ -45,10 +88,11 @@ func resolving(name string) []string {
|
||||
"laptop": {"127.0.0.1", "10.10.0.4"}}[name]
|
||||
}
|
||||
|
||||
// What the controller says of its tools and machines is read whole: the seats' verbs, the modules' own tools,
|
||||
// every machine with its domains and addresses — a loopback address never one of them.
|
||||
func TestTheMeshsToolsAreAskedOfTheController(t *testing.T) {
|
||||
m, err := AskMeshTools(askingAController(t), resolving)
|
||||
// Issue 1126 B13: what the mesh says of its tools and machines is read from the bus whole, never asked of the
|
||||
// controller: the seats' verbs, the placed modules' own tools, every machine with its domains and addresses — a
|
||||
// loopback address never one of them.
|
||||
func TestTheMeshsToolsAreReadFromTheBus(t *testing.T) {
|
||||
m, err := ReadMeshToolsFrom(aMesh(t), someSeats(), resolving)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -56,7 +100,7 @@ func TestTheMeshsToolsAreAskedOfTheController(t *testing.T) {
|
||||
for _, r := range m.Replacements {
|
||||
addresses = append(addresses, r.Address)
|
||||
}
|
||||
if got := strings.Join(addresses, ","); got != "mesh-controller.node,node-service-manager.status,node-service-manager.journal,docker.docker_logs" {
|
||||
if got := strings.Join(addresses, ","); got != "mesh-placement.placement,node-service-manager.status,node-service-manager.journal,docker.docker_logs" {
|
||||
t.Errorf("replacements: %s", got)
|
||||
}
|
||||
if len(m.Machines) != 2 || strings.Join(m.Machines[0].Domains, ",") != "anchor.internal,anchor.example" ||
|
||||
@@ -71,12 +115,12 @@ func TestTheMeshsToolsAreAskedOfTheController(t *testing.T) {
|
||||
// **The table is generated, not written** (novox/hq ADR 0245): what the records say a verb replaces is a row,
|
||||
// the machines' seats first, and nothing is rendered where the records say nothing.
|
||||
func TestTheInsteadOfTableIsGeneratedFromTheRecords(t *testing.T) {
|
||||
m, _ := AskMeshTools(askingAController(t), resolving)
|
||||
m, _ := ReadMeshToolsFrom(aMesh(t), someSeats(), resolving)
|
||||
table := InsteadOf(&m)
|
||||
for _, want := range []string{
|
||||
"| `<node>/node-service-manager.` | `status` — `systemctl status`, `systemctl is-active` · `journal` — `journalctl` |",
|
||||
"| `<node>/docker.` | `docker_logs` — `docker logs` |",
|
||||
"| `mesh-controller.` | `node` — `hostnamectl`, `uptime` |",
|
||||
"| `mesh-placement.` | `placement` — `hostnamectl`, `uptime` |",
|
||||
"Never `ssh` to a mesh machine",
|
||||
} {
|
||||
if !strings.Contains(table, want) {
|
||||
@@ -84,14 +128,14 @@ func TestTheInsteadOfTableIsGeneratedFromTheRecords(t *testing.T) {
|
||||
}
|
||||
}
|
||||
if strings.Index(table, "node-service-manager.") > strings.Index(table, "docker.") ||
|
||||
strings.Index(table, "docker.") > strings.Index(table, "`mesh-controller.`") {
|
||||
strings.Index(table, "docker.") > strings.Index(table, "`mesh-placement.`") {
|
||||
t.Errorf("not the machines' seats, then modules, then the mesh's:\n%s", table)
|
||||
}
|
||||
// What the guard here named most comes first: its row, and in its row its verb.
|
||||
m.Refused = map[string]int{"node-service-manager.journal": 3, "mesh-controller.node": 1}
|
||||
m.Refused = map[string]int{"node-service-manager.journal": 3, "mesh-placement.placement": 1}
|
||||
ranked := InsteadOf(&m)
|
||||
if !strings.Contains(ranked, "| `<node>/node-service-manager.` | `journal` — `journalctl` · `status`") ||
|
||||
strings.Index(ranked, "`mesh-controller.`") > strings.Index(ranked, "`<node>/docker.`") {
|
||||
strings.Index(ranked, "`mesh-placement.`") > strings.Index(ranked, "`<node>/docker.`") {
|
||||
t.Errorf("not ranked by the guard's record:\n%s", ranked)
|
||||
}
|
||||
m.Refused = nil
|
||||
@@ -121,7 +165,7 @@ func TestTheInsteadOfTableIsGeneratedFromTheRecords(t *testing.T) {
|
||||
// Asked again with the same answer, nothing is written and nothing renders.
|
||||
func TestTheMeshsToolsAreKeptOnlyWhenTheyChange(t *testing.T) {
|
||||
p := Paths{State: t.TempDir()}
|
||||
m, _ := AskMeshTools(askingAController(t), resolving)
|
||||
m, _ := ReadMeshToolsFrom(aMesh(t), someSeats(), resolving)
|
||||
if !KeepMeshTools(p, m) {
|
||||
t.Fatal("the first answer was not kept")
|
||||
}
|
||||
@@ -138,18 +182,66 @@ func TestTheMeshsToolsAreKeptOnlyWhenTheyChange(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Where claude-code runs is read from the controller's `modules` answer as it comes — JSON, from `module list
|
||||
// --json` — and from the printed list too.
|
||||
func TestTheMachinesRunningTheModuleAreReadFromTheControllersAnswer(t *testing.T) {
|
||||
asJSON := json.RawMessage(`{"ok":true,"output":"[...]","answer":[{"module":"zsh","on":["a"]},{"module":"claude-code","on":["a","b"]}]}`)
|
||||
if got := strings.Join(NodesRunning(asJSON, "claude-code"), ","); got != "a,b" {
|
||||
t.Errorf("from JSON: %q", got)
|
||||
// Where a module runs is read from the seat mesh-placement's placement (issue 1126 B13), never the controller's answer.
|
||||
func TestTheMachinesRunningAModuleAreReadFromPlacement(t *testing.T) {
|
||||
b := aMesh(t)
|
||||
if got, err := NodesRunning(b, "zsh"); err != nil || strings.Join(got, ",") != "anchor" {
|
||||
t.Errorf("zsh runs on %v (%v)", got, err)
|
||||
}
|
||||
printed := json.RawMessage(`{"ok":true,"output":"zsh 1 built x on a\nclaude-code 1 built y on a, b\n"}`)
|
||||
if got := strings.Join(NodesRunning(printed, "claude-code"), ","); got != "a,b" {
|
||||
t.Errorf("from the printed list: %q", got)
|
||||
if got, err := NodesRunning(b, "claude-code"); err != nil || got != nil {
|
||||
t.Errorf("a module that runs nowhere: %v (%v)", got, err)
|
||||
}
|
||||
if got := NodesRunning(json.RawMessage(`{"ok":true,"answer":[{"module":"zsh","on":["a"]}]}`), "claude-code"); got != nil {
|
||||
t.Errorf("a module that runs nowhere: %v", got)
|
||||
if _, err := NodesRunning(aBus{}, "zsh"); err == nil {
|
||||
t.Error("an unreadable placement was read as no machine")
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0615 §1: the tools and the placement come from the owners' first views and their views events, never their state.
|
||||
// A reader whose first views are answered from the fixture reads what the state read gave, and a views event changes
|
||||
// what it reads (CAT-CC3).
|
||||
func TestTheMeshIsReadFromTheOwnersFirstViews(t *testing.T) {
|
||||
bus := aMesh(t)
|
||||
r := newMeshViews()
|
||||
for _, name := range []string{ModulesRead, ManifestsRead, PlacementRead} {
|
||||
state := strings.TrimPrefix(name, strings.Split(name, ".")[0]+".")
|
||||
values := map[string]json.RawMessage{}
|
||||
for k, v := range bus[name] {
|
||||
values[k] = v
|
||||
}
|
||||
f := r.Follow(strings.Split(name, ".")[0], state, "x")
|
||||
f.Ask = func(context.Context, string) (views.Answer, error) {
|
||||
return views.Answer{V: views.Version, Owner: f.Owner, State: state, Revision: 5, Created: time.Unix(1, 0), Values: values, At: time.Now()}, nil
|
||||
}
|
||||
if err := f.Start(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
want, err := ReadMeshToolsFrom(bus, seats.List(), resolving)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := ReadMeshToolsFrom(viewStates{r}, seats.List(), resolving)
|
||||
if err != nil || !reflect.DeepEqual(got, want) || len(got.Replacements) == 0 {
|
||||
t.Fatalf("from the views: %+v, %v\nfrom the state: %+v", got, err, want)
|
||||
}
|
||||
// A reader not yet answered says so, and reads nothing.
|
||||
if _, err := ReadMeshToolsFrom(viewStates{newMeshViews()}, seats.List(), resolving); err == nil || !strings.Contains(err.Error(), "no first view") {
|
||||
t.Fatalf("an unanswered first view: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1126 B13 AC1 (ADR 0590): over the seats the mesh defines, the rendered instructions, their table among them,
|
||||
// name no address of the controller, which is not placed: each job is named by the seat that owns it now.
|
||||
func TestTheRenderedInstructionsNameNoControllerAddress(t *testing.T) {
|
||||
m, err := ReadMeshToolsFrom(aMesh(t), seats.List(), resolving)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
md := RenderWithMesh(Facts{Node: "w", Console: "x"}, Settings{}, nil, "/h", nil, Config{}, &m)["CLAUDE.md"]
|
||||
if !strings.Contains(md, "## Instead of a shell command") {
|
||||
t.Fatalf("no table rendered:\n%s", md)
|
||||
}
|
||||
if strings.Contains(md, "mesh-controller") {
|
||||
t.Errorf("the instructions name the controller:\n%s", md)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,11 +2,12 @@ package main
|
||||
|
||||
// instructionsText is the managed instruction file, generated from the TypeScript renderer it replaced so
|
||||
// the file under the agent's managed directory did not change by a byte when the module moved to Go;
|
||||
// a test holds it to that renderer's own output (testdata/rendered-by-typescript.json).
|
||||
// a test holds it to that renderer's output (testdata/rendered-by-typescript.json), changed since only
|
||||
// where the controller's verbs were named (issue 1126 B13: the controller is not placed, ADR 0590).
|
||||
func instructionsText(node, role string) string {
|
||||
return "# This machine is a node of a Novox mesh\n\nWritten by the mesh's `claude-code` module. Edit the module's settings or the catalog, never this file:\nit is rewritten whenever the module renders.\n\n## Who this node is\n\n- **Node:** `" +
|
||||
node +
|
||||
"`\n- **Role:** " +
|
||||
role +
|
||||
"\n- The other nodes, their roles and what runs where: ask the controller (`mesh-controller.nodes`,\n `mesh-controller.node`). Nothing here lists them, because a copy drifts.\n\n## How a session on this mesh works\n\nThe console is the only way to the mesh: the MCP server named `mesh`. It offers five tools, and\neverything else is an address you find and call through them:\n\n- `mesh_search` — words in, matching addresses out. `mesh_describe` — one address's arguments.\n- `mesh_call` — call an address. A seat the mesh holds once is `<seat>.<verb>` (the mesh's own verbs\n are `mesh-controller.<verb>`: `status`, `plan`, `node`, `assign`, `push`, `settings`);\n a module on a machine is `<node>/<module>.<tool>`.\n- `mesh_overview` and `mesh_machine` — the mesh's seats and machines, and what one machine runs.\n\n- **Symptom first.** For an error, a failing service or anything unexpected, search the record with the\n literal text before forming a hypothesis: the records module's `records_search`, then\n `records_read`.\n- **Ask the mesh before changing it**, and change it through the controller's verbs or the catalog.\n- **A licence** through the `anthropic-licence-manager` seat's verbs. Never edit the agent's credentials\n file by hand, never print or ask for a token.\n\n## Hard rules\n\n- A file the mesh manages is changed through the verb or the catalog that owns it, never on disk. If\n unsure, `mesh-controller.plan` for the node says what the mesh writes there.\n- Never write to a store's database by hand; schema changes are numbered migrations.\n- Never push to a main branch: a branch, a pull request, and a human approval for every merge.\n- The mesh creates no symlinks, and nobody else does either.\n- A package is declared in a module, never installed by hand.\n\n## Conventions\n\n- Commit messages are concise, in the imperative, about why.\n- Test before pushing: nodes update unattended.\n- The playbooks in the record say how research, decisions, designs, issues and hand-offs are done.\n"
|
||||
"\n- The other nodes, their roles and what runs where: ask the placement seat (`mesh-placement.placement`),\n or `mesh_overview` and `mesh_machine`. Nothing here lists them, because a copy drifts.\n\n## How a session on this mesh works\n\nThe console is the only way to the mesh: the MCP server named `mesh`. It offers five tools, and\neverything else is an address you find and call through them:\n\n- `mesh_search` — words in, matching addresses out. `mesh_describe` — one address's arguments.\n- `mesh_call` — call an address. A seat the mesh holds once is `<seat>.<verb>` (the mesh's own seats\n include `mesh-placement`, `mesh-delivery`, `mesh-config` and `mesh-conditions`);\n a module on a machine is `<node>/<module>.<tool>`.\n- `mesh_overview` and `mesh_machine` — the mesh's seats and machines, and what one machine runs.\n\n- **Symptom first.** For an error, a failing service or anything unexpected, search the record with the\n literal text before forming a hypothesis: the records module's `records_search`, then\n `records_read`.\n- **Ask the mesh before changing it**, and change it through the seats' verbs or the catalog.\n- **A licence** through the `anthropic-licence-manager` seat's verbs. Never edit the agent's credentials\n file by hand, never print or ask for a token.\n\n## Hard rules\n\n- A file the mesh manages is changed through the verb or the catalog that owns it, never on disk. If\n unsure, `<node>/node-engine.composition` for the module says what the mesh writes there.\n- Never write to a store's database by hand; schema changes are numbered migrations.\n- Never push to a main branch: a branch, a pull request, and a human approval for every merge.\n- The mesh creates no symlinks, and nobody else does either.\n- A package is declared in a module, never installed by hand.\n\n## Conventions\n\n- Commit messages are concise, in the imperative, about why.\n- Test before pushing: nodes update unattended.\n- The playbooks in the record say how research, decisions, designs, issues and hand-offs are done.\n"
|
||||
}
|
||||
|
||||
@@ -71,6 +71,17 @@ type Judged struct {
|
||||
Pending bool `json:"pending,omitempty"`
|
||||
}
|
||||
|
||||
// JudgedEvent is the body of `judged`: one per judgement asked with a bound past JudgeBound, said when it ends, with
|
||||
// its answer or why there is none. The question and the schema are never in it.
|
||||
type JudgedEvent struct {
|
||||
Ticket string `json:"ticket"`
|
||||
Node string `json:"node"`
|
||||
Model string `json:"model,omitempty"`
|
||||
Seconds float64 `json:"seconds,omitempty"`
|
||||
Answer json.RawMessage `json:"answer,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// running is one judgement asked with a bound past JudgeBound, until it is collected.
|
||||
type running struct {
|
||||
done chan struct{}
|
||||
@@ -92,6 +103,9 @@ type Judge struct {
|
||||
// Wait is how long a call waits for a long judgement before it answers its ticket; JudgeWait unless a test
|
||||
// says less.
|
||||
Wait time.Duration
|
||||
// Say tells the bus a ticketed judgement ended: the event `judged`, once, under the message id it is given so a
|
||||
// repeat is dropped. Nil says nothing (a test's judge); the ticket is collected by claude_code_judged all the same.
|
||||
Say func(body JudgedEvent, id string) error
|
||||
turn chan struct{}
|
||||
|
||||
mu sync.Mutex
|
||||
@@ -208,10 +222,28 @@ func (j *Judge) start(schema, model, question string, bound time.Duration) (*Jud
|
||||
go func() {
|
||||
r.out, r.err = j.run(schema, model, question, bound)
|
||||
close(r.done)
|
||||
j.said(ticket, r)
|
||||
}()
|
||||
return j.Collect(ticket)
|
||||
}
|
||||
|
||||
// said tells the bus that the judgement under a ticket ended (the event `judged`), once. A failure to say is logged: the
|
||||
// ticket stays collectible by claude_code_judged, and the asker's own bound ends its wait.
|
||||
func (j *Judge) said(ticket string, r *running) {
|
||||
if j.Say == nil {
|
||||
return
|
||||
}
|
||||
e := JudgedEvent{Ticket: ticket, Node: j.P.Node}
|
||||
if r.err != nil {
|
||||
e.Error = r.err.Error()
|
||||
} else {
|
||||
e.Model, e.Seconds, e.Answer = r.out.Model, r.out.Seconds, r.out.Answer
|
||||
}
|
||||
if err := j.Say(e, "judged-"+j.P.Node+"-"+ticket); err != nil {
|
||||
say("the judgement under ticket %s ended and could not be said on the bus: %v", ticket, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Collect waits as long as a call may for the judgement under a ticket: its answer, why it failed, or that it is
|
||||
// still pending. A ticket is answered once; an answer nobody collects is kept JudgedKept past its bound.
|
||||
func (j *Judge) Collect(ticket string) (*Judged, error) {
|
||||
|
||||
@@ -133,7 +133,7 @@ func TestAJudgementWaitsItsTurnWithinTheBound(t *testing.T) {
|
||||
if err == nil || !strings.Contains(err.Error(), "another judgement") {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
if time.Since(start) > 2*time.Second {
|
||||
if time.Since(start) > 10*time.Second {
|
||||
t.Fatal("waited past the bound")
|
||||
}
|
||||
}
|
||||
@@ -175,13 +175,16 @@ func TestALongJudgementIsCollectedByItsTicket(t *testing.T) {
|
||||
j := testJudge(true, true, func(ctx context.Context, _ []string, _ []byte) ([]byte, error) {
|
||||
deadline, _ = ctx.Deadline()
|
||||
select {
|
||||
case <-time.After(400 * time.Millisecond):
|
||||
case <-time.After(1500 * time.Millisecond): // well past Wait, however loaded the machine
|
||||
return []byte(`{"structured_output":{"verdict":"none"}}`), nil
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
})
|
||||
j.Wait = 50 * time.Millisecond
|
||||
said := make(chan JudgedEvent, 4)
|
||||
ids := make(chan string, 4)
|
||||
j.Say = func(e JudgedEvent, id string) error { said <- e; ids <- id; return nil }
|
||||
start := time.Now()
|
||||
out, err := j.Judge(JudgeAsk{Question: "a long record", Schema: verdictSchema, Within: 90 * time.Second})
|
||||
if err != nil {
|
||||
@@ -191,7 +194,7 @@ func TestALongJudgementIsCollectedByItsTicket(t *testing.T) {
|
||||
t.Fatalf("not a ticket: %+v", out)
|
||||
}
|
||||
var got *Judged
|
||||
for i := 0; i < 100; i++ {
|
||||
for i := 0; i < 600; i++ {
|
||||
got, err = j.Collect(out.Ticket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -203,6 +206,21 @@ func TestALongJudgementIsCollectedByItsTicket(t *testing.T) {
|
||||
if got.Pending || string(got.Answer) != `{"verdict":"none"}` || got.Ticket != out.Ticket {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
// One `judged` per ticket, with the answer, under an id of the ticket's own; none for a judgement with no ticket.
|
||||
select {
|
||||
case e := <-said:
|
||||
if e.Ticket != out.Ticket || e.Node != "control" || string(e.Answer) != `{"verdict":"none"}` || e.Error != "" || e.Model != "sonnet" {
|
||||
t.Fatalf("judged: %+v", e)
|
||||
}
|
||||
if id := <-ids; id != "judged-control-"+out.Ticket {
|
||||
t.Fatalf("judged under %q", id)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the ended judgement was not said")
|
||||
}
|
||||
if len(said) != 0 {
|
||||
t.Fatal("judged said twice")
|
||||
}
|
||||
// Read once the judgement is collected: its end is what orders the agent's write before this read.
|
||||
if left := deadline.Sub(start); left < 80*time.Second || left > 91*time.Second {
|
||||
t.Fatalf("the judgement's bound is %s, not the 90s asked", left)
|
||||
@@ -223,12 +241,25 @@ func TestALongJudgementIsBoundedAndSaysWhyItFailed(t *testing.T) {
|
||||
t.Fatalf("a bound past the most taken: %v", err)
|
||||
}
|
||||
j.Wait = 20 * time.Millisecond
|
||||
j.Bound = 30 * time.Millisecond // the test's stand-in for JudgeBound: within is past it, so a ticket
|
||||
out, err := j.Judge(JudgeAsk{Question: "q", Schema: verdictSchema, Within: 200 * time.Millisecond})
|
||||
j.Bound = time.Second // the test's stand-in for JudgeBound: within is past it, so a ticket; far past Wait on a loaded machine
|
||||
failed := make(chan JudgedEvent, 1)
|
||||
j.Say = func(e JudgedEvent, _ string) error { failed <- e; return nil }
|
||||
out, err := j.Judge(JudgeAsk{Question: "q", Schema: verdictSchema, Within: 2 * time.Second})
|
||||
if err != nil || !out.Pending {
|
||||
t.Fatalf("%+v %v", out, err)
|
||||
}
|
||||
for i := 0; i < 100; i++ {
|
||||
defer func() {
|
||||
// A failed judgement is said too, with why and no answer.
|
||||
select {
|
||||
case e := <-failed:
|
||||
if e.Ticket != out.Ticket || e.Answer != nil || !strings.Contains(e.Error, "did not answer within") {
|
||||
t.Errorf("judged: %+v", e)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Error("the failed judgement was not said")
|
||||
}
|
||||
}()
|
||||
for i := 0; i < 3000; i++ {
|
||||
got, err := j.Collect(out.Ticket)
|
||||
if err != nil {
|
||||
if !strings.Contains(err.Error(), "did not answer within") {
|
||||
|
||||
@@ -12,6 +12,7 @@ package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -21,8 +22,13 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
occ "git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats"
|
||||
oc "git.novox.be/novox/mesh-contracts/go/seats/operatorchannel"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
func say(format string, args ...any) {
|
||||
@@ -179,12 +185,12 @@ func (s memoryStateOf) Delete(key string) error { return s.s.Delete(key) }
|
||||
var memoryCarrier *MemoryCarrier
|
||||
|
||||
// sayCondition says a condition on the operator-channel seat under this module's name (novox/hq ADR 0437 §9).
|
||||
func sayCondition(c asks.Condition) error {
|
||||
subject, err := asks.ConditionSubject("claude-code")
|
||||
func sayCondition(c oc.Condition) error {
|
||||
subject, err := occ.ConditionSubject("claude-code")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
raw, err := asks.EncodeCondition(c)
|
||||
raw, err := occ.EncodeCondition(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -204,55 +210,45 @@ func (s stateOf) Get(key string) (json.RawMessage, bool, error) {
|
||||
return e.Value, true, nil
|
||||
}
|
||||
|
||||
// nodesRunningMe is the nodes claude-code runs on, from the controller's list of modules — for the register
|
||||
// tool's question.
|
||||
func nodesRunningMe() ([]string, error) {
|
||||
raw, err := ask("seat:mesh-controller.modules", map[string]any{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return NodesRunning(raw, "claude-code"), nil
|
||||
// nodesRunningMe is the nodes claude-code runs on, from the assigner's placement — for the register tool's question.
|
||||
func nodesRunningMe() ([]string, error) { return NodesRunning(viewStates{meshViews}, "claude-code") }
|
||||
|
||||
// The owners' read tools of the states this module follows, which the manifest invokes (novox/hq ADR 0615 §1).
|
||||
const (
|
||||
ModulesTool = delivery.IndexerModule + ".indexer_read_modules"
|
||||
ManifestsTool = delivery.IndexerModule + ".indexer_read_manifests"
|
||||
PlacementTool = delivery.AssignerModule + ".assigner_read_placement"
|
||||
)
|
||||
|
||||
// meshViews is this module's picture of the indexer's modules and manifests and the assigner's placement: each asked
|
||||
// once of the owner's read tool, then followed on the owner's views family. It reads none of their state.
|
||||
var meshViews *views.Reader
|
||||
|
||||
// meshChanged is told, without waiting, that a followed state changed: the instructions' table is read again.
|
||||
var meshChanged = make(chan struct{}, 1)
|
||||
|
||||
// newMeshViews builds the reader of those three states; Start asks them.
|
||||
func newMeshViews() *views.Reader {
|
||||
v := &views.Reader{Module: Module, Logf: func(f string, a ...any) { say(f, a...) }, Findings: views.OwnFindings{},
|
||||
Acts: handacts.Recorder{Ask: stdio.Ask},
|
||||
Changed: func(string, string, json.RawMessage, bool) {
|
||||
select {
|
||||
case meshChanged <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}}
|
||||
v.Follow(delivery.IndexerModule, "modules", ModulesTool)
|
||||
v.Follow(delivery.IndexerModule, "manifests", ManifestsTool)
|
||||
v.Follow(delivery.AssignerModule, "placement", PlacementTool)
|
||||
return v
|
||||
}
|
||||
|
||||
// NodesRunning reads where a module runs from the controller's `modules` answer. The verb runs `module list
|
||||
// --json`, so the answer is a JSON list of {module, on}: the lines of the printed list it was once read as
|
||||
// never came, and every "also on" hint and every `nodes: "all"` named no machine. The printed form, `<module>
|
||||
// … on a, b`, is still read when that is what came.
|
||||
func NodesRunning(raw json.RawMessage, module string) []string {
|
||||
var listed []struct {
|
||||
Module string `json:"module"`
|
||||
On []string `json:"on"`
|
||||
}
|
||||
if json.Unmarshal(verbAnswer(raw), &listed) == nil {
|
||||
for _, m := range listed {
|
||||
if m.Module == module {
|
||||
return m.On
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
text := outputOf(raw)
|
||||
if text == "" {
|
||||
text = string(raw)
|
||||
}
|
||||
for _, line := range strings.Split(text, "\n") {
|
||||
if !strings.HasPrefix(line, module+" ") {
|
||||
continue
|
||||
}
|
||||
_, on, ok := strings.Cut(line, " on ")
|
||||
if !ok || strings.TrimSpace(on) == "nothing" {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, n := range strings.Split(on, ",") {
|
||||
if n = strings.TrimSpace(n); n != "" {
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
return nil
|
||||
}
|
||||
// viewStates reads the followed states from the reader's pictures, never from the owners' state.
|
||||
type viewStates struct{ v *views.Reader }
|
||||
|
||||
func (s viewStates) Keys(read string) ([]string, error) { return s.v.Keys(read) }
|
||||
|
||||
func (s viewStates) Get(read, key string) ([]byte, bool, error) { return s.v.Get(read, key) }
|
||||
|
||||
func fingerprintOfFile(path string) any {
|
||||
raw, err := os.ReadFile(path)
|
||||
@@ -320,11 +316,11 @@ func tools(p Paths, servers ServerState, view *ServerView, config ConfigState, c
|
||||
Description: "Claude Code on this machine as the mesh configured it: the licence it holds and when its token expires, what it reports holding, the managed files, the MCP servers registered here. Fingerprints only, never a token.",
|
||||
Run: func(map[string]any) (any, error) { return status(p), nil }},
|
||||
{Name: "claude_code_render",
|
||||
Description: "Write Claude Code's managed directory now, from the mesh's facts, this module's settings and the servers registered here — after asking the controller again what each tool replaces and which machines the mesh has (the instructions' table and the shell's guard).",
|
||||
Description: "Write Claude Code's managed directory now, from the mesh's facts, this module's settings and the servers registered here — after reading the bus again for what each tool replaces and which machines the mesh has (the instructions' table and the shell's guard).",
|
||||
Run: func(map[string]any) (any, error) {
|
||||
answer := map[string]any{}
|
||||
if m, err := AskMeshTools(ask, lookup); err != nil {
|
||||
answer["asked"] = "the controller did not answer in full (" + err.Error() + "); what was kept is rendered"
|
||||
if m, err := ReadMeshToolsFrom(viewStates{meshViews}, seats.List(), lookup); err != nil {
|
||||
answer["asked"] = "the bus could not be read in full (" + err.Error() + "); what was kept is rendered"
|
||||
} else {
|
||||
KeepMeshTools(p, m)
|
||||
}
|
||||
@@ -404,7 +400,12 @@ func tools(p Paths, servers ServerState, view *ServerView, config ConfigState, c
|
||||
out = append(out, proposalTools(p)...)
|
||||
out = append(out, waitTools(p)...)
|
||||
judge := NewJudge(p)
|
||||
// The event `judged`, one per ticket when its judgement ends: the asker hears it instead of asking again.
|
||||
judge.Say = func(e JudgedEvent, id string) error {
|
||||
return stdio.Emit("judged", e, stdio.EmitOptions{MessageID: id})
|
||||
}
|
||||
out = append(out, judgeTool(judge), judgedTool(judge))
|
||||
out = append(out, meshViews.Tools("claude_code")...)
|
||||
return append(out, configTools(p, config, configView)...)
|
||||
}
|
||||
|
||||
@@ -450,6 +451,7 @@ func main() {
|
||||
return
|
||||
}
|
||||
p, launched := PathsFrom(os.Getenv)
|
||||
meshViews = newMeshViews()
|
||||
if launched {
|
||||
fsys = FilesFor(p)
|
||||
}
|
||||
@@ -479,6 +481,7 @@ func main() {
|
||||
}
|
||||
}
|
||||
}
|
||||
meshViews.Start(context.Background())
|
||||
go refreshMeshTools(p)
|
||||
servers := stateOf{stdio.State("servers")}
|
||||
view := NewServerView(p)
|
||||
|
||||
@@ -950,11 +950,11 @@ func TestTheWatchHandlerReturnsAtOnceAndDrainAppliesWhatItNoted(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatalf("the watch handler waited for the look or the state (%d of 400 returned)", i)
|
||||
}
|
||||
}
|
||||
if took := time.Since(start); took > time.Second {
|
||||
if took := time.Since(start); took > 30*time.Second {
|
||||
t.Fatalf("400 values took %s to hand over", took)
|
||||
}
|
||||
shanks.mu.Unlock()
|
||||
|
||||
@@ -4,21 +4,27 @@ package main
|
||||
// shell need it (novox/hq ADR 0245): every seat verb and module tool that says what shell command it
|
||||
// replaces, and the mesh's machines by every name and address a session could reach one by.
|
||||
//
|
||||
// Asked of the controller — `tools` for the seats' verbs, `modules` for the modules' own tools, `nodes`
|
||||
// and `node` for the machines — at start and every few minutes, kept in the module's state so a render
|
||||
// never waits for the bus, and rendered whenever it changes. Nothing here is written by hand: a verb that
|
||||
// gains a `replaces` in the records is in the next render's table and the guard's next refusal.
|
||||
// Read from the bus at start and every few minutes (issue 1126 B13; the controller that answered this before is
|
||||
// not placed, novox/hq ADR 0590): the seats' verbs from the seats' contract (mesh-contracts), the modules'
|
||||
// own tools from the manifests the indexer keeps (`mesh-delivery-indexer.modules` and `.manifests`), and the
|
||||
// machines and where each module runs from the seat mesh-placement's `placement`. Kept in the module's state so
|
||||
// a render never waits for the bus, and rendered whenever it changes. Nothing here is written by hand: a verb
|
||||
// that gains a `replaces` is in the next render's table and the guard's next refusal.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Replacement is one verb or tool and the shell commands it is the mesh's way to do.
|
||||
@@ -41,11 +47,11 @@ type Machine struct {
|
||||
Addresses []string `json:"addresses,omitempty"`
|
||||
}
|
||||
|
||||
// MeshTools is what the controller said, as kept in the module's state.
|
||||
// MeshTools is what the bus said, as kept in the module's state.
|
||||
type MeshTools struct {
|
||||
Replacements []Replacement `json:"replacements"`
|
||||
Machines []Machine `json:"machines"`
|
||||
// Asked is when the controller last answered.
|
||||
// Asked is when the bus was last read whole.
|
||||
Asked string `json:"asked,omitempty"`
|
||||
// Refused is how often the guard here named each tool instead of a work-around, from its record: the
|
||||
// table's order. Never kept — read from the record at each render.
|
||||
@@ -78,42 +84,120 @@ func ReadMeshTools(p Paths) *MeshTools {
|
||||
return &m
|
||||
}
|
||||
|
||||
// verbAnswer is a controller verb's answer: the command's printed output, and its JSON when it printed one.
|
||||
func verbAnswer(raw json.RawMessage) json.RawMessage {
|
||||
var r struct {
|
||||
Output string `json:"output"`
|
||||
Answer json.RawMessage `json:"answer"`
|
||||
// The states read, each declared in the manifest's reads.
|
||||
const (
|
||||
PlacementRead = delivery.PlacementRead
|
||||
ModulesRead = delivery.IndexerModule + ".modules"
|
||||
ManifestsRead = delivery.IndexerModule + ".manifests"
|
||||
)
|
||||
|
||||
// States reads the module states the manifest declares in its reads: a state's keys, and one key's value.
|
||||
type States interface {
|
||||
Keys(read string) ([]string, error)
|
||||
Get(read, key string) ([]byte, bool, error)
|
||||
}
|
||||
|
||||
// ModuleTools is one module's own tools that replace a command, as its manifest says (`replaces`), and where it runs.
|
||||
type ModuleTools struct {
|
||||
Module string
|
||||
On []string
|
||||
Replaces map[string][]string
|
||||
}
|
||||
|
||||
// placements is every node's placement, from the seat mesh-placement's state.
|
||||
func placements(st States) (map[string]delivery.Placement, error) {
|
||||
keys, err := st.Keys(PlacementRead)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", PlacementRead, err)
|
||||
}
|
||||
if json.Unmarshal(raw, &r) != nil {
|
||||
return raw
|
||||
}
|
||||
if len(r.Answer) > 0 && string(r.Answer) != "null" {
|
||||
return r.Answer
|
||||
}
|
||||
if r.Output != "" {
|
||||
t := strings.TrimSpace(r.Output)
|
||||
for _, open := range []string{"[", "{"} {
|
||||
if strings.HasPrefix(t, open) {
|
||||
return json.RawMessage(t)
|
||||
}
|
||||
if i := strings.Index(t, "\n"+open); i >= 0 {
|
||||
return json.RawMessage(strings.TrimSpace(t[i+1:]))
|
||||
}
|
||||
read := delivery.NewPlacementKeys()
|
||||
for _, k := range keys {
|
||||
v, found, err := st.Get(PlacementRead, k)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s %s: %w", PlacementRead, k, err)
|
||||
}
|
||||
if found {
|
||||
_, _ = read.Put(k, v, 0, false) // a key that does not read places nothing
|
||||
}
|
||||
return json.RawMessage(nil)
|
||||
}
|
||||
return raw
|
||||
return read.All(), nil
|
||||
}
|
||||
|
||||
func outputOf(raw json.RawMessage) string {
|
||||
var r struct {
|
||||
Output string `json:"output"`
|
||||
// runsOn is, by module, the nodes placement puts it on, sorted.
|
||||
func runsOn(placed map[string]delivery.Placement) map[string][]string {
|
||||
out := map[string][]string{}
|
||||
for node, p := range placed {
|
||||
for _, m := range p.Modules {
|
||||
out[m.Module] = append(out[m.Module], node)
|
||||
}
|
||||
}
|
||||
_ = json.Unmarshal(raw, &r)
|
||||
return r.Output
|
||||
for _, nodes := range out {
|
||||
sort.Strings(nodes)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
var publicDomain = regexp.MustCompile(`(?m)^\s*public domain\s+(\S+)\s*$`)
|
||||
// NodesRunning is the nodes placement puts module on.
|
||||
func NodesRunning(st States, module string) ([]string, error) {
|
||||
placed, err := placements(st)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return runsOn(placed)[module], nil
|
||||
}
|
||||
|
||||
// modulesTools is every placed module's own tools that replace a command, by module, from the manifest the indexer
|
||||
// keeps of it. A module whose manifest cannot be read is left out and said in the error.
|
||||
func modulesTools(st States, on map[string][]string) ([]ModuleTools, error) {
|
||||
var out []ModuleTools
|
||||
var problems []error
|
||||
modules := make([]string, 0, len(on))
|
||||
for m := range on {
|
||||
modules = append(modules, m)
|
||||
}
|
||||
sort.Strings(modules)
|
||||
for _, module := range modules {
|
||||
key, err := delivery.ModulesKey(module)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
raw, found, err := st.Get(ModulesRead, key)
|
||||
if err != nil || !found {
|
||||
if err != nil {
|
||||
problems = append(problems, fmt.Errorf("%s %s: %w", ModulesRead, module, err))
|
||||
}
|
||||
continue
|
||||
}
|
||||
m, err := delivery.DecodeModule(raw)
|
||||
if err != nil || m.Manifest == "" {
|
||||
continue
|
||||
}
|
||||
mkey, err := delivery.ManifestsKey(m.Manifest)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
raw, found, err = st.Get(ManifestsRead, mkey)
|
||||
if err != nil || !found {
|
||||
if err != nil {
|
||||
problems = append(problems, fmt.Errorf("%s of %s: %w", ManifestsRead, module, err))
|
||||
}
|
||||
continue
|
||||
}
|
||||
kept, err := delivery.DecodeManifest(raw)
|
||||
if err != nil {
|
||||
problems = append(problems, fmt.Errorf("the manifest of %s: %w", module, err))
|
||||
continue
|
||||
}
|
||||
var man struct {
|
||||
Replaces map[string][]string `json:"replaces"`
|
||||
}
|
||||
if json.Unmarshal([]byte(kept.JSON), &man) != nil || len(man.Replaces) == 0 {
|
||||
continue
|
||||
}
|
||||
out = append(out, ModuleTools{Module: module, On: on[module], Replaces: man.Replaces})
|
||||
}
|
||||
return out, errors.Join(problems...)
|
||||
}
|
||||
|
||||
// firstSentence is what a verb does, short: its description to the first full stop, at most 70 characters.
|
||||
func firstSentence(s string) string {
|
||||
@@ -131,52 +215,33 @@ func firstSentence(s string) string {
|
||||
return s
|
||||
}
|
||||
|
||||
// ReplacementsOf reads the controller's `tools` and `modules` answers: the seats' verbs first, in the
|
||||
// records' order, then the modules' own tools by module and tool.
|
||||
func ReplacementsOf(tools, modules json.RawMessage) []Replacement {
|
||||
// ReplacementsOf is every seat verb and module tool that says it replaces a command: the seats' verbs first, in the
|
||||
// seats' order, then the modules' own tools by module and tool.
|
||||
func ReplacementsOf(defined []seats.Seat, modules []ModuleTools) []Replacement {
|
||||
var out []Replacement
|
||||
var seats struct {
|
||||
Seats []struct {
|
||||
Seat string `json:"seat"`
|
||||
Scope string `json:"scope"`
|
||||
Tools []struct {
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
Replaces []string `json:"replaces"`
|
||||
} `json:"tools"`
|
||||
} `json:"seats"`
|
||||
}
|
||||
if json.Unmarshal(tools, &seats) == nil {
|
||||
for _, s := range seats.Seats {
|
||||
for _, t := range s.Tools {
|
||||
if len(t.Replaces) == 0 {
|
||||
continue
|
||||
}
|
||||
out = append(out, Replacement{Address: s.Seat + "." + t.Name, NodeScoped: s.Scope == "node", Seat: true,
|
||||
Does: firstSentence(t.Description), Replaces: t.Replaces})
|
||||
for _, s := range defined {
|
||||
for _, v := range s.Serves {
|
||||
if len(v.Replaces) == 0 {
|
||||
continue
|
||||
}
|
||||
out = append(out, Replacement{Address: s.Name + "." + v.Name, NodeScoped: s.Scope == "node", Seat: true,
|
||||
Does: firstSentence(v.Description), Replaces: v.Replaces})
|
||||
}
|
||||
}
|
||||
var listed []struct {
|
||||
Module string `json:"module"`
|
||||
On []string `json:"on"`
|
||||
Replaces map[string][]string `json:"replaces"`
|
||||
}
|
||||
if json.Unmarshal(modules, &listed) == nil {
|
||||
sort.SliceStable(listed, func(i, j int) bool { return listed[i].Module < listed[j].Module })
|
||||
for _, m := range listed {
|
||||
names := make([]string, 0, len(m.Replaces))
|
||||
for t := range m.Replaces {
|
||||
names = append(names, t)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, t := range names {
|
||||
if len(m.Replaces[t]) == 0 {
|
||||
continue
|
||||
}
|
||||
out = append(out, Replacement{Address: m.Module + "." + t, NodeScoped: true, Replaces: m.Replaces[t], On: m.On,
|
||||
Does: strings.ReplaceAll(strings.TrimPrefix(t, strings.ReplaceAll(m.Module, "-", "_")+"_"), "_", " ")})
|
||||
sorted := append([]ModuleTools(nil), modules...)
|
||||
sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].Module < sorted[j].Module })
|
||||
for _, m := range sorted {
|
||||
names := make([]string, 0, len(m.Replaces))
|
||||
for t := range m.Replaces {
|
||||
names = append(names, t)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, t := range names {
|
||||
if len(m.Replaces[t]) == 0 {
|
||||
continue
|
||||
}
|
||||
out = append(out, Replacement{Address: m.Module + "." + t, NodeScoped: true, Replaces: m.Replaces[t], On: m.On,
|
||||
Does: strings.ReplaceAll(strings.TrimPrefix(t, strings.ReplaceAll(m.Module, "-", "_")+"_"), "_", " ")})
|
||||
}
|
||||
}
|
||||
return out
|
||||
@@ -190,36 +255,24 @@ func lookup(name string) []string {
|
||||
return addrs
|
||||
}
|
||||
|
||||
// AskMeshTools asks the controller now. A part that does not answer is left out and said in the error;
|
||||
// what did answer is kept.
|
||||
func AskMeshTools(ask Ask, resolve func(string) []string) (MeshTools, error) {
|
||||
// ReadMeshToolsFrom reads the bus now: the seats' verbs, the placed modules' own tools, and every node placement names
|
||||
// with its domains and addresses. A part that cannot be read is left out and said in the error; what was read is kept.
|
||||
func ReadMeshToolsFrom(st States, defined []seats.Seat, resolve func(string) []string) (MeshTools, error) {
|
||||
var m MeshTools
|
||||
tools, terr := ask("seat:mesh-controller.tools", map[string]any{})
|
||||
modules, merr := ask("seat:mesh-controller.modules", map[string]any{})
|
||||
if terr != nil && merr != nil {
|
||||
return m, terr
|
||||
}
|
||||
m.Replacements = ReplacementsOf(verbAnswer(tools), verbAnswer(modules))
|
||||
nodesRaw, err := ask("seat:mesh-controller.nodes", map[string]any{})
|
||||
placed, err := placements(st)
|
||||
if err != nil {
|
||||
m.Replacements = ReplacementsOf(defined, nil)
|
||||
return m, err
|
||||
}
|
||||
var nodes []struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
_ = json.Unmarshal(verbAnswer(nodesRaw), &nodes)
|
||||
for _, n := range nodes {
|
||||
if n.Name == "" {
|
||||
continue
|
||||
}
|
||||
machine := Machine{Name: n.Name, Domains: []string{n.Name + InternalSuffix}}
|
||||
if shown, err := ask("seat:mesh-controller.node", map[string]any{"node": n.Name}); err == nil {
|
||||
if d := publicDomain.FindStringSubmatch(outputOf(shown)); d != nil {
|
||||
machine.Domains = append(machine.Domains, strings.ToLower(d[1]))
|
||||
}
|
||||
modules, merr := modulesTools(st, runsOn(placed))
|
||||
m.Replacements = ReplacementsOf(defined, modules)
|
||||
for node, p := range placed {
|
||||
machine := Machine{Name: node, Domains: []string{node + InternalSuffix}}
|
||||
if d := strings.ToLower(strings.TrimSpace(p.Decided.PublicDomain)); d != "" {
|
||||
machine.Domains = append(machine.Domains, d)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, name := range append([]string{n.Name}, machine.Domains...) {
|
||||
for _, name := range append([]string{node}, machine.Domains...) {
|
||||
for _, a := range resolve(name) {
|
||||
if !seen[a] && !net.ParseIP(a).IsLoopback() {
|
||||
seen[a] = true
|
||||
@@ -231,7 +284,7 @@ func AskMeshTools(ask Ask, resolve func(string) []string) (MeshTools, error) {
|
||||
m.Machines = append(m.Machines, machine)
|
||||
}
|
||||
sort.Slice(m.Machines, func(i, j int) bool { return m.Machines[i].Name < m.Machines[j].Name })
|
||||
return m, nil
|
||||
return m, merr
|
||||
}
|
||||
|
||||
// KeepMeshTools writes what was asked to the module's state when it differs from what was kept, and
|
||||
@@ -281,24 +334,42 @@ func guardFiles(p Paths, m *MeshTools) *GuardFiles {
|
||||
return &GuardFiles{Exe: exe, Data: GuardDataOf(p, m), WaitsDir: WaitsDirPath(p)}
|
||||
}
|
||||
|
||||
// MeshToolsEvery is how often the controller is asked again.
|
||||
// MeshToolsEvery is how often the machines' addresses are resolved again; the tools and the placement are read again
|
||||
// when their followed states change (a views event), not on this timer.
|
||||
const MeshToolsEvery = 10 * time.Minute
|
||||
|
||||
// refreshMeshTools asks the controller at start and every few minutes, and renders when what it said changed.
|
||||
// MeshToolsFirst is how long the first read waits for the first views to be answered.
|
||||
const MeshToolsFirst = 2 * time.Minute
|
||||
|
||||
// refreshMeshTools reads the pictures once the first views are answered, and again whenever one changes (and the
|
||||
// machines' addresses every few minutes), and renders when what it said changed.
|
||||
func refreshMeshTools(p Paths) {
|
||||
select {
|
||||
case <-meshViews.Ready():
|
||||
case <-time.After(MeshToolsFirst):
|
||||
say("the first views of the tools and the placement were not answered in %s; reading what is held", MeshToolsFirst)
|
||||
}
|
||||
for {
|
||||
m, err := AskMeshTools(ask, lookup)
|
||||
m, err := ReadMeshToolsFrom(viewStates{meshViews}, seats.List(), lookup)
|
||||
if err != nil {
|
||||
say("asking the controller what each tool replaces: %v", err)
|
||||
say("reading what each tool replaces and which machines the mesh has: %v", err)
|
||||
}
|
||||
if (err == nil || len(m.Replacements) > 0) && KeepMeshTools(p, m) {
|
||||
if _, err := RenderNow(p, writeManaged); err != nil {
|
||||
say("rendering what the controller said of its tools: %v", err)
|
||||
say("rendering what the bus said of the tools: %v", err)
|
||||
} else {
|
||||
say("the mesh's tools changed: %d replace a command, %d machines; rendered", len(m.Replacements), len(m.Machines))
|
||||
}
|
||||
}
|
||||
time.Sleep(MeshToolsEvery)
|
||||
select {
|
||||
case <-meshChanged:
|
||||
time.Sleep(2 * time.Second) // a burst of events is one read
|
||||
select {
|
||||
case <-meshChanged:
|
||||
default:
|
||||
}
|
||||
case <-time.After(MeshToolsEvery):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -306,7 +377,7 @@ func refreshMeshTools(p Paths) {
|
||||
func GuardStatus(p Paths, last int) map[string]any {
|
||||
m := ReadMeshTools(p)
|
||||
d := GuardDataOf(p, m)
|
||||
asked := "never: the controller has not answered yet"
|
||||
asked := "never: the bus has not been read whole yet"
|
||||
if m != nil {
|
||||
asked = m.Asked
|
||||
}
|
||||
|
||||
@@ -9,8 +9,8 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
oc "git.novox.be/novox/mesh-contracts/go/seats/operatorchannel"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
)
|
||||
|
||||
// Proposing is what the proposal tools reach: the `proposals` state, the operator channel, the controller's
|
||||
@@ -71,9 +71,13 @@ func (d *Proposing) watch(p Paths) {
|
||||
}
|
||||
}
|
||||
go persist("hearing the operator's answers", func() error {
|
||||
return stdio.Subscribe(asks.Seat+".decided."+Module, func(e stdio.Envelope) error {
|
||||
var w asks.Warrant
|
||||
if json.Unmarshal(e.Body, &w) == nil {
|
||||
return stdio.Subscribe(oc.Seat+".decided."+Module, func(e stdio.Envelope) error {
|
||||
var w oc.Warrant
|
||||
if err := json.Unmarshal(e.Body, &w); err == nil {
|
||||
if _, err := warrantAt(w); err != nil {
|
||||
say("not taking the router's word on %s: %v", w.Ask, err)
|
||||
return nil
|
||||
}
|
||||
if strings.HasPrefix(w.Ask, QuestionPrefix) {
|
||||
HeardQuestionEnd(d.questions(), w)
|
||||
return nil
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user