Compare commits
42
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
237a534ca4 | ||
|
|
dd458cb155 | ||
|
|
6c09be4a70 | ||
|
|
448f1ef481 | ||
|
|
093967af9b | ||
|
|
be749c9513 | ||
|
|
4026392a6e | ||
|
|
d632071317 | ||
|
|
5fe7e3e6ba | ||
|
|
be24f5ac38 | ||
|
|
f2eb548204 | ||
|
|
e6de426303 | ||
|
|
5fab8f118e | ||
|
|
d5237091a6 | ||
|
|
1a35e5cf4d | ||
|
|
e2fad6a50a | ||
|
|
04ab9bc603 | ||
|
|
333ab9de22 | ||
|
|
e84b9d13c4 | ||
|
|
9c36d4edd4 | ||
|
|
4c137795a8 | ||
|
|
281f586621 | ||
|
|
843da56d09 | ||
|
|
2d34ff468b | ||
|
|
230230ce0e | ||
|
|
34ce2895dd | ||
|
|
0aa319154e | ||
|
|
90ff598207 | ||
|
|
1d1d5ec443 | ||
|
|
3112692def | ||
|
|
4915ff5d29 | ||
|
|
dd4c53eefb | ||
|
|
32dc74788e | ||
|
|
1fb32928ae | ||
|
|
0b2d393b11 | ||
|
|
134baf0e3b | ||
|
|
7c78fd2e0c | ||
|
|
9239177efd | ||
|
|
a1d98c2438 | ||
|
|
f469eb387a | ||
|
|
ee75d4327c | ||
|
|
45084af849 |
@@ -5,6 +5,8 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// The tree's manifests are ones a judge can read, and no module is named twice. A row is a catalogue of its own: a
|
||||
@@ -48,3 +50,45 @@ func TestEveryManifestDecodesAndNamesOneModule(t *testing.T) {
|
||||
t.Fatalf("the catalogue holds manifests no judge can read:\n%s", strings.Join(real, "\n"))
|
||||
}
|
||||
}
|
||||
|
||||
// No manifest of this catalogue claims the seat mesh-catalog: the indexer of the delivery holds it, and the module that
|
||||
// held it before has dropped its claim (ADR 0604 §6). A row is a tree of its own: a manifest claiming it is named.
|
||||
func TestNoManifestClaimsTheSeatMeshCatalog(t *testing.T) {
|
||||
claimants := func(root string) []string {
|
||||
files, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out []string
|
||||
for _, f := range files {
|
||||
raw, err := os.ReadFile(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.Decode(raw)
|
||||
if err != nil {
|
||||
continue // Problems names it
|
||||
}
|
||||
for _, c := range m.Claims {
|
||||
if c.Name == "mesh-catalog" {
|
||||
out = append(out, m.Module)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
row := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(row, "modules", "a"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(row, "modules", "a", "module.json"),
|
||||
[]byte(`{"module":"a","version":"1","claims":[{"name":"mesh-catalog","scope":"mesh"}]}`), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := claimants(row); len(got) != 1 || got[0] != "a" {
|
||||
t.Fatalf("a manifest claiming the seat is named, got %v", got)
|
||||
}
|
||||
if got := claimants(filepath.Join("..", "..")); len(got) != 0 {
|
||||
t.Errorf("the catalogue's manifests %v claim the seat mesh-catalog: the indexer of the delivery holds it", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -61,7 +61,7 @@ record names leaves its platform manifests unrecorded with it.
|
||||
**What the definitions pin** (hq issue 323). A definition may pin an image of the store the mesh did not
|
||||
make: pushed before the mesh built it, and pinned under the store's current name or a name it had before.
|
||||
No record of the controller names it, yet a machine runs it, so `artifact_store_references` reads every
|
||||
definition the catalogue holds (`mesh-catalog.catalog_modules`, then `catalog_module` for each, eight at
|
||||
definition the catalogue holds (the seat mesh-catalog's verbs `modules`, then `module` for each, eight at
|
||||
a time and within twenty seconds) and finds every `<name>/<repository>@sha256:…` in any of its strings,
|
||||
under each of the store's names: `artifact-store://`, the store's address as the controller answers it,
|
||||
and the names it had before. Those are the mesh's setting `older-store-names` (space-separated, `none` by
|
||||
|
||||
@@ -23,10 +23,11 @@ import (
|
||||
// manifest's.
|
||||
const NamesFileEnv = "MESH_ARTIFACT_STORE_NAMES_FILE"
|
||||
|
||||
// CatalogueModules and CatalogueModule are the catalog's tools these definitions are read through.
|
||||
// CatalogueModules and CatalogueModule are the verbs of the seat mesh-catalog these definitions are read through
|
||||
// (ADR 0604 §3); the answers are catalogue.CatalogModules and catalogue.CatalogOne.
|
||||
const (
|
||||
CatalogueModules = "mesh-catalog.catalog_modules"
|
||||
CatalogueModule = "mesh-catalog.catalog_module"
|
||||
CatalogueModules = "seat:mesh-catalog.modules"
|
||||
CatalogueModule = "seat:mesh-catalog.module"
|
||||
)
|
||||
|
||||
// DefinitionsTimeout bounds reading every definition, below the runtime's thirty-second call limit; what
|
||||
@@ -279,11 +280,8 @@ func (c *Catalogue) definition(m catalogueEntry, names StoreNames, nameKey strin
|
||||
return pins, ""
|
||||
}
|
||||
}
|
||||
args := map[string]any{"module": m.Module}
|
||||
if m.Commit != "" {
|
||||
args["commit"] = m.Commit
|
||||
}
|
||||
raw, err := c.Ask(CatalogueModule, args)
|
||||
// The verb takes no commit: it answers the trunk's module, and the answer names the commit it is at.
|
||||
raw, err := c.Ask(CatalogueModule, map[string]any{"module": m.Module})
|
||||
if err != nil {
|
||||
return nil, "could not be asked: " + err.Error()
|
||||
}
|
||||
@@ -293,6 +291,7 @@ func (c *Catalogue) definition(m catalogueEntry, names StoreNames, nameKey strin
|
||||
}
|
||||
var one struct {
|
||||
Module *struct {
|
||||
Commit string `json:"commit"`
|
||||
Manifest json.RawMessage `json:"manifest"`
|
||||
} `json:"module"`
|
||||
Error string `json:"error"`
|
||||
@@ -301,9 +300,11 @@ func (c *Catalogue) definition(m catalogueEntry, names StoreNames, nameKey strin
|
||||
return nil, "not readable: " + firstOf(one.Error, errText(err), "no manifest in the answer")
|
||||
}
|
||||
pins := PinsIn(one.Module.Manifest, names)
|
||||
if m.Commit != "" {
|
||||
// Kept by the commit the answer names, which may be newer than the list's: a definition and its commit never
|
||||
// change together.
|
||||
if at := firstOf(one.Module.Commit, m.Commit); at != "" {
|
||||
c.mu.Lock()
|
||||
c.kept[key] = pins
|
||||
c.kept[m.Module+"@"+at+"|"+nameKey] = pins
|
||||
c.mu.Unlock()
|
||||
}
|
||||
return pins, ""
|
||||
|
||||
@@ -211,7 +211,7 @@ func TestReferencesReadTheDefinitionsUnderTheStoresOlderNames(t *testing.T) {
|
||||
}
|
||||
}
|
||||
if before != 2 || after != 2 {
|
||||
t.Errorf("asked catalog_module %d times, then %d: each definition at its commit is read once", before, after)
|
||||
t.Errorf("asked the seat's module %d times, then %d: each definition at its commit is read once", before, after)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -366,8 +366,8 @@ func (a *asked) ask(key string, body any) (json.RawMessage, error) {
|
||||
if a.failing[name] {
|
||||
return nil, fmt.Errorf("no answer")
|
||||
}
|
||||
if args["commit"] != "c-"+name {
|
||||
return nil, fmt.Errorf("asked %s at %v", name, args["commit"])
|
||||
if _, has := args["commit"]; has {
|
||||
return nil, fmt.Errorf("asked %s at %v: the seat's verb takes no commit", name, args["commit"])
|
||||
}
|
||||
manifest, has := a.definitions[name]
|
||||
if !has {
|
||||
@@ -375,7 +375,8 @@ func (a *asked) ask(key string, body any) (json.RawMessage, error) {
|
||||
break
|
||||
}
|
||||
// The catalogue's tool answers through the runtime as the protocol's content list.
|
||||
raw, _ := json.Marshal(map[string]any{"module": map[string]any{"module": name, "manifest": json.RawMessage(manifest)}})
|
||||
raw, _ := json.Marshal(catalogue.CatalogOne{Module: catalogue.CatalogModule{
|
||||
CatalogEntry: catalogue.CatalogEntry{Module: name, Commit: "c-" + name}, Manifest: json.RawMessage(manifest)}})
|
||||
return json.Marshal(map[string]any{"content": []map[string]any{{"type": "text", "text": string(raw)}}})
|
||||
case "collect":
|
||||
if args["confirm"] == "true" {
|
||||
|
||||
@@ -3,7 +3,7 @@ module artifactstoretools
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -19,8 +19,8 @@
|
||||
],
|
||||
"invokes": [
|
||||
"seat:mesh-hand-acts.hand-act",
|
||||
"mesh-catalog.catalog_modules",
|
||||
"mesh-catalog.catalog_module"
|
||||
"seat:mesh-catalog.modules",
|
||||
"seat:mesh-catalog.module"
|
||||
],
|
||||
"reads": [
|
||||
"mesh-delivery-assigner.wants",
|
||||
|
||||
@@ -33,7 +33,7 @@ All six only read.
|
||||
| `disk_filesystems` | Each filesystem that stores data, said once however many places it is mounted: size, used and free GiB, used % as `df` counts it, inodes used % | `df -h`, `df -i` |
|
||||
| `disk_devices` | Each disk without sampling: model, size, whether it spins, scheduler, queue limit, what it is built on, and since boot the GiB read and written, the busy share and the wait per read and per write | `lsblk`, `cat /proc/diskstats` |
|
||||
| `disk_folders` | Which folders fill a disk, like `du -x`: the largest folders under `path` (default `/`) to `depth` levels (default 2, at most 3), `limit` of them (default 20, at most 100), with bytes and GiB. Sizes are allocated bytes, a hard link once. It stays on the path's one filesystem: a mount below it is named, never entered. It reads every folder the tool runner can read (docker's included) and no content; folders it cannot read are counted and named. After 25 s (the tool runner gives a call 30 s) it answers what it has measured, marked `partial`, with `stopped_at` (novox/hq issue 1804) | `du -x`, `ncdu` |
|
||||
| `cpu_top` | Samples a window (`seconds`, default 3, at most 10). It answers the machine's CPU in use (% of all CPUs and as CPUs busy, from `/proc/stat`), then `by=unit` (the default) ranks systemd units and containers by CPU, from their cgroups' cpu.stat, busiest first (`limit`, default 15), with the busy time no unit accounts for; `by=container` ranks containers by name | `top`, `systemd-cgtop` |
|
||||
| `cpu_top` | Samples a window (`seconds`, default 3, at most 10). It answers the machine's CPU in use (% of all CPUs and as CPUs busy, from `/proc/stat`), then `by=unit` (the default) ranks systemd units and containers by CPU, from their cgroups' cpu.stat, busiest first (`limit`, default 15), with the busy time no unit accounts for; `by=container` ranks containers by name With `unit` (a unit name such as `session-c1.scope`, or a cgroup path) it ranks that unit's processes instead (pid, command, cpu_pct, share_pct of the unit's busy time, threads), from `/proc/<pid>/stat` | `top`, `systemd-cgtop` |
|
||||
|
||||
### How each number is read
|
||||
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// clockTicks is USER_HZ, the unit of utime and stime in /proc/<pid>/stat: 100 on every Linux the mesh runs.
|
||||
const clockTicks = 100.0
|
||||
|
||||
// ProcessCPU is one process of a unit and its share of CPU over the window (cpu_top with unit set).
|
||||
type ProcessCPU struct {
|
||||
PID int `json:"pid"`
|
||||
Command string `json:"command"` // comm, then the first 120 characters of the command line
|
||||
CPUPct float64 `json:"cpu_pct"` // % of one CPU: 100 = one CPU busy the whole window
|
||||
SharePct float64 `json:"share_pct"` // this process's share of the unit's busy time
|
||||
Threads int `json:"threads"`
|
||||
raw float64
|
||||
}
|
||||
|
||||
// procTicks is a process's utime+stime ticks and thread count from its /proc/<pid>/stat, whose comm
|
||||
// field may hold spaces and parentheses, so the fields are counted from the last ")".
|
||||
func parseProcStat(s string) (comm string, ticks uint64, threads int, start string, ok bool) {
|
||||
l, r := strings.Index(s, "("), strings.LastIndex(s, ")")
|
||||
if l < 0 || r < l {
|
||||
return "", 0, 0, "", false
|
||||
}
|
||||
f := strings.Fields(s[r+1:])
|
||||
if len(f) < 20 {
|
||||
return "", 0, 0, "", false
|
||||
}
|
||||
ut, e1 := strconv.ParseUint(f[11], 10, 64)
|
||||
st, e2 := strconv.ParseUint(f[12], 10, 64)
|
||||
th, e3 := strconv.Atoi(f[17])
|
||||
if e1 != nil || e2 != nil || e3 != nil {
|
||||
return "", 0, 0, "", false
|
||||
}
|
||||
return s[l+1 : r], ut + st, th, f[19], true
|
||||
}
|
||||
|
||||
// cgroupOfUnit is the cgroup path (relative to /sys/fs/cgroup) a unit name or a path names.
|
||||
func (m *Machine) cgroupOfUnit(unit string) (string, error) {
|
||||
base := m.path("/sys/fs/cgroup")
|
||||
if strings.Contains(unit, "/") {
|
||||
rel := strings.Trim(strings.TrimPrefix(unit, "/sys/fs/cgroup"), "/")
|
||||
if strings.Contains(rel, "..") || filepath.Clean("/"+rel) != "/"+rel {
|
||||
return "", fmt.Errorf("cgroup %q is not a path under /sys/fs/cgroup", unit)
|
||||
}
|
||||
if rel == "" || !m.exists("/sys/fs/cgroup/"+rel) {
|
||||
return "", fmt.Errorf("no cgroup %q", unit)
|
||||
}
|
||||
return rel, nil
|
||||
}
|
||||
var found []string
|
||||
filepath.WalkDir(base, func(p string, d fs.DirEntry, err error) error {
|
||||
if err == nil && d.IsDir() && d.Name() == unit {
|
||||
rel, _ := filepath.Rel(base, p)
|
||||
found = append(found, filepath.ToSlash(rel))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
switch len(found) {
|
||||
case 0:
|
||||
return "", fmt.Errorf("no unit %q in the cgroup tree", unit)
|
||||
case 1:
|
||||
return found[0], nil
|
||||
}
|
||||
return "", fmt.Errorf("unit %q is %d cgroups (%s): give the cgroup path", unit, len(found), strings.Join(found, ", "))
|
||||
}
|
||||
|
||||
type procReading struct {
|
||||
ticks uint64
|
||||
comm string
|
||||
threads int
|
||||
pid int
|
||||
start string // starttime, to tell a reused pid
|
||||
}
|
||||
|
||||
// unitProcs reads every process in a cgroup and its child cgroups: its ticks, name and threads.
|
||||
func (m *Machine) unitProcs(cgroup string) map[int]procReading {
|
||||
out := map[int]procReading{}
|
||||
base := m.path("/sys/fs/cgroup/" + cgroup)
|
||||
filepath.WalkDir(base, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() || d.Name() != "cgroup.procs" {
|
||||
return nil
|
||||
}
|
||||
for _, f := range strings.Fields(m.read(strings.TrimPrefix(p, filepath.Clean(m.Root)))) {
|
||||
pid, err := strconv.Atoi(f)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
comm, ticks, th, start, ok := parseProcStat(m.read("/proc/" + f + "/stat"))
|
||||
if ok {
|
||||
out[pid] = procReading{ticks: ticks, comm: comm, threads: th, pid: pid, start: start}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// commandLine is a process's program path (argv[0], cut to 120 characters), never its arguments:
|
||||
// they can carry a password.
|
||||
func (m *Machine) commandLine(pid int) string {
|
||||
s, _, _ := strings.Cut(m.read("/proc/"+strconv.Itoa(pid)+"/cmdline"), "\x00")
|
||||
s, _, _ = strings.Cut(s, " ") // a process that rewrites its title puts its whole command line here
|
||||
s, _, _ = strings.Cut(s, "\t")
|
||||
s = strings.TrimSpace(s)
|
||||
if r := []rune(s); len(r) > 120 {
|
||||
s = string(r[:120])
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// topProcessCPU ranks a unit's processes by CPU between two readings; a process missing from either
|
||||
// reading (started or ended in the window) is skipped.
|
||||
func (m *Machine) topProcessCPU(a, b map[int]procReading, window time.Duration, limit int) []ProcessCPU {
|
||||
var out []ProcessCPU
|
||||
var sum float64
|
||||
for pid, end := range b {
|
||||
start, ok := a[pid]
|
||||
if !ok || end.start != start.start || end.ticks <= start.ticks {
|
||||
continue
|
||||
}
|
||||
pct := float64(end.ticks-start.ticks) / clockTicks / window.Seconds() * 100
|
||||
sum += pct
|
||||
out = append(out, ProcessCPU{PID: pid, Command: end.comm, Threads: end.threads, raw: pct})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].raw != out[j].raw {
|
||||
return out[i].raw > out[j].raw
|
||||
}
|
||||
return out[i].PID < out[j].PID
|
||||
})
|
||||
if len(out) > limit {
|
||||
out = out[:limit]
|
||||
}
|
||||
for i := range out {
|
||||
out[i].CPUPct = round1(out[i].raw)
|
||||
out[i].SharePct = round1(out[i].raw / sum * 100)
|
||||
if cl := m.commandLine(out[i].PID); cl != "" {
|
||||
out[i].Command += ": " + cl
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -180,3 +180,52 @@ func TestCPUTopKeepsCgroupsWithTheSameUnitNameAsSeparateRows(t *testing.T) {
|
||||
t.Errorf("the user session's init.scope: %+v", byCgroup)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCPUTopRanksAUnitsProcesses is the unit argument: a session scope with a child cgroup, ranked by
|
||||
// the utime+stime each process used over a 2 s window; one process ends, one starts, one comm holds ")".
|
||||
func TestCPUTopRanksAUnitsProcesses(t *testing.T) {
|
||||
f := aCPUHog(t)
|
||||
f.file("/sys/fs/cgroup/user.slice/session-c1.scope/cgroup.procs", "10\n11\n12\n")
|
||||
f.file("/sys/fs/cgroup/user.slice/session-c1.scope/app/cgroup.procs", "13\n")
|
||||
st := func(pid int, comm string, ticks, threads int) {
|
||||
f.then("/proc/"+itoa(pid)+"/stat", itoa(pid)+" ("+comm+") S 1 1 1 0 -1 0 0 0 0 0 "+itoa(ticks)+" 0 0 0 20 0 "+itoa(threads)+" 0 1 1 1\n")
|
||||
}
|
||||
for pid, c := range map[int][2]string{10: {"firefox", "1"}, 11: {"sh (x)", "1"}, 12: {"gone", "1"}} {
|
||||
f.file("/proc/"+itoa(pid)+"/stat", itoa(pid)+" ("+c[0]+") S 1 1 1 0 -1 0 0 0 0 0 0 0 0 0 20 0 "+c[1]+" 0 1 1 1\n")
|
||||
}
|
||||
f.file("/proc/13/stat", "13 (deep) S 1 1 1 0 -1 0 0 0 0 0 0 0 0 0 20 0 4 0 1 1 1\n")
|
||||
f.file("/proc/10/cmdline", "/usr/bin/firefox\x00--new-window\x00")
|
||||
f.file("/proc/11/cmdline", "mysql\x00--password\x00hunter2\x00")
|
||||
f.file("/proc/13/cmdline", "/usr/bin/tool --password hunter2 -x") // a rewritten title: one string, spaces, no NUL
|
||||
st(10, "firefox", 150, 40) // 75 % of one CPU over 2 s
|
||||
st(11, "sh (x)", 40, 1) // 20 %
|
||||
st(13, "deep", 10, 4) // 5 %, in the child cgroup
|
||||
f.then("/sys/fs/cgroup/user.slice/session-c1.scope/cgroup.procs", "10\n11\n14\n")
|
||||
f.file("/sys/fs/cgroup/user.slice/session-c1.scope/app/cgroup.procs", "13\n15\n")
|
||||
f.file("/proc/15/stat", "15 (old) S 1 1 1 0 -1 0 0 0 0 0 0 0 0 0 20 0 1 0 1 1 1\n")
|
||||
f.then("/proc/15/stat", "15 (reused) S 1 1 1 0 -1 0 0 0 0 0 900 0 0 0 20 0 1 0 77 1 1\n") // new starttime: another process
|
||||
st(14, "newborn", 500, 1) // not in the first reading: skipped
|
||||
|
||||
out, err := tool(t, f.machine(), "cpu_top").Run(map[string]any{"unit": "session-c1.scope", "seconds": 2.0, "limit": 3.0})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
top := out.(map[string]any)["top"].([]ProcessCPU)
|
||||
if len(top) != 3 || top[2].PID != 13 || top[2].CPUPct != 5 || top[0].PID != 10 || top[0].CPUPct != 75 || top[0].SharePct != 75 || top[0].Threads != 40 ||
|
||||
top[0].Command != "firefox: /usr/bin/firefox" || top[1].PID != 11 || top[1].CPUPct != 20 || top[1].Command != "sh (x): mysql" || top[2].Command != "deep: /usr/bin/tool" {
|
||||
t.Errorf("processes of the unit: %+v", top)
|
||||
}
|
||||
for _, p := range top {
|
||||
if p.PID == 15 || strings.Contains(p.Command, "hunter2") {
|
||||
t.Errorf("a reused pid was counted, or an argument leaked: %+v", p)
|
||||
}
|
||||
}
|
||||
for _, u := range []string{"../../etc", "/sys/fs/cgroup/../../proc", "user.slice/../.."} {
|
||||
if _, err := tool(t, f.machine(), "cpu_top").Run(map[string]any{"unit": u}); err == nil {
|
||||
t.Errorf("cgroup path %q escaping the root was accepted", u)
|
||||
}
|
||||
}
|
||||
if _, err := tool(t, f.machine(), "cpu_top").Run(map[string]any{"unit": "nothing.scope"}); err == nil {
|
||||
t.Error("an unknown unit was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,11 +90,14 @@ func Tools(m *Machine) []stdio.Tool {
|
||||
Description: "What used the most CPU over a few seconds, from each unit's cgroup cpu.stat, which needs no root: the machine's " +
|
||||
"CPU in use (as % of all CPUs and as CPUs busy, from /proc/stat), then by=unit (the default) each systemd unit and " +
|
||||
"container ranked by CPU, busiest first, with the busy time no unit accounts for; by=container only the containers, " +
|
||||
"by name. Replaces top, systemd-cgtop. (r)",
|
||||
"by name. With unit (a systemd unit such as session-c1.scope, or its cgroup path) it ranks that unit's processes instead, CPU by process, the top process: " +
|
||||
"pid, command, cpu_pct, share_pct of the unit's busy time and threads, from /proc/<pid>/stat, needing no root; a desktop session is one unit, " +
|
||||
"and this names the program in it. Replaces top, systemd-cgtop. (r)",
|
||||
Input: map[string]any{
|
||||
"seconds": map[string]any{"type": "integer", "description": "how long to sample, in seconds (default 3, at most 10)"},
|
||||
"limit": map[string]any{"type": "integer", "description": "how many units or containers to name (default 15, at most 50)"},
|
||||
"by": map[string]any{"type": "string", "enum": []string{"unit", "container"}, "description": "what to rank (default unit)"},
|
||||
"unit": map[string]any{"type": "string", "description": "a systemd unit name (session-c1.scope) or cgroup path: rank that unit's processes by CPU instead"},
|
||||
},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
n, err := count(args, "seconds", 3, 1, 10)
|
||||
@@ -118,6 +121,25 @@ func Tools(m *Machine) []stdio.Tool {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if unit := text(args, "unit"); unit != "" {
|
||||
cg, err := m.cgroupOfUnit(unit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pa := m.unitProcs(cg)
|
||||
m.Wait(w)
|
||||
bc, err := parseCPU(m.read("/proc/stat"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{
|
||||
"window_s": w.Seconds(),
|
||||
"by": "process",
|
||||
"cgroup": cg,
|
||||
"cpu": machineCPUOver(ac, bc),
|
||||
"top": m.topProcessCPU(pa, m.unitProcs(cg), w, limit),
|
||||
}, nil
|
||||
}
|
||||
a := m.unitCPU()
|
||||
m.Wait(w)
|
||||
bc, err := parseCPU(m.read("/proc/stat"))
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# mesh-catalog
|
||||
|
||||
The `mesh-catalog` module of the mesh catalogue: claims the seat `mesh-catalog`.
|
||||
The `mesh-catalog` module of the mesh catalogue: deprecated. It no longer claims the seat `mesh-catalog`: the indexer of the delivery claims it and serves its verbs from the bus (ADR 0604). This module stays, with its tools, until it is retired.
|
||||
|
||||
## Files it does not know
|
||||
|
||||
|
||||
@@ -5,12 +5,6 @@
|
||||
"capabilities": [
|
||||
"container-runtime"
|
||||
],
|
||||
"claims": [
|
||||
{
|
||||
"name": "mesh-catalog",
|
||||
"scope": "mesh"
|
||||
}
|
||||
],
|
||||
"requires": [
|
||||
"postgres-database"
|
||||
],
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sdk "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The holder, end to end over fakes (novox/hq ADR 0239 *How it is checked*, cut by ADR 0539 §17): a pull request's head
|
||||
@@ -238,6 +240,17 @@ func TestAMergedGroupEndsMergedWithSourceMainMoved(t *testing.T) {
|
||||
if w.source.main == nil || w.source.main.Heads["novox/a"] != strings.Repeat("p", 12) || w.source.main.Heads["novox/b"] != strings.Repeat("q", 12) {
|
||||
t.Fatalf("source.main is %+v, wanted both merges", w.source.main)
|
||||
}
|
||||
// A merge that leaves no module.json carries no trees (issue 1367): the gate reads it, and refuses nothing.
|
||||
msg := sdk.Forge{Kind: sdk.ForgeMerged, Repository: "novox/a", Number: 9, Head: strings.Repeat("e", 40),
|
||||
Merge: strings.Repeat("f", 40), Root: "r"}
|
||||
body, err := sdk.EncodeForge(msg)
|
||||
if err != nil {
|
||||
t.Fatalf("a merged message with no trees is refused: %v", err)
|
||||
}
|
||||
w.h.Forged("mesh-delivery-forge-bridge.novox.a.merged", body)
|
||||
if len(w.h.refusals) != 0 {
|
||||
t.Errorf("the gate refused a merged message with no trees: %v", w.h.refusals)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0419 §2, issue 978: a description's `after:` line is not read, and says so once on its pull request. The
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-gate
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103174208-b7abe203a929
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103174208-b7abe203a929 h1:RtAFbUJRP1qO9AIMM2ue+cXNzqj7WZ7IOlSqNvttwuU=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103174208-b7abe203a929/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -970,7 +970,11 @@ func (c *Client) report(ctx context.Context, kvs map[string]jetstream.KeyValue,
|
||||
logf("[provisioner-client] the report of %s: %v", r.Module, err)
|
||||
return
|
||||
}
|
||||
if _, err := kvs[AppliedBucket].Put(ctx, c.Node+"."+r.Module, body); err != nil {
|
||||
// Kept under a context the client's stop does not cancel, bounded: a report made as the client stops is still kept,
|
||||
// or s.applied would never name the module and a later pass would never remove it (leftModules).
|
||||
keep, done := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
||||
defer done()
|
||||
if _, err := kvs[AppliedBucket].Put(keep, c.Node+"."+r.Module, body); err != nil {
|
||||
logf("[provisioner-client] the report of %s could not be kept: %v", r.Module, err)
|
||||
} else {
|
||||
s.applied[r.Module] = true
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-provisioner-client
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
golang.org/x/sys v0.48.0
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
{
|
||||
"module": "mesh-provisioner-client",
|
||||
"version": "1",
|
||||
"version": "2",
|
||||
"claims": [
|
||||
{
|
||||
"name": "node-provisioner",
|
||||
"scope": "node"
|
||||
}
|
||||
],
|
||||
"core-credentials": true,
|
||||
"own-secrets": {
|
||||
"provisioner-broker": "${dir:state}/broker"
|
||||
},
|
||||
|
||||
@@ -18,6 +18,12 @@ package main
|
||||
// - **Retiring is asked** (clause 5): a key no longer needed is `retiring`, one approve ask under mesh-provisioner's
|
||||
// name says what goes from which provider on which node; the warrant goes into the key and the provider acts on it;
|
||||
// a decline is kept and not asked again while nothing changes; a key needed again returns to its outcome's state.
|
||||
// - **Retiring is judged from a whole picture only** (issue 940, met before as 937 and 1332): no pass judges a
|
||||
// retirement before every watched read has loaded once, nor while a module on the node has no composition published
|
||||
// at its build; a key whose consumer still has a want or a have on the node, and no plan read in this pass, is left
|
||||
// as it is; a pass that would retire most of a node's consumers at once retires nothing and says it as a fault of
|
||||
// the mesh (provisioner_health), never as a question; at most askOpenAtMost asks stand open at once, and an ask the
|
||||
// router does not hold is raised again.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
@@ -62,7 +68,10 @@ const (
|
||||
ModulesRead = delivery.IndexerModule + ".modules"
|
||||
ManifestsRead = delivery.IndexerModule + ".manifests"
|
||||
HaveRead = "node-engine.have"
|
||||
ComposedRead = "mesh-host.composed"
|
||||
// BuildsRead is the builder's builds, `<module>.<fingerprint part>`: the key of the build a provider runs names the
|
||||
// digest of the manifest it was built from (novox/hq ADR 0603 clause 5).
|
||||
BuildsRead = delivery.BuilderModule + ".builds"
|
||||
ComposedRead = "mesh-host.composed"
|
||||
)
|
||||
|
||||
// BoundReads are the providers' entries this module's manifest reads, `<module>.bound`, whose changes it hears. Which
|
||||
@@ -84,7 +93,10 @@ type Composer func(ctx context.Context, node string, keptAt map[string]string) C
|
||||
// Composition is a node's plans at its wanted builds and at its newest, and where the time went.
|
||||
type Composition struct {
|
||||
AtWants, Newest HeldPlans
|
||||
Took []Phase
|
||||
// Unpublished is each module whose composition at the build it is read at is not published yet: the node's picture
|
||||
// is not whole, so no retirement is judged on it (issue 940).
|
||||
Unpublished []string
|
||||
Took []Phase
|
||||
}
|
||||
|
||||
// HeldPlans is one scope's plans by module, the modules held with why, and a failure that is no single module's.
|
||||
@@ -114,8 +126,16 @@ type Holder struct {
|
||||
Asks Asker
|
||||
Now func() time.Time
|
||||
Logf func(format string, args ...any)
|
||||
// Loaded says which watched reads have not loaded once yet, "" when all have (nil: all have). A pass before then
|
||||
// judges nothing (issue 940: a first read that was not whole retired 21 routes of running apps).
|
||||
Loaded func() string
|
||||
|
||||
mu sync.Mutex
|
||||
// faults is, by node, the retirement a pass held back as a fault of the mesh (issue 940), said by provisioner_health.
|
||||
faults map[string]string
|
||||
// unpublished is, by node and module, since when the module is held because its composition at this build is not
|
||||
// published, with the hold's words: one held longer than unpublishedFor is said by provisioner_health.
|
||||
unpublished map[string]map[string]unpublishedHold
|
||||
}
|
||||
|
||||
// Composed is what one node's composition did.
|
||||
@@ -149,7 +169,13 @@ func TookSaid(took []Took) string {
|
||||
}
|
||||
|
||||
// ComposeNode composes one node: its plans and its provisions.
|
||||
func (h *Holder) ComposeNode(ctx context.Context, node string) (out Composed, err error) {
|
||||
func (h *Holder) ComposeNode(ctx context.Context, node string) (Composed, error) {
|
||||
return h.ComposeNodeWith(ctx, node, false)
|
||||
}
|
||||
|
||||
// ComposeNodeWith composes one node; retireMany lets this one pass retire more of the node's consumers at once than
|
||||
// the bound (manyAtOnce) allows, each still asked: the operator's word that a large removal is meant.
|
||||
func (h *Holder) ComposeNodeWith(ctx context.Context, node string, retireMany bool) (out Composed, err error) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
out = Composed{Node: node}
|
||||
@@ -163,13 +189,29 @@ func (h *Holder) ComposeNode(ctx context.Context, node string) (out Composed, er
|
||||
start = now
|
||||
}
|
||||
defer func() { mark("whole", time.Since(began)) }()
|
||||
if h.Loaded != nil {
|
||||
if missing := h.Loaded(); missing != "" {
|
||||
return out, fmt.Errorf("%s is not composed: %s has not loaded once yet, and a pass over a part judges nothing", node, missing)
|
||||
}
|
||||
}
|
||||
wanted, err := h.wantedOn(node)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
running, err := h.runningOn(node)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
// present is every module with a want or a have on the node: one of them with no plan read in this pass is not
|
||||
// unneeded (issue 940).
|
||||
present := map[string]bool{}
|
||||
for module := range wanted {
|
||||
present[module] = true
|
||||
}
|
||||
// A module held on the node with no want (the bus on the control-node) is composed at the build its have says it
|
||||
// runs, so it has a plan too and its client keeps filling it (review of 2026-10-30).
|
||||
for module, fp := range h.runningOn(node) {
|
||||
for module, fp := range running {
|
||||
present[module] = true
|
||||
if _, ok := wanted[module]; !ok {
|
||||
wanted[module] = fp
|
||||
}
|
||||
@@ -200,6 +242,9 @@ func (h *Holder) ComposeNode(ctx context.Context, node string) (out Composed, er
|
||||
if err := h.writeRefused(node, heldAtWants); err != nil {
|
||||
out.Problems = append(out.Problems, err.Error())
|
||||
}
|
||||
// A module whose composition is not published at its build is held (heldAtWants): its own plan and keys are kept,
|
||||
// and nothing else on the node waits for it (review 1 of issue 940). A hold that lasts is said on the health.
|
||||
h.countUnpublished(node, c.Unpublished, heldAtWants)
|
||||
if err := h.writePlans(node, atWants, heldAtWants, &out); err != nil {
|
||||
out.Problems = append(out.Problems, err.Error())
|
||||
}
|
||||
@@ -259,8 +304,21 @@ func (h *Holder) ComposeNode(ctx context.Context, node string) (out Composed, er
|
||||
}
|
||||
}
|
||||
}
|
||||
// A module with a want or a have on the node whose plan this pass did not read is left as it is: its keys are not
|
||||
// judged unneeded (issue 940).
|
||||
untouched := map[string]string{}
|
||||
for m, why := range heldAtWants {
|
||||
untouched[m] = why
|
||||
}
|
||||
for m := range present {
|
||||
_, a := atWants[m]
|
||||
_, n := newest[m]
|
||||
if _, u := untouched[m]; !a && !n && !u {
|
||||
untouched[m] = "it has a want or a have on " + node + " and no plan of it was read"
|
||||
}
|
||||
}
|
||||
h.judgeElsewhere(elsewhere, &out)
|
||||
h.judgeNode(node, needed, ahead, heldAtWants, &out)
|
||||
h.judgeNode(node, needed, ahead, untouched, retireMany, &out)
|
||||
lap("provisions judged")
|
||||
return out, nil
|
||||
}
|
||||
@@ -419,11 +477,11 @@ func (h *Holder) wantedOn(node string) (map[string]string, error) {
|
||||
|
||||
// runningOn is the fingerprint each module's have on a node says it runs, by module: what a module held without a want
|
||||
// is composed at.
|
||||
func (h *Holder) runningOn(node string) map[string]string {
|
||||
func (h *Holder) runningOn(node string) (map[string]string, error) {
|
||||
out := map[string]string{}
|
||||
keys, err := h.Store.Keys(HaveRead)
|
||||
if err != nil {
|
||||
return out
|
||||
return nil, fmt.Errorf("the haves cannot be read: %w", err)
|
||||
}
|
||||
for _, k := range keys {
|
||||
n, module, isNode, err := delivery.ParseHaveKey(k)
|
||||
@@ -438,7 +496,7 @@ func (h *Holder) runningOn(node string) map[string]string {
|
||||
out[module] = have.FP
|
||||
}
|
||||
}
|
||||
return out
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// writePlans writes each plan that differs and removes the node's plans of modules no longer composed there; a held
|
||||
@@ -489,14 +547,18 @@ func (h *Holder) writePlans(node string, plans map[string]catalogue.Plan, held m
|
||||
return errors.Join(problems...)
|
||||
}
|
||||
|
||||
// judgeNode brings the node's provision keys to what is needed and what each provider said. A key of a module held on
|
||||
// the node (heldModules) that its composition no longer names is left as it is: not composed is not unneeded.
|
||||
// judgeNode brings the node's provision keys to what is needed and what each provider said. A key of a module left
|
||||
// untouched (held on the node, or with a want or a have and no plan read in this pass) that its composition no longer
|
||||
// names is left as it is: not composed is not unneeded. A pass that would retire most of the node's consumers at once retires none, said as a fault of
|
||||
// the mesh (issue 940), unless retireMany.
|
||||
func (h *Holder) judgeNode(node string, needed map[string]provisions.Provision, ahead map[string]string,
|
||||
heldModules map[string]string, out *Composed) {
|
||||
untouched map[string]string, retireMany bool, out *Composed) {
|
||||
held := h.held(node)
|
||||
consumers := map[string]bool{}
|
||||
for key, have := range held {
|
||||
consumers[have.Consumer.Module] = true
|
||||
if _, isNeeded := needed[key]; !isNeeded && have.Consumer.Node == node {
|
||||
if _, notComposed := heldModules[have.Consumer.Module]; notComposed {
|
||||
if _, left := untouched[have.Consumer.Module]; left {
|
||||
delete(held, key)
|
||||
}
|
||||
}
|
||||
@@ -510,6 +572,37 @@ func (h *Holder) judgeNode(node string, needed map[string]provisions.Provision,
|
||||
}
|
||||
}
|
||||
}
|
||||
// The keys this pass would turn retiring, and whose they are.
|
||||
var newly []string
|
||||
leaving := map[string]bool{}
|
||||
for key, have := range held {
|
||||
if _, isNeeded := needed[key]; !isNeeded && have.State != provisions.Retiring {
|
||||
newly = append(newly, key)
|
||||
leaving[have.Consumer.Module] = true
|
||||
}
|
||||
}
|
||||
sort.Strings(newly)
|
||||
hold := func(why string) {
|
||||
for _, key := range newly {
|
||||
delete(held, key)
|
||||
}
|
||||
out.Problems = append(out.Problems, why)
|
||||
}
|
||||
|
||||
switch {
|
||||
case len(newly) == 0:
|
||||
h.clearFault(node)
|
||||
case manyAtOnce(len(leaving), len(consumers)) && !retireMany:
|
||||
fault := fmt.Sprintf("On %s, one pass would retire the provisions of %d of the %d consumers that hold any (%s): "+
|
||||
"most likely a read stopped listing them (a node's compositions published in part, a want, a have or the "+
|
||||
"placement not read whole), not their removal. Nothing is retired and nothing is asked; the keys stay as they "+
|
||||
"were. If the removal is meant, provisioner_compose with retire-many retires them, each asked.",
|
||||
node, len(leaving), len(consumers), strings.Join(sortedKeys(leaving), ", "))
|
||||
hold(fault)
|
||||
h.setFault(node, fault)
|
||||
default:
|
||||
h.clearFault(node)
|
||||
}
|
||||
for _, key := range unionKeys(needed, held) {
|
||||
want, isNeeded := needed[key]
|
||||
have, isHeld := held[key]
|
||||
@@ -532,12 +625,12 @@ func (h *Holder) judgeNode(node string, needed map[string]provisions.Provision,
|
||||
}
|
||||
case have.State == provisions.Retiring:
|
||||
next = have
|
||||
if next.Ask != "" && next.Warrant == nil && !h.askHeld(next) {
|
||||
out.Problems = append(out.Problems, "the router holds no ask "+next.Ask+" of "+key+": it is raised again")
|
||||
next.Ask = ""
|
||||
}
|
||||
if next.Ask == "" && next.Warrant == nil {
|
||||
if err := h.ask(&next, key); err != nil {
|
||||
out.Problems = append(out.Problems, err.Error())
|
||||
} else {
|
||||
out.Asked = append(out.Asked, key)
|
||||
}
|
||||
h.raise(&next, key, out)
|
||||
}
|
||||
if gone, why := h.retired(key, have); gone {
|
||||
if err := h.Store.Delete(ProvisionsState, key); err != nil {
|
||||
@@ -552,11 +645,7 @@ func (h *Holder) judgeNode(node string, needed map[string]provisions.Provision,
|
||||
next.State, next.Why = provisions.Retiring, fmt.Sprintf("no build wanted or placed on %s requires %s of %s on %s "+
|
||||
"any more", node, have.Provision, have.Provider.Module, have.Provider.Node)
|
||||
next.Warrant = nil
|
||||
if err := h.ask(&next, key); err != nil {
|
||||
out.Problems = append(out.Problems, err.Error())
|
||||
} else {
|
||||
out.Asked = append(out.Asked, key)
|
||||
}
|
||||
h.raise(&next, key, out)
|
||||
}
|
||||
if isHeld && same(have, next) {
|
||||
continue
|
||||
@@ -733,6 +822,14 @@ func (h *Holder) haveOutcome(p *provisions.Provision) {
|
||||
case have.Phase == delivery.PhaseFailed || have.Health == delivery.HealthFailed:
|
||||
p.State, p.Why = provisions.Failed, orSaid(have.Detail, where+" failed")
|
||||
case have.Phase == delivery.PhaseRunning && have.Health == delivery.HealthOK && (wantFP == "" || have.FP == wantFP):
|
||||
// A provider's have counts as its outcome only from a build whose manifest offers the provision (ADR 0603 clause 5).
|
||||
// Refused only when that manifest is known and does not offer it: a have with no builder key (an old-form build, an
|
||||
// adopted module) or an unreadable manifest keeps the outcome it had.
|
||||
if offers, known := h.buildOffers(p.Provider.Module, have.FP, p.Provision); known && !offers {
|
||||
p.State, p.Why = provisions.Asked, "waits for "+where+" to run a build that offers "+p.Provision+": the build it runs ("+
|
||||
have.FP+") does not"
|
||||
return
|
||||
}
|
||||
p.State, p.Why = provisions.Provided, ""
|
||||
default:
|
||||
p.State, p.Why = provisions.Asked, fmt.Sprintf("waits for %s to run healthy at the build its want names (it is %s, "+
|
||||
@@ -740,6 +837,56 @@ func (h *Holder) haveOutcome(p *provisions.Provision) {
|
||||
}
|
||||
}
|
||||
|
||||
// buildOffers says whether the build of a module at a fingerprint, as the builder keyed it, was made from a manifest that
|
||||
// offers the provision; known is false when the builder holds no such key, the key names no manifest, or the indexer holds
|
||||
// no such manifest (the caller then does not refuse).
|
||||
func (h *Holder) buildOffers(module, fingerprint, provision string) (offers, known bool) {
|
||||
key, err := delivery.BuilderBuildKey(module, fingerprint)
|
||||
if err != nil {
|
||||
return false, false
|
||||
}
|
||||
be, err := h.Store.Get(BuildsRead, key)
|
||||
if err != nil || be == nil {
|
||||
return false, false
|
||||
}
|
||||
b, err := delivery.DecodeBuilderBuild(be.Value)
|
||||
if err != nil || b.Manifest == "" {
|
||||
return false, false
|
||||
}
|
||||
mk, err := delivery.ManifestsKey(b.Manifest)
|
||||
if err != nil {
|
||||
return false, false
|
||||
}
|
||||
me, err := h.Store.Get(ManifestsRead, mk)
|
||||
if err != nil || me == nil {
|
||||
return false, false
|
||||
}
|
||||
man, err := delivery.DecodeManifest(me.Value)
|
||||
if err != nil {
|
||||
return false, false
|
||||
}
|
||||
var decl struct {
|
||||
Provides []json.RawMessage `json:"provides"`
|
||||
}
|
||||
if json.Unmarshal([]byte(man.JSON), &decl) != nil {
|
||||
return false, false
|
||||
}
|
||||
for _, raw := range decl.Provides {
|
||||
var name string
|
||||
if json.Unmarshal(raw, &name) != nil {
|
||||
var full struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &full)
|
||||
name = full.Name
|
||||
}
|
||||
if name == provision {
|
||||
return true, true
|
||||
}
|
||||
}
|
||||
return false, true
|
||||
}
|
||||
|
||||
// keepsEntries says whether a provider's outcome for one provision is its entries (its manifest at the trunk declares the
|
||||
// state `bound`) rather than its own have. When any offer of the provider says `answered-by: helper`, only those offers
|
||||
// are answered by its entries: a module running the helper for one provision and making another some other way (gitea:
|
||||
@@ -878,7 +1025,33 @@ func retireAct(key, id, option string) asks.Act {
|
||||
return asks.Act{"verb": "mesh-provisioner.retire", "provision": key, "ask": id, "option": option}
|
||||
}
|
||||
|
||||
// ask raises the retiring ask of a key, under mesh-provisioner's own name; one already raised stands.
|
||||
// askOpenAtMost is how many of its asks mesh-provisioner keeps open at once: the router shows at most three per asker
|
||||
// and refuses the rest (issue 940: 18 of 21 retiring asks were refused so, yet counted asked). The next is raised as
|
||||
// one ends (Decide).
|
||||
const askOpenAtMost = 3
|
||||
|
||||
// askHeldWithin is how long the router may take to hold an ask before an ask it holds no record of is raised again.
|
||||
const askHeldWithin = time.Minute
|
||||
|
||||
// askWaits is what a retiring key's why says while its ask waits for an open one to end.
|
||||
const askWaits = "its ask waits until fewer than 3 of mesh-provisioner's asks stand open, the router showing at most 3 per asker"
|
||||
|
||||
// errAskWaits is an ask held back by askOpenAtMost: no fault, raised once one ends.
|
||||
var errAskWaits = errors.New(askWaits)
|
||||
|
||||
// raise asks a retiring key and says so in out: asked, waiting for an open ask to end, or a problem.
|
||||
func (h *Holder) raise(p *provisions.Provision, key string, out *Composed) {
|
||||
switch err := h.ask(p, key); {
|
||||
case err == nil:
|
||||
out.Asked = append(out.Asked, key)
|
||||
case errors.Is(err, errAskWaits):
|
||||
default:
|
||||
out.Problems = append(out.Problems, err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// ask raises the retiring ask of a key, under mesh-provisioner's own name; one already raised stands. While
|
||||
// askOpenAtMost asks stand open it raises none and the key's why says so.
|
||||
func (h *Holder) ask(p *provisions.Provision, key string) error {
|
||||
if p.Ask != "" {
|
||||
return nil
|
||||
@@ -886,6 +1059,12 @@ func (h *Holder) ask(p *provisions.Provision, key string) error {
|
||||
if h.Asks == nil {
|
||||
return errors.New("the operator cannot be asked from here, so " + key + " is retiring and nothing is removed")
|
||||
}
|
||||
if h.openAsks() >= askOpenAtMost {
|
||||
if !strings.Contains(p.Why, askWaits) {
|
||||
p.Why = strings.TrimPrefix(p.Why+"; "+askWaits, "; ")
|
||||
}
|
||||
return errAskWaits
|
||||
}
|
||||
now := h.Now().UTC()
|
||||
id := "retire-" + strings.ReplaceAll(key, ".", "-") + "-" + fmt.Sprint(now.Unix())
|
||||
a := retiringAsk(key, *p, now, id)
|
||||
@@ -896,9 +1075,37 @@ func (h *Holder) ask(p *provisions.Provision, key string) error {
|
||||
return fmt.Errorf("the retiring ask of %s could not be published: %w", key, err)
|
||||
}
|
||||
p.Ask = id
|
||||
p.Why = strings.TrimSuffix(strings.TrimSuffix(p.Why, askWaits), "; ")
|
||||
return nil
|
||||
}
|
||||
|
||||
// openAsks is how many retiring keys hold an ask with no answer yet.
|
||||
func (h *Holder) openAsks() int {
|
||||
n := 0
|
||||
keys, _ := h.Store.Keys(ProvisionsState)
|
||||
for _, k := range keys {
|
||||
if e, err := h.Store.Get(ProvisionsState, k); err == nil && e != nil {
|
||||
if p, err := provisions.Decode(e.Value); err == nil && p.State == provisions.Retiring && p.Ask != "" &&
|
||||
p.Warrant == nil {
|
||||
n++
|
||||
}
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// askHeld says the router holds a key's ask, or may not have taken it yet: false only when it has no record of it
|
||||
// askHeldWithin after it was raised (issue 940: an ask the router refused was counted asked for good).
|
||||
func (h *Holder) askHeld(p provisions.Provision) bool {
|
||||
if h.Asks == nil || p.Ask == "" {
|
||||
return true
|
||||
}
|
||||
if _, found, err := h.Asks.Record(p.Ask); err != nil || found {
|
||||
return true
|
||||
}
|
||||
return h.Now().Sub(p.At) < askHeldWithin
|
||||
}
|
||||
|
||||
// Decide reads the router's record of each retiring key's ask and writes the warrant into the key (clause 5): the
|
||||
// provider acts on an Approve; a Decline is kept, and not asked again while nothing changes.
|
||||
func (h *Holder) Decide() []string {
|
||||
@@ -922,6 +1129,14 @@ func (h *Holder) Decide() []string {
|
||||
continue
|
||||
}
|
||||
raw, found, err := h.Asks.Record(p.Ask)
|
||||
if err == nil && !found && h.Now().Sub(p.At) >= askHeldWithin {
|
||||
// The router holds no record of it: not asked, raised again below once an open ask allows.
|
||||
p.Ask = ""
|
||||
p.Why = strings.TrimPrefix(p.Why+"; "+askWaits, "; ")
|
||||
h.put(k, p)
|
||||
said = append(said, k+": the router holds no ask of it; raised again")
|
||||
continue
|
||||
}
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
@@ -963,9 +1178,119 @@ func (h *Holder) Decide() []string {
|
||||
h.put(k, p)
|
||||
said = append(said, k+": "+option+" by "+by)
|
||||
}
|
||||
return append(said, h.raiseWaiting()...)
|
||||
}
|
||||
|
||||
// raiseWaiting raises the asks that wait for an open one to end, as many as askOpenAtMost allows: an answer, which
|
||||
// wakes Decide, is what frees a place.
|
||||
func (h *Holder) raiseWaiting() []string {
|
||||
keys, err := h.Store.Keys(ProvisionsState)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var said []string
|
||||
for _, k := range keys {
|
||||
e, err := h.Store.Get(ProvisionsState, k)
|
||||
if err != nil || e == nil {
|
||||
continue
|
||||
}
|
||||
p, err := provisions.Decode(e.Value)
|
||||
if err != nil || p.State != provisions.Retiring || p.Ask != "" || p.Warrant != nil || !strings.Contains(p.Why, askWaits) {
|
||||
continue
|
||||
}
|
||||
switch err := h.ask(&p, k); {
|
||||
case errors.Is(err, errAskWaits):
|
||||
return said
|
||||
case err != nil:
|
||||
said = append(said, k+": "+err.Error())
|
||||
default:
|
||||
h.put(k, p)
|
||||
said = append(said, k+": asked")
|
||||
}
|
||||
}
|
||||
return said
|
||||
}
|
||||
|
||||
// manyAtOnce says a pass retiring the provisions of leaving consumers, of all the consumers holding a key on the node,
|
||||
// retires most of them at once: more than 3, or more than half and at least 2 (one consumer leaving is a removal, not
|
||||
// a fault).
|
||||
func manyAtOnce(leaving, all int) bool {
|
||||
return leaving > 3 || (leaving >= 2 && 2*leaving > all)
|
||||
}
|
||||
|
||||
func (h *Holder) setFault(node, why string) {
|
||||
if h.faults == nil {
|
||||
h.faults = map[string]string{}
|
||||
}
|
||||
if h.faults[node] != why {
|
||||
h.Logf("[%s] %s", Module, why)
|
||||
}
|
||||
h.faults[node] = why
|
||||
}
|
||||
|
||||
func (h *Holder) clearFault(node string) {
|
||||
if _, ok := h.faults[node]; ok {
|
||||
h.Logf("[%s] the retirement held back on %s is no longer judged", Module, node)
|
||||
delete(h.faults, node)
|
||||
}
|
||||
}
|
||||
|
||||
// unpublishedFor is how long a module may be held unpublished at one build before the health says it: longer than a
|
||||
// build takes (the build step's bound is 30 minutes), so a newly placed module waiting for its build is no fault. A
|
||||
// pass runs on every change of its node, so passes are no measure of it (review 2 of issue 940).
|
||||
const unpublishedFor = 45 * time.Minute
|
||||
|
||||
type unpublishedHold struct {
|
||||
since time.Time
|
||||
why string // the hold's words, which name the build
|
||||
}
|
||||
|
||||
// countUnpublished keeps, for each module held unpublished on node, since when it is held at that build; a module
|
||||
// published again, or held at another build, starts again.
|
||||
func (h *Holder) countUnpublished(node string, modules []string, held map[string]string) {
|
||||
if h.unpublished == nil {
|
||||
h.unpublished = map[string]map[string]unpublishedHold{}
|
||||
}
|
||||
was := h.unpublished[node]
|
||||
now := map[string]unpublishedHold{}
|
||||
for _, m := range modules {
|
||||
u, ok := was[m]
|
||||
if !ok || u.why != held[m] {
|
||||
u = unpublishedHold{since: h.Now().UTC(), why: held[m]}
|
||||
}
|
||||
now[m] = u
|
||||
}
|
||||
h.unpublished[node] = now
|
||||
}
|
||||
|
||||
// Faults is, by node, each retirement a pass held back as a fault of the mesh.
|
||||
func (h *Holder) Faults() map[string]string {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
out := make(map[string]string, len(h.faults))
|
||||
for n, why := range h.faults {
|
||||
out[n] = why
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// LastingHolds is each module held unpublished at one build for longer than unpublishedFor, as "<node>.<module>" with
|
||||
// the hold's words, which name its build, and since when.
|
||||
func (h *Holder) LastingHolds() map[string]string {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
out := map[string]string{}
|
||||
now := h.Now()
|
||||
for n, mods := range h.unpublished {
|
||||
for m, u := range mods {
|
||||
if now.Sub(u.since) > unpublishedFor {
|
||||
out[n+"."+m] = fmt.Sprintf("%s (held since %s UTC)", u.why, u.since.Format("2006-01-02 15:04"))
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (h *Holder) put(key string, p provisions.Provision) {
|
||||
p.At = h.Now().UTC()
|
||||
if raw, err := provisions.Encode(p); err == nil {
|
||||
@@ -1123,7 +1448,9 @@ func orSaid(words ...string) string {
|
||||
// builds a module is read at the fingerprint keptAt names, else at the indexer's newest (FP2); at the newest, each placed
|
||||
// module at FP2. A key whose engine refused the module holds it with the engine's words; a key not published yet holds
|
||||
// it at its wanted build (its last plan kept, as for a refusal) and leaves it out of the newest, whose build the engine
|
||||
// writes once it is done (ADR 0593 §6). A plan absent from a key is a module that hands the provisioner nothing.
|
||||
// writes once it is done (ADR 0593 §6), and says it unpublished, so the pass judges no retirement on the node (issue
|
||||
// 940). A placed module no build of which is known is held too. A plan absent from a key is a module that hands the
|
||||
// provisioner nothing.
|
||||
func FromCompositions(s Store) Composer {
|
||||
return func(_ context.Context, node string, keptAt map[string]string) Composition {
|
||||
start := time.Now()
|
||||
@@ -1134,7 +1461,11 @@ func FromCompositions(s Store) Composer {
|
||||
at.Err, newest.Err = err, err
|
||||
return Composition{AtWants: at, Newest: newest, Took: []Phase{{Name: "compositions read", Took: time.Since(start)}}}
|
||||
}
|
||||
placed, fp2 := placedModules(s, node)
|
||||
placed, fp2, err := placedModules(s, node)
|
||||
if err != nil {
|
||||
at.Err, newest.Err = err, err
|
||||
return Composition{AtWants: at, Newest: newest, Took: []Phase{{Name: "compositions read", Took: time.Since(start)}}}
|
||||
}
|
||||
atWants := map[string]string{}
|
||||
for _, m := range placed {
|
||||
atWants[m] = fp2[m]
|
||||
@@ -1142,14 +1473,19 @@ func FromCompositions(s Store) Composer {
|
||||
for m, fp := range keptAt {
|
||||
atWants[m] = fp
|
||||
}
|
||||
var unpublished []string
|
||||
for _, m := range sortedKeys(atWants) {
|
||||
fp := atWants[m]
|
||||
if fp == "" {
|
||||
// Placed, with no want, no have and no newest build the indexer names: no build of it is known, which is
|
||||
// not a module handing nothing (issue 940).
|
||||
at.Held[m] = fmt.Sprintf("no build of %s on %s is known: no want, no have, and the indexer names no newest build", m, node)
|
||||
continue
|
||||
}
|
||||
c, ok := published[composedAt(node, m, fp)]
|
||||
switch {
|
||||
case !ok:
|
||||
unpublished = append(unpublished, m)
|
||||
at.Held[m] = fmt.Sprintf("the composition of %s at %s is not published by the node-engine of %s yet", m, fp, node)
|
||||
case c.Refused != "":
|
||||
at.Held[m] = c.Refused
|
||||
@@ -1170,7 +1506,8 @@ func FromCompositions(s Store) Composer {
|
||||
newest.Plans[m] = *c.Plan
|
||||
}
|
||||
}
|
||||
return Composition{AtWants: at, Newest: newest, Took: []Phase{{Name: "compositions read", Took: time.Since(start)}}}
|
||||
return Composition{AtWants: at, Newest: newest, Unpublished: unpublished,
|
||||
Took: []Phase{{Name: "compositions read", Took: time.Since(start)}}}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1216,16 +1553,18 @@ func publishedOn(s Store, node string) (map[string]publishedPlan, error) {
|
||||
}
|
||||
|
||||
// placedModules is every module placed on node, sorted, and the indexer's newest fingerprint (FP2) of each.
|
||||
func placedModules(s Store, node string) ([]string, map[string]string) {
|
||||
func placedModules(s Store, node string) ([]string, map[string]string, error) {
|
||||
raw := map[string][]byte{}
|
||||
if keys, err := s.Keys(PlacementRead); err == nil {
|
||||
for _, k := range keys {
|
||||
if n, _, _, err := delivery.ParsePlacementKey(k); err != nil || n != node {
|
||||
continue
|
||||
}
|
||||
if e, err := s.Get(PlacementRead, k); err == nil && e != nil {
|
||||
raw[k] = e.Value
|
||||
}
|
||||
keys, err := s.Keys(PlacementRead)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the placement cannot be read: %w", err)
|
||||
}
|
||||
for _, k := range keys {
|
||||
if n, _, _, err := delivery.ParsePlacementKey(k); err != nil || n != node {
|
||||
continue
|
||||
}
|
||||
if e, err := s.Get(PlacementRead, k); err == nil && e != nil {
|
||||
raw[k] = e.Value
|
||||
}
|
||||
}
|
||||
placements, _ := delivery.PlacementsOf(raw)
|
||||
@@ -1244,7 +1583,7 @@ func placedModules(s Store, node string) ([]string, map[string]string) {
|
||||
}
|
||||
}
|
||||
sort.Strings(placed)
|
||||
return placed, fp2
|
||||
return placed, fp2, nil
|
||||
}
|
||||
|
||||
// WatchedReads are the states the holder reads whole, each from the watched copy (ADR 0580 clause 2): the placement,
|
||||
|
||||
@@ -224,6 +224,34 @@ func TestTheHolderPlansProvidesAndAsksOnTheBus(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settled(HaveRead, node+".postgres")
|
||||
// The running build's manifest, as the builder keyed it: one that does not offer the provision keeps the key asked
|
||||
// (ADR 0603 clause 5), one that does lets the have count.
|
||||
buildsKey := mustKey(delivery.BuilderBuildKey("postgres", fpsOf["postgres"]))
|
||||
offered := func(provides string) {
|
||||
raw := `{"module":"postgres","version":"1","provides":` + provides + `}`
|
||||
digest := delivery.ManifestDigest([]byte(raw))
|
||||
value, _ := delivery.EncodeManifest(delivery.Manifest{Module: "postgres", JSON: raw})
|
||||
if err := s.Put(ManifestsRead, mustKey(delivery.ManifestsKey(digest)), json.RawMessage(value)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
build, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: digest})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Put(BuildsRead, buildsKey, json.RawMessage(build)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
offered(`["other"]`)
|
||||
for i := 0; i < 2; i++ {
|
||||
if _, err := h.ComposeNode(ctx, node); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if p := state(key); p.State != provisions.Asked {
|
||||
t.Fatalf("a build not offering the provision turned it %s", p.State)
|
||||
}
|
||||
offered(`["postgres-database"]`)
|
||||
for i := 0; i < 2; i++ { // the credential judged first, then what depends on it
|
||||
if _, err := h.ComposeNode(ctx, node); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
@@ -19,14 +20,19 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
)
|
||||
|
||||
// router is the operator's channel as the holder asks on it.
|
||||
// router is the operator's channel as the holder asks on it: it holds an ask open, or, like the router past its open
|
||||
// asks per asker, holds no record of it (drop).
|
||||
type router struct {
|
||||
asked map[string]asks.Ask
|
||||
records map[string][]byte
|
||||
drop bool
|
||||
}
|
||||
|
||||
func (r *router) Publish(_ string, body any, id string) error {
|
||||
r.asked[id] = body.(asks.Ask)
|
||||
if !r.drop {
|
||||
r.records[id], _ = json.Marshal(map[string]any{"state": "open", "ask": body})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -58,20 +64,22 @@ type mesh struct {
|
||||
atWant map[string]map[string]catalogue.Plan
|
||||
atNewest map[string]map[string]catalogue.Plan
|
||||
heldAt map[string]map[string]string // by node, the modules its composition at the wanted builds holds
|
||||
composed []string
|
||||
// unpublished is, by node, the modules whose composition is not published yet.
|
||||
unpublished map[string][]string
|
||||
composed []string
|
||||
}
|
||||
|
||||
func newMesh() *mesh {
|
||||
m := &mesh{s: newStore(), r: &router{asked: map[string]asks.Ask{}, records: map[string][]byte{}},
|
||||
atWant: map[string]map[string]catalogue.Plan{}, atNewest: map[string]map[string]catalogue.Plan{},
|
||||
heldAt: map[string]map[string]string{}}
|
||||
heldAt: map[string]map[string]string{}, unpublished: map[string][]string{}}
|
||||
now := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC)
|
||||
m.h = &Holder{Store: m.s, Asks: m.r, Logf: func(string, ...any) {},
|
||||
Now: func() time.Time { now = now.Add(time.Second); return now },
|
||||
Compose: func(_ context.Context, node string, keptAt map[string]string) Composition {
|
||||
m.composed = append(m.composed, node)
|
||||
return Composition{AtWants: HeldPlans{Plans: m.atWant[node], Held: m.heldAt[node]},
|
||||
Newest: HeldPlans{Plans: m.atNewest[node]}}
|
||||
Newest: HeldPlans{Plans: m.atNewest[node]}, Unpublished: m.unpublished[node]}
|
||||
}}
|
||||
// What each provider's manifest declares: postgres, keycloak and the vault keep entries, the others report by their
|
||||
// have.
|
||||
@@ -181,6 +189,28 @@ func TestAKeysStateFollowsItsProvidersOutcome(t *testing.T) {
|
||||
t.Errorf("the provider's have %s: %+v, want %s", row.have, p, row.state)
|
||||
}
|
||||
}
|
||||
// A have counts as the provider's outcome only from a build whose manifest offers the provision (ADR 0603 clause 5):
|
||||
// refused when the build's manifest is known and does not offer it, provided when it does.
|
||||
bfp := "src2:" + strings.Repeat("bb", 32)
|
||||
buildKey := mustKey(delivery.BuilderBuildKey("systemd-resolved", bfp))
|
||||
for _, row := range []struct{ name, provides, state, why string }{
|
||||
{"a build whose manifest does not offer it", `["other"]`, provisions.Asked, "offers split-dns"},
|
||||
{"a build whose manifest offers it", `[{"name":"split-dns"}]`, provisions.Provided, ""},
|
||||
} {
|
||||
raw := `{"module":"systemd-resolved","version":"1","provides":` + row.provides + `}`
|
||||
digest := delivery.ManifestDigest([]byte(raw))
|
||||
value, _ := delivery.EncodeManifest(delivery.Manifest{Module: "systemd-resolved", JSON: raw})
|
||||
_ = m.s.Put(ManifestsRead, mustKey(delivery.ManifestsKey(digest)), json.RawMessage(value))
|
||||
build, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: digest})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = m.s.Put(BuildsRead, buildKey, json.RawMessage(build))
|
||||
m.compose(t, "g14")
|
||||
if p := m.state(t, split); p.State != row.state || !strings.Contains(p.Why, row.why) {
|
||||
t.Errorf("%s: %+v, want %s", row.name, p, row.state)
|
||||
}
|
||||
}
|
||||
if e, _ := m.s.Get(PlansState, "g14.gitea"); e == nil {
|
||||
t.Error("the plan is not written")
|
||||
}
|
||||
@@ -566,3 +596,191 @@ func TestAKeyAheadOfItsWantCarriesItsPlanEntryAndSaysWhy(t *testing.T) {
|
||||
t.Errorf("the planned key: %+v", pg)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 940 (met before as 937 and 1332): a retirement is judged from a whole picture only. At 2026-11-02 22:55:58 a
|
||||
// first pass after a new membership read a part, counted the modules missing from it as handing nothing, and asked the
|
||||
// operator to remove the routes of 21 running web apps; 3 s later the next pass put every plan back.
|
||||
func TestARetirementIsJudgedOnlyFromAWholePicture(t *testing.T) {
|
||||
fp := "src2:" + strings.Repeat("ab", 32)
|
||||
route := func(n int) catalogue.PlanProvision {
|
||||
return catalogue.PlanProvision{Provision: "route", Provider: "novox", ProviderModule: "keycloak", Local: fmt.Sprint(n)}
|
||||
}
|
||||
apps := []string{"de-spiegel", "invoicing", "link2pay", "nextcloud", "photos", "website"}
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
consumer int // how many of apps hold a key, each one
|
||||
gone int // how many of them the pass no longer composes
|
||||
change func(m *mesh) // what else the pass sees
|
||||
many bool // the operator's retire-many
|
||||
retiring int // keys retiring after the pass
|
||||
asked int // asks raised by the pass
|
||||
problem string // a part of a problem the pass says
|
||||
fault string // a part of the health's why, "" for healthy
|
||||
after func(*testing.T, *mesh) // a further check
|
||||
}{
|
||||
{name: "a read not loaded once: nothing judged", consumer: 6, gone: 6,
|
||||
change: func(m *mesh) { m.h.Loaded = func() string { return WantsRead } }},
|
||||
{name: "another module's composition not published: a removed consumer still retires", consumer: 2, gone: 1,
|
||||
change: func(m *mesh) {
|
||||
m.unpublished["novox"] = []string{"gitea"}
|
||||
m.heldAt["novox"] = map[string]string{"gitea": "the composition of gitea at " + fp + " is not published yet"}
|
||||
}, retiring: 1, asked: 1, after: func(t *testing.T, m *mesh) {
|
||||
// Held at one build: healthy over many passes within the bound (a build takes minutes), unhealthy past it,
|
||||
// naming the module, its build and since when; another build starts the clock again.
|
||||
now := m.h.Now()
|
||||
m.h.Now = func() time.Time { return now }
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
after time.Duration
|
||||
healthy bool
|
||||
}{
|
||||
{"many passes within the bound", 0, true},
|
||||
{"still within the bound", unpublishedFor - time.Minute, true},
|
||||
{"past the bound", unpublishedFor + time.Minute, false},
|
||||
} {
|
||||
now = now.Add(row.after)
|
||||
for range 5 {
|
||||
m.compose(t, "novox")
|
||||
}
|
||||
a := m.h.health()
|
||||
if a.Healthy != row.healthy || (!row.healthy && (!strings.Contains(a.Why, "novox.gitea") ||
|
||||
!strings.Contains(a.Why, fp) || !strings.Contains(a.Why, "held since"))) {
|
||||
t.Errorf("%s: %+v", row.name, a)
|
||||
}
|
||||
}
|
||||
m.heldAt["novox"] = map[string]string{"gitea": "the composition of gitea at src2:" + strings.Repeat("cd", 32) +
|
||||
" is not published yet"}
|
||||
m.compose(t, "novox")
|
||||
if a := m.h.health(); !a.Healthy {
|
||||
t.Errorf("another build did not start the clock again: %+v", a)
|
||||
}
|
||||
m.unpublished["novox"], m.heldAt["novox"] = nil, nil
|
||||
m.compose(t, "novox")
|
||||
if a := m.h.health(); !a.Healthy {
|
||||
t.Errorf("the hold is said after it ended: %+v", a)
|
||||
}
|
||||
}},
|
||||
{name: "a consumer that still has a want: not retired", consumer: 2, gone: 1,
|
||||
change: func(m *mesh) {
|
||||
m.s.write(t, WantsRead, "want.novox.de-spiegel", `{"v":1,"fp":"`+fp+`","self":true}`)
|
||||
}},
|
||||
{name: "a consumer that still has a have: not retired", consumer: 2, gone: 1,
|
||||
change: func(m *mesh) {
|
||||
m.s.write(t, HaveRead, "novox.de-spiegel", `{"v":1,"fp":"`+fp+`","phase":"running","health":"ok"}`)
|
||||
}},
|
||||
{name: "one consumer of two leaving: retired and asked", consumer: 2, gone: 1, retiring: 1, asked: 1},
|
||||
{name: "most consumers at once: a fault, not a question", consumer: 6, gone: 6, problem: "a read stopped listing them",
|
||||
fault: "6 of the 6 consumers"},
|
||||
{name: "more than 3 at once, though fewer than half", consumer: 6, gone: 4, fault: "4 of the 6"},
|
||||
{name: "the operator's retire-many: retired, at most 3 asked at once", consumer: 6, gone: 6, many: true,
|
||||
retiring: 6, asked: askOpenAtMost, after: func(t *testing.T, m *mesh) {
|
||||
var waiting int
|
||||
for _, a := range apps {
|
||||
if strings.Contains(m.state(t, "novox."+a+".route.1").Why, askWaits) {
|
||||
waiting++
|
||||
}
|
||||
}
|
||||
if waiting != 6-askOpenAtMost {
|
||||
t.Errorf("%d keys say their ask waits, want %d", waiting, 6-askOpenAtMost)
|
||||
}
|
||||
}},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
m := newMesh()
|
||||
plans := map[string]catalogue.Plan{}
|
||||
for i := range row.consumer {
|
||||
plans[apps[i]] = planOf(route(1))
|
||||
}
|
||||
m.atWant["novox"], m.atNewest["novox"] = plans, plans
|
||||
m.compose(t, "novox")
|
||||
left := map[string]catalogue.Plan{}
|
||||
for i := row.gone; i < row.consumer; i++ {
|
||||
left[apps[i]] = plans[apps[i]]
|
||||
}
|
||||
m.atWant["novox"], m.atNewest["novox"] = left, left
|
||||
if row.change != nil {
|
||||
row.change(m)
|
||||
}
|
||||
c, err := m.h.ComposeNodeWith(context.Background(), "novox", row.many)
|
||||
if m.h.Loaded != nil {
|
||||
if err == nil || !strings.Contains(err.Error(), WantsRead) {
|
||||
t.Errorf("a pass before every read loaded: %v", err)
|
||||
}
|
||||
} else if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
retiring := 0
|
||||
for i := range row.consumer {
|
||||
if m.state(t, "novox."+apps[i]+".route.1").State == provisions.Retiring {
|
||||
retiring++
|
||||
}
|
||||
}
|
||||
if retiring != row.retiring || len(c.Asked) != row.asked || len(m.r.asked) != row.asked {
|
||||
t.Errorf("%d retiring, %d asked (%d published), want %d and %d: %+v", retiring, len(c.Asked),
|
||||
len(m.r.asked), row.retiring, row.asked, c)
|
||||
}
|
||||
if row.problem != "" && !strings.Contains(strings.Join(c.Problems, " "), row.problem) {
|
||||
t.Errorf("the problems %v do not say %q", c.Problems, row.problem)
|
||||
}
|
||||
if a := m.h.health(); (row.fault == "") != a.Healthy || !strings.Contains(a.Why, row.fault) ||
|
||||
(row.fault != "" && !strings.Contains(a.Why, "retire-many")) {
|
||||
t.Errorf("the health is %+v, want a fault saying %q", a, row.fault)
|
||||
}
|
||||
if row.fault != "" {
|
||||
// The read whole again: the fault clears.
|
||||
m.atWant["novox"], m.atNewest["novox"] = plans, plans
|
||||
m.compose(t, "novox")
|
||||
if a := m.h.health(); !a.Healthy {
|
||||
t.Errorf("the fault stands once the picture is whole: %+v", a)
|
||||
}
|
||||
}
|
||||
if row.after != nil {
|
||||
row.after(t, m)
|
||||
}
|
||||
})
|
||||
}
|
||||
// A placed module no build of which is known is held, not counted as handing nothing; one unpublished says so.
|
||||
var c today
|
||||
readJSON(t, "testdata/today-composed.json", &c)
|
||||
s := goldenMesh(t, c)
|
||||
published(t, s, c)
|
||||
s.write(t, PlacementRead, "placement."+c.Node, `{"v":1,"node":"`+c.Node+`","decided":{},"modules":[{"module":"store"},`+
|
||||
`{"module":"postgres"},{"module":"mesh-provisioner-client"},{"module":"restic"},{"module":"de-spiegel"}]}`)
|
||||
got := FromCompositions(s)(context.Background(), c.Node, map[string]string{"photos": fp})
|
||||
if !strings.Contains(got.AtWants.Held["de-spiegel"], "no build") || !slices.Equal(got.Unpublished, []string{"photos"}) {
|
||||
t.Errorf("held %v, unpublished %v", got.AtWants.Held, got.Unpublished)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 940: an ask the router does not hold is not counted asked; it is raised again once an open ask ends.
|
||||
func TestAnAskTheRouterDoesNotHoldIsRaisedAgain(t *testing.T) {
|
||||
m := newMesh()
|
||||
key := "g14.gitea.postgres-database"
|
||||
m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB), "dunst": planOf(oidc)}
|
||||
m.compose(t, "g14")
|
||||
m.atWant["g14"] = map[string]catalogue.Plan{"dunst": planOf(oidc)}
|
||||
m.r.drop = true
|
||||
m.compose(t, "g14")
|
||||
first := m.state(t, key).Ask
|
||||
if first == "" {
|
||||
t.Fatal("not asked")
|
||||
}
|
||||
m.r.drop = false
|
||||
steps := []struct {
|
||||
name string
|
||||
later time.Duration
|
||||
held bool // the router holds an ask of the key after the step
|
||||
}{
|
||||
{"within a minute the router may not have taken it yet", 0, false},
|
||||
{"a minute on, with no record: raised again", 2 * time.Minute, true},
|
||||
}
|
||||
for _, s := range steps {
|
||||
now := m.h.Now()
|
||||
m.h.Now = func() time.Time { now = now.Add(time.Second); return now.Add(s.later) }
|
||||
m.h.Decide()
|
||||
p := m.state(t, key)
|
||||
if _, held := m.r.records[p.Ask]; held != s.held || (s.held && p.Ask == first) {
|
||||
t.Errorf("%s: the key's ask is %q, the router holds it: %v", s.name, p.Ask, held)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
package main
|
||||
|
||||
// The holder's tools (novox/hq ADR 0570 Consequences, ADR 0564 §12's rule): a read of every key of `provisions` and
|
||||
// `plans`, one key whole, each state's fill, making one node's plans now, and reading the operator's answers now.
|
||||
// `plans`, one key whole, each state's fill, making one node's plans now (retiring more at once on the operator's
|
||||
// word), reading the operator's answers now, and its health: a retirement held back as a fault of the mesh (issue 940).
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -10,6 +11,7 @@ import (
|
||||
"strings"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
)
|
||||
|
||||
@@ -19,6 +21,7 @@ const (
|
||||
PlansTool = "provisioner_plans"
|
||||
ComposeTool = "provisioner_compose"
|
||||
DecideTool = "provisioner_decide"
|
||||
HealthTool = "provisioner_health"
|
||||
)
|
||||
|
||||
func str(d string) map[string]any { return map[string]any{"type": "string", "description": d} }
|
||||
@@ -57,22 +60,50 @@ func holderTools(h *Holder) []stdio.Tool {
|
||||
{Name: ComposeTool,
|
||||
Description: "Make one node's plans and provisions now from the node-engine's published compositions and the " +
|
||||
"providers' outcomes, as a change event would (it composes nothing), and say what was written, removed, " +
|
||||
"changed and asked, and where the time went.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{"node": str("the node")},
|
||||
"changed and asked, and where the time went. A pass that would retire the provisions of most of the " +
|
||||
"node's consumers at once retires none and says it as a fault (provisioner_health); retire-many: true " +
|
||||
"is the operator's word that such a removal is meant, and retires them in this pass, each asked.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{"node": str("the node"),
|
||||
"retire-many": map[string]any{"type": "boolean", "description": "retire past the bound in this pass"}},
|
||||
"required": []string{"node"}},
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
if arg(a, "node") == "" {
|
||||
return nil, fmt.Errorf("compose names the node")
|
||||
}
|
||||
return h.ComposeNode(context.Background(), arg(a, "node"))
|
||||
many, _ := a["retire-many"].(bool)
|
||||
return h.ComposeNodeWith(context.Background(), arg(a, "node"), many)
|
||||
}},
|
||||
{Name: DecideTool,
|
||||
Description: "Read the operator's answers to every retiring provision's ask now, writing each warrant into its " +
|
||||
"key, as the router's word on an ask would.",
|
||||
Run: func(map[string]any) (any, error) { return map[string]any{"said": h.Decide()}, nil }},
|
||||
{Name: HealthTool,
|
||||
Description: "The provisioner's runtime health, as the node-engine asks it (novox/hq ADR 0366): unhealthy while " +
|
||||
"a pass held back a retirement of most of a node's consumers at once as a fault of the mesh (issue 940), " +
|
||||
"naming the node, the consumers and the likely cause, or while a module's composition has stayed " +
|
||||
"unpublished at one build for longer than 45 minutes (longer than a build takes), naming the module, " +
|
||||
"the build and since when.",
|
||||
Run: func(map[string]any) (any, error) { return h.health(), nil }},
|
||||
}
|
||||
}
|
||||
|
||||
// health is the holder's runtime health: unhealthy while a retirement is held back as a fault of the mesh.
|
||||
func (h *Holder) health() health.Answer {
|
||||
faults, holds := h.Faults(), h.LastingHolds()
|
||||
if len(faults)+len(holds) == 0 {
|
||||
return health.Healthy("No pass holds back a retirement of most of a node's consumers, and no module's " +
|
||||
"composition has stayed unpublished at one build for longer than " + unpublishedFor.String() + ".")
|
||||
}
|
||||
var said []string
|
||||
for _, n := range sortedKeys(faults) {
|
||||
said = append(said, faults[n])
|
||||
}
|
||||
for _, k := range sortedKeys(holds) {
|
||||
said = append(said, k+": "+holds[k]+"; its plan and its provisions are kept as they were.")
|
||||
}
|
||||
return health.Unhealthy(strings.Join(said, " "))
|
||||
}
|
||||
|
||||
// read is a state's keys narrowed, or one whole, with the state's fill.
|
||||
func (h *Holder) read(state, key, node, want string) (any, error) {
|
||||
if key != "" {
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -68,6 +69,52 @@ func (runtimeAsks) Record(id string) ([]byte, bool, error) {
|
||||
return e.Value, true, nil
|
||||
}
|
||||
|
||||
// loads is which of the reads a pass composes from have loaded once: their watch took every current value.
|
||||
type loads struct {
|
||||
mu sync.Mutex
|
||||
want []string
|
||||
loaded map[string]bool
|
||||
}
|
||||
|
||||
func newLoads(states ...string) *loads { return &loads{want: states, loaded: map[string]bool{}} }
|
||||
|
||||
func (l *loads) done(state string) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
l.loaded[state] = true
|
||||
}
|
||||
|
||||
// missing names the reads not loaded yet, "" once all are.
|
||||
func (l *loads) missing() string {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
var out []string
|
||||
for _, s := range l.want {
|
||||
if !l.loaded[s] {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return strings.Join(out, ", ")
|
||||
}
|
||||
|
||||
// heldBack is, once every read has loaded, every node the placement names: the passes their loads held back.
|
||||
func (l *loads) heldBack(h *Holder) []string {
|
||||
if l.missing() != "" {
|
||||
return nil
|
||||
}
|
||||
keys, err := h.Store.Keys(PlacementRead)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
nodes := map[string]bool{}
|
||||
for _, k := range keys {
|
||||
for _, n := range NodesOfPlacement(k) {
|
||||
nodes[n] = true
|
||||
}
|
||||
}
|
||||
return sortedKeys(nodes)
|
||||
}
|
||||
|
||||
// work is the nodes a change touched, composed one at a time by one worker: no timer, a change's event is the wake.
|
||||
type work struct {
|
||||
mu sync.Mutex
|
||||
@@ -128,14 +175,25 @@ func main() {
|
||||
h := &Holder{Store: Watching(s, ws), Asks: runtimeAsks{}, Now: time.Now, Logf: logf}
|
||||
h.Compose = FromCompositions(h.Store) // the node-engines' published compositions, from the copy (ADR 0593 §8)
|
||||
w := &work{pending: map[string]bool{}, wake: make(chan struct{}, 1)}
|
||||
// No pass judges before every read it composes from has loaded once (issue 940); each load wakes every node it
|
||||
// held back, through its follower's handler.
|
||||
loads := newLoads(PlacementRead, WantsRead, ComposedRead, HaveRead, ModulesRead)
|
||||
h.Loaded = loads.missing
|
||||
go w.run(h)
|
||||
go starting(h)
|
||||
// Each watch's handler runs after the copy took its change, so a composition it wakes reads the copy that change made.
|
||||
go ws.Follow(PlacementRead, func(c stdio.StateChange) error { w.add(NodesOfPlacement(c.Key)...); return nil })
|
||||
go ws.Follow(WantsRead, func(c stdio.StateChange) error { w.add(NodesOfWant(c.Key)...); return nil })
|
||||
go ws.Follow(ComposedRead, func(c stdio.StateChange) error { w.add(NodesOfNodeKey(c.Key)...); return nil })
|
||||
go ws.Follow(HaveRead, func(c stdio.StateChange) error { w.add(NodesOfNodeKey(c.Key)...); return nil })
|
||||
go ws.Follow(ModulesRead, func(c stdio.StateChange) error {
|
||||
follow := func(state string, then func(stdio.StateChange) error) {
|
||||
go func() {
|
||||
ws.Follow(state, then)
|
||||
loads.done(state)
|
||||
w.add(loads.heldBack(h)...)
|
||||
}()
|
||||
}
|
||||
follow(PlacementRead, func(c stdio.StateChange) error { w.add(NodesOfPlacement(c.Key)...); return nil })
|
||||
follow(WantsRead, func(c stdio.StateChange) error { w.add(NodesOfWant(c.Key)...); return nil })
|
||||
follow(ComposedRead, func(c stdio.StateChange) error { w.add(NodesOfNodeKey(c.Key)...); return nil })
|
||||
follow(HaveRead, func(c stdio.StateChange) error { w.add(NodesOfNodeKey(c.Key)...); return nil })
|
||||
follow(ModulesRead, func(c stdio.StateChange) error {
|
||||
if module, err := delivery.ParseModulesKey(c.Key); err == nil {
|
||||
w.add(h.NodesOfModule(module)...)
|
||||
}
|
||||
|
||||
@@ -8,8 +8,6 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// memStore is the bus as mesh-provisioner reaches it, in memory: the states it keeps.
|
||||
@@ -326,13 +324,15 @@ func TestTheManifestDeclaresWhatTheCodeKeeps(t *testing.T) {
|
||||
t.Errorf("the manifest lists %s, which is not served", tool)
|
||||
}
|
||||
}
|
||||
for _, r := range append(slices.Clone(WatchedReads), ManifestsRead) {
|
||||
// The builds read serves only haveOutcome's offer check (ADR 0603 clause 5: which manifest the build a provider runs
|
||||
// was made from) and makes no plan, so ADR 0593 §8 ("it composes nothing") holds.
|
||||
for _, r := range append(slices.Clone(WatchedReads), ManifestsRead, BuildsRead) {
|
||||
if !slices.Contains(m.Reads, r) {
|
||||
t.Errorf("the manifest does not read %s, which the holder reads", r)
|
||||
}
|
||||
}
|
||||
// It composes nothing (ADR 0593 §8): no build, and no config value, is read to make a plan.
|
||||
for _, r := range []string{delivery.BuilderModule + ".builds", "mesh-config.resolved"} {
|
||||
// It composes nothing (ADR 0593 §8): no config value is read to make a plan.
|
||||
for _, r := range []string{"mesh-config.resolved"} {
|
||||
if slices.Contains(m.Reads, r) {
|
||||
t.Errorf("the manifest reads %s, which only a composer reads", r)
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-provisioner
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"module": "mesh-provisioner",
|
||||
"version": "1",
|
||||
"version": "2",
|
||||
"runs-as": "mesh-provisioner",
|
||||
"claims": [
|
||||
{
|
||||
@@ -20,6 +20,7 @@
|
||||
"mesh-delivery-indexer.modules",
|
||||
"mesh-delivery-indexer.manifests",
|
||||
"node-engine.have",
|
||||
"mesh-delivery-builder.builds",
|
||||
"mesh-host.composed",
|
||||
"gitea.bound",
|
||||
"keycloak.bound",
|
||||
@@ -33,7 +34,8 @@
|
||||
"provisioner_provisions",
|
||||
"provisioner_plans",
|
||||
"provisioner_compose",
|
||||
"provisioner_decide"
|
||||
"provisioner_decide",
|
||||
"provisioner_health"
|
||||
],
|
||||
"state": [
|
||||
"registry",
|
||||
@@ -44,10 +46,19 @@
|
||||
"plans",
|
||||
"refused"
|
||||
],
|
||||
"core-credentials": true,
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
},
|
||||
"secrets-owner": "mesh-provisioner",
|
||||
"runtime-health": {
|
||||
"kind": "tool",
|
||||
"tool": "provisioner_health",
|
||||
"interval": "60s",
|
||||
"timeout": "10s",
|
||||
"looks": 2,
|
||||
"grace": "60s"
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"id": "account",
|
||||
|
||||
@@ -24,10 +24,6 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// CoreModules are the core's modules whose own secrets the controller carries sealed by the vault: the same fixed
|
||||
// list as the controller's (ADR 0349 §16). The node-engine and the controller have no own secrets of their own here.
|
||||
var CoreModules = []string{"mesh-provisioner", "mesh-provisioner-client", "nats"}
|
||||
|
||||
// nodeKeyFile is a machine's sealing key as its engine stated it.
|
||||
type nodeKeyFile struct {
|
||||
Node string `json:"node"`
|
||||
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
"encoding/pem"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -56,13 +55,6 @@ func clientsBroker(sealingKey string) MoveOf {
|
||||
Generation: "g-1", SealingKey: sealingKey}
|
||||
}
|
||||
|
||||
// The list is the controller's (mesh-controller's core list, ADR 0349 §16): named here so a change to either is seen.
|
||||
func TestTheCoreIsTheBusTheProvisionerAndItsClient(t *testing.T) {
|
||||
if want := []string{"mesh-provisioner", "mesh-provisioner-client", "nats"}; !reflect.DeepEqual(CoreModules, want) {
|
||||
t.Fatalf("the core's modules are %v, want %v", CoreModules, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A node-engine states its machine's sealing key in its own name at the move, and only its own machine's.
|
||||
func TestAMoveRecordsTheSealingKeyItsNodeEngineStates(t *testing.T) {
|
||||
v := newVault(t, nil)
|
||||
|
||||
@@ -224,6 +224,9 @@ const ProcessSecretPrefix = "broker-"
|
||||
// `broker-<process id>`. The vault cannot tell core or moved from a module's manifest; the controller, which can, is the
|
||||
// only caller that may hand one in, and the name keeps what it hands to these.
|
||||
func Covered(secret string) bool {
|
||||
if _, _, ok := ledgerAccountOf(secret); ok {
|
||||
return true
|
||||
}
|
||||
parts := strings.Split(secret, "/")
|
||||
if len(parts) != 4 || parts[0] != "own" {
|
||||
return false
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
package main
|
||||
|
||||
// The ledger accounts of a machine and the tool runner's own secret `ledgers` (novox/hq ADR 0597 §9, design 25 § 12.4,
|
||||
// issue 1966 V1 and V2). Nothing here is the custody ledger of ledger.go (ADR 0085): this file is about the bus account
|
||||
// of kind ledger, one per module assignment that asks for a ledger.
|
||||
//
|
||||
// The tool runner of a machine holds its machine's ledger accounts together as its own secret `ledgers`
|
||||
// (`own/<node>/node-tools/ledgers`): a JSON map of module to credential, `{}` when there is none, so its composition
|
||||
// never refuses a secret the vault did not make. The vault makes it as `{}` for every machine whose tool runner is
|
||||
// wanted, and rewrites it on each ledger account minted or retired, saying `secret-changed` once per rewrite.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
)
|
||||
|
||||
// toolRunnerModule is the module whose own secret `ledgers` is: the tool runner (mesh-tools node-tools).
|
||||
const toolRunnerModule = "node-tools"
|
||||
|
||||
// ledgersSecret is the vault's name of node's `ledgers`.
|
||||
func ledgersSecret(node string) (string, error) {
|
||||
return OwnSecret(node, toolRunnerModule, buscompose.LedgersSecret)
|
||||
}
|
||||
|
||||
// toolRunnerNodes are the machines whose tool runner is wanted, sorted.
|
||||
func (v *Vault) toolRunnerNodes() []string {
|
||||
v.minting.mu.Lock()
|
||||
defer v.minting.mu.Unlock()
|
||||
var out []string
|
||||
for _, w := range v.minting.wants {
|
||||
if w.Kind == buscompose.KindNodeTools {
|
||||
out = append(out, w.Node)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// LedgersPass makes `ledgers` as `{}` on every machine whose tool runner is wanted and for which the vault holds none,
|
||||
// at the vault's minute pass (and when the tool runner's want is heard).
|
||||
func (v *Vault) LedgersPass() {
|
||||
for _, node := range v.toolRunnerNodes() {
|
||||
v.ensureLedgers(node)
|
||||
}
|
||||
}
|
||||
|
||||
// ensureLedgers keeps node's `ledgers` as `{}` when the vault holds none: a held one is never touched here.
|
||||
func (v *Vault) ensureLedgers(node string) {
|
||||
secret, err := ledgersSecret(node)
|
||||
if err != nil {
|
||||
v.log("%s's ledgers are not made: %v", node, err)
|
||||
return
|
||||
}
|
||||
_, held, err := v.Store.Read(secret)
|
||||
if err != nil || held {
|
||||
if err != nil {
|
||||
v.log("%s is not read, so its ledgers are not made: %v", secret, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
empty, _ := json.Marshal(map[string]json.RawMessage{})
|
||||
if _, err := v.keepLocked(secret, string(empty), "the vault", "", OriginMade, keepOptions{neverReplace: true}); err != nil {
|
||||
v.log("%s is not made, tried again at the next pass: %v", secret, err)
|
||||
return
|
||||
}
|
||||
v.log("made %s as {}: no ledger account is held for its machine yet", secret)
|
||||
}
|
||||
|
||||
// ledgerAccountPrefix starts the name a ledger account is kept under: `ledger/<machine>/<module>`. It is the account's
|
||||
// own record, never an own secret (so `own` hands no module its ledger account); the machine's `ledgers` is made of them.
|
||||
const ledgerAccountPrefix = "ledger/"
|
||||
|
||||
// LedgerAccountSecret is the vault's name of the credential of the ledger account of module on node.
|
||||
func LedgerAccountSecret(node, module string) (string, error) {
|
||||
for what, v := range map[string]string{"the machine": node, "the module": module} {
|
||||
if !namePart.MatchString(v) {
|
||||
return "", fmt.Errorf("%s %q is not a name the mesh writes", what, v)
|
||||
}
|
||||
}
|
||||
return ledgerAccountPrefix + node + "/" + module, nil
|
||||
}
|
||||
|
||||
// ledgerAccountOf says secret is a ledger account's name, with its machine and module.
|
||||
func ledgerAccountOf(secret string) (node, module string, ok bool) {
|
||||
parts := strings.Split(secret, "/")
|
||||
if len(parts) != 3 || parts[0] != "ledger" || !namePart.MatchString(parts[1]) || !namePart.MatchString(parts[2]) {
|
||||
return "", "", false
|
||||
}
|
||||
return parts[1], parts[2], true
|
||||
}
|
||||
|
||||
// rewriteLedgers makes node's `ledgers` of the ledger accounts that stand: held, not retired, and wanted (once the wanted
|
||||
// accounts are read whole; before that every one held stands, so a start never drops what it has not heard yet). It
|
||||
// says `secret-changed` only when the map changed.
|
||||
func (v *Vault) rewriteLedgers(node string) {
|
||||
v.minting.ledgersMu.Lock()
|
||||
defer v.minting.ledgersMu.Unlock()
|
||||
secret, err := ledgersSecret(node)
|
||||
if err != nil {
|
||||
v.log("%s's ledgers are not rewritten: %v", node, err)
|
||||
return
|
||||
}
|
||||
records, err := v.Store.ListPrefix(ledgerAccountPrefix + node + "/")
|
||||
if err != nil {
|
||||
v.log("%s is not rewritten, the ledger accounts of %s could not be read: %v", secret, node, err)
|
||||
return
|
||||
}
|
||||
v.minting.mu.Lock()
|
||||
read := v.minting.retiring.wantsRead
|
||||
v.minting.mu.Unlock()
|
||||
standing := map[string]json.RawMessage{}
|
||||
for _, r := range records {
|
||||
_, module, ok := ledgerAccountOf(r.Name)
|
||||
if !ok || r.Retired != nil {
|
||||
continue
|
||||
}
|
||||
if _, wanted := v.wanted("ledger." + node + "." + module); read && !wanted {
|
||||
continue
|
||||
}
|
||||
value, err := openAtRest(r, v)
|
||||
if err != nil || !BusAccountValue(r.Name, value) {
|
||||
v.log("%s is not an account the vault can open, so it is left out of %s", r.Name, secret)
|
||||
continue
|
||||
}
|
||||
standing[module] = json.RawMessage(value)
|
||||
}
|
||||
raw, err := json.Marshal(standing)
|
||||
if err != nil {
|
||||
v.log("%s is not rewritten: %v", secret, err)
|
||||
return
|
||||
}
|
||||
if _, err := v.keepLocked(secret, string(raw), "the vault", "", OriginMade, keepOptions{}); err != nil {
|
||||
v.log("%s is NOT REWRITTEN (%d ledger accounts stand): %v", secret, len(standing), err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/seal"
|
||||
)
|
||||
|
||||
// Ledger accounts and the tool runner's own secret `ledgers` (novox/hq ADR 0597 §9, issue 1966). Not the custody
|
||||
// ledger of ledger.go (ADR 0085), which is another thing.
|
||||
|
||||
// nodeToolsWant is the wanted account of a machine's tool runner, as the grants part writes it.
|
||||
func nodeToolsWant(node string) buscompose.WantedAccount {
|
||||
w := aWant("node-tools")
|
||||
w.Kind, w.Node, w.User = buscompose.KindNodeTools, node, node+".node-tools"
|
||||
return w
|
||||
}
|
||||
|
||||
// ledgerWant is the wanted ledger account of a module on a machine.
|
||||
func ledgerWant(node, module string) buscompose.WantedAccount {
|
||||
w := aWant(module)
|
||||
w.Kind, w.Node, w.User, w.Claims = buscompose.KindLedger, node, "ledger."+node+"."+module, nil
|
||||
return w
|
||||
}
|
||||
|
||||
// ledgersOf is what the vault's `own` answers the machine as node-tools' `ledgers`: the map of module to credential,
|
||||
// and whether the secret is there at all.
|
||||
func ledgersOf(t *testing.T, v testVault, priv string) (map[string]json.RawMessage, bool) {
|
||||
t.Helper()
|
||||
own, err := v.OwnOfNode("node.ace")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, o := range own.Own {
|
||||
if o.Module == "node-tools" && o.Name == buscompose.LedgersSecret {
|
||||
value, err := seal.Open(o.Sealed, priv)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := map[string]json.RawMessage{}
|
||||
if err := json.Unmarshal([]byte(value), &got); err != nil {
|
||||
t.Fatalf("ledgers is not a JSON map: %v", err)
|
||||
}
|
||||
return got, true
|
||||
}
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// changesOf counts the secret-changed events said for name since the first of events.
|
||||
func changesOf(v testVault, since int, name string) int {
|
||||
n := 0
|
||||
for _, e := range v.bus.events[since:] {
|
||||
if body, _ := e["body"].(map[string]any); e["event"] == ChangedEvent && body["name"] == name {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// redacted says a credential's shape without its password, for a failure message.
|
||||
func redacted(raw json.RawMessage) string {
|
||||
var c map[string]any
|
||||
if json.Unmarshal(raw, &c) != nil {
|
||||
return "not JSON"
|
||||
}
|
||||
delete(c, "password")
|
||||
out, _ := json.Marshal(c)
|
||||
return string(out)
|
||||
}
|
||||
@@ -350,6 +350,7 @@ func main() {
|
||||
}
|
||||
env.Vault.HashOwed()
|
||||
env.Vault.MintOwed()
|
||||
env.Vault.LedgersPass()
|
||||
env.Vault.RetirePass()
|
||||
env.Vault.RotationPass()
|
||||
env.Vault.LoginsPass()
|
||||
|
||||
@@ -59,6 +59,8 @@ type minting struct {
|
||||
retiring retiring
|
||||
// mintMu keeps one mint at a time: a want heard, the switch turning on, and issue-account.
|
||||
mintMu sync.Mutex
|
||||
// ledgersMu keeps one rewrite of a machine's `ledgers` at a time (ledgers.go).
|
||||
ledgersMu sync.Mutex
|
||||
}
|
||||
|
||||
// Minted is what one mint did: the account and its secret, never its value or its hash.
|
||||
@@ -169,7 +171,11 @@ func (v *Vault) busAccount(secret, value string) bool {
|
||||
}
|
||||
|
||||
// AccountSecret is the vault's name of the own secret a wanted account is held in.
|
||||
// A ledger account is the exception: it is kept as LedgerAccountSecret, and its machine's `ledgers` is made of it.
|
||||
func AccountSecret(w buscompose.WantedAccount) (string, error) {
|
||||
if w.Kind == buscompose.KindLedger {
|
||||
return LedgerAccountSecret(w.Node, w.Module)
|
||||
}
|
||||
return OwnSecret(w.Node, w.Module, w.OwnSecret())
|
||||
}
|
||||
|
||||
@@ -193,9 +199,16 @@ func (v *Vault) WantHeard(user string, raw []byte, deleted bool) {
|
||||
v.minting.wants[user] = w
|
||||
v.minting.mu.Unlock()
|
||||
v.wantBack(user)
|
||||
if w.Kind == buscompose.KindNodeTools {
|
||||
v.ensureLedgers(w.Node)
|
||||
}
|
||||
if _, err := v.mintWanted(w, "the grants part's want of "+user); err != nil {
|
||||
v.log("%s is not minted: %v", user, err)
|
||||
}
|
||||
if w.Kind == buscompose.KindLedger {
|
||||
// Wanted again, or held already: the machine's ledgers say what is wanted now (nothing said when unchanged).
|
||||
v.rewriteLedgers(w.Node)
|
||||
}
|
||||
}
|
||||
|
||||
// errHeldBack is a want the vault does not mint now, with why: not an error of the vault.
|
||||
@@ -303,7 +316,12 @@ func (v *Vault) mintOnce(w buscompose.WantedAccount, by string) (Minted, error)
|
||||
}
|
||||
}
|
||||
}
|
||||
v.announce(c)
|
||||
if w.Kind == buscompose.KindLedger {
|
||||
// A ledger account is no own secret of its module: its machine's `ledgers` is rewritten and says secret-changed.
|
||||
v.rewriteLedgers(w.Node)
|
||||
} else {
|
||||
v.announce(c)
|
||||
}
|
||||
v.minting.mu.Lock()
|
||||
if v.minting.minted == nil {
|
||||
v.minting.minted = map[string]Minted{}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/seal"
|
||||
@@ -99,6 +100,109 @@ func TestTheVaultMintsAWantedAccountOnceAndNothingItHolds(t *testing.T) {
|
||||
if st := v.MintState(); len(st.HeldBack) != 0 || len(st.Minted) != 1 {
|
||||
t.Errorf("vault_status says %+v: an adopted user is not held back, one account minted", st)
|
||||
}
|
||||
t.Run("ledgers", func(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
run func(t *testing.T, v testVault, accounts *fakeAccounts, priv string)
|
||||
}{
|
||||
// 1966/AC24 (issue 1966 V1): a machine carrying node-tools, with no ledger account, has `ledgers` as {}.
|
||||
{"an empty ledgers on a machine with node-tools", func(t *testing.T, v testVault, accounts *fakeAccounts, priv string) {
|
||||
if _, ok := ledgersOf(t, v, priv); ok {
|
||||
t.Fatal("ledgers is there before node-tools is wanted")
|
||||
}
|
||||
w := nodeToolsWant("ace")
|
||||
v.WantHeard(w.User, wantRaw(t, w), false)
|
||||
before := len(v.bus.events)
|
||||
v.LedgersPass()
|
||||
got, ok := ledgersOf(t, v, priv)
|
||||
if !ok || len(got) != 0 {
|
||||
t.Fatalf("ledgers is %v (held %v), want {}", got, ok)
|
||||
}
|
||||
v.LedgersPass()
|
||||
if n := changesOf(v, before, "own/ace/node-tools/ledgers"); n != 0 {
|
||||
t.Errorf("a second pass said secret-changed %d times for a value that did not change", n)
|
||||
}
|
||||
}},
|
||||
// 1966/AC25 (issue 1966 V2): wanted ledger accounts for m and k on a machine are minted and hashed like every
|
||||
// kind, kept together in the machine's `ledgers`, each mint said once as a change of `ledgers` and never of
|
||||
// the account's own record.
|
||||
{"ledger accounts minted into ledgers", func(t *testing.T, v testVault, accounts *fakeAccounts, priv string) {
|
||||
tools := nodeToolsWant("ace")
|
||||
v.WantHeard(tools.User, wantRaw(t, tools), false)
|
||||
v.LedgersPass()
|
||||
for i, module := range []string{"m", "k"} {
|
||||
w := ledgerWant("ace", module)
|
||||
before := len(v.bus.events)
|
||||
v.WantHeard(w.User, wantRaw(t, w), false)
|
||||
if n := changesOf(v, before, "own/ace/node-tools/ledgers"); n != 1 {
|
||||
t.Errorf("minting %s said secret-changed %d times for ledgers, want once", module, n)
|
||||
}
|
||||
if n := eventsNamed(v, before, ChangedEvent); n != 1 {
|
||||
t.Errorf("minting %s said %d secret-changed in all, want the one of ledgers", module, n)
|
||||
}
|
||||
got, _ := ledgersOf(t, v, priv)
|
||||
if len(got) != i+1 {
|
||||
t.Fatalf("after %s ledgers holds %d credentials, want %d", module, len(got), i+1)
|
||||
}
|
||||
}
|
||||
got, _ := ledgersOf(t, v, priv)
|
||||
for _, module := range []string{"m", "k"} {
|
||||
var cred struct{ User, Password string }
|
||||
if err := json.Unmarshal(got[module], &cred); err != nil || cred.User != "ledger.ace."+module || len(cred.Password) != 40 {
|
||||
t.Fatalf("ledgers[%s] is %s (%v)", module, redacted(got[module]), err)
|
||||
}
|
||||
if !accounts.hashOf(cred.User, cred.Password, 1) {
|
||||
t.Errorf("%s's hash is not in bus-accounts or not of the minted password", cred.User)
|
||||
}
|
||||
if _, held, _ := v.Store.Read("own/ace/" + module + "/ledgers"); held {
|
||||
t.Errorf("the ledger account of %s is also kept as an own secret of its module", module)
|
||||
}
|
||||
}
|
||||
}},
|
||||
// 1966/AC26: m's wanted key deleted: m leaves ledgers at once, its hash is retired within ADR 0230's bounds,
|
||||
// k stays; wanted again, m is back in ledgers.
|
||||
{"a ledger account no longer wanted leaves ledgers", func(t *testing.T, v testVault, accounts *fakeAccounts, priv string) {
|
||||
clock := time.Date(2026, 11, 3, 9, 0, 0, 0, time.UTC)
|
||||
v.Now = func() time.Time { return clock }
|
||||
for _, w := range []buscompose.WantedAccount{nodeToolsWant("ace"), ledgerWant("ace", "m"), ledgerWant("ace", "k")} {
|
||||
v.WantHeard(w.User, wantRaw(t, w), false)
|
||||
}
|
||||
v.WantsRead()
|
||||
before := len(v.bus.events)
|
||||
v.WantHeard("ledger.ace.m", nil, true)
|
||||
got, _ := ledgersOf(t, v, priv)
|
||||
if _, in := got["m"]; in || len(got) != 1 {
|
||||
t.Fatalf("after m's want went, ledgers holds %d credentials, m among them: %v", len(got), in)
|
||||
}
|
||||
if n := changesOf(v, before, "own/ace/node-tools/ledgers"); n != 1 {
|
||||
t.Errorf("m's retirement said secret-changed %d times for ledgers, want once", n)
|
||||
}
|
||||
for i := 0; i < 6; i++ {
|
||||
clock = clock.Add(3 * time.Minute)
|
||||
v.RetirePass()
|
||||
}
|
||||
if _, ok := accounts.keys["ledger.ace.m"]; ok {
|
||||
t.Errorf("m's hash is still in bus-accounts: %+v", v.RetireState())
|
||||
}
|
||||
if _, ok := accounts.keys["ledger.ace.k"]; !ok {
|
||||
t.Error("k's hash was retired with m's")
|
||||
}
|
||||
if got, _ := ledgersOf(t, v, priv); len(got) != 1 {
|
||||
t.Errorf("the retirement changed ledgers to %d credentials", len(got))
|
||||
}
|
||||
w := ledgerWant("ace", "m")
|
||||
v.WantHeard(w.User, wantRaw(t, w), false)
|
||||
if got, _ := ledgersOf(t, v, priv); len(got) != 2 || accounts.keys["ledger.ace.m"].Hash == "" {
|
||||
t.Errorf("wanted again, m is not back in ledgers (%d) with its hash", len(got))
|
||||
}
|
||||
}},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
v, accounts, priv := mintingVault(t)
|
||||
c.run(t, v, accounts, priv)
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// 774/AC8, after research 072's gap "vault mints after the lease": the vault is the one issuer, so a wanted account is
|
||||
|
||||
@@ -292,7 +292,10 @@ func (v *Vault) StateNodeKey(caller, key string) error {
|
||||
}
|
||||
for _, r := range all {
|
||||
parts := strings.Split(r.Name, "/")
|
||||
if len(parts) == 4 && parts[0] == "own" && parts[1] == node && slices.Contains(CoreModules, parts[2]) {
|
||||
// Every own secret of the machine is said changed: which modules carry theirs sealed by the vault is what their
|
||||
// manifests say (core-credentials), read by the node-engine, which composes again and reseals those alone; the vault
|
||||
// keeps no list of modules (issue 774, the operator 2026-11-03).
|
||||
if len(parts) == 4 && parts[0] == "own" && parts[1] == node {
|
||||
v.announce(Changed{Name: r.Name, Generation: r.Generation, At: v.Now().UTC(), By: caller})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ package main
|
||||
|
||||
// Issue 786 (ADR 0372): a machine's node-engine states its own sealing key to the vault, in its own name, at start and
|
||||
// when it changes, whether or not it ever moved a secret in. The caller is the bus user the runtime proved; a changed
|
||||
// key (or a first one) raises `secret-changed` for the machine's core secrets, the same key says nothing.
|
||||
// key (or a first one) raises `secret-changed` for the machine's own secrets, the same key says nothing.
|
||||
|
||||
import (
|
||||
"git.novox.be/novox/mesh-sdk/go/seal"
|
||||
@@ -76,7 +76,8 @@ func TestANodeEngineStatesItsOwnKeyAndAChangeRaisesItsCoreSecrets(t *testing.T)
|
||||
t.Error("a key was recorded for g14 by ace's engine")
|
||||
}
|
||||
got := changedNames(v.bus)[before:]
|
||||
want := []string{"own/ace/mesh-provisioner-client/provisioner-broker", "own/ace/nats/broker"}
|
||||
// Every own secret of the machine: which carry the vault's box their manifests say, read by the node-engine (issue 774).
|
||||
want := []string{"own/ace/mesh-provisioner-client/provisioner-broker", "own/ace/nats/broker", "own/ace/postgres/superuser-password"}
|
||||
if strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Errorf("secret-changed was raised for %v, want %v", got, want)
|
||||
}
|
||||
@@ -85,16 +86,16 @@ func TestANodeEngineStatesItsOwnKeyAndAChangeRaisesItsCoreSecrets(t *testing.T)
|
||||
if _, err := tool.runAs("node.ace", map[string]any{"key": pub}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := len(changedNames(v.bus)); n != before+2 {
|
||||
t.Errorf("the same key raised %d more", n-before-2)
|
||||
if n := len(changedNames(v.bus)); n != before+3 {
|
||||
t.Errorf("the same key raised %d more", n-before-3)
|
||||
}
|
||||
// A different key raises them again.
|
||||
other, _ := aSealingKey(t)
|
||||
if _, err := tool.runAs("node.ace", map[string]any{"key": other}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := len(changedNames(v.bus)); n != before+4 {
|
||||
t.Errorf("a changed key raised %d, want 2", n-before-2)
|
||||
if n := len(changedNames(v.bus)); n != before+6 {
|
||||
t.Errorf("a changed key raised %d, want 3", n-before-3)
|
||||
}
|
||||
if got, _, _ := v.Store.NodeKey("ace"); got != other {
|
||||
t.Error("the changed key was not recorded")
|
||||
|
||||
@@ -10,6 +10,7 @@ package main
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -424,3 +425,72 @@ func hands(raw []byte) bool {
|
||||
h, err := handsOf(raw)
|
||||
return err == nil && h
|
||||
}
|
||||
|
||||
// HandedJudged is how the vault judges one module of a machine for `own`: handed (its own secrets left out, for the
|
||||
// provisioner's client to fill from `get`), and why, as the plan in its assignment says it; or why it could not judge.
|
||||
type HandedJudged struct {
|
||||
Module string `json:"module"`
|
||||
Handed bool `json:"handed"`
|
||||
Why string `json:"why"`
|
||||
}
|
||||
|
||||
// HandedOf is the vault's judgement of each module of node, as `own` would answer it now (issue 1966): every module it
|
||||
// keeps an own secret of there and every module placed there, sorted. Read from the same plans `own` reads, so what it
|
||||
// says is what the next answer leaves out. A node composes by its manifest, its build and the machine, and the plans it
|
||||
// makes are what this reads next: a module named here as not handed for want of a plan is handed by its node's next
|
||||
// composition, where the machine takes it.
|
||||
func (v *Vault) HandedOf(node string) ([]HandedJudged, error) {
|
||||
if !namePart.MatchString(node) {
|
||||
return nil, fmt.Errorf("%q is not a machine's name", node)
|
||||
}
|
||||
all, err := v.Store.ListPrefix("own/" + node + "/")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
modules := map[string]bool{}
|
||||
for _, r := range all {
|
||||
if parts := strings.Split(r.Name, "/"); len(parts) == 4 {
|
||||
modules[parts[2]] = true
|
||||
}
|
||||
}
|
||||
plan := v.plansOf(node)
|
||||
placed, placedErr := plan.placedModules()
|
||||
for _, m := range placed {
|
||||
modules[m] = true
|
||||
}
|
||||
out := make([]HandedJudged, 0, len(modules))
|
||||
for _, module := range slices.Sorted(maps.Keys(modules)) {
|
||||
out = append(out, plan.judged(module))
|
||||
}
|
||||
if placedErr != nil && len(out) == 0 {
|
||||
return nil, fmt.Errorf("the placement of %s could not be read and the vault keeps no own secret there: %w", node, placedErr)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// judged is handed with its why, as HandedOf says it.
|
||||
func (p *planReader) judged(module string) HandedJudged {
|
||||
j := HandedJudged{Module: module}
|
||||
raw, why, err := p.read(module)
|
||||
switch {
|
||||
case why != "":
|
||||
j.Why = "not judged: " + why
|
||||
return j
|
||||
case err != nil:
|
||||
j.Why = fmt.Sprintf("not judged: the assignment %s.%s could not be read: %v", p.node, module, err)
|
||||
return j
|
||||
case raw == nil:
|
||||
j.Why = "no assignment holds a plan of it yet: its own secrets are answered to its node"
|
||||
return j
|
||||
}
|
||||
hands, err := handsOf(raw)
|
||||
switch {
|
||||
case err != nil:
|
||||
j.Why = fmt.Sprintf("not judged: the assignment %s.%s is not the controller's: %v", p.node, module, err)
|
||||
case hands:
|
||||
j.Handed, j.Why = true, "its plan hands it: its own secrets are its provisioner client's, from get"
|
||||
default:
|
||||
j.Why = "its plan hands nothing (none, or provisions alone): its own secrets are answered to its node"
|
||||
}
|
||||
return j
|
||||
}
|
||||
|
||||
@@ -291,4 +291,27 @@ func TestOwnLeavesOutOnlyAModuleWhosePlanHandsIt(t *testing.T) {
|
||||
if want := map[string]bool{"grafana": false, "web": true, "db": true}; !reflect.DeepEqual(got.Handed, want) {
|
||||
t.Errorf("own judged %v handed, want %v", got.Handed, want)
|
||||
}
|
||||
// Issue 1966: the vault's judgement is read by a tool, each module with why; one with no plan yet is not handed, for
|
||||
// want of a plan its node's next composition makes.
|
||||
if _, err := moveIn(t, v, "node.anchor", MoveOf{Node: "anchor", Module: "builder", Name: "token", Origin: "made",
|
||||
Generation: "g-1", SealingKey: pub}, "v-builder"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
answer, err := toolNamed(t, definitions(&Env{Vault: v.Vault}), "own_handed").Run(map[string]any{"node": "anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
judged := map[string]HandedJudged{}
|
||||
for _, j := range answer.(map[string]any)["modules"].([]HandedJudged) {
|
||||
judged[j.Module] = j
|
||||
}
|
||||
for module, want := range map[string][2]string{"builder": {"false", "no assignment holds a plan"},
|
||||
"grafana": {"false", "hands nothing"}, "web": {"true", "its plan hands it"}, "db": {"true", "its plan hands it"}} {
|
||||
if j := judged[module]; fmt.Sprint(j.Handed) != want[0] || !strings.Contains(j.Why, want[1]) {
|
||||
t.Errorf("own_handed says %s is %+v, want handed %s, why saying %q", module, j, want[0], want[1])
|
||||
}
|
||||
}
|
||||
if len(judged) != 4 {
|
||||
t.Errorf("own_handed names %v, want builder, db, grafana and web", judged)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,8 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
)
|
||||
|
||||
// retiring is what the passes keep of the accounts no longer wanted, under minting.mu.
|
||||
@@ -64,9 +66,14 @@ func (v *Vault) WantsRead() {
|
||||
// wantGone is a wanted account's key deleted: forgotten as wanted, its retirement left to the passes.
|
||||
func (v *Vault) wantGone(user string) {
|
||||
v.minting.mu.Lock()
|
||||
defer v.minting.mu.Unlock()
|
||||
gone, was := v.minting.wants[user]
|
||||
delete(v.minting.wants, user)
|
||||
delete(v.minting.heldBack, user)
|
||||
v.minting.mu.Unlock()
|
||||
if was && gone.Kind == buscompose.KindLedger {
|
||||
// Its credential leaves the machine's ledgers at once; its hash is retired by the passes, as every kind's.
|
||||
v.rewriteLedgers(gone.Node)
|
||||
}
|
||||
}
|
||||
|
||||
// wantBack is a wanted account's key written: a retired account of it is re-enabled at once.
|
||||
@@ -221,6 +228,9 @@ func (v *Vault) retireOne(user, why, by string) error {
|
||||
v.minting.mu.Lock()
|
||||
v.minting.retiring.retired = append(v.minting.retiring.retired, user)
|
||||
v.minting.mu.Unlock()
|
||||
if node, _, ok := ledgerAccountOf(a.Of); ok {
|
||||
v.rewriteLedgers(node)
|
||||
}
|
||||
v.log("retired the bus account %s (%s): its hash is out of %s and its value kept, marked; %s", user, a.Of,
|
||||
BusAccountsState, why)
|
||||
return nil
|
||||
|
||||
@@ -399,6 +399,23 @@ func definitions(env *Env) []toolDef {
|
||||
given, _ := a["fingerprint"].(string)
|
||||
return verifyAnswer(env, as, given), nil
|
||||
}},
|
||||
{Name: "own_handed",
|
||||
Description: "Which modules of a machine the vault judges handed to the provisioner, as its verb `own` answers that " +
|
||||
"machine's node-engine now (issue 1966): for every module it keeps an own secret of there and every module " +
|
||||
"placed there, handed or not and why (its plan hands it, hands nothing, no plan yet, or why it could not " +
|
||||
"judge). A handed module's own secrets are left out of `own`, for its client to fill from `get`. Never a value.",
|
||||
Input: map[string]any{"node": str("the machine")},
|
||||
run: func(_ string, a map[string]any) (any, error) {
|
||||
v, err := env.ready()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
judged, err := v.HandedOf(arg(a, "node"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"node": arg(a, "node"), "modules": judged}, nil
|
||||
}},
|
||||
{Name: "secret_handed",
|
||||
Description: "Who was handed which secret by `get`, and when, and who was refused, newest first, as the " +
|
||||
"vault's own store records it. Never a value.",
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-vault
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103174208-b7abe203a929
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
golang.org/x/crypto v0.57.0
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103174208-b7abe203a929 h1:RtAFbUJRP1qO9AIMM2ue+cXNzqj7WZ7IOlSqNvttwuU=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103174208-b7abe203a929/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -173,6 +173,7 @@
|
||||
"secret_holders",
|
||||
"secret_verify",
|
||||
"secret_handed",
|
||||
"own_handed",
|
||||
"secret_own",
|
||||
"account_hash",
|
||||
"account_adopt",
|
||||
|
||||
@@ -32,6 +32,13 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) {
|
||||
} `json:"backup"`
|
||||
} `json:"own"`
|
||||
} `json:"data"`
|
||||
Provides []struct {
|
||||
Name string `json:"name"`
|
||||
Scope string `json:"scope"`
|
||||
Credential struct {
|
||||
Form string `json:"form"`
|
||||
} `json:"credential"`
|
||||
} `json:"provides"`
|
||||
Tools []string `json:"tools"`
|
||||
Emits []string `json:"emits"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
@@ -43,6 +50,16 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) {
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The bus provides "bus-state" beside "mesh-bus" (ADR 0614 §5): a module whose manifest keeps bus state requires it.
|
||||
provided := map[string]string{}
|
||||
for _, p := range m.Provides {
|
||||
provided[p.Name] = p.Scope + "/" + p.Credential.Form
|
||||
}
|
||||
for _, name := range []string{"mesh-bus", "bus-state"} {
|
||||
if provided[name] != "mesh/none" {
|
||||
t.Errorf("provision %q is %q: the bus provides it mesh-wide with no credential", name, provided[name])
|
||||
}
|
||||
}
|
||||
if m.Build.On != nil {
|
||||
t.Error("build.on is declared: nothing is built on a base image any more")
|
||||
}
|
||||
|
||||
+1
-1
@@ -3,7 +3,7 @@ module nats-tools
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8
|
||||
github.com/klauspost/compress v1.20.0
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
|
||||
+2
-2
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"version": "1",
|
||||
"upgrade": {
|
||||
"policy": "record",
|
||||
"why": "the bus: replaced only as a planned step a person starts (`bus upgrade`), its streams snapshotted first and checked after (hq ADR 0236); the controller holds this whatever is said here"
|
||||
"why": "the bus keeps the streams of the whole mesh: each move waits for a person's yes, which the operator's Approve of the release that carries its build gives, and its node backs its streams up before it applies (hq ADR 0236, ADR 0601)"
|
||||
},
|
||||
"provides": [
|
||||
{
|
||||
@@ -12,6 +12,13 @@
|
||||
"credential": {
|
||||
"form": "none"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "bus-state",
|
||||
"scope": "mesh",
|
||||
"credential": {
|
||||
"form": "none"
|
||||
}
|
||||
}
|
||||
],
|
||||
"claims": [
|
||||
@@ -21,6 +28,7 @@
|
||||
}
|
||||
],
|
||||
"bus-users": "/var/lib/nats-module/conf/accounts.conf",
|
||||
"core-credentials": true,
|
||||
"own-secrets": {
|
||||
"broker": "${dir:mesh-state}/broker",
|
||||
"broker-nats-tools": "${dir:tools-secret}/broker-nats-tools"
|
||||
|
||||
Reference in New Issue
Block a user