Compare commits

..
Author SHA1 Message Date
jschoubben 4919a4e076 Probe: the placement files for seed-9 (issue 1126); not to be merged
mesh/delivery closed: the pull request closed unmerged
mesh/check checking 4919a4e07654 on 69056948bfa8
2026-10-28 14:31:28 +01:00
17 changed files with 40 additions and 562 deletions
+6 -8
View File
@@ -16,14 +16,12 @@ credential and an environment built from scratch.
The account's subordinate user and group ranges are not declared anywhere: the node-engine makes every `user` account
with `useradd`, which allocates them. The module's health reads `/etc/subuid` and `/etc/subgid` and is unhealthy
when the module cannot run: without a range for the account, without buildah, when a user namespace cannot be made,
or when the disk is under its floor. A node where `/dev/net/tun` cannot be opened stays **healthy** and is **not
eligible** (buildah makes a RUN line's network with pasta, which needs the tun driver; on 2026-10-28 a build node had
the device node and no driver, so every image build there failed with "Failed to open() /dev/net/tun: No such
device"): the seat's puller judges by a capacity of 0 with that reason (`capacity.<node>`, which `build_usage` shows),
so the asks go to a node that can build them, and its health says "not eligible" with the cause, so a node that
cannot build here now never holds the module's delivery for every node. The kernel module loads `tun` at boot; a look
that passes again wakes the puller.
without a range for the account, without buildah, when a user namespace cannot be made, or when `/dev/net/tun`
cannot be opened (buildah makes a RUN line's network with pasta, which needs the tun driver; on 2026-10-28 a build node had the
device node and no driver, so every image build there failed with "Failed to open() /dev/net/tun: No such device"). The
kernel module loads `tun` at boot. Whatever the health refuses, the node is **not eligible**: the seat's puller judges
by a capacity of 0 with that reason (`capacity.<node>`, which `build_usage` shows), so the asks go to a node that can
build them; a look that passes again wakes the puller.
The Dockerfile's steps may use the network (ADR 0487 §6); nothing a step can read holds a key.
@@ -2,7 +2,6 @@ package main
import (
"context"
"errors"
"fmt"
"net/http"
"sync"
@@ -98,25 +97,11 @@ func (m *Module) look(ctx context.Context) (Look, error) {
return l, &buildstep.Refusal{Text: err.Error(), Have: l.Buildah}
}
if err := m.tunDevice(); err != nil {
return l, &notEligible{&buildstep.Refusal{Text: err.Error(), Have: l.Buildah}}
return l, &buildstep.Refusal{Text: err.Error(), Have: l.Buildah}
}
return l, nil
}
// notEligible is a refusal of a machine that runs the module fine but cannot build here now (no tun device: its asks go
// to another node). It is a *buildstep.Refusal for a step, and the health stays healthy for it (issue 1126): the
// assigner reads unhealthy as a failed delivery and holds the build for every node. What cannot run (buildah missing,
// no ranges, no namespace, no runtime directory, no room) stays unhealthy.
type notEligible struct{ *buildstep.Refusal }
func (e *notEligible) Unwrap() error { return e.Refusal }
// isNotEligible says whether err is a refusal that leaves the module healthy.
func isNotEligible(err error) bool {
var n *notEligible
return errors.As(err, &n)
}
// Check is the guard: nil when the machine can build, else the refusal. Every look is kept for Eligible, and a change
// wakes the puller, so a node that can build again takes asks at once.
func (m *Module) Check(ctx context.Context) error {
@@ -135,13 +120,6 @@ func (m *Module) keep(err error) {
changed := m.last != why
m.last = why
m.lastMu.Unlock()
if changed && m.Say != nil {
if why == "" {
m.Say("this machine is eligible to build images again")
} else {
m.Say("this machine takes no image build now: %s", why)
}
}
if changed && m.Held != nil {
m.Held.Wake()
}
@@ -177,16 +155,12 @@ func (m *Module) Start(ctx context.Context) {
func (m *Module) Health(ctx context.Context) health.Answer {
l, err := m.look(ctx)
m.keep(err)
if err != nil && !isNotEligible(err) {
if err != nil {
return health.Unhealthy(err.Error())
}
if err := m.room(); err != nil {
return health.Unhealthy(err.Error())
}
if err != nil {
// Not eligible: the module runs, it takes no image ask here now, and says why; the capacity record says it too.
return health.Healthy("not eligible to build images, so it takes no image ask: " + err.Error())
}
return health.Healthy(fmt.Sprintf("buildah %s installed; %s holds %d subordinate uids and %d gids; a user namespace can be made; %s opens", l.Buildah, l.Account, l.UIDs, l.GIDs, m.tunPath()))
}
@@ -194,7 +168,7 @@ func (m *Module) Health(ctx context.Context) health.Answer {
func (m *Module) Status(ctx context.Context) map[string]any {
l, err := m.look(ctx)
out := map[string]any{"node": m.Node, "account": m.Account, "buildah": l.Buildah, "subuids": l.UIDs, "subgids": l.GIDs, "refusing": err != nil,
"eligible": err == nil, "cpu": m.cpuStatus()}
"cpu": m.cpuStatus()}
if err != nil {
out["why"] = err.Error()
}
@@ -2,9 +2,7 @@ package main
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
@@ -56,6 +54,7 @@ func TestHealthIsUnhealthyWithoutWhatABuildNeeds(t *testing.T) {
}{
{"subuid", func(m *Module, p string) { m.SubUIDFile = p }, []string{"/subuid", "mesh-build-image"}},
{"subgid", func(m *Module, p string) { m.SubGIDFile = p }, []string{"/subgid", "mesh-build-image"}},
{"tun", func(m *Module, p string) { m.Tun = p + "/dev/net/tun" }, []string{"not eligible: ", "/dev/net/tun cannot be opened", "tun"}},
} {
f := newFake(t)
m := moduleWith(t, f, newStates())
@@ -75,61 +74,6 @@ func TestHealthIsUnhealthyWithoutWhatABuildNeeds(t *testing.T) {
}
}
// 1126: a node that cannot build here now (no tun device) stays healthy, saying "not eligible" and why; the assigner reads
// unhealthy as a failed delivery and held the new build for every node. The eligibility, the status and the log say it too.
func TestANodeWithoutTunIsHealthyAndSaysItIsNotEligible(t *testing.T) {
f := newFake(t)
m := moduleWith(t, f, newStates())
var said []string
m.Say = func(format string, a ...any) { said = append(said, fmt.Sprintf(format, a...)) }
m.Tun = t.TempDir() + "/dev/net/tun"
a := m.Health(context.Background())
if a.State != health.StateHealthy || !a.Healthy {
t.Fatalf("health = %+v, want healthy", a)
}
for _, want := range []string{"not eligible", "/dev/net/tun cannot be opened", "tun"} {
if !strings.Contains(a.Why, want) {
t.Errorf("health says %q, lacks %q", a.Why, want)
}
}
if got := m.Eligible(); !strings.Contains(got, "not eligible: ") {
t.Errorf("eligible says %q", got)
}
if s := m.Status(context.Background()); s["eligible"] != false || s["refusing"] != true {
t.Errorf("status = %v, want not eligible", s)
}
// One log line for the change, none for the same look again; one when it can build again.
m.Health(context.Background())
if len(said) != 1 || !strings.Contains(said[0], "takes no image build now") {
t.Errorf("said = %q, want one line for the change", said)
}
m.Tun = filepath.Join(t.TempDir(), "tun")
os.WriteFile(m.Tun, nil, 0o644)
if a := m.Health(context.Background()); !a.Healthy || strings.Contains(a.Why, "not eligible") {
t.Errorf("health = %+v, want plain healthy", a)
}
if len(said) != 2 || !strings.Contains(said[1], "eligible to build images again") {
t.Errorf("said = %q, want the return said", said)
}
// A step still ends refused with the cause named.
m.Tun = t.TempDir() + "/none"
if err := m.Check(context.Background()); err == nil || !strings.Contains(err.Error(), "not eligible") {
t.Errorf("check = %v", err)
}
}
// 1126: what cannot run stays unhealthy even beside a missing tun: buildah missing is not "not eligible".
func TestWithoutBuildahTheNodeIsUnhealthyNotIneligible(t *testing.T) {
m := moduleWith(t, newFake(t), newStates())
m.Tun = t.TempDir() + "/none"
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
return forgekey.Ran{Status: 127, Err: "exec: \"buildah\": executable file not found in $PATH"}
}
if a := m.Health(context.Background()); a.Healthy {
t.Errorf("health = %+v, want unhealthy", a)
}
}
func TestHealthIsUnhealthyWhenANamespaceCannotBeMade(t *testing.T) {
f := newFake(t)
m := moduleWith(t, f, newStates())
@@ -54,7 +54,7 @@ func (m *Module) Tools(steps []StepTool) []stdio.Tool {
Run: func(map[string]any) (any, error) { return m.Status(m.Ctx), nil }},
stdio.Tool{Name: HealthTool,
Description: "How the image build module is: unhealthy while buildah cannot be run, while its account holds no " +
"subordinate user or group range, or while a user namespace cannot be made, saying which; healthy otherwise. A machine that cannot build here now (no tun device) stays healthy and says in its words \"not eligible\", with the cause; it takes no image ask meanwhile (issue 1126). Read-only.",
"subordinate user or group range, or while a user namespace cannot be made, saying which; healthy otherwise. Read-only.",
Run: func(map[string]any) (any, error) { return m.Health(m.Ctx), nil }},
)
}
@@ -4,9 +4,6 @@ import (
"encoding/json"
"fmt"
"strings"
"sync"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// Who may ask `get` for which secret (novox/hq ADR 0349 §11, design 57 V4, issue 647).
@@ -111,9 +108,6 @@ func fieldSet(f json.RawMessage) bool {
// that the module's plan there names.
type Declared struct {
Read AssignmentReader
// Confirm, when set, reads an assignment directly (over the bus): a module Read does not hold is confirmed through it
// before a credential is refused, since a watched mirror can be behind (issue 1126, review round 1).
Confirm AssignmentReader
// Store is the vault's store, whose record of who holds a provision's one credential (pair.go) the rule reads.
Store *Store
}
@@ -213,77 +207,3 @@ func (d Declared) May(caller, name string) (bool, string) {
// bus error rather than an answer. The vault says "holds no secret named ... yet", which the client waits on and asks
// again, and records no refusal.
const notYetPrefix = "not yet: "
// WatchedAssignments is the assignments mirror as the vault's watch of it hands it over (issue 1126, 2026-10-28: novox's
// `own` read one assignment over the bus per own secret and per credential, 203 reads, and answered after 26 s, past
// the node-engine's 15 s). The state is the bus's; the vault watches it, never polls it, and reads it from memory once the
// watch has handed every current value. Until then, and when the watch could not be begun, a read goes to the bus
// (Fallback), as before.
type WatchedAssignments struct {
mu sync.Mutex
held map[string][]byte
caught bool
attempt uint64
Fallback AssignmentReader
}
// WatchAttempt is one attempt at the watch: what it hands over counts only while it is the latest attempt, so a watch
// that failed partway and is begun again keeps nothing it took (a key deleted meanwhile is never handed again), and a
// late delivery to an older attempt changes nothing (issue 1126, review round 1).
type WatchAttempt struct {
w *WatchedAssignments
n uint64
}
// Begin starts a new attempt from nothing held: reads go to the bus until it has caught up.
func (w *WatchedAssignments) Begin() WatchAttempt {
w.mu.Lock()
defer w.mu.Unlock()
w.attempt++
w.held, w.caught = map[string][]byte{}, false
return WatchAttempt{w: w, n: w.attempt}
}
// Take keeps one change the attempt handed over, a value put or a key deleted, while it is the latest attempt.
func (a WatchAttempt) Take(c stdio.StateChange) error {
w := a.w
w.mu.Lock()
defer w.mu.Unlock()
if a.n != w.attempt {
return nil
}
if c.Op == "delete" || c.Op == "purge" || len(c.Value) == 0 {
delete(w.held, c.Key)
return nil
}
w.held[c.Key] = append([]byte(nil), c.Value...)
return nil
}
// Caught says the attempt handed every current value: reads are answered from memory from now on, if it is the latest.
func (a WatchAttempt) Caught() {
w := a.w
w.mu.Lock()
defer w.mu.Unlock()
if a.n == w.attempt {
w.caught = true
}
}
// Read is one value of the mirror, `<node>.<module>`: nil when there is none.
func (w *WatchedAssignments) Read(key string) ([]byte, error) {
w.mu.Lock()
if w.caught {
raw, ok := w.held[key]
w.mu.Unlock()
if !ok {
return nil, nil
}
return append([]byte(nil), raw...), nil
}
w.mu.Unlock()
if w.Fallback == nil {
return nil, fmt.Errorf("the assignments mirror is not watched yet, and nothing reads it from the bus")
}
return w.Fallback(key)
}
+2 -29
View File
@@ -49,37 +49,13 @@ func build() *Env {
}
// Who may ask (askers.go): a node's provisioner client, for what its module's plan there names.
// Where it sees a consumer bind a provider (pair.go): the same mirror, every key.
// The assignments as the vault's watch keeps them (watchAssignments), read over the bus until it has caught up.
assigned = &WatchedAssignments{Fallback: readAssignment}
env.Vault = &Vault{Store: store, Own: own, Askers: Declared{Read: assigned.Read, Confirm: readAssignment, Store: store},
env.Vault = &Vault{Store: store, Own: own, Askers: Declared{Read: readAssignment, Store: store},
Assignments: Mirror{Keys: assignmentKeys, Read: readAssignment}, Emit: emit, Ask: stdio.Ask,
Now: time.Now, Log: say, Accounts: busAccounts{}, Haves: readHave,
HaveKeys: func() ([]string, error) { return stdio.State(HaveRead).Keys() }}
return env
}
// assigned is the assignments mirror as the vault's watch keeps it.
var assigned *WatchedAssignments
// watchAssignments begins the watch of the assignments mirror the vault's rules read (issue 1126, 2026-10-28): a
// watch, never a poll, so `own` and `get` read no assignment over the bus once it has caught up. It answers whether the
// watch began; one that did not is begun again at the next minute pass, and reads go to the bus meanwhile.
func watchAssignments() bool {
if assigned == nil {
return true
}
// Each attempt begins from nothing held: one that failed partway leaves nothing a later one would not hand again.
attempt := assigned.Begin()
if err := stdio.State(AssignmentsRead).Watch("", attempt.Take); err != nil {
assigned.Begin() // what the failed attempt took counts for nothing, and reads go to the bus
say("the assignments are not watched, so each is read over the bus; begun again at the next pass: %v", err)
return false
}
attempt.Caught()
say("the assignments are watched: the vault's rules read them from memory")
return true
}
// readHave is one key of the node-engines' have bucket, read through the runtime under the vault's own account (its
// manifest `reads` it), or nil when there is none.
func readHave(key string) ([]byte, error) {
@@ -173,14 +149,11 @@ func main() {
written, failed := env.Vault.HashHeld()
say("hashed %d bus accounts at start; %d could not be, tried again every minute: %v", len(written), len(failed), failed)
}
watched, watching := watchHandings(env), watchAssignments()
watched := watchHandings(env)
for {
if !watched {
watched = watchHandings(env)
}
if !watching {
watching = watchAssignments()
}
env.pass()
if env.Vault != nil {
env.Vault.HashOwed()
+15 -74
View File
@@ -70,7 +70,7 @@ func credentialNames(c CredentialAsked) []string {
// credentialRefusal says why node may not have c, or "": a consumer's credential to the machine its consuming module
// stands on, a provider's to the providing machine, and nothing to a machine that is neither.
func (v *Vault) credentialRefusal(node string, c CredentialAsked, plan *planReader) string {
func (v *Vault) credentialRefusal(node string, c CredentialAsked) string {
for _, part := range []string{c.Provision, c.Consumer, c.ConsumerModule, c.Provider} {
if !namePart.MatchString(part) {
return fmt.Sprintf("%q is not a name the mesh writes", part)
@@ -85,19 +85,14 @@ func (v *Vault) credentialRefusal(node string, c CredentialAsked, plan *planRead
return fmt.Sprintf("%s asks as a consumer on %s, and is not that machine", node, c.Consumer)
}
// The consuming module stands on the machine: read, never assumed (an assignment that cannot be read refuses).
raw, why, err := plan.read(c.ConsumerModule)
read, why := v.assignmentReader(node)
if why != "" {
return why
}
raw, err := read(node + "." + c.ConsumerModule)
if err != nil {
return fmt.Sprintf(notYetPrefix+"the assignment %s.%s could not be read: %v", node, c.ConsumerModule, err)
}
if raw == nil {
// Confirmed directly before refusing: the mirror may not hold a module assigned a moment ago.
if raw, err = plan.confirmed(c.ConsumerModule); err != nil {
return fmt.Sprintf(notYetPrefix+"the assignment %s.%s could not be read: %v", node, c.ConsumerModule, err)
}
}
if raw == nil {
return fmt.Sprintf("%s is not on %s: neither assigned there nor required by a module there", c.ConsumerModule, node)
}
@@ -111,56 +106,6 @@ func (v *Vault) credentialRefusal(node string, c CredentialAsked, plan *planRead
return fmt.Sprintf("the role %q is not consumer or provider", c.Role)
}
// planReader reads the assignments of one machine's modules for one answer of `own`, each module's once (issue 1126,
// 2026-10-28: one read per own secret and per credential, 203 on novox, held the answer past the node-engine's bound).
type planReader struct {
node string
reader AssignmentReader
confirm AssignmentReader
why string
got map[string]planRead
}
type planRead struct {
raw []byte
err error
}
func (v *Vault) plansOf(node string) *planReader {
read, why := v.assignmentReader(node)
var confirm AssignmentReader
if d, ok := v.Askers.(Declared); ok && d.Confirm != nil {
confirm = standing(d.Confirm)
}
return &planReader{node: node, confirm: confirm, reader: read, why: why, got: map[string]planRead{}}
}
// read is module's assignment on the machine, nil when there is none, or why the vault reads none.
func (p *planReader) read(module string) ([]byte, string, error) {
if p.why != "" {
return nil, p.why, nil
}
r, ok := p.got[module]
if !ok {
r.raw, r.err = p.reader(p.node + "." + module)
p.got[module] = r
}
return r.raw, "", r.err
}
// confirmed is module's assignment read directly, for one the reader does not hold (a watched mirror can be behind),
// nil when there is none or nothing reads directly.
func (p *planReader) confirmed(module string) ([]byte, error) {
if p.confirm == nil {
return nil, nil
}
raw, err := p.confirm(p.node + "." + module)
if err == nil && raw != nil {
p.got[module] = planRead{raw: raw}
}
return raw, err
}
// assignmentReader is the vault's reader of the assignments, or why it has none: the rule `own` keeps reads them,
// and without them it refuses rather than hands (a rule that cannot be read is no rule).
func (v *Vault) assignmentReader(node string) (AssignmentReader, string) {
@@ -232,21 +177,17 @@ func (v *Vault) OwnOfNode(caller string, asked ...CredentialAsked) (OwnAnswer, e
if err != nil {
return OwnAnswer{}, fmt.Errorf("%s's sealing key as recorded is not usable: %w", node, err)
}
// Only this machine's own records are read, each module's assignment once, and every hand-out recorded in one synced
// write after all are sealed: none is handed when that record cannot be written (issue 1126, 2026-10-28).
all, err := v.Store.ListPrefix("own/" + node + "/")
all, err := v.Store.List()
if err != nil {
return OwnAnswer{}, err
}
plan := v.plansOf(node)
var handings []Handing
out := OwnAnswer{Node: node, Key: key}
for _, r := range all {
parts := strings.Split(r.Name, "/")
if len(parts) != 4 || parts[0] != "own" || parts[1] != node {
continue
}
handed, err := plan.handed(parts[2])
handed, err := v.handed(node, parts[2])
if err != nil {
return OwnAnswer{}, err
}
@@ -261,7 +202,9 @@ func (v *Vault) OwnOfNode(caller string, asked ...CredentialAsked) (OwnAnswer, e
if err != nil {
return OwnAnswer{}, err
}
handings = append(handings, Handing{To: caller, Name: r.Name, Generation: r.Generation, Outcome: "handed"})
if err := v.recordHanding(Handing{To: caller, Name: r.Name, Generation: r.Generation, Outcome: "handed"}); err != nil {
return OwnAnswer{}, fmt.Errorf("%s is not handed out: who was handed it could not be recorded: %w", r.Name, err)
}
out.Own = append(out.Own, NodeOwnSecret{Module: parts[2], Name: parts[3], Origin: composeOrigin(r.Origin),
Generation: r.Generation, Sealed: sealed})
}
@@ -272,7 +215,7 @@ func (v *Vault) OwnOfNode(caller string, asked ...CredentialAsked) (OwnAnswer, e
return out.Own[i].Name < out.Own[j].Name
})
for _, c := range asked {
if why := v.credentialRefusal(node, c, plan); why != "" {
if why := v.credentialRefusal(node, c); why != "" {
return OwnAnswer{}, fmt.Errorf("%s may not ask for the credential of %s from %s to %s on %s: %s", caller,
c.Provision, c.Provider, c.ConsumerModule, c.Consumer, why)
}
@@ -305,7 +248,9 @@ func (v *Vault) OwnOfNode(caller string, asked ...CredentialAsked) (OwnAnswer, e
if err != nil {
return OwnAnswer{}, err
}
handings = append(handings, Handing{To: caller, Name: name, Generation: r.Generation, Outcome: "handed"})
if err := v.recordHanding(Handing{To: caller, Name: name, Generation: r.Generation, Outcome: "handed"}); err != nil {
return OwnAnswer{}, fmt.Errorf("%s is not handed out: who was handed it could not be recorded: %w", name, err)
}
out.Credentials = append(out.Credentials, NodeCredential{CredentialAsked: c, Name: name, Generation: r.Generation, Sealed: sealed})
found = true
break
@@ -314,10 +259,6 @@ func (v *Vault) OwnOfNode(caller string, asked ...CredentialAsked) (OwnAnswer, e
out.Missing = append(out.Missing, c)
}
}
if err := v.recordHandings(handings); err != nil {
return OwnAnswer{}, fmt.Errorf("nothing is handed out to %s: who was handed its %d values could not be recorded: %w",
caller, len(handings), err)
}
v.log("handed %s's node-engine the %d own secrets of its modules and %d credentials (%d not held), sealed to its key",
node, len(out.Own), len(out.Credentials), len(out.Missing))
return out, nil
@@ -326,12 +267,12 @@ func (v *Vault) OwnOfNode(caller string, asked ...CredentialAsked) (OwnAnswer, e
// handed says the plan of module on node hands it to the provisioner, so its secrets are filled from `get`, not composed.
// A plan that cannot be read is an error, never "not handed": `own` refuses rather than hands a secret the provisioner
// holds.
func (p *planReader) handed(module string) (bool, error) {
node := p.node
raw, why, err := p.read(module)
func (v *Vault) handed(node, module string) (bool, error) {
read, why := v.assignmentReader(node)
if why != "" {
return false, errors.New(why)
}
raw, err := read(node + "." + module)
if err != nil {
return false, fmt.Errorf(notYetPrefix+"the assignment %s.%s could not be read, so whether it is handed to the "+
"provisioner is not known: %w", node, module, err)
@@ -2,12 +2,9 @@ package main
import (
"errors"
"os"
"path/filepath"
"strings"
"testing"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/seal"
)
@@ -122,117 +119,3 @@ func TestOwnAnswersAMachineItsModulesSecretsSealedToItsKey(t *testing.T) {
t.Errorf("own with no reader of the assignments: %v", err)
}
}
// Issue 1126, 2026-10-28 (seed-9's probe: novox's 57 own secrets and 146 credentials took 26 s, past the node-engine's
// 15 s): one answer of `own` reads each module's assignment once, and records every hand-out in one synced write.
func TestOwnReadsEachAssignmentOnceAndSyncsItsRecordOnce(t *testing.T) {
plans := assignments{"anchor.web": `{"node":"anchor","module":"web"}`, "anchor.db": `{"node":"anchor","module":"db"}`}
reads := map[string]int{}
v := newVault(t, nil)
v.Askers = Declared{Read: func(key string) ([]byte, error) { reads[key]++; return plans.read(key) }, Store: v.Store}
pub, _ := aSealingKey(t)
for _, m := range []MoveOf{
{Node: "anchor", Module: "web", Name: "password", Origin: "made", Generation: "g-1", SealingKey: pub},
{Node: "anchor", Module: "web", Name: "api-key", Origin: "made", Generation: "g-1", SealingKey: pub},
{Node: "anchor", Module: "web", Name: "cookie", Origin: "made", Generation: "g-1", SealingKey: pub},
{Node: "anchor", Module: "db", Name: "root", Origin: "made", Generation: "g-1", SealingKey: pub},
{Provider: "anchor", Provision: "postgres", Consumer: "anchor", ConsumerModule: "web", Origin: "made", Generation: "g-1", SealingKey: pub},
{Provider: "anchor", Provision: "redis", Consumer: "anchor", ConsumerModule: "web", Origin: "made", Generation: "g-1", SealingKey: pub},
} {
if _, err := moveIn(t, v, "node.anchor", m, "v-"+m.Name+m.Provision); err != nil {
t.Fatal(err)
}
}
for k := range reads {
delete(reads, k)
}
syncs, handed := v.Store.syncs, len(mustHanded(t, v))
got, err := v.OwnOfNode("node.anchor",
CredentialAsked{Role: "consumer", Provision: "postgres", Consumer: "anchor", ConsumerModule: "web", Provider: "anchor"},
CredentialAsked{Role: "consumer", Provision: "redis", Consumer: "anchor", ConsumerModule: "web", Provider: "anchor"})
if err != nil || len(got.Own) != 4 || len(got.Credentials) != 2 {
t.Fatalf("anchor was answered %+v, %v", got, err)
}
for key, n := range reads {
if n != 1 {
t.Errorf("the assignment %s was read %d times in one answer", key, n)
}
}
if len(reads) != 2 {
t.Errorf("the answer read %v, want each of anchor's two modules once", reads)
}
if n := v.Store.syncs - syncs; n != 1 {
t.Errorf("one answer synced the record %d times", n)
}
if n := len(mustHanded(t, v)) - handed; n != 6 {
t.Errorf("one answer recorded %d hand-outs, want 6", n)
}
// Review round 1, note (b): a consumer module the mirror does not hold yet (its watch behind) is read directly before
// the credential is refused.
v.Askers = Declared{Read: func(string) ([]byte, error) { return nil, nil }, Confirm: plans.read, Store: v.Store}
if got, err := v.OwnOfNode("node.anchor", CredentialAsked{Role: "consumer", Provision: "postgres", Consumer: "anchor",
ConsumerModule: "web", Provider: "anchor"}); err != nil || len(got.Credentials) != 1 {
t.Errorf("a module the mirror does not hold yet, read directly: %+v, %v", got, err)
}
v.Askers = Declared{Read: plans.read, Store: v.Store}
// A record that cannot be written hands nothing, the batch as one.
if err := os.Remove(filepath.Join(v.dir, "store", "handed.jsonl")); err != nil {
t.Fatal(err)
}
if err := os.Mkdir(filepath.Join(v.dir, "store", "handed.jsonl"), 0o700); err != nil {
t.Fatal(err)
}
if a, err := v.OwnOfNode("node.anchor"); err == nil {
t.Errorf("handed with no record of it: %+v", a)
}
}
func mustHanded(t *testing.T, v testVault) []Handing {
t.Helper()
h, err := v.Store.Handed(1000)
if err != nil {
t.Fatal(err)
}
return h
}
// Issue 1126, 2026-10-28: the vault keeps the assignments mirror as its watch hands it over, so `own` and `get` read
// no assignment over the bus once the watch has caught up; before that a read goes to the bus.
func TestTheWatchedAssignmentsAnswerWithoutTheBusOnceCaughtUp(t *testing.T) {
asked := 0
w := &WatchedAssignments{Fallback: func(key string) ([]byte, error) { asked++; return []byte(`{"from":"bus"}`), nil }}
if raw, err := w.Read("anchor.web"); err != nil || string(raw) != `{"from":"bus"}` || asked != 1 {
t.Errorf("before the watch caught up: %s, %v, %d asks", raw, err, asked)
}
// A first watch takes two values and fails before it caught up; a late value still arrives on it.
first := w.Begin()
for _, c := range []stdio.StateChange{
{Key: "anchor.web", Op: "put", Value: []byte(`{"node":"anchor","module":"web"}`), Current: true},
{Key: "anchor.db", Op: "put", Value: []byte(`{"node":"anchor","module":"db"}`), Current: true},
} {
if err := first.Take(c); err != nil {
t.Fatal(err)
}
}
// Review round 1: the watch begun again holds anchor.db no more (removed meanwhile); what the first took is gone.
second := w.Begin()
_ = second.Take(stdio.StateChange{Key: "anchor.web", Op: "put", Value: []byte(`{"node":"anchor","module":"web"}`), Current: true})
_ = first.Take(stdio.StateChange{Key: "anchor.db", Op: "put", Value: []byte(`{"node":"anchor","module":"db"}`)})
first.Caught()
if raw, _ := w.Read("anchor.db"); string(raw) != `{"from":"bus"}` {
t.Errorf("the failed watch's caught up counted: %s", raw)
}
second.Caught()
_ = first.Take(stdio.StateChange{Key: "anchor.db", Op: "put", Value: []byte(`{"node":"anchor","module":"db"}`)})
_ = second.Take(stdio.StateChange{Key: "anchor.web", Op: "put", Value: []byte(`{"node":"anchor","module":"web","v":2}`)})
asked = 0
for key, want := range map[string]string{"anchor.web": `{"node":"anchor","module":"web","v":2}`, "anchor.db": "", "laptop.x": ""} {
raw, err := w.Read(key)
if err != nil || string(raw) != want || (want == "") != (raw == nil) {
t.Errorf("%s read %q, %v; want %q", key, raw, err, want)
}
}
if asked != 0 {
t.Errorf("caught up, the mirror asked the bus %d times", asked)
}
}
+7 -27
View File
@@ -81,8 +81,6 @@ type Store struct {
handedMu sync.Mutex
rolled int
// syncs counts the record's synced appends since the vault started: how a test proves one sync per answer.
syncs int
// asksMu serialises the record of asks, so two asks for one secret at once do not both pass its bounds.
asksMu sync.Mutex
}
@@ -175,37 +173,22 @@ type Handing struct {
// RecordHanding appends one handing to the store's record, synced before it returns. It says whether the
// record was rolled to make room.
func (s *Store) RecordHanding(h Handing) (rolled bool, err error) {
return s.RecordHandings([]Handing{h})
}
// RecordHandings appends every handing of one answer to the store's record in one write, synced once before it returns
// (issue 1126, 2026-10-28: one sync per secret, under the one lock every node's answers share, held `own` past the
// node-engine's bound). It says whether the record was rolled to make room; nothing is written when one cannot be.
func (s *Store) RecordHandings(hs []Handing) (rolled bool, err error) {
if len(hs) == 0 {
return false, nil
}
var raw []byte
for _, h := range hs {
line, err := json.Marshal(h)
if err != nil {
return false, err
}
raw = append(append(raw, line...), '\n')
raw, err := json.Marshal(h)
if err != nil {
return false, err
}
s.handedMu.Lock()
defer s.handedMu.Unlock()
path := filepath.Join(s.dir, "handed.jsonl")
max := s.HandedBound()
if info, err := os.Stat(path); err == nil && info.Mode().IsRegular() && info.Size()+int64(len(raw)) > max {
if info, err := os.Stat(path); err == nil && info.Mode().IsRegular() && info.Size()+int64(len(raw))+1 > max {
if err := s.roll(path); err != nil {
return false, err
}
s.rolled++
rolled = true
}
s.syncs++
return rolled, appendSynced(path, raw)
return rolled, appendSynced(path, append(raw, '\n'))
}
// roll moves handed.jsonl to .1, .1 to .2 and so on, dropping what falls past HandedKept.
@@ -283,10 +266,7 @@ func (s *Store) HandedBound() int64 {
}
// List is every record, by name.
func (s *Store) List() ([]Record, error) { return s.ListPrefix("") }
// ListPrefix is every record whose name begins with prefix, by name: only those files are read.
func (s *Store) ListPrefix(prefix string) ([]Record, error) {
func (s *Store) List() ([]Record, error) {
entries, err := os.ReadDir(filepath.Join(s.dir, "secrets"))
if err != nil {
return nil, err
@@ -297,7 +277,7 @@ func (s *Store) ListPrefix(prefix string) ([]Record, error) {
continue
}
name, err := url.PathUnescape(strings.TrimSuffix(e.Name(), ".json"))
if err != nil || !strings.HasPrefix(name, prefix) {
if err != nil {
continue
}
r, ok, err := s.Read(name)
+4 -17
View File
@@ -235,24 +235,11 @@ func (v *Vault) announce(c Changed) {
// recordHanding keeps who asked for what, and what they got, in the vault's own store, and says loudly
// when the record had to be rolled to make room.
func (v *Vault) recordHanding(h Handing) error { return v.recordHandings([]Handing{h}) }
// recordHandings keeps every handing of one answer in one synced write, as recordHanding keeps one.
func (v *Vault) recordHandings(hs []Handing) error {
if len(hs) == 0 {
return nil
}
at := v.Now().UTC()
for i := range hs {
hs[i].At = at
}
rolled, err := v.Store.RecordHandings(hs)
func (v *Vault) recordHanding(h Handing) error {
h.At = v.Now().UTC()
rolled, err := v.Store.RecordHanding(h)
if err != nil {
names := make([]string, len(hs))
for i, h := range hs {
names[i] = h.Name
}
v.log("the record of %s asking for %s could not be written: %v", hs[0].To, strings.Join(names, ", "), err)
v.log("the record of %s asking for %s could not be written: %v", h.To, h.Name, err)
return err
}
if rolled {
@@ -167,10 +167,8 @@ func TestNoBucketHoldsASecret(t *testing.T) {
code = []byte(strings.Replace(string(code), "stdio.State(BusAccountsState).Put(", "", 1))
code = []byte(strings.Replace(string(code), "stdio.State(BusAccountsState).Keys(", "", 1))
}
// The haves, in main.go alone: one read, one watch and their keys (own_drift), never a write; and one watch of
// the assignments, which the vault's rules read from memory once it caught up (issue 1126).
// The haves, in main.go alone: one read, one watch and their keys (own_drift), never a write.
if f == "main.go" {
code = []byte(strings.Replace(string(code), "stdio.State(AssignmentsRead).Watch(", "", 1))
code = []byte(strings.Replace(string(code), "stdio.State(HaveRead).Get(", "", 1))
code = []byte(strings.Replace(string(code), "stdio.State(HaveRead).Watch(", "", 1))
code = []byte(strings.Replace(string(code), "stdio.State(HaveRead).Keys(", "", 1))
-4
View File
@@ -14,7 +14,6 @@ states on the bus.
| `/usr/local/lib/mesh-power/bin/power-moment` | the runner: each module's piece on its own, as root, with `sh`, bounded (`MESH_POWER_BOUND`, default 30 s); every outcome in the journal under `mesh-power` |
| five units | `mesh-power-after-boot`, `-before-shutdown` (its stop is the shutdown, while the network is still up), `-before-sleep`, `-after-wake`, `-supply` |
| `mesh-power-draw` (process) | samples the power drawn for the module's bundle, which puts it on the bus as `power.draw` (below) |
| `reboot.target.d`, `poweroff.target.d`, `halt.target.d` (`mesh-power.conf`) | a shutdown is bounded at ten minutes, not systemd's thirty; the target keeps its own forcing action (issue 1126: on ace's reboot of 2026-10-28 the bus was started again during the shutdown, nothing stopped it a second time, and the reboot waited 29 minutes) |
| `/etc/udev/rules.d/90-mesh-power.rules` | a power supply's change starts `mesh-power-supply`, which runs `on-mains` or `on-battery` once per change of source |
## The moments, and how a module adds code
@@ -147,7 +146,4 @@ Each directory or archive this module declares says how it treats a file it did
- `/etc/systemd/system/hibernate.target.d`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
- `/etc/systemd/system/hybrid-sleep.target.d`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
- `/etc/systemd/system/suspend-then-hibernate.target.d`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
- `/etc/systemd/system/reboot.target.d`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
- `/etc/systemd/system/poweroff.target.d`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
- `/etc/systemd/system/halt.target.d`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
- `/run/mesh-power`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
-50
View File
@@ -127,53 +127,3 @@ func TestTheBarBlocksAreTwoItemsOfTheBarsCollection(t *testing.T) {
}
}
}
// A shutdown is bounded at ten minutes, not systemd's thirty (issue 1126, ace's reboot of 2026-10-28): every unit had
// stopped within 17 s, yet the bus and the authorization manager were started again during the shutdown, nothing
// stopped them a second time, and the reboot waited on its job's own timeout of 30 min with nothing on the console.
// The bound keeps each target's own forcing action (reboot-force and the like): systemd's last phase still ends
// every process, unmounts and syncs. Ten minutes stays above the longest stop the catalogue declares (120 s) chained.
func TestEveryShutdownIsBoundedAtTenMinutes(t *testing.T) {
raw, err := os.ReadFile("../../module.json")
if err != nil {
t.Fatal(err)
}
var m struct {
Resources []map[string]any `json:"resources"`
}
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
byPath := map[string]map[string]any{}
for _, r := range m.Resources {
if p, ok := r["path"].(string); ok {
byPath[p] = r
}
}
for _, target := range []string{"reboot", "poweroff", "halt"} {
dir := "/etc/systemd/system/" + target + ".target.d"
if d := byPath[dir]; d == nil || d["type"] != "directory" || d["unknown"] != "program" {
t.Errorf("%s: no directory resource %s whose unknown files are the program's: %v", target, dir, d)
}
f := byPath[dir+"/mesh-power.conf"]
if f == nil || f["type"] != "file" {
t.Errorf("%s: no file resource %s/mesh-power.conf", target, dir)
continue
}
name, _ := f["template"].(string)
text, err := os.ReadFile(filepath.Join("../..", name))
if err != nil {
t.Errorf("%s: %v", target, err)
continue
}
var lines []string
for _, l := range strings.Split(string(text), "\n") {
if l != "" && !strings.HasPrefix(l, "#") {
lines = append(lines, l)
}
}
if strings.Join(lines, "\n") != "[Unit]\nJobTimeoutSec=10min" {
t.Errorf("%s: %s sets %q, want only [Unit] JobTimeoutSec=10min (the target keeps its own action)", target, name, lines)
}
}
}
-42
View File
@@ -314,48 +314,6 @@
"mode": "0644",
"template": "templates/systemd/system/suspend-then-hibernate.target.d/mesh-power.conf"
},
{
"id": "reboot-drop-ins",
"type": "directory",
"path": "/etc/systemd/system/reboot.target.d",
"mode": "0755",
"unknown": "program"
},
{
"id": "reboot-bound",
"type": "file",
"path": "/etc/systemd/system/reboot.target.d/mesh-power.conf",
"mode": "0644",
"template": "templates/systemd/system/reboot.target.d/mesh-power.conf"
},
{
"id": "poweroff-drop-ins",
"type": "directory",
"path": "/etc/systemd/system/poweroff.target.d",
"mode": "0755",
"unknown": "program"
},
{
"id": "poweroff-bound",
"type": "file",
"path": "/etc/systemd/system/poweroff.target.d/mesh-power.conf",
"mode": "0644",
"template": "templates/systemd/system/poweroff.target.d/mesh-power.conf"
},
{
"id": "halt-drop-ins",
"type": "directory",
"path": "/etc/systemd/system/halt.target.d",
"mode": "0755",
"unknown": "program"
},
{
"id": "halt-bound",
"type": "file",
"path": "/etc/systemd/system/halt.target.d/mesh-power.conf",
"mode": "0644",
"template": "templates/systemd/system/halt.target.d/mesh-power.conf"
},
{
"id": "udev-rule",
"type": "file",
@@ -1,8 +0,0 @@
# Written by the mesh (module power, novox/hq ADR 0211; issue 1126). Replaced on every push.
#
# A shutdown is bounded at ten minutes, not systemd's thirty. A unit started again during the shutdown
# (on ace, 2026-10-28, the bus and the authorization manager) is never stopped a second time, and the
# shutdown then waits on this job's timeout with nothing on the console. The target keeps its own
# forcing action: the last phase still ends every process, unmounts and syncs.
[Unit]
JobTimeoutSec=10min
@@ -1,8 +0,0 @@
# Written by the mesh (module power, novox/hq ADR 0211; issue 1126). Replaced on every push.
#
# A shutdown is bounded at ten minutes, not systemd's thirty. A unit started again during the shutdown
# (on ace, 2026-10-28, the bus and the authorization manager) is never stopped a second time, and the
# shutdown then waits on this job's timeout with nothing on the console. The target keeps its own
# forcing action: the last phase still ends every process, unmounts and syncs.
[Unit]
JobTimeoutSec=10min
@@ -1,8 +0,0 @@
# Written by the mesh (module power, novox/hq ADR 0211; issue 1126). Replaced on every push.
#
# A shutdown is bounded at ten minutes, not systemd's thirty. A unit started again during the shutdown
# (on ace, 2026-10-28, the bus and the authorization manager) is never stopped a second time, and the
# shutdown then waits on this job's timeout with nothing on the console. The target keeps its own
# forcing action: the last phase still ends every process, unmounts and syncs.
[Unit]
JobTimeoutSec=10min