Compare commits

..
3 Commits
Author SHA1 Message Date
jochen bf82d30865 Keep a fill warning for an unreadable bucket or log only when its condition was read open
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery ready: it delivers once merged
Second review of #231: when neither the stream nor its condition can be
read, nothing is known of its fill, so none is said; the unread warning
covers it. Only a condition read back as open is kept, with its own words.
2026-10-11 19:34:42 +02:00
jochen c432b9b5f6 Name the fill condition for buckets and logs, and judge each one on its own
mesh/delivery superseded: a newer head of the same pull request
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
Review of #231: "store" is the glossary's word for the postgres server, so
the condition is bucket-or-log-filling. And one stream that cannot be read
no longer fails the whole check: it is said as unread with its reason, an
open fill condition for it is kept, and every other bucket and log is
still judged (ADR 0297 §6).
2026-10-11 19:30:59 +02:00
jochen 7bd61332cb Let a module declare a log the controller creates and never removes (ADR 0297)
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery superseded: a newer head of the same pull request
The issue tracker's history lasts seven days on the events stream and its
bucket fills toward a cap with no warning (issues 501, 502). A module may
now declare `logs`; the controller creates each as a stream on the bus
(create-or-update, never deleted or recreated, reported when undeclared),
issues it to the owner's assignments in their membership, grants the
runtime exactly the append and direct-read subjects, and raises a
condition when any module's bucket or log reaches 75% of its cap, cleared
below 70%.
2026-10-11 19:22:43 +02:00
84 changed files with 1337 additions and 5285 deletions
-3
View File
@@ -3,6 +3,3 @@
# A built binary. The lab writes one here when pointed at the repository root by mistake;
# built artifacts belong in build/, which is already ignored.
/mesh-builder
/mesh-controller
/postgres-provisioner
/redis-provisioner
-3
View File
@@ -408,9 +408,6 @@ func comparisonOf(module string, c comparison, opts takeOptions) (preview string
switch {
case sec.Origin == inventory.OriginAccepted:
fmt.Fprintf(&b, " %s %s: accepted from a person, carried in as it is\n", what, name)
case sec.Origin == inventory.OriginIssued:
// Taken from the provider's software (novox/hq ADR 0305): the value the running service uses.
fmt.Fprintf(&b, " %s %s: issued by the provider's software, carried in as it is\n", what, name)
case opts.Mint[sec.Name]:
fmt.Fprintf(&b, " %s %s: minted by the mesh; the service takes the new value, as --mint said\n", what, name)
case foundData:
+2 -53
View File
@@ -367,13 +367,7 @@ func keepBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan,
}
closes, latest := windowOf(merges, b.Created, window, atMost)
b.Delivery.Merges = named
var closed bool
var withheld []string
if was := b.Delivery.Batch; was != nil {
closed, withheld = was.Closed, was.Withheld // a stop's, which no look takes back (novox/hq ADR 0299)
}
b.Delivery.Batch = &inventory.PlanBatch{ClosesAt: closes, AtMost: latest, Behind: behind, Own: b.OwnPath(),
Closed: closed, Withheld: withheld}
b.Delivery.Batch = &inventory.PlanBatch{ClosesAt: closes, AtMost: latest, Behind: behind, Own: b.OwnPath()}
b.State = inventory.PlanAssembling
if behind != "" && windowClosed(*b, now) {
b.State = inventory.PlanQueued
@@ -408,7 +402,7 @@ func windowClosed(b inventory.Plan, now time.Time) bool {
return true
}
w := b.Delivery.Batch
return w.Closed || !now.Before(w.ClosesAt) || !now.Before(w.AtMost)
return !now.Before(w.ClosesAt) || !now.Before(w.AtMost)
}
// carriedOf is a set of merges as a walk carries them: the latest merge of each repository's branch, and
@@ -768,18 +762,7 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
}
sort.Strings(also)
ordered := groupOrderEdges(members, entries, read, edges)
var withheld []string
plan := planOfMoves(moved, append(append([]inventory.Edge{}, edges...), ordered...))
// **Withheld, a stopped merge's modules** (novox/hq ADR 0299): a module is built at its branch, which holds the
// merge a stop left out of the walk these merges were walked again from — built here only as a dependent, it
// would deliver that merge. One a merge of this batch moves is built: that merge carries the stopped change,
// and the stop named it.
if batch.Delivery != nil && batch.Delivery.Batch != nil {
if out := withhold(&plan, batch.Delivery.Batch.Withheld, moved); len(out) > 0 {
withheld = out
fmt.Printf(" withheld, a stopped merge's modules built only as dependents: %s\n", strings.Join(out, ", "))
}
}
plan.ID, plan.Revision, plan.Created, plan.Epoch = batch.ID, batch.Revision, now, batch.Epoch
plan.Commits = commits
newest := newestCommit(commits)
@@ -790,7 +773,6 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
}
plan.Delivery.Merges = named
plan.Delivery.Alone = batch.Delivery != nil && batch.Delivery.Alone
plan.Delivery.Withheld = withheld
// **Its moments and its class** (novox/hq ADR 0282 decision 6): measured, never acted on.
plan.Times = walkTimesAtCut(*batch, plan, entries, now)
if len(moved) == 0 {
@@ -841,39 +823,6 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
return nil
}
// withhold takes from a plan every module named that its merges do not move, and the tiers left empty: the
// modules taken.
func withhold(plan *inventory.Plan, names, moved []string) []string {
var out []string
for _, n := range names {
if _, in := plan.Modules[n]; in && !slices.Contains(moved, n) {
delete(plan.Modules, n)
out = append(out, n)
}
}
if len(out) == 0 {
return nil
}
var tiers [][]string
for _, t := range plan.Tiers {
var kept []string
for _, n := range t {
if !slices.Contains(out, n) {
kept = append(kept, n)
}
}
if len(kept) > 0 {
tiers = append(tiers, kept)
}
}
if tiers == nil {
tiers = [][]string{}
}
plan.Tiers = tiers
sort.Strings(out)
return out
}
// walkTimesAtCut is a walk's own moments as it is cut (novox/hq ADR 0282 decision 6): when its batch's window
// closed — no merge for the window's length, or its maximum, whichever came first — when it was cut, and its
// class. A merge walked alone had no window.
+207
View File
@@ -0,0 +1,207 @@
package main
import (
"context"
"errors"
"fmt"
"sort"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/conditions"
)
// **A filling bucket or log is said before it is full** (novox/hq ADR 0297 §6, issue 501).
//
// A module's bucket and its log each have a cap on the bus, and when either is full the bus refuses
// the next write: the module stops doing what it writes for, and nothing said so before. On
// 2026-10-11 the issue tracker's bucket held a sixth of its cap after two days, growing toward a stop
// nobody would have heard coming. So the self-check reads every module's bucket and log, and one at
// fillRaiseAt of its cap or more raises a condition naming the module, the bucket or log, and how full
// it is.
//
// **Cleared by observation below fillClearBelow, not below fillRaiseAt**: a bucket or log hovering at
// its threshold would otherwise be raised and cleared on every run. Between the two, an open condition
// is kept and none is raised.
//
// **One that cannot be read is said, and the rest are still judged.** An unanswered question about one
// stream is no reason to know nothing of the others: it is a finding of its own, naming the bucket or
// log and why, and a fill condition already open for it is kept, because not knowing is not a pass.
// The fill thresholds, in percent of a bucket's or log's cap.
const (
fillRaiseAt = 75
fillClearBelow = 70
)
// The conditions the fill probe raises: one filling, and one that could not be read.
const (
kindBucketOrLogFilling = "bucket-or-log-filling"
kindBucketOrLogUnread = "bucket-or-log-unread"
)
// probeFillID is the probe's id in the registry.
const probeFillID = "D-fill"
// bucketOrLog is one module's bucket or log as the fill probe reads it.
type bucketOrLog struct {
Module string
// Kind is `bucket` or `log`; Name its local name; Stream the stream it is on the bus.
Kind, Name, Stream string
}
// filled is how full one bucket or log is, read from the bus.
type filled struct {
Bytes, Max uint64
}
// percent is how full, in whole percent, rounded down.
func (f filled) percent() uint64 {
if f.Max == 0 {
return 0
}
return f.Bytes * 100 / f.Max
}
// fillKey is where a bucket's or log's fill condition is kept.
func fillKey(s bucketOrLog) string { return conditions.Key(conditions.ScopeBus, s.Stream, "filling") }
// fillObservation is what one bucket's or log's fill says: a finding at fillRaiseAt or above, and,
// while its condition is open, at fillClearBelow or above too; nothing otherwise, which is what clears
// it. The operator's words are the kind's (plain_words.go); the summary and the evidence name the
// module, the bucket or log, and the fill.
func fillObservation(s bucketOrLog, f filled, open bool) (conditions.Observation, bool) {
if f.Max == 0 {
return conditions.Observation{}, false // one with no cap cannot fill
}
// Compared in bytes, not rounded percent: 74.9% is not 75%.
atRaise := f.Bytes*100 >= f.Max*fillRaiseAt
aboveClear := f.Bytes*100 >= f.Max*fillClearBelow
if !atRaise && !(open && aboveClear) {
return conditions.Observation{}, false
}
pct := f.percent()
return conditions.Observation{
Scope: conditions.ScopeBus, ID: s.Stream, Token: "filling", Kind: kindBucketOrLogFilling,
Severity: conditions.Warning,
Summary: fmt.Sprintf("%s's %s %s holds %s of %s, %d%%: at its cap the bus refuses the module's next write",
s.Module, s.Kind, s.Name, mibWords(f.Bytes), mibWords(f.Max), pct),
Said: fmt.Sprintf("module %s, %s %s (stream %s): %d of %d bytes, %d%%", s.Module, s.Kind, s.Name,
s.Stream, f.Bytes, f.Max, pct),
}, true
}
// unreadObservation says one bucket or log could not be read, and why. Asked over the network, so one
// look can be wrong: raised on the second look in a row (confirm.go).
func unreadObservation(s bucketOrLog, why error) conditions.Observation {
return conditions.Observation{
Scope: conditions.ScopeBus, ID: s.Stream, Token: "unread", Kind: kindBucketOrLogUnread,
Severity: conditions.Warning, Confirm: true,
Summary: fmt.Sprintf("%s's %s %s could not be read, so how full it is is not known", s.Module, s.Kind, s.Name),
Said: fmt.Sprintf("module %s, %s %s (stream %s): %v", s.Module, s.Kind, s.Name, s.Stream, why),
}
}
// keptFilling is an open fill condition said again for a bucket or log that could not be read this
// time: not knowing how full it is now is no reason to say it has room. Only ever made from a condition
// read back as open — its own summary and severity, never words made up for it.
func keptFilling(s bucketOrLog, open conditions.Condition, why error) conditions.Observation {
return conditions.Observation{
Scope: conditions.ScopeBus, ID: s.Stream, Token: "filling", Kind: kindBucketOrLogFilling,
Severity: open.Severity, Summary: open.Summary,
Said: fmt.Sprintf("module %s, %s %s (stream %s): not read this time (%v); kept open as it was",
s.Module, s.Kind, s.Name, s.Stream, why),
}
}
// mibWords is a size as a person reads it, in MiB with one decimal.
func mibWords(b uint64) string {
return fmt.Sprintf("%.1f MiB", float64(b)/(1024*1024))
}
// readFill is how full one bucket or log is on the bus; found false when it is not on the bus.
type readFill func(ctx context.Context, s bucketOrLog) (f filled, found bool, err error)
// openFill is the fill condition open under a key, if one is, or why it could not be read.
type openFill func(ctx context.Context, key string) (conditions.Condition, bool, error)
// judgeFills judges every bucket and log on its own: one that cannot be read is said as unread, with
// its fill condition kept only when that condition was read back and is open — when it cannot be read
// either, nothing is said of its fill, which the unread finding already covers. Every other is judged
// by its fill; for one of those, a condition that cannot be read is taken as open, so an unknown never
// clears a fill measured between fillClearBelow and fillRaiseAt.
func judgeFills(ctx context.Context, all []bucketOrLog, read readFill, open openFill) []conditions.Observation {
var out []conditions.Observation
for _, s := range all {
f, found, err := read(ctx, s)
if err != nil {
out = append(out, unreadObservation(s, err))
if c, isOpen, cerr := open(ctx, fillKey(s)); cerr == nil && isOpen {
out = append(out, keptFilling(s, c, err))
}
continue
}
if !found {
continue // not on the bus: created on the controller's next raise, and nothing there can fill
}
_, isOpen, cerr := open(ctx, fillKey(s))
isOpen = isOpen || cerr != nil
if o, said := fillObservation(s, f, isOpen); said {
out = append(out, o)
}
}
sort.SliceStable(out, func(i, j int) bool { return out[i].Key() < out[j].Key() })
return out
}
// declaredBucketsAndLogs is every module's bucket and log the catalogue declares, by its stream.
func declaredBucketsAndLogs(ctx context.Context, d *doctor) ([]bucketOrLog, error) {
buckets, err := d.open.inventory.DeclaredBuckets(ctx)
if err != nil {
return nil, err
}
logs, err := d.open.inventory.DeclaredLogs(ctx)
if err != nil {
return nil, err
}
var out []bucketOrLog
for _, b := range buckets {
out = append(out, bucketOrLog{Module: b.Module, Kind: "bucket", Name: b.Name, Stream: "KV_" + b.Bucket()})
}
for _, l := range logs {
out = append(out, bucketOrLog{Module: l.Module, Kind: "log", Name: l.Name, Stream: l.Stream()})
}
return out, nil
}
// probeFill reads every module's bucket and log on the bus and says each one filling toward its cap,
// and each one that could not be read.
func probeFill(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
all, err := declaredBucketsAndLogs(ctx, d)
if err != nil {
return nil, err
}
js := d.js.Context()
read := func(ctx context.Context, s bucketOrLog) (filled, bool, error) {
if err := ctx.Err(); err != nil {
return filled{}, false, err
}
info, err := js.StreamInfo(s.Stream, nats.Context(ctx))
switch {
case errors.Is(err, nats.ErrStreamNotFound):
return filled{}, false, nil
case err != nil:
return filled{}, false, err
case info.Config.MaxBytes <= 0:
return filled{}, true, nil
}
return filled{Bytes: info.State.Bytes, Max: uint64(info.Config.MaxBytes)}, true, nil
}
open := func(ctx context.Context, key string) (conditions.Condition, bool, error) {
if d.keeper == nil {
return conditions.Condition{}, false, nil
}
return d.keeper.Get(ctx, key)
}
return judgeFills(ctx, all, read, open), nil
}
+165
View File
@@ -0,0 +1,165 @@
package main
import (
"context"
"errors"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/conditions"
)
// **A filling log or bucket is said at three quarters of its cap and cleared below seven tenths**
// (novox/hq ADR 0297 §6): raised at 75% naming the module, the bucket or log and its fill; not at 74.9%;
// kept between 70% and 75% while it is open, and not raised there when it is not; and cleared — said no
// more — once it reads below 70%, open or not.
func TestAFillingBucketOrLogIsRaisedAtThreeQuartersAndClearedBelowSevenTenths(t *testing.T) {
const mib = 1024 * 1024
log := bucketOrLog{Module: "mesh-issues", Kind: "log", Name: "changes", Stream: "LOG_mesh-issues_changes"}
bucket := bucketOrLog{Module: "mesh-issues", Kind: "bucket", Name: "issues", Stream: "KV_mesh-issues_issues"}
for _, c := range []struct {
name string
of bucketOrLog
bytes uint64
max uint64
open bool
said bool
}{
{"a log at exactly 75%", log, 768 * mib, 1024 * mib, false, true},
{"a bucket at exactly 75%", bucket, 48 * mib, 64 * mib, false, true},
{"a bucket full", bucket, 64 * mib, 64 * mib, false, true},
{"a log just under 75%", log, 768*mib - 1, 1024 * mib, false, false},
{"a bucket at 72%, not open", bucket, 64 * mib * 72 / 100, 64 * mib, false, false},
{"a bucket at 72%, open: kept", bucket, 64 * mib * 72 / 100, 64 * mib, true, true},
{"a log at exactly 70%, open: kept", log, 700 * mib, 1000 * mib, true, true},
{"a log just under 70%, open: cleared", log, 700*mib - 1, 1000 * mib, true, false},
{"a bucket at 10%, open: cleared", bucket, 64 * mib / 10, 64 * mib, true, false},
{"a bucket with no cap", bucket, 100, 0, true, false},
} {
o, said := fillObservation(c.of, filled{Bytes: c.bytes, Max: c.max}, c.open)
if said != c.said {
t.Errorf("%s: said %v, want %v", c.name, said, c.said)
continue
}
if !said {
continue
}
if o.Key() != fillKey(c.of) || o.Kind != kindBucketOrLogFilling || o.Severity != conditions.Warning {
t.Errorf("%s: raised as %s (%s, %s)", c.name, o.Key(), o.Kind, o.Severity)
}
for _, part := range []string{c.of.Module, c.of.Kind + " " + c.of.Name, "%", " of "} {
if !strings.Contains(o.Summary, part) {
t.Errorf("%s: does not name %q: %q", c.name, part, o.Summary)
}
}
if !strings.Contains(o.Said, "bytes") {
t.Errorf("%s: the evidence does not say the fill in bytes: %q", c.name, o.Said)
}
}
o, _ := fillObservation(log, filled{Bytes: 800 * mib, Max: 1024 * mib}, false)
if !strings.Contains(o.Summary, "800.0 MiB of 1024.0 MiB, 78%") {
t.Errorf("the fill is said as %q", o.Summary)
}
}
// **One bucket or log that cannot be read does not stop the others being judged** (review of #231): it is
// said as unread with its reason, a fill condition open for it is kept, and every other bucket and log
// is judged by its own fill.
func TestAnUnreadableBucketOrLogIsSaidAndTheRestAreStillJudged(t *testing.T) {
const mib = 1024 * 1024
broken := bucketOrLog{Module: "a", Kind: "bucket", Name: "broken", Stream: "KV_a_broken"}
brokenOpen := bucketOrLog{Module: "a", Kind: "log", Name: "was-filling", Stream: "LOG_a_was-filling"}
full := bucketOrLog{Module: "b", Kind: "log", Name: "changes", Stream: "LOG_b_changes"}
empty := bucketOrLog{Module: "c", Kind: "bucket", Name: "quiet", Stream: "KV_c_quiet"}
absent := bucketOrLog{Module: "d", Kind: "bucket", Name: "not-yet", Stream: "KV_d_not-yet"}
refusal := errors.New("the bus did not answer")
read := func(_ context.Context, s bucketOrLog) (filled, bool, error) {
switch s {
case broken, brokenOpen:
return filled{}, false, refusal
case full:
return filled{Bytes: 900 * mib, Max: 1000 * mib}, true, nil
case empty:
return filled{Bytes: 1, Max: 64 * mib}, true, nil
}
return filled{}, false, nil
}
open := func(_ context.Context, key string) (conditions.Condition, bool, error) {
if key == fillKey(brokenOpen) {
return conditions.Condition{Key: key, Severity: conditions.Warning,
Summary: "a's log was-filling holds 800.0 MiB of 1024.0 MiB, 78%"}, true, nil
}
return conditions.Condition{}, false, nil
}
got := map[string]conditions.Observation{}
for _, o := range judgeFills(context.Background(), []bucketOrLog{broken, brokenOpen, full, empty, absent}, read, open) {
got[o.Key()] = o
}
for _, s := range []bucketOrLog{broken, brokenOpen} {
o, said := got[conditions.Key(conditions.ScopeBus, s.Stream, "unread")]
if !said || o.Kind != kindBucketOrLogUnread || !o.Confirm || !strings.Contains(o.Said, refusal.Error()) {
t.Errorf("%s could not be read and was said as %+v", s.Stream, o)
}
}
if o, kept := got[fillKey(brokenOpen)]; !kept || !strings.Contains(o.Said, "kept open") {
t.Errorf("an open fill condition of a log not read this time was not kept: %+v", o)
}
if _, said := got[fillKey(broken)]; said {
t.Error("a bucket not read and not filling was said filling")
}
if o, said := got[fillKey(full)]; !said || o.Kind != kindBucketOrLogFilling {
t.Errorf("a log at 90%% beside an unreadable one was not judged: %+v", got)
}
if len(got) != 4 {
t.Errorf("said %d findings, want 4 (two unread, one kept, one filling): %v", len(got), got)
}
}
// Both kinds have plain words of their own, which hold to the plain rule (novox/hq ADR 0253).
func TestAFillingOrUnreadBucketOrLogIsSaidInPlainWords(t *testing.T) {
for _, kind := range []string{kindBucketOrLogFilling, kindBucketOrLogUnread} {
wording, has := plainWordings[kind]
if !has {
t.Errorf("%s has no plain words", kind)
continue
}
if why, ok := conditions.PlainWords(wording(conditions.Observation{Kind: kind})); !ok {
t.Errorf("%s: %s", kind, why)
}
}
}
// **When neither the bucket or log nor its condition can be read, nothing is said of its fill** (second
// review of #231): the unread finding covers it, and no fill is invented for it. For one that is read and
// measured between seven tenths and three quarters, a condition that cannot be read is taken as open, so
// an unknown never clears it.
func TestNothingIsSaidOfAFillWhenNeitherItNorItsConditionCanBeRead(t *testing.T) {
const mib = 1024 * 1024
broken := bucketOrLog{Module: "a", Kind: "log", Name: "changes", Stream: "LOG_a_changes"}
between := bucketOrLog{Module: "b", Kind: "bucket", Name: "issues", Stream: "KV_b_issues"}
read := func(_ context.Context, s bucketOrLog) (filled, bool, error) {
if s == broken {
return filled{}, false, errors.New("the bus did not answer")
}
return filled{Bytes: 72 * mib, Max: 100 * mib}, true, nil
}
open := func(context.Context, string) (conditions.Condition, bool, error) {
return conditions.Condition{}, false, errors.New("the conditions bucket did not answer")
}
got := map[string]conditions.Observation{}
for _, o := range judgeFills(context.Background(), []bucketOrLog{broken, between}, read, open) {
got[o.Key()] = o
}
if o, said := got[fillKey(broken)]; said {
t.Fatalf("a fill was said for a log whose fill and condition could not be read: %+v", o)
}
if _, said := got[conditions.Key(conditions.ScopeBus, broken.Stream, "unread")]; !said {
t.Error("the log that could not be read was not said as unread")
}
if _, kept := got[fillKey(between)]; !kept {
t.Error("a bucket at 72% whose condition could not be read was cleared")
}
if len(got) != 2 {
t.Errorf("said %d findings, want 2: %v", len(got), got)
}
}
-7
View File
@@ -835,9 +835,6 @@ type answers struct {
// public name on the machine went dark. The holds were correct; they were recorded only in the
// machine's own state file, and the one visible symptom was a count that did not add up.
untaken map[string]map[string]int
// leftOut is, per machine, every module of its set its composition leaves out, and why (novox/hq issue
// 380): assigned and not applied, which every push said only in passing. Not well while there is any.
leftOut map[string][]leftOutModule
// unheld is every module on a machine whose resources are applied through a seat nothing on
// that machine holds (novox/hq ADR 0207), with the modules that could hold it. Reported, not
// refused, until the switch — and while there is any, the mesh is not all well: the order the
@@ -866,10 +863,6 @@ type answers struct {
// applied, is not well: an assignment somebody made is not made yet, or will not be.
pending []inventory.PendingAssignment
pendingUnread string
// lastSent is every machine's last recorded send, by name, with what the machine answered of it (novox/hq
// issue 485); lastSentUnread why it could not be read.
lastSent map[string]inventory.Send
lastSentUnread string
}
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
-24
View File
@@ -2,7 +2,6 @@ package main
import (
"fmt"
"slices"
"sort"
"strings"
@@ -29,14 +28,6 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
for _, e := range entries {
byName[e.Manifest.Module] = e
}
// The machine holding the bus, whose declaration carries every module's grants (novox/hq issue 490).
holder := ""
for _, e := range entries {
if e.Manifest.BusUsers != "" && e.Manifest.ClaimsSeat(catalogue.BrokerSeat) && len(e.On) > 0 {
holder = e.On[0]
}
}
var granting []string
machines := map[string]*link.MachinePlan{}
machine := func(name string) *link.MachinePlan {
if machines[name] == nil {
@@ -59,10 +50,6 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
machine(on).Receives = append(machine(on).Receives, name)
}
}
if !waits && holder != "" && !(len(e.On) == 1 && e.On[0] == holder) && len(e.On) > 0 &&
!slices.Contains(granting, name) {
granting = append(granting, name)
}
switch {
case (name == "nats" || catalogue.ProvidesBus(e.Manifest)) && len(e.On) > 0:
// Said before the merge (novox/hq issue 336): a new bus build holds every send to the bus's machine
@@ -89,13 +76,6 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
}
}
}
if len(granting) > 0 {
// Said before the merge (novox/hq issue 490): what the walk does when the change alters the bus's
// user list. Whether it does is known only once the builds are registered, so it is said as a rule.
p.Steps = append(p.Steps, fmt.Sprintf("%s: if this changes what the bus's user list says (a new tool, "+
"subject or user), %s is sent that list alone, every build there kept, before %s is judged on its first "+
"machine", grantsStepWord, holder, strings.Join(granting, ", ")))
}
var names []string
for name := range machines {
names = append(names, name)
@@ -108,10 +88,6 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
return p
}
// grantsStepWord names the grants step (novox/hq issue 490): the bus's user list sent alone to the machine
// holding the bus ahead of a walk's gate. Not the bus step, which replaces the bus itself.
const grantsStepWord = "grants step"
// busUpgradeNeeded is how a change plan says that merging it holds the bus's machine for a person's step.
const busUpgradeNeeded = "merging this needs a person's bus upgrade"
+8
View File
@@ -179,6 +179,14 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
}
fmt.Fprintf(out, ", keeps state %s", strings.Join(kept, ", "))
}
// And the logs it keeps, with their caps (novox/hq ADR 0297).
if len(m.Logs) > 0 {
kept := make([]string, 0, len(m.Logs))
for _, l := range m.Logs {
kept = append(kept, fmt.Sprintf("%s (%d MiB)", l.Name, l.Cap()))
}
fmt.Fprintf(out, ", keeps log %s", strings.Join(kept, ", "))
}
if len(m.Reads) > 0 {
fmt.Fprintf(out, ", reads %s", strings.Join(m.Reads, ", "))
}
+13 -404
View File
@@ -117,411 +117,28 @@ func letGo(ctx context.Context, inv *inventory.Inventory, id, by, why string) (i
// stopWalk ends a walk on its delivery's word: failed, said as stopped by whom, why. What it asked still
// builds and registers; nothing further is asked or sent.
func stopWalk(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, error) {
p, _, err := stopWalkWithout(ctx, inv, id, by, why, nil, time.Now().UTC())
return p, err
}
// stopAnswer is what a stop that leaves merges out says, as data (novox/hq ADR 0299): the record stopped, the
// batch that walks its other merges again with those merges, every merge left out, the modules the stopped walk
// already sent that stay on the machines named, the modules the batch's walk withholds, and every merge elsewhere
// that will deliver a stopped change when it is walked — a later merge of its repository, or one moving its modules.
type stopAnswer struct {
Stopped string `json:"stopped"`
WalkedAgainBy string `json:"walked_again_by,omitempty"`
WalkedAgain []string `json:"walked_again,omitempty"`
LeftOut []inventory.PlanLeftOut `json:"left_out,omitempty"`
Kept []inventory.PlanKept `json:"kept,omitempty"`
Withheld []string `json:"withheld,omitempty"`
StillCarriedBy []string `json:"still_carried_by,omitempty"`
}
// stopWalkWithout ends a walk on its delivery's word and, given merges to leave out, walks every other merge it
// answered again without them (novox/hq issue 459, ADR 0299): one walk carries every merge of its batch, so ending
// it for one group member ended every other delivery on it.
//
// - Left out are the merges named, each later merge of their repository's branch, which contains one, and each
// merge moving a module a left-out merge moves: a module is built at its branch, which holds the stopped merge.
// - The others join the open batch of their kind — merges heard while the walk ran wait there, and an older
// commit never walks after a newer one — or a new batch when none is open; its window is closed, and its walk
// withholds the left-out merges' modules where it would build them only as dependents.
// - Given a batch not yet cut, the merges left out leave it for a stopped record of their own, and the batch
// goes on without them; with nothing left in it, the batch itself is stopped.
// - The stopped record, the batch and every merge moved are written in one transaction (decision 7).
//
// With nothing left to walk, or no merge named, the walk is only ended. A merge named that the record does not
// answer refuses the stop whole.
func stopWalkWithout(ctx context.Context, inv *inventory.Inventory, id, by, why string, without []string,
now time.Time) (inventory.Plan, stopAnswer, error) {
said := stopAnswer{Stopped: id}
release, err := inv.HoldPlans(ctx, true)
if err != nil {
return inventory.Plan{}, said, err
return inventory.Plan{}, err
}
defer release()
p, err := inv.PlanByID(ctx, id)
if err != nil {
return inventory.Plan{}, said, err
}
batch := p.Batch()
if !p.Open() && !(batch && len(without) > 0) {
return p, said, fmt.Errorf("%s is already %s", p.ID, p.State)
}
var merges, again []inventory.BatchedMerge
if len(without) > 0 {
if merges, err = inv.MergesOf(ctx, p.ID); err != nil {
return p, said, err
}
if again, said.LeftOut, err = leaveOut(merges, without); err != nil {
return p, said, fmt.Errorf("%s is not stopped: %w", p.ID, err)
}
}
stopped := func(r *inventory.Plan) {
if r.Delivery == nil {
r.Delivery = &inventory.PlanDelivery{}
}
r.Delivery.Stopped, r.Delivery.StoppedWhy, r.Delivery.LeftOut = by, why, said.LeftOut
r.State = inventory.PlanFailed
r.Note = fmt.Sprintf("stopped by %s at tier %d: %s", by, r.Tier, why)
}
withheld := movesOf(leftOutMerges(merges, said.LeftOut))
said.Withheld = withheld
if said.StillCarriedBy, err = stillCarried(ctx, inv, p.ID, merges, said.LeftOut, withheld); err != nil {
return p, said, err
}
if batch {
return p, said, stopInBatch(ctx, inv, &p, merges, again, &said, stopped, now)
}
entries, err := inv.Catalogued(ctx)
if err != nil {
return p, said, err
}
running := map[string][]string{}
for _, e := range entries {
running[e.Manifest.Module] = e.On
}
said.Kept = keptOf(p, movesOf(again), running)
stopped(&p)
p.Delivery.Kept = said.Kept
if len(again) == 0 {
if len(said.LeftOut) > 0 {
p.Note += fmt.Sprintf("; left out %s, nothing else to walk", mergesLeftOutWords(said.LeftOut))
}
return p, said, inv.SavePlan(ctx, &p)
}
target, err := batchToWalkAgain(ctx, inv, again, now)
if err != nil {
return p, said, fmt.Errorf("%s is not stopped: %w", p.ID, err)
}
target.Delivery.Batch.Closed = true
target.Delivery.Batch.Withheld = union(target.Delivery.Batch.Withheld, withheld)
sort.Strings(target.Delivery.Batch.Withheld)
said.WalkedAgainBy = target.ID
var moves []inventory.MergeMove
for _, m := range again {
said.WalkedAgain = append(said.WalkedAgain, m.Repository+"@"+m.Commit)
moves = append(moves, inventory.MergeMove{Repository: m.Repository, Commit: m.Commit, Plan: target.ID, Alone: m.Alone})
}
p.Delivery.WalkedAgainBy = target.ID
p.Note += fmt.Sprintf("; its other merges are walked again by %s: %s", target.ID, strings.Join(said.WalkedAgain, ", "))
if len(said.LeftOut) > 0 {
p.Note += "; left out " + mergesLeftOutWords(said.LeftOut)
}
if len(said.Kept) > 0 {
p.Note += "; " + keptWords(said.Kept)
}
// The batch first, so no merge names a plan the store does not hold; all of it, or nothing (decision 7).
if err := inv.SavePlansMoving(ctx, []*inventory.Plan{&target, &p}, moves); err != nil {
return p, said, fmt.Errorf("%s is not stopped, and none of its merges moved: %w", p.ID, err)
}
// What the batch holds and names now: written again by the cutter's next look should this fail.
if err := keepBatch(ctx, inv, &target, now, ""); err != nil {
fmt.Printf(" %s holds its merges; its record is written again at the next look: %v\n", target.ID, err)
}
fmt.Printf("%s stopped by %s; %s walks its other merges again: %s\n", p.ID, by, target.ID,
strings.Join(said.WalkedAgain, ", "))
return p, said, nil
}
// stopInBatch leaves merges out of a batch not yet cut (ADR 0299): they go to a stopped record of their own, which
// names the batch as walking the others, and the batch withholds their modules; a batch left with none is stopped
// itself.
func stopInBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan, merges, again []inventory.BatchedMerge,
said *stopAnswer, stopped func(*inventory.Plan), now time.Time) error {
if len(again) == 0 {
stopped(b)
b.Delivery.Batch = nil
b.Note = "a batch whose every merge was left out by a stop: " + b.Note
return inv.SavePlan(ctx, b)
}
left := leftOutMerges(merges, said.LeftOut)
first := left[0]
r := inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch,
Commit: first.Commit, Merged: first.Merged, Created: now, Tiers: [][]string{},
Modules: map[string]*inventory.PlanModule{}}
if r.ID == b.ID {
r.ID += "-left"
}
stopped(&r)
r.Delivery.WalkedAgainBy = b.ID
r.Note += fmt.Sprintf("; left out of %s before it was walked: %s; %s walks the others", b.ID,
mergesLeftOutWords(said.LeftOut), b.ID)
b.Delivery.Batch.Withheld = union(b.Delivery.Batch.Withheld, said.Withheld)
sort.Strings(b.Delivery.Batch.Withheld)
var moves []inventory.MergeMove
for _, m := range left {
moves = append(moves, inventory.MergeMove{Repository: m.Repository, Commit: m.Commit, Plan: r.ID, Alone: m.Alone})
}
for _, m := range again {
said.WalkedAgain = append(said.WalkedAgain, m.Repository+"@"+m.Commit)
}
said.Stopped, said.WalkedAgainBy = r.ID, b.ID
if err := inv.SavePlansMoving(ctx, []*inventory.Plan{&r, b}, moves); err != nil {
return fmt.Errorf("%s is not changed, and none of its merges moved: %w", b.ID, err)
}
if err := keepBatch(ctx, inv, b, now, ""); err != nil {
fmt.Printf(" %s holds its merges; its record is written again at the next look: %v\n", b.ID, err)
}
fmt.Printf("%s: %s left out by a stop, kept by %s\n", b.ID, mergesLeftOutWords(said.LeftOut), r.ID)
return nil
}
// batchToWalkAgain is the batch a stopped walk's merges join: the open batch of their kind, or a new one, not yet
// written. The kind is read from what the merges moved.
func batchToWalkAgain(ctx context.Context, inv *inventory.Inventory, again []inventory.BatchedMerge,
now time.Time) (inventory.Plan, error) {
own := false
for _, n := range movesOf(again) {
if _, on := onTheControllersPath[n]; on {
own = true
}
}
batches, err := inv.Batches(ctx)
if err != nil {
return inventory.Plan{}, err
}
for _, b := range batches {
if b.OwnPath() == own {
if b.Delivery == nil {
b.Delivery = &inventory.PlanDelivery{}
}
if b.Delivery.Batch == nil {
b.Delivery.Batch = &inventory.PlanBatch{Own: own}
}
return b, nil
}
if !p.Open() {
return p, fmt.Errorf("%s is already %s", p.ID, p.State)
}
first := again[0]
return inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch,
Commit: first.Commit, Merged: first.Merged, Created: now, State: inventory.PlanAssembling,
Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{},
Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: own}}}, nil
}
// leaveOut splits a walk's merges by the merges named to leave out, each `<repository>@<commit>` (a commit's
// first characters will do): the merges walked again, and those left out — each named one; each later merge of
// its repository's branch, which contains it; and each merge moving a module a left-out merge moves, which is
// built at its branch and so holds the stopped merge — until none more is. A name the walk does not answer is
// refused.
func leaveOut(merges []inventory.BatchedMerge, without []string) ([]inventory.BatchedMerge, []inventory.PlanLeftOut, error) {
var stopped []inventory.BatchedMerge
for _, w := range without {
repository, commit, ok := strings.Cut(strings.TrimSpace(w), "@")
if !ok || repository == "" || len(commit) < 7 {
return nil, nil, fmt.Errorf("%q names no merge: <repository>@<commit>", w)
}
i := slices.IndexFunc(merges, func(m inventory.BatchedMerge) bool {
return strings.EqualFold(m.Repository, repository) && strings.HasPrefix(m.Commit, commit)
})
if i < 0 {
return nil, nil, fmt.Errorf("it answers no merge %s@%s", repository, commit)
}
stopped = append(stopped, merges[i])
if p.Delivery == nil {
p.Delivery = &inventory.PlanDelivery{}
}
same := func(a, b inventory.BatchedMerge) bool {
return strings.EqualFold(a.Repository, b.Repository) && a.Commit == b.Commit
p.Delivery.Stopped, p.Delivery.StoppedWhy = by, why
p.State = inventory.PlanFailed
p.Note = fmt.Sprintf("stopped by %s at tier %d: %s", by, p.Tier, why)
if err := inv.SavePlan(ctx, &p); err != nil {
return p, err
}
out := map[int]inventory.PlanLeftOut{}
for i, m := range merges {
if slices.ContainsFunc(stopped, func(s inventory.BatchedMerge) bool { return same(s, m) }) {
out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit}
}
}
for grew := true; grew; {
grew = false
for i, m := range merges {
if _, left := out[i]; left {
continue
}
for j, o := range out {
l := merges[j]
contains := o.Contains
if contains == "" {
contains = l.Commit
}
if strings.EqualFold(l.Repository, m.Repository) && l.Branch == m.Branch && laterMerge(m, l) {
out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit, Contains: contains}
} else if shared := sharedMoves(m, l); len(shared) > 0 {
out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit, Contains: contains,
Shares: shared}
} else {
continue
}
grew = true
break
}
}
}
var again []inventory.BatchedMerge
var left []inventory.PlanLeftOut
for i, m := range merges {
if o, is := out[i]; is {
left = append(left, o)
} else {
again = append(again, m)
}
}
sort.SliceStable(left, func(i, j int) bool { return left[i].Contains == "" && left[j].Contains != "" })
return again, left, nil
}
// sharedMoves are the modules two kept merges both moved when they were heard.
func sharedMoves(a, b inventory.BatchedMerge) []string {
var out []string
theirs := announcedOf(b).Moves
for _, n := range announcedOf(a).Moves {
if slices.Contains(theirs, n) && !slices.Contains(out, n) {
out = append(out, n)
}
}
sort.Strings(out)
return out
}
// leftOutMerges are the kept merges a stop left out.
func leftOutMerges(merges []inventory.BatchedMerge, out []inventory.PlanLeftOut) []inventory.BatchedMerge {
var left []inventory.BatchedMerge
for _, m := range merges {
if slices.ContainsFunc(out, func(o inventory.PlanLeftOut) bool {
return strings.EqualFold(o.Repository, m.Repository) && o.Commit == m.Commit
}) {
left = append(left, m)
}
}
return left
}
// movesOf is every module kept merges moved when they were heard, sorted.
func movesOf(merges []inventory.BatchedMerge) []string {
var out []string
for _, m := range merges {
out = union(out, announcedOf(m).Moves)
}
sort.Strings(out)
return out
}
// keptOf are the modules a stopped walk sent that no merge walked again moves, and the machines each reached:
// those keep running the stopped walk's build. A module sent to its first machines reached those, and the rest it
// was sent to after them; one sent with no first machine (its policy sends it to all together, or a later tier is
// built by it) reached every machine running it.
func keptOf(p inventory.Plan, walkedAgain []string, running map[string][]string) []inventory.PlanKept {
var out []inventory.PlanKept
for name, m := range p.Modules {
if m == nil || slices.Contains(walkedAgain, name) {
continue
}
var machines []string
if m.FirstAt != nil || m.SentAt != nil {
machines = union(machines, m.First)
}
if m.SentAt != nil && len(m.First) == 0 {
machines = union(machines, running[name])
}
if m.Gate != nil && m.Gate.Since != nil {
machines = union(machines, m.Gate.Machines)
}
for machine := range m.Rest {
machines = union(machines, []string{machine})
}
if len(machines) == 0 {
continue
}
sort.Strings(machines)
out = append(out, inventory.PlanKept{Module: name, Machines: machines})
}
sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module })
return out
}
// keptWords is what a stopped walk leaves on the machines, as a person reads it.
func keptWords(kept []inventory.PlanKept) string {
var words []string
for _, k := range kept {
words = append(words, k.Module+" on "+strings.Join(k.Machines, ", "))
}
return "left running the stopped walk's build, which nothing walks again: " + strings.Join(words, "; ")
}
// stillCarried are the merges outside the stopped record, in a batch or an open walk, that will deliver a stopped
// change when they are walked: a later merge of its repository's branch, which contains it, or one that moves a
// module it moves. Holding a merged change back for good is a revert's (ADR 0299).
func stillCarried(ctx context.Context, inv *inventory.Inventory, id string, merges []inventory.BatchedMerge,
out []inventory.PlanLeftOut, withheld []string) ([]string, error) {
var named []inventory.BatchedMerge
for _, m := range leftOutMerges(merges, out) {
if slices.ContainsFunc(out, func(o inventory.PlanLeftOut) bool { return o.Commit == m.Commit && o.Contains == "" }) {
named = append(named, m)
}
}
if len(named) == 0 {
return nil, nil
}
open, err := inv.OpenPlans(ctx)
if err != nil {
return nil, err
}
batches, err := inv.Batches(ctx)
if err != nil {
return nil, err
}
var carried []string
for _, p := range append(open, batches...) {
if p.ID == id {
continue
}
ms, err := inv.MergesOf(ctx, p.ID)
if err != nil {
return nil, err
}
for _, m := range ms {
later := slices.ContainsFunc(named, func(s inventory.BatchedMerge) bool {
return strings.EqualFold(s.Repository, m.Repository) && s.Branch == m.Branch && laterMerge(m, s)
})
shares := slices.ContainsFunc(announcedOf(m).Moves, func(n string) bool { return slices.Contains(withheld, n) })
if later || shares {
carried = append(carried, fmt.Sprintf("%s@%s (in %s)", m.Repository, m.Commit, p.ID))
}
}
}
sort.Strings(carried)
return carried, nil
}
// mergesLeftOutWords is the merges a stop left out, as a person reads them: each that contains a stopped one, or
// shares its modules, says so.
func mergesLeftOutWords(out []inventory.PlanLeftOut) string {
var words []string
for _, o := range out {
s := o.Repository + "@" + short(o.Commit)
switch {
case len(o.Shares) > 0:
s += " (which moves " + strings.Join(o.Shares, ", ") + " as " + short(o.Contains) + " does)"
case o.Contains != "":
s += " (which contains " + short(o.Contains) + ")"
}
words = append(words, s)
}
return strings.Join(words, ", ")
return p, nil
}
// orderMember is one member of a group, as mesh-delivery says it.
@@ -849,8 +466,6 @@ func deliveryCommand(ctx context.Context, args []string) error {
group := set.String("group", "", "a delivery group's id")
by := set.String("by", catalogue.DeliverySeat, "who says it")
why := set.String("why", "", "why")
without := set.String("without", "", "merges a stop leaves out, <repository>@<commit>, comma-separated: the walk's "+
"other merges are walked again without them (novox/hq ADR 0299)")
limit := set.Int("n", 50, "how many ended walks to answer beside the open ones")
planID := set.String("plan", "", "one walk")
positionals, err := parseAround(set, rest)
@@ -937,21 +552,15 @@ func deliveryCommand(ctx context.Context, args []string) error {
return nil
case "stop":
if len(positionals) != 1 || strings.TrimSpace(*why) == "" {
return errors.New("delivery stop <plan> --why <text> [--by <who>] [--without <repository>@<commit>,…]")
return errors.New("delivery stop <plan> --why <text> [--by <who>]")
}
var p inventory.Plan
var said stopAnswer
if err := sayingOnTheBus(ctx, func() (err error) {
p, said, err = stopWalkWithout(ctx, inv, positionals[0], *by, strings.TrimSpace(*why), splitList(*without),
time.Now().UTC())
p, err = stopWalk(ctx, inv, positionals[0], *by, strings.TrimSpace(*why))
return err
}); err != nil {
return err
}
if *without != "" {
// What it left out and what it walks again, as data: the delivery's owner reads it (ADR 0299).
return answer(said)
}
fmt.Printf("%s stopped by %s at tier %d of %d; what was asked still builds and registers, nothing further "+
"is asked or sent\n", p.ID, *by, p.Tier, len(p.Tiers))
return nil
+10 -8
View File
@@ -74,12 +74,8 @@ type probe struct {
// probeRegistry is the registry, in to-be 45's order. **The registry is the design's live form**: a
// probe added to a design is a row added here.
var probeRegistry = []probe{
// D1 also says every module assigned and left out of a machine's declaration (novox/hq issue 380), from the
// resolution it already makes: needs-operator for a setting nobody gave, left-out for any other cause.
{ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation; no module " +
"assigned to a machine is left out of its declaration unsaid",
From: "issues 236, 263, 275, 380", Kind: "declaration-refused",
Raises: []string{kindAwaitingPush, kindLeftOut, kindNeedsOperator}, Phase: 1,
{ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation",
From: "issues 236, 263, 275", Kind: "declaration-refused", Raises: []string{kindAwaitingPush}, Phase: 1,
run: probeDeclarations},
{ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6",
From: "issue 262", Kind: "resolver-wrong", Phase: 1, run: probeResolvers},
@@ -122,8 +118,8 @@ var probeRegistry = []probe{
Kind: kindDeliveryStalled, Phase: 3, run: probeDeliveries},
// The delivery budgets (novox/hq ADR 0282 decision 7): the newest delivery of each class within its budget,
// read from mesh-delivery's `times`; a measurement said, never a delivery held.
{ID: probeBudgetsID, Asserts: "the newest delivery of a leaf module ran on every machine within three minutes of " +
"its merge, and of a core module within ten: mesh-delivery's `times`", From: "ADR 0282, ADR 0304",
{ID: probeBudgetsID, Asserts: "the newest delivery of a leaf module ran on every machine within five minutes of " +
"its merge, and of a core module within ten: mesh-delivery's `times`", From: "ADR 0282",
Kind: kindOverBudget, Phase: 3, run: probeBudgets},
// A client of the bus reconnecting in a loop (novox/hq issue 327), from the server's record of closed
// connections, which the bus's own module reads.
@@ -140,6 +136,12 @@ var probeRegistry = []probe{
{ID: "D-root", Asserts: "no agent can become root without a person on a machine where the router or a channel " +
"proving its sender runs: not by its own account, and not through a tool that runs its command as an account " +
"that can", From: "ADR 0259 §8", Kind: kindRootNotFree, Phase: 2, run: probeAgentRoot},
// A module's bucket or log filling toward its cap (novox/hq ADR 0297 §6): said at three quarters, cleared
// below seven tenths, so one at its threshold is not raised and cleared on every run. One that cannot be
// read is said on its own, and every other is still judged.
{ID: probeFillID, Asserts: "every module's bucket and log holds less than three quarters of its cap; one " +
"said filling is cleared once it holds less than seven tenths", From: "ADR 0297, issue 501",
Kind: kindBucketOrLogFilling, Raises: []string{kindBucketOrLogUnread}, Phase: 1, run: probeFill},
{ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals",
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs},
// The core's health definitions (novox/hq to-be 45 §8, ADR 0236): what a core component's new build is
+9 -66
View File
@@ -27,10 +27,8 @@ import (
// **"Reported applied" is not enough.** ADR 0218 sent a module to one machine first and the rest once
// that machine reported it applied. A build that applies and then does nothing, crashes, serves no
// tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by
// the component's health — the core's health definitions, or a module's own — within ten minutes of the
// apply: a core module passing three times over at least two minutes, a leaf module once, as soon as the
// node-engine states it healthy (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it). Only then
// are the rest sent.
// the component's health — the core's health definitions, or a module's own — passing three times over
// at least two minutes, within ten minutes of the apply. Only then are the rest sent.
//
// **A failing gate stops the plan and puts the previous build back there.** The build is marked failed
// at its gate — registration refuses it and nothing sends it again on its own — the module's registered
@@ -45,38 +43,11 @@ var (
gateSettle = 2 * time.Minute
// gateBound is how long after the apply the build may take to become healthy.
gateBound = 10 * time.Minute
// gatePasses is how many consecutive judgings must find a core module healthy, gateEvery apart at the
// least, after gateSettle. gateLeafPasses is a leaf module's: one, taken once the node-engine states every
// resource of it healthy by its own check, with no settle (novox/hq ADR 0282 decision 4 as ADR 0304
// decision 3 changed it, issue 524).
gatePasses = 3
gateLeafPasses = 1
gateLeafSettles = false
gateEvery = 40 * time.Second
// gatePasses is how many consecutive judgings must find it healthy, gateEvery apart at the least.
gatePasses = 3
gateEvery = 40 * time.Second
)
// coreModule is whether a module is judged as the core (novox/hq ADR 0282 decision 1): one on the controller's
// own path, or one holding the mesh's resolver. Every other module is a leaf.
func coreModule(module string, m catalogue.Manifest) bool {
if _, own := onTheControllersPath[module]; own {
return true
}
for _, c := range m.Claims {
if c.Name == resolverSeat {
return true
}
}
return coreComponent(module) != ""
}
// gateWants is how many passes a module needs, and whether it must stay healthy gateSettle first.
func gateWants(module string, m catalogue.Manifest) (passes int, settles bool) {
if coreModule(module, m) {
return gatePasses, true
}
return gateLeafPasses, gateLeafSettles
}
// gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions
// are raised by a plan as it judges, and kept or cleared by the probe on every run.
const gateProbe = "DG"
@@ -637,24 +608,8 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
}
}
settled := now.Sub(*g.Since) >= gateSettle
// What the send needs is what its most demanding module needs: a send carrying a core module is judged
// as the core.
wants, settles := 0, false
for _, m := range modules {
p, s := gateWants(m, shelf[m])
wants, settles = max(wants, p), settles || s
}
if len(modules) == 0 {
// A send judged on no machine of its modules (its first machine not running them) has no class to
// be judged by: it is judged as the core, never passed at once without a reading.
wants, settles = gatePasses, true
}
g.Wants = wants
// A module's own pass is its own class's: a leaf beside a core module that fails keeps its build after
// its one reading, and only what did not pass is put back (issue 318's rule, by class since issue 524).
passedAlone := func(m string) bool {
p, s := gateWants(m, shelf[m])
return (settled || !s) && reading[m] <= healthPerson && g.Healthy[m] >= p
return settled && reading[m] <= healthPerson && g.Healthy[m] >= gatePasses
}
// fail decides the send failed: what passed on its own keeps its pass, everything else is put back.
fail := func(why string) {
@@ -702,7 +657,7 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
g.Passes++
g.LastPass, g.Last, g.Failing = &now, "", nil
g.Read(now, true, "")
if g.Passes >= wants && (settled || !settles) {
if g.Passes >= gatePasses && settled {
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now)
}
@@ -1385,9 +1340,9 @@ func gateLine(g *inventory.PlanGate) string {
if g.Why != "" {
line += ": " + g.Why
} else if g.Last != "" {
line += fmt.Sprintf(" (%d of %d passes; wanting: %s)", g.Passes, gateWanted(g), g.Last)
line += fmt.Sprintf(" (%d of %d passes; wanting: %s)", g.Passes, gatePasses, g.Last)
} else if g.Verdict == "" {
line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gateWanted(g))
line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gatePasses)
}
if g.Verdict == "" && len(g.Waits) > 0 {
line += waitsSaid(g.Waits)
@@ -1398,10 +1353,6 @@ func gateLine(g *inventory.PlanGate) string {
if g.Rollback != "" {
line += "; " + g.Rollback
}
// Before the send it judges (novox/hq issue 490).
if said := grantsSaid(g.Grants); said != "" {
line += "; " + said
}
return line
}
@@ -1430,11 +1381,3 @@ func witnessObservation(node string, r lease.Rollback) conditions.Observation {
Summary: fmt.Sprintf("the witness on %s judged the %s %s not healthy %s at %s: %s", node, r.Component,
short(r.From), what, r.At.UTC().Format(time.RFC3339), r.Why)}
}
// gateWanted is the passes a gate's send needs, three where the gate was judged before it was said.
func gateWanted(g *inventory.PlanGate) int {
if g.Wants > 0 {
return g.Wants
}
return gatePasses
}
@@ -53,7 +53,6 @@ func healthyContainer(module string) inventory.ResourceHealth {
// tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass,
// counted over the gate's spacing — before it says its own. The broken one alone is put back.
func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) {
judgeLeavesAsTheCore(t)
open := aMesh(t)
ctx := t.Context()
withGateBounds(t, 150*time.Millisecond, 20*time.Millisecond, 10*time.Second)
@@ -285,7 +284,6 @@ func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) {
// tools are marked and put back in the very judging that found them broken — their registered build is the
// one before, and the laptop is sent it — while `app` goes on being judged, recovers, and keeps its own pass.
func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testing.T) {
judgeLeavesAsTheCore(t)
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
+1 -56
View File
@@ -153,7 +153,6 @@ func (g *gateMesh) plan(t *testing.T) inventory.Plan {
// registered and sent to it again — and never reaches the second machine; it is marked, said as a
// condition and an event, never registered or rolled back again.
func TestABuildThatFailsItsGateIsRolledBackOnItsFirstMachineAndGoesNoFurther(t *testing.T) {
judgeLeavesAsTheCore(t)
g := aGateMesh(t)
ctx := t.Context()
inv := g.open.inventory
@@ -256,7 +255,7 @@ func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) {
t.Fatalf("sent %v", g.sent)
}
gate := p.Modules["app"].Gate
if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gateWanted(gate) {
if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gatePasses {
t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate)
}
if v, found, err := g.open.inventory.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed {
@@ -267,52 +266,8 @@ func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) {
}
}
// A leaf module passes its gate on one reading, as soon as its first machine is healthy, without waiting out the
// core's settle time (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it, issue 524).
func TestALeafModulePassesOnOneHealthyReading(t *testing.T) {
g := aGateMesh(t)
ctx := t.Context()
gateSettle = time.Hour // a core module could not pass for an hour
advancePlans(ctx, g.open) // the first machine is sent the new build
advancePlans(ctx, g.open) // one healthy judging
p := g.plan(t)
gate := p.Modules["app"].Gate
if gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes != 1 || gate.Wants != 1 {
t.Fatalf("a leaf module's gate after one healthy judging: %+v (plan %s: %s)", gate, p.State, p.Note)
}
if !strings.Contains(gate.Why, "healthy 1 times") {
t.Errorf("the verdict says %q", gate.Why)
}
}
// Which modules are judged as the core: those on the controller's own path and the resolver's holder, three
// readings after the settle time; every other module one (novox/hq ADR 0282 decision 1, ADR 0304 decision 3).
func TestAModuleIsJudgedByItsClass(t *testing.T) {
for _, c := range []struct {
module string
claims []catalogue.Claim
passes int
settles bool
}{
{"app", nil, 1, false},
{"nzbget", nil, 1, false},
{"mesh-controller", nil, 3, true},
{"mesh-host", nil, 3, true},
{"node-tools", nil, 3, true},
{"nats", nil, 3, true},
{catalogue.DeliverySeat, nil, 3, true},
{"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, true},
} {
p, s := gateWants(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims})
if p != c.passes || s != c.settles {
t.Errorf("%s: %d passes, settles %v; want %d, %v", c.module, p, s, c.passes, c.settles)
}
}
}
// A first machine that never becomes healthy fails its gate at the bound, and is put back.
func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) {
judgeLeavesAsTheCore(t)
g := aGateMesh(t)
ctx := t.Context()
advancePlans(ctx, g.open)
@@ -638,13 +593,3 @@ func TestAWitnessesVerdictIsAConditionWhileItsReportsSayIt(t *testing.T) {
t.Fatalf("a verdict the reports no longer carry is still said: %+v %v", obs, err)
}
}
// judgeLeavesAsTheCore makes a test's leaf modules judged as a core module is, three readings after the settle
// time, for a test of the gate's other rules (a fault, the bound, a supersession) that needs a judging still
// open after a healthy reading.
func judgeLeavesAsTheCore(t *testing.T) {
t.Helper()
wasPasses, wasSettles := gateLeafPasses, gateLeafSettles
gateLeafPasses, gateLeafSettles = gatePasses, true
t.Cleanup(func() { gateLeafPasses, gateLeafSettles = wasPasses, wasSettles })
}
+8 -86
View File
@@ -202,99 +202,21 @@ func modulesWords(modules []string) string {
return strings.Join(modules, ", ")
}
// writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234), and
// what it answered (issue 485): nothing when the mesh has not recorded a send to it.
// writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234):
// nothing when the mesh has not recorded a send to it.
func writeLastSend(ctx context.Context, w io.Writer, inv *inventory.Inventory, node string) error {
s, found, err := inv.LastSend(ctx, node)
if err != nil || !found {
return err
}
fmt.Fprintf(w, "%s:\n", node)
for _, line := range lastSendLines(s) {
fmt.Fprintln(w, line)
}
return nil
}
// lastSendLines is a machine's last send in full, as `node show` and `plan` say it (novox/hq issue 485): when,
// by whom, under which lease epoch, from which assignment generation, naming which modules, and what the
// machine answered of it — the refusal in its own words.
func lastSendLines(s inventory.Send) []string {
const stamp = "2006-01-02 15:04:05"
epoch := "under no lease epoch"
if s.Epoch > 0 {
epoch = fmt.Sprintf("under lease epoch %d", s.Epoch)
}
generation := "no assignment generation recorded"
generation := "no generation recorded"
if s.Generation > 0 {
generation = fmt.Sprintf("assignment generation %d", s.Generation)
}
lines := []string{
fmt.Sprintf(" last sent sequence %d at %s", s.Sequence, s.SentAt.Local().Format(stamp)),
fmt.Sprintf(" by %s", s.Sender),
fmt.Sprintf(" %s, composed from %s", epoch, generation),
fmt.Sprintf(" naming %s", modulesWords(s.Modules)),
fmt.Fprintf(w, "%s was last sent sequence %d at %s by %s, composed from %s, naming %s\n", node, s.Sequence, s.SentAt.Local().Format("2006-01-02 15:04:05"), s.Sender, generation, modulesWords(s.Modules))
if s.RefusedAt != nil {
fmt.Fprintf(w, " and refused it at %s: it had applied generation %d\n",
s.RefusedAt.Local().Format("2006-01-02 15:04:05"), s.RefusedApplied)
}
answer := func(words string) { lines = append(lines, " its answer "+words) }
switch a := s.Answer; {
case s.RefusedAt != nil:
// Before the report: a refusal for the generation is kept on the send itself and is the machine's
// answer to this send in particular.
answer(fmt.Sprintf("refused it at %s: it had applied assignment generation %d, newer than this one",
s.RefusedAt.Local().Format(stamp), s.RefusedApplied))
case a.Outcome == "" || a.At == nil:
answer("the machine has not reported on it yet")
case a.Outcome == inventory.OutcomeApplied:
answer("applied it at " + a.At.Local().Format(stamp))
case a.Outcome == inventory.OutcomeRefused:
answer("refused it at " + a.At.Local().Format(stamp) + ", in its words:")
for _, line := range strings.Split(strings.TrimRight(a.Refused, "\n"), "\n") {
lines = append(lines, " "+strings.TrimSpace(line))
}
default:
answer(fmt.Sprintf("%s at %s: %d resource(s) failed — `status` says which", a.Outcome,
a.At.Local().Format(stamp), a.Failed))
}
return lines
}
// sendAnswerWord is what a machine answered of a send, in a word or few, as status says it in one line.
func sendAnswerWord(s inventory.Send) string {
switch a := s.Answer; {
case s.RefusedAt != nil:
return fmt.Sprintf("refused: it had applied generation %d", s.RefusedApplied)
case a.Outcome == "" || a.At == nil:
return "not reported on yet"
case a.Outcome == inventory.OutcomeFailed:
return fmt.Sprintf("failed (%d resource(s))", a.Failed)
default:
return a.Outcome
}
}
// printLastSends says, one line per machine, when it was last sent a declaration, by whom, from which assignment
// generation and what it answered (novox/hq issue 485). `node show <node>` says the send in full.
func printLastSends(nodes []inventory.Node, sends map[string]inventory.Send, unread string) {
if unread != "" {
fmt.Printf("what each machine was last sent could NOT be read: %s\n\n", unread)
return
}
if len(nodes) == 0 {
return
}
fmt.Println("last sent, per machine:")
for _, n := range nodes {
s, found := sends[n.Name]
if !found {
fmt.Printf(" %-12s no send recorded\n", n.Name)
continue
}
generation := "no generation"
if s.Generation > 0 {
generation = fmt.Sprintf("generation %d", s.Generation)
}
fmt.Printf(" %-12s %s by %s, %s — %s\n", n.Name, s.SentAt.Local().Format("2006-01-02 15:04"), s.Sender,
generation, sendAnswerWord(s))
}
fmt.Printf("\n `node show <node>` says a machine's last send in full\n\n")
return nil
}
@@ -1,198 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/identity"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The grants step on a store (novox/hq issue 490): the machine holding the bus is sent its new user list,
// and nothing else — not the new build of a module it runs, not a module newly assigned there.
//
// The restic shape: anchor holds the bus and runs restic at c1, as does laptop; restic's c2 gives it a
// health tool. The grants step's send to anchor composes restic at c1, carries the user list that grants
// laptop's node-engine c2's tool, and records that anchor still runs c1. A module newly assigned to anchor
// makes the step fail, said, rather than install it unjudged.
func TestTheGrantsStepSendsTheUserListAndNothingElse(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
start := time.Now().Add(-time.Hour)
build := func(module, commit string, at time.Time, manifest map[string]any) link.BuildResult {
manifest["module"], manifest["version"] = module, "1"
raw, _ := json.Marshal(manifest)
return link.BuildResult{ID: link.NewBuildID(at), Repository: "novox/mesh-catalog", Path: "modules/" + module,
On: "anchor", Module: module, Commit: commit, Manifest: raw,
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
}
for _, b := range []link.BuildResult{
build("nats", "n1", start, natsFixture()),
build("restic", "c1", start.Add(time.Second), resticFixture(false)),
build("wallpaper", "w1", start.Add(2*time.Second), wallpaperFixture()),
} {
if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil {
t.Fatal(err)
}
}
for _, a := range [][2]string{{"anchor", "nats"}, {"anchor", "restic"}, {"laptop", "restic"}} {
if _, err := inv.Assign(ctx, a[0], a[1]); err != nil {
t.Fatal(err)
}
}
for _, n := range []string{"anchor", "laptop"} {
if _, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: "node." + n, Kind: inventory.BusNode, Node: n}); err != nil {
t.Fatal(err)
}
}
wasEpoch := epochForActs
epochForActs = func(context.Context) (uint64, error) { return 7, nil }
t.Cleanup(func() { epochForActs = wasEpoch })
// A send to anchor composed as sendToEach composes it: numbered, planned, declared, stamped.
compose := func(under context.Context) (catalogue.Resolution, sendable) {
t.Helper()
numbered, err := allot(under, inv, "anchor")
if err != nil {
t.Fatal(err)
}
plan, settings, err := planFor(under, open, "anchor")
if err != nil {
t.Fatal(err)
}
gens, err := generators(under, open)
if err != nil {
t.Fatal(err)
}
declared, err := declarationWith(under, open, "anchor", plan, settings, gens, Allocating)
if err != nil {
t.Fatal(err)
}
numbered.stamp(&declared)
return plan, declared
}
record := func(declared sendable) {
t.Helper()
body, err := declared.Body()
if err != nil {
t.Fatal(err)
}
if _, err := recordSent(ctx, inv, "anchor", body, declared.Builds, declared.Epoch,
sentRecordOf(ctx, declared)); err != nil {
t.Fatal(err)
}
}
// Kept for anchor alone. laptop is never recorded as sent, on purpose: composing anchor resolves laptop
// too (who is on the private network), and a step that kept every machine's builds refused anchor's
// composition for want of laptop's last send (repo-check of #230 at 0f853e93).
kept := keepingEveryBuild(keepingRecorded(ctx), "anchor")
// What anchor was last sent: everything at the builds of before the merge, as an ordinary send sends it.
_, before := compose(keepingRecorded(ctx))
record(before)
plan, declared := compose(kept)
if only, err := grantsOnlyIn(ctx, open, "anchor", plan, declared); err != nil || only != "" {
t.Fatalf("with nothing changed, the step reads %q, %v", only, err)
}
// The merge: restic's c2 gives it a health tool.
if _, _, err := takeIn(ctx, inv, build("restic", "c2", start.Add(time.Minute),
resticFixture(true))); err != nil {
t.Fatal(err)
}
// The step's send to anchor: restic as anchor runs it, c1, and the user list granting c2's tool.
generation, err := inv.AssignmentGeneration(ctx)
if err != nil {
t.Fatal(err)
}
plan, declared = compose(kept)
for _, m := range plan.Modules {
if m.Module == "restic" && len(m.Tools) > 0 {
t.Fatalf("the grants step composed restic's new build on anchor: tools %v", m.Tools)
}
}
if declared.Builds["restic"] != "c1" {
t.Fatalf("the step's send records it carries restic %q; want c1, which anchor runs", declared.Builds["restic"])
}
if !strings.Contains(declared.BusUsers, "mesh.mod.restic.tool.backup_health.laptop") {
t.Errorf("the step's declaration does not grant laptop's node-engine restic's new tool:\n%s", declared.BusUsers)
}
// The generation it carries is the current one (novox/hq issue 234), and the step does not raise it.
if declared.composedFrom != generation {
t.Errorf("the step's send was composed from generation %d; the mesh is at %d", declared.composedFrom, generation)
}
// The guard, on the very declaration the send sends: only the user list differs from the last send.
if only, err := grantsOnlyIn(ctx, open, "anchor", plan, declared); err != nil || only != "" {
t.Fatalf("the step reads as changing more than the user list: %q, %v", only, err)
}
// And the gate's refusal still reads the step's send: it moves nothing there.
if names, err := ungatedIn(kept, open, []string{"anchor"}, ""); err != nil || strings.Join(names, ",") != "anchor" {
t.Fatalf("the step's send is refused as a move: %v, %v", names, err)
}
// Recorded as the send records it: anchor still runs restic c1, its gate still to come there.
record(declared)
if sent, _, err := inv.SentBuilds(ctx, "anchor"); err != nil || sent["restic"] != "c1" {
t.Fatalf("after the step anchor reads as sent restic %q (%v); want c1", sent["restic"], err)
}
if after, err := inv.AssignmentGeneration(ctx); err != nil || after != generation {
t.Fatalf("the step moved the assignment generation from %d to %d (%v)", generation, after, err)
}
// A module newly assigned to anchor: the step would install it, unjudged, so it is refused unsent.
if _, err := inv.Assign(ctx, "anchor", "wallpaper"); err != nil {
t.Fatal(err)
}
plan, declared = compose(kept)
only, err := grantsOnlyIn(ctx, open, "anchor", plan, declared)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(only, "wallpaper newly assigned") || !strings.Contains(only, "+wallpaper") {
t.Fatalf("a new assignment on the bus's machine reads %q; want the step refused, naming it", only)
}
// And the send itself refuses it, unsent: judged where it is composed, before the bus is dialled.
identity.ForTest(t)
if _, err := sendToEach(kept, open, []string{"anchor"}); !errors.Is(err, errNotGrantsOnly) ||
!strings.Contains(err.Error(), "wallpaper") {
t.Fatalf("the grants step's send of a new assignment was not refused by the send: %v", err)
}
}
// fixtureImage is the image every fixture container of the grants step's tests runs.
var fixtureImage = "registry.invalid:5000/restic/backup@sha256:" + strings.Repeat("b", 64)
// resticFixture is restic's manifest in the shape of mesh-catalog #210: at c2 (withTool) its backup judged by
// its new tool `backup_health`, beside a check of another kind it does not run itself — a tool check alone
// is refused (ADR 0227 rule 8, ADR 0240 rule 2) — and at c1 neither.
func resticFixture(withTool bool) map[string]any {
backup := map[string]any{"id": "backup", "type": "container", "name": "restic-backup", "image": fixtureImage}
measure := map[string]any{"id": "measure", "type": "container", "name": "restic-measure", "image": fixtureImage}
m := map[string]any{"resources": []any{backup, measure}}
if withTool {
backup["health"] = map[string]any{"kind": catalogue.HealthTool, "tool": "backup_health"}
measure["health"] = map[string]any{"kind": "runtime"}
m["tools"] = []string{"backup_health"}
}
return m
}
// natsFixture is the bus's module: it holds mesh-broker and is sent the user list.
func natsFixture() map[string]any {
return map[string]any{"bus-users": "/var/lib/nats-module/conf/accounts.conf",
"claims": []any{map[string]any{"name": catalogue.BrokerSeat, "scope": catalogue.ScopeMesh}}}
}
// wallpaperFixture is a module the bus's machine is newly assigned.
func wallpaperFixture() map[string]any {
return map[string]any{"resources": []any{
map[string]any{"id": "paper", "type": "container", "name": "wallpaper", "image": fixtureImage}}}
}
-226
View File
@@ -1,226 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The grants reach the bus before the gate (novox/hq issue 490).
//
// On 2026-10-11 a merge gave restic, assigned to every machine, a new health tool. Its walk sent restic to
// its first machine and judged it there; that machine's node-engine asked the new tool and the bus refused
// it, because the grant was in the bus's user list, which only the machine holding the bus carries — and
// that machine runs restic too, so it was left out of the send while restic's new build waited there for
// the very gate that could not pass. A person pushed it by hand, carrying restic's new build there unjudged.
// aSend is one send the walk made: to which machines, and whether every build there was kept.
type aSend struct {
names []string
keepsAll bool
judged []string
}
// sendsRecorded replaces the walk's send and the reading of the bus's user list for one test: the machine
// holding the bus is novox, and its list is behind as behind says.
func sendsRecorded(t *testing.T, holder string, behind bool, refuse error) *[]aSend {
t.Helper()
var sends []aSend
wasSend, wasBehind := sendRollout, brokerBehindOf
t.Cleanup(func() { sendRollout, brokerBehindOf = wasSend, wasBehind })
brokerBehindOf = func(_ context.Context, _ *stores, names []string) (string, bool, error) {
if slices.Contains(names, holder) {
return holder, false, nil
}
return holder, behind, nil
}
sendRollout = func(ctx context.Context, _ *stores, names []string) ([]string, error) {
s := aSend{names: append([]string(nil), names...), keepsAll: len(names) == 1 && everyBuildKept(ctx, names[0])}
for n := range scopeOf(ctx).judged {
s.judged = append(s.judged, n)
}
sends = append(sends, s)
if refuse != nil && s.keepsAll {
return nil, refuse
}
return names, nil
}
return &sends
}
// The restic shape: a new tool on a module on every machine, its first machine ace, the bus on novox.
func TestAWalkSendsTheGrantsToTheBusBeforeTheFirstMachineIsJudged(t *testing.T) {
sends := sendsRecorded(t, "novox", true, nil)
sent, grants, err := sendJudged(t.Context(), nil, "ace")
if err != nil {
t.Fatal(err)
}
if len(*sends) != 2 {
t.Fatalf("the walk made %d send(s), want the grants step and then the first machine: %+v", len(*sends), *sends)
}
first, then := (*sends)[0], (*sends)[1]
if strings.Join(first.names, ",") != "novox" || !first.keepsAll || len(first.judged) != 0 {
t.Errorf("the first send is %+v, want novox alone, every build there kept, judging nothing", first)
}
if strings.Join(then.names, ",") != "ace" || then.keepsAll || strings.Join(then.judged, ",") != "ace" {
t.Errorf("the second send is %+v, want ace, judged", then)
}
if strings.Join(sent, ",") != "ace" {
t.Errorf("the gate's machines are %v, want ace alone: the bus's machine is not judged", sent)
}
if grants == nil || grants.Node != "novox" || grants.At == nil || grants.Failed != "" {
t.Fatalf("the gate keeps %+v as its grants step", grants)
}
line := gateLine(&inventory.PlanGate{Machines: sent, Grants: grants})
if !strings.Contains(line, "grants step: novox sent the bus's user list first, its builds kept") {
t.Errorf("plans says the gate as %q", line)
}
}
// Nothing more when there is nothing to carry: the list unchanged, or the first machine holds the bus.
func TestAWalkTakesNoGrantsStepWhenTheListIsAlreadyThere(t *testing.T) {
for _, c := range []struct {
name string
node string
behind bool
}{{"the list unchanged", "ace", false}, {"the first machine holds the bus", "novox", true}} {
t.Run(c.name, func(t *testing.T) {
sends := sendsRecorded(t, "novox", c.behind, nil)
_, grants, err := sendJudged(t.Context(), nil, c.node)
if err != nil {
t.Fatal(err)
}
if len(*sends) != 1 || (*sends)[0].keepsAll || grants != nil {
t.Errorf("the walk made %+v with grants %+v, want the judged send alone", *sends, grants)
}
})
}
}
// A grants step that cannot be sent is said on the gate, and the walk goes on as it did before.
func TestAGrantsStepThatFailsIsSaidAndTheSendGoesOn(t *testing.T) {
sends := sendsRecorded(t, "novox", true, errors.New("a bus upgrade waits for a person"))
sent, grants, err := sendJudged(t.Context(), nil, "ace")
if err != nil {
t.Fatal(err)
}
if strings.Join(sent, ",") != "ace" || len(*sends) != 2 {
t.Errorf("the walk made %+v", *sends)
}
if grants == nil || grants.Failed == "" || grants.At != nil {
t.Fatalf("the gate keeps %+v", grants)
}
if line := gateLine(&inventory.PlanGate{Machines: sent, Grants: grants}); !strings.Contains(line,
"grants step to novox FAILED, sent without it: a bus upgrade waits for a person") {
t.Errorf("plans says the gate as %q", line)
}
}
// A step whose send would change more than the user list is refused unsent by the send (sendToEach): the
// gate says it FAILED with what differed, and the walk goes on.
func TestAGrantsStepThatWouldCarryMoreIsNotSent(t *testing.T) {
refusal := fmt.Errorf("%w: its send would change more than the bus's user list: -postgres.login-n8n", errNotGrantsOnly)
sends := sendsRecorded(t, "novox", true, refusal)
sent, grants, err := sendJudged(t.Context(), nil, "ace")
if err != nil {
t.Fatal(err)
}
if strings.Join(sent, ",") != "ace" || len(*sends) != 2 {
t.Errorf("the walk made %+v, want the refused step and then ace's judged send", *sends)
}
if grants == nil || grants.At != nil || !strings.Contains(grants.Failed, "-postgres.login-n8n") {
t.Fatalf("the gate keeps %+v", grants)
}
}
// The step's exemption is the list's content alone: its path or mode moving is a change like any other.
func TestOnlyTheUserListsContentIsExempt(t *testing.T) {
list := "nats.bus-users"
file := func(fields map[string]any) sentResource {
r := map[string]any{"id": list, "type": "file", "path": "/conf/accounts.conf", "mode": "0600", "content": "a"}
for k, v := range fields {
r[k] = v
}
body, _ := json.Marshal(map[string]any{"resources": []any{r}})
s, err := summarize(body)
if err != nil {
t.Fatal(err)
}
return s[0]
}
was := file(nil)
if other := otherThanTheList(diffSent([]sentResource{was}, []sentResource{file(map[string]any{"content": "b"})}), list); len(other) != 0 {
t.Errorf("the list's new content reads as more: %v", other)
}
for _, f := range []map[string]any{{"mode": "0644"}, {"path": "/elsewhere"}, {"content": "b", "owner": "nats"}} {
other := otherThanTheList(diffSent([]sentResource{was}, []sentResource{file(f)}), list)
if len(other) != 1 || !strings.HasPrefix(other[0], "~"+list) {
t.Errorf("the list's resource changed by %v reads %v; want it refused", f, other)
}
}
}
// The step keeps the builds of the machine it sends alone: the others its composition resolves are composed
// as any send composes them.
func TestTheGrantsStepKeepsOneMachinesBuilds(t *testing.T) {
ctx := keepingEveryBuild(t.Context(), "novox")
if !everyBuildKept(ctx, "novox") || everyBuildKept(ctx, "ace") || everyBuildKept(t.Context(), "novox") {
t.Error("the grants step keeps the builds of a machine it does not send")
}
}
// The store test's manifests are manifests the controller takes in: checked here, where no store is needed,
// so a fixture the module rules refuse fails before the store test is run (repo-check of #230 at 325575b2).
func TestTheGrantsStepFixturesAreManifests(t *testing.T) {
for name, m := range map[string]map[string]any{"nats": natsFixture(), "restic-c1": resticFixture(false),
"restic-c2": resticFixture(true), "wallpaper": wallpaperFixture()} {
m["module"], m["version"] = strings.SplitN(name, "-", 2)[0], "1"
raw, err := json.Marshal(m)
if err != nil {
t.Fatal(err)
}
if _, err := catalogue.ParseManifest(raw); err != nil {
t.Errorf("%s: %v", name, err)
}
}
}
// The delivery plan says the step before the merge.
func TestAChangePlanSaysTheGrantsStep(t *testing.T) {
const catalogue_ = "http://forge.internal:20000/novox/mesh-catalog.git"
entry := func(name string, on ...string) inventory.Entry {
e := fromRepo(name, catalogue_, "modules/"+name)
e.Source.BuiltFrom = "old"
e.On = on
return e
}
nats := entry("nats", "novox")
nats.Manifest.BusUsers = "/etc/nats/users.conf"
nats.Manifest.Claims = []catalogue.Claim{{Name: catalogue.BrokerSeat, Scope: catalogue.ScopeMesh}}
restic := entry("restic", "ace", "novox", "shanks")
only := entry("bus-tools", "novox")
entries := []inventory.Entry{nats, restic, only}
rollsOut := func(string) (inventory.Upgrade, bool) { return inventory.Upgrade{RollOut: true}, true }
plan := func(paths ...string) link.ChangePlan {
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: paths}
return changePlanOf("novox/mesh-catalog", "main", "head", reachOfMerge(m, entries, nil, nil), entries, rollsOut)
}
text := planText(plan("modules/restic/module.json"))
if !strings.Contains(text, "grants step: if this changes what the bus's user list says (a new tool, subject or "+
"user), novox is sent that list alone, every build there kept, before restic is judged on its first machine") {
t.Errorf("the change plan does not say the grants step:\n%s", text)
}
// A module only on the bus's machine goes there with the list in its own send: no step of its own.
if text := planText(plan("modules/bus-tools/module.json")); strings.Contains(text, "grants step") {
t.Errorf("a module on the bus's machine alone reads:\n%s", text)
}
}
-3
View File
@@ -84,9 +84,6 @@ var handActVerbs = []handActVerb{
"deliberate test, never a repair", DecidedFor: []string{causeDrill}},
// A drill: something broken on purpose to see the mesh raise and clear it. A person's test, never a
// repair, however often it is run.
// A credential a provider's software issued, taken from the provider and kept for its consumers (novox/hq ADR
// 0305): the mesh working as decided, however often the software changes it.
{Verb: handActIssued, Decision: "a credential the provider's software issued, kept for its consumers (ADR 0305)"},
{Verb: "hand-act drill", Decision: "a drill is a person's deliberate test of the mesh, never a repair"},
// A person's decisions by design.
{Verb: "retire approve", Decision: "nothing is retired past its bound without a person (ADR 0230)"},
+1 -4
View File
@@ -25,7 +25,6 @@ import (
// another send reaches anchor, unreported; the gate still passes. And a send that moves a judged module
// to another build supersedes the judging: no verdict, nothing put back.
func TestReplay352(t *testing.T) {
judgeLeavesAsTheCore(t)
t.Run("a newer send to the judged machine does not unreport the gate's", testANewerSendDoesNotUnreportTheGatesOwn)
t.Run("a send that moves the module supersedes the judging", testASendThatMovesTheModuleSupersedesTheJudging)
}
@@ -67,11 +66,10 @@ func testANewerSendDoesNotUnreportTheGatesOwn(t *testing.T) {
// A plan's own first send waits while a release judges the same module on that machine with another build.
func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) {
judgeLeavesAsTheCore(t)
b := aBacklog(t)
ctx := t.Context()
advancePlans(ctx, b.open) // the release judges app c2 on anchor
_, _, _, err := gatedSend(ctx, b.open, "anchor", []inventory.CarriedMove{{Module: "app", Node: "anchor", From: "c2", To: "c3", Build: "build-app-c3"}})
_, _, err := gatedSend(ctx, b.open, "anchor", []inventory.CarriedMove{{Module: "app", Node: "anchor", From: "c2", To: "c3", Build: "build-app-c3"}})
if !errors.Is(err, errWalkedElsewhere) || !strings.Contains(err.Error(), "release-") {
t.Fatalf("a newer build of a judged module was sent under the release's gate: %v", err)
}
@@ -82,7 +80,6 @@ func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) {
// A merge plan's judging is superseded the same way: another send moved its module on the first machine.
func TestAPlansJudgingIsSupersededByASendThatMovesItsModule(t *testing.T) {
judgeLeavesAsTheCore(t)
g := aGateMesh(t)
ctx := t.Context()
inv := g.open.inventory
-427
View File
@@ -1,427 +0,0 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"reflect"
"slices"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
)
// novox/hq issue 459, ADR 0299: since the merge window one walk carries every merge of its batch, so a stop of one
// group member through `delivery-stop` ended the walk of every other delivery in it. The stop now names the merges
// to leave out: the walk is ended, and every other merge it answered is walked again in a fresh batch without them
// — the stopped merge, and each later merge of its repository's branch, which contains it and ends with it.
// endsItsLine ends, as the test ends, a line the controller's output left open (a walk's ask prints " tier N: " and
// the asked build the rest, which the tests' fake ask never prints): `go test -v` would put the test's "--- PASS"
// after it, and the store-tests step of merge-check.sh, which reads that marker at a line's start, would say the
// test never ran.
func endsItsLine(t *testing.T) {
t.Cleanup(func() { fmt.Println() })
}
// leaved is a kept merge of a repository, made at a moment.
func leaved(repository, commit string, made time.Time) inventory.BatchedMerge {
return inventory.BatchedMerge{Repository: repository, Branch: "main", Commit: commit, Merged: made, Heard: made}
}
// The stopped merge is left out, and so is each later merge of its repository's branch, naming the merge it
// contains; an earlier merge of that repository and every other repository's merge are walked again.
func TestLeftOutAreTheStoppedMergeAndTheMergesThatContainIt(t *testing.T) {
c1 := leaved("novox/mesh-catalog", "c1aaaaaaaaaa", t0)
c2 := leaved("novox/mesh-catalog", "c2bbbbbbbbbb", t0.Add(10*time.Second))
c3 := leaved("novox/mesh-catalog", "c3cccccccccc", t0.Add(20*time.Second))
o1 := leaved("novox/one", "o1dddddddddd", t0.Add(5*time.Second))
again, out, err := leaveOut([]inventory.BatchedMerge{c1, o1, c2, c3}, []string{"novox/Mesh-Catalog@c2bbbbbbbbbb"})
if err != nil {
t.Fatal(err)
}
var walked []string
for _, m := range again {
walked = append(walked, m.Commit)
}
if !reflect.DeepEqual(walked, []string{c1.Commit, o1.Commit}) {
t.Fatalf("walked again: %v; want the earlier catalogue merge and the other repository's", walked)
}
want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: c2.Commit},
{Repository: "novox/mesh-catalog", Commit: c3.Commit, Contains: c2.Commit}}
if !reflect.DeepEqual(out, want) {
t.Fatalf("left out: %+v; want %+v", out, want)
}
// A merge the walk does not answer is refused, naming it: nothing ends.
if _, _, err := leaveOut([]inventory.BatchedMerge{c1, o1}, []string{"novox/mesh-catalog@ffffffffffff"}); err == nil ||
!strings.Contains(err.Error(), "ffffffffffff") {
t.Fatalf("a merge the walk does not answer was left out: %v", err)
}
if _, _, err := leaveOut([]inventory.BatchedMerge{c1}, []string{"no-commit"}); err == nil {
t.Fatal("a merge without its commit was taken")
}
}
// moving is a kept merge that moved modules when it was heard.
func moving(m inventory.BatchedMerge, modules ...string) inventory.BatchedMerge {
m.Event, _ = json.Marshal(keptMerge{Moves: modules})
return m
}
// A merge of another repository that moves a module the stopped merge moves is left out too, naming the modules
// it shares: the module is built at its branch, which holds the stopped merge (review of #232). And so on, until
// none more is: a merge sharing a module with that one is left out as well.
func TestAMergeSharingAStoppedMergesModuleIsLeftOut(t *testing.T) {
stopped := moving(leaved("novox/mesh-catalog", "c1aaaaaaaaaa", t0), "app")
reader := moving(leaved("novox/one", "o1bbbbbbbbbb", t0.Add(time.Second)), "app", "one")
next := moving(leaved("novox/two", "t1cccccccccc", t0.Add(2*time.Second)), "one")
free := moving(leaved("novox/three", "h1dddddddddd", t0.Add(3*time.Second)), "three")
again, out, err := leaveOut([]inventory.BatchedMerge{stopped, reader, next, free}, []string{"novox/mesh-catalog@c1aaaaaaa"})
if err != nil {
t.Fatal(err)
}
if len(again) != 1 || again[0].Commit != free.Commit {
t.Fatalf("walked again: %+v; want the merge that shares nothing alone", again)
}
want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: stopped.Commit},
{Repository: "novox/one", Commit: reader.Commit, Contains: stopped.Commit, Shares: []string{"app"}},
{Repository: "novox/two", Commit: next.Commit, Contains: stopped.Commit, Shares: []string{"one"}}}
if !reflect.DeepEqual(out, want) {
t.Fatalf("left out %+v; want %+v", out, want)
}
}
// A walk withholds a module a stopped merge moved where it would build it only as a dependent; one a merge of its
// own batch moves is built, and empty tiers go.
func TestAWalkWithholdsAStoppedMergesDependents(t *testing.T) {
plan := inventory.Plan{Tiers: [][]string{{"one"}, {"app"}, {"notes", "app2"}},
Modules: map[string]*inventory.PlanModule{"one": {}, "app": {}, "notes": {}, "app2": {}}}
got := withhold(&plan, []string{"app", "notes", "absent"}, []string{"notes"})
if !reflect.DeepEqual(got, []string{"app"}) || !reflect.DeepEqual(plan.Tiers, [][]string{{"one"}, {"notes", "app2"}}) {
t.Fatalf("withheld %v, tiers %v", got, plan.Tiers)
}
if _, in := plan.Modules["app"]; in {
t.Fatal("a withheld module is still in the plan")
}
}
// The seat verb passes the merges to leave out to the command.
func TestDeliveryStopTakesTheMergesToLeaveOut(t *testing.T) {
got, err := argvFor("delivery-stop", map[string]any{"plan": "plan-1", "why": "w", "by": "jochen",
"without": "novox/mesh-lab@b1,novox/mesh-lab@b2"})
want := []string{"delivery", "stop", "plan-1", "--why", "w", "--by", "mesh-delivery for jochen",
"--without", "novox/mesh-lab@b1,novox/mesh-lab@b2"}
if err != nil || !reflect.DeepEqual(got, want) {
t.Fatalf("delivery-stop with without → %v %v, wanted %v", got, err, want)
}
}
// A walk of the catalogue's merge and another repository's is stopped without the catalogue's: it ends stopped,
// names the batch that walks the other merge again and the merge it left out, and that batch is cut into a walk
// that builds the other repository's module alone.
func TestAStopWithoutAMergeWalksTheOthersAgain(t *testing.T) {
open := windowed(t)
endsItsLine(t)
asksRecorded(t)
ctx := t.Context()
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
one := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second))
hear(t, open, cat, t0.Add(time.Second))
hear(t, open, one, t0.Add(6*time.Second))
cutAt(t, open, t0.Add(2*time.Minute))
ws, _ := walks(t, open)
if len(ws) != 1 || !ws[0].Open() {
t.Fatalf("no open walk to stop: %+v", ws)
}
stopped := ws[0]
p, said, err := stopWalkWithout(ctx, open.inventory, stopped.ID, "mesh-delivery for jochen", "its group stopped",
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute))
if err != nil {
t.Fatal(err)
}
if p.State != inventory.PlanFailed || p.Delivery.Stopped == "" || p.Delivery.WalkedAgainBy == "" ||
!reflect.DeepEqual(p.Delivery.LeftOut, []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: cat.Commit}}) {
t.Fatalf("the stopped walk reads %s %+v", p.State, p.Delivery)
}
if said.WalkedAgainBy != p.Delivery.WalkedAgainBy || !reflect.DeepEqual(said.WalkedAgain, []string{"novox/one@" + one.Commit}) {
t.Fatalf("the stop answered %+v", said)
}
again, err := open.inventory.MergesOf(ctx, p.Delivery.WalkedAgainBy)
if err != nil || len(again) != 1 || again[0].Commit != one.Commit {
t.Fatalf("the fresh batch answers %+v (%v); want the other repository's merge alone", again, err)
}
if left, _ := open.inventory.MergesOf(ctx, stopped.ID); len(left) != 1 || left[0].Commit != cat.Commit {
t.Fatalf("the stopped walk still answers %+v; want the left-out merge alone", left)
}
cutAt(t, open, t0.Add(4*time.Minute))
fresh, err := open.inventory.PlanByID(ctx, p.Delivery.WalkedAgainBy)
if err != nil || fresh.Batch() || !fresh.Open() {
t.Fatalf("the fresh batch was not cut into a walk: %s %v", fresh.State, err)
}
if _, in := fresh.Modules["one"]; !in {
t.Fatalf("the fresh walk does not build one: %v", fresh.Modules)
}
if _, in := fresh.Modules["app"]; in {
t.Fatalf("the fresh walk builds app, whose merge was left out: %v", fresh.Modules)
}
if len(fresh.Delivery.Merges) != 1 || fresh.Delivery.Merges[0].Commit != one.Commit {
t.Fatalf("the fresh walk answers %+v", fresh.Delivery.Merges)
}
// A stopped walk starts no search of its own (ADR 0276 decision 3 is for a failed one).
cutAt(t, open, t0.Add(5*time.Minute))
if alone, _ := open.inventory.AloneMerges(ctx); len(alone) != 0 {
t.Fatalf("a merge of the stopped walk waits to be walked alone: %+v", alone)
}
}
// A later merge of the stopped merge's repository contains it: it is left out too, named with the merge it
// contains, and only the other repository's merge is walked again. With nothing left to walk, the walk is only
// ended. A merge the walk does not answer is refused, and nothing ends.
func TestAStopWithoutAMergeEndsTheMergesThatContainIt(t *testing.T) {
open := windowed(t)
endsItsLine(t)
asksRecorded(t)
ctx := t.Context()
first := catalogueMerge("cat1aaaaaaaa", "app", t0)
later := catalogueMerge("cat2cccccccc", "notes", t0.Add(10*time.Second))
one := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second))
hear(t, open, first, t0.Add(time.Second))
hear(t, open, one, t0.Add(6*time.Second))
hear(t, open, later, t0.Add(11*time.Second))
cutAt(t, open, t0.Add(2*time.Minute))
ws, _ := walks(t, open)
walk := ws[0]
if _, _, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "x",
[]string{"novox/mesh-catalog@ffffffffffff"}, t0.Add(3*time.Minute)); err == nil {
t.Fatal("a stop leaving out a merge the walk does not answer was taken")
}
if p, _ := open.inventory.PlanByID(ctx, walk.ID); !p.Open() {
t.Fatalf("a refused stop ended the walk: %s", p.State)
}
p, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped",
[]string{"novox/mesh-catalog@" + first.Commit}, t0.Add(3*time.Minute))
if err != nil {
t.Fatal(err)
}
want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: first.Commit},
{Repository: "novox/mesh-catalog", Commit: later.Commit, Contains: first.Commit}}
if !reflect.DeepEqual(p.Delivery.LeftOut, want) || !reflect.DeepEqual(said.LeftOut, want) {
t.Fatalf("left out %+v, said %+v; want %+v", p.Delivery.LeftOut, said.LeftOut, want)
}
if !strings.Contains(p.Note, later.Commit[:8]) || !strings.Contains(p.Note, p.Delivery.WalkedAgainBy) {
t.Fatalf("the stopped walk's note does not name the merge ended with it and the fresh batch: %q", p.Note)
}
again, _ := open.inventory.MergesOf(ctx, p.Delivery.WalkedAgainBy)
if len(again) != 1 || again[0].Commit != one.Commit {
t.Fatalf("walked again: %+v; want the other repository's merge alone", again)
}
// Nothing left once the other repository's merge is left out too: the fresh walk is stopped, and nothing more.
cutAt(t, open, t0.Add(4*time.Minute))
fresh, _ := open.inventory.PlanByID(ctx, p.Delivery.WalkedAgainBy)
q, said, err := stopWalkWithout(ctx, open.inventory, fresh.ID, "mesh-delivery for jochen", "that one too",
[]string{"novox/one@" + one.Commit}, t0.Add(5*time.Minute))
if err != nil || q.State != inventory.PlanFailed || q.Delivery.WalkedAgainBy != "" || said.WalkedAgainBy != "" {
t.Fatalf("a stop leaving nothing to walk: %s %+v %+v %v", q.State, q.Delivery, said, err)
}
if _, bs := walks(t, open); len(bs) != 0 {
t.Fatalf("a batch of nothing was made: %+v", bs)
}
}
// A merge heard while the walk ran waits in the open batch, at a later commit of the same repository. The merge
// walked again joins that batch, never a batch of its own that would walk the older commit after the newer one
// (review of #232): its window closes, and its walk is at the newer commit, answering both.
func TestAStopWalksTheOthersAgainInTheOpenBatch(t *testing.T) {
open := windowed(t)
endsItsLine(t)
asksRecorded(t)
ctx := t.Context()
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
one1 := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second))
hear(t, open, cat, t0.Add(time.Second))
hear(t, open, one1, t0.Add(6*time.Second))
cutAt(t, open, t0.Add(2*time.Minute))
ws, _ := walks(t, open)
walk := ws[0]
one2 := repoMerge("one", "one2cccccccc", t0.Add(3*time.Minute))
hear(t, open, one2, t0.Add(3*time.Minute))
_, bs := walks(t, open)
if len(bs) != 1 {
t.Fatalf("the merge heard while the walk ran is in %d batches", len(bs))
}
waiting := bs[0]
p, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped",
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second))
if err != nil {
t.Fatal(err)
}
if p.Delivery.WalkedAgainBy != waiting.ID || said.WalkedAgainBy != waiting.ID {
t.Fatalf("walked again by %q (said %q); want the open batch %s", p.Delivery.WalkedAgainBy, said.WalkedAgainBy, waiting.ID)
}
if _, bs := walks(t, open); len(bs) != 1 {
t.Fatalf("%d batches; want the open one alone", len(bs))
}
b, _ := open.inventory.PlanByID(ctx, waiting.ID)
if b.Delivery.Batch == nil || !b.Delivery.Batch.Closed || !slices.Contains(b.Delivery.Batch.Withheld, "app") {
t.Fatalf("the open batch reads %+v; want its window closed and app withheld", b.Delivery.Batch)
}
// Its window closed by the stop, not by the clock: cut at once.
cutAt(t, open, t0.Add(3*time.Minute+11*time.Second))
fresh, _ := open.inventory.PlanByID(ctx, waiting.ID)
if fresh.Batch() || fresh.CommitOf("novox/one") != one2.Commit {
t.Fatalf("the batch is %s at %q; want it cut, walking the newer commit", fresh.State, fresh.CommitOf("novox/one"))
}
var answered []string
for _, m := range fresh.Delivery.Merges {
answered = append(answered, m.Commit+">"+m.Carried)
}
if !slices.Contains(answered, one1.Commit+">"+one2.Commit) || !slices.Contains(answered, one2.Commit+">") {
t.Fatalf("the walk answers %v; want the earlier merge carried by the later", answered)
}
}
// A stop names what the stopped walk already sent and nothing walks again, on which machines; and a later merge of
// the stopped merge's repository that waits in a batch, which will deliver the stopped change. A stopped walk
// that walked its merges again is refused a retry, saying why.
func TestAStopNamesWhatStaysOnTheMachinesAndWhatStillCarriesIt(t *testing.T) {
open := windowed(t)
endsItsLine(t)
asksRecorded(t)
ctx := t.Context()
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
one1 := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second))
hear(t, open, cat, t0.Add(time.Second))
hear(t, open, one1, t0.Add(6*time.Second))
cutAt(t, open, t0.Add(2*time.Minute))
ws, _ := walks(t, open)
walk := ws[0]
sent := t0.Add(150 * time.Second)
for name, m := range walk.Modules {
if name == "app" || name == "one" {
m.First, m.FirstAt = []string{"anchor"}, &sent
}
}
if err := open.inventory.SavePlan(ctx, &walk); err != nil {
t.Fatal(err)
}
cat2 := catalogueMerge("cat2dddddddd", "notes", t0.Add(3*time.Minute))
hear(t, open, cat2, t0.Add(3*time.Minute))
_, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped",
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second))
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(said.Kept, []inventory.PlanKept{{Module: "app", Machines: []string{"anchor"}}}) {
t.Fatalf("kept %+v; want app on anchor alone (one is walked again)", said.Kept)
}
p, _ := open.inventory.PlanByID(ctx, walk.ID)
if !reflect.DeepEqual(p.Delivery.Kept, said.Kept) || !strings.Contains(p.Note, "app on anchor") {
t.Fatalf("the stopped walk's record does not name what stays: %+v %q", p.Delivery.Kept, p.Note)
}
if len(said.StillCarriedBy) != 1 || !strings.Contains(said.StillCarriedBy[0], cat2.Commit) {
t.Fatalf("still carried by %v; want the later catalogue merge waiting in the batch", said.StillCarriedBy)
}
if _, err := retryPlan(ctx, open, walk.ID); err == nil || !strings.Contains(err.Error(), "was stopped without") {
t.Fatalf("a walk stopped without some merges was retried, or refused for another reason: %v", err)
}
}
// A batch not yet cut is stopped without a merge: the merge goes to a stopped record of its own, the batch goes on
// with the others and withholds the stopped merge's modules; with every merge left out, the batch is stopped.
func TestAStopWithoutAMergeLeavesItOutOfABatch(t *testing.T) {
open := windowed(t)
endsItsLine(t)
asksRecorded(t)
ctx := t.Context()
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
one := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second))
hear(t, open, cat, t0.Add(time.Second))
hear(t, open, one, t0.Add(6*time.Second))
_, bs := walks(t, open)
b := bs[0]
r, said, err := stopWalkWithout(ctx, open.inventory, b.ID, "mesh-delivery for jochen", "its group stopped",
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(10*time.Second))
if err != nil {
t.Fatal(err)
}
if said.WalkedAgainBy != b.ID || said.Stopped == b.ID {
t.Fatalf("said %+v; want a stopped record of its own, and the batch walking the others", said)
}
if left, _ := open.inventory.MergesOf(ctx, said.Stopped); len(left) != 1 || left[0].Commit != cat.Commit {
t.Fatalf("the stopped record holds %+v", left)
}
if stopped, _ := open.inventory.PlanByID(ctx, said.Stopped); stopped.State != inventory.PlanFailed ||
stopped.Delivery.Stopped == "" || stopped.Delivery.WalkedAgainBy != b.ID {
t.Fatalf("the stopped record reads %s %+v", stopped.State, stopped.Delivery)
}
_ = r
cutAt(t, open, t0.Add(3*time.Minute))
w, _ := open.inventory.PlanByID(ctx, b.ID)
if _, in := w.Modules["app"]; in || w.Batch() {
t.Fatalf("the batch's walk is %s and builds %v; want it cut without app", w.State, w.Modules)
}
// A batch whose every merge is left out is stopped itself.
hear(t, open, catalogueMerge("cat3eeeeeeee", "notes", t0.Add(4*time.Minute)), t0.Add(4*time.Minute))
_, bs = walks(t, open)
last := bs[0]
if _, said, err := stopWalkWithout(ctx, open.inventory, last.ID, "mesh-delivery for jochen", "x",
[]string{"novox/mesh-catalog@cat3eeeeeeee"}, t0.Add(4*time.Minute+time.Second)); err != nil || said.Stopped != last.ID {
t.Fatalf("a batch left with nothing: %+v %v", said, err)
}
if p, _ := open.inventory.PlanByID(ctx, last.ID); p.State != inventory.PlanFailed || p.Batch() {
t.Fatalf("the emptied batch is %s", p.State)
}
}
// The stopped walk, the batch and the merges moved are one act (ADR 0299 decision 7): a plan written by somebody
// else since it was read refuses all of it, and no merge moves, no plan is written.
func TestSavingPlansAndMovingMergesIsOneAct(t *testing.T) {
open := windowed(t)
endsItsLine(t)
ctx := t.Context()
hear(t, open, catalogueMerge("cat1aaaaaaaa", "app", t0), t0.Add(time.Second))
_, bs := walks(t, open)
stale := bs[0]
moved := stale
moved.Note = "written by somebody else"
if err := open.inventory.SavePlan(ctx, &moved); err != nil {
t.Fatal(err)
}
fresh := inventory.Plan{ID: "plan-fresh", Repository: "novox/mesh-catalog", Branch: "main", Commit: "x", Created: t0,
State: inventory.PlanAssembling, Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{}}
err := open.inventory.SavePlansMoving(ctx, []*inventory.Plan{&fresh, &stale},
[]inventory.MergeMove{{Repository: "novox/mesh-catalog", Commit: "cat1aaaaaaaa", Plan: "plan-fresh"}})
if !errors.Is(err, inventory.ErrPlanMoved) {
t.Fatalf("a stale plan in the act: %v", err)
}
if _, err := open.inventory.PlanByID(ctx, "plan-fresh"); err == nil {
t.Fatal("the new plan was written though the act was refused")
}
if m, _, _ := open.inventory.MergeOf(ctx, "novox/mesh-catalog", "cat1aaaaaaaa"); m.Plan != stale.ID {
t.Fatalf("the merge moved to %q though the act was refused", m.Plan)
}
}
// What a stopped walk leaves on the machines names every machine its sends reached (review of #232): a module sent
// with no first machine reached every machine running it; one sent first reached those, and the rest it was sent
// to; a module a merge walked again moves is sent again, and is not named.
func TestKeptNamesEveryMachineASendReached(t *testing.T) {
at := t0
p := inventory.Plan{Modules: map[string]*inventory.PlanModule{
"together": {SentAt: &at},
"first": {First: []string{"anchor"}, FirstAt: &at, Rest: map[string]inventory.SentDeclaration{"laptop": {}}},
"again": {SentAt: &at},
"unsent": {},
}}
got := keptOf(p, []string{"again"}, map[string][]string{"together": {"laptop", "anchor"}, "again": {"anchor"}})
want := []inventory.PlanKept{{Module: "first", Machines: []string{"anchor", "laptop"}},
{Module: "together", Machines: []string{"anchor", "laptop"}}}
if !reflect.DeepEqual(got, want) {
t.Fatalf("kept %+v; want %+v", got, want)
}
}
-272
View File
@@ -1,272 +0,0 @@
package main
import (
"context"
"errors"
"fmt"
"sort"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/secrets"
)
// `secret issued <provision>`: the credential a provider's software issued, taken from the provider and kept as
// the provision's one credential for every consumer (novox/hq ADR 0305, issue 526).
//
// **The value comes from the provider, never from whoever asks.** A module served by its machine's runtime calls
// under the runtime's one bus account, which carries that machine's agents' calls too, so a value handed to a
// verb could be anybody's. This line carries none: it asks the provider's own tool, on the provider's machine, in
// the controller's own name — the subject that names its caller, which the bus lets the controller alone publish
// (issue 365) — with a sealing key made for this one call. The tool answers only the controller, the value sealed
// to that key; the controller opens it, seals it to every holder and forgets it, as the desk's value is (ADR 0277
// §23). Asking again fetches the same value again, and a value every holder already holds is not sealed anew.
//
// **What this trusts: the provider's machine.** The answer is taken from whoever answers the provider's tool
// subject on the provider's machine first. On that machine its runtime serves the subject for every module it
// carries, so anything that runtime runs, or anything else granted that subject there, could answer before the
// provider does and hand the controller a value of its own. ADR 0305 accepts that by trusting the provider's
// machine, as the provider itself is trusted; the bus keeps every other machine from answering it.
// handActIssued is the verb a change of an issued value is recorded under: the mesh working as decided.
const handActIssued = "secret issued"
// issuedAsk is how long the provider's tool is given to answer.
const issuedAsk = 20 * time.Second
// issuedProvider is one assignment of a module whose offer says its software issues the provision's credential.
type issuedProvider struct {
Node, Module, Tool string
}
// issuedTaker is the line's work, with what it reaches handed in: the store, the bus and the sends.
type issuedTaker struct {
// providers are the assignments to ask, and whether any module offers the provision so at all.
providers func(provision string) ([]issuedProvider, bool, error)
ask func(p issuedProvider, args map[string]any) (link.Answer, error)
keep func(provision, provider, value string) (inventory.IssuedKept, error)
record func(act link.HandAct) error
send func(machines []string) error
}
// take asks every provider of the provision for its software's value and keeps each; it answers what it did, in
// words that never carry the value or its fingerprint.
func (t issuedTaker) take(provision string) (string, error) {
providers, offered, err := t.providers(provision)
if err != nil {
return "", err
}
if !offered {
return "", fmt.Errorf("no module offers %q with a credential its software issues (\"credential\": "+
"{\"issued-by\": \"software\", \"tool\": …}, ADR 0305), so there is nothing to take from a provider; "+
"nothing was asked", provision)
}
if len(providers) == 0 {
return fmt.Sprintf("no consumer holds %s from a provider whose software issues it yet, so there is nothing "+
"to seal; a consumer that binds holds a value the mesh made until the provider next asks", provision), nil
}
var said []string
var failed []string
for _, p := range providers {
words, err := t.takeFrom(provision, p)
if err != nil {
failed = append(failed, fmt.Sprintf("%s on %s: %v", p.Module, p.Node, err))
continue
}
said = append(said, words)
}
out := strings.Join(said, "\n")
if len(failed) > 0 {
return out, fmt.Errorf("%s was not taken from every provider:\n %s", provision, strings.Join(failed, "\n "))
}
return out, nil
}
// takeFrom is one provider: asked, opened, kept, recorded and sent when it changed.
func (t issuedTaker) takeFrom(provision string, p issuedProvider) (string, error) {
public, private, err := secrets.Keypair()
if err != nil {
return "", fmt.Errorf("no key could be made to take the value: %w", err)
}
answer, err := t.ask(p, map[string]any{"provision": provision, "seal_to": public})
if err != nil {
return "", fmt.Errorf("its tool %s could not be asked: %w", p.Tool, err)
}
if answer.Error != "" {
return "", fmt.Errorf("its tool %s refused: %s", p.Tool, answer.Error)
}
var given struct {
Sealed string `json:"sealed"`
}
if err := unmarshalAnswer(answer, &given); err != nil || given.Sealed == "" {
return "", fmt.Errorf("its tool %s answered no sealed value", p.Tool)
}
opened, err := secrets.Open(private, given.Sealed)
if err != nil {
// Never the value, never what failed to open: only that it was not sealed to this call.
return "", fmt.Errorf("its tool %s answered a value not sealed to this call; nothing was taken", p.Tool)
}
value := strings.TrimSpace(string(opened))
for i := range opened {
opened[i] = 0
}
if value == "" {
return "", fmt.Errorf("its tool %s answered an empty value; nothing was taken", p.Tool)
}
kept, err := t.keep(provision, p.Node, value)
value = ""
if err != nil {
return "", err
}
holders := make([]string, 0, len(kept.Holders))
for _, h := range kept.Holders {
holders = append(holders, fmt.Sprintf("%s on %s%s", h.ConsumerModule, h.Consumer, asLocal(h.Local)))
}
words := ""
if len(kept.Unsealable) > 0 {
var names []string
for _, h := range kept.Unsealable {
names = append(names, fmt.Sprintf("%s on %s", h.ConsumerModule, h.Consumer))
}
words = fmt.Sprintf("\n not sealed to %s: no sealing key on that machine", strings.Join(names, ", "))
}
if !kept.Changed {
return fmt.Sprintf("%s from %s on %s: every holder already holds the value its software issued (%s); "+
"nothing was sent%s", provision, p.Module, p.Node, strings.Join(holders, ", "), words), nil
}
sealedTo := make([]string, 0, len(kept.Sealed))
for _, h := range kept.Sealed {
sealedTo = append(sealedTo, fmt.Sprintf("%s on %s%s", h.ConsumerModule, h.Consumer, asLocal(h.Local)))
}
recorded := ""
if err := t.record(link.HandAct{Verb: handActIssued, Args: []string{provision},
Why: fmt.Sprintf("%s on %s gave the credential its software issued for %s; sealed to %s", p.Module, p.Node,
provision, strings.Join(sealedTo, ", ")),
Cause: "issued-by-software"}); err != nil {
recorded = fmt.Sprintf("\n this was NOT recorded in the hand-act log, and is done anyway: %v", err)
}
if err := t.send(kept.Machines); err != nil {
return "", fmt.Errorf("%s from %s on %s is sealed to %s and could not be sent: %w; `push --behind` sends it",
provision, p.Module, p.Node, strings.Join(sealedTo, ", "), err)
}
return fmt.Sprintf("%s from %s on %s: the value its software issued is sealed to %s; sent %s, and each of "+
"them starts again on it%s%s", provision, p.Module, p.Node, strings.Join(sealedTo, ", "),
strings.Join(kept.Machines, ", "), words, recorded), nil
}
// issuedProvidersOf are the assignments to ask for a provision: every machine providing it to a pair the mesh
// holds, and on it the assigned module whose offer says its software issues the credential (ADR 0305 §1).
func issuedProvidersOf(ctx context.Context, inv *inventory.Inventory, provision string) ([]issuedProvider, bool, error) {
shelf, err := inv.Catalogue(ctx)
if err != nil {
return nil, false, err
}
if !offersIssued(shelf, provision) {
return nil, false, nil
}
holders, err := inv.HoldersOf(ctx, provision, "")
if err != nil {
return nil, true, err
}
nodes := map[string]bool{}
for _, h := range holders {
nodes[h.Provider] = true
}
names := make([]string, 0, len(nodes))
for n := range nodes {
names = append(names, n)
}
sort.Strings(names)
var out []issuedProvider
for _, node := range names {
assigned, err := inv.Assigned(ctx, node)
if err != nil {
return nil, true, err
}
out = append(out, issuedOn(shelf, node, assigned, provision)...)
}
return out, true, nil
}
// offersIssued is whether any module offers the provision with a credential its software issues.
func offersIssued(shelf map[string]catalogue.Manifest, provision string) bool {
for _, m := range shelf {
if _, issued := m.IssuedCredentialOf(provision); issued {
return true
}
}
return false
}
// issuedOn are the modules assigned to one machine whose offer says their software issues the provision's
// credential, each with the tool that answers it.
func issuedOn(shelf map[string]catalogue.Manifest, node string, assigned []string, provision string) []issuedProvider {
var out []issuedProvider
for _, module := range assigned {
if tool, issued := shelf[module].IssuedCredentialOf(provision); issued {
out = append(out, issuedProvider{Node: node, Module: module, Tool: tool})
}
}
return out
}
// issuedRefusesRotation refuses `rotate` of a provision its provider's software issues (ADR 0305 §9): a value the
// mesh made would replace one the software holds, and every consumer would hold a dead credential until the
// provider next asks.
func issuedRefusesRotation(shelf map[string]catalogue.Manifest, provision string) error {
if !offersIssued(shelf, provision) {
return nil
}
return fmt.Errorf("%s's credential is issued by its provider's software, which the mesh cannot make or "+
"rotate (ADR 0305): change it in that software, and `secret issued %s` takes the value it holds now. "+
"Nothing was rotated", provision, provision)
}
// secretIssued is `secret issued <provision>`, on this controller's stores and bus.
func secretIssued(ctx context.Context, args []string) error {
if len(args) != 1 || args[0] == "" || strings.HasPrefix(args[0], "-") {
return errors.New("secret issued <provision>")
}
provision := args[0]
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
t := issuedTaker{
providers: func(provision string) ([]issuedProvider, bool, error) {
return issuedProvidersOf(ctx, open.inventory, provision)
},
ask: func(p issuedProvider, args map[string]any) (link.Answer, error) {
var answer link.Answer
err := onTheBus(func(conn *nats.Conn) error {
var err error
answer, err = link.AskModuleToolOnAs(ctx, conn, broker.ControllerName, p.Module, p.Tool, p.Node,
args, issuedAsk)
return err
})
return answer, err
},
keep: func(provision, provider, value string) (inventory.IssuedKept, error) {
return open.inventory.KeepIssuedSecret(ctx, provision, provider, value)
},
record: func(act link.HandAct) error {
return onTheBus(func(conn *nats.Conn) error {
_, err := link.RecordHandAct(ctx, conn, act)
return err
})
},
send: func(machines []string) error { return sendTo(ctx, open, machines) },
}
words, err := t.take(provision)
if words != "" {
fmt.Println(words)
}
return err
}
-218
View File
@@ -1,218 +0,0 @@
package main
import (
"encoding/json"
"errors"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/secrets"
)
// The `issued` verb runs `secret issued <provision>` and nothing else, and that line is the one `secret` line
// beside rotate and the desk's that a verb may run (novox/hq ADR 0305 §3, §10, issue 526).
func TestTheIssuedVerbRunsTheOneLineThatCarriesNoValue(t *testing.T) {
argv, err := argvFor("issued", map[string]any{"provision": "plex-api"})
if err != nil {
t.Fatal(err)
}
if strings.Join(argv, " ") != "secret issued plex-api" {
t.Fatalf("the verb composed %q", argv)
}
if _, err := argvFor("issued", map[string]any{}); err == nil {
t.Fatal("the verb ran with no provision")
}
if _, err := argvFor("issued", map[string]any{"provision": "plex-api", "secret": "a value"}); err == nil {
t.Fatal("the verb took a value beside the provision")
}
for _, line := range [][]string{
{"secret", "issued", "plex-api", "--from", "/tmp/x"},
{"secret", "issued", "--from"},
{"secret", "issued"},
{"secret", "accept", "ace", "kometa", "plex-api", "--provider", "ace"},
{"secret", "recover", "ace", "kometa", "plex-api"},
} {
if err := terminalOnly(line); err == nil {
t.Errorf("%q ran through a verb", line)
}
}
if err := terminalOnly([]string{"secret", "issued", "plex-api"}); err != nil {
t.Fatalf("the issued line was refused through its verb: %v", err)
}
found := false
for _, v := range catalogue.ControllerVerbs {
if v.Name == "issued" {
found = true
}
}
if !found {
t.Fatal("the controller's seat does not declare the issued verb")
}
}
// A provider's tool answering the value sealed to the call's key: what a provider does (ADR 0305 §2).
func issuedSealed(t *testing.T, value string) func(issuedProvider, map[string]any) (link.Answer, error) {
return func(_ issuedProvider, args map[string]any) (link.Answer, error) {
key, _ := args["seal_to"].(string)
if key == "" {
t.Fatal("the provider was asked without a key to seal to")
}
if _, carried := args["value"]; carried {
t.Fatal("the ask carried a value")
}
sealed, err := secrets.Seal(key, []byte(value))
if err != nil {
t.Fatal(err)
}
raw, _ := json.Marshal(map[string]string{"sealed": sealed})
return link.Answer{Result: raw}, nil
}
}
type issuedRecorder struct {
kept []string
acts []link.HandAct
sent [][]string
again bool // the value is held by every holder already
}
func (r *issuedRecorder) taker(t *testing.T, ask func(issuedProvider, map[string]any) (link.Answer, error)) issuedTaker {
return issuedTaker{
providers: func(provision string) ([]issuedProvider, bool, error) {
return []issuedProvider{{Node: "ace", Module: "plex", Tool: "plex_issued_credential"}}, true, nil
},
ask: ask,
keep: func(provision, provider, value string) (inventory.IssuedKept, error) {
r.kept = append(r.kept, value)
holders := []inventory.Holder{{Provision: provision, Consumer: "ace", ConsumerModule: "kometa", Provider: provider},
{Provision: provision, Consumer: "ace", ConsumerModule: "tautulli", Provider: provider}}
if r.again {
return inventory.IssuedKept{Holders: holders}, nil
}
return inventory.IssuedKept{Changed: true, Holders: holders, Sealed: holders, Machines: []string{"ace"}}, nil
},
record: func(act link.HandAct) error { r.acts = append(r.acts, act); return nil },
send: func(machines []string) error { r.sent = append(r.sent, machines); return nil },
}
}
// The value the provider's tool sealed to the call is kept, recorded and sent, and the answer never says it.
func TestAnIssuedValueIsTakenSealedFromTheProviderAndSent(t *testing.T) {
var r issuedRecorder
words, err := r.taker(t, issuedSealed(t, "the-servers-own-token")).take("plex-api")
if err != nil {
t.Fatal(err)
}
if len(r.kept) != 1 || r.kept[0] != "the-servers-own-token" {
t.Fatalf("the provider's value was not kept: %d kept", len(r.kept))
}
if len(r.sent) != 1 || strings.Join(r.sent[0], ",") != "ace" {
t.Fatalf("the holders' machines were not sent: %v", r.sent)
}
if len(r.acts) != 1 || r.acts[0].Verb != handActIssued {
t.Fatalf("the change was not recorded in the hand-act log: %+v", r.acts)
}
for _, said := range []string{words, r.acts[0].Why, strings.Join(r.acts[0].Args, " ")} {
if strings.Contains(said, "the-servers-own-token") || strings.Contains(said, inventory.IssuedStamp("the-servers-own-token")) {
t.Fatalf("the value or its fingerprint was said: %q", said)
}
}
if !strings.Contains(words, "kometa on ace") || !strings.Contains(words, "tautulli on ace") {
t.Fatalf("the answer does not name the holders: %q", words)
}
}
// The same value again writes nothing, records nothing and sends nobody (ADR 0305 §5).
func TestAnIssuedValueEveryHolderHoldsSendsNobody(t *testing.T) {
r := issuedRecorder{again: true}
words, err := r.taker(t, issuedSealed(t, "the-servers-own-token")).take("plex-api")
if err != nil {
t.Fatal(err)
}
if len(r.sent) != 0 || len(r.acts) != 0 {
t.Fatalf("an unchanged value was sent or recorded: %v %v", r.sent, r.acts)
}
if !strings.Contains(words, "nothing was sent") {
t.Fatalf("the answer does not say nothing was sent: %q", words)
}
}
// A value not sealed to the call's key is not taken, nor is a refusal, nor an empty answer (ADR 0305 §2).
func TestOnlyAValueSealedToTheCallIsTaken(t *testing.T) {
other, _, _ := secrets.Keypair()
elsewhere, _ := secrets.Seal(other, []byte("somebody's value"))
raw, _ := json.Marshal(map[string]string{"sealed": elsewhere})
plain, _ := json.Marshal(map[string]string{"value": "the-servers-own-token"})
for name, ask := range map[string]func(issuedProvider, map[string]any) (link.Answer, error){
"sealed to another key": func(issuedProvider, map[string]any) (link.Answer, error) { return link.Answer{Result: raw}, nil },
"answered in the clear": func(issuedProvider, map[string]any) (link.Answer, error) { return link.Answer{Result: plain}, nil },
"refused": func(issuedProvider, map[string]any) (link.Answer, error) {
return link.Answer{Error: "the controller alone"}, nil
},
"not reached": func(issuedProvider, map[string]any) (link.Answer, error) {
return link.Answer{}, errors.New("503 no responders")
},
"sealed empty": issuedSealed(t, " "),
} {
var r issuedRecorder
if _, err := r.taker(t, ask).take("plex-api"); err == nil {
t.Errorf("%s: taken", name)
}
if len(r.kept) != 0 || len(r.sent) != 0 {
t.Errorf("%s: something was kept or sent", name)
}
}
}
// A provision no offer says its software issues is refused before anything is asked.
func TestAProvisionNoOfferSaysIsIssuedIsRefused(t *testing.T) {
asked := false
taker := issuedTaker{
providers: func(string) ([]issuedProvider, bool, error) { return nil, false, nil },
ask: func(issuedProvider, map[string]any) (link.Answer, error) {
asked = true
return link.Answer{}, nil
},
}
if _, err := taker.take("postgres-database"); err == nil || !strings.Contains(err.Error(), "ADR 0305") {
t.Fatalf("expected a refusal naming the record, got %v", err)
}
if asked {
t.Fatal("a provider was asked for a provision no offer says its software issues")
}
}
// Which assignments are asked: the modules on the providing machine whose offer says so, with their tool.
func TestTheProvidersAskedAreTheOffersThatSaySo(t *testing.T) {
shelf := map[string]catalogue.Manifest{
"plex": {Module: "plex", Provides: []catalogue.Offer{{Name: "plex-api", Credential: &catalogue.OfferCredential{
IssuedBy: catalogue.IssuedBySoftware, Tool: "plex_issued_credential"}}}},
"jellyfin": {Module: "jellyfin", Provides: []catalogue.Offer{{Name: "plex-api"}}},
}
got := issuedOn(shelf, "ace", []string{"jellyfin", "plex", "kometa"}, "plex-api")
if len(got) != 1 || got[0] != (issuedProvider{Node: "ace", Module: "plex", Tool: "plex_issued_credential"}) {
t.Fatalf("the providers asked: %+v", got)
}
if err := issuedRefusesRotation(shelf, "plex-api"); err == nil || !strings.Contains(err.Error(), "secret issued plex-api") {
t.Fatalf("rotate of an issued provision was not refused with the way to take it: %v", err)
}
if err := issuedRefusesRotation(shelf, "postgres-database"); err != nil {
t.Fatalf("rotate of an ordinary provision was refused: %v", err)
}
}
// A change of an issued value is the mesh working as decided, never a repair (ADR 0305 §8).
func TestAnIssuedValueIsADecisionInTheHandActLog(t *testing.T) {
for _, v := range handActVerbs {
if v.Verb == handActIssued {
if v.Decision == "" {
t.Fatal("secret issued counts as a repair")
}
return
}
}
t.Fatal("secret issued is not in the hand-act table")
}
-160
View File
@@ -1,160 +0,0 @@
package main
import (
"encoding/json"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
)
// novox/hq issue 485: the mesh records every send — sender, lease epoch, generation, modules — and only `plan`
// said it. `status` says per machine when it was last sent a declaration and by whom, in one line; `node show`
// says the last send in full and what the machine answered; and `status --json` carries it as fields, since
// the store's reader may not read the controller's tables and these verbs are the only window onto them.
func aLastSend(answer inventory.SendAnswer) inventory.Send {
sent := time.Date(2026, 10, 11, 18, 40, 5, 0, time.Local)
return inventory.Send{NodeName: "laptop", Sequence: 12, Epoch: 57,
Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12",
Modules: []string{"docker", "sudo"}, SentAt: sent, Recorded: true, Answer: answer}
}
func TestStatusSaysEachMachinesLastSendInOneLine(t *testing.T) {
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.Local)
asked := answers{
nodes: []inventory.Node{{Name: "anchor", LastSeen: time.Now()}, {Name: "laptop", LastSeen: time.Now()}},
lastSent: map[string]inventory.Send{"laptop": aLastSend(inventory.SendAnswer{
Outcome: inventory.OutcomeApplied, At: &at})},
}
shown := printed(t, func() error { return printStatus(asked) })
var laptop, anchor []string
for _, line := range strings.Split(shown, "\n") {
fields := strings.Fields(line)
if len(fields) > 0 && fields[0] == "laptop" {
laptop = append(laptop, line)
}
if len(fields) > 0 && fields[0] == "anchor" {
anchor = append(anchor, line)
}
}
if len(laptop) != 1 || !strings.Contains(laptop[0], "2026-10-11 18:40") ||
!strings.Contains(laptop[0], "by a one-shot push by jochen at a shell on anchor") ||
!strings.Contains(laptop[0], "generation 40") || !strings.Contains(laptop[0], "applied") {
t.Fatalf("status does not say laptop's last send in one line:\n%s", shown)
}
if len(anchor) != 1 || !strings.Contains(anchor[0], "no send recorded") {
t.Fatalf("status does not say anchor has no send recorded, in one line:\n%s", shown)
}
}
func TestStatusSaysTheLastSendsCouldNotBeRead(t *testing.T) {
asked := answers{nodes: []inventory.Node{{Name: "laptop", LastSeen: time.Now()}},
lastSentUnread: "the store went away"}
if shown := printed(t, func() error { return printStatus(asked) }); !strings.Contains(shown, "the store went away") {
t.Fatalf("a record of sends that could not be read is not said:\n%s", shown)
}
if asked.well() {
t.Error("a mesh whose sends could not be read is called well")
}
}
func TestNodeShowSaysTheLastSendInFull(t *testing.T) {
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.Local)
refused := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeRefused,
Refused: "unknown field \"generation\"\nsecond line", At: &at})), "\n")
for _, want := range []string{"sequence 12", "2026-10-11 18:40:05", "a one-shot push by jochen at a shell on anchor",
"lease epoch 57", "generation 40", "docker, sudo", "refused", "unknown field \"generation\""} {
if !strings.Contains(refused, want) {
t.Errorf("node show's last send does not say %q:\n%s", want, refused)
}
}
if applied := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeApplied,
At: &at})), "\n"); !strings.Contains(applied, "applied it at 2026-10-11 18:41:00") {
t.Errorf("an applied send is not said applied:\n%s", applied)
}
if waiting := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{})), "\n"); !strings.Contains(waiting,
"not reported on it yet") {
t.Errorf("a send not reported on is not said so:\n%s", waiting)
}
stale := aLastSend(inventory.SendAnswer{})
when := at
stale.RefusedAt, stale.RefusedApplied = &when, 44
if s := strings.Join(lastSendLines(stale), "\n"); !strings.Contains(s, "refused it at 2026-10-11 18:41:00") ||
!strings.Contains(s, "generation 44") {
t.Errorf("a send refused for its generation is not said so:\n%s", s)
}
}
func TestStatusJSONCarriesEachMachinesLastSend(t *testing.T) {
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.UTC)
asked := answers{nodes: []inventory.Node{{Name: "anchor"}, {Name: "laptop"}},
lastSent: map[string]inventory.Send{"laptop": aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeRefused,
Refused: "unknown field", At: &at})}}
body, err := statusAsJSON(asked)
if err != nil {
t.Fatal(err)
}
var got struct {
LastSent []map[string]any `json:"lastSent"`
}
if err := json.Unmarshal(body, &got); err != nil {
t.Fatal(err)
}
if len(got.LastSent) != 1 {
t.Fatalf("status --json carries %d last sends, want laptop's only:\n%s", len(got.LastSent), body)
}
s := got.LastSent[0]
if s["node"] != "laptop" || s["sequence"] != float64(12) || s["epoch"] != float64(57) ||
s["generation"] != float64(40) || s["sender"] != "a one-shot push by jochen at a shell on anchor" ||
s["digest"] != "d12" || s["sentAt"] == nil || s["answer"] != "refused" || s["refused"] != "unknown field" ||
s["answeredAt"] == nil {
t.Fatalf("laptop's last send reads %v", s)
}
if modules, _ := s["modules"].([]any); len(modules) != 2 || modules[0] != "docker" {
t.Fatalf("the modules it named read %v", s["modules"])
}
}
// And through the store: a send recorded and refused by the machine is what `node show` and `status --json`
// read back.
func TestNodeShowAndStatusReadTheLastSendFromTheStore(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
record, err := open.inventory.NodeByName(ctx, "laptop")
if err != nil {
t.Fatal(err)
}
if err := open.inventory.RecordSentWith(ctx, record.ID, "d12", nil, 57, 40, inventory.Send{Sequence: 12,
Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12",
Modules: []string{"docker", "sudo"}}); err != nil {
t.Fatal(err)
}
if _, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeRefused,
Declared: "d12", Refused: "unknown field \"generation\""}); err != nil {
t.Fatal(err)
}
shown := printed(t, func() error { return showNode(ctx, open, "laptop") })
for _, want := range []string{"a one-shot push by jochen at a shell on anchor", "lease epoch 57", "generation 40",
"docker, sudo", "unknown field \"generation\""} {
if !strings.Contains(shown, want) {
t.Errorf("node show laptop does not say %q:\n%s", want, shown)
}
}
body := printed(t, func() error { return statusFor(ctx, open, true) })
var got struct {
LastSent []struct {
Node string `json:"node"`
Sender string `json:"sender"`
Answer string `json:"answer"`
} `json:"lastSent"`
}
if err := json.Unmarshal([]byte(body), &got); err != nil {
t.Fatalf("%v:\n%s", err, body)
}
if len(got.LastSent) != 1 || got.LastSent[0].Node != "laptop" || got.LastSent[0].Answer != "refused" ||
got.LastSent[0].Sender != "a one-shot push by jochen at a shell on anchor" {
t.Fatalf("status --json's last sends read %+v", got.LastSent)
}
}
-222
View File
@@ -1,222 +0,0 @@
package main
import (
"errors"
"fmt"
"sort"
"strings"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
)
// A module assigned to a machine and left out of its declaration is said (novox/hq issue 380).
//
// **An omission is a finding, never a refusal of the push** (ADR 0163, rule 6): the machine is sent everything
// else, and the module's held things are kept. Until issue 380 the only place it showed was `plan`: nfs-server was
// assigned to the home server for days, every send left it out for a setting nobody gave, and the operator believed
// it ran. So the self-check (D1) judges every machine as the next send would (Resolution.LeftOutBecause, the send's
// own judgement) and raises a condition on that machine for each module it leaves out:
//
// - a setting nobody gave (catalogue.UnsetSettingError) is the operator's to give: kind needs-operator, naming
// the module, the setting and the command that sets it;
// - any other cause is said as a module not working is: kind left-out, a warning with the reason as evidence.
//
// Never urgent and never escalated by age (as ADR 0283 decision 5): nothing the machine ran was undone. It clears
// on the first run that no longer finds it — the module composed again, or no longer assigned. `status` and
// `node show` list the same modules under "assigned, not applied" with the same reason.
//
// A module left out because its stored manifest has a key this controller does not know is not raised here: the
// catalogue's own unknown-field condition says it once for the whole mesh (ADR 0262); it is still listed.
const (
// probeLeftOutID is the self-check's probe that raises and clears these conditions: D1, which already
// resolves every machine (probeDeclarations), so the judgement costs no resolution of its own.
probeLeftOutID = "D1"
// kindLeftOut is a module left out for any cause but a setting nobody gave; it is also every such condition's
// token, whichever its kind, so a cause that changes is the same condition said anew.
kindLeftOut = "left-out"
)
// leftOutModule is one module of a machine's set that its declaration leaves out, and why.
type leftOutModule struct {
Module string
// Setting is the setting nobody gave, when that is the cause.
Setting string
// Why is the declaration's own reason, whole.
Why string
// Unread is a stored manifest this controller cannot read whole (said by the catalogue's condition).
Unread bool
}
// leftOutOf is every module of a machine's resolution that a push would leave out, sorted. Pure: the judgement
// the push makes (catalogue.Resolution.LeftOutBecause), nothing allocated.
func leftOutOf(plan catalogue.Resolution, settings catalogue.SettingsBy, adopted bool) []leftOutModule {
because := plan.LeftOutBecause(settings, adopted)
out := make([]leftOutModule, 0, len(because))
for module, why := range because {
l := leftOutModule{Module: module, Why: oneLine(why.Error())}
var unset *catalogue.UnsetSettingError
var unread *catalogue.UnreadManifestError
switch {
case errors.As(why, &unset):
l.Setting = unset.Setting
case errors.As(why, &unread):
l.Unread = true
}
out = append(out, l)
}
sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module })
return out
}
// settingCommand is the command that gives a module's setting on one machine.
func settingCommand(module, setting, node string) string {
return fmt.Sprintf("`settings set %s '{%q: …}' --node %s`", module, setting, node)
}
// reason is why a module is left out, as `status`, `node show` and the condition's summary say it: for a setting
// nobody gave, the setting and the command that gives it; otherwise the declaration's own words.
func (l leftOutModule) reason(node string) string {
if l.Setting != "" {
return fmt.Sprintf("nothing sets its setting %q, so every send leaves it out of its declaration — %s sets it", l.Setting,
settingCommand(l.Module, l.Setting, node))
}
return "every send leaves it out of its declaration: " + l.Why
}
// leftOutObservations are the conditions a machine's left-out modules raise, one each.
func leftOutObservations(node string, left []leftOutModule) []conditions.Observation {
var out []conditions.Observation
for _, l := range left {
if l.Unread {
continue
}
out = append(out, leftOutObservation(node, l))
}
return out
}
// leftOutObservation is one module left out of one machine's declaration: needs-operator for a setting nobody
// gave, left-out otherwise; a warning either way, the operator's to resolve.
func leftOutObservation(node string, l leftOutModule) conditions.Observation {
o := conditions.Observation{Scope: conditions.ScopeModule, ID: l.Module + "." + node, Token: kindLeftOut,
Kind: kindLeftOut, Machine: node, Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("%s is assigned to %s and not applied: %s", l.Module, node, l.reason(node)),
Said: l.Why}
w := leftOutWords(l.Module, node, l.Setting)
if l.Setting != "" {
o.Kind = kindNeedsOperator
} else {
// The words of why may name a path or an address, which the operator's channel withholds: the
// summary sends them to the evidence, and `plan` says them in full.
o.Summary = fmt.Sprintf("%s is assigned to %s and not applied: every send leaves it out of its declaration, "+
"because what is set for it does not fit its manifest — the evidence and `plan %s` say why", l.Module, node, node)
}
o.Headline, o.Explanation, o.Needs, o.Resolved = w.Headline, w.Explanation, w.Needs, w.Resolved
return o
}
// leftOutWords is what the operator reads of a module left out (ADR 0253): plain, the act named.
func leftOutWords(module, node, setting string) words {
w := words{
Headline: fmt.Sprintf("%s is not applied on %s", module, node),
Explanation: fmt.Sprintf("%s is assigned to %s, and every send to %s leaves it out of the declaration because what "+
"is set for it does not fit its manifest. Nothing of it changes there; the rest of %s is sent as usual.",
module, node, node, node),
Needs: fmt.Sprintf("read why in the details, then change what is set for %s or unassign it.", module),
Resolved: fmt.Sprintf("%s on %s is no longer left out", module, node),
}
if setting != "" {
w.Explanation = fmt.Sprintf("%s is assigned to %s, and every send to %s leaves it out of the declaration because "+
"nothing sets its setting %s. Nothing of it runs there until it is set; the rest of %s is sent as usual.",
module, node, node, setting, node)
w.Needs = fmt.Sprintf("set %s for %s on %s, or approve it when it is proposed to you.", setting, module, node)
// A setting's name that is not plain (a dotted key) is in the summary instead.
if _, ok := conditions.PlainWords(w, node); !ok {
w.Explanation = fmt.Sprintf("%s is assigned to %s, and every send to %s leaves it out of the declaration because a "+
"setting it needs is not set. Nothing of it runs there until it is set; the details name it.",
module, node, node)
w.Needs = fmt.Sprintf("set what %s needs on %s; the details name the setting.", module, node)
}
}
return w
}
// notAppliedLines is a machine's "assigned, not applied" as `node show` prints it: one line a module, with the
// reason its condition says.
func notAppliedLines(node string, left []leftOutModule) []string {
if len(left) == 0 {
return nil
}
lines := []string{"", " assigned, not applied:"}
for _, l := range left {
lines = append(lines, fmt.Sprintf(" %-22s %s", l.Module, l.reason(node)))
}
return lines
}
// machineNotApplied is one module assigned to a machine and left out of its declaration, in `status --json`.
type machineNotApplied struct {
Node string `json:"node"`
Module string `json:"module"`
Setting string `json:"setting,omitempty"`
Reason string `json:"reason"`
// UnreadManifest is a module left out because this controller cannot read its manifest whole: it raises no
// condition of its own on the machine, since the catalogue's unknown-field condition says it once (ADR 0262).
UnreadManifest bool `json:"unread-manifest,omitempty"`
}
// notApplied is every machine's left-out modules, in a stated order, as `status --json` carries them.
func notApplied(left map[string][]leftOutModule) []machineNotApplied {
names := make([]string, 0, len(left))
for name := range left {
names = append(names, name)
}
sort.Strings(names)
var out []machineNotApplied
for _, name := range names {
for _, l := range left[name] {
out = append(out, machineNotApplied{Node: name, Module: l.Module, Setting: l.Setting, Reason: l.reason(name),
UnreadManifest: l.Unread})
}
}
return out
}
// printNotApplied is status's "assigned, not applied", every machine's.
func printNotApplied(left map[string][]leftOutModule) {
rows := notApplied(left)
if len(rows) == 0 {
return
}
unread := 0
for _, r := range rows {
if r.UnreadManifest {
unread++
}
}
// Which raise a condition is said, not implied (review of #223): one whose manifest this controller cannot
// read is listed here and raises none of its own on the machine — the catalogue's condition says it.
raises := "each raises a condition on its machine"
switch {
case unread == len(rows):
raises = "none raises a condition on its machine: this controller cannot read their manifests, which the " +
"catalogue's own condition says"
case unread > 0:
raises += fmt.Sprintf(", except the %d whose manifest this controller cannot read, which the catalogue's own "+
"condition says", unread)
}
fmt.Printf("%d module(s) assigned, not applied — every send leaves them out of their declaration; %s:\n",
len(rows), raises)
for _, r := range rows {
fmt.Printf(" %-12s %-22s %s\n", r.Node, r.Module, r.Reason)
}
fmt.Println()
}
// isLeftOutCondition is whether a condition is a module left out of its declaration, which the machine's health
// statements neither raise nor clear: by its token, which every one carries whatever its kind.
func isLeftOutCondition(c conditions.Condition) bool {
return c.Subject.Scope == conditions.ScopeModule && strings.HasSuffix(c.Key, "."+kindLeftOut)
}
-259
View File
@@ -1,259 +0,0 @@
package main
import (
"context"
"encoding/json"
"os"
"slices"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// novox/hq issue 380: nfs-server was assigned to the home server for days and every send left it out — "nfs-server
// has a file that says ${setting:shares}, and nothing sets shares for it" — and nothing but `plan` said so. The
// manifest is the catalogue's own at the commit that added it (mesh-catalog 48fba44), which still says
// ${setting:shares}; the reason below is the one the live push printed.
// leftOutNFS is the resolution of a machine assigned that nfs-server, with the settings given.
func leftOutNFS(t *testing.T) catalogue.Resolution {
t.Helper()
raw, err := os.ReadFile("testdata/left-out/nfs-server.json")
if err != nil {
t.Fatal(err)
}
m, err := catalogue.ParseManifest(raw)
if err != nil {
t.Fatal(err)
}
return catalogue.Resolution{Modules: []catalogue.Manifest{m}}
}
// sharesGiven is the operator's setting for it on the machine.
func sharesGiven(value string) catalogue.SettingsBy {
return catalogue.SettingsBy{"nfs-server": {{From: "anchor", Values: map[string]any{"shares": value}}}}
}
func TestAModuleLeftOutForASettingNobodyGaveNeedsTheOperatorNamingTheSettingAndTheCommand(t *testing.T) {
left := leftOutOf(leftOutNFS(t), nil, false)
if len(left) != 1 || left[0].Module != "nfs-server" || left[0].Setting != "shares" {
t.Fatalf("left out: %+v", left)
}
if !strings.Contains(left[0].Why, `nothing sets "shares" for it`) {
t.Fatalf("the reason is not the push's own: %s", left[0].Why)
}
obs := leftOutObservations("anchor", left)
if len(obs) != 1 {
t.Fatalf("raised %+v", obs)
}
o := obs[0]
if o.Key() != "module.nfs-server.anchor.left-out" || o.Kind != kindNeedsOperator || o.Machine != "anchor" ||
o.Severity != conditions.Warning || o.Resolver != conditions.ResolverOperator {
t.Fatalf("raised %s as %s, %s, by %s, on %q", o.Key(), o.Kind, o.Severity, o.Resolver, o.Machine)
}
for _, want := range []string{"nfs-server", "anchor", `"shares"`, "`settings set nfs-server '{\"shares\": …}' --node anchor`"} {
if !strings.Contains(o.Summary, want) {
t.Errorf("the summary does not name %s: %s", want, o.Summary)
}
}
if o.Said != left[0].Why {
t.Errorf("the evidence is not the reason the send gives: %s", o.Said)
}
plainExample(t, o, "nfs-server is not applied on anchor",
"Needs you: set shares for nfs-server on anchor, or approve it when it is proposed to you. nfs-server is assigned to "+
"anchor, and every send to anchor leaves it out of the declaration because nothing sets its setting shares. "+
"Nothing of it runs there until it is set; the rest of anchor is sent as usual.")
}
func TestAModuleLeftOutForAnotherCauseIsAWarningWithTheReasonAsEvidence(t *testing.T) {
// A setting stored that its manifest can no longer take: one with a line break (issue 339).
left := leftOutOf(leftOutNFS(t), sharesGiven("library=/srv/library\nmedia=/srv/media"), false)
if len(left) != 1 || left[0].Setting != "" {
t.Fatalf("left out: %+v", left)
}
obs := leftOutObservations("anchor", left)
if len(obs) != 1 {
t.Fatalf("raised %+v", obs)
}
o := obs[0]
if o.Key() != "module.nfs-server.anchor.left-out" || o.Kind != kindLeftOut || o.Severity != conditions.Warning {
t.Fatalf("raised %s as %s, %s", o.Key(), o.Kind, o.Severity)
}
if !strings.Contains(o.Said, "holds a line break") || strings.Contains(o.Summary, "/srv/") {
t.Fatalf("the reason is not the evidence, or the summary carries it to the channel:\n%s\n%s", o.Summary, o.Said)
}
plainExample(t, o, "nfs-server is not applied on anchor",
"Needs you: read why in the details, then change what is set for nfs-server or unassign it. nfs-server is assigned to "+
"anchor, and every send to anchor leaves it out of the declaration because what is set for it does not fit "+
"its manifest. Nothing of it changes there; the rest of anchor is sent as usual.")
}
// The self-check raises it, keeps it a warning however long it stands, and clears it when the module composes
// again or is no longer assigned; a machine's health statement neither clears nor raises it.
func TestALeftOutModulesConditionClearsWhenItComposesAgainOrIsUnassigned(t *testing.T) {
plan, settings := leftOutNFS(t), catalogue.SettingsBy(nil)
withProbes(t, probe{ID: probeLeftOutID, Asserts: "the test's", Kind: kindLeftOut, Phase: 1,
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
return leftOutObservations("anchor", leftOutOf(plan, settings, false)), nil
}})
store := conditions.NewInMemory()
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
defer k.Close(context.Background())
d := &doctor{keeper: k, teller: &conditions.Told{}, host: "anchor"}
key := "module.nfs-server.anchor.left-out"
openOnes := func() map[string]conditions.Condition {
t.Helper()
open, err := k.Open(t.Context())
if err != nil {
t.Fatal(err)
}
out := map[string]conditions.Condition{}
for _, c := range open {
out[c.Key] = c
}
return out
}
d.runOnce(t.Context(), "a test")
c, raised := openOnes()[key]
if !raised || c.Kind != kindNeedsOperator || c.Severity != conditions.Warning {
t.Fatalf("a module left out raised %+v", openOnes())
}
// A statement from the machine that says nothing of it — the module runs nothing there — leaves it open.
if err := judgeModuleHealth(t.Context(), nil, k, "anchor", map[string][]inventory.ResourceHealth{}, nil,
time.Now().Add(72*time.Hour)); err != nil {
t.Fatal(err)
}
if _, still := openOnes()[key]; !still {
t.Fatal("a health statement that says nothing of the module cleared its left-out condition")
}
d.runOnce(t.Context(), "a test")
if c := openOnes()[key]; c.Severity != conditions.Warning {
t.Fatalf("after a health statement and days, it is %+v", openOnes())
}
// The setting given: it composes, and the condition clears.
settings = sharesGiven("library=/srv/library")
d.runOnce(t.Context(), "a test")
if _, still := openOnes()[key]; still {
t.Fatalf("composed again, still open: %+v", openOnes())
}
// Left out again, then unassigned: no longer in the machine's set, and it clears.
settings = nil
d.runOnce(t.Context(), "a test")
if _, raised := openOnes()[key]; !raised {
t.Fatal("left out again and not raised")
}
plan = catalogue.Resolution{}
d.runOnce(t.Context(), "a test")
if _, still := openOnes()[key]; still {
t.Fatalf("unassigned, still open: %+v", openOnes())
}
}
func TestTheLeftOutProbeIsInTheRegistry(t *testing.T) {
for _, p := range probeRegistry {
if p.ID == probeLeftOutID {
if p.run == nil || !slices.Contains(p.Raises, kindLeftOut) || !slices.Contains(p.Raises, kindNeedsOperator) {
t.Fatalf("%+v", p)
}
return
}
}
t.Fatalf("no probe %s: a module left out is said nowhere", probeLeftOutID)
}
// status and node show list it under "assigned, not applied" with the reason the condition says, and status is
// not well while there is one.
func TestStatusAndNodeListAModuleAssignedAndNotApplied(t *testing.T) {
left := map[string][]leftOutModule{"anchor": leftOutOf(leftOutNFS(t), nil, false)}
reason := leftOutObservations("anchor", left["anchor"])[0].Summary
asked := answers{leftOut: left}
if asked.well() {
t.Fatal("a mesh with a module assigned and not applied is called well")
}
shown := printed(t, func() error { return printStatus(asked) })
if !strings.Contains(shown, "1 module(s) assigned, not applied") || !strings.Contains(shown, "nfs-server") ||
!strings.Contains(shown, "`settings set nfs-server '{\"shares\": …}' --node anchor` sets it") {
t.Fatalf("status says:\n%s", shown)
}
if !strings.Contains(reason, left["anchor"][0].reason("anchor")) {
t.Fatalf("status and the condition say different reasons:\n%s\n%s", shown, reason)
}
body, err := statusAsJSON(asked)
if err != nil {
t.Fatal(err)
}
var doc struct {
NotApplied []machineNotApplied `json:"not-applied"`
}
if err := json.Unmarshal(body, &doc); err != nil || len(doc.NotApplied) != 1 ||
doc.NotApplied[0].Setting != "shares" || doc.NotApplied[0].Node != "anchor" {
t.Fatalf("status --json: %v %s", err, body)
}
lines := strings.Join(notAppliedLines("anchor", left["anchor"]), "\n")
if !strings.Contains(lines, "assigned, not applied:") || !strings.Contains(lines, "nfs-server") ||
!strings.Contains(lines, `nothing sets its setting "shares"`) {
t.Fatalf("node show says:\n%s", lines)
}
}
// The skip is this probe's alone (review of #223): a part waiting for the operator (ADR 0283) keeps its own
// needs-operator condition, `module.<m>.<node>.needs-operator`, which still clears on the first statement that no
// longer names it — beside a left-out condition on the same machine, which stays.
func TestAWaitsNeedsOperatorStillClearsWhenItsStatementStopsNamingItBesideALeftOutOne(t *testing.T) {
store := conditions.NewInMemory()
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
defer k.Close(context.Background())
left := leftOutObservations("anchor", leftOutOf(leftOutNFS(t), nil, false))
if err := k.Reconcile(t.Context(), probeLeftOutID, left); err != nil {
t.Fatal(err)
}
waiting := map[string][]inventory.ResourceHealth{"notes": {{Module: "notes", Resource: "server", State: link.StateWaiting,
Waits: []inventory.Wait{{Setting: "domain", What: "the domain it serves"}}}}}
if err := judgeModuleHealth(t.Context(), nil, k, "anchor", waiting, map[string]int{"notes": 2}, time.Now()); err != nil {
t.Fatal(err)
}
waitKey, leftKey := needsOperatorKey("notes", "anchor"), "module.nfs-server.anchor.left-out"
open := func() map[string]conditions.Condition {
t.Helper()
list, err := k.Open(t.Context())
if err != nil {
t.Fatal(err)
}
out := map[string]conditions.Condition{}
for _, c := range list {
out[c.Key] = c
}
return out
}
if c, raised := open()[waitKey]; !raised || c.Kind != kindNeedsOperator {
t.Fatalf("the wait raised %+v", open())
}
if c := open()[leftKey]; c.Kind != kindNeedsOperator {
t.Fatalf("the left-out condition is not open as needs-operator: %+v", open())
}
// D1 runs again and still finds nfs-server left out: its reconcile clears only what D1 raised, never the wait,
// which the machine's statement raised.
if err := k.Reconcile(t.Context(), probeLeftOutID, left); err != nil {
t.Fatal(err)
}
if _, still := open()[waitKey]; !still {
t.Fatalf("D1's reconcile cleared the wait's needs-operator: %+v", open())
}
if err := judgeModuleHealth(t.Context(), nil, k, "anchor", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil {
t.Fatal(err)
}
if _, still := open()[waitKey]; still {
t.Fatalf("the statement no longer names the wait, and its needs-operator is still open: %+v", open())
}
if _, still := open()[leftKey]; !still {
t.Fatalf("the left-out condition was cleared by a health statement: %+v", open())
}
}
+1 -3
View File
@@ -147,11 +147,9 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
}
standing := map[string]conditions.Condition{}
for _, c := range open {
// A module left out of the composition is the self-check's to raise and clear, never a statement's
// (novox/hq issue 380): its needs-operator is not cleared for not being in what the machine runs.
if (c.Kind == kindModuleUnhealthy || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound ||
c.Kind == kindNeedsOperator) &&
c.Subject.Machine == node && !isLeftOutCondition(c) {
c.Subject.Machine == node {
standing[c.Key] = c
}
}
+2 -27
View File
@@ -44,7 +44,7 @@ func nodeCommand(ctx context.Context, args []string) error {
if len(args) != 2 {
return errors.New("node show <name>")
}
return showNode(ctx, open, args[1])
return showNode(ctx, inv, args[1])
case "add":
return addNode(ctx, inv, args[1:])
@@ -787,8 +787,7 @@ func roughly(d time.Duration) string {
//
// It is also where "what should it be configured as" is read. The same line that gates an
// assignment carries `card1-DP-1`, and a person composing settings for that machine needs it.
func showNode(ctx context.Context, stored *stores, name string) error {
inv := stored.inventory
func showNode(ctx context.Context, inv *inventory.Inventory, name string) error {
node, err := inv.NodeByName(ctx, name)
if err != nil {
return err
@@ -804,19 +803,6 @@ func showNode(ctx context.Context, stored *stores, name string) error {
if err := showMode(ctx, inv, node); err != nil {
return err
}
// What it was last sent, by whom, under which epoch, from which generation, naming which modules, and what
// it answered (novox/hq issue 485): the record of sends is read through this verb, never the store's reader.
// Unreadable is said, not taken for none.
switch last, found, err := inv.LastSend(ctx, name); {
case err != nil:
fmt.Printf(" what it was last sent could NOT be read: %v\n", err)
case !found:
fmt.Printf(" last sent no send recorded since the mesh began keeping them\n")
default:
for _, line := range lastSendLines(last) {
fmt.Println(line)
}
}
// Whom agents run as here, and whether that account can become root without a person (ADR 0266).
for _, line := range agentAccountLines(ctx, inv, node) {
fmt.Println(line)
@@ -903,17 +889,6 @@ func showNode(ctx context.Context, stored *stores, name string) error {
if len(assigned) > 0 {
fmt.Printf("\n assigned: %s\n", strings.Join(assigned, ", "))
}
// And which of them a push leaves out, and why (novox/hq issue 380): judged as the push judges it, the same
// reason its condition says. Not computable is said, never read as "all applied".
plan, settings, err := planFor(ctx, stored, name)
switch {
case err != nil:
fmt.Printf("\n whether a send leaves any assigned module out is NOT known: %s\n", oneLine(err.Error()))
default:
for _, line := range notAppliedLines(name, leftOutOf(plan, settings, node.Adopted)) {
fmt.Println(line)
}
}
return nil
}
+4 -5
View File
@@ -17,8 +17,8 @@ import (
// A delivery over its budget is loud (novox/hq ADR 0282 decision 7, issue 382): the self-check reads
// mesh-delivery's `times` and raises the warning `delivery.<class>.over-budget` when the newest delivery of a
// class whose delivery time is known took longer than its class's budget — three minutes for a leaf module (ADR
// 0304 decision 4), ten for a core module — naming the delivery and its longest phase. It clears when the next delivery of that class
// class whose delivery time is known took longer than its class's budget — five minutes for a leaf module, ten
// for a core module — naming the delivery and its longest phase. It clears when the next delivery of that class
// lands within its budget. Measurement only: the condition says, it never holds or acts on a delivery.
// probeBudgetsID is the probe that reads the delivery times.
@@ -27,9 +27,8 @@ const probeBudgetsID = "D16"
// kindOverBudget is what a delivery over its class's budget raises.
const kindOverBudget = "over-budget"
// deliveryBudgets are the budgets by class (ADR 0282 decision 1, the leaf's as ADR 0304 decision 4 changed it);
// the probe raises nothing for another class.
var deliveryBudgets = map[string]time.Duration{inventory.ClassLeaf: 3 * time.Minute, inventory.ClassCore: 10 * time.Minute}
// deliveryBudgets are the budgets by class (ADR 0282 decision 1); the probe raises nothing for another class.
var deliveryBudgets = map[string]time.Duration{inventory.ClassLeaf: 5 * time.Minute, inventory.ClassCore: 10 * time.Minute}
// timesAnswer is what mesh-delivery's `times` answers, as far as the probe reads it.
type timesAnswer struct {
+4 -5
View File
@@ -10,12 +10,11 @@ import (
"github.com/novox/mesh-controller/internal/inventory"
)
// A leaf delivery of 3 minutes 10 seconds raises delivery.leaf.over-budget naming its longest phase; one of
// exactly three minutes, a core one of nine and a class without a budget raise nothing (ADR 0282 decision 7,
// the leaf's budget as ADR 0304 decision 4 changed it).
// A leaf delivery of 5 minutes 10 seconds raises delivery.leaf.over-budget naming its longest phase; one of
// exactly five minutes, a core one of nine and a class without a budget raise nothing (ADR 0282 decision 7).
func TestADeliveryOverItsBudgetIsLoud(t *testing.T) {
a := &timesAnswer{Classes: []timesClass{
{Class: inventory.ClassLeaf, Latest: &timesLatest{ID: "novox/mesh-catalog@abc", TookMS: (3*time.Minute + 10*time.Second).Milliseconds(),
{Class: inventory.ClassLeaf, Latest: &timesLatest{ID: "novox/mesh-catalog@abc", TookMS: (5*time.Minute + 10*time.Second).Milliseconds(),
Longest: "judgement 2m40s"}},
{Class: inventory.ClassCore, Latest: &timesLatest{ID: "novox/mesh-controller@def", TookMS: (9 * time.Minute).Milliseconds(),
Longest: "build 1m"}},
@@ -31,7 +30,7 @@ func TestADeliveryOverItsBudgetIsLoud(t *testing.T) {
t.Fatalf("the condition names its delivery and longest phase: %s — %s", o.Key(), o.Summary)
}
// The next leaf delivery within its budget clears it: the probe raises nothing for the class.
a.Classes[0].Latest = &timesLatest{ID: "novox/mesh-catalog@ghi", TookMS: (3 * time.Minute).Milliseconds()}
a.Classes[0].Latest = &timesLatest{ID: "novox/mesh-catalog@ghi", TookMS: (5 * time.Minute).Milliseconds()}
if obs := overBudgetObservations(a); len(obs) != 0 {
t.Fatalf("a delivery of exactly its budget is within it: %+v", obs)
}
+13 -8
View File
@@ -231,14 +231,6 @@ var plainWordings = map[string]func(conditions.Observation) words{
w := needsOperatorWords(orModule(module), node, nil)
return w
}),
kindLeftOut: worded(func(o conditions.Observation) words {
// The observation carries its own words (novox/hq issue 380); these are its kind's alone.
module := ""
if o.Scope == conditions.ScopeModule && o.Machine != "" {
module = strings.TrimSuffix(o.ID, "."+o.Machine)
}
return leftOutWords(orModule(module), machineOr(o, "a machine"), "")
}),
kindProviderFailing: worded(func(o conditions.Observation) words {
thing, consumer := conditions.ThingWords(o), idPart(o, 2)
if consumer == "" {
@@ -614,6 +606,19 @@ var plainWordings = map[string]func(conditions.Observation) words{
Explanation: "The bus refused messages from a part of the mesh, so what they carried did not happen.",
Resolved: "Resolved: the bus takes the messages again"}
}),
kindBucketOrLogFilling: worded(func(o conditions.Observation) words {
return words{Headline: "A module's bucket or log on the bus is filling up",
Needs: "decide whether to raise its cap or have the module keep less.",
Explanation: "A bucket or log a module keeps on the bus holds three quarters of its cap or more. When it " +
"is full, the bus refuses what the module writes there.",
Resolved: "It has room again"}
}),
kindBucketOrLogUnread: worded(func(o conditions.Observation) words {
return words{Headline: "A module's bucket or log could not be read",
Explanation: "The controller could not read how full a bucket or log a module keeps on the bus is, so it " +
"cannot say whether it is filling up. It asks again at its next check.",
Resolved: "It can be read again"}
}),
"stream-wrong": worded(func(o conditions.Observation) words {
return words{Headline: "Part of the bus's storage is wrong",
Needs: "check the machine the bus runs on; the details say what is missing.",
-7
View File
@@ -278,13 +278,6 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
plans = append(plans, recent...)
// **A failed walk whose earlier merges are walked alone is not retried** (novox/hq ADR 0276): the search for
// the merge that brought the failure answers them now, and a retried walk would name them twice.
// **A walk stopped without some merges is not retried** (novox/hq ADR 0299): the merges it left out were stopped,
// and the others are another walk's now.
if p.Delivery != nil && (p.Delivery.WalkedAgainBy != "" || len(p.Delivery.LeftOut) > 0) {
return "", fmt.Errorf("%s was stopped without %s: those merges were stopped, and %s walks the others; a "+
"newer merge, or `rebuild <module>`, builds again", p.ID, mergesLeftOutWords(p.Delivery.LeftOut),
orWords(p.Delivery.WalkedAgainBy, "nothing"))
}
if p.Delivery != nil && len(p.Delivery.Merges) > 0 {
kept, err := inv.MergesOf(ctx, p.ID)
if err != nil {
-5
View File
@@ -78,11 +78,6 @@ func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation
return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err)
}
var problems, foreseen []string
// Each module the next send leaves out of this machine's declaration (novox/hq issue 380), judged from this
// resolution as the send judges it: a finding, never a refusal.
if err == nil {
out = append(out, leftOutObservations(n.Name, leftOutOf(plan, settings, n.Adopted))...)
}
if err == nil && gensErr == nil {
var declared sendable
if declared, err = declarationWith(ctx, open, n.Name, plan, settings, gens, Foreseeing); err == nil {
+25 -12
View File
@@ -1236,17 +1236,6 @@ func sendToEach(ctx context.Context, open *stores, names []string) ([]string, er
continue
}
numbered.stamp(&declared)
// **The grants step sends the user list and nothing else** (novox/hq issue 490): judged on this very
// declaration, the one sent, so nothing composed between a check and the send can slip through.
if everyBuildKept(ctx, name) {
other, err := grantsOnlyIn(ctx, open, name, plan, declared)
if err != nil {
return nil, err
}
if other != "" {
return nil, fmt.Errorf("%w: %s", errNotGrantsOnly, other)
}
}
reportLeftOut(name, declared)
sending = append(sending, readyNode{name, declared})
}
@@ -1311,6 +1300,15 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, se
if _, err := broker.RaiseBuckets(broker.OnConn(bus.Conn), buckets); err != nil {
return fmt.Errorf("the modules' state could not be asserted on the bus: %w", err)
}
// **And every declared log, for the same reason** (novox/hq ADR 0297 §2): a membership names its
// logs, and a module whose log does not exist fails its first append.
logs, err := open.inventory.DeclaredLogs(ctx)
if err != nil {
return fmt.Errorf("the modules' logs could not be read, so no log was asserted: %w", err)
}
if _, err := broker.RaiseLogs(broker.OnConn(bus.Conn), logs); err != nil {
return fmt.Errorf("the modules' logs could not be asserted on the bus: %w", err)
}
// Every membership is tried, and the first failure named once.
issued := 0
refused := map[string]error{}
@@ -1494,13 +1492,28 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string)
fmt.Printf("the bus holds state nothing declares any more, kept because it is data: %s — "+
"removing it is a person's act\n", strings.Join(undeclared, ", "))
}
// Every module's log (novox/hq ADR 0297), from the catalogue, as its buckets: one that nothing
// declares any more is said and kept — a log is a module's record, and no path of the mesh removes it.
logs, err := inv.DeclaredLogs(ctx)
if err != nil {
return err
}
unlogged, err := broker.RaiseLogs(js, logs)
if err != nil {
return err
}
if len(unlogged) > 0 {
fmt.Printf("the bus holds logs nothing declares any more, kept because they are data: %s — "+
"removing one is a person's act\n", strings.Join(unlogged, ", "))
}
// And how every module hears what it consumes: asserted with the rest above, counted here.
hearing, err := moduleConsumerCount(ctx, inv)
if err != nil {
return err
}
fmt.Printf("the bus at %s has its streams, %d machine(s) can hear a declaration, %d module(s) "+
"can hear what they consume, and %d bucket(s) of state\n", broker.BareAddress(address), len(names), hearing, len(buckets))
"can hear what they consume, %d bucket(s) of state and %d log(s)\n", broker.BareAddress(address), len(names), hearing,
len(buckets), len(logs))
return nil
}
-49
View File
@@ -69,9 +69,6 @@ type meshStatus struct {
// **A document without this said an outage was a well mesh.** Read from what each machine
// reported, so it is the machine's account and not the mesh's take-time listing.
Untaken []machineUntaken `json:"untaken,omitempty"`
// NotApplied is every module assigned to a machine and left out of its composition, with why (novox/hq
// issue 380). Absent when every module composes.
NotApplied []machineNotApplied `json:"not-applied,omitempty"`
// Filtered is every converged machine that is not filtered by the mesh alone (novox/hq ADR
// 0168), one entry per rule set the mesh did not write — the found firewall in force again,
// or a chain nobody speaks for. Absent when every converged machine is filtered by the mesh
@@ -109,33 +106,6 @@ type meshStatus struct {
// why they could not be read.
Pending []pendingStatus `json:"pending,omitempty"`
PendingUnread string `json:"pendingUnread,omitempty"`
// LastSent is every machine's last recorded send — who sent it, under which lease epoch, from which
// assignment generation, naming which modules — and what the machine answered of it (novox/hq issue 485).
// A machine the mesh has recorded no send to is absent. LastSentUnread says why it could not be read.
LastSent []machineLastSent `json:"lastSent"`
LastSentUnread string `json:"lastSentUnread,omitempty"`
}
// machineLastSent is one machine's last send as status says it.
type machineLastSent struct {
Node string `json:"node"`
Sequence int64 `json:"sequence"`
SentAt time.Time `json:"sentAt"`
Sender string `json:"sender"`
Epoch int64 `json:"epoch"`
Generation int64 `json:"generation"`
Digest string `json:"digest"`
Modules []string `json:"modules"`
// Answer is what the machine said of it: applied, failed or refused, as its report has it; empty when it
// has not reported on it. Refused is its words when it refused it whole, Failed how many resources failed.
Answer string `json:"answer"`
Refused string `json:"refused,omitempty"`
Failed int `json:"failed,omitempty"`
AnsweredAt *time.Time `json:"answeredAt,omitempty"`
// RefusedForGeneration is when the machine refused it for coming from an older assignment generation than
// it applied, and AppliedGeneration that generation; absent when it did not.
RefusedForGeneration *time.Time `json:"refusedForGeneration,omitempty"`
AppliedGeneration int64 `json:"appliedGeneration,omitempty"`
}
// pendingStatus is one pending assignment as status says it.
@@ -281,7 +251,6 @@ func statusAsJSON(asked answers) ([]byte, error) {
}
}
out.Unheld = asked.unheld
out.NotApplied = notApplied(asked.leftOut)
out.HandActsThisWeek, out.HandActsUnread = asked.handActs, asked.handActsUnread
out.HealsThisWeek, out.HealsUnread = asked.heals, asked.healsUnread
// In brief, as `conditions` lists them: status leads with every open condition, and their whole
@@ -294,7 +263,6 @@ func statusAsJSON(asked answers) ([]byte, error) {
out.Pending = append(out.Pending, pendingStatus{Node: p.Node, Module: p.Module, State: p.State,
Build: p.Build, Repository: p.Repository, Path: p.Path, Since: p.Since, Settled: p.Settled, Note: p.Note})
}
out.LastSent, out.LastSentUnread = lastSentStatus(asked.lastSent), asked.lastSentUnread
for name := range asked.refused {
out.Unresolved = append(out.Unresolved, machineUnresolved{
Node: name, Problem: asked.refused[name]})
@@ -351,20 +319,3 @@ func say(value any) error {
fmt.Println(string(body))
return nil
}
// lastSentStatus is every machine's last send as status --json says it, by machine name (novox/hq issue 485).
func lastSentStatus(sends map[string]inventory.Send) []machineLastSent {
out := []machineLastSent{}
for _, s := range sends {
modules := s.Modules
if modules == nil {
modules = []string{}
}
out = append(out, machineLastSent{Node: s.NodeName, Sequence: s.Sequence, SentAt: s.SentAt, Sender: s.Sender,
Epoch: s.Epoch, Generation: s.Generation, Digest: s.Digest, Modules: modules, Answer: s.Answer.Outcome,
Refused: s.Answer.Refused, Failed: s.Answer.Failed, AnsweredAt: s.Answer.At,
RefusedForGeneration: s.RefusedAt, AppliedGeneration: s.RefusedApplied})
}
sort.Slice(out, func(i, j int) bool { return out[i].Node < out[j].Node })
return out
}
+3 -44
View File
@@ -61,57 +61,20 @@ func keptExcept(ctx context.Context) (map[string]bool, bool) {
return skip, on
}
type keepEveryBuildKey struct{}
// keepingEveryBuild is a context whose sends compose every module of one machine — recorded or rolling out —
// at the build that machine was last sent (novox/hq issue 490): the grants step, which sends the machine
// holding the bus its new user list and nothing of any module's new code. That code waits there for a gate
// like any other move; the list must not, or the first machine's gate is judged against a bus that refuses
// what the change newly grants.
//
// **That machine alone.** Composing one machine resolves the others too — who is on the private network,
// who answers a requirement — on the same context, and those are no part of the step's send: they are
// composed as any send composes them. Kept for every machine, a machine whose last send is not known
// refused the bus's machine's composition.
func keepingEveryBuild(ctx context.Context, node string) context.Context {
return context.WithValue(ctx, keepEveryBuildKey{}, node)
}
// everyBuildKept is whether this context keeps every module of this machine at the build it runs.
func everyBuildKept(ctx context.Context, node string) bool {
on, _ := ctx.Value(keepEveryBuildKey{}).(string)
return on != "" && on == node
}
// recordedKept is, for a send under keepingRecorded, every recorded module the machine was last sent a
// build of that the mesh's build is not identical to: module → the commit it keeps. Nil when the context
// keeps nothing, or when what the machine was last sent is not known (it is then held whole elsewhere —
// ADR 0221).
//
// Under keepingEveryBuild, every module the machine was sent is kept, whatever its policy (novox/hq issue
// 490), and a machine whose last send is not known refuses the send: there is nothing to keep it at, and
// composing the mesh's builds would be the very move the grants step must not make.
func recordedKept(ctx context.Context, open *stores, node string) (map[string]string, error) {
every := everyBuildKept(ctx, node)
skip, on := keptExcept(ctx)
if !on && !every {
if !on {
return nil, nil
}
if every {
skip = nil
}
inv := open.inventory
sent, known, err := inv.SentBuilds(ctx, node)
if err != nil {
if err != nil || !known {
return nil, err
}
if !known {
if every {
return nil, fmt.Errorf("what %s was last sent is not known, so the bus's user list cannot be sent "+
"there alone with every build kept (novox/hq issue 490)", node)
}
return nil, nil
}
current, err := inv.CurrentBuilds(ctx)
if err != nil {
return nil, err
@@ -120,7 +83,7 @@ func recordedKept(ctx context.Context, open *stores, node string) (map[string]st
out := map[string]string{}
for m, was := range sent {
now, held := current[m]
if !held || (now.RollOut && !every) || skip[m] || was == "" || sameCommit(was, now.Commit) {
if !held || now.RollOut || skip[m] || was == "" || sameCommit(was, now.Commit) {
continue
}
if f == nil {
@@ -155,10 +118,6 @@ func keepRecorded(ctx context.Context, open *stores, node string, shelf map[stri
if err != nil {
return nil, err
}
if !found && everyBuildKept(ctx, node) {
return nil, fmt.Errorf("%s runs %s's build %s, which the build records no longer hold: the bus's user "+
"list cannot be sent there alone with it kept (novox/hq issue 490)", node, m, short(kept[m]))
}
if !found {
return nil, fmt.Errorf("%s records rather than rolls out, and %s runs its build %s, which the build "+
"records no longer hold: this send cannot keep it and does not move it — `push %s` sends the new "+
+12 -190
View File
@@ -2,7 +2,6 @@ package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
@@ -254,19 +253,8 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
if err != nil {
return nil, err
}
// The grants step composes every module at the build its machine was last sent (novox/hq issue 490):
// a move it keeps is not made. Read, not assumed, so a move it could not keep is still refused here.
var keeps map[string]string
if everyBuildKept(ctx, n) {
if keeps, err = recordedKept(ctx, open, n); err != nil {
return nil, err
}
}
var waiting []string
for _, mv := range moves {
if was, kept := keeps[mv.Module]; kept && sameCommit(was, mv.From) {
continue
}
if !scope.judged[n] && !scope.modules[mv.Module] {
waiting = append(waiting, fmt.Sprintf("%s %s → %s", mv.Module, short(mv.From), short(mv.To)))
}
@@ -295,18 +283,15 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
// owns are the moves the send exists for — every module of a plan's tier whose first machine this is, in
// one send (novox/hq issue 281): a send carries the machine's whole declaration (ADR 0221), so a send per
// module was the same declaration sent again and again, each one setting aside the one before.
//
// And the grants step first, when the send changes what the bus's user list must say (novox/hq issue 490):
// answered for the gate to keep, nil when there was none.
func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, []string, *inventory.GrantsStep, error) {
func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, []string, error) {
inv := open.inventory
f, err := readMoveFacts(ctx, inv)
if err != nil {
return nil, nil, nil, err
return nil, nil, err
}
moves, err := machineMoves(ctx, open, f, node, true)
if err != nil {
return nil, nil, nil, err
return nil, nil, err
}
own := func(module string) bool {
return slices.ContainsFunc(owns, func(o inventory.CarriedMove) bool { return o.Module == module })
@@ -316,7 +301,7 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
continue
}
if id := f.walkedBy(mv.Module, node, mv.To); id != "" {
return nil, nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
return nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
mv.Module, short(mv.To), node, id)
}
}
@@ -324,7 +309,7 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
// plan, or a release, on this machine is not crossed with a newer build; this send waits for its gate.
for _, o := range owns {
if id := f.walkedBy(o.Module, node, o.To); id != "" {
return nil, nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
return nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
o.Module, short(o.To), node, id)
}
}
@@ -338,22 +323,22 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
}
}
if len(moves) == 0 && len(owns) == 0 {
return nil, nil, nil, nil
return nil, nil, nil
}
for i := range moves {
if moves[i].Build == "" {
if moves[i].Build, err = inv.BuildOf(ctx, moves[i].Module, moves[i].To); err != nil {
return nil, nil, nil, err
return nil, nil, err
}
}
}
sort.Slice(moves, func(i, j int) bool { return moves[i].Module < moves[j].Module })
sayRecreations(ctx, open, moves)
sent, grants, err := sendJudged(ctx, open, node)
sent, err := sendRollout(withScope(ctx, sendScope{judged: map[string]bool{node: true}}), open, []string{node})
if err != nil {
return nil, nil, nil, err
return nil, nil, err
}
return moves, sent, grants, nil
return moves, sent, nil
}
// passCarried keeps a pass as the verdict of every build the gate judged beside its own module.
@@ -689,7 +674,7 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
r.Next++
continue
}
moves, sent, grants, err := gatedSend(ctx, open, node, nil)
moves, sent, err := gatedSend(ctx, open, node, nil)
if errors.Is(err, errWalkedElsewhere) {
note := fmt.Sprintf("waiting before %s: %v", node, err)
changed := p.Note != note
@@ -704,12 +689,8 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
r.Next++
continue
}
r.Gate = &inventory.PlanGate{Machines: sent, Since: &now, Carried: moves, Sent: sentNow(ctx, open.inventory, sent),
Grants: grants}
r.Gate = &inventory.PlanGate{Machines: sent, Since: &now, Carried: moves, Sent: sentNow(ctx, open.inventory, sent)}
p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves))
if said := grantsSaid(grants); said != "" {
p.Note += "; " + said
}
if said := recreationsSaid(moves); said != "" {
p.Note += "; " + said
}
@@ -935,162 +916,3 @@ func recreationsSaid(moves []inventory.CarriedMove) string {
}
return strings.Join(said, "; ")
}
// The grants step (novox/hq issue 490).
//
// The bus's user list — every module's grants, composed mesh-wide from the catalogue — travels only in the
// declaration of the machine holding the bus. A send to any other machine that changed it put that machine
// first (issue 249), unless a build waited there for a gate: then ungatedIn left it out, and said so. On
// 2026-10-11 a merge gave restic a new health tool; its walk sent restic to its first machine and judged it
// there, the machine's node-engine asked the tool and the bus refused it ("this host's grant does not name
// …"), because the machine holding the bus also runs restic, whose new build waited for that very gate. The
// gate could not pass; a person pushed the bus's machine by hand, which carried restic's new build there
// unjudged.
//
// So before a gated send, when the user list composed now is not the one the machine holding the bus was
// last sent, that machine is sent its declaration with **every build kept at the one it runs**: the new
// list, and nothing of any module's new code. Said on the gate (`plans`), and, when it cannot be sent, said
// with why and passed over: the send goes on as it did before, and its gate says what it finds.
// brokerBehindOf is brokerBehind: a variable so a test of the walk needs no store.
var brokerBehindOf = brokerBehind
// grantsStep sends the machine holding the bus its user list alone, ahead of a gated send to node, when the
// list changed; answers what it did, or nil when there was nothing to send.
func grantsStep(ctx context.Context, open *stores, node string) *inventory.GrantsStep {
holder, behind, err := brokerBehindOf(ctx, open, []string{node})
if err != nil {
fmt.Printf("grants step before %s: whether the bus's user list changed could not be read: %v\n", node, err)
return &inventory.GrantsStep{Node: "the machine holding the bus", Failed: err.Error()}
}
if holder == "" || holder == node || !behind {
// No bus with a user list, the gate's own machine holds it (its send carries the list first), or the
// list is the one already sent.
return nil
}
if _, err := sendRollout(keepingEveryBuild(withScope(ctx, sendScope{}), holder), open, []string{holder}); err != nil {
fmt.Printf("grants step: the bus's user list could not be sent to %s ahead of %s, which is sent without "+
"it — the bus may refuse what the send newly grants: %v\n", holder, node, err)
return &inventory.GrantsStep{Node: holder, Failed: err.Error()}
}
now := time.Now().UTC()
fmt.Printf("grants step: %s sent the bus's user list alone, every build there kept, before %s is judged\n",
holder, node)
return &inventory.GrantsStep{Node: holder, At: &now}
}
// errNotGrantsOnly is a grants step refused unsent: its declaration would change more than the user list.
var errNotGrantsOnly = errors.New("the grants step would change more than the bus's user list, and is not sent")
// grantsOnlyIn is what a grants step's declaration, composed for sending, would change on the machine holding
// the bus besides its user list, against what it was last sent (novox/hq issue 490); empty when nothing else.
// Judged by sendToEach on the very declaration it then sends — one composition, judged, then sent or refused
// unsent. Kept builds are not the whole of a declaration: a new assignment, a settings change, an assignment
// taken away, a provision's logins are composed as the mesh holds them now, and a step that carried any of
// it would be the unjudged change this step exists to avoid. Compared resource by resource with the summary
// the last send kept (ADR 0217): exact, rather than a list of the cases that can differ. A module there that
// was never sent is a new assignment, refused whatever its resources.
func grantsOnlyIn(ctx context.Context, open *stores, holder string, plan catalogue.Resolution, declared sendable) (string, error) {
inv := open.inventory
sent, known, err := inv.SentBuilds(ctx, holder)
if err != nil {
return "", err
}
if !known {
return fmt.Sprintf("what %s was last sent is not known", holder), nil
}
var other []string
listID := ""
for _, m := range plan.Modules {
if _, was := sent[m.Module]; !was {
other = append(other, m.Module+" newly assigned")
}
if m.BusUsers != "" && m.ClaimsSeat(catalogue.BrokerSeat) {
listID = m.Module + "." + catalogue.BusUsersID()
}
}
var facts *moveFacts
for m, was := range sent {
now, carried := declared.Builds[m]
if !carried || sameCommit(now, was) {
continue
}
if facts == nil {
f, err := readMoveFacts(ctx, inv)
if err != nil {
return "", err
}
facts = &f
}
if !facts.identical(m, was, now) {
other = append(other, fmt.Sprintf("%s %s → %s", m, short(was), short(now)))
}
}
for _, f := range declared.foreseen {
other = append(other, f+" would be minted")
}
body, err := declared.Body()
if err != nil {
return "", err
}
after, err := summarize(body)
if err != nil {
return "", err
}
prev, err := inv.SentSummary(ctx, holder)
if err != nil {
return "", err
}
if len(prev) == 0 {
return fmt.Sprintf("what %s was last sent is not kept for comparison", holder), nil
}
var before []sentResource
if err := json.Unmarshal(prev, &before); err != nil {
return "", fmt.Errorf("what %s was last sent is kept in a form this version does not read: %w", holder, err)
}
other = append(other, otherThanTheList(diffSent(before, after), listID)...)
sort.Strings(other)
if len(other) == 0 {
return "", nil
}
return "its send would change more than the bus's user list: " + strings.Join(other, "; "), nil
}
// otherThanTheList is what a diff against the last send changes besides the user list's content: every
// resource added, removed or changed, but the list's own resource when its content is all that changed —
// its path, mode and every other field must be as last sent (novox/hq issue 490).
func otherThanTheList(d sentDiff, listID string) []string {
var other []string
for _, id := range d.Added {
other = append(other, "+"+id)
}
for _, id := range d.Removed {
other = append(other, "-"+id)
}
for _, c := range d.Changed {
if c.ID == listID && len(c.Fields) == 1 && c.Fields[0] == "content" {
continue
}
other = append(other, fmt.Sprintf("~%s (%s)", c.ID, strings.Join(c.Fields, ", ")))
}
return other
}
// sendJudged is a gated send's sends: the grants step when the user list changed, then the machine judged.
func sendJudged(ctx context.Context, open *stores, node string) ([]string, *inventory.GrantsStep, error) {
grants := grantsStep(ctx, open, node)
sent, err := sendRollout(withScope(ctx, sendScope{judged: map[string]bool{node: true}}), open, []string{node})
return sent, grants, err
}
// grantsSaid is a grants step as `plans` and a walk's note say it.
func grantsSaid(g *inventory.GrantsStep) string {
switch {
case g == nil:
return ""
case g.Failed != "":
return fmt.Sprintf(grantsStepWord+" to %s FAILED, sent without it: %s", g.Node, g.Failed)
default:
return fmt.Sprintf(grantsStepWord+": %s sent the bus's user list first, its builds kept", g.Node)
}
}
+1 -6
View File
@@ -971,7 +971,7 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
}
// A gated send (ADR 0236): everything waiting on the machine goes with the tier, and the gate judges
// all of it there.
carried, sent, grants, err := gatedSend(ctx, open, node, owns)
carried, sent, err := gatedSend(ctx, open, node, owns)
if err != nil {
return err
}
@@ -992,7 +992,6 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
s.GatedBy = ""
if m == lead {
s.Gate.Carried = carried
s.Gate.Grants = grants
} else {
s.GatedBy = lead
}
@@ -1002,10 +1001,6 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
strings.Join(sent, ", "))
fmt.Printf("%s: tier %d built; sent %d module(s) to %s first in one send (%s), the rest once its gate passes\n",
p.ID, p.Tier, len(modules), strings.Join(sent, ", "), strings.Join(modules, ", "))
// The grants step taken before it (novox/hq issue 490), said with it.
if said := grantsSaid(grants); said != "" {
p.Note += "; " + said
}
// What the send recreates, said with it (novox/hq ADR 0245).
if said := recreationsSaid(carried); said != "" {
p.Note += "; " + said
-1
View File
@@ -160,7 +160,6 @@ func (b *backlogMesh) release(t *testing.T) inventory.Plan {
// The backlog goes out one machine at a time: the first judged before the second is sent, each build's
// pass kept; a rebuild with the same bytes is no move at all; and a send outside a gate is refused.
func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) {
judgeLeavesAsTheCore(t)
b := aBacklog(t)
ctx := t.Context()
inv := b.open.inventory
-1
View File
@@ -77,7 +77,6 @@ func testAWalkStoppedAfterItsGatePassedKeepsThePass(t *testing.T) {
// the first send is older than the wait for a first machine's report. Read before the pass is taken over, the
// carried module's step said the first machine never applied it, and the passed build was put back.
func TestACarriedModuleTakesItsLeadsPassBeforeItsStepIsRead(t *testing.T) {
judgeLeavesAsTheCore(t)
tm := aTierMesh(t, "m01", "m02")
ctx := t.Context()
inv := tm.open.inventory
-9
View File
@@ -55,15 +55,6 @@ func rotateCommand(ctx context.Context, args []string) error {
defer open.Close()
inv := open.inventory
// A credential the provider's software issued is not the mesh's to remake (novox/hq ADR 0305 §9).
shelf, err := inv.Catalogue(ctx)
if err != nil {
return err
}
if err := issuedRefusesRotation(shelf, provision); err != nil {
return err
}
holders, err := inv.HoldersOf(ctx, provision, *only)
if err != nil {
return err
-1
View File
@@ -162,7 +162,6 @@ func TestABusRebuiltFromAnUnchangedSourceDemandsNoBusStep(t *testing.T) {
// A plan's build made from the source every machine running the module runs is not sent and not
// judged: nothing moves. A build of a changed source is sent to its first machine, gated, as before.
func TestAPlanSendsNothingForAnUnchangedSource(t *testing.T) {
judgeLeavesAsTheCore(t)
for _, tc := range []struct {
name string
source string
+1 -20
View File
@@ -410,10 +410,6 @@ func (a *verbArguments) commandLine() ([]string, error) {
if b := str("by"); b != "" {
argv = append(argv, "--by", catalogue.DeliverySeat+" for "+b)
}
// The merges to leave out: the walk's other merges are walked again without them (novox/hq ADR 0299).
if w := str("without"); w != "" {
argv = append(argv, "--without", w)
}
return argv, nil
case "delivery-walks":
argv := []string{"delivery", "walks"}
@@ -777,13 +773,6 @@ func (a *verbArguments) commandLine() ([]string, error) {
return nil, err
}
return []string{"secret", "ask", str("node"), str("module"), str("secret"), "--at", str("at")}, nil
case "issued":
// The credential a provider's software issued, taken from the provider (novox/hq ADR 0305): no value in
// the arguments, the provision alone.
if err := need("provision"); err != nil {
return nil, err
}
return []string{"secret", "issued", str("provision")}, nil
case "secret-ask":
// A module's own secret asked for, typed by the operator at the desk (novox/hq ADR 0277): never a value
// in the arguments. The desk is the module's machine unless at names another.
@@ -1590,14 +1579,6 @@ func givenAtTheDesk(argv []string) bool {
return argv[5] == "--at" && argv[6] != "" && !strings.HasPrefix(argv[6], "-")
}
// takenFromTheProvider is exactly the line the `issued` verb composes, and nothing beside it: `secret issued
// <provision>` (novox/hq ADR 0305 §10). It carries no value — the value comes from the provider's own tool, sealed to
// the call — and answers none.
func takenFromTheProvider(argv []string) bool {
return len(argv) == 3 && argv[0] == "secret" && argv[1] == "issued" && argv[2] != "" &&
!strings.HasPrefix(argv[2], "-")
}
// terminalOnly refuses, through any verb, a command that is the operator's at the controller's terminal
// alone (novox/hq ADR 0266). **Every `node` subcommand that is not a read**: `node account` and
// `node agent-account` above all. Whoever may call a verb includes agents, and an agent that named itself
@@ -1614,7 +1595,7 @@ func terminalOnly(argv []string) error {
// Of a secret's commands only rotation, which seals the new value to the machine that uses it, and the
// `give` verb's own line: an own secret typed by the operator into the desk's hidden prompt, sealed to this
// call and then to the module's machine, so no value travels in the verb or its answer (hq ADR 0259 §10).
if argv[0] == "secret" && (len(argv) < 2 || argv[1] != "rotate") && !givenAtTheDesk(argv) && !takenFromTheProvider(argv) {
if argv[0] == "secret" && (len(argv) < 2 || argv[1] != "rotate") && !givenAtTheDesk(argv) {
return terminalRefusal("secret %s is run at the controller's terminal only, never through a verb: accepting, "+
"recovering or exporting a secret hands it to whoever asks, and that includes agents (novox/hq ADR "+
"0266). Nothing was done", strings.Join(argv[1:], " "))
-4
View File
@@ -41,9 +41,6 @@ func secretCommand(ctx context.Context, args []string) error {
case "accept":
case "ask":
return secretAsk(ctx, args[1:])
case "issued":
// The credential a provider's software issued, taken from the provider (novox/hq ADR 0305).
return secretIssued(ctx, args[1:])
case "rotate":
return secretRotate(ctx, args[1:])
case "recover":
@@ -171,7 +168,6 @@ func secretAsk(ctx context.Context, args []string) error {
const secretUsage = "secret rotate <node> <module> <name> [--why <text> [--cause <word>]]\n" +
"secret ask <node> <module> <name> [--at <machine>]\n" +
"secret issued <provision>\n" +
"secret accept <node> <module> <name> [--from <file> | --at-desk <machine>] [--provider <node> [--local <name>]]\n" +
"secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>] [--provider <node>]\n" +
"secret export [--out <file>]"
+1 -1
View File
@@ -66,7 +66,7 @@ func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
}
// Both machines' `node show` name it: where the provider runs, and where the consumer is.
for _, node := range []string{"anchor", "laptop"} {
shown := printed(t, func() error { return showNode(ctx, open, node) })
shown := printed(t, func() error { return showNode(ctx, open.inventory, node) })
if !strings.Contains(shown, "open condition(s) about this machine") || !strings.Contains(shown, "mesh_laptop_dashboard") {
t.Fatalf("node show %s does not name it:\n%s", node, shown)
}
+2 -24
View File
@@ -289,10 +289,6 @@ func printStatus(asked answers) error {
fmt.Printf("\n `take <node> <module>` compares what runs against what it declares, and runs it\n\n")
}
// Assigned and not applied (novox/hq issue 380): before what is merely reported, because it reads like work
// finished and is none.
printNotApplied(asked.leftOut)
if len(asked.unheld) > 0 {
// **Reported, and not refused yet** (novox/hq ADR 0207 §4). Each machine still resolves and
// is sent what it would be; this says which of its modules depend on a seat nothing there
@@ -341,10 +337,6 @@ func printStatus(asked answers) error {
"lists them, and each is in its condition's tried\n\n", asked.heals.Acts, asked.heals.Escalated)
}
// What each machine was last sent and by whom, one line each (novox/hq issue 485). Not a fault, so it does not
// break "all well"; a record that could not be read does.
printLastSends(nodes, asked.lastSent, asked.lastSentUnread)
if adopted := adoptedNodes(nodes); len(adopted) > 0 {
// Said, because nothing forces the flip: a node left adopted is visible here rather than
// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
@@ -464,13 +456,6 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
return answers{}, err
}
plans[n.Name] = planned{plan, settings}
// And which of its modules a push leaves out (novox/hq issue 380), judged as the push judges it.
if left := leftOutOf(plan, settings, n.Adopted); len(left) > 0 {
if out.leftOut == nil {
out.leftOut = map[string][]leftOutModule{}
}
out.leftOut[n.Name] = left
}
out.unheld = append(out.unheld, plan.Unheld...)
// And which of its modules a provider leaves out of its grants, for an identity too long
// for what the provision keeps (novox/hq ADR 0225) — judged from the consumer's own
@@ -489,12 +474,6 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
out.pendingUnread = err.Error()
err = nil
}
// And what each machine was last sent, by whom (novox/hq issue 485): the record of sends is read through
// the controller's verbs alone, the store's reader being shut out of its tables. Unreadable is said.
if out.lastSent, err = inv.LastSends(ctx); err != nil {
out.lastSentUnread = err.Error()
err = nil
}
out.plans, err = inv.RecentPlans(ctx, 5)
if err != nil {
return answers{}, err
@@ -625,10 +604,9 @@ func untakenModules(ctx context.Context, inv *inventory.Inventory, nodes []inven
// read as success for the whole of the edge cut-over outage (novox/hq 04-ISSUES/125).
func (a answers) well() bool {
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 && len(a.leftOut) == 0 &&
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 &&
len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.overflowing) == 0 &&
len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingOpen(a.pending) &&
a.lastSentUnread == ""
len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingOpen(a.pending)
}
// pendingOpen is whether any pending assignment is still to be made. One that ended without being made is
-142
View File
@@ -1,142 +0,0 @@
{
"module": "nfs-server",
"version": "1",
"upgrade": {
"policy": "record",
"why": "the folders other machines mount: a build that breaks the exports leaves every client's mount hanging or refused, and the gate on this one machine does not see the clients (hq ADR 0236, ADR 0263)"
},
"capabilities": [
"package-manager",
"service-manager"
],
"provides": [
{
"name": "nfs-share",
"scope": "mesh",
"identity": false
}
],
"data": {
"consumers": {
"nfs-share": {
"class": "none",
"why": "the shared folders are the operator's data (hq ADR 0051): what a client writes lands in them, and they are protected where the operator declares them, never by this module, which keeps nothing of a consumer's"
}
}
},
"claims": [
{
"name": "node-nfs-server",
"scope": "node",
"serves": [
"exports",
"clients",
"test",
"reload",
"adopt"
]
}
],
"state": [
{
"name": "exports",
"ttl-seconds": 120,
"per-machine": true
}
],
"tools": [
"nfs_health"
],
"listens": [
{
"name": "nfs",
"port": 2049,
"protocol": "tcp",
"from": "mesh",
"fixed": true,
"why": "the shares, to the mesh's machines only (hq ADR 0263): NFS version 4 alone, which needs no other port, and never the home network, where a device that is not a node could claim any user id"
}
],
"resources": [
{
"id": "package",
"type": "package",
"package": "nfs-utils"
},
{
"id": "nfs-conf",
"type": "file",
"path": "/etc/nfs.conf.d/50-mesh.conf",
"mode": "0644",
"content": "# Written by the mesh (module nfs-server, novox/hq ADR 0263). Replaced on every push; a drop-in of\n# the operator's that sorts after this one overrides it, and is theirs.\n#\n# NFS version 4 only: a client needs port 2049 and nothing else, so the module opens nothing more\n# than that, to the private network. Version 3 needs rpcbind and mountd, on ports the mesh does not open.\n[nfsd]\nvers2=n\nvers3=n\nvers4=y\nvers4.0=n\nvers4.1=y\nvers4.2=y\n"
},
{
"id": "config-dir",
"type": "directory",
"path": "/etc/nfs-server",
"mode": "0755"
},
{
"id": "config",
"type": "file",
"path": "/etc/nfs-server/shares.conf",
"mode": "0644",
"content": "# Written by the mesh (module nfs-server, novox/hq ADR 0263) from this machine's assignment.\n# Replaced on every push; change the `shares` setting, never this file.\n#\n# The shares: name=folder, or name=folder:ro, one share per folder. The module's process exports each\n# to the private network's range below, every client mapped to the folder's owner.\nshares=${setting:shares}\nrange=${machine:mesh-range}\n"
},
{
"id": "run-dir",
"type": "directory",
"path": "/run/nfs-server",
"mode": "0755"
},
{
"id": "server",
"type": "service",
"unit": "nfs-server.service",
"state": "running",
"boot": "enabled",
"restart-on": [
"nfs-conf"
],
"health": {
"kind": "unit"
}
},
{
"id": "exports",
"type": "process",
"name": "nfs-server-exports",
"artifact": "tools",
"run": [
"./nfs-server",
"exports"
],
"restart-on": [
"config"
],
"health": {
"kind": "tool",
"tool": "nfs_health",
"interval": "60s",
"timeout": "10s",
"looks": 2,
"grace": "90s"
}
}
],
"build": {
"artifacts": [
{
"name": "tools",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/nfs-server",
"binary": "nfs-server",
"loads": [
"nfs-server"
]
}
]
}
}
-39
View File
@@ -167,7 +167,6 @@ func (tm *tierMesh) raise(machine, kind, summary string) {
// The incident's tier, small: a dozen modules whose first machine is the same are sent there in one
// send, judged by one gate, and the rest of their machines are sent once — two sends, not two dozen.
func TestATierOfManyModulesIsSentToEachMachineOnce(t *testing.T) {
judgeLeavesAsTheCore(t)
var modules []string
for i := 1; i <= 12; i++ {
modules = append(modules, fmt.Sprintf("m%02d", i))
@@ -269,7 +268,6 @@ func TestACoreBehindWhileTheNodeToolsSettleFailsNoOtherModule(t *testing.T) {
// A machine-level condition about anything else holds the machine back as a whole: everything the send
// moved there fails together, once, with that reason — not one module for it.
func TestAMachineUnhealthyAsAWholeFailsTheWholeSendTogether(t *testing.T) {
judgeLeavesAsTheCore(t)
tm := aTierMesh(t, "app1", "app2", "app3")
ctx := t.Context()
inv := tm.open.inventory
@@ -332,7 +330,6 @@ func TestTheConditionsAboutAMachineItself(t *testing.T) {
// rollback that failed; and `plans retry` asks it again rather than refusing a build that was never
// judged (the incident's "NOT put back: no build kept").
func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) {
judgeLeavesAsTheCore(t)
g := aGateMesh(t)
ctx := t.Context()
inv := g.open.inventory
@@ -387,7 +384,6 @@ func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) {
// them, they are registered back and marked in the judging that finds them broken, the plan goes on judging
// the other module to its own pass, and only then fails.
func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) {
judgeLeavesAsTheCore(t)
for _, order := range [][]string{{broker.RuntimeModule, "app1"}, {"app1", broker.RuntimeModule}} {
t.Run("gate kept on "+order[0], func(t *testing.T) {
tm := aTierMesh(t, order...)
@@ -439,38 +435,3 @@ func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) {
})
}
}
// **A send carrying a core module waits for the core's readings** (issue 524 review): the leaf beside the node
// tools has its one reading at once, but the send is judged as the core: not before the settle time, and three
// readings.
func TestASendCarryingTheCoreIsJudgedAsTheCore(t *testing.T) {
tm := aTierMesh(t, broker.RuntimeModule, "app1")
ctx := t.Context()
// One reading counted per advance, as every test with a gate left open does: a gate judged with no time
// between readings would count a pass at every look and never let an advance rest.
gateEvery, gateSettle = time.Hour, time.Hour
for i := 0; i < 6; i++ {
advancePlans(ctx, tm.open)
}
p := tm.plan(t)
lead := p.Modules[broker.RuntimeModule].Gate
if lead == nil {
lead = p.Modules["app1"].Gate
}
if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) || !p.Open() || lead == nil || lead.Verdict != "" || lead.Wants != gatePasses {
t.Fatalf("passed before the core's settle time: sent %v, plan %s (%s), gate %+v", tm.sent, p.State, p.Note, lead)
}
gateEvery, gateSettle = 0, 0
for i := 0; i < 6; i++ {
advancePlans(ctx, tm.open)
}
p = tm.plan(t)
if p.State != inventory.PlanDone || !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"laptop"}}) {
t.Fatalf("the plan is %s (%s), sent %v", p.State, p.Note, tm.sent)
}
for _, m := range []string{broker.RuntimeModule, "app1"} {
if g := p.Modules[m].Gate; g != nil && (g.Verdict != inventory.GatePassed || g.Passes < gatePasses) {
t.Fatalf("%s's gate: %+v; want passed on %d readings", m, g, gatePasses)
}
}
}
-53
View File
@@ -1,53 +0,0 @@
package broker
import "testing"
// The controller asks a tool that answers a software-issued credential in its own name, and nothing else under
// `call` (novox/hq ADR 0305 §2, issue 526): the grant is composed from the records, so the installer's first user
// list, a controller with no records, carries none.
func TestTheControllerAsksAnIssuedCredentialsToolInItsOwnNameAlone(t *testing.T) {
r := Records{Nodes: []string{"n", "ace"}, Assigned: map[string][]Declared{
"n": {{Module: "x", Issued: []string{"t"}}},
"ace": {{Module: "plex", Issued: []string{"plex_issued_credential"}}, {Module: "kometa"}},
}}
users, err := Users(r)
if err != nil {
t.Fatal(err)
}
controller := users[0]
if controller.Kind != KindController {
t.Fatalf("the first user is %s", controller.Username())
}
perms, err := PermissionsFor(controller)
if err != nil {
t.Fatal(err)
}
for _, s := range []string{
"mesh.mod.x.call.t.n.controller",
"mesh.mod.plex.call.plex_issued_credential.ace.controller",
} {
if !MayPublish(perms, s) {
t.Errorf("the controller may not publish %s", s)
}
}
for _, s := range []string{
"mesh.mod.x.call.t.n.ace.node-tools",
"mesh.mod.x.call.t.n." + CallerToken("ace.node-tools"),
"mesh.mod.plex.call.plex_issued_credential.ace." + CallerToken("ace.node-tools"),
"mesh.mod.plex.call.plex_status.ace.controller",
"mesh.mod.kometa.call.t.ace.controller",
"mesh.seat.mesh-controller.call.status.controller",
} {
if MayPublish(perms, s) {
t.Errorf("the controller may publish %s", s)
}
}
bare, err := PermissionsFor(Principal{Kind: KindController})
if err != nil {
t.Fatal(err)
}
if MayPublish(bare, "mesh.mod.x.call.t.n.controller") {
t.Fatal("a controller composed with no records is granted a caller-named call")
}
}
+38
View File
@@ -448,3 +448,41 @@ func (j *JetStream) BucketNames() ([]string, error) {
}
return out, lister.Error()
}
// EnsureLog creates a module's log if it is absent and brings its configuration to match if it is
// present (novox/hq ADR 0297).
//
// **An update, never a delete and recreate**, for a bucket's reason: recreating discards what the log
// holds, and a log is a module's record. A configuration the server will not change in place (its
// storage, say, on a stream made by hand) is said as this assertion's error and the stream is left as
// it is — never removed to be made again.
func (j *JetStream) EnsureLog(l Log) error {
js, err := jetstream.New(j.conn)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if _, err := js.CreateOrUpdateStream(ctx, l.Config()); err != nil {
return fmt.Errorf("asserting log %s: %w", l.Stream(), err)
}
return nil
}
// LogStreams is every log stream on the server: every stream whose name starts with LogStreamPrefix.
func (j *JetStream) LogStreams() ([]string, error) {
js, err := jetstream.New(j.conn)
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
lister := js.StreamNames(ctx)
var out []string
for name := range lister.Name() {
if strings.HasPrefix(name, LogStreamPrefix) {
out = append(out, name)
}
}
return out, lister.Err()
}
+195
View File
@@ -0,0 +1,195 @@
package broker
import (
"fmt"
"sort"
"strings"
"github.com/nats-io/nats.go/jetstream"
)
// A module's log on the bus (novox/hq ADR 0297): its record of operations, each entry appended under
// a key and kept as long as the log is.
//
// A log follows a bucket in every respect (ADR 0201): the module names it locally, the mesh derives
// its stream and subjects, the controller creates it from the catalogue on every raise and from
// registration, never a module, and **no path of the mesh removes one** — a log whose declaration
// is gone is reported, as a bucket is. Unlike a bucket, a log is its owner's alone: no other module
// reads it, so there is no read of a log to grant or issue.
//
// Pure, but for the stream's configuration, which is the server's own type so that what is tested
// is exactly what is sent; jetstream.go is the part that asks a server.
// The mesh's caps on a log: what an entry's value may weigh, what a message on the log's stream may
// weigh — the value and its headers, which the server counts in a message's size, so a value of the
// full size still fits with the expected-last-sequence header an append carries — and what a log holds
// when its module says nothing and at most.
const (
LogMaxEntryBytes = 256 * 1024
LogMaxMessageBytes = LogMaxEntryBytes + 4*1024
LogDefaultMiB = 1024
LogMostMiB = 8192
)
// A Log is one module's declared log as the bus holds it.
type Log struct {
Module string
Name string
// MaxMiB is its cap in MiB; zero is LogDefaultMiB.
MaxMiB int
}
// LogStreamPrefix starts every log's stream name, which no other stream of the mesh's starts with.
const LogStreamPrefix = "LOG_"
// LogStreamName is the stream a module's log lives in: `LOG_<module>_<name>`. The module and the
// local name are each one token with no underscore, so two modules can never derive one stream.
func LogStreamName(module, name string) string { return LogStreamPrefix + module + "_" + name }
// LogSubject is the subject a log's entries are published under, without the key: an entry for key K
// is on `<LogSubject>.<K>`.
func LogSubject(module, name string) string { return "mesh.log." + module + "." + name }
// Stream is this log's stream name.
func (l Log) Stream() string { return LogStreamName(l.Module, l.Name) }
// Subject is this log's subject, without the key.
func (l Log) Subject() string { return LogSubject(l.Module, l.Name) }
// MaxBytes is this log's cap in bytes.
func (l Log) MaxBytes() int64 {
mib := l.MaxMiB
if mib <= 0 {
mib = LogDefaultMiB
}
return int64(mib) * 1024 * 1024
}
// Why is carried into the server's description of the stream, so somebody reading the server's own
// state finds whose it is and why it is kept.
func (l Log) Why() string {
return fmt.Sprintf("%s's log %q (novox/hq ADR 0297): its record of operations, one entry per operation "+
"under its key, appended by %s alone and kept as long as the log; never removed by the mesh, because "+
"it is data", l.Module, l.Name, l.Module)
}
// Config is the stream a log is, field by field as novox/hq ADR 0297 fixes it: a file stream kept by
// limits, with no maximum age and no cap per key, whose message is an entry's value and 4 KiB of
// headers at most, that refuses a new entry when full rather than
// drop an old one, and that refuses deleting an entry or purging it. Direct gets are allowed, which
// is how the runtime reads it.
func (l Log) Config() jetstream.StreamConfig {
return jetstream.StreamConfig{
Name: l.Stream(),
Description: l.Why(),
Subjects: []string{l.Subject() + ".>"},
Storage: jetstream.FileStorage,
Retention: jetstream.LimitsPolicy,
MaxAge: 0,
MaxMsgs: -1,
MaxMsgsPerSubject: -1,
MaxBytes: l.MaxBytes(),
MaxMsgSize: LogMaxMessageBytes,
Discard: jetstream.DiscardNew,
AllowDirect: true,
DenyDelete: true,
DenyPurge: true,
Replicas: 1,
}
}
// LogIssued is one log an assignment may reach, by the name its module uses for it (novox/hq ADR
// 0297): its stream, the subject its entries go under, and whether it may append. Only the owner's
// instances are issued a log, so Writes is always true today; it is said so the runtime need not
// assume it.
type LogIssued struct {
Name string `json:"name"`
Stream string `json:"stream"`
Subject string `json:"subject"`
Writes bool `json:"writes"`
}
// logsIssuedFor is every log a module's code may reach, as its membership lists them: its own.
func logsIssuedFor(d Declared) []LogIssued {
var out []LogIssued
for _, l := range d.Logs {
if !safeSubject.MatchString(d.Module) || !safeSubject.MatchString(l.Name) {
continue
}
out = append(out, LogIssued{Name: l.Name, Stream: LogStreamName(d.Module, l.Name),
Subject: LogSubject(d.Module, l.Name), Writes: true})
}
return out
}
// logGrants is what a principal publishes to reach the logs its module keeps: for each, appending
// under the log's subjects, binding to its stream, and reading it directly. Nothing more — the
// runtime reads a log by direct gets alone and makes no consumer on it — and nothing of any other
// module's log, because a log is its owner's alone. Replies come on the principal's inbox, as for a
// bucket.
func logGrants(module string, names []string) []string {
if !safeSubject.MatchString(module) {
return nil
}
var out []string
for _, name := range names {
if !safeSubject.MatchString(name) {
continue
}
stream := LogStreamName(module, name)
out = append(out,
LogSubject(module, name)+".>",
"$JS.API.STREAM.INFO."+stream,
"$JS.API.DIRECT.GET."+stream,
"$JS.API.DIRECT.GET."+stream+".>")
}
return out
}
// logNames is the local names of a module's logs.
func logNames(logs []Log) []string {
out := make([]string, 0, len(logs))
for _, l := range logs {
out = append(out, l.Name)
}
return out
}
// A LogAsserter is the part of a JetStream connection log assertion needs. It has no way to remove a
// log, by design: nothing the mesh runs asks for one.
type LogAsserter interface {
// EnsureLog creates the log's stream if absent and brings its configuration to match if present,
// never deleting or recreating it.
EnsureLog(l Log) error
// LogStreams is every log stream on the server: every stream named with LogStreamPrefix.
LogStreams() ([]string, error)
}
// RaiseLogs asserts every declared log and answers the log streams on the server that nothing
// declares any more.
//
// **Those are reported, never removed** (novox/hq ADR 0297 §2, ADR 0201 §15–16): a log is a module's
// record, and a manifest edited, a module renamed or unassigned is an ordinary day's work that must
// not take a record with it. Removing one is a person's act, outside the mesh.
func RaiseLogs(a LogAsserter, logs []Log) (undeclared []string, err error) {
sorted := append([]Log(nil), logs...)
sort.Slice(sorted, func(i, j int) bool { return sorted[i].Stream() < sorted[j].Stream() })
declared := map[string]bool{}
for _, l := range sorted {
if err := a.EnsureLog(l); err != nil {
return nil, fmt.Errorf("asserting %s's log %q: %w", l.Module, l.Name, err)
}
declared[l.Stream()] = true
}
names, err := a.LogStreams()
if err != nil {
return nil, fmt.Errorf("listing the bus's logs: %w", err)
}
for _, n := range names {
if strings.HasPrefix(n, LogStreamPrefix) && !declared[n] {
undeclared = append(undeclared, n)
}
}
sort.Strings(undeclared)
return undeclared, nil
}
+294
View File
@@ -0,0 +1,294 @@
package broker
import (
"context"
"encoding/json"
"go/ast"
"go/parser"
"go/token"
"io/fs"
"path/filepath"
"slices"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go/jetstream"
)
// **The stream a log is, field by field** (novox/hq ADR 0297 §2, the shared contract): a file stream
// kept by limits, no maximum age, no cap per key, the declared cap, a message of at most 260 KiB (a
// value of 256 KiB and 4 KiB of headers),
// refusing what comes next when full, read directly, refusing a delete or a purge, one replica, and
// a description that says whose it is and why.
func TestALogsStreamIsAsTheDecisionFixesIt(t *testing.T) {
got := Log{Module: "mesh-issues", Name: "changes"}.Config()
want := jetstream.StreamConfig{
Name: "LOG_mesh-issues_changes",
Description: got.Description,
Subjects: []string{"mesh.log.mesh-issues.changes.>"},
Storage: jetstream.FileStorage,
Retention: jetstream.LimitsPolicy,
MaxAge: 0,
MaxMsgs: -1,
MaxMsgsPerSubject: -1,
MaxBytes: 1024 * 1024 * 1024,
MaxMsgSize: 260 * 1024,
Discard: jetstream.DiscardNew,
AllowDirect: true,
DenyDelete: true,
DenyPurge: true,
Replicas: 1,
}
a, _ := json.Marshal(got)
b, _ := json.Marshal(want)
if string(a) != string(b) {
t.Fatalf("the log's stream is\n %s\nwant\n %s", a, b)
}
if !strings.Contains(got.Description, "mesh-issues") || !strings.Contains(got.Description, "ADR 0297") {
t.Fatalf("the description does not say whose the log is and why: %q", got.Description)
}
if c := (Log{Module: "m", Name: "n", MaxMiB: 8192}).Config(); c.MaxBytes != 8192*1024*1024 {
t.Fatalf("a cap of 8192 MiB is %d bytes", c.MaxBytes)
}
}
// **The membership names each of the owner's logs** with exactly the field names the runtime reads:
// `logs`, and in each `name`, `stream`, `subject`, `writes`. A module with no log is issued none, and
// the field is absent.
func TestAMembershipListsItsModulesLogs(t *testing.T) {
m := MembershipFor("one", Declared{Module: "mesh-issues",
Logs: []Log{{Module: "mesh-issues", Name: "changes"}}}, Placements{})
raw, err := json.Marshal(m)
if err != nil {
t.Fatal(err)
}
var back map[string]json.RawMessage
if err := json.Unmarshal(raw, &back); err != nil {
t.Fatal(err)
}
if got := string(back["logs"]); got !=
`[{"name":"changes","stream":"LOG_mesh-issues_changes","subject":"mesh.log.mesh-issues.changes","writes":true}]` {
t.Fatalf("the membership's logs are %s", got)
}
none, _ := json.Marshal(MembershipFor("one", Declared{Module: "audit"}, Placements{}))
if strings.Contains(string(none), `"logs"`) {
t.Fatalf("a module with no log was issued logs: %s", none)
}
}
// logGrantsOf is the grants of a principal that are about logs.
func logGrantsOf(publish []string) []string {
var out []string
for _, s := range publish {
if strings.HasPrefix(s, "mesh.log.") || strings.Contains(s, ".LOG_") {
out = append(out, s)
}
}
slices.Sort(out)
return out
}
// **The runtime is granted exactly the contract's subjects for each log it carries, and nothing else
// of any log**: appending under the log's subjects, binding to its stream, reading it directly. No
// consumer, no delete, no purge, and nothing of a log its modules do not keep.
func TestTheRuntimeIsGrantedItsModulesLogsAndNoMore(t *testing.T) {
perms, err := PermissionsFor(Principal{Kind: KindNodeTools, Node: "one", Module: RuntimeModule,
Carries: []Declared{
{Module: "mesh-issues", Logs: []Log{{Module: "mesh-issues", Name: "changes"}}},
{Module: "audit"},
}, PasswordHash: "x"})
if err != nil {
t.Fatal(err)
}
want := []string{
"$JS.API.DIRECT.GET.LOG_mesh-issues_changes",
"$JS.API.DIRECT.GET.LOG_mesh-issues_changes.>",
"$JS.API.STREAM.INFO.LOG_mesh-issues_changes",
"mesh.log.mesh-issues.changes.>",
}
if got := logGrantsOf(perms.Publish); !slices.Equal(got, want) {
t.Fatalf("the runtime is granted\n %q\nwant\n %q", got, want)
}
for _, s := range perms.Subscribe {
if strings.HasPrefix(s, "mesh.log.") || strings.Contains(s, "LOG_") {
t.Fatalf("the runtime subscribes a log's subjects: %q", s)
}
}
// A module running on its own account is granted the same for its own logs.
own, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "mesh-issues",
Logs: []string{"changes"}, PasswordHash: "x"})
if err != nil {
t.Fatal(err)
}
if got := logGrantsOf(own.Publish); !slices.Equal(got, want) {
t.Fatalf("the module's own account is granted\n %q\nwant\n %q", got, want)
}
// And a module with no log, nothing of any.
none, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "audit", PasswordHash: "x"})
if err != nil {
t.Fatal(err)
}
if got := logGrantsOf(none.Publish); len(got) != 0 {
t.Fatalf("a module with no log is granted %q", got)
}
}
// A log name or module that is not one plain token is issued and granted nothing rather than a
// pattern that happens to parse.
func TestALogThatNamesNoStreamGrantsNothing(t *testing.T) {
if got := logGrants("a", []string{"x.y", "x>", "*", ""}); len(got) != 0 {
t.Fatalf("granted %v for logs that name no stream", got)
}
if got := logGrants("a.b", []string{"c"}); len(got) != 0 {
t.Fatalf("granted %v for a module that is not one token", got)
}
}
type logs struct {
ensured []string
on []string
}
func (l *logs) EnsureLog(x Log) error { l.ensured = append(l.ensured, x.Stream()); return nil }
func (l *logs) LogStreams() ([]string, error) { return l.on, nil }
// Every declared log is asserted; one on the server that nothing declares is said, not removed — and
// the asserter has no way to remove one.
func TestRaisingLogsReportsWhatNothingDeclares(t *testing.T) {
l := &logs{on: []string{"LOG_mesh-issues_changes", "LOG_gone_old", "KV_not_a_log"}}
undeclared, err := RaiseLogs(l, []Log{{Module: "mesh-issues", Name: "changes"}, {Module: "a", Name: "b"}})
if err != nil {
t.Fatal(err)
}
if !slices.Equal(l.ensured, []string{"LOG_a_b", "LOG_mesh-issues_changes"}) {
t.Fatalf("asserted %v", l.ensured)
}
if !slices.Equal(undeclared, []string{"LOG_gone_old"}) {
t.Fatalf("reported %v", undeclared)
}
}
// **No path of the mesh deletes or purges a log or a bucket, or recreates one** (novox/hq ADR 0297 §2,
// ADR 0201 §15–16): no code of this repository outside its tests calls the client's stream or bucket
// delete, or purges a stream, but for the controller lease's own stream, which purges its own history
// below a sequence (internal/lease). A new call is a decision, not a refactor.
func TestNoPathDeletesALogOrABucket(t *testing.T) {
removers := map[string]bool{"DeleteStream": true, "DeleteKeyValue": true, "PurgeStream": true,
"DeleteObjectStore": true}
root := filepath.Join("..", "..")
var found []string
for _, dir := range []string{"cmd", "internal"} {
err := filepath.WalkDir(filepath.Join(root, dir), func(path string, e fs.DirEntry, err error) error {
if err != nil || e.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
return err
}
f, err := parser.ParseFile(token.NewFileSet(), path, nil, 0)
if err != nil {
return err
}
ast.Inspect(f, func(n ast.Node) bool {
call, ok := n.(*ast.CallExpr)
if !ok {
return true
}
sel, ok := call.Fun.(*ast.SelectorExpr)
if !ok {
return true
}
name := sel.Sel.Name
if removers[name] || (name == "Purge" && !strings.Contains(filepath.ToSlash(path), "internal/lease/")) {
found = append(found, path+": "+name)
}
return true
})
for _, lit := range stringsIn(f) {
if strings.Contains(lit, "$JS.API.STREAM.DELETE") || strings.Contains(lit, "$JS.API.STREAM.PURGE") {
// Only the writers table names it, as a subject no one but the controller may publish.
if !strings.HasSuffix(filepath.ToSlash(path), "internal/broker/writers.go") {
found = append(found, path+": "+lit)
}
}
}
return nil
})
if err != nil {
t.Fatal(err)
}
}
if len(found) > 0 {
t.Fatalf("a path of the mesh removes a stream, a bucket or what one holds:\n %s", strings.Join(found, "\n "))
}
}
// stringsIn is every string literal of a file.
func stringsIn(f *ast.File) []string {
var out []string
ast.Inspect(f, func(n ast.Node) bool {
if lit, ok := n.(*ast.BasicLit); ok && lit.Kind == token.STRING {
out = append(out, lit.Value)
}
return true
})
return out
}
// Against a real server: a log is created as its configuration says, asserting it again keeps what
// it holds, a changed cap is brought to match in place, an entry cannot be deleted from it, and one
// nothing declares any more is reported and stays.
func TestALogIsAssertedInPlaceAndNeverRemoved(t *testing.T) {
bus := aLiveBus(t)
l := Log{Module: "logtest", Name: "changes", MaxMiB: 1}
if _, err := RaiseLogs(bus, []Log{l}); err != nil {
t.Fatalf("a real server refused a module's log: %v", err)
}
js, err := jetstream.New(bus.Conn())
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
ack, err := js.Publish(ctx, l.Subject()+".7", []byte(`{"op":"opened"}`))
if err != nil {
t.Fatalf("an append to the log was refused: %v", err)
}
stream, err := js.Stream(ctx, l.Stream())
if err != nil {
t.Fatal(err)
}
have := stream.CachedInfo().Config
want := l.Config()
if have.Storage != want.Storage || have.Retention != want.Retention || have.MaxAge != 0 ||
have.MaxMsgsPerSubject != -1 || have.MaxBytes != want.MaxBytes || have.MaxMsgSize != want.MaxMsgSize ||
have.Discard != jetstream.DiscardNew || !have.AllowDirect || !have.DenyDelete || !have.DenyPurge ||
have.Replicas != 1 || !slices.Equal(have.Subjects, want.Subjects) {
t.Fatalf("the server holds the log as %+v", have)
}
if err := stream.DeleteMsg(ctx, ack.Sequence); err == nil {
t.Fatal("an entry was deleted from a log")
}
l.MaxMiB = 2
if _, err := RaiseLogs(bus, []Log{l}); err != nil {
t.Fatalf("asserting the log again failed, so a restart would: %v", err)
}
info, err := stream.Info(ctx)
if err != nil {
t.Fatal(err)
}
if info.Config.MaxBytes != 2*1024*1024 {
t.Fatalf("the changed cap was not brought to match: %d", info.Config.MaxBytes)
}
if info.State.Msgs < 1 {
t.Fatal("asserting the log again lost what it held")
}
undeclared, err := RaiseLogs(bus, nil)
if err != nil {
t.Fatal(err)
}
if !slices.Contains(undeclared, l.Stream()) {
t.Fatalf("a log nothing declares was not reported: %v", undeclared)
}
if _, err := js.Stream(ctx, l.Stream()); err != nil {
t.Fatalf("a log nothing declares is gone: %v", err)
}
}
+5
View File
@@ -51,6 +51,10 @@ type Membership struct {
// and refuses, with the reason, what is not on it — the bus enforces only the union over every
// module on the machine.
State []StateIssued `json:"state,omitempty"`
// Logs is every log this module's code may reach, by the name it uses for each (novox/hq ADR 0297):
// its own and no other's, since a log is its owner's alone. The runtime answers a bundle's log verbs
// from this list and refuses, with the reason, a log not on it.
Logs []LogIssued `json:"logs,omitempty"`
// SeatTraffic is what this module's code may submit, say, hear, take, ask, answer and read on seats
// that name their caller or their kind (novox/hq ADR 0259 §3). The runtime carrying the module
// publishes, takes and answers for it only what is listed here: the bus enforces only the union
@@ -121,6 +125,7 @@ func MembershipFor(node string, d Declared, where Placements) Membership {
}
}
m.State = stateIssuedFor(d, node)
m.Logs = logsIssuedFor(d)
t := SeatTrafficOf(d.Module, d.Holds, d.Uses, d.Watches)
for _, s := range append(append([]Seat{}, d.Uses...), d.Watches...) {
if !s.Kinded {
+10 -18
View File
@@ -123,13 +123,6 @@ type Principal struct {
// permission and nothing beside it.
Invokes []string
// AsksIssued are the tools the controller asks in its own name for a credential a provider's software
// issued, each `<module>.<tool>` (novox/hq ADR 0305): on the subject that names its caller, so the tool
// knows the controller asks. Composed from the records, so the installer's first user list — a controller
// with no records — carries none, and moves with issue 464 like the rest of the controller's caller-named
// grants.
AsksIssued []string
// State is the local names of the state this principal's module keeps, and Reads the state of
// others it reads as `<module>.<name>` (novox/hq ADR 0201): a bucket each, kept by the owner's
// instances and read by whoever declares it.
@@ -139,6 +132,8 @@ type Principal struct {
// no other; KeyedReads the keys of others' state it reads one key at a time (novox/hq ADR 0260).
PerMachine []string
KeyedReads []KeyedRead
// Logs is the local names of the logs this principal's module keeps (novox/hq ADR 0297).
Logs []string
// SnapshotsTheBus is the bus's own module, the one holding mesh-broker (novox/hq ADR 0235). Its
// whole authority is BusSnapshotGrants: it copies the streams for the night's backup and nothing
@@ -753,6 +748,8 @@ func PermissionsFor(p Principal) (Permissions, error) {
// watched, its own written too.
pub = append(pub, stateGrants(stateAccess{Module: p.Module, Node: p.Node, Keeps: p.State,
PerMachine: p.PerMachine, Reads: p.Reads, KeyedReads: p.KeyedReads})...)
// And its logs (novox/hq ADR 0297): appended to and read directly, its own alone.
pub = append(pub, logGrants(p.Module, p.Logs)...)
// 6. Its traffic on seats that name their caller or their kind, ask proofs or keep records
// (novox/hq ADR 0259 §3).
@@ -840,6 +837,12 @@ func PermissionsFor(p Principal) (Permissions, error) {
pub = append(pub, stateGrants(stateAccess{Module: d.Module, Node: p.Node, Keeps: stateNames(d.State),
PerMachine: perMachineNames(d.State), Reads: d.Reads, KeyedReads: d.KeyedReads})...)
}
// **And it keeps the logs of the modules it carries** (novox/hq ADR 0297): each module's own, the
// union over them. That one module's code does not append to another's log through it is the
// runtime's to keep, from the logs each membership lists.
for _, d := range p.Carries {
pub = append(pub, logGrants(d.Module, logNames(d.Logs))...)
}
// **Never the traffic of a trusted holder** (novox/hq ADR 0259 §8): the machine's runtime runs as the
// operator's account, which every agent runs as, so a module saying warrants or speaking for a kind
// that proves its sender is never composed into it — refused here, naming it, whatever registration
@@ -897,17 +900,6 @@ func PermissionsFor(p Principal) (Permissions, error) {
// and moves with issue 464.
if p.Kind != KindController {
pub, sub = callerNamed(p.Username(), pub, sub)
} else {
// **But the tools that answer a software-issued credential** (novox/hq ADR 0305 §2): those answer the
// controller alone, and know it only from the subject's last token, so it asks them in its own name —
// those tools, on any machine, and nothing else under `call`.
for _, key := range p.AsksIssued {
module, tool, ok := strings.Cut(key, ".")
if !ok || module == "" || tool == "" || strings.ContainsAny(key, "*> ") {
continue
}
pub = append(pub, "mesh.mod."+module+"."+CallKind+"."+tool+".*."+CallerToken(p.Username()))
}
}
sort.Strings(pub)
+4 -23
View File
@@ -29,15 +29,14 @@ type Declared struct {
Holds []Seat
// Uses are the seats this module sends to.
Uses []Seat
// Issued are the tools that answer a credential its software issued, one per offer that says so (novox/hq
// ADR 0305): the controller is granted to ask each in its own name, and no other tool so.
Issued []string
// Watches are the seats whose events it consumes.
Watches []Seat
// Invokes are the tools it calls, `<module>.<tool>` or `*` (novox/hq ADR 0152).
Invokes []string
// State is the state it keeps, each a bucket its instances write (novox/hq ADR 0201).
State []Bucket
// Logs are the logs it keeps, each a stream its instances append to (novox/hq ADR 0297).
Logs []Log
// Reads are other modules' state it reads, each `<module>.<name>` (novox/hq ADR 0201).
Reads []string
// KeyedReads are keys of other modules' state it reads, each one key alone: what a seat's holder is
@@ -85,7 +84,7 @@ type Records struct {
// is a mesh that cannot be told anything, and there is no state of the records in which that is
// correct.
func Users(r Records) ([]Principal, error) {
out := []Principal{{Kind: KindController, Uses: asksSeatOf(r), AsksIssued: issuedOf(r)}}
out := []Principal{{Kind: KindController, Uses: asksSeatOf(r)}}
for _, node := range sortedCopy(r.Nodes) {
witness := false
@@ -127,6 +126,7 @@ func Users(r Records) ([]Principal, error) {
Holds: d.Holds, Uses: d.Uses, Watches: d.Watches, Invokes: d.Invokes,
State: stateNames(d.State), Reads: d.Reads, SnapshotsTheBus: d.SnapshotsTheBus,
PerMachine: perMachineNames(d.State), KeyedReads: d.KeyedReads,
Logs: logNames(d.Logs),
})
}
if runtimeHere {
@@ -206,25 +206,6 @@ const controllerModule = "mesh-controller"
// asksSeatOf is the seat an ask is made on, as its holder declares it (novox/hq ADR 0259): the controller
// asks the operator through it like any other user, and is granted what its declaration names for a caller.
// None while nothing holds it.
// issuedOf is every tool of a module assigned anywhere that answers a credential its software issued, as
// `<module>.<tool>` (novox/hq ADR 0305): what the controller asks in its own name, and nothing beside.
func issuedOf(r Records) []string {
seen := map[string]bool{}
var out []string
for _, node := range sortedCopy(r.Nodes) {
for _, d := range r.Assigned[node] {
for _, t := range d.Issued {
if key := d.Module + "." + t; !seen[key] {
seen[key] = true
out = append(out, key)
}
}
}
}
sort.Strings(out)
return out
}
func asksSeatOf(r Records) []Seat {
for _, node := range sortedCopy(r.Nodes) {
for _, d := range r.Assigned[node] {
@@ -79,38 +79,23 @@ func TestAHolderMeetsItsOwnContribution(t *testing.T) {
}
}
func TestTwoModulesMayDeclareOnePackage(t *testing.T) {
func TestTwoModulesDeclaringOnePackageAreRefusedBeforeAnythingIsRecorded(t *testing.T) {
pacman := withResources(mod("pacman", nil, nil, nil, Claim{Name: PackageManagerSeat}),
res("package", "pacman-contrib"))
bar := withResources(mod("bar", nil, nil, nil), res("package", "bar"), res("package", "pacman-contrib"))
catalogue := map[string]Manifest{"pacman": pacman, "bar": bar}
if err := CollisionRefusal(catalogue, "laptop", []string{"pacman"}, []string{"bar"}); err != nil {
t.Fatalf("a second declaration of a package was refused (novox/hq ADR 0303): %v", err)
}
}
func TestAPackageDeclaredPresentAndAbsentIsRefusedNamingBoth(t *testing.T) {
gone := res("package", "ufw")
gone["absent"] = true
filter := withResources(mod("nftables", nil, nil, nil), gone)
wants := withResources(mod("legacy", nil, nil, nil), res("package", "ufw"))
other := withResources(mod("other", nil, nil, nil), res("package", "other"))
catalogue := map[string]Manifest{"nftables": filter, "legacy": wants, "other": other}
catalogue := map[string]Manifest{"pacman": pacman, "bar": bar, "other": other}
err := CollisionRefusal(catalogue, "laptop", []string{"nftables"}, []string{"legacy"})
if err == nil || !strings.Contains(err.Error(), "ufw") || !strings.Contains(err.Error(), "nftables") ||
!strings.Contains(err.Error(), "legacy") || !strings.Contains(err.Error(), "ADR 0303") {
t.Fatalf("a package declared present and absent was not refused naming both: %v", err)
err := CollisionRefusal(catalogue, "laptop", []string{"pacman"}, []string{"bar"})
if err == nil || !strings.Contains(err.Error(), "pacman-contrib") || !strings.Contains(err.Error(), "bar") {
t.Fatalf("the second owner of a package was not refused by name: %v", err)
}
if strings.Contains(err.Error(), "one owner per node") {
t.Errorf("the refusal gives the one-owner rule, which no longer holds for packages: %v", err)
if err := CollisionRefusal(catalogue, "laptop", []string{"pacman"}, []string{"other"}); err != nil {
t.Errorf("a module declaring nothing shared is refused: %v", err)
}
if err := CollisionRefusal(catalogue, "laptop", []string{"nftables"}, []string{"other"}); err != nil {
t.Errorf("a module declaring nothing contradictory is refused: %v", err)
}
// A contradiction already on the node is status's, not a reason to refuse an unrelated assignment.
if err := CollisionRefusal(catalogue, "laptop", []string{"nftables", "legacy"}, []string{"other"}); err != nil {
t.Errorf("an unrelated assignment is refused for a contradiction already there: %v", err)
// A collision already on the node is status's, not a reason to refuse an unrelated assignment.
if err := CollisionRefusal(catalogue, "laptop", []string{"pacman", "bar"}, []string{"other"}); err != nil {
t.Errorf("an unrelated assignment is refused for a collision already there: %v", err)
}
}
+2 -22
View File
@@ -342,38 +342,18 @@ func (e *NotMadeError) Error() string {
// compose. Empty when every module composes. The same judgement SetSettings makes before storing.
func (r Resolution) LeftOut(settings SettingsBy, adopted bool) map[string]string {
out := map[string]string{}
for module, why := range r.LeftOutBecause(settings, adopted) {
out[module] = why.Error()
}
return out
}
// LeftOutBecause is LeftOut with each reason as the error it was, so a reader can tell a setting nobody
// gave (an *UnsetSettingError) from any other cause and say it as the operator's to give (novox/hq issue
// 380). An UnreadManifestError for a stored manifest this controller cannot read whole.
func (r Resolution) LeftOutBecause(settings SettingsBy, adopted bool) map[string]error {
out := map[string]error{}
for _, m := range r.Modules {
if why := UnknownFieldReason(m); why != "" {
out[m.Module] = &UnreadManifestError{Module: m.Module, said: why}
out[m.Module] = why
continue
}
if err := JudgeSettings(m, settings[m.Module], adopted); err != nil {
out[m.Module] = err
out[m.Module] = err.Error()
}
}
return out
}
// UnreadManifestError is a module left out because its stored manifest has a key this controller does not know
// (novox/hq ADR 0262): said once for the whole mesh, by the catalogue's own condition, not per machine.
type UnreadManifestError struct {
Module string
said string
}
func (e *UnreadManifestError) Error() string { return e.said }
// Compose is Declaration with the owner of every resource said.
func (r Resolution) Compose(with Rendering) (Composed, error) {
owner := map[string]string{}
+1 -1
View File
@@ -21,7 +21,7 @@ func deliveryVerbs() []Verb {
// Delivery time against the delivery budgets (novox/hq ADR 0282): what probe D16 reads, optional until its
// holder serves it.
{Name: "times", Description: "Delivery time: from a merge to every machine of its walk running the build. " +
"Each class's delivery budget (a leaf module three minutes, a core module ten), the last days' median and " +
"Each class's delivery budget (a leaf module five minutes, a core module ten), the last days' median and " +
"worst, how many within and over, the newest delivery of each class, every delivery over its budget with its " +
"longest phase, and the delivered ones whose time is not known. Given a delivery's id, its time phase by phase.",
Input: schema(map[string]string{"days": "how many days back (default 7)", "id": "one delivery's id: its phases"}, nil),
-71
View File
@@ -1,71 +0,0 @@
package catalogue
import (
"go/ast"
"go/parser"
"go/token"
"path/filepath"
"slices"
"strconv"
"testing"
"github.com/novox/mesh-controller/internal/beside"
)
// The controller refuses a definition's resolved path by the node-engine's own rules, no more (novox/hq issue
// 496): the same lists, read here from mesh-host's internal/apply/placement_guard.go — in a merge check the clone
// beside it at the commit the mesh runs, elsewhere the copy captured in testdata/beside. When the engine's lists
// move, this fails until the controller's move with them, so the gate never refuses what the engine applies nor
// passes what it refuses by path alone.
func TestTheResolvedPathRulesAreTheNodeEnginesOwn(t *testing.T) {
file := filepath.Join(beside.Dir(t, "mesh-host"), "internal", "apply", "placement_guard.go")
parsed, err := parser.ParseFile(token.NewFileSet(), file, nil, 0)
if err != nil {
t.Fatalf("the node-engine's guard does not parse: %v", err)
}
lists := map[string][]string{}
consts := map[string]string{}
ast.Inspect(parsed, func(n ast.Node) bool {
spec, ok := n.(*ast.ValueSpec)
if !ok {
return true
}
for i, name := range spec.Names {
if i >= len(spec.Values) {
continue
}
switch v := spec.Values[i].(type) {
case *ast.CompositeLit:
for _, elt := range v.Elts {
if lit, ok := elt.(*ast.BasicLit); ok && lit.Kind == token.STRING {
s, _ := strconv.Unquote(lit.Value)
lists[name.Name] = append(lists[name.Name], s)
}
}
case *ast.BasicLit:
if v.Kind == token.STRING {
consts[name.Name], _ = strconv.Unquote(v.Value)
}
}
}
return true
})
for name, ours := range map[string][]string{
"protectedRoots": protectedRoots, "forbiddenBelow": forbiddenBelow, "engineTrees": engineTrees,
} {
theirs := lists[name]
if len(theirs) == 0 {
t.Errorf("the node-engine's guard names no %s any more; read it and say where its rule went", name)
continue
}
a, b := slices.Clone(ours), slices.Clone(theirs)
slices.Sort(a)
slices.Sort(b)
if !slices.Equal(a, b) {
t.Errorf("%s differs from the node-engine's:\n controller %v\n node-engine %v", name, a, b)
}
}
if consts["engineModule"] != engineModule {
t.Errorf("the node-engine's own module is %q there and %q here", consts["engineModule"], engineModule)
}
}
@@ -1,42 +0,0 @@
package catalogue
import (
"strings"
"testing"
)
// A provision's credential may be issued by its provider's software (novox/hq ADR 0305, issue 526): the offer
// names the tool that answers it and no own secret, and every other shape is refused at parse.
func TestAnOfferMaySayItsSoftwareIssuesTheCredential(t *testing.T) {
m, err := ParseManifest([]byte(`{"module":"media","version":"1",
"provides":[{"name":"media-api","scope":"mesh","credential":{"issued-by":"software","tool":"media_issued_credential"}}],
"serves":{"media-api":{"port":32400}}}`))
if err != nil {
t.Fatal(err)
}
if tool, issued := m.IssuedCredentialOf("media-api"); !issued || tool != "media_issued_credential" {
t.Fatalf("the offer's issued credential was not read: %q %v", tool, issued)
}
if _, shared := m.SharedCredentialOf("media-api"); shared {
t.Fatal("an issued credential was read as an own secret shared")
}
if _, issued := m.IssuedCredentialOf("other"); issued {
t.Fatal("a provision the module does not offer reads as issued")
}
for want, raw := range map[string]string{
"the one issuer an offer may name": `{"module":"d","version":"1",
"provides":[{"name":"d-api","credential":{"issued-by":"vendor","tool":"d_token"}}]}`,
"names no tool that answers it": `{"module":"d","version":"1",
"provides":[{"name":"d-api","credential":{"issued-by":"software"}}]}`,
"names its own secret": `{"module":"d","version":"1","own-secrets":{"token":{"path":"/p","taken":"at-start"}},
"provides":[{"name":"d-api","credential":{"issued-by":"software","tool":"d_token","own":"token"}}]}`,
"the one issuer an offer may name ": `{"module":"d","version":"1",
"provides":[{"name":"d-api","credential":{"tool":"d_token"}}]}`,
} {
want = strings.TrimSpace(want)
if _, err := ParseManifest([]byte(raw)); err == nil || !strings.Contains(err.Error(), want) {
t.Errorf("expected a refusal saying %q, got %v", want, err)
}
}
}
+105
View File
@@ -0,0 +1,105 @@
package catalogue
import (
"bytes"
"encoding/json"
"fmt"
)
// What a module may call its logs (novox/hq ADR 0297).
//
// A log is a module's record of operations: entries appended under a key, each kept as long as the
// log is, in the order they came. A module names each log it owns **locally** — `changes`, never a
// stream or a subject (ADR 0201 §4) — and the mesh derives the stream from the module and the local
// name, as it derives a bucket. So the rule for a log's name is a state name's: one plain token, and
// a module that keeps a log has a name that is one plain token too.
// The mesh's caps on a log, in MiB: what a module may ask, and what it gets when it asks nothing.
const (
LogLeastMiB = 1
LogMostMiB = 8192
LogDefaultMiB = 1024
)
// LogDeclaration is one log a module owns: its local name, and how large it may grow.
type LogDeclaration struct {
Name string `json:"name"`
// MaxMiB is the log's cap in MiB; zero is LogDefaultMiB. When full, the log refuses new entries
// and never drops old ones.
MaxMiB int `json:"max-mib,omitempty"`
}
// Cap is the log's cap in MiB, the default where none is said.
func (l LogDeclaration) Cap() int {
if l.MaxMiB == 0 {
return LogDefaultMiB
}
return l.MaxMiB
}
// UnmarshalJSON reads a log as its bare name, or as {name, max-mib}.
func (l *LogDeclaration) UnmarshalJSON(raw []byte) error {
trimmed := bytes.TrimSpace(raw)
if len(trimmed) > 0 && trimmed[0] == '"' {
return json.Unmarshal(trimmed, &l.Name)
}
var full struct {
Name string `json:"name"`
MaxMiB *int `json:"max-mib"`
}
dec := json.NewDecoder(bytes.NewReader(trimmed))
dec.DisallowUnknownFields()
if err := dec.Decode(&full); err != nil {
return fmt.Errorf("a log is either a name or {name, max-mib}: %w", typedUnknown(err))
}
l.Name = full.Name
l.MaxMiB = 0
if full.MaxMiB != nil {
// Said, and said as nothing: refused rather than read as the default, which it did not say.
if *full.MaxMiB == 0 {
return fmt.Errorf("log %q: max-mib is between %d and %d, not 0", full.Name, LogLeastMiB, LogMostMiB)
}
l.MaxMiB = *full.MaxMiB
}
return nil
}
// MarshalJSON writes back the short form when there is nothing else to say.
func (l LogDeclaration) MarshalJSON() ([]byte, error) {
if l.MaxMiB == 0 {
return json.Marshal(l.Name)
}
type plain LogDeclaration
return json.Marshal(plain(l))
}
// LogProblems is what is wrong with a manifest's logs.
//
// Refused at registration, for a bucket's reason: a stream name the bus cannot hold, or a cap the
// mesh would not grant, is a module that installs, starts, and is refused on its first append.
func LogProblems(m Manifest) []string {
var problems []string
if len(m.Logs) > 0 && !stateName.MatchString(m.Module) {
problems = append(problems, fmt.Sprintf(
"%s keeps a log, and a module's name is part of its logs' names, which take one plain "+
"name — no dot (novox/hq ADR 0297)", m.Module))
}
seen := map[string]bool{}
for _, l := range m.Logs {
switch {
case !stateName.MatchString(l.Name):
problems = append(problems, fmt.Sprintf(
"%s keeps log %q: a log is named locally — lower-case letters, digits and hyphens, "+
"no dot and no underscore; the mesh derives the stream (novox/hq ADR 0297)", m.Module, l.Name))
case seen[l.Name]:
problems = append(problems, fmt.Sprintf("%s keeps log %q twice", m.Module, l.Name))
}
seen[l.Name] = true
if l.MaxMiB != 0 && (l.MaxMiB < LogLeastMiB || l.MaxMiB > LogMostMiB) {
problems = append(problems, fmt.Sprintf(
"%s caps log %q at %d MiB; a log holds between %d and %d MiB (novox/hq ADR 0297)",
m.Module, l.Name, l.MaxMiB, LogLeastMiB, LogMostMiB))
}
}
return problems
}
+68
View File
@@ -0,0 +1,68 @@
package catalogue
import (
"encoding/json"
"strings"
"testing"
)
// A module declares the logs it keeps (novox/hq ADR 0297 §1): a log by its bare name, or with its cap
// in MiB; the default cap where none is said.
func TestAManifestMaySayWhatLogsItKeeps(t *testing.T) {
m, err := ParseManifest([]byte(`{"module":"mesh-issues","version":"1",` +
`"logs":["changes",{"name":"moves","max-mib":2048}]}`))
if err != nil {
t.Fatal(err)
}
if len(m.Logs) != 2 || m.Logs[0].Name != "changes" || m.Logs[1].Name != "moves" {
t.Fatalf("logs not read: %+v", m.Logs)
}
if m.Logs[0].Cap() != LogDefaultMiB || m.Logs[0].Cap() != 1024 || m.Logs[1].Cap() != 2048 {
t.Fatalf("caps read as %d and %d", m.Logs[0].Cap(), m.Logs[1].Cap())
}
out, _ := json.Marshal(m.Logs)
if string(out) != `["changes",{"name":"moves","max-mib":2048}]` {
t.Fatalf("written back as %s", out)
}
for _, edge := range []string{`{"name":"a","max-mib":1}`, `{"name":"a","max-mib":8192}`} {
if _, err := ParseManifest([]byte(`{"module":"a","version":"1","logs":[` + edge + `]}`)); err != nil {
t.Errorf("%s is inside the caps and was refused: %v", edge, err)
}
}
}
// The catalogue check refuses a log outside the caps, with a name that is not one plain token, with a
// field the mesh does not know, or kept by a module whose own name is not one plain token.
func TestALogIsNamedLocallyAndCapped(t *testing.T) {
for _, c := range []struct{ manifest, says string }{
{`{"module":"a","version":"1","logs":["mesh.changes"]}`, `keeps log "mesh.changes": a log is named locally`},
{`{"module":"a","version":"1","logs":["my_changes"]}`, `keeps log "my_changes"`},
{`{"module":"a","version":"1","logs":["Changes"]}`, `keeps log "Changes"`},
{`{"module":"a","version":"1","logs":["c","c"]}`, `keeps log "c" twice`},
{`{"module":"a","version":"1","logs":[{"name":"c","max-mib":8193}]}`, `between 1 and 8192 MiB`},
{`{"module":"a","version":"1","logs":[{"name":"c","max-mib":-1}]}`, `between 1 and 8192 MiB`},
{`{"module":"a","version":"1","logs":[{"name":"c","max-mib":0}]}`, `max-mib is between 1 and 8192, not 0`},
{`{"module":"a","version":"1","logs":[{"name":"c","stream":"LOG_x"}]}`, `{name, max-mib}`},
{`{"module":"a.b","version":"1","logs":["c"]}`, `no dot`},
} {
_, err := ParseManifest([]byte(c.manifest))
if err == nil {
t.Errorf("%s was accepted", c.manifest)
continue
}
if !strings.Contains(err.Error(), c.says) {
t.Errorf("%s refused for the wrong reason: %v", c.manifest, err)
}
}
}
// **Across the whole catalogue**: every log is named locally and capped within the mesh's caps.
func TestEveryManifestsLogsAreLocalAndCapped(t *testing.T) {
var problems []string
for _, m := range theCatalogue(t) {
problems = append(problems, LogProblems(m)...)
}
if len(problems) > 0 {
t.Fatalf("the catalogue's logs are not what ADR 0297 says:\n %s", strings.Join(problems, "\n "))
}
}
+11 -42
View File
@@ -268,29 +268,9 @@ func KeepsConsumerData(catalogue map[string]Manifest, provision string) bool {
// MachineReach is whether a provision is usable only on its provider's own machine.
func (o Offer) MachineReach() bool { return o.Reach == ReachMachine }
// OfferCredential names which of the provider's own secrets a provision's consumers receive — or, with
// IssuedBy, that the provider's software issues the provision's one credential itself (novox/hq ADR 0305,
// issue 526): a media server's sign-in token from its vendor, which the mesh can neither make nor rotate.
// Tool is then the provider's own tool that answers that value, sealed to a key made for the controller's
// one call, and to the controller alone.
// OfferCredential names which of the provider's own secrets a provision's consumers receive.
type OfferCredential struct {
Own string `json:"own,omitempty"`
IssuedBy string `json:"issued-by,omitempty"`
Tool string `json:"tool,omitempty"`
}
// IssuedBySoftware says a provision's credential is issued by its provider's software (novox/hq ADR 0305).
const IssuedBySoftware = "software"
// IssuedCredentialOf is the tool through which this module answers the credential its software issued for a
// provision, and whether its offer says the software issues it (novox/hq ADR 0305).
func (m Manifest) IssuedCredentialOf(provision string) (string, bool) {
for _, o := range m.Provides {
if o.Name == provision && o.Credential != nil && o.Credential.IssuedBy == IssuedBySoftware && o.Credential.Tool != "" {
return o.Credential.Tool, true
}
}
return "", false
Own string `json:"own"`
}
// SharedCredentialOf is the own secret an offer of this module names as the provision's credential,
@@ -485,6 +465,11 @@ type Manifest struct {
// (novox/hq ADR 0201). Not history — that is an event — and never a secret, sealed or not.
State []StateDeclaration `json:"state,omitempty"`
// Logs are the logs of operations this module keeps on the bus, by local name: each a stream the
// controller creates and never removes, which every instance of the module appends to and reads
// (novox/hq ADR 0297). A log is its owner's alone: no other module reads it.
Logs []LogDeclaration `json:"logs,omitempty"`
// Settings are the defaults this module gives its settings (novox/hq ADR 0262): each key a file,
// a contribution or a served fact asks for as `${setting:<key>}`, its default, and why that
// default. Only a preference has one — a font size, a width, a number of workers — and a value
@@ -1527,7 +1512,7 @@ func ParseManifest(raw []byte) (Manifest, error) {
name string
n int
}{{"consumes", len(m.Consumes)}, {"uses", len(m.Uses)}, {"invokes", len(m.Invokes)},
{"state", len(m.State)}, {"reads", len(m.Reads)}} {
{"state", len(m.State)}, {"logs", len(m.Logs)}, {"reads", len(m.Reads)}} {
if f.n > 0 {
said = append(said, f.name)
}
@@ -1570,24 +1555,7 @@ func ParseManifest(raw []byte) (Manifest, error) {
"makes is %d", m.Module, p, id.Max, shortest))
}
}
if c := offer.Credential; c != nil && (c.IssuedBy != "" || c.Tool != "") {
// The software issues it (novox/hq ADR 0305): named by the tool that answers it, never by an own
// secret, which would be a second copy for the mesh to keep in step with the software's.
switch {
case c.IssuedBy != IssuedBySoftware:
problems = append(problems, fmt.Sprintf(
"%s provides %q with a credential issued by %q; the one issuer an offer may name is %q (ADR 0305)",
m.Module, p, c.IssuedBy, IssuedBySoftware))
case c.Own != "":
problems = append(problems, fmt.Sprintf(
"%s provides %q with a credential its software issues and names its own secret %q too; the "+
"software's value is the one, so the offer names no own secret (ADR 0305)", m.Module, p, c.Own))
case strings.TrimSpace(c.Tool) == "":
problems = append(problems, fmt.Sprintf(
"%s provides %q with a credential its software issues and names no tool that answers it: "+
"\"tool\": \"<its tool>\" (ADR 0305)", m.Module, p))
}
} else if offer.Credential != nil {
if offer.Credential != nil {
own, declared := m.OwnSecrets[offer.Credential.Own]
switch {
case offer.Credential.Own == "":
@@ -1655,6 +1623,8 @@ func ParseManifest(raw []byte) (Manifest, error) {
problems = append(problems, EventProblems(m)...)
// And what it may call its state, and whose it may read (state.go, novox/hq ADR 0201).
problems = append(problems, StateProblems(m)...)
// And what it may call its logs, and how large it may ask them to grow (logs.go, novox/hq ADR 0297).
problems = append(problems, LogProblems(m)...)
// And the defaults it gives its settings (setting_defaults.go, novox/hq ADR 0262).
problems = append(problems, SettingProblems(m)...)
wellFormed := true
@@ -2175,7 +2145,6 @@ func ParseManifest(raw []byte) (Manifest, error) {
problems = append(problems, m.undeclaredMounts()...)
problems = append(problems, m.unknownDirRefs()...)
problems = append(problems, m.unknownAccessRefs()...)
problems = append(problems, m.resolvedPathProblems()...)
problems = append(problems, m.jailProblems()...)
// What a module adds to the account's environment and to the login shell, and the holder's
// placeholders for them (novox/hq ADR 0203, ADR 0204) — here, so the catalogue check refuses
-99
View File
@@ -92,105 +92,6 @@ func systemPath(path string) string {
return ""
}
// Where no directory or file a module's definition resolves to may be (novox/hq issue 496).
//
// mesh-catalog #205 gave docker's `state` directory `"place": "."`, which resolves to <root>/docker: with the
// default root, /var/lib/docker, every container's filesystem. systemPath judged only the `places` and `accesses`
// settings, so the default layout and a definition's own paths reached the merge gate unjudged; it composed every
// machine and passed, and only the node-engine refused the directory, at apply, failing the walk.
//
// **Judged where a definition is judged, never where a machine is composed.** resolvedPathProblems runs in
// ParseManifest, which every route a definition takes into the mesh passes: `module check`, registration of a
// build (the builder's and the registry verbs'), and the merge gate's reading of the changed repository. A
// refusal there stops one definition before it reaches any machine. Composition reads registered manifests
// without ParseManifest, and judges nothing of this: refusing there would fail the whole machine's declaration and
// freeze every module on it for one module's path, where the node-engine fails only that resource.
//
// **The node-engine's rules, no more** (mesh-host's internal/apply/placement_guard.go, with files judged as
// directories are after novox/hq issue 495, rule 6). A path is refused when it is one of protectedRoots or holds
// one, when it is at or below a tree in forbiddenBelow, or at or below one of engineTrees and its module is not
// the node-engine's. What the engine judges with what only the machine knows stays the engine's: where the
// runtimes really keep their data, links, the accounts' homes, a directory's owner below /etc. The lists are the
// engine's own words, and a test (engine_guard_test.go) holds them equal to mesh-host's beside this repository.
// systemPath stays the stricter rule for a setting: a setting is an operator's word about one machine, and the
// trees it lists (/etc, /usr, /run, the mesh's own) are where the mesh's own modules write by design.
var (
protectedRoots = []string{"/", "/bin", "/boot", "/dev", "/etc", "/home", "/lib", "/lib32", "/lib64",
"/media", "/mnt", "/opt", "/proc", "/root", "/run", "/sbin", "/srv", "/sys", "/tmp", "/usr", "/usr/bin",
"/usr/lib", "/usr/lib64", "/usr/local", "/usr/local/bin", "/usr/local/lib", "/usr/local/sbin", "/usr/sbin",
"/usr/share", "/var", "/var/cache", "/var/lib", "/var/lib/mesh", "/var/log", "/var/run", "/var/tmp",
"/var/spool"}
forbiddenBelow = []string{"/proc", "/sys", "/dev", "/boot", "/root", "/var/spool", "/opt", "/var/lib/docker",
"/var/lib/containers", "/var/lib/containerd"}
engineTrees = []string{"/var/lib/mesh-host", "/usr/lib/nox-mesh-host"}
)
// engineModule is the node-engine's own module, the one that places in engineTrees.
const engineModule = "mesh-host"
// enginePath says why the node-engine refuses a directory or file at path for module, or "".
func enginePath(path, module string) string {
path = filepath.Clean(path)
if !filepath.IsAbs(path) {
return ""
}
atOrBelow := func(tree string) bool { return path == tree || strings.HasPrefix(path, tree+"/") }
for _, root := range protectedRoots {
if path == root || path == "/" || strings.HasPrefix(root, path+"/") {
return root + " is one of the machine's own directories, and owning it is owning everything in it"
}
}
for _, tree := range forbiddenBelow {
if atOrBelow(tree) {
return "nothing is placed in " + tree
}
}
if module != engineModule {
for _, tree := range engineTrees {
if atOrBelow(tree) {
return tree + " is the node-engine's own, placed in by its own module alone"
}
}
}
return ""
}
// resolvedPathProblems is every directory and file of the definition whose path, resolved as a node with the
// default root resolves it, the node-engine would refuse (novox/hq issue 496). A path still holding a placeholder
// only a machine fills (a setting, an access the definition gives no default) is the engine's to judge.
func (m Manifest) resolvedPathProblems() []string {
dirs := dirsFor(m, Rendering{})
accesses := map[string]string{}
for _, a := range m.Accesses {
if a.ID != "" && a.Path != "" {
accesses[a.ID] = a.Path
}
}
var problems []string
for _, r := range m.Resources {
kind := fmt.Sprint(r["type"])
if kind != "directory" && kind != "file" {
continue
}
id := fmt.Sprint(r["id"])
path, _ := r["path"].(string)
if kind == "directory" && path == "" {
path = dirs[id]
}
path, _ = dirFill(path, dirs, m.Module)
path, _ = accessFill(path, accesses, m.Module)
if path == "" || strings.Contains(path, "${") {
continue
}
if why := enginePath(path, m.Module); why != "" {
problems = append(problems, fmt.Sprintf("%s's %s %q resolves to %s, which the node-engine refuses: %s. "+
"A module's directories and files are judged when its definition is, so the merge gate refuses it "+
"before any machine does (novox/hq issue 496)", m.Module, kind, id, filepath.Clean(path), why))
}
}
return problems
}
// accessRef is how a module names one of its accesses: ${access:<id>}.
var accessRef = regexp.MustCompile(`\$\{access:([a-z0-9][a-z0-9-]*)\}`)
+1 -33
View File
@@ -960,7 +960,6 @@ func checkResources(modules []Manifest) []string {
var problems []string
owner := map[string]string{}
ownedPath := map[string]string{} // path → owning module, for the access check below
packagePresent, packageAbsent := map[string][]string{}, map[string][]string{}
for _, m := range modules {
// Compared placed, not as written (novox/hq ADR 0112): ${dir:state} is the same six
@@ -993,22 +992,7 @@ func checkResources(modules []Manifest) []string {
}
continue
}
if fmt.Sprint(r["type"]) == "package" {
// **A package may be declared by several modules** (novox/hq ADR 0303): declaring it says
// it is needed, and the host installs a missing one, never removes one a declaration
// stopped naming, so a second declaration changes nothing on the machine. What two
// modules can contradict is presence: one declaring it present and another absent
// (ADR 0180) would have the host install and remove it in turn, so that pair is refused.
if name, _ := r["package"].(string); name != "" {
if r["absent"] == true {
packageAbsent[name] = append(packageAbsent[name], m.Module)
} else {
packagePresent[name] = append(packagePresent[name], m.Module)
}
}
continue
}
for _, field := range []string{"path", "unit", "name"} {
for _, field := range []string{"path", "unit", "name", "package"} {
value, ok := r[field].(string)
if !ok || value == "" {
if field == "path" && fmt.Sprint(r["type"]) == "directory" {
@@ -1054,12 +1038,6 @@ func checkResources(modules []Manifest) []string {
ownedPath[fact.Path] = m.Module
}
}
for _, name := range packageNames(packageAbsent) {
if wanted := packagePresent[name]; len(wanted) > 0 {
problems = append(problems, fmt.Sprintf("%s declares the package %q absent and %s declare it present (novox/hq ADR 0303)",
strings.Join(packageAbsent[name], ", "), name, strings.Join(wanted, ", ")))
}
}
// An accessed path is the operator's, so no module may declare it as one of its own
// (novox/hq ADR 0051). Refused here rather than silently tolerated: an owner would create and
@@ -1330,13 +1308,3 @@ func answeredElsewhere(want string, node Node, world World, brokered, keeps map[
holder, held := HolderAmong(want, world.Offered[want], world.Held)
return held && holder.Node != node.Name
}
// packageNames are the packages a map names, in order.
func packageNames(m map[string][]string) []string {
names := make([]string, 0, len(m))
for n := range m {
names = append(names, n)
}
sort.Strings(names)
return names
}
-127
View File
@@ -1,127 +0,0 @@
package catalogue
// A definition's directories and files are judged at their resolved paths when the definition is (novox/hq issue
// 496). mesh-catalog #205 gave docker's `state` directory `"place": "."`, which resolves to <root>/docker —
// /var/lib/docker, every container's filesystem. The merge gate composed every machine and passed it; only the
// node-engine refused it, at apply, and failed the walk. ParseManifest is what `module check`, registration and the
// merge gate's reading of a changed repository all run, so a refusal here is a refusal at each of them.
import (
"strings"
"testing"
)
func TestADirectoryPlacedInDockersDataIsRefusedWhereTheDefinitionIsJudged(t *testing.T) {
// The #205 shape, as it was merged.
_, err := ParseManifest([]byte(`{"module": "docker", "version": "1",
"resources": [{"id": "state", "type": "directory", "place": "."}]}`))
if err == nil {
t.Fatal("a directory resolving to /var/lib/docker was accepted; the node-engine refuses it at apply")
}
for _, said := range []string{"docker", `"state"`, "/var/lib/docker", "issue 496"} {
if !strings.Contains(err.Error(), said) {
t.Errorf("the refusal names the module, the resource, the path and why; %q is missing from %q", said, err)
}
}
}
func TestTheMeshsPlaceForDockerIsAccepted(t *testing.T) {
// The fix #205 needed: the mesh's own directory for the module, <root>/mesh/docker.
m, err := ParseManifest([]byte(`{"module": "docker", "version": "1", "resources": [
{"id": "state", "type": "directory", "place": "mesh"},
{"id": "marker", "type": "file", "path": "${dir:state}/applied", "content": "x"}]}`))
if err != nil {
t.Fatalf("place %q is in the mesh's tree, where the mesh writes for every module: %v", "mesh", err)
}
if got := dirsFor(m, Rendering{}); got["state"] != "/var/lib/mesh/docker" {
t.Fatalf("got %v", got)
}
}
// The node-engine's rules, for directories and — as the engine judges them since novox/hq issue 495 — files.
func TestADefinitionIsRefusedWhereTheNodeEngineRefusesItsPaths(t *testing.T) {
refused := map[string]string{
"a stated directory in docker's data": `{"module": "sidecar", "version": "1", "resources": [
{"id": "volumes", "type": "directory", "path": "/var/lib/docker/volumes/x"}]}`,
"a file in containerd's data": `{"module": "images", "version": "1", "resources": [
{"id": "f", "type": "file", "path": "/var/lib/containerd/x", "content": "x"}]}`,
"a file beneath a placed directory that climbs out of it": `{"module": "docker", "version": "1", "resources": [
{"id": "state", "type": "directory", "place": "mesh"},
{"id": "f", "type": "file", "path": "${dir:state}/../../containers/x", "content": "x"}]}`,
"a file in /boot": `{"module": "grub", "version": "1", "resources": [
{"id": "cfg", "type": "file", "path": "/boot/grub/custom.cfg", "content": "x"}]}`,
"a directory that is the mesh's whole tree": `{"module": "mesh", "version": "1", "resources": [
{"id": "all", "type": "directory", "place": "."}]}`,
"a directory that holds /etc": `{"module": "x", "version": "1", "resources": [
{"id": "d", "type": "directory", "path": "/"}]}`,
"a directory in the node-engine's own tree, by another module": `{"module": "intruder", "version": "1",
"resources": [{"id": "d", "type": "directory", "path": "/var/lib/mesh-host/x"}]}`,
}
for name, raw := range refused {
if _, err := ParseManifest([]byte(raw)); err == nil || !strings.Contains(err.Error(), "issue 496") {
t.Errorf("%s: accepted, or refused for another reason: %v", name, err)
}
}
// What the engine applies, the mesh's own modules' paths among them (read from every machine's live plan):
// the machine's configuration, run directories, programs below /usr/local, the node-engine's own trees by its
// own module, an account's keys, a module's own root, and — not on the engine's lists, so not refused here —
// below /lib and at /storage, /data, /services and /var/lock.
accepted := map[string]string{
"a unit in /etc": `{"module": "power", "version": "1", "resources": [
{"id": "d", "type": "directory", "path": "/etc/systemd/system/x.service.d"},
{"id": "u", "type": "file", "path": "/etc/systemd/system/x.service.d/a.conf", "content": "x"}]}`,
"a run directory": `{"module": "fail2ban", "version": "1", "resources": [
{"id": "run-dir", "type": "directory", "path": "/var/run/fail2ban"}]}`,
"a program in /usr/local/bin": `{"module": "claude-code", "version": "1", "resources": [
{"id": "start", "type": "file", "path": "/usr/local/bin/claude-agent", "content": "x"}]}`,
"the node-engine's launcher and state, by the node-engine": `{"module": "mesh-host", "version": "1", "resources": [
{"id": "launcher", "type": "file", "path": "/usr/lib/nox-mesh-host/launch", "content": "x"},
{"id": "state", "type": "directory", "path": "/var/lib/mesh-host"}]}`,
"an account's .ssh": `{"module": "ssh-client", "version": "1", "resources": [
{"id": "ssh-dir", "type": "directory", "path": "/home/someone/.ssh"},
{"id": "config", "type": "file", "path": "/home/someone/.ssh/config", "content": "x"}]}`,
"a module's own root": `{"module": "mailu", "version": "1", "resources": [
{"id": "state", "type": "directory", "place": "."}]}`,
"a file below /lib": `{"module": "udev", "version": "1", "resources": [
{"id": "rule", "type": "file", "path": "/lib/udev/rules.d/99-x.rules", "content": "x"}]}`,
"directories at /storage, /data, /services and /var/lock": `{"module": "roots", "version": "1", "resources": [
{"id": "a", "type": "directory", "path": "/storage"}, {"id": "b", "type": "directory", "path": "/data"},
{"id": "c", "type": "directory", "path": "/services"}, {"id": "d", "type": "directory", "path": "/var/lock"}]}`,
}
for name, raw := range accepted {
if _, err := ParseManifest([]byte(raw)); err != nil {
t.Errorf("%s: refused: %v", name, err)
}
}
}
func TestAPathThroughAnAccessIsJudgedWithTheAccessFilledIn(t *testing.T) {
// A file's path may name an access; the access's default path is the definition's, so it is judged with it.
_, err := ParseManifest([]byte(`{"module": "backup", "version": "1",
"accesses": [{"id": "images", "path": "/var/lib/docker/volumes"}],
"resources": [{"id": "marker", "type": "file", "path": "${access:images}/marker", "content": "x"}]}`))
if err == nil || !strings.Contains(err.Error(), "/var/lib/docker/volumes/marker") ||
!strings.Contains(err.Error(), "issue 496") {
t.Fatalf("a file reaching Docker's data through an access's default path was accepted: %v", err)
}
// An access the definition gives no path is placed by a setting, which Places and AccessPlaces judge, and on
// the machine by the node-engine: nothing here to resolve it against, so nothing is refused for it.
if _, err := ParseManifest([]byte(`{"module": "backup", "version": "1",
"accesses": [{"id": "images"}],
"resources": [{"id": "marker", "type": "file", "path": "${access:images}/marker", "content": "x"}]}`)); err != nil {
t.Fatalf("an access placed only by a setting cannot be judged at the definition: %v", err)
}
}
func TestComposingAMachineIsNeverStoppedByOneModulesPath(t *testing.T) {
// A module registered from outside the catalogue never passes the gate. Refusing its path while a machine's
// declaration is composed would fail the whole declaration and freeze every module on that machine; the
// node-engine fails only the one resource. So composition leaves it to the engine.
m := Manifest{Module: "docker", Version: "1", Resources: []map[string]any{
{"id": "state", "type": "directory", "place": "."},
}}
if _, err := (Resolution{Node: "anchor", Modules: []Manifest{m}}).Declaration(Rendering{}); err != nil {
t.Fatalf("the machine's declaration failed for one module's path: %v", err)
}
}
-4
View File
@@ -349,10 +349,6 @@ func CollisionRefusal(catalogue map[string]Manifest, node string, assigned, addi
if before[p] {
continue // on the node already; not this assignment's doing
}
if strings.Contains(p, "ADR 0303") {
problems = append(problems, p) // a package declared present and absent says its own rule
continue
}
problems = append(problems, p+" (novox/hq ADR 0210: one owner per node; the other module "+
"depends on the owner's seat instead)")
}
+4 -15
View File
@@ -38,17 +38,6 @@ func settingsUsed(content string) []string {
return keys
}
// UnsetSettingError is a module whose definition says ${setting:<key>} where nothing sets that key: typed, so
// that whoever reads why a module was left out of a machine can tell a setting nobody gave — the operator's
// to give, named with the command that gives it — from any other reason (novox/hq issue 380). Its words are
// the refusal's, unchanged.
type UnsetSettingError struct {
Module, Setting string
said string
}
func (e *UnsetSettingError) Error() string { return e.said }
// settingInto fills a file's ${setting:…} placeholders from the layers over a module.
//
// The last layer setting a key wins, which is the node's over the mesh's over the module's own
@@ -69,12 +58,12 @@ func settingInto(resource map[string]any, layers []Layer, module string) error {
for _, key := range settingsUsed(content) {
value, set := settingValue(layers, key)
if !set {
return &UnsetSettingError{Module: module, Setting: key, said: fmt.Sprintf(
return fmt.Errorf(
"%s has a file that says ${setting:%s}, and nothing sets %q for it — an operator's "+
"value is the assignment's, never the definition's (novox/hq ADR 0112), and only a "+
"preference has a default in the definition (ADR 0262): "+
"`settings set %s <file>` with {%q: …}%s",
module, key, key, module, key, orNoSettings(layers))}
module, key, key, module, key, orNoSettings(layers))
}
content = strings.ReplaceAll(content, "${setting:"+key+"}", plainly(value))
}
@@ -125,11 +114,11 @@ func settingIntoUnit(resource map[string]any, layers []Layer, module string) err
for _, key := range settingsUsed(unit) {
value, set := settingValue(layers, key)
if !set {
return &UnsetSettingError{Module: module, Setting: key, said: fmt.Sprintf(
return fmt.Errorf(
"%s has a service whose unit says ${setting:%s}, and nothing sets %q for it — an operator's "+
"value is the assignment's, never the definition's (novox/hq ADR 0112): "+
"`settings set %s <file>` with {%q: …}%s",
module, key, key, module, key, orNoSettings(layers))}
module, key, key, module, key, orNoSettings(layers))
}
v := plainly(value)
if !unitPart.MatchString(v) {
+2 -17
View File
@@ -160,14 +160,8 @@ var ControllerVerbs = []Verb{
Input: schema(map[string]string{"plan": "the walk's id", "why": "what lets it go: its turn, a person's release"},
[]string{"plan"})},
{Name: "delivery-stop", Description: "End a walk on its delivery's word (novox/hq ADR 0239): failed, said as stopped " +
"by whom and why; what it asked still builds and registers, nothing further is asked or sent. Given merges to " +
"leave out, every other merge the walk answered is walked again — in the open batch of its kind, or a new one — " +
"without them, without each later merge of their repository, which contains one, and without each merge moving " +
"a module one moves, each left out too and named (novox/hq ADR 0299); a batch not yet cut drops the merges. The " +
"answer names the batch, the merges walked again, the merges left out, the modules the stopped walk sent that " +
"stay on their machines, and the merges elsewhere that will still deliver a stopped change.",
Input: schema(map[string]string{"plan": "the walk's id", "why": "why", "by": "who stopped the delivery",
"without": "merges to leave out, <repository>@<commit>, comma-separated"},
"by whom and why; what it asked still builds and registers, nothing further is asked or sent.",
Input: schema(map[string]string{"plan": "the walk's id", "why": "why", "by": "who stopped the delivery"},
[]string{"plan", "why"})},
{Name: "delivery-walks", Description: "The walks the controller keeps (novox/hq ADR 0239): every open one and the " +
"last ended ones, each whole — its tiers, each module's state, first machines and first-node gate with its readings, and its " +
@@ -249,15 +243,6 @@ var ControllerVerbs = []Verb{
"secret": "the own secret's name in the module's definition",
"at": "the machine the operator sits at, where the prompt opens",
}, []string{"node", "module", "secret", "at"})},
{Name: "issued", Description: "Take the credential a provider's software issued for a provision — a media " +
"server's sign-in token — and keep it as that provision's one credential for every consumer (novox/hq ADR " +
"0305): the controller asks the provider's own tool on its machine, in the controller's own name, for the " +
"value sealed to a key made for this call alone, seals it to every holder and forgets it, and sends their " +
"machines when it changed. Carries no value and answers none; a value every holder already holds is not " +
"sealed again, so asking again restarts nothing. Refused for a provision no offer says its software issues.",
Input: schema(map[string]string{
"provision": "the provision whose provider's software issues its credential, e.g. plex-api",
}, []string{"provision"})},
{Name: "secret-ask", Description: "Ask the operator for a module's own secret (novox/hq ADR 0277): a prompt " +
"that shows nothing of what is typed opens at the desk — the module's machine, or the one at names — " +
"naming the module, the secret and who asked; the operator types the value there, never on a channel " +
+25 -15
View File
@@ -3,7 +3,6 @@ package inventory
import (
"context"
"fmt"
"slices"
"strings"
"github.com/novox/mesh-controller/internal/broker"
@@ -167,13 +166,12 @@ func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaratio
// And the state it keeps and reads (novox/hq ADR 0201).
State: bucketsOf(m),
Reads: m.Reads,
// And the logs it keeps (novox/hq ADR 0297).
Logs: logsOf(m),
// And the tools its health asks (novox/hq ADR 0240): the machine's node-engine is granted them.
Checks: catalogue.HealthChecks(m),
// And whether it runs as an account of its own (novox/hq ADR 0259 §8).
RunsAs: m.RunsAs,
// And the tools that answer a credential its software issued, which the controller asks in its own
// name (novox/hq ADR 0305).
Issued: issuedTools(m),
}
// Whether it can be given an account at all: delivered as its own secret named broker, so one
// that declares none has nowhere to read it (novox/hq issue 195).
@@ -201,17 +199,6 @@ func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaratio
return d
}
// issuedTools are the tools through which a module answers the credentials its software issues (novox/hq ADR 0305).
func issuedTools(m catalogue.Manifest) []string {
var out []string
for _, o := range m.Provides {
if tool, issued := m.IssuedCredentialOf(o.Name); issued && !slices.Contains(out, tool) {
out = append(out, tool)
}
}
return out
}
// bucketsOf is the state a module keeps, as the bus holds it.
func bucketsOf(m catalogue.Manifest) []broker.Bucket {
var out []broker.Bucket
@@ -237,6 +224,29 @@ func (i *Inventory) DeclaredBuckets(ctx context.Context) ([]broker.Bucket, error
return out, nil
}
// logsOf is the logs a module keeps, as the bus holds them.
func logsOf(m catalogue.Manifest) []broker.Log {
var out []broker.Log
for _, l := range m.Logs {
out = append(out, broker.Log{Module: m.Module, Name: l.Name, MaxMiB: l.Cap()})
}
return out
}
// DeclaredLogs is every log the catalogue declares, registered modules assigned or not: a log exists
// from registration, as a bucket does (novox/hq ADR 0297 §2, ADR 0201 §6).
func (i *Inventory) DeclaredLogs(ctx context.Context) ([]broker.Log, error) {
declared, err := i.Catalogue(ctx)
if err != nil {
return nil, fmt.Errorf("cannot read the catalogue: %w", err)
}
var out []broker.Log
for _, m := range declared {
out = append(out, logsOf(m)...)
}
return out, nil
}
func asSeat(s catalogue.SeatDeclaration, declarer string) broker.Seat {
seat := broker.Seat{Name: s.Name, Scope: s.Scope, Accepts: s.Accepts, Emits: s.Emits,
Serves: catalogue.VerbNames(s.Serves), Kinded: s.Kinded, ByCaller: s.ByCaller, Proofs: s.Proofs,
-155
View File
@@ -1,155 +0,0 @@
package inventory
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"sort"
"strings"
"github.com/novox/mesh-controller/internal/secrets"
)
// A credential a provider's software issued, kept as the provision's one credential (novox/hq ADR 0305, issue
// 526).
//
// **The software made it, so the mesh carries it.** A media server's sign-in token comes from its vendor; the mesh
// can neither make nor rotate it. The controller fetches it from the provider's own tool, sealed to a key made for
// that one call, and hands the value here, which seals it to every consumer that holds the provision from that
// provider — in the pair credential's shape, so a consumer's definition does not know where it came from (ADR 0158
// §5) — and to the operator, and forgets it.
//
// **Asked often, written seldom.** The provider asks hourly and on every start. The mesh keeps no plaintext, so it
// cannot compare values; it stamps every row with the value's fingerprint instead, and a value whose stamp every
// row already carries is not sealed again — sealing anew would change every consumer's declaration and restart
// each of them for nothing.
// OriginIssued is a pair credential the provider's software issued and the provider gave (novox/hq ADR 0305).
const OriginIssued = "issued"
// IssuedStamp is the generation a value issued by a provider's software is kept under: a fingerprint of the value,
// so the same value fetched again is known as the same without being kept.
func IssuedStamp(value string) string {
sum := sha256.Sum256([]byte(value))
return "issued:sha256:" + hex.EncodeToString(sum[:8])
}
// IssuedKept is what keeping an issued value did: whether it was sealed anew, the pairs that hold it, and the
// machines a send must reach — the provider's and every holder's — when it was.
type IssuedKept struct {
Changed bool
// Holders are every pair that holds the provision from the provider; Sealed those sealed anew by this act.
Holders []Holder
Sealed []Holder
Machines []string
// Unsealable are the holders whose machine has no sealing key: left as they are, said by name.
Unsealable []Holder
}
// KeepIssuedSecret seals a value a provider's software issued to every pair of provision from provider that does
// not hold it yet, under one stamp; a pair that does is left as it is, and when every pair does nothing is written
// (novox/hq ADR 0305 §4, §5).
func (i *Inventory) KeepIssuedSecret(ctx context.Context, provision, provider, value string) (IssuedKept, error) {
if strings.TrimSpace(value) == "" {
return IssuedKept{}, fmt.Errorf("there is nothing to seal")
}
providerNode, err := i.NodeByName(ctx, provider)
if err != nil {
return IssuedKept{}, err
}
providerKey, err := i.SealingKeyOf(ctx, provider)
if err != nil {
return IssuedKept{}, err
}
if providerKey == "" {
return IssuedKept{}, fmt.Errorf("%s has no sealing key, so %s cannot be sealed to it", provider, provision)
}
stamp := IssuedStamp(value)
type row struct {
holder Holder
consumerID any
generation, heldC, heldP string
consumerKey string
}
rows, err := i.store.Pool().Query(ctx,
`select s.consumer, c.name, s.consumer_module, s.local, coalesce(s.generation, ''),
coalesce(s.consumer_key, ''), coalesce(s.provider_key, ''), coalesce(c.sealing_key, '')
from secret s join node c on c.id = s.consumer
where s.provider = $1 and s.name = $2
order by c.name, s.consumer_module, s.local`, providerNode.ID, provision)
if err != nil {
return IssuedKept{}, err
}
var held []row
for rows.Next() {
var r row
if err := rows.Scan(&r.consumerID, &r.holder.Consumer, &r.holder.ConsumerModule, &r.holder.Local,
&r.generation, &r.heldC, &r.heldP, &r.consumerKey); err != nil {
rows.Close()
return IssuedKept{}, err
}
r.holder.Provision, r.holder.Provider = provision, provider
held = append(held, r)
}
rows.Close()
if err := rows.Err(); err != nil {
return IssuedKept{}, err
}
var kept IssuedKept
var stale []row
for _, r := range held {
if r.consumerKey == "" {
kept.Unsealable = append(kept.Unsealable, r.holder)
continue
}
kept.Holders = append(kept.Holders, r.holder)
// Only a pair that does not hold this value, sealed to the keys its ends have now, is sealed anew: a
// holder that already does is left byte for byte as it is, so it is not restarted for another's sake.
if r.generation != stamp || r.heldC != r.consumerKey || r.heldP != providerKey {
stale = append(stale, r)
}
}
if len(stale) == 0 {
return kept, nil
}
forOperator, operatorKey, err := i.operatorSeal(ctx, value)
if err != nil {
return IssuedKept{}, err
}
tx, err := i.store.Pool().Begin(ctx)
if err != nil {
return IssuedKept{}, err
}
defer func() { _ = tx.Rollback(ctx) }()
machines := map[string]bool{provider: true}
for _, r := range stale {
sealed, err := secrets.Accept(value, r.consumerKey, providerKey)
if err != nil {
return IssuedKept{}, err
}
if _, err := tx.Exec(ctx,
`update secret set for_consumer = $6, for_provider = $7, consumer_key = $8, provider_key = $9,
operator_sealed = $10, operator_key = $11, origin = $12, generation = $13, created_at = now()
where name = $1 and consumer = $2 and consumer_module = $3 and provider = $4 and local = $5`,
provision, r.consumerID, r.holder.ConsumerModule, providerNode.ID, r.holder.Local,
sealed.ForConsumer, sealed.ForProvider, sealed.ConsumerKey, sealed.ProviderKey,
forOperator, operatorKey, OriginIssued, stamp); err != nil {
return IssuedKept{}, err
}
machines[r.holder.Consumer] = true
kept.Sealed = append(kept.Sealed, r.holder)
}
if err := tx.Commit(ctx); err != nil {
return IssuedKept{}, err
}
kept.Changed = true
for m := range machines {
kept.Machines = append(kept.Machines, m)
}
sort.Strings(kept.Machines)
return kept, nil
}
-178
View File
@@ -1,178 +0,0 @@
package inventory
import (
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// A value a provider's software issued is sealed to every pair that holds the provision from that provider, under
// one stamp of the value, and the same value given again writes nothing and sends nobody (novox/hq ADR 0305 §4,
// §5, issue 526).
func TestAnIssuedValueIsSealedToEveryHolderOnceAndNotAgain(t *testing.T) {
inv, ctx := twoNodesWithKeys(t)
consumerKey, consumerOpens := aSealingKey(t)
cn, _ := inv.NodeByName(ctx, "consumer")
if err := inv.RecordSealingKey(ctx, cn.ID, consumerKey); err != nil {
t.Fatal(err)
}
for _, m := range []string{"kometa", "tautulli"} {
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1", Requires: []string{"plex-api"},
Secrets: map[string]string{"plex-api": "/run/plex-api.secret"}}, Source{}); err != nil {
t.Fatal(err)
}
// What the mesh made before the provider gave its value: a random value per pair (issue 164).
if _, err := inv.SecretFor(ctx, "plex-api", "consumer", m, "provider", ""); err != nil {
t.Fatal(err)
}
}
kept, err := inv.KeepIssuedSecret(ctx, "plex-api", "provider", "the-servers-own-token")
if err != nil {
t.Fatal(err)
}
if !kept.Changed || len(kept.Holders) != 2 {
t.Fatalf("the value was not sealed to both holders: %+v", kept)
}
if len(kept.Machines) != 2 || kept.Machines[0] != "consumer" || kept.Machines[1] != "provider" {
t.Fatalf("the machines to send are the provider's and every holder's: %v", kept.Machines)
}
for _, m := range []string{"kometa", "tautulli"} {
got, err := inv.SecretFor(ctx, "plex-api", "consumer", m, "provider", "")
if err != nil {
t.Fatal(err)
}
if got.Origin != OriginIssued {
t.Fatalf("%s's pair says it came from %q, not the provider's software", m, got.Origin)
}
opened, ok := consumerOpens(got.ForConsumer)
if !ok || string(opened) != "the-servers-own-token" {
t.Fatalf("%s's copy is not the provider's value", m)
}
var generation string
pn, _ := inv.NodeByName(ctx, "provider")
if err := inv.store.Pool().QueryRow(ctx, `select generation from secret where name = 'plex-api' and consumer = $1
and consumer_module = $2 and provider = $3`, cn.ID, m, pn.ID).Scan(&generation); err != nil {
t.Fatal(err)
}
if generation != IssuedStamp("the-servers-own-token") {
t.Fatalf("%s's pair is not stamped with the value's fingerprint: %q", m, generation)
}
}
before, _ := inv.SecretFor(ctx, "plex-api", "consumer", "kometa", "provider", "")
// A consumer that binds later gets a pair of its own, and no holder's value is remade for it: the offer names
// no own secret, so ADR 0158 §6's remaking on a bind does not reach it (ADR 0305 §7).
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "overseerr", Version: "1", Requires: []string{"plex-api"},
Secrets: map[string]string{"plex-api": "/run/plex-api.secret"}}, Source{}); err != nil {
t.Fatal(err)
}
late, err := inv.SecretFor(ctx, "plex-api", "consumer", "overseerr", "provider", "")
if err != nil {
t.Fatal(err)
}
if late.Origin == OriginIssued {
t.Fatal("a consumer that bound after the ask holds the issued value without an ask")
}
if held, _ := inv.SecretFor(ctx, "plex-api", "consumer", "kometa", "provider", ""); held.ForConsumer != before.ForConsumer {
t.Fatal("a later bind remade a holder's issued value")
}
// The next ask seals the provider's value to the late consumer alone; the holders that had it are not touched.
again, err := inv.KeepIssuedSecret(ctx, "plex-api", "provider", "the-servers-own-token")
if err != nil {
t.Fatal(err)
}
if !again.Changed || len(again.Sealed) != 1 || again.Sealed[0].ConsumerModule != "overseerr" {
t.Fatalf("the late consumer was not sealed to, or not alone: %+v", again)
}
if held, _ := inv.SecretFor(ctx, "plex-api", "consumer", "kometa", "provider", ""); held.ForConsumer != before.ForConsumer {
t.Fatal("sealing the late consumer changed what a holder already held")
}
if got, _ := inv.SecretFor(ctx, "plex-api", "consumer", "overseerr", "provider", ""); got.Origin != OriginIssued {
t.Fatal("the late consumer does not hold the issued value after the ask")
}
// And once every holder holds it, the same value writes nothing and sends nobody.
still, err := inv.KeepIssuedSecret(ctx, "plex-api", "provider", "the-servers-own-token")
if err != nil {
t.Fatal(err)
}
if still.Changed || len(still.Machines) != 0 {
t.Fatalf("the same value was sealed again, which restarts every consumer for nothing: %+v", still)
}
after, _ := inv.SecretFor(ctx, "plex-api", "consumer", "kometa", "provider", "")
if after.ForConsumer != before.ForConsumer {
t.Fatal("the same value given again changed what the consumer holds")
}
changed, err := inv.KeepIssuedSecret(ctx, "plex-api", "provider", "a-token-plex-changed")
if err != nil {
t.Fatal(err)
}
if !changed.Changed {
t.Fatal("a changed value was not sealed")
}
got, _ := inv.SecretFor(ctx, "plex-api", "consumer", "tautulli", "provider", "")
if opened, ok := consumerOpens(got.ForConsumer); !ok || string(opened) != "a-token-plex-changed" {
t.Fatal("the changed value did not reach the holder")
}
if _, err := inv.KeepIssuedSecret(ctx, "plex-api", "provider", " "); err == nil {
t.Fatal("an empty value was kept")
}
}
// A consumer whose sealing key changed holds a value the mesh made in place of the issued one, and the next ask
// seals the issued value to it again; so does a pair a person accepted over it (novox/hq ADR 0305 §5, the review
// of mesh-controller #239).
func TestAnIssuedValueIsSealedAgainAfterAKeyChangeOrAnAcceptance(t *testing.T) {
inv, ctx := twoNodesWithKeys(t)
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "kometa", Version: "1", Requires: []string{"plex-api"},
Secrets: map[string]string{"plex-api": "/run/plex-api.secret"}}, Source{}); err != nil {
t.Fatal(err)
}
if _, err := inv.SecretFor(ctx, "plex-api", "consumer", "kometa", "provider", ""); err != nil {
t.Fatal(err)
}
if kept, err := inv.KeepIssuedSecret(ctx, "plex-api", "provider", "the-servers-own-token"); err != nil || !kept.Changed {
t.Fatalf("the first ask did not seal: %+v %v", kept, err)
}
// The consumer rejoins with a new key: the mesh remakes the pair, a value the software refuses.
newKey, opens := aSealingKey(t)
cn, _ := inv.NodeByName(ctx, "consumer")
if err := inv.RecordSealingKey(ctx, cn.ID, newKey); err != nil {
t.Fatal(err)
}
remade, err := inv.SecretFor(ctx, "plex-api", "consumer", "kometa", "provider", "")
if err != nil {
t.Fatal(err)
}
if remade.Origin != OriginMade {
t.Fatalf("a value the mesh remade says it came from %q", remade.Origin)
}
again, err := inv.KeepIssuedSecret(ctx, "plex-api", "provider", "the-servers-own-token")
if err != nil {
t.Fatal(err)
}
if !again.Changed || len(again.Sealed) != 1 {
t.Fatalf("the issued value was not sealed again after the key changed: %+v", again)
}
got, _ := inv.SecretFor(ctx, "plex-api", "consumer", "kometa", "provider", "")
if opened, ok := opens(got.ForConsumer); !ok || string(opened) != "the-servers-own-token" {
t.Fatal("the consumer does not hold the issued value under its new key")
}
// A person accepts another value over it: the next ask seals the software's again.
if err := inv.AcceptSecretForPair(ctx, "plex-api", "consumer", "kometa", "provider", "", "a-typed-copy"); err != nil {
t.Fatal(err)
}
after, err := inv.KeepIssuedSecret(ctx, "plex-api", "provider", "the-servers-own-token")
if err != nil {
t.Fatal(err)
}
if !after.Changed {
t.Fatal("an accepted value over the issued one was read as still holding it")
}
}
+48 -142
View File
@@ -111,12 +111,6 @@ type PlanBatch struct {
// a merge never shares a batch with one that waits for mesh-delivery's (ADR 0276, decided during the
// build, 2026-10-10).
Own bool `json:"own,omitempty"`
// Closed says the window was closed by a stop that walks a stopped walk's merges again in this batch: they
// were heard before, and an older commit never waits behind a newer one (novox/hq ADR 0299). Withheld are
// the modules a stopped merge moves: built at the branch, which holds that merge, they would deliver it, so
// the batch's walk does not build them.
Closed bool `json:"closed,omitempty"`
Withheld []string `json:"withheld,omitempty"`
}
// OwnPath says the record is a batch of merges on the controller's own path.
@@ -193,17 +187,6 @@ type PlanDelivery struct {
// TakenOverBy names the walk a walk folded before it started was taken over by: its merges are that
// walk's now.
TakenOverBy string `json:"taken_over_by,omitempty"`
// WalkedAgainBy names, on a walk stopped without some of its merges, the batch that walks its other merges
// again, and LeftOut the merges it left out (novox/hq issue 459, ADR 0299): a delivery of a merge left out
// ends stopped with the walk, one of a merge walked again follows that batch's walk.
WalkedAgainBy string `json:"walked_again_by,omitempty"`
LeftOut []PlanLeftOut `json:"left_out,omitempty"`
// Kept are the modules the stopped walk already sent and nothing walks again: the machines named keep running
// the stopped walk's build (novox/hq ADR 0299).
Kept []PlanKept `json:"kept,omitempty"`
// Withheld are, on a walk cut from a batch a stop walked merges again in, the modules it did not build: a
// stopped merge moves them, and built only as dependents at the branch they would deliver it (ADR 0299).
Withheld []string `json:"withheld,omitempty"`
// Batch is the window of a batch; nil once it is cut into a walk.
Batch *PlanBatch `json:"batch,omitempty"`
// Alone says the walk walks one merge on its own commit, after a failed walk carried it in a later one:
@@ -211,26 +194,6 @@ type PlanDelivery struct {
Alone bool `json:"alone,omitempty"`
}
// PlanLeftOut is a merge a stop left out of the batch that walks a stopped walk's merges again (novox/hq ADR
// 0299): a merge stopped, or a later merge of its repository's branch, which contains the stopped one and so
// cannot be walked without delivering it.
type PlanLeftOut struct {
Repository string `json:"repository"`
Commit string `json:"commit"`
// Contains is the stopped merge a later merge contains, or whose modules it shares; empty for a merge
// stopped itself.
Contains string `json:"contains,omitempty"`
// Shares are the modules it moves that a stopped merge moves too: built at the branch, which holds the stopped
// merge, they would deliver it.
Shares []string `json:"shares,omitempty"`
}
// PlanKept is a module a stopped walk sent and nothing walks again, and the machines it reached.
type PlanKept struct {
Module string `json:"module"`
Machines []string `json:"machines"`
}
// Waiting is whether the walk waits for its delivery's word.
func (p Plan) Waiting() bool {
return p.Delivery != nil && p.Delivery.Awaits != "" && p.Delivery.Go == nil
@@ -302,10 +265,7 @@ type PlanGate struct {
Sent map[string]SentDeclaration `json:"sent,omitempty"`
// Passes counts the consecutive judgings that found it healthy, LastPass the newest; a judging that
// does not resets them.
Passes int `json:"passes,omitempty"`
// Wants is how many passes the send needs: a core module's three, a leaf module's one (novox/hq ADR 0282
// decision 4 as ADR 0304 decision 3 changed it). Zero in a gate judged before it was said: three.
Wants int `json:"wants,omitempty"`
Passes int `json:"passes,omitempty"`
LastPass *time.Time `json:"last_pass,omitempty"`
// Last is what the newest judging found wanting, while it still may pass.
Last string `json:"last,omitempty"`
@@ -345,18 +305,6 @@ type PlanGate struct {
// Readings are the judging's readings with their times (novox/hq ADR 0282 decision 6): every reading that
// counted a pass, and the first that did not after one that did. At most maxReadings, the newest kept.
Readings []GateReading `json:"readings,omitempty"`
// Grants is the grants step taken before this gate's send (novox/hq issue 490): the bus's user list sent
// alone to the machine holding the bus, every build there kept, so what the send newly grants is on the
// bus before the gate judges it. Nil when the list did not change, or the gate's machine holds the bus.
Grants *GrantsStep `json:"grants,omitempty"`
}
// GrantsStep is the bus's user list sent to the machine holding the bus ahead of a gated send (novox/hq
// issue 490): to which machine, when, and, when it could not be sent, why — the send went on without it.
type GrantsStep struct {
Node string `json:"node"`
At *time.Time `json:"at,omitempty"`
Failed string `json:"failed,omitempty"`
}
// GateReading is one reading of a first-node gate.
@@ -437,28 +385,39 @@ func (p Plan) Open() bool { return p.State == PlanBuilding || p.State == PlanRol
// with ErrPlanMoved otherwise; and only by a process that may act (ActsUnder), whose epoch it records.
// On success p's revision and epoch are the ones written, so the caller may save it again.
func (i *Inventory) SavePlan(ctx context.Context, p *Plan) error {
return i.SavePlansMoving(ctx, []*Plan{p}, nil)
}
// MergeMove is a merge given to the plan that answers it now (novox/hq ADR 0276).
type MergeMove struct {
Repository, Commit, Plan string
Alone bool
}
// SavePlansMoving writes plans, as SavePlan does each, and gives merges to the plans that answer them, all in one
// transaction: every write and move, or none (novox/hq ADR 0299 decision 7 — a stop that walks merges again never
// leaves a merge answered by two plans, nor a walk stopped whose merges were not moved). A plan written by
// somebody else since it was read refuses the whole act with ErrPlanMoved. Each plan is said (PlanSaved) once
// the transaction is committed, in order.
func (i *Inventory) SavePlansMoving(ctx context.Context, plans []*Plan, moves []MergeMove) error {
epoch, err := i.actingEpoch(ctx)
if err != nil {
if len(plans) > 0 {
return fmt.Errorf("the plan for %s %s is not written: %w", plans[0].Repository, plans[0].Commit, err)
}
return fmt.Errorf("the plan for %s %s is not written: %w", p.Repository, p.Commit, err)
}
tiers, err := json.Marshal(p.Tiers)
if err != nil {
return err
}
modules, err := json.Marshal(p.Modules)
if err != nil {
return err
}
var release, delivery, commits, times []byte
if p.Times != nil {
if times, err = json.Marshal(p.Times); err != nil {
return err
}
}
if len(p.Commits) > 0 {
if commits, err = json.Marshal(p.Commits); err != nil {
return err
}
}
if p.Release != nil {
if release, err = json.Marshal(p.Release); err != nil {
return err
}
}
if p.Delivery != nil {
if delivery, err = json.Marshal(p.Delivery); err != nil {
return err
}
}
// **And how long the tier it left took** (novox/hq to-be 45 Phase 0): measured here, where the
// plan moves, in the same transaction as the move, so no save can move a tier unmeasured or
// measure one twice.
@@ -467,77 +426,9 @@ func (i *Inventory) SavePlansMoving(ctx context.Context, plans []*Plan, moves []
return err
}
defer func() { _ = tx.Rollback(ctx) }()
type written struct {
revision int64
entered time.Time
}
done := make([]written, len(plans))
for n, p := range plans {
revision, entered, err := writePlan(ctx, tx, p, epoch)
if err != nil {
return err
}
done[n] = written{revision, entered}
}
for _, m := range moves {
if _, err := tx.Exec(ctx,
`update batched_merge set plan_id = nullif($3, ''), alone = $4 where repository = lower($1) and commit_hash = $2`,
m.Repository, m.Commit, m.Plan, m.Alone); err != nil {
return err
}
}
if err := tx.Commit(ctx); err != nil {
return err
}
for n, p := range plans {
p.Revision, p.TierEntered = done[n].revision, done[n].entered
if epoch != nil {
p.Epoch = uint64(*epoch)
} else {
p.Epoch = 0
}
if PlanSaved != nil {
PlanSaved(*p)
}
}
return nil
}
// writePlan writes one plan within a transaction, by compare-and-set on its revision: its new revision, and when
// it entered its tier.
func writePlan(ctx context.Context, tx pgx.Tx, p *Plan, epoch *int64) (int64, time.Time, error) {
tiers, err := json.Marshal(p.Tiers)
if err != nil {
return 0, time.Time{}, err
}
modules, err := json.Marshal(p.Modules)
if err != nil {
return 0, time.Time{}, err
}
var release, delivery, commits, times []byte
if p.Times != nil {
if times, err = json.Marshal(p.Times); err != nil {
return 0, time.Time{}, err
}
}
if len(p.Commits) > 0 {
if commits, err = json.Marshal(p.Commits); err != nil {
return 0, time.Time{}, err
}
}
if p.Release != nil {
if release, err = json.Marshal(p.Release); err != nil {
return 0, time.Time{}, err
}
}
if p.Delivery != nil {
if delivery, err = json.Marshal(p.Delivery); err != nil {
return 0, time.Time{}, err
}
}
entered, err := planTierLeft(ctx, tx, *p, time.Now())
if err != nil {
return 0, time.Time{}, err
return err
}
var revision int64
err = tx.QueryRow(ctx,
@@ -558,9 +449,24 @@ func writePlan(ctx context.Context, tx pgx.Tx, p *Plan, epoch *int64) (int64, ti
// The row is there and at another revision — moved since this was read, or there already
// when this one is new: either way not this writer's to overwrite. (A plan saved before plans
// had revisions is at zero, and its first save here is from a read at zero.)
return 0, time.Time{}, fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved)
return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved)
}
return revision, entered, err
if err != nil {
return err
}
if err := tx.Commit(ctx); err != nil {
return err
}
p.Revision, p.TierEntered = revision, entered
if epoch != nil {
p.Epoch = uint64(*epoch)
} else {
p.Epoch = 0
}
if PlanSaved != nil {
PlanSaved(*p)
}
return nil
}
// short is a commit as a person reads it.
+3 -6
View File
@@ -58,9 +58,7 @@ const (
// **Remade when either end's sealing key changes.** A node that rejoined generated a new key and
// can no longer open what was sealed to the old one, so keeping the blob would deliver something
// unreadable for ever. The new secret reaches both ends in the same push, which is the only
// moment they can be changed together. A remade value is the mesh's, whatever the row held: its
// origin says made and it carries no stamp, so a value a provider's software issued is sealed to it
// again at the provider's next ask (novox/hq ADR 0305 §5) rather than read as still held.
// moment they can be changed together.
func (i *Inventory) SecretFor(ctx context.Context, name, consumer, consumerModule, provider, local string) (
Secret, error) {
consumerKey, err := i.SealingKeyOf(ctx, consumer)
@@ -123,8 +121,7 @@ func (i *Inventory) SecretFor(ctx context.Context, name, consumer, consumerModul
for_consumer = excluded.for_consumer, for_provider = excluded.for_provider,
consumer_key = excluded.consumer_key, provider_key = excluded.provider_key,
created_at = now(),
operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key,
origin = 'made', generation = null`,
operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key`,
name, consumerNode.ID, consumerModule, providerNode.ID,
made.ForConsumer, made.ForProvider, made.ConsumerKey, made.ProviderKey, forOperator, operatorKey, local)
if err != nil {
@@ -205,7 +202,7 @@ func (i *Inventory) AcceptSecretForPair(ctx context.Context, name, consumer, con
on conflict (name, local, consumer, consumer_module, provider) do update set
for_consumer = excluded.for_consumer, for_provider = excluded.for_provider,
consumer_key = excluded.consumer_key, provider_key = excluded.provider_key,
created_at = now(), origin = excluded.origin, generation = null,
created_at = now(), origin = excluded.origin,
operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key`,
name, consumerNode.ID, consumerModule, providerNode.ID,
sealed.ForConsumer, sealed.ForProvider, sealed.ConsumerKey, sealed.ProviderKey,
+9 -62
View File
@@ -100,23 +100,6 @@ type Send struct {
CounterRaisedTo int64
// Recorded is false for a refusal of a sequence the mesh has no send for: its sender is unknown.
Recorded bool
// Answer is what the machine last reported of this send; filled only where a last send is read back
// (LastSend, LastSends), the zero answer elsewhere.
Answer SendAnswer
}
// SendAnswer is what a machine reported of one send, read from its last report when that report is about the
// send's declaration (novox/hq issue 485): whether it applied it, failed part of it or refused it, and why.
// The zero answer is a machine that has not reported on this send — yet, or since a later report replaced it.
type SendAnswer struct {
// Outcome is applied, failed or refused, as the machine reported it; empty when it has not reported on it.
Outcome string
// Refused is the machine's own words when it refused the declaration whole.
Refused string
// Failed is how many of its resources failed, when some did.
Failed int
// At is when the machine reported it; nil when it has not.
At *time.Time
}
// unknownSender is the sender of a refused sequence the mesh has no record of sending.
@@ -226,56 +209,20 @@ func scanSends(rows pgx.Rows) ([]Send, error) {
return out, rows.Err()
}
// LastSend is the last declaration a machine was sent, by its name, with what the machine answered of it; false
// when none was recorded. A refusal of a sequence the mesh has no send for is not a send, and is not it.
// LastSend is the last declaration a machine was sent, by its name; false when none was recorded. A refusal
// of a sequence the mesh has no send for is not a send, and is not it.
func (i *Inventory) LastSend(ctx context.Context, name string) (Send, bool, error) {
sends, err := i.lastSends(ctx, `n.name = $1`, name)
rows, err := i.store.Pool().Query(ctx, `select `+sendColumns+`
from declaration_send s join node n on n.id = s.node
where n.name = $1 and s.recorded order by s.id desc limit 1`, name)
if err != nil {
return Send{}, false, err
}
s, found := sends[name]
return s, found, nil
}
// LastSends is every machine's last send, by machine name, each with what the machine answered of it: what
// `status` says per machine (novox/hq issue 485). A machine never sent anything has no entry.
func (i *Inventory) LastSends(ctx context.Context) (map[string]Send, error) {
return i.lastSends(ctx, `true`)
}
// lastSends reads the newest recorded send of each machine matching where, beside the machine's last report
// when that report is about the send (novox/hq issue 485). It names the send's declaration (its digest), and:
// - from an ordered node-engine, one that reports the epoch and sequence it acted on, those are the send's
// own. A periodic report about an identical earlier send names the same digest and may arrive after a later
// send the machine never received; only the sequence tells them apart. It also holds for a fast machine
// whose report lands before the send's row is stamped.
// - from an engine that reports no order, the report came after the send — both times the store's own — so
// a machine sent again a declaration it once applied does not read as having applied the new send.
func (i *Inventory) lastSends(ctx context.Context, where string, args ...any) (map[string]Send, error) {
rows, err := i.store.Pool().Query(ctx, `select distinct on (n.name) `+sendColumns+`,
coalesce(r.outcome, ''), coalesce(r.refused, ''), case when jsonb_typeof(r.failed) = 'array' then jsonb_array_length(r.failed) else 0 end, r.at
from declaration_send s join node n on n.id = s.node
left join node_report r on r.node = s.node and r.declared <> '' and r.declared = s.digest
and case when r.reported_sequence is not null
then r.reported_sequence = s.sequence and coalesce(r.reported_epoch, 0) = coalesce(s.epoch, 0)
else r.at >= s.sent_at end
where s.recorded and `+where+`
order by n.name, s.id desc`, args...)
if err != nil {
return nil, err
sends, err := scanSends(rows)
if err != nil || len(sends) == 0 {
return Send{}, false, err
}
defer rows.Close()
out := map[string]Send{}
for rows.Next() {
var s Send
if err := rows.Scan(&s.Node, &s.NodeName, &s.Sequence, &s.Epoch, &s.Sender, &s.Generation, &s.Digest,
&s.Modules, &s.SentAt, &s.RefusedAt, &s.RefusedApplied, &s.CounterRaisedTo, &s.Recorded,
&s.Answer.Outcome, &s.Answer.Refused, &s.Answer.Failed, &s.Answer.At); err != nil {
return nil, err
}
out[s.NodeName] = s
}
return out, rows.Err()
return sends[0], true, nil
}
// RefusedSend keeps a machine's refusal of the send of a sequence for the generation it came from, and
-141
View File
@@ -205,144 +205,3 @@ func TestWhetherAMachineReadsAGenerationIsItsLatestWord(t *testing.T) {
}
}
}
// **The last send says what the machine answered of it** (novox/hq issue 485): applied, refused with its
// words, or not answered yet — read from the machine's report only when that report is about the send's
// declaration and came after it, so a report about an earlier send is never this one's answer. And every
// machine's last send is read at once, for status, by name; a machine never sent anything has none.
func TestTheLastSendSaysWhatTheMachineAnswered(t *testing.T) {
inv, node := aNodeWithModules(t)
ctx := t.Context()
record, err := inv.NodeByName(ctx, node)
if err != nil {
t.Fatal(err)
}
if _, err := inv.AddNode(ctx, "laptop"); err != nil {
t.Fatal(err)
}
send := func(sequence int64, digest string) {
t.Helper()
if err := inv.RecordSentWith(ctx, record.ID, digest, nil, 57, 40, Send{Sequence: sequence, Epoch: 57,
Sender: "the controller's daemon (pid 7 on anchor)", Generation: 40, Digest: digest,
Modules: []string{"docker", "sudo"}}); err != nil {
t.Fatal(err)
}
}
report := func(d Doing) {
t.Helper()
if _, err := inv.RecordDoing(ctx, record.ID, d); err != nil {
t.Fatal(err)
}
}
answer := func() SendAnswer {
t.Helper()
last, found, err := inv.LastSend(ctx, node)
if err != nil || !found {
t.Fatalf("the last send is not read back: %v, %v", found, err)
}
return last.Answer
}
send(11, "d11")
if a := answer(); a.Outcome != "" || a.At != nil {
t.Fatalf("a send not reported on reads as answered: %+v", a)
}
report(Doing{Outcome: OutcomeApplied, Declared: "d11", Applied: 3})
if a := answer(); a.Outcome != OutcomeApplied || a.At == nil {
t.Fatalf("a send the machine reported applying reads %+v", a)
}
send(12, "d12")
if a := answer(); a.Outcome != "" {
t.Fatalf("the report about the send before is taken for this one's answer: %+v", a)
}
report(Doing{Outcome: OutcomeRefused, Declared: "d12", Refused: "unknown field \"generation\"\nand more"})
if a := answer(); a.Outcome != OutcomeRefused || a.Refused != "unknown field \"generation\"\nand more" {
t.Fatalf("a send the machine refused reads %+v", a)
}
// Sent again a declaration it reported applying before the send: not answered until it reports again.
report(Doing{Outcome: OutcomeApplied, Declared: "d12", Applied: 3})
send(13, "d12")
if a := answer(); a.Outcome != "" {
t.Fatalf("a report from before the send is taken for its answer: %+v", a)
}
all, err := inv.LastSends(ctx)
if err != nil {
t.Fatal(err)
}
if len(all) != 1 || all[node].Sequence != 13 || all[node].Epoch != 57 || all[node].Generation != 40 ||
!slices.Equal(all[node].Modules, []string{"docker", "sudo"}) {
t.Fatalf("every machine's last send reads %+v", all)
}
if _, found := all["laptop"]; found {
t.Error("a machine never sent anything has a last send")
}
}
// **A report about another declaration, or another send of the same one, is not the last send's answer**
// (novox/hq issue 485, review of mesh-controller #234). A report naming an earlier send's digest that arrives
// after a later send is not the later send's answer. And from an ordered node-engine, a periodic report about
// an identical earlier send — the same digest, arriving after the later send — is not its answer either: only
// the report's epoch and sequence say which send it is about.
func TestAReportIsTheLastSendsAnswerOnlyWhenItIsAboutThatSend(t *testing.T) {
inv, node := aNodeWithModules(t)
ctx := t.Context()
record, err := inv.NodeByName(ctx, node)
if err != nil {
t.Fatal(err)
}
send := func(sequence int64, digest string) {
t.Helper()
if err := inv.RecordSentWith(ctx, record.ID, digest, nil, 57, 40, Send{Sequence: sequence, Epoch: 57,
Sender: "the controller's daemon (pid 7 on anchor)", Generation: 40, Digest: digest}); err != nil {
t.Fatal(err)
}
}
ordered := func(epoch, sequence int64, digest string) {
t.Helper()
if _, err := inv.RecordOrderedDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: digest, Applied: 1},
ReportOrder{Epoch: epoch, Sequence: sequence}, func(ReportOrder) bool { return false }); err != nil {
t.Fatal(err)
}
}
answer := func() string {
t.Helper()
last, found, err := inv.LastSend(ctx, node)
if err != nil || !found {
t.Fatalf("the last send is not read back: %v, %v", found, err)
}
return last.Answer.Outcome
}
// An engine that reports no order: the digest decides.
send(11, "d11")
send(12, "d12")
if _, err := inv.RecordDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: "d11"}); err != nil {
t.Fatal(err)
}
if a := answer(); a != "" {
t.Fatalf("a report about d11, after d12 was sent, reads as d12's answer: %q", a)
}
// An ordered engine: the epoch and sequence decide, whatever the digest and the time.
send(20, "d20")
ordered(57, 20, "d20")
if a := answer(); a != OutcomeApplied {
t.Fatalf("an ordered report about sequence 20 is not its answer: %q", a)
}
send(21, "d20")
ordered(57, 20, "d20")
if a := answer(); a != "" {
t.Fatalf("a periodic report about sequence 20, after 21 was sent with the same digest, reads as 21's answer: %q", a)
}
ordered(56, 21, "d20")
if a := answer(); a != "" {
t.Fatalf("a report about sequence 21 of another epoch reads as this send's answer: %q", a)
}
ordered(57, 21, "d20")
if a := answer(); a != OutcomeApplied {
t.Fatalf("an ordered report about sequence 21 is not its answer: %q", a)
}
}
+8 -17
View File
@@ -123,24 +123,15 @@ func TestNatsABuildIsTakenAndItsOutcomeReachesEverybody(t *testing.T) {
t.Fatalf("line %d came back as %s (%v)", want, msg.Data, err)
}
}
// And it is in the stream for a reader who comes later. **Waited for, with a bound, rather
// than read once** (novox/hq issue 507): a line is said with a plain publish, so the server
// hands it to a live subscriber and stores it in the stream independently, and under load the
// subscriber above had both lines while the stream still held one, or none.
var held map[string]uint64
for until := time.Now().Add(5 * time.Second); ; {
info, err := js.Context().StreamInfo(broker.EventsStream, &nats.StreamInfoRequest{SubjectsFilter: BuildLog("b-1")})
if err != nil {
t.Fatal(err)
}
held = info.State.Subjects
if held[BuildLog("b-1")] >= 2 || time.Now().After(until) {
break
}
time.Sleep(10 * time.Millisecond)
// And it is in the stream for a reader who comes later.
info, err := js.Context().StreamInfo(broker.EventsStream, &nats.StreamInfoRequest{SubjectsFilter: BuildLog("b-1")})
if err != nil {
t.Fatal(err)
}
if held[BuildLog("b-1")] != 2 {
t.Fatalf("the stream holds %v under the build's subject, want 2", held)
if info.State.Subjects[BuildLog("b-1")] != 2 {
t.Fatalf("the stream holds %v under the build's subject, want 2", info.State.Subjects)
}
if err == nil {
}
if result.ID != "b-1" || result.Commit != "abc1234" {
t.Fatalf("the asker got %+v", result)
+1 -19
View File
@@ -173,25 +173,6 @@ func ModuleToolOn(module, tool, node string) string { return ToolSubject(module,
// AskModuleToolOn asks one machine's instance of a module one of its tools and reads its answer.
func AskModuleToolOn(ctx context.Context, conn *nats.Conn, module, tool, node string, args any,
timeout time.Duration) (Answer, error) {
return askModuleToolAt(ctx, conn, ModuleToolOn(module, tool, node), module, tool, node, args, timeout)
}
// AskModuleToolOnAs is AskModuleToolOn in the caller's own name: on the subject that names its caller (novox/hq
// issue 365), which the bus lets only that user publish, so the tool knows who asks from the subject alone. The
// one way a tool can answer the controller and nobody else (novox/hq ADR 0305 §2).
func AskModuleToolOnAs(ctx context.Context, conn *nats.Conn, caller, module, tool, node string, args any,
timeout time.Duration) (Answer, error) {
subject := broker.CalledSubject(ModuleToolOn(module, tool, node), caller)
if subject == "" {
return Answer{}, fmt.Errorf("%q cannot ask %s.%s on %s in its own name: it is not a bus user's name", caller,
module, tool, node)
}
return askModuleToolAt(ctx, conn, subject, module, tool, node, args, timeout)
}
// askModuleToolAt asks one machine's instance of a module one of its tools on subject and reads its answer.
func askModuleToolAt(ctx context.Context, conn *nats.Conn, subject, module, tool, node string, args any,
timeout time.Duration) (Answer, error) {
body, err := json.Marshal(args)
if err != nil {
@@ -199,6 +180,7 @@ func askModuleToolAt(ctx context.Context, conn *nats.Conn, subject, module, tool
}
asking, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
subject := ModuleToolOn(module, tool, node)
refused, stop := refusalsOf(conn, subject)
defer stop()
type replied struct {
-82
View File
@@ -18,34 +18,6 @@
set -eu
export GOFLAGS=-mod=vendor GOPROXY=off
# judge_store_tests says each listed test of one package as `go test -v` reported it, and counts in $missing each
# that did not pass: $1 the package, $2 go test's exit status, $3 the file holding its output, then the names.
# A result is found anywhere in a line, not only at its start: when a test's last output leaves a line open,
# `go test -v` writes `--- PASS: <name> (…)` on that same line (novox/hq issue 515). A package whose go test
# exits non-zero fails even when each listed test said PASS — a TestMain or a leak checker can fail after them.
judge_store_tests() {
pkg=$1 status=$2 out=$3
shift 3
for name in "$@"; do
line=$(grep -oE -- "--- (PASS|FAIL|SKIP): $name \([^)]*\)" "$out" | tail -n 1 || true)
echo " $pkg ${line:-"--- NOT RUN: $name"}"
case "$line" in "--- PASS: "*) ;; *) missing=$((missing + 1)) ;; esac
done
if [ "$status" -ne 0 ]; then
echo " $pkg FAIL: go test exited $status"
missing=$((missing + 1))
fi
}
# `sh merge-check.sh judge-store-tests <package> <status> <output> <name>…` runs only the judging, to try it on a
# recorded `go test -v` output.
if [ "${1:-}" = judge-store-tests ]; then
shift
missing=0
judge_store_tests "$@"
[ "$missing" -eq 0 ]
exit
fi
unformatted=$(gofmt -l cmd internal examples)
if [ -n "$unformatted" ]; then
echo "not gofmt'd:"
@@ -60,57 +32,3 @@ else
echo "NOT RACE-CHECKED: the toolchain holds no C compiler; the suite runs without the race detector"
CGO_ENABLED=0 go test -count=1 -timeout 30m ./...
fi
# **The store tests listed ran** (novox/hq issue 459): a test that needs the store skips without one, and a skip
# passes unseen. With the store this check is given, every test named in testdata/store-tests must have run against
# it and passed, each said by name; without one, that is said. A pull request adds its own store tests to the list.
listed=$(grep -v '^[[:space:]]*\(#\|$\)' testdata/store-tests || true)
# Each line is exactly `<package> <TestName>`: a line of one field ran nothing and said nothing (novox/hq issue 515).
malformed=$(printf '%s\n' "$listed" | awk 'NF != 2 && NF != 0' || true)
if [ -n "$malformed" ]; then
echo "testdata/store-tests: a line is not <package> <TestName>:"
echo "$malformed"
exit 1
fi
# **A change does not shorten the list it is judged by** (novox/hq issue 515): the list is in the change's own tree,
# so an entry main lists stays listed while its test function exists; removing the function removes the entry.
if ! git rev-parse --verify --quiet origin/main^{commit} >/dev/null; then
echo "origin/main cannot be read, and its testdata/store-tests says which store tests must stay listed"
exit 1
fi
dropped=0
for entry in $(git show origin/main:testdata/store-tests 2>/dev/null | grep -v '^[[:space:]]*\(#\|$\)' |
awk 'NF == 2 {print $1 "|" $2}'); do
pkg=${entry%%|*} name=${entry#*|}
printf '%s\n' "$listed" | awk -v p="$pkg" -v n="$name" '$1 == p && $2 == n {f = 1} END {exit !f}' && continue
if cat "$pkg"/*_test.go 2>/dev/null | grep -q "^func $name("; then
echo "testdata/store-tests: $pkg $name is listed on main and its test still exists, but the change drops it"
dropped=$((dropped + 1))
fi
done
[ "$dropped" -eq 0 ] || exit 1
if [ -z "$listed" ]; then
echo "store tests: none listed in testdata/store-tests"
elif [ -n "${MESH_TEST_POSTGRES:-}" ]; then
echo "store tests, each run against the store:"
missing=0
out=$(mktemp)
trap 'rm -f "$out"' EXIT
for pkg in $(printf '%s\n' "$listed" | awk '{print $1}' | sort -u); do
names=$(printf '%s\n' "$listed" | awk -v p="$pkg" '$1 == p {print $2}')
pattern="^($(printf '%s\n' "$names" | paste -sd '|' -))\$"
status=0
CGO_ENABLED=0 go test -count=1 -v -run "$pattern" "$pkg" >"$out" 2>&1 || status=$?
# shellcheck disable=SC2086 # the names are one word each, split on purpose
judge_store_tests "$pkg" "$status" "$out" $names
done
if [ "$missing" -ne 0 ]; then
echo "$missing store test(s) listed in testdata/store-tests did not run and pass against the store"
exit 1
fi
else
echo "NOT STORE-CHECKED: no MESH_TEST_POSTGRES, so the store tests listed in testdata/store-tests skipped"
fi
BIN
View File
Binary file not shown.
-1
View File
@@ -77,7 +77,6 @@
"drill",
"warranted",
"secret-ask",
"issued",
"hand-acts",
"durations",
"conditions",
BIN
View File
Binary file not shown.
BIN
View File
Binary file not shown.
-8
View File
@@ -25,11 +25,3 @@ and write the commits here. The SDK is captured at the commit go.mod pins for gi
rm -rf testdata/beside/mesh-sdk && mkdir -p testdata/beside/mesh-sdk
git -C ../mesh-sdk archive <commit> conformance/events | tar -x -C testdata/beside/mesh-sdk
And from mesh-host at the same commit as its line above, the node-engine's placement guard, which
internal/catalogue's engine_guard_test.go holds the controller's resolved-path rules to (novox/hq issue 496),
kept as .captured so no Go tool reads it as this repository's code:
mkdir -p testdata/beside/mesh-host/internal/apply
git -C ../mesh-host show <commit>:internal/apply/placement_guard.go \
> testdata/beside/mesh-host/internal/apply/placement_guard.go.captured
@@ -1,549 +0,0 @@
package apply
// Where the node-engine places nothing and mounts nothing, whoever asks (novox/hq issue 339).
//
// A directory names its path, and the controller resolves part of that path from what was set for the
// module: its `places` setting moves a directory anywhere, with an owner it names, and its `accesses` setting
// says which of the machine's paths are mounted into its container. The engine runs as root, so a path it
// accepts blindly is a path anyone who could change those settings hands to any account: a directory at /etc
// owned by a caller's account gives it /etc, and an access at / mounts the machine's root into a container.
// The controller refuses both where a setting is made; the engine refuses them again where it applies,
// because a guard in one place is a guard one change away from gone. Whatever the declaration says:
//
// 1. **No directory, access or mount source is one of the machine's own roots, or holds one**: /, /etc,
// /usr, /var, /var/lib, /home, /run and the rest of protectedRoots. Modules place directories BELOW /etc
// or /var/lib, never the root itself; owning one is owning everything in it.
// 2. **Nothing is placed in /proc, /sys, /dev, /boot, /root, /var/spool, /opt or the container runtimes' data
// (/var/lib/docker, /var/lib/containers, /var/lib/containerd, and where the runtimes' configuration moves
// them: runtimeDataRoots), nor in the node-engine's
// own trees** (its state, its identity, its installed builds) but by its own module; nothing is mounted from
// those but /proc, /sys and /dev. A mount of a kernel file, a device or the clock is the plumbing
// systemPath names.
// 2a. **An account's .ssh is never an access or a mount**, and is a directory only below its account's home,
// as that account's (rule 4).
// 3. **A directory below /etc, /usr or /run is root's.** The machine's configuration and programs are read
// as root's word; a directory there owned by another account is that account writing root's word. An
// access is never there at all: the operator's data is not the machine's configuration.
// 4. **Below a person's or an agent's home, a directory is that account's.** Root's or another account's
// directory there is one the account does not control in a tree whose every parent it does. A home
// itself may be a module's directory (a backup repository kept as an account's home is one), and as
// every directory the mesh did not make, it is used as found: never chowned or chmodded (applyDirectory).
// 5. **A mount source that is a refused directory or a refused access is refused with it**: the container
// would otherwise bind the very path the engine would not place, and the runtime creates a missing one
// as root.
//
// Each is a failed resource with its reason in the node's report; nothing is touched. Paths are judged as
// declared and again with every link in them resolved, so a link at /srv/x pointing at /etc places nothing.
import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"sync"
"time"
"github.com/novox/mesh-host/internal/declaration"
)
// protectedRoots are paths no directory, access or mount source may be, nor hold.
var protectedRoots = []string{"/", "/bin", "/boot", "/dev", "/etc", "/home", "/lib", "/lib32", "/lib64",
"/media", "/mnt", "/opt", "/proc", "/root", "/run", "/sbin", "/srv", "/sys", "/tmp", "/usr", "/usr/bin",
"/usr/lib", "/usr/lib64", "/usr/local", "/usr/local/bin", "/usr/local/lib", "/usr/local/sbin", "/usr/sbin",
"/usr/share", "/var", "/var/cache", "/var/lib", "/var/lib/mesh", "/var/log", "/var/run", "/var/tmp",
"/var/spool"}
// forbiddenBelow are trees nothing is placed in or mounted from: the kernel's, the boot loader's and root's
// home. engineTrees are the node-engine's own, which only its own module places in.
var (
forbiddenBelow = []string{"/proc", "/sys", "/dev", "/boot", "/root", "/var/spool", "/opt", "/var/lib/docker",
"/var/lib/containers", "/var/lib/containerd"}
// forbiddenBelowMount are the trees no container mounts from: a mount of the kernel's files and devices is
// the plumbing a container may need (systemPath); root's home and the boot loader's are no plumbing.
forbiddenBelowMount = []string{"/boot", "/root", "/var/spool", "/opt", "/var/lib/docker", "/var/lib/containers",
"/var/lib/containerd"}
engineTrees = []string{"/var/lib/mesh-host", "/usr/lib/nox-mesh-host"}
// rootsOnly are trees a directory below is root's, and an access is never in.
rootsOnly = []string{"/etc", "/usr", "/run", "/var/run"}
)
// runtimeFiles are where the container runtimes say where they keep their data: dockerd's daemon.json, its
// unit and the unit's drop-ins (an ExecStart with --data-root, or the older -g/--graph, continued over lines,
// quoted, through Environment= or EnvironmentFile=, or a --config-file naming another daemon.json), and podman's
// storage.conf (graphroot, a basic or a literal string). The running runtimes are asked first. containerd keeps its own under /var/lib/containerd, which forbiddenBelow names; a
// containerd configured elsewhere, and podman's rootless stores under each account's home, are not read: the
// first is no runtime this mesh runs, and the second is below a home, which rule 4 already keeps for its
// account. A variable so a test names its own.
type runtimeFiles struct {
daemonJSON string
units []string
dropInDirs []string
storageConf string
}
var runtimeConfigs = runtimeFiles{
daemonJSON: "/etc/docker/daemon.json",
units: []string{"/etc/systemd/system/docker.service", "/usr/lib/systemd/system/docker.service", "/lib/systemd/system/docker.service"},
dropInDirs: []string{"/etc/systemd/system/docker.service.d", "/run/systemd/system/docker.service.d", "/usr/lib/systemd/system/docker.service.d"},
storageConf: "/etc/containers/storage.conf",
}
var (
dataRootFlag = regexp.MustCompile(`(?:--data-root|--graph|-g)(?:=|\s+)(\S+)`)
configFlag = regexp.MustCompile(`--config-file(?:=|\s+)(\S+)`)
graphRoot = regexp.MustCompile(`(?m)^\s*graphroot\s*=\s*(?:"([^"]+)"|'([^']+)')`)
envVar = regexp.MustCompile(`\$\{([A-Za-z_][A-Za-z0-9_]*)\}|\$([A-Za-z_][A-Za-z0-9_]*)`)
)
// runtimeRoots is where the container runtimes keep their data, as the last apply read it (readRuntimeRoots);
// nil until an apply has read it, when the guard reads the files itself.
var (
runtimeRootsMu sync.Mutex
runtimeRoots []string
)
// AskRuntimes is how the engine asks the running container runtimes where they keep their data: set by the engine
// to run the commands, nil in a test, which then reads the files alone. Its own, never the apply's runner, whose
// commands a test reads back as what the apply did.
var AskRuntimes Runner
// refreshRuntimeRoots reads where the runtimes keep their data once, at the start of an apply.
func refreshRuntimeRoots(ctx context.Context) {
roots := readRuntimeRoots(ctx, AskRuntimes)
runtimeRootsMu.Lock()
runtimeRoots = roots
runtimeRootsMu.Unlock()
}
// runtimeDataRoots is every place the container runtimes keep their data, beyond the default trees forbiddenBelow
// names: every container's filesystem is there.
func runtimeDataRoots() []string {
runtimeRootsMu.Lock()
roots := runtimeRoots
runtimeRootsMu.Unlock()
if roots != nil {
return roots
}
return readRuntimeRoots(context.Background(), nil)
}
// readRuntimeRoots asks the running runtimes where they keep their data, when run is given (`docker info`,
// `podman info`), and reads their configuration besides: an answer from a runtime that is running is what it
// really does, and the files say what it will do when it starts again. Both count. A runtime that is not running
// or not installed answers nothing, which is no error.
func readRuntimeRoots(ctx context.Context, run Runner) []string {
seen := map[string]bool{}
var out []string
add := func(p string) {
p = strings.Trim(strings.TrimSpace(p), `"'`)
if !filepath.IsAbs(p) {
return
}
p = filepath.Clean(p)
if !seen[p] {
seen[p] = true
out = append(out, p)
}
}
if run != nil {
// Each runtime has its own ten seconds: one that hangs costs the other nothing.
for _, q := range [][]string{{"docker", "info", "--format", "{{.DockerRootDir}}"},
{"podman", "info", "--format", "{{.Store.GraphRoot}}"}} {
ask, cancel := context.WithTimeout(ctx, 10*time.Second)
if root, err := run(ask, q[0], q[1:]...); err == nil {
add(root)
}
cancel()
}
}
daemonJSON := func(path string) {
raw, err := os.ReadFile(path)
if err != nil {
return
}
var c struct {
DataRoot string `json:"data-root"`
Graph string `json:"graph"`
}
if json.Unmarshal(raw, &c) == nil {
add(c.DataRoot)
add(c.Graph)
}
}
daemonJSON(runtimeConfigs.daemonJSON)
units := append([]string(nil), runtimeConfigs.units...)
for _, dir := range runtimeConfigs.dropInDirs {
matches, _ := filepath.Glob(filepath.Join(dir, "*.conf"))
units = append(units, matches...)
}
// The unit and its drop-ins are one unit to systemd: a variable set in one is seen by an ExecStart in another.
env := map[string]string{}
var execs []string
for _, u := range units {
raw, err := os.ReadFile(u)
if err != nil {
continue
}
e, x := unitLines(string(raw))
for k, v := range e {
env[k] = v
}
execs = append(execs, x...)
}
for _, line := range execs {
line = envVar.ReplaceAllStringFunc(line, func(ref string) string {
m := envVar.FindStringSubmatch(ref)
if v, ok := env[m[1]+m[2]]; ok {
return v
}
return ref
})
for _, m := range dataRootFlag.FindAllStringSubmatch(line, -1) {
add(m[1])
}
for _, m := range configFlag.FindAllStringSubmatch(line, -1) {
daemonJSON(strings.Trim(m[1], `"'`))
}
}
if raw, err := os.ReadFile(runtimeConfigs.storageConf); err == nil {
for _, m := range graphRoot.FindAllStringSubmatch(string(raw), -1) {
add(m[1] + m[2])
}
}
return out
}
// unitLines reads a unit file as systemd does for what matters here: a line ending in a backslash continues on the
// next, Environment= sets variables (quoted or not, several to a line), EnvironmentFile= (a leading - says it may
// be missing) reads KEY=value lines, and every ExecStart line is returned whole.
func unitLines(text string) (map[string]string, []string) {
var joined []string
var cur strings.Builder
for _, line := range strings.Split(text, "\n") {
trimmed := strings.TrimRight(line, " \t")
if strings.HasSuffix(trimmed, "\\") {
cur.WriteString(strings.TrimSuffix(trimmed, "\\") + " ")
continue
}
cur.WriteString(line)
joined = append(joined, cur.String())
cur.Reset()
}
if cur.Len() > 0 {
joined = append(joined, cur.String())
}
env := map[string]string{}
setPairs := func(s string) {
for _, f := range splitQuoted(s) {
if k, v, ok := strings.Cut(f, "="); ok {
env[strings.TrimSpace(k)] = strings.Trim(strings.TrimSpace(v), `"'`)
}
}
}
var execs []string
for _, line := range joined {
l := strings.TrimSpace(line)
switch {
case strings.HasPrefix(l, "Environment="):
setPairs(strings.TrimPrefix(l, "Environment="))
case strings.HasPrefix(l, "EnvironmentFile="):
path := strings.TrimPrefix(strings.TrimSpace(strings.TrimPrefix(l, "EnvironmentFile=")), "-")
if raw, err := os.ReadFile(path); err == nil {
for _, kv := range strings.Split(string(raw), "\n") {
kv = strings.TrimSpace(kv)
if kv == "" || strings.HasPrefix(kv, "#") {
continue
}
setPairs(kv)
}
}
case strings.HasPrefix(l, "ExecStart"):
execs = append(execs, l)
}
}
return env, execs
}
// splitQuoted splits on blanks outside double or single quotes, keeping the quotes' contents whole.
func splitQuoted(s string) []string {
var out []string
var cur strings.Builder
var quote rune
for _, r := range s {
switch {
case quote != 0 && r == quote:
quote = 0
case quote == 0 && (r == '"' || r == '\''):
quote = r
case quote == 0 && (r == ' ' || r == '\t'):
if cur.Len() > 0 {
out = append(out, cur.String())
cur.Reset()
}
default:
cur.WriteRune(r)
}
}
if cur.Len() > 0 {
out = append(out, cur.String())
}
return out
}
// engineModule is the module whose resources may place in the engine's own trees.
const engineModule = "mesh-host"
// passwdFile is the user database homes are read from. A variable so a test names its own.
var passwdFile = "/etc/passwd"
// PlacementRefusedError is a resource the engine will not place, or mount, where it says.
type PlacementRefusedError struct {
Path, Why string
}
func (e *PlacementRefusedError) Error() string {
return fmt.Sprintf("%s is not placed: %s (novox/hq issue 339); nothing was touched", e.Path, e.Why)
}
// homeAccount is a person's or an agent's account and its home.
type homeAccount struct {
Name string
UID int
}
// accountsOfHomes is each person's or agent's home and the account it belongs to, from the user database: an
// account with a uid of 1000 or more, or a home under /home. A variable so a test names its own.
var accountsOfHomes = func() map[string]homeAccount {
f, err := os.Open(passwdFile)
if err != nil {
return nil
}
defer f.Close()
out := map[string]homeAccount{}
sc := bufio.NewScanner(f)
for sc.Scan() {
fields := strings.Split(sc.Text(), ":")
if len(fields) < 6 {
continue
}
uid, err := strconv.Atoi(fields[2])
if err != nil {
continue
}
home := filepath.Clean(fields[5])
if home == "/" || home == "." || home == "" || home == "/nonexistent" {
continue
}
if (uid >= 1000 && uid != 65534) || strings.HasPrefix(home, "/home/") {
out[home] = homeAccount{Name: fields[0], UID: uid}
}
}
return out
}
// below says whether path is strictly below dir.
func below(path, dir string) bool {
if dir == "/" {
return path != "/"
}
return strings.HasPrefix(path, dir+"/")
}
// atOrBelow says whether path is dir or below it.
func atOrBelow(path, dir string) bool { return path == dir || below(path, dir) }
// resolved is a path with every link in it followed, as far as the path exists, and the rest as declared.
func resolved(path string) string {
rest := ""
for p := path; ; p = filepath.Dir(p) {
if real, err := filepath.EvalSymlinks(p); err == nil {
return filepath.Clean(filepath.Join(real, rest))
}
if filepath.Dir(p) == p {
return path
}
rest = filepath.Join(filepath.Base(p), rest)
}
}
// ownedByAccount says whether a declared owner is that account: by name, or by its uid ("1001", "1001:1001").
func ownedByAccount(owner string, a homeAccount) bool {
if owner == a.Name {
return true
}
user, _, _ := strings.Cut(owner, ":")
if uid, err := strconv.Atoi(user); err == nil {
return uid == a.UID
}
return false
}
// rootOwner says whether a declared owner is root: none, "root", or uid 0.
func rootOwner(owner string) bool {
if owner == "" || owner == "root" {
return true
}
user, _, _ := strings.Cut(owner, ":")
return user == "0"
}
// what a guarded path is, for the words of a refusal.
type placing int
const (
placingDirectory placing = iota
placingAccess
placingMount
)
// refusePath says why a path is not placed or mounted; nil when it may be. module is the resource's module,
// owner a directory's declared owner.
func refusePath(path string, kind placing, module, owner string) error {
clean := filepath.Clean(path)
if !filepath.IsAbs(clean) {
return nil // the declaration refuses a relative path already; a named volume is not a path
}
for _, p := range []string{clean, resolved(clean)} {
if err := refuseOne(p, kind, module, owner); err != nil {
if p != clean {
err.Why = fmt.Sprintf("through a link, it is %s, and %s", p, err.Why)
err.Path = clean
}
return err
}
}
return nil
}
func refuseOne(path string, kind placing, module, owner string) *PlacementRefusedError {
for _, root := range protectedRoots {
if path == root || below(root, path) {
return &PlacementRefusedError{Path: path, Why: root + " is one of the machine's own directories, " +
"and owning or mounting it would be owning or mounting everything in it"}
}
}
trees := append([]string(nil), forbiddenBelow...)
if kind == placingMount {
// A mount is the machine's plumbing as often as a module's data — the clock, a kernel file, /dev/null
// (systemPath) — and what a setting can mount at all is a directory or an access, refused above it.
trees = append([]string(nil), forbiddenBelowMount...)
}
// The container runtimes' data, wherever the machine keeps it: every container's filesystem is there.
trees = append(trees, runtimeDataRoots()...)
for _, tree := range trees {
if atOrBelow(path, tree) {
return &PlacementRefusedError{Path: path, Why: "nothing is placed in or mounted from " + tree}
}
}
if module != engineModule {
for _, tree := range engineTrees {
if atOrBelow(path, tree) {
return &PlacementRefusedError{Path: path, Why: tree + " is the node-engine's own, placed in by " +
"its own module alone"}
}
}
}
for _, tree := range rootsOnly {
if !below(path, tree) {
continue
}
switch {
case kind == placingAccess:
return &PlacementRefusedError{Path: path, Why: "an access is the operator's data, and " + tree +
" is the machine's own"}
case kind == placingDirectory && !rootOwner(owner):
return &PlacementRefusedError{Path: path, Why: fmt.Sprintf("a directory below %s is root's, and this "+
"one is declared %s's", tree, owner)}
}
}
ssh := false
for _, part := range strings.Split(path, "/") {
ssh = ssh || part == ".ssh"
}
if ssh && kind != placingDirectory {
return &PlacementRefusedError{Path: path, Why: "it is an account's .ssh, which holds its keys and who may " +
"log in as it, and is never an access or a mount"}
}
if kind != placingDirectory {
return nil
}
homes := accountsOfHomes()
var deepest string
for home := range homes {
if below(path, home) && len(home) > len(deepest) {
deepest = home
}
}
if ssh && deepest == "" {
return &PlacementRefusedError{Path: path, Why: "a .ssh directory is placed only below its account's home, " +
"as that account's"}
}
if deepest != "" {
if a := homes[deepest]; !ownedByAccount(owner, a) {
if owner == "" {
owner = "root"
}
return &PlacementRefusedError{Path: path, Why: fmt.Sprintf("it is below %s's home and declared %s's; "+
"below a home only that account's directories are placed", a.Name, owner)}
}
}
return nil
}
// moduleOfID is the module a resource id names, or "".
func moduleOfID(id string) string {
module, _ := moduleOf(id)
return module
}
// refusedPlaces judges every directory and access of a declaration before anything is applied, and answers
// each refusal by path: what is refused is refused again as a container's mount source.
func refusedPlaces(resources []declaration.Resource) map[string]error {
out := map[string]error{}
for _, r := range resources {
var err error
switch res := r.(type) {
case *declaration.Directory:
err = refusePath(res.Path, placingDirectory, moduleOfID(res.ID), res.Owner)
case *declaration.Access:
err = refusePath(res.Path, placingAccess, moduleOfID(res.ID), "")
default:
continue
}
if err != nil {
out[filepath.Clean(r.Target())] = err
}
}
return out
}
// refuseMounts says why a container's mounts are not made; nil when they may be.
func refuseMounts(c *declaration.Container, refused map[string]error) error {
for _, v := range c.Volumes {
src := mountSource(v)
if !strings.HasPrefix(src, "/") {
continue // a named volume, which the runtime keeps in its own tree
}
src = filepath.Clean(src)
for path, why := range refused {
if atOrBelow(src, path) {
var refusal *PlacementRefusedError
if errors.As(why, &refusal) {
return &PlacementRefusedError{Path: src, Why: "it is mounted from " + path +
", which is refused: " + refusal.Why}
}
return why
}
}
if err := refusePath(src, placingMount, moduleOfID(c.ID), ""); err != nil {
return err
}
}
return nil
}
-14
View File
@@ -1,14 +0,0 @@
# The tests that need the throwaway store and must have run against it (novox/hq issue 459): merge-check.sh runs
# each one named here, by package, and fails unless every one passed — a test that needs the store skips without
# one, and a skip passes unseen. One per line: <package> <TestName>. Lines starting with # are said nothing of.
./cmd/mesh-controller TestAStopWithoutAMergeWalksTheOthersAgain
./cmd/mesh-controller TestAStopWithoutAMergeEndsTheMergesThatContainIt
./cmd/mesh-controller TestAStopWalksTheOthersAgainInTheOpenBatch
./cmd/mesh-controller TestAStopNamesWhatStaysOnTheMachinesAndWhatStillCarriesIt
./cmd/mesh-controller TestAStopWithoutAMergeLeavesItOutOfABatch
./cmd/mesh-controller TestSavingPlansAndMovingMergesIsOneAct
./internal/inventory TestTheLastSendSaysWhatTheMachineAnswered
./internal/inventory TestAReportIsTheLastSendsAnswerOnlyWhenItIsAboutThatSend
./cmd/mesh-controller TestNodeShowAndStatusReadTheLastSendFromTheStore
./internal/inventory TestAnIssuedValueIsSealedToEveryHolderOnceAndNotAgain
./internal/inventory TestAnIssuedValueIsSealedAgainAfterAKeyChangeOrAnAcceptance