Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bf82d30865 | ||
|
|
c432b9b5f6 | ||
|
|
7bd61332cb |
@@ -0,0 +1,207 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// **A filling bucket or log is said before it is full** (novox/hq ADR 0297 §6, issue 501).
|
||||
//
|
||||
// A module's bucket and its log each have a cap on the bus, and when either is full the bus refuses
|
||||
// the next write: the module stops doing what it writes for, and nothing said so before. On
|
||||
// 2026-10-11 the issue tracker's bucket held a sixth of its cap after two days, growing toward a stop
|
||||
// nobody would have heard coming. So the self-check reads every module's bucket and log, and one at
|
||||
// fillRaiseAt of its cap or more raises a condition naming the module, the bucket or log, and how full
|
||||
// it is.
|
||||
//
|
||||
// **Cleared by observation below fillClearBelow, not below fillRaiseAt**: a bucket or log hovering at
|
||||
// its threshold would otherwise be raised and cleared on every run. Between the two, an open condition
|
||||
// is kept and none is raised.
|
||||
//
|
||||
// **One that cannot be read is said, and the rest are still judged.** An unanswered question about one
|
||||
// stream is no reason to know nothing of the others: it is a finding of its own, naming the bucket or
|
||||
// log and why, and a fill condition already open for it is kept, because not knowing is not a pass.
|
||||
|
||||
// The fill thresholds, in percent of a bucket's or log's cap.
|
||||
const (
|
||||
fillRaiseAt = 75
|
||||
fillClearBelow = 70
|
||||
)
|
||||
|
||||
// The conditions the fill probe raises: one filling, and one that could not be read.
|
||||
const (
|
||||
kindBucketOrLogFilling = "bucket-or-log-filling"
|
||||
kindBucketOrLogUnread = "bucket-or-log-unread"
|
||||
)
|
||||
|
||||
// probeFillID is the probe's id in the registry.
|
||||
const probeFillID = "D-fill"
|
||||
|
||||
// bucketOrLog is one module's bucket or log as the fill probe reads it.
|
||||
type bucketOrLog struct {
|
||||
Module string
|
||||
// Kind is `bucket` or `log`; Name its local name; Stream the stream it is on the bus.
|
||||
Kind, Name, Stream string
|
||||
}
|
||||
|
||||
// filled is how full one bucket or log is, read from the bus.
|
||||
type filled struct {
|
||||
Bytes, Max uint64
|
||||
}
|
||||
|
||||
// percent is how full, in whole percent, rounded down.
|
||||
func (f filled) percent() uint64 {
|
||||
if f.Max == 0 {
|
||||
return 0
|
||||
}
|
||||
return f.Bytes * 100 / f.Max
|
||||
}
|
||||
|
||||
// fillKey is where a bucket's or log's fill condition is kept.
|
||||
func fillKey(s bucketOrLog) string { return conditions.Key(conditions.ScopeBus, s.Stream, "filling") }
|
||||
|
||||
// fillObservation is what one bucket's or log's fill says: a finding at fillRaiseAt or above, and,
|
||||
// while its condition is open, at fillClearBelow or above too; nothing otherwise, which is what clears
|
||||
// it. The operator's words are the kind's (plain_words.go); the summary and the evidence name the
|
||||
// module, the bucket or log, and the fill.
|
||||
func fillObservation(s bucketOrLog, f filled, open bool) (conditions.Observation, bool) {
|
||||
if f.Max == 0 {
|
||||
return conditions.Observation{}, false // one with no cap cannot fill
|
||||
}
|
||||
// Compared in bytes, not rounded percent: 74.9% is not 75%.
|
||||
atRaise := f.Bytes*100 >= f.Max*fillRaiseAt
|
||||
aboveClear := f.Bytes*100 >= f.Max*fillClearBelow
|
||||
if !atRaise && !(open && aboveClear) {
|
||||
return conditions.Observation{}, false
|
||||
}
|
||||
pct := f.percent()
|
||||
return conditions.Observation{
|
||||
Scope: conditions.ScopeBus, ID: s.Stream, Token: "filling", Kind: kindBucketOrLogFilling,
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s's %s %s holds %s of %s, %d%%: at its cap the bus refuses the module's next write",
|
||||
s.Module, s.Kind, s.Name, mibWords(f.Bytes), mibWords(f.Max), pct),
|
||||
Said: fmt.Sprintf("module %s, %s %s (stream %s): %d of %d bytes, %d%%", s.Module, s.Kind, s.Name,
|
||||
s.Stream, f.Bytes, f.Max, pct),
|
||||
}, true
|
||||
}
|
||||
|
||||
// unreadObservation says one bucket or log could not be read, and why. Asked over the network, so one
|
||||
// look can be wrong: raised on the second look in a row (confirm.go).
|
||||
func unreadObservation(s bucketOrLog, why error) conditions.Observation {
|
||||
return conditions.Observation{
|
||||
Scope: conditions.ScopeBus, ID: s.Stream, Token: "unread", Kind: kindBucketOrLogUnread,
|
||||
Severity: conditions.Warning, Confirm: true,
|
||||
Summary: fmt.Sprintf("%s's %s %s could not be read, so how full it is is not known", s.Module, s.Kind, s.Name),
|
||||
Said: fmt.Sprintf("module %s, %s %s (stream %s): %v", s.Module, s.Kind, s.Name, s.Stream, why),
|
||||
}
|
||||
}
|
||||
|
||||
// keptFilling is an open fill condition said again for a bucket or log that could not be read this
|
||||
// time: not knowing how full it is now is no reason to say it has room. Only ever made from a condition
|
||||
// read back as open — its own summary and severity, never words made up for it.
|
||||
func keptFilling(s bucketOrLog, open conditions.Condition, why error) conditions.Observation {
|
||||
return conditions.Observation{
|
||||
Scope: conditions.ScopeBus, ID: s.Stream, Token: "filling", Kind: kindBucketOrLogFilling,
|
||||
Severity: open.Severity, Summary: open.Summary,
|
||||
Said: fmt.Sprintf("module %s, %s %s (stream %s): not read this time (%v); kept open as it was",
|
||||
s.Module, s.Kind, s.Name, s.Stream, why),
|
||||
}
|
||||
}
|
||||
|
||||
// mibWords is a size as a person reads it, in MiB with one decimal.
|
||||
func mibWords(b uint64) string {
|
||||
return fmt.Sprintf("%.1f MiB", float64(b)/(1024*1024))
|
||||
}
|
||||
|
||||
// readFill is how full one bucket or log is on the bus; found false when it is not on the bus.
|
||||
type readFill func(ctx context.Context, s bucketOrLog) (f filled, found bool, err error)
|
||||
|
||||
// openFill is the fill condition open under a key, if one is, or why it could not be read.
|
||||
type openFill func(ctx context.Context, key string) (conditions.Condition, bool, error)
|
||||
|
||||
// judgeFills judges every bucket and log on its own: one that cannot be read is said as unread, with
|
||||
// its fill condition kept only when that condition was read back and is open — when it cannot be read
|
||||
// either, nothing is said of its fill, which the unread finding already covers. Every other is judged
|
||||
// by its fill; for one of those, a condition that cannot be read is taken as open, so an unknown never
|
||||
// clears a fill measured between fillClearBelow and fillRaiseAt.
|
||||
func judgeFills(ctx context.Context, all []bucketOrLog, read readFill, open openFill) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, s := range all {
|
||||
f, found, err := read(ctx, s)
|
||||
if err != nil {
|
||||
out = append(out, unreadObservation(s, err))
|
||||
if c, isOpen, cerr := open(ctx, fillKey(s)); cerr == nil && isOpen {
|
||||
out = append(out, keptFilling(s, c, err))
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !found {
|
||||
continue // not on the bus: created on the controller's next raise, and nothing there can fill
|
||||
}
|
||||
_, isOpen, cerr := open(ctx, fillKey(s))
|
||||
isOpen = isOpen || cerr != nil
|
||||
if o, said := fillObservation(s, f, isOpen); said {
|
||||
out = append(out, o)
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].Key() < out[j].Key() })
|
||||
return out
|
||||
}
|
||||
|
||||
// declaredBucketsAndLogs is every module's bucket and log the catalogue declares, by its stream.
|
||||
func declaredBucketsAndLogs(ctx context.Context, d *doctor) ([]bucketOrLog, error) {
|
||||
buckets, err := d.open.inventory.DeclaredBuckets(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logs, err := d.open.inventory.DeclaredLogs(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []bucketOrLog
|
||||
for _, b := range buckets {
|
||||
out = append(out, bucketOrLog{Module: b.Module, Kind: "bucket", Name: b.Name, Stream: "KV_" + b.Bucket()})
|
||||
}
|
||||
for _, l := range logs {
|
||||
out = append(out, bucketOrLog{Module: l.Module, Kind: "log", Name: l.Name, Stream: l.Stream()})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// probeFill reads every module's bucket and log on the bus and says each one filling toward its cap,
|
||||
// and each one that could not be read.
|
||||
func probeFill(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
all, err := declaredBucketsAndLogs(ctx, d)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js := d.js.Context()
|
||||
read := func(ctx context.Context, s bucketOrLog) (filled, bool, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return filled{}, false, err
|
||||
}
|
||||
info, err := js.StreamInfo(s.Stream, nats.Context(ctx))
|
||||
switch {
|
||||
case errors.Is(err, nats.ErrStreamNotFound):
|
||||
return filled{}, false, nil
|
||||
case err != nil:
|
||||
return filled{}, false, err
|
||||
case info.Config.MaxBytes <= 0:
|
||||
return filled{}, true, nil
|
||||
}
|
||||
return filled{Bytes: info.State.Bytes, Max: uint64(info.Config.MaxBytes)}, true, nil
|
||||
}
|
||||
open := func(ctx context.Context, key string) (conditions.Condition, bool, error) {
|
||||
if d.keeper == nil {
|
||||
return conditions.Condition{}, false, nil
|
||||
}
|
||||
return d.keeper.Get(ctx, key)
|
||||
}
|
||||
return judgeFills(ctx, all, read, open), nil
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// **A filling log or bucket is said at three quarters of its cap and cleared below seven tenths**
|
||||
// (novox/hq ADR 0297 §6): raised at 75% naming the module, the bucket or log and its fill; not at 74.9%;
|
||||
// kept between 70% and 75% while it is open, and not raised there when it is not; and cleared — said no
|
||||
// more — once it reads below 70%, open or not.
|
||||
func TestAFillingBucketOrLogIsRaisedAtThreeQuartersAndClearedBelowSevenTenths(t *testing.T) {
|
||||
const mib = 1024 * 1024
|
||||
log := bucketOrLog{Module: "mesh-issues", Kind: "log", Name: "changes", Stream: "LOG_mesh-issues_changes"}
|
||||
bucket := bucketOrLog{Module: "mesh-issues", Kind: "bucket", Name: "issues", Stream: "KV_mesh-issues_issues"}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
of bucketOrLog
|
||||
bytes uint64
|
||||
max uint64
|
||||
open bool
|
||||
said bool
|
||||
}{
|
||||
{"a log at exactly 75%", log, 768 * mib, 1024 * mib, false, true},
|
||||
{"a bucket at exactly 75%", bucket, 48 * mib, 64 * mib, false, true},
|
||||
{"a bucket full", bucket, 64 * mib, 64 * mib, false, true},
|
||||
{"a log just under 75%", log, 768*mib - 1, 1024 * mib, false, false},
|
||||
{"a bucket at 72%, not open", bucket, 64 * mib * 72 / 100, 64 * mib, false, false},
|
||||
{"a bucket at 72%, open: kept", bucket, 64 * mib * 72 / 100, 64 * mib, true, true},
|
||||
{"a log at exactly 70%, open: kept", log, 700 * mib, 1000 * mib, true, true},
|
||||
{"a log just under 70%, open: cleared", log, 700*mib - 1, 1000 * mib, true, false},
|
||||
{"a bucket at 10%, open: cleared", bucket, 64 * mib / 10, 64 * mib, true, false},
|
||||
{"a bucket with no cap", bucket, 100, 0, true, false},
|
||||
} {
|
||||
o, said := fillObservation(c.of, filled{Bytes: c.bytes, Max: c.max}, c.open)
|
||||
if said != c.said {
|
||||
t.Errorf("%s: said %v, want %v", c.name, said, c.said)
|
||||
continue
|
||||
}
|
||||
if !said {
|
||||
continue
|
||||
}
|
||||
if o.Key() != fillKey(c.of) || o.Kind != kindBucketOrLogFilling || o.Severity != conditions.Warning {
|
||||
t.Errorf("%s: raised as %s (%s, %s)", c.name, o.Key(), o.Kind, o.Severity)
|
||||
}
|
||||
for _, part := range []string{c.of.Module, c.of.Kind + " " + c.of.Name, "%", " of "} {
|
||||
if !strings.Contains(o.Summary, part) {
|
||||
t.Errorf("%s: does not name %q: %q", c.name, part, o.Summary)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(o.Said, "bytes") {
|
||||
t.Errorf("%s: the evidence does not say the fill in bytes: %q", c.name, o.Said)
|
||||
}
|
||||
}
|
||||
o, _ := fillObservation(log, filled{Bytes: 800 * mib, Max: 1024 * mib}, false)
|
||||
if !strings.Contains(o.Summary, "800.0 MiB of 1024.0 MiB, 78%") {
|
||||
t.Errorf("the fill is said as %q", o.Summary)
|
||||
}
|
||||
}
|
||||
|
||||
// **One bucket or log that cannot be read does not stop the others being judged** (review of #231): it is
|
||||
// said as unread with its reason, a fill condition open for it is kept, and every other bucket and log
|
||||
// is judged by its own fill.
|
||||
func TestAnUnreadableBucketOrLogIsSaidAndTheRestAreStillJudged(t *testing.T) {
|
||||
const mib = 1024 * 1024
|
||||
broken := bucketOrLog{Module: "a", Kind: "bucket", Name: "broken", Stream: "KV_a_broken"}
|
||||
brokenOpen := bucketOrLog{Module: "a", Kind: "log", Name: "was-filling", Stream: "LOG_a_was-filling"}
|
||||
full := bucketOrLog{Module: "b", Kind: "log", Name: "changes", Stream: "LOG_b_changes"}
|
||||
empty := bucketOrLog{Module: "c", Kind: "bucket", Name: "quiet", Stream: "KV_c_quiet"}
|
||||
absent := bucketOrLog{Module: "d", Kind: "bucket", Name: "not-yet", Stream: "KV_d_not-yet"}
|
||||
refusal := errors.New("the bus did not answer")
|
||||
read := func(_ context.Context, s bucketOrLog) (filled, bool, error) {
|
||||
switch s {
|
||||
case broken, brokenOpen:
|
||||
return filled{}, false, refusal
|
||||
case full:
|
||||
return filled{Bytes: 900 * mib, Max: 1000 * mib}, true, nil
|
||||
case empty:
|
||||
return filled{Bytes: 1, Max: 64 * mib}, true, nil
|
||||
}
|
||||
return filled{}, false, nil
|
||||
}
|
||||
open := func(_ context.Context, key string) (conditions.Condition, bool, error) {
|
||||
if key == fillKey(brokenOpen) {
|
||||
return conditions.Condition{Key: key, Severity: conditions.Warning,
|
||||
Summary: "a's log was-filling holds 800.0 MiB of 1024.0 MiB, 78%"}, true, nil
|
||||
}
|
||||
return conditions.Condition{}, false, nil
|
||||
}
|
||||
got := map[string]conditions.Observation{}
|
||||
for _, o := range judgeFills(context.Background(), []bucketOrLog{broken, brokenOpen, full, empty, absent}, read, open) {
|
||||
got[o.Key()] = o
|
||||
}
|
||||
for _, s := range []bucketOrLog{broken, brokenOpen} {
|
||||
o, said := got[conditions.Key(conditions.ScopeBus, s.Stream, "unread")]
|
||||
if !said || o.Kind != kindBucketOrLogUnread || !o.Confirm || !strings.Contains(o.Said, refusal.Error()) {
|
||||
t.Errorf("%s could not be read and was said as %+v", s.Stream, o)
|
||||
}
|
||||
}
|
||||
if o, kept := got[fillKey(brokenOpen)]; !kept || !strings.Contains(o.Said, "kept open") {
|
||||
t.Errorf("an open fill condition of a log not read this time was not kept: %+v", o)
|
||||
}
|
||||
if _, said := got[fillKey(broken)]; said {
|
||||
t.Error("a bucket not read and not filling was said filling")
|
||||
}
|
||||
if o, said := got[fillKey(full)]; !said || o.Kind != kindBucketOrLogFilling {
|
||||
t.Errorf("a log at 90%% beside an unreadable one was not judged: %+v", got)
|
||||
}
|
||||
if len(got) != 4 {
|
||||
t.Errorf("said %d findings, want 4 (two unread, one kept, one filling): %v", len(got), got)
|
||||
}
|
||||
}
|
||||
|
||||
// Both kinds have plain words of their own, which hold to the plain rule (novox/hq ADR 0253).
|
||||
func TestAFillingOrUnreadBucketOrLogIsSaidInPlainWords(t *testing.T) {
|
||||
for _, kind := range []string{kindBucketOrLogFilling, kindBucketOrLogUnread} {
|
||||
wording, has := plainWordings[kind]
|
||||
if !has {
|
||||
t.Errorf("%s has no plain words", kind)
|
||||
continue
|
||||
}
|
||||
if why, ok := conditions.PlainWords(wording(conditions.Observation{Kind: kind})); !ok {
|
||||
t.Errorf("%s: %s", kind, why)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **When neither the bucket or log nor its condition can be read, nothing is said of its fill** (second
|
||||
// review of #231): the unread finding covers it, and no fill is invented for it. For one that is read and
|
||||
// measured between seven tenths and three quarters, a condition that cannot be read is taken as open, so
|
||||
// an unknown never clears it.
|
||||
func TestNothingIsSaidOfAFillWhenNeitherItNorItsConditionCanBeRead(t *testing.T) {
|
||||
const mib = 1024 * 1024
|
||||
broken := bucketOrLog{Module: "a", Kind: "log", Name: "changes", Stream: "LOG_a_changes"}
|
||||
between := bucketOrLog{Module: "b", Kind: "bucket", Name: "issues", Stream: "KV_b_issues"}
|
||||
read := func(_ context.Context, s bucketOrLog) (filled, bool, error) {
|
||||
if s == broken {
|
||||
return filled{}, false, errors.New("the bus did not answer")
|
||||
}
|
||||
return filled{Bytes: 72 * mib, Max: 100 * mib}, true, nil
|
||||
}
|
||||
open := func(context.Context, string) (conditions.Condition, bool, error) {
|
||||
return conditions.Condition{}, false, errors.New("the conditions bucket did not answer")
|
||||
}
|
||||
got := map[string]conditions.Observation{}
|
||||
for _, o := range judgeFills(context.Background(), []bucketOrLog{broken, between}, read, open) {
|
||||
got[o.Key()] = o
|
||||
}
|
||||
if o, said := got[fillKey(broken)]; said {
|
||||
t.Fatalf("a fill was said for a log whose fill and condition could not be read: %+v", o)
|
||||
}
|
||||
if _, said := got[conditions.Key(conditions.ScopeBus, broken.Stream, "unread")]; !said {
|
||||
t.Error("the log that could not be read was not said as unread")
|
||||
}
|
||||
if _, kept := got[fillKey(between)]; !kept {
|
||||
t.Error("a bucket at 72% whose condition could not be read was cleared")
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Errorf("said %d findings, want 2: %v", len(got), got)
|
||||
}
|
||||
}
|
||||
@@ -835,9 +835,6 @@ type answers struct {
|
||||
// public name on the machine went dark. The holds were correct; they were recorded only in the
|
||||
// machine's own state file, and the one visible symptom was a count that did not add up.
|
||||
untaken map[string]map[string]int
|
||||
// leftOut is, per machine, every module of its set its composition leaves out, and why (novox/hq issue
|
||||
// 380): assigned and not applied, which every push said only in passing. Not well while there is any.
|
||||
leftOut map[string][]leftOutModule
|
||||
// unheld is every module on a machine whose resources are applied through a seat nothing on
|
||||
// that machine holds (novox/hq ADR 0207), with the modules that could hold it. Reported, not
|
||||
// refused, until the switch — and while there is any, the mesh is not all well: the order the
|
||||
|
||||
@@ -179,6 +179,14 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
}
|
||||
fmt.Fprintf(out, ", keeps state %s", strings.Join(kept, ", "))
|
||||
}
|
||||
// And the logs it keeps, with their caps (novox/hq ADR 0297).
|
||||
if len(m.Logs) > 0 {
|
||||
kept := make([]string, 0, len(m.Logs))
|
||||
for _, l := range m.Logs {
|
||||
kept = append(kept, fmt.Sprintf("%s (%d MiB)", l.Name, l.Cap()))
|
||||
}
|
||||
fmt.Fprintf(out, ", keeps log %s", strings.Join(kept, ", "))
|
||||
}
|
||||
if len(m.Reads) > 0 {
|
||||
fmt.Fprintf(out, ", reads %s", strings.Join(m.Reads, ", "))
|
||||
}
|
||||
|
||||
@@ -74,12 +74,8 @@ type probe struct {
|
||||
// probeRegistry is the registry, in to-be 45's order. **The registry is the design's live form**: a
|
||||
// probe added to a design is a row added here.
|
||||
var probeRegistry = []probe{
|
||||
// D1 also says every module assigned and left out of a machine's declaration (novox/hq issue 380), from the
|
||||
// resolution it already makes: needs-operator for a setting nobody gave, left-out for any other cause.
|
||||
{ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation; no module " +
|
||||
"assigned to a machine is left out of its declaration unsaid",
|
||||
From: "issues 236, 263, 275, 380", Kind: "declaration-refused",
|
||||
Raises: []string{kindAwaitingPush, kindLeftOut, kindNeedsOperator}, Phase: 1,
|
||||
{ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation",
|
||||
From: "issues 236, 263, 275", Kind: "declaration-refused", Raises: []string{kindAwaitingPush}, Phase: 1,
|
||||
run: probeDeclarations},
|
||||
{ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6",
|
||||
From: "issue 262", Kind: "resolver-wrong", Phase: 1, run: probeResolvers},
|
||||
@@ -140,6 +136,12 @@ var probeRegistry = []probe{
|
||||
{ID: "D-root", Asserts: "no agent can become root without a person on a machine where the router or a channel " +
|
||||
"proving its sender runs: not by its own account, and not through a tool that runs its command as an account " +
|
||||
"that can", From: "ADR 0259 §8", Kind: kindRootNotFree, Phase: 2, run: probeAgentRoot},
|
||||
// A module's bucket or log filling toward its cap (novox/hq ADR 0297 §6): said at three quarters, cleared
|
||||
// below seven tenths, so one at its threshold is not raised and cleared on every run. One that cannot be
|
||||
// read is said on its own, and every other is still judged.
|
||||
{ID: probeFillID, Asserts: "every module's bucket and log holds less than three quarters of its cap; one " +
|
||||
"said filling is cleared once it holds less than seven tenths", From: "ADR 0297, issue 501",
|
||||
Kind: kindBucketOrLogFilling, Raises: []string{kindBucketOrLogUnread}, Phase: 1, run: probeFill},
|
||||
{ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals",
|
||||
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs},
|
||||
// The core's health definitions (novox/hq to-be 45 §8, ADR 0236): what a core component's new build is
|
||||
|
||||
@@ -1,222 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// A module assigned to a machine and left out of its declaration is said (novox/hq issue 380).
|
||||
//
|
||||
// **An omission is a finding, never a refusal of the push** (ADR 0163, rule 6): the machine is sent everything
|
||||
// else, and the module's held things are kept. Until issue 380 the only place it showed was `plan`: nfs-server was
|
||||
// assigned to the home server for days, every send left it out for a setting nobody gave, and the operator believed
|
||||
// it ran. So the self-check (D1) judges every machine as the next send would (Resolution.LeftOutBecause, the send's
|
||||
// own judgement) and raises a condition on that machine for each module it leaves out:
|
||||
//
|
||||
// - a setting nobody gave (catalogue.UnsetSettingError) is the operator's to give: kind needs-operator, naming
|
||||
// the module, the setting and the command that sets it;
|
||||
// - any other cause is said as a module not working is: kind left-out, a warning with the reason as evidence.
|
||||
//
|
||||
// Never urgent and never escalated by age (as ADR 0283 decision 5): nothing the machine ran was undone. It clears
|
||||
// on the first run that no longer finds it — the module composed again, or no longer assigned. `status` and
|
||||
// `node show` list the same modules under "assigned, not applied" with the same reason.
|
||||
//
|
||||
// A module left out because its stored manifest has a key this controller does not know is not raised here: the
|
||||
// catalogue's own unknown-field condition says it once for the whole mesh (ADR 0262); it is still listed.
|
||||
|
||||
const (
|
||||
// probeLeftOutID is the self-check's probe that raises and clears these conditions: D1, which already
|
||||
// resolves every machine (probeDeclarations), so the judgement costs no resolution of its own.
|
||||
probeLeftOutID = "D1"
|
||||
// kindLeftOut is a module left out for any cause but a setting nobody gave; it is also every such condition's
|
||||
// token, whichever its kind, so a cause that changes is the same condition said anew.
|
||||
kindLeftOut = "left-out"
|
||||
)
|
||||
|
||||
// leftOutModule is one module of a machine's set that its declaration leaves out, and why.
|
||||
type leftOutModule struct {
|
||||
Module string
|
||||
// Setting is the setting nobody gave, when that is the cause.
|
||||
Setting string
|
||||
// Why is the declaration's own reason, whole.
|
||||
Why string
|
||||
// Unread is a stored manifest this controller cannot read whole (said by the catalogue's condition).
|
||||
Unread bool
|
||||
}
|
||||
|
||||
// leftOutOf is every module of a machine's resolution that a push would leave out, sorted. Pure: the judgement
|
||||
// the push makes (catalogue.Resolution.LeftOutBecause), nothing allocated.
|
||||
func leftOutOf(plan catalogue.Resolution, settings catalogue.SettingsBy, adopted bool) []leftOutModule {
|
||||
because := plan.LeftOutBecause(settings, adopted)
|
||||
out := make([]leftOutModule, 0, len(because))
|
||||
for module, why := range because {
|
||||
l := leftOutModule{Module: module, Why: oneLine(why.Error())}
|
||||
var unset *catalogue.UnsetSettingError
|
||||
var unread *catalogue.UnreadManifestError
|
||||
switch {
|
||||
case errors.As(why, &unset):
|
||||
l.Setting = unset.Setting
|
||||
case errors.As(why, &unread):
|
||||
l.Unread = true
|
||||
}
|
||||
out = append(out, l)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module })
|
||||
return out
|
||||
}
|
||||
|
||||
// settingCommand is the command that gives a module's setting on one machine.
|
||||
func settingCommand(module, setting, node string) string {
|
||||
return fmt.Sprintf("`settings set %s '{%q: …}' --node %s`", module, setting, node)
|
||||
}
|
||||
|
||||
// reason is why a module is left out, as `status`, `node show` and the condition's summary say it: for a setting
|
||||
// nobody gave, the setting and the command that gives it; otherwise the declaration's own words.
|
||||
func (l leftOutModule) reason(node string) string {
|
||||
if l.Setting != "" {
|
||||
return fmt.Sprintf("nothing sets its setting %q, so every send leaves it out of its declaration — %s sets it", l.Setting,
|
||||
settingCommand(l.Module, l.Setting, node))
|
||||
}
|
||||
return "every send leaves it out of its declaration: " + l.Why
|
||||
}
|
||||
|
||||
// leftOutObservations are the conditions a machine's left-out modules raise, one each.
|
||||
func leftOutObservations(node string, left []leftOutModule) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, l := range left {
|
||||
if l.Unread {
|
||||
continue
|
||||
}
|
||||
out = append(out, leftOutObservation(node, l))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// leftOutObservation is one module left out of one machine's declaration: needs-operator for a setting nobody
|
||||
// gave, left-out otherwise; a warning either way, the operator's to resolve.
|
||||
func leftOutObservation(node string, l leftOutModule) conditions.Observation {
|
||||
o := conditions.Observation{Scope: conditions.ScopeModule, ID: l.Module + "." + node, Token: kindLeftOut,
|
||||
Kind: kindLeftOut, Machine: node, Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s is assigned to %s and not applied: %s", l.Module, node, l.reason(node)),
|
||||
Said: l.Why}
|
||||
w := leftOutWords(l.Module, node, l.Setting)
|
||||
if l.Setting != "" {
|
||||
o.Kind = kindNeedsOperator
|
||||
} else {
|
||||
// The words of why may name a path or an address, which the operator's channel withholds: the
|
||||
// summary sends them to the evidence, and `plan` says them in full.
|
||||
o.Summary = fmt.Sprintf("%s is assigned to %s and not applied: every send leaves it out of its declaration, "+
|
||||
"because what is set for it does not fit its manifest — the evidence and `plan %s` say why", l.Module, node, node)
|
||||
}
|
||||
o.Headline, o.Explanation, o.Needs, o.Resolved = w.Headline, w.Explanation, w.Needs, w.Resolved
|
||||
return o
|
||||
}
|
||||
|
||||
// leftOutWords is what the operator reads of a module left out (ADR 0253): plain, the act named.
|
||||
func leftOutWords(module, node, setting string) words {
|
||||
w := words{
|
||||
Headline: fmt.Sprintf("%s is not applied on %s", module, node),
|
||||
Explanation: fmt.Sprintf("%s is assigned to %s, and every send to %s leaves it out of the declaration because what "+
|
||||
"is set for it does not fit its manifest. Nothing of it changes there; the rest of %s is sent as usual.",
|
||||
module, node, node, node),
|
||||
Needs: fmt.Sprintf("read why in the details, then change what is set for %s or unassign it.", module),
|
||||
Resolved: fmt.Sprintf("%s on %s is no longer left out", module, node),
|
||||
}
|
||||
if setting != "" {
|
||||
w.Explanation = fmt.Sprintf("%s is assigned to %s, and every send to %s leaves it out of the declaration because "+
|
||||
"nothing sets its setting %s. Nothing of it runs there until it is set; the rest of %s is sent as usual.",
|
||||
module, node, node, setting, node)
|
||||
w.Needs = fmt.Sprintf("set %s for %s on %s, or approve it when it is proposed to you.", setting, module, node)
|
||||
// A setting's name that is not plain (a dotted key) is in the summary instead.
|
||||
if _, ok := conditions.PlainWords(w, node); !ok {
|
||||
w.Explanation = fmt.Sprintf("%s is assigned to %s, and every send to %s leaves it out of the declaration because a "+
|
||||
"setting it needs is not set. Nothing of it runs there until it is set; the details name it.",
|
||||
module, node, node)
|
||||
w.Needs = fmt.Sprintf("set what %s needs on %s; the details name the setting.", module, node)
|
||||
}
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// notAppliedLines is a machine's "assigned, not applied" as `node show` prints it: one line a module, with the
|
||||
// reason its condition says.
|
||||
func notAppliedLines(node string, left []leftOutModule) []string {
|
||||
if len(left) == 0 {
|
||||
return nil
|
||||
}
|
||||
lines := []string{"", " assigned, not applied:"}
|
||||
for _, l := range left {
|
||||
lines = append(lines, fmt.Sprintf(" %-22s %s", l.Module, l.reason(node)))
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
// machineNotApplied is one module assigned to a machine and left out of its declaration, in `status --json`.
|
||||
type machineNotApplied struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Setting string `json:"setting,omitempty"`
|
||||
Reason string `json:"reason"`
|
||||
// UnreadManifest is a module left out because this controller cannot read its manifest whole: it raises no
|
||||
// condition of its own on the machine, since the catalogue's unknown-field condition says it once (ADR 0262).
|
||||
UnreadManifest bool `json:"unread-manifest,omitempty"`
|
||||
}
|
||||
|
||||
// notApplied is every machine's left-out modules, in a stated order, as `status --json` carries them.
|
||||
func notApplied(left map[string][]leftOutModule) []machineNotApplied {
|
||||
names := make([]string, 0, len(left))
|
||||
for name := range left {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
var out []machineNotApplied
|
||||
for _, name := range names {
|
||||
for _, l := range left[name] {
|
||||
out = append(out, machineNotApplied{Node: name, Module: l.Module, Setting: l.Setting, Reason: l.reason(name),
|
||||
UnreadManifest: l.Unread})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// printNotApplied is status's "assigned, not applied", every machine's.
|
||||
func printNotApplied(left map[string][]leftOutModule) {
|
||||
rows := notApplied(left)
|
||||
if len(rows) == 0 {
|
||||
return
|
||||
}
|
||||
unread := 0
|
||||
for _, r := range rows {
|
||||
if r.UnreadManifest {
|
||||
unread++
|
||||
}
|
||||
}
|
||||
// Which raise a condition is said, not implied (review of #223): one whose manifest this controller cannot
|
||||
// read is listed here and raises none of its own on the machine — the catalogue's condition says it.
|
||||
raises := "each raises a condition on its machine"
|
||||
switch {
|
||||
case unread == len(rows):
|
||||
raises = "none raises a condition on its machine: this controller cannot read their manifests, which the " +
|
||||
"catalogue's own condition says"
|
||||
case unread > 0:
|
||||
raises += fmt.Sprintf(", except the %d whose manifest this controller cannot read, which the catalogue's own "+
|
||||
"condition says", unread)
|
||||
}
|
||||
fmt.Printf("%d module(s) assigned, not applied — every send leaves them out of their declaration; %s:\n",
|
||||
len(rows), raises)
|
||||
for _, r := range rows {
|
||||
fmt.Printf(" %-12s %-22s %s\n", r.Node, r.Module, r.Reason)
|
||||
}
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
// isLeftOutCondition is whether a condition is a module left out of its declaration, which the machine's health
|
||||
// statements neither raise nor clear: by its token, which every one carries whatever its kind.
|
||||
func isLeftOutCondition(c conditions.Condition) bool {
|
||||
return c.Subject.Scope == conditions.ScopeModule && strings.HasSuffix(c.Key, "."+kindLeftOut)
|
||||
}
|
||||
@@ -1,259 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// novox/hq issue 380: nfs-server was assigned to the home server for days and every send left it out — "nfs-server
|
||||
// has a file that says ${setting:shares}, and nothing sets shares for it" — and nothing but `plan` said so. The
|
||||
// manifest is the catalogue's own at the commit that added it (mesh-catalog 48fba44), which still says
|
||||
// ${setting:shares}; the reason below is the one the live push printed.
|
||||
|
||||
// leftOutNFS is the resolution of a machine assigned that nfs-server, with the settings given.
|
||||
func leftOutNFS(t *testing.T) catalogue.Resolution {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile("testdata/left-out/nfs-server.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return catalogue.Resolution{Modules: []catalogue.Manifest{m}}
|
||||
}
|
||||
|
||||
// sharesGiven is the operator's setting for it on the machine.
|
||||
func sharesGiven(value string) catalogue.SettingsBy {
|
||||
return catalogue.SettingsBy{"nfs-server": {{From: "anchor", Values: map[string]any{"shares": value}}}}
|
||||
}
|
||||
|
||||
func TestAModuleLeftOutForASettingNobodyGaveNeedsTheOperatorNamingTheSettingAndTheCommand(t *testing.T) {
|
||||
left := leftOutOf(leftOutNFS(t), nil, false)
|
||||
if len(left) != 1 || left[0].Module != "nfs-server" || left[0].Setting != "shares" {
|
||||
t.Fatalf("left out: %+v", left)
|
||||
}
|
||||
if !strings.Contains(left[0].Why, `nothing sets "shares" for it`) {
|
||||
t.Fatalf("the reason is not the push's own: %s", left[0].Why)
|
||||
}
|
||||
obs := leftOutObservations("anchor", left)
|
||||
if len(obs) != 1 {
|
||||
t.Fatalf("raised %+v", obs)
|
||||
}
|
||||
o := obs[0]
|
||||
if o.Key() != "module.nfs-server.anchor.left-out" || o.Kind != kindNeedsOperator || o.Machine != "anchor" ||
|
||||
o.Severity != conditions.Warning || o.Resolver != conditions.ResolverOperator {
|
||||
t.Fatalf("raised %s as %s, %s, by %s, on %q", o.Key(), o.Kind, o.Severity, o.Resolver, o.Machine)
|
||||
}
|
||||
for _, want := range []string{"nfs-server", "anchor", `"shares"`, "`settings set nfs-server '{\"shares\": …}' --node anchor`"} {
|
||||
if !strings.Contains(o.Summary, want) {
|
||||
t.Errorf("the summary does not name %s: %s", want, o.Summary)
|
||||
}
|
||||
}
|
||||
if o.Said != left[0].Why {
|
||||
t.Errorf("the evidence is not the reason the send gives: %s", o.Said)
|
||||
}
|
||||
plainExample(t, o, "nfs-server is not applied on anchor",
|
||||
"Needs you: set shares for nfs-server on anchor, or approve it when it is proposed to you. nfs-server is assigned to "+
|
||||
"anchor, and every send to anchor leaves it out of the declaration because nothing sets its setting shares. "+
|
||||
"Nothing of it runs there until it is set; the rest of anchor is sent as usual.")
|
||||
}
|
||||
|
||||
func TestAModuleLeftOutForAnotherCauseIsAWarningWithTheReasonAsEvidence(t *testing.T) {
|
||||
// A setting stored that its manifest can no longer take: one with a line break (issue 339).
|
||||
left := leftOutOf(leftOutNFS(t), sharesGiven("library=/srv/library\nmedia=/srv/media"), false)
|
||||
if len(left) != 1 || left[0].Setting != "" {
|
||||
t.Fatalf("left out: %+v", left)
|
||||
}
|
||||
obs := leftOutObservations("anchor", left)
|
||||
if len(obs) != 1 {
|
||||
t.Fatalf("raised %+v", obs)
|
||||
}
|
||||
o := obs[0]
|
||||
if o.Key() != "module.nfs-server.anchor.left-out" || o.Kind != kindLeftOut || o.Severity != conditions.Warning {
|
||||
t.Fatalf("raised %s as %s, %s", o.Key(), o.Kind, o.Severity)
|
||||
}
|
||||
if !strings.Contains(o.Said, "holds a line break") || strings.Contains(o.Summary, "/srv/") {
|
||||
t.Fatalf("the reason is not the evidence, or the summary carries it to the channel:\n%s\n%s", o.Summary, o.Said)
|
||||
}
|
||||
plainExample(t, o, "nfs-server is not applied on anchor",
|
||||
"Needs you: read why in the details, then change what is set for nfs-server or unassign it. nfs-server is assigned to "+
|
||||
"anchor, and every send to anchor leaves it out of the declaration because what is set for it does not fit "+
|
||||
"its manifest. Nothing of it changes there; the rest of anchor is sent as usual.")
|
||||
}
|
||||
|
||||
// The self-check raises it, keeps it a warning however long it stands, and clears it when the module composes
|
||||
// again or is no longer assigned; a machine's health statement neither clears nor raises it.
|
||||
func TestALeftOutModulesConditionClearsWhenItComposesAgainOrIsUnassigned(t *testing.T) {
|
||||
plan, settings := leftOutNFS(t), catalogue.SettingsBy(nil)
|
||||
withProbes(t, probe{ID: probeLeftOutID, Asserts: "the test's", Kind: kindLeftOut, Phase: 1,
|
||||
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
|
||||
return leftOutObservations("anchor", leftOutOf(plan, settings, false)), nil
|
||||
}})
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||
defer k.Close(context.Background())
|
||||
d := &doctor{keeper: k, teller: &conditions.Told{}, host: "anchor"}
|
||||
key := "module.nfs-server.anchor.left-out"
|
||||
openOnes := func() map[string]conditions.Condition {
|
||||
t.Helper()
|
||||
open, err := k.Open(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := map[string]conditions.Condition{}
|
||||
for _, c := range open {
|
||||
out[c.Key] = c
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
d.runOnce(t.Context(), "a test")
|
||||
c, raised := openOnes()[key]
|
||||
if !raised || c.Kind != kindNeedsOperator || c.Severity != conditions.Warning {
|
||||
t.Fatalf("a module left out raised %+v", openOnes())
|
||||
}
|
||||
// A statement from the machine that says nothing of it — the module runs nothing there — leaves it open.
|
||||
if err := judgeModuleHealth(t.Context(), nil, k, "anchor", map[string][]inventory.ResourceHealth{}, nil,
|
||||
time.Now().Add(72*time.Hour)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, still := openOnes()[key]; !still {
|
||||
t.Fatal("a health statement that says nothing of the module cleared its left-out condition")
|
||||
}
|
||||
d.runOnce(t.Context(), "a test")
|
||||
if c := openOnes()[key]; c.Severity != conditions.Warning {
|
||||
t.Fatalf("after a health statement and days, it is %+v", openOnes())
|
||||
}
|
||||
|
||||
// The setting given: it composes, and the condition clears.
|
||||
settings = sharesGiven("library=/srv/library")
|
||||
d.runOnce(t.Context(), "a test")
|
||||
if _, still := openOnes()[key]; still {
|
||||
t.Fatalf("composed again, still open: %+v", openOnes())
|
||||
}
|
||||
|
||||
// Left out again, then unassigned: no longer in the machine's set, and it clears.
|
||||
settings = nil
|
||||
d.runOnce(t.Context(), "a test")
|
||||
if _, raised := openOnes()[key]; !raised {
|
||||
t.Fatal("left out again and not raised")
|
||||
}
|
||||
plan = catalogue.Resolution{}
|
||||
d.runOnce(t.Context(), "a test")
|
||||
if _, still := openOnes()[key]; still {
|
||||
t.Fatalf("unassigned, still open: %+v", openOnes())
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheLeftOutProbeIsInTheRegistry(t *testing.T) {
|
||||
for _, p := range probeRegistry {
|
||||
if p.ID == probeLeftOutID {
|
||||
if p.run == nil || !slices.Contains(p.Raises, kindLeftOut) || !slices.Contains(p.Raises, kindNeedsOperator) {
|
||||
t.Fatalf("%+v", p)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("no probe %s: a module left out is said nowhere", probeLeftOutID)
|
||||
}
|
||||
|
||||
// status and node show list it under "assigned, not applied" with the reason the condition says, and status is
|
||||
// not well while there is one.
|
||||
func TestStatusAndNodeListAModuleAssignedAndNotApplied(t *testing.T) {
|
||||
left := map[string][]leftOutModule{"anchor": leftOutOf(leftOutNFS(t), nil, false)}
|
||||
reason := leftOutObservations("anchor", left["anchor"])[0].Summary
|
||||
asked := answers{leftOut: left}
|
||||
if asked.well() {
|
||||
t.Fatal("a mesh with a module assigned and not applied is called well")
|
||||
}
|
||||
shown := printed(t, func() error { return printStatus(asked) })
|
||||
if !strings.Contains(shown, "1 module(s) assigned, not applied") || !strings.Contains(shown, "nfs-server") ||
|
||||
!strings.Contains(shown, "`settings set nfs-server '{\"shares\": …}' --node anchor` sets it") {
|
||||
t.Fatalf("status says:\n%s", shown)
|
||||
}
|
||||
if !strings.Contains(reason, left["anchor"][0].reason("anchor")) {
|
||||
t.Fatalf("status and the condition say different reasons:\n%s\n%s", shown, reason)
|
||||
}
|
||||
body, err := statusAsJSON(asked)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var doc struct {
|
||||
NotApplied []machineNotApplied `json:"not-applied"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &doc); err != nil || len(doc.NotApplied) != 1 ||
|
||||
doc.NotApplied[0].Setting != "shares" || doc.NotApplied[0].Node != "anchor" {
|
||||
t.Fatalf("status --json: %v %s", err, body)
|
||||
}
|
||||
lines := strings.Join(notAppliedLines("anchor", left["anchor"]), "\n")
|
||||
if !strings.Contains(lines, "assigned, not applied:") || !strings.Contains(lines, "nfs-server") ||
|
||||
!strings.Contains(lines, `nothing sets its setting "shares"`) {
|
||||
t.Fatalf("node show says:\n%s", lines)
|
||||
}
|
||||
}
|
||||
|
||||
// The skip is this probe's alone (review of #223): a part waiting for the operator (ADR 0283) keeps its own
|
||||
// needs-operator condition, `module.<m>.<node>.needs-operator`, which still clears on the first statement that no
|
||||
// longer names it — beside a left-out condition on the same machine, which stays.
|
||||
func TestAWaitsNeedsOperatorStillClearsWhenItsStatementStopsNamingItBesideALeftOutOne(t *testing.T) {
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||
defer k.Close(context.Background())
|
||||
left := leftOutObservations("anchor", leftOutOf(leftOutNFS(t), nil, false))
|
||||
if err := k.Reconcile(t.Context(), probeLeftOutID, left); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waiting := map[string][]inventory.ResourceHealth{"notes": {{Module: "notes", Resource: "server", State: link.StateWaiting,
|
||||
Waits: []inventory.Wait{{Setting: "domain", What: "the domain it serves"}}}}}
|
||||
if err := judgeModuleHealth(t.Context(), nil, k, "anchor", waiting, map[string]int{"notes": 2}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitKey, leftKey := needsOperatorKey("notes", "anchor"), "module.nfs-server.anchor.left-out"
|
||||
open := func() map[string]conditions.Condition {
|
||||
t.Helper()
|
||||
list, err := k.Open(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := map[string]conditions.Condition{}
|
||||
for _, c := range list {
|
||||
out[c.Key] = c
|
||||
}
|
||||
return out
|
||||
}
|
||||
if c, raised := open()[waitKey]; !raised || c.Kind != kindNeedsOperator {
|
||||
t.Fatalf("the wait raised %+v", open())
|
||||
}
|
||||
if c := open()[leftKey]; c.Kind != kindNeedsOperator {
|
||||
t.Fatalf("the left-out condition is not open as needs-operator: %+v", open())
|
||||
}
|
||||
// D1 runs again and still finds nfs-server left out: its reconcile clears only what D1 raised, never the wait,
|
||||
// which the machine's statement raised.
|
||||
if err := k.Reconcile(t.Context(), probeLeftOutID, left); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, still := open()[waitKey]; !still {
|
||||
t.Fatalf("D1's reconcile cleared the wait's needs-operator: %+v", open())
|
||||
}
|
||||
if err := judgeModuleHealth(t.Context(), nil, k, "anchor", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, still := open()[waitKey]; still {
|
||||
t.Fatalf("the statement no longer names the wait, and its needs-operator is still open: %+v", open())
|
||||
}
|
||||
if _, still := open()[leftKey]; !still {
|
||||
t.Fatalf("the left-out condition was cleared by a health statement: %+v", open())
|
||||
}
|
||||
}
|
||||
@@ -147,11 +147,9 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
}
|
||||
standing := map[string]conditions.Condition{}
|
||||
for _, c := range open {
|
||||
// A module left out of the composition is the self-check's to raise and clear, never a statement's
|
||||
// (novox/hq issue 380): its needs-operator is not cleared for not being in what the machine runs.
|
||||
if (c.Kind == kindModuleUnhealthy || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound ||
|
||||
c.Kind == kindNeedsOperator) &&
|
||||
c.Subject.Machine == node && !isLeftOutCondition(c) {
|
||||
c.Subject.Machine == node {
|
||||
standing[c.Key] = c
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,7 +44,7 @@ func nodeCommand(ctx context.Context, args []string) error {
|
||||
if len(args) != 2 {
|
||||
return errors.New("node show <name>")
|
||||
}
|
||||
return showNode(ctx, open, args[1])
|
||||
return showNode(ctx, inv, args[1])
|
||||
case "add":
|
||||
return addNode(ctx, inv, args[1:])
|
||||
|
||||
@@ -787,8 +787,7 @@ func roughly(d time.Duration) string {
|
||||
//
|
||||
// It is also where "what should it be configured as" is read. The same line that gates an
|
||||
// assignment carries `card1-DP-1`, and a person composing settings for that machine needs it.
|
||||
func showNode(ctx context.Context, stored *stores, name string) error {
|
||||
inv := stored.inventory
|
||||
func showNode(ctx context.Context, inv *inventory.Inventory, name string) error {
|
||||
node, err := inv.NodeByName(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -890,17 +889,6 @@ func showNode(ctx context.Context, stored *stores, name string) error {
|
||||
if len(assigned) > 0 {
|
||||
fmt.Printf("\n assigned: %s\n", strings.Join(assigned, ", "))
|
||||
}
|
||||
// And which of them a push leaves out, and why (novox/hq issue 380): judged as the push judges it, the same
|
||||
// reason its condition says. Not computable is said, never read as "all applied".
|
||||
plan, settings, err := planFor(ctx, stored, name)
|
||||
switch {
|
||||
case err != nil:
|
||||
fmt.Printf("\n whether a send leaves any assigned module out is NOT known: %s\n", oneLine(err.Error()))
|
||||
default:
|
||||
for _, line := range notAppliedLines(name, leftOutOf(plan, settings, node.Adopted)) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -231,14 +231,6 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
||||
w := needsOperatorWords(orModule(module), node, nil)
|
||||
return w
|
||||
}),
|
||||
kindLeftOut: worded(func(o conditions.Observation) words {
|
||||
// The observation carries its own words (novox/hq issue 380); these are its kind's alone.
|
||||
module := ""
|
||||
if o.Scope == conditions.ScopeModule && o.Machine != "" {
|
||||
module = strings.TrimSuffix(o.ID, "."+o.Machine)
|
||||
}
|
||||
return leftOutWords(orModule(module), machineOr(o, "a machine"), "")
|
||||
}),
|
||||
kindProviderFailing: worded(func(o conditions.Observation) words {
|
||||
thing, consumer := conditions.ThingWords(o), idPart(o, 2)
|
||||
if consumer == "" {
|
||||
@@ -614,6 +606,19 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
||||
Explanation: "The bus refused messages from a part of the mesh, so what they carried did not happen.",
|
||||
Resolved: "Resolved: the bus takes the messages again"}
|
||||
}),
|
||||
kindBucketOrLogFilling: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A module's bucket or log on the bus is filling up",
|
||||
Needs: "decide whether to raise its cap or have the module keep less.",
|
||||
Explanation: "A bucket or log a module keeps on the bus holds three quarters of its cap or more. When it " +
|
||||
"is full, the bus refuses what the module writes there.",
|
||||
Resolved: "It has room again"}
|
||||
}),
|
||||
kindBucketOrLogUnread: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A module's bucket or log could not be read",
|
||||
Explanation: "The controller could not read how full a bucket or log a module keeps on the bus is, so it " +
|
||||
"cannot say whether it is filling up. It asks again at its next check.",
|
||||
Resolved: "It can be read again"}
|
||||
}),
|
||||
"stream-wrong": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Part of the bus's storage is wrong",
|
||||
Needs: "check the machine the bus runs on; the details say what is missing.",
|
||||
|
||||
@@ -78,11 +78,6 @@ func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation
|
||||
return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err)
|
||||
}
|
||||
var problems, foreseen []string
|
||||
// Each module the next send leaves out of this machine's declaration (novox/hq issue 380), judged from this
|
||||
// resolution as the send judges it: a finding, never a refusal.
|
||||
if err == nil {
|
||||
out = append(out, leftOutObservations(n.Name, leftOutOf(plan, settings, n.Adopted))...)
|
||||
}
|
||||
if err == nil && gensErr == nil {
|
||||
var declared sendable
|
||||
if declared, err = declarationWith(ctx, open, n.Name, plan, settings, gens, Foreseeing); err == nil {
|
||||
|
||||
@@ -1300,6 +1300,15 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, se
|
||||
if _, err := broker.RaiseBuckets(broker.OnConn(bus.Conn), buckets); err != nil {
|
||||
return fmt.Errorf("the modules' state could not be asserted on the bus: %w", err)
|
||||
}
|
||||
// **And every declared log, for the same reason** (novox/hq ADR 0297 §2): a membership names its
|
||||
// logs, and a module whose log does not exist fails its first append.
|
||||
logs, err := open.inventory.DeclaredLogs(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the modules' logs could not be read, so no log was asserted: %w", err)
|
||||
}
|
||||
if _, err := broker.RaiseLogs(broker.OnConn(bus.Conn), logs); err != nil {
|
||||
return fmt.Errorf("the modules' logs could not be asserted on the bus: %w", err)
|
||||
}
|
||||
// Every membership is tried, and the first failure named once.
|
||||
issued := 0
|
||||
refused := map[string]error{}
|
||||
@@ -1483,13 +1492,28 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string)
|
||||
fmt.Printf("the bus holds state nothing declares any more, kept because it is data: %s — "+
|
||||
"removing it is a person's act\n", strings.Join(undeclared, ", "))
|
||||
}
|
||||
// Every module's log (novox/hq ADR 0297), from the catalogue, as its buckets: one that nothing
|
||||
// declares any more is said and kept — a log is a module's record, and no path of the mesh removes it.
|
||||
logs, err := inv.DeclaredLogs(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
unlogged, err := broker.RaiseLogs(js, logs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(unlogged) > 0 {
|
||||
fmt.Printf("the bus holds logs nothing declares any more, kept because they are data: %s — "+
|
||||
"removing one is a person's act\n", strings.Join(unlogged, ", "))
|
||||
}
|
||||
// And how every module hears what it consumes: asserted with the rest above, counted here.
|
||||
hearing, err := moduleConsumerCount(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("the bus at %s has its streams, %d machine(s) can hear a declaration, %d module(s) "+
|
||||
"can hear what they consume, and %d bucket(s) of state\n", broker.BareAddress(address), len(names), hearing, len(buckets))
|
||||
"can hear what they consume, %d bucket(s) of state and %d log(s)\n", broker.BareAddress(address), len(names), hearing,
|
||||
len(buckets), len(logs))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -69,9 +69,6 @@ type meshStatus struct {
|
||||
// **A document without this said an outage was a well mesh.** Read from what each machine
|
||||
// reported, so it is the machine's account and not the mesh's take-time listing.
|
||||
Untaken []machineUntaken `json:"untaken,omitempty"`
|
||||
// NotApplied is every module assigned to a machine and left out of its composition, with why (novox/hq
|
||||
// issue 380). Absent when every module composes.
|
||||
NotApplied []machineNotApplied `json:"not-applied,omitempty"`
|
||||
// Filtered is every converged machine that is not filtered by the mesh alone (novox/hq ADR
|
||||
// 0168), one entry per rule set the mesh did not write — the found firewall in force again,
|
||||
// or a chain nobody speaks for. Absent when every converged machine is filtered by the mesh
|
||||
@@ -254,7 +251,6 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
||||
}
|
||||
}
|
||||
out.Unheld = asked.unheld
|
||||
out.NotApplied = notApplied(asked.leftOut)
|
||||
out.HandActsThisWeek, out.HandActsUnread = asked.handActs, asked.handActsUnread
|
||||
out.HealsThisWeek, out.HealsUnread = asked.heals, asked.healsUnread
|
||||
// In brief, as `conditions` lists them: status leads with every open condition, and their whole
|
||||
|
||||
@@ -66,7 +66,7 @@ func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
|
||||
}
|
||||
// Both machines' `node show` name it: where the provider runs, and where the consumer is.
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
shown := printed(t, func() error { return showNode(ctx, open, node) })
|
||||
shown := printed(t, func() error { return showNode(ctx, open.inventory, node) })
|
||||
if !strings.Contains(shown, "open condition(s) about this machine") || !strings.Contains(shown, "mesh_laptop_dashboard") {
|
||||
t.Fatalf("node show %s does not name it:\n%s", node, shown)
|
||||
}
|
||||
|
||||
@@ -289,10 +289,6 @@ func printStatus(asked answers) error {
|
||||
fmt.Printf("\n `take <node> <module>` compares what runs against what it declares, and runs it\n\n")
|
||||
}
|
||||
|
||||
// Assigned and not applied (novox/hq issue 380): before what is merely reported, because it reads like work
|
||||
// finished and is none.
|
||||
printNotApplied(asked.leftOut)
|
||||
|
||||
if len(asked.unheld) > 0 {
|
||||
// **Reported, and not refused yet** (novox/hq ADR 0207 §4). Each machine still resolves and
|
||||
// is sent what it would be; this says which of its modules depend on a seat nothing there
|
||||
@@ -460,13 +456,6 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
||||
return answers{}, err
|
||||
}
|
||||
plans[n.Name] = planned{plan, settings}
|
||||
// And which of its modules a push leaves out (novox/hq issue 380), judged as the push judges it.
|
||||
if left := leftOutOf(plan, settings, n.Adopted); len(left) > 0 {
|
||||
if out.leftOut == nil {
|
||||
out.leftOut = map[string][]leftOutModule{}
|
||||
}
|
||||
out.leftOut[n.Name] = left
|
||||
}
|
||||
out.unheld = append(out.unheld, plan.Unheld...)
|
||||
// And which of its modules a provider leaves out of its grants, for an identity too long
|
||||
// for what the provision keeps (novox/hq ADR 0225) — judged from the consumer's own
|
||||
@@ -615,7 +604,7 @@ func untakenModules(ctx context.Context, inv *inventory.Inventory, nodes []inven
|
||||
// read as success for the whole of the edge cut-over outage (novox/hq 04-ISSUES/125).
|
||||
func (a answers) well() bool {
|
||||
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
|
||||
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 && len(a.leftOut) == 0 &&
|
||||
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 &&
|
||||
len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.overflowing) == 0 &&
|
||||
len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingOpen(a.pending)
|
||||
}
|
||||
|
||||
@@ -1,142 +0,0 @@
|
||||
{
|
||||
"module": "nfs-server",
|
||||
"version": "1",
|
||||
"upgrade": {
|
||||
"policy": "record",
|
||||
"why": "the folders other machines mount: a build that breaks the exports leaves every client's mount hanging or refused, and the gate on this one machine does not see the clients (hq ADR 0236, ADR 0263)"
|
||||
},
|
||||
"capabilities": [
|
||||
"package-manager",
|
||||
"service-manager"
|
||||
],
|
||||
"provides": [
|
||||
{
|
||||
"name": "nfs-share",
|
||||
"scope": "mesh",
|
||||
"identity": false
|
||||
}
|
||||
],
|
||||
"data": {
|
||||
"consumers": {
|
||||
"nfs-share": {
|
||||
"class": "none",
|
||||
"why": "the shared folders are the operator's data (hq ADR 0051): what a client writes lands in them, and they are protected where the operator declares them, never by this module, which keeps nothing of a consumer's"
|
||||
}
|
||||
}
|
||||
},
|
||||
"claims": [
|
||||
{
|
||||
"name": "node-nfs-server",
|
||||
"scope": "node",
|
||||
"serves": [
|
||||
"exports",
|
||||
"clients",
|
||||
"test",
|
||||
"reload",
|
||||
"adopt"
|
||||
]
|
||||
}
|
||||
],
|
||||
"state": [
|
||||
{
|
||||
"name": "exports",
|
||||
"ttl-seconds": 120,
|
||||
"per-machine": true
|
||||
}
|
||||
],
|
||||
"tools": [
|
||||
"nfs_health"
|
||||
],
|
||||
"listens": [
|
||||
{
|
||||
"name": "nfs",
|
||||
"port": 2049,
|
||||
"protocol": "tcp",
|
||||
"from": "mesh",
|
||||
"fixed": true,
|
||||
"why": "the shares, to the mesh's machines only (hq ADR 0263): NFS version 4 alone, which needs no other port, and never the home network, where a device that is not a node could claim any user id"
|
||||
}
|
||||
],
|
||||
"resources": [
|
||||
{
|
||||
"id": "package",
|
||||
"type": "package",
|
||||
"package": "nfs-utils"
|
||||
},
|
||||
{
|
||||
"id": "nfs-conf",
|
||||
"type": "file",
|
||||
"path": "/etc/nfs.conf.d/50-mesh.conf",
|
||||
"mode": "0644",
|
||||
"content": "# Written by the mesh (module nfs-server, novox/hq ADR 0263). Replaced on every push; a drop-in of\n# the operator's that sorts after this one overrides it, and is theirs.\n#\n# NFS version 4 only: a client needs port 2049 and nothing else, so the module opens nothing more\n# than that, to the private network. Version 3 needs rpcbind and mountd, on ports the mesh does not open.\n[nfsd]\nvers2=n\nvers3=n\nvers4=y\nvers4.0=n\nvers4.1=y\nvers4.2=y\n"
|
||||
},
|
||||
{
|
||||
"id": "config-dir",
|
||||
"type": "directory",
|
||||
"path": "/etc/nfs-server",
|
||||
"mode": "0755"
|
||||
},
|
||||
{
|
||||
"id": "config",
|
||||
"type": "file",
|
||||
"path": "/etc/nfs-server/shares.conf",
|
||||
"mode": "0644",
|
||||
"content": "# Written by the mesh (module nfs-server, novox/hq ADR 0263) from this machine's assignment.\n# Replaced on every push; change the `shares` setting, never this file.\n#\n# The shares: name=folder, or name=folder:ro, one share per folder. The module's process exports each\n# to the private network's range below, every client mapped to the folder's owner.\nshares=${setting:shares}\nrange=${machine:mesh-range}\n"
|
||||
},
|
||||
{
|
||||
"id": "run-dir",
|
||||
"type": "directory",
|
||||
"path": "/run/nfs-server",
|
||||
"mode": "0755"
|
||||
},
|
||||
{
|
||||
"id": "server",
|
||||
"type": "service",
|
||||
"unit": "nfs-server.service",
|
||||
"state": "running",
|
||||
"boot": "enabled",
|
||||
"restart-on": [
|
||||
"nfs-conf"
|
||||
],
|
||||
"health": {
|
||||
"kind": "unit"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "exports",
|
||||
"type": "process",
|
||||
"name": "nfs-server-exports",
|
||||
"artifact": "tools",
|
||||
"run": [
|
||||
"./nfs-server",
|
||||
"exports"
|
||||
],
|
||||
"restart-on": [
|
||||
"config"
|
||||
],
|
||||
"health": {
|
||||
"kind": "tool",
|
||||
"tool": "nfs_health",
|
||||
"interval": "60s",
|
||||
"timeout": "10s",
|
||||
"looks": 2,
|
||||
"grace": "90s"
|
||||
}
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/nfs-server",
|
||||
"binary": "nfs-server",
|
||||
"loads": [
|
||||
"nfs-server"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -448,3 +448,41 @@ func (j *JetStream) BucketNames() ([]string, error) {
|
||||
}
|
||||
return out, lister.Error()
|
||||
}
|
||||
|
||||
// EnsureLog creates a module's log if it is absent and brings its configuration to match if it is
|
||||
// present (novox/hq ADR 0297).
|
||||
//
|
||||
// **An update, never a delete and recreate**, for a bucket's reason: recreating discards what the log
|
||||
// holds, and a log is a module's record. A configuration the server will not change in place (its
|
||||
// storage, say, on a stream made by hand) is said as this assertion's error and the stream is left as
|
||||
// it is — never removed to be made again.
|
||||
func (j *JetStream) EnsureLog(l Log) error {
|
||||
js, err := jetstream.New(j.conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if _, err := js.CreateOrUpdateStream(ctx, l.Config()); err != nil {
|
||||
return fmt.Errorf("asserting log %s: %w", l.Stream(), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// LogStreams is every log stream on the server: every stream whose name starts with LogStreamPrefix.
|
||||
func (j *JetStream) LogStreams() ([]string, error) {
|
||||
js, err := jetstream.New(j.conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
lister := js.StreamNames(ctx)
|
||||
var out []string
|
||||
for name := range lister.Name() {
|
||||
if strings.HasPrefix(name, LogStreamPrefix) {
|
||||
out = append(out, name)
|
||||
}
|
||||
}
|
||||
return out, lister.Err()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// A module's log on the bus (novox/hq ADR 0297): its record of operations, each entry appended under
|
||||
// a key and kept as long as the log is.
|
||||
//
|
||||
// A log follows a bucket in every respect (ADR 0201): the module names it locally, the mesh derives
|
||||
// its stream and subjects, the controller creates it from the catalogue on every raise and from
|
||||
// registration, never a module, and **no path of the mesh removes one** — a log whose declaration
|
||||
// is gone is reported, as a bucket is. Unlike a bucket, a log is its owner's alone: no other module
|
||||
// reads it, so there is no read of a log to grant or issue.
|
||||
//
|
||||
// Pure, but for the stream's configuration, which is the server's own type so that what is tested
|
||||
// is exactly what is sent; jetstream.go is the part that asks a server.
|
||||
|
||||
// The mesh's caps on a log: what an entry's value may weigh, what a message on the log's stream may
|
||||
// weigh — the value and its headers, which the server counts in a message's size, so a value of the
|
||||
// full size still fits with the expected-last-sequence header an append carries — and what a log holds
|
||||
// when its module says nothing and at most.
|
||||
const (
|
||||
LogMaxEntryBytes = 256 * 1024
|
||||
LogMaxMessageBytes = LogMaxEntryBytes + 4*1024
|
||||
LogDefaultMiB = 1024
|
||||
LogMostMiB = 8192
|
||||
)
|
||||
|
||||
// A Log is one module's declared log as the bus holds it.
|
||||
type Log struct {
|
||||
Module string
|
||||
Name string
|
||||
// MaxMiB is its cap in MiB; zero is LogDefaultMiB.
|
||||
MaxMiB int
|
||||
}
|
||||
|
||||
// LogStreamPrefix starts every log's stream name, which no other stream of the mesh's starts with.
|
||||
const LogStreamPrefix = "LOG_"
|
||||
|
||||
// LogStreamName is the stream a module's log lives in: `LOG_<module>_<name>`. The module and the
|
||||
// local name are each one token with no underscore, so two modules can never derive one stream.
|
||||
func LogStreamName(module, name string) string { return LogStreamPrefix + module + "_" + name }
|
||||
|
||||
// LogSubject is the subject a log's entries are published under, without the key: an entry for key K
|
||||
// is on `<LogSubject>.<K>`.
|
||||
func LogSubject(module, name string) string { return "mesh.log." + module + "." + name }
|
||||
|
||||
// Stream is this log's stream name.
|
||||
func (l Log) Stream() string { return LogStreamName(l.Module, l.Name) }
|
||||
|
||||
// Subject is this log's subject, without the key.
|
||||
func (l Log) Subject() string { return LogSubject(l.Module, l.Name) }
|
||||
|
||||
// MaxBytes is this log's cap in bytes.
|
||||
func (l Log) MaxBytes() int64 {
|
||||
mib := l.MaxMiB
|
||||
if mib <= 0 {
|
||||
mib = LogDefaultMiB
|
||||
}
|
||||
return int64(mib) * 1024 * 1024
|
||||
}
|
||||
|
||||
// Why is carried into the server's description of the stream, so somebody reading the server's own
|
||||
// state finds whose it is and why it is kept.
|
||||
func (l Log) Why() string {
|
||||
return fmt.Sprintf("%s's log %q (novox/hq ADR 0297): its record of operations, one entry per operation "+
|
||||
"under its key, appended by %s alone and kept as long as the log; never removed by the mesh, because "+
|
||||
"it is data", l.Module, l.Name, l.Module)
|
||||
}
|
||||
|
||||
// Config is the stream a log is, field by field as novox/hq ADR 0297 fixes it: a file stream kept by
|
||||
// limits, with no maximum age and no cap per key, whose message is an entry's value and 4 KiB of
|
||||
// headers at most, that refuses a new entry when full rather than
|
||||
// drop an old one, and that refuses deleting an entry or purging it. Direct gets are allowed, which
|
||||
// is how the runtime reads it.
|
||||
func (l Log) Config() jetstream.StreamConfig {
|
||||
return jetstream.StreamConfig{
|
||||
Name: l.Stream(),
|
||||
Description: l.Why(),
|
||||
Subjects: []string{l.Subject() + ".>"},
|
||||
Storage: jetstream.FileStorage,
|
||||
Retention: jetstream.LimitsPolicy,
|
||||
MaxAge: 0,
|
||||
MaxMsgs: -1,
|
||||
MaxMsgsPerSubject: -1,
|
||||
MaxBytes: l.MaxBytes(),
|
||||
MaxMsgSize: LogMaxMessageBytes,
|
||||
Discard: jetstream.DiscardNew,
|
||||
AllowDirect: true,
|
||||
DenyDelete: true,
|
||||
DenyPurge: true,
|
||||
Replicas: 1,
|
||||
}
|
||||
}
|
||||
|
||||
// LogIssued is one log an assignment may reach, by the name its module uses for it (novox/hq ADR
|
||||
// 0297): its stream, the subject its entries go under, and whether it may append. Only the owner's
|
||||
// instances are issued a log, so Writes is always true today; it is said so the runtime need not
|
||||
// assume it.
|
||||
type LogIssued struct {
|
||||
Name string `json:"name"`
|
||||
Stream string `json:"stream"`
|
||||
Subject string `json:"subject"`
|
||||
Writes bool `json:"writes"`
|
||||
}
|
||||
|
||||
// logsIssuedFor is every log a module's code may reach, as its membership lists them: its own.
|
||||
func logsIssuedFor(d Declared) []LogIssued {
|
||||
var out []LogIssued
|
||||
for _, l := range d.Logs {
|
||||
if !safeSubject.MatchString(d.Module) || !safeSubject.MatchString(l.Name) {
|
||||
continue
|
||||
}
|
||||
out = append(out, LogIssued{Name: l.Name, Stream: LogStreamName(d.Module, l.Name),
|
||||
Subject: LogSubject(d.Module, l.Name), Writes: true})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// logGrants is what a principal publishes to reach the logs its module keeps: for each, appending
|
||||
// under the log's subjects, binding to its stream, and reading it directly. Nothing more — the
|
||||
// runtime reads a log by direct gets alone and makes no consumer on it — and nothing of any other
|
||||
// module's log, because a log is its owner's alone. Replies come on the principal's inbox, as for a
|
||||
// bucket.
|
||||
func logGrants(module string, names []string) []string {
|
||||
if !safeSubject.MatchString(module) {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, name := range names {
|
||||
if !safeSubject.MatchString(name) {
|
||||
continue
|
||||
}
|
||||
stream := LogStreamName(module, name)
|
||||
out = append(out,
|
||||
LogSubject(module, name)+".>",
|
||||
"$JS.API.STREAM.INFO."+stream,
|
||||
"$JS.API.DIRECT.GET."+stream,
|
||||
"$JS.API.DIRECT.GET."+stream+".>")
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// logNames is the local names of a module's logs.
|
||||
func logNames(logs []Log) []string {
|
||||
out := make([]string, 0, len(logs))
|
||||
for _, l := range logs {
|
||||
out = append(out, l.Name)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// A LogAsserter is the part of a JetStream connection log assertion needs. It has no way to remove a
|
||||
// log, by design: nothing the mesh runs asks for one.
|
||||
type LogAsserter interface {
|
||||
// EnsureLog creates the log's stream if absent and brings its configuration to match if present,
|
||||
// never deleting or recreating it.
|
||||
EnsureLog(l Log) error
|
||||
// LogStreams is every log stream on the server: every stream named with LogStreamPrefix.
|
||||
LogStreams() ([]string, error)
|
||||
}
|
||||
|
||||
// RaiseLogs asserts every declared log and answers the log streams on the server that nothing
|
||||
// declares any more.
|
||||
//
|
||||
// **Those are reported, never removed** (novox/hq ADR 0297 §2, ADR 0201 §15–16): a log is a module's
|
||||
// record, and a manifest edited, a module renamed or unassigned is an ordinary day's work that must
|
||||
// not take a record with it. Removing one is a person's act, outside the mesh.
|
||||
func RaiseLogs(a LogAsserter, logs []Log) (undeclared []string, err error) {
|
||||
sorted := append([]Log(nil), logs...)
|
||||
sort.Slice(sorted, func(i, j int) bool { return sorted[i].Stream() < sorted[j].Stream() })
|
||||
declared := map[string]bool{}
|
||||
for _, l := range sorted {
|
||||
if err := a.EnsureLog(l); err != nil {
|
||||
return nil, fmt.Errorf("asserting %s's log %q: %w", l.Module, l.Name, err)
|
||||
}
|
||||
declared[l.Stream()] = true
|
||||
}
|
||||
names, err := a.LogStreams()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("listing the bus's logs: %w", err)
|
||||
}
|
||||
for _, n := range names {
|
||||
if strings.HasPrefix(n, LogStreamPrefix) && !declared[n] {
|
||||
undeclared = append(undeclared, n)
|
||||
}
|
||||
}
|
||||
sort.Strings(undeclared)
|
||||
return undeclared, nil
|
||||
}
|
||||
@@ -0,0 +1,294 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// **The stream a log is, field by field** (novox/hq ADR 0297 §2, the shared contract): a file stream
|
||||
// kept by limits, no maximum age, no cap per key, the declared cap, a message of at most 260 KiB (a
|
||||
// value of 256 KiB and 4 KiB of headers),
|
||||
// refusing what comes next when full, read directly, refusing a delete or a purge, one replica, and
|
||||
// a description that says whose it is and why.
|
||||
func TestALogsStreamIsAsTheDecisionFixesIt(t *testing.T) {
|
||||
got := Log{Module: "mesh-issues", Name: "changes"}.Config()
|
||||
want := jetstream.StreamConfig{
|
||||
Name: "LOG_mesh-issues_changes",
|
||||
Description: got.Description,
|
||||
Subjects: []string{"mesh.log.mesh-issues.changes.>"},
|
||||
Storage: jetstream.FileStorage,
|
||||
Retention: jetstream.LimitsPolicy,
|
||||
MaxAge: 0,
|
||||
MaxMsgs: -1,
|
||||
MaxMsgsPerSubject: -1,
|
||||
MaxBytes: 1024 * 1024 * 1024,
|
||||
MaxMsgSize: 260 * 1024,
|
||||
Discard: jetstream.DiscardNew,
|
||||
AllowDirect: true,
|
||||
DenyDelete: true,
|
||||
DenyPurge: true,
|
||||
Replicas: 1,
|
||||
}
|
||||
a, _ := json.Marshal(got)
|
||||
b, _ := json.Marshal(want)
|
||||
if string(a) != string(b) {
|
||||
t.Fatalf("the log's stream is\n %s\nwant\n %s", a, b)
|
||||
}
|
||||
if !strings.Contains(got.Description, "mesh-issues") || !strings.Contains(got.Description, "ADR 0297") {
|
||||
t.Fatalf("the description does not say whose the log is and why: %q", got.Description)
|
||||
}
|
||||
if c := (Log{Module: "m", Name: "n", MaxMiB: 8192}).Config(); c.MaxBytes != 8192*1024*1024 {
|
||||
t.Fatalf("a cap of 8192 MiB is %d bytes", c.MaxBytes)
|
||||
}
|
||||
}
|
||||
|
||||
// **The membership names each of the owner's logs** with exactly the field names the runtime reads:
|
||||
// `logs`, and in each `name`, `stream`, `subject`, `writes`. A module with no log is issued none, and
|
||||
// the field is absent.
|
||||
func TestAMembershipListsItsModulesLogs(t *testing.T) {
|
||||
m := MembershipFor("one", Declared{Module: "mesh-issues",
|
||||
Logs: []Log{{Module: "mesh-issues", Name: "changes"}}}, Placements{})
|
||||
raw, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var back map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &back); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := string(back["logs"]); got !=
|
||||
`[{"name":"changes","stream":"LOG_mesh-issues_changes","subject":"mesh.log.mesh-issues.changes","writes":true}]` {
|
||||
t.Fatalf("the membership's logs are %s", got)
|
||||
}
|
||||
none, _ := json.Marshal(MembershipFor("one", Declared{Module: "audit"}, Placements{}))
|
||||
if strings.Contains(string(none), `"logs"`) {
|
||||
t.Fatalf("a module with no log was issued logs: %s", none)
|
||||
}
|
||||
}
|
||||
|
||||
// logGrantsOf is the grants of a principal that are about logs.
|
||||
func logGrantsOf(publish []string) []string {
|
||||
var out []string
|
||||
for _, s := range publish {
|
||||
if strings.HasPrefix(s, "mesh.log.") || strings.Contains(s, ".LOG_") {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// **The runtime is granted exactly the contract's subjects for each log it carries, and nothing else
|
||||
// of any log**: appending under the log's subjects, binding to its stream, reading it directly. No
|
||||
// consumer, no delete, no purge, and nothing of a log its modules do not keep.
|
||||
func TestTheRuntimeIsGrantedItsModulesLogsAndNoMore(t *testing.T) {
|
||||
perms, err := PermissionsFor(Principal{Kind: KindNodeTools, Node: "one", Module: RuntimeModule,
|
||||
Carries: []Declared{
|
||||
{Module: "mesh-issues", Logs: []Log{{Module: "mesh-issues", Name: "changes"}}},
|
||||
{Module: "audit"},
|
||||
}, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{
|
||||
"$JS.API.DIRECT.GET.LOG_mesh-issues_changes",
|
||||
"$JS.API.DIRECT.GET.LOG_mesh-issues_changes.>",
|
||||
"$JS.API.STREAM.INFO.LOG_mesh-issues_changes",
|
||||
"mesh.log.mesh-issues.changes.>",
|
||||
}
|
||||
if got := logGrantsOf(perms.Publish); !slices.Equal(got, want) {
|
||||
t.Fatalf("the runtime is granted\n %q\nwant\n %q", got, want)
|
||||
}
|
||||
for _, s := range perms.Subscribe {
|
||||
if strings.HasPrefix(s, "mesh.log.") || strings.Contains(s, "LOG_") {
|
||||
t.Fatalf("the runtime subscribes a log's subjects: %q", s)
|
||||
}
|
||||
}
|
||||
// A module running on its own account is granted the same for its own logs.
|
||||
own, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "mesh-issues",
|
||||
Logs: []string{"changes"}, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := logGrantsOf(own.Publish); !slices.Equal(got, want) {
|
||||
t.Fatalf("the module's own account is granted\n %q\nwant\n %q", got, want)
|
||||
}
|
||||
// And a module with no log, nothing of any.
|
||||
none, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "audit", PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := logGrantsOf(none.Publish); len(got) != 0 {
|
||||
t.Fatalf("a module with no log is granted %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A log name or module that is not one plain token is issued and granted nothing rather than a
|
||||
// pattern that happens to parse.
|
||||
func TestALogThatNamesNoStreamGrantsNothing(t *testing.T) {
|
||||
if got := logGrants("a", []string{"x.y", "x>", "*", ""}); len(got) != 0 {
|
||||
t.Fatalf("granted %v for logs that name no stream", got)
|
||||
}
|
||||
if got := logGrants("a.b", []string{"c"}); len(got) != 0 {
|
||||
t.Fatalf("granted %v for a module that is not one token", got)
|
||||
}
|
||||
}
|
||||
|
||||
type logs struct {
|
||||
ensured []string
|
||||
on []string
|
||||
}
|
||||
|
||||
func (l *logs) EnsureLog(x Log) error { l.ensured = append(l.ensured, x.Stream()); return nil }
|
||||
func (l *logs) LogStreams() ([]string, error) { return l.on, nil }
|
||||
|
||||
// Every declared log is asserted; one on the server that nothing declares is said, not removed — and
|
||||
// the asserter has no way to remove one.
|
||||
func TestRaisingLogsReportsWhatNothingDeclares(t *testing.T) {
|
||||
l := &logs{on: []string{"LOG_mesh-issues_changes", "LOG_gone_old", "KV_not_a_log"}}
|
||||
undeclared, err := RaiseLogs(l, []Log{{Module: "mesh-issues", Name: "changes"}, {Module: "a", Name: "b"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Equal(l.ensured, []string{"LOG_a_b", "LOG_mesh-issues_changes"}) {
|
||||
t.Fatalf("asserted %v", l.ensured)
|
||||
}
|
||||
if !slices.Equal(undeclared, []string{"LOG_gone_old"}) {
|
||||
t.Fatalf("reported %v", undeclared)
|
||||
}
|
||||
}
|
||||
|
||||
// **No path of the mesh deletes or purges a log or a bucket, or recreates one** (novox/hq ADR 0297 §2,
|
||||
// ADR 0201 §15–16): no code of this repository outside its tests calls the client's stream or bucket
|
||||
// delete, or purges a stream, but for the controller lease's own stream, which purges its own history
|
||||
// below a sequence (internal/lease). A new call is a decision, not a refactor.
|
||||
func TestNoPathDeletesALogOrABucket(t *testing.T) {
|
||||
removers := map[string]bool{"DeleteStream": true, "DeleteKeyValue": true, "PurgeStream": true,
|
||||
"DeleteObjectStore": true}
|
||||
root := filepath.Join("..", "..")
|
||||
var found []string
|
||||
for _, dir := range []string{"cmd", "internal"} {
|
||||
err := filepath.WalkDir(filepath.Join(root, dir), func(path string, e fs.DirEntry, err error) error {
|
||||
if err != nil || e.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
|
||||
return err
|
||||
}
|
||||
f, err := parser.ParseFile(token.NewFileSet(), path, nil, 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
call, ok := n.(*ast.CallExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
sel, ok := call.Fun.(*ast.SelectorExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
name := sel.Sel.Name
|
||||
if removers[name] || (name == "Purge" && !strings.Contains(filepath.ToSlash(path), "internal/lease/")) {
|
||||
found = append(found, path+": "+name)
|
||||
}
|
||||
return true
|
||||
})
|
||||
for _, lit := range stringsIn(f) {
|
||||
if strings.Contains(lit, "$JS.API.STREAM.DELETE") || strings.Contains(lit, "$JS.API.STREAM.PURGE") {
|
||||
// Only the writers table names it, as a subject no one but the controller may publish.
|
||||
if !strings.HasSuffix(filepath.ToSlash(path), "internal/broker/writers.go") {
|
||||
found = append(found, path+": "+lit)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if len(found) > 0 {
|
||||
t.Fatalf("a path of the mesh removes a stream, a bucket or what one holds:\n %s", strings.Join(found, "\n "))
|
||||
}
|
||||
}
|
||||
|
||||
// stringsIn is every string literal of a file.
|
||||
func stringsIn(f *ast.File) []string {
|
||||
var out []string
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
if lit, ok := n.(*ast.BasicLit); ok && lit.Kind == token.STRING {
|
||||
out = append(out, lit.Value)
|
||||
}
|
||||
return true
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// Against a real server: a log is created as its configuration says, asserting it again keeps what
|
||||
// it holds, a changed cap is brought to match in place, an entry cannot be deleted from it, and one
|
||||
// nothing declares any more is reported and stays.
|
||||
func TestALogIsAssertedInPlaceAndNeverRemoved(t *testing.T) {
|
||||
bus := aLiveBus(t)
|
||||
l := Log{Module: "logtest", Name: "changes", MaxMiB: 1}
|
||||
if _, err := RaiseLogs(bus, []Log{l}); err != nil {
|
||||
t.Fatalf("a real server refused a module's log: %v", err)
|
||||
}
|
||||
js, err := jetstream.New(bus.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
ack, err := js.Publish(ctx, l.Subject()+".7", []byte(`{"op":"opened"}`))
|
||||
if err != nil {
|
||||
t.Fatalf("an append to the log was refused: %v", err)
|
||||
}
|
||||
stream, err := js.Stream(ctx, l.Stream())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
have := stream.CachedInfo().Config
|
||||
want := l.Config()
|
||||
if have.Storage != want.Storage || have.Retention != want.Retention || have.MaxAge != 0 ||
|
||||
have.MaxMsgsPerSubject != -1 || have.MaxBytes != want.MaxBytes || have.MaxMsgSize != want.MaxMsgSize ||
|
||||
have.Discard != jetstream.DiscardNew || !have.AllowDirect || !have.DenyDelete || !have.DenyPurge ||
|
||||
have.Replicas != 1 || !slices.Equal(have.Subjects, want.Subjects) {
|
||||
t.Fatalf("the server holds the log as %+v", have)
|
||||
}
|
||||
if err := stream.DeleteMsg(ctx, ack.Sequence); err == nil {
|
||||
t.Fatal("an entry was deleted from a log")
|
||||
}
|
||||
l.MaxMiB = 2
|
||||
if _, err := RaiseLogs(bus, []Log{l}); err != nil {
|
||||
t.Fatalf("asserting the log again failed, so a restart would: %v", err)
|
||||
}
|
||||
info, err := stream.Info(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.Config.MaxBytes != 2*1024*1024 {
|
||||
t.Fatalf("the changed cap was not brought to match: %d", info.Config.MaxBytes)
|
||||
}
|
||||
if info.State.Msgs < 1 {
|
||||
t.Fatal("asserting the log again lost what it held")
|
||||
}
|
||||
undeclared, err := RaiseLogs(bus, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Contains(undeclared, l.Stream()) {
|
||||
t.Fatalf("a log nothing declares was not reported: %v", undeclared)
|
||||
}
|
||||
if _, err := js.Stream(ctx, l.Stream()); err != nil {
|
||||
t.Fatalf("a log nothing declares is gone: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -51,6 +51,10 @@ type Membership struct {
|
||||
// and refuses, with the reason, what is not on it — the bus enforces only the union over every
|
||||
// module on the machine.
|
||||
State []StateIssued `json:"state,omitempty"`
|
||||
// Logs is every log this module's code may reach, by the name it uses for each (novox/hq ADR 0297):
|
||||
// its own and no other's, since a log is its owner's alone. The runtime answers a bundle's log verbs
|
||||
// from this list and refuses, with the reason, a log not on it.
|
||||
Logs []LogIssued `json:"logs,omitempty"`
|
||||
// SeatTraffic is what this module's code may submit, say, hear, take, ask, answer and read on seats
|
||||
// that name their caller or their kind (novox/hq ADR 0259 §3). The runtime carrying the module
|
||||
// publishes, takes and answers for it only what is listed here: the bus enforces only the union
|
||||
@@ -121,6 +125,7 @@ func MembershipFor(node string, d Declared, where Placements) Membership {
|
||||
}
|
||||
}
|
||||
m.State = stateIssuedFor(d, node)
|
||||
m.Logs = logsIssuedFor(d)
|
||||
t := SeatTrafficOf(d.Module, d.Holds, d.Uses, d.Watches)
|
||||
for _, s := range append(append([]Seat{}, d.Uses...), d.Watches...) {
|
||||
if !s.Kinded {
|
||||
|
||||
@@ -132,6 +132,8 @@ type Principal struct {
|
||||
// no other; KeyedReads the keys of others' state it reads one key at a time (novox/hq ADR 0260).
|
||||
PerMachine []string
|
||||
KeyedReads []KeyedRead
|
||||
// Logs is the local names of the logs this principal's module keeps (novox/hq ADR 0297).
|
||||
Logs []string
|
||||
|
||||
// SnapshotsTheBus is the bus's own module, the one holding mesh-broker (novox/hq ADR 0235). Its
|
||||
// whole authority is BusSnapshotGrants: it copies the streams for the night's backup and nothing
|
||||
@@ -746,6 +748,8 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// watched, its own written too.
|
||||
pub = append(pub, stateGrants(stateAccess{Module: p.Module, Node: p.Node, Keeps: p.State,
|
||||
PerMachine: p.PerMachine, Reads: p.Reads, KeyedReads: p.KeyedReads})...)
|
||||
// And its logs (novox/hq ADR 0297): appended to and read directly, its own alone.
|
||||
pub = append(pub, logGrants(p.Module, p.Logs)...)
|
||||
|
||||
// 6. Its traffic on seats that name their caller or their kind, ask proofs or keep records
|
||||
// (novox/hq ADR 0259 §3).
|
||||
@@ -833,6 +837,12 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
pub = append(pub, stateGrants(stateAccess{Module: d.Module, Node: p.Node, Keeps: stateNames(d.State),
|
||||
PerMachine: perMachineNames(d.State), Reads: d.Reads, KeyedReads: d.KeyedReads})...)
|
||||
}
|
||||
// **And it keeps the logs of the modules it carries** (novox/hq ADR 0297): each module's own, the
|
||||
// union over them. That one module's code does not append to another's log through it is the
|
||||
// runtime's to keep, from the logs each membership lists.
|
||||
for _, d := range p.Carries {
|
||||
pub = append(pub, logGrants(d.Module, logNames(d.Logs))...)
|
||||
}
|
||||
// **Never the traffic of a trusted holder** (novox/hq ADR 0259 §8): the machine's runtime runs as the
|
||||
// operator's account, which every agent runs as, so a module saying warrants or speaking for a kind
|
||||
// that proves its sender is never composed into it — refused here, naming it, whatever registration
|
||||
|
||||
@@ -35,6 +35,8 @@ type Declared struct {
|
||||
Invokes []string
|
||||
// State is the state it keeps, each a bucket its instances write (novox/hq ADR 0201).
|
||||
State []Bucket
|
||||
// Logs are the logs it keeps, each a stream its instances append to (novox/hq ADR 0297).
|
||||
Logs []Log
|
||||
// Reads are other modules' state it reads, each `<module>.<name>` (novox/hq ADR 0201).
|
||||
Reads []string
|
||||
// KeyedReads are keys of other modules' state it reads, each one key alone: what a seat's holder is
|
||||
@@ -124,6 +126,7 @@ func Users(r Records) ([]Principal, error) {
|
||||
Holds: d.Holds, Uses: d.Uses, Watches: d.Watches, Invokes: d.Invokes,
|
||||
State: stateNames(d.State), Reads: d.Reads, SnapshotsTheBus: d.SnapshotsTheBus,
|
||||
PerMachine: perMachineNames(d.State), KeyedReads: d.KeyedReads,
|
||||
Logs: logNames(d.Logs),
|
||||
})
|
||||
}
|
||||
if runtimeHere {
|
||||
|
||||
@@ -342,38 +342,18 @@ func (e *NotMadeError) Error() string {
|
||||
// compose. Empty when every module composes. The same judgement SetSettings makes before storing.
|
||||
func (r Resolution) LeftOut(settings SettingsBy, adopted bool) map[string]string {
|
||||
out := map[string]string{}
|
||||
for module, why := range r.LeftOutBecause(settings, adopted) {
|
||||
out[module] = why.Error()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// LeftOutBecause is LeftOut with each reason as the error it was, so a reader can tell a setting nobody
|
||||
// gave (an *UnsetSettingError) from any other cause and say it as the operator's to give (novox/hq issue
|
||||
// 380). An UnreadManifestError for a stored manifest this controller cannot read whole.
|
||||
func (r Resolution) LeftOutBecause(settings SettingsBy, adopted bool) map[string]error {
|
||||
out := map[string]error{}
|
||||
for _, m := range r.Modules {
|
||||
if why := UnknownFieldReason(m); why != "" {
|
||||
out[m.Module] = &UnreadManifestError{Module: m.Module, said: why}
|
||||
out[m.Module] = why
|
||||
continue
|
||||
}
|
||||
if err := JudgeSettings(m, settings[m.Module], adopted); err != nil {
|
||||
out[m.Module] = err
|
||||
out[m.Module] = err.Error()
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// UnreadManifestError is a module left out because its stored manifest has a key this controller does not know
|
||||
// (novox/hq ADR 0262): said once for the whole mesh, by the catalogue's own condition, not per machine.
|
||||
type UnreadManifestError struct {
|
||||
Module string
|
||||
said string
|
||||
}
|
||||
|
||||
func (e *UnreadManifestError) Error() string { return e.said }
|
||||
|
||||
// Compose is Declaration with the owner of every resource said.
|
||||
func (r Resolution) Compose(with Rendering) (Composed, error) {
|
||||
owner := map[string]string{}
|
||||
|
||||
@@ -1,71 +0,0 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/beside"
|
||||
)
|
||||
|
||||
// The controller refuses a definition's resolved path by the node-engine's own rules, no more (novox/hq issue
|
||||
// 496): the same lists, read here from mesh-host's internal/apply/placement_guard.go — in a merge check the clone
|
||||
// beside it at the commit the mesh runs, elsewhere the copy captured in testdata/beside. When the engine's lists
|
||||
// move, this fails until the controller's move with them, so the gate never refuses what the engine applies nor
|
||||
// passes what it refuses by path alone.
|
||||
func TestTheResolvedPathRulesAreTheNodeEnginesOwn(t *testing.T) {
|
||||
file := filepath.Join(beside.Dir(t, "mesh-host"), "internal", "apply", "placement_guard.go")
|
||||
parsed, err := parser.ParseFile(token.NewFileSet(), file, nil, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("the node-engine's guard does not parse: %v", err)
|
||||
}
|
||||
lists := map[string][]string{}
|
||||
consts := map[string]string{}
|
||||
ast.Inspect(parsed, func(n ast.Node) bool {
|
||||
spec, ok := n.(*ast.ValueSpec)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
for i, name := range spec.Names {
|
||||
if i >= len(spec.Values) {
|
||||
continue
|
||||
}
|
||||
switch v := spec.Values[i].(type) {
|
||||
case *ast.CompositeLit:
|
||||
for _, elt := range v.Elts {
|
||||
if lit, ok := elt.(*ast.BasicLit); ok && lit.Kind == token.STRING {
|
||||
s, _ := strconv.Unquote(lit.Value)
|
||||
lists[name.Name] = append(lists[name.Name], s)
|
||||
}
|
||||
}
|
||||
case *ast.BasicLit:
|
||||
if v.Kind == token.STRING {
|
||||
consts[name.Name], _ = strconv.Unquote(v.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
for name, ours := range map[string][]string{
|
||||
"protectedRoots": protectedRoots, "forbiddenBelow": forbiddenBelow, "engineTrees": engineTrees,
|
||||
} {
|
||||
theirs := lists[name]
|
||||
if len(theirs) == 0 {
|
||||
t.Errorf("the node-engine's guard names no %s any more; read it and say where its rule went", name)
|
||||
continue
|
||||
}
|
||||
a, b := slices.Clone(ours), slices.Clone(theirs)
|
||||
slices.Sort(a)
|
||||
slices.Sort(b)
|
||||
if !slices.Equal(a, b) {
|
||||
t.Errorf("%s differs from the node-engine's:\n controller %v\n node-engine %v", name, a, b)
|
||||
}
|
||||
}
|
||||
if consts["engineModule"] != engineModule {
|
||||
t.Errorf("the node-engine's own module is %q there and %q here", consts["engineModule"], engineModule)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// What a module may call its logs (novox/hq ADR 0297).
|
||||
//
|
||||
// A log is a module's record of operations: entries appended under a key, each kept as long as the
|
||||
// log is, in the order they came. A module names each log it owns **locally** — `changes`, never a
|
||||
// stream or a subject (ADR 0201 §4) — and the mesh derives the stream from the module and the local
|
||||
// name, as it derives a bucket. So the rule for a log's name is a state name's: one plain token, and
|
||||
// a module that keeps a log has a name that is one plain token too.
|
||||
|
||||
// The mesh's caps on a log, in MiB: what a module may ask, and what it gets when it asks nothing.
|
||||
const (
|
||||
LogLeastMiB = 1
|
||||
LogMostMiB = 8192
|
||||
LogDefaultMiB = 1024
|
||||
)
|
||||
|
||||
// LogDeclaration is one log a module owns: its local name, and how large it may grow.
|
||||
type LogDeclaration struct {
|
||||
Name string `json:"name"`
|
||||
// MaxMiB is the log's cap in MiB; zero is LogDefaultMiB. When full, the log refuses new entries
|
||||
// and never drops old ones.
|
||||
MaxMiB int `json:"max-mib,omitempty"`
|
||||
}
|
||||
|
||||
// Cap is the log's cap in MiB, the default where none is said.
|
||||
func (l LogDeclaration) Cap() int {
|
||||
if l.MaxMiB == 0 {
|
||||
return LogDefaultMiB
|
||||
}
|
||||
return l.MaxMiB
|
||||
}
|
||||
|
||||
// UnmarshalJSON reads a log as its bare name, or as {name, max-mib}.
|
||||
func (l *LogDeclaration) UnmarshalJSON(raw []byte) error {
|
||||
trimmed := bytes.TrimSpace(raw)
|
||||
if len(trimmed) > 0 && trimmed[0] == '"' {
|
||||
return json.Unmarshal(trimmed, &l.Name)
|
||||
}
|
||||
var full struct {
|
||||
Name string `json:"name"`
|
||||
MaxMiB *int `json:"max-mib"`
|
||||
}
|
||||
dec := json.NewDecoder(bytes.NewReader(trimmed))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&full); err != nil {
|
||||
return fmt.Errorf("a log is either a name or {name, max-mib}: %w", typedUnknown(err))
|
||||
}
|
||||
l.Name = full.Name
|
||||
l.MaxMiB = 0
|
||||
if full.MaxMiB != nil {
|
||||
// Said, and said as nothing: refused rather than read as the default, which it did not say.
|
||||
if *full.MaxMiB == 0 {
|
||||
return fmt.Errorf("log %q: max-mib is between %d and %d, not 0", full.Name, LogLeastMiB, LogMostMiB)
|
||||
}
|
||||
l.MaxMiB = *full.MaxMiB
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarshalJSON writes back the short form when there is nothing else to say.
|
||||
func (l LogDeclaration) MarshalJSON() ([]byte, error) {
|
||||
if l.MaxMiB == 0 {
|
||||
return json.Marshal(l.Name)
|
||||
}
|
||||
type plain LogDeclaration
|
||||
return json.Marshal(plain(l))
|
||||
}
|
||||
|
||||
// LogProblems is what is wrong with a manifest's logs.
|
||||
//
|
||||
// Refused at registration, for a bucket's reason: a stream name the bus cannot hold, or a cap the
|
||||
// mesh would not grant, is a module that installs, starts, and is refused on its first append.
|
||||
func LogProblems(m Manifest) []string {
|
||||
var problems []string
|
||||
if len(m.Logs) > 0 && !stateName.MatchString(m.Module) {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s keeps a log, and a module's name is part of its logs' names, which take one plain "+
|
||||
"name — no dot (novox/hq ADR 0297)", m.Module))
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, l := range m.Logs {
|
||||
switch {
|
||||
case !stateName.MatchString(l.Name):
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s keeps log %q: a log is named locally — lower-case letters, digits and hyphens, "+
|
||||
"no dot and no underscore; the mesh derives the stream (novox/hq ADR 0297)", m.Module, l.Name))
|
||||
case seen[l.Name]:
|
||||
problems = append(problems, fmt.Sprintf("%s keeps log %q twice", m.Module, l.Name))
|
||||
}
|
||||
seen[l.Name] = true
|
||||
if l.MaxMiB != 0 && (l.MaxMiB < LogLeastMiB || l.MaxMiB > LogMostMiB) {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s caps log %q at %d MiB; a log holds between %d and %d MiB (novox/hq ADR 0297)",
|
||||
m.Module, l.Name, l.MaxMiB, LogLeastMiB, LogMostMiB))
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A module declares the logs it keeps (novox/hq ADR 0297 §1): a log by its bare name, or with its cap
|
||||
// in MiB; the default cap where none is said.
|
||||
func TestAManifestMaySayWhatLogsItKeeps(t *testing.T) {
|
||||
m, err := ParseManifest([]byte(`{"module":"mesh-issues","version":"1",` +
|
||||
`"logs":["changes",{"name":"moves","max-mib":2048}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(m.Logs) != 2 || m.Logs[0].Name != "changes" || m.Logs[1].Name != "moves" {
|
||||
t.Fatalf("logs not read: %+v", m.Logs)
|
||||
}
|
||||
if m.Logs[0].Cap() != LogDefaultMiB || m.Logs[0].Cap() != 1024 || m.Logs[1].Cap() != 2048 {
|
||||
t.Fatalf("caps read as %d and %d", m.Logs[0].Cap(), m.Logs[1].Cap())
|
||||
}
|
||||
out, _ := json.Marshal(m.Logs)
|
||||
if string(out) != `["changes",{"name":"moves","max-mib":2048}]` {
|
||||
t.Fatalf("written back as %s", out)
|
||||
}
|
||||
for _, edge := range []string{`{"name":"a","max-mib":1}`, `{"name":"a","max-mib":8192}`} {
|
||||
if _, err := ParseManifest([]byte(`{"module":"a","version":"1","logs":[` + edge + `]}`)); err != nil {
|
||||
t.Errorf("%s is inside the caps and was refused: %v", edge, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue check refuses a log outside the caps, with a name that is not one plain token, with a
|
||||
// field the mesh does not know, or kept by a module whose own name is not one plain token.
|
||||
func TestALogIsNamedLocallyAndCapped(t *testing.T) {
|
||||
for _, c := range []struct{ manifest, says string }{
|
||||
{`{"module":"a","version":"1","logs":["mesh.changes"]}`, `keeps log "mesh.changes": a log is named locally`},
|
||||
{`{"module":"a","version":"1","logs":["my_changes"]}`, `keeps log "my_changes"`},
|
||||
{`{"module":"a","version":"1","logs":["Changes"]}`, `keeps log "Changes"`},
|
||||
{`{"module":"a","version":"1","logs":["c","c"]}`, `keeps log "c" twice`},
|
||||
{`{"module":"a","version":"1","logs":[{"name":"c","max-mib":8193}]}`, `between 1 and 8192 MiB`},
|
||||
{`{"module":"a","version":"1","logs":[{"name":"c","max-mib":-1}]}`, `between 1 and 8192 MiB`},
|
||||
{`{"module":"a","version":"1","logs":[{"name":"c","max-mib":0}]}`, `max-mib is between 1 and 8192, not 0`},
|
||||
{`{"module":"a","version":"1","logs":[{"name":"c","stream":"LOG_x"}]}`, `{name, max-mib}`},
|
||||
{`{"module":"a.b","version":"1","logs":["c"]}`, `no dot`},
|
||||
} {
|
||||
_, err := ParseManifest([]byte(c.manifest))
|
||||
if err == nil {
|
||||
t.Errorf("%s was accepted", c.manifest)
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("%s refused for the wrong reason: %v", c.manifest, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **Across the whole catalogue**: every log is named locally and capped within the mesh's caps.
|
||||
func TestEveryManifestsLogsAreLocalAndCapped(t *testing.T) {
|
||||
var problems []string
|
||||
for _, m := range theCatalogue(t) {
|
||||
problems = append(problems, LogProblems(m)...)
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
t.Fatalf("the catalogue's logs are not what ADR 0297 says:\n %s", strings.Join(problems, "\n "))
|
||||
}
|
||||
}
|
||||
@@ -465,6 +465,11 @@ type Manifest struct {
|
||||
// (novox/hq ADR 0201). Not history — that is an event — and never a secret, sealed or not.
|
||||
State []StateDeclaration `json:"state,omitempty"`
|
||||
|
||||
// Logs are the logs of operations this module keeps on the bus, by local name: each a stream the
|
||||
// controller creates and never removes, which every instance of the module appends to and reads
|
||||
// (novox/hq ADR 0297). A log is its owner's alone: no other module reads it.
|
||||
Logs []LogDeclaration `json:"logs,omitempty"`
|
||||
|
||||
// Settings are the defaults this module gives its settings (novox/hq ADR 0262): each key a file,
|
||||
// a contribution or a served fact asks for as `${setting:<key>}`, its default, and why that
|
||||
// default. Only a preference has one — a font size, a width, a number of workers — and a value
|
||||
@@ -1507,7 +1512,7 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
||||
name string
|
||||
n int
|
||||
}{{"consumes", len(m.Consumes)}, {"uses", len(m.Uses)}, {"invokes", len(m.Invokes)},
|
||||
{"state", len(m.State)}, {"reads", len(m.Reads)}} {
|
||||
{"state", len(m.State)}, {"logs", len(m.Logs)}, {"reads", len(m.Reads)}} {
|
||||
if f.n > 0 {
|
||||
said = append(said, f.name)
|
||||
}
|
||||
@@ -1618,6 +1623,8 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
||||
problems = append(problems, EventProblems(m)...)
|
||||
// And what it may call its state, and whose it may read (state.go, novox/hq ADR 0201).
|
||||
problems = append(problems, StateProblems(m)...)
|
||||
// And what it may call its logs, and how large it may ask them to grow (logs.go, novox/hq ADR 0297).
|
||||
problems = append(problems, LogProblems(m)...)
|
||||
// And the defaults it gives its settings (setting_defaults.go, novox/hq ADR 0262).
|
||||
problems = append(problems, SettingProblems(m)...)
|
||||
wellFormed := true
|
||||
@@ -2138,7 +2145,6 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
||||
problems = append(problems, m.undeclaredMounts()...)
|
||||
problems = append(problems, m.unknownDirRefs()...)
|
||||
problems = append(problems, m.unknownAccessRefs()...)
|
||||
problems = append(problems, m.resolvedPathProblems()...)
|
||||
problems = append(problems, m.jailProblems()...)
|
||||
// What a module adds to the account's environment and to the login shell, and the holder's
|
||||
// placeholders for them (novox/hq ADR 0203, ADR 0204) — here, so the catalogue check refuses
|
||||
|
||||
@@ -92,105 +92,6 @@ func systemPath(path string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Where no directory or file a module's definition resolves to may be (novox/hq issue 496).
|
||||
//
|
||||
// mesh-catalog #205 gave docker's `state` directory `"place": "."`, which resolves to <root>/docker: with the
|
||||
// default root, /var/lib/docker, every container's filesystem. systemPath judged only the `places` and `accesses`
|
||||
// settings, so the default layout and a definition's own paths reached the merge gate unjudged; it composed every
|
||||
// machine and passed, and only the node-engine refused the directory, at apply, failing the walk.
|
||||
//
|
||||
// **Judged where a definition is judged, never where a machine is composed.** resolvedPathProblems runs in
|
||||
// ParseManifest, which every route a definition takes into the mesh passes: `module check`, registration of a
|
||||
// build (the builder's and the registry verbs'), and the merge gate's reading of the changed repository. A
|
||||
// refusal there stops one definition before it reaches any machine. Composition reads registered manifests
|
||||
// without ParseManifest, and judges nothing of this: refusing there would fail the whole machine's declaration and
|
||||
// freeze every module on it for one module's path, where the node-engine fails only that resource.
|
||||
//
|
||||
// **The node-engine's rules, no more** (mesh-host's internal/apply/placement_guard.go, with files judged as
|
||||
// directories are after novox/hq issue 495, rule 6). A path is refused when it is one of protectedRoots or holds
|
||||
// one, when it is at or below a tree in forbiddenBelow, or at or below one of engineTrees and its module is not
|
||||
// the node-engine's. What the engine judges with what only the machine knows stays the engine's: where the
|
||||
// runtimes really keep their data, links, the accounts' homes, a directory's owner below /etc. The lists are the
|
||||
// engine's own words, and a test (engine_guard_test.go) holds them equal to mesh-host's beside this repository.
|
||||
// systemPath stays the stricter rule for a setting: a setting is an operator's word about one machine, and the
|
||||
// trees it lists (/etc, /usr, /run, the mesh's own) are where the mesh's own modules write by design.
|
||||
var (
|
||||
protectedRoots = []string{"/", "/bin", "/boot", "/dev", "/etc", "/home", "/lib", "/lib32", "/lib64",
|
||||
"/media", "/mnt", "/opt", "/proc", "/root", "/run", "/sbin", "/srv", "/sys", "/tmp", "/usr", "/usr/bin",
|
||||
"/usr/lib", "/usr/lib64", "/usr/local", "/usr/local/bin", "/usr/local/lib", "/usr/local/sbin", "/usr/sbin",
|
||||
"/usr/share", "/var", "/var/cache", "/var/lib", "/var/lib/mesh", "/var/log", "/var/run", "/var/tmp",
|
||||
"/var/spool"}
|
||||
forbiddenBelow = []string{"/proc", "/sys", "/dev", "/boot", "/root", "/var/spool", "/opt", "/var/lib/docker",
|
||||
"/var/lib/containers", "/var/lib/containerd"}
|
||||
engineTrees = []string{"/var/lib/mesh-host", "/usr/lib/nox-mesh-host"}
|
||||
)
|
||||
|
||||
// engineModule is the node-engine's own module, the one that places in engineTrees.
|
||||
const engineModule = "mesh-host"
|
||||
|
||||
// enginePath says why the node-engine refuses a directory or file at path for module, or "".
|
||||
func enginePath(path, module string) string {
|
||||
path = filepath.Clean(path)
|
||||
if !filepath.IsAbs(path) {
|
||||
return ""
|
||||
}
|
||||
atOrBelow := func(tree string) bool { return path == tree || strings.HasPrefix(path, tree+"/") }
|
||||
for _, root := range protectedRoots {
|
||||
if path == root || path == "/" || strings.HasPrefix(root, path+"/") {
|
||||
return root + " is one of the machine's own directories, and owning it is owning everything in it"
|
||||
}
|
||||
}
|
||||
for _, tree := range forbiddenBelow {
|
||||
if atOrBelow(tree) {
|
||||
return "nothing is placed in " + tree
|
||||
}
|
||||
}
|
||||
if module != engineModule {
|
||||
for _, tree := range engineTrees {
|
||||
if atOrBelow(tree) {
|
||||
return tree + " is the node-engine's own, placed in by its own module alone"
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// resolvedPathProblems is every directory and file of the definition whose path, resolved as a node with the
|
||||
// default root resolves it, the node-engine would refuse (novox/hq issue 496). A path still holding a placeholder
|
||||
// only a machine fills (a setting, an access the definition gives no default) is the engine's to judge.
|
||||
func (m Manifest) resolvedPathProblems() []string {
|
||||
dirs := dirsFor(m, Rendering{})
|
||||
accesses := map[string]string{}
|
||||
for _, a := range m.Accesses {
|
||||
if a.ID != "" && a.Path != "" {
|
||||
accesses[a.ID] = a.Path
|
||||
}
|
||||
}
|
||||
var problems []string
|
||||
for _, r := range m.Resources {
|
||||
kind := fmt.Sprint(r["type"])
|
||||
if kind != "directory" && kind != "file" {
|
||||
continue
|
||||
}
|
||||
id := fmt.Sprint(r["id"])
|
||||
path, _ := r["path"].(string)
|
||||
if kind == "directory" && path == "" {
|
||||
path = dirs[id]
|
||||
}
|
||||
path, _ = dirFill(path, dirs, m.Module)
|
||||
path, _ = accessFill(path, accesses, m.Module)
|
||||
if path == "" || strings.Contains(path, "${") {
|
||||
continue
|
||||
}
|
||||
if why := enginePath(path, m.Module); why != "" {
|
||||
problems = append(problems, fmt.Sprintf("%s's %s %q resolves to %s, which the node-engine refuses: %s. "+
|
||||
"A module's directories and files are judged when its definition is, so the merge gate refuses it "+
|
||||
"before any machine does (novox/hq issue 496)", m.Module, kind, id, filepath.Clean(path), why))
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// accessRef is how a module names one of its accesses: ${access:<id>}.
|
||||
var accessRef = regexp.MustCompile(`\$\{access:([a-z0-9][a-z0-9-]*)\}`)
|
||||
|
||||
|
||||
@@ -1,127 +0,0 @@
|
||||
package catalogue
|
||||
|
||||
// A definition's directories and files are judged at their resolved paths when the definition is (novox/hq issue
|
||||
// 496). mesh-catalog #205 gave docker's `state` directory `"place": "."`, which resolves to <root>/docker —
|
||||
// /var/lib/docker, every container's filesystem. The merge gate composed every machine and passed it; only the
|
||||
// node-engine refused it, at apply, and failed the walk. ParseManifest is what `module check`, registration and the
|
||||
// merge gate's reading of a changed repository all run, so a refusal here is a refusal at each of them.
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestADirectoryPlacedInDockersDataIsRefusedWhereTheDefinitionIsJudged(t *testing.T) {
|
||||
// The #205 shape, as it was merged.
|
||||
_, err := ParseManifest([]byte(`{"module": "docker", "version": "1",
|
||||
"resources": [{"id": "state", "type": "directory", "place": "."}]}`))
|
||||
if err == nil {
|
||||
t.Fatal("a directory resolving to /var/lib/docker was accepted; the node-engine refuses it at apply")
|
||||
}
|
||||
for _, said := range []string{"docker", `"state"`, "/var/lib/docker", "issue 496"} {
|
||||
if !strings.Contains(err.Error(), said) {
|
||||
t.Errorf("the refusal names the module, the resource, the path and why; %q is missing from %q", said, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheMeshsPlaceForDockerIsAccepted(t *testing.T) {
|
||||
// The fix #205 needed: the mesh's own directory for the module, <root>/mesh/docker.
|
||||
m, err := ParseManifest([]byte(`{"module": "docker", "version": "1", "resources": [
|
||||
{"id": "state", "type": "directory", "place": "mesh"},
|
||||
{"id": "marker", "type": "file", "path": "${dir:state}/applied", "content": "x"}]}`))
|
||||
if err != nil {
|
||||
t.Fatalf("place %q is in the mesh's tree, where the mesh writes for every module: %v", "mesh", err)
|
||||
}
|
||||
if got := dirsFor(m, Rendering{}); got["state"] != "/var/lib/mesh/docker" {
|
||||
t.Fatalf("got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The node-engine's rules, for directories and — as the engine judges them since novox/hq issue 495 — files.
|
||||
func TestADefinitionIsRefusedWhereTheNodeEngineRefusesItsPaths(t *testing.T) {
|
||||
refused := map[string]string{
|
||||
"a stated directory in docker's data": `{"module": "sidecar", "version": "1", "resources": [
|
||||
{"id": "volumes", "type": "directory", "path": "/var/lib/docker/volumes/x"}]}`,
|
||||
"a file in containerd's data": `{"module": "images", "version": "1", "resources": [
|
||||
{"id": "f", "type": "file", "path": "/var/lib/containerd/x", "content": "x"}]}`,
|
||||
"a file beneath a placed directory that climbs out of it": `{"module": "docker", "version": "1", "resources": [
|
||||
{"id": "state", "type": "directory", "place": "mesh"},
|
||||
{"id": "f", "type": "file", "path": "${dir:state}/../../containers/x", "content": "x"}]}`,
|
||||
"a file in /boot": `{"module": "grub", "version": "1", "resources": [
|
||||
{"id": "cfg", "type": "file", "path": "/boot/grub/custom.cfg", "content": "x"}]}`,
|
||||
"a directory that is the mesh's whole tree": `{"module": "mesh", "version": "1", "resources": [
|
||||
{"id": "all", "type": "directory", "place": "."}]}`,
|
||||
"a directory that holds /etc": `{"module": "x", "version": "1", "resources": [
|
||||
{"id": "d", "type": "directory", "path": "/"}]}`,
|
||||
"a directory in the node-engine's own tree, by another module": `{"module": "intruder", "version": "1",
|
||||
"resources": [{"id": "d", "type": "directory", "path": "/var/lib/mesh-host/x"}]}`,
|
||||
}
|
||||
for name, raw := range refused {
|
||||
if _, err := ParseManifest([]byte(raw)); err == nil || !strings.Contains(err.Error(), "issue 496") {
|
||||
t.Errorf("%s: accepted, or refused for another reason: %v", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
// What the engine applies, the mesh's own modules' paths among them (read from every machine's live plan):
|
||||
// the machine's configuration, run directories, programs below /usr/local, the node-engine's own trees by its
|
||||
// own module, an account's keys, a module's own root, and — not on the engine's lists, so not refused here —
|
||||
// below /lib and at /storage, /data, /services and /var/lock.
|
||||
accepted := map[string]string{
|
||||
"a unit in /etc": `{"module": "power", "version": "1", "resources": [
|
||||
{"id": "d", "type": "directory", "path": "/etc/systemd/system/x.service.d"},
|
||||
{"id": "u", "type": "file", "path": "/etc/systemd/system/x.service.d/a.conf", "content": "x"}]}`,
|
||||
"a run directory": `{"module": "fail2ban", "version": "1", "resources": [
|
||||
{"id": "run-dir", "type": "directory", "path": "/var/run/fail2ban"}]}`,
|
||||
"a program in /usr/local/bin": `{"module": "claude-code", "version": "1", "resources": [
|
||||
{"id": "start", "type": "file", "path": "/usr/local/bin/claude-agent", "content": "x"}]}`,
|
||||
"the node-engine's launcher and state, by the node-engine": `{"module": "mesh-host", "version": "1", "resources": [
|
||||
{"id": "launcher", "type": "file", "path": "/usr/lib/nox-mesh-host/launch", "content": "x"},
|
||||
{"id": "state", "type": "directory", "path": "/var/lib/mesh-host"}]}`,
|
||||
"an account's .ssh": `{"module": "ssh-client", "version": "1", "resources": [
|
||||
{"id": "ssh-dir", "type": "directory", "path": "/home/someone/.ssh"},
|
||||
{"id": "config", "type": "file", "path": "/home/someone/.ssh/config", "content": "x"}]}`,
|
||||
"a module's own root": `{"module": "mailu", "version": "1", "resources": [
|
||||
{"id": "state", "type": "directory", "place": "."}]}`,
|
||||
"a file below /lib": `{"module": "udev", "version": "1", "resources": [
|
||||
{"id": "rule", "type": "file", "path": "/lib/udev/rules.d/99-x.rules", "content": "x"}]}`,
|
||||
"directories at /storage, /data, /services and /var/lock": `{"module": "roots", "version": "1", "resources": [
|
||||
{"id": "a", "type": "directory", "path": "/storage"}, {"id": "b", "type": "directory", "path": "/data"},
|
||||
{"id": "c", "type": "directory", "path": "/services"}, {"id": "d", "type": "directory", "path": "/var/lock"}]}`,
|
||||
}
|
||||
for name, raw := range accepted {
|
||||
if _, err := ParseManifest([]byte(raw)); err != nil {
|
||||
t.Errorf("%s: refused: %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPathThroughAnAccessIsJudgedWithTheAccessFilledIn(t *testing.T) {
|
||||
// A file's path may name an access; the access's default path is the definition's, so it is judged with it.
|
||||
_, err := ParseManifest([]byte(`{"module": "backup", "version": "1",
|
||||
"accesses": [{"id": "images", "path": "/var/lib/docker/volumes"}],
|
||||
"resources": [{"id": "marker", "type": "file", "path": "${access:images}/marker", "content": "x"}]}`))
|
||||
if err == nil || !strings.Contains(err.Error(), "/var/lib/docker/volumes/marker") ||
|
||||
!strings.Contains(err.Error(), "issue 496") {
|
||||
t.Fatalf("a file reaching Docker's data through an access's default path was accepted: %v", err)
|
||||
}
|
||||
// An access the definition gives no path is placed by a setting, which Places and AccessPlaces judge, and on
|
||||
// the machine by the node-engine: nothing here to resolve it against, so nothing is refused for it.
|
||||
if _, err := ParseManifest([]byte(`{"module": "backup", "version": "1",
|
||||
"accesses": [{"id": "images"}],
|
||||
"resources": [{"id": "marker", "type": "file", "path": "${access:images}/marker", "content": "x"}]}`)); err != nil {
|
||||
t.Fatalf("an access placed only by a setting cannot be judged at the definition: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComposingAMachineIsNeverStoppedByOneModulesPath(t *testing.T) {
|
||||
// A module registered from outside the catalogue never passes the gate. Refusing its path while a machine's
|
||||
// declaration is composed would fail the whole declaration and freeze every module on that machine; the
|
||||
// node-engine fails only the one resource. So composition leaves it to the engine.
|
||||
m := Manifest{Module: "docker", Version: "1", Resources: []map[string]any{
|
||||
{"id": "state", "type": "directory", "place": "."},
|
||||
}}
|
||||
if _, err := (Resolution{Node: "anchor", Modules: []Manifest{m}}).Declaration(Rendering{}); err != nil {
|
||||
t.Fatalf("the machine's declaration failed for one module's path: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -38,17 +38,6 @@ func settingsUsed(content string) []string {
|
||||
return keys
|
||||
}
|
||||
|
||||
// UnsetSettingError is a module whose definition says ${setting:<key>} where nothing sets that key: typed, so
|
||||
// that whoever reads why a module was left out of a machine can tell a setting nobody gave — the operator's
|
||||
// to give, named with the command that gives it — from any other reason (novox/hq issue 380). Its words are
|
||||
// the refusal's, unchanged.
|
||||
type UnsetSettingError struct {
|
||||
Module, Setting string
|
||||
said string
|
||||
}
|
||||
|
||||
func (e *UnsetSettingError) Error() string { return e.said }
|
||||
|
||||
// settingInto fills a file's ${setting:…} placeholders from the layers over a module.
|
||||
//
|
||||
// The last layer setting a key wins, which is the node's over the mesh's over the module's own
|
||||
@@ -69,12 +58,12 @@ func settingInto(resource map[string]any, layers []Layer, module string) error {
|
||||
for _, key := range settingsUsed(content) {
|
||||
value, set := settingValue(layers, key)
|
||||
if !set {
|
||||
return &UnsetSettingError{Module: module, Setting: key, said: fmt.Sprintf(
|
||||
return fmt.Errorf(
|
||||
"%s has a file that says ${setting:%s}, and nothing sets %q for it — an operator's "+
|
||||
"value is the assignment's, never the definition's (novox/hq ADR 0112), and only a "+
|
||||
"preference has a default in the definition (ADR 0262): "+
|
||||
"`settings set %s <file>` with {%q: …}%s",
|
||||
module, key, key, module, key, orNoSettings(layers))}
|
||||
module, key, key, module, key, orNoSettings(layers))
|
||||
}
|
||||
content = strings.ReplaceAll(content, "${setting:"+key+"}", plainly(value))
|
||||
}
|
||||
@@ -125,11 +114,11 @@ func settingIntoUnit(resource map[string]any, layers []Layer, module string) err
|
||||
for _, key := range settingsUsed(unit) {
|
||||
value, set := settingValue(layers, key)
|
||||
if !set {
|
||||
return &UnsetSettingError{Module: module, Setting: key, said: fmt.Sprintf(
|
||||
return fmt.Errorf(
|
||||
"%s has a service whose unit says ${setting:%s}, and nothing sets %q for it — an operator's "+
|
||||
"value is the assignment's, never the definition's (novox/hq ADR 0112): "+
|
||||
"`settings set %s <file>` with {%q: …}%s",
|
||||
module, key, key, module, key, orNoSettings(layers))}
|
||||
module, key, key, module, key, orNoSettings(layers))
|
||||
}
|
||||
v := plainly(value)
|
||||
if !unitPart.MatchString(v) {
|
||||
|
||||
@@ -166,6 +166,8 @@ func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaratio
|
||||
// And the state it keeps and reads (novox/hq ADR 0201).
|
||||
State: bucketsOf(m),
|
||||
Reads: m.Reads,
|
||||
// And the logs it keeps (novox/hq ADR 0297).
|
||||
Logs: logsOf(m),
|
||||
// And the tools its health asks (novox/hq ADR 0240): the machine's node-engine is granted them.
|
||||
Checks: catalogue.HealthChecks(m),
|
||||
// And whether it runs as an account of its own (novox/hq ADR 0259 §8).
|
||||
@@ -222,6 +224,29 @@ func (i *Inventory) DeclaredBuckets(ctx context.Context) ([]broker.Bucket, error
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// logsOf is the logs a module keeps, as the bus holds them.
|
||||
func logsOf(m catalogue.Manifest) []broker.Log {
|
||||
var out []broker.Log
|
||||
for _, l := range m.Logs {
|
||||
out = append(out, broker.Log{Module: m.Module, Name: l.Name, MaxMiB: l.Cap()})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// DeclaredLogs is every log the catalogue declares, registered modules assigned or not: a log exists
|
||||
// from registration, as a bucket does (novox/hq ADR 0297 §2, ADR 0201 §6).
|
||||
func (i *Inventory) DeclaredLogs(ctx context.Context) ([]broker.Log, error) {
|
||||
declared, err := i.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot read the catalogue: %w", err)
|
||||
}
|
||||
var out []broker.Log
|
||||
for _, m := range declared {
|
||||
out = append(out, logsOf(m)...)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func asSeat(s catalogue.SeatDeclaration, declarer string) broker.Seat {
|
||||
seat := broker.Seat{Name: s.Name, Scope: s.Scope, Accepts: s.Accepts, Emits: s.Emits,
|
||||
Serves: catalogue.VerbNames(s.Serves), Kinded: s.Kinded, ByCaller: s.ByCaller, Proofs: s.Proofs,
|
||||
|
||||
Vendored
-8
@@ -25,11 +25,3 @@ and write the commits here. The SDK is captured at the commit go.mod pins for gi
|
||||
|
||||
rm -rf testdata/beside/mesh-sdk && mkdir -p testdata/beside/mesh-sdk
|
||||
git -C ../mesh-sdk archive <commit> conformance/events | tar -x -C testdata/beside/mesh-sdk
|
||||
|
||||
And from mesh-host at the same commit as its line above, the node-engine's placement guard, which
|
||||
internal/catalogue's engine_guard_test.go holds the controller's resolved-path rules to (novox/hq issue 496),
|
||||
kept as .captured so no Go tool reads it as this repository's code:
|
||||
|
||||
mkdir -p testdata/beside/mesh-host/internal/apply
|
||||
git -C ../mesh-host show <commit>:internal/apply/placement_guard.go \
|
||||
> testdata/beside/mesh-host/internal/apply/placement_guard.go.captured
|
||||
|
||||
@@ -1,549 +0,0 @@
|
||||
package apply
|
||||
|
||||
// Where the node-engine places nothing and mounts nothing, whoever asks (novox/hq issue 339).
|
||||
//
|
||||
// A directory names its path, and the controller resolves part of that path from what was set for the
|
||||
// module: its `places` setting moves a directory anywhere, with an owner it names, and its `accesses` setting
|
||||
// says which of the machine's paths are mounted into its container. The engine runs as root, so a path it
|
||||
// accepts blindly is a path anyone who could change those settings hands to any account: a directory at /etc
|
||||
// owned by a caller's account gives it /etc, and an access at / mounts the machine's root into a container.
|
||||
// The controller refuses both where a setting is made; the engine refuses them again where it applies,
|
||||
// because a guard in one place is a guard one change away from gone. Whatever the declaration says:
|
||||
//
|
||||
// 1. **No directory, access or mount source is one of the machine's own roots, or holds one**: /, /etc,
|
||||
// /usr, /var, /var/lib, /home, /run and the rest of protectedRoots. Modules place directories BELOW /etc
|
||||
// or /var/lib, never the root itself; owning one is owning everything in it.
|
||||
// 2. **Nothing is placed in /proc, /sys, /dev, /boot, /root, /var/spool, /opt or the container runtimes' data
|
||||
// (/var/lib/docker, /var/lib/containers, /var/lib/containerd, and where the runtimes' configuration moves
|
||||
// them: runtimeDataRoots), nor in the node-engine's
|
||||
// own trees** (its state, its identity, its installed builds) but by its own module; nothing is mounted from
|
||||
// those but /proc, /sys and /dev. A mount of a kernel file, a device or the clock is the plumbing
|
||||
// systemPath names.
|
||||
// 2a. **An account's .ssh is never an access or a mount**, and is a directory only below its account's home,
|
||||
// as that account's (rule 4).
|
||||
// 3. **A directory below /etc, /usr or /run is root's.** The machine's configuration and programs are read
|
||||
// as root's word; a directory there owned by another account is that account writing root's word. An
|
||||
// access is never there at all: the operator's data is not the machine's configuration.
|
||||
// 4. **Below a person's or an agent's home, a directory is that account's.** Root's or another account's
|
||||
// directory there is one the account does not control in a tree whose every parent it does. A home
|
||||
// itself may be a module's directory (a backup repository kept as an account's home is one), and as
|
||||
// every directory the mesh did not make, it is used as found: never chowned or chmodded (applyDirectory).
|
||||
// 5. **A mount source that is a refused directory or a refused access is refused with it**: the container
|
||||
// would otherwise bind the very path the engine would not place, and the runtime creates a missing one
|
||||
// as root.
|
||||
//
|
||||
// Each is a failed resource with its reason in the node's report; nothing is touched. Paths are judged as
|
||||
// declared and again with every link in them resolved, so a link at /srv/x pointing at /etc places nothing.
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
)
|
||||
|
||||
// protectedRoots are paths no directory, access or mount source may be, nor hold.
|
||||
var protectedRoots = []string{"/", "/bin", "/boot", "/dev", "/etc", "/home", "/lib", "/lib32", "/lib64",
|
||||
"/media", "/mnt", "/opt", "/proc", "/root", "/run", "/sbin", "/srv", "/sys", "/tmp", "/usr", "/usr/bin",
|
||||
"/usr/lib", "/usr/lib64", "/usr/local", "/usr/local/bin", "/usr/local/lib", "/usr/local/sbin", "/usr/sbin",
|
||||
"/usr/share", "/var", "/var/cache", "/var/lib", "/var/lib/mesh", "/var/log", "/var/run", "/var/tmp",
|
||||
"/var/spool"}
|
||||
|
||||
// forbiddenBelow are trees nothing is placed in or mounted from: the kernel's, the boot loader's and root's
|
||||
// home. engineTrees are the node-engine's own, which only its own module places in.
|
||||
var (
|
||||
forbiddenBelow = []string{"/proc", "/sys", "/dev", "/boot", "/root", "/var/spool", "/opt", "/var/lib/docker",
|
||||
"/var/lib/containers", "/var/lib/containerd"}
|
||||
// forbiddenBelowMount are the trees no container mounts from: a mount of the kernel's files and devices is
|
||||
// the plumbing a container may need (systemPath); root's home and the boot loader's are no plumbing.
|
||||
forbiddenBelowMount = []string{"/boot", "/root", "/var/spool", "/opt", "/var/lib/docker", "/var/lib/containers",
|
||||
"/var/lib/containerd"}
|
||||
engineTrees = []string{"/var/lib/mesh-host", "/usr/lib/nox-mesh-host"}
|
||||
// rootsOnly are trees a directory below is root's, and an access is never in.
|
||||
rootsOnly = []string{"/etc", "/usr", "/run", "/var/run"}
|
||||
)
|
||||
|
||||
// runtimeFiles are where the container runtimes say where they keep their data: dockerd's daemon.json, its
|
||||
// unit and the unit's drop-ins (an ExecStart with --data-root, or the older -g/--graph, continued over lines,
|
||||
// quoted, through Environment= or EnvironmentFile=, or a --config-file naming another daemon.json), and podman's
|
||||
// storage.conf (graphroot, a basic or a literal string). The running runtimes are asked first. containerd keeps its own under /var/lib/containerd, which forbiddenBelow names; a
|
||||
// containerd configured elsewhere, and podman's rootless stores under each account's home, are not read: the
|
||||
// first is no runtime this mesh runs, and the second is below a home, which rule 4 already keeps for its
|
||||
// account. A variable so a test names its own.
|
||||
type runtimeFiles struct {
|
||||
daemonJSON string
|
||||
units []string
|
||||
dropInDirs []string
|
||||
storageConf string
|
||||
}
|
||||
|
||||
var runtimeConfigs = runtimeFiles{
|
||||
daemonJSON: "/etc/docker/daemon.json",
|
||||
units: []string{"/etc/systemd/system/docker.service", "/usr/lib/systemd/system/docker.service", "/lib/systemd/system/docker.service"},
|
||||
dropInDirs: []string{"/etc/systemd/system/docker.service.d", "/run/systemd/system/docker.service.d", "/usr/lib/systemd/system/docker.service.d"},
|
||||
storageConf: "/etc/containers/storage.conf",
|
||||
}
|
||||
|
||||
var (
|
||||
dataRootFlag = regexp.MustCompile(`(?:--data-root|--graph|-g)(?:=|\s+)(\S+)`)
|
||||
configFlag = regexp.MustCompile(`--config-file(?:=|\s+)(\S+)`)
|
||||
graphRoot = regexp.MustCompile(`(?m)^\s*graphroot\s*=\s*(?:"([^"]+)"|'([^']+)')`)
|
||||
envVar = regexp.MustCompile(`\$\{([A-Za-z_][A-Za-z0-9_]*)\}|\$([A-Za-z_][A-Za-z0-9_]*)`)
|
||||
)
|
||||
|
||||
// runtimeRoots is where the container runtimes keep their data, as the last apply read it (readRuntimeRoots);
|
||||
// nil until an apply has read it, when the guard reads the files itself.
|
||||
var (
|
||||
runtimeRootsMu sync.Mutex
|
||||
runtimeRoots []string
|
||||
)
|
||||
|
||||
// AskRuntimes is how the engine asks the running container runtimes where they keep their data: set by the engine
|
||||
// to run the commands, nil in a test, which then reads the files alone. Its own, never the apply's runner, whose
|
||||
// commands a test reads back as what the apply did.
|
||||
var AskRuntimes Runner
|
||||
|
||||
// refreshRuntimeRoots reads where the runtimes keep their data once, at the start of an apply.
|
||||
func refreshRuntimeRoots(ctx context.Context) {
|
||||
roots := readRuntimeRoots(ctx, AskRuntimes)
|
||||
runtimeRootsMu.Lock()
|
||||
runtimeRoots = roots
|
||||
runtimeRootsMu.Unlock()
|
||||
}
|
||||
|
||||
// runtimeDataRoots is every place the container runtimes keep their data, beyond the default trees forbiddenBelow
|
||||
// names: every container's filesystem is there.
|
||||
func runtimeDataRoots() []string {
|
||||
runtimeRootsMu.Lock()
|
||||
roots := runtimeRoots
|
||||
runtimeRootsMu.Unlock()
|
||||
if roots != nil {
|
||||
return roots
|
||||
}
|
||||
return readRuntimeRoots(context.Background(), nil)
|
||||
}
|
||||
|
||||
// readRuntimeRoots asks the running runtimes where they keep their data, when run is given (`docker info`,
|
||||
// `podman info`), and reads their configuration besides: an answer from a runtime that is running is what it
|
||||
// really does, and the files say what it will do when it starts again. Both count. A runtime that is not running
|
||||
// or not installed answers nothing, which is no error.
|
||||
func readRuntimeRoots(ctx context.Context, run Runner) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
add := func(p string) {
|
||||
p = strings.Trim(strings.TrimSpace(p), `"'`)
|
||||
if !filepath.IsAbs(p) {
|
||||
return
|
||||
}
|
||||
p = filepath.Clean(p)
|
||||
if !seen[p] {
|
||||
seen[p] = true
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
if run != nil {
|
||||
// Each runtime has its own ten seconds: one that hangs costs the other nothing.
|
||||
for _, q := range [][]string{{"docker", "info", "--format", "{{.DockerRootDir}}"},
|
||||
{"podman", "info", "--format", "{{.Store.GraphRoot}}"}} {
|
||||
ask, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
if root, err := run(ask, q[0], q[1:]...); err == nil {
|
||||
add(root)
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
daemonJSON := func(path string) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var c struct {
|
||||
DataRoot string `json:"data-root"`
|
||||
Graph string `json:"graph"`
|
||||
}
|
||||
if json.Unmarshal(raw, &c) == nil {
|
||||
add(c.DataRoot)
|
||||
add(c.Graph)
|
||||
}
|
||||
}
|
||||
daemonJSON(runtimeConfigs.daemonJSON)
|
||||
units := append([]string(nil), runtimeConfigs.units...)
|
||||
for _, dir := range runtimeConfigs.dropInDirs {
|
||||
matches, _ := filepath.Glob(filepath.Join(dir, "*.conf"))
|
||||
units = append(units, matches...)
|
||||
}
|
||||
// The unit and its drop-ins are one unit to systemd: a variable set in one is seen by an ExecStart in another.
|
||||
env := map[string]string{}
|
||||
var execs []string
|
||||
for _, u := range units {
|
||||
raw, err := os.ReadFile(u)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
e, x := unitLines(string(raw))
|
||||
for k, v := range e {
|
||||
env[k] = v
|
||||
}
|
||||
execs = append(execs, x...)
|
||||
}
|
||||
for _, line := range execs {
|
||||
line = envVar.ReplaceAllStringFunc(line, func(ref string) string {
|
||||
m := envVar.FindStringSubmatch(ref)
|
||||
if v, ok := env[m[1]+m[2]]; ok {
|
||||
return v
|
||||
}
|
||||
return ref
|
||||
})
|
||||
for _, m := range dataRootFlag.FindAllStringSubmatch(line, -1) {
|
||||
add(m[1])
|
||||
}
|
||||
for _, m := range configFlag.FindAllStringSubmatch(line, -1) {
|
||||
daemonJSON(strings.Trim(m[1], `"'`))
|
||||
}
|
||||
}
|
||||
if raw, err := os.ReadFile(runtimeConfigs.storageConf); err == nil {
|
||||
for _, m := range graphRoot.FindAllStringSubmatch(string(raw), -1) {
|
||||
add(m[1] + m[2])
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// unitLines reads a unit file as systemd does for what matters here: a line ending in a backslash continues on the
|
||||
// next, Environment= sets variables (quoted or not, several to a line), EnvironmentFile= (a leading - says it may
|
||||
// be missing) reads KEY=value lines, and every ExecStart line is returned whole.
|
||||
func unitLines(text string) (map[string]string, []string) {
|
||||
var joined []string
|
||||
var cur strings.Builder
|
||||
for _, line := range strings.Split(text, "\n") {
|
||||
trimmed := strings.TrimRight(line, " \t")
|
||||
if strings.HasSuffix(trimmed, "\\") {
|
||||
cur.WriteString(strings.TrimSuffix(trimmed, "\\") + " ")
|
||||
continue
|
||||
}
|
||||
cur.WriteString(line)
|
||||
joined = append(joined, cur.String())
|
||||
cur.Reset()
|
||||
}
|
||||
if cur.Len() > 0 {
|
||||
joined = append(joined, cur.String())
|
||||
}
|
||||
env := map[string]string{}
|
||||
setPairs := func(s string) {
|
||||
for _, f := range splitQuoted(s) {
|
||||
if k, v, ok := strings.Cut(f, "="); ok {
|
||||
env[strings.TrimSpace(k)] = strings.Trim(strings.TrimSpace(v), `"'`)
|
||||
}
|
||||
}
|
||||
}
|
||||
var execs []string
|
||||
for _, line := range joined {
|
||||
l := strings.TrimSpace(line)
|
||||
switch {
|
||||
case strings.HasPrefix(l, "Environment="):
|
||||
setPairs(strings.TrimPrefix(l, "Environment="))
|
||||
case strings.HasPrefix(l, "EnvironmentFile="):
|
||||
path := strings.TrimPrefix(strings.TrimSpace(strings.TrimPrefix(l, "EnvironmentFile=")), "-")
|
||||
if raw, err := os.ReadFile(path); err == nil {
|
||||
for _, kv := range strings.Split(string(raw), "\n") {
|
||||
kv = strings.TrimSpace(kv)
|
||||
if kv == "" || strings.HasPrefix(kv, "#") {
|
||||
continue
|
||||
}
|
||||
setPairs(kv)
|
||||
}
|
||||
}
|
||||
case strings.HasPrefix(l, "ExecStart"):
|
||||
execs = append(execs, l)
|
||||
}
|
||||
}
|
||||
return env, execs
|
||||
}
|
||||
|
||||
// splitQuoted splits on blanks outside double or single quotes, keeping the quotes' contents whole.
|
||||
func splitQuoted(s string) []string {
|
||||
var out []string
|
||||
var cur strings.Builder
|
||||
var quote rune
|
||||
for _, r := range s {
|
||||
switch {
|
||||
case quote != 0 && r == quote:
|
||||
quote = 0
|
||||
case quote == 0 && (r == '"' || r == '\''):
|
||||
quote = r
|
||||
case quote == 0 && (r == ' ' || r == '\t'):
|
||||
if cur.Len() > 0 {
|
||||
out = append(out, cur.String())
|
||||
cur.Reset()
|
||||
}
|
||||
default:
|
||||
cur.WriteRune(r)
|
||||
}
|
||||
}
|
||||
if cur.Len() > 0 {
|
||||
out = append(out, cur.String())
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// engineModule is the module whose resources may place in the engine's own trees.
|
||||
const engineModule = "mesh-host"
|
||||
|
||||
// passwdFile is the user database homes are read from. A variable so a test names its own.
|
||||
var passwdFile = "/etc/passwd"
|
||||
|
||||
// PlacementRefusedError is a resource the engine will not place, or mount, where it says.
|
||||
type PlacementRefusedError struct {
|
||||
Path, Why string
|
||||
}
|
||||
|
||||
func (e *PlacementRefusedError) Error() string {
|
||||
return fmt.Sprintf("%s is not placed: %s (novox/hq issue 339); nothing was touched", e.Path, e.Why)
|
||||
}
|
||||
|
||||
// homeAccount is a person's or an agent's account and its home.
|
||||
type homeAccount struct {
|
||||
Name string
|
||||
UID int
|
||||
}
|
||||
|
||||
// accountsOfHomes is each person's or agent's home and the account it belongs to, from the user database: an
|
||||
// account with a uid of 1000 or more, or a home under /home. A variable so a test names its own.
|
||||
var accountsOfHomes = func() map[string]homeAccount {
|
||||
f, err := os.Open(passwdFile)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer f.Close()
|
||||
out := map[string]homeAccount{}
|
||||
sc := bufio.NewScanner(f)
|
||||
for sc.Scan() {
|
||||
fields := strings.Split(sc.Text(), ":")
|
||||
if len(fields) < 6 {
|
||||
continue
|
||||
}
|
||||
uid, err := strconv.Atoi(fields[2])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
home := filepath.Clean(fields[5])
|
||||
if home == "/" || home == "." || home == "" || home == "/nonexistent" {
|
||||
continue
|
||||
}
|
||||
if (uid >= 1000 && uid != 65534) || strings.HasPrefix(home, "/home/") {
|
||||
out[home] = homeAccount{Name: fields[0], UID: uid}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// below says whether path is strictly below dir.
|
||||
func below(path, dir string) bool {
|
||||
if dir == "/" {
|
||||
return path != "/"
|
||||
}
|
||||
return strings.HasPrefix(path, dir+"/")
|
||||
}
|
||||
|
||||
// atOrBelow says whether path is dir or below it.
|
||||
func atOrBelow(path, dir string) bool { return path == dir || below(path, dir) }
|
||||
|
||||
// resolved is a path with every link in it followed, as far as the path exists, and the rest as declared.
|
||||
func resolved(path string) string {
|
||||
rest := ""
|
||||
for p := path; ; p = filepath.Dir(p) {
|
||||
if real, err := filepath.EvalSymlinks(p); err == nil {
|
||||
return filepath.Clean(filepath.Join(real, rest))
|
||||
}
|
||||
if filepath.Dir(p) == p {
|
||||
return path
|
||||
}
|
||||
rest = filepath.Join(filepath.Base(p), rest)
|
||||
}
|
||||
}
|
||||
|
||||
// ownedByAccount says whether a declared owner is that account: by name, or by its uid ("1001", "1001:1001").
|
||||
func ownedByAccount(owner string, a homeAccount) bool {
|
||||
if owner == a.Name {
|
||||
return true
|
||||
}
|
||||
user, _, _ := strings.Cut(owner, ":")
|
||||
if uid, err := strconv.Atoi(user); err == nil {
|
||||
return uid == a.UID
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// rootOwner says whether a declared owner is root: none, "root", or uid 0.
|
||||
func rootOwner(owner string) bool {
|
||||
if owner == "" || owner == "root" {
|
||||
return true
|
||||
}
|
||||
user, _, _ := strings.Cut(owner, ":")
|
||||
return user == "0"
|
||||
}
|
||||
|
||||
// what a guarded path is, for the words of a refusal.
|
||||
type placing int
|
||||
|
||||
const (
|
||||
placingDirectory placing = iota
|
||||
placingAccess
|
||||
placingMount
|
||||
)
|
||||
|
||||
// refusePath says why a path is not placed or mounted; nil when it may be. module is the resource's module,
|
||||
// owner a directory's declared owner.
|
||||
func refusePath(path string, kind placing, module, owner string) error {
|
||||
clean := filepath.Clean(path)
|
||||
if !filepath.IsAbs(clean) {
|
||||
return nil // the declaration refuses a relative path already; a named volume is not a path
|
||||
}
|
||||
for _, p := range []string{clean, resolved(clean)} {
|
||||
if err := refuseOne(p, kind, module, owner); err != nil {
|
||||
if p != clean {
|
||||
err.Why = fmt.Sprintf("through a link, it is %s, and %s", p, err.Why)
|
||||
err.Path = clean
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func refuseOne(path string, kind placing, module, owner string) *PlacementRefusedError {
|
||||
for _, root := range protectedRoots {
|
||||
if path == root || below(root, path) {
|
||||
return &PlacementRefusedError{Path: path, Why: root + " is one of the machine's own directories, " +
|
||||
"and owning or mounting it would be owning or mounting everything in it"}
|
||||
}
|
||||
}
|
||||
trees := append([]string(nil), forbiddenBelow...)
|
||||
if kind == placingMount {
|
||||
// A mount is the machine's plumbing as often as a module's data — the clock, a kernel file, /dev/null
|
||||
// (systemPath) — and what a setting can mount at all is a directory or an access, refused above it.
|
||||
trees = append([]string(nil), forbiddenBelowMount...)
|
||||
}
|
||||
// The container runtimes' data, wherever the machine keeps it: every container's filesystem is there.
|
||||
trees = append(trees, runtimeDataRoots()...)
|
||||
for _, tree := range trees {
|
||||
if atOrBelow(path, tree) {
|
||||
return &PlacementRefusedError{Path: path, Why: "nothing is placed in or mounted from " + tree}
|
||||
}
|
||||
}
|
||||
if module != engineModule {
|
||||
for _, tree := range engineTrees {
|
||||
if atOrBelow(path, tree) {
|
||||
return &PlacementRefusedError{Path: path, Why: tree + " is the node-engine's own, placed in by " +
|
||||
"its own module alone"}
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, tree := range rootsOnly {
|
||||
if !below(path, tree) {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case kind == placingAccess:
|
||||
return &PlacementRefusedError{Path: path, Why: "an access is the operator's data, and " + tree +
|
||||
" is the machine's own"}
|
||||
case kind == placingDirectory && !rootOwner(owner):
|
||||
return &PlacementRefusedError{Path: path, Why: fmt.Sprintf("a directory below %s is root's, and this "+
|
||||
"one is declared %s's", tree, owner)}
|
||||
}
|
||||
}
|
||||
ssh := false
|
||||
for _, part := range strings.Split(path, "/") {
|
||||
ssh = ssh || part == ".ssh"
|
||||
}
|
||||
if ssh && kind != placingDirectory {
|
||||
return &PlacementRefusedError{Path: path, Why: "it is an account's .ssh, which holds its keys and who may " +
|
||||
"log in as it, and is never an access or a mount"}
|
||||
}
|
||||
if kind != placingDirectory {
|
||||
return nil
|
||||
}
|
||||
homes := accountsOfHomes()
|
||||
var deepest string
|
||||
for home := range homes {
|
||||
if below(path, home) && len(home) > len(deepest) {
|
||||
deepest = home
|
||||
}
|
||||
}
|
||||
if ssh && deepest == "" {
|
||||
return &PlacementRefusedError{Path: path, Why: "a .ssh directory is placed only below its account's home, " +
|
||||
"as that account's"}
|
||||
}
|
||||
if deepest != "" {
|
||||
if a := homes[deepest]; !ownedByAccount(owner, a) {
|
||||
if owner == "" {
|
||||
owner = "root"
|
||||
}
|
||||
return &PlacementRefusedError{Path: path, Why: fmt.Sprintf("it is below %s's home and declared %s's; "+
|
||||
"below a home only that account's directories are placed", a.Name, owner)}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// moduleOfID is the module a resource id names, or "".
|
||||
func moduleOfID(id string) string {
|
||||
module, _ := moduleOf(id)
|
||||
return module
|
||||
}
|
||||
|
||||
// refusedPlaces judges every directory and access of a declaration before anything is applied, and answers
|
||||
// each refusal by path: what is refused is refused again as a container's mount source.
|
||||
func refusedPlaces(resources []declaration.Resource) map[string]error {
|
||||
out := map[string]error{}
|
||||
for _, r := range resources {
|
||||
var err error
|
||||
switch res := r.(type) {
|
||||
case *declaration.Directory:
|
||||
err = refusePath(res.Path, placingDirectory, moduleOfID(res.ID), res.Owner)
|
||||
case *declaration.Access:
|
||||
err = refusePath(res.Path, placingAccess, moduleOfID(res.ID), "")
|
||||
default:
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
out[filepath.Clean(r.Target())] = err
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// refuseMounts says why a container's mounts are not made; nil when they may be.
|
||||
func refuseMounts(c *declaration.Container, refused map[string]error) error {
|
||||
for _, v := range c.Volumes {
|
||||
src := mountSource(v)
|
||||
if !strings.HasPrefix(src, "/") {
|
||||
continue // a named volume, which the runtime keeps in its own tree
|
||||
}
|
||||
src = filepath.Clean(src)
|
||||
for path, why := range refused {
|
||||
if atOrBelow(src, path) {
|
||||
var refusal *PlacementRefusedError
|
||||
if errors.As(why, &refusal) {
|
||||
return &PlacementRefusedError{Path: src, Why: "it is mounted from " + path +
|
||||
", which is refused: " + refusal.Why}
|
||||
}
|
||||
return why
|
||||
}
|
||||
}
|
||||
if err := refusePath(src, placingMount, moduleOfID(c.ID), ""); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user