Let a module declare a log the controller creates and never removes (ADR 0297)
The issue tracker's history lasts seven days on the events stream and its bucket fills toward a cap with no warning (issues 501, 502). A module may now declare `logs`; the controller creates each as a stream on the bus (create-or-update, never deleted or recreated, reported when undeclared), issues it to the owner's assignments in their membership, grants the runtime exactly the append and direct-read subjects, and raises a condition when any module's bucket or log reaches 75% of its cap, cleared below 70%.
This commit is contained in:
@@ -179,6 +179,14 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
}
|
||||
fmt.Fprintf(out, ", keeps state %s", strings.Join(kept, ", "))
|
||||
}
|
||||
// And the logs it keeps, with their caps (novox/hq ADR 0297).
|
||||
if len(m.Logs) > 0 {
|
||||
kept := make([]string, 0, len(m.Logs))
|
||||
for _, l := range m.Logs {
|
||||
kept = append(kept, fmt.Sprintf("%s (%d MiB)", l.Name, l.Cap()))
|
||||
}
|
||||
fmt.Fprintf(out, ", keeps log %s", strings.Join(kept, ", "))
|
||||
}
|
||||
if len(m.Reads) > 0 {
|
||||
fmt.Fprintf(out, ", reads %s", strings.Join(m.Reads, ", "))
|
||||
}
|
||||
|
||||
@@ -136,6 +136,11 @@ var probeRegistry = []probe{
|
||||
{ID: "D-root", Asserts: "no agent can become root without a person on a machine where the router or a channel " +
|
||||
"proving its sender runs: not by its own account, and not through a tool that runs its command as an account " +
|
||||
"that can", From: "ADR 0259 §8", Kind: kindRootNotFree, Phase: 2, run: probeAgentRoot},
|
||||
// A module's bucket or log filling toward its cap (novox/hq ADR 0297 §6): said at three quarters, cleared
|
||||
// below seven tenths, so a store at its threshold is not raised and cleared on every run.
|
||||
{ID: probeStoreFillID, Asserts: "every module's bucket and log holds less than three quarters of its cap; one " +
|
||||
"said filling is cleared once it holds less than seven tenths", From: "ADR 0297, issue 501",
|
||||
Kind: kindStoreFilling, Phase: 1, run: probeStoreFill},
|
||||
{ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals",
|
||||
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs},
|
||||
// The core's health definitions (novox/hq to-be 45 §8, ADR 0236): what a core component's new build is
|
||||
|
||||
@@ -606,6 +606,13 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
||||
Explanation: "The bus refused messages from a part of the mesh, so what they carried did not happen.",
|
||||
Resolved: "Resolved: the bus takes the messages again"}
|
||||
}),
|
||||
kindStoreFilling: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A module's store on the bus is filling up",
|
||||
Needs: "decide whether to raise its cap or have the module keep less.",
|
||||
Explanation: "A bucket or log a module keeps on the bus holds three quarters of its cap or more. When it " +
|
||||
"is full, the bus refuses what the module writes there.",
|
||||
Resolved: "The store has room again"}
|
||||
}),
|
||||
"stream-wrong": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Part of the bus's storage is wrong",
|
||||
Needs: "check the machine the bus runs on; the details say what is missing.",
|
||||
|
||||
@@ -1300,6 +1300,15 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, se
|
||||
if _, err := broker.RaiseBuckets(broker.OnConn(bus.Conn), buckets); err != nil {
|
||||
return fmt.Errorf("the modules' state could not be asserted on the bus: %w", err)
|
||||
}
|
||||
// **And every declared log, for the same reason** (novox/hq ADR 0297 §2): a membership names its
|
||||
// logs, and a module whose log does not exist fails its first append.
|
||||
logs, err := open.inventory.DeclaredLogs(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the modules' logs could not be read, so no log was asserted: %w", err)
|
||||
}
|
||||
if _, err := broker.RaiseLogs(broker.OnConn(bus.Conn), logs); err != nil {
|
||||
return fmt.Errorf("the modules' logs could not be asserted on the bus: %w", err)
|
||||
}
|
||||
// Every membership is tried, and the first failure named once.
|
||||
issued := 0
|
||||
refused := map[string]error{}
|
||||
@@ -1483,13 +1492,28 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string)
|
||||
fmt.Printf("the bus holds state nothing declares any more, kept because it is data: %s — "+
|
||||
"removing it is a person's act\n", strings.Join(undeclared, ", "))
|
||||
}
|
||||
// Every module's log (novox/hq ADR 0297), from the catalogue, as its buckets: one that nothing
|
||||
// declares any more is said and kept — a log is a module's record, and no path of the mesh removes it.
|
||||
logs, err := inv.DeclaredLogs(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
unlogged, err := broker.RaiseLogs(js, logs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(unlogged) > 0 {
|
||||
fmt.Printf("the bus holds logs nothing declares any more, kept because they are data: %s — "+
|
||||
"removing one is a person's act\n", strings.Join(unlogged, ", "))
|
||||
}
|
||||
// And how every module hears what it consumes: asserted with the rest above, counted here.
|
||||
hearing, err := moduleConsumerCount(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("the bus at %s has its streams, %d machine(s) can hear a declaration, %d module(s) "+
|
||||
"can hear what they consume, and %d bucket(s) of state\n", broker.BareAddress(address), len(names), hearing, len(buckets))
|
||||
"can hear what they consume, %d bucket(s) of state and %d log(s)\n", broker.BareAddress(address), len(names), hearing,
|
||||
len(buckets), len(logs))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// **A filling bucket or log is said before it is full** (novox/hq ADR 0297 §6, issue 501).
|
||||
//
|
||||
// A module's bucket and its log each have a cap on the bus, and when either is full the bus refuses
|
||||
// the next write: the module stops doing what it writes for, and nothing said so before. On
|
||||
// 2026-10-11 the issue tracker's bucket held a sixth of its cap after two days, growing toward a stop
|
||||
// nobody would have heard coming. So the self-check reads every module's bucket and log, and one at
|
||||
// fillRaiseAt of its cap or more raises a condition naming the module, the store and how full it is.
|
||||
//
|
||||
// **Cleared by observation below fillClearBelow, not below fillRaiseAt**: a store hovering at its
|
||||
// threshold would otherwise be raised and cleared on every run. Between the two, an open condition is
|
||||
// kept and none is raised.
|
||||
|
||||
// The fill thresholds, in percent of a store's cap.
|
||||
const (
|
||||
fillRaiseAt = 75
|
||||
fillClearBelow = 70
|
||||
)
|
||||
|
||||
// kindStoreFilling is the condition a filling bucket or log raises.
|
||||
const kindStoreFilling = "store-filling"
|
||||
|
||||
// probeStoreFillID is the probe's id in the registry.
|
||||
const probeStoreFillID = "D-fill"
|
||||
|
||||
// moduleStore is one module's bucket or log as the fill probe reads it.
|
||||
type moduleStore struct {
|
||||
Module string
|
||||
// Kind is `bucket` or `log`; Name its local name; Stream the stream it is on the bus.
|
||||
Kind, Name, Stream string
|
||||
}
|
||||
|
||||
// filled is how full one store is, read from the bus.
|
||||
type filled struct {
|
||||
Bytes, Max uint64
|
||||
}
|
||||
|
||||
// percent is how full, in whole percent, rounded down.
|
||||
func (f filled) percent() uint64 {
|
||||
if f.Max == 0 {
|
||||
return 0
|
||||
}
|
||||
return f.Bytes * 100 / f.Max
|
||||
}
|
||||
|
||||
// fillKey is where a store's fill condition is kept.
|
||||
func fillKey(s moduleStore) string { return conditions.Key(conditions.ScopeBus, s.Stream, "filling") }
|
||||
|
||||
// fillObservation is what one store's fill says: a finding at fillRaiseAt or above, and, while its
|
||||
// condition is open, at fillClearBelow or above too; nothing otherwise, which is what clears it.
|
||||
func fillObservation(s moduleStore, f filled, open bool) (conditions.Observation, bool) {
|
||||
if f.Max == 0 {
|
||||
return conditions.Observation{}, false // a store with no cap cannot fill
|
||||
}
|
||||
// Compared in bytes, not rounded percent: 74.9% is not 75%.
|
||||
atRaise := f.Bytes*100 >= f.Max*fillRaiseAt
|
||||
aboveClear := f.Bytes*100 >= f.Max*fillClearBelow
|
||||
if !atRaise && !(open && aboveClear) {
|
||||
return conditions.Observation{}, false
|
||||
}
|
||||
pct := f.percent()
|
||||
fill := fmt.Sprintf("%s of %s, %d%%", mibWords(f.Bytes), mibWords(f.Max), pct)
|
||||
return conditions.Observation{
|
||||
Scope: conditions.ScopeBus, ID: s.Stream, Token: "filling", Kind: kindStoreFilling,
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s's %s %s holds %s: at its cap the bus refuses the module's next write",
|
||||
s.Module, s.Kind, s.Name, fill),
|
||||
Headline: fillHeadline(s, pct),
|
||||
Explanation: fmt.Sprintf("The %s %s of %s holds %s. When it is full, the bus refuses what %s writes "+
|
||||
"there, and %s stops doing what it writes for.", s.Kind, s.Name, s.Module, fill, s.Module, s.Module),
|
||||
Needs: fmt.Sprintf("decide whether to raise the cap of this %s or have %s keep less.", s.Kind, s.Module),
|
||||
Resolved: fillResolved(s),
|
||||
Said: fmt.Sprintf("module %s, %s %s (stream %s): %d of %d bytes, %d%%", s.Module, s.Kind, s.Name,
|
||||
s.Stream, f.Bytes, f.Max, pct),
|
||||
}, true
|
||||
}
|
||||
|
||||
// fillHeadline names the store and how full it is, in at most a headline's length.
|
||||
func fillHeadline(s moduleStore, pct uint64) string {
|
||||
for _, h := range []string{
|
||||
fmt.Sprintf("%s's %s %s is %d%% full", s.Module, s.Kind, s.Name, pct),
|
||||
fmt.Sprintf("A %s of %s is %d%% full", s.Kind, s.Module, pct),
|
||||
} {
|
||||
if len(h) <= conditions.HeadlineMax {
|
||||
return h
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("A module's %s is %d%% full", s.Kind, pct)
|
||||
}
|
||||
|
||||
// fillResolved is the line said when the store has room again, in at most a resolved line's length.
|
||||
func fillResolved(s moduleStore) string {
|
||||
if h := fmt.Sprintf("%s's %s %s has room again", s.Module, s.Kind, s.Name); len(h) <= conditions.HeadlineMax {
|
||||
return h
|
||||
}
|
||||
return fmt.Sprintf("A %s of %s has room again", s.Kind, s.Module)
|
||||
}
|
||||
|
||||
// mibWords is a size as a person reads it, in MiB with one decimal.
|
||||
func mibWords(b uint64) string {
|
||||
return fmt.Sprintf("%.1f MiB", float64(b)/(1024*1024))
|
||||
}
|
||||
|
||||
// declaredStores is every module's bucket and log the catalogue declares, by the stream each is.
|
||||
func declaredStores(ctx context.Context, d *doctor) ([]moduleStore, error) {
|
||||
buckets, err := d.open.inventory.DeclaredBuckets(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logs, err := d.open.inventory.DeclaredLogs(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []moduleStore
|
||||
for _, b := range buckets {
|
||||
out = append(out, moduleStore{Module: b.Module, Kind: "bucket", Name: b.Name, Stream: "KV_" + b.Bucket()})
|
||||
}
|
||||
for _, l := range logs {
|
||||
out = append(out, moduleStore{Module: l.Module, Kind: "log", Name: l.Name, Stream: l.Stream()})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// probeStoreFill reads every module's bucket and log on the bus and says each one filling toward its
|
||||
// cap. One not on the bus says nothing here: the controller creates it on its next raise, and what is
|
||||
// not there cannot fill.
|
||||
func probeStoreFill(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
stores, err := declaredStores(ctx, d)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js := d.js.Context()
|
||||
var out []conditions.Observation
|
||||
for _, s := range stores {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
info, err := js.StreamInfo(s.Stream, nats.Context(ctx))
|
||||
switch {
|
||||
case errors.Is(err, nats.ErrStreamNotFound):
|
||||
continue
|
||||
case err != nil:
|
||||
return nil, fmt.Errorf("%s's %s %s cannot be read: %w", s.Module, s.Kind, s.Name, err)
|
||||
}
|
||||
if info.Config.MaxBytes <= 0 {
|
||||
continue
|
||||
}
|
||||
f := filled{Bytes: info.State.Bytes, Max: uint64(info.Config.MaxBytes)}
|
||||
if o, found := fillObservation(s, f, isOpen(ctx, d.keeper, fillKey(s))); found {
|
||||
out = append(out, o)
|
||||
}
|
||||
}
|
||||
return sortedFound(out), nil
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// **A filling log or bucket is said at three quarters of its cap and cleared below seven tenths**
|
||||
// (novox/hq ADR 0297 §6): raised at 75% naming the module, the store and its fill; not at 74.9%; kept
|
||||
// between 70% and 75% while it is open, and not raised there when it is not; and cleared — said no
|
||||
// more — once it reads below 70%, open or not.
|
||||
func TestAFillingStoreIsRaisedAtThreeQuartersAndClearedBelowSevenTenths(t *testing.T) {
|
||||
const mib = 1024 * 1024
|
||||
log := moduleStore{Module: "mesh-issues", Kind: "log", Name: "changes", Stream: "LOG_mesh-issues_changes"}
|
||||
bucket := moduleStore{Module: "mesh-issues", Kind: "bucket", Name: "issues", Stream: "KV_mesh-issues_issues"}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
store moduleStore
|
||||
bytes uint64
|
||||
max uint64
|
||||
open bool
|
||||
said bool
|
||||
}{
|
||||
{"a log at exactly 75%", log, 768 * mib, 1024 * mib, false, true},
|
||||
{"a bucket at exactly 75%", bucket, 48 * mib, 64 * mib, false, true},
|
||||
{"a bucket full", bucket, 64 * mib, 64 * mib, false, true},
|
||||
{"a log just under 75%", log, 768*mib - 1, 1024 * mib, false, false},
|
||||
{"a bucket at 72%, not open", bucket, 64 * mib * 72 / 100, 64 * mib, false, false},
|
||||
{"a bucket at 72%, open: kept", bucket, 64 * mib * 72 / 100, 64 * mib, true, true},
|
||||
{"a log at exactly 70%, open: kept", log, 700 * mib, 1000 * mib, true, true},
|
||||
{"a log just under 70%, open: cleared", log, 700*mib - 1, 1000 * mib, true, false},
|
||||
{"a bucket at 10%, open: cleared", bucket, 64 * mib / 10, 64 * mib, true, false},
|
||||
{"a store with no cap", bucket, 100, 0, true, false},
|
||||
} {
|
||||
o, said := fillObservation(c.store, filled{Bytes: c.bytes, Max: c.max}, c.open)
|
||||
if said != c.said {
|
||||
t.Errorf("%s: said %v, want %v", c.name, said, c.said)
|
||||
continue
|
||||
}
|
||||
if !said {
|
||||
continue
|
||||
}
|
||||
if o.Key() != fillKey(c.store) || o.Kind != kindStoreFilling || o.Severity != conditions.Warning {
|
||||
t.Errorf("%s: raised as %s (%s, %s)", c.name, o.Key(), o.Kind, o.Severity)
|
||||
}
|
||||
for _, part := range []string{c.store.Module, c.store.Kind + " " + c.store.Name, "%"} {
|
||||
if !strings.Contains(o.Summary, part) || !strings.Contains(o.Headline, part) {
|
||||
t.Errorf("%s: does not name %q: %q / %q", c.name, part, o.Summary, o.Headline)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(o.Summary, " of ") || !strings.Contains(o.Said, "bytes") {
|
||||
t.Errorf("%s: does not say the fill as bytes of the cap: %q / %q", c.name, o.Summary, o.Said)
|
||||
}
|
||||
}
|
||||
o, _ := fillObservation(log, filled{Bytes: 800 * mib, Max: 1024 * mib}, false)
|
||||
if !strings.Contains(o.Summary, "800.0 MiB of 1024.0 MiB, 78%") {
|
||||
t.Errorf("the fill is said as %q", o.Summary)
|
||||
}
|
||||
}
|
||||
|
||||
// The words the operator reads are plain, for every store a module may name.
|
||||
func TestAFillingStoreIsSaidInPlainWords(t *testing.T) {
|
||||
for _, s := range []moduleStore{
|
||||
{Module: "mesh-issues", Kind: "log", Name: "changes", Stream: "LOG_mesh-issues_changes"},
|
||||
{Module: "a-module-with-a-very-long-name-indeed", Kind: "bucket", Name: "a-bucket-with-a-long-name",
|
||||
Stream: "KV_a-module-with-a-very-long-name-indeed_a-bucket-with-a-long-name"},
|
||||
} {
|
||||
o, _ := fillObservation(s, filled{Bytes: 90, Max: 100}, false)
|
||||
if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Explanation: o.Explanation,
|
||||
Resolved: o.Resolved, Needs: o.Needs}); !ok {
|
||||
t.Errorf("%s: %s", s.Stream, why)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user