Compare commits
52
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5ddc59cd32 | ||
|
|
cdbbd9ec4b | ||
|
|
e92a3fe237 | ||
|
|
98ef7bac6e | ||
|
|
db953e7da8 | ||
|
|
7597294ff8 | ||
|
|
9a7ac3af46 | ||
|
|
e7dca6c280 | ||
|
|
c37d7742ba | ||
|
|
ea09f074db | ||
|
|
6a0d84b3f6 | ||
|
|
dd668ec887 | ||
|
|
7dbe80ad63 | ||
|
|
697395af32 | ||
|
|
036b6cc873 | ||
|
|
96c34c52dc | ||
|
|
3808634a9f | ||
|
|
4f80d87750 | ||
|
|
ea92af2a7d | ||
|
|
4469cab7f4 | ||
|
|
875a4e5758 | ||
|
|
bf11a8baac | ||
|
|
65610f2ea2 | ||
|
|
85d664438f | ||
|
|
13b6fc6d97 | ||
|
|
d851573793 | ||
|
|
f6aea4bfbb | ||
|
|
e550c95543 | ||
|
|
1fdff00794 | ||
|
|
5efe999733 | ||
|
|
cdf30349a7 | ||
|
|
40e42606cf | ||
|
|
b8bbf9c79f | ||
|
|
8bcf787258 | ||
|
|
a5a132ac15 | ||
|
|
7f25666cee | ||
|
|
4291fee68e | ||
|
|
b98fd0f396 | ||
|
|
863ebd4277 | ||
|
|
2799e95035 | ||
|
|
582f4a082a | ||
|
|
6c7af5c63f | ||
|
|
9d15f3a39e | ||
|
|
725fcd977e | ||
|
|
1cc6a2d759 | ||
|
|
5d3e52219b | ||
|
|
099c176fa9 | ||
|
|
ec3769a8a6 | ||
|
|
8b2abd08cd | ||
|
|
858b4672dd | ||
|
|
3d7ccc8aeb | ||
|
|
b39eaa485a |
@@ -720,6 +720,8 @@ type answers struct {
|
|||||||
plans []inventory.Plan
|
plans []inventory.Plan
|
||||||
// paused is whether the build seat takes work, which a plan waiting on it says (ADR 0219).
|
// paused is whether the build seat takes work, which a plan waiting on it says (ADR 0219).
|
||||||
paused pauseView
|
paused pauseView
|
||||||
|
// tierBounds is how long each repository's plan tier may take before it is late (novox/hq issue 296).
|
||||||
|
tierBounds tierBounds
|
||||||
// refused is why a machine cannot be worked out at all, by name. A different thing from every
|
// refused is why a machine cannot be worked out at all, by name. A different thing from every
|
||||||
// other answer here: those are about a machine that was told something, and this is about one
|
// other answer here: those are about a machine that was told something, and this is about one
|
||||||
// that cannot be told anything — it never reaches waiting, because nothing was computed for it
|
// that cannot be told anything — it never reaches waiting, because nothing was computed for it
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
)
|
)
|
||||||
@@ -107,6 +108,28 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
|||||||
names = append(names, name)
|
names = append(names, name)
|
||||||
}
|
}
|
||||||
sort.Strings(names)
|
sort.Strings(names)
|
||||||
|
|
||||||
|
// **Every long-running resource says how it is ready** (novox/hq ADR 0240 rule 8): warned until the
|
||||||
|
// date, refused from it. The count is the catalogue's: its merge check keeps the number and lets a
|
||||||
|
// change lower it, never raise it.
|
||||||
|
undeclared := 0
|
||||||
|
required := !checkNow().Before(catalogue.HealthRequiredFrom)
|
||||||
|
for _, name := range names {
|
||||||
|
missing := catalogue.Undeclared(shelf[name])
|
||||||
|
undeclared += len(missing)
|
||||||
|
if len(missing) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if required {
|
||||||
|
for _, id := range missing {
|
||||||
|
fmt.Fprintf(out, "%s: %s stays up and does not say how it is ready: a long-running resource declares "+
|
||||||
|
"health since %s (novox/hq ADR 0240 rule 8)\n", name, id, catalogue.HealthRequiredFrom.Format("2006-01-02"))
|
||||||
|
}
|
||||||
|
failed += len(missing)
|
||||||
|
faulted[name] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
for _, name := range names {
|
for _, name := range names {
|
||||||
m := shelf[name]
|
m := shelf[name]
|
||||||
if faulted[name] {
|
if faulted[name] {
|
||||||
@@ -138,8 +161,24 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
|||||||
}
|
}
|
||||||
fmt.Fprintf(out, ", keeps %s", strings.Join(kept, ", "))
|
fmt.Fprintf(out, ", keeps %s", strings.Join(kept, ", "))
|
||||||
}
|
}
|
||||||
|
// How what it runs is ready (ADR 0240): each declared check, and what is judged by liveness alone.
|
||||||
|
var checks []string
|
||||||
|
for _, r := range m.Resources {
|
||||||
|
if h, has, _ := catalogue.ReadHealth(r); has {
|
||||||
|
checks = append(checks, fmt.Sprintf("%v by %s", r["id"], catalogue.HealthWords(h)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(checks) > 0 {
|
||||||
|
fmt.Fprintf(out, ", ready: %s", strings.Join(checks, "; "))
|
||||||
|
}
|
||||||
|
if missing := catalogue.Undeclared(m); len(missing) > 0 {
|
||||||
|
fmt.Fprintf(out, "; WARNING: %s stay(s) up and say(s) not how it is ready — judged by liveness alone, "+
|
||||||
|
"refused from %s (ADR 0240 rule 8)", strings.Join(missing, ", "), catalogue.HealthRequiredFrom.Format("2006-01-02"))
|
||||||
|
}
|
||||||
fmt.Fprintln(out)
|
fmt.Fprintln(out)
|
||||||
}
|
}
|
||||||
|
// The count the catalogue keeps (ADR 0240 rule 8), in a line its merge check reads.
|
||||||
|
fmt.Fprintf(out, "%s %d\n", UndeclaredHealthLine, undeclared)
|
||||||
if failed > 0 {
|
if failed > 0 {
|
||||||
return fmt.Errorf("%d problem(s) in %d manifest(s)", failed, len(paths))
|
return fmt.Errorf("%d problem(s) in %d manifest(s)", failed, len(paths))
|
||||||
}
|
}
|
||||||
@@ -150,6 +189,13 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// UndeclaredHealthLine starts the line `module check` says the count of long-running resources without
|
||||||
|
// `health` in, over the manifests given: the catalogue's merge check compares it with the number it keeps.
|
||||||
|
const UndeclaredHealthLine = "long-running resources without health:"
|
||||||
|
|
||||||
|
// checkNow is the clock `module check` judges the date by; a test sets it.
|
||||||
|
var checkNow = time.Now
|
||||||
|
|
||||||
func joinInvokes(invokes []string) string {
|
func joinInvokes(invokes []string) string {
|
||||||
if len(invokes) == 1 && invokes[0] == "*" {
|
if len(invokes) == 1 && invokes[0] == "*" {
|
||||||
return "every tool"
|
return "every tool"
|
||||||
|
|||||||
@@ -406,11 +406,11 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return snapshot.Facts{}, err
|
return snapshot.Facts{}, err
|
||||||
}
|
}
|
||||||
mod := snapshot.Module{Name: e.Manifest.Module, Repository: e.Source.Repository, Path: e.Source.Path,
|
mod := snapshot.Module{Name: e.Manifest.Module, Repository: snapshot.RepositoryName(e.Source.Repository), Path: e.Source.Path,
|
||||||
Commit: e.Source.BuiltFrom, Provided: e.Provided, RollOut: current[e.Manifest.Module].RollOut,
|
Commit: e.Source.BuiltFrom, Provided: e.Provided, RollOut: current[e.Manifest.Module].RollOut,
|
||||||
Manifest: raw}
|
Manifest: raw}
|
||||||
for _, r := range read[e.Manifest.Module] {
|
for _, r := range read[e.Manifest.Module] {
|
||||||
mod.Reads = append(mod.Reads, r.Repository)
|
mod.Reads = append(mod.Reads, snapshot.RepositoryName(r.Repository))
|
||||||
}
|
}
|
||||||
f.Modules = append(f.Modules, mod)
|
f.Modules = append(f.Modules, mod)
|
||||||
if e.Provided || e.Source.Repository == "" {
|
if e.Provided || e.Source.Repository == "" {
|
||||||
@@ -426,7 +426,8 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot
|
|||||||
if commit == "" {
|
if commit == "" {
|
||||||
commit = s.BuiltFrom
|
commit = s.BuiltFrom
|
||||||
}
|
}
|
||||||
f.Sources = append(f.Sources, snapshot.Source{Repository: repository, Commit: commit, Modules: count[repository]})
|
f.Sources = append(f.Sources, snapshot.Source{Repository: snapshot.RepositoryName(repository), Commit: commit,
|
||||||
|
Modules: count[repository]})
|
||||||
}
|
}
|
||||||
for _, e := range edges {
|
for _, e := range edges {
|
||||||
f.Edges = append(f.Edges, snapshot.Edge{From: e.From, To: e.To, Kind: e.Kind})
|
f.Edges = append(f.Edges, snapshot.Edge{From: e.From, To: e.To, Kind: e.Kind})
|
||||||
|
|||||||
@@ -77,6 +77,10 @@ type health int
|
|||||||
|
|
||||||
const (
|
const (
|
||||||
healthGood health = iota
|
healthGood health = iota
|
||||||
|
// healthWaiting is not a pass and not a fault: what the module's checks find waits on an unhealthy
|
||||||
|
// provider (ADR 0240 rule 5), so the judging waits — past the bound too — rather than putting back a
|
||||||
|
// build for something it did not do.
|
||||||
|
healthWaiting
|
||||||
healthNotYet
|
healthNotYet
|
||||||
healthBroken
|
healthBroken
|
||||||
)
|
)
|
||||||
@@ -105,6 +109,12 @@ type gateFacts struct {
|
|||||||
// holder is who holds the controller lease, for judging the controller.
|
// holder is who holds the controller lease, for judging the controller.
|
||||||
holder *lease.Holder
|
holder *lease.Holder
|
||||||
holderErr error
|
holderErr error
|
||||||
|
// health is each machine's newest health statement (ADR 0240); a machine absent never stated one.
|
||||||
|
// healthErr is why they could not be read.
|
||||||
|
health map[string]inventory.NodeHealth
|
||||||
|
healthErr error
|
||||||
|
// heldOn is, per "<module>@<machine>", the provider its findings are held under (ADR 0240 rule 5).
|
||||||
|
heldOn map[string]string
|
||||||
}
|
}
|
||||||
|
|
||||||
// gatherGateFacts reads what a judging needs, from the store, the bus and this controller's memory. A
|
// gatherGateFacts reads what a judging needs, from the store, the bus and this controller's memory. A
|
||||||
@@ -127,6 +137,9 @@ var gatherGateFacts = func(ctx context.Context, open *stores, component string)
|
|||||||
for _, n := range nodes {
|
for _, n := range nodes {
|
||||||
f.engines[n.Name] = n.HostVersion
|
f.engines[n.Name] = n.HostVersion
|
||||||
}
|
}
|
||||||
|
// What each machine says of its long-running resources (ADR 0240): unreadable is said, never read as
|
||||||
|
// healthy.
|
||||||
|
f.health, f.healthErr = inv.Healths(ctx)
|
||||||
if d := doctorFrom; d != nil {
|
if d := doctorFrom; d != nil {
|
||||||
if d.keeper != nil {
|
if d.keeper != nil {
|
||||||
f.judged = true
|
f.judged = true
|
||||||
@@ -140,6 +153,17 @@ var gatherGateFacts = func(ctx context.Context, open *stores, component string)
|
|||||||
} else {
|
} else {
|
||||||
f.servedErr = errors.New("this process does not serve the mesh, so it cannot ask the bus who serves what")
|
f.servedErr = errors.New("this process does not serve the mesh, so it cannot ask the bus who serves what")
|
||||||
}
|
}
|
||||||
|
// Whose findings wait on an unhealthy provider (ADR 0240 rule 5): their gates wait, not fail.
|
||||||
|
if f.healthErr == nil && f.openErr == nil {
|
||||||
|
if hold, err := readHolding(ctx, inv, f.open); err == nil {
|
||||||
|
f.heldOn = map[string]string{}
|
||||||
|
for machine := range f.health {
|
||||||
|
for module, p := range hold.heldModules(machine) {
|
||||||
|
f.heldOn[module+"@"+machine] = p.Module + " on " + p.Node
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
if theLease != nil {
|
if theLease != nil {
|
||||||
h, found, err := theLease.holder(ctx)
|
h, found, err := theLease.holder(ctx)
|
||||||
switch {
|
switch {
|
||||||
@@ -193,7 +217,9 @@ func judgeHealth(module, component string, m catalogue.Manifest, machine string,
|
|||||||
if !onIt {
|
if !onIt {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if c.Subject.Scope == conditions.ScopeMachine || slices.Contains(strings.Split(c.Subject.ID, "."), module) {
|
// A module's own health condition names it whole, its name's dots and all (ADR 0240).
|
||||||
|
ownHealth := c.Subject.Scope == conditions.ScopeModule && c.Subject.ID == module+"."+machine
|
||||||
|
if c.Subject.Scope == conditions.ScopeMachine || ownHealth || slices.Contains(strings.Split(c.Subject.ID, "."), module) {
|
||||||
return healthNotYet, fmt.Sprintf("raised since it was sent: %s — %s", c.Key, c.Summary)
|
return healthNotYet, fmt.Sprintf("raised since it was sent: %s — %s", c.Key, c.Summary)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -241,6 +267,11 @@ func judgeHealth(module, component string, m catalogue.Manifest, machine string,
|
|||||||
return healthNotYet, fmt.Sprintf("the node tools on %s do not serve %s's tools", machine, module)
|
return healthNotYet, fmt.Sprintf("the node tools on %s do not serve %s's tools", machine, module)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// **And what it runs is stated healthy** (ADR 0240 §4): every long-running resource of it on that
|
||||||
|
// machine, in a statement heard since the send. A resource still starting makes the judging wait.
|
||||||
|
if h, why := moduleHealthWord(module, machine, since, f); h != healthGood {
|
||||||
|
return h, why
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return healthGood, ""
|
return healthGood, ""
|
||||||
}
|
}
|
||||||
@@ -252,6 +283,8 @@ type machineWord struct {
|
|||||||
facts gateFacts
|
facts gateFacts
|
||||||
on map[string]string
|
on map[string]string
|
||||||
whole string
|
whole string
|
||||||
|
// waiting is what holds the machine on something shown to be another's (ADR 0241): the judging waits.
|
||||||
|
waiting string
|
||||||
}
|
}
|
||||||
|
|
||||||
// kindCoreBehind is D10's kind: a machine runs core components older than the mesh holds, or has not
|
// kindCoreBehind is D10's kind: a machine runs core components older than the mesh holds, or has not
|
||||||
@@ -315,6 +348,13 @@ func aboutTheMachine(machine string, moved []string, since time.Time, f gateFact
|
|||||||
}
|
}
|
||||||
case coreBehind:
|
case coreBehind:
|
||||||
// Not what the send moved: said nowhere against it.
|
// Not what the send moved: said nowhere against it.
|
||||||
|
case c.Kind == kindNetworkRewritten || c.Kind == kindNetworkUnreachable && c.Subject.ID != machine:
|
||||||
|
// **Shown to be somebody else's** (ADR 0241): another program rewrote the resolver file the send
|
||||||
|
// did not move, or the machine cannot reach another that is down. Nothing the send did; the
|
||||||
|
// judging waits for it rather than putting back a build at the bound.
|
||||||
|
if w.waiting == "" {
|
||||||
|
w.waiting = machine + "'s network: " + said
|
||||||
|
}
|
||||||
default:
|
default:
|
||||||
if w.whole == "" {
|
if w.whole == "" {
|
||||||
w.whole = machine + " as a whole: " + said
|
w.whole = machine + " as a whole: " + said
|
||||||
@@ -402,6 +442,8 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
|||||||
h, said = healthNotYet, on
|
h, said = healthNotYet, on
|
||||||
} else if w.whole != "" {
|
} else if w.whole != "" {
|
||||||
h, said = healthNotYet, w.whole
|
h, said = healthNotYet, w.whole
|
||||||
|
} else if w.waiting != "" {
|
||||||
|
h, said = healthWaiting, w.waiting
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if h != healthGood && !slices.Contains(failing, j.module) {
|
if h != healthGood && !slices.Contains(failing, j.module) {
|
||||||
@@ -421,6 +463,10 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
|||||||
// What broke is put back; what was only not yet healthy beside it is too — they moved together.
|
// What broke is put back; what was only not yet healthy beside it is too — they moved together.
|
||||||
g.Failing = failing
|
g.Failing = failing
|
||||||
decide(g, inventory.GateFailed, why, now)
|
decide(g, inventory.GateFailed, why, now)
|
||||||
|
case worst == healthWaiting:
|
||||||
|
// Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either —
|
||||||
|
// the provider's own condition says what is wrong (ADR 0240 rule 5).
|
||||||
|
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
||||||
case worst == healthNotYet:
|
case worst == healthNotYet:
|
||||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
||||||
if now.Sub(*g.Since) > gateBound {
|
if now.Sub(*g.Since) > gateBound {
|
||||||
|
|||||||
@@ -346,6 +346,48 @@ func TestTheHealthDefinitions(t *testing.T) {
|
|||||||
!strings.Contains(why, "first run") {
|
!strings.Contains(why, "first run") {
|
||||||
t.Errorf("a controller not ready is %v (%s)", h, why)
|
t.Errorf("a controller not ready is %v (%s)", h, why)
|
||||||
}
|
}
|
||||||
|
// What the module runs (novox/hq ADR 0240 §4): a judging passes only when every long-running
|
||||||
|
// resource of it on that machine is stated healthy since the send; starting and unhealthy wait.
|
||||||
|
f = applied("anchor")
|
||||||
|
stated := func(state, reason string, heard time.Time) {
|
||||||
|
f.health = map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Contract: 1, SaidAt: heard, HeardAt: heard,
|
||||||
|
Resources: []inventory.ResourceHealth{
|
||||||
|
{Module: "app", Resource: "app.server", Kind: "container", Target: "app-server", State: state, Reason: reason},
|
||||||
|
{Module: "other", Resource: "other.server", Kind: "container", State: link.StateUnhealthy, Reason: "down"}}}}
|
||||||
|
}
|
||||||
|
plain := catalogue.Manifest{Module: "app"}
|
||||||
|
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthGood {
|
||||||
|
t.Errorf("a machine whose engine states no health is judged as before, not %v (%s)", h, why)
|
||||||
|
}
|
||||||
|
stated(link.StateStarting, "", now)
|
||||||
|
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "starting") {
|
||||||
|
t.Errorf("a resource still starting is not yet a pass: %v (%s)", h, why)
|
||||||
|
}
|
||||||
|
stated(link.StateUnhealthy, "restarting", now)
|
||||||
|
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "restarting") {
|
||||||
|
t.Errorf("a resource unhealthy fails the judging: %v (%s)", h, why)
|
||||||
|
}
|
||||||
|
stated(link.StateHealthy, "", since.Add(-time.Minute))
|
||||||
|
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet {
|
||||||
|
t.Errorf("a statement from before the send says nothing of the new build: %v (%s)", h, why)
|
||||||
|
}
|
||||||
|
stated(link.StateHealthy, "", now)
|
||||||
|
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthGood {
|
||||||
|
t.Errorf("every resource of it healthy since the send — another module's state is not its — is %v (%s)", h, why)
|
||||||
|
}
|
||||||
|
f.healthErr = errors.New("the store is away")
|
||||||
|
if h, _ := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet {
|
||||||
|
t.Errorf("health that cannot be read is never read as healthy: %v", h)
|
||||||
|
}
|
||||||
|
// And the condition it raises after the send holds it, as every condition about it does.
|
||||||
|
f.healthErr = nil
|
||||||
|
f.judged = true
|
||||||
|
f.open = []conditions.Condition{{Key: "module.app.anchor.unhealthy", Kind: kindModuleUnhealthy, Subject: conditions.Subject{
|
||||||
|
Scope: conditions.ScopeModule, ID: "app.anchor", Machine: "anchor"}, Raised: now, Summary: "app on anchor is not healthy"}}
|
||||||
|
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "module.app.anchor.unhealthy") {
|
||||||
|
t.Errorf("a module held on its own unhealthy condition is %v (%s)", h, why)
|
||||||
|
}
|
||||||
|
|
||||||
// A machine that refused what it was sent: broken.
|
// A machine that refused what it was sent: broken.
|
||||||
f = applied("anchor")
|
f = applied("anchor")
|
||||||
r := f.reports["anchor"]
|
r := f.reports["anchor"]
|
||||||
|
|||||||
@@ -36,19 +36,29 @@ type handActVerb struct {
|
|||||||
Decision string
|
Decision string
|
||||||
// DecidedFor limits Decision to these causes; empty, it holds for every act of the verb.
|
// DecidedFor limits Decision to these causes; empty, it holds for every act of the verb.
|
||||||
DecidedFor []string
|
DecidedFor []string
|
||||||
|
// DecidedWhen limits Decision to the acts it answers true for: what the controller read the act to
|
||||||
|
// be from what it did (a push's kind), never a word the person gave.
|
||||||
|
DecidedWhen func(link.HandAct) bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// causeLeakedInLogs is the cause a rotation after a value was printed into a log gives.
|
// causeLeakedInLogs is the cause a rotation after a value was printed into a log gives.
|
||||||
const causeLeakedInLogs = "leaked-in-logs"
|
const causeLeakedInLogs = "leaked-in-logs"
|
||||||
|
|
||||||
|
// causeDrill is the cause of every act `hand-act drill` records, and the one cause `hand-act record`
|
||||||
|
// refuses: a drill has its own verb, so whether an act was a drill is said by the verb a person chose,
|
||||||
|
// never by a word typed into a repair's cause (novox/hq issue 292).
|
||||||
|
const causeDrill = "drill"
|
||||||
|
|
||||||
// handActVerbs is every verb that writes the hand-act log (novox/hq to-be 45 §7). **S15 reads it**:
|
// handActVerbs is every verb that writes the hand-act log (novox/hq to-be 45 §7). **S15 reads it**:
|
||||||
// an act recorded by a verb whose entry names a decision is the mesh working as decided, never a
|
// an act recorded by a verb whose entry names a decision is the mesh working as decided, never a
|
||||||
// repair, and does not count toward `healer-wanted` — whatever cause it gives. A verb not listed, or
|
// repair, and does not count toward `healer-wanted` — whatever cause it gives. A verb not listed, or
|
||||||
// listed without a decision, counts, so a new verb is a repair until its entry says otherwise.
|
// listed without a decision, counts, so a new verb is a repair until its entry says otherwise.
|
||||||
var handActVerbs = []handActVerb{
|
var handActVerbs = []handActVerb{
|
||||||
// Repairs: each repeated is a healer the mesh lacks. A push by hand is exactly what roll-out by
|
// Repairs: each repeated is a healer the mesh lacks. A push by hand is exactly what roll-out by
|
||||||
// default (ADR 0236) exists to end.
|
// default (ADR 0236) exists to end — except a push that only moved builds a `record` policy held for
|
||||||
{Verb: "push"},
|
// a person's word (ADR 0242), which the push itself reads from what it carried (recorded_push.go).
|
||||||
|
{Verb: "push", Decision: "a recorded build moves only by a person's push: that push is the word its " +
|
||||||
|
"upgrade policy asks for (ADR 0242)", DecidedWhen: pushedRecorded},
|
||||||
{Verb: "plans stop"},
|
{Verb: "plans stop"},
|
||||||
{Verb: "plans close"},
|
{Verb: "plans close"},
|
||||||
// A walk started by a person instead of its delivery's owner (novox/hq ADR 0239): the owner down, or
|
// A walk started by a person instead of its delivery's owner (novox/hq ADR 0239): the owner down, or
|
||||||
@@ -57,8 +67,14 @@ var handActVerbs = []handActVerb{
|
|||||||
{Verb: "broker consumer-reset"},
|
{Verb: "broker consumer-reset"},
|
||||||
// Silencing the same condition twice says the condition, or what it watches, wants mending.
|
// Silencing the same condition twice says the condition, or what it watches, wants mending.
|
||||||
{Verb: "conditions silence"},
|
{Verb: "conditions silence"},
|
||||||
// An act done outside the mesh: the mesh cannot tell a repair from a decision there, so it counts.
|
// An act done outside the mesh: the mesh cannot tell a repair from a decision there, so it counts —
|
||||||
{Verb: "hand-act record"},
|
// except a drill recorded through it before `hand-act drill` existed (2026-10-07). It refuses the
|
||||||
|
// cause since, so no act recorded through it now carries it.
|
||||||
|
{Verb: "hand-act record", Decision: "a drill recorded before `hand-act drill` existed: a person's " +
|
||||||
|
"deliberate test, never a repair", DecidedFor: []string{causeDrill}},
|
||||||
|
// A drill: something broken on purpose to see the mesh raise and clear it. A person's test, never a
|
||||||
|
// repair, however often it is run.
|
||||||
|
{Verb: "hand-act drill", Decision: "a drill is a person's deliberate test of the mesh, never a repair"},
|
||||||
// A person's decisions by design.
|
// A person's decisions by design.
|
||||||
{Verb: "retire approve", Decision: "nothing is retired past its bound without a person (ADR 0230)"},
|
{Verb: "retire approve", Decision: "nothing is retired past its bound without a person (ADR 0230)"},
|
||||||
{Verb: "retire reject", Decision: "keeping a consumer active is a person's word (ADR 0230)"},
|
{Verb: "retire reject", Decision: "keeping a consumer active is a person's word (ADR 0230)"},
|
||||||
@@ -82,7 +98,8 @@ func personsDecision(a link.HandAct) bool {
|
|||||||
if v.Verb != a.Verb {
|
if v.Verb != a.Verb {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
return v.Decision != "" && (len(v.DecidedFor) == 0 || slices.Contains(v.DecidedFor, a.Cause))
|
return v.Decision != "" && (len(v.DecidedFor) == 0 || slices.Contains(v.DecidedFor, a.Cause)) &&
|
||||||
|
(v.DecidedWhen == nil || v.DecidedWhen(a))
|
||||||
}
|
}
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
@@ -142,6 +159,19 @@ func (f handActFlags) record(ctx context.Context, verb string, args []string) {
|
|||||||
}
|
}
|
||||||
act := link.HandAct{Verb: verb, Args: args, Why: strings.TrimSpace(*f.why),
|
act := link.HandAct{Verb: verb, Args: args, Why: strings.TrimSpace(*f.why),
|
||||||
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
|
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
|
||||||
|
// A push naming one machine says whether it only moves recorded builds (novox/hq issue 301):
|
||||||
|
// read from what it carries, before it is sent.
|
||||||
|
if verb == "push" && len(args) == 1 && !strings.HasPrefix(args[0], "-") {
|
||||||
|
switch carried, why, err := recordedPushOf(ctx, args[0]); {
|
||||||
|
case err != nil:
|
||||||
|
fmt.Fprintf(os.Stderr, "whether this push only moves recorded builds could not be read, so it is "+
|
||||||
|
"recorded as a push by hand: %v\n", err)
|
||||||
|
case len(carried) > 0:
|
||||||
|
act.Kind, act.Carried = link.KindRecordedBuilds, carried
|
||||||
|
default:
|
||||||
|
fmt.Printf("a push by hand, not of recorded builds only: %s\n", why)
|
||||||
|
}
|
||||||
|
}
|
||||||
err := onTheBus(func(conn *nats.Conn) error {
|
err := onTheBus(func(conn *nats.Conn) error {
|
||||||
written, err := link.RecordHandAct(ctx, conn, act)
|
written, err := link.RecordHandAct(ctx, conn, act)
|
||||||
act = written
|
act = written
|
||||||
@@ -151,6 +181,12 @@ func (f handActFlags) record(ctx context.Context, verb string, args []string) {
|
|||||||
fmt.Fprintf(os.Stderr, "this act by hand could NOT be recorded in the hand-act log, and is done anyway: %v\n", err)
|
fmt.Fprintf(os.Stderr, "this act by hand could NOT be recorded in the hand-act log, and is done anyway: %v\n", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if act.Kind == link.KindRecordedBuilds {
|
||||||
|
fmt.Printf("recorded as %s in the hand-act log: a push of recorded builds (%s) by %s, because %q "+
|
||||||
|
"— the person's word their upgrade policy asks for, which no healer is wanted for\n", act.ID,
|
||||||
|
strings.Join(act.Carried, "; "), act.By, act.Why)
|
||||||
|
return
|
||||||
|
}
|
||||||
fmt.Printf("recorded as %s in the hand-act log: %s, because %q (cause: %s)\n", act.ID, act.By, act.Why, act.Cause)
|
fmt.Printf("recorded as %s in the hand-act log: %s, because %q (cause: %s)\n", act.ID, act.By, act.Why, act.Cause)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -174,6 +210,10 @@ func handActCommand(ctx context.Context, args []string) error {
|
|||||||
return errors.New("hand-act record says the cause too: --cause <word>, the word a second " +
|
return errors.New("hand-act record says the cause too: --cause <word>, the word a second " +
|
||||||
"act for the same reason will use — it is how a repair done twice is found")
|
"act for the same reason will use — it is how a repair done twice is found")
|
||||||
}
|
}
|
||||||
|
if strings.EqualFold(strings.TrimSpace(*f.cause), causeDrill) {
|
||||||
|
return errors.New("a drill is not recorded as a repair: hand-act drill <what was done> --why <text> " +
|
||||||
|
"records it as the person's deliberate test it is, which no healer is wanted for. Nothing was recorded")
|
||||||
|
}
|
||||||
act := link.HandAct{Verb: "hand-act record", Args: []string{what}, Why: strings.TrimSpace(*f.why),
|
act := link.HandAct{Verb: "hand-act record", Args: []string{what}, Why: strings.TrimSpace(*f.why),
|
||||||
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
|
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
|
||||||
return onTheBus(func(conn *nats.Conn) error {
|
return onTheBus(func(conn *nats.Conn) error {
|
||||||
@@ -186,8 +226,12 @@ func handActCommand(ctx context.Context, args []string) error {
|
|||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
if len(args) > 0 && args[0] == "drill" {
|
||||||
|
return handActDrill(ctx, args[1:])
|
||||||
|
}
|
||||||
if len(args) > 0 && args[0] != "list" && !strings.HasPrefix(args[0], "-") {
|
if len(args) > 0 && args[0] != "list" && !strings.HasPrefix(args[0], "-") {
|
||||||
return errors.New("hand-act record <what> --why <text> --cause <word> | hand-acts [--days N] [--json]")
|
return errors.New("hand-act record <what> --why <text> --cause <word> | hand-act drill <what> --why <text> " +
|
||||||
|
"| hand-acts [--days N] [--json]")
|
||||||
}
|
}
|
||||||
if len(args) > 0 && args[0] == "list" {
|
if len(args) > 0 && args[0] == "list" {
|
||||||
args = args[1:]
|
args = args[1:]
|
||||||
@@ -225,6 +269,13 @@ func handActCommand(ctx context.Context, args []string) error {
|
|||||||
fmt.Printf(", condition %s", a.Condition)
|
fmt.Printf(", condition %s", a.Condition)
|
||||||
}
|
}
|
||||||
fmt.Println(")")
|
fmt.Println(")")
|
||||||
|
if pushedRecorded(a) {
|
||||||
|
carried := strings.Join(a.Carried, "; ")
|
||||||
|
if carried == "" {
|
||||||
|
carried = recordedBefore[a.ID]
|
||||||
|
}
|
||||||
|
fmt.Printf(" a push of recorded builds, no repair: %s\n", carried)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if len(repeated) > 0 {
|
if len(repeated) > 0 {
|
||||||
causes := make([]string, 0, len(repeated))
|
causes := make([]string, 0, len(repeated))
|
||||||
@@ -239,6 +290,38 @@ func handActCommand(ctx context.Context, args []string) error {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// handActDrill is `hand-act drill`: an act done on purpose to test the mesh — a module stopped, a
|
||||||
|
// process killed — recorded so the conditions it raises are read as the drill they are. Its cause is
|
||||||
|
// always causeDrill and S15 never counts it (handActVerbs).
|
||||||
|
func handActDrill(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("hand-act drill", flag.ContinueOnError)
|
||||||
|
why := set.String("why", "", "what the drill tests — recorded in the hand-act log (novox/hq to-be 45 §7)")
|
||||||
|
condition := set.String("condition", "", "the key of the condition the drill is meant to raise, if any")
|
||||||
|
positionals, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
what := strings.TrimSpace(strings.Join(positionals, " "))
|
||||||
|
if what == "" {
|
||||||
|
return errors.New("hand-act drill <what was done on purpose> --why <what it tests> [--condition <key>]")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(*why) == "" {
|
||||||
|
return errors.New("a drill says what it tests: --why <text> (recorded in the hand-act log, novox/hq " +
|
||||||
|
"to-be 45 §7). Nothing was recorded")
|
||||||
|
}
|
||||||
|
act := link.HandAct{Verb: "hand-act drill", Args: []string{what}, Why: strings.TrimSpace(*why),
|
||||||
|
Cause: causeDrill, Condition: strings.TrimSpace(*condition)}
|
||||||
|
return onTheBus(func(conn *nats.Conn) error {
|
||||||
|
written, err := link.RecordHandAct(ctx, conn, act)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the drill could not be recorded: %w", err)
|
||||||
|
}
|
||||||
|
fmt.Printf("recorded as %s: %s drilled %q, because %q — a drill, which no healer is wanted for\n",
|
||||||
|
written.ID, written.By, what, written.Why)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
// repairs are the acts that are not a person's decision by design: what S15 counts.
|
// repairs are the acts that are not a person's decision by design: what S15 counts.
|
||||||
func repairs(acts []link.HandAct) []link.HandAct {
|
func repairs(acts []link.HandAct) []link.HandAct {
|
||||||
out := make([]link.HandAct, 0, len(acts))
|
out := make([]link.HandAct, 0, len(acts))
|
||||||
|
|||||||
@@ -92,3 +92,32 @@ func TestDurationsAreSummarisedPerSubject(t *testing.T) {
|
|||||||
t.Fatalf("a minute between words suggests %q", got[1].Suggests)
|
t.Fatalf("a minute between words suggests %q", got[1].Suggests)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **A drill has its own verb** — `hand-act drill`, the seat's `drill` — and `hand-act record` refuses
|
||||||
|
// the cause, so a repair cannot pass for a drill by the word it gives.
|
||||||
|
func TestADrillIsRecordedThroughItsOwnVerb(t *testing.T) {
|
||||||
|
err := handActCommand(context.Background(), []string{"record", "stopped searxng", "--why", "a test", "--cause", "drill"})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "hand-act drill") {
|
||||||
|
t.Errorf("hand-act record --cause drill was not sent to the drill verb: %v", err)
|
||||||
|
}
|
||||||
|
if err := handActCommand(context.Background(), []string{"drill", "stopped searxng"}); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "--why") {
|
||||||
|
t.Errorf("a drill without what it tests: %v", err)
|
||||||
|
}
|
||||||
|
if err := handActCommand(context.Background(), []string{"drill", "--why", "a test"}); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "hand-act drill <what") {
|
||||||
|
t.Errorf("a drill without what was done: %v", err)
|
||||||
|
}
|
||||||
|
argv, err := argvFor("drill", map[string]any{"what": "stopped searxng", "why": "ADR 0240 phase A",
|
||||||
|
"condition": "machine.ace.module.searxng.unhealthy"})
|
||||||
|
if want := "hand-act drill stopped searxng --why ADR 0240 phase A --condition machine.ace.module.searxng.unhealthy"; err != nil ||
|
||||||
|
strings.Join(argv, " ") != want {
|
||||||
|
t.Errorf("the seat's drill: %v %v, want %q", argv, err, want)
|
||||||
|
}
|
||||||
|
if _, err := argvFor("drill", map[string]any{"what": "stopped searxng"}); err == nil {
|
||||||
|
t.Error("the seat's drill without why was not refused")
|
||||||
|
}
|
||||||
|
if repairingCommand([]string{"hand-act", "drill", "x"}) != "hand-act drill" {
|
||||||
|
t.Error("a drill through `command` is not held to why")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,47 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Every catalogue module that runs something long-lived declares how it is ready (novox/hq ADR 0240 rule
|
||||||
|
// 8): `module check` warns and counts the undeclared before the date, and refuses them from it.
|
||||||
|
func TestModuleCheckCountsTheUndeclaredAndRefusesThemFromTheDate(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
digest := "@sha256:" + strings.Repeat("a", 64)
|
||||||
|
path := filepath.Join(dir, "module.json")
|
||||||
|
os.WriteFile(path, []byte(`{"module":"web","listens":[{"name":"web","port":80,"from":"mesh"}],"resources":[
|
||||||
|
{"id":"server","type":"container","name":"web","image":"registry.example/web`+digest+`","ports":["80"],
|
||||||
|
"health":{"kind":"http","endpoint":"web"}},
|
||||||
|
{"id":"worker","type":"container","name":"web-worker","image":"registry.example/web`+digest+`"},
|
||||||
|
{"id":"seed","type":"container","name":"web-seed","image":"registry.example/web`+digest+`","run-once":true}]}`), 0o600)
|
||||||
|
defer func() { checkNow = time.Now }()
|
||||||
|
|
||||||
|
checkNow = func() time.Time { return catalogue.HealthRequiredFrom.Add(-time.Hour) }
|
||||||
|
var out bytes.Buffer
|
||||||
|
if err := moduleCheck([]string{path}, &out); err != nil {
|
||||||
|
t.Fatalf("refused before the date: %v\n%s", err, out.String())
|
||||||
|
}
|
||||||
|
for _, want := range []string{"ready: server by http / on web every 30s", "WARNING: worker stay(s) up",
|
||||||
|
catalogue.HealthRequiredFrom.Format("2006-01-02"), UndeclaredHealthLine + " 1"} {
|
||||||
|
if !strings.Contains(out.String(), want) {
|
||||||
|
t.Errorf("the check does not say %q:\n%s", want, out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
checkNow = func() time.Time { return catalogue.HealthRequiredFrom }
|
||||||
|
out.Reset()
|
||||||
|
if err := moduleCheck([]string{path}, &out); err == nil {
|
||||||
|
t.Fatalf("a long-running resource without health passed after the date:\n%s", out.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(out.String(), "web: worker stays up and does not say how it is ready") {
|
||||||
|
t.Errorf("the refusal does not name the resource:\n%s", out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,94 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The live case of 2026-10-07 (novox/hq issue 299), replayed: the node-uplink seat gained its first
|
||||||
|
// verbs, `resolvers` and `links`, both optional while its holders catch up, and the holders serve
|
||||||
|
// neither. The working set is read from the store, whose rows carry no optional mark, so the seat comes
|
||||||
|
// back through UseSeats as it does live. Its holder on every machine answers nothing for it on the bus,
|
||||||
|
// and must not be judged silent: a seat whose verbs are all optional asks no holder to answer.
|
||||||
|
func TestAHolderOfASeatWhoseVerbsAreAllOptionalIsNotSilent(t *testing.T) {
|
||||||
|
defer catalogue.UseSeats(catalogue.DefaultSeats())
|
||||||
|
var rows []catalogue.Seat
|
||||||
|
for _, s := range catalogue.DefaultSeats() {
|
||||||
|
stored := s
|
||||||
|
stored.Serves = nil
|
||||||
|
for _, v := range s.Serves {
|
||||||
|
v.Optional = false // the store never keeps the mark
|
||||||
|
stored.Serves = append(stored.Serves, v)
|
||||||
|
}
|
||||||
|
rows = append(rows, stored)
|
||||||
|
}
|
||||||
|
catalogue.UseSeats(rows)
|
||||||
|
|
||||||
|
uplink, ok := catalogue.SeatNamed("node-uplink")
|
||||||
|
if !ok || len(uplink.Serves) == 0 {
|
||||||
|
t.Fatalf("node-uplink must serve verbs for this replay: %+v", uplink)
|
||||||
|
}
|
||||||
|
for _, v := range uplink.Serves {
|
||||||
|
if !v.Optional {
|
||||||
|
t.Fatalf("node-uplink's %s is required; this replay is of a seat whose verbs are all optional", v.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
nodes := []string{"anchor", "home-server", "workstation", "laptop"}
|
||||||
|
heard := map[string]bool{}
|
||||||
|
var recorded []catalogue.Held
|
||||||
|
for _, n := range nodes {
|
||||||
|
heard[n] = true
|
||||||
|
recorded = append(recorded, catalogue.Held{Claim: "node-uplink", Scope: catalogue.ScopeNode, Node: n,
|
||||||
|
Module: "networkmanager"})
|
||||||
|
}
|
||||||
|
expected := holdersToHear(catalogue.SeatsWithAProtocol(), recorded, nil, heard, nil, time.Now())
|
||||||
|
if _, asked := expected["node-uplink"]; asked {
|
||||||
|
t.Fatalf("node-uplink's holders are expected to answer though every verb of the seat is optional: %v",
|
||||||
|
expected["node-uplink"])
|
||||||
|
}
|
||||||
|
// Nothing answers for the seat, as live: no silence is said about it.
|
||||||
|
for _, o := range silentHolders(expected, map[string]map[string]bool{}) {
|
||||||
|
if o.ID == "node-uplink."+o.Machine {
|
||||||
|
t.Errorf("a holder serving none of a seat's optional verbs was judged silent: %s", o.Summary)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Silence is still judged on a required verb: a seat with one required and one optional verb, held on a
|
||||||
|
// machine that is heard from, whose holder answers nothing — silent.
|
||||||
|
func TestAHolderServingNoRequiredVerbIsStillSilent(t *testing.T) {
|
||||||
|
seat := catalogue.Seat{Name: "example-seat", Scope: catalogue.ScopeNode, Serves: []catalogue.Verb{
|
||||||
|
{Name: "state"}, {Name: "later", Optional: true}}}
|
||||||
|
recorded := []catalogue.Held{{Claim: "example-seat", Scope: catalogue.ScopeNode, Node: "laptop", Module: "m"}}
|
||||||
|
expected := holdersToHear([]catalogue.Seat{seat}, recorded, nil, map[string]bool{"laptop": true}, nil, time.Now())
|
||||||
|
if !expected["example-seat"]["laptop"] {
|
||||||
|
t.Fatalf("a holder of a seat with a required verb is not expected to answer: %v", expected)
|
||||||
|
}
|
||||||
|
out := silentHolders(expected, map[string]map[string]bool{})
|
||||||
|
if len(out) != 1 || out[0].ID != "example-seat.laptop" || out[0].Token != "silent" {
|
||||||
|
t.Fatalf("a holder serving no required verb is not said to be silent: %+v", out)
|
||||||
|
}
|
||||||
|
if got := silentHolders(expected, map[string]map[string]bool{"example-seat": {"laptop": true}}); len(got) != 0 {
|
||||||
|
t.Fatalf("a holder that answers is said to be silent: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHoldingNeedsAnAnswer(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
serves []catalogue.Verb
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"no verb", nil, false},
|
||||||
|
{"only optional verbs", []catalogue.Verb{{Name: "a", Optional: true}, {Name: "b", Optional: true}}, false},
|
||||||
|
{"a required verb among optional ones", []catalogue.Verb{{Name: "a", Optional: true}, {Name: "b"}}, true},
|
||||||
|
{"only required verbs", []catalogue.Verb{{Name: "a"}}, true},
|
||||||
|
} {
|
||||||
|
if got := holdingNeedsAnAnswer(catalogue.Seat{Name: "s", Serves: c.serves}); got != c.want {
|
||||||
|
t.Errorf("%s: holdingNeedsAnAnswer = %v, want %v", c.name, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,359 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"slices"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A machine says how its network is (novox/hq ADR 0241, which extends ADR 0240 from what a module runs
|
||||||
|
// to the machine it runs on).
|
||||||
|
//
|
||||||
|
// **Every machine's node-engine judges its own networking** — the resolver file the uplink holder
|
||||||
|
// declared, the names through every resolver it lists, the tunnel's handshake with the hub, the bus, the
|
||||||
|
// default route — on the two-look rule, and states it beside its resources. The controller keeps the
|
||||||
|
// newest statement per machine and raises from all of them together:
|
||||||
|
//
|
||||||
|
// - **an outside writer of the resolver file is its own finding**, `machine.<m>.<owner>.rewritten`:
|
||||||
|
// the file the uplink holder declares was rewritten by another program, named where the engine could
|
||||||
|
// name it. The names failing through what that program wrote are that finding's consequence, said in
|
||||||
|
// it — never a second condition;
|
||||||
|
// - **what is the machine's own** — its route, its tunnel, its resolvers answering wrong, a resolver
|
||||||
|
// that is no mesh machine — is `machine.<m>.network`;
|
||||||
|
// - **what points at another machine is said once, there** (the provider hold of ADR 0240 rule 5, for
|
||||||
|
// the network): a failure toward the hub or toward a mesh resolver is held under that machine when it
|
||||||
|
// is down on the record — silent, or its own network unhealthy — or when a second machine finds the
|
||||||
|
// same; then `machine.<x>.unreachable` names every machine that cannot reach it, and none of them
|
||||||
|
// raises anything of its own for it. One machine alone failing toward a healthy one is its own.
|
||||||
|
//
|
||||||
|
// Each is a warning; urgent on the control node, or when the bus cannot be reached, or for the hub.
|
||||||
|
// Cleared on the first statement that no longer says it. An engine older than this judging says nothing
|
||||||
|
// of its network, and nothing is raised for it.
|
||||||
|
|
||||||
|
// The conditions a machine's network raises.
|
||||||
|
const (
|
||||||
|
kindMachineNetwork = "machine-network"
|
||||||
|
kindNetworkRewritten = "network-rewritten"
|
||||||
|
kindNetworkUnreachable = "network-unreachable"
|
||||||
|
sourceNetwork = "network"
|
||||||
|
)
|
||||||
|
|
||||||
|
// networkKinds are the kinds this judging owns: every open one it no longer says, it clears.
|
||||||
|
var networkKinds = []string{kindMachineNetwork, kindNetworkRewritten, kindNetworkUnreachable}
|
||||||
|
|
||||||
|
// networkFacts is what one judging of every machine's network reads.
|
||||||
|
type networkFacts struct {
|
||||||
|
healths map[string]inventory.NodeHealth
|
||||||
|
// byAddress is each machine's address on the private network; hub the hub's name; control the
|
||||||
|
// control node's.
|
||||||
|
byAddress map[string]string
|
||||||
|
hub string
|
||||||
|
control string
|
||||||
|
// silent is every machine whose silence is an open condition.
|
||||||
|
silent map[string]bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// judgeNetworks raises and clears every machine's network conditions from every machine's newest
|
||||||
|
// statement, after one machine's statement was kept.
|
||||||
|
func judgeNetworks(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, now time.Time) error {
|
||||||
|
healths, err := inv.Healths(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
overlays, err := inv.Overlays(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
open, err := k.Open(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
f := networkFacts{healths: healths, byAddress: map[string]string{}, control: controlHost(ctx, inv),
|
||||||
|
silent: map[string]bool{}}
|
||||||
|
for _, o := range overlays {
|
||||||
|
if o.Address != "" {
|
||||||
|
f.byAddress[o.Address] = o.Name
|
||||||
|
}
|
||||||
|
if o.Hub {
|
||||||
|
f.hub = o.Name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, c := range open {
|
||||||
|
if c.Subject.Scope == conditions.ScopeMachine && c.Kind == "silent" {
|
||||||
|
f.silent[c.Subject.ID] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var problems []string
|
||||||
|
said := map[string]bool{}
|
||||||
|
for _, o := range networkObservations(f) {
|
||||||
|
said[o.Key()] = true
|
||||||
|
if _, err := k.Observe(ctx, o); err != nil {
|
||||||
|
problems = append(problems, err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, c := range open {
|
||||||
|
if !slices.Contains(networkKinds, c.Kind) || said[c.Key] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
why := "no machine says it any more"
|
||||||
|
if c.Subject.Machine != "" {
|
||||||
|
why = c.Subject.Machine + "'s network no longer says it"
|
||||||
|
}
|
||||||
|
if _, err := k.Clear(ctx, c.Key, why); err != nil {
|
||||||
|
problems = append(problems, err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(problems) > 0 {
|
||||||
|
return fmt.Errorf("%s", strings.Join(problems, "; "))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// pointed is one machine's failing part that points at another machine.
|
||||||
|
type pointed struct {
|
||||||
|
from string
|
||||||
|
part inventory.NetworkPart
|
||||||
|
}
|
||||||
|
|
||||||
|
// networkObservations is every network condition the statements say now. Pure.
|
||||||
|
func networkObservations(f networkFacts) []conditions.Observation {
|
||||||
|
machines := make([]string, 0, len(f.healths))
|
||||||
|
for m := range f.healths {
|
||||||
|
machines = append(machines, m)
|
||||||
|
}
|
||||||
|
sort.Strings(machines)
|
||||||
|
|
||||||
|
unhealthy := func(m string) []inventory.NetworkPart {
|
||||||
|
h := f.healths[m]
|
||||||
|
if h.Network == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var out []inventory.NetworkPart
|
||||||
|
for _, p := range h.Network.Parts {
|
||||||
|
if p.State == link.StateUnhealthy {
|
||||||
|
out = append(out, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
// The machines a part points at, other than its own: the hub, and each mesh resolver by its address.
|
||||||
|
// An address that is no mesh machine's — the resolver a VPN client wrote in — is the machine's own.
|
||||||
|
targets := func(m string, p inventory.NetworkPart) ([]string, bool) {
|
||||||
|
if len(p.Toward) == 0 {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
var out []string
|
||||||
|
for _, t := range p.Toward {
|
||||||
|
x := f.byAddress[t]
|
||||||
|
if t == link.TowardHub {
|
||||||
|
x = f.hub
|
||||||
|
}
|
||||||
|
if x == "" || x == m {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
if !slices.Contains(out, x) {
|
||||||
|
out = append(out, x)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// First pass: what points at whom.
|
||||||
|
pointing := map[string][]pointed{}
|
||||||
|
for _, m := range machines {
|
||||||
|
for _, p := range unhealthy(m) {
|
||||||
|
if xs, ok := targets(m, p); ok {
|
||||||
|
for _, x := range xs {
|
||||||
|
pointing[x] = append(pointing[x], pointed{from: m, part: p})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
from := func(x string) []string {
|
||||||
|
var out []string
|
||||||
|
for _, pt := range pointing[x] {
|
||||||
|
if !slices.Contains(out, pt.from) {
|
||||||
|
out = append(out, pt.from)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
// A machine is down on the record when its silence is open or its own network is unhealthy, or when
|
||||||
|
// two machines find it unreachable: then what points at it is held there.
|
||||||
|
down := func(x string) bool {
|
||||||
|
return f.silent[x] || len(unhealthy(x)) > 0 || len(from(x)) >= 2
|
||||||
|
}
|
||||||
|
|
||||||
|
var out []conditions.Observation
|
||||||
|
saysOwn := map[string]bool{}
|
||||||
|
for _, m := range machines {
|
||||||
|
parts := unhealthy(m)
|
||||||
|
if len(parts) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var own []inventory.NetworkPart
|
||||||
|
var rewritten *inventory.NetworkPart
|
||||||
|
for i, p := range parts {
|
||||||
|
if p.Part == link.PartResolvConf {
|
||||||
|
rewritten = &parts[i]
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if xs, ok := targets(m, p); ok && allDown(xs, down) {
|
||||||
|
continue // held at the machines it points at
|
||||||
|
}
|
||||||
|
own = append(own, p)
|
||||||
|
}
|
||||||
|
if rewritten != nil {
|
||||||
|
out = append(out, rewrittenObservation(m, *rewritten, parts, f))
|
||||||
|
// The names failing through what another program wrote are that finding's, said in it.
|
||||||
|
kept := own[:0]
|
||||||
|
for _, p := range own {
|
||||||
|
if p.Part != link.PartNames {
|
||||||
|
kept = append(kept, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
own = kept
|
||||||
|
}
|
||||||
|
if len(own) > 0 {
|
||||||
|
out = append(out, machineNetworkObservation(m, own, f, from(m)))
|
||||||
|
saysOwn[m] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Said once, at the machine everybody points at — unless its own network condition already says it
|
||||||
|
// (listed there), or its silence does.
|
||||||
|
targetsSorted := make([]string, 0, len(pointing))
|
||||||
|
for x := range pointing {
|
||||||
|
targetsSorted = append(targetsSorted, x)
|
||||||
|
}
|
||||||
|
sort.Strings(targetsSorted)
|
||||||
|
for _, x := range targetsSorted {
|
||||||
|
if !down(x) || saysOwn[x] || f.silent[x] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, unreachableObservation(x, pointing[x], from(x), f))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func allDown(xs []string, down func(string) bool) bool {
|
||||||
|
for _, x := range xs {
|
||||||
|
if !down(x) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return len(xs) > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// rewrittenObservation is the resolver file rewritten by another program: its own finding, naming the
|
||||||
|
// writer where the engine could, and what it costs the machine.
|
||||||
|
func rewrittenObservation(m string, p inventory.NetworkPart, all []inventory.NetworkPart, f networkFacts) conditions.Observation {
|
||||||
|
writer := ""
|
||||||
|
if p.Writer != "" {
|
||||||
|
writer = " (" + p.Writer + ")"
|
||||||
|
}
|
||||||
|
cost := "the names through it are not yet judged"
|
||||||
|
said := []string{p.Part + ": " + p.Said}
|
||||||
|
for _, q := range all {
|
||||||
|
if q.Part == link.PartNames {
|
||||||
|
cost = strings.TrimSuffix(q.Reason, ".")
|
||||||
|
said = append(said, q.Part+": "+q.Said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if cost == "the names through it are not yet judged" {
|
||||||
|
cost = "the names still resolve through what it wrote"
|
||||||
|
}
|
||||||
|
id := m
|
||||||
|
if p.Owner != "" {
|
||||||
|
// Named by the module whose file it is: a send that moved that module is what the gate
|
||||||
|
// holds it on (issue 281's rule — what names a moved module is that module's).
|
||||||
|
id = m + "." + p.Owner
|
||||||
|
}
|
||||||
|
severity := conditions.Warning
|
||||||
|
if m == f.control {
|
||||||
|
severity = conditions.Urgent
|
||||||
|
}
|
||||||
|
summary := fmt.Sprintf("the resolver file on %s was rewritten by another program%s — %s until the "+
|
||||||
|
"node-engine writes it back at its next reconcile, or that program gives it back", m, writer, cost)
|
||||||
|
if p.Owner != "" {
|
||||||
|
summary = fmt.Sprintf("the resolver file %s writes on %s was rewritten by another program%s — %s until "+
|
||||||
|
"the node-engine writes it back at its next reconcile, or that program gives it back", p.Owner, m, writer, cost)
|
||||||
|
}
|
||||||
|
return conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: "rewritten", Kind: kindNetworkRewritten,
|
||||||
|
Machine: m, Severity: severity, Source: sourceNetwork, Summary: summary,
|
||||||
|
Said: fmt.Sprintf("since %s: %s", p.Since.UTC().Format("2006-01-02 15:04:05 MST"), strings.Join(said, " | "))}
|
||||||
|
}
|
||||||
|
|
||||||
|
// machineNetworkObservation is what is wrong with a machine's own networking.
|
||||||
|
func machineNetworkObservation(m string, parts []inventory.NetworkPart, f networkFacts, waiting []string) conditions.Observation {
|
||||||
|
var words, said []string
|
||||||
|
severity := conditions.Warning
|
||||||
|
for _, p := range parts {
|
||||||
|
words = append(words, p.Reason)
|
||||||
|
said = append(said, fmt.Sprintf("%s since %s: %s", p.Part, p.Since.UTC().Format("2006-01-02 15:04:05 MST"), p.Said))
|
||||||
|
if p.Part == link.PartBus {
|
||||||
|
severity = conditions.Urgent
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if m == f.control || m == f.hub {
|
||||||
|
severity = conditions.Urgent
|
||||||
|
}
|
||||||
|
summary := fmt.Sprintf("%s's network is not healthy: %s", m, strings.Join(words, "; "))
|
||||||
|
if len(waiting) > 0 {
|
||||||
|
severity = conditions.Urgent
|
||||||
|
summary += fmt.Sprintf("; %s cannot reach it", strings.Join(waiting, ", "))
|
||||||
|
}
|
||||||
|
return conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Token: "network", Kind: kindMachineNetwork,
|
||||||
|
Machine: m, Severity: severity, Source: sourceNetwork, Summary: summary, Said: strings.Join(said, " | ")}
|
||||||
|
}
|
||||||
|
|
||||||
|
// unreachableObservation is one machine others cannot reach, said once there.
|
||||||
|
func unreachableObservation(x string, pts []pointed, from []string, f networkFacts) conditions.Observation {
|
||||||
|
var what []string
|
||||||
|
var said []string
|
||||||
|
for _, pt := range pts {
|
||||||
|
w := map[string]string{link.PartTunnel: "the tunnel to it", link.PartBus: "the bus on it",
|
||||||
|
link.PartNames: "its resolver"}[pt.part.Part]
|
||||||
|
if w == "" {
|
||||||
|
w = pt.part.Part
|
||||||
|
}
|
||||||
|
if !slices.Contains(what, w) {
|
||||||
|
what = append(what, w)
|
||||||
|
}
|
||||||
|
said = append(said, fmt.Sprintf("%s: %s: %s", pt.from, pt.part.Part, pt.part.Said))
|
||||||
|
}
|
||||||
|
severity := conditions.Warning
|
||||||
|
if x == f.hub || x == f.control || slices.Contains(what, "the bus on it") {
|
||||||
|
severity = conditions.Urgent
|
||||||
|
}
|
||||||
|
return conditions.Observation{Scope: conditions.ScopeMachine, ID: x, Token: "unreachable", Kind: kindNetworkUnreachable,
|
||||||
|
Machine: x, Also: from, Severity: severity, Source: sourceNetwork,
|
||||||
|
Summary: fmt.Sprintf("%s cannot be reached from %s: %s", x, strings.Join(from, ", "), strings.Join(what, ", ")),
|
||||||
|
Said: strings.Join(said, " | ")}
|
||||||
|
}
|
||||||
|
|
||||||
|
// networkLines is what `node show` says of a machine's networking.
|
||||||
|
func networkLines(h inventory.NodeHealth, had bool, now time.Time) []string {
|
||||||
|
if !had || h.Network == nil {
|
||||||
|
return []string{" its node-engine does not say how its network is — it is older than that judging (ADR 0241)"}
|
||||||
|
}
|
||||||
|
out := []string{fmt.Sprintf(" its network: %s since %s", h.Network.State, h.Network.Since.Local().Format("2006-01-02 15:04"))}
|
||||||
|
for _, p := range h.Network.Parts {
|
||||||
|
line := fmt.Sprintf(" %-10s %s", p.State, p.Part)
|
||||||
|
if p.Reason != "" {
|
||||||
|
line += " — " + p.Reason
|
||||||
|
}
|
||||||
|
if p.Writer != "" {
|
||||||
|
line += " (" + p.Writer + ")"
|
||||||
|
}
|
||||||
|
out = append(out, line)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,306 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A machine says how its network is (novox/hq ADR 0241, "how it is checked"): the resolver file rewritten
|
||||||
|
// by another program is one finding, naming the writer, with the names it costs said in it; what is the
|
||||||
|
// machine's own is `machine.<m>.network`; what points at another machine that is down is said once, there;
|
||||||
|
// one machine alone failing toward a healthy one is its own; the control node, the hub and the bus are
|
||||||
|
// urgent; an engine that says nothing of its network raises nothing; and the gate waits on what is shown to
|
||||||
|
// be another's.
|
||||||
|
|
||||||
|
func aNetwork(state string, parts ...inventory.NetworkPart) *inventory.NetworkHealth {
|
||||||
|
for i := range parts {
|
||||||
|
if parts[i].State == "" {
|
||||||
|
parts[i].State = link.StateUnhealthy
|
||||||
|
}
|
||||||
|
parts[i].Since = h0
|
||||||
|
}
|
||||||
|
return &inventory.NetworkHealth{State: state, Since: h0, Parts: parts}
|
||||||
|
}
|
||||||
|
|
||||||
|
func netFacts(healths map[string]*inventory.NetworkHealth) networkFacts {
|
||||||
|
f := networkFacts{healths: map[string]inventory.NodeHealth{}, hub: "anchor", control: "anchor",
|
||||||
|
byAddress: map[string]string{"10.77.0.1": "anchor", "10.77.0.2": "laptop", "10.77.0.3": "spare"},
|
||||||
|
silent: map[string]bool{}}
|
||||||
|
for m, n := range healths {
|
||||||
|
f.healths[m] = inventory.NodeHealth{Node: m, Network: n}
|
||||||
|
}
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
rewrittenByVPN = inventory.NetworkPart{Part: link.PartResolvConf, Reason: "the resolver file was rewritten by another program",
|
||||||
|
Said: "/etc/resolv.conf lists 172.16.5.5 where 10.77.0.1 is declared", Writer: "FortiClient", Owner: "networkmanager"}
|
||||||
|
namesThroughVPN = inventory.NetworkPart{Part: link.PartNames, Reason: "mesh names do not resolve",
|
||||||
|
Said: "172.16.5.5 — anchor.internal (IPv4): says no such name", Toward: []string{"172.16.5.5"}}
|
||||||
|
tunnelDown = inventory.NetworkPart{Part: link.PartTunnel, Reason: "the tunnel to the hub has not handshaken for over five minutes",
|
||||||
|
Said: "mesh0's newest handshake with the hub was 9m0s ago", Toward: []string{link.TowardHub}}
|
||||||
|
noRoute = inventory.NetworkPart{Part: link.PartRoute, Reason: "the machine has no default route", Said: "no default route"}
|
||||||
|
)
|
||||||
|
|
||||||
|
func keysOf(obs []conditions.Observation) []string {
|
||||||
|
var keys []string
|
||||||
|
for _, o := range obs {
|
||||||
|
keys = append(keys, o.Key())
|
||||||
|
}
|
||||||
|
return keys
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAResolverFileRewrittenIsOneFindingNamingItsWriterAndWhatItCosts(t *testing.T) {
|
||||||
|
obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
|
||||||
|
"anchor": aNetwork(link.StateHealthy),
|
||||||
|
"laptop": aNetwork(link.StateUnhealthy, rewrittenByVPN, namesThroughVPN),
|
||||||
|
}))
|
||||||
|
if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" {
|
||||||
|
t.Fatalf("want one finding, the rewrite, got %v", keys)
|
||||||
|
}
|
||||||
|
o := obs[0]
|
||||||
|
for _, want := range []string{"laptop", "rewritten by another program (FortiClient)", "mesh names do not resolve",
|
||||||
|
"next reconcile"} {
|
||||||
|
if !strings.Contains(o.Summary, want) {
|
||||||
|
t.Errorf("the summary does not say %q: %s", want, o.Summary)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(o.Summary, "172.16.") || strings.Contains(o.Summary, "/etc/") {
|
||||||
|
t.Errorf("an address or a path reached the summary: %s", o.Summary)
|
||||||
|
}
|
||||||
|
if !strings.Contains(o.Said, "172.16.5.5") || o.Severity != conditions.Warning || o.Machine != "laptop" {
|
||||||
|
t.Errorf("the evidence or the severity is wrong: %+v", o)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOneMachineFailingTowardAHealthyHubIsItsOwn(t *testing.T) {
|
||||||
|
obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
|
||||||
|
"anchor": aNetwork(link.StateHealthy),
|
||||||
|
"laptop": aNetwork(link.StateUnhealthy, tunnelDown),
|
||||||
|
"spare": aNetwork(link.StateHealthy),
|
||||||
|
}))
|
||||||
|
if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" {
|
||||||
|
t.Fatalf("want the laptop's own, got %v", keys)
|
||||||
|
}
|
||||||
|
if obs[0].Severity != conditions.Warning {
|
||||||
|
t.Fatalf("a laptop's own tunnel is a warning, got %s", obs[0].Severity)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTwoMachinesThatCannotReachTheHubAreSaidOnceAtTheHub(t *testing.T) {
|
||||||
|
obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
|
||||||
|
"anchor": aNetwork(link.StateHealthy),
|
||||||
|
"laptop": aNetwork(link.StateUnhealthy, tunnelDown),
|
||||||
|
"spare": aNetwork(link.StateUnhealthy, tunnelDown),
|
||||||
|
}))
|
||||||
|
if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.unreachable" {
|
||||||
|
t.Fatalf("want one condition at the hub, got %v", keys)
|
||||||
|
}
|
||||||
|
o := obs[0]
|
||||||
|
if o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop, spare") || len(o.Also) != 2 {
|
||||||
|
t.Fatalf("the hub's condition is %+v", o)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWhatPointsAtASilentHubIsHeldUnderItsSilence(t *testing.T) {
|
||||||
|
f := netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, tunnelDown)})
|
||||||
|
f.silent["anchor"] = true
|
||||||
|
if obs := networkObservations(f); len(obs) != 0 {
|
||||||
|
t.Fatalf("the hub's silence says it; got %v", keysOf(obs))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAHubWhoseOwnNetworkIsUnhealthyListsWhoCannotReachIt(t *testing.T) {
|
||||||
|
obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
|
||||||
|
"anchor": aNetwork(link.StateUnhealthy, noRoute),
|
||||||
|
"laptop": aNetwork(link.StateUnhealthy, tunnelDown),
|
||||||
|
}))
|
||||||
|
if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.network" {
|
||||||
|
t.Fatalf("want the hub's own, holding the laptop's, got %v", keys)
|
||||||
|
}
|
||||||
|
if o := obs[0]; o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop cannot reach it") {
|
||||||
|
t.Fatalf("the hub's condition is %+v", o)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOneMachineFailingAHealthyMeshResolverIsItsOwnAndTwoAreTheResolvers(t *testing.T) {
|
||||||
|
silentResolver := inventory.NetworkPart{Part: link.PartNames, Reason: "1 of its 2 resolvers do not answer as the mesh's do",
|
||||||
|
Said: "10.77.0.3 — no answer within 1s", Toward: []string{"10.77.0.3"}}
|
||||||
|
obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
|
||||||
|
"anchor": aNetwork(link.StateHealthy), "spare": aNetwork(link.StateHealthy),
|
||||||
|
"laptop": aNetwork(link.StateUnhealthy, silentResolver),
|
||||||
|
}))
|
||||||
|
if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" {
|
||||||
|
t.Fatalf("one machine alone: want its own, got %v", keys)
|
||||||
|
}
|
||||||
|
obs = networkObservations(netFacts(map[string]*inventory.NetworkHealth{
|
||||||
|
"anchor": aNetwork(link.StateUnhealthy, silentResolver), "spare": aNetwork(link.StateHealthy),
|
||||||
|
"laptop": aNetwork(link.StateUnhealthy, silentResolver),
|
||||||
|
}))
|
||||||
|
if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.spare.unreachable" {
|
||||||
|
t.Fatalf("two machines: want it said once at the resolver's machine, got %v", keys)
|
||||||
|
}
|
||||||
|
if !strings.Contains(obs[0].Summary, "its resolver") {
|
||||||
|
t.Fatalf("the resolver's machine is said %s", obs[0].Summary)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheControlNodeAndTheBusAreUrgent(t *testing.T) {
|
||||||
|
f := netFacts(map[string]*inventory.NetworkHealth{"anchor": aNetwork(link.StateUnhealthy, rewrittenByVPN)})
|
||||||
|
if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent {
|
||||||
|
t.Fatalf("the control node's rewritten file: %+v", obs)
|
||||||
|
}
|
||||||
|
bus := inventory.NetworkPart{Part: link.PartBus, Reason: "the bus cannot be reached", Said: "no link"}
|
||||||
|
f = netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, bus, noRoute)})
|
||||||
|
if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent {
|
||||||
|
t.Fatalf("the bus unreachable from the laptop: %+v", obs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnEngineThatSaysNothingOfItsNetworkRaisesNothing(t *testing.T) {
|
||||||
|
f := netFacts(map[string]*inventory.NetworkHealth{"laptop": nil, "anchor": aNetwork(link.StateHealthy)})
|
||||||
|
if obs := networkObservations(f); len(obs) != 0 {
|
||||||
|
t.Fatalf("got %v", keysOf(obs))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Through the store and the keeper: the statement kept, the rewrite raised from it, cleared when the file
|
||||||
|
// is written back, and node show saying it.
|
||||||
|
func TestARewrittenResolverFileIsRaisedFromTheStatementAndClearedWhenWrittenBack(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv, k := open.inventory, conditionsFrom
|
||||||
|
say := func(at time.Time, n *link.NetworkHealth) {
|
||||||
|
t.Helper()
|
||||||
|
if err := stateHealth(ctx, inv, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
openKeys := func() []string {
|
||||||
|
t.Helper()
|
||||||
|
list, err := k.Open(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var keys []string
|
||||||
|
for _, c := range list {
|
||||||
|
keys = append(keys, c.Key)
|
||||||
|
}
|
||||||
|
return keys
|
||||||
|
}
|
||||||
|
healthy := &link.NetworkHealth{State: link.StateHealthy, Since: h0, Parts: []link.NetworkPart{
|
||||||
|
{Part: link.PartResolvConf, State: link.StateHealthy, Since: h0}}}
|
||||||
|
rewritten := &link.NetworkHealth{State: link.StateUnhealthy, Since: h0.Add(time.Minute), Parts: []link.NetworkPart{
|
||||||
|
{Part: link.PartResolvConf, State: link.StateUnhealthy, Reason: rewrittenByVPN.Reason, Said: rewrittenByVPN.Said,
|
||||||
|
Writer: "FortiClient", Owner: "networkmanager", Since: h0.Add(time.Minute)},
|
||||||
|
{Part: link.PartNames, State: link.StateUnhealthy, Reason: namesThroughVPN.Reason, Said: namesThroughVPN.Said,
|
||||||
|
Toward: namesThroughVPN.Toward, Since: h0.Add(time.Minute)}}}
|
||||||
|
|
||||||
|
say(h0, healthy)
|
||||||
|
if keys := openKeys(); len(keys) != 0 {
|
||||||
|
t.Fatalf("a healthy network raised %v", keys)
|
||||||
|
}
|
||||||
|
say(h0.Add(time.Minute), rewritten)
|
||||||
|
if keys := openKeys(); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" {
|
||||||
|
t.Fatalf("the rewrite raised %v", keys)
|
||||||
|
}
|
||||||
|
kept, had, err := inv.HealthOf(ctx, "laptop")
|
||||||
|
if err != nil || !had || kept.Network == nil || kept.Network.Parts[0].Writer != "FortiClient" {
|
||||||
|
t.Fatalf("the statement's network was not kept: %+v %v", kept.Network, err)
|
||||||
|
}
|
||||||
|
if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "FortiClient") ||
|
||||||
|
!strings.Contains(lines, "unhealthy resolv-conf") {
|
||||||
|
t.Fatalf("node show says:\n%s", lines)
|
||||||
|
}
|
||||||
|
say(h0.Add(2*time.Minute), healthy)
|
||||||
|
if keys := openKeys(); len(keys) != 0 {
|
||||||
|
t.Fatalf("written back, still open: %v", keys)
|
||||||
|
}
|
||||||
|
// An engine older than the judging says no network: nothing raised, and node show says it is not known.
|
||||||
|
say(h0.Add(3*time.Minute), nil)
|
||||||
|
kept, had, _ = inv.HealthOf(ctx, "laptop")
|
||||||
|
if keys := openKeys(); len(keys) != 0 || kept.Network != nil {
|
||||||
|
t.Fatalf("an older engine: %v %+v", keys, kept.Network)
|
||||||
|
}
|
||||||
|
if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "older than that judging") {
|
||||||
|
t.Fatalf("node show says:\n%s", lines)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The gate: a resolver file another program rewrote waits the judging rather than failing it at the
|
||||||
|
// bound; the same, when the send moved the module whose file it is, is that module's; a machine's own
|
||||||
|
// network fault holds the machine as a whole, as before.
|
||||||
|
func TestTheGateWaitsOnARewriteItDidNotMakeAndHoldsTheOwnerOnOneItMoved(t *testing.T) {
|
||||||
|
since := h0
|
||||||
|
rewrite := conditions.Condition{Key: "machine.laptop.networkmanager.rewritten", Kind: kindNetworkRewritten,
|
||||||
|
Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop.networkmanager", Machine: "laptop"},
|
||||||
|
Summary: "the resolver file was rewritten", Raised: since.Add(time.Minute), Source: sourceNetwork}
|
||||||
|
f := gateFacts{judged: true, open: []conditions.Condition{rewrite}}
|
||||||
|
if w := aboutTheMachine("laptop", []string{"letta"}, since, f); w.waiting == "" || w.whole != "" || len(w.on) != 0 {
|
||||||
|
t.Fatalf("a rewrite the send did not make: %+v", w)
|
||||||
|
}
|
||||||
|
if w := aboutTheMachine("laptop", []string{"networkmanager", "letta"}, since, f); w.on["networkmanager"] == "" ||
|
||||||
|
w.on["letta"] != "" || w.waiting != "" {
|
||||||
|
t.Fatalf("a rewrite of the file a moved module owns: %+v", w)
|
||||||
|
}
|
||||||
|
own := conditions.Condition{Key: "machine.laptop.network", Kind: kindMachineNetwork,
|
||||||
|
Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop", Machine: "laptop"},
|
||||||
|
Summary: "laptop's network is not healthy", Raised: since.Add(time.Minute), Source: sourceNetwork}
|
||||||
|
if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true,
|
||||||
|
open: []conditions.Condition{own}}); w.whole == "" || w.waiting != "" {
|
||||||
|
t.Fatalf("the machine's own network: %+v", w)
|
||||||
|
}
|
||||||
|
unreachable := conditions.Condition{Key: "machine.anchor.unreachable", Kind: kindNetworkUnreachable,
|
||||||
|
Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor", Also: []string{"laptop", "spare"}},
|
||||||
|
Summary: "anchor cannot be reached", Raised: since.Add(time.Minute), Source: sourceNetwork}
|
||||||
|
if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true,
|
||||||
|
open: []conditions.Condition{unreachable}}); w.waiting == "" || w.whole != "" {
|
||||||
|
t.Fatalf("a machine that cannot reach the hub: %+v", w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestTheDrillsStatementsRaiseAndClearTheRewrite replays the drill of ADR 0241 (mesh-host
|
||||||
|
// internal/network TestDrill…): what a node-engine said in a throwaway container while its resolver file
|
||||||
|
// was declared, rewritten as a VPN client rewrites it, and written back — recorded, with the mesh's names
|
||||||
|
// and addresses replaced by this test mesh's. Healthy raises nothing; the rewrite, on its second look, is
|
||||||
|
// raised as one finding naming the writer; written back, it clears.
|
||||||
|
func TestTheDrillsStatementsRaiseAndClearTheRewrite(t *testing.T) {
|
||||||
|
raw, err := os.ReadFile("testdata/network-drill.json")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var said []link.Health
|
||||||
|
if err := json.Unmarshal(raw, &said); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
k := conditionsFrom
|
||||||
|
var raisedAt []int
|
||||||
|
for i, h := range said {
|
||||||
|
if err := stateHealth(ctx, open.inventory, k, "laptop", h, h.At); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
list, err := k.Open(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, c := range list {
|
||||||
|
if c.Key != "machine.laptop.networkmanager.rewritten" || !strings.Contains(c.Summary, "(FortiClient)") {
|
||||||
|
t.Fatalf("statement %d raised %s: %s", i, c.Key, c.Summary)
|
||||||
|
}
|
||||||
|
raisedAt = append(raisedAt, i)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Five statements: healthy, healthy, one failing look (still healthy), unhealthy, written back.
|
||||||
|
if len(raisedAt) != 1 || raisedAt[0] != 3 {
|
||||||
|
t.Fatalf("the rewrite was open after statements %v; want after the fourth alone", raisedAt)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,342 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A module says how it is healthy, and the node-engine judges it (novox/hq ADR 0240, to-be 48 §4 and §5,
|
||||||
|
// Phase A).
|
||||||
|
//
|
||||||
|
// **The node-engine owns every verdict; the controller keeps the last word and raises the condition.** A
|
||||||
|
// machine states, in every report and as an event between reports, the state of every long-running
|
||||||
|
// resource it runs for a module. The controller keeps the newest statement per machine (node_health), and
|
||||||
|
// raises `module.<module>.<machine>.unhealthy` when two statements in a row say a resource of the module
|
||||||
|
// is unhealthy — one statement is listed as unconfirmed, as the self-check does a finding one look can be
|
||||||
|
// wrong about (to-be 45 §4, issue 277) — and clears it on the first that does not. The release gate reads
|
||||||
|
// the stated health: a judging passes a module only when every long-running resource of it on that
|
||||||
|
// machine is stated healthy, so a resource still starting is not yet a pass.
|
||||||
|
//
|
||||||
|
// **An engine older than the judging states nothing**, and its machine's health is not known: never
|
||||||
|
// healthy, never a reason to raise anything, and the gate judges it as it did before.
|
||||||
|
|
||||||
|
// The condition a module's health raises.
|
||||||
|
const (
|
||||||
|
kindModuleUnhealthy = "module-unhealthy"
|
||||||
|
// sourceHealth is what raised it: the machine's own statement.
|
||||||
|
sourceHealth = "health"
|
||||||
|
// moduleUnhealthyUrgentAfter is how long it stands before it is urgent (to-be 48 §4).
|
||||||
|
moduleUnhealthyUrgentAfter = 4 * time.Hour
|
||||||
|
// moduleUnhealthyAfter is how many statements in a row raise it.
|
||||||
|
moduleUnhealthyAfter = 2
|
||||||
|
)
|
||||||
|
|
||||||
|
// healthRefused counts the statements refused as older than the one kept, for the log and a test.
|
||||||
|
var healthRefused atomic.Int64
|
||||||
|
|
||||||
|
// moduleHealth keeps what the machines state, for the link (link.Healths).
|
||||||
|
type moduleHealth struct {
|
||||||
|
inv *inventory.Inventory
|
||||||
|
keeper func() *conditions.Keeper
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m moduleHealth) Stated(ctx context.Context, node string, h link.Health) error {
|
||||||
|
return stateHealth(ctx, m.inv, m.keeper(), node, h, time.Now())
|
||||||
|
}
|
||||||
|
|
||||||
|
// stateHealth keeps one machine's statement and raises or clears its modules' conditions from it. An
|
||||||
|
// older statement than the one kept is refused, by when the engine looked.
|
||||||
|
func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, node string, h link.Health,
|
||||||
|
now time.Time) error {
|
||||||
|
if h.Contract == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
prev, had, err := inv.HealthOf(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if had && h.At.Before(prev.SaidAt) {
|
||||||
|
healthRefused.Add(1)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
unhealthy := map[string][]inventory.ResourceHealth{}
|
||||||
|
resources := make([]inventory.ResourceHealth, 0, len(h.Resources))
|
||||||
|
for _, r := range h.Resources {
|
||||||
|
kept := inventory.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target,
|
||||||
|
State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts,
|
||||||
|
Check: r.Check, Needs: r.Needs}
|
||||||
|
resources = append(resources, kept)
|
||||||
|
if r.State == link.StateUnhealthy && r.Module != "" {
|
||||||
|
unhealthy[r.Module] = append(unhealthy[r.Module], kept)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
streaks := map[string]int{}
|
||||||
|
for module := range unhealthy {
|
||||||
|
streaks[module] = prev.Streaks[module] + 1
|
||||||
|
}
|
||||||
|
var network *inventory.NetworkHealth
|
||||||
|
if h.Network != nil {
|
||||||
|
network = &inventory.NetworkHealth{State: h.Network.State, Since: h.Network.Since, Parts: []inventory.NetworkPart{}}
|
||||||
|
for _, p := range h.Network.Parts {
|
||||||
|
network.Parts = append(network.Parts, inventory.NetworkPart{Part: p.Part, State: p.State, Reason: p.Reason,
|
||||||
|
Said: p.Said, Writer: p.Writer, Owner: p.Owner, Toward: p.Toward, Since: p.Since, Streak: p.Streak})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
stored, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: node, Contract: h.Contract, SaidAt: h.At,
|
||||||
|
HeardAt: now, Resources: resources, Streaks: streaks, Network: network})
|
||||||
|
if err != nil || !stored {
|
||||||
|
if err == nil {
|
||||||
|
healthRefused.Add(1)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if k == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
err = judgeModuleHealth(ctx, inv, k, node, unhealthy, streaks, now)
|
||||||
|
// And every machine's network, from every machine's newest statement (ADR 0241): a statement about one
|
||||||
|
// machine can hold another's finding, or release it.
|
||||||
|
if nerr := judgeNetworks(ctx, inv, k, now); nerr != nil {
|
||||||
|
if err == nil {
|
||||||
|
return nerr
|
||||||
|
}
|
||||||
|
return fmt.Errorf("%w; %v", err, nerr)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// judgeModuleHealth raises a module's condition on a machine on the second statement in a row that says a
|
||||||
|
// resource of it is unhealthy — or on the first while it is already open — and clears every one this
|
||||||
|
// statement no longer says. **A consumer whose findings wait on an unhealthy provider is held** (to-be 48
|
||||||
|
// §6): raised as nothing of its own, listed at the provider's condition, which is urgent while anyone
|
||||||
|
// waits on it.
|
||||||
|
func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, node string,
|
||||||
|
unhealthy map[string][]inventory.ResourceHealth, streaks map[string]int, now time.Time) error {
|
||||||
|
open, err := k.Open(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
standing := map[string]conditions.Condition{}
|
||||||
|
for _, c := range open {
|
||||||
|
if c.Kind == kindModuleUnhealthy && c.Subject.Machine == node {
|
||||||
|
standing[c.Key] = c
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var hold *holding
|
||||||
|
if inv != nil {
|
||||||
|
if hold, err = readHolding(ctx, inv, open); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var problems []string
|
||||||
|
modules := make([]string, 0, len(unhealthy))
|
||||||
|
for m := range unhealthy {
|
||||||
|
modules = append(modules, m)
|
||||||
|
}
|
||||||
|
sort.Strings(modules)
|
||||||
|
seen := map[string]bool{}
|
||||||
|
heldOn := map[string]string{}
|
||||||
|
providers := map[catalogue.Chosen]bool{}
|
||||||
|
for _, m := range modules {
|
||||||
|
o := moduleUnhealthyObservation(m, node, unhealthy[m])
|
||||||
|
if hold != nil {
|
||||||
|
if p, held := hold.heldUnder(node, m, unhealthy[m]); held {
|
||||||
|
// Held under the provider's condition: nothing of its own, and the provider's says it waits.
|
||||||
|
heldOn[o.Key()] = p.Module + " on " + p.Node
|
||||||
|
providers[p] = true
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if waiters := hold.waitersOn(catalogue.Chosen{Node: node, Module: m}); len(waiters) > 0 {
|
||||||
|
o.Severity = conditions.Urgent
|
||||||
|
o.Said += "; " + waitingWords(waiters)
|
||||||
|
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
seen[o.Key()] = true
|
||||||
|
c, isOpen := standing[o.Key()]
|
||||||
|
if streaks[m] < moduleUnhealthyAfter && !isOpen {
|
||||||
|
continue // unconfirmed: one statement can be wrong; `node show` lists it
|
||||||
|
}
|
||||||
|
if isOpen && now.Sub(c.Raised) >= moduleUnhealthyUrgentAfter {
|
||||||
|
o.Severity = conditions.Urgent
|
||||||
|
}
|
||||||
|
if _, err := k.Observe(ctx, o); err != nil {
|
||||||
|
problems = append(problems, err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for key, c := range standing {
|
||||||
|
if seen[key] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
module := strings.TrimSuffix(c.Subject.ID, "."+node)
|
||||||
|
why := fmt.Sprintf("%s says no resource of %s is unhealthy", node, module)
|
||||||
|
if on, held := heldOn[key]; held {
|
||||||
|
why = fmt.Sprintf("what %s finds on %s waits on %s, which is unhealthy: held under its condition", module, node, on)
|
||||||
|
}
|
||||||
|
if _, err := k.Clear(ctx, key, why); err != nil {
|
||||||
|
problems = append(problems, err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// And each provider a consumer here now waits on, when its own condition is open: said again with who
|
||||||
|
// waits on it, so the wait is listed at the provider whichever machine's statement arrived first.
|
||||||
|
for p := range providers {
|
||||||
|
if err := sayWaiters(ctx, k, hold, p, now); err != nil {
|
||||||
|
problems = append(problems, err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(problems) > 0 {
|
||||||
|
return fmt.Errorf("%s", strings.Join(problems, "; "))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// sayWaiters observes a provider's open condition again, with who waits on it, from its machine's newest
|
||||||
|
// statement. Nothing when its condition is not open: it is raised by its own statements, on its own looks.
|
||||||
|
func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p catalogue.Chosen, now time.Time) error {
|
||||||
|
var raisedAt *conditions.Condition
|
||||||
|
for i, c := range hold.open {
|
||||||
|
if c.Key == moduleUnhealthyKey(p.Module, p.Node) {
|
||||||
|
raisedAt = &hold.open[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if raisedAt == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var rs []inventory.ResourceHealth
|
||||||
|
for _, r := range hold.healths[p.Node].Resources {
|
||||||
|
if r.Module == p.Module && r.State == link.StateUnhealthy {
|
||||||
|
rs = append(rs, r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(rs) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
o := moduleUnhealthyObservation(p.Module, p.Node, rs)
|
||||||
|
if waiters := hold.waitersOn(p); len(waiters) > 0 {
|
||||||
|
o.Severity = conditions.Urgent
|
||||||
|
o.Said += "; " + waitingWords(waiters)
|
||||||
|
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
|
||||||
|
}
|
||||||
|
_, err := k.Observe(ctx, o)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module,
|
||||||
|
// the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence.
|
||||||
|
func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {
|
||||||
|
var words, said []string
|
||||||
|
for _, r := range rs {
|
||||||
|
words = append(words, fmt.Sprintf("its %s %s %s", r.Kind, r.Resource, reasonWords(r)))
|
||||||
|
said = append(said, fmt.Sprintf("%s (%s %s): %s, %d look(s) in a row, %d restart(s) counted, since %s",
|
||||||
|
r.Resource, r.Kind, r.Target, orNotSaid(r.Reason), r.Streak, r.Restarts,
|
||||||
|
r.Since.UTC().Format("2006-01-02 15:04:05 MST")))
|
||||||
|
}
|
||||||
|
return conditions.Observation{Scope: conditions.ScopeModule, ID: module + "." + node, Token: "unhealthy",
|
||||||
|
Kind: kindModuleUnhealthy, Machine: node, Severity: conditions.Warning, Source: sourceHealth,
|
||||||
|
Summary: fmt.Sprintf("%s on %s is not healthy: %s", module, node, strings.Join(words, "; ")),
|
||||||
|
Said: strings.Join(said, "; ")}
|
||||||
|
}
|
||||||
|
|
||||||
|
// reasonWords is why a resource is unhealthy, as a person reads it.
|
||||||
|
func reasonWords(r inventory.ResourceHealth) string {
|
||||||
|
switch r.Reason {
|
||||||
|
case "restarting":
|
||||||
|
return fmt.Sprintf("keeps restarting (%d restart(s) counted)", r.Restarts)
|
||||||
|
case "down":
|
||||||
|
return "is not running"
|
||||||
|
case "":
|
||||||
|
return "is unhealthy"
|
||||||
|
}
|
||||||
|
// What a declared check found says an endpoint, a path or an address: evidence, never the summary the
|
||||||
|
// operator's channel carries (ADR 0234 §6). The summary names the check.
|
||||||
|
if r.Check != "" {
|
||||||
|
return "fails its " + r.Check + " check"
|
||||||
|
}
|
||||||
|
return "is unhealthy: " + r.Reason
|
||||||
|
}
|
||||||
|
|
||||||
|
func orNotSaid(s string) string {
|
||||||
|
if s == "" {
|
||||||
|
return "no reason said"
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// moduleHealthWord is the gate's reading of a module's stated health on a machine (ADR 0240 §4, ADR 0236
|
||||||
|
// §2 as amended): good when every long-running resource of it is stated healthy in a statement heard since
|
||||||
|
// the send; not yet otherwise, saying which. A machine that never stated health is judged as before.
|
||||||
|
func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (health, string) {
|
||||||
|
if f.healthErr != nil {
|
||||||
|
return healthNotYet, "what " + machine + " says of its resources' health cannot be read: " + firstLine(f.healthErr.Error())
|
||||||
|
}
|
||||||
|
h, states := f.health[machine]
|
||||||
|
if !states {
|
||||||
|
return healthGood, ""
|
||||||
|
}
|
||||||
|
if h.HeardAt.Before(since) {
|
||||||
|
return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module)
|
||||||
|
}
|
||||||
|
for _, r := range h.Resources {
|
||||||
|
if r.Module != module {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
switch r.State {
|
||||||
|
case link.StateHealthy:
|
||||||
|
case link.StateStarting:
|
||||||
|
return healthNotYet, fmt.Sprintf("its %s %s on %s is still starting", r.Kind, r.Resource, machine)
|
||||||
|
case link.StateUnhealthy:
|
||||||
|
if on, held := f.heldOn[module+"@"+machine]; held {
|
||||||
|
return healthWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which is unhealthy", r.Kind,
|
||||||
|
r.Resource, machine, on)
|
||||||
|
}
|
||||||
|
return healthNotYet, fmt.Sprintf("its %s %s on %s %s", r.Kind, r.Resource, machine, reasonWords(r))
|
||||||
|
default:
|
||||||
|
return healthNotYet, fmt.Sprintf("its %s %s on %s is %s%s", r.Kind, r.Resource, machine, r.State,
|
||||||
|
reasonAfter(r.Reason))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return healthGood, ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func reasonAfter(s string) string {
|
||||||
|
if s == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return ": " + s
|
||||||
|
}
|
||||||
|
|
||||||
|
// healthLines is what `node show` says of a machine's long-running resources: each with its state and
|
||||||
|
// since when, an unhealthy one said once marked unconfirmed.
|
||||||
|
func healthLines(h inventory.NodeHealth, had bool, now time.Time) []string {
|
||||||
|
if !had {
|
||||||
|
return []string{" its node-engine does not say how what it runs is — it is older than the judging (ADR 0240)"}
|
||||||
|
}
|
||||||
|
if len(h.Resources) == 0 {
|
||||||
|
return []string{fmt.Sprintf(" it runs nothing long-lived for a module (said %s ago)", roughly(now.Sub(h.HeardAt)))}
|
||||||
|
}
|
||||||
|
out := []string{fmt.Sprintf(" what it runs, as it said %s ago:", roughly(now.Sub(h.HeardAt)))}
|
||||||
|
for _, r := range h.Resources {
|
||||||
|
line := fmt.Sprintf(" %-10s %-34s %s %s, since %s", r.State, r.Resource, r.Kind, r.Target,
|
||||||
|
r.Since.Local().Format("2006-01-02 15:04"))
|
||||||
|
if r.Reason != "" {
|
||||||
|
line += " — " + r.Reason
|
||||||
|
}
|
||||||
|
if r.Restarts > 0 {
|
||||||
|
line += fmt.Sprintf(", %d restart(s) counted", r.Restarts)
|
||||||
|
}
|
||||||
|
if r.State == link.StateUnhealthy && h.Streaks[r.Module] < moduleUnhealthyAfter {
|
||||||
|
line += " (unconfirmed: said once)"
|
||||||
|
}
|
||||||
|
out = append(out, line)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,196 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A module's stated health, as the controller keeps it and raises from it (novox/hq ADR 0240, "how it is
|
||||||
|
// checked", rule 4): one unhealthy statement raises nothing and is listed unconfirmed; two raise; a
|
||||||
|
// healthy one clears; a condition from before the send clears at the new build's start; an older
|
||||||
|
// statement is refused; and an engine that states nothing raises nothing.
|
||||||
|
|
||||||
|
var h0 = time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC)
|
||||||
|
|
||||||
|
func aStatement(at time.Time, states ...string) link.Health {
|
||||||
|
h := link.Health{Contract: link.LivenessContract, At: at}
|
||||||
|
for i, s := range states {
|
||||||
|
r := link.ResourceHealth{Module: "letta", Resource: "letta.server", Kind: "container", Target: "letta-server",
|
||||||
|
State: s, Since: at}
|
||||||
|
if i > 0 {
|
||||||
|
r.Module, r.Resource, r.Target = "mqtt", "mqtt.broker", "mosquitto.service"
|
||||||
|
}
|
||||||
|
if s == link.StateUnhealthy {
|
||||||
|
r.Reason, r.Restarts, r.Streak = "restarting", 4, 2
|
||||||
|
}
|
||||||
|
h.Resources = append(h.Resources, r)
|
||||||
|
}
|
||||||
|
return h
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTwoUnhealthyStatementsRaiseTheModulesConditionAndAHealthyOneClearsIt(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
k := conditionsFrom
|
||||||
|
const key = "module.letta.anchor.unhealthy"
|
||||||
|
openKeys := func() []string {
|
||||||
|
t.Helper()
|
||||||
|
list, err := k.Open(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var keys []string
|
||||||
|
for _, c := range list {
|
||||||
|
keys = append(keys, c.Key)
|
||||||
|
}
|
||||||
|
return keys
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0, link.StateHealthy, link.StateHealthy), h0); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// One statement: nothing raised, and `node show` lists it as unconfirmed.
|
||||||
|
if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(time.Minute), link.StateUnhealthy, link.StateHealthy),
|
||||||
|
h0.Add(time.Minute)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if keys := openKeys(); len(keys) != 0 {
|
||||||
|
t.Fatalf("one statement raised %v", keys)
|
||||||
|
}
|
||||||
|
kept, had, err := inv.HealthOf(ctx, "anchor")
|
||||||
|
if err != nil || !had {
|
||||||
|
t.Fatalf("the statement was not kept: %v %v", had, err)
|
||||||
|
}
|
||||||
|
if lines := strings.Join(healthLines(kept, had, h0.Add(time.Minute)), "\n"); !strings.Contains(lines, "unconfirmed") ||
|
||||||
|
!strings.Contains(lines, "letta.server") {
|
||||||
|
t.Fatalf("node show does not list the first statement as unconfirmed:\n%s", lines)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The second in a row raises it — the module's own, never the other module's on the machine.
|
||||||
|
if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(2*time.Minute), link.StateUnhealthy, link.StateHealthy),
|
||||||
|
h0.Add(2*time.Minute)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if keys := openKeys(); len(keys) != 1 || keys[0] != key {
|
||||||
|
t.Fatalf("two statements raised %v, not %s", keys, key)
|
||||||
|
}
|
||||||
|
c, _, _ := k.Get(ctx, key)
|
||||||
|
if c.Severity != conditions.Warning || c.Resolver != conditions.ResolverSelf || c.Subject.Machine != "anchor" ||
|
||||||
|
!strings.Contains(c.Summary, "letta on anchor") || !strings.Contains(c.Summary, "keeps restarting") ||
|
||||||
|
!strings.Contains(c.Evidence[0].Said, "letta-server") {
|
||||||
|
t.Fatalf("the condition does not say it in words with its evidence: %+v", c)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Standing four hours, it is urgent.
|
||||||
|
if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(5*time.Hour), link.StateUnhealthy, link.StateHealthy),
|
||||||
|
time.Now().Add(5*time.Hour)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if c, _, _ := k.Get(ctx, key); c.Severity != conditions.Urgent {
|
||||||
|
t.Fatalf("unhealthy for four hours is still %s", c.Severity)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A new build's start — every start begins in `starting` — clears it: what follows is the new build's.
|
||||||
|
if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(6*time.Hour), link.StateStarting, link.StateHealthy),
|
||||||
|
h0.Add(6*time.Hour)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if keys := openKeys(); len(keys) != 0 {
|
||||||
|
t.Fatalf("the first statement that says no resource is unhealthy did not clear it: %v", keys)
|
||||||
|
}
|
||||||
|
// And the streak starts again: one unhealthy statement after it raises nothing.
|
||||||
|
if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(7*time.Hour), link.StateUnhealthy), h0.Add(7*time.Hour)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if keys := openKeys(); len(keys) != 0 {
|
||||||
|
t.Fatalf("one statement after a clearing raised %v", keys)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnOlderHealthStatementIsRefused(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
before := healthRefused.Load()
|
||||||
|
if err := stateHealth(ctx, inv, nil, "anchor", aStatement(h0.Add(time.Minute), link.StateHealthy), h0); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// An event said before the report that overtook it arrives late.
|
||||||
|
if err := stateHealth(ctx, inv, nil, "anchor", aStatement(h0, link.StateUnhealthy), h0); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
kept, _, err := inv.HealthOf(ctx, "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !kept.SaidAt.Equal(h0.Add(time.Minute)) || kept.Resources[0].State != link.StateHealthy {
|
||||||
|
t.Fatalf("the older statement replaced the newer: %+v", kept)
|
||||||
|
}
|
||||||
|
if healthRefused.Load() != before+1 {
|
||||||
|
t.Fatalf("the refusal was not counted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **An engine older than the judging states nothing**: its reports raise nothing, keep nothing, and the
|
||||||
|
// gate judges its machine as before — never healthy for having said nothing, never unhealthy.
|
||||||
|
func TestAReportWithNoHealthRaisesAndKeepsNothing(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
l := nudgingListener{Enrolment: link.Enrolment{Inventory: open.inventory}}
|
||||||
|
if _, err := l.Heard(ctx, link.Report{Node: "anchor", Declared: "d1", Applied: []string{"letta.server"}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, had, err := open.inventory.HealthOf(ctx, "anchor"); err != nil || had {
|
||||||
|
t.Fatalf("health kept for an engine that said none: %v %v", had, err)
|
||||||
|
}
|
||||||
|
if lines := healthLines(inventory.NodeHealth{}, false, time.Now()); !strings.Contains(lines[0], "older than the judging") {
|
||||||
|
t.Fatalf("node show: %v", lines)
|
||||||
|
}
|
||||||
|
// And one that does, through the report, is kept.
|
||||||
|
h := aStatement(time.Now().UTC(), link.StateHealthy)
|
||||||
|
if _, err := l.Heard(ctx, link.Report{Node: "anchor", Declared: "d1", Health: &h}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if kept, had, err := open.inventory.HealthOf(ctx, "anchor"); err != nil || !had || len(kept.Resources) != 1 {
|
||||||
|
t.Fatalf("the report's health was not kept: %+v %v %v", kept, had, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A declared `health` is sent only to an engine whose own statement says it reads it (novox/hq ADR 0240
|
||||||
|
// Phase B): an older engine is strict and would refuse the whole declaration for the field.
|
||||||
|
func TestHealthIsSentOnlyToAnEngineThatSaysItReadsIt(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
reads := func() bool {
|
||||||
|
t.Helper()
|
||||||
|
got, err := engineReadsHealth(ctx, inv, "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return got
|
||||||
|
}
|
||||||
|
if reads() {
|
||||||
|
t.Fatal("an engine that never stated anything is sent health")
|
||||||
|
}
|
||||||
|
if err := stateHealth(ctx, inv, nil, "anchor", aStatement(h0, link.StateHealthy), h0); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if reads() {
|
||||||
|
t.Fatal("an engine judging liveness alone is sent health")
|
||||||
|
}
|
||||||
|
later := aStatement(h0.Add(time.Minute), link.StateHealthy)
|
||||||
|
later.Contract = link.ReadinessContract
|
||||||
|
if err := stateHealth(ctx, inv, nil, "anchor", later, h0.Add(time.Minute)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !reads() {
|
||||||
|
t.Fatal("an engine that reads health is not sent it")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -182,6 +182,9 @@ func moduleCommand(ctx context.Context, args []string) error {
|
|||||||
Requires []string `json:"requires,omitempty"`
|
Requires []string `json:"requires,omitempty"`
|
||||||
Claims []string `json:"claims,omitempty"`
|
Claims []string `json:"claims,omitempty"`
|
||||||
Capabilities []string `json:"capabilities,omitempty"`
|
Capabilities []string `json:"capabilities,omitempty"`
|
||||||
|
// Replaces is what each of its own tools replaces (novox/hq ADR 0245), for the console's
|
||||||
|
// search and the agent's instructions.
|
||||||
|
Replaces map[string][]string `json:"replaces,omitempty"`
|
||||||
}
|
}
|
||||||
out := make([]listed, 0, len(entries))
|
out := make([]listed, 0, len(entries))
|
||||||
for _, e := range entries {
|
for _, e := range entries {
|
||||||
@@ -189,7 +192,7 @@ func moduleCommand(ctx context.Context, args []string) error {
|
|||||||
l := listed{Module: m.Module, Version: m.Version, Built: e.Source.BuiltFrom, Head: e.Source.Head,
|
l := listed{Module: m.Module, Version: m.Version, Built: e.Source.BuiltFrom, Head: e.Source.Head,
|
||||||
Current: e.Provided || e.Source.Repository == "" || e.Source.Current(), Provided: e.Provided,
|
Current: e.Provided || e.Source.Repository == "" || e.Source.Current(), Provided: e.Provided,
|
||||||
On: append([]string{}, e.On...), Provides: m.Offers(), Requires: m.Requires,
|
On: append([]string{}, e.On...), Provides: m.Offers(), Requires: m.Requires,
|
||||||
Capabilities: m.Capabilities, Tools: declaresTools(m)}
|
Capabilities: m.Capabilities, Tools: declaresTools(m), Replaces: m.Replaces}
|
||||||
for _, c := range m.Claims {
|
for _, c := range m.Claims {
|
||||||
l.Claims = append(l.Claims, c.At()+"/"+c.Name)
|
l.Claims = append(l.Claims, c.At()+"/"+c.Name)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,94 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// asksWithPaths records every build a merge asks for, as repository, path and ref.
|
||||||
|
func asksWithPaths(t *testing.T) *[][3]string {
|
||||||
|
t.Helper()
|
||||||
|
var asked [][3]string
|
||||||
|
was := askABuild
|
||||||
|
askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) {
|
||||||
|
asked = append(asked, [3]string{source.Repository, path, ref})
|
||||||
|
return "b-" + path, nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { askABuild = was })
|
||||||
|
return &asked
|
||||||
|
}
|
||||||
|
|
||||||
|
// The live case of 2026-10-07 (novox/hq issue 300), replayed: a merge of the catalogue adds one module's
|
||||||
|
// directory and touches nothing else the mesh holds. Its delivery plan said "builds new: …, sent nowhere";
|
||||||
|
// the merge said it changed nothing any module the mesh holds is built from, and built nothing, so the
|
||||||
|
// module was never registered and `assign` refused it. The merge asks for its build, at the branch merged
|
||||||
|
// into, from the repository as the mesh spells it — and opens no plan, since nothing it holds moved.
|
||||||
|
func TestAMergeBuildsTheNewModuleItAdds(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
asked := asksWithPaths(t)
|
||||||
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "networkmanager", Version: "1"},
|
||||||
|
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/networkmanager", Ref: "main",
|
||||||
|
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "3da80a4b00",
|
||||||
|
Paths: []string{"modules/systemd-resolved/module.json", "modules/systemd-resolved/cmd/main.go"},
|
||||||
|
ModuleDirs: []string{"modules/systemd-resolved"}, ModuleDirsSaid: true}
|
||||||
|
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := [3]string{"novox/mesh-catalog", "modules/systemd-resolved", "main"}
|
||||||
|
if len(*asked) != 1 || (*asked)[0] != want {
|
||||||
|
t.Fatalf("the merge adding modules/systemd-resolved asked for %v, not %v", *asked, want)
|
||||||
|
}
|
||||||
|
if plans, err := inv.OpenPlans(ctx); err != nil || len(plans) != 0 {
|
||||||
|
t.Fatalf("a merge moving nothing the mesh holds opened a plan: %+v %v", plans, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A merge that changes a held module and adds a new one does both: the held one walks its plan, the new
|
||||||
|
// one is asked for beside it. A merge adding nothing builds nothing new.
|
||||||
|
func TestAMergeBuildsItsNewModuleBesideItsPlan(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
asked := asksWithPaths(t)
|
||||||
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"},
|
||||||
|
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main",
|
||||||
|
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1aaaaaaaa",
|
||||||
|
Paths: []string{"modules/app/index.ts", "modules/fresh/module.json"},
|
||||||
|
ModuleDirs: []string{"modules/app", "modules/fresh"}, ModuleDirsSaid: true}
|
||||||
|
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
paths := map[string]bool{}
|
||||||
|
for _, a := range *asked {
|
||||||
|
paths[a[1]] = true
|
||||||
|
}
|
||||||
|
if !paths["modules/fresh"] || !paths["modules/app"] || len(*asked) != 2 {
|
||||||
|
t.Fatalf("asked %v; want the new module and the plan's first tier", *asked)
|
||||||
|
}
|
||||||
|
plans, err := inv.OpenPlans(ctx)
|
||||||
|
if err != nil || len(plans) != 1 || plans[0].Modules["fresh"] != nil || plans[0].Modules["app"] == nil {
|
||||||
|
t.Fatalf("the plan should walk app alone: %+v %v", plans, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
*asked = nil
|
||||||
|
m2 := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c2aaaaaaaa",
|
||||||
|
Paths: []string{"README.md"}, ModuleDirsSaid: true}
|
||||||
|
if err := (following{open: open}).SourceMoved(ctx, m2); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(*asked) != 0 {
|
||||||
|
t.Fatalf("a merge adding no module asked for %v", *asked)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -537,6 +537,20 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// What it runs for its modules, and how each is (novox/hq ADR 0240): "is it working" answered without
|
||||||
|
// a terminal on the machine.
|
||||||
|
if h, had, err := inv.HealthOf(ctx, name); err != nil {
|
||||||
|
fmt.Printf("\n what it says of what it runs could NOT be read: %v\n", err)
|
||||||
|
} else {
|
||||||
|
fmt.Println()
|
||||||
|
for _, line := range healthLines(h, had, time.Now()) {
|
||||||
|
fmt.Println(line)
|
||||||
|
}
|
||||||
|
for _, line := range networkLines(h, had, time.Now()) {
|
||||||
|
fmt.Println(line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
held, err := inv.Profile(ctx, name)
|
held, err := inv.Profile(ctx, name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ import (
|
|||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/licences"
|
"github.com/novox/mesh-controller/internal/licences"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
"github.com/novox/mesh-controller/internal/overlay"
|
"github.com/novox/mesh-controller/internal/overlay"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -126,6 +127,12 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
|
|||||||
return catalogue.Resolution{}, nil, err
|
return catalogue.Resolution{}, nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A recorded module is composed at the build this machine runs, on any send but a person's push
|
||||||
|
// (novox/hq issue 295, ADR 0245).
|
||||||
|
if _, err := keepRecorded(ctx, open, nodeName, shelf); err != nil {
|
||||||
|
return catalogue.Resolution{}, nil, err
|
||||||
|
}
|
||||||
|
|
||||||
resolved, err := catalogue.Resolve(shelf, assigned,
|
resolved, err := catalogue.Resolve(shelf, assigned,
|
||||||
catalogue.Node{Name: nodeName, Site: site, Capabilities: capabilities,
|
catalogue.Node{Name: nodeName, Site: site, Capabilities: capabilities,
|
||||||
At: onNetwork[nodeName], PublicDomain: publicDomain,
|
At: onNetwork[nodeName], PublicDomain: publicDomain,
|
||||||
@@ -432,6 +439,16 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
|||||||
names = append(names, m.Module)
|
names = append(names, m.Module)
|
||||||
}
|
}
|
||||||
out.Builds = carriedBuilds(names, composed.LeftOut, current, before)
|
out.Builds = carriedBuilds(names, composed.LeftOut, current, before)
|
||||||
|
// A recorded module kept at the build the machine runs is recorded as carrying that one (ADR 0245).
|
||||||
|
kept, err := recordedKept(ctx, open, node)
|
||||||
|
if err != nil {
|
||||||
|
return sendable{}, err
|
||||||
|
}
|
||||||
|
for m, was := range kept {
|
||||||
|
if _, carried := out.Builds[m]; carried {
|
||||||
|
out.Builds[m] = was
|
||||||
|
}
|
||||||
|
}
|
||||||
out.Bindings = boundToData(plan, composed.LeftOut)
|
out.Bindings = boundToData(plan, composed.LeftOut)
|
||||||
}
|
}
|
||||||
return out, nil
|
return out, nil
|
||||||
@@ -860,7 +877,12 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
|||||||
if artifactStore != "" {
|
if artifactStore != "" {
|
||||||
reach = map[string]string{"mesh-artifact-store": artifactStore}
|
reach = map[string]string{"mesh-artifact-store": artifactStore}
|
||||||
}
|
}
|
||||||
|
readsHealth, err := engineReadsHealth(ctx, inv, node)
|
||||||
|
if err != nil {
|
||||||
|
return catalogue.Rendering{}, inventory.Node{}, err
|
||||||
|
}
|
||||||
return catalogue.Rendering{
|
return catalogue.Rendering{
|
||||||
|
ReadsHealth: readsHealth,
|
||||||
BusMembership: memberships[node],
|
BusMembership: memberships[node],
|
||||||
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Foreseen: foreseen, Ports: ports,
|
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Foreseen: foreseen, Ports: ports,
|
||||||
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
||||||
@@ -872,6 +894,17 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
|||||||
}, record, nil
|
}, record, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// engineReadsHealth says whether a machine's node-engine reads a declared `health` (novox/hq ADR 0240
|
||||||
|
// Phase B), by its own newest statement: one older, or one that never stated anything, is not sent the
|
||||||
|
// field, because it parses strictly and would refuse the whole declaration for it.
|
||||||
|
func engineReadsHealth(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) {
|
||||||
|
stated, had, err := inv.HealthOf(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return had && stated.Contract >= link.ReadinessContract, nil
|
||||||
|
}
|
||||||
|
|
||||||
// zonesInTheMesh is every zone a module in the mesh declares, where the mesh placed it (novox/hq ADR
|
// zonesInTheMesh is every zone a module in the mesh declares, where the mesh placed it (novox/hq ADR
|
||||||
// 0199): the zone settled from that node's settings, the node's private address, the port the
|
// 0199): the zone settled from that node's settings, the node's private address, the port the
|
||||||
// answering listen is published on there.
|
// answering listen is published on there.
|
||||||
|
|||||||
@@ -0,0 +1,121 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq issue 296: a plan saved a few seconds ago that has stood in its tier for forty minutes says
|
||||||
|
// forty minutes, and LATE — the line counted from the last save, which every advance made.
|
||||||
|
func TestAPlansLineCountsFromItsTierNotFromItsLastSave(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 7, 20, 30, 0, 0, time.UTC)
|
||||||
|
asked := now.Add(-40 * time.Minute)
|
||||||
|
p := inventory.Plan{ID: "plan-t", Repository: "novox/mesh-catalog", Commit: "3ad22356", State: inventory.PlanBuilding,
|
||||||
|
Created: now.Add(-40 * time.Minute), Updated: now.Add(-3 * time.Second), TierEntered: now.Add(-40 * time.Minute),
|
||||||
|
Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked}}}
|
||||||
|
|
||||||
|
line := planLine(p, now)
|
||||||
|
if !strings.Contains(line, "building for 40m0s") || !strings.Contains(line, "LATE") {
|
||||||
|
t.Fatalf("a plan in its tier for 40m, saved 3s ago, reads %q", line)
|
||||||
|
}
|
||||||
|
st := planStatuses([]inventory.Plan{p}, now, pauseView{}, nil)[0]
|
||||||
|
if !st.Late || !st.Since.Equal(p.TierEntered) {
|
||||||
|
t.Errorf("status --json says %+v", st)
|
||||||
|
}
|
||||||
|
if _, late := openPlans([]inventory.Plan{p}, now, pauseView{}, nil); late != 1 {
|
||||||
|
t.Errorf("status counted %d late", late)
|
||||||
|
}
|
||||||
|
|
||||||
|
// LATE is the stalled condition said on the line: the same start and the same bound, measured or not.
|
||||||
|
bounds := boundsOf([]inventory.Duration{{Subject: p.Repository, Took: 20 * time.Minute}})
|
||||||
|
if got := bounds.of(p.Repository); got != time.Hour {
|
||||||
|
t.Fatalf("a repository whose tiers took 20m is given %s", got)
|
||||||
|
}
|
||||||
|
if got := bounds.of("novox/other"); got != tierAtLeast {
|
||||||
|
t.Fatalf("a repository with nothing measured is given %s", got)
|
||||||
|
}
|
||||||
|
for _, in := range []time.Duration{40 * time.Minute, 70 * time.Minute} {
|
||||||
|
q := p
|
||||||
|
q.TierEntered = now.Add(-in)
|
||||||
|
bound := bounds.of(q.Repository)
|
||||||
|
late := strings.Contains(planLineWith(q, now, pauseView{}, bound), "LATE")
|
||||||
|
stalled := len(watchPlans(&signalFacts{now: now, plans: []planFacts{{id: q.ID, repository: q.Repository,
|
||||||
|
commit: q.Commit, tiers: 1, entered: inTierSince(q), bound: bound}}})) == 1
|
||||||
|
if late != stalled || late != (in > bound) {
|
||||||
|
t.Errorf("in its tier %s with a bound of %s: LATE %v, stalled %v", in, bound, late, stalled)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A plan read without its tier's stamp counts from its last save, the only time it has.
|
||||||
|
old := p
|
||||||
|
old.TierEntered = time.Time{}
|
||||||
|
if line := planLine(old, now); !strings.Contains(line, "for 3s") || strings.Contains(line, "LATE") {
|
||||||
|
t.Errorf("a plan without its tier's stamp reads %q", line)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A walk that waited for its delivery's word counts its wait from when it was opened, and its tier
|
||||||
|
// from the word.
|
||||||
|
waiting := p
|
||||||
|
waiting.Delivery = &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}
|
||||||
|
waiting.Modules = map[string]*inventory.PlanModule{}
|
||||||
|
if line := planLine(waiting, now); !strings.Contains(line, "for 40m0s") || strings.Contains(line, "LATE") {
|
||||||
|
t.Errorf("a walk waiting 40m reads %q", line)
|
||||||
|
}
|
||||||
|
if st := planStatuses([]inventory.Plan{waiting}, now, pauseView{}, nil)[0]; st.Late {
|
||||||
|
t.Errorf("a waiting walk is called late: %+v", st)
|
||||||
|
}
|
||||||
|
let := now.Add(-5 * time.Minute)
|
||||||
|
gone := p
|
||||||
|
gone.Delivery = &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat, Go: &let}
|
||||||
|
if line := planLine(gone, now); !strings.Contains(line, "building for 5m0s") || strings.Contains(line, "LATE") {
|
||||||
|
t.Errorf("a walk let go 5m ago, opened 40m ago, reads %q", line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq issue 296: a plan whose tier stands still is not written again by every advance — its revision,
|
||||||
|
// its last save and its `plan-moved` stay where its last change left them.
|
||||||
|
func TestAPlanStandingStillIsNotSavedAgain(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
asked := asksRecorded(t)
|
||||||
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"},
|
||||||
|
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main",
|
||||||
|
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1aaaaaaaa",
|
||||||
|
Paths: []string{"modules/app/index.ts"}, ModuleDirs: []string{"modules/app"}, ModuleDirsSaid: true}
|
||||||
|
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
advanceHeld(ctx, open)
|
||||||
|
recent, err := inv.RecentPlans(ctx, 1)
|
||||||
|
if err != nil || len(recent) != 1 || len(*asked) != 1 {
|
||||||
|
t.Fatalf("no plan building: %v %v, asked %v", recent, err, *asked)
|
||||||
|
}
|
||||||
|
before := recent[0]
|
||||||
|
if before.State != inventory.PlanBuilding || before.Modules["app"] == nil || before.Modules["app"].State != "asked" {
|
||||||
|
t.Fatalf("the plan is not waiting on its build: %+v", before)
|
||||||
|
}
|
||||||
|
|
||||||
|
for range 3 {
|
||||||
|
advanceHeld(ctx, open)
|
||||||
|
}
|
||||||
|
after, err := inv.PlanByID(ctx, before.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if after.Revision != before.Revision || !after.Updated.Equal(before.Updated) {
|
||||||
|
t.Fatalf("a plan that did not move was saved again: revision %d → %d, saved %s → %s",
|
||||||
|
before.Revision, after.Revision, before.Updated, after.Updated)
|
||||||
|
}
|
||||||
|
if len(*asked) != 1 {
|
||||||
|
t.Fatalf("advancing asked again: %v", *asked)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -419,8 +419,9 @@ func askResolver(ctx context.Context, at, name string, kind dnsmessage.Type) ([]
|
|||||||
return addresses, answer.RCode, nil
|
return addresses, answer.RCode, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// probeHolders is D3: every seat that serves verbs has, on every machine that holds it and is heard
|
// probeHolders is D3: every seat with a verb its holder must serve has, on every machine that holds it
|
||||||
// from, a holder answering the bus's discovery for that seat. A machine past its heartbeat's bound is
|
// and is heard from, a holder answering the bus's discovery for that seat. A seat whose verbs are all
|
||||||
|
// still optional is not asked about (holdingNeedsAnAnswer). A machine past its heartbeat's bound is
|
||||||
// S1's, and is not asked about here.
|
// S1's, and is not asked about here.
|
||||||
func probeHolders(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
func probeHolders(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
entries, err := d.open.inventory.Catalogued(ctx)
|
entries, err := d.open.inventory.Catalogued(ctx)
|
||||||
@@ -436,44 +437,7 @@ func probeHolders(ctx context.Context, d *doctor) ([]conditions.Observation, err
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
now := time.Now()
|
expected := holdersToHear(catalogue.SeatsWithAProtocol(), recorded, entries, heard, delivered, time.Now())
|
||||||
expected := map[string]map[string]bool{} // seat → machine
|
|
||||||
add := func(seat, node string) {
|
|
||||||
if expected[seat] == nil {
|
|
||||||
expected[seat] = map[string]bool{}
|
|
||||||
}
|
|
||||||
expected[seat][node] = true
|
|
||||||
}
|
|
||||||
for _, s := range catalogue.SeatsWithAProtocol() {
|
|
||||||
if len(s.Serves) == 0 || s.Name == catalogue.ControllerSeatName {
|
|
||||||
continue // a seat with no verb has nothing to answer with; this controller is answering now
|
|
||||||
}
|
|
||||||
onRecord := false
|
|
||||||
for _, h := range recorded {
|
|
||||||
if h.Claim == s.Name && heard[h.Node] {
|
|
||||||
add(s.Name, h.Node)
|
|
||||||
onRecord = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if onRecord && s.Scope == catalogue.ScopeMesh {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
for _, e := range entries {
|
|
||||||
for _, c := range e.Manifest.Claims {
|
|
||||||
if c.Name != s.Name {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
for _, node := range e.On {
|
|
||||||
// Assigned is not there yet: a holder is expected once its machine was sent it and
|
|
||||||
// had time to report, never between `assign` and `push` (novox/hq issue 275).
|
|
||||||
if heard[node] && (s.Scope == catalogue.ScopeNode || !onRecord) &&
|
|
||||||
delivered[node].settled(e.Manifest.Module, now) {
|
|
||||||
add(s.Name, node)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(expected) == 0 {
|
if len(expected) == 0 {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
@@ -509,6 +473,11 @@ func probeHolders(ctx context.Context, d *doctor) ([]conditions.Observation, err
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return silentHolders(expected, answering), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// silentHolders is every expected holder the discovery did not hear, as D3's observations.
|
||||||
|
func silentHolders(expected, answering map[string]map[string]bool) []conditions.Observation {
|
||||||
var out []conditions.Observation
|
var out []conditions.Observation
|
||||||
for seat, nodes := range expected {
|
for seat, nodes := range expected {
|
||||||
for node := range nodes {
|
for node := range nodes {
|
||||||
@@ -521,7 +490,71 @@ func probeHolders(ctx context.Context, d *doctor) ([]conditions.Observation, err
|
|||||||
Said: fmt.Sprintf("no answer for %s from %s to the bus's discovery", seat, node)})
|
Said: fmt.Sprintf("no answer for %s from %s to the bus's discovery", seat, node)})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return sortedFound(out), nil
|
return sortedFound(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// holdersToHear is D3's expectation: seat → every machine whose holder of it must answer the bus's
|
||||||
|
// discovery. A seat is listed only if holding it requires serving a verb (holdingNeedsAnAnswer); a holder
|
||||||
|
// is expected on a machine that is heard from and that holds it on record or — for a node seat, or a mesh
|
||||||
|
// seat nobody holds on record — was sent the module that claims it and had time to report.
|
||||||
|
func holdersToHear(seats []catalogue.Seat, recorded []catalogue.Held, entries []inventory.Entry,
|
||||||
|
heard map[string]bool, delivered map[string]lastSend, now time.Time) map[string]map[string]bool {
|
||||||
|
expected := map[string]map[string]bool{} // seat → machine
|
||||||
|
add := func(seat, node string) {
|
||||||
|
if expected[seat] == nil {
|
||||||
|
expected[seat] = map[string]bool{}
|
||||||
|
}
|
||||||
|
expected[seat][node] = true
|
||||||
|
}
|
||||||
|
for _, s := range seats {
|
||||||
|
if s.Name == catalogue.ControllerSeatName {
|
||||||
|
continue // this controller is answering now
|
||||||
|
}
|
||||||
|
if !holdingNeedsAnAnswer(s) {
|
||||||
|
continue // nothing its holder must serve, so nothing its silence would say (novox/hq issue 299)
|
||||||
|
}
|
||||||
|
onRecord := false
|
||||||
|
for _, h := range recorded {
|
||||||
|
if h.Claim == s.Name && heard[h.Node] {
|
||||||
|
add(s.Name, h.Node)
|
||||||
|
onRecord = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if onRecord && s.Scope == catalogue.ScopeMesh {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, e := range entries {
|
||||||
|
for _, c := range e.Manifest.Claims {
|
||||||
|
if c.Name != s.Name {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, node := range e.On {
|
||||||
|
// Assigned is not there yet: a holder is expected once its machine was sent it and
|
||||||
|
// had time to report, never between `assign` and `push` (novox/hq issue 275).
|
||||||
|
if heard[node] && (s.Scope == catalogue.ScopeNode || !onRecord) &&
|
||||||
|
delivered[node].settled(e.Manifest.Module, now) {
|
||||||
|
add(s.Name, node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return expected
|
||||||
|
}
|
||||||
|
|
||||||
|
// holdingNeedsAnAnswer says whether a seat's holder is judged silent when the bus's discovery hears
|
||||||
|
// nothing from it: only when the seat has a verb its holder must serve. **Silence is judged on required
|
||||||
|
// verbs alone** (novox/hq issue 299). An optional verb is one the seat's holders are still catching up
|
||||||
|
// to (Verb.Optional; design 33 §7, novox/hq ADR 0246): a holder that does not serve it yet holds the seat
|
||||||
|
// rightly, and a holder serving nothing because every verb is still optional is exactly that holder, not
|
||||||
|
// a silent one. A seat with no verb at all has nothing to answer with either.
|
||||||
|
func holdingNeedsAnAnswer(s catalogue.Seat) bool {
|
||||||
|
for _, v := range s.Serves {
|
||||||
|
if !v.Optional {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// reportGrace is how long a machine is given, after it was sent a declaration, to apply it and report
|
// reportGrace is how long a machine is given, after it was sent a declaration, to apply it and report
|
||||||
|
|||||||
@@ -0,0 +1,192 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A provider down is said once, at the provider (novox/hq ADR 0240 rule 5, to-be 48 §6, Phase C).
|
||||||
|
//
|
||||||
|
// With twelve consumers of the database provision and thirty-six of a route, one provider down would be
|
||||||
|
// twelve conditions for one fault and twelve gates failed for something none of them did. So a consumer's
|
||||||
|
// check names, in `needs`, the provision it exercises; while **that provision's provider for this
|
||||||
|
// consumer** — the one the controller composed the consumer against: its recorded binding (ADR 0232), or
|
||||||
|
// the provider its credential for the provision is from — is unhealthy on the record, what the check finds
|
||||||
|
// is held under the provider's condition: listed there as waiting on it, raised as nothing of its own, and
|
||||||
|
// the consumer's gate waits rather than fails. The provider's condition is urgent while consumers wait.
|
||||||
|
//
|
||||||
|
// **Only what the check finds is held.** A consumer that is down or restarting is its own, whatever its
|
||||||
|
// provider does; so is anything a check that names no provision finds, and anything found while the
|
||||||
|
// provider is healthy. A machine-level fault is never pinned on a module (issue 281), and this does not
|
||||||
|
// change that.
|
||||||
|
|
||||||
|
// holding is what one reading of the record needs to say who waits on whom: every machine's newest
|
||||||
|
// statement, the open conditions, and the catalogue — read once, asked many times.
|
||||||
|
type holding struct {
|
||||||
|
ctx context.Context
|
||||||
|
inv *inventory.Inventory
|
||||||
|
healths map[string]inventory.NodeHealth
|
||||||
|
open []conditions.Condition
|
||||||
|
shelf map[string]catalogue.Manifest
|
||||||
|
// providers memoises providerFor by machine, consumer and provision.
|
||||||
|
providers map[string]providerLookup
|
||||||
|
}
|
||||||
|
|
||||||
|
type providerLookup struct {
|
||||||
|
chosen catalogue.Chosen
|
||||||
|
ok bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// readHolding reads what the hold is judged from.
|
||||||
|
func readHolding(ctx context.Context, inv *inventory.Inventory, open []conditions.Condition) (*holding, error) {
|
||||||
|
healths, err := inv.Healths(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &holding{ctx: ctx, inv: inv, healths: healths, open: open, providers: map[string]providerLookup{}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// heldFinding says a resource's state is a finding of its declared check that names a provision: what
|
||||||
|
// may be held. Down and restarting are liveness, the resource's own.
|
||||||
|
func heldFinding(r inventory.ResourceHealth) bool {
|
||||||
|
return r.State == link.StateUnhealthy && r.Check != "" && r.Needs != "" &&
|
||||||
|
r.Reason != "down" && r.Reason != "restarting"
|
||||||
|
}
|
||||||
|
|
||||||
|
// providerFor is the provider composed for a consumer's provision: its recorded binding, else the
|
||||||
|
// machine its credential for the provision comes from and the module there that provides it.
|
||||||
|
func (h *holding) providerFor(machine, consumer, provision string) (catalogue.Chosen, bool) {
|
||||||
|
key := machine + "\x00" + consumer + "\x00" + provision
|
||||||
|
if p, known := h.providers[key]; known {
|
||||||
|
return p.chosen, p.ok
|
||||||
|
}
|
||||||
|
chosen, ok := h.lookUpProvider(machine, consumer, provision)
|
||||||
|
h.providers[key] = providerLookup{chosen, ok}
|
||||||
|
return chosen, ok
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *holding) lookUpProvider(machine, consumer, provision string) (catalogue.Chosen, bool) {
|
||||||
|
if bound, err := h.inv.BindingsFor(h.ctx, machine); err == nil {
|
||||||
|
if c, ok := bound[consumer][provision]; ok && c.Node != "" && c.Module != "" {
|
||||||
|
return c, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
secrets, err := h.inv.SecretsOf(h.ctx, machine, consumer)
|
||||||
|
if err != nil {
|
||||||
|
return catalogue.Chosen{}, false
|
||||||
|
}
|
||||||
|
for _, s := range secrets {
|
||||||
|
if s.Name != provision || s.Provider == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if h.shelf == nil {
|
||||||
|
if h.shelf, err = h.inv.Catalogue(h.ctx); err != nil {
|
||||||
|
return catalogue.Chosen{}, false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assigned, err := h.inv.Assigned(h.ctx, s.Provider)
|
||||||
|
if err != nil {
|
||||||
|
return catalogue.Chosen{}, false
|
||||||
|
}
|
||||||
|
for _, module := range assigned {
|
||||||
|
for _, offer := range h.shelf[module].Offers() {
|
||||||
|
if offer == provision {
|
||||||
|
return catalogue.Chosen{Node: s.Provider, Module: module}, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return catalogue.Chosen{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// unhealthy says a provider is unhealthy on the record: its condition is open, or its machine's newest
|
||||||
|
// statement says a resource of it is unhealthy.
|
||||||
|
func (h *holding) unhealthy(p catalogue.Chosen) bool {
|
||||||
|
key := moduleUnhealthyKey(p.Module, p.Node)
|
||||||
|
for _, c := range h.open {
|
||||||
|
if c.Key == key {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, r := range h.healths[p.Node].Resources {
|
||||||
|
if r.Module == p.Module && r.State == link.StateUnhealthy {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// heldUnder is the provider a consumer's unhealthy resources wait on: when every one of them is a finding
|
||||||
|
// of a check naming a provision whose provider for this consumer is unhealthy on the record. False when any
|
||||||
|
// is the consumer's own.
|
||||||
|
func (h *holding) heldUnder(machine, module string, rs []inventory.ResourceHealth) (catalogue.Chosen, bool) {
|
||||||
|
var on catalogue.Chosen
|
||||||
|
for _, r := range rs {
|
||||||
|
if r.State != link.StateUnhealthy {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !heldFinding(r) {
|
||||||
|
return catalogue.Chosen{}, false
|
||||||
|
}
|
||||||
|
p, ok := h.providerFor(machine, module, r.Needs)
|
||||||
|
if !ok || (p.Node == machine && p.Module == module) || !h.unhealthy(p) {
|
||||||
|
return catalogue.Chosen{}, false
|
||||||
|
}
|
||||||
|
on = p
|
||||||
|
}
|
||||||
|
return on, on.Module != ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitersOn is every consumer held under a provider, as "<module> on <machine>", sorted.
|
||||||
|
func (h *holding) waitersOn(p catalogue.Chosen) []string {
|
||||||
|
var out []string
|
||||||
|
for machine, nh := range h.healths {
|
||||||
|
byModule := map[string][]inventory.ResourceHealth{}
|
||||||
|
for _, r := range nh.Resources {
|
||||||
|
if r.Module != "" && r.State == link.StateUnhealthy {
|
||||||
|
byModule[r.Module] = append(byModule[r.Module], r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for module, rs := range byModule {
|
||||||
|
if on, held := h.heldUnder(machine, module, rs); held && on == p {
|
||||||
|
out = append(out, module+" on "+machine)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// heldModules is, for one machine's statement, each module whose finding is held, with the provider.
|
||||||
|
func (h *holding) heldModules(machine string) map[string]catalogue.Chosen {
|
||||||
|
out := map[string]catalogue.Chosen{}
|
||||||
|
byModule := map[string][]inventory.ResourceHealth{}
|
||||||
|
for _, r := range h.healths[machine].Resources {
|
||||||
|
if r.Module != "" && r.State == link.StateUnhealthy {
|
||||||
|
byModule[r.Module] = append(byModule[r.Module], r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for module, rs := range byModule {
|
||||||
|
if on, held := h.heldUnder(machine, module, rs); held {
|
||||||
|
out[module] = on
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// moduleUnhealthyKey is a module's health condition's key on a machine.
|
||||||
|
func moduleUnhealthyKey(module, machine string) string {
|
||||||
|
return conditions.ScopeModule + "." + module + "." + machine + ".unhealthy"
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitingWords is the provider's evidence that consumers wait on it.
|
||||||
|
func waitingWords(waiters []string) string {
|
||||||
|
return fmt.Sprintf("waiting on it — %s", strings.Join(waiters, ", "))
|
||||||
|
}
|
||||||
@@ -0,0 +1,172 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A provider down is said once, at the provider (novox/hq ADR 0240 rule 5, "how it is checked"): one
|
||||||
|
// unhealthy database provider and three consumers failing their checks that need it — one condition, at the
|
||||||
|
// provider, urgent, the consumers listed as waiting; their gates wait, not fail; once the provider is
|
||||||
|
// healthy, a consumer still failing is its own. A consumer failing while its provider is healthy is raised
|
||||||
|
// on its own from the start.
|
||||||
|
func TestOneProviderDownAndThreeConsumersFailingAreOneCondition(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
k := conditionsFrom
|
||||||
|
register(t, open, catalogue.Manifest{Module: "db", Version: "1",
|
||||||
|
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}})
|
||||||
|
consumers := []string{"shop", "wiki", "crm"}
|
||||||
|
for _, c := range consumers {
|
||||||
|
register(t, open, catalogue.Manifest{Module: c, Version: "1", Requires: []string{"postgres-database"}})
|
||||||
|
}
|
||||||
|
if _, err := inv.Assign(ctx, "anchor", "db"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
machineOf := map[string]string{"shop": "laptop", "wiki": "laptop", "crm": "anchor"}
|
||||||
|
for _, c := range consumers {
|
||||||
|
if _, err := inv.Assign(ctx, machineOf[c], c); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.RecordBindings(ctx, machineOf[c], []inventory.Binding{{Machine: machineOf[c], Consumer: c,
|
||||||
|
Provision: "postgres-database", Provider: catalogue.Chosen{Node: "anchor", Module: "db"}}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
at := h0
|
||||||
|
say := func(machine string, rs ...link.ResourceHealth) {
|
||||||
|
t.Helper()
|
||||||
|
at = at.Add(time.Second)
|
||||||
|
for i := range rs {
|
||||||
|
rs[i].Since = at
|
||||||
|
}
|
||||||
|
if err := stateHealth(ctx, inv, k, machine, link.Health{Contract: link.ReadinessContract, At: at, Resources: rs}, at); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dbDown := link.ResourceHealth{Module: "db", Resource: "db.server", Kind: "container", Target: "db", State: link.StateUnhealthy,
|
||||||
|
Reason: "its command: the database refuses connections", Check: "exec"}
|
||||||
|
dbUp := dbDown
|
||||||
|
dbUp.State, dbUp.Reason = link.StateHealthy, ""
|
||||||
|
failing := func(module string) link.ResourceHealth {
|
||||||
|
return link.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module,
|
||||||
|
State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"}
|
||||||
|
}
|
||||||
|
fine := func(module string) link.ResourceHealth {
|
||||||
|
r := failing(module)
|
||||||
|
r.State, r.Reason = link.StateHealthy, ""
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
openKeys := func() []conditions.Condition {
|
||||||
|
t.Helper()
|
||||||
|
list, err := k.Open(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return list
|
||||||
|
}
|
||||||
|
|
||||||
|
// Two looks of the provider down and its consumers failing, in either order on each machine.
|
||||||
|
for look := 0; look < 2; look++ {
|
||||||
|
say("laptop", failing("shop"), failing("wiki"))
|
||||||
|
say("anchor", dbDown, failing("crm"))
|
||||||
|
}
|
||||||
|
raised := openKeys()
|
||||||
|
if len(raised) != 1 || raised[0].Key != "module.db.anchor.unhealthy" {
|
||||||
|
var keys []string
|
||||||
|
for _, c := range raised {
|
||||||
|
keys = append(keys, c.Key)
|
||||||
|
}
|
||||||
|
t.Fatalf("one provider down and three consumers failing raised %v; want the provider's alone", keys)
|
||||||
|
}
|
||||||
|
c := raised[0]
|
||||||
|
if c.Severity != conditions.Urgent {
|
||||||
|
t.Errorf("consumers wait on the provider and its condition is %s", c.Severity)
|
||||||
|
}
|
||||||
|
said := c.Evidence[0].Said
|
||||||
|
for _, w := range []string{"shop on laptop", "wiki on laptop", "crm on anchor"} {
|
||||||
|
if !strings.Contains(said, w) {
|
||||||
|
t.Errorf("the provider's condition does not list %s as waiting on it: %s", w, said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Their gates wait, not fail: the judging is neither a pass nor a fault, past the bound too.
|
||||||
|
f, err := gatherGateFacts(ctx, open, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
f.open, f.openErr, f.judged = raised, nil, false
|
||||||
|
for _, m := range consumers {
|
||||||
|
h, why := moduleHealthWord(m, machineOf[m], h0, f)
|
||||||
|
if h != healthWaiting || !strings.Contains(why, "waits on db on anchor") {
|
||||||
|
t.Errorf("%s's gate: %v %q; want it waiting on its provider", m, h, why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// And a whole judging past the bound puts nothing back: it waits.
|
||||||
|
long := h0.Add(-time.Hour)
|
||||||
|
for _, m := range []string{"anchor", "laptop"} {
|
||||||
|
f.reports[m] = inventory.Reported{Node: m, Outcome: inventory.OutcomeApplied, At: &f.now, Current: true}
|
||||||
|
}
|
||||||
|
gatherWas := gatherGateFacts
|
||||||
|
defer func() { gatherGateFacts = gatherWas }()
|
||||||
|
gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) { return f, nil }
|
||||||
|
g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &long}
|
||||||
|
verdict, err := judgeMoves(ctx, open, g, []judged{{module: "shop", node: "laptop"}}, time.Now())
|
||||||
|
if err != nil || verdict != "" || !strings.Contains(g.Last, "waits on db on anchor") {
|
||||||
|
t.Fatalf("a held consumer's gate past its bound: verdict %q (%v), last %q; want it waiting", verdict, err, g.Last)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The provider healthy again: a consumer still failing is now its own, at once (its streak stood).
|
||||||
|
say("anchor", dbUp, fine("crm"))
|
||||||
|
say("laptop", failing("shop"), fine("wiki"))
|
||||||
|
var keys []string
|
||||||
|
for _, c := range openKeys() {
|
||||||
|
keys = append(keys, c.Key)
|
||||||
|
}
|
||||||
|
if !slices.Equal(keys, []string{"module.shop.laptop.unhealthy"}) {
|
||||||
|
t.Fatalf("after the provider recovered: %v; want shop's own and nothing else", keys)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A consumer failing while its provider is healthy is raised on its own.
|
||||||
|
func TestAConsumerFailingBesideAHealthyProviderIsItsOwn(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
register(t, open, catalogue.Manifest{Module: "db", Version: "1",
|
||||||
|
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}})
|
||||||
|
register(t, open, catalogue.Manifest{Module: "shop", Version: "1", Requires: []string{"postgres-database"}})
|
||||||
|
for _, a := range [][2]string{{"anchor", "db"}, {"laptop", "shop"}} {
|
||||||
|
if _, err := inv.Assign(ctx, a[0], a[1]); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := inv.RecordBindings(ctx, "laptop", []inventory.Binding{{Machine: "laptop", Consumer: "shop",
|
||||||
|
Provision: "postgres-database", Provider: catalogue.Chosen{Node: "anchor", Module: "db"}}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
healthy := link.ResourceHealth{Module: "db", Resource: "db.server", Kind: "container", Target: "db", State: link.StateHealthy, Check: "exec"}
|
||||||
|
shop := link.ResourceHealth{Module: "shop", Resource: "shop.web", Kind: "container", Target: "shop", State: link.StateUnhealthy,
|
||||||
|
Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"}
|
||||||
|
for i := 1; i <= 2; i++ {
|
||||||
|
at := h0.Add(time.Duration(i) * time.Minute)
|
||||||
|
_ = stateHealth(ctx, inv, conditionsFrom, "anchor", link.Health{Contract: 2, At: at, Resources: []link.ResourceHealth{healthy}}, at)
|
||||||
|
if err := stateHealth(ctx, inv, conditionsFrom, "laptop", link.Health{Contract: 2, At: at, Resources: []link.ResourceHealth{shop}}, at); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
list, _ := conditionsFrom.Open(ctx)
|
||||||
|
if len(list) != 1 || list[0].Key != "module.shop.laptop.unhealthy" {
|
||||||
|
t.Fatalf("a consumer failing beside a healthy provider raised %v", list)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -128,6 +128,10 @@ func serve(ctx context.Context) (err error) {
|
|||||||
stopActing()
|
stopActing()
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
}
|
}
|
||||||
|
// Stopping, whichever way: a verb arriving from here is refused as a handover, so its caller
|
||||||
|
// asks the controller after this one rather than have a command started and killed with this
|
||||||
|
// process (novox/hq issue 289).
|
||||||
|
handingOver.Store(true)
|
||||||
}()
|
}()
|
||||||
defer func() {
|
defer func() {
|
||||||
select {
|
select {
|
||||||
@@ -182,6 +186,9 @@ func serve(ctx context.Context) (err error) {
|
|||||||
if err := server.Watches(standings{keeper: func() *conditions.Keeper { return conditionsFrom }}); err != nil {
|
if err := server.Watches(standings{keeper: func() *conditions.Keeper { return conditionsFrom }}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
// And what each machine says of what it runs between its reports, kept and raised from (novox/hq ADR
|
||||||
|
// 0240): the gate, `node show` and the conditions read it.
|
||||||
|
server.Hears(moduleHealth{inv: inv, keeper: func() *conditions.Keeper { return conditionsFrom }})
|
||||||
// And what they say about consumers the mesh stopped asking for: one waiting for a person is an
|
// And what they say about consumers the mesh stopped asking for: one waiting for a person is an
|
||||||
// urgent condition, and an act asked of a provider some other way is recorded by hand (ADR 0230).
|
// urgent condition, and an act asked of a provider some other way is recorded by hand (ADR 0230).
|
||||||
if err := server.KeepsRetirements(retirements{keeper: func() *conditions.Keeper { return conditionsFrom },
|
if err := server.KeepsRetirements(retirements{keeper: func() *conditions.Keeper { return conditionsFrom },
|
||||||
@@ -580,6 +587,9 @@ func pushCommand(ctx context.Context, args []string) error {
|
|||||||
})
|
})
|
||||||
|
|
||||||
defer release()
|
defer release()
|
||||||
|
// **What it recreates, said before it is sent** (novox/hq ADR 0245): a person's push moves every
|
||||||
|
// module whose build changed, and recreates what changed in it — a gated send said so, a push did not.
|
||||||
|
sayWhatAPushRecreates(ctx, open, sending, os.Stdout)
|
||||||
// Each machine's memberships first, then the declarations (novox/hq issue 249, ADR 0160): a push
|
// Each machine's memberships first, then the declarations (novox/hq issue 249, ADR 0160): a push
|
||||||
// is the one most operators run, and on 2026-10-01 it was the one path that issued none.
|
// is the one most operators run, and on 2026-10-01 it was the one path that issued none.
|
||||||
bus := overTheBus{open: open, server: server, signer: ident}
|
bus := overTheBus{open: open, server: server, signer: ident}
|
||||||
@@ -1075,6 +1085,13 @@ func sendToEach(ctx context.Context, open *stores, names []string) ([]string, er
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **A recorded build moves only by a person's push** (novox/hq issue 295, ADR 0245): this send — a
|
||||||
|
// plan's, a release plan's, a rollback's, a healer's, a rotation's — composes every recorded module at
|
||||||
|
// the build its machine runs. The bus step is a person's word for the bus alone.
|
||||||
|
if ctx, err = sendKeeps(ctx, inv); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
// Held from composing to sending (novox/hq ADR 0100); a caller that holds them already —
|
// Held from composing to sending (novox/hq ADR 0100); a caller that holds them already —
|
||||||
// converge, which flips the node and then sends it — is not made to wait on itself.
|
// converge, which flips the node and then sends it — is not made to wait on itself.
|
||||||
ctx, release, err := holdNodes(ctx, open, names)
|
ctx, release, err := holdNodes(ctx, open, names)
|
||||||
|
|||||||
@@ -0,0 +1,84 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The live case of 2026-10-07, 20:27 UTC: a person's `push` of one machine moved mail to a new build whose
|
||||||
|
// automx container had a new image, and recreated it; the push's answer said nothing of it. The push now
|
||||||
|
// says, per machine, every module it moves and what that recreates — before it is sent. A module whose
|
||||||
|
// build did not change, and one new to the machine, are not moves.
|
||||||
|
func TestAPushSaysWhatItRecreates(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
start := time.Now().Add(-time.Hour)
|
||||||
|
image := func(c string) string { return "registry.invalid:5000/mailu/x@sha256:" + strings.Repeat(c, 64) }
|
||||||
|
for _, b := range []link.BuildResult{
|
||||||
|
aContainerBuild(t, "mailu", "c74f407a", catalogue.PolicyRecord, start,
|
||||||
|
map[string][2]string{"smtp": {image("a"), ""}, "automx": {image("c"), ""}}),
|
||||||
|
aContainerBuild(t, "postgres", "c1111111", "", start.Add(time.Second),
|
||||||
|
map[string][2]string{"server": {image("e"), ""}}),
|
||||||
|
} {
|
||||||
|
if _, _, err := takeIn(ctx, inv, b); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, m := range []string{"mailu", "postgres"} {
|
||||||
|
if _, err := inv.Assign(ctx, "anchor", m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor",
|
||||||
|
map[string]string{"mailu": "c74f407a", "postgres": "c1111111"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Mail's new build: automx's image changes, smtp's does not.
|
||||||
|
if _, _, err := takeIn(ctx, inv, aContainerBuild(t, "mailu", "1ab84e69", catalogue.PolicyRecord,
|
||||||
|
start.Add(time.Minute), map[string][2]string{"smtp": {image("a"), ""}, "automx": {image("d"), ""}})); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Composed as a person's push composes it.
|
||||||
|
plan, settings, err := planFor(ctx, open, "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
gens, err := generators(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
declared, err := declarationWith(ctx, open, "anchor", plan, settings, gens, Allocating)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if declared.Builds["mailu"] != "1ab84e69" {
|
||||||
|
t.Fatalf("a person's push carries mail at %q", declared.Builds["mailu"])
|
||||||
|
}
|
||||||
|
|
||||||
|
var said bytes.Buffer
|
||||||
|
sayWhatAPushRecreates(ctx, open, []readyNode{{node: "anchor", declared: declared}}, &said)
|
||||||
|
out := said.String()
|
||||||
|
for _, want := range []string{"anchor moves 1 module(s)", "mailu", "c74f407a → 1ab84e69",
|
||||||
|
"recreates 1 of mailu's 2 container(s) (with a new image: automx)"} {
|
||||||
|
if !strings.Contains(out, want) {
|
||||||
|
t.Fatalf("the push says %q; it does not say %q", out, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(out, "postgres") {
|
||||||
|
t.Fatalf("a module whose build did not change is said to move: %q", out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A machine never sent anything before has no moves to say.
|
||||||
|
said.Reset()
|
||||||
|
sayWhatAPushRecreates(ctx, open, []readyNode{{node: "laptop", declared: declared}}, &said)
|
||||||
|
if said.Len() != 0 {
|
||||||
|
t.Fatalf("a machine with no last send is said to move: %q", said.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -276,21 +276,21 @@ func TestAPlanWaitingOnAPausedSeatSaysSoAndIsNotLate(t *testing.T) {
|
|||||||
Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked}}}
|
Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked}}}
|
||||||
|
|
||||||
all := pauseOf([]string{"g14", "ace"}, map[string]link.HolderState{"ace": {Paused: true}, "g14": {Paused: true}})
|
all := pauseOf([]string{"g14", "ace"}, map[string]link.HolderState{"ace": {Paused: true}, "g14": {Paused: true}})
|
||||||
line := planLineWith(p, now, all)
|
line := planLineWith(p, now, all, tierAtLeast)
|
||||||
if !strings.Contains(line, "waiting: the build seat is paused on ace, g14 (asked 12m0s ago)") || strings.Contains(line, "LATE") {
|
if !strings.Contains(line, "waiting: the build seat is paused on ace, g14 (asked 12m0s ago)") || strings.Contains(line, "LATE") {
|
||||||
t.Errorf("a plan on a paused seat reads %q", line)
|
t.Errorf("a plan on a paused seat reads %q", line)
|
||||||
}
|
}
|
||||||
if st := planStatuses([]inventory.Plan{p}, now, all)[0]; st.Late || !strings.Contains(st.Waiting, "paused on ace, g14") {
|
if st := planStatuses([]inventory.Plan{p}, now, all, nil)[0]; st.Late || !strings.Contains(st.Waiting, "paused on ace, g14") {
|
||||||
t.Errorf("status --json says %+v", st)
|
t.Errorf("status --json says %+v", st)
|
||||||
}
|
}
|
||||||
if _, late := openPlans([]inventory.Plan{p}, all); late != 0 {
|
if _, late := openPlans([]inventory.Plan{p}, now, all, nil); late != 0 {
|
||||||
t.Errorf("a plan waiting on a paused seat counted late")
|
t.Errorf("a plan waiting on a paused seat counted late")
|
||||||
}
|
}
|
||||||
|
|
||||||
longAgo := now.Add(-25 * time.Hour)
|
longAgo := now.Add(-25 * time.Hour)
|
||||||
forgotten := p
|
forgotten := p
|
||||||
forgotten.Modules = map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &longAgo}}
|
forgotten.Modules = map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &longAgo}}
|
||||||
if line := planLineWith(forgotten, now, all); !strings.Contains(line, "PAUSED OVER A DAY") || strings.Contains(line, "LATE") {
|
if line := planLineWith(forgotten, now, all, tierAtLeast); !strings.Contains(line, "PAUSED OVER A DAY") || strings.Contains(line, "LATE") {
|
||||||
t.Errorf("a plan paused over a day reads %q", line)
|
t.Errorf("a plan paused over a day reads %q", line)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -298,10 +298,10 @@ func TestAPlanWaitingOnAPausedSeatSaysSoAndIsNotLate(t *testing.T) {
|
|||||||
if some.All || !reflect.DeepEqual(some.Nodes, []string{"ace"}) {
|
if some.All || !reflect.DeepEqual(some.Nodes, []string{"ace"}) {
|
||||||
t.Fatalf("%+v", some)
|
t.Fatalf("%+v", some)
|
||||||
}
|
}
|
||||||
if line := planLineWith(p, now, some); !strings.Contains(line, "LATE") {
|
if line := planLineWith(p, now, some, tierAtLeast); !strings.Contains(line, "LATE") {
|
||||||
t.Errorf("a seat paused on one holder of two made the plan not late: %q", line)
|
t.Errorf("a seat paused on one holder of two made the plan not late: %q", line)
|
||||||
}
|
}
|
||||||
if st := planStatuses([]inventory.Plan{p}, now, some)[0]; !st.Late {
|
if st := planStatuses([]inventory.Plan{p}, now, some, nil)[0]; !st.Late {
|
||||||
t.Errorf("status --json: %+v", st)
|
t.Errorf("status --json: %+v", st)
|
||||||
}
|
}
|
||||||
// A plan rolling out, or with nothing asked, is not waiting on the seat.
|
// A plan rolling out, or with nothing asked, is not waiting on the seat.
|
||||||
|
|||||||
@@ -199,7 +199,7 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
|||||||
out := meshStatus{Machines: len(nodes), Wrong: []machineDoing{},
|
out := meshStatus{Machines: len(nodes), Wrong: []machineDoing{},
|
||||||
Quiet: []machineQuiet{}, Behind: []moduleBehind{}, Waiting: []machineWaiting{},
|
Quiet: []machineQuiet{}, Behind: []moduleBehind{}, Waiting: []machineWaiting{},
|
||||||
Reported: []machineReported{}, Unresolved: []machineUnresolved{},
|
Reported: []machineReported{}, Unresolved: []machineUnresolved{},
|
||||||
Network: asked.network, Adopted: adoptedNodes(nodes), Plans: planStatuses(asked.plans, time.Now(), asked.paused)}
|
Network: asked.network, Adopted: adoptedNodes(nodes), Plans: planStatuses(asked.plans, time.Now(), asked.paused, asked.tierBounds)}
|
||||||
// In a stated order, so two readings of an unchanged mesh are the same document.
|
// In a stated order, so two readings of an unchanged mesh are the same document.
|
||||||
untakenNodes := make([]string, 0, len(asked.untaken))
|
untakenNodes := make([]string, 0, len(asked.untaken))
|
||||||
for name := range asked.untaken {
|
for name := range asked.untaken {
|
||||||
|
|||||||
@@ -0,0 +1,129 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A recorded build reaches a machine only by a person's push (novox/hq issue 295, ADR 0245).
|
||||||
|
//
|
||||||
|
// **A send carries the machine's whole declaration** (ADR 0221), composed from the build the mesh holds
|
||||||
|
// of every module on it. A module whose upgrade policy records — postgres, mongodb, keycloak, the
|
||||||
|
// network path — has its new build registered at its merge and sent nowhere, "until a person pushes".
|
||||||
|
// But every other send to its machine composed it too: on 2026-10-07 a catalogue merge adopting the
|
||||||
|
// images' health checks rebuilt postgres and mongodb with the rest, and the plan's gated send to the
|
||||||
|
// control node for mail — and to the anchor for the spreadsheet app — carried both, recreating the
|
||||||
|
// providers every consumer on those machines drops with. No gate judged them (the gate judges only what
|
||||||
|
// rolls out), and nobody had pushed.
|
||||||
|
//
|
||||||
|
// So **every send but a person's push composes a recorded module at the build its machine was last
|
||||||
|
// sent**: the manifest of that build, from the build records, in place of the one the mesh holds. The
|
||||||
|
// machine runs what it ran; the send records that it still carries that build; `status` keeps saying
|
||||||
|
// the machine is behind, and `push <node>` — a person's word — sends the new one. The bus step is a
|
||||||
|
// person's too, and carries the bus; any other recorded module waiting on the bus's machine stays.
|
||||||
|
//
|
||||||
|
// A recorded module the machine was never sent (a new assignment) is composed as the mesh holds it —
|
||||||
|
// there is nothing running to keep. One whose kept build is no longer in the records refuses the send,
|
||||||
|
// said: composing the new build would be the very move this exists to stop.
|
||||||
|
|
||||||
|
type keepRecordedKey struct{}
|
||||||
|
|
||||||
|
// sendKeeps is the context a send that is not a person's push composes under: every recorded module
|
||||||
|
// kept at the build its machine runs — except, on the bus step, the bus.
|
||||||
|
func sendKeeps(ctx context.Context, inv *inventory.Inventory) (context.Context, error) {
|
||||||
|
if !busStepSending(ctx) {
|
||||||
|
return keepingRecorded(ctx), nil
|
||||||
|
}
|
||||||
|
bus, err := pendingBus(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return keepingRecorded(ctx, bus.module), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// keepingRecorded is a context whose sends compose every recorded module at the build its machine runs,
|
||||||
|
// except the modules named (the bus, on the bus step).
|
||||||
|
func keepingRecorded(ctx context.Context, except ...string) context.Context {
|
||||||
|
skip := map[string]bool{}
|
||||||
|
for _, m := range except {
|
||||||
|
skip[m] = true
|
||||||
|
}
|
||||||
|
return context.WithValue(ctx, keepRecordedKey{}, skip)
|
||||||
|
}
|
||||||
|
|
||||||
|
// keptExcept is whether this context keeps recorded modules, and the modules it lets move.
|
||||||
|
func keptExcept(ctx context.Context) (map[string]bool, bool) {
|
||||||
|
skip, on := ctx.Value(keepRecordedKey{}).(map[string]bool)
|
||||||
|
return skip, on
|
||||||
|
}
|
||||||
|
|
||||||
|
// recordedKept is, for a send under keepingRecorded, every recorded module the machine was last sent a
|
||||||
|
// build of that the mesh's build is not identical to: module → the commit it keeps. Nil when the context
|
||||||
|
// keeps nothing, or when what the machine was last sent is not known (it is then held whole elsewhere —
|
||||||
|
// ADR 0221).
|
||||||
|
func recordedKept(ctx context.Context, open *stores, node string) (map[string]string, error) {
|
||||||
|
skip, on := keptExcept(ctx)
|
||||||
|
if !on {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
inv := open.inventory
|
||||||
|
sent, known, err := inv.SentBuilds(ctx, node)
|
||||||
|
if err != nil || !known {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
current, err := inv.CurrentBuilds(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var f *moveFacts
|
||||||
|
out := map[string]string{}
|
||||||
|
for m, was := range sent {
|
||||||
|
now, held := current[m]
|
||||||
|
if !held || now.RollOut || skip[m] || was == "" || sameCommit(was, now.Commit) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if f == nil {
|
||||||
|
read, err := readMoveFacts(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
f = &read
|
||||||
|
}
|
||||||
|
if f.identical(m, was, now.Commit) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out[m] = was
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// keepRecorded puts, in a shelf about to be resolved for a machine, the build each recorded module there
|
||||||
|
// runs in place of the one the mesh holds; it answers what it kept, module → commit.
|
||||||
|
func keepRecorded(ctx context.Context, open *stores, node string, shelf map[string]catalogue.Manifest) (map[string]string, error) {
|
||||||
|
kept, err := recordedKept(ctx, open, node)
|
||||||
|
if err != nil || len(kept) == 0 {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(kept))
|
||||||
|
for m := range kept {
|
||||||
|
names = append(names, m)
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
for _, m := range names {
|
||||||
|
ran, found, err := open.inventory.ManifestAt(ctx, m, kept[m])
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
return nil, fmt.Errorf("%s records rather than rolls out, and %s runs its build %s, which the build "+
|
||||||
|
"records no longer hold: this send cannot keep it and does not move it — `push %s` sends the new "+
|
||||||
|
"one on a person's word (novox/hq ADR 0245)", m, node, short(kept[m]), node)
|
||||||
|
}
|
||||||
|
shelf[m] = ran
|
||||||
|
}
|
||||||
|
return kept, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,223 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A recorded build reaches a machine only by a person's push (novox/hq issue 295, ADR 0245).
|
||||||
|
|
||||||
|
// aContainerBuild is a build outcome of a module of containers, each named by id with the image and the
|
||||||
|
// health it is given; a policy when one is said.
|
||||||
|
func aContainerBuild(t *testing.T, module, commit, policy string, asked time.Time, containers map[string][2]string) link.BuildResult {
|
||||||
|
t.Helper()
|
||||||
|
var resources []any
|
||||||
|
for id, c := range containers {
|
||||||
|
r := map[string]any{"id": id, "type": "container", "name": module + "-" + id, "image": c[0]}
|
||||||
|
if c[1] != "" {
|
||||||
|
r["health"] = map[string]any{"kind": c[1]}
|
||||||
|
}
|
||||||
|
resources = append(resources, r)
|
||||||
|
}
|
||||||
|
m := map[string]any{"module": module, "version": "1", "resources": resources}
|
||||||
|
if policy != "" {
|
||||||
|
m["upgrade"] = map[string]any{"policy": policy, "why": "a provider whose restart drops every consumer"}
|
||||||
|
}
|
||||||
|
manifest, _ := json.Marshal(m)
|
||||||
|
return link.BuildResult{ID: link.NewBuildID(asked), Repository: "novox/mesh-catalog", Path: "modules/" + module,
|
||||||
|
On: "anchor", Module: module, Commit: commit, Manifest: manifest,
|
||||||
|
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// imageOf is the image the composed plan of a machine gives one of a module's containers.
|
||||||
|
func imageOf(t *testing.T, plan catalogue.Resolution, module, id string) string {
|
||||||
|
t.Helper()
|
||||||
|
for _, m := range plan.Modules {
|
||||||
|
if m.Module != module {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, r := range m.Resources {
|
||||||
|
if r["id"] == id {
|
||||||
|
image, _ := r["image"].(string)
|
||||||
|
return image
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("%s has no container %s in the plan", module, id)
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tonight's case, 2026-10-07: a catalogue merge adopting the images' own health checks rebuilt the
|
||||||
|
// database (policy record) with mail (policy roll). The plan's gated send for mail carried the
|
||||||
|
// database's new build to the control node and recreated it, unjudged, with nobody's word. A send that
|
||||||
|
// is not a person's push now composes the database at the build the machine runs and records that it
|
||||||
|
// still carries it; a person's push composes the new one; the bus step moves the bus alone.
|
||||||
|
func TestARecordedBuildIsCarriedOnlyByAPersonsPush(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
start := time.Now().Add(-time.Hour)
|
||||||
|
pgImage := "registry.invalid:5000/postgres/server@sha256:" + strings.Repeat("a", 64)
|
||||||
|
mailImage := "registry.invalid:5000/mailu/smtp@sha256:" + strings.Repeat("c", 64)
|
||||||
|
for _, b := range []link.BuildResult{
|
||||||
|
aContainerBuild(t, "postgres", "c1111111", catalogue.PolicyRecord, start,
|
||||||
|
map[string][2]string{"server": {pgImage, ""}}),
|
||||||
|
aContainerBuild(t, "mailu", "c1111111", "", start.Add(time.Second),
|
||||||
|
map[string][2]string{"smtp": {mailImage, ""}, "imap": {mailImage, ""}}),
|
||||||
|
} {
|
||||||
|
if _, _, err := takeIn(ctx, inv, b); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, m := range []string{"postgres", "mailu"} {
|
||||||
|
if _, err := inv.Assign(ctx, "anchor", m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor",
|
||||||
|
map[string]string{"postgres": "c1111111", "mailu": "c1111111"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// The merge: both adopt a health check; the images are the same.
|
||||||
|
for _, b := range []link.BuildResult{
|
||||||
|
aContainerBuild(t, "postgres", "c2222222", catalogue.PolicyRecord, start.Add(time.Minute),
|
||||||
|
map[string][2]string{"server": {pgImage, "runtime"}}),
|
||||||
|
aContainerBuild(t, "mailu", "c2222222", "", start.Add(time.Minute+time.Second),
|
||||||
|
map[string][2]string{"smtp": {mailImage, "runtime"}, "imap": {mailImage, "runtime"}}),
|
||||||
|
} {
|
||||||
|
if _, _, err := takeIn(ctx, inv, b); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
healthOf := func(plan catalogue.Resolution, module, id string) any {
|
||||||
|
for _, m := range plan.Modules {
|
||||||
|
for _, r := range m.Resources {
|
||||||
|
if m.Module == module && r["id"] == id {
|
||||||
|
return r["health"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// A plan's, a release plan's, a healer's send: the database is kept as it runs, mail moves.
|
||||||
|
kept := keepingRecorded(ctx)
|
||||||
|
plan, settings, err := planFor(kept, open, "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if healthOf(plan, "postgres", "server") != nil {
|
||||||
|
t.Fatal("a send that is not a person's push composed the recorded module's new build")
|
||||||
|
}
|
||||||
|
if healthOf(plan, "mailu", "smtp") == nil {
|
||||||
|
t.Fatal("the module that rolls out was not composed at its new build")
|
||||||
|
}
|
||||||
|
if imageOf(t, plan, "postgres", "server") != pgImage {
|
||||||
|
t.Fatal("the recorded module's container lost its image")
|
||||||
|
}
|
||||||
|
gens, err := generators(kept, open)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
declared, err := declarationWith(kept, open, "anchor", plan, settings, gens, Allocating)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if declared.Builds["postgres"] != "c1111111" || declared.Builds["mailu"] != "c2222222" {
|
||||||
|
t.Fatalf("the send records it carries %v; want postgres still at c1111111 and mailu at c2222222", declared.Builds)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A person's push: the recorded module's new build.
|
||||||
|
plan, _, err = planFor(ctx, open, "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if healthOf(plan, "postgres", "server") == nil {
|
||||||
|
t.Fatal("a person's push did not compose the recorded module's new build")
|
||||||
|
}
|
||||||
|
|
||||||
|
// The bus step moves the bus alone: a recorded module it is told it may move moves, the others stay.
|
||||||
|
plan, _, err = planFor(keepingRecorded(ctx, "postgres"), open, "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if healthOf(plan, "postgres", "server") == nil {
|
||||||
|
t.Fatal("the module a send is let move was kept")
|
||||||
|
}
|
||||||
|
|
||||||
|
// What a send composes under (sendToEach): every recorded module kept; on the bus step, the bus moves.
|
||||||
|
if under, err := sendKeeps(ctx, inv); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
} else if skip, on := keptExcept(under); !on || len(skip) != 0 {
|
||||||
|
t.Fatalf("an ordinary send keeps %v %v", on, skip)
|
||||||
|
}
|
||||||
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "nats", Version: "2",
|
||||||
|
Provides: []catalogue.Offer{{Name: "mesh-bus"}}}, inventory.Source{Repository: "novox/mesh-catalog",
|
||||||
|
Seat: "git", Path: "modules/nats", BuiltFrom: "n2", Head: "n2"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if under, err := sendKeeps(withBusStep(ctx), inv); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
} else if skip, on := keptExcept(under); !on || !skip["nats"] || len(skip) != 1 {
|
||||||
|
t.Fatalf("the bus step keeps %v %v; want everything recorded but the bus", on, skip)
|
||||||
|
}
|
||||||
|
|
||||||
|
// And a recorded module whose kept build the records no longer hold refuses the send, said.
|
||||||
|
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor",
|
||||||
|
map[string]string{"postgres": "c0000000", "mailu": "c2222222"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, _, err := planFor(kept, open, "anchor"); err == nil || !strings.Contains(err.Error(), "push anchor") {
|
||||||
|
t.Fatalf("a recorded build that cannot be kept did not refuse the send with its remedy: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// And the gated send for mail says what it recreates: both containers, no new image, at once.
|
||||||
|
moves := []inventory.CarriedMove{{Module: "mailu", Node: "anchor", From: "c1111111", To: "c2222222"}}
|
||||||
|
sayRecreations(ctx, open, moves)
|
||||||
|
if !strings.Contains(moves[0].Recreates, "recreates 2 of mailu's 2") || !strings.Contains(moves[0].Recreates, "no new image") {
|
||||||
|
t.Fatalf("the send says %q of mail's containers", moves[0].Recreates)
|
||||||
|
}
|
||||||
|
if said := recreationsSaid(moves); !strings.HasPrefix(said, "on anchor recreates") {
|
||||||
|
t.Fatalf("the plan's note says %q", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The send says what it recreates (ADR 0245): mail's health checks adopted recreate both its
|
||||||
|
// containers, with no new image, every one at once.
|
||||||
|
func TestASendSaysWhatItRecreates(t *testing.T) {
|
||||||
|
image := "registry.invalid:5000/mailu/smtp@sha256:" + strings.Repeat("c", 64)
|
||||||
|
from := catalogue.Manifest{Module: "mailu", Resources: []map[string]any{
|
||||||
|
{"id": "smtp", "type": "container", "image": image},
|
||||||
|
{"id": "imap", "type": "container", "image": image},
|
||||||
|
{"id": "redis", "type": "container", "image": image},
|
||||||
|
{"id": "config", "type": "file", "path": "/etc/x"}}}
|
||||||
|
to := catalogue.Manifest{Module: "mailu", Resources: []map[string]any{
|
||||||
|
{"id": "smtp", "type": "container", "image": image, "health": map[string]any{"kind": "runtime"}},
|
||||||
|
{"id": "imap", "type": "container", "image": image, "health": map[string]any{"kind": "runtime"}},
|
||||||
|
{"id": "redis", "type": "container", "image": image},
|
||||||
|
{"id": "config", "type": "file", "path": "/etc/y"}}}
|
||||||
|
r := catalogue.Recreates(from, to)
|
||||||
|
if !r.SpecOnly() || len(r.Recreated) != 2 || r.Containers != 3 {
|
||||||
|
t.Fatalf("recreation %+v", r)
|
||||||
|
}
|
||||||
|
said := r.Say("mailu")
|
||||||
|
for _, want := range []string{"recreates 2 of mailu's 3", "no new image", "imap, smtp", "interrupted"} {
|
||||||
|
if !strings.Contains(said, want) {
|
||||||
|
t.Fatalf("%q does not say %q", said, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if catalogue.Recreates(from, from).Say("mailu") != "" {
|
||||||
|
t.Fatal("a move that recreates nothing said something")
|
||||||
|
}
|
||||||
|
to.Resources[2]["image"] = strings.Replace(image, "c", "d", 1)
|
||||||
|
if r := catalogue.Recreates(from, to); r.SpecOnly() || len(r.Images) != 1 {
|
||||||
|
t.Fatalf("a new image read as a declaration only: %+v", r)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,153 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A push of recorded builds is no repair (novox/hq issue 301, ADR 0242, to-be 45 §7).
|
||||||
|
//
|
||||||
|
// **A recorded build moves only by a person's push** (ADR 0242): a module whose upgrade policy is
|
||||||
|
// `record` — the network path, the providers whose restart costs, mail — has its new build registered
|
||||||
|
// at its merge and sent by nothing but `push <node>`. That push is the person's word the policy asks
|
||||||
|
// for: the mesh working as decided. The hand-act log counted it as a repair all the same, because a push
|
||||||
|
// by hand is the repair roll-out by default exists to end. On 2026-10-07 the operator approved, node by
|
||||||
|
// node, new builds of the resolver, the FortiClient adapter, the network managers and the packet filter;
|
||||||
|
// ten pushes later S15 wanted a healer for `split-dns`, `words` and `uplink-verbs`.
|
||||||
|
//
|
||||||
|
// **The push says what it was, from what it carried.** Never from a word the person gives — that is
|
||||||
|
// how a repair could pass for anything (issue 292). Before it is recorded, a push that names one machine
|
||||||
|
// reads what that machine was last sent against what the mesh holds. It is a **push of recorded builds**
|
||||||
|
// when:
|
||||||
|
//
|
||||||
|
// - what the machine was last sent is known;
|
||||||
|
// - its last report is no failure or refusal, so the push does not re-send to mend one;
|
||||||
|
// - it moves at least one module's build; and
|
||||||
|
// - every module it moves records rather than rolls out: a held new build of one the machine runs, or
|
||||||
|
// the first build of one newly assigned there (whose `assign` says "run `push <node>` to send it").
|
||||||
|
//
|
||||||
|
// Anything else — a build that rolls out, a module taken off, nothing moved at all — is a push by hand
|
||||||
|
// as before, and counts. The act is written with `kind: recorded-builds` and what it carried; the table
|
||||||
|
// of verbs (handActVerbs) reads a push of that kind as a person's decision, and S15 never counts it.
|
||||||
|
//
|
||||||
|
// What is not compared: settings and grants. A push of a recorded build carries whatever else
|
||||||
|
// changed in the machine's declaration with it, as any push does.
|
||||||
|
|
||||||
|
// recordedPush answers what a push to one machine carries, one "module from → to" each, when every
|
||||||
|
// build it moves is a recorded one held for a person's word; otherwise nil and why it is not.
|
||||||
|
//
|
||||||
|
// `modules` is the machine's set; `sent` and `known` what it was last sent (Inventory.SentBuilds);
|
||||||
|
// `current` what the mesh holds; `identical` whether two builds of a module put the same thing on a
|
||||||
|
// machine (moveFacts.identical); `failing` the machine's last report when it was a failure or a refusal.
|
||||||
|
func recordedPush(modules []string, sent map[string]string, known bool, current map[string]inventory.CurrentBuild,
|
||||||
|
identical func(module, a, b string) bool, failing string) ([]string, string) {
|
||||||
|
if !known {
|
||||||
|
return nil, "what the machine was last sent is not known"
|
||||||
|
}
|
||||||
|
if failing != "" {
|
||||||
|
return nil, "its last report was " + failing + ": the push sends again what failed"
|
||||||
|
}
|
||||||
|
in := map[string]bool{}
|
||||||
|
var carried []string
|
||||||
|
for _, m := range modules {
|
||||||
|
in[m] = true
|
||||||
|
// A module the mesh holds no build of, or holds without a source, has no build to move.
|
||||||
|
now, held := current[m]
|
||||||
|
if !held || now.Commit == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
was, ran := sent[m]
|
||||||
|
if ran && identical(m, was, now.Commit) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if now.RollOut {
|
||||||
|
return nil, fmt.Sprintf("%s would move %sto %s, and it rolls out: its build is a plan's to send", m,
|
||||||
|
fromBuild(was, ran), buildName(now.Commit))
|
||||||
|
}
|
||||||
|
carried = append(carried, fmt.Sprintf("%s %s→ %s", m, fromBuild(was, ran), buildName(now.Commit)))
|
||||||
|
}
|
||||||
|
for m := range sent {
|
||||||
|
if !in[m] {
|
||||||
|
return nil, m + " is taken off the machine"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(carried) == 0 {
|
||||||
|
return nil, "it moves no build: a send again"
|
||||||
|
}
|
||||||
|
sort.Strings(carried)
|
||||||
|
return carried, ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// recordedPushOf reads, for a push about to name one machine, whether it is a push of recorded builds: what
|
||||||
|
// it carries,
|
||||||
|
// or nil and why not. An error is that it could not be read; the push is then recorded as a push by hand,
|
||||||
|
// as every push was before, and says why.
|
||||||
|
func recordedPushOf(ctx context.Context, node string) ([]string, string, error) {
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
plan, _, err := planFor(ctx, open, node)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", fmt.Errorf("its set cannot be worked out: %w", err)
|
||||||
|
}
|
||||||
|
modules := make([]string, 0, len(plan.Modules))
|
||||||
|
for _, m := range plan.Modules {
|
||||||
|
modules = append(modules, m.Module)
|
||||||
|
}
|
||||||
|
sent, known, err := inv.SentBuilds(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
f, err := readMoveFacts(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
doing, said, err := inv.DoingOf(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
failing := ""
|
||||||
|
if said && (doing.Outcome == inventory.OutcomeFailed || doing.Outcome == inventory.OutcomeRefused) {
|
||||||
|
failing = doing.Outcome
|
||||||
|
}
|
||||||
|
carried, why := recordedPush(modules, sent, known, f.current, f.identical, failing)
|
||||||
|
return carried, why, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// recordedBefore are the pushes recorded before a push said its kind, each a push of recorded builds by
|
||||||
|
// what it carried (novox/hq issue 301): the operator's word, machine by machine, for new builds of modules
|
||||||
|
// whose upgrade policy records — systemd-resolved, forticlient, networkmanager, systemd-networkd,
|
||||||
|
// nftables, mailu. The log did not keep what a push carried then, so the record names them; no push
|
||||||
|
// recorded since needs it, because every push now says its kind. S15 reads these as it reads a push of
|
||||||
|
// that kind, and the three `healer-wanted` they opened clear on the next tick.
|
||||||
|
var recordedBefore = map[string]string{
|
||||||
|
"act-1791404241010309198-1": "systemd-resolved, first build on its machine (ADR 0247)",
|
||||||
|
"act-1791404587689907257-1": "systemd-resolved, first build; nftables, held build (catalogue #111)",
|
||||||
|
"act-1791404809925218992-1": "nftables, held build (catalogue #111)",
|
||||||
|
"act-1791404844504887009-1": "mailu and nftables, held builds (catalogue #106, #111)",
|
||||||
|
"act-1791408072745552019-1": "forticlient, held build (catalogue #114)",
|
||||||
|
"act-1791408101286318350-1": "forticlient, held build (catalogue #114)",
|
||||||
|
"act-1791409209593713522-1": "networkmanager, held build (catalogue #107)",
|
||||||
|
"act-1791409280125022264-1": "networkmanager, held build (catalogue #107)",
|
||||||
|
"act-1791409336586475835-1": "networkmanager, held build (catalogue #107)",
|
||||||
|
"act-1791409393494198352-1": "systemd-networkd, held build (catalogue #107)",
|
||||||
|
}
|
||||||
|
|
||||||
|
// pushedRecorded is whether an act in the log is a push of recorded builds.
|
||||||
|
func pushedRecorded(a link.HandAct) bool {
|
||||||
|
if a.Verb != "push" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if a.Kind == link.KindRecordedBuilds {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
_, before := recordedBefore[a.ID]
|
||||||
|
return before && a.Kind == ""
|
||||||
|
}
|
||||||
@@ -0,0 +1,124 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **A push says whether it pushed recorded builds only, from what it carries** (novox/hq issue 301):
|
||||||
|
// every module it moves records rather than rolls out, the machine's last send is known and its last
|
||||||
|
// report no failure. Anything else is a push by hand, as before.
|
||||||
|
func TestAPushIsOfRecordedBuildsOnlyWhenEveryBuildItMovesWasHeldForAPersonsWord(t *testing.T) {
|
||||||
|
current := map[string]inventory.CurrentBuild{
|
||||||
|
"resolver": {Commit: "r2222222"}, "nftables": {Commit: "n2222222"}, "mail": {Commit: "m2222222"},
|
||||||
|
"web": {Commit: "w2222222", RollOut: true}, "proxy": {Commit: "p1111111", RollOut: true},
|
||||||
|
}
|
||||||
|
same := func(_, a, b string) bool { return a == b }
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
modules []string
|
||||||
|
sent map[string]string
|
||||||
|
known bool
|
||||||
|
failing string
|
||||||
|
carried string // what it carried, joined; empty when it is not a push of recorded builds
|
||||||
|
why string
|
||||||
|
}{
|
||||||
|
{name: "one held build", modules: []string{"resolver", "proxy"},
|
||||||
|
sent: map[string]string{"resolver": "r1111111", "proxy": "p1111111"}, known: true,
|
||||||
|
carried: "resolver from r1111111 → r2222222"},
|
||||||
|
{name: "two held builds", modules: []string{"mail", "nftables", "proxy"},
|
||||||
|
sent: map[string]string{"mail": "m1111111", "nftables": "n1111111", "proxy": "p1111111"}, known: true,
|
||||||
|
carried: "mail from m1111111 → m2222222; nftables from n1111111 → n2222222"},
|
||||||
|
{name: "a recorded module's first build, newly assigned", modules: []string{"resolver", "proxy"},
|
||||||
|
sent: map[string]string{"proxy": "p1111111"}, known: true,
|
||||||
|
carried: "resolver (never sent it) → r2222222"},
|
||||||
|
{name: "a build that rolls out moves too", modules: []string{"resolver", "web"},
|
||||||
|
sent: map[string]string{"resolver": "r1111111", "web": "w1111111"}, known: true, why: "rolls out"},
|
||||||
|
{name: "nothing moves", modules: []string{"resolver", "proxy"},
|
||||||
|
sent: map[string]string{"resolver": "r2222222", "proxy": "p1111111"}, known: true, why: "a send again"},
|
||||||
|
{name: "a module taken off", modules: []string{"resolver"},
|
||||||
|
sent: map[string]string{"resolver": "r1111111", "proxy": "p1111111"}, known: true, why: "taken off"},
|
||||||
|
{name: "what it was last sent is not known", modules: []string{"resolver"}, why: "not known"},
|
||||||
|
{name: "its last report failed", modules: []string{"resolver"},
|
||||||
|
sent: map[string]string{"resolver": "r1111111"}, known: true, failing: "failed", why: "failed"},
|
||||||
|
} {
|
||||||
|
carried, why := recordedPush(c.modules, c.sent, c.known, current, same, c.failing)
|
||||||
|
if got := strings.Join(carried, "; "); got != c.carried {
|
||||||
|
t.Errorf("%s: carried %q, want %q (%s)", c.name, got, c.carried, why)
|
||||||
|
}
|
||||||
|
if c.why != "" && !strings.Contains(why, c.why) {
|
||||||
|
t.Errorf("%s: not of recorded builds because %q, want it to say %q", c.name, why, c.why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The live case, as the log holds it** (novox/hq issue 301). The ten pushes of 2026-10-07 recorded
|
||||||
|
// before a push said its kind pushed recorded builds only; the three `healer-wanted` they opened clear.
|
||||||
|
// The five pushes for the settings beside them, and any push that says no kind, still count; and a push
|
||||||
|
// recorded as one of recorded builds since, repeated, never does.
|
||||||
|
func TestThePushesOfRecordedBuildsOf20261007WantNoHealer(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC)
|
||||||
|
pushed := func(id, cause string, at time.Time) link.HandAct {
|
||||||
|
return link.HandAct{ID: id, At: at, By: "operator", Verb: "push", Args: []string{"laptop"},
|
||||||
|
Why: "operator's yes", Cause: cause}
|
||||||
|
}
|
||||||
|
var acts []link.HandAct
|
||||||
|
at := now.Add(-12 * time.Hour)
|
||||||
|
causes := map[string]string{
|
||||||
|
"act-1791404241010309198-1": "split-dns", "act-1791404587689907257-1": "split-dns",
|
||||||
|
"act-1791408072745552019-1": "split-dns", "act-1791408101286318350-1": "split-dns",
|
||||||
|
"act-1791404809925218992-1": "words", "act-1791404844504887009-1": "words",
|
||||||
|
"act-1791409209593713522-1": "uplink-verbs", "act-1791409280125022264-1": "uplink-verbs",
|
||||||
|
"act-1791409336586475835-1": "uplink-verbs", "act-1791409393494198352-1": "uplink-verbs",
|
||||||
|
}
|
||||||
|
if len(causes) != len(recordedBefore) {
|
||||||
|
t.Fatalf("the live case has %d acts and the record names %d", len(causes), len(recordedBefore))
|
||||||
|
}
|
||||||
|
for id, cause := range causes {
|
||||||
|
if _, named := recordedBefore[id]; !named {
|
||||||
|
t.Fatalf("%s is not named a push of recorded builds", id)
|
||||||
|
}
|
||||||
|
acts = append(acts, pushed(id, cause, at))
|
||||||
|
at = at.Add(time.Minute)
|
||||||
|
}
|
||||||
|
for i := range 2 {
|
||||||
|
acts = append(acts, pushed("act-settings-"+string(rune('a'+i)), "push", at))
|
||||||
|
at = at.Add(time.Minute)
|
||||||
|
}
|
||||||
|
f := calm(now)
|
||||||
|
f.handActs = acts
|
||||||
|
got := watchHandActs(f)
|
||||||
|
if len(got) != 1 || got[0].ID != "hand-acts.push" {
|
||||||
|
t.Fatalf("S15 saw %+v; want only the settings pushes, counted as before", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A push that says it carried recorded builds, repeated, wants no healer; one recorded with the same word and no
|
||||||
|
// kind still counts, as does an act of another verb whose ID such a push once had.
|
||||||
|
ofRecorded := func(at time.Time) link.HandAct {
|
||||||
|
a := pushed("", "split-dns", at)
|
||||||
|
a.Kind, a.Carried = link.KindRecordedBuilds, []string{"resolver from r1111111 → r2222222"}
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
f = calm(now)
|
||||||
|
f.handActs = []link.HandAct{ofRecorded(now.Add(-26 * time.Hour)), ofRecorded(now.Add(-time.Hour)),
|
||||||
|
ofRecorded(now.Add(-time.Minute))}
|
||||||
|
if got := watchHandActs(f); len(got) != 0 {
|
||||||
|
t.Errorf("pushes of recorded builds repeated asked for a healer: %+v", got)
|
||||||
|
}
|
||||||
|
f = calm(now)
|
||||||
|
f.handActs = []link.HandAct{pushed("a", "split-dns", now.Add(-time.Hour)), pushed("b", "split-dns", now.Add(-time.Minute))}
|
||||||
|
if got := watchHandActs(f); len(got) != 1 {
|
||||||
|
t.Errorf("pushes that said no kind were not counted: %+v", got)
|
||||||
|
}
|
||||||
|
other := actOf(now.Add(-time.Hour), "plans close", "split-dns")
|
||||||
|
other.ID = "act-1791404241010309198-1"
|
||||||
|
f = calm(now)
|
||||||
|
f.handActs = []link.HandAct{other, actOf(now.Add(-time.Minute), "plans close", "split-dns")}
|
||||||
|
if got := watchHandActs(f); len(got) != 1 {
|
||||||
|
t.Errorf("another verb under such a push's ID passed for one: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"slices"
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -246,7 +247,7 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(refused) > 0 {
|
if len(refused) > 0 {
|
||||||
return nil, fmt.Errorf("%w: %s — a release plan sends them, one machine at a time, each judged (novox/hq ADR "+
|
return nil, fmt.Errorf("%w: %s — a walk sends them, one machine at a time, each judged (novox/hq ADR "+
|
||||||
"0236); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; "))
|
"0236); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; "))
|
||||||
}
|
}
|
||||||
return kept, nil
|
return kept, nil
|
||||||
@@ -301,6 +302,7 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
sort.Slice(moves, func(i, j int) bool { return moves[i].Module < moves[j].Module })
|
sort.Slice(moves, func(i, j int) bool { return moves[i].Module < moves[j].Module })
|
||||||
|
sayRecreations(ctx, open, moves)
|
||||||
sent, err := sendRollout(withScope(ctx, sendScope{judged: map[string]bool{node: true}}), open, []string{node})
|
sent, err := sendRollout(withScope(ctx, sendScope{judged: map[string]bool{node: true}}), open, []string{node})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
@@ -552,6 +554,9 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
|||||||
}
|
}
|
||||||
r.Gate = &inventory.PlanGate{Machines: sent, Since: &now, Carried: moves}
|
r.Gate = &inventory.PlanGate{Machines: sent, Since: &now, Carried: moves}
|
||||||
p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves))
|
p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves))
|
||||||
|
if said := recreationsSaid(moves); said != "" {
|
||||||
|
p.Note += "; " + said
|
||||||
|
}
|
||||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
@@ -665,3 +670,88 @@ func backlogCommand(ctx context.Context, sub string, args []string) error {
|
|||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// sayRecreations says, on each move a send carries, what it does to the module's containers (novox/hq
|
||||||
|
// ADR 0245): the build the machine ran against the one it is sent, container by container. Left unsaid
|
||||||
|
// for a move whose earlier build is not in the records.
|
||||||
|
func sayRecreations(ctx context.Context, open *stores, moves []inventory.CarriedMove) {
|
||||||
|
if len(moves) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
shelf, err := open.inventory.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for i, mv := range moves {
|
||||||
|
to, held := shelf[mv.Module]
|
||||||
|
if !held || mv.From == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
from, found, err := open.inventory.ManifestAt(ctx, mv.Module, mv.From)
|
||||||
|
if err != nil || !found {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
moves[i].Recreates = catalogue.Recreates(from, to).Say(mv.Module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// sayWhatAPushRecreates says, per machine a push is about to send, every module the send moves and what
|
||||||
|
// the move recreates (novox/hq ADR 0245) — the same words a gated send keeps on its plan. A person's push
|
||||||
|
// carries every module's new build, whatever its policy and whether or not a gate saw it, so it is the
|
||||||
|
// send that most needs to say so: on 2026-10-07 a `push` of one machine moved mail to a new build and
|
||||||
|
// recreated one of its containers with a new image, and the answer said nothing about it. A machine whose
|
||||||
|
// last send's builds are not known is not said, and neither is a failure to read: the push goes on.
|
||||||
|
func sayWhatAPushRecreates(ctx context.Context, open *stores, sending []readyNode, w io.Writer) {
|
||||||
|
if len(sending) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
f, err := readMoveFacts(ctx, open.inventory)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, r := range sending {
|
||||||
|
sent, known, err := open.inventory.SentBuilds(ctx, r.node)
|
||||||
|
if err != nil || !known {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
moves := pushMoves(f, r.node, r.declared.Builds, sent)
|
||||||
|
if len(moves) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
sayRecreations(ctx, open, moves)
|
||||||
|
fmt.Fprintf(w, "%s moves %d module(s):\n", r.node, len(moves))
|
||||||
|
for _, mv := range moves {
|
||||||
|
said := mv.Recreates
|
||||||
|
if said == "" {
|
||||||
|
said = "recreates none of its containers, or what it ran is not in the records"
|
||||||
|
}
|
||||||
|
fmt.Fprintf(w, " %-28s %s → %s: %s\n", mv.Module, short(mv.From), short(mv.To), said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// pushMoves is what a send carrying these builds moves on a machine last sent `sent`: every module it ran
|
||||||
|
// before at a build not identical to the one it is now sent. A module new to the machine is not a move.
|
||||||
|
func pushMoves(f moveFacts, node string, carries, sent map[string]string) []inventory.CarriedMove {
|
||||||
|
var out []inventory.CarriedMove
|
||||||
|
for m, to := range carries {
|
||||||
|
was, ran := sent[m]
|
||||||
|
if !ran || was == "" || to == "" || f.identical(m, was, to) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, inventory.CarriedMove{Module: m, Node: node, From: was, To: to})
|
||||||
|
}
|
||||||
|
sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module })
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// recreationsSaid is every recreation a send's moves say, joined: what a plan's note carries.
|
||||||
|
func recreationsSaid(moves []inventory.CarriedMove) string {
|
||||||
|
var said []string
|
||||||
|
for _, mv := range moves {
|
||||||
|
if mv.Recreates != "" {
|
||||||
|
said = append(said, fmt.Sprintf("on %s %s", mv.Node, mv.Recreates))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return strings.Join(said, "; ")
|
||||||
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -506,23 +507,33 @@ func advanceHeld(ctx context.Context, open *stores) {
|
|||||||
for i := range plans {
|
for i := range plans {
|
||||||
p := &plans[i]
|
p := &plans[i]
|
||||||
for p.Open() {
|
for p.Open() {
|
||||||
|
// **Saved only when the step changed it** (novox/hq issue 296). Every step was saved, and
|
||||||
|
// the steps run on every tick and after every build outcome on the mesh: a plan whose tier
|
||||||
|
// stood still for half an hour was written every few seconds, its revision moved, `plan-moved`
|
||||||
|
// said on the bus, and its last save read as when it began building.
|
||||||
|
before := planSnapshot(*p)
|
||||||
moved, err := advanceOnce(ctx, open, p, edges, rollsOut)
|
moved, err := advanceOnce(ctx, open, p, edges, rollsOut)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Printf("%s: %v\n", p.ID, err)
|
|
||||||
// Kept in the plan, so `plans` says why it has not moved rather than the log alone;
|
// Kept in the plan, so `plans` says why it has not moved rather than the log alone;
|
||||||
// the state is left as it was and the step is tried again on the next tick.
|
// the state is left as it was and the step is tried again on the next tick. Said and
|
||||||
|
// kept when it is new: the same refusal on every tick is one fact, not one per tick.
|
||||||
p.Note = "tier " + fmt.Sprint(p.Tier) + ": " + err.Error() + " — tried again"
|
p.Note = "tier " + fmt.Sprint(p.Tier) + ": " + err.Error() + " — tried again"
|
||||||
if err := inv.SavePlan(ctx, p); err != nil {
|
if planSnapshot(*p) != before {
|
||||||
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
|
fmt.Printf("%s: %v\n", p.ID, err)
|
||||||
|
if err := inv.SavePlan(ctx, p); err != nil {
|
||||||
|
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
if p.State == inventory.PlanFailed {
|
if p.State == inventory.PlanFailed {
|
||||||
sayUnsent(p, rollsOut)
|
sayUnsent(p, rollsOut)
|
||||||
}
|
}
|
||||||
if err := inv.SavePlan(ctx, p); err != nil {
|
if planSnapshot(*p) != before {
|
||||||
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
|
if err := inv.SavePlan(ctx, p); err != nil {
|
||||||
break
|
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
|
||||||
|
break
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if !moved {
|
if !moved {
|
||||||
break
|
break
|
||||||
@@ -531,6 +542,18 @@ func advanceHeld(ctx context.Context, open *stores) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// planSnapshot is a plan as a save would write it, to compare before and after a step: what the store
|
||||||
|
// stamps itself (when it was saved, its revision and epoch, when its tier was entered) is left out.
|
||||||
|
func planSnapshot(p inventory.Plan) string {
|
||||||
|
p.Updated, p.Revision, p.Epoch, p.TierEntered = time.Time{}, 0, 0, time.Time{}
|
||||||
|
b, err := json.Marshal(p)
|
||||||
|
if err != nil {
|
||||||
|
// Unreadable is never the same: the plan is saved, as every step was before.
|
||||||
|
return fmt.Sprintf("unmarshallable %p %v", &p, err)
|
||||||
|
}
|
||||||
|
return string(b)
|
||||||
|
}
|
||||||
|
|
||||||
// advanceOnce takes one step of one plan and says whether anything changed.
|
// advanceOnce takes one step of one plan and says whether anything changed.
|
||||||
func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||||
edges []inventory.Edge, rollsOut func(string) bool) (bool, error) {
|
edges []inventory.Edge, rollsOut func(string) bool) (bool, error) {
|
||||||
@@ -904,6 +927,11 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
|||||||
strings.Join(sent, ", "))
|
strings.Join(sent, ", "))
|
||||||
fmt.Printf("%s: tier %d built; sent %d module(s) to %s first in one send (%s), the rest once its gate passes\n",
|
fmt.Printf("%s: tier %d built; sent %d module(s) to %s first in one send (%s), the rest once its gate passes\n",
|
||||||
p.ID, p.Tier, len(modules), strings.Join(sent, ", "), strings.Join(modules, ", "))
|
p.ID, p.Tier, len(modules), strings.Join(sent, ", "), strings.Join(modules, ", "))
|
||||||
|
// What the send recreates, said with it (novox/hq ADR 0245).
|
||||||
|
if said := recreationsSaid(carried); said != "" {
|
||||||
|
p.Note += "; " + said
|
||||||
|
fmt.Printf("%s: %s\n", p.ID, said)
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1065,12 +1093,37 @@ func planTicker(ctx context.Context, open *stores) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// planLine is one plan as `status` says it.
|
// planLine is one plan as `status` says it, given the least bound of a tier.
|
||||||
func planLine(p inventory.Plan, now time.Time) string { return planLineWith(p, now, pauseView{}) }
|
func planLine(p inventory.Plan, now time.Time) string {
|
||||||
|
return planLineWith(p, now, pauseView{}, tierAtLeast)
|
||||||
|
}
|
||||||
|
|
||||||
|
// inTierSince is when a plan began waiting on what it waits on now, which `plans`, `status` and the
|
||||||
|
// watchdog of a plan's progress (S3) all count from (novox/hq issue 296): a walk waiting for its
|
||||||
|
// delivery's word from when it was opened, as S16 counts; any other plan from when it entered its tier,
|
||||||
|
// or, when the delivery let it go after that, from the go. Never from the plan's last save: a plan is
|
||||||
|
// saved for many reasons while its tier stands still, and a clock reset by each save said a build queued
|
||||||
|
// for minutes had been building for a few seconds. A plan read without the stamp (one from before it
|
||||||
|
// was kept) counts from its last save, the only time it has.
|
||||||
|
func inTierSince(p inventory.Plan) time.Time {
|
||||||
|
if p.Waiting() {
|
||||||
|
return p.Created
|
||||||
|
}
|
||||||
|
since := p.TierEntered
|
||||||
|
if since.IsZero() {
|
||||||
|
since = p.Updated
|
||||||
|
}
|
||||||
|
if d := p.Delivery; d != nil && d.Go != nil && d.Go.After(since) {
|
||||||
|
since = *d.Go
|
||||||
|
}
|
||||||
|
return since
|
||||||
|
}
|
||||||
|
|
||||||
// planLineWith is planLine knowing whether the build seat is paused (novox/hq ADR 0219): a plan
|
// planLineWith is planLine knowing whether the build seat is paused (novox/hq ADR 0219): a plan
|
||||||
// waiting on builds nobody will take until a person resumes the seat says so, and is not late.
|
// waiting on builds nobody will take until a person resumes the seat says so, and is not late. bound is
|
||||||
func planLineWith(p inventory.Plan, now time.Time, pause pauseView) string {
|
// the plan's tier bound, the one its stalled condition is raised at (tierBounds): LATE is that condition
|
||||||
|
// said on the line (novox/hq issue 296).
|
||||||
|
func planLineWith(p inventory.Plan, now time.Time, pause pauseView, bound time.Duration) string {
|
||||||
where := fmt.Sprintf("tier %d of %d", min(p.Tier+1, len(p.Tiers)), len(p.Tiers))
|
where := fmt.Sprintf("tier %d of %d", min(p.Tier+1, len(p.Tiers)), len(p.Tiers))
|
||||||
switch p.State {
|
switch p.State {
|
||||||
case inventory.PlanDone:
|
case inventory.PlanDone:
|
||||||
@@ -1080,7 +1133,7 @@ func planLineWith(p inventory.Plan, now time.Time, pause pauseView) string {
|
|||||||
case inventory.PlanSuperseded:
|
case inventory.PlanSuperseded:
|
||||||
return fmt.Sprintf("%s %s %s", p.Repository, short(p.Commit), p.Note)
|
return fmt.Sprintf("%s %s %s", p.Repository, short(p.Commit), p.Note)
|
||||||
}
|
}
|
||||||
since := now.Sub(p.Updated).Round(time.Second)
|
since := now.Sub(inTierSince(p)).Round(time.Second)
|
||||||
if p.Waiting() {
|
if p.Waiting() {
|
||||||
// Waiting for its delivery's word is no lateness of the walk's (novox/hq ADR 0239).
|
// Waiting for its delivery's word is no lateness of the walk's (novox/hq ADR 0239).
|
||||||
return fmt.Sprintf("%s %s %s, %s, for %s", p.Repository, short(p.Commit), where, waitingNote(p), since)
|
return fmt.Sprintf("%s %s %s, %s, for %s", p.Repository, short(p.Commit), where, waitingNote(p), since)
|
||||||
@@ -1089,7 +1142,7 @@ func planLineWith(p inventory.Plan, now time.Time, pause pauseView) string {
|
|||||||
return fmt.Sprintf("%s %s %s, %s", p.Repository, short(p.Commit), where, waiting)
|
return fmt.Sprintf("%s %s %s, %s", p.Repository, short(p.Commit), where, waiting)
|
||||||
}
|
}
|
||||||
late := ""
|
late := ""
|
||||||
if since > planWaitBound {
|
if since > bound {
|
||||||
late = " — LATE"
|
late = " — LATE"
|
||||||
}
|
}
|
||||||
what := "building"
|
what := "building"
|
||||||
@@ -1162,17 +1215,19 @@ type planStatus struct {
|
|||||||
Late bool `json:"late"`
|
Late bool `json:"late"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func planStatuses(plans []inventory.Plan, now time.Time, pause pauseView) []planStatus {
|
func planStatuses(plans []inventory.Plan, now time.Time, pause pauseView, bounds tierBounds) []planStatus {
|
||||||
out := make([]planStatus, 0, len(plans))
|
out := make([]planStatus, 0, len(plans))
|
||||||
for _, p := range plans {
|
for _, p := range plans {
|
||||||
ps := planStatus{ID: p.ID, Repository: p.Repository, Commit: p.Commit, State: p.State,
|
ps := planStatus{ID: p.ID, Repository: p.Repository, Commit: p.Commit, State: p.State,
|
||||||
Tier: p.Tier, Tiers: len(p.Tiers), Since: p.Updated}
|
Tier: p.Tier, Tiers: len(p.Tiers), Since: p.Updated}
|
||||||
if p.Open() {
|
if p.Open() {
|
||||||
|
ps.Since = inTierSince(p)
|
||||||
ps.Waiting = p.Note
|
ps.Waiting = p.Note
|
||||||
if ps.Waiting == "" {
|
if ps.Waiting == "" {
|
||||||
ps.Waiting = "builds of tier " + fmt.Sprint(p.Tier)
|
ps.Waiting = "builds of tier " + fmt.Sprint(p.Tier)
|
||||||
}
|
}
|
||||||
ps.Late = now.Sub(p.Updated) > planWaitBound
|
// A walk waiting for its delivery's word is no tier late (ADR 0239): S16 is its watchdog.
|
||||||
|
ps.Late = !p.Waiting() && now.Sub(ps.Since) > bounds.of(p.Repository)
|
||||||
// Paused is a person's decision, not lateness (novox/hq ADR 0219).
|
// Paused is a person's decision, not lateness (novox/hq ADR 0219).
|
||||||
if waiting, paused := pausedWaiting(p, pause, now); paused {
|
if waiting, paused := pausedWaiting(p, pause, now); paused {
|
||||||
ps.Waiting, ps.Late = waiting, false
|
ps.Waiting, ps.Late = waiting, false
|
||||||
@@ -1184,16 +1239,16 @@ func planStatuses(plans []inventory.Plan, now time.Time, pause pauseView) []plan
|
|||||||
}
|
}
|
||||||
|
|
||||||
// openPlans is the open plans among the recent ones, and how many have waited past the bound.
|
// openPlans is the open plans among the recent ones, and how many have waited past the bound.
|
||||||
func openPlans(plans []inventory.Plan, pause pauseView) ([]inventory.Plan, int) {
|
func openPlans(plans []inventory.Plan, now time.Time, pause pauseView, bounds tierBounds) ([]inventory.Plan, int) {
|
||||||
var open []inventory.Plan
|
var open []inventory.Plan
|
||||||
late := 0
|
late := 0
|
||||||
for _, p := range plans {
|
for _, p := range plans {
|
||||||
if p.Open() {
|
if p.Open() {
|
||||||
open = append(open, p)
|
open = append(open, p)
|
||||||
if _, paused := pausedWaiting(p, pause, time.Now()); paused {
|
if _, paused := pausedWaiting(p, pause, now); paused || p.Waiting() {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if time.Since(p.Updated) > planWaitBound {
|
if now.Sub(inTierSince(p)) > bounds.of(p.Repository) {
|
||||||
late++
|
late++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1235,7 +1290,9 @@ func plansCommand(ctx context.Context, args []string) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
fmt.Printf("%s — %s\n", p.ID, planLineWith(p, now, buildSeatPause(ctx, inv, []inventory.Plan{p})))
|
bounds := readTierBounds(ctx, inv, now)
|
||||||
|
fmt.Printf("%s — %s\n", p.ID, planLineWith(p, now, buildSeatPause(ctx, inv, []inventory.Plan{p}),
|
||||||
|
bounds.of(p.Repository)))
|
||||||
if r := p.Release; r != nil {
|
if r := p.Release; r != nil {
|
||||||
// A release plan's walk (ADR 0236): machines done, the one judged, those to come.
|
// A release plan's walk (ADR 0236): machines done, the one judged, those to come.
|
||||||
fmt.Printf(" machines in order: %s; done: %s; skipped: %s\n", strings.Join(r.Order, ", "),
|
fmt.Printf(" machines in order: %s; done: %s; skipped: %s\n", strings.Join(r.Order, ", "),
|
||||||
@@ -1244,6 +1301,9 @@ func plansCommand(ctx context.Context, args []string) error {
|
|||||||
fmt.Printf(" %s\n", gateLine(r.Gate))
|
fmt.Printf(" %s\n", gateLine(r.Gate))
|
||||||
for _, c := range r.Gate.Carried {
|
for _, c := range r.Gate.Carried {
|
||||||
fmt.Printf(" %-22s %s → %s\n", c.Module, short(c.From), short(c.To))
|
fmt.Printf(" %-22s %s → %s\n", c.Module, short(c.From), short(c.To))
|
||||||
|
if c.Recreates != "" {
|
||||||
|
fmt.Printf(" %-22s %s\n", "", c.Recreates)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
@@ -1279,6 +1339,14 @@ func plansCommand(ctx context.Context, args []string) error {
|
|||||||
case s != nil && s.Gate != nil:
|
case s != nil && s.Gate != nil:
|
||||||
fmt.Printf(" %-22s %s\n", "", gateLine(s.Gate))
|
fmt.Printf(" %-22s %s\n", "", gateLine(s.Gate))
|
||||||
}
|
}
|
||||||
|
// What the send to its first machine did to its containers (ADR 0245).
|
||||||
|
if s != nil && s.Gate != nil {
|
||||||
|
for _, c := range s.Gate.Carried {
|
||||||
|
if c.Recreates != "" {
|
||||||
|
fmt.Printf(" %-22s on %s %s\n", "", c.Node, c.Recreates)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
@@ -1353,8 +1421,9 @@ func plansCommand(ctx context.Context, args []string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
pause := buildSeatPause(ctx, inv, plans)
|
pause := buildSeatPause(ctx, inv, plans)
|
||||||
|
bounds := readTierBounds(ctx, inv, now)
|
||||||
for _, p := range plans {
|
for _, p := range plans {
|
||||||
fmt.Printf("%-28s %s\n", p.ID, planLineWith(p, now, pause))
|
fmt.Printf("%-28s %s\n", p.ID, planLineWith(p, now, pause, bounds.of(p.Repository)))
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,9 +1,22 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
"github.com/novox/mesh-controller/internal/testbus"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The replays of the controller's incidents (novox/hq to-be 45 §9, M9): each a scripted replay of what
|
// The replays of the controller's incidents (novox/hq to-be 45 §9, M9): each a scripted replay of what
|
||||||
@@ -111,3 +124,462 @@ func TestReplay273AConsumerBesideItsStoreStaysBoundToIt(t *testing.T) {
|
|||||||
t.Fatalf("the resolver was bound to %q; its seat is held on anchor (issue 258)", network)
|
t.Fatalf("the resolver was bound to %q; its seat is held on anchor (issue 258)", network)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **R-crashloop — a module that crash-loops after it applied fails its gate on its first machine** (novox/hq
|
||||||
|
// ADR 0240; research 032 §6). On the home server the agent server restarted about a hundred times while the
|
||||||
|
// mesh read it applied, its tools served and no condition raised — the gate judged a module by what the
|
||||||
|
// mesh saw from outside, and nothing looked at what it ran. The outcome asserted: a build whose container
|
||||||
|
// exits at start never passes its gate on the first machine, is put back there at the bound, and never
|
||||||
|
// reaches the second.
|
||||||
|
//
|
||||||
|
// The machine is heard as a node-engine says it: its report of the apply, then the same account said again
|
||||||
|
// later, each carrying what the engine states of what it runs — `starting` as the apply ends, then the
|
||||||
|
// crash loop. What it states of the crash loop is MESH_REPLAY_STATEMENT when the lab's replay ran the
|
||||||
|
// engine against a real container (mesh-lab replays, R-crashloop), and otherwise what the engine said of
|
||||||
|
// one, kept below. Heard as bytes, so an older controller reads them as it reads any report — this file is
|
||||||
|
// written only with what the controller had before the judging, for the prover to lay over that commit.
|
||||||
|
func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
|
||||||
|
crashLoop := []byte(`{"contract":1,"at":"2026-10-07T00:00:00Z","resources":[{"module":"app","resource":"app.server",` +
|
||||||
|
`"kind":"container","target":"app-server","state":"unhealthy","reason":"restarting","since":"2026-10-07T00:00:00Z",` +
|
||||||
|
`"streak":5,"restarts":2}]}`)
|
||||||
|
if path := os.Getenv("MESH_REPLAY_STATEMENT"); path != "" {
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("the engine's statement of the crash loop: %v", err)
|
||||||
|
}
|
||||||
|
crashLoop = raw
|
||||||
|
}
|
||||||
|
// `null` is an engine that states nothing — older than the judging — and its reports carry no health.
|
||||||
|
var stated map[string]any
|
||||||
|
if err := json.Unmarshal(crashLoop, &stated); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, b := range []inventory.Build{
|
||||||
|
{ID: "build-1", Module: "app", Commit: "c1", Repository: "novox/mesh-catalog", Path: "modules/app",
|
||||||
|
Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)},
|
||||||
|
{ID: "build-2", Module: "app", Commit: "c2", Repository: "novox/mesh-catalog", Path: "modules/app",
|
||||||
|
Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)},
|
||||||
|
} {
|
||||||
|
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: b.Commit})
|
||||||
|
b.Manifest = manifest
|
||||||
|
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
registerAt := func(commit string, asked time.Time) {
|
||||||
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: commit},
|
||||||
|
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: commit,
|
||||||
|
Head: commit, Asked: asked}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
registerAt("c1", time.Now().Add(-2*time.Hour))
|
||||||
|
for _, n := range []string{"anchor", "laptop"} {
|
||||||
|
if _, err := inv.Assign(ctx, n, "app"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"app": "c1"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
registerAt("c2", time.Now().Add(-time.Minute))
|
||||||
|
|
||||||
|
// The machine: what it is sent is applied, and its report says what runs is starting.
|
||||||
|
listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}}
|
||||||
|
sequence := int64(0)
|
||||||
|
say := func(node, digest, state string) {
|
||||||
|
t.Helper()
|
||||||
|
sequence++
|
||||||
|
health := map[string]any{}
|
||||||
|
for k, v := range stated {
|
||||||
|
health[k] = v
|
||||||
|
}
|
||||||
|
health["at"] = time.Now().UTC().Format(time.RFC3339Nano)
|
||||||
|
if state != "" && stated != nil {
|
||||||
|
resources := []any{}
|
||||||
|
for _, r := range stated["resources"].([]any) {
|
||||||
|
kept := map[string]any{}
|
||||||
|
for k, v := range r.(map[string]any) {
|
||||||
|
kept[k] = v
|
||||||
|
}
|
||||||
|
kept["state"], kept["reason"] = state, ""
|
||||||
|
resources = append(resources, kept)
|
||||||
|
}
|
||||||
|
health["resources"] = resources
|
||||||
|
}
|
||||||
|
said := map[string]any{"node": node, "applied": []string{"app.server"}, "declared": digest,
|
||||||
|
"report_sequence": sequence}
|
||||||
|
if stated != nil {
|
||||||
|
said["health"] = health
|
||||||
|
}
|
||||||
|
body, _ := json.Marshal(said)
|
||||||
|
var report link.Report
|
||||||
|
if err := json.Unmarshal(body, &report); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := listener.Heard(ctx, report); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var sent [][]string
|
||||||
|
last := map[string]string{}
|
||||||
|
n := 0
|
||||||
|
wasSend := sendRollout
|
||||||
|
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||||
|
sent = append(sent, append([]string(nil), names...))
|
||||||
|
current, err := open.inventory.CurrentBuilds(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, node := range names {
|
||||||
|
n++
|
||||||
|
digest := fmt.Sprintf("d-%s-%d", node, n)
|
||||||
|
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, map[string]string{"app": current["app"].Commit}); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
last[node] = digest
|
||||||
|
say(node, digest, "starting")
|
||||||
|
}
|
||||||
|
return names, nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { sendRollout = wasSend })
|
||||||
|
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
||||||
|
gateSettle, gateEvery = 0, 0
|
||||||
|
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
|
||||||
|
|
||||||
|
built := time.Now().UTC()
|
||||||
|
plan := inventory.Plan{ID: "plan-crashloop", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
|
||||||
|
Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
|
||||||
|
Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-2"}}}
|
||||||
|
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
advancePlans(ctx, open) // the first machine is sent the new build, and starts it
|
||||||
|
if len(sent) == 0 || !slices.Equal(sent[0], []string{"anchor"}) {
|
||||||
|
t.Fatalf("sent %v, not the first machine first", sent)
|
||||||
|
}
|
||||||
|
advancePlans(ctx, open) // judged while it starts
|
||||||
|
// Its container exits at start, and the runtime restarts it: the machine says so, again and again.
|
||||||
|
for i := 0; i < 6 && len(sent) == 1; i++ {
|
||||||
|
say("anchor", last["anchor"], "")
|
||||||
|
advancePlans(ctx, open)
|
||||||
|
}
|
||||||
|
gateBound = -time.Second // and the bound passes
|
||||||
|
advancePlans(ctx, open)
|
||||||
|
|
||||||
|
p, err := inv.PlanByID(ctx, "plan-crashloop")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
gate := p.Modules["app"].Gate
|
||||||
|
for _, names := range sent {
|
||||||
|
if slices.Contains(names, "laptop") {
|
||||||
|
t.Fatalf("the crash loop passed its gate on anchor and was sent to laptop: sent %v, the gate %+v", sent, gate)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if gate == nil || gate.Verdict != inventory.GateFailed || p.State != inventory.PlanFailed {
|
||||||
|
t.Fatalf("a crash-looping build did not fail its gate on the first machine: the plan is %s (%s), its gate %+v",
|
||||||
|
p.State, p.Note, gate)
|
||||||
|
}
|
||||||
|
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" {
|
||||||
|
t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **R145 — a web application that accepts TCP and answers nothing is raised within two looks** (novox/hq
|
||||||
|
// ADR 0240 rule 4 and Phase B, issue 145). For eleven hours a web application's port was open and its
|
||||||
|
// program ran while every request hung, and the mesh said its machine was healthy; a person found it.
|
||||||
|
// Liveness cannot see it and a TCP check cannot either: the port is open. The module's declared HTTP check
|
||||||
|
// can. The engine's half (mesh-host internal/liveness TestReplaySilentWebAppIsSaidUnhealthy) states what it
|
||||||
|
// found looking at such a program; here the controller hears that statement on two looks in a row and
|
||||||
|
// raises the module's condition — the second, never the first. `null` is an engine older than the
|
||||||
|
// readiness check: it states the program alive, and nothing is raised.
|
||||||
|
func TestReplaySilentWebAppIsRaisedWithinTwoLooks(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
silent := []byte(`{"contract":2,"at":"2026-10-07T00:00:00Z","resources":[{"module":"app","resource":"app.server",` +
|
||||||
|
`"kind":"container","target":"app-server","state":"unhealthy","reason":"http / on web: no answer within 5s",` +
|
||||||
|
`"since":"2026-10-07T00:00:00Z","streak":3,"check":"http"}]}`)
|
||||||
|
if path := os.Getenv("MESH_REPLAY_STATEMENT"); path != "" {
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("the engine's statement of the silent web application: %v", err)
|
||||||
|
}
|
||||||
|
silent = raw
|
||||||
|
}
|
||||||
|
var h link.Health
|
||||||
|
if err := json.Unmarshal(silent, &h); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if h.Contract == 0 {
|
||||||
|
t.Fatal("the engine states nothing of the silent web application: it is older than the judging")
|
||||||
|
}
|
||||||
|
open1 := func() []conditions.Condition {
|
||||||
|
t.Helper()
|
||||||
|
list, err := conditionsFrom.Open(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return list
|
||||||
|
}
|
||||||
|
for look := 1; look <= 2; look++ {
|
||||||
|
at := time.Now().Add(time.Duration(look) * time.Second)
|
||||||
|
h.At = at
|
||||||
|
if err := stateHealth(ctx, open.inventory, conditionsFrom, "anchor", h, at); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
raised := open1()
|
||||||
|
switch {
|
||||||
|
case look == 1 && len(raised) != 0:
|
||||||
|
t.Fatalf("one look raised %v", raised[0].Key)
|
||||||
|
case look == 2 && (len(raised) != 1 || raised[0].Key != "module.app.anchor.unhealthy"):
|
||||||
|
t.Fatalf("two looks in a row did not raise the module's condition: %v", raised)
|
||||||
|
case look == 2:
|
||||||
|
t.Logf("raised on the second look: %s", raised[0].Summary)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **R296 — a plan's clock counts from its tier, not from its last save.** On 2026-10-07 a plan for a
|
||||||
|
// merge to the catalogue, its one module's build asked at 19:48:41 and queued at the build seat for
|
||||||
|
// minutes, read "tier 1 of 1, building for 1s", then 4s, then 9s: the line counted from the plan's last
|
||||||
|
// save, and every advance — on the 30-second tick and after every build outcome on the mesh — saved it
|
||||||
|
// whether or not anything moved. A clock reset by each save is never LATE, so `plans` and `status` could
|
||||||
|
// not say a plan was stuck. The outcome asserted: a plan in its tier for forty minutes, saved seconds ago,
|
||||||
|
// says forty minutes and LATE.
|
||||||
|
func TestReplay296APlansLineCountsFromWhenItEnteredItsTier(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 7, 20, 30, 0, 0, time.UTC)
|
||||||
|
entered := time.Date(2026, 10, 7, 19, 48, 41, 0, time.UTC)
|
||||||
|
asked := entered
|
||||||
|
p := inventory.Plan{ID: "plan-296", Repository: "novox/mesh-catalog", Branch: "main", Commit: "3ad22356",
|
||||||
|
State: inventory.PlanBuilding, Created: entered, TierEntered: entered, Updated: now.Add(-4 * time.Second),
|
||||||
|
Tiers: [][]string{{"app"}}, Modules: map[string]*inventory.PlanModule{"app": {State: "asked", AskedAt: &asked}}}
|
||||||
|
line := planLine(p, now)
|
||||||
|
if !strings.Contains(line, "for 41m19s") || !strings.Contains(line, "LATE") {
|
||||||
|
t.Fatalf("a plan in its tier since 19:48:41, read at 20:30:00 and saved 4s before, reads %q", line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **R296, the save.** The same live case from the store's side: a plan whose only build was asked and
|
||||||
|
// still waits is advanced as the tick and the mesh's build outcomes advance it, and is not written again —
|
||||||
|
// its revision and its last save stay where its last change left them. Before the fix each advance saved
|
||||||
|
// it, and its revision went from 3 to 6.
|
||||||
|
func TestReplay296APlanStandingStillInItsTierIsNotSavedAgain(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
asked := asksRecorded(t)
|
||||||
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"},
|
||||||
|
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main",
|
||||||
|
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "3ad22356aa",
|
||||||
|
Paths: []string{"modules/app/index.ts"}, ModuleDirs: []string{"modules/app"}, ModuleDirsSaid: true}
|
||||||
|
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
advanceHeld(ctx, open)
|
||||||
|
recent, err := inv.RecentPlans(ctx, 1)
|
||||||
|
if err != nil || len(recent) != 1 || len(*asked) != 1 {
|
||||||
|
t.Fatalf("no plan waiting on its build: %v %v, asked %v", recent, err, *asked)
|
||||||
|
}
|
||||||
|
before := recent[0]
|
||||||
|
if before.State != inventory.PlanBuilding || before.Modules["app"] == nil || before.Modules["app"].State != "asked" {
|
||||||
|
t.Fatalf("the plan is not waiting on its build: %+v", before)
|
||||||
|
}
|
||||||
|
for range 3 {
|
||||||
|
advanceHeld(ctx, open)
|
||||||
|
}
|
||||||
|
after, err := inv.PlanByID(ctx, before.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if after.Revision != before.Revision || !after.Updated.Equal(before.Updated) {
|
||||||
|
t.Fatalf("a plan that did not move was saved again: revision %d → %d, saved %s → %s",
|
||||||
|
before.Revision, after.Revision, before.Updated, after.Updated)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **R299 — a holder of a seat whose verbs are all optional is not silent.** On 2026-10-07 the
|
||||||
|
// node-uplink seat gained its first verbs, `resolvers` and `links`, both optional while its holders catch
|
||||||
|
// up (ADR 0246, step 1); its holders served neither, so they registered nothing on the bus's discovery for
|
||||||
|
// it, and the self-check's D3 raised `seat.node-uplink.<machine>.silent` on every machine of the mesh,
|
||||||
|
// which the healer then acted on. The outcome asserted: through D3 itself — the seats read back from the
|
||||||
|
// store as the controller reads them, a holder on record on every machine heard from, and nothing
|
||||||
|
// answering the discovery — no holder of a seat whose verbs are all optional is said silent.
|
||||||
|
//
|
||||||
|
// The live seat is replayed while its verbs are all optional; beside it, a seat whose verbs are all
|
||||||
|
// optional by definition, so the outcome is still held once node-uplink's verbs are required (step 3).
|
||||||
|
func TestReplay299AHolderOfASeatWhoseVerbsAreAllOptionalIsNotSilent(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
catchingUp := catalogue.Seat{Name: "node-catching-up", Scope: catalogue.ScopeNode, Decision: "novox/hq issue 299",
|
||||||
|
Serves: []catalogue.Verb{{Name: "first", Optional: true}, {Name: "second", Optional: true}}}
|
||||||
|
if _, err := inv.SeedSeats(ctx, append(catalogue.DefaultSeats(), catchingUp)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rows, err := inv.Seats(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// The working set as the serving controller holds it: the store's rows, the compiled seats' marks.
|
||||||
|
// A seat the compiled set does not know keeps its mark only as defined here.
|
||||||
|
for i := range rows {
|
||||||
|
if rows[i].Name == catchingUp.Name {
|
||||||
|
rows[i].Serves = catchingUp.Serves
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catalogue.UseSeats(rows)
|
||||||
|
t.Cleanup(func() { catalogue.UseSeats(catalogue.DefaultSeats()) })
|
||||||
|
|
||||||
|
seats := []string{catchingUp.Name}
|
||||||
|
if uplink, ok := catalogue.SeatNamed("node-uplink"); ok && len(uplink.Serves) > 0 {
|
||||||
|
allOptional := true
|
||||||
|
for _, v := range uplink.Serves {
|
||||||
|
allOptional = allOptional && v.Optional
|
||||||
|
}
|
||||||
|
if allOptional {
|
||||||
|
seats = append(seats, "node-uplink")
|
||||||
|
} else {
|
||||||
|
t.Logf("node-uplink's verbs are required at this commit; the live seat is not replayed, its kind is")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1"})
|
||||||
|
now := time.Now()
|
||||||
|
var heard []machineFacts
|
||||||
|
for _, node := range []string{"anchor", "laptop"} {
|
||||||
|
if _, err := inv.Assign(ctx, node, "networkmanager"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, seat := range seats {
|
||||||
|
if err := inv.HoldSeat(ctx, seat, catalogue.ScopeNode, node, "networkmanager"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
heard = append(heard, machineFacts{name: node, lastHeard: now})
|
||||||
|
}
|
||||||
|
js, err := broker.Dial(testbus.URL(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(js.Close)
|
||||||
|
d := &doctor{open: open, js: js, watchdogs: &watchdogs{last: &signalFacts{now: now, machines: heard}}}
|
||||||
|
|
||||||
|
found, err := probeHolders(ctx, d)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, o := range found {
|
||||||
|
for _, seat := range seats {
|
||||||
|
if strings.HasPrefix(o.ID, seat+".") {
|
||||||
|
t.Errorf("a holder serving none of a seat's optional verbs was said silent: %s (%s)", o.Summary, o.Said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **R300 — a merge that adds a module builds it.** On 2026-10-07 a pull request to the catalogue added
|
||||||
|
// one module, systemd-resolved; its delivery plan said "builds new: modules/systemd-resolved, sent
|
||||||
|
// nowhere", and at the merge the controller said it "changed nothing any module the mesh holds is built
|
||||||
|
// from" and built nothing, so the module was never registered and `assign` refused it until a person
|
||||||
|
// built it by hand. The outcome asserted: the merge asks the build seat for the new module's directory, at
|
||||||
|
// the branch merged into, from the repository as the mesh spells it, and opens no plan for it.
|
||||||
|
func TestReplay300AMergeAddingAModuleAsksForItsBuild(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
var asked [][3]string
|
||||||
|
was := askABuild
|
||||||
|
askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) {
|
||||||
|
asked = append(asked, [3]string{source.Repository, path, ref})
|
||||||
|
return "build-" + path, nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { askABuild = was })
|
||||||
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "networkmanager", Version: "1"},
|
||||||
|
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/networkmanager", Ref: "main",
|
||||||
|
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "3da80a4b00",
|
||||||
|
Paths: []string{"modules/systemd-resolved/module.json", "modules/systemd-resolved/cmd/main.go"},
|
||||||
|
ModuleDirs: []string{"modules/systemd-resolved"}, ModuleDirsSaid: true}
|
||||||
|
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := [3]string{"novox/mesh-catalog", "modules/systemd-resolved", "main"}
|
||||||
|
if len(asked) != 1 || asked[0] != want {
|
||||||
|
t.Fatalf("the merge adding modules/systemd-resolved asked the build seat for %v, not %v", asked, want)
|
||||||
|
}
|
||||||
|
if plans, err := inv.OpenPlans(ctx); err != nil || len(plans) != 0 {
|
||||||
|
t.Fatalf("a merge moving nothing the mesh holds opened a plan: %+v %v", plans, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **R301 — a person's push of a held recorded build is no repair.** On 2026-10-07 the operator approved,
|
||||||
|
// machine by machine, new builds of modules whose upgrade policy records rather than rolls out — the
|
||||||
|
// resolver, the FortiClient adapter, the network managers, the packet filter, mail. A recorded build
|
||||||
|
// moves only by a person's push (ADR 0242), so each was `push <machine>` through the seat with the
|
||||||
|
// operator's yes as its why and a word for the change as its cause. The hand-act log counted every one
|
||||||
|
// as a repair, and S15 raised `mesh.hand-acts.split-dns.healer-wanted` ("repaired by hand 4 times in 14
|
||||||
|
// days … a healer is wanted"), and the same for `words` and `uplink-verbs`. The outcome asserted: a push
|
||||||
|
// that carries only held recorded builds, recorded as the push records it, wants no healer however
|
||||||
|
// often the person gives that word — through S15 itself, reading the log as the watchdog reads it.
|
||||||
|
func TestReplay301APersonsPushOfAHeldRecordedBuildIsNoRepair(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
start := time.Now().Add(-time.Hour)
|
||||||
|
image := "registry.invalid:5000/resolver/server@sha256:" + strings.Repeat("e", 64)
|
||||||
|
if _, _, err := takeIn(ctx, inv, aContainerBuild(t, "resolver", "c1111111", catalogue.PolicyRecord, start,
|
||||||
|
map[string][2]string{"server": {image, ""}})); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, node := range []string{"anchor", "laptop"} {
|
||||||
|
if _, err := inv.Assign(ctx, node, "resolver"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.RecordSent(ctx, nodeID(t, open, node), "d-"+node, map[string]string{"resolver": "c1111111"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The merge: the resolver's new build, registered and held for a person's word.
|
||||||
|
if _, _, err := takeIn(ctx, inv, aContainerBuild(t, "resolver", "c2222222", catalogue.PolicyRecord,
|
||||||
|
start.Add(time.Minute), map[string][2]string{"server": {image, "runtime"}})); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
conn := onATestBus(t)
|
||||||
|
// The operator's yes, machine by machine, as the seat's push records it before it sends.
|
||||||
|
for _, node := range []string{"anchor", "laptop"} {
|
||||||
|
set := flag.NewFlagSet("push", flag.ContinueOnError)
|
||||||
|
why := addHandActFlags(set)
|
||||||
|
if err := set.Parse([]string{"--why", "operator's yes: the resolver's new build on " + node,
|
||||||
|
"--cause", "split-dns"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
why.record(ctx, "push", []string{node})
|
||||||
|
}
|
||||||
|
acts, err := link.HandActs(ctx, conn, start)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(acts) != 2 {
|
||||||
|
t.Fatalf("the two pushes were recorded as %d act(s): %+v", len(acts), acts)
|
||||||
|
}
|
||||||
|
f := calm(time.Now())
|
||||||
|
f.handActs = acts
|
||||||
|
for _, o := range watchHandActs(f) {
|
||||||
|
if strings.Contains(o.ID, "split-dns") {
|
||||||
|
t.Fatalf("a person's push of a held recorded build was counted as a repair: %s — %s", o.ID, o.Summary)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -456,6 +456,15 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
|||||||
argv = append(argv, "--condition", c)
|
argv = append(argv, "--condition", c)
|
||||||
}
|
}
|
||||||
return argv, nil
|
return argv, nil
|
||||||
|
case "drill":
|
||||||
|
if err := need("what", "why"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
argv := []string{"hand-act", "drill", str("what"), "--why", str("why")}
|
||||||
|
if c := str("condition"); c != "" {
|
||||||
|
argv = append(argv, "--condition", c)
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
case "hand-acts":
|
case "hand-acts":
|
||||||
argv := []string{"hand-acts", "--json"}
|
argv := []string{"hand-acts", "--json"}
|
||||||
if d := str("days"); d != "" {
|
if d := str("days"); d != "" {
|
||||||
@@ -723,6 +732,8 @@ func repairingCommand(argv []string) string {
|
|||||||
return "plans " + argv[1]
|
return "plans " + argv[1]
|
||||||
case argv[0] == "broker" && len(argv) > 1 && argv[1] == "consumer-reset":
|
case argv[0] == "broker" && len(argv) > 1 && argv[1] == "consumer-reset":
|
||||||
return "broker consumer-reset"
|
return "broker consumer-reset"
|
||||||
|
case argv[0] == "hand-act" && len(argv) > 1 && argv[1] == "drill":
|
||||||
|
return "hand-act drill"
|
||||||
case argv[0] == "hand-act":
|
case argv[0] == "hand-act":
|
||||||
return "hand-act record"
|
return "hand-act record"
|
||||||
case argv[0] == "conditions" && len(argv) > 1 && argv[1] == "silence":
|
case argv[0] == "conditions" && len(argv) > 1 && argv[1] == "silence":
|
||||||
@@ -740,12 +751,18 @@ func isWhyFlag(word string) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// runVerb runs this binary with the given command line and gathers what it said.
|
// runVerb runs this binary with the given command line and gathers what it said.
|
||||||
|
//
|
||||||
|
// **This binary is the image this process runs, never the file at the path it started from**
|
||||||
|
// (novox/hq issue 289): the node-engine's witness moves a running build aside when it places the next
|
||||||
|
// one, into a directory only it may enter, and deletes it once the next is proved — while this process
|
||||||
|
// may still be serving. A verb run from the path then failed with "permission denied" for the seconds
|
||||||
|
// the old controller still answered. selfCommand runs what this process is running, wherever its file
|
||||||
|
// went; a verb it still cannot start is refused as a handover, which the caller asks again.
|
||||||
func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
||||||
self, err := os.Executable()
|
if handingOver.Load() {
|
||||||
if err != nil {
|
return verbAnswer{}, fmt.Errorf("%w: this controller is stopping and runs no new command", link.ErrHandingOver)
|
||||||
return verbAnswer{}, err
|
|
||||||
}
|
}
|
||||||
cmd := exec.CommandContext(ctx, self, argv...)
|
cmd := selfCommand(ctx, argv)
|
||||||
// The same environment: the stores' credentials, the bus, the broker — everything a command run
|
// The same environment: the stores' credentials, the bus, the broker — everything a command run
|
||||||
// from a shell in this container would have, because it is that.
|
// from a shell in this container would have, because it is that.
|
||||||
cmd.Env = os.Environ()
|
cmd.Env = os.Environ()
|
||||||
@@ -773,6 +790,12 @@ func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
|||||||
var exit *exec.ExitError
|
var exit *exec.ExitError
|
||||||
if runErr != nil && !errors.As(runErr, &exit) {
|
if runErr != nil && !errors.As(runErr, &exit) {
|
||||||
// Not the command refusing — the command not running at all, which is this process's fault.
|
// Not the command refusing — the command not running at all, which is this process's fault.
|
||||||
|
if errors.Is(runErr, os.ErrPermission) || errors.Is(runErr, os.ErrNotExist) {
|
||||||
|
// Its own image unreachable: a build replaced under a process that has not yet stopped.
|
||||||
|
// Refused as a handover, so the caller asks the controller that follows.
|
||||||
|
return answer, fmt.Errorf("%w: could not run %s from this controller's own build: %v",
|
||||||
|
link.ErrHandingOver, strings.Join(argv, " "), runErr)
|
||||||
|
}
|
||||||
return answer, fmt.Errorf("could not run %s: %w", strings.Join(argv, " "), runErr)
|
return answer, fmt.Errorf("could not run %s: %w", strings.Join(argv, " "), runErr)
|
||||||
}
|
}
|
||||||
return answer, nil
|
return answer, nil
|
||||||
@@ -947,9 +970,15 @@ func seatTools() map[string]any {
|
|||||||
}
|
}
|
||||||
var tools []map[string]any
|
var tools []map[string]any
|
||||||
for _, v := range s.Serves {
|
for _, v := range s.Serves {
|
||||||
tools = append(tools, map[string]any{
|
tool := map[string]any{
|
||||||
"name": v.Name, "description": v.Description, "input": v.Input, "output": v.Output,
|
"name": v.Name, "description": v.Description, "input": v.Input, "output": v.Output,
|
||||||
})
|
}
|
||||||
|
// What the verb is the mesh's way to do (novox/hq ADR 0245): read by the console's search and
|
||||||
|
// the agent's instructions.
|
||||||
|
if len(v.Replaces) > 0 {
|
||||||
|
tool["replaces"] = v.Replaces
|
||||||
|
}
|
||||||
|
tools = append(tools, tool)
|
||||||
}
|
}
|
||||||
seats = append(seats, map[string]any{"seat": s.Name, "scope": s.Scope, "tools": tools})
|
seats = append(seats, map[string]any{"seat": s.Name, "scope": s.Scope, "tools": tools})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -144,6 +144,23 @@ func TestToolsAnswersTheSeatsRecords(t *testing.T) {
|
|||||||
if !found {
|
if !found {
|
||||||
t.Fatal("the mesh-controller seat is not in the listing")
|
t.Fatal("the mesh-controller seat is not in the listing")
|
||||||
}
|
}
|
||||||
|
// And what a verb replaces travels with it (novox/hq ADR 0245): the console and the agent's
|
||||||
|
// instructions read it from here.
|
||||||
|
var journal []string
|
||||||
|
for _, s := range seats {
|
||||||
|
if s["seat"] != catalogue.ServiceManagerSeat {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
tools, _ := s["tools"].([]map[string]any)
|
||||||
|
for _, tool := range tools {
|
||||||
|
if tool["name"] == "journal" {
|
||||||
|
journal, _ = tool["replaces"].([]string)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(journal) == 0 || journal[0] != "journalctl" {
|
||||||
|
t.Fatalf("the service manager's journal does not say it replaces journalctl: %v", journal)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A JSON verb's answer is parsed from what the command wrote to standard output alone; a warning it
|
// A JSON verb's answer is parsed from what the command wrote to standard output alone; a warning it
|
||||||
|
|||||||
@@ -0,0 +1,46 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"runtime"
|
||||||
|
"sync/atomic"
|
||||||
|
)
|
||||||
|
|
||||||
|
// startedFrom is the path this process's executable had when it started: what a verb's command line
|
||||||
|
// is named in a process listing, and what is run where the image cannot be named otherwise.
|
||||||
|
var startedFrom, _ = os.Executable()
|
||||||
|
|
||||||
|
// ownImage is how a process names the executable it is running, as long as it runs, wherever the file
|
||||||
|
// has gone since. On Linux the kernel keeps it: /proc/self/exe is the running image itself, not a path,
|
||||||
|
// so it is valid after the file is renamed into a directory this process may not enter, or deleted —
|
||||||
|
// which is what the node-engine's witness does to a build it replaces (novox/hq issue 289). Read in the
|
||||||
|
// child, it names the child's image, which until the exec is this process's.
|
||||||
|
//
|
||||||
|
// os.Executable reads the same link and returns the path it points at *now*: correct at start and
|
||||||
|
// wrong the moment the file moves, which is the fault. A variable so a test can name another.
|
||||||
|
var ownImage = func() string {
|
||||||
|
if runtime.GOOS == "linux" {
|
||||||
|
if _, err := os.Stat("/proc/self/exe"); err == nil {
|
||||||
|
return "/proc/self/exe"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return startedFrom
|
||||||
|
}
|
||||||
|
|
||||||
|
// selfCommand is this binary run with a command line: the image this process runs, named in a process
|
||||||
|
// listing as the path it started from.
|
||||||
|
func selfCommand(ctx context.Context, argv []string) *exec.Cmd {
|
||||||
|
cmd := exec.CommandContext(ctx, ownImage(), argv...)
|
||||||
|
if startedFrom != "" {
|
||||||
|
cmd.Args[0] = startedFrom
|
||||||
|
}
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
// handingOver is set when the serving controller begins to stop — a signal from its supervisor, a lease
|
||||||
|
// lost. From then a verb that would run a command is refused as a handover rather than started and
|
||||||
|
// killed with this process: the caller asks again, and the controller after this one answers
|
||||||
|
// (novox/hq issue 289).
|
||||||
|
var handingOver atomic.Bool
|
||||||
@@ -0,0 +1,177 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The roles a copy of this test binary plays in TestAVerbRunsWhileItsBuildIsMovedOrDeleted.
|
||||||
|
const selfExecRole = "MESH_CONTROLLER_SELFEXEC_ROLE"
|
||||||
|
|
||||||
|
// TestSelfExecServer is a controller standing in, when run as one: it waits until its build has been
|
||||||
|
// moved, then runs a verb as the seat runs one, and prints what came of it as JSON.
|
||||||
|
func TestSelfExecServer(t *testing.T) {
|
||||||
|
if os.Getenv(selfExecRole) != "server" {
|
||||||
|
t.Skip("run by TestAVerbRunsWhileItsBuildIsMovedOrDeleted")
|
||||||
|
}
|
||||||
|
line, _ := bufio.NewReader(os.Stdin).ReadString('\n')
|
||||||
|
if strings.TrimSpace(line) != "go" {
|
||||||
|
t.Fatalf("told %q", line)
|
||||||
|
}
|
||||||
|
if err := os.Setenv(selfExecRole, "verb"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
answer, err := runVerb(t.Context(), []string{"-test.run", "^TestSelfExecVerb$", "-test.v"})
|
||||||
|
said := map[string]any{"ok": answer.OK, "output": answer.Output}
|
||||||
|
if err != nil {
|
||||||
|
said["error"] = err.Error()
|
||||||
|
}
|
||||||
|
body, _ := json.Marshal(said)
|
||||||
|
fmt.Println("ANSWER " + string(body))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSelfExecVerb is the verb, when run as one.
|
||||||
|
func TestSelfExecVerb(t *testing.T) {
|
||||||
|
if os.Getenv(selfExecRole) != "verb" {
|
||||||
|
t.Skip("run by TestSelfExecServer")
|
||||||
|
}
|
||||||
|
fmt.Println("the verb ran")
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A verb runs while the build it was started from is moved where its user may not go, or deleted**
|
||||||
|
// (novox/hq issue 289). The node-engine's witness moves a running controller's build into a directory
|
||||||
|
// only it may enter as it places the next, and deletes it once the next is proved; the controller
|
||||||
|
// still serving in between ran its verbs from the path and answered "permission denied".
|
||||||
|
//
|
||||||
|
// A copy of this test binary is the controller: started from one place, moved into a directory closed
|
||||||
|
// to everyone (or deleted), and only then asked to run a verb.
|
||||||
|
func TestAVerbRunsWhileItsBuildIsMovedOrDeleted(t *testing.T) {
|
||||||
|
if runtime.GOOS != "linux" {
|
||||||
|
t.Skip("the image is named through /proc on Linux only")
|
||||||
|
}
|
||||||
|
self, err := os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, how := range []string{"moved into a closed directory", "deleted"} {
|
||||||
|
t.Run(how, func(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
placed := filepath.Join(root, "mesh-controller", "mesh-controller")
|
||||||
|
if err := os.MkdirAll(filepath.Dir(placed), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
copyFile(t, self, placed)
|
||||||
|
|
||||||
|
server := exec.Command(placed, "-test.run", "^TestSelfExecServer$", "-test.v")
|
||||||
|
server.Env = append(os.Environ(), selfExecRole+"=server")
|
||||||
|
stdin, err := server.StdinPipe()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
stdout, err := server.StdoutPipe()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
server.Stderr = os.Stderr
|
||||||
|
if err := server.Start(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = server.Process.Kill(); _ = server.Wait() })
|
||||||
|
|
||||||
|
// What the witness does to a running build, once the process runs.
|
||||||
|
switch how {
|
||||||
|
case "deleted":
|
||||||
|
if err := os.RemoveAll(filepath.Dir(placed)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
kept := filepath.Join(root, ".witness", "mesh-controller", "previous")
|
||||||
|
if err := os.MkdirAll(filepath.Dir(kept), 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Rename(filepath.Dir(placed), kept); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Chmod(filepath.Join(root, ".witness"), 0); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = os.Chmod(filepath.Join(root, ".witness"), 0o700) })
|
||||||
|
}
|
||||||
|
if _, err := io.WriteString(stdin, "go\n"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
out, _ := io.ReadAll(stdout)
|
||||||
|
_ = server.Wait()
|
||||||
|
var said struct {
|
||||||
|
OK bool `json:"ok"`
|
||||||
|
Output string `json:"output"`
|
||||||
|
Error string `json:"error"`
|
||||||
|
}
|
||||||
|
found := false
|
||||||
|
for _, line := range strings.Split(string(out), "\n") {
|
||||||
|
if rest, ok := strings.CutPrefix(line, "ANSWER "); ok {
|
||||||
|
found = json.Unmarshal([]byte(rest), &said) == nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatalf("the controller standing in answered nothing:\n%s", out)
|
||||||
|
}
|
||||||
|
if said.Error != "" || !said.OK || !strings.Contains(said.Output, "the verb ran") {
|
||||||
|
t.Fatalf("the verb did not run from the controller's own image after its build was %s: %+v", how, said)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A verb the controller cannot start from its own build is refused as a handover — marked so its
|
||||||
|
// caller asks again — never a permission error; and so is one arriving once the controller is stopping.
|
||||||
|
func TestAVerbThatCannotRunIsRefusedAsAHandover(t *testing.T) {
|
||||||
|
closed := filepath.Join(t.TempDir(), "closed")
|
||||||
|
if err := os.MkdirAll(closed, 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
was := ownImage
|
||||||
|
ownImage = func() string { return filepath.Join(closed, "gone", "mesh-controller") }
|
||||||
|
t.Cleanup(func() { ownImage = was })
|
||||||
|
_, err := runVerb(t.Context(), []string{"status"})
|
||||||
|
if !errors.Is(err, link.ErrHandingOver) {
|
||||||
|
t.Fatalf("a build that is not there was answered %v, not a handover", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ownImage = was
|
||||||
|
handingOver.Store(true)
|
||||||
|
t.Cleanup(func() { handingOver.Store(false) })
|
||||||
|
if _, err := runVerb(t.Context(), []string{"-test.run", "^$"}); !errors.Is(err, link.ErrHandingOver) {
|
||||||
|
t.Fatalf("a controller stopping ran a verb: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyFile(t *testing.T, from, to string) {
|
||||||
|
t.Helper()
|
||||||
|
in, err := os.Open(from)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer in.Close()
|
||||||
|
out, err := os.OpenFile(to, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o755)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := io.Copy(out, in); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := out.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -550,3 +550,47 @@ func TestAWalkWaitingFourHoursIsUrgentAndNamesTheWayOn(t *testing.T) {
|
|||||||
t.Fatalf("it does not say how on: %q", got[0].Summary)
|
t.Fatalf("it does not say how on: %q", got[0].Summary)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **A drill is a person's deliberate test, never a repair** — the log of 2026-10-07: two drills of ADR
|
||||||
|
// 0240 recorded through `hand-act record --cause drill` before `hand-act drill` existed raised
|
||||||
|
// `mesh.hand-acts.drill.healer-wanted`. A drill through its own verb never counts, the two recorded
|
||||||
|
// before it clear on the next tick, and the cause word alone on any other verb still counts — whether
|
||||||
|
// an act is a drill is said by the verb chosen, not by a word typed into a repair's cause.
|
||||||
|
func TestADrillIsNoRepairAndItsHealerWantedClears(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC)
|
||||||
|
f := calm(now)
|
||||||
|
f.handActs = []link.HandAct{actOf(now.Add(-26*time.Hour), "hand-act drill", causeDrill),
|
||||||
|
actOf(now.Add(-time.Hour), "hand-act drill", causeDrill), actOf(now.Add(-time.Minute), "hand-act drill", causeDrill)}
|
||||||
|
if got := watchHandActs(f); len(got) != 0 {
|
||||||
|
t.Errorf("drills repeated asked for a healer: %+v", got)
|
||||||
|
}
|
||||||
|
for _, verb := range []string{"push", "conditions silence", "plans close", "a verb nobody listed"} {
|
||||||
|
f := calm(now)
|
||||||
|
f.handActs = []link.HandAct{actOf(now.Add(-26*time.Hour), verb, causeDrill), actOf(now.Add(-time.Hour), verb, causeDrill)}
|
||||||
|
if got := watchHandActs(f); len(got) != 1 {
|
||||||
|
t.Errorf("%s with the cause %q passed for a drill: %+v", verb, causeDrill, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
store := conditions.NewInMemory()
|
||||||
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||||
|
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
||||||
|
defer k.Close(context.Background())
|
||||||
|
before := []conditions.Observation{{Scope: conditions.ScopeMesh, ID: "hand-acts." + causeDrill,
|
||||||
|
Token: "healer-wanted", Kind: "healer-wanted", Severity: conditions.Warning,
|
||||||
|
Summary: "\"drill\" was repaired by hand 2 times in 14 days"}}
|
||||||
|
if err := k.Reconcile(t.Context(), "S15", before); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if open, _ := k.Open(t.Context()); len(open) != 1 {
|
||||||
|
t.Fatalf("the condition of the build before was not open: %+v", open)
|
||||||
|
}
|
||||||
|
f = calm(now)
|
||||||
|
f.handActs = []link.HandAct{actOf(now.Add(-11*time.Hour), "hand-act record", causeDrill),
|
||||||
|
actOf(now.Add(-30*time.Minute), "hand-act record", causeDrill)}
|
||||||
|
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
||||||
|
w.see(t.Context(), f)
|
||||||
|
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||||
|
t.Fatalf("a healer-wanted for two drills stayed open: %+v", open)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -143,14 +143,14 @@ func printStatus(asked answers) error {
|
|||||||
len(quiet), strings.Join(said, "\n "))
|
len(quiet), strings.Join(said, "\n "))
|
||||||
}
|
}
|
||||||
|
|
||||||
if open, late := openPlans(asked.plans, asked.paused); len(open) > 0 {
|
if open, late := openPlans(asked.plans, time.Now(), asked.paused, asked.tierBounds); len(open) > 0 {
|
||||||
fmt.Printf("%d plan(s) open", len(open))
|
fmt.Printf("%d plan(s) open", len(open))
|
||||||
if late > 0 {
|
if late > 0 {
|
||||||
fmt.Printf(", %d waiting past %s", late, planWaitBound)
|
fmt.Printf(", %d in a tier past its bound", late)
|
||||||
}
|
}
|
||||||
fmt.Println(":")
|
fmt.Println(":")
|
||||||
for _, p := range open {
|
for _, p := range open {
|
||||||
fmt.Printf(" %s\n", planLineWith(p, time.Now(), asked.paused))
|
fmt.Printf(" %s\n", planLineWith(p, time.Now(), asked.paused, asked.tierBounds.of(p.Repository)))
|
||||||
}
|
}
|
||||||
fmt.Println()
|
fmt.Println()
|
||||||
}
|
}
|
||||||
@@ -472,6 +472,8 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
|||||||
}
|
}
|
||||||
// Whether the build seat takes work, for a plan waiting on it (novox/hq ADR 0219).
|
// Whether the build seat takes work, for a plan waiting on it (novox/hq ADR 0219).
|
||||||
out.paused = buildSeatPause(ctx, inv, out.plans)
|
out.paused = buildSeatPause(ctx, inv, out.plans)
|
||||||
|
// And how long a tier may take, the bound a plan is late at (novox/hq issue 296).
|
||||||
|
out.tierBounds = readTierBounds(ctx, inv, time.Now())
|
||||||
// And how many repairs were done by hand this week (novox/hq to-be 45 §7) — where there is a bus
|
// And how many repairs were done by hand this week (novox/hq to-be 45 §7) — where there is a bus
|
||||||
// to read the log from; a process with none has no log to count.
|
// to read the log from; a process with none has no log to count.
|
||||||
if _, onBus := broker.BusAddress(); onBus == nil {
|
if _, onBus := broker.BusAddress(); onBus == nil {
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"os"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -204,6 +205,15 @@ func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, e
|
|||||||
if news {
|
if news {
|
||||||
l.summary.nudge()
|
l.summary.nudge()
|
||||||
}
|
}
|
||||||
|
// What it says of its long-running resources (novox/hq ADR 0240): kept, and its modules' conditions
|
||||||
|
// raised or cleared from it. Only from an account of the machine the store took.
|
||||||
|
if err == nil && report.Health != nil && report.Superseded == "" && report.Rekey == nil && report.Node != "" &&
|
||||||
|
l.Enrolment.Inventory != nil {
|
||||||
|
if herr := stateHealth(ctx, l.Enrolment.Inventory, conditionsFrom, report.Node, *report.Health, now); herr != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "mesh-controller: could not keep what %s says of its resources' health: %v\n",
|
||||||
|
report.Node, herr)
|
||||||
|
}
|
||||||
|
}
|
||||||
if err == nil && l.open != nil && startedWell(report) {
|
if err == nil && l.open != nil && startedWell(report) {
|
||||||
// Off the report's path: replacing a given value sends the machine, and a report waits for
|
// Off the report's path: replacing a given value sends the machine, and a report waits for
|
||||||
// nothing it caused (novox/hq ADR 0228).
|
// nothing it caused (novox/hq ADR 0228).
|
||||||
|
|||||||
+170
@@ -0,0 +1,170 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"contract": 2,
|
||||||
|
"at": "2026-10-07T16:48:58.691388404Z",
|
||||||
|
"resources": [],
|
||||||
|
"network": {
|
||||||
|
"state": "healthy",
|
||||||
|
"since": "2026-10-07T16:48:58.691388404Z",
|
||||||
|
"parts": [
|
||||||
|
{
|
||||||
|
"part": "resolv-conf",
|
||||||
|
"state": "healthy",
|
||||||
|
"since": "2026-10-07T16:48:58.691388404Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"part": "names",
|
||||||
|
"state": "healthy",
|
||||||
|
"since": "2026-10-07T16:48:58.691388404Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"part": "bus",
|
||||||
|
"state": "healthy",
|
||||||
|
"since": "2026-10-07T16:48:58.691388404Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"part": "route",
|
||||||
|
"state": "healthy",
|
||||||
|
"since": "2026-10-07T16:48:58.691388404Z"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"contract": 2,
|
||||||
|
"at": "2026-10-07T16:49:28.691388404Z",
|
||||||
|
"resources": [],
|
||||||
|
"network": {
|
||||||
|
"state": "healthy",
|
||||||
|
"since": "2026-10-07T16:48:58.691388404Z",
|
||||||
|
"parts": [
|
||||||
|
{
|
||||||
|
"part": "resolv-conf",
|
||||||
|
"state": "healthy",
|
||||||
|
"since": "2026-10-07T16:48:58.691388404Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"part": "names",
|
||||||
|
"state": "healthy",
|
||||||
|
"since": "2026-10-07T16:48:58.691388404Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"part": "bus",
|
||||||
|
"state": "healthy",
|
||||||
|
"since": "2026-10-07T16:48:58.691388404Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"part": "route",
|
||||||
|
"state": "healthy",
|
||||||
|
"since": "2026-10-07T16:48:58.691388404Z"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"contract": 2,
|
||||||
|
"at": "2026-10-07T16:49:58.691388404Z",
|
||||||
|
"resources": [],
|
||||||
|
"network": {
|
||||||
|
"state": "healthy",
|
||||||
|
"since": "2026-10-07T16:48:58.691388404Z",
|
||||||
|
"parts": [
|
||||||
|
{
|
||||||
|
"part": "resolv-conf",
|
||||||
|
"state": "healthy",
|
||||||
|
"since": "2026-10-07T16:48:58.691388404Z",
|
||||||
|
"streak": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"part": "names",
|
||||||
|
"state": "healthy",
|
||||||
|
"since": "2026-10-07T16:48:58.691388404Z",
|
||||||
|
"streak": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"part": "bus",
|
||||||
|
"state": "healthy",
|
||||||
|
"since": "2026-10-07T16:48:58.691388404Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"part": "route",
|
||||||
|
"state": "healthy",
|
||||||
|
"since": "2026-10-07T16:48:58.691388404Z"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"contract": 2,
|
||||||
|
"at": "2026-10-07T16:50:28.691388404Z",
|
||||||
|
"resources": [],
|
||||||
|
"network": {
|
||||||
|
"state": "unhealthy",
|
||||||
|
"since": "2026-10-07T16:50:28.691388404Z",
|
||||||
|
"parts": [
|
||||||
|
{
|
||||||
|
"part": "resolv-conf",
|
||||||
|
"state": "unhealthy",
|
||||||
|
"reason": "the resolver file was rewritten by another program",
|
||||||
|
"said": "/etc/resolv.conf differs from what networkmanager declares: it lists 192.0.2.53 where 10.77.0.2, 10.77.0.1 is declared; changed 2026-10-07T16:48:28Z; its own header names FortiClient",
|
||||||
|
"writer": "FortiClient",
|
||||||
|
"owner": "networkmanager",
|
||||||
|
"since": "2026-10-07T16:50:28.691388404Z",
|
||||||
|
"streak": 2
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"part": "names",
|
||||||
|
"state": "unhealthy",
|
||||||
|
"reason": "neither mesh names nor public names resolve",
|
||||||
|
"said": "within 1s: 192.0.2.53 — anchor.internal (IPv4): no answer within 1s; anchor.internal (IPv6): no answer within 1s; example.com (IPv4): no answer within 1s",
|
||||||
|
"toward": [
|
||||||
|
"192.0.2.53"
|
||||||
|
],
|
||||||
|
"since": "2026-10-07T16:50:28.691388404Z",
|
||||||
|
"streak": 2
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"part": "bus",
|
||||||
|
"state": "healthy",
|
||||||
|
"since": "2026-10-07T16:48:58.691388404Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"part": "route",
|
||||||
|
"state": "healthy",
|
||||||
|
"since": "2026-10-07T16:48:58.691388404Z"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"contract": 2,
|
||||||
|
"at": "2026-10-07T16:50:58.691388404Z",
|
||||||
|
"resources": [],
|
||||||
|
"network": {
|
||||||
|
"state": "healthy",
|
||||||
|
"since": "2026-10-07T16:50:58.691388404Z",
|
||||||
|
"parts": [
|
||||||
|
{
|
||||||
|
"part": "resolv-conf",
|
||||||
|
"state": "healthy",
|
||||||
|
"since": "2026-10-07T16:50:58.691388404Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"part": "names",
|
||||||
|
"state": "healthy",
|
||||||
|
"since": "2026-10-07T16:50:58.691388404Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"part": "bus",
|
||||||
|
"state": "healthy",
|
||||||
|
"since": "2026-10-07T16:48:58.691388404Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"part": "route",
|
||||||
|
"state": "healthy",
|
||||||
|
"since": "2026-10-07T16:48:58.691388404Z"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
@@ -345,7 +345,7 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
|||||||
e.Manifest.Module, m.Owner, m.Repo, e.Source.Ref, m.Base)
|
e.Manifest.Module, m.Owner, m.Repo, e.Source.Ref, m.Base)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
touched := whatTheMergeTouched(from, entries, m)
|
touched, added, _ := touchedBy(from, entries, m)
|
||||||
// **A module the merge deleted is not built** (novox/hq ADR 0236): its manifest is gone, so the build
|
// **A module the merge deleted is not built** (novox/hq ADR 0236): its manifest is gone, so the build
|
||||||
// seat finds nothing saying what it is, and the plan failed on it (`has no module.json at …`) with
|
// seat finds nothing saying what it is, and the plan failed on it (`has no module.json at …`) with
|
||||||
// every other module of its tier left unsent. It is forgotten where nothing holds it, said otherwise.
|
// every other module of its tier left unsent. It is forgotten where nothing holds it, said otherwise.
|
||||||
@@ -358,8 +358,17 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
|||||||
return notNow(err)
|
return notNow(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// **A new module is built and registered, and sent nowhere** (novox/hq issue 300): the delivery plan
|
||||||
|
// says so, and assigning it is a person's act, which needs it registered first.
|
||||||
|
built := askNewModules(ctx, m, from, added)
|
||||||
moved := append(append([]inventory.Entry{}, touched...), packaging...)
|
moved := append(append([]inventory.Entry{}, touched...), packaging...)
|
||||||
if len(moved) == 0 {
|
if len(moved) == 0 {
|
||||||
|
if len(built) > 0 {
|
||||||
|
fmt.Printf("%s/%s merged into %s (%.8s); it changed no module the mesh holds, and adds %s: "+
|
||||||
|
"built, registered when the build lands, and sent nowhere\n", m.Owner, m.Repo, m.Base, m.Commit,
|
||||||
|
strings.Join(built, ", "))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
fmt.Printf("%s/%s merged into %s (%.8s); it changed nothing any module the mesh holds is "+
|
fmt.Printf("%s/%s merged into %s (%.8s); it changed nothing any module the mesh holds is "+
|
||||||
"built from\n", m.Owner, m.Repo, m.Base, m.Commit)
|
"built from\n", m.Owner, m.Repo, m.Base, m.Commit)
|
||||||
return nil
|
return nil
|
||||||
@@ -471,6 +480,41 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// askNewModules asks the build seat for every module a merge adds to the repository — a directory holding a
|
||||||
|
// module.json that no module of the mesh is known from (touchedBy's `added`) — at the branch merged into,
|
||||||
|
// not waited for: the build's take-in registers it, as a hand `build` does, and sends it nowhere, since no
|
||||||
|
// machine is assigned it (novox/hq issue 300). Before this a merge said "it changed nothing any module
|
||||||
|
// the mesh holds is built from", the module was never registered, and `assign` refused it as unknown.
|
||||||
|
//
|
||||||
|
// The repository is spelled, and found on the seat, as the modules already built from it are; `from` is
|
||||||
|
// those of them this merge moves, so a merge read as history, or of a branch nothing follows, builds no
|
||||||
|
// new module either. Outside the plan: the plan walks modules the catalogue holds, and a new one has no
|
||||||
|
// machine to send to and nothing standing on it. What could not be asked is said, and left to `build`.
|
||||||
|
// Returns the directories asked for.
|
||||||
|
func askNewModules(ctx context.Context, m link.SourceMoved, from []inventory.Entry, added []string) []string {
|
||||||
|
if len(added) == 0 || len(from) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
source := buildSource{Repository: from[0].Source.Repository, Seat: from[0].Source.Seat}
|
||||||
|
var asked []string
|
||||||
|
for _, dir := range added {
|
||||||
|
path := dir
|
||||||
|
if path == "." {
|
||||||
|
path = ""
|
||||||
|
}
|
||||||
|
id, err := askABuild(ctx, source, path, m.Base)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf(" %s is a new module in %s/%s and could not be built: %v — a hand `build` of it "+
|
||||||
|
"asks again\n", dir, m.Owner, m.Repo, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fmt.Printf(" %s is a new module in %s/%s: build %s asked at %s; registered when it lands, assigned "+
|
||||||
|
"nowhere\n", dir, m.Owner, m.Repo, id, m.Base)
|
||||||
|
asked = append(asked, dir)
|
||||||
|
}
|
||||||
|
return asked
|
||||||
|
}
|
||||||
|
|
||||||
// actingOnMerges keeps one merge acted on at a time (novox/hq issue 266).
|
// actingOnMerges keeps one merge acted on at a time (novox/hq issue 266).
|
||||||
var actingOnMerges sync.Mutex
|
var actingOnMerges sync.Mutex
|
||||||
|
|
||||||
@@ -655,7 +699,8 @@ func whatTheMergeTouched(candidates, known []inventory.Entry, m link.SourceMoved
|
|||||||
//
|
//
|
||||||
// `added` is the directories the change holds a module in that no module of this repository is known
|
// `added` is the directories the change holds a module in that no module of this repository is known
|
||||||
// from — said by the announcer at the head (issue 278), or a module.json among the changed files — `.`
|
// from — said by the announcer at the head (issue 278), or a module.json among the changed files — `.`
|
||||||
// for the root: a new module, which a check judges before it merges and a merge does not build.
|
// for the root: a new module, which a check judges before it merges and the merge builds and registers,
|
||||||
|
// sending it nowhere (askNewModules, novox/hq issue 300).
|
||||||
//
|
//
|
||||||
// Nothing said about the files, or not all of them said, is still everything: what is not known cannot
|
// Nothing said about the files, or not all of them said, is still everything: what is not known cannot
|
||||||
// be narrowed.
|
// be narrowed.
|
||||||
|
|||||||
@@ -439,13 +439,9 @@ func gatherPlans(ctx context.Context, inv *inventory.Inventory, now time.Time) (
|
|||||||
if len(plans) == 0 {
|
if len(plans) == 0 {
|
||||||
return nil, nil, nil
|
return nil, nil, nil
|
||||||
}
|
}
|
||||||
tiers, err := inv.Durations(ctx, inventory.DurationPlanTier, now.Add(-14*24*time.Hour))
|
bounds, err := measuredTierBounds(ctx, inv, now)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, fmt.Errorf("the measured plan tiers cannot be read: %w", err)
|
return nil, nil, err
|
||||||
}
|
|
||||||
measured := map[string][]time.Duration{}
|
|
||||||
for _, d := range tiers {
|
|
||||||
measured[d.Subject] = append(measured[d.Subject], d.Took)
|
|
||||||
}
|
}
|
||||||
pause := buildSeatPause(ctx, inv, plans)
|
pause := buildSeatPause(ctx, inv, plans)
|
||||||
var out []planFacts
|
var out []planFacts
|
||||||
@@ -459,13 +455,59 @@ func gatherPlans(ctx context.Context, inv *inventory.Inventory, now time.Time) (
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
_, paused := pausedWaiting(p, pause, now)
|
_, paused := pausedWaiting(p, pause, now)
|
||||||
|
bound := bounds.of(p.Repository)
|
||||||
out = append(out, planFacts{id: p.ID, repository: p.Repository, commit: p.Commit, tier: p.Tier,
|
out = append(out, planFacts{id: p.ID, repository: p.Repository, commit: p.Commit, tier: p.Tier,
|
||||||
tiers: len(p.Tiers), entered: p.TierEntered, bound: max(tierAtLeast, 3*p90(measured[p.Repository])),
|
tiers: len(p.Tiers), entered: inTierSince(p), bound: bound,
|
||||||
waiting: planLineWith(p, now, pause), paused: paused})
|
waiting: planLineWith(p, now, pause, bound), paused: paused})
|
||||||
}
|
}
|
||||||
return out, waits, nil
|
return out, waits, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// tierBounds is how long a plan's tier may take before it is stalled (S3) and `plans` and `status` call
|
||||||
|
// it LATE, by repository: three times the ninetieth percentile of the tiers measured for it, and never
|
||||||
|
// less than tierAtLeast. One bound for the watchdog and the lines a person reads, so the two cannot
|
||||||
|
// disagree about one plan (novox/hq issue 296). A repository nothing was measured for, or a nil
|
||||||
|
// tierBounds, is given tierAtLeast.
|
||||||
|
type tierBounds map[string]time.Duration
|
||||||
|
|
||||||
|
func (b tierBounds) of(repository string) time.Duration {
|
||||||
|
if d, ok := b[repository]; ok {
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
return tierAtLeast
|
||||||
|
}
|
||||||
|
|
||||||
|
// measuredTierBounds is the tier bounds from the last fourteen days of measured plan tiers.
|
||||||
|
func measuredTierBounds(ctx context.Context, inv *inventory.Inventory, now time.Time) (tierBounds, error) {
|
||||||
|
tiers, err := inv.Durations(ctx, inventory.DurationPlanTier, now.Add(-14*24*time.Hour))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the measured plan tiers cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
return boundsOf(tiers), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// readTierBounds is measuredTierBounds for a line a person reads: what cannot be read is said, and
|
||||||
|
// every plan is then given the least bound, as the watchdog gives a repository nothing was measured for.
|
||||||
|
func readTierBounds(ctx context.Context, inv *inventory.Inventory, now time.Time) tierBounds {
|
||||||
|
bounds, err := measuredTierBounds(ctx, inv, now)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "%v; a plan is called late past %s\n", err, tierAtLeast)
|
||||||
|
}
|
||||||
|
return bounds
|
||||||
|
}
|
||||||
|
|
||||||
|
func boundsOf(tiers []inventory.Duration) tierBounds {
|
||||||
|
measured := map[string][]time.Duration{}
|
||||||
|
for _, d := range tiers {
|
||||||
|
measured[d.Subject] = append(measured[d.Subject], d.Took)
|
||||||
|
}
|
||||||
|
out := tierBounds{}
|
||||||
|
for repository, took := range measured {
|
||||||
|
out[repository] = max(tierAtLeast, 3*p90(took))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// p90 is the ninetieth percentile of measurements; zero for none.
|
// p90 is the ninetieth percentile of measurements; zero for none.
|
||||||
func p90(took []time.Duration) time.Duration {
|
func p90(took []time.Duration) time.Duration {
|
||||||
if len(took) == 0 {
|
if len(took) == 0 {
|
||||||
|
|||||||
@@ -35,4 +35,4 @@ require (
|
|||||||
// committed. Every build (the build agent's `go build`, the Dockerfile) compiles from vendor/ and
|
// committed. Every build (the build agent's `go build`, the Dockerfile) compiles from vendor/ and
|
||||||
// fetches nothing; go refuses to build when vendor/ and this file disagree, so a pin moved without
|
// fetches nothing; go refuses to build when vendor/ and this file disagree, so a pin moved without
|
||||||
// `go mod vendor` fails loudly, at once, everywhere.
|
// `go mod vendor` fails loudly, at once, everywhere.
|
||||||
replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261006095519-3e80b7ae325e
|
replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261007120832-bdd44154ccac
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
git.novox.be/novox/mesh-host v0.0.0-20261006095519-3e80b7ae325e h1:g9h4QRaAMg5yaJLwqtb0FoOs23DVGUYpW6qvnQ3oY5A=
|
git.novox.be/novox/mesh-host v0.0.0-20261006095519-3e80b7ae325e h1:g9h4QRaAMg5yaJLwqtb0FoOs23DVGUYpW6qvnQ3oY5A=
|
||||||
git.novox.be/novox/mesh-host v0.0.0-20261006095519-3e80b7ae325e/go.mod h1:VlilMCRZ5yyNXg7SNigNBLr0Gt32jrGw5KSNq5JAVYs=
|
git.novox.be/novox/mesh-host v0.0.0-20261006095519-3e80b7ae325e/go.mod h1:VlilMCRZ5yyNXg7SNigNBLr0Gt32jrGw5KSNq5JAVYs=
|
||||||
|
git.novox.be/novox/mesh-host v0.0.0-20261007120832-bdd44154ccac h1:KvnKtJ2rWeIE/t4GweK+JL0OjKSNxsrVP3/nMdpii8o=
|
||||||
|
git.novox.be/novox/mesh-host v0.0.0-20261007120832-bdd44154ccac/go.mod h1:VlilMCRZ5yyNXg7SNigNBLr0Gt32jrGw5KSNq5JAVYs=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
|||||||
@@ -122,6 +122,10 @@ type Principal struct {
|
|||||||
// key, and nothing else of the bucket, so it can judge a new controller build and put the previous
|
// key, and nothing else of the bucket, so it can judge a new controller build and put the previous
|
||||||
// one back.
|
// one back.
|
||||||
WitnessesController bool
|
WitnessesController bool
|
||||||
|
// Checks are the tools a machine principal's node-engine asks as a declared health check, each
|
||||||
|
// `<module>.<tool>` (novox/hq ADR 0240, to-be 48 §3): asked of the instance on its own machine and
|
||||||
|
// nowhere else.
|
||||||
|
Checks []string
|
||||||
|
|
||||||
// PasswordHash is the bcrypt hash the mesh minted. The plaintext is sealed to the principal
|
// PasswordHash is the bcrypt hash the mesh minted. The plaintext is sealed to the principal
|
||||||
// and never appears here: this file is written to a node's disk and read by a server, and a
|
// and never appears here: this file is written to a node's disk and read by a server, and a
|
||||||
@@ -1021,5 +1025,26 @@ func WitnessSubjects(p Principal) []string {
|
|||||||
if p.WitnessesController {
|
if p.WitnessesController {
|
||||||
out = append(out, lease.LeaseReadSubject(LeaseBucket))
|
out = append(out, lease.LeaseReadSubject(LeaseBucket))
|
||||||
}
|
}
|
||||||
|
return append(out, CheckSubjects(p)...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// CheckSubjects are the tools a machine's node-engine asks as declared health checks (novox/hq ADR 0240,
|
||||||
|
// to-be 48 §3): each `<module>.<tool>` on this machine's instance — `mesh.mod.<module>.tool.<tool>.<node>`
|
||||||
|
// — and never the plain subject, which any machine's instance may answer. Sorted and once each.
|
||||||
|
func CheckSubjects(p Principal) []string {
|
||||||
|
seen := map[string]bool{}
|
||||||
|
var out []string
|
||||||
|
for _, c := range p.Checks {
|
||||||
|
module, tool, ok := strings.Cut(c, ".")
|
||||||
|
if !ok || !safeSubject.MatchString(module) || !safeSubject.MatchString(tool) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
subject := "mesh.mod." + module + ".tool." + tool + "." + p.Node
|
||||||
|
if !seen[subject] {
|
||||||
|
seen[subject] = true
|
||||||
|
out = append(out, subject)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -44,6 +44,9 @@ type Declared struct {
|
|||||||
// SnapshotsTheBus says the module holds mesh-broker — it is the bus — and so is the one module
|
// SnapshotsTheBus says the module holds mesh-broker — it is the bus — and so is the one module
|
||||||
// granted the snapshot API, to copy the bus's streams for the night's backup (novox/hq ADR 0235).
|
// granted the snapshot API, to copy the bus's streams for the night's backup (novox/hq ADR 0235).
|
||||||
SnapshotsTheBus bool
|
SnapshotsTheBus bool
|
||||||
|
// Checks are the module's own tools its health asks, each `<module>.<tool>` (novox/hq ADR 0240, to-be
|
||||||
|
// 48 §3): the machine's node-engine asks them of its own node tools, and is granted that and no more.
|
||||||
|
Checks []string
|
||||||
}
|
}
|
||||||
|
|
||||||
// Records is what composing a user list needs to know about the mesh, and nothing more.
|
// Records is what composing a user list needs to know about the mesh, and nothing more.
|
||||||
@@ -73,12 +76,14 @@ func Users(r Records) ([]Principal, error) {
|
|||||||
|
|
||||||
for _, node := range sortedCopy(r.Nodes) {
|
for _, node := range sortedCopy(r.Nodes) {
|
||||||
witness := false
|
witness := false
|
||||||
|
var checks []string
|
||||||
for _, d := range r.Assigned[node] {
|
for _, d := range r.Assigned[node] {
|
||||||
if d.Module == controllerModule {
|
if d.Module == controllerModule {
|
||||||
witness = true
|
witness = true
|
||||||
}
|
}
|
||||||
|
checks = append(checks, d.Checks...)
|
||||||
}
|
}
|
||||||
out = append(out, Principal{Kind: KindNode, Node: node, WitnessesController: witness})
|
out = append(out, Principal{Kind: KindNode, Node: node, WitnessesController: witness, Checks: checks})
|
||||||
// **Where the runtime is assigned, the machine gets one runtime principal in place of the
|
// **Where the runtime is assigned, the machine gets one runtime principal in place of the
|
||||||
// runtime module's own** (novox/hq ADR 0175, to-be 38). It carries every module on the
|
// runtime module's own** (novox/hq ADR 0175, to-be 38). It carries every module on the
|
||||||
// node: its serving grants are the union of theirs. Every other module keeps its own
|
// node: its serving grants are the union of theirs. Every other module keeps its own
|
||||||
|
|||||||
@@ -35,3 +35,31 @@ func TestTheWitnessIsGrantedWhatItReadsAndNoMore(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A module's health that asks one of its own tools is asked by the machine's node-engine, of the instance
|
||||||
|
// on its own machine and nowhere else (novox/hq ADR 0240, to-be 48 §3).
|
||||||
|
func TestTheEngineIsGrantedTheToolsItsModulesHealthAsks(t *testing.T) {
|
||||||
|
users, err := Users(Records{Nodes: []string{"control", "edge"},
|
||||||
|
Assigned: map[string][]Declared{"edge": {{Module: "keycloak", Checks: []string{"keycloak.keycloak_admin_health"}}}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, u := range users {
|
||||||
|
if u.Kind != KindNode {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
perms, err := PermissionsFor(u)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
mine := "mesh.mod.keycloak.tool.keycloak_admin_health.edge"
|
||||||
|
if got := slices.Contains(perms.Publish, mine); got != (u.Node == "edge") {
|
||||||
|
t.Errorf("%s may ask keycloak's health tool on edge: %v", u.Node, got)
|
||||||
|
}
|
||||||
|
for _, p := range perms.Publish {
|
||||||
|
if p == "mesh.mod.keycloak.tool.keycloak_admin_health" || p == "mesh.mod.keycloak.tool.>" {
|
||||||
|
t.Errorf("%s may ask the tool of any machine: %s", u.Node, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -86,7 +86,9 @@ var WritersTable = []WriterRow{
|
|||||||
Others: "read", Subjects: []string{"mesh.node.*.declare"}, Writes: isController},
|
Others: "read", Subjects: []string{"mesh.node.*.declare"}, Writes: isController},
|
||||||
{State: "a machine's applied state and its report", Writer: "the node-engine's apply queue",
|
{State: "a machine's applied state and its report", Writer: "the node-engine's apply queue",
|
||||||
KeptIn: "the machine; the report on the bus", Others: "the reconcile and a delivery enqueue, never apply",
|
KeptIn: "the machine; the report on the bus", Others: "the reconcile and a delivery enqueue, never apply",
|
||||||
Subjects: []string{"mesh.control.*.report"}, Writes: ownMachine},
|
// And its health statement between reports (novox/hq ADR 0240): the same writer stating the same
|
||||||
|
// machine, inside the grant it already had (`mesh.control.<its own>.>`).
|
||||||
|
Subjects: []string{"mesh.control.*.report", "mesh.control.*.health"}, Writes: ownMachine},
|
||||||
{State: "the controller lease", Writer: "the controller instance holding it", KeptIn: "key-value " + LeaseBucket,
|
{State: "the controller lease", Writer: "the controller instance holding it", KeptIn: "key-value " + LeaseBucket,
|
||||||
Others: "a candidate waits", Subjects: kvOf(LeaseBucket), Writes: isController},
|
Others: "a candidate waits", Subjects: kvOf(LeaseBucket), Writes: isController},
|
||||||
{State: "plans and their tiers", Writer: "controller (lease holder), compare-and-set on the plan's revision",
|
{State: "plans and their tiers", Writer: "controller (lease holder), compare-and-set on the plan's revision",
|
||||||
|
|||||||
+162
-30
@@ -39,6 +39,11 @@ import (
|
|||||||
// quality, run when present in the toolchain it declares (`# mesh-check-toolchain: go|typescript`
|
// quality, run when present in the toolchain it declares (`# mesh-check-toolchain: go|typescript`
|
||||||
// among its first lines; go when it declares none). A repository that reaches the build seat with
|
// among its first lines; go when it declares none). A repository that reaches the build seat with
|
||||||
// none is said as a warning: it is the mesh's, and nothing of its own is tested before it merges.
|
// none is said as a warning: it is the mesh's, and nothing of its own is tested before it merges.
|
||||||
|
// **A repository in two languages declares the other part too** (`# mesh-check-also: <toolchain>
|
||||||
|
// <script>`, one line each, among the same first lines): each named script is run from the root in
|
||||||
|
// its own toolchain's container, after merge-check.sh, and the layer passes only when every part
|
||||||
|
// does (novox/hq issue 302 — the lab's Go replays went unchecked because its script runs in the
|
||||||
|
// TypeScript toolchain, which holds no Go compiler).
|
||||||
//
|
//
|
||||||
// One check:
|
// One check:
|
||||||
//
|
//
|
||||||
@@ -178,6 +183,32 @@ const noScript = "the repository declares no " + CheckScript + ": none of its ow
|
|||||||
// toolchainLine is how a merge-check.sh declares the toolchain it runs in.
|
// toolchainLine is how a merge-check.sh declares the toolchain it runs in.
|
||||||
var toolchainLine = regexp.MustCompile(`^#\s*mesh-check-toolchain:\s*([a-z0-9-]+)\s*$`)
|
var toolchainLine = regexp.MustCompile(`^#\s*mesh-check-toolchain:\s*([a-z0-9-]+)\s*$`)
|
||||||
|
|
||||||
|
// alsoLine is how a merge-check.sh declares a further part of the repository's check, run in a toolchain
|
||||||
|
// of its own: `# mesh-check-also: <toolchain> <script>`.
|
||||||
|
var alsoLine = regexp.MustCompile(`^#\s*mesh-check-also:\s*([a-z0-9-]+)\s+(\S+)\s*$`)
|
||||||
|
|
||||||
|
// ScriptPart is one part of a repository's own check: a script, run from the repository's root, and the
|
||||||
|
// toolchain it runs in.
|
||||||
|
type ScriptPart struct {
|
||||||
|
Toolchain string
|
||||||
|
Script string
|
||||||
|
}
|
||||||
|
|
||||||
|
// ScriptParts is every part of a repository's own check: its merge-check.sh in the toolchain it declares,
|
||||||
|
// then each further part it declares among its first twenty lines (`# mesh-check-also: <toolchain>
|
||||||
|
// <script>`), in the order declared. A declared script that is not a path inside the repository is
|
||||||
|
// answered as a part all the same, for the check to refuse by name rather than to drop.
|
||||||
|
func ScriptParts(script []byte) []ScriptPart {
|
||||||
|
parts := []ScriptPart{{Toolchain: ScriptToolchain(script), Script: CheckScript}}
|
||||||
|
lines := bufio.NewScanner(bytes.NewReader(script))
|
||||||
|
for i := 0; i < 20 && lines.Scan(); i++ {
|
||||||
|
if m := alsoLine.FindStringSubmatch(strings.TrimSpace(lines.Text())); m != nil {
|
||||||
|
parts = append(parts, ScriptPart{Toolchain: m[1], Script: m[2]})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return parts
|
||||||
|
}
|
||||||
|
|
||||||
// ScriptToolchain is the language a merge-check.sh declares it runs in, among its first twenty lines;
|
// ScriptToolchain is the language a merge-check.sh declares it runs in, among its first twenty lines;
|
||||||
// go when it declares none.
|
// go when it declares none.
|
||||||
func ScriptToolchain(script []byte) string {
|
func ScriptToolchain(script []byte) string {
|
||||||
@@ -298,6 +329,13 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
|||||||
say("check", "the facts of %s (%s): %d machine(s), the bus at %s, the store at %s", f.Taken.Format(time.RFC3339),
|
say("check", "the facts of %s (%s): %d machine(s), the bus at %s, the store at %s", f.Taken.Format(time.RFC3339),
|
||||||
short(strings.TrimPrefix(digest, "sha256:")), len(f.Machines), f.Versions.Bus, f.Versions.Store)
|
short(strings.TrimPrefix(digest, "sha256:")), len(f.Machines), f.Versions.Bus, f.Versions.Store)
|
||||||
|
|
||||||
|
// **What an earlier delivery of this same ask left is removed first** (novox/hq issue 285): an ask is
|
||||||
|
// redelivered when the holder that took it stopped mid-check — the build agent itself updated by the
|
||||||
|
// rollout it is checking — and its containers, named by the ask's id, are still there. Raising the
|
||||||
|
// store again under that name was refused, and the check said it could not run.
|
||||||
|
if n, err := RemoveContainersOf(ctx, run, spec.ID); err == nil && n > 0 {
|
||||||
|
say("check", "removed %d throwaway container(s) an earlier delivery of this check left", n)
|
||||||
|
}
|
||||||
// The throwaway store and bus, of the versions the mesh runs, removed whatever happens.
|
// The throwaway store and bus, of the versions the mesh runs, removed whatever happens.
|
||||||
defer func() {
|
defer func() {
|
||||||
removing, done := context.WithTimeout(context.Background(), time.Minute)
|
removing, done := context.WithTimeout(context.Background(), time.Minute)
|
||||||
@@ -409,32 +447,11 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
|||||||
case timedOut():
|
case timedOut():
|
||||||
v.Repo = &Layer{Verdict: "error", Summary: fmt.Sprintf("the check ran past %s before its %s ran", CheckTimeout, CheckScript)}
|
v.Repo = &Layer{Verdict: "error", Summary: fmt.Sprintf("the check ran past %s before its %s ran", CheckTimeout, CheckScript)}
|
||||||
default:
|
default:
|
||||||
language := ScriptToolchain(script)
|
v.Repo = ownCheck(ctx, spec, ScriptParts(script), tree, &out, timedOut, func(image string, script string) *exec.Cmd {
|
||||||
image := spec.Toolchains[language]
|
return exec.CommandContext(ctx, "docker", LabelledArgs("docker", in(image, tree, env, "sh", script), spec.ID)...)
|
||||||
if language == "go" && image == "" {
|
}, say)
|
||||||
image = spec.Toolchain
|
if v.Repo == nil {
|
||||||
}
|
|
||||||
if image == "" {
|
|
||||||
v.Repo = &Layer{Verdict: "error", Summary: fmt.Sprintf("its %s runs in the %s toolchain, which the mesh does "+
|
|
||||||
"not hold", CheckScript, language)}
|
|
||||||
break
|
|
||||||
}
|
|
||||||
say("check", "running its %s in the mesh's %s toolchain", CheckScript, language)
|
|
||||||
fmt.Fprintf(&out, "--- its %s (%s toolchain)\n", CheckScript, language)
|
|
||||||
var own tail
|
|
||||||
cmd := exec.CommandContext(ctx, "docker", LabelledArgs("docker", in(image, tree, env, "sh", CheckScript), spec.ID)...)
|
|
||||||
inItsOwnGroup(cmd)
|
|
||||||
w := io.MultiWriter(&out, &own)
|
|
||||||
cmd.Stdout, cmd.Stderr = w, w
|
|
||||||
switch err := cmd.Run(); {
|
|
||||||
case timedOut():
|
|
||||||
v.Repo = &Layer{Verdict: "error", Summary: fmt.Sprintf("its %s ran past %s and was ended", CheckScript, CheckTimeout)}
|
|
||||||
case ctx.Err() != nil:
|
|
||||||
return v, ctx.Err()
|
return v, ctx.Err()
|
||||||
case err != nil:
|
|
||||||
v.Repo = &Layer{Verdict: "fail", Summary: "its " + CheckScript + " failed: " + whatFailed(own.String())}
|
|
||||||
default:
|
|
||||||
v.Repo = &Layer{Verdict: "pass", Summary: "its " + CheckScript + " passed"}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -445,6 +462,66 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
|||||||
return v, nil
|
return v, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ownCheck runs the repository's own check, part by part, each in its toolchain's own container: the
|
||||||
|
// layer's verdict. Nil when the check was ended from outside (not past its bound), which the caller
|
||||||
|
// answers as the context's error.
|
||||||
|
func ownCheck(ctx context.Context, spec CheckSpec, parts []ScriptPart, tree string, out *tail, timedOut func() bool,
|
||||||
|
command func(image, script string) *exec.Cmd, say func(step, format string, args ...any)) *Layer {
|
||||||
|
var ran []string
|
||||||
|
for i, part := range parts {
|
||||||
|
named := part.Script
|
||||||
|
if i > 0 {
|
||||||
|
named = CheckScript + "'s part " + part.Script
|
||||||
|
}
|
||||||
|
image := spec.Toolchains[part.Toolchain]
|
||||||
|
if part.Toolchain == "go" && image == "" {
|
||||||
|
image = spec.Toolchain
|
||||||
|
}
|
||||||
|
if image == "" {
|
||||||
|
return &Layer{Verdict: "error", Summary: fmt.Sprintf("%sits %s runs in the %s toolchain, which the mesh does "+
|
||||||
|
"not hold", passedSoFar(ran), named, part.Toolchain)}
|
||||||
|
}
|
||||||
|
if i > 0 && !filepath.IsLocal(part.Script) {
|
||||||
|
return &Layer{Verdict: "fail", Summary: fmt.Sprintf("%s declares a part %q that is not a path inside the "+
|
||||||
|
"repository", CheckScript, part.Script)}
|
||||||
|
}
|
||||||
|
if i > 0 {
|
||||||
|
if info, err := os.Stat(filepath.Join(tree, part.Script)); err != nil || info.IsDir() {
|
||||||
|
return &Layer{Verdict: "fail", Summary: fmt.Sprintf("%s declares a part %s, which the repository does "+
|
||||||
|
"not hold", CheckScript, part.Script)}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
say("check", "running its %s in the mesh's %s toolchain", part.Script, part.Toolchain)
|
||||||
|
fmt.Fprintf(out, "--- its %s (%s toolchain)\n", part.Script, part.Toolchain)
|
||||||
|
var own tail
|
||||||
|
cmd := command(image, part.Script)
|
||||||
|
inItsOwnGroup(cmd)
|
||||||
|
w := io.MultiWriter(out, &own)
|
||||||
|
cmd.Stdout, cmd.Stderr = w, w
|
||||||
|
switch err := cmd.Run(); {
|
||||||
|
case timedOut():
|
||||||
|
return &Layer{Verdict: "error", Summary: fmt.Sprintf("its %s ran past %s and was ended", part.Script, CheckTimeout)}
|
||||||
|
case ctx.Err() != nil:
|
||||||
|
return nil
|
||||||
|
case err != nil:
|
||||||
|
return &Layer{Verdict: "fail", Summary: passedSoFar(ran) + "its " + part.Script + " failed: " + whatFailed(own.String())}
|
||||||
|
}
|
||||||
|
ran = append(ran, fmt.Sprintf("%s (%s)", part.Script, part.Toolchain))
|
||||||
|
}
|
||||||
|
if len(ran) == 1 {
|
||||||
|
return &Layer{Verdict: "pass", Summary: "its " + CheckScript + " passed"}
|
||||||
|
}
|
||||||
|
return &Layer{Verdict: "pass", Summary: "its " + CheckScript + " passed, each part in its toolchain: " + strings.Join(ran, ", ")}
|
||||||
|
}
|
||||||
|
|
||||||
|
// passedSoFar is what of a check's parts passed before the one that did not, said first.
|
||||||
|
func passedSoFar(ran []string) string {
|
||||||
|
if len(ran) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return strings.Join(ran, ", ") + " passed; "
|
||||||
|
}
|
||||||
|
|
||||||
// gateLayer runs the gate: the touched manifests through `module check`, every machine composed with the
|
// gateLayer runs the gate: the touched manifests through `module check`, every machine composed with the
|
||||||
// change, and the replays of what the mesh runs. It answers the gate's verdict and summary.
|
// change, and the replays of what the mesh runs. It answers the gate's verdict and summary.
|
||||||
func gateLayer(ctx context.Context, spec CheckSpec, tree, root, gate, verdictFile string, env []string,
|
func gateLayer(ctx context.Context, spec CheckSpec, tree, root, gate, verdictFile string, env []string,
|
||||||
@@ -526,14 +603,17 @@ func gateLayer(ctx context.Context, spec CheckSpec, tree, root, gate, verdictFil
|
|||||||
// The gate could not judge: not the change's fault, and never a pass.
|
// The gate could not judge: not the change's fault, and never a pass.
|
||||||
return "error", "the merge gate could not judge the change: " + lastLine(out.String())
|
return "error", "the merge gate could not judge the change: " + lastLine(out.String())
|
||||||
}
|
}
|
||||||
var said struct {
|
|
||||||
Verdict string `json:"verdict"`
|
|
||||||
Summary string `json:"summary"`
|
|
||||||
}
|
|
||||||
raw, err := os.ReadFile(verdictFile)
|
raw, err := os.ReadFile(verdictFile)
|
||||||
if err != nil || json.Unmarshal(raw, &said) != nil || said.Verdict == "" {
|
if err != nil {
|
||||||
return "error", "the merge gate said no verdict"
|
return "error", "the merge gate said no verdict"
|
||||||
}
|
}
|
||||||
|
said, ok := readGateVerdict(raw)
|
||||||
|
if !ok {
|
||||||
|
return "error", "the merge gate said no verdict"
|
||||||
|
}
|
||||||
|
if verdict, summary, raised := gateRaisedTheMesh(said); !raised {
|
||||||
|
return verdict, summary
|
||||||
|
}
|
||||||
|
|
||||||
// 3. **The replays of what the mesh runs** (to-be 45 §9, M9): mesh-lab's, from its main — reviewed code,
|
// 3. **The replays of what the mesh runs** (to-be 45 §9, M9): mesh-lab's, from its main — reviewed code,
|
||||||
// so given the container runtime the resolver replay raises containers with — against the bus of the
|
// so given the container runtime the resolver replay raises containers with — against the bus of the
|
||||||
@@ -683,6 +763,58 @@ func newProblems(change, base string) []string {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// gateVerdict is what of the gate's verdict the seat reads: the verdict, and every machine — whether it
|
||||||
|
// composes on the mesh and whether it composed in the gate's store without the change.
|
||||||
|
type gateVerdict struct {
|
||||||
|
Verdict string `json:"verdict"`
|
||||||
|
Summary string `json:"summary"`
|
||||||
|
Machines []struct {
|
||||||
|
Described string `json:"described"`
|
||||||
|
Live bool `json:"live-composes"`
|
||||||
|
Base struct {
|
||||||
|
Composes bool `json:"composes"`
|
||||||
|
} `json:"base"`
|
||||||
|
} `json:"machines"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// readGateVerdict reads the verdict the gate wrote, from its first line that opens a JSON document: a
|
||||||
|
// judge from before the verdict was alone on its output printed what composition said ahead of it.
|
||||||
|
func readGateVerdict(raw []byte) (gateVerdict, bool) {
|
||||||
|
var v gateVerdict
|
||||||
|
text := string(raw)
|
||||||
|
if i := strings.Index(text, "\n{"); i >= 0 && !strings.HasPrefix(strings.TrimSpace(text), "{") {
|
||||||
|
text = text[i+1:]
|
||||||
|
}
|
||||||
|
if json.Unmarshal([]byte(text), &v) != nil || v.Verdict == "" {
|
||||||
|
return gateVerdict{}, false
|
||||||
|
}
|
||||||
|
return v, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// gateRaisedTheMesh is the seat's own reading of a verdict that passes (novox/hq issue 285): **a machine
|
||||||
|
// the mesh composes that did not compose in the gate's store without the change makes the gate an error,
|
||||||
|
// never a pass** — the change was judged against a machine that is not the mesh's, broken against broken.
|
||||||
|
// The judge says so itself since issue 285, but the judge is the controller the mesh runs, and one from
|
||||||
|
// before it passed such a verdict; read here too, the rule holds whatever judged. It answers false, with
|
||||||
|
// the verdict to report, when the gate did not raise the mesh.
|
||||||
|
func gateRaisedTheMesh(v gateVerdict) (string, string, bool) {
|
||||||
|
if v.Verdict != "pass" && v.Verdict != "warning" {
|
||||||
|
return "", "", true
|
||||||
|
}
|
||||||
|
var unraised []string
|
||||||
|
for _, m := range v.Machines {
|
||||||
|
if m.Live && !m.Base.Composes {
|
||||||
|
unraised = append(unraised, m.Described)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(unraised) == 0 {
|
||||||
|
return "", "", true
|
||||||
|
}
|
||||||
|
return "error", fmt.Sprintf("the mesh as it is could not be raised, so the change cannot be judged against it: "+
|
||||||
|
"%d of %d machines compose on the mesh and not in the gate (the first: %s) — novox/hq issue 285",
|
||||||
|
len(unraised), len(v.Machines), unraised[0]), false
|
||||||
|
}
|
||||||
|
|
||||||
// gitShow is a file as a ref has it.
|
// gitShow is a file as a ref has it.
|
||||||
func gitShow(ctx context.Context, dir, ref, file string) ([]byte, error) {
|
func gitShow(ctx context.Context, dir, ref, file string) ([]byte, error) {
|
||||||
cmd := exec.CommandContext(ctx, "git", "show", ref+":"+filepath.ToSlash(file))
|
cmd := exec.CommandContext(ctx, "git", "show", ref+":"+filepath.ToSlash(file))
|
||||||
|
|||||||
@@ -429,3 +429,127 @@ func TestAFailedScriptIsSaidByWhatFailed(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **Issue 285**: a judge from before the rule passed "every machine composes with the change as it did
|
||||||
|
// without (0 of 4 compose)". The seat reads the machines itself: a machine the mesh composes that did not
|
||||||
|
// compose in the gate's store makes the gate an error, whatever judged it.
|
||||||
|
func TestTheSeatCallsAGateThatRaisedNoMachineAnError(t *testing.T) {
|
||||||
|
old := "bus users without a credential: controller\n" + `{"verdict":"pass","summary":"every machine composes with the change as it did without (0 of 2 compose)",
|
||||||
|
"machines":[{"described":"the hub","live-composes":true,"base":{"composes":false}},
|
||||||
|
{"described":"a machine","live-composes":true,"base":{"composes":false}}]}`
|
||||||
|
v, ok := readGateVerdict([]byte(old))
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("a verdict after a line of composition's was not read")
|
||||||
|
}
|
||||||
|
verdict, summary, raised := gateRaisedTheMesh(v)
|
||||||
|
if raised || verdict != "error" || !strings.Contains(summary, "2 of 2 machines") || !strings.Contains(summary, "the hub") {
|
||||||
|
t.Errorf("0 of 2 composing is %q %q", verdict, summary)
|
||||||
|
}
|
||||||
|
good := `{"verdict":"pass","summary":"(2 of 2 compose)","machines":[{"described":"the hub","live-composes":true,"base":{"composes":true}},
|
||||||
|
{"described":"a laptop","live-composes":false,"base":{"composes":false}}]}`
|
||||||
|
v, _ = readGateVerdict([]byte(good))
|
||||||
|
if _, _, raised := gateRaisedTheMesh(v); !raised {
|
||||||
|
t.Error("a machine that does not compose on the mesh either was read as the gate's failure")
|
||||||
|
}
|
||||||
|
failed := `{"verdict":"fail","summary":"x","machines":[{"described":"the hub","live-composes":true,"base":{"composes":false}}]}`
|
||||||
|
v, _ = readGateVerdict([]byte(failed))
|
||||||
|
if _, _, raised := gateRaisedTheMesh(v); !raised {
|
||||||
|
t.Error("a failing verdict is the change's, and stands")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Issue 285**: an ask redelivered after its holder stopped mid-check found its own throwaway store still
|
||||||
|
// there under its name, and the check said it could not run. What an earlier delivery left goes first.
|
||||||
|
func TestARedeliveredCheckRemovesWhatItsEarlierDeliveryLeft(t *testing.T) {
|
||||||
|
registry := checkEnvironment(t)
|
||||||
|
repo, head := aCheckedRepository(t, map[string]string{CheckScript: "echo checked\n"})
|
||||||
|
id := fmt.Sprintf("check-again-%d", time.Now().UnixNano())
|
||||||
|
if out, err := exec.Command("docker", "run", "-d", "--rm", "--label", BuildLabel+"="+id, "--name", id+"-store",
|
||||||
|
"postgres:17-alpine", "sleep", "300").CombinedOutput(); err != nil {
|
||||||
|
t.Skipf("no container for the earlier delivery: %v %s", err, out)
|
||||||
|
}
|
||||||
|
v, err := Check(t.Context(), Command, CheckSpec{ID: id, Repository: repo, Ref: head, Owner: "novox",
|
||||||
|
Repo: "hq", Number: 7, Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("a redelivered check could not run: %v", err)
|
||||||
|
}
|
||||||
|
if v.Repo == nil || v.Repo.Verdict != "pass" {
|
||||||
|
t.Errorf("the repository's check answered %+v", v.Repo)
|
||||||
|
}
|
||||||
|
if left := labelled(id); len(left) > 0 {
|
||||||
|
t.Errorf("the check left %d container(s) behind", len(left))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A repository in two languages checks both, each in its own toolchain** (novox/hq issue 302): the lab's
|
||||||
|
// merge-check.sh runs in the TypeScript toolchain, and its Go replays were said "NOT CHECKED HERE" on every
|
||||||
|
// pull request, so a register that did not compile would have merged green. Its script declares the Go
|
||||||
|
// part; the check runs it in the Go toolchain's container after the script, and the layer passes only
|
||||||
|
// when both do.
|
||||||
|
func TestARepositoryInTwoLanguagesIsCheckedInBoth(t *testing.T) {
|
||||||
|
script := "#!/bin/sh\n# mesh-check-toolchain: typescript\n# mesh-check-also: go replays/merge-check.sh\nnpm test\n"
|
||||||
|
parts := ScriptParts([]byte(script))
|
||||||
|
if len(parts) != 2 || parts[0] != (ScriptPart{"typescript", CheckScript}) ||
|
||||||
|
parts[1] != (ScriptPart{"go", "replays/merge-check.sh"}) {
|
||||||
|
t.Fatalf("the parts read as %+v", parts)
|
||||||
|
}
|
||||||
|
if got := ScriptParts([]byte("#!/bin/sh\nset -eu\n")); len(got) != 1 || got[0] != (ScriptPart{"go", CheckScript}) {
|
||||||
|
t.Fatalf("a script declaring nothing reads as %+v", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Each part run where its toolchain is: here, the image the part was given is what the command says.
|
||||||
|
held := CheckSpec{Toolchain: "go-image", Toolchains: map[string]string{"typescript": "ts-image", "go": "go-image"}}
|
||||||
|
run := func(t *testing.T, spec CheckSpec, files map[string]string, parts []ScriptPart) (*Layer, []string) {
|
||||||
|
t.Helper()
|
||||||
|
tree := t.TempDir()
|
||||||
|
for name, body := range files {
|
||||||
|
if err := os.MkdirAll(filepath.Dir(filepath.Join(tree, name)), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(tree, name), []byte(body), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var images []string
|
||||||
|
var out tail
|
||||||
|
layer := ownCheck(t.Context(), spec, parts, tree, &out, func() bool { return false },
|
||||||
|
func(image, script string) *exec.Cmd {
|
||||||
|
images = append(images, image+" "+script)
|
||||||
|
cmd := exec.CommandContext(t.Context(), "sh", script)
|
||||||
|
cmd.Dir = tree
|
||||||
|
return cmd
|
||||||
|
}, func(string, string, ...any) {})
|
||||||
|
return layer, images
|
||||||
|
}
|
||||||
|
twoParts := []ScriptPart{{"typescript", CheckScript}, {"go", "replays/merge-check.sh"}}
|
||||||
|
|
||||||
|
layer, images := run(t, held, map[string]string{CheckScript: "exit 0\n", "replays/merge-check.sh": "exit 0\n"}, twoParts)
|
||||||
|
if layer.Verdict != "pass" || !strings.Contains(layer.Summary, "replays/merge-check.sh (go)") ||
|
||||||
|
strings.Join(images, ", ") != "ts-image merge-check.sh, go-image replays/merge-check.sh" {
|
||||||
|
t.Errorf("both parts passing answered %+v, ran %v", layer, images)
|
||||||
|
}
|
||||||
|
layer, _ = run(t, held, map[string]string{CheckScript: "exit 0\n",
|
||||||
|
"replays/merge-check.sh": "echo 'not gofmt'\"'\"'d:'; echo register.go; exit 1\n"}, twoParts)
|
||||||
|
if layer.Verdict != "fail" || !strings.Contains(layer.Summary, "replays/merge-check.sh failed") ||
|
||||||
|
!strings.Contains(layer.Summary, "register.go") || !strings.Contains(layer.Summary, "merge-check.sh (typescript) passed") {
|
||||||
|
t.Errorf("a failing Go part answered %+v", layer)
|
||||||
|
}
|
||||||
|
layer, images = run(t, held, map[string]string{CheckScript: "exit 2\n", "replays/merge-check.sh": "exit 0\n"}, twoParts)
|
||||||
|
if layer.Verdict != "fail" || len(images) != 1 {
|
||||||
|
t.Errorf("a failing first part answered %+v and ran %v", layer, images)
|
||||||
|
}
|
||||||
|
layer, _ = run(t, held, map[string]string{CheckScript: "exit 0\n"}, twoParts)
|
||||||
|
if layer.Verdict != "fail" || !strings.Contains(layer.Summary, "does not hold") {
|
||||||
|
t.Errorf("a declared part the repository lacks answered %+v", layer)
|
||||||
|
}
|
||||||
|
layer, _ = run(t, held, map[string]string{CheckScript: "exit 0\n"}, []ScriptPart{{"typescript", CheckScript},
|
||||||
|
{"go", "../elsewhere.sh"}})
|
||||||
|
if layer.Verdict != "fail" || !strings.Contains(layer.Summary, "not a path inside") {
|
||||||
|
t.Errorf("a part outside the repository answered %+v", layer)
|
||||||
|
}
|
||||||
|
layer, _ = run(t, CheckSpec{Toolchains: map[string]string{"typescript": "ts-image"}},
|
||||||
|
map[string]string{CheckScript: "exit 0\n", "replays/merge-check.sh": "exit 0\n"}, twoParts)
|
||||||
|
if layer.Verdict != "error" || !strings.Contains(layer.Summary, "go toolchain") {
|
||||||
|
t.Errorf("a part in a toolchain the mesh does not hold answered %+v — never a pass", layer)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -235,6 +235,12 @@ type Rendering struct {
|
|||||||
// mounts and environment. A node setting fixed at installation; empty means the default,
|
// mounts and environment. A node setting fixed at installation; empty means the default,
|
||||||
// /var/lib — see dir_into.go.
|
// /var/lib — see dir_into.go.
|
||||||
DataRoot string
|
DataRoot string
|
||||||
|
|
||||||
|
// ReadsHealth says this machine's node-engine reads a resource's `health` (novox/hq ADR 0240 Phase B:
|
||||||
|
// its statement's contract is link.ReadinessContract or later). An older engine parses strictly and
|
||||||
|
// refuses a field it does not know, whole — so to it the field is not sent, and what it runs is
|
||||||
|
// judged by liveness alone.
|
||||||
|
ReadsHealth bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// machinePort is where a module's port lives on this machine, or the port itself when the mesh has
|
// machinePort is where a module's port lives on this machine, or the port itself when the mesh has
|
||||||
@@ -957,6 +963,9 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
publishedOn(copied, m.Module, with)
|
publishedOn(copied, m.Module, with)
|
||||||
|
// How it is ready, in the node-engine's words: its endpoint as the port this machine
|
||||||
|
// published it on — or not sent at all to an engine older than the field (ADR 0240).
|
||||||
|
healthInto(copied, m, with)
|
||||||
// The account's environment and every module's shell code, where this module holds the
|
// The account's environment and every module's shell code, where this module holds the
|
||||||
// seat that places them (novox/hq ADR 0203, ADR 0204). Gathered from every module on
|
// seat that places them (novox/hq ADR 0203, ADR 0204). Gathered from every module on
|
||||||
// the node, as the jails are, and **last of every placeholder pass**: shell code is a
|
// the node, as the jails are, and **last of every placeholder pass**: shell code is a
|
||||||
@@ -1026,7 +1035,9 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
|||||||
// plane's; making a name resolve is the module's software. Emitted as ordinary files under
|
// plane's; making a name resolve is the module's software. Emitted as ordinary files under
|
||||||
// this module's name, so they are applied, reported and removed exactly as anything else
|
// this module's name, so they are applied, reported and removed exactly as anything else
|
||||||
// it declares.
|
// it declares.
|
||||||
given, err := FactsFrom(m, r, with)
|
// The resolver file is the node-resolver holder's where one is held here, and the uplink's
|
||||||
|
// holder steps back from it (novox/hq ADR 0247).
|
||||||
|
given, err := FactsFrom(stepsBack(m, r.Modules), r, with)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ package catalogue
|
|||||||
// DeliverySeat is the seat mesh-delivery holds.
|
// DeliverySeat is the seat mesh-delivery holds.
|
||||||
const DeliverySeat = "mesh-delivery"
|
const DeliverySeat = "mesh-delivery"
|
||||||
|
|
||||||
// deliveryVerbs are the delivery seat's tools: five that read, and the acts of a person and of healer H2.
|
// deliveryVerbs are the delivery seat's tools: six that read, and the acts of a person and of healer H2.
|
||||||
func deliveryVerbs() []Verb {
|
func deliveryVerbs() []Verb {
|
||||||
return []Verb{
|
return []Verb{
|
||||||
{Name: "deliveries", Description: "Every delivery not final, and those that ended in the last day, one line " +
|
{Name: "deliveries", Description: "Every delivery not final, and those that ended in the last day, one line " +
|
||||||
@@ -32,6 +32,16 @@ func deliveryVerbs() []Verb {
|
|||||||
Input: schema(map[string]string{"repository": "owner/repository",
|
Input: schema(map[string]string{"repository": "owner/repository",
|
||||||
"paths": "the files it changes, comma-separated, from the repository's root",
|
"paths": "the files it changes, comma-separated, from the repository's root",
|
||||||
"base": "the branch it merges into (default main)"}, []string{"repository", "paths"})},
|
"base": "the branch it merges into (default main)"}, []string{"repository", "paths"})},
|
||||||
|
{Name: "checks", Description: "What the mesh's checks said of a pull request's head or of one commit: each " +
|
||||||
|
"of the commit's mesh statuses (mesh/merge-gate, mesh/repo-check, mesh/delivery, …) with its state, " +
|
||||||
|
"description and when it was set; the merge check's full verdict as the controller said it — each " +
|
||||||
|
"layer's summary, the machine that ran it, when, its build id and its report; and whether the branch's " +
|
||||||
|
"protection would let it merge, every required status being success.",
|
||||||
|
Input: schema(map[string]string{"repository": "owner/repository",
|
||||||
|
"number": "a pull request's number: its head is read",
|
||||||
|
"commit": "a commit's sha, or the start of one, instead of a pull request"}, []string{"repository"}),
|
||||||
|
// Added after the seat's first holder shipped: optional until mesh-delivery serves it everywhere.
|
||||||
|
Optional: true},
|
||||||
{Name: "table", Description: "The state table every delivery runs by: each transition with its guard, " +
|
{Name: "table", Description: "The state table every delivery runs by: each transition with its guard, " +
|
||||||
"each state's bound and what healer H2 may do once it has passed; and the machine steps' table.",
|
"each state's bound and what healer H2 may do once it has passed; and the machine steps' table.",
|
||||||
Input: schema(map[string]string{}, nil)},
|
Input: schema(map[string]string{}, nil)},
|
||||||
|
|||||||
@@ -0,0 +1,60 @@
|
|||||||
|
package catalogue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The delivery seat answers "what did the mesh's checks say of this pull request?" as a verb of its own,
|
||||||
|
// `checks` (novox/hq ADR 0239): read by repository and a pull request's number or a commit.
|
||||||
|
func TestTheDeliverySeatPromisesChecks(t *testing.T) {
|
||||||
|
seat, ok := SeatNamed(DeliverySeat)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("the delivery seat is not in the set")
|
||||||
|
}
|
||||||
|
var checks *Verb
|
||||||
|
for i := range seat.Serves {
|
||||||
|
if seat.Serves[i].Name == "checks" {
|
||||||
|
checks = &seat.Serves[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if checks == nil {
|
||||||
|
t.Fatalf("the delivery seat promises %v and not checks", VerbNames(seat.Serves))
|
||||||
|
}
|
||||||
|
props, _ := checks.Input["properties"].(map[string]any)
|
||||||
|
for _, arg := range []string{"repository", "number", "commit"} {
|
||||||
|
if _, has := props[arg]; !has {
|
||||||
|
t.Errorf("checks takes no %q", arg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if req, _ := checks.Input["required"].([]string); !reflect.DeepEqual(req, []string{"repository"}) {
|
||||||
|
t.Errorf("checks requires %v, wanted only the repository", req)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Added after the seat's holder shipped, it is optional: the holder serving the ten verbs before it still
|
||||||
|
// holds the seat, and one serving all eleven does too — so the controller and the catalogue can move in
|
||||||
|
// either order, and no check of the catalogue fails on a module nobody touched between the two.
|
||||||
|
if !checks.Optional {
|
||||||
|
t.Fatal("checks is a condition of holding before its holder serves it")
|
||||||
|
}
|
||||||
|
var before []string
|
||||||
|
for _, v := range VerbNames(seat.Serves) {
|
||||||
|
if v != "checks" {
|
||||||
|
before = append(before, v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
m := Manifest{Module: "mesh-delivery", Claims: []Claim{{Name: DeliverySeat, Scope: ScopeMesh, Serves: before}}}
|
||||||
|
if err := CanHold(m, seat); err != nil {
|
||||||
|
t.Fatalf("a holder without checks yet: %v", err)
|
||||||
|
}
|
||||||
|
m.Claims[0].Serves = VerbNames(seat.Serves)
|
||||||
|
if err := CanHold(m, seat); err != nil {
|
||||||
|
t.Fatalf("a holder serving every verb: %v", err)
|
||||||
|
}
|
||||||
|
// Every other verb is still a condition of holding.
|
||||||
|
m.Claims[0].Serves = append([]string{"checks"}, before[1:]...)
|
||||||
|
if err := CanHold(m, seat); err == nil || !strings.Contains(err.Error(), before[0]) {
|
||||||
|
t.Fatalf("a holder without %s: %v", before[0], err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,460 @@
|
|||||||
|
package catalogue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A module says how each long-running resource is ready (novox/hq ADR 0240 rule 2, to-be 48 §2 and §8,
|
||||||
|
// Phase B).
|
||||||
|
//
|
||||||
|
// **On the resource, beside its other fields**: one kind and its timing. The node-engine judges every
|
||||||
|
// long-running resource alive with no declaration (Phase A); this is how a module says what *ready* means
|
||||||
|
// for one — the image's own check adopted by name, an HTTP request to a declared endpoint, a TCP connect,
|
||||||
|
// a command in the container, the unit's own readiness, or one of the module's own tools.
|
||||||
|
//
|
||||||
|
// **An endpoint is named, never a port or an address**: the check follows the machine's port for that
|
||||||
|
// endpoint as the endpoint does, so a port this machine gave elsewhere moves the check with it. The
|
||||||
|
// controller composes the name into the port the machine published it on, and sends the field only to a
|
||||||
|
// node-engine that reads it (link.ReadinessContract): an older, strict engine refuses a field it does not
|
||||||
|
// know, whole.
|
||||||
|
//
|
||||||
|
// **The bounds are the record's**: an interval not under ten seconds, a timeout under the interval, at
|
||||||
|
// least two failing looks in a row, and a grace plus the failing looks within five minutes — so a
|
||||||
|
// resource broken from its start is said within the gate's ten. Refused here, near the author, and again
|
||||||
|
// by the node-engine, far away, in the same words.
|
||||||
|
|
||||||
|
// HealthField is the resource field.
|
||||||
|
const HealthField = "health"
|
||||||
|
|
||||||
|
// The kinds of check (to-be 48 §2).
|
||||||
|
const (
|
||||||
|
HealthRuntime = "runtime"
|
||||||
|
HealthHTTP = "http"
|
||||||
|
HealthTCP = "tcp"
|
||||||
|
HealthExec = "exec"
|
||||||
|
HealthUnit = "unit"
|
||||||
|
HealthTool = "tool"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The bounds and the defaults (ADR 0240 rule 2).
|
||||||
|
const (
|
||||||
|
HealthIntervalDefault = 30 * time.Second
|
||||||
|
HealthIntervalFloor = 10 * time.Second
|
||||||
|
HealthTimeoutDefault = 5 * time.Second
|
||||||
|
HealthLooksDefault = 3
|
||||||
|
HealthLooksFloor = 2
|
||||||
|
HealthGraceDefault = 60 * time.Second
|
||||||
|
// HealthWithin is the most a grace and the failing looks may take together: a resource broken
|
||||||
|
// from its start is said within the gate's ten minutes with room for its judgings.
|
||||||
|
HealthWithin = 5 * time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
|
// HealthRequiredFrom is when `module check` refuses a long-running resource without `health` (ADR 0240
|
||||||
|
// rule 8): six weeks after liveness was first judged live (2026-10-07), unless the catalogue's count of
|
||||||
|
// undeclared resources reached zero first — which its own counter enforces by never letting it rise.
|
||||||
|
var HealthRequiredFrom = time.Date(2026, 11, 18, 0, 0, 0, 0, time.UTC)
|
||||||
|
|
||||||
|
// Health is one resource's declaration, read.
|
||||||
|
type Health struct {
|
||||||
|
Kind string
|
||||||
|
// Endpoint is the `listens` name an http or tcp check looks at.
|
||||||
|
Endpoint string
|
||||||
|
// Path, Status, Body and Scheme are an http check's: the path asked, the status expected (zero: any
|
||||||
|
// status under 400), a text the answer must hold, and http or https.
|
||||||
|
Path string
|
||||||
|
Status int
|
||||||
|
Body string
|
||||||
|
Scheme string
|
||||||
|
// Command is an exec check's command, run by the container's shell.
|
||||||
|
Command string
|
||||||
|
// Tool is a tool check's tool, one of the module's own.
|
||||||
|
Tool string
|
||||||
|
// The timing, with the defaults applied.
|
||||||
|
Interval, Timeout, Grace time.Duration
|
||||||
|
Looks int
|
||||||
|
// Needs is the provision the check exercises (to-be 48 §6): while its provider for this consumer is
|
||||||
|
// unhealthy, what this check finds is held under the provider's condition.
|
||||||
|
Needs string
|
||||||
|
}
|
||||||
|
|
||||||
|
// healthKeys are the keys a `health` field may carry; anything else is refused by name.
|
||||||
|
var healthKeys = map[string]bool{"kind": true, "endpoint": true, "path": true, "status": true, "body": true,
|
||||||
|
"scheme": true, "command": true, "tool": true, "interval": true, "timeout": true, "looks": true,
|
||||||
|
"grace": true, "needs": true}
|
||||||
|
|
||||||
|
// healthAddressKeys are what a check may not be aimed by: a port or an address does not follow the
|
||||||
|
// machine's port for the endpoint, and a manifest is the same on every machine.
|
||||||
|
var healthAddressKeys = map[string]bool{"port": true, "address": true, "host": true, "url": true, "ip": true}
|
||||||
|
|
||||||
|
// LongRunning says whether a manifest resource stays up: a container that is no step and on no schedule,
|
||||||
|
// a service stated running, a process that is no step and on no schedule (ADR 0240 rule 1).
|
||||||
|
func LongRunning(r map[string]any) bool {
|
||||||
|
switch fmt.Sprint(r["type"]) {
|
||||||
|
case "container", "process":
|
||||||
|
if once, _ := r["run-once"].(bool); once {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return r["schedule"] == nil
|
||||||
|
case "service":
|
||||||
|
return fmt.Sprint(r["state"]) == "running"
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadHealth reads a resource's `health` field, defaults applied. False when it carries none.
|
||||||
|
func ReadHealth(r map[string]any) (Health, bool, []string) {
|
||||||
|
raw, present := r[HealthField]
|
||||||
|
if !present {
|
||||||
|
return Health{}, false, nil
|
||||||
|
}
|
||||||
|
id := fmt.Sprint(r["id"])
|
||||||
|
fields, ok := raw.(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
return Health{}, true, []string{fmt.Sprintf("%s: health is %T; it is an object with a kind and its timing", id, raw)}
|
||||||
|
}
|
||||||
|
var problems []string
|
||||||
|
keys := make([]string, 0, len(fields))
|
||||||
|
for k := range fields {
|
||||||
|
keys = append(keys, k)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
for _, k := range keys {
|
||||||
|
switch {
|
||||||
|
case healthAddressKeys[k]:
|
||||||
|
problems = append(problems, fmt.Sprintf("%s: its health names a %s; a check names an endpoint the module "+
|
||||||
|
"declares under listens, by its name, so it follows the port this machine gives it (ADR 0240 rule 2)", id, k))
|
||||||
|
case !healthKeys[k]:
|
||||||
|
problems = append(problems, fmt.Sprintf("%s: its health says %q, which a health check does not have", id, k))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
text := func(key string) string {
|
||||||
|
v, present := fields[key]
|
||||||
|
if !present {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
s, ok := v.(string)
|
||||||
|
if !ok {
|
||||||
|
problems = append(problems, fmt.Sprintf("%s: its health's %s is %T; it is text", id, key, v))
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(s)
|
||||||
|
}
|
||||||
|
duration := func(key string, fallback time.Duration) time.Duration {
|
||||||
|
s := text(key)
|
||||||
|
if s == "" {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
d, err := time.ParseDuration(s)
|
||||||
|
if err != nil || d < 0 {
|
||||||
|
problems = append(problems, fmt.Sprintf("%s: its health's %s is %q; it is a duration such as \"30s\"", id, key, s))
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
number := func(key string, fallback int) int {
|
||||||
|
v, present := fields[key]
|
||||||
|
if !present {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
f, ok := v.(float64)
|
||||||
|
if !ok || f != float64(int(f)) {
|
||||||
|
problems = append(problems, fmt.Sprintf("%s: its health's %s is %v; it is a whole number", id, key, v))
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
return int(f)
|
||||||
|
}
|
||||||
|
h := Health{Kind: text("kind"), Endpoint: text("endpoint"), Path: text("path"), Body: text("body"),
|
||||||
|
Scheme: text("scheme"), Command: text("command"), Tool: text("tool"), Needs: text("needs"),
|
||||||
|
Interval: duration("interval", HealthIntervalDefault), Timeout: duration("timeout", HealthTimeoutDefault),
|
||||||
|
Grace: duration("grace", HealthGraceDefault), Looks: number("looks", HealthLooksDefault),
|
||||||
|
Status: number("status", 0)}
|
||||||
|
return h, true, problems
|
||||||
|
}
|
||||||
|
|
||||||
|
// healthProblems is everything wrong with a manifest's `health` fields, in the manifest's words (to-be
|
||||||
|
// 48 §8): a field on something that does not stay up, a kind its resource cannot have, an endpoint the
|
||||||
|
// module does not declare, a port or an address, a timing outside its bounds, a tool the module does not
|
||||||
|
// serve, and a tool check with no check of another kind beside it on the module.
|
||||||
|
func healthProblems(m Manifest) []string {
|
||||||
|
var problems []string
|
||||||
|
endpoints := map[string]Listening{}
|
||||||
|
for _, l := range m.Listens {
|
||||||
|
if n := strings.TrimSpace(l.Name); n != "" {
|
||||||
|
endpoints[n] = l
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tools := map[string]bool{}
|
||||||
|
for _, t := range m.Tools {
|
||||||
|
tools[t] = true
|
||||||
|
}
|
||||||
|
wants := map[string]bool{}
|
||||||
|
for _, w := range m.Wants() {
|
||||||
|
wants[w] = true
|
||||||
|
}
|
||||||
|
var toolChecks []string
|
||||||
|
otherKinds := 0
|
||||||
|
for _, r := range m.Resources {
|
||||||
|
h, has, read := ReadHealth(r)
|
||||||
|
if !has {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
id, typ := fmt.Sprint(r["id"]), fmt.Sprint(r["type"])
|
||||||
|
where := m.Module + ": " + id
|
||||||
|
for _, p := range read {
|
||||||
|
problems = append(problems, m.Module+": "+p)
|
||||||
|
}
|
||||||
|
if !LongRunning(r) {
|
||||||
|
problems = append(problems, fmt.Sprintf("%s declares health and does not stay up: a step, anything on a "+
|
||||||
|
"schedule and a service not stated running are judged by their step and their schedule (ADR 0240 rule 1)", where))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
switch h.Kind {
|
||||||
|
case HealthRuntime, HealthExec:
|
||||||
|
if typ != "container" {
|
||||||
|
problems = append(problems, fmt.Sprintf("%s is a %s and its health is %q, which only a container has: "+
|
||||||
|
"the runtime runs it inside the container", where, typ, h.Kind))
|
||||||
|
}
|
||||||
|
case HealthUnit:
|
||||||
|
if typ == "container" {
|
||||||
|
problems = append(problems, fmt.Sprintf("%s is a container and its health is %q, which is a service's "+
|
||||||
|
"or a process's own readiness", where, h.Kind))
|
||||||
|
}
|
||||||
|
case HealthHTTP, HealthTCP, HealthTool:
|
||||||
|
case "":
|
||||||
|
problems = append(problems, fmt.Sprintf("%s declares health with no kind: %s", where, healthKindsWords()))
|
||||||
|
default:
|
||||||
|
problems = append(problems, fmt.Sprintf("%s declares health of kind %q: %s", where, h.Kind, healthKindsWords()))
|
||||||
|
}
|
||||||
|
switch h.Kind {
|
||||||
|
case HealthHTTP, HealthTCP:
|
||||||
|
l, declared := endpoints[h.Endpoint]
|
||||||
|
switch {
|
||||||
|
case h.Endpoint == "":
|
||||||
|
problems = append(problems, fmt.Sprintf("%s's %s check names no endpoint: it names one the module "+
|
||||||
|
"declares under listens, by its name", where, h.Kind))
|
||||||
|
case !declared:
|
||||||
|
problems = append(problems, fmt.Sprintf("%s's %s check names the endpoint %q, which %s does not declare "+
|
||||||
|
"under listens (%s)", where, h.Kind, h.Endpoint, m.Module, namedEndpointsWords(endpoints)))
|
||||||
|
case l.At() != "tcp":
|
||||||
|
problems = append(problems, fmt.Sprintf("%s's %s check names %q, which is %s: a check connects over tcp",
|
||||||
|
where, h.Kind, h.Endpoint, l.At()))
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
if h.Endpoint != "" {
|
||||||
|
problems = append(problems, fmt.Sprintf("%s's %s check names an endpoint, which only an http or tcp check "+
|
||||||
|
"looks at", where, h.Kind))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if h.Kind != HealthHTTP && (h.Path != "" || h.Status != 0 || h.Body != "" || h.Scheme != "") {
|
||||||
|
problems = append(problems, fmt.Sprintf("%s's %s check says a path, a status, a body or a scheme, which "+
|
||||||
|
"only an http check has", where, h.Kind))
|
||||||
|
}
|
||||||
|
if h.Kind == HealthHTTP {
|
||||||
|
if h.Path != "" && !strings.HasPrefix(h.Path, "/") {
|
||||||
|
problems = append(problems, fmt.Sprintf("%s's http check asks %q; a path starts with /", where, h.Path))
|
||||||
|
}
|
||||||
|
if h.Status != 0 && (h.Status < 100 || h.Status > 599) {
|
||||||
|
problems = append(problems, fmt.Sprintf("%s's http check expects status %d, which is not one", where, h.Status))
|
||||||
|
}
|
||||||
|
if h.Scheme != "" && h.Scheme != "http" && h.Scheme != "https" {
|
||||||
|
problems = append(problems, fmt.Sprintf("%s's http check is over %q; it is http or https", where, h.Scheme))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (h.Command != "") != (h.Kind == HealthExec) {
|
||||||
|
if h.Kind == HealthExec {
|
||||||
|
problems = append(problems, fmt.Sprintf("%s's exec check says no command", where))
|
||||||
|
} else {
|
||||||
|
problems = append(problems, fmt.Sprintf("%s's %s check says a command, which only an exec check runs",
|
||||||
|
where, h.Kind))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if h.Kind == HealthTool {
|
||||||
|
switch {
|
||||||
|
case h.Tool == "":
|
||||||
|
problems = append(problems, fmt.Sprintf("%s's tool check names no tool", where))
|
||||||
|
case !tools[h.Tool]:
|
||||||
|
problems = append(problems, fmt.Sprintf("%s's tool check asks %q, which %s does not serve (its tools: %s)",
|
||||||
|
where, h.Tool, m.Module, orNoneWords(m.Tools)))
|
||||||
|
}
|
||||||
|
toolChecks = append(toolChecks, id)
|
||||||
|
} else {
|
||||||
|
if h.Tool != "" {
|
||||||
|
problems = append(problems, fmt.Sprintf("%s's %s check names a tool, which only a tool check asks", where, h.Kind))
|
||||||
|
}
|
||||||
|
if h.Kind != "" {
|
||||||
|
otherKinds++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if h.Needs != "" && !wants[h.Needs] {
|
||||||
|
problems = append(problems, fmt.Sprintf("%s's check needs %q, which %s does not require: a check names "+
|
||||||
|
"the provision it exercises, among those the module requires", where, h.Needs, m.Module))
|
||||||
|
}
|
||||||
|
problems = append(problems, healthTimingProblems(where, h)...)
|
||||||
|
}
|
||||||
|
if len(toolChecks) > 0 && otherKinds == 0 {
|
||||||
|
problems = append(problems, fmt.Sprintf("%s judges itself only by its own tool (%s): a tool check is for "+
|
||||||
|
"function no endpoint shows, and only beside a check of another kind the module does not run itself "+
|
||||||
|
"(ADR 0227 rule 8, ADR 0240 rule 2)", m.Module, strings.Join(toolChecks, ", ")))
|
||||||
|
}
|
||||||
|
return problems
|
||||||
|
}
|
||||||
|
|
||||||
|
// healthTimingProblems holds a check's timing to its bounds.
|
||||||
|
func healthTimingProblems(where string, h Health) []string {
|
||||||
|
var problems []string
|
||||||
|
if h.Interval < HealthIntervalFloor {
|
||||||
|
problems = append(problems, fmt.Sprintf("%s looks every %s; a check looks no more often than every %s — "+
|
||||||
|
"the mesh is a guest on the machine (ADR 0240 rule 2)", where, h.Interval, HealthIntervalFloor))
|
||||||
|
}
|
||||||
|
if h.Timeout <= 0 || h.Timeout >= h.Interval {
|
||||||
|
problems = append(problems, fmt.Sprintf("%s gives a look %s, which must be more than nothing and under its "+
|
||||||
|
"interval of %s", where, h.Timeout, h.Interval))
|
||||||
|
}
|
||||||
|
if h.Looks < HealthLooksFloor {
|
||||||
|
problems = append(problems, fmt.Sprintf("%s is unhealthy after %d failing look(s); it is at least %d — one "+
|
||||||
|
"look can be wrong (issue 277)", where, h.Looks, HealthLooksFloor))
|
||||||
|
}
|
||||||
|
if h.Grace < 0 {
|
||||||
|
problems = append(problems, fmt.Sprintf("%s has a grace of %s", where, h.Grace))
|
||||||
|
}
|
||||||
|
if h.Looks >= HealthLooksFloor && h.Interval >= HealthIntervalFloor {
|
||||||
|
if took := h.Grace + time.Duration(h.Looks)*h.Interval; took > HealthWithin {
|
||||||
|
problems = append(problems, fmt.Sprintf("%s is said unhealthy at the earliest %s after it starts (a grace "+
|
||||||
|
"of %s and %d looks every %s); it is at most %s, so a resource broken from its start is said within "+
|
||||||
|
"the gate's bound", where, took, h.Grace, h.Looks, h.Interval, HealthWithin))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return problems
|
||||||
|
}
|
||||||
|
|
||||||
|
func healthKindsWords() string {
|
||||||
|
return "a check is runtime (the image's own, adopted by name), http, tcp, exec, unit or tool"
|
||||||
|
}
|
||||||
|
|
||||||
|
func namedEndpointsWords(endpoints map[string]Listening) string {
|
||||||
|
if len(endpoints) == 0 {
|
||||||
|
return "it names none"
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(endpoints))
|
||||||
|
for n := range endpoints {
|
||||||
|
names = append(names, n)
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
return "it names " + strings.Join(names, ", ")
|
||||||
|
}
|
||||||
|
|
||||||
|
func orNoneWords(names []string) string {
|
||||||
|
if len(names) == 0 {
|
||||||
|
return "none"
|
||||||
|
}
|
||||||
|
return strings.Join(names, ", ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Undeclared is every long-running resource of the manifest without `health`, by id (ADR 0240 rule 8):
|
||||||
|
// what the catalogue's count counts.
|
||||||
|
func Undeclared(m Manifest) []string {
|
||||||
|
var out []string
|
||||||
|
for _, r := range m.Resources {
|
||||||
|
if !LongRunning(r) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, has := r[HealthField]; !has {
|
||||||
|
out = append(out, fmt.Sprint(r["id"]))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// HealthChecks is every tool a module's health asks, as `<module>.<tool>`: what a machine's node-engine
|
||||||
|
// is granted to ask of its own node tools (to-be 48 §3).
|
||||||
|
func HealthChecks(m Manifest) []string {
|
||||||
|
var out []string
|
||||||
|
for _, r := range m.Resources {
|
||||||
|
h, has, _ := ReadHealth(r)
|
||||||
|
if has && h.Kind == HealthTool && h.Tool != "" && LongRunning(r) {
|
||||||
|
out = append(out, m.Module+"."+h.Tool)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// healthInto composes a resource's `health` into the node-engine's words, or takes it away (to-be 48 §2,
|
||||||
|
// §3): for an engine that reads it, the endpoint becomes the port this machine published it on and the
|
||||||
|
// defaults are written out; for one that does not — older and strict — the field is not sent, and the
|
||||||
|
// resource is judged by liveness alone, as before.
|
||||||
|
func healthInto(resource map[string]any, m Manifest, with Rendering) {
|
||||||
|
if _, has := resource[HealthField]; !has {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !with.ReadsHealth {
|
||||||
|
delete(resource, HealthField)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
h, _, _ := ReadHealth(resource)
|
||||||
|
out := map[string]any{"kind": h.Kind, "interval": h.Interval.String(), "timeout": h.Timeout.String(),
|
||||||
|
"looks": h.Looks, "grace": h.Grace.String()}
|
||||||
|
if h.Endpoint != "" {
|
||||||
|
out["endpoint"] = h.Endpoint
|
||||||
|
if port, ok := EndpointPort(m, h.Endpoint); ok {
|
||||||
|
out["port"] = with.machinePort(m.Module, port)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if h.Kind == HealthHTTP {
|
||||||
|
path := h.Path
|
||||||
|
if path == "" {
|
||||||
|
path = "/"
|
||||||
|
}
|
||||||
|
out["path"] = path
|
||||||
|
if h.Status != 0 {
|
||||||
|
out["status"] = h.Status
|
||||||
|
}
|
||||||
|
if h.Body != "" {
|
||||||
|
out["body"] = h.Body
|
||||||
|
}
|
||||||
|
if h.Scheme != "" {
|
||||||
|
out["scheme"] = h.Scheme
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if h.Command != "" {
|
||||||
|
out["command"] = h.Command
|
||||||
|
}
|
||||||
|
if h.Tool != "" {
|
||||||
|
out["tool"] = h.Tool
|
||||||
|
}
|
||||||
|
if h.Needs != "" {
|
||||||
|
out["needs"] = h.Needs
|
||||||
|
}
|
||||||
|
resource[HealthField] = out
|
||||||
|
}
|
||||||
|
|
||||||
|
// HealthWords is a declared check in a line, for `module check` and `node show`.
|
||||||
|
func HealthWords(h Health) string {
|
||||||
|
var what string
|
||||||
|
switch h.Kind {
|
||||||
|
case HealthHTTP:
|
||||||
|
what = "http " + orSlash(h.Path) + " on " + h.Endpoint
|
||||||
|
if h.Status != 0 {
|
||||||
|
what += " expecting " + strconv.Itoa(h.Status)
|
||||||
|
}
|
||||||
|
case HealthTCP:
|
||||||
|
what = "tcp on " + h.Endpoint
|
||||||
|
case HealthTool:
|
||||||
|
what = "its tool " + h.Tool
|
||||||
|
case HealthRuntime:
|
||||||
|
what = "its image's own check"
|
||||||
|
default:
|
||||||
|
what = h.Kind
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s every %s", what, h.Interval)
|
||||||
|
}
|
||||||
|
|
||||||
|
func orSlash(p string) string {
|
||||||
|
if p == "" {
|
||||||
|
return "/"
|
||||||
|
}
|
||||||
|
return p
|
||||||
|
}
|
||||||
@@ -0,0 +1,217 @@
|
|||||||
|
package catalogue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/validate"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A module says how each long-running resource is ready (novox/hq ADR 0240 rule 2, to-be 48 §8): `module
|
||||||
|
// check` refuses each part out of its bounds, each endpoint named by a port or an address, a tool the
|
||||||
|
// module does not serve, and a tool check alone — a test per refusal.
|
||||||
|
|
||||||
|
const healthDigest = "@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||||
|
|
||||||
|
// healthManifest is a module of one web container, one running service and one step, with the health
|
||||||
|
// given on the container (or on the resource named by on).
|
||||||
|
func healthManifest(t *testing.T, on string, health map[string]any, more ...map[string]any) []byte {
|
||||||
|
t.Helper()
|
||||||
|
resources := []map[string]any{
|
||||||
|
{"id": "server", "type": "container", "name": "app-server", "image": "registry.example/app" + healthDigest,
|
||||||
|
"ports": []any{"8080"}},
|
||||||
|
{"id": "daemon", "type": "service", "unit": "app.service", "state": "running"},
|
||||||
|
{"id": "seed", "type": "container", "name": "app-seed", "image": "registry.example/app" + healthDigest,
|
||||||
|
"run-once": true},
|
||||||
|
{"id": "sweep", "type": "container", "name": "app-sweep", "image": "registry.example/app" + healthDigest,
|
||||||
|
"schedule": "0 3 * * *"},
|
||||||
|
}
|
||||||
|
resources = append(resources, more...)
|
||||||
|
for _, r := range resources {
|
||||||
|
if r["id"] == on && health != nil {
|
||||||
|
r["health"] = health
|
||||||
|
}
|
||||||
|
}
|
||||||
|
m := map[string]any{"module": "app", "requires": []any{"postgres-database"}, "tools": []any{"app_status"},
|
||||||
|
"listens": []any{
|
||||||
|
map[string]any{"name": "web", "port": 8080, "from": "mesh"},
|
||||||
|
map[string]any{"name": "beacon", "port": 9999, "protocol": "udp", "from": "mesh"},
|
||||||
|
},
|
||||||
|
"resources": resources}
|
||||||
|
raw, err := json.Marshal(m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAWellFormedHealthIsAccepted(t *testing.T) {
|
||||||
|
for _, h := range []map[string]any{
|
||||||
|
{"kind": "http", "endpoint": "web", "path": "/healthz", "status": 200, "body": "ok", "needs": "postgres-database"},
|
||||||
|
{"kind": "tcp", "endpoint": "web", "interval": "10s", "timeout": "2s", "looks": 2, "grace": "0s"},
|
||||||
|
{"kind": "runtime"},
|
||||||
|
{"kind": "exec", "command": "pg_isready -q", "grace": "4m", "looks": 2, "interval": "30s"},
|
||||||
|
} {
|
||||||
|
if _, err := ParseManifest(healthManifest(t, "server", h)); err != nil {
|
||||||
|
t.Errorf("%v was refused: %v", h, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := ParseManifest(healthManifest(t, "daemon", map[string]any{"kind": "unit"})); err != nil {
|
||||||
|
t.Errorf("a service's own readiness was refused: %v", err)
|
||||||
|
}
|
||||||
|
// A tool beside a check of another kind on the module.
|
||||||
|
raw := healthManifest(t, "server", map[string]any{"kind": "http", "endpoint": "web"},
|
||||||
|
map[string]any{"id": "admin", "type": "service", "unit": "app-admin.service", "state": "running",
|
||||||
|
"health": map[string]any{"kind": "tool", "tool": "app_status"}})
|
||||||
|
if _, err := ParseManifest(raw); err != nil {
|
||||||
|
t.Errorf("a tool check beside an http check was refused: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEveryOutOfBoundsHealthIsRefusedByName(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
on string
|
||||||
|
health map[string]any
|
||||||
|
says string
|
||||||
|
}{
|
||||||
|
{"no kind", "server", map[string]any{"endpoint": "web"}, "with no kind"},
|
||||||
|
{"an unknown kind", "server", map[string]any{"kind": "ping"}, `of kind "ping"`},
|
||||||
|
{"a port", "server", map[string]any{"kind": "tcp", "port": 8080}, "names a port"},
|
||||||
|
{"an address", "server", map[string]any{"kind": "http", "endpoint": "web", "address": "127.0.0.1"}, "names a address"},
|
||||||
|
{"a url", "server", map[string]any{"kind": "http", "url": "http://localhost:8080/"}, "names a url"},
|
||||||
|
{"an unknown key", "server", map[string]any{"kind": "tcp", "endpoint": "web", "retries": 3}, `"retries"`},
|
||||||
|
{"no endpoint", "server", map[string]any{"kind": "http"}, "names no endpoint"},
|
||||||
|
{"an undeclared endpoint", "server", map[string]any{"kind": "tcp", "endpoint": "admin"}, "does not declare"},
|
||||||
|
{"a udp endpoint", "server", map[string]any{"kind": "tcp", "endpoint": "beacon"}, "a check connects over tcp"},
|
||||||
|
{"an interval under the floor", "server", map[string]any{"kind": "tcp", "endpoint": "web", "interval": "5s", "timeout": "1s"}, "no more often than every 10s"},
|
||||||
|
{"a timeout of the interval", "server", map[string]any{"kind": "tcp", "endpoint": "web", "interval": "10s", "timeout": "10s"}, "under its interval"},
|
||||||
|
{"one failing look", "server", map[string]any{"kind": "tcp", "endpoint": "web", "looks": 1}, "at least 2"},
|
||||||
|
{"a grace and looks past five minutes", "server", map[string]any{"kind": "tcp", "endpoint": "web", "grace": "4m", "looks": 3, "interval": "30s"}, "at most 5m0s"},
|
||||||
|
{"a duration that is not one", "server", map[string]any{"kind": "tcp", "endpoint": "web", "interval": "often"}, "is a duration"},
|
||||||
|
{"looks that are not a number", "server", map[string]any{"kind": "tcp", "endpoint": "web", "looks": "three"}, "whole number"},
|
||||||
|
{"a path without a slash", "server", map[string]any{"kind": "http", "endpoint": "web", "path": "health"}, "starts with /"},
|
||||||
|
{"a status that is not one", "server", map[string]any{"kind": "http", "endpoint": "web", "status": 700}, "is not one"},
|
||||||
|
{"a scheme that is not one", "server", map[string]any{"kind": "http", "endpoint": "web", "scheme": "ftp"}, "http or https"},
|
||||||
|
{"a status on a tcp check", "server", map[string]any{"kind": "tcp", "endpoint": "web", "status": 200}, "only an http check has"},
|
||||||
|
{"an exec with no command", "server", map[string]any{"kind": "exec"}, "says no command"},
|
||||||
|
{"a command on an http check", "server", map[string]any{"kind": "http", "endpoint": "web", "command": "true"}, "only an exec check runs"},
|
||||||
|
{"a runtime check on a service", "daemon", map[string]any{"kind": "runtime"}, "only a container has"},
|
||||||
|
{"a unit check on a container", "server", map[string]any{"kind": "unit"}, "a service's or a process's"},
|
||||||
|
{"a tool the module does not serve", "server", map[string]any{"kind": "tool", "tool": "app_admin"}, "does not serve"},
|
||||||
|
{"a tool check alone", "server", map[string]any{"kind": "tool", "tool": "app_status"}, "judges itself only by its own tool"},
|
||||||
|
{"needs not required", "server", map[string]any{"kind": "tcp", "endpoint": "web", "needs": "redis"}, "does not require"},
|
||||||
|
{"health on a step", "seed", map[string]any{"kind": "runtime"}, "does not stay up"},
|
||||||
|
{"health on a schedule", "sweep", map[string]any{"kind": "runtime"}, "does not stay up"},
|
||||||
|
{"health that is not an object", "server", nil, "is an object"},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
raw := healthManifest(t, c.on, c.health)
|
||||||
|
if c.health == nil {
|
||||||
|
raw = []byte(strings.Replace(string(raw), `"name":"app-server"`, `"name":"app-server","health":"tcp"`, 1))
|
||||||
|
}
|
||||||
|
_, err := ParseManifest(raw)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("%s was accepted", c.name)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), c.says) {
|
||||||
|
t.Errorf("refused, but not for saying %q: %v", c.says, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHealthIsComposedAsThePortThisMachineGaveTheEndpoint(t *testing.T) {
|
||||||
|
m, err := ParseManifest(healthManifest(t, "server", map[string]any{"kind": "http", "endpoint": "web", "path": "/healthz",
|
||||||
|
"needs": "postgres-database"}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
compose := func(with Rendering) map[string]any {
|
||||||
|
out, err := Resolution{Node: "laptop", Modules: []Manifest{m}}.Declaration(with)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
at := indexOfID(out, "app.server")
|
||||||
|
if at < 0 {
|
||||||
|
t.Fatal("the container was lost")
|
||||||
|
}
|
||||||
|
return out[at]
|
||||||
|
}
|
||||||
|
// An engine older than the field is not sent it: it would refuse the whole declaration.
|
||||||
|
if h, sent := compose(Rendering{Ports: map[string]map[int]int{"app": {8080: 31001}}})["health"]; sent {
|
||||||
|
t.Fatalf("health was sent to an engine that does not read it: %v", h)
|
||||||
|
}
|
||||||
|
got := compose(Rendering{ReadsHealth: true, Ports: map[string]map[int]int{"app": {8080: 31001}}})["health"].(map[string]any)
|
||||||
|
if got["port"] != 31001 || got["endpoint"] != "web" || got["path"] != "/healthz" || got["needs"] != "postgres-database" {
|
||||||
|
t.Errorf("composed as %v", got)
|
||||||
|
}
|
||||||
|
if got["interval"] != "30s" || got["timeout"] != "5s" || got["looks"] != 3 || got["grace"] != "1m0s" {
|
||||||
|
t.Errorf("the defaults were not written out: %v", got)
|
||||||
|
}
|
||||||
|
// The port this machine gives the endpoint moves, and the check moves with it.
|
||||||
|
moved := compose(Rendering{ReadsHealth: true, Ports: map[string]map[int]int{"app": {8080: 31002}}})["health"].(map[string]any)
|
||||||
|
if moved["port"] != 31002 {
|
||||||
|
t.Errorf("after the port moved the check still dials %v", moved["port"])
|
||||||
|
}
|
||||||
|
// And the catalogue's manifest is untouched by composing it.
|
||||||
|
if _, ok := m.Resources[0]["health"].(map[string]any)["port"]; ok {
|
||||||
|
t.Error("composing wrote the port into the catalogue's own manifest")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheUndeclaredAreTheLongRunningWithoutHealth(t *testing.T) {
|
||||||
|
m, err := ParseManifest(healthManifest(t, "server", map[string]any{"kind": "runtime"}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := strings.Join(Undeclared(m), ","); got != "daemon" {
|
||||||
|
t.Errorf("undeclared: %q; the step and the schedule are not long-running, the server declares", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestATooledHealthIsGrantedToTheEngine(t *testing.T) {
|
||||||
|
raw := healthManifest(t, "server", map[string]any{"kind": "http", "endpoint": "web"},
|
||||||
|
map[string]any{"id": "admin", "type": "service", "unit": "app-admin.service", "state": "running",
|
||||||
|
"health": map[string]any{"kind": "tool", "tool": "app_status"}})
|
||||||
|
m, err := ParseManifest(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := HealthChecks(m); len(got) != 1 || got[0] != "app.app_status" {
|
||||||
|
t.Errorf("checks %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// What the controller composes the node-engine takes: every kind, composed for an engine that reads it,
|
||||||
|
// passes the engine's own validator (mesh-host/validate) — one set of words on both sides.
|
||||||
|
func TestEveryComposedHealthIsOneTheNodeEngineTakes(t *testing.T) {
|
||||||
|
for _, h := range []map[string]any{
|
||||||
|
{"kind": "http", "endpoint": "web", "path": "/healthz", "status": 200, "body": "ok", "needs": "postgres-database"},
|
||||||
|
{"kind": "tcp", "endpoint": "web", "interval": "10s", "timeout": "2s", "looks": 2, "grace": "0s"},
|
||||||
|
{"kind": "runtime"},
|
||||||
|
{"kind": "exec", "command": "pg_isready -q"},
|
||||||
|
} {
|
||||||
|
m, err := ParseManifest(healthManifest(t, "server", h,
|
||||||
|
map[string]any{"id": "admin", "type": "service", "unit": "app-admin.service", "state": "running",
|
||||||
|
"health": map[string]any{"kind": "tool", "tool": "app_status"}}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m.Resources[1]["health"] = map[string]any{"kind": "unit"}
|
||||||
|
out, err := Resolution{Node: "laptop", Modules: []Manifest{m}}.Declaration(Rendering{ReadsHealth: true,
|
||||||
|
Ports: map[string]map[int]int{"app": {8080: 31001}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
body, err := json.Marshal(map[string]any{"declaration": validate.Version, "resources": out})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if problems := validate.Declaration(body); len(problems) > 0 {
|
||||||
|
t.Errorf("%v composed into something the node-engine refuses: %v", h, problems)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -426,6 +426,13 @@ type Manifest struct {
|
|||||||
// module claiming a seat answers what that seat's protocol promises (novox/hq ADR 0118).
|
// module claiming a seat answers what that seat's protocol promises (novox/hq ADR 0118).
|
||||||
Tools []string `json:"tools,omitempty"`
|
Tools []string `json:"tools,omitempty"`
|
||||||
|
|
||||||
|
// Replaces says, for a tool of this module's own, the shell commands it is the mesh's way to do —
|
||||||
|
// `{"docker_logs": ["docker logs"]}` — as a seat's verb says it in its definition (novox/hq ADR
|
||||||
|
// 0241). A seat's verb carries its own: what a role replaces is the role's, so a module never says it
|
||||||
|
// for a verb it serves under a claim. The console's search, the agent's instructions and the guard on
|
||||||
|
// its shell are built from both.
|
||||||
|
Replaces map[string][]string `json:"replaces,omitempty"`
|
||||||
|
|
||||||
// Instances says whether this module's instances are the same anywhere — `interchangeable` —
|
// Instances says whether this module's instances are the same anywhere — `interchangeable` —
|
||||||
// so a call that names no machine may be answered by any of them (novox/hq ADR 0160). A fact
|
// so a call that names no machine may be answered by any of them (novox/hq ADR 0160). A fact
|
||||||
// about the software, not about the bus: a stateless web tool says it; a database does not,
|
// about the software, not about the bus: a stateless web tool says it; a database does not,
|
||||||
@@ -1692,6 +1699,7 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
problems = append(problems, invokeProblems(m)...)
|
problems = append(problems, invokeProblems(m)...)
|
||||||
|
problems = append(problems, replacesProblems(m)...)
|
||||||
problems = append(problems, endpointNameProblems(m)...)
|
problems = append(problems, endpointNameProblems(m)...)
|
||||||
problems = append(problems, RouteProblems(m)...)
|
problems = append(problems, RouteProblems(m)...)
|
||||||
for _, port := range m.Guards {
|
for _, port := range m.Guards {
|
||||||
@@ -2036,6 +2044,9 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
problems = append(problems, zoneProblems(m)...)
|
problems = append(problems, zoneProblems(m)...)
|
||||||
|
// How each long-running resource is ready (novox/hq ADR 0240 rule 2): said near its author, in the
|
||||||
|
// words the node-engine would refuse it in far away.
|
||||||
|
problems = append(problems, healthProblems(m)...)
|
||||||
if len(problems) > 0 {
|
if len(problems) > 0 {
|
||||||
sort.Strings(problems)
|
sort.Strings(problems)
|
||||||
return Manifest{}, fmt.Errorf("this manifest cannot be used:\n - %s",
|
return Manifest{}, fmt.Errorf("this manifest cannot be used:\n - %s",
|
||||||
@@ -2303,6 +2314,53 @@ func EndpointPort(m Manifest, name string) (int, bool) {
|
|||||||
return 0, false
|
return 0, false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// MaxReplaced is the longest command a tool may say it replaces: a command's name and the words that
|
||||||
|
// make it this command, never a script.
|
||||||
|
const MaxReplaced = 80
|
||||||
|
|
||||||
|
// replacesProblems judges what a module says its tools replace (novox/hq ADR 0245): each key a tool it
|
||||||
|
// declares, each entry a command line of one line, short, and said once.
|
||||||
|
func replacesProblems(m Manifest) []string {
|
||||||
|
var problems []string
|
||||||
|
tools := map[string]bool{}
|
||||||
|
for _, t := range m.Tools {
|
||||||
|
tools[t] = true
|
||||||
|
}
|
||||||
|
keys := make([]string, 0, len(m.Replaces))
|
||||||
|
for k := range m.Replaces {
|
||||||
|
keys = append(keys, k)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
for _, tool := range keys {
|
||||||
|
if !tools[tool] {
|
||||||
|
problems = append(problems, fmt.Sprintf("%s says what %q replaces, and declares no such tool: "+
|
||||||
|
"a module says it for a tool in its `tools`; a seat's verb says it in the seat's definition "+
|
||||||
|
"(novox/hq ADR 0245)", m.Module, tool))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(m.Replaces[tool]) == 0 {
|
||||||
|
problems = append(problems, fmt.Sprintf("%s says %s replaces nothing: leave it out", m.Module, tool))
|
||||||
|
}
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for _, c := range m.Replaces[tool] {
|
||||||
|
c = strings.TrimSpace(c)
|
||||||
|
switch {
|
||||||
|
case c == "":
|
||||||
|
problems = append(problems, fmt.Sprintf("%s: %s replaces an empty command", m.Module, tool))
|
||||||
|
case strings.ContainsAny(c, "\n\r"):
|
||||||
|
problems = append(problems, fmt.Sprintf("%s: %s replaces %q, which is more than one line", m.Module, tool, c))
|
||||||
|
case len(c) > MaxReplaced:
|
||||||
|
problems = append(problems, fmt.Sprintf("%s: %s replaces %q, longer than %d characters: "+
|
||||||
|
"the command's name and the words that make it this command", m.Module, tool, c, MaxReplaced))
|
||||||
|
case seen[c]:
|
||||||
|
problems = append(problems, fmt.Sprintf("%s: %s replaces %q twice", m.Module, tool, c))
|
||||||
|
}
|
||||||
|
seen[c] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return problems
|
||||||
|
}
|
||||||
|
|
||||||
// invokeProblems judges what a module says it calls (novox/hq ADR 0152).
|
// invokeProblems judges what a module says it calls (novox/hq ADR 0152).
|
||||||
//
|
//
|
||||||
// Refused here, in the manifest's words, rather than at the next composition of the bus's user
|
// Refused here, in the manifest's words, rather than at the next composition of the bus's user
|
||||||
|
|||||||
@@ -0,0 +1,124 @@
|
|||||||
|
package catalogue
|
||||||
|
|
||||||
|
// The machine's own resolver (novox/hq ADR 0247).
|
||||||
|
//
|
||||||
|
// **Most machines have none.** Every machine lists the mesh's resolvers in /etc/resolv.conf and nothing
|
||||||
|
// else, and the module holding its uplink writes that file (ADR 0223). A machine with a VPN client that
|
||||||
|
// pushes its own resolvers for its own domains needs a third answer: those domains to the VPN's servers,
|
||||||
|
// over the VPN's link, and every other name to the mesh's resolvers as before. One file cannot list both
|
||||||
|
// sets of servers — musl takes the first reply, glibc the first server's "no such name" — so the domains
|
||||||
|
// are routed by a resolver on the machine itself: the `node-resolver` seat's holder.
|
||||||
|
//
|
||||||
|
// **Where it is held, it owns the resolver file.** The file then names the machine's own resolver, which
|
||||||
|
// routes; the uplink's holder steps back from writing it on that machine, by the rule below and not by
|
||||||
|
// two modules writing one path. There are two uplink holders (NetworkManager's and systemd-networkd's),
|
||||||
|
// and the resolver is its own module so it is written once, not once in each.
|
||||||
|
//
|
||||||
|
// **It knows nothing of any VPN.** Its verbs route a set of domains to a set of servers over one link,
|
||||||
|
// list what is routed and remove a route. A module wrapping a VPN client that writes /etc/resolv.conf
|
||||||
|
// itself carries its own adapter and calls those verbs; a VPN that tells systemd-resolved its link's DNS
|
||||||
|
// itself needs none.
|
||||||
|
|
||||||
|
// ResolverSeat is the machine's own resolver (novox/hq ADR 0247).
|
||||||
|
const ResolverSeat = "node-resolver"
|
||||||
|
|
||||||
|
// ResolverFile is the machine's resolver file: the uplink holder's, or the node-resolver holder's where
|
||||||
|
// one is held.
|
||||||
|
const ResolverFile = "/etc/resolv.conf"
|
||||||
|
|
||||||
|
// resolverVerbs is the contract every holder of node-resolver serves (novox/hq ADR 0247): what is routed
|
||||||
|
// where, route a set of domains, and take a route away. The same verbs are served on the machine itself to
|
||||||
|
// the modules there, so what a VPN pushed never crosses the bus to be routed; on the mesh they are the
|
||||||
|
// operator's way to read and correct the same.
|
||||||
|
func resolverVerbs() []Verb {
|
||||||
|
return []Verb{
|
||||||
|
{Name: "routes", Description: "What this machine's own resolver sends where: the mesh's resolvers, " +
|
||||||
|
"which answer every name not routed elsewhere, and each link given servers of its own with the " +
|
||||||
|
"domains routed to them. Also the resolver file's outside writes it kept, newest first: when, who " +
|
||||||
|
"wrote it as far as the file says, whether a module took it, and when the resolver's own file was " +
|
||||||
|
"put back.",
|
||||||
|
Input: schema(map[string]string{}, nil),
|
||||||
|
Replaces: []string{"resolvectl status", "resolvectl dns", "resolvectl domain"}},
|
||||||
|
{Name: "route", Description: "Send these domains, and every name under them, to these servers over " +
|
||||||
|
"this link — and only them: the link is never the machine's default route for names, and the " +
|
||||||
|
"mesh's own domain is refused. Replaces whatever the link was given before. A link that goes away " +
|
||||||
|
"takes its route with it.",
|
||||||
|
Input: schema(map[string]string{
|
||||||
|
"link": "the network link the servers are reached over, by name (a VPN's tunnel interface)",
|
||||||
|
"domains": "the domains to route there, separated by spaces or commas",
|
||||||
|
"servers": "the servers' addresses, separated by spaces or commas",
|
||||||
|
}, []string{"link", "domains", "servers"}),
|
||||||
|
Replaces: []string{"resolvectl dns", "resolvectl domain", "resolvectl default-route"}},
|
||||||
|
{Name: "unroute", Description: "Take one link's route away: its domains go back to the mesh's " +
|
||||||
|
"resolvers. Nothing changes when the link has none.",
|
||||||
|
Input: schema(map[string]string{"link": "the network link, by name"}, []string{"link"}),
|
||||||
|
Replaces: []string{"resolvectl revert"}},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// holdsSeat says whether a module claims a seat, by its current name.
|
||||||
|
func holdsSeat(m Manifest, seat string) bool {
|
||||||
|
for _, c := range m.Claims {
|
||||||
|
if canonicalSeat(c.Name) == seat {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// rendersResolverFile says whether a module asks the mesh to render the machine's resolver file.
|
||||||
|
func rendersResolverFile(m Manifest) bool {
|
||||||
|
for _, f := range m.Facts {
|
||||||
|
if !f.Home && f.Path == ResolverFile {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResolverFileOwner is the module that writes the machine's resolver file among the modules on one
|
||||||
|
// machine (novox/hq ADR 0247): the holder of node-resolver when one renders it, else nobody is named here
|
||||||
|
// and the file is whoever's it always was — the uplink holder's (ADR 0223).
|
||||||
|
func ResolverFileOwner(modules []Manifest) string {
|
||||||
|
for _, m := range modules {
|
||||||
|
if holdsSeat(m, ResolverSeat) && rendersResolverFile(m) {
|
||||||
|
return m.Module
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// stepsBack is the module as it composes on a machine whose resolver file is the node-resolver holder's:
|
||||||
|
// **the uplink holder's rendering of that file is left out**, and nothing else of it changes. Only the
|
||||||
|
// uplink's holder steps back — any other module rendering the file beside the resolver's is still two
|
||||||
|
// owners of one path, and is refused as before.
|
||||||
|
func stepsBack(m Manifest, modules []Manifest) Manifest {
|
||||||
|
if !holdsSeat(m, "node-uplink") || !rendersResolverFile(m) {
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
owner := ResolverFileOwner(modules)
|
||||||
|
if owner == "" || owner == m.Module {
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
facts := make(map[string]RosterFile, len(m.Facts))
|
||||||
|
for name, f := range m.Facts {
|
||||||
|
if !f.Home && f.Path == ResolverFile {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
facts[name] = f
|
||||||
|
}
|
||||||
|
m.Facts = facts
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
// steppedBack is every module of one machine as it composes there (stepsBack, each).
|
||||||
|
func steppedBack(modules []Manifest) []Manifest {
|
||||||
|
if ResolverFileOwner(modules) == "" {
|
||||||
|
return modules
|
||||||
|
}
|
||||||
|
out := make([]Manifest, len(modules))
|
||||||
|
for i, m := range modules {
|
||||||
|
out[i] = stepsBack(m, modules)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,216 @@
|
|||||||
|
package catalogue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Defends novox/hq ADR 0247: a machine's own resolver is a node seat, held only where something requires
|
||||||
|
// `split-dns`; where it is held it writes the resolver file and the uplink's holder steps back from it.
|
||||||
|
|
||||||
|
// The seat: node-scoped, decided by ADR 0247, and its three verbs required of every holder — a holder
|
||||||
|
// exists only once the module serving them does, so nothing has to be optional while it catches up.
|
||||||
|
func TestTheMachinesOwnResolverIsANodeSeatWithItsVerbs(t *testing.T) {
|
||||||
|
s, ok := SeatNamed(ResolverSeat)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("%s is not in the mesh's set", ResolverSeat)
|
||||||
|
}
|
||||||
|
if s.Scope != ScopeNode || s.Decision != "novox/hq ADR 0247" || s.Delivers != "" || s.Replicated {
|
||||||
|
t.Errorf("%s is %+v; a node seat under ADR 0247 that delivers nothing", ResolverSeat, s)
|
||||||
|
}
|
||||||
|
var got []string
|
||||||
|
for _, v := range s.Serves {
|
||||||
|
got = append(got, v.Name)
|
||||||
|
if v.Optional {
|
||||||
|
t.Errorf("%s.%s is optional; its first holder serves it", ResolverSeat, v.Name)
|
||||||
|
}
|
||||||
|
if v.Description == "" || v.Input["type"] != "object" || len(v.Replaces) == 0 {
|
||||||
|
t.Errorf("%s.%s has no description, no object schema or says it replaces nothing", ResolverSeat, v.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Join(got, " ") != "routes route unroute" {
|
||||||
|
t.Errorf("%s serves %v, not routes, route and unroute", ResolverSeat, got)
|
||||||
|
}
|
||||||
|
// The verbs name a link, domains and servers, and never a VPN: the resolver knows nothing of one.
|
||||||
|
for _, v := range s.Serves {
|
||||||
|
if strings.Contains(strings.ToLower(v.Description), "forti") {
|
||||||
|
t.Errorf("%s.%s names a VPN client: %s", ResolverSeat, v.Name, v.Description)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// localResolver is a stand-in holder: it claims the seat and renders the resolver file naming the
|
||||||
|
// machine's own address. What is under test is the controller's rule, not the catalogue's module.
|
||||||
|
func localResolver() Manifest {
|
||||||
|
return Manifest{Module: "local-resolver", Version: "1",
|
||||||
|
Claims: []Claim{{Name: ResolverSeat, Scope: ScopeNode}},
|
||||||
|
Facts: map[string]RosterFile{"resolvers": {Path: ResolverFile,
|
||||||
|
Template: "# Managed by the mesh\n{{range .Machines}}{{if eq .Name $.Node}}nameserver {{.Address}}\n{{end}}{{end}}"}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func splitDNSLaptop() Node {
|
||||||
|
return Node{Name: "laptop", At: "laptop.internal", Capabilities: map[string]bool{
|
||||||
|
"package-manager": true, "service-manager": true, "uplink-systemd-networkd": true}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Where a module holds node-resolver, the machine is composed one resolver file, the holder's, naming
|
||||||
|
// the machine's own address; the uplink's holder composes everything else it declares, and not that file.
|
||||||
|
func TestWhereTheResolverIsHeldItWritesTheFileAndTheUplinkStepsBack(t *testing.T) {
|
||||||
|
shelf := resolverShelf(t)
|
||||||
|
shelf["local-resolver"] = localResolver()
|
||||||
|
got, err := Resolve(shelf, []string{"dnsmasq", "systemd-networkd", "local-resolver"}, splitDNSLaptop(), World{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("the resolver's holder and the uplink's were refused together: %v", err)
|
||||||
|
}
|
||||||
|
out, err := got.Declaration(Rendering{Names: twoMachines, Machines: twoMachines, Suffix: "internal",
|
||||||
|
Holders: map[string]map[string]string{"mesh-dns-resolver": {"anchor.internal": "10.42.0.1"}},
|
||||||
|
Needed: map[string]map[string]string{"dnsmasq": {"broker": "sealed"}},
|
||||||
|
Settings: SettingsBy{"dnsmasq": {{From: "the mesh", Values: map[string]any{"listen-addresses": "127.0.0.1"}}}},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var files []string
|
||||||
|
for _, r := range out {
|
||||||
|
if r["path"] == ResolverFile {
|
||||||
|
files = append(files, r["id"].(string))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Join(files, " ") != "local-resolver.fact-resolvers" {
|
||||||
|
t.Fatalf("the resolver file is composed as %v; once, the resolver's", files)
|
||||||
|
}
|
||||||
|
content := byID(out)["local-resolver.fact-resolvers"]["content"].(string)
|
||||||
|
if !strings.Contains(content, "nameserver 10.42.0.2\n") || strings.Contains(content, "10.42.0.1") {
|
||||||
|
t.Errorf("the resolver file does not name this machine's own resolver alone:\n%s", content)
|
||||||
|
}
|
||||||
|
// The uplink's holder is still composed: only the one file moved.
|
||||||
|
uplinkComposed := false
|
||||||
|
for _, r := range out {
|
||||||
|
if id, _ := r["id"].(string); strings.HasPrefix(id, "systemd-networkd.") {
|
||||||
|
uplinkComposed = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !uplinkComposed {
|
||||||
|
t.Errorf("the uplink's holder composed nothing once the resolver was held")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Where nobody holds it, nothing changes: the uplink's holder writes the file, listing the mesh's
|
||||||
|
// resolvers (ADR 0223) — every machine but the one that requires split-dns.
|
||||||
|
func TestWithoutTheResolverTheUplinkWritesTheFileAsBefore(t *testing.T) {
|
||||||
|
got, err := Resolve(resolverShelf(t), []string{"dnsmasq", "systemd-networkd"}, splitDNSLaptop(), World{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
out, err := got.Declaration(Rendering{Names: twoMachines, Machines: twoMachines, Suffix: "internal",
|
||||||
|
Holders: map[string]map[string]string{"mesh-dns-resolver": {"anchor.internal": "10.42.0.1"}},
|
||||||
|
Needed: map[string]map[string]string{"dnsmasq": {"broker": "sealed"}},
|
||||||
|
Settings: SettingsBy{"dnsmasq": {{From: "the mesh", Values: map[string]any{"listen-addresses": "127.0.0.1"}}}},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if r := byID(out)["systemd-networkd.fact-resolvers"]; r == nil || r["path"] != ResolverFile {
|
||||||
|
t.Fatalf("without the resolver, the uplink's holder no longer writes the resolver file: %v", r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only the uplink's holder steps back. A third module rendering or declaring the file beside the
|
||||||
|
// resolver's is two owners of one path, refused as before; and a module rendering it without holding
|
||||||
|
// the seat is not the resolver, so the uplink's holder does not step back for it.
|
||||||
|
func TestOnlyTheUplinkStepsBackForTheResolver(t *testing.T) {
|
||||||
|
uplink := Manifest{Module: "uplink", Version: "1", Claims: []Claim{{Name: "node-uplink"}},
|
||||||
|
Facts: map[string]RosterFile{"resolvers": {Path: ResolverFile, Template: "nameserver 10.42.0.1\n"}}}
|
||||||
|
if p := checkResources([]Manifest{uplink, localResolver()}); len(p) != 0 {
|
||||||
|
t.Errorf("the uplink's holder and the resolver's were refused together: %v", p)
|
||||||
|
}
|
||||||
|
other := Manifest{Module: "other", Version: "1",
|
||||||
|
Facts: map[string]RosterFile{"mine": {Path: ResolverFile, Template: "nameserver 10.42.0.9\n"}}}
|
||||||
|
if p := checkResources([]Manifest{uplink, localResolver(), other}); len(p) == 0 {
|
||||||
|
t.Error("a third module wrote the resolver file beside the resolver's")
|
||||||
|
}
|
||||||
|
if p := checkResources([]Manifest{uplink, other}); len(p) == 0 {
|
||||||
|
t.Error("a module that does not hold node-resolver took the resolver file from the uplink's holder")
|
||||||
|
}
|
||||||
|
declared := Manifest{Module: "declared", Version: "1", Resources: []map[string]any{
|
||||||
|
{"id": "mine", "type": "file", "path": ResolverFile, "content": "nameserver 10.42.0.9\n"}}}
|
||||||
|
if p := checkResources([]Manifest{uplink, localResolver(), declared}); len(p) == 0 {
|
||||||
|
t.Error("a module declaring the resolver file was let beside the resolver's")
|
||||||
|
}
|
||||||
|
// What steps back is the one file: the uplink's other facts stay.
|
||||||
|
uplink.Facts["hosts"] = RosterFile{Path: "/etc/elsewhere", Template: "x"}
|
||||||
|
if got := stepsBack(uplink, []Manifest{uplink, localResolver()}); len(got.Facts) != 1 || got.Facts["hosts"].Path == "" {
|
||||||
|
t.Errorf("the uplink's holder lost more than the resolver file: %v", got.Facts)
|
||||||
|
}
|
||||||
|
if got := stepsBack(uplink, []Manifest{uplink}); len(got.Facts) != 2 {
|
||||||
|
t.Errorf("the uplink's holder stepped back with no resolver held: %v", got.Facts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The catalogue as it is: every holder of node-resolver renders the resolver file as the mesh's own
|
||||||
|
// (its header is how the uplink's verb and the node-engine read a file as the mesh's), and provides
|
||||||
|
// split-dns at the machine's reach — a requirement is answered only on the same machine and never pulls
|
||||||
|
// the resolver in. Skipped while the catalogue has no holder.
|
||||||
|
func TestTheCataloguesResolverHoldersWriteTheFileAndProvideSplitDNS(t *testing.T) {
|
||||||
|
held := 0
|
||||||
|
for _, m := range theCatalogue(t) {
|
||||||
|
if !holdsSeat(m, ResolverSeat) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
held++
|
||||||
|
f, ok := m.Facts["resolvers"]
|
||||||
|
if !ok || f.Path != ResolverFile || !strings.HasPrefix(f.Template, "# Managed by the mesh") {
|
||||||
|
t.Errorf("%s holds %s and does not render the resolver file as the mesh's: %+v", m.Module, ResolverSeat, f)
|
||||||
|
}
|
||||||
|
provides := false
|
||||||
|
for _, o := range m.Provides {
|
||||||
|
if o.Name == "split-dns" && o.Reach == ReachMachine {
|
||||||
|
provides = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !provides {
|
||||||
|
t.Errorf("%s holds %s and does not provide split-dns at the machine's reach", m.Module, ResolverSeat)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if held == 0 {
|
||||||
|
t.Skip("no module of the catalogue holds node-resolver yet")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The catalogue's systemd-resolved, composed on a machine beside its uplink: the resolver file names the
|
||||||
|
// machine's own private address alone, its kept copy is the same file, and resolved is given every mesh
|
||||||
|
// resolver as its default route and the private address to listen on. Skipped without the catalogue.
|
||||||
|
func TestTheCataloguesResolverComposesOnAMachine(t *testing.T) {
|
||||||
|
shelf := resolverShelf(t)
|
||||||
|
shelf["systemd-resolved"] = catalogueManifest(t, "systemd-resolved")
|
||||||
|
got, err := Resolve(shelf, []string{"dnsmasq", "systemd-networkd", "systemd-resolved"}, splitDNSLaptop(), World{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
out, err := got.Declaration(Rendering{Names: twoMachines, Machines: twoMachines, Suffix: "internal",
|
||||||
|
Holders: map[string]map[string]string{"mesh-dns-resolver": {"anchor.internal": "10.42.0.1", "home.internal": "10.42.0.3"}},
|
||||||
|
Needed: map[string]map[string]string{"dnsmasq": {"broker": "sealed"}},
|
||||||
|
Settings: SettingsBy{"dnsmasq": {{From: "the mesh", Values: map[string]any{"listen-addresses": "127.0.0.1"}}}},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ids := byID(out)
|
||||||
|
resolv, _ := ids["systemd-resolved.fact-resolvers"]["content"].(string)
|
||||||
|
kept, _ := ids["systemd-resolved.fact-kept"]["content"].(string)
|
||||||
|
if !strings.Contains(resolv, "\nnameserver 10.42.0.2\noptions timeout:1 attempts:2 edns0\n") || resolv != kept {
|
||||||
|
t.Errorf("the resolver file does not name this machine's own resolver alone, or its copy differs:\n%s", resolv)
|
||||||
|
}
|
||||||
|
if ids["systemd-networkd.fact-resolvers"] != nil {
|
||||||
|
t.Error("the uplink's holder still writes the resolver file beside the resolver's")
|
||||||
|
}
|
||||||
|
conf, _ := ids["systemd-resolved.fact-resolved"]["content"].(string)
|
||||||
|
for _, want := range []string{"\nDNS=10.42.0.1 10.42.0.3 \n", "\nDNSStubListenerExtra=10.42.0.2\n", "\nFallbackDNS=\n"} {
|
||||||
|
if !strings.Contains(conf, want) {
|
||||||
|
t.Errorf("resolved's drop-in lacks %q:\n%s", want, conf)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if s, _ := ids["systemd-resolved.fact-suffix"]["content"].(string); s != "internal\n" {
|
||||||
|
t.Errorf("the mesh's domain is rendered as %q", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -42,7 +42,7 @@ func TestTheServiceManagerSeatServesTheUnitVerbs(t *testing.T) {
|
|||||||
if seat.Scope != ScopeNode {
|
if seat.Scope != ScopeNode {
|
||||||
t.Fatalf("the service manager is a role each machine has once, and the seat is %s-scoped", seat.Scope)
|
t.Fatalf("the service manager is a role each machine has once, and the seat is %s-scoped", seat.Scope)
|
||||||
}
|
}
|
||||||
want := []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal"}
|
want := []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal", "failed"}
|
||||||
var got []string
|
var got []string
|
||||||
for _, v := range seat.Serves {
|
for _, v := range seat.Serves {
|
||||||
got = append(got, v.Name)
|
got = append(got, v.Name)
|
||||||
@@ -58,3 +58,53 @@ func TestTheServiceManagerSeatServesTheUnitVerbs(t *testing.T) {
|
|||||||
t.Fatalf("the seat serves %v, not %v", got, want)
|
t.Fatalf("the seat serves %v, not %v", got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **`failed` joins the seat optional** (the operator's direction 2026-10-07): the holder running today,
|
||||||
|
// which does not serve it, still holds the seat; the holder that serves it is not refused; and a verb
|
||||||
|
// the seat does not promise is still refused, and one it requires is still required.
|
||||||
|
func TestFailedIsAnOptionalVerbOfTheServiceManager(t *testing.T) {
|
||||||
|
seat, _ := SeatNamed(ServiceManagerSeat)
|
||||||
|
holder := func(serves ...string) Manifest {
|
||||||
|
return Manifest{Module: "systemd", Version: "1", Claims: []Claim{{Name: ServiceManagerSeat, Scope: ScopeNode, Serves: serves}}}
|
||||||
|
}
|
||||||
|
eight := []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal"}
|
||||||
|
if err := CanHold(holder(eight...), seat); err != nil {
|
||||||
|
t.Fatalf("today's holder, without failed, is refused: %v", err)
|
||||||
|
}
|
||||||
|
if err := CanHold(holder(append(eight, "failed")...), seat); err != nil {
|
||||||
|
t.Fatalf("a holder serving failed is refused: %v", err)
|
||||||
|
}
|
||||||
|
if err := CanHold(holder(append(eight, "fail")...), seat); err == nil || !strings.Contains(err.Error(), "does not promise") {
|
||||||
|
t.Fatalf("a verb the seat does not promise was accepted: %v", err)
|
||||||
|
}
|
||||||
|
if err := CanHold(holder(eight[1:]...), seat); err == nil || !strings.Contains(err.Error(), "does not serve units") {
|
||||||
|
t.Fatalf("a holder missing a required verb was accepted: %v", err)
|
||||||
|
}
|
||||||
|
for _, v := range seat.Serves {
|
||||||
|
if v.Optional != (v.Name == "failed") {
|
||||||
|
t.Errorf("%s optional: %v", v.Name, v.Optional)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The optional mark is not stored, so a seat set read back from the store's rows takes it from the
|
||||||
|
// compiled seat: otherwise `failed`, seeded into the row, would come back required.
|
||||||
|
func TestAnOptionalVerbStaysOptionalInASetReadFromTheStore(t *testing.T) {
|
||||||
|
defer UseSeats(DefaultSeats())
|
||||||
|
var rows []Seat
|
||||||
|
for _, s := range DefaultSeats() {
|
||||||
|
row := s
|
||||||
|
row.Serves = nil
|
||||||
|
for _, v := range s.Serves {
|
||||||
|
row.Serves = append(row.Serves, Verb{Name: v.Name, Description: v.Description, Input: v.Input})
|
||||||
|
}
|
||||||
|
rows = append(rows, row)
|
||||||
|
}
|
||||||
|
UseSeats(rows)
|
||||||
|
seat, _ := SeatNamed(ServiceManagerSeat)
|
||||||
|
holder := Manifest{Module: "systemd", Version: "1", Claims: []Claim{{Name: ServiceManagerSeat, Scope: ScopeNode,
|
||||||
|
Serves: []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal"}}}}
|
||||||
|
if err := CanHold(holder, seat); err != nil {
|
||||||
|
t.Fatalf("a set read from rows refuses today's holder: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,111 @@
|
|||||||
|
package catalogue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"reflect"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What a move does to a module's containers (novox/hq ADR 0245).
|
||||||
|
//
|
||||||
|
// A container whose declaration changes is recreated, whatever changed in it: an image, an environment
|
||||||
|
// variable, a health check adopted. A module whose containers are recreated in one send is down while
|
||||||
|
// they start again — on 2026-10-07 a catalogue change that only adopted the images' own health checks
|
||||||
|
// recreated nine of mail's containers at once, and the operator's phone could not reach the mail. So a
|
||||||
|
// send says, per module, how many of its containers it recreates and whether any image changed, before
|
||||||
|
// it is sent and in the plan that sent it.
|
||||||
|
|
||||||
|
// Recreation is what moving a module from one build's manifest to another's does to its containers.
|
||||||
|
type Recreation struct {
|
||||||
|
// Containers is how many containers the new build declares.
|
||||||
|
Containers int
|
||||||
|
// Recreated are the ids of the containers whose declaration differs — changed, added or gone —
|
||||||
|
// sorted.
|
||||||
|
Recreated []string
|
||||||
|
// Images are the ids among them whose image changed (or that are added or gone).
|
||||||
|
Images []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recreates compares two builds of a module, container by container, by resource id.
|
||||||
|
func Recreates(from, to Manifest) Recreation {
|
||||||
|
before := containersByID(from)
|
||||||
|
after := containersByID(to)
|
||||||
|
var r Recreation
|
||||||
|
r.Containers = len(after)
|
||||||
|
for id, now := range after {
|
||||||
|
was, held := before[id]
|
||||||
|
switch {
|
||||||
|
case !held:
|
||||||
|
r.Recreated = append(r.Recreated, id)
|
||||||
|
r.Images = append(r.Images, id)
|
||||||
|
case !sameDeclaration(was, now):
|
||||||
|
r.Recreated = append(r.Recreated, id)
|
||||||
|
if !sameDeclaration(was["image"], now["image"]) {
|
||||||
|
r.Images = append(r.Images, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for id := range before {
|
||||||
|
if _, kept := after[id]; !kept {
|
||||||
|
r.Recreated = append(r.Recreated, id)
|
||||||
|
r.Images = append(r.Images, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(r.Recreated)
|
||||||
|
sort.Strings(r.Images)
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// SpecOnly is whether the move recreates containers without any new image: a change to how they are
|
||||||
|
// declared only — a health check adopted, a variable — which a person may well not expect to interrupt.
|
||||||
|
func (r Recreation) SpecOnly() bool { return len(r.Recreated) > 0 && len(r.Images) == 0 }
|
||||||
|
|
||||||
|
// Say is the recreation in the mesh's words, for a module: empty when the move recreates nothing.
|
||||||
|
func (r Recreation) Say(module string) string {
|
||||||
|
if len(r.Recreated) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
what := "with a new image"
|
||||||
|
switch {
|
||||||
|
case r.SpecOnly():
|
||||||
|
what = "no new image, only their declaration"
|
||||||
|
case len(r.Images) < len(r.Recreated):
|
||||||
|
what = fmt.Sprintf("%d with a new image", len(r.Images))
|
||||||
|
}
|
||||||
|
whole := ""
|
||||||
|
if len(r.Recreated) > 1 && len(r.Recreated) >= r.Containers {
|
||||||
|
whole = ", every one at once: its service is interrupted until they are up again"
|
||||||
|
} else if len(r.Recreated) > 1 {
|
||||||
|
whole = ", at once: what they serve is interrupted until they are up again"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("recreates %d of %s's %d container(s) (%s: %s)%s", len(r.Recreated), module, r.Containers,
|
||||||
|
what, strings.Join(r.Recreated, ", "), whole)
|
||||||
|
}
|
||||||
|
|
||||||
|
func containersByID(m Manifest) map[string]map[string]any {
|
||||||
|
out := map[string]map[string]any{}
|
||||||
|
for _, r := range m.Resources {
|
||||||
|
if t, _ := r["type"].(string); t != "container" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
id, _ := r["id"].(string)
|
||||||
|
out[id] = r
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// sameDeclaration compares two declarations as JSON, so a number read as an int and one read as a
|
||||||
|
// float are one value.
|
||||||
|
func sameDeclaration(a, b any) bool {
|
||||||
|
ra, errA := json.Marshal(a)
|
||||||
|
rb, errB := json.Marshal(b)
|
||||||
|
if errA != nil || errB != nil {
|
||||||
|
return reflect.DeepEqual(a, b)
|
||||||
|
}
|
||||||
|
var na, nb any
|
||||||
|
_ = json.Unmarshal(ra, &na)
|
||||||
|
_ = json.Unmarshal(rb, &nb)
|
||||||
|
return reflect.DeepEqual(na, nb)
|
||||||
|
}
|
||||||
@@ -0,0 +1,119 @@
|
|||||||
|
package catalogue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The verbs an agent works around most say what they replace (novox/hq ADR 0245): the journal, a
|
||||||
|
// restart, a unit's status, the hosts file — each the command the agent would otherwise type over ssh.
|
||||||
|
func TestTheVerbsWorkedAroundSayWhatTheyReplace(t *testing.T) {
|
||||||
|
want := map[string]string{
|
||||||
|
ServiceManagerSeat + ".journal": "journalctl",
|
||||||
|
ServiceManagerSeat + ".restart": "systemctl restart",
|
||||||
|
ServiceManagerSeat + ".status": "systemctl status",
|
||||||
|
"node-hostname.add": "edit /etc/hosts",
|
||||||
|
ControllerSeatName + ".node": "hostnamectl",
|
||||||
|
}
|
||||||
|
for _, s := range DefaultSeats() {
|
||||||
|
for _, v := range s.Serves {
|
||||||
|
cmd, ok := want[s.Name+"."+v.Name]
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
delete(want, s.Name+"."+v.Name)
|
||||||
|
found := false
|
||||||
|
for _, r := range v.Replaces {
|
||||||
|
found = found || r == cmd
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Errorf("%s.%s does not say it replaces %q: %v", s.Name, v.Name, cmd, v.Replaces)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for verb := range want {
|
||||||
|
t.Errorf("%s is not a verb of the compiled seats", verb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every command a seat's verb replaces is one short line, said once — what the guard and the search match.
|
||||||
|
func TestEveryReplacedCommandIsOneShortLine(t *testing.T) {
|
||||||
|
for _, s := range DefaultSeats() {
|
||||||
|
for _, v := range s.Serves {
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for _, r := range v.Replaces {
|
||||||
|
if strings.TrimSpace(r) == "" || strings.ContainsAny(r, "\n\r") || len(r) > MaxReplaced || seen[r] {
|
||||||
|
t.Errorf("%s.%s replaces %q, which is not one short line said once", s.Name, v.Name, r)
|
||||||
|
}
|
||||||
|
seen[r] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A verb's definition in a manifest may say what it replaces, like the mesh's own.
|
||||||
|
func TestAVerbDefinitionCarriesWhatItReplaces(t *testing.T) {
|
||||||
|
m, err := ParseManifest([]byte(`{"module":"till","version":"1","tools":["refund"],` +
|
||||||
|
`"seats":[{"name":"shop-till","serves":[{"name":"refund","replaces":["psql -c refund"]}]}],` +
|
||||||
|
`"claims":[{"name":"shop-till","scope":"mesh"}]}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := m.DefinesSeats[0].Serves[0].Replaces; len(got) != 1 || got[0] != "psql -c refund" {
|
||||||
|
t.Fatalf("replaces not read: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A module says what its own tools replace, and only for a tool it declares.
|
||||||
|
func TestAModuleSaysWhatItsOwnToolsReplace(t *testing.T) {
|
||||||
|
m, err := ParseManifest([]byte(`{"module":"docker","version":"1","tools":["docker_logs","docker_list"],` +
|
||||||
|
`"replaces":{"docker_logs":["docker logs"],"docker_list":["docker ps"]}}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if m.Replaces["docker_logs"][0] != "docker logs" || m.Replaces["docker_list"][0] != "docker ps" {
|
||||||
|
t.Fatalf("replaces not read: %v", m.Replaces)
|
||||||
|
}
|
||||||
|
for _, c := range []struct{ manifest, says string }{
|
||||||
|
{`{"module":"docker","version":"1","tools":["docker_logs"],"replaces":{"docker_lgos":["docker logs"]}}`,
|
||||||
|
"declares no such tool"},
|
||||||
|
{`{"module":"docker","version":"1","tools":["docker_logs"],"replaces":{"docker_logs":[]}}`, "replaces nothing"},
|
||||||
|
{`{"module":"docker","version":"1","tools":["docker_logs"],"replaces":{"docker_logs":[" "]}}`, "empty command"},
|
||||||
|
{`{"module":"docker","version":"1","tools":["docker_logs"],"replaces":{"docker_logs":["docker logs\nrm -rf /"]}}`,
|
||||||
|
"more than one line"},
|
||||||
|
{`{"module":"docker","version":"1","tools":["docker_logs"],"replaces":{"docker_logs":["docker logs","docker logs"]}}`,
|
||||||
|
"twice"},
|
||||||
|
{`{"module":"docker","version":"1","tools":["docker_logs"],"replaces":{"docker_logs":["` + strings.Repeat("x", MaxReplaced+1) + `"]}}`,
|
||||||
|
"longer than"},
|
||||||
|
} {
|
||||||
|
if _, err := ParseManifest([]byte(c.manifest)); err == nil || !strings.Contains(err.Error(), c.says) {
|
||||||
|
t.Errorf("%s: want a refusal saying %q, got %v", c.manifest, c.says, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A seat row the store seeded before `replaces` existed carries none; the working set takes the compiled
|
||||||
|
// one, so the `tools` answer says it on a mesh whose rows exist.
|
||||||
|
func TestASeededRowTakesWhatItsVerbsReplaceFromTheCompiledSeat(t *testing.T) {
|
||||||
|
was := Seats()
|
||||||
|
t.Cleanup(func() { UseSeats(was) })
|
||||||
|
var row Seat
|
||||||
|
for _, s := range DefaultSeats() {
|
||||||
|
if s.Name == ServiceManagerSeat {
|
||||||
|
row = s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
stored := make([]Verb, len(row.Serves))
|
||||||
|
for i, v := range row.Serves {
|
||||||
|
v.Replaces = nil
|
||||||
|
stored[i] = v
|
||||||
|
}
|
||||||
|
row.Serves = stored
|
||||||
|
UseSeats([]Seat{row})
|
||||||
|
got, _ := SeatNamed(ServiceManagerSeat)
|
||||||
|
for _, v := range got.Serves {
|
||||||
|
if v.Name == "journal" && (len(v.Replaces) == 0 || v.Replaces[0] != "journalctl") {
|
||||||
|
t.Fatalf("the seeded journal verb says it replaces %v", v.Replaces)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -930,6 +930,9 @@ func canonicalSeat(name string) string {
|
|||||||
// resolves with no refusal. What is refused is the contradiction — a path one module owns and
|
// resolves with no refusal. What is refused is the contradiction — a path one module owns and
|
||||||
// another merely accesses — because shared data is the operator's and nobody's to own.
|
// another merely accesses — because shared data is the operator's and nobody's to own.
|
||||||
func checkResources(modules []Manifest) []string {
|
func checkResources(modules []Manifest) []string {
|
||||||
|
// Judged as the machine composes them: where the node-resolver holder writes the resolver file, the
|
||||||
|
// uplink holder's rendering of it is not composed, so it is not a second owner (novox/hq ADR 0247).
|
||||||
|
modules = steppedBack(modules)
|
||||||
var problems []string
|
var problems []string
|
||||||
owner := map[string]string{}
|
owner := map[string]string{}
|
||||||
ownedPath := map[string]string{} // path → owning module, for the access check below
|
ownedPath := map[string]string{} // path → owning module, for the access check below
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ func TestTheResolverConfigSeatIsGone(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// The catalogue as it is: nothing claims the retired seat, and every manager the mesh knows holds the
|
// The catalogue as it is: nothing claims the retired seat, and every manager the mesh knows holds the
|
||||||
// uplink and writes the resolver file.
|
// uplink and writes the resolver file; beside them only a holder of node-resolver does (ADR 0247).
|
||||||
func TestTheCataloguesUplinksWriteTheResolverFileAndNothingElseDoes(t *testing.T) {
|
func TestTheCataloguesUplinksWriteTheResolverFileAndNothingElseDoes(t *testing.T) {
|
||||||
cat := map[string]Manifest{}
|
cat := map[string]Manifest{}
|
||||||
for _, m := range theCatalogue(t) {
|
for _, m := range theCatalogue(t) {
|
||||||
@@ -41,8 +41,10 @@ func TestTheCataloguesUplinksWriteTheResolverFileAndNothingElseDoes(t *testing.T
|
|||||||
isUplink = isUplink || u == name
|
isUplink = isUplink || u == name
|
||||||
}
|
}
|
||||||
for _, f := range m.Facts {
|
for _, f := range m.Facts {
|
||||||
if f.Path == "/etc/resolv.conf" && !isUplink {
|
// Or the machine's own resolver's, where it is held (novox/hq ADR 0247): the uplink's holder
|
||||||
t.Errorf("%s writes /etc/resolv.conf and does not hold the uplink", name)
|
// steps back from the file there, and nowhere else.
|
||||||
|
if f.Path == "/etc/resolv.conf" && !isUplink && !holdsSeat(m, ResolverSeat) {
|
||||||
|
t.Errorf("%s writes /etc/resolv.conf and holds neither the uplink nor the machine's resolver", name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, r := range m.Resources {
|
for _, r := range m.Resources {
|
||||||
|
|||||||
+116
-26
@@ -102,10 +102,10 @@ var defaultSeats = append([]Seat{
|
|||||||
{Name: "mesh-store", Scope: ScopeMesh, Delivers: "postgres-database", Decision: "novox/hq ADR 0079",
|
{Name: "mesh-store", Scope: ScopeMesh, Delivers: "postgres-database", Decision: "novox/hq ADR 0079",
|
||||||
Serves: []Verb{
|
Serves: []Verb{
|
||||||
{Name: "databases", Description: "Every database the store holds, with its on-disk size.",
|
{Name: "databases", Description: "Every database the store holds, with its on-disk size.",
|
||||||
Input: schema(map[string]string{}, nil)},
|
Input: schema(map[string]string{}, nil), Replaces: []string{"psql -l"}},
|
||||||
{Name: "query", Description: "One read-only statement against one database the store holds.",
|
{Name: "query", Description: "One read-only statement against one database the store holds.",
|
||||||
Input: schema(map[string]string{"database": "the database to query", "sql": "the read-only statement"},
|
Input: schema(map[string]string{"database": "the database to query", "sql": "the read-only statement"},
|
||||||
[]string{"database", "sql"})},
|
[]string{"database", "sql"}), Replaces: []string{"psql"}},
|
||||||
}},
|
}},
|
||||||
// **Delivers the mesh's own bus, not `amqp`.** Those were the same word until
|
// **Delivers the mesh's own bus, not `amqp`.** Those were the same word until
|
||||||
// ADR 0127 separated them: `amqp` is a backing service a module may require, and this seat is
|
// ADR 0127 separated them: `amqp` is a backing service a module may require, and this seat is
|
||||||
@@ -174,15 +174,16 @@ var defaultSeats = append([]Seat{
|
|||||||
Serves: []Verb{
|
Serves: []Verb{
|
||||||
{Name: "entries", Description: "Every line of this machine's /etc/hosts, each marked whose it is: " +
|
{Name: "entries", Description: "Every line of this machine's /etc/hosts, each marked whose it is: " +
|
||||||
"the operator's, or the block of the module or tool that writes it.",
|
"the operator's, or the block of the module or tool that writes it.",
|
||||||
Input: schema(map[string]string{}, nil)},
|
Input: schema(map[string]string{}, nil), Replaces: []string{"cat /etc/hosts", "getent hosts"}},
|
||||||
{Name: "add", Description: "Add one address and its names to the operator's lines of this machine's " +
|
{Name: "add", Description: "Add one address and its names to the operator's lines of this machine's " +
|
||||||
"/etc/hosts — a name for this machine's own programs, not the mesh's.",
|
"/etc/hosts — a name for this machine's own programs, not the mesh's.",
|
||||||
Input: schema(map[string]string{"address": "the IPv4 or IPv6 address",
|
Input: schema(map[string]string{"address": "the IPv4 or IPv6 address",
|
||||||
"names": "the names for it, separated by spaces"}, []string{"address", "names"})},
|
"names": "the names for it, separated by spaces"}, []string{"address", "names"}),
|
||||||
|
Replaces: []string{"edit /etc/hosts", "HOSTALIASES"}},
|
||||||
{Name: "remove", Description: "Remove one name, or every line of one address, from the operator's " +
|
{Name: "remove", Description: "Remove one name, or every line of one address, from the operator's " +
|
||||||
"lines of this machine's /etc/hosts. A line a module writes is refused, naming the module.",
|
"lines of this machine's /etc/hosts. A line a module writes is refused, naming the module.",
|
||||||
Input: schema(map[string]string{"name": "a host name, or an address to remove every line of"},
|
Input: schema(map[string]string{"name": "a host name, or an address to remove every line of"},
|
||||||
[]string{"name"})},
|
[]string{"name"}), Replaces: []string{"sed -i /etc/hosts"}},
|
||||||
}},
|
}},
|
||||||
// The intrusion prevention's verbs (novox/hq ADR 0179): what a person asks a machine's ban list
|
// The intrusion prevention's verbs (novox/hq ADR 0179): what a person asks a machine's ban list
|
||||||
// whatever keeps it — who is banned and why, ban one address, let one go. Every holder serves all
|
// whatever keeps it — who is banned and why, ban one address, let one go. Every holder serves all
|
||||||
@@ -191,15 +192,17 @@ var defaultSeats = append([]Seat{
|
|||||||
Serves: []Verb{
|
Serves: []Verb{
|
||||||
{Name: "status", Description: "Every jail on this machine with how many it is watching and " +
|
{Name: "status", Description: "Every jail on this machine with how many it is watching and " +
|
||||||
"holding now, and the totals since the jail started; one jail's detail when named.",
|
"holding now, and the totals since the jail started; one jail's detail when named.",
|
||||||
Input: schema(map[string]string{"jail": "one jail (optional)"}, nil)},
|
Input: schema(map[string]string{"jail": "one jail (optional)"}, nil), Replaces: []string{"fail2ban-client status"}},
|
||||||
{Name: "banned", Description: "Every address banned on this machine right now, with the jail " +
|
{Name: "banned", Description: "Every address banned on this machine right now, with the jail " +
|
||||||
"that holds it and when the ban ends.",
|
"that holds it and when the ban ends.",
|
||||||
Input: schema(map[string]string{"jail": "one jail (optional)"}, nil)},
|
Input: schema(map[string]string{"jail": "one jail (optional)"}, nil), Replaces: []string{"fail2ban-client banned"}},
|
||||||
{Name: "ban", Description: "Ban one address in one jail now, for the jail's ban time — an " +
|
{Name: "ban", Description: "Ban one address in one jail now, for the jail's ban time — an " +
|
||||||
"operator's act on the live ban list, which the mesh never writes itself.",
|
"operator's act on the live ban list, which the mesh never writes itself.",
|
||||||
Input: schema(map[string]string{"ip": "the address", "jail": "the jail to hold it"}, []string{"ip", "jail"})},
|
Input: schema(map[string]string{"ip": "the address", "jail": "the jail to hold it"}, []string{"ip", "jail"}),
|
||||||
|
Replaces: []string{"fail2ban-client set banip"}},
|
||||||
{Name: "unban", Description: "Let one address go, from one jail or from every jail when none is named.",
|
{Name: "unban", Description: "Let one address go, from one jail or from every jail when none is named.",
|
||||||
Input: schema(map[string]string{"ip": "the address", "jail": "one jail (optional)"}, []string{"ip"})},
|
Input: schema(map[string]string{"ip": "the address", "jail": "one jail (optional)"}, []string{"ip"}),
|
||||||
|
Replaces: []string{"fail2ban-client unban", "fail2ban-client set unbanip"}},
|
||||||
}},
|
}},
|
||||||
// The packet filter's verbs (novox/hq ADR 0170): what a person asks a machine's filter whatever
|
// The packet filter's verbs (novox/hq ADR 0170): what a person asks a machine's filter whatever
|
||||||
// filter answers — the rules as enforced, reload the mesh's own, remove one thing the mesh did
|
// filter answers — the rules as enforced, reload the mesh's own, remove one thing the mesh did
|
||||||
@@ -210,15 +213,17 @@ var defaultSeats = append([]Seat{
|
|||||||
"ruleset and, where the tool exists, the legacy filter's listings. Narrowed to one table or " +
|
"ruleset and, where the tool exists, the legacy filter's listings. Narrowed to one table or " +
|
||||||
"chain when asked.",
|
"chain when asked.",
|
||||||
Input: schema(map[string]string{"table": "one nftables table, as `family name` (optional)",
|
Input: schema(map[string]string{"table": "one nftables table, as `family name` (optional)",
|
||||||
"chain": "one chain of that table (optional)"}, nil)},
|
"chain": "one chain of that table (optional)"}, nil),
|
||||||
|
Replaces: []string{"nft list ruleset", "iptables -L", "iptables-save"}},
|
||||||
{Name: "reload", Description: "Load the mesh's own filter again from the file the mesh writes, " +
|
{Name: "reload", Description: "Load the mesh's own filter again from the file the mesh writes, " +
|
||||||
"and answer with the mesh's table as loaded.",
|
"and answer with the mesh's table as loaded.",
|
||||||
Input: schema(map[string]string{}, nil)},
|
Input: schema(map[string]string{}, nil), Replaces: []string{"nft -f"}},
|
||||||
{Name: "remove", Description: "Remove one rule set the mesh did not write, named exactly as the " +
|
{Name: "remove", Description: "Remove one rule set the mesh did not write, named exactly as the " +
|
||||||
"host reports it (novox/hq ADR 0168) — `chain X (iptables-legacy)` or `table ip6 filter, chain " +
|
"host reports it (novox/hq ADR 0168) — `chain X (iptables-legacy)` or `table ip6 filter, chain " +
|
||||||
"DOCKER-USER`. Refuses the mesh's tables, the runtime's own chains, a built-in chain and an " +
|
"DOCKER-USER`. Refuses the mesh's tables, the runtime's own chains, a built-in chain and an " +
|
||||||
"active found firewall's chains. An operator's act, by name, never a flush.",
|
"active found firewall's chains. An operator's act, by name, never a flush.",
|
||||||
Input: schema(map[string]string{"where": "the rule set, as `node show` lists it"}, []string{"where"})},
|
Input: schema(map[string]string{"where": "the rule set, as `node show` lists it"}, []string{"where"}),
|
||||||
|
Replaces: []string{"nft delete", "iptables -X"}},
|
||||||
}},
|
}},
|
||||||
// The machine's service manager (novox/hq ADR 0177). The host applies every declared unit,
|
// The machine's service manager (novox/hq ADR 0177). The host applies every declared unit,
|
||||||
// system or user scope; the holder answers questions and operator acts about them, each verb
|
// system or user scope; the holder answers questions and operator acts about them, each verb
|
||||||
@@ -281,7 +286,17 @@ var defaultSeats = append([]Seat{
|
|||||||
// channel a fix could arrive on. A seat
|
// channel a fix could arrive on. A seat
|
||||||
// rather than a condition in the resolver's module, so a machine running two managers is
|
// rather than a condition in the resolver's module, so a machine running two managers is
|
||||||
// refused at assignment instead of found by the resolver being rewritten (novox/hq ADR 0117).
|
// refused at assignment instead of found by the resolver being rewritten (novox/hq ADR 0117).
|
||||||
{Name: "node-uplink", Scope: ScopeNode, Decision: "novox/hq ADR 0117"},
|
//
|
||||||
|
// Its verbs say what the machine resolves through and over which links (novox/hq ADR 0241): the
|
||||||
|
// resolver file as it is and who wrote it, and every link with its addresses, routes and the resolvers
|
||||||
|
// the manager knows for it — the same for every holder, so a question about a machine's names is asked
|
||||||
|
// the same way whatever manages its network. Optional until both holders serve them.
|
||||||
|
{Name: "node-uplink", Scope: ScopeNode, Decision: "novox/hq ADR 0117", Serves: uplinkVerbs()},
|
||||||
|
// The machine's own resolver (novox/hq ADR 0247): held only where something requires `split-dns` — a
|
||||||
|
// VPN client whose domains must go to its own servers while every other name goes to the mesh's.
|
||||||
|
// Where it is held, its holder writes the resolver file and the uplink's holder steps back from it
|
||||||
|
// (node_resolver.go). It knows nothing of any VPN: its verbs route domains to servers over a link.
|
||||||
|
{Name: ResolverSeat, Scope: ScopeNode, Decision: "novox/hq ADR 0247", Serves: resolverVerbs()},
|
||||||
},
|
},
|
||||||
// The graphical session's roles (novox/hq ADR 0208), last because they are a workstation's.
|
// The graphical session's roles (novox/hq ADR 0208), last because they are a workstation's.
|
||||||
graphicalSessionSeats()...)
|
graphicalSessionSeats()...)
|
||||||
@@ -338,12 +353,48 @@ func UseSeats(s []Seat) {
|
|||||||
if d, known := byName[row.Name]; known {
|
if d, known := byName[row.Name]; known {
|
||||||
row.Receives = d.Receives
|
row.Receives = d.Receives
|
||||||
row.Replicated = d.Replicated
|
row.Replicated = d.Replicated
|
||||||
|
row.Serves = optionalAsCompiled(row.Serves, d.Serves)
|
||||||
}
|
}
|
||||||
merged = append(merged, row)
|
merged = append(merged, row)
|
||||||
}
|
}
|
||||||
seats = merged
|
seats = merged
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// optionalAsCompiled is a row's verbs with each one the compiled seat marks optional marked so. The mark
|
||||||
|
// is never stored (Verb.Optional), and a row is what the working set holds once the store is read: a verb
|
||||||
|
// the seeding added to the row — `failed`, `checks` — would otherwise come back required, and every holder
|
||||||
|
// not serving it yet would be refused at registration and at handover, which the mark exists to prevent.
|
||||||
|
//
|
||||||
|
// **And what a verb replaces is the compiled one** (novox/hq ADR 0245): a row seeded before the field
|
||||||
|
// existed carries none, and re-seeding widens a verb's arguments only, so read from the row alone the
|
||||||
|
// `tools` answer — what the mesh MCP server's search and the agent's instructions are built from — would
|
||||||
|
// say no verb replaces anything on a mesh whose rows exist. The mesh's statement of what a role replaces
|
||||||
|
// is this binary's, like the optional mark.
|
||||||
|
func optionalAsCompiled(row, compiled []Verb) []Verb {
|
||||||
|
optional := map[string]bool{}
|
||||||
|
replaces := map[string][]string{}
|
||||||
|
for _, v := range compiled {
|
||||||
|
if v.Optional {
|
||||||
|
optional[v.Name] = true
|
||||||
|
}
|
||||||
|
if len(v.Replaces) > 0 {
|
||||||
|
replaces[v.Name] = v.Replaces
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(optional) == 0 && len(replaces) == 0 {
|
||||||
|
return row
|
||||||
|
}
|
||||||
|
out := make([]Verb, len(row))
|
||||||
|
for i, v := range row {
|
||||||
|
v.Optional = optional[v.Name]
|
||||||
|
if r, ok := replaces[v.Name]; ok {
|
||||||
|
v.Replaces = r
|
||||||
|
}
|
||||||
|
out[i] = v
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// aliases maps a seat's former names to its current canonical name (novox/hq ADR 0122). Loaded from
|
// aliases maps a seat's former names to its current canonical name (novox/hq ADR 0122). Loaded from
|
||||||
// the store alongside the set, so a reference to a name a seat used to have — a manifest's claim, a
|
// the store alongside the set, so a reference to a name a seat used to have — a manifest's claim, a
|
||||||
// held record — still resolves to it after a rename, and nothing downstream has to change.
|
// held record — still resolves to it after a rename, and nothing downstream has to change.
|
||||||
@@ -561,7 +612,8 @@ func SeatsWithAProtocol() []Seat {
|
|||||||
// serviceManagerVerbs is the contract every holder of node-service-manager serves (novox/hq ADR
|
// serviceManagerVerbs is the contract every holder of node-service-manager serves (novox/hq ADR
|
||||||
// 0177): the units on the machine in both scopes, read and acted on by name. Every verb takes an
|
// 0177): the units on the machine in both scopes, read and acted on by name. Every verb takes an
|
||||||
// optional scope — "system" when absent, "user" for the operator account's own manager — so a
|
// optional scope — "system" when absent, "user" for the operator account's own manager — so a
|
||||||
// caller asks for a user unit the way it asks for a system one.
|
// caller asks for a user unit the way it asks for a system one; `failed` alone reads both managers
|
||||||
|
// when none is named, and is optional (Verb.Optional).
|
||||||
func serviceManagerVerbs() []Verb {
|
func serviceManagerVerbs() []Verb {
|
||||||
scoped := func(more map[string]string, required []string) map[string]any {
|
scoped := func(more map[string]string, required []string) map[string]any {
|
||||||
props := map[string]string{"scope": "\"system\" (the default) or \"user\": the operator account's own manager"}
|
props := map[string]string{"scope": "\"system\" (the default) or \"user\": the operator account's own manager"}
|
||||||
@@ -573,21 +625,59 @@ func serviceManagerVerbs() []Verb {
|
|||||||
unit := map[string]string{"unit": "the unit's name, as the service manager knows it"}
|
unit := map[string]string{"unit": "the unit's name, as the service manager knows it"}
|
||||||
return []Verb{
|
return []Verb{
|
||||||
{Name: "units", Description: "The units the service manager knows in a scope, each with its load, active and sub state; narrowed to a pattern when asked.",
|
{Name: "units", Description: "The units the service manager knows in a scope, each with its load, active and sub state; narrowed to a pattern when asked.",
|
||||||
Input: scoped(map[string]string{"pattern": "a glob the unit's name must match (optional)"}, nil)},
|
Input: scoped(map[string]string{"pattern": "a glob the unit's name must match (optional)"}, nil), Replaces: []string{"systemctl list-units"}},
|
||||||
{Name: "status", Description: "One unit as the service manager sees it now: its states, whether it starts at boot, its main process, and whether the mesh declares it.",
|
{Name: "status", Description: "One unit as the service manager sees it now: its states, whether it starts at boot, its main process, and whether the mesh declares it.",
|
||||||
Input: scoped(unit, []string{"unit"})},
|
Input: scoped(unit, []string{"unit"}), Replaces: []string{"systemctl status", "systemctl is-active"}},
|
||||||
{Name: "start", Description: "Start one unit. For a unit the mesh declares, the answer says the host will restore what its declaration says at the next apply.",
|
{Name: "start", Description: "Start one unit. For a unit the mesh declares, the answer says the host will restore what its declaration says at the next apply.",
|
||||||
Input: scoped(unit, []string{"unit"})},
|
Input: scoped(unit, []string{"unit"}), Replaces: []string{"systemctl start"}},
|
||||||
{Name: "stop", Description: "Stop one unit; for a mesh-declared unit the answer says the host will restore its declared state.",
|
{Name: "stop", Description: "Stop one unit; for a mesh-declared unit the answer says the host will restore its declared state.",
|
||||||
Input: scoped(unit, []string{"unit"})},
|
Input: scoped(unit, []string{"unit"}), Replaces: []string{"systemctl stop"}},
|
||||||
{Name: "restart", Description: "Restart one unit.",
|
{Name: "restart", Description: "Restart one unit.",
|
||||||
Input: scoped(unit, []string{"unit"})},
|
Input: scoped(unit, []string{"unit"}), Replaces: []string{"systemctl restart"}},
|
||||||
{Name: "enable", Description: "Make one unit start at boot (or at the account's login, in user scope).",
|
{Name: "enable", Description: "Make one unit start at boot (or at the account's login, in user scope).",
|
||||||
Input: scoped(unit, []string{"unit"})},
|
Input: scoped(unit, []string{"unit"}), Replaces: []string{"systemctl enable"}},
|
||||||
{Name: "disable", Description: "Stop one unit starting at boot (or at login, in user scope).",
|
{Name: "disable", Description: "Stop one unit starting at boot (or at login, in user scope).",
|
||||||
Input: scoped(unit, []string{"unit"})},
|
Input: scoped(unit, []string{"unit"}), Replaces: []string{"systemctl disable"}},
|
||||||
{Name: "journal", Description: "The last lines of one unit's journal.",
|
// **A window, not only a tail** (the operator's direction 2026-10-07): an incident is read for the
|
||||||
Input: scoped(map[string]string{"unit": unit["unit"], "lines": "how many lines from the end (default 100)"}, []string{"unit"})},
|
// minutes it happened in, and with no window on the verb a person reached for a shell. Every
|
||||||
|
// argument is the holder's to validate — passed to journalctl as one word of its own, never through
|
||||||
|
// a shell — and what the unit printed of a secret is redacted before it is answered.
|
||||||
|
{Name: "journal", Description: "The last lines of one unit's journal (at most 2000), in a time window and " +
|
||||||
|
"narrowed to a priority and to lines holding a text when asked. A secret the unit printed is shown as " +
|
||||||
|
"[redacted: <what it was>].",
|
||||||
|
Input: scoped(map[string]string{
|
||||||
|
"unit": unit["unit"],
|
||||||
|
"lines": "how many lines from the end of what matches (default 100, at most 2000)",
|
||||||
|
"since": "the window's start: an RFC 3339 time (2026-10-07T09:30:00Z) or relative to now (-30min, -2h, yesterday) (optional)",
|
||||||
|
"until": "the window's end, in the same forms (optional; now when absent)",
|
||||||
|
"match": "only the lines holding this text, as written — a fixed string, not a pattern (optional)",
|
||||||
|
"priority": "only entries this severe or more: 0-7 or emerg, alert, crit, err, warning, notice, info, debug (optional)",
|
||||||
|
}, []string{"unit"}),
|
||||||
|
Replaces: []string{"journalctl"}},
|
||||||
|
// What has failed, on the seat rather than as one holder's own tool: whatever holds the role answers
|
||||||
|
// it, so a caller asks every machine the same way. **Optional while its holders catch up**: the
|
||||||
|
// systemd module running today serves it as its own systemd_failed, and a required verb would
|
||||||
|
// refuse it before the version serving `failed` could be delivered.
|
||||||
|
{Name: "failed", Optional: true, Description: "Every failed unit on this machine, in the system manager and in the operator " +
|
||||||
|
"account's; a manager that does not answer is reported with its error, never as nothing failed.",
|
||||||
|
Input: schema(map[string]string{"scope": "\"system\" or \"user\": only that manager (both when absent)"}, nil),
|
||||||
|
Replaces: []string{"systemctl --failed"}},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// uplinkVerbs is the contract every holder of node-uplink serves (novox/hq ADR 0241): read-only answers
|
||||||
|
// about what the machine resolves through and over which links, whatever program manages its network.
|
||||||
|
func uplinkVerbs() []Verb {
|
||||||
|
return []Verb{
|
||||||
|
{Name: "resolvers", Description: "The machine's resolver file as it is now: the resolvers, search " +
|
||||||
|
"domains and options it lists, whether it is the file the mesh declares, and when it is not, who " +
|
||||||
|
"wrote it as far as the machine shows — its header, a backup a VPN client left beside it, a link " +
|
||||||
|
"in its place, a known writer running.",
|
||||||
|
Input: schema(nil, nil), Optional: true},
|
||||||
|
{Name: "links", Description: "Every network link on the machine: its state, its addresses, whether " +
|
||||||
|
"the default route leaves through it, and the resolvers and search domains the network manager " +
|
||||||
|
"knows for it — a VPN's tunnel included.",
|
||||||
|
Input: schema(nil, nil), Optional: true},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -600,10 +690,10 @@ func backupVerbs() []Verb {
|
|||||||
return []Verb{
|
return []Verb{
|
||||||
{Name: "backed-up", Description: "What this machine backs up: each module, what it declared, " +
|
{Name: "backed-up", Description: "What this machine backs up: each module, what it declared, " +
|
||||||
"its last good night, how many restore points are kept and the repository's size.",
|
"its last good night, how many restore points are kept and the repository's size.",
|
||||||
Input: schema(map[string]string{"module": "one module (optional)"}, nil)},
|
Input: schema(map[string]string{"module": "one module (optional)"}, nil), Replaces: []string{"restic snapshots"}},
|
||||||
{Name: "now", Description: "Take a backup now, of one module or of every module on this " +
|
{Name: "now", Description: "Take a backup now, of one module or of every module on this " +
|
||||||
"machine — before a migration, a retirement or anything else that could go wrong.",
|
"machine — before a migration, a retirement or anything else that could go wrong.",
|
||||||
Input: schema(map[string]string{"module": "one module (optional)"}, nil)},
|
Input: schema(map[string]string{"module": "one module (optional)"}, nil), Replaces: []string{"restic backup"}},
|
||||||
{Name: "restore", Description: "Restore one module's data from a restore point BESIDE the live " +
|
{Name: "restore", Description: "Restore one module's data from a restore point BESIDE the live " +
|
||||||
"data, never over it: each directory as <path>.restored-<date>. Swapping it in is a " +
|
"data, never over it: each directory as <path>.restored-<date>. Swapping it in is a " +
|
||||||
"person's act. Lists the restore points when none is named.",
|
"person's act. Lists the restore points when none is named.",
|
||||||
@@ -611,7 +701,7 @@ func backupVerbs() []Verb {
|
|||||||
"module": "the module",
|
"module": "the module",
|
||||||
"snapshot": "the restore point (from `backed-up`; the newest when omitted)",
|
"snapshot": "the restore point (from `backed-up`; the newest when omitted)",
|
||||||
"path": "one of the module's directories (all of them when omitted)",
|
"path": "one of the module's directories (all of them when omitted)",
|
||||||
}, []string{"module"})},
|
}, []string{"module"}), Replaces: []string{"restic restore"}},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -46,7 +46,7 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
|
|||||||
delivered[s.Delivers] = s.Name
|
delivered[s.Delivers] = s.Name
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Thirty-seven with mesh-delivery (novox/hq ADR 0239); thirty-six since node-resolver-config retired
|
// Thirty-eight with node-resolver (novox/hq ADR 0247); thirty-seven with mesh-delivery (novox/hq ADR 0239); thirty-six since node-resolver-config retired
|
||||||
// into node-uplink (novox/hq ADR 0223); thirty-seven
|
// into node-uplink (novox/hq ADR 0223); thirty-seven
|
||||||
// since the retired node-dns-resolver went (novox/hq ADR 0220); thirty-eight with
|
// since the retired node-dns-resolver went (novox/hq ADR 0220); thirty-eight with
|
||||||
// node-backup (novox/hq ADR 0214); thirty-seven with node-message-bus (novox/hq ADR 0215);
|
// node-backup (novox/hq ADR 0214); thirty-seven with node-message-bus (novox/hq ADR 0215);
|
||||||
@@ -57,8 +57,8 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
|
|||||||
// node-container-runtime (ADR 0207); nineteen with node-environment and node-login-shell (ADR 0203,
|
// node-container-runtime (ADR 0207); nineteen with node-environment and node-login-shell (ADR 0203,
|
||||||
// ADR 0204); seventeen with node-build-agent (ADR 0190). One fewer once the retired
|
// ADR 0204); seventeen with node-build-agent (ADR 0190). One fewer once the retired
|
||||||
// mesh-build-machine row goes, when no registered manifest claims it.
|
// mesh-build-machine row goes, when no registered manifest claims it.
|
||||||
if len(Seats()) != 37 {
|
if len(Seats()) != 38 {
|
||||||
t.Errorf("the mesh defines %d seats rather than 37; the set is closed, so a change here is "+
|
t.Errorf("the mesh defines %d seats rather than 38; the set is closed, so a change here is "+
|
||||||
"a decision (novox/hq ADR 0110): %s", len(Seats()), seatNames())
|
"a decision (novox/hq ADR 0110): %s", len(Seats()), seatNames())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,62 @@
|
|||||||
|
package catalogue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The node-uplink seat's verbs (novox/hq ADR 0241): promised and routed, optional — not yet a condition of
|
||||||
|
// holding — so the holders of today still hold it, a holder may name them, and nothing it does not promise;
|
||||||
|
// and still optional once the store's row is read back (optionalAsCompiled).
|
||||||
|
|
||||||
|
func uplinkSeat(t *testing.T) Seat {
|
||||||
|
t.Helper()
|
||||||
|
for _, s := range Seats() {
|
||||||
|
if s.Name == "node-uplink" {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatal("no node-uplink seat")
|
||||||
|
return Seat{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheUplinkSeatPromisesItsVerbsAndRequiresNoneYet(t *testing.T) {
|
||||||
|
seat := uplinkSeat(t)
|
||||||
|
var names []string
|
||||||
|
for _, v := range seat.Serves {
|
||||||
|
names = append(names, v.Name)
|
||||||
|
if !v.Optional {
|
||||||
|
t.Errorf("%s is required already", v.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !slices.Equal(names, []string{"resolvers", "links"}) {
|
||||||
|
t.Fatalf("node-uplink promises %v", names)
|
||||||
|
}
|
||||||
|
today := Manifest{Module: "networkmanager", Version: "1", Claims: []Claim{{Name: "node-uplink", Scope: ScopeNode}}}
|
||||||
|
if err := CanHold(today, seat); err != nil {
|
||||||
|
t.Fatalf("a holder that serves no verb yet is refused: %v", err)
|
||||||
|
}
|
||||||
|
serving := today
|
||||||
|
serving.Claims = []Claim{{Name: "node-uplink", Scope: ScopeNode, Serves: []string{"resolvers", "links"}}}
|
||||||
|
if err := CanHold(serving, seat); err != nil {
|
||||||
|
t.Fatalf("a holder serving both is refused: %v", err)
|
||||||
|
}
|
||||||
|
typo := today
|
||||||
|
typo.Claims = []Claim{{Name: "node-uplink", Scope: ScopeNode, Serves: []string{"resolver"}}}
|
||||||
|
if err := CanHold(typo, seat); err == nil || !strings.Contains(err.Error(), "does not promise") {
|
||||||
|
t.Fatalf("a verb the seat does not promise was accepted: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheUplinkVerbsReadBackFromTheRowStayOptional(t *testing.T) {
|
||||||
|
before := seats
|
||||||
|
t.Cleanup(func() { seats = before })
|
||||||
|
// The row as seeding stores it: both verbs, the mark not a column.
|
||||||
|
UseSeats([]Seat{{Name: "node-uplink", Scope: ScopeNode, Serves: []Verb{{Name: "resolvers"}, {Name: "links"}}}})
|
||||||
|
seat := uplinkSeat(t)
|
||||||
|
holder := Manifest{Module: "systemd-networkd", Version: "1", Claims: []Claim{{Name: "node-uplink", Scope: ScopeNode}}}
|
||||||
|
if err := CanHold(holder, seat); err != nil {
|
||||||
|
t.Fatalf("read back from the row, the verbs are required: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -22,6 +22,21 @@ type Verb struct {
|
|||||||
Description string `json:"description,omitempty"`
|
Description string `json:"description,omitempty"`
|
||||||
Input map[string]any `json:"input,omitempty"`
|
Input map[string]any `json:"input,omitempty"`
|
||||||
Output map[string]any `json:"output,omitempty"`
|
Output map[string]any `json:"output,omitempty"`
|
||||||
|
// Optional marks a verb added to a mesh seat whose holder lives in another repository (design 33 §7,
|
||||||
|
// additive within a version): a holder that serves it is accepted, and one that does not yet still
|
||||||
|
// holds the seat. Without it the addition would be a deadlock — this controller refusing the holder that
|
||||||
|
// does not serve the verb, the controller before it refusing the holder that does — and every check of
|
||||||
|
// the catalogue between the two would fail on a module nobody touched. Once every holder serves it,
|
||||||
|
// the mark is removed and the verb is a condition of holding like the rest. Never stored or said: the
|
||||||
|
// seat set's protocol is the compiled one.
|
||||||
|
Optional bool `json:"-"`
|
||||||
|
// Replaces are the shell commands this verb is the mesh's way to do, each as it is typed —
|
||||||
|
// `journalctl`, `systemctl restart` — so the agent finds the verb by the command it would have run
|
||||||
|
// and is told to call it instead (novox/hq ADR 0245): the mesh MCP server's search matches them, the agent's
|
||||||
|
// instructions list them, and the guard on its shell names the verb when it refuses a work-around.
|
||||||
|
// A command line is matched by its words in order, the first being the command's name; `edit
|
||||||
|
// /etc/hosts` and `HOSTALIASES` name the guard's two local work-arounds for a mesh name.
|
||||||
|
Replaces []string `json:"replaces,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (v *Verb) UnmarshalJSON(raw []byte) error {
|
func (v *Verb) UnmarshalJSON(raw []byte) error {
|
||||||
@@ -86,7 +101,8 @@ var ControllerVerbs = []Verb{
|
|||||||
{Name: "nodes", Description: "Every machine the mesh knows, with whether it is converged or adopted.",
|
{Name: "nodes", Description: "Every machine the mesh knows, with whether it is converged or adopted.",
|
||||||
Input: schema(nil, nil)},
|
Input: schema(nil, nil)},
|
||||||
{Name: "node", Description: "What one machine reported it can do, what it is assigned, and why.",
|
{Name: "node", Description: "What one machine reported it can do, what it is assigned, and why.",
|
||||||
Input: schema(map[string]string{"node": "the machine's name"}, []string{"node"})},
|
Input: schema(map[string]string{"node": "the machine's name"}, []string{"node"}),
|
||||||
|
Replaces: []string{"hostnamectl", "uptime"}},
|
||||||
{Name: "modules", Description: "Every module the mesh holds: version, the commit it was built from, " +
|
{Name: "modules", Description: "Every module the mesh holds: version, the commit it was built from, " +
|
||||||
"and which machines run it.",
|
"and which machines run it.",
|
||||||
Input: schema(nil, nil)},
|
Input: schema(nil, nil)},
|
||||||
@@ -262,6 +278,14 @@ var ControllerVerbs = []Verb{
|
|||||||
"cause": "the cause in a word, or a condition's kind — the word a second act for the same reason uses",
|
"cause": "the cause in a word, or a condition's kind — the word a second act for the same reason uses",
|
||||||
"condition": "the key of the condition it addressed, if any (optional)",
|
"condition": "the key of the condition it addressed, if any (optional)",
|
||||||
}, []string{"what", "why", "cause"})},
|
}, []string{"what", "why", "cause"})},
|
||||||
|
{Name: "drill", Description: "Record a drill — something broken on purpose to see the mesh raise and clear it: " +
|
||||||
|
"a module stopped, a process killed — with what it tests, in the hand-act log. A drill is a person's " +
|
||||||
|
"deliberate test, never a repair: no healer is wanted for it however often it is run (S15).",
|
||||||
|
Input: schema(map[string]string{
|
||||||
|
"what": "what was done on purpose, in a line",
|
||||||
|
"why": "what the drill tests",
|
||||||
|
"condition": "the key of the condition the drill is meant to raise, if any (optional)",
|
||||||
|
}, []string{"what", "why"})},
|
||||||
{Name: "hand-acts", Description: "What was done by hand lately — pushes, plans ended, consumers re-made, acts " +
|
{Name: "hand-acts", Description: "What was done by hand lately — pushes, plans ended, consumers re-made, acts " +
|
||||||
"recorded — who, why and the cause of each, and which causes repeat: each repeat is a healer the mesh lacks.",
|
"recorded — who, why and the cause of each, and which causes repeat: each repeat is a healer the mesh lacks.",
|
||||||
Input: schema(map[string]string{"days": "how many days back (default 14)"}, nil)},
|
Input: schema(map[string]string{"days": "how many days back (default 14)"}, nil)},
|
||||||
@@ -434,7 +458,7 @@ func unservedVerbs(tools []string, promised []Verb) []string {
|
|||||||
}
|
}
|
||||||
var missing []string
|
var missing []string
|
||||||
for _, v := range promised {
|
for _, v := range promised {
|
||||||
if !has[v.Name] {
|
if !has[v.Name] && !v.Optional {
|
||||||
missing = append(missing, v.Name)
|
missing = append(missing, v.Name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -46,11 +46,14 @@ const (
|
|||||||
ScopeMesh = "mesh"
|
ScopeMesh = "mesh"
|
||||||
// ScopeDelivery is a delivery mesh-delivery owns, by its id (novox/hq ADR 0239).
|
// ScopeDelivery is a delivery mesh-delivery owns, by its id (novox/hq ADR 0239).
|
||||||
ScopeDelivery = "delivery"
|
ScopeDelivery = "delivery"
|
||||||
|
// ScopeModule is a module on a machine, by `<module>.<machine>`: what it runs is not healthy there
|
||||||
|
// (novox/hq ADR 0240).
|
||||||
|
ScopeModule = "module"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Scopes is every scope, in the order a person reads them.
|
// Scopes is every scope, in the order a person reads them.
|
||||||
var Scopes = []string{ScopeMachine, ScopePlan, ScopeCall, ScopeBuild, ScopeMerge, ScopeProvider,
|
var Scopes = []string{ScopeMachine, ScopePlan, ScopeCall, ScopeBuild, ScopeMerge, ScopeProvider,
|
||||||
ScopeSeat, ScopeBus, ScopeCore, ScopeProbe, ScopeMesh, ScopeDelivery}
|
ScopeSeat, ScopeBus, ScopeCore, ScopeProbe, ScopeMesh, ScopeDelivery, ScopeModule}
|
||||||
|
|
||||||
// Who resolves a condition.
|
// Who resolves a condition.
|
||||||
const (
|
const (
|
||||||
|
|||||||
@@ -172,3 +172,18 @@ func TestAWithheldPathStaysAPath(t *testing.T) {
|
|||||||
t.Errorf("a key became %q", got)
|
t.Errorf("a key became %q", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **Issue 288**: a repository was kept as the URL it was cloned from, the forge's address with it.
|
||||||
|
func TestARepositoryIsNamedWithoutTheForge(t *testing.T) {
|
||||||
|
for in, want := range map[string]string{
|
||||||
|
"http://forge.internal:3000/owner/repo.git": "owner/repo",
|
||||||
|
"ssh://git@forge.internal:222/owner/repo": "owner/repo",
|
||||||
|
"git@forge.internal:owner/repo.git": "owner/repo",
|
||||||
|
"owner/repo": "owner/repo",
|
||||||
|
"": "",
|
||||||
|
} {
|
||||||
|
if got := RepositoryName(in); got != want {
|
||||||
|
t.Errorf("%q is named %q, not %q", in, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -321,3 +321,31 @@ func standIn(run string) string {
|
|||||||
}
|
}
|
||||||
return run
|
return run
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RepositoryName is a repository as `owner/repository`, without the forge's address it was cloned from:
|
||||||
|
// http://forge.internal:3000/owner/repo.git → owner/repo (novox/hq issue 288). What a check matches a
|
||||||
|
// pull request's repository by is its owner and name, never the forge's address, so nothing is lost.
|
||||||
|
func RepositoryName(repository string) string {
|
||||||
|
r := strings.TrimSuffix(strings.TrimSuffix(strings.TrimSpace(repository), "/"), ".git")
|
||||||
|
if r == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
if _, rest, found := strings.Cut(r, "://"); found {
|
||||||
|
r = rest
|
||||||
|
if _, path, found := strings.Cut(r, "/"); found {
|
||||||
|
r = path
|
||||||
|
} else {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
} else if at := strings.Index(r, "@"); at >= 0 {
|
||||||
|
// scp-like: git@forge:owner/repo
|
||||||
|
if _, path, found := strings.Cut(r[at+1:], ":"); found {
|
||||||
|
r = path
|
||||||
|
}
|
||||||
|
}
|
||||||
|
parts := strings.Split(strings.Trim(r, "/"), "/")
|
||||||
|
if len(parts) >= 2 {
|
||||||
|
return parts[len(parts)-2] + "/" + parts[len(parts)-1]
|
||||||
|
}
|
||||||
|
return parts[len(parts)-1]
|
||||||
|
}
|
||||||
|
|||||||
@@ -137,6 +137,8 @@ func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaratio
|
|||||||
// And the state it keeps and reads (novox/hq ADR 0201).
|
// And the state it keeps and reads (novox/hq ADR 0201).
|
||||||
State: bucketsOf(m),
|
State: bucketsOf(m),
|
||||||
Reads: m.Reads,
|
Reads: m.Reads,
|
||||||
|
// And the tools its health asks (novox/hq ADR 0240): the machine's node-engine is granted them.
|
||||||
|
Checks: catalogue.HealthChecks(m),
|
||||||
}
|
}
|
||||||
// Whether it can be given an account at all: delivered as its own secret named broker, so one
|
// Whether it can be given an account at all: delivered as its own secret named broker, so one
|
||||||
// that declares none has nowhere to read it (novox/hq issue 195).
|
// that declares none has nowhere to read it (novox/hq issue 195).
|
||||||
|
|||||||
@@ -307,3 +307,38 @@ func (i *Inventory) BuildOf(ctx context.Context, module, commit string) (string,
|
|||||||
}
|
}
|
||||||
return id, err
|
return id, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ManifestAt is the manifest a module was registered with at a commit (or a commit it abbreviates): the
|
||||||
|
// newest successful build from it, with the artifacts of the build standing for it when its source was
|
||||||
|
// unchanged (issue 280) — what a machine last sent that build runs. False when no such build, or none
|
||||||
|
// with a manifest, is kept (novox/hq issue 295: a recorded module is composed at the build its machine
|
||||||
|
// runs until a person's push moves it).
|
||||||
|
func (i *Inventory) ManifestAt(ctx context.Context, module, commit string) (catalogue.Manifest, bool, error) {
|
||||||
|
if commit == "" {
|
||||||
|
return catalogue.Manifest{}, false, nil
|
||||||
|
}
|
||||||
|
var id string
|
||||||
|
var raw []byte
|
||||||
|
err := i.store.Pool().QueryRow(ctx,
|
||||||
|
`select id, manifest from build
|
||||||
|
where module = $1 and failed = '' and manifest is not null and manifest::text <> 'null'
|
||||||
|
and (commit_hash = $2 or starts_with(commit_hash, $2))
|
||||||
|
order by at desc limit 1`, module, commit).Scan(&id, &raw)
|
||||||
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
return catalogue.Manifest{}, false, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return catalogue.Manifest{}, false, err
|
||||||
|
}
|
||||||
|
if stands, same, err := i.StandingBuild(ctx, module, id); err != nil {
|
||||||
|
return catalogue.Manifest{}, false, err
|
||||||
|
} else if stands != "" && stands != id && len(same) > 0 {
|
||||||
|
raw = same
|
||||||
|
}
|
||||||
|
m, err := catalogue.ParseManifest(raw)
|
||||||
|
if err != nil {
|
||||||
|
return catalogue.Manifest{}, false, fmt.Errorf("%s's build from %s is not a manifest the mesh can compose: %w",
|
||||||
|
module, commit, err)
|
||||||
|
}
|
||||||
|
return m, true, nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,159 @@
|
|||||||
|
package inventory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What each machine says of its long-running resources (novox/hq ADR 0240, to-be 48 §4): the newest
|
||||||
|
// statement per machine, and per module how many statements in a row said a resource of it was unhealthy.
|
||||||
|
|
||||||
|
// ResourceHealth is one long-running resource's state as the node-engine said it.
|
||||||
|
type ResourceHealth struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
Resource string `json:"resource"`
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Target string `json:"target"`
|
||||||
|
State string `json:"state"`
|
||||||
|
Reason string `json:"reason,omitempty"`
|
||||||
|
Since time.Time `json:"since"`
|
||||||
|
Streak int `json:"streak,omitempty"`
|
||||||
|
Restarts int `json:"restarts,omitempty"`
|
||||||
|
// Check is the declared check's kind, empty for liveness alone; Needs the provision it exercises
|
||||||
|
// (ADR 0240 Phase B, to-be 48 §6).
|
||||||
|
Check string `json:"check,omitempty"`
|
||||||
|
Needs string `json:"needs,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// NodeHealth is a machine's newest statement, as kept.
|
||||||
|
type NodeHealth struct {
|
||||||
|
Node string
|
||||||
|
Contract int
|
||||||
|
// SaidAt is when the engine looked (the machine's clock); HeardAt when this controller heard it.
|
||||||
|
SaidAt time.Time
|
||||||
|
HeardAt time.Time
|
||||||
|
Resources []ResourceHealth
|
||||||
|
// Streaks is, per module, how many statements in a row said a resource of it was unhealthy.
|
||||||
|
Streaks map[string]int
|
||||||
|
// Network is the machine's own networking as its engine said it (novox/hq ADR 0241); nil from an
|
||||||
|
// engine older than that judging.
|
||||||
|
Network *NetworkHealth
|
||||||
|
}
|
||||||
|
|
||||||
|
// NetworkHealth is a machine's networking as its engine said it (ADR 0241).
|
||||||
|
type NetworkHealth struct {
|
||||||
|
State string `json:"state"`
|
||||||
|
Since time.Time `json:"since"`
|
||||||
|
Parts []NetworkPart `json:"parts"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// NetworkPart is one part of it: resolv-conf, names, tunnel, bus or route.
|
||||||
|
type NetworkPart struct {
|
||||||
|
Part string `json:"part"`
|
||||||
|
State string `json:"state"`
|
||||||
|
Reason string `json:"reason,omitempty"`
|
||||||
|
Said string `json:"said,omitempty"`
|
||||||
|
Writer string `json:"writer,omitempty"`
|
||||||
|
Owner string `json:"owner,omitempty"`
|
||||||
|
Toward []string `json:"toward,omitempty"`
|
||||||
|
Since time.Time `json:"since"`
|
||||||
|
Streak int `json:"streak,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// HealthOf is a machine's newest statement; false when its node-engine has never stated one.
|
||||||
|
func (i *Inventory) HealthOf(ctx context.Context, nodeName string) (NodeHealth, bool, error) {
|
||||||
|
all, err := i.healths(ctx, nodeName)
|
||||||
|
if err != nil {
|
||||||
|
return NodeHealth{}, false, err
|
||||||
|
}
|
||||||
|
h, ok := all[nodeName]
|
||||||
|
return h, ok, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Healths is every machine's newest statement, by machine. A machine absent never stated one: its
|
||||||
|
// node-engine is older than the judging, and its health is not known.
|
||||||
|
func (i *Inventory) Healths(ctx context.Context) (map[string]NodeHealth, error) {
|
||||||
|
return i.healths(ctx, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHealth, error) {
|
||||||
|
rows, err := i.store.Pool().Query(ctx,
|
||||||
|
`select n.name, h.contract, h.said_at, h.heard_at, h.resources, h.streaks, h.network
|
||||||
|
from node_health h join node n on n.id = h.node
|
||||||
|
where $1 = '' or n.name = $1`, only)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
out := map[string]NodeHealth{}
|
||||||
|
for rows.Next() {
|
||||||
|
var h NodeHealth
|
||||||
|
var resources, streaks, network []byte
|
||||||
|
if err := rows.Scan(&h.Node, &h.Contract, &h.SaidAt, &h.HeardAt, &resources, &streaks, &network); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(resources, &h.Resources); err != nil {
|
||||||
|
return nil, fmt.Errorf("%s's health cannot be read: %w", h.Node, err)
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(streaks, &h.Streaks); err != nil {
|
||||||
|
return nil, fmt.Errorf("%s's health cannot be read: %w", h.Node, err)
|
||||||
|
}
|
||||||
|
if len(network) > 0 {
|
||||||
|
if err := json.Unmarshal(network, &h.Network); err != nil {
|
||||||
|
return nil, fmt.Errorf("%s's network health cannot be read: %w", h.Node, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out[h.Node] = h
|
||||||
|
}
|
||||||
|
return out, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordHealth keeps a machine's statement in place of the one kept — unless the one kept is newer, by
|
||||||
|
// when the engine looked: then nothing is written, and false says it was refused as older.
|
||||||
|
func (i *Inventory) RecordHealth(ctx context.Context, h NodeHealth) (bool, error) {
|
||||||
|
if h.Resources == nil {
|
||||||
|
h.Resources = []ResourceHealth{}
|
||||||
|
}
|
||||||
|
if h.Streaks == nil {
|
||||||
|
h.Streaks = map[string]int{}
|
||||||
|
}
|
||||||
|
resources, err := json.Marshal(h.Resources)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
streaks, err := json.Marshal(h.Streaks)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
var network []byte
|
||||||
|
if h.Network != nil {
|
||||||
|
if network, err = json.Marshal(h.Network); err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
heard := h.HeardAt
|
||||||
|
if heard.IsZero() {
|
||||||
|
heard = time.Now()
|
||||||
|
}
|
||||||
|
var node string
|
||||||
|
err = i.store.Pool().QueryRow(ctx,
|
||||||
|
`insert into node_health (node, contract, said_at, heard_at, resources, streaks, network)
|
||||||
|
select id, $2, $3, $4, $5, $6, $7 from node where name = $1
|
||||||
|
on conflict (node) do update set contract = excluded.contract, said_at = excluded.said_at,
|
||||||
|
heard_at = excluded.heard_at, resources = excluded.resources, streaks = excluded.streaks,
|
||||||
|
network = excluded.network
|
||||||
|
where node_health.said_at <= excluded.said_at
|
||||||
|
returning node`, h.Node, h.Contract, h.SaidAt, heard, resources, streaks, network).Scan(&node)
|
||||||
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
if _, nerr := i.NodeByName(ctx, h.Node); nerr != nil {
|
||||||
|
return false, nerr
|
||||||
|
}
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
return err == nil, err
|
||||||
|
}
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
-- A module says how it is healthy, and the node-engine judges it (novox/hq ADR 0240, to-be 48 Phase A).
|
||||||
|
--
|
||||||
|
-- Every machine's node-engine states the health of every long-running resource it runs for a module — a
|
||||||
|
-- container that stays up, a process that stays up, a service stated running — in every report and as an
|
||||||
|
-- event between reports. The controller keeps the newest statement per machine, here, so the release
|
||||||
|
-- gate, `node show` and a controller started again all read the same word; and with it, per module, how
|
||||||
|
-- many statements in a row said a resource of it was unhealthy — `module.<module>.<machine>.unhealthy` is
|
||||||
|
-- raised on the second (ADR 0240 §4).
|
||||||
|
--
|
||||||
|
-- One row per machine, replaced, as node_report is: the question is the machine's state now. A machine
|
||||||
|
-- whose node-engine is older than the judging has no row, and its health is not known — never healthy,
|
||||||
|
-- never unhealthy.
|
||||||
|
create table node_health (
|
||||||
|
node uuid primary key references node(id) on delete cascade,
|
||||||
|
-- The statement's version (the engine's liveness contract).
|
||||||
|
contract int not null,
|
||||||
|
-- When the node-engine looked, on the machine's clock: the order of its statements. An older one
|
||||||
|
-- than this is refused.
|
||||||
|
said_at timestamptz not null,
|
||||||
|
-- When this controller heard it, on its own: what "since the send" is judged by.
|
||||||
|
heard_at timestamptz not null default now(),
|
||||||
|
-- [{module, resource, kind, target, state, reason, since, streak, restarts}], as the engine said them.
|
||||||
|
resources jsonb not null default '[]',
|
||||||
|
-- {module: statements in a row saying a resource of it is unhealthy}.
|
||||||
|
streaks jsonb not null default '{}'
|
||||||
|
);
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
-- A machine says how its network is (novox/hq ADR 0241, which extends ADR 0240 from what a module runs to
|
||||||
|
-- the machine it runs on).
|
||||||
|
--
|
||||||
|
-- Beside the state of every long-running resource, each machine's node-engine states its own networking:
|
||||||
|
-- the resolver file the uplink holder declared and who rewrote it, the names through every resolver it
|
||||||
|
-- lists, the tunnel's handshake with the hub, the bus and the default route — the worst of them, since
|
||||||
|
-- when, and each part. Kept with the machine's newest statement, replaced with it, so `node show`, the
|
||||||
|
-- gate and a controller started again read the same word. Null for a machine whose node-engine is older
|
||||||
|
-- than this judging: its network is not known — never healthy, never a reason to raise anything.
|
||||||
|
alter table node_health add column network jsonb;
|
||||||
@@ -152,6 +152,10 @@ type CarriedMove struct {
|
|||||||
From string `json:"from,omitempty"`
|
From string `json:"from,omitempty"`
|
||||||
To string `json:"to"`
|
To string `json:"to"`
|
||||||
Build string `json:"build,omitempty"`
|
Build string `json:"build,omitempty"`
|
||||||
|
// Recreates is what the send does to the module's containers, in the mesh's words — how many it
|
||||||
|
// recreates, and whether with a new image or only their declaration (novox/hq ADR 0245); empty when
|
||||||
|
// it recreates none, or when the build the machine ran is not known.
|
||||||
|
Recreates string `json:"recreates,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// PlanRelease is a release plan's walk through the machines (novox/hq ADR 0236): every module build
|
// PlanRelease is a release plan's walk through the machines (novox/hq ADR 0236): every module build
|
||||||
|
|||||||
@@ -126,6 +126,16 @@ func (i *Inventory) widenProtocol(ctx context.Context, s catalogue.Seat) error {
|
|||||||
if s.Name == catalogue.ControllerSeatName && !sameVerb(row.Serves[at], v) {
|
if s.Name == catalogue.ControllerSeatName && !sameVerb(row.Serves[at], v) {
|
||||||
row.Serves[at] = v
|
row.Serves[at] = v
|
||||||
changed = true
|
changed = true
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// **Any other seat's verb gains the arguments the binary names and the row lacks** — additive,
|
||||||
|
// as the rest of the protocol is. The console refuses an argument the row does not name (issue
|
||||||
|
// 244), so a holder taught a new argument (the service manager's journal window) would be
|
||||||
|
// unreachable through it while the row kept the older schema. Nothing the row has is removed or
|
||||||
|
// made required; the description is the binary's, since it describes the arguments added.
|
||||||
|
if widened, ok := widenInput(row.Serves[at], v); ok {
|
||||||
|
row.Serves[at] = widened
|
||||||
|
changed = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !changed {
|
if !changed {
|
||||||
@@ -296,6 +306,45 @@ func (i *Inventory) Holdings(ctx context.Context) ([]catalogue.Held, error) {
|
|||||||
return out, rows.Err()
|
return out, rows.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// widenInput is the row's verb with every input property the binary's names and the row's lacks, and
|
||||||
|
// the binary's description; and whether anything was added.
|
||||||
|
func widenInput(row, binary catalogue.Verb) (catalogue.Verb, bool) {
|
||||||
|
want, _ := binary.Input["properties"].(map[string]any)
|
||||||
|
if len(want) == 0 {
|
||||||
|
return row, false
|
||||||
|
}
|
||||||
|
have, _ := row.Input["properties"].(map[string]any)
|
||||||
|
added := map[string]any{}
|
||||||
|
for name, p := range want {
|
||||||
|
if _, kept := have[name]; !kept {
|
||||||
|
added[name] = p
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(added) == 0 {
|
||||||
|
return row, false
|
||||||
|
}
|
||||||
|
input := map[string]any{}
|
||||||
|
for k, v := range row.Input {
|
||||||
|
input[k] = v
|
||||||
|
}
|
||||||
|
if input["type"] == nil {
|
||||||
|
input["type"] = "object"
|
||||||
|
}
|
||||||
|
props := map[string]any{}
|
||||||
|
for k, v := range have {
|
||||||
|
props[k] = v
|
||||||
|
}
|
||||||
|
for k, v := range added {
|
||||||
|
props[k] = v
|
||||||
|
}
|
||||||
|
input["properties"] = props
|
||||||
|
row.Input = input
|
||||||
|
if binary.Description != "" {
|
||||||
|
row.Description = binary.Description
|
||||||
|
}
|
||||||
|
return row, true
|
||||||
|
}
|
||||||
|
|
||||||
// sameVerb is whether two definitions of a verb say the same, read as the row stores them.
|
// sameVerb is whether two definitions of a verb say the same, read as the row stores them.
|
||||||
func sameVerb(a, b catalogue.Verb) bool {
|
func sameVerb(a, b catalogue.Verb) bool {
|
||||||
ja, errA := json.Marshal(a)
|
ja, errA := json.Marshal(a)
|
||||||
|
|||||||
@@ -156,3 +156,74 @@ func TestTheControllersVerbsInTheRowAreTheBinarys(t *testing.T) {
|
|||||||
t.Fatal("a verb this binary adds was not added")
|
t.Fatal("a verb this binary adds was not added")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **A seat's verb gains the arguments a newer binary names** (the service manager's journal window,
|
||||||
|
// 2026-10-07): the console refuses an argument the row does not name, so a row seeded before them would
|
||||||
|
// keep the holder's new arguments unreachable. Added, never removed — an argument only the row has stays,
|
||||||
|
// and nothing becomes required — and a verb the binary adds optional is optional in the working set read back.
|
||||||
|
func TestASeatsVerbGainsTheArgumentsTheBinaryNames(t *testing.T) {
|
||||||
|
inv := ForTest(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
old := `[{"name":"journal","description":"The last lines of one unit's journal.","input":{"type":"object",
|
||||||
|
"properties":{"unit":{"type":"string"},"lines":{"type":"string"},"scope":{"type":"string"},"kept":{"type":"string"}},
|
||||||
|
"required":["unit"]}}]`
|
||||||
|
if _, err := inv.store.Pool().Exec(ctx, `update seat set serves = $1 where name = $2`,
|
||||||
|
[]byte(old), catalogue.ServiceManagerSeat); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
seats, err := inv.Seats(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
verbs := map[string]catalogue.Verb{}
|
||||||
|
for _, s := range seats {
|
||||||
|
if s.Name == catalogue.ServiceManagerSeat {
|
||||||
|
for _, v := range s.Serves {
|
||||||
|
verbs[v.Name] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
props, _ := verbs["journal"].Input["properties"].(map[string]any)
|
||||||
|
for _, arg := range []string{"since", "until", "match", "priority", "unit", "lines", "scope", "kept"} {
|
||||||
|
if _, has := props[arg]; !has {
|
||||||
|
t.Errorf("journal in the row does not take %s: %v", arg, props)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if req, _ := verbs["journal"].Input["required"].([]any); len(req) != 1 || req[0] != "unit" {
|
||||||
|
t.Errorf("what journal requires changed: %v", verbs["journal"].Input["required"])
|
||||||
|
}
|
||||||
|
if _, added := verbs["failed"]; !added {
|
||||||
|
t.Fatal("failed was not added to the row")
|
||||||
|
}
|
||||||
|
// The optional mark is not stored; the working set read from the rows takes it from the compiled seat,
|
||||||
|
// so today's holder, which does not serve failed, still holds the seat.
|
||||||
|
catalogue.UseSeats(seats)
|
||||||
|
defer catalogue.UseSeats(catalogue.DefaultSeats())
|
||||||
|
live, _ := catalogue.SeatNamed(catalogue.ServiceManagerSeat)
|
||||||
|
holder := catalogue.Manifest{Module: "systemd", Version: "1", Claims: []catalogue.Claim{{Name: catalogue.ServiceManagerSeat,
|
||||||
|
Scope: catalogue.ScopeNode, Serves: []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal"}}}}
|
||||||
|
if err := catalogue.CanHold(holder, live); err != nil {
|
||||||
|
t.Fatalf("the seat read from the store refuses today's holder: %v", err)
|
||||||
|
}
|
||||||
|
// Seeding again changes nothing more.
|
||||||
|
before := verbs["journal"]
|
||||||
|
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
after, _ := inv.Seats(ctx)
|
||||||
|
for _, s := range after {
|
||||||
|
if s.Name == catalogue.ServiceManagerSeat {
|
||||||
|
for _, v := range s.Serves {
|
||||||
|
if v.Name == "journal" && !sameVerb(v, before) {
|
||||||
|
t.Fatalf("re-seeding changed journal again: %+v", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -75,8 +75,16 @@ const (
|
|||||||
// ToolsAliveSubjects is every machine's node tools saying they are there (novox/hq to-be 45 §3,
|
// ToolsAliveSubjects is every machine's node tools saying they are there (novox/hq to-be 45 §3,
|
||||||
// S11): core NATS like the host's, for the same reason.
|
// S11): core NATS like the host's, for the same reason.
|
||||||
ToolsAliveSubjects = "mesh.control.*.tools-alive"
|
ToolsAliveSubjects = "mesh.control.*.tools-alive"
|
||||||
|
|
||||||
|
// HealthSubjects is every machine's health statement between its reports (novox/hq ADR 0240): core
|
||||||
|
// NATS like the heartbeat, because a statement lost is said again within a minute while anything is
|
||||||
|
// not healthy, and the next report carries it whatever happens.
|
||||||
|
HealthSubjects = "mesh.control.*.health"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// HealthSubject is one machine's health statement.
|
||||||
|
func HealthSubject(node string) string { return "mesh.control." + node + ".health" }
|
||||||
|
|
||||||
// ReportSubject is where one node says what it did. On the CONTROL stream, because it is the
|
// ReportSubject is where one node says what it did. On the CONTROL stream, because it is the
|
||||||
// message the store-window guarantee is about (ADR 0083).
|
// message the store-window guarantee is about (ADR 0083).
|
||||||
func ReportSubject(node string) string { return "mesh.control." + node + ".report" }
|
func ReportSubject(node string) string { return "mesh.control." + node + ".report" }
|
||||||
|
|||||||
+13
-1
@@ -29,6 +29,15 @@ import (
|
|||||||
// then and with "still running as call <id>" when it does not; and every call is kept here, with
|
// then and with "still running as call <id>" when it does not; and every call is kept here, with
|
||||||
// what came of it, including an answer the bus refused, so `calls` can say it.
|
// what came of it, including an answer the bus refused, so `calls` can say it.
|
||||||
|
|
||||||
|
// ErrHandingOver is a call refused because the controller answering it is being replaced: stopping, or
|
||||||
|
// unable to run its own build because the node-engine has moved it aside (novox/hq issue 289). Nothing
|
||||||
|
// was done, and the controller after it answers the same call — so the answer carries
|
||||||
|
// `"retry": RetryHandingOver`, and a caller asks once more.
|
||||||
|
var ErrHandingOver = errors.New("the controller is handing over to the next one; nothing was done, ask again")
|
||||||
|
|
||||||
|
// RetryHandingOver is the `retry` mark of an answer refused by ErrHandingOver.
|
||||||
|
const RetryHandingOver = "handing-over"
|
||||||
|
|
||||||
// AnswerWithin is how long a call runs before its caller is answered that it is still running. Well
|
// AnswerWithin is how long a call runs before its caller is answered that it is still running. Well
|
||||||
// inside the shortest wait of a caller the mesh ships (the console's thirty seconds) and the bus's
|
// inside the shortest wait of a caller the mesh ships (the console's thirty seconds) and the bus's
|
||||||
// own window for an answer (broker.ResponseTTL), so the one answer a call has is never late for
|
// own window for an answer (broker.ResponseTTL), so the one answer a call has is never late for
|
||||||
@@ -524,7 +533,10 @@ func (l *CallLog) serveCall(seat, verb string, args json.RawMessage, reply strin
|
|||||||
var body []byte
|
var body []byte
|
||||||
result, err := handle(ctx, args)
|
result, err := handle(ctx, args)
|
||||||
failed := err != nil
|
failed := err != nil
|
||||||
if err != nil {
|
if errors.Is(err, ErrHandingOver) {
|
||||||
|
// Marked as well as said, so a caller asks again without reading the words (issue 289).
|
||||||
|
body, _ = json.Marshal(map[string]any{"error": err.Error(), "retry": RetryHandingOver})
|
||||||
|
} else if err != nil {
|
||||||
body, _ = json.Marshal(map[string]any{"error": err.Error()})
|
body, _ = json.Marshal(map[string]any{"error": err.Error()})
|
||||||
} else if body, err = json.Marshal(map[string]any{"result": result}); err != nil {
|
} else if body, err = json.Marshal(map[string]any{"result": result}); err != nil {
|
||||||
failed = true
|
failed = true
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -177,3 +178,24 @@ func recentOf(l *CallLog) []Call {
|
|||||||
out, _ := l.Recent()
|
out, _ := l.Recent()
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A call refused because its controller is handing over says so in a mark as well as in words, so the
|
||||||
|
// caller asks once more without parsing a sentence (novox/hq issue 289).
|
||||||
|
func TestAHandoverRefusalIsMarkedRetryable(t *testing.T) {
|
||||||
|
l, a := NewCallLog(), newAnswers(t)
|
||||||
|
l.serveCall("mesh-controller", "rotate", nil, "_INBOX.x.9", func(context.Context, json.RawMessage) (any, error) {
|
||||||
|
return nil, fmt.Errorf("%w: stopping", ErrHandingOver)
|
||||||
|
}, a.respond, nil)
|
||||||
|
got := a.only()
|
||||||
|
if got["retry"] != RetryHandingOver || !strings.Contains(fmt.Sprint(got["error"]), "handing over") {
|
||||||
|
t.Fatalf("answered %v", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
l, a = NewCallLog(), newAnswers(t)
|
||||||
|
l.serveCall("mesh-controller", "rotate", nil, "_INBOX.x.10", func(context.Context, json.RawMessage) (any, error) {
|
||||||
|
return nil, errors.New("refused for its own reason")
|
||||||
|
}, a.respond, nil)
|
||||||
|
if _, marked := a.only()["retry"]; marked {
|
||||||
|
t.Fatal("an ordinary refusal was marked to be asked again")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -35,6 +35,10 @@ var Contracts = map[string]Contract{
|
|||||||
KindHeartbeat: {Unordered: "a word that the machine is there: the newest heard is the newest said, and one " +
|
KindHeartbeat: {Unordered: "a word that the machine is there: the newest heard is the newest said, and one " +
|
||||||
"lost is the next one"},
|
"lost is the next one"},
|
||||||
KindToolsHeartbeat: {Unordered: "a word that the node tools are there, as a machine's heartbeat"},
|
KindToolsHeartbeat: {Unordered: "a word that the node tools are there, as a machine's heartbeat"},
|
||||||
|
KindHealth: {Ordered: "by the time the node-engine looked (`at`, on the machine's clock): a statement older " +
|
||||||
|
"than the one kept for that machine is refused, so an event arriving after a newer report does not undo it " +
|
||||||
|
"(novox/hq ADR 0240)",
|
||||||
|
Tests: []string{"TestAnOlderHealthStatementIsRefused"}},
|
||||||
KindEnrolment: {Unordered: "a request answered once, under a token spent once: a second presentation is " +
|
KindEnrolment: {Unordered: "a request answered once, under a token spent once: a second presentation is " +
|
||||||
"refused by the token, not by an order (issue 083)",
|
"refused by the token, not by an order (issue 083)",
|
||||||
Tests: []string{"TestAnEnrolmentMetByAHeldTokenIsAskedToTryAgain"}},
|
Tests: []string{"TestAnEnrolmentMetByAHeldTokenIsAskedToTryAgain"}},
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ import (
|
|||||||
func TestEveryConsumedKindHasAContract(t *testing.T) {
|
func TestEveryConsumedKindHasAContract(t *testing.T) {
|
||||||
// What the controller can be handed, from the subjects it is granted and the ones it derives.
|
// What the controller can be handed, from the subjects it is granted and the ones it derives.
|
||||||
subjects := []string{EnrolSubject, BuiltSubject, ReportSubject("anchor"), AliveSubject("anchor"),
|
subjects := []string{EnrolSubject, BuiltSubject, ReportSubject("anchor"), AliveSubject("anchor"),
|
||||||
ToolsAliveSubject("anchor"), "mesh.mod.postgres.event.provisioner.failing"}
|
ToolsAliveSubject("anchor"), HealthSubject("anchor"), "mesh.mod.postgres.event.provisioner.failing"}
|
||||||
subjects = append(subjects, broker.ControllerFollows...)
|
subjects = append(subjects, broker.ControllerFollows...)
|
||||||
kinds := map[string]bool{}
|
kinds := map[string]bool{}
|
||||||
for _, s := range subjects {
|
for _, s := range subjects {
|
||||||
|
|||||||
@@ -43,8 +43,18 @@ type HandAct struct {
|
|||||||
Cause string `json:"cause"`
|
Cause string `json:"cause"`
|
||||||
// Condition is the condition's key the act addresses, when it names one.
|
// Condition is the condition's key the act addresses, when it names one.
|
||||||
Condition string `json:"condition,omitempty"`
|
Condition string `json:"condition,omitempty"`
|
||||||
|
// Kind is what the controller read the act to be from what it did, never a word a person gives:
|
||||||
|
// KindRecordedBuilds for a push that carried only builds a `record` policy held back for a person's
|
||||||
|
// word (novox/hq ADR 0242), empty for everything else. Absent from entries written before it existed.
|
||||||
|
Kind string `json:"kind,omitempty"`
|
||||||
|
// Carried is what such a push moved, one "module from → to" per module, so the log says it.
|
||||||
|
Carried []string `json:"carried,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// KindRecordedBuilds is a push that only moved recorded builds: the person's word their `record` policy
|
||||||
|
// asks for, which is no repair (novox/hq ADR 0242, to-be 45 §7).
|
||||||
|
const KindRecordedBuilds = "recorded-builds"
|
||||||
|
|
||||||
// CallerVar carries a seat call's caller to the command the controller runs for it, so an act done
|
// CallerVar carries a seat call's caller to the command the controller runs for it, so an act done
|
||||||
// through the console says who asked rather than "the controller".
|
// through the console says who asked rather than "the controller".
|
||||||
const CallerVar = "MESH_CALLER"
|
const CallerVar = "MESH_CALLER"
|
||||||
|
|||||||
@@ -0,0 +1,56 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A machine's health statement (novox/hq ADR 0240, to-be 48 §4).
|
||||||
|
//
|
||||||
|
// **The node-engine owns every verdict; the controller keeps the last word per machine.** A statement
|
||||||
|
// arrives in every report and as an event between reports — on each change, and again every minute while
|
||||||
|
// a resource is not healthy. What the controller does with it — keep it, refuse an older one, raise
|
||||||
|
// `module.<module>.<machine>.unhealthy` on the second statement that says so — is the Healths it is given.
|
||||||
|
|
||||||
|
// Healths keeps what machines state of their long-running resources.
|
||||||
|
type Healths interface {
|
||||||
|
// Stated keeps one machine's statement, from a report or the event. An older statement than the one
|
||||||
|
// kept is refused there, by its time on the machine.
|
||||||
|
Stated(ctx context.Context, node string, h Health) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Hears says where the machines' health statements are kept. Their subscription is the heartbeats':
|
||||||
|
// core, and always made, so nothing is asked of the bus here.
|
||||||
|
func (s *Server) Hears(h Healths) { s.healths = h }
|
||||||
|
|
||||||
|
// healthSaid acts on one health event. Core NATS, so there is nothing to hold: a statement that could not
|
||||||
|
// be kept is said in the log, and the next one — a minute away while anything is not healthy — is kept.
|
||||||
|
func (s *Server) healthSaid(ctx context.Context, m Control) {
|
||||||
|
defer func() { _ = m.Took() }()
|
||||||
|
var said HealthSaid
|
||||||
|
if err := json.Unmarshal(m.Body(), &said); err != nil || said.Health.Contract == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// The machine is the one in the subject the bus let it publish on, never the body's.
|
||||||
|
node, ok := nodeOfHealth(m.Subject())
|
||||||
|
if !ok || s.healths == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := s.healths.Stated(ctx, node, said.Health); err != nil {
|
||||||
|
s.log.Printf("could not keep what %s says of its resources' health: %v", node, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// nodeOfHealth is the machine a health statement names in its subject.
|
||||||
|
func nodeOfHealth(subject string) (string, bool) {
|
||||||
|
rest, ok := strings.CutPrefix(subject, "mesh.control.")
|
||||||
|
if !ok {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
node, kind, ok := strings.Cut(rest, ".")
|
||||||
|
if !ok || kind != "health" || node == "" || strings.Contains(node, ".") {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
return node, true
|
||||||
|
}
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A health event is kept as the machine its subject names says it, whatever its body claims, and taken
|
||||||
|
// (novox/hq ADR 0240): core NATS, nothing to hold.
|
||||||
|
|
||||||
|
type keptHealths struct{ by map[string]Health }
|
||||||
|
|
||||||
|
func (k *keptHealths) Stated(_ context.Context, node string, h Health) error {
|
||||||
|
k.by[node] = h
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAHealthEventIsKeptAsTheMachineInItsSubject(t *testing.T) {
|
||||||
|
s, in := serving()
|
||||||
|
kept := &keptHealths{by: map[string]Health{}}
|
||||||
|
s.Hears(kept)
|
||||||
|
to := &settled{}
|
||||||
|
m := in.sends(t, to, KindHealth, HealthSaid{Node: "laptop", Health: Health{Contract: LivenessContract, At: time.Now(),
|
||||||
|
Resources: []ResourceHealth{{Module: "letta", Resource: "letta.server", State: StateUnhealthy}}}}).(*fakeControl)
|
||||||
|
m.subject = HealthSubject("anchor")
|
||||||
|
s.act(t.Context(), m)
|
||||||
|
if !to.acked {
|
||||||
|
t.Fatal("a health event was not taken")
|
||||||
|
}
|
||||||
|
if _, lied := kept.by["laptop"]; lied || len(kept.by["anchor"].Resources) != 1 {
|
||||||
|
t.Fatalf("kept %+v: the machine is the subject's, never the body's", kept.by)
|
||||||
|
}
|
||||||
|
if kind, ok := kindOfSubject(HealthSubject("anchor")); !ok || kind != KindHealth {
|
||||||
|
t.Fatalf("%s is not read as a health statement", HealthSubject("anchor"))
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -262,6 +262,13 @@ type Report struct {
|
|||||||
// `witness` and `not-reversible` are sent only to a machine whose report carries it.
|
// `witness` and `not-reversible` are sent only to a machine whose report carries it.
|
||||||
Witness int `json:"witness,omitempty"`
|
Witness int `json:"witness,omitempty"`
|
||||||
|
|
||||||
|
// Health is the machine's word on every long-running resource it runs for a module (novox/hq ADR
|
||||||
|
// 0240, to-be 48 §4; mesh-host's internal/liveness): its state, since when, its failing streak and
|
||||||
|
// the restarts its node-engine counted. **Absent from an engine older than the judging**, which is
|
||||||
|
// read as "not known" — never as healthy, never as a reason to raise anything; present with no
|
||||||
|
// resources from a machine that runs nothing long-lived.
|
||||||
|
Health *Health `json:"health,omitempty"`
|
||||||
|
|
||||||
// Rekey is a node taking a found tunnel's key as its overlay key after enrolment (novox/hq
|
// Rekey is a node taking a found tunnel's key as its overlay key after enrolment (novox/hq
|
||||||
// ADR 0105). A report carrying one is not an account of the machine: it moves the node's
|
// ADR 0105). A report carrying one is not an account of the machine: it moves the node's
|
||||||
// overlay key and tunnel and nothing else.
|
// overlay key and tunnel and nothing else.
|
||||||
@@ -404,3 +411,93 @@ func EnrolProof(secret string, public []byte, overlay, sealing, serving string)
|
|||||||
return []byte("novox-mesh-enrol\x00" + secret + "\x00" + base64.StdEncoding.EncodeToString(public) +
|
return []byte("novox-mesh-enrol\x00" + secret + "\x00" + base64.StdEncoding.EncodeToString(public) +
|
||||||
"\x00" + overlay + "\x00" + sealing + "\x00" + serving)
|
"\x00" + overlay + "\x00" + sealing + "\x00" + serving)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// LivenessContract is the version of the health statement this controller reads (ADR 0240 Phase A).
|
||||||
|
const LivenessContract = 1
|
||||||
|
|
||||||
|
// ReadinessContract is the statement of an engine that also reads a resource's declared `health` and
|
||||||
|
// judges it (ADR 0240 Phase B): only to an engine whose statement says this or later is the field sent,
|
||||||
|
// because an older one parses strictly and would refuse the whole declaration for it.
|
||||||
|
const ReadinessContract = 2
|
||||||
|
|
||||||
|
// Health is one statement of a machine's long-running resources (to-be 48 §4): in every report, as the
|
||||||
|
// event HealthSubject between reports on each change, and again every minute while one is not healthy.
|
||||||
|
// The node-engine's own (mesh-host internal/link Health); a test on each side holds the field names.
|
||||||
|
type Health struct {
|
||||||
|
Contract int `json:"contract"`
|
||||||
|
// At is when the engine looked, on the machine's clock: the order of its statements.
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
Resources []ResourceHealth `json:"resources"`
|
||||||
|
// Network is the machine's own networking, judged by its engine (novox/hq ADR 0241); nil from an engine
|
||||||
|
// older than that judging, which is "not known", never healthy.
|
||||||
|
Network *NetworkHealth `json:"network,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// NetworkHealth is a machine's networking in one statement (ADR 0241): the worst of its parts, since
|
||||||
|
// when, and each part. The node-engine's own (mesh-host internal/link NetworkHealth).
|
||||||
|
type NetworkHealth struct {
|
||||||
|
State string `json:"state"`
|
||||||
|
Since time.Time `json:"since"`
|
||||||
|
Parts []NetworkPart `json:"parts"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// The parts of a machine's networking its engine judges (ADR 0241).
|
||||||
|
const (
|
||||||
|
PartResolvConf = "resolv-conf"
|
||||||
|
PartNames = "names"
|
||||||
|
PartTunnel = "tunnel"
|
||||||
|
PartBus = "bus"
|
||||||
|
PartRoute = "route"
|
||||||
|
// TowardHub is what a part failing toward the hub names in Toward.
|
||||||
|
TowardHub = "hub"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NetworkPart is one part, as the engine judged it on its second look.
|
||||||
|
type NetworkPart struct {
|
||||||
|
Part string `json:"part"`
|
||||||
|
State string `json:"state"`
|
||||||
|
// Reason is in words with no address, path or domain; Said is the detail, kept as evidence.
|
||||||
|
Reason string `json:"reason,omitempty"`
|
||||||
|
Said string `json:"said,omitempty"`
|
||||||
|
// Writer is the program that rewrote the resolver file, when the engine could name it; Owner the
|
||||||
|
// module whose file it is.
|
||||||
|
Writer string `json:"writer,omitempty"`
|
||||||
|
Owner string `json:"owner,omitempty"`
|
||||||
|
// Toward is what a failure points at: "hub", or each resolver's address that failed.
|
||||||
|
Toward []string `json:"toward,omitempty"`
|
||||||
|
Since time.Time `json:"since"`
|
||||||
|
Streak int `json:"streak,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// The states a resource is said in (ADR 0240 §4).
|
||||||
|
const (
|
||||||
|
StateHealthy = "healthy"
|
||||||
|
StateUnhealthy = "unhealthy"
|
||||||
|
StateStarting = "starting"
|
||||||
|
StateHeld = "held"
|
||||||
|
StateUnknown = "unknown"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ResourceHealth is one long-running resource's state.
|
||||||
|
type ResourceHealth struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
Resource string `json:"resource"`
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Target string `json:"target"`
|
||||||
|
State string `json:"state"`
|
||||||
|
Reason string `json:"reason,omitempty"`
|
||||||
|
Since time.Time `json:"since"`
|
||||||
|
Streak int `json:"streak,omitempty"`
|
||||||
|
Restarts int `json:"restarts,omitempty"`
|
||||||
|
// Check is the declared check's kind (ADR 0240 Phase B), empty for a resource judged by liveness
|
||||||
|
// alone; Needs is the provision the check exercises (to-be 48 §6).
|
||||||
|
Check string `json:"check,omitempty"`
|
||||||
|
Needs string `json:"needs,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// HealthSaid is the health event's body: the machine and its statement. The machine is read from the
|
||||||
|
// subject the bus let it publish on, never from here.
|
||||||
|
type HealthSaid struct {
|
||||||
|
Node string `json:"node"`
|
||||||
|
Health Health `json:"health"`
|
||||||
|
}
|
||||||
|
|||||||
@@ -25,6 +25,21 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) {
|
|||||||
[]string{"protocol", "address", "port", "by", "published", "container-port"}},
|
[]string{"protocol", "address", "port", "by", "published", "container-port"}},
|
||||||
{EnrolRequest{Node: "n", Secret: "s", PublicKey: []byte("k")},
|
{EnrolRequest{Node: "n", Secret: "s", PublicKey: []byte("k")},
|
||||||
[]string{"node", "secret", "public_key"}},
|
[]string{"node", "secret", "public_key"}},
|
||||||
|
// novox/hq ADR 0240: every long-running resource's health, in every report and in its own event.
|
||||||
|
{Report{Node: "n", Health: &Health{Contract: LivenessContract}}, []string{"node", "health"}},
|
||||||
|
{Health{Contract: LivenessContract, Resources: []ResourceHealth{}}, []string{"contract", "at", "resources"}},
|
||||||
|
{ResourceHealth{Module: "m", Resource: "m.r", Kind: "container", Target: "t", State: StateUnhealthy,
|
||||||
|
Reason: "restarting", Streak: 2, Restarts: 3, Check: "http", Needs: "postgres-database"},
|
||||||
|
[]string{"module", "resource", "kind", "target", "state", "reason", "since", "streak", "restarts",
|
||||||
|
"check", "needs"}},
|
||||||
|
{HealthSaid{Node: "n"}, []string{"node", "health"}},
|
||||||
|
// novox/hq ADR 0241: the machine's own networking, beside its resources.
|
||||||
|
{Health{Contract: ReadinessContract, Resources: []ResourceHealth{}, Network: &NetworkHealth{State: "unhealthy",
|
||||||
|
Parts: []NetworkPart{}}}, []string{"contract", "at", "resources", "network"}},
|
||||||
|
{NetworkHealth{State: "unhealthy", Parts: []NetworkPart{}}, []string{"state", "since", "parts"}},
|
||||||
|
{NetworkPart{Part: "resolv-conf", State: "unhealthy", Reason: "r", Said: "s", Writer: "w", Owner: "o",
|
||||||
|
Toward: []string{"hub"}, Streak: 2}, []string{"part", "state", "reason", "said", "writer", "owner", "toward",
|
||||||
|
"since", "streak"}},
|
||||||
} {
|
} {
|
||||||
raw, err := json.Marshal(c.value)
|
raw, err := json.Marshal(c.value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -30,8 +30,10 @@ const (
|
|||||||
KindHeartbeat = "heartbeat"
|
KindHeartbeat = "heartbeat"
|
||||||
// KindToolsHeartbeat is a machine's node tools saying they are there (novox/hq to-be 45 S11).
|
// KindToolsHeartbeat is a machine's node tools saying they are there (novox/hq to-be 45 S11).
|
||||||
KindToolsHeartbeat = "tools-heartbeat"
|
KindToolsHeartbeat = "tools-heartbeat"
|
||||||
KindBuilt = "built"
|
// KindHealth is a machine's statement of its long-running resources' health (novox/hq ADR 0240).
|
||||||
KindModuleMoved = "module-moved"
|
KindHealth = "health"
|
||||||
|
KindBuilt = "built"
|
||||||
|
KindModuleMoved = "module-moved"
|
||||||
// KindSourceMoved is the forge announcing a merge: a source moved, and what it produces is
|
// KindSourceMoved is the forge announcing a merge: a source moved, and what it produces is
|
||||||
// built without anybody telling the mesh (novox/hq 04-ISSUES/131).
|
// built without anybody telling the mesh (novox/hq 04-ISSUES/131).
|
||||||
KindSourceMoved = "source-moved"
|
KindSourceMoved = "source-moved"
|
||||||
|
|||||||
@@ -108,6 +108,13 @@ func (n *natsInbound) Receive(ctx context.Context, act func(context.Context, Con
|
|||||||
return fmt.Errorf("subscribing to the node tools' heartbeats: %w", err)
|
return fmt.Errorf("subscribing to the node tools' heartbeats: %w", err)
|
||||||
}
|
}
|
||||||
defer func() { _ = toolsAlive.Unsubscribe() }()
|
defer func() { _ = toolsAlive.Unsubscribe() }()
|
||||||
|
// And what each machine says of its long-running resources between reports (novox/hq ADR 0240): core
|
||||||
|
// too, and said again while it matters.
|
||||||
|
health, err := conn.ChanSubscribe(HealthSubjects, beats)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("subscribing to the machines' health: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = health.Unsubscribe() }()
|
||||||
|
|
||||||
// The events the controller follows, when something is listening for them.
|
// The events the controller follows, when something is listening for them.
|
||||||
var events chan *nats.Msg
|
var events chan *nats.Msg
|
||||||
@@ -239,6 +246,8 @@ func kindOfSubject(subject string) (string, bool) {
|
|||||||
return KindHeartbeat, true
|
return KindHeartbeat, true
|
||||||
case "tools-alive":
|
case "tools-alive":
|
||||||
return KindToolsHeartbeat, true
|
return KindToolsHeartbeat, true
|
||||||
|
case "health":
|
||||||
|
return KindHealth, true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
switch subject {
|
switch subject {
|
||||||
|
|||||||
@@ -94,6 +94,8 @@ type Server struct {
|
|||||||
retirements Retirements
|
retirements Retirements
|
||||||
// checker asks for a pull request's merge check (novox/hq to-be 45 §9).
|
// checker asks for a pull request's merge check (novox/hq to-be 45 §9).
|
||||||
checker Checker
|
checker Checker
|
||||||
|
// healths keeps what machines say of their long-running resources (novox/hq ADR 0240).
|
||||||
|
healths Healths
|
||||||
|
|
||||||
log *log.Logger
|
log *log.Logger
|
||||||
// giveUp is how long one message is held for the store; zero means GiveUpAfter.
|
// giveUp is how long one message is held for the store; zero means GiveUpAfter.
|
||||||
@@ -194,6 +196,8 @@ func (s *Server) act(ctx context.Context, m Control) {
|
|||||||
s.heartbeat(m)
|
s.heartbeat(m)
|
||||||
case KindToolsHeartbeat:
|
case KindToolsHeartbeat:
|
||||||
s.toolsHeartbeat(m)
|
s.toolsHeartbeat(m)
|
||||||
|
case KindHealth:
|
||||||
|
s.healthSaid(ctx, m)
|
||||||
case KindBuilt:
|
case KindBuilt:
|
||||||
s.wasBuilt(ctx, m)
|
s.wasBuilt(ctx, m)
|
||||||
case KindModuleMoved:
|
case KindModuleMoved:
|
||||||
|
|||||||
@@ -60,6 +60,7 @@
|
|||||||
"pause",
|
"pause",
|
||||||
"resume",
|
"resume",
|
||||||
"hand-act",
|
"hand-act",
|
||||||
|
"drill",
|
||||||
"hand-acts",
|
"hand-acts",
|
||||||
"durations",
|
"durations",
|
||||||
"conditions",
|
"conditions",
|
||||||
|
|||||||
+41
-3
@@ -590,6 +590,23 @@ type Process struct {
|
|||||||
// For a process that stays up; a step or a scheduled run is not running a moment later by
|
// For a process that stays up; a step or a scheduled run is not running a moment later by
|
||||||
// design, so there is nothing to hand over to.
|
// design, so there is nothing to hand over to.
|
||||||
Replaces []string `json:"replaces,omitempty"`
|
Replaces []string `json:"replaces,omitempty"`
|
||||||
|
|
||||||
|
// Witness is how the node-engine judges a new build of this process, and restores the build
|
||||||
|
// before it when the new one is not healthy in bound (novox/hq to-be 45 §8): "lease" — the
|
||||||
|
// controller this machine started holds the controller's lease; "ping" — this machine's runtime
|
||||||
|
// answers the services protocol's PING; "none". Absent is the default for the process's name:
|
||||||
|
// the mesh's two core processes are judged, nothing else is. For a process that stays up.
|
||||||
|
Witness string `json:"witness,omitempty"`
|
||||||
|
|
||||||
|
// NotReversible says why this build may not be rolled back, when it may not: the build before it
|
||||||
|
// would run against what this one changes — a migration it runs that the older build cannot read
|
||||||
|
// (to-be 45 §8, rule 8). A build so declared that is not healthy in bound is left running and said
|
||||||
|
// as urgent; the build before it is never started against the newer data.
|
||||||
|
NotReversible string `json:"not-reversible,omitempty"`
|
||||||
|
|
||||||
|
// Health is how this resource is ready (novox/hq ADR 0240 rule 2, Phase B): one kind and its
|
||||||
|
// timing, judged by the node-engine beside liveness. Absent: judged alive or not, and nothing more.
|
||||||
|
Health *Health `json:"health,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *Process) Identity() string { return d.ID }
|
func (d *Process) Identity() string { return d.ID }
|
||||||
@@ -621,7 +638,7 @@ func ProcessNameProblem(name string) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (d *Process) validate(where string, _ bool) []string {
|
func (d *Process) validate(where string, _ bool) []string {
|
||||||
var problems []string
|
problems := d.Health.problems(where, false, !d.RunOnce && d.Schedule == "")
|
||||||
if problem := ProcessNameProblem(d.Name); problem != "" {
|
if problem := ProcessNameProblem(d.Name); problem != "" {
|
||||||
problems = append(problems, where+": "+problem)
|
problems = append(problems, where+": "+problem)
|
||||||
}
|
}
|
||||||
@@ -637,6 +654,19 @@ func (d *Process) validate(where string, _ bool) []string {
|
|||||||
if len(d.Run) == 0 {
|
if len(d.Run) == 0 {
|
||||||
problems = append(problems, where+": a process needs to say what to run")
|
problems = append(problems, where+": a process needs to say what to run")
|
||||||
}
|
}
|
||||||
|
switch d.Witness {
|
||||||
|
case "", "lease", "ping", "none":
|
||||||
|
default:
|
||||||
|
problems = append(problems, fmt.Sprintf("%s: witness %q is not one this host keeps: lease, ping or none",
|
||||||
|
where, d.Witness))
|
||||||
|
}
|
||||||
|
if d.Witness != "" && d.Witness != "none" && (d.RunOnce || d.Schedule != "") {
|
||||||
|
problems = append(problems, where+": a witness judges a process that stays up; a step or a "+
|
||||||
|
"scheduled run is not running between its runs")
|
||||||
|
}
|
||||||
|
if strings.ContainsAny(d.NotReversible, "\n\r") {
|
||||||
|
problems = append(problems, where+": not-reversible is one line")
|
||||||
|
}
|
||||||
for _, part := range d.Run {
|
for _, part := range d.Run {
|
||||||
if part == "" {
|
if part == "" {
|
||||||
problems = append(problems, where+": a process command has an empty element")
|
problems = append(problems, where+": a process command has an empty element")
|
||||||
@@ -756,6 +786,10 @@ type Service struct {
|
|||||||
// said by the controller, which knows the found tunnel's key is this node's own: without that,
|
// said by the controller, which knows the found tunnel's key is this node's own: without that,
|
||||||
// starting this unit on the found one's port would drop every peer's packets.
|
// starting this unit on the found one's port would drop every peer's packets.
|
||||||
TakesOver *TakeOver `json:"takes-over,omitempty"`
|
TakesOver *TakeOver `json:"takes-over,omitempty"`
|
||||||
|
|
||||||
|
// Health is how this resource is ready (novox/hq ADR 0240 rule 2, Phase B): one kind and its
|
||||||
|
// timing, judged by the node-engine beside liveness. Absent: judged alive or not, and nothing more.
|
||||||
|
Health *Health `json:"health,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// TakeOver is a found tunnel a service replaces: its interface, the unit that raised it, and its
|
// TakeOver is a found tunnel a service replaces: its interface, the unit that raised it, and its
|
||||||
@@ -784,7 +818,7 @@ const (
|
|||||||
func (s *Service) UserScoped() bool { return s.Scope == ScopeUser }
|
func (s *Service) UserScoped() bool { return s.Scope == ScopeUser }
|
||||||
|
|
||||||
func (s *Service) validate(where string, _ bool) []string {
|
func (s *Service) validate(where string, _ bool) []string {
|
||||||
var problems []string
|
problems := s.Health.problems(where, false, s.State == "running")
|
||||||
if s.Unit == "" {
|
if s.Unit == "" {
|
||||||
problems = append(problems, where+": a service needs a unit")
|
problems = append(problems, where+": a service needs a unit")
|
||||||
}
|
}
|
||||||
@@ -1096,6 +1130,10 @@ type Container struct {
|
|||||||
// offline job says *before*, not *instead of*; a recurring window is the case order cannot
|
// offline job says *before*, not *instead of*; a recurring window is the case order cannot
|
||||||
// express, and the only one this serves.
|
// express, and the only one this serves.
|
||||||
WhileStopped []string `json:"while-stopped,omitempty"`
|
WhileStopped []string `json:"while-stopped,omitempty"`
|
||||||
|
|
||||||
|
// Health is how this resource is ready (novox/hq ADR 0240 rule 2, Phase B): one kind and its
|
||||||
|
// timing, judged by the node-engine beside liveness. Absent: judged alive or not, and nothing more.
|
||||||
|
Health *Health `json:"health,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Container) Identity() string { return c.ID }
|
func (c *Container) Identity() string { return c.ID }
|
||||||
@@ -1103,7 +1141,7 @@ func (c *Container) Kind() Type { return TypeContainer }
|
|||||||
func (c *Container) Target() string { return c.Name }
|
func (c *Container) Target() string { return c.Name }
|
||||||
|
|
||||||
func (c *Container) validate(where string, _ bool) []string {
|
func (c *Container) validate(where string, _ bool) []string {
|
||||||
var problems []string
|
problems := c.Health.problems(where, true, !c.RunOnce && c.Schedule == "")
|
||||||
if c.Name == "" {
|
if c.Name == "" {
|
||||||
problems = append(problems, where+": a container needs a name")
|
problems = append(problems, where+": a container needs a name")
|
||||||
}
|
}
|
||||||
|
|||||||
+188
@@ -0,0 +1,188 @@
|
|||||||
|
package declaration
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Health is how a long-running resource is ready, as the controller composed it from the module's
|
||||||
|
// `health` (novox/hq ADR 0240 rule 2, to-be 48 §2–§3, Phase B): one kind and its timing, the endpoint
|
||||||
|
// already the port this machine published it on.
|
||||||
|
//
|
||||||
|
// **The node-engine runs every kind and owns every verdict.** http and tcp it makes itself, from the
|
||||||
|
// machine to the port; unit it reads from the service manager it already reads; exec and runtime it hands
|
||||||
|
// to the container runtime as the container's own check, with this timing, and reads the state; tool it
|
||||||
|
// asks of its own node tools. Nothing else on the machine sets a container's check.
|
||||||
|
//
|
||||||
|
// Refused here as the controller refuses it near the author, in the same bounds: an engine that took a
|
||||||
|
// check it could not judge would say a module ready that nothing looked at.
|
||||||
|
type Health struct {
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
// Endpoint is the module's name for what Port is: for the words a verdict is said in.
|
||||||
|
Endpoint string `json:"endpoint,omitempty"`
|
||||||
|
Port int `json:"port,omitempty"`
|
||||||
|
Path string `json:"path,omitempty"`
|
||||||
|
Status int `json:"status,omitempty"`
|
||||||
|
Body string `json:"body,omitempty"`
|
||||||
|
Scheme string `json:"scheme,omitempty"`
|
||||||
|
Command string `json:"command,omitempty"`
|
||||||
|
Tool string `json:"tool,omitempty"`
|
||||||
|
Interval string `json:"interval"`
|
||||||
|
Timeout string `json:"timeout"`
|
||||||
|
Looks int `json:"looks"`
|
||||||
|
Grace string `json:"grace"`
|
||||||
|
// Needs is the provision the check exercises (to-be 48 §6): said with every verdict, so the
|
||||||
|
// controller can hold what it finds under the provider's own condition.
|
||||||
|
Needs string `json:"needs,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// UnmarshalJSON reads a health strictly, as everything a declaration carries is read: a field this host
|
||||||
|
// does not know is a part of the check the controller believes it asked for, and nothing would look at it.
|
||||||
|
func (h *Health) UnmarshalJSON(raw []byte) error {
|
||||||
|
type plain Health
|
||||||
|
var p plain
|
||||||
|
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||||
|
dec.DisallowUnknownFields()
|
||||||
|
if err := dec.Decode(&p); err != nil {
|
||||||
|
return fmt.Errorf("health: %w", err)
|
||||||
|
}
|
||||||
|
*h = Health(p)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// The kinds.
|
||||||
|
const (
|
||||||
|
HealthRuntime = "runtime"
|
||||||
|
HealthHTTP = "http"
|
||||||
|
HealthTCP = "tcp"
|
||||||
|
HealthExec = "exec"
|
||||||
|
HealthUnit = "unit"
|
||||||
|
HealthTool = "tool"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The bounds (ADR 0240 rule 2) — the controller's, held again here.
|
||||||
|
const (
|
||||||
|
HealthIntervalFloor = 10 * time.Second
|
||||||
|
HealthLooksFloor = 2
|
||||||
|
HealthWithin = 5 * time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
|
// Every, Within and GraceOf are the timing, read. Validated on arrival, so a parse error here is
|
||||||
|
// impossible on a declaration that was accepted; it reads as zero.
|
||||||
|
func (h *Health) Every() time.Duration { d, _ := time.ParseDuration(h.Interval); return d }
|
||||||
|
func (h *Health) Within() time.Duration { d, _ := time.ParseDuration(h.Timeout); return d }
|
||||||
|
func (h *Health) GraceOf() time.Duration { d, _ := time.ParseDuration(h.Grace); return d }
|
||||||
|
|
||||||
|
// RunByRuntime says the container runtime runs this check as the container's own: exec and runtime.
|
||||||
|
func (h *Health) RunByRuntime() bool {
|
||||||
|
return h != nil && (h.Kind == HealthExec || h.Kind == HealthRuntime)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Words is the check in a few words, as a verdict is said: "http /healthz on web".
|
||||||
|
func (h *Health) Words() string {
|
||||||
|
switch h.Kind {
|
||||||
|
case HealthHTTP:
|
||||||
|
return "http " + h.Path + " on " + orPort(h.Endpoint, h.Port)
|
||||||
|
case HealthTCP:
|
||||||
|
return "tcp on " + orPort(h.Endpoint, h.Port)
|
||||||
|
case HealthTool:
|
||||||
|
return "its tool " + h.Tool
|
||||||
|
case HealthRuntime:
|
||||||
|
return "its image's own check"
|
||||||
|
case HealthExec:
|
||||||
|
return "its command"
|
||||||
|
case HealthUnit:
|
||||||
|
return "its unit"
|
||||||
|
}
|
||||||
|
return h.Kind
|
||||||
|
}
|
||||||
|
|
||||||
|
func orPort(endpoint string, port int) string {
|
||||||
|
if endpoint != "" {
|
||||||
|
return endpoint
|
||||||
|
}
|
||||||
|
return fmt.Sprint(port)
|
||||||
|
}
|
||||||
|
|
||||||
|
// problems holds a resource's health to its kind and bounds. container says whether the resource is a
|
||||||
|
// container; longRunning whether it stays up.
|
||||||
|
func (h *Health) problems(where string, container, longRunning bool) []string {
|
||||||
|
if h == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var problems []string
|
||||||
|
say := func(format string, args ...any) {
|
||||||
|
problems = append(problems, where+": "+fmt.Sprintf(format, args...))
|
||||||
|
}
|
||||||
|
if !longRunning {
|
||||||
|
say("health is judged on what stays up; a step or a scheduled run is judged by its own outcome")
|
||||||
|
}
|
||||||
|
switch h.Kind {
|
||||||
|
case HealthHTTP, HealthTCP:
|
||||||
|
if h.Port < 1 || h.Port > 65535 {
|
||||||
|
say("a %s check needs the port it looks at", h.Kind)
|
||||||
|
}
|
||||||
|
case HealthExec:
|
||||||
|
if !container {
|
||||||
|
say("an exec check runs inside a container")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(h.Command) == "" {
|
||||||
|
say("an exec check needs a command")
|
||||||
|
}
|
||||||
|
case HealthRuntime:
|
||||||
|
if !container {
|
||||||
|
say("a runtime check is a container image's own")
|
||||||
|
}
|
||||||
|
case HealthUnit:
|
||||||
|
if container {
|
||||||
|
say("a unit check is a service's or a process's own")
|
||||||
|
}
|
||||||
|
case HealthTool:
|
||||||
|
if strings.TrimSpace(h.Tool) == "" {
|
||||||
|
say("a tool check names the tool")
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
say("health of kind %q; it is runtime, http, tcp, exec, unit or tool", h.Kind)
|
||||||
|
}
|
||||||
|
if h.Kind == HealthHTTP {
|
||||||
|
if !strings.HasPrefix(h.Path, "/") {
|
||||||
|
say("an http check asks a path starting with /")
|
||||||
|
}
|
||||||
|
if h.Status != 0 && (h.Status < 100 || h.Status > 599) {
|
||||||
|
say("an http check expects status %d, which is not one", h.Status)
|
||||||
|
}
|
||||||
|
if h.Scheme != "" && h.Scheme != "http" && h.Scheme != "https" {
|
||||||
|
say("an http check is over http or https, not %q", h.Scheme)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.ContainsAny(h.Command, "\n\r") {
|
||||||
|
say("an exec check's command is one line")
|
||||||
|
}
|
||||||
|
every, everyErr := time.ParseDuration(h.Interval)
|
||||||
|
within, withinErr := time.ParseDuration(h.Timeout)
|
||||||
|
grace, graceErr := time.ParseDuration(h.Grace)
|
||||||
|
switch {
|
||||||
|
case everyErr != nil || withinErr != nil || graceErr != nil:
|
||||||
|
say("health's interval, timeout and grace are durations")
|
||||||
|
default:
|
||||||
|
if every < HealthIntervalFloor {
|
||||||
|
say("a check looks no more often than every %s, not every %s", HealthIntervalFloor, every)
|
||||||
|
}
|
||||||
|
if within <= 0 || within >= every {
|
||||||
|
say("a look takes more than nothing and less than its interval")
|
||||||
|
}
|
||||||
|
if grace < 0 {
|
||||||
|
say("a grace is not negative")
|
||||||
|
}
|
||||||
|
if h.Looks >= HealthLooksFloor && grace+time.Duration(h.Looks)*every > HealthWithin {
|
||||||
|
say("a grace and the failing looks take at most %s", HealthWithin)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if h.Looks < HealthLooksFloor {
|
||||||
|
say("a check is unhealthy after at least %d failing looks, not %d", HealthLooksFloor, h.Looks)
|
||||||
|
}
|
||||||
|
return problems
|
||||||
|
}
|
||||||
Vendored
+2
-2
@@ -75,7 +75,7 @@ github.com/nats-io/nkeys
|
|||||||
# github.com/nats-io/nuid v1.0.1
|
# github.com/nats-io/nuid v1.0.1
|
||||||
## explicit
|
## explicit
|
||||||
github.com/nats-io/nuid
|
github.com/nats-io/nuid
|
||||||
# github.com/novox/mesh-host v0.0.0 => git.novox.be/novox/mesh-host v0.0.0-20261006095519-3e80b7ae325e
|
# github.com/novox/mesh-host v0.0.0 => git.novox.be/novox/mesh-host v0.0.0-20261007120832-bdd44154ccac
|
||||||
## explicit; go 1.26.0
|
## explicit; go 1.26.0
|
||||||
github.com/novox/mesh-host/internal/declaration
|
github.com/novox/mesh-host/internal/declaration
|
||||||
github.com/novox/mesh-host/validate
|
github.com/novox/mesh-host/validate
|
||||||
@@ -133,4 +133,4 @@ golang.org/x/text/width
|
|||||||
# golang.org/x/time v0.15.0
|
# golang.org/x/time v0.15.0
|
||||||
## explicit; go 1.25.0
|
## explicit; go 1.25.0
|
||||||
golang.org/x/time/rate
|
golang.org/x/time/rate
|
||||||
# github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261006095519-3e80b7ae325e
|
# github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261007120832-bdd44154ccac
|
||||||
|
|||||||
Reference in New Issue
Block a user