Compare commits
13
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4469cab7f4 | ||
|
|
65610f2ea2 | ||
|
|
85d664438f | ||
|
|
13b6fc6d97 | ||
|
|
d851573793 | ||
|
|
f6aea4bfbb | ||
|
|
e550c95543 | ||
|
|
1fdff00794 | ||
|
|
5efe999733 | ||
|
|
cdf30349a7 | ||
|
|
40e42606cf | ||
|
|
b8bbf9c79f | ||
|
|
8bcf787258 |
@@ -283,6 +283,8 @@ type machineWord struct {
|
||||
facts gateFacts
|
||||
on map[string]string
|
||||
whole string
|
||||
// waiting is what holds the machine on something shown to be another's (ADR 0241): the judging waits.
|
||||
waiting string
|
||||
}
|
||||
|
||||
// kindCoreBehind is D10's kind: a machine runs core components older than the mesh holds, or has not
|
||||
@@ -346,6 +348,13 @@ func aboutTheMachine(machine string, moved []string, since time.Time, f gateFact
|
||||
}
|
||||
case coreBehind:
|
||||
// Not what the send moved: said nowhere against it.
|
||||
case c.Kind == kindNetworkRewritten || c.Kind == kindNetworkUnreachable && c.Subject.ID != machine:
|
||||
// **Shown to be somebody else's** (ADR 0241): another program rewrote the resolver file the send
|
||||
// did not move, or the machine cannot reach another that is down. Nothing the send did; the
|
||||
// judging waits for it rather than putting back a build at the bound.
|
||||
if w.waiting == "" {
|
||||
w.waiting = machine + "'s network: " + said
|
||||
}
|
||||
default:
|
||||
if w.whole == "" {
|
||||
w.whole = machine + " as a whole: " + said
|
||||
@@ -433,6 +442,8 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
h, said = healthNotYet, on
|
||||
} else if w.whole != "" {
|
||||
h, said = healthNotYet, w.whole
|
||||
} else if w.waiting != "" {
|
||||
h, said = healthWaiting, w.waiting
|
||||
}
|
||||
}
|
||||
if h != healthGood && !slices.Contains(failing, j.module) {
|
||||
|
||||
@@ -0,0 +1,359 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A machine says how its network is (novox/hq ADR 0241, which extends ADR 0240 from what a module runs
|
||||
// to the machine it runs on).
|
||||
//
|
||||
// **Every machine's node-engine judges its own networking** — the resolver file the uplink holder
|
||||
// declared, the names through every resolver it lists, the tunnel's handshake with the hub, the bus, the
|
||||
// default route — on the two-look rule, and states it beside its resources. The controller keeps the
|
||||
// newest statement per machine and raises from all of them together:
|
||||
//
|
||||
// - **an outside writer of the resolver file is its own finding**, `machine.<m>.<owner>.rewritten`:
|
||||
// the file the uplink holder declares was rewritten by another program, named where the engine could
|
||||
// name it. The names failing through what that program wrote are that finding's consequence, said in
|
||||
// it — never a second condition;
|
||||
// - **what is the machine's own** — its route, its tunnel, its resolvers answering wrong, a resolver
|
||||
// that is no mesh machine — is `machine.<m>.network`;
|
||||
// - **what points at another machine is said once, there** (the provider hold of ADR 0240 rule 5, for
|
||||
// the network): a failure toward the hub or toward a mesh resolver is held under that machine when it
|
||||
// is down on the record — silent, or its own network unhealthy — or when a second machine finds the
|
||||
// same; then `machine.<x>.unreachable` names every machine that cannot reach it, and none of them
|
||||
// raises anything of its own for it. One machine alone failing toward a healthy one is its own.
|
||||
//
|
||||
// Each is a warning; urgent on the control node, or when the bus cannot be reached, or for the hub.
|
||||
// Cleared on the first statement that no longer says it. An engine older than this judging says nothing
|
||||
// of its network, and nothing is raised for it.
|
||||
|
||||
// The conditions a machine's network raises.
|
||||
const (
|
||||
kindMachineNetwork = "machine-network"
|
||||
kindNetworkRewritten = "network-rewritten"
|
||||
kindNetworkUnreachable = "network-unreachable"
|
||||
sourceNetwork = "network"
|
||||
)
|
||||
|
||||
// networkKinds are the kinds this judging owns: every open one it no longer says, it clears.
|
||||
var networkKinds = []string{kindMachineNetwork, kindNetworkRewritten, kindNetworkUnreachable}
|
||||
|
||||
// networkFacts is what one judging of every machine's network reads.
|
||||
type networkFacts struct {
|
||||
healths map[string]inventory.NodeHealth
|
||||
// byAddress is each machine's address on the private network; hub the hub's name; control the
|
||||
// control node's.
|
||||
byAddress map[string]string
|
||||
hub string
|
||||
control string
|
||||
// silent is every machine whose silence is an open condition.
|
||||
silent map[string]bool
|
||||
}
|
||||
|
||||
// judgeNetworks raises and clears every machine's network conditions from every machine's newest
|
||||
// statement, after one machine's statement was kept.
|
||||
func judgeNetworks(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, now time.Time) error {
|
||||
healths, err := inv.Healths(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
overlays, err := inv.Overlays(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
open, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f := networkFacts{healths: healths, byAddress: map[string]string{}, control: controlHost(ctx, inv),
|
||||
silent: map[string]bool{}}
|
||||
for _, o := range overlays {
|
||||
if o.Address != "" {
|
||||
f.byAddress[o.Address] = o.Name
|
||||
}
|
||||
if o.Hub {
|
||||
f.hub = o.Name
|
||||
}
|
||||
}
|
||||
for _, c := range open {
|
||||
if c.Subject.Scope == conditions.ScopeMachine && c.Kind == "silent" {
|
||||
f.silent[c.Subject.ID] = true
|
||||
}
|
||||
}
|
||||
var problems []string
|
||||
said := map[string]bool{}
|
||||
for _, o := range networkObservations(f) {
|
||||
said[o.Key()] = true
|
||||
if _, err := k.Observe(ctx, o); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
}
|
||||
for _, c := range open {
|
||||
if !slices.Contains(networkKinds, c.Kind) || said[c.Key] {
|
||||
continue
|
||||
}
|
||||
why := "no machine says it any more"
|
||||
if c.Subject.Machine != "" {
|
||||
why = c.Subject.Machine + "'s network no longer says it"
|
||||
}
|
||||
if _, err := k.Clear(ctx, c.Key, why); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
return fmt.Errorf("%s", strings.Join(problems, "; "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// pointed is one machine's failing part that points at another machine.
|
||||
type pointed struct {
|
||||
from string
|
||||
part inventory.NetworkPart
|
||||
}
|
||||
|
||||
// networkObservations is every network condition the statements say now. Pure.
|
||||
func networkObservations(f networkFacts) []conditions.Observation {
|
||||
machines := make([]string, 0, len(f.healths))
|
||||
for m := range f.healths {
|
||||
machines = append(machines, m)
|
||||
}
|
||||
sort.Strings(machines)
|
||||
|
||||
unhealthy := func(m string) []inventory.NetworkPart {
|
||||
h := f.healths[m]
|
||||
if h.Network == nil {
|
||||
return nil
|
||||
}
|
||||
var out []inventory.NetworkPart
|
||||
for _, p := range h.Network.Parts {
|
||||
if p.State == link.StateUnhealthy {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
// The machines a part points at, other than its own: the hub, and each mesh resolver by its address.
|
||||
// An address that is no mesh machine's — the resolver a VPN client wrote in — is the machine's own.
|
||||
targets := func(m string, p inventory.NetworkPart) ([]string, bool) {
|
||||
if len(p.Toward) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
var out []string
|
||||
for _, t := range p.Toward {
|
||||
x := f.byAddress[t]
|
||||
if t == link.TowardHub {
|
||||
x = f.hub
|
||||
}
|
||||
if x == "" || x == m {
|
||||
return nil, false
|
||||
}
|
||||
if !slices.Contains(out, x) {
|
||||
out = append(out, x)
|
||||
}
|
||||
}
|
||||
return out, true
|
||||
}
|
||||
|
||||
// First pass: what points at whom.
|
||||
pointing := map[string][]pointed{}
|
||||
for _, m := range machines {
|
||||
for _, p := range unhealthy(m) {
|
||||
if xs, ok := targets(m, p); ok {
|
||||
for _, x := range xs {
|
||||
pointing[x] = append(pointing[x], pointed{from: m, part: p})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
from := func(x string) []string {
|
||||
var out []string
|
||||
for _, pt := range pointing[x] {
|
||||
if !slices.Contains(out, pt.from) {
|
||||
out = append(out, pt.from)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
// A machine is down on the record when its silence is open or its own network is unhealthy, or when
|
||||
// two machines find it unreachable: then what points at it is held there.
|
||||
down := func(x string) bool {
|
||||
return f.silent[x] || len(unhealthy(x)) > 0 || len(from(x)) >= 2
|
||||
}
|
||||
|
||||
var out []conditions.Observation
|
||||
saysOwn := map[string]bool{}
|
||||
for _, m := range machines {
|
||||
parts := unhealthy(m)
|
||||
if len(parts) == 0 {
|
||||
continue
|
||||
}
|
||||
var own []inventory.NetworkPart
|
||||
var rewritten *inventory.NetworkPart
|
||||
for i, p := range parts {
|
||||
if p.Part == link.PartResolvConf {
|
||||
rewritten = &parts[i]
|
||||
continue
|
||||
}
|
||||
if xs, ok := targets(m, p); ok && allDown(xs, down) {
|
||||
continue // held at the machines it points at
|
||||
}
|
||||
own = append(own, p)
|
||||
}
|
||||
if rewritten != nil {
|
||||
out = append(out, rewrittenObservation(m, *rewritten, parts, f))
|
||||
// The names failing through what another program wrote are that finding's, said in it.
|
||||
kept := own[:0]
|
||||
for _, p := range own {
|
||||
if p.Part != link.PartNames {
|
||||
kept = append(kept, p)
|
||||
}
|
||||
}
|
||||
own = kept
|
||||
}
|
||||
if len(own) > 0 {
|
||||
out = append(out, machineNetworkObservation(m, own, f, from(m)))
|
||||
saysOwn[m] = true
|
||||
}
|
||||
}
|
||||
// Said once, at the machine everybody points at — unless its own network condition already says it
|
||||
// (listed there), or its silence does.
|
||||
targetsSorted := make([]string, 0, len(pointing))
|
||||
for x := range pointing {
|
||||
targetsSorted = append(targetsSorted, x)
|
||||
}
|
||||
sort.Strings(targetsSorted)
|
||||
for _, x := range targetsSorted {
|
||||
if !down(x) || saysOwn[x] || f.silent[x] {
|
||||
continue
|
||||
}
|
||||
out = append(out, unreachableObservation(x, pointing[x], from(x), f))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func allDown(xs []string, down func(string) bool) bool {
|
||||
for _, x := range xs {
|
||||
if !down(x) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(xs) > 0
|
||||
}
|
||||
|
||||
// rewrittenObservation is the resolver file rewritten by another program: its own finding, naming the
|
||||
// writer where the engine could, and what it costs the machine.
|
||||
func rewrittenObservation(m string, p inventory.NetworkPart, all []inventory.NetworkPart, f networkFacts) conditions.Observation {
|
||||
writer := ""
|
||||
if p.Writer != "" {
|
||||
writer = " (" + p.Writer + ")"
|
||||
}
|
||||
cost := "the names through it are not yet judged"
|
||||
said := []string{p.Part + ": " + p.Said}
|
||||
for _, q := range all {
|
||||
if q.Part == link.PartNames {
|
||||
cost = strings.TrimSuffix(q.Reason, ".")
|
||||
said = append(said, q.Part+": "+q.Said)
|
||||
}
|
||||
}
|
||||
if cost == "the names through it are not yet judged" {
|
||||
cost = "the names still resolve through what it wrote"
|
||||
}
|
||||
id := m
|
||||
if p.Owner != "" {
|
||||
// Named by the module whose file it is: a send that moved that module is what the gate
|
||||
// holds it on (issue 281's rule — what names a moved module is that module's).
|
||||
id = m + "." + p.Owner
|
||||
}
|
||||
severity := conditions.Warning
|
||||
if m == f.control {
|
||||
severity = conditions.Urgent
|
||||
}
|
||||
summary := fmt.Sprintf("the resolver file on %s was rewritten by another program%s — %s until the "+
|
||||
"node-engine writes it back at its next reconcile, or that program gives it back", m, writer, cost)
|
||||
if p.Owner != "" {
|
||||
summary = fmt.Sprintf("the resolver file %s writes on %s was rewritten by another program%s — %s until "+
|
||||
"the node-engine writes it back at its next reconcile, or that program gives it back", p.Owner, m, writer, cost)
|
||||
}
|
||||
return conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: "rewritten", Kind: kindNetworkRewritten,
|
||||
Machine: m, Severity: severity, Source: sourceNetwork, Summary: summary,
|
||||
Said: fmt.Sprintf("since %s: %s", p.Since.UTC().Format("2006-01-02 15:04:05 MST"), strings.Join(said, " | "))}
|
||||
}
|
||||
|
||||
// machineNetworkObservation is what is wrong with a machine's own networking.
|
||||
func machineNetworkObservation(m string, parts []inventory.NetworkPart, f networkFacts, waiting []string) conditions.Observation {
|
||||
var words, said []string
|
||||
severity := conditions.Warning
|
||||
for _, p := range parts {
|
||||
words = append(words, p.Reason)
|
||||
said = append(said, fmt.Sprintf("%s since %s: %s", p.Part, p.Since.UTC().Format("2006-01-02 15:04:05 MST"), p.Said))
|
||||
if p.Part == link.PartBus {
|
||||
severity = conditions.Urgent
|
||||
}
|
||||
}
|
||||
if m == f.control || m == f.hub {
|
||||
severity = conditions.Urgent
|
||||
}
|
||||
summary := fmt.Sprintf("%s's network is not healthy: %s", m, strings.Join(words, "; "))
|
||||
if len(waiting) > 0 {
|
||||
severity = conditions.Urgent
|
||||
summary += fmt.Sprintf("; %s cannot reach it", strings.Join(waiting, ", "))
|
||||
}
|
||||
return conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Token: "network", Kind: kindMachineNetwork,
|
||||
Machine: m, Severity: severity, Source: sourceNetwork, Summary: summary, Said: strings.Join(said, " | ")}
|
||||
}
|
||||
|
||||
// unreachableObservation is one machine others cannot reach, said once there.
|
||||
func unreachableObservation(x string, pts []pointed, from []string, f networkFacts) conditions.Observation {
|
||||
var what []string
|
||||
var said []string
|
||||
for _, pt := range pts {
|
||||
w := map[string]string{link.PartTunnel: "the tunnel to it", link.PartBus: "the bus on it",
|
||||
link.PartNames: "its resolver"}[pt.part.Part]
|
||||
if w == "" {
|
||||
w = pt.part.Part
|
||||
}
|
||||
if !slices.Contains(what, w) {
|
||||
what = append(what, w)
|
||||
}
|
||||
said = append(said, fmt.Sprintf("%s: %s: %s", pt.from, pt.part.Part, pt.part.Said))
|
||||
}
|
||||
severity := conditions.Warning
|
||||
if x == f.hub || x == f.control || slices.Contains(what, "the bus on it") {
|
||||
severity = conditions.Urgent
|
||||
}
|
||||
return conditions.Observation{Scope: conditions.ScopeMachine, ID: x, Token: "unreachable", Kind: kindNetworkUnreachable,
|
||||
Machine: x, Also: from, Severity: severity, Source: sourceNetwork,
|
||||
Summary: fmt.Sprintf("%s cannot be reached from %s: %s", x, strings.Join(from, ", "), strings.Join(what, ", ")),
|
||||
Said: strings.Join(said, " | ")}
|
||||
}
|
||||
|
||||
// networkLines is what `node show` says of a machine's networking.
|
||||
func networkLines(h inventory.NodeHealth, had bool, now time.Time) []string {
|
||||
if !had || h.Network == nil {
|
||||
return []string{" its node-engine does not say how its network is — it is older than that judging (ADR 0241)"}
|
||||
}
|
||||
out := []string{fmt.Sprintf(" its network: %s since %s", h.Network.State, h.Network.Since.Local().Format("2006-01-02 15:04"))}
|
||||
for _, p := range h.Network.Parts {
|
||||
line := fmt.Sprintf(" %-10s %s", p.State, p.Part)
|
||||
if p.Reason != "" {
|
||||
line += " — " + p.Reason
|
||||
}
|
||||
if p.Writer != "" {
|
||||
line += " (" + p.Writer + ")"
|
||||
}
|
||||
out = append(out, line)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,306 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A machine says how its network is (novox/hq ADR 0241, "how it is checked"): the resolver file rewritten
|
||||
// by another program is one finding, naming the writer, with the names it costs said in it; what is the
|
||||
// machine's own is `machine.<m>.network`; what points at another machine that is down is said once, there;
|
||||
// one machine alone failing toward a healthy one is its own; the control node, the hub and the bus are
|
||||
// urgent; an engine that says nothing of its network raises nothing; and the gate waits on what is shown to
|
||||
// be another's.
|
||||
|
||||
func aNetwork(state string, parts ...inventory.NetworkPart) *inventory.NetworkHealth {
|
||||
for i := range parts {
|
||||
if parts[i].State == "" {
|
||||
parts[i].State = link.StateUnhealthy
|
||||
}
|
||||
parts[i].Since = h0
|
||||
}
|
||||
return &inventory.NetworkHealth{State: state, Since: h0, Parts: parts}
|
||||
}
|
||||
|
||||
func netFacts(healths map[string]*inventory.NetworkHealth) networkFacts {
|
||||
f := networkFacts{healths: map[string]inventory.NodeHealth{}, hub: "anchor", control: "anchor",
|
||||
byAddress: map[string]string{"10.77.0.1": "anchor", "10.77.0.2": "laptop", "10.77.0.3": "spare"},
|
||||
silent: map[string]bool{}}
|
||||
for m, n := range healths {
|
||||
f.healths[m] = inventory.NodeHealth{Node: m, Network: n}
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
var (
|
||||
rewrittenByVPN = inventory.NetworkPart{Part: link.PartResolvConf, Reason: "the resolver file was rewritten by another program",
|
||||
Said: "/etc/resolv.conf lists 172.16.5.5 where 10.77.0.1 is declared", Writer: "FortiClient", Owner: "networkmanager"}
|
||||
namesThroughVPN = inventory.NetworkPart{Part: link.PartNames, Reason: "mesh names do not resolve",
|
||||
Said: "172.16.5.5 — anchor.internal (IPv4): says no such name", Toward: []string{"172.16.5.5"}}
|
||||
tunnelDown = inventory.NetworkPart{Part: link.PartTunnel, Reason: "the tunnel to the hub has not handshaken for over five minutes",
|
||||
Said: "mesh0's newest handshake with the hub was 9m0s ago", Toward: []string{link.TowardHub}}
|
||||
noRoute = inventory.NetworkPart{Part: link.PartRoute, Reason: "the machine has no default route", Said: "no default route"}
|
||||
)
|
||||
|
||||
func keysOf(obs []conditions.Observation) []string {
|
||||
var keys []string
|
||||
for _, o := range obs {
|
||||
keys = append(keys, o.Key())
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
func TestAResolverFileRewrittenIsOneFindingNamingItsWriterAndWhatItCosts(t *testing.T) {
|
||||
obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
|
||||
"anchor": aNetwork(link.StateHealthy),
|
||||
"laptop": aNetwork(link.StateUnhealthy, rewrittenByVPN, namesThroughVPN),
|
||||
}))
|
||||
if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" {
|
||||
t.Fatalf("want one finding, the rewrite, got %v", keys)
|
||||
}
|
||||
o := obs[0]
|
||||
for _, want := range []string{"laptop", "rewritten by another program (FortiClient)", "mesh names do not resolve",
|
||||
"next reconcile"} {
|
||||
if !strings.Contains(o.Summary, want) {
|
||||
t.Errorf("the summary does not say %q: %s", want, o.Summary)
|
||||
}
|
||||
}
|
||||
if strings.Contains(o.Summary, "172.16.") || strings.Contains(o.Summary, "/etc/") {
|
||||
t.Errorf("an address or a path reached the summary: %s", o.Summary)
|
||||
}
|
||||
if !strings.Contains(o.Said, "172.16.5.5") || o.Severity != conditions.Warning || o.Machine != "laptop" {
|
||||
t.Errorf("the evidence or the severity is wrong: %+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOneMachineFailingTowardAHealthyHubIsItsOwn(t *testing.T) {
|
||||
obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
|
||||
"anchor": aNetwork(link.StateHealthy),
|
||||
"laptop": aNetwork(link.StateUnhealthy, tunnelDown),
|
||||
"spare": aNetwork(link.StateHealthy),
|
||||
}))
|
||||
if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" {
|
||||
t.Fatalf("want the laptop's own, got %v", keys)
|
||||
}
|
||||
if obs[0].Severity != conditions.Warning {
|
||||
t.Fatalf("a laptop's own tunnel is a warning, got %s", obs[0].Severity)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTwoMachinesThatCannotReachTheHubAreSaidOnceAtTheHub(t *testing.T) {
|
||||
obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
|
||||
"anchor": aNetwork(link.StateHealthy),
|
||||
"laptop": aNetwork(link.StateUnhealthy, tunnelDown),
|
||||
"spare": aNetwork(link.StateUnhealthy, tunnelDown),
|
||||
}))
|
||||
if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.unreachable" {
|
||||
t.Fatalf("want one condition at the hub, got %v", keys)
|
||||
}
|
||||
o := obs[0]
|
||||
if o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop, spare") || len(o.Also) != 2 {
|
||||
t.Fatalf("the hub's condition is %+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhatPointsAtASilentHubIsHeldUnderItsSilence(t *testing.T) {
|
||||
f := netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, tunnelDown)})
|
||||
f.silent["anchor"] = true
|
||||
if obs := networkObservations(f); len(obs) != 0 {
|
||||
t.Fatalf("the hub's silence says it; got %v", keysOf(obs))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAHubWhoseOwnNetworkIsUnhealthyListsWhoCannotReachIt(t *testing.T) {
|
||||
obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
|
||||
"anchor": aNetwork(link.StateUnhealthy, noRoute),
|
||||
"laptop": aNetwork(link.StateUnhealthy, tunnelDown),
|
||||
}))
|
||||
if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.network" {
|
||||
t.Fatalf("want the hub's own, holding the laptop's, got %v", keys)
|
||||
}
|
||||
if o := obs[0]; o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop cannot reach it") {
|
||||
t.Fatalf("the hub's condition is %+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOneMachineFailingAHealthyMeshResolverIsItsOwnAndTwoAreTheResolvers(t *testing.T) {
|
||||
silentResolver := inventory.NetworkPart{Part: link.PartNames, Reason: "1 of its 2 resolvers do not answer as the mesh's do",
|
||||
Said: "10.77.0.3 — no answer within 1s", Toward: []string{"10.77.0.3"}}
|
||||
obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
|
||||
"anchor": aNetwork(link.StateHealthy), "spare": aNetwork(link.StateHealthy),
|
||||
"laptop": aNetwork(link.StateUnhealthy, silentResolver),
|
||||
}))
|
||||
if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" {
|
||||
t.Fatalf("one machine alone: want its own, got %v", keys)
|
||||
}
|
||||
obs = networkObservations(netFacts(map[string]*inventory.NetworkHealth{
|
||||
"anchor": aNetwork(link.StateUnhealthy, silentResolver), "spare": aNetwork(link.StateHealthy),
|
||||
"laptop": aNetwork(link.StateUnhealthy, silentResolver),
|
||||
}))
|
||||
if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.spare.unreachable" {
|
||||
t.Fatalf("two machines: want it said once at the resolver's machine, got %v", keys)
|
||||
}
|
||||
if !strings.Contains(obs[0].Summary, "its resolver") {
|
||||
t.Fatalf("the resolver's machine is said %s", obs[0].Summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheControlNodeAndTheBusAreUrgent(t *testing.T) {
|
||||
f := netFacts(map[string]*inventory.NetworkHealth{"anchor": aNetwork(link.StateUnhealthy, rewrittenByVPN)})
|
||||
if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent {
|
||||
t.Fatalf("the control node's rewritten file: %+v", obs)
|
||||
}
|
||||
bus := inventory.NetworkPart{Part: link.PartBus, Reason: "the bus cannot be reached", Said: "no link"}
|
||||
f = netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, bus, noRoute)})
|
||||
if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent {
|
||||
t.Fatalf("the bus unreachable from the laptop: %+v", obs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnEngineThatSaysNothingOfItsNetworkRaisesNothing(t *testing.T) {
|
||||
f := netFacts(map[string]*inventory.NetworkHealth{"laptop": nil, "anchor": aNetwork(link.StateHealthy)})
|
||||
if obs := networkObservations(f); len(obs) != 0 {
|
||||
t.Fatalf("got %v", keysOf(obs))
|
||||
}
|
||||
}
|
||||
|
||||
// Through the store and the keeper: the statement kept, the rewrite raised from it, cleared when the file
|
||||
// is written back, and node show saying it.
|
||||
func TestARewrittenResolverFileIsRaisedFromTheStatementAndClearedWhenWrittenBack(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv, k := open.inventory, conditionsFrom
|
||||
say := func(at time.Time, n *link.NetworkHealth) {
|
||||
t.Helper()
|
||||
if err := stateHealth(ctx, inv, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
openKeys := func() []string {
|
||||
t.Helper()
|
||||
list, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keys []string
|
||||
for _, c := range list {
|
||||
keys = append(keys, c.Key)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
healthy := &link.NetworkHealth{State: link.StateHealthy, Since: h0, Parts: []link.NetworkPart{
|
||||
{Part: link.PartResolvConf, State: link.StateHealthy, Since: h0}}}
|
||||
rewritten := &link.NetworkHealth{State: link.StateUnhealthy, Since: h0.Add(time.Minute), Parts: []link.NetworkPart{
|
||||
{Part: link.PartResolvConf, State: link.StateUnhealthy, Reason: rewrittenByVPN.Reason, Said: rewrittenByVPN.Said,
|
||||
Writer: "FortiClient", Owner: "networkmanager", Since: h0.Add(time.Minute)},
|
||||
{Part: link.PartNames, State: link.StateUnhealthy, Reason: namesThroughVPN.Reason, Said: namesThroughVPN.Said,
|
||||
Toward: namesThroughVPN.Toward, Since: h0.Add(time.Minute)}}}
|
||||
|
||||
say(h0, healthy)
|
||||
if keys := openKeys(); len(keys) != 0 {
|
||||
t.Fatalf("a healthy network raised %v", keys)
|
||||
}
|
||||
say(h0.Add(time.Minute), rewritten)
|
||||
if keys := openKeys(); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" {
|
||||
t.Fatalf("the rewrite raised %v", keys)
|
||||
}
|
||||
kept, had, err := inv.HealthOf(ctx, "laptop")
|
||||
if err != nil || !had || kept.Network == nil || kept.Network.Parts[0].Writer != "FortiClient" {
|
||||
t.Fatalf("the statement's network was not kept: %+v %v", kept.Network, err)
|
||||
}
|
||||
if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "FortiClient") ||
|
||||
!strings.Contains(lines, "unhealthy resolv-conf") {
|
||||
t.Fatalf("node show says:\n%s", lines)
|
||||
}
|
||||
say(h0.Add(2*time.Minute), healthy)
|
||||
if keys := openKeys(); len(keys) != 0 {
|
||||
t.Fatalf("written back, still open: %v", keys)
|
||||
}
|
||||
// An engine older than the judging says no network: nothing raised, and node show says it is not known.
|
||||
say(h0.Add(3*time.Minute), nil)
|
||||
kept, had, _ = inv.HealthOf(ctx, "laptop")
|
||||
if keys := openKeys(); len(keys) != 0 || kept.Network != nil {
|
||||
t.Fatalf("an older engine: %v %+v", keys, kept.Network)
|
||||
}
|
||||
if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "older than that judging") {
|
||||
t.Fatalf("node show says:\n%s", lines)
|
||||
}
|
||||
}
|
||||
|
||||
// The gate: a resolver file another program rewrote waits the judging rather than failing it at the
|
||||
// bound; the same, when the send moved the module whose file it is, is that module's; a machine's own
|
||||
// network fault holds the machine as a whole, as before.
|
||||
func TestTheGateWaitsOnARewriteItDidNotMakeAndHoldsTheOwnerOnOneItMoved(t *testing.T) {
|
||||
since := h0
|
||||
rewrite := conditions.Condition{Key: "machine.laptop.networkmanager.rewritten", Kind: kindNetworkRewritten,
|
||||
Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop.networkmanager", Machine: "laptop"},
|
||||
Summary: "the resolver file was rewritten", Raised: since.Add(time.Minute), Source: sourceNetwork}
|
||||
f := gateFacts{judged: true, open: []conditions.Condition{rewrite}}
|
||||
if w := aboutTheMachine("laptop", []string{"letta"}, since, f); w.waiting == "" || w.whole != "" || len(w.on) != 0 {
|
||||
t.Fatalf("a rewrite the send did not make: %+v", w)
|
||||
}
|
||||
if w := aboutTheMachine("laptop", []string{"networkmanager", "letta"}, since, f); w.on["networkmanager"] == "" ||
|
||||
w.on["letta"] != "" || w.waiting != "" {
|
||||
t.Fatalf("a rewrite of the file a moved module owns: %+v", w)
|
||||
}
|
||||
own := conditions.Condition{Key: "machine.laptop.network", Kind: kindMachineNetwork,
|
||||
Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop", Machine: "laptop"},
|
||||
Summary: "laptop's network is not healthy", Raised: since.Add(time.Minute), Source: sourceNetwork}
|
||||
if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true,
|
||||
open: []conditions.Condition{own}}); w.whole == "" || w.waiting != "" {
|
||||
t.Fatalf("the machine's own network: %+v", w)
|
||||
}
|
||||
unreachable := conditions.Condition{Key: "machine.anchor.unreachable", Kind: kindNetworkUnreachable,
|
||||
Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor", Also: []string{"laptop", "spare"}},
|
||||
Summary: "anchor cannot be reached", Raised: since.Add(time.Minute), Source: sourceNetwork}
|
||||
if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true,
|
||||
open: []conditions.Condition{unreachable}}); w.waiting == "" || w.whole != "" {
|
||||
t.Fatalf("a machine that cannot reach the hub: %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTheDrillsStatementsRaiseAndClearTheRewrite replays the drill of ADR 0241 (mesh-host
|
||||
// internal/network TestDrill…): what a node-engine said in a throwaway container while its resolver file
|
||||
// was declared, rewritten as a VPN client rewrites it, and written back — recorded, with the mesh's names
|
||||
// and addresses replaced by this test mesh's. Healthy raises nothing; the rewrite, on its second look, is
|
||||
// raised as one finding naming the writer; written back, it clears.
|
||||
func TestTheDrillsStatementsRaiseAndClearTheRewrite(t *testing.T) {
|
||||
raw, err := os.ReadFile("testdata/network-drill.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var said []link.Health
|
||||
if err := json.Unmarshal(raw, &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
k := conditionsFrom
|
||||
var raisedAt []int
|
||||
for i, h := range said {
|
||||
if err := stateHealth(ctx, open.inventory, k, "laptop", h, h.At); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
list, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range list {
|
||||
if c.Key != "machine.laptop.networkmanager.rewritten" || !strings.Contains(c.Summary, "(FortiClient)") {
|
||||
t.Fatalf("statement %d raised %s: %s", i, c.Key, c.Summary)
|
||||
}
|
||||
raisedAt = append(raisedAt, i)
|
||||
}
|
||||
}
|
||||
// Five statements: healthy, healthy, one failing look (still healthy), unhealthy, written back.
|
||||
if len(raisedAt) != 1 || raisedAt[0] != 3 {
|
||||
t.Fatalf("the rewrite was open after statements %v; want after the fourth alone", raisedAt)
|
||||
}
|
||||
}
|
||||
@@ -83,8 +83,16 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke
|
||||
for module := range unhealthy {
|
||||
streaks[module] = prev.Streaks[module] + 1
|
||||
}
|
||||
var network *inventory.NetworkHealth
|
||||
if h.Network != nil {
|
||||
network = &inventory.NetworkHealth{State: h.Network.State, Since: h.Network.Since, Parts: []inventory.NetworkPart{}}
|
||||
for _, p := range h.Network.Parts {
|
||||
network.Parts = append(network.Parts, inventory.NetworkPart{Part: p.Part, State: p.State, Reason: p.Reason,
|
||||
Said: p.Said, Writer: p.Writer, Owner: p.Owner, Toward: p.Toward, Since: p.Since, Streak: p.Streak})
|
||||
}
|
||||
}
|
||||
stored, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: node, Contract: h.Contract, SaidAt: h.At,
|
||||
HeardAt: now, Resources: resources, Streaks: streaks})
|
||||
HeardAt: now, Resources: resources, Streaks: streaks, Network: network})
|
||||
if err != nil || !stored {
|
||||
if err == nil {
|
||||
healthRefused.Add(1)
|
||||
@@ -94,7 +102,16 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke
|
||||
if k == nil {
|
||||
return nil
|
||||
}
|
||||
return judgeModuleHealth(ctx, inv, k, node, unhealthy, streaks, now)
|
||||
err = judgeModuleHealth(ctx, inv, k, node, unhealthy, streaks, now)
|
||||
// And every machine's network, from every machine's newest statement (ADR 0241): a statement about one
|
||||
// machine can hold another's finding, or release it.
|
||||
if nerr := judgeNetworks(ctx, inv, k, now); nerr != nil {
|
||||
if err == nil {
|
||||
return nerr
|
||||
}
|
||||
return fmt.Errorf("%w; %v", err, nerr)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// judgeModuleHealth raises a module's condition on a machine on the second statement in a row that says a
|
||||
|
||||
@@ -182,6 +182,9 @@ func moduleCommand(ctx context.Context, args []string) error {
|
||||
Requires []string `json:"requires,omitempty"`
|
||||
Claims []string `json:"claims,omitempty"`
|
||||
Capabilities []string `json:"capabilities,omitempty"`
|
||||
// Replaces is what each of its own tools replaces (novox/hq ADR 0245), for the console's
|
||||
// search and the agent's instructions.
|
||||
Replaces map[string][]string `json:"replaces,omitempty"`
|
||||
}
|
||||
out := make([]listed, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
@@ -189,7 +192,7 @@ func moduleCommand(ctx context.Context, args []string) error {
|
||||
l := listed{Module: m.Module, Version: m.Version, Built: e.Source.BuiltFrom, Head: e.Source.Head,
|
||||
Current: e.Provided || e.Source.Repository == "" || e.Source.Current(), Provided: e.Provided,
|
||||
On: append([]string{}, e.On...), Provides: m.Offers(), Requires: m.Requires,
|
||||
Capabilities: m.Capabilities, Tools: declaresTools(m)}
|
||||
Capabilities: m.Capabilities, Tools: declaresTools(m), Replaces: m.Replaces}
|
||||
for _, c := range m.Claims {
|
||||
l.Claims = append(l.Claims, c.At()+"/"+c.Name)
|
||||
}
|
||||
|
||||
@@ -546,6 +546,9 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
|
||||
for _, line := range healthLines(h, had, time.Now()) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
for _, line := range networkLines(h, had, time.Now()) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
}
|
||||
|
||||
held, err := inv.Profile(ctx, name)
|
||||
|
||||
@@ -127,6 +127,12 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
|
||||
return catalogue.Resolution{}, nil, err
|
||||
}
|
||||
|
||||
// A recorded module is composed at the build this machine runs, on any send but a person's push
|
||||
// (novox/hq issue 295, ADR 0245).
|
||||
if _, err := keepRecorded(ctx, open, nodeName, shelf); err != nil {
|
||||
return catalogue.Resolution{}, nil, err
|
||||
}
|
||||
|
||||
resolved, err := catalogue.Resolve(shelf, assigned,
|
||||
catalogue.Node{Name: nodeName, Site: site, Capabilities: capabilities,
|
||||
At: onNetwork[nodeName], PublicDomain: publicDomain,
|
||||
@@ -433,6 +439,16 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
||||
names = append(names, m.Module)
|
||||
}
|
||||
out.Builds = carriedBuilds(names, composed.LeftOut, current, before)
|
||||
// A recorded module kept at the build the machine runs is recorded as carrying that one (ADR 0245).
|
||||
kept, err := recordedKept(ctx, open, node)
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
for m, was := range kept {
|
||||
if _, carried := out.Builds[m]; carried {
|
||||
out.Builds[m] = was
|
||||
}
|
||||
}
|
||||
out.Bindings = boundToData(plan, composed.LeftOut)
|
||||
}
|
||||
return out, nil
|
||||
|
||||
@@ -1082,6 +1082,13 @@ func sendToEach(ctx context.Context, open *stores, names []string) ([]string, er
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// **A recorded build moves only by a person's push** (novox/hq issue 295, ADR 0245): this send — a
|
||||
// plan's, a release plan's, a rollback's, a healer's, a rotation's — composes every recorded module at
|
||||
// the build its machine runs. The bus step is a person's word for the bus alone.
|
||||
if ctx, err = sendKeeps(ctx, inv); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Held from composing to sending (novox/hq ADR 0100); a caller that holds them already —
|
||||
// converge, which flips the node and then sends it — is not made to wait on itself.
|
||||
ctx, release, err := holdNodes(ctx, open, names)
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A recorded build reaches a machine only by a person's push (novox/hq issue 295, ADR 0245).
|
||||
//
|
||||
// **A send carries the machine's whole declaration** (ADR 0221), composed from the build the mesh holds
|
||||
// of every module on it. A module whose upgrade policy records — postgres, mongodb, keycloak, the
|
||||
// network path — has its new build registered at its merge and sent nowhere, "until a person pushes".
|
||||
// But every other send to its machine composed it too: on 2026-10-07 a catalogue merge adopting the
|
||||
// images' health checks rebuilt postgres and mongodb with the rest, and the plan's gated send to the
|
||||
// control node for mail — and to the anchor for the spreadsheet app — carried both, recreating the
|
||||
// providers every consumer on those machines drops with. No gate judged them (the gate judges only what
|
||||
// rolls out), and nobody had pushed.
|
||||
//
|
||||
// So **every send but a person's push composes a recorded module at the build its machine was last
|
||||
// sent**: the manifest of that build, from the build records, in place of the one the mesh holds. The
|
||||
// machine runs what it ran; the send records that it still carries that build; `status` keeps saying
|
||||
// the machine is behind, and `push <node>` — a person's word — sends the new one. The bus step is a
|
||||
// person's too, and carries the bus; any other recorded module waiting on the bus's machine stays.
|
||||
//
|
||||
// A recorded module the machine was never sent (a new assignment) is composed as the mesh holds it —
|
||||
// there is nothing running to keep. One whose kept build is no longer in the records refuses the send,
|
||||
// said: composing the new build would be the very move this exists to stop.
|
||||
|
||||
type keepRecordedKey struct{}
|
||||
|
||||
// sendKeeps is the context a send that is not a person's push composes under: every recorded module
|
||||
// kept at the build its machine runs — except, on the bus step, the bus.
|
||||
func sendKeeps(ctx context.Context, inv *inventory.Inventory) (context.Context, error) {
|
||||
if !busStepSending(ctx) {
|
||||
return keepingRecorded(ctx), nil
|
||||
}
|
||||
bus, err := pendingBus(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return keepingRecorded(ctx, bus.module), nil
|
||||
}
|
||||
|
||||
// keepingRecorded is a context whose sends compose every recorded module at the build its machine runs,
|
||||
// except the modules named (the bus, on the bus step).
|
||||
func keepingRecorded(ctx context.Context, except ...string) context.Context {
|
||||
skip := map[string]bool{}
|
||||
for _, m := range except {
|
||||
skip[m] = true
|
||||
}
|
||||
return context.WithValue(ctx, keepRecordedKey{}, skip)
|
||||
}
|
||||
|
||||
// keptExcept is whether this context keeps recorded modules, and the modules it lets move.
|
||||
func keptExcept(ctx context.Context) (map[string]bool, bool) {
|
||||
skip, on := ctx.Value(keepRecordedKey{}).(map[string]bool)
|
||||
return skip, on
|
||||
}
|
||||
|
||||
// recordedKept is, for a send under keepingRecorded, every recorded module the machine was last sent a
|
||||
// build of that the mesh's build is not identical to: module → the commit it keeps. Nil when the context
|
||||
// keeps nothing, or when what the machine was last sent is not known (it is then held whole elsewhere —
|
||||
// ADR 0221).
|
||||
func recordedKept(ctx context.Context, open *stores, node string) (map[string]string, error) {
|
||||
skip, on := keptExcept(ctx)
|
||||
if !on {
|
||||
return nil, nil
|
||||
}
|
||||
inv := open.inventory
|
||||
sent, known, err := inv.SentBuilds(ctx, node)
|
||||
if err != nil || !known {
|
||||
return nil, err
|
||||
}
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var f *moveFacts
|
||||
out := map[string]string{}
|
||||
for m, was := range sent {
|
||||
now, held := current[m]
|
||||
if !held || now.RollOut || skip[m] || was == "" || sameCommit(was, now.Commit) {
|
||||
continue
|
||||
}
|
||||
if f == nil {
|
||||
read, err := readMoveFacts(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f = &read
|
||||
}
|
||||
if f.identical(m, was, now.Commit) {
|
||||
continue
|
||||
}
|
||||
out[m] = was
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// keepRecorded puts, in a shelf about to be resolved for a machine, the build each recorded module there
|
||||
// runs in place of the one the mesh holds; it answers what it kept, module → commit.
|
||||
func keepRecorded(ctx context.Context, open *stores, node string, shelf map[string]catalogue.Manifest) (map[string]string, error) {
|
||||
kept, err := recordedKept(ctx, open, node)
|
||||
if err != nil || len(kept) == 0 {
|
||||
return nil, err
|
||||
}
|
||||
names := make([]string, 0, len(kept))
|
||||
for m := range kept {
|
||||
names = append(names, m)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, m := range names {
|
||||
ran, found, err := open.inventory.ManifestAt(ctx, m, kept[m])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("%s records rather than rolls out, and %s runs its build %s, which the build "+
|
||||
"records no longer hold: this send cannot keep it and does not move it — `push %s` sends the new "+
|
||||
"one on a person's word (novox/hq ADR 0245)", m, node, short(kept[m]), node)
|
||||
}
|
||||
shelf[m] = ran
|
||||
}
|
||||
return kept, nil
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A recorded build reaches a machine only by a person's push (novox/hq issue 295, ADR 0245).
|
||||
|
||||
// aContainerBuild is a build outcome of a module of containers, each named by id with the image and the
|
||||
// health it is given; a policy when one is said.
|
||||
func aContainerBuild(t *testing.T, module, commit, policy string, asked time.Time, containers map[string][2]string) link.BuildResult {
|
||||
t.Helper()
|
||||
var resources []any
|
||||
for id, c := range containers {
|
||||
r := map[string]any{"id": id, "type": "container", "name": module + "-" + id, "image": c[0]}
|
||||
if c[1] != "" {
|
||||
r["health"] = map[string]any{"kind": c[1]}
|
||||
}
|
||||
resources = append(resources, r)
|
||||
}
|
||||
m := map[string]any{"module": module, "version": "1", "resources": resources}
|
||||
if policy != "" {
|
||||
m["upgrade"] = map[string]any{"policy": policy, "why": "a provider whose restart drops every consumer"}
|
||||
}
|
||||
manifest, _ := json.Marshal(m)
|
||||
return link.BuildResult{ID: link.NewBuildID(asked), Repository: "novox/mesh-catalog", Path: "modules/" + module,
|
||||
On: "anchor", Module: module, Commit: commit, Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
||||
}
|
||||
|
||||
// imageOf is the image the composed plan of a machine gives one of a module's containers.
|
||||
func imageOf(t *testing.T, plan catalogue.Resolution, module, id string) string {
|
||||
t.Helper()
|
||||
for _, m := range plan.Modules {
|
||||
if m.Module != module {
|
||||
continue
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
if r["id"] == id {
|
||||
image, _ := r["image"].(string)
|
||||
return image
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Fatalf("%s has no container %s in the plan", module, id)
|
||||
return ""
|
||||
}
|
||||
|
||||
// Tonight's case, 2026-10-07: a catalogue merge adopting the images' own health checks rebuilt the
|
||||
// database (policy record) with mail (policy roll). The plan's gated send for mail carried the
|
||||
// database's new build to the control node and recreated it, unjudged, with nobody's word. A send that
|
||||
// is not a person's push now composes the database at the build the machine runs and records that it
|
||||
// still carries it; a person's push composes the new one; the bus step moves the bus alone.
|
||||
func TestARecordedBuildIsCarriedOnlyByAPersonsPush(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
start := time.Now().Add(-time.Hour)
|
||||
pgImage := "registry.invalid:5000/postgres/server@sha256:" + strings.Repeat("a", 64)
|
||||
mailImage := "registry.invalid:5000/mailu/smtp@sha256:" + strings.Repeat("c", 64)
|
||||
for _, b := range []link.BuildResult{
|
||||
aContainerBuild(t, "postgres", "c1111111", catalogue.PolicyRecord, start,
|
||||
map[string][2]string{"server": {pgImage, ""}}),
|
||||
aContainerBuild(t, "mailu", "c1111111", "", start.Add(time.Second),
|
||||
map[string][2]string{"smtp": {mailImage, ""}, "imap": {mailImage, ""}}),
|
||||
} {
|
||||
if _, _, err := takeIn(ctx, inv, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, m := range []string{"postgres", "mailu"} {
|
||||
if _, err := inv.Assign(ctx, "anchor", m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor",
|
||||
map[string]string{"postgres": "c1111111", "mailu": "c1111111"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The merge: both adopt a health check; the images are the same.
|
||||
for _, b := range []link.BuildResult{
|
||||
aContainerBuild(t, "postgres", "c2222222", catalogue.PolicyRecord, start.Add(time.Minute),
|
||||
map[string][2]string{"server": {pgImage, "runtime"}}),
|
||||
aContainerBuild(t, "mailu", "c2222222", "", start.Add(time.Minute+time.Second),
|
||||
map[string][2]string{"smtp": {mailImage, "runtime"}, "imap": {mailImage, "runtime"}}),
|
||||
} {
|
||||
if _, _, err := takeIn(ctx, inv, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
healthOf := func(plan catalogue.Resolution, module, id string) any {
|
||||
for _, m := range plan.Modules {
|
||||
for _, r := range m.Resources {
|
||||
if m.Module == module && r["id"] == id {
|
||||
return r["health"]
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// A plan's, a release plan's, a healer's send: the database is kept as it runs, mail moves.
|
||||
kept := keepingRecorded(ctx)
|
||||
plan, settings, err := planFor(kept, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if healthOf(plan, "postgres", "server") != nil {
|
||||
t.Fatal("a send that is not a person's push composed the recorded module's new build")
|
||||
}
|
||||
if healthOf(plan, "mailu", "smtp") == nil {
|
||||
t.Fatal("the module that rolls out was not composed at its new build")
|
||||
}
|
||||
if imageOf(t, plan, "postgres", "server") != pgImage {
|
||||
t.Fatal("the recorded module's container lost its image")
|
||||
}
|
||||
gens, err := generators(kept, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationWith(kept, open, "anchor", plan, settings, gens, Allocating)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if declared.Builds["postgres"] != "c1111111" || declared.Builds["mailu"] != "c2222222" {
|
||||
t.Fatalf("the send records it carries %v; want postgres still at c1111111 and mailu at c2222222", declared.Builds)
|
||||
}
|
||||
|
||||
// A person's push: the recorded module's new build.
|
||||
plan, _, err = planFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if healthOf(plan, "postgres", "server") == nil {
|
||||
t.Fatal("a person's push did not compose the recorded module's new build")
|
||||
}
|
||||
|
||||
// The bus step moves the bus alone: a recorded module it is told it may move moves, the others stay.
|
||||
plan, _, err = planFor(keepingRecorded(ctx, "postgres"), open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if healthOf(plan, "postgres", "server") == nil {
|
||||
t.Fatal("the module a send is let move was kept")
|
||||
}
|
||||
|
||||
// What a send composes under (sendToEach): every recorded module kept; on the bus step, the bus moves.
|
||||
if under, err := sendKeeps(ctx, inv); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if skip, on := keptExcept(under); !on || len(skip) != 0 {
|
||||
t.Fatalf("an ordinary send keeps %v %v", on, skip)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "nats", Version: "2",
|
||||
Provides: []catalogue.Offer{{Name: "mesh-bus"}}}, inventory.Source{Repository: "novox/mesh-catalog",
|
||||
Seat: "git", Path: "modules/nats", BuiltFrom: "n2", Head: "n2"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if under, err := sendKeeps(withBusStep(ctx), inv); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if skip, on := keptExcept(under); !on || !skip["nats"] || len(skip) != 1 {
|
||||
t.Fatalf("the bus step keeps %v %v; want everything recorded but the bus", on, skip)
|
||||
}
|
||||
|
||||
// And a recorded module whose kept build the records no longer hold refuses the send, said.
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor",
|
||||
map[string]string{"postgres": "c0000000", "mailu": "c2222222"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := planFor(kept, open, "anchor"); err == nil || !strings.Contains(err.Error(), "push anchor") {
|
||||
t.Fatalf("a recorded build that cannot be kept did not refuse the send with its remedy: %v", err)
|
||||
}
|
||||
|
||||
// And the gated send for mail says what it recreates: both containers, no new image, at once.
|
||||
moves := []inventory.CarriedMove{{Module: "mailu", Node: "anchor", From: "c1111111", To: "c2222222"}}
|
||||
sayRecreations(ctx, open, moves)
|
||||
if !strings.Contains(moves[0].Recreates, "recreates 2 of mailu's 2") || !strings.Contains(moves[0].Recreates, "no new image") {
|
||||
t.Fatalf("the send says %q of mail's containers", moves[0].Recreates)
|
||||
}
|
||||
if said := recreationsSaid(moves); !strings.HasPrefix(said, "on anchor recreates") {
|
||||
t.Fatalf("the plan's note says %q", said)
|
||||
}
|
||||
}
|
||||
|
||||
// The send says what it recreates (ADR 0245): mail's health checks adopted recreate both its
|
||||
// containers, with no new image, every one at once.
|
||||
func TestASendSaysWhatItRecreates(t *testing.T) {
|
||||
image := "registry.invalid:5000/mailu/smtp@sha256:" + strings.Repeat("c", 64)
|
||||
from := catalogue.Manifest{Module: "mailu", Resources: []map[string]any{
|
||||
{"id": "smtp", "type": "container", "image": image},
|
||||
{"id": "imap", "type": "container", "image": image},
|
||||
{"id": "redis", "type": "container", "image": image},
|
||||
{"id": "config", "type": "file", "path": "/etc/x"}}}
|
||||
to := catalogue.Manifest{Module: "mailu", Resources: []map[string]any{
|
||||
{"id": "smtp", "type": "container", "image": image, "health": map[string]any{"kind": "runtime"}},
|
||||
{"id": "imap", "type": "container", "image": image, "health": map[string]any{"kind": "runtime"}},
|
||||
{"id": "redis", "type": "container", "image": image},
|
||||
{"id": "config", "type": "file", "path": "/etc/y"}}}
|
||||
r := catalogue.Recreates(from, to)
|
||||
if !r.SpecOnly() || len(r.Recreated) != 2 || r.Containers != 3 {
|
||||
t.Fatalf("recreation %+v", r)
|
||||
}
|
||||
said := r.Say("mailu")
|
||||
for _, want := range []string{"recreates 2 of mailu's 3", "no new image", "imap, smtp", "interrupted"} {
|
||||
if !strings.Contains(said, want) {
|
||||
t.Fatalf("%q does not say %q", said, want)
|
||||
}
|
||||
}
|
||||
if catalogue.Recreates(from, from).Say("mailu") != "" {
|
||||
t.Fatal("a move that recreates nothing said something")
|
||||
}
|
||||
to.Resources[2]["image"] = strings.Replace(image, "c", "d", 1)
|
||||
if r := catalogue.Recreates(from, to); r.SpecOnly() || len(r.Images) != 1 {
|
||||
t.Fatalf("a new image read as a declaration only: %+v", r)
|
||||
}
|
||||
}
|
||||
@@ -301,6 +301,7 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
|
||||
}
|
||||
}
|
||||
sort.Slice(moves, func(i, j int) bool { return moves[i].Module < moves[j].Module })
|
||||
sayRecreations(ctx, open, moves)
|
||||
sent, err := sendRollout(withScope(ctx, sendScope{judged: map[string]bool{node: true}}), open, []string{node})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
@@ -552,6 +553,9 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
||||
}
|
||||
r.Gate = &inventory.PlanGate{Machines: sent, Since: &now, Carried: moves}
|
||||
p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves))
|
||||
if said := recreationsSaid(moves); said != "" {
|
||||
p.Note += "; " + said
|
||||
}
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
return true, nil
|
||||
}
|
||||
@@ -665,3 +669,38 @@ func backlogCommand(ctx context.Context, sub string, args []string) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// sayRecreations says, on each move a send carries, what it does to the module's containers (novox/hq
|
||||
// ADR 0245): the build the machine ran against the one it is sent, container by container. Left unsaid
|
||||
// for a move whose earlier build is not in the records.
|
||||
func sayRecreations(ctx context.Context, open *stores, moves []inventory.CarriedMove) {
|
||||
if len(moves) == 0 {
|
||||
return
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for i, mv := range moves {
|
||||
to, held := shelf[mv.Module]
|
||||
if !held || mv.From == "" {
|
||||
continue
|
||||
}
|
||||
from, found, err := open.inventory.ManifestAt(ctx, mv.Module, mv.From)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
moves[i].Recreates = catalogue.Recreates(from, to).Say(mv.Module)
|
||||
}
|
||||
}
|
||||
|
||||
// recreationsSaid is every recreation a send's moves say, joined: what a plan's note carries.
|
||||
func recreationsSaid(moves []inventory.CarriedMove) string {
|
||||
var said []string
|
||||
for _, mv := range moves {
|
||||
if mv.Recreates != "" {
|
||||
said = append(said, fmt.Sprintf("on %s %s", mv.Node, mv.Recreates))
|
||||
}
|
||||
}
|
||||
return strings.Join(said, "; ")
|
||||
}
|
||||
|
||||
@@ -904,6 +904,11 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
||||
strings.Join(sent, ", "))
|
||||
fmt.Printf("%s: tier %d built; sent %d module(s) to %s first in one send (%s), the rest once its gate passes\n",
|
||||
p.ID, p.Tier, len(modules), strings.Join(sent, ", "), strings.Join(modules, ", "))
|
||||
// What the send recreates, said with it (novox/hq ADR 0245).
|
||||
if said := recreationsSaid(carried); said != "" {
|
||||
p.Note += "; " + said
|
||||
fmt.Printf("%s: %s\n", p.ID, said)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1244,6 +1249,9 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
fmt.Printf(" %s\n", gateLine(r.Gate))
|
||||
for _, c := range r.Gate.Carried {
|
||||
fmt.Printf(" %-22s %s → %s\n", c.Module, short(c.From), short(c.To))
|
||||
if c.Recreates != "" {
|
||||
fmt.Printf(" %-22s %s\n", "", c.Recreates)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
@@ -1279,6 +1287,14 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
case s != nil && s.Gate != nil:
|
||||
fmt.Printf(" %-22s %s\n", "", gateLine(s.Gate))
|
||||
}
|
||||
// What the send to its first machine did to its containers (ADR 0245).
|
||||
if s != nil && s.Gate != nil {
|
||||
for _, c := range s.Gate.Carried {
|
||||
if c.Recreates != "" {
|
||||
fmt.Printf(" %-22s on %s %s\n", "", c.Node, c.Recreates)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -970,9 +970,15 @@ func seatTools() map[string]any {
|
||||
}
|
||||
var tools []map[string]any
|
||||
for _, v := range s.Serves {
|
||||
tools = append(tools, map[string]any{
|
||||
tool := map[string]any{
|
||||
"name": v.Name, "description": v.Description, "input": v.Input, "output": v.Output,
|
||||
})
|
||||
}
|
||||
// What the verb is the mesh's way to do (novox/hq ADR 0245): read by the console's search and
|
||||
// the agent's instructions.
|
||||
if len(v.Replaces) > 0 {
|
||||
tool["replaces"] = v.Replaces
|
||||
}
|
||||
tools = append(tools, tool)
|
||||
}
|
||||
seats = append(seats, map[string]any{"seat": s.Name, "scope": s.Scope, "tools": tools})
|
||||
}
|
||||
|
||||
@@ -144,6 +144,23 @@ func TestToolsAnswersTheSeatsRecords(t *testing.T) {
|
||||
if !found {
|
||||
t.Fatal("the mesh-controller seat is not in the listing")
|
||||
}
|
||||
// And what a verb replaces travels with it (novox/hq ADR 0245): the console and the agent's
|
||||
// instructions read it from here.
|
||||
var journal []string
|
||||
for _, s := range seats {
|
||||
if s["seat"] != catalogue.ServiceManagerSeat {
|
||||
continue
|
||||
}
|
||||
tools, _ := s["tools"].([]map[string]any)
|
||||
for _, tool := range tools {
|
||||
if tool["name"] == "journal" {
|
||||
journal, _ = tool["replaces"].([]string)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(journal) == 0 || journal[0] != "journalctl" {
|
||||
t.Fatalf("the service manager's journal does not say it replaces journalctl: %v", journal)
|
||||
}
|
||||
}
|
||||
|
||||
// A JSON verb's answer is parsed from what the command wrote to standard output alone; a warning it
|
||||
|
||||
+170
@@ -0,0 +1,170 @@
|
||||
[
|
||||
{
|
||||
"contract": 2,
|
||||
"at": "2026-10-07T16:48:58.691388404Z",
|
||||
"resources": [],
|
||||
"network": {
|
||||
"state": "healthy",
|
||||
"since": "2026-10-07T16:48:58.691388404Z",
|
||||
"parts": [
|
||||
{
|
||||
"part": "resolv-conf",
|
||||
"state": "healthy",
|
||||
"since": "2026-10-07T16:48:58.691388404Z"
|
||||
},
|
||||
{
|
||||
"part": "names",
|
||||
"state": "healthy",
|
||||
"since": "2026-10-07T16:48:58.691388404Z"
|
||||
},
|
||||
{
|
||||
"part": "bus",
|
||||
"state": "healthy",
|
||||
"since": "2026-10-07T16:48:58.691388404Z"
|
||||
},
|
||||
{
|
||||
"part": "route",
|
||||
"state": "healthy",
|
||||
"since": "2026-10-07T16:48:58.691388404Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"contract": 2,
|
||||
"at": "2026-10-07T16:49:28.691388404Z",
|
||||
"resources": [],
|
||||
"network": {
|
||||
"state": "healthy",
|
||||
"since": "2026-10-07T16:48:58.691388404Z",
|
||||
"parts": [
|
||||
{
|
||||
"part": "resolv-conf",
|
||||
"state": "healthy",
|
||||
"since": "2026-10-07T16:48:58.691388404Z"
|
||||
},
|
||||
{
|
||||
"part": "names",
|
||||
"state": "healthy",
|
||||
"since": "2026-10-07T16:48:58.691388404Z"
|
||||
},
|
||||
{
|
||||
"part": "bus",
|
||||
"state": "healthy",
|
||||
"since": "2026-10-07T16:48:58.691388404Z"
|
||||
},
|
||||
{
|
||||
"part": "route",
|
||||
"state": "healthy",
|
||||
"since": "2026-10-07T16:48:58.691388404Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"contract": 2,
|
||||
"at": "2026-10-07T16:49:58.691388404Z",
|
||||
"resources": [],
|
||||
"network": {
|
||||
"state": "healthy",
|
||||
"since": "2026-10-07T16:48:58.691388404Z",
|
||||
"parts": [
|
||||
{
|
||||
"part": "resolv-conf",
|
||||
"state": "healthy",
|
||||
"since": "2026-10-07T16:48:58.691388404Z",
|
||||
"streak": 1
|
||||
},
|
||||
{
|
||||
"part": "names",
|
||||
"state": "healthy",
|
||||
"since": "2026-10-07T16:48:58.691388404Z",
|
||||
"streak": 1
|
||||
},
|
||||
{
|
||||
"part": "bus",
|
||||
"state": "healthy",
|
||||
"since": "2026-10-07T16:48:58.691388404Z"
|
||||
},
|
||||
{
|
||||
"part": "route",
|
||||
"state": "healthy",
|
||||
"since": "2026-10-07T16:48:58.691388404Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"contract": 2,
|
||||
"at": "2026-10-07T16:50:28.691388404Z",
|
||||
"resources": [],
|
||||
"network": {
|
||||
"state": "unhealthy",
|
||||
"since": "2026-10-07T16:50:28.691388404Z",
|
||||
"parts": [
|
||||
{
|
||||
"part": "resolv-conf",
|
||||
"state": "unhealthy",
|
||||
"reason": "the resolver file was rewritten by another program",
|
||||
"said": "/etc/resolv.conf differs from what networkmanager declares: it lists 192.0.2.53 where 10.77.0.2, 10.77.0.1 is declared; changed 2026-10-07T16:48:28Z; its own header names FortiClient",
|
||||
"writer": "FortiClient",
|
||||
"owner": "networkmanager",
|
||||
"since": "2026-10-07T16:50:28.691388404Z",
|
||||
"streak": 2
|
||||
},
|
||||
{
|
||||
"part": "names",
|
||||
"state": "unhealthy",
|
||||
"reason": "neither mesh names nor public names resolve",
|
||||
"said": "within 1s: 192.0.2.53 — anchor.internal (IPv4): no answer within 1s; anchor.internal (IPv6): no answer within 1s; example.com (IPv4): no answer within 1s",
|
||||
"toward": [
|
||||
"192.0.2.53"
|
||||
],
|
||||
"since": "2026-10-07T16:50:28.691388404Z",
|
||||
"streak": 2
|
||||
},
|
||||
{
|
||||
"part": "bus",
|
||||
"state": "healthy",
|
||||
"since": "2026-10-07T16:48:58.691388404Z"
|
||||
},
|
||||
{
|
||||
"part": "route",
|
||||
"state": "healthy",
|
||||
"since": "2026-10-07T16:48:58.691388404Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"contract": 2,
|
||||
"at": "2026-10-07T16:50:58.691388404Z",
|
||||
"resources": [],
|
||||
"network": {
|
||||
"state": "healthy",
|
||||
"since": "2026-10-07T16:50:58.691388404Z",
|
||||
"parts": [
|
||||
{
|
||||
"part": "resolv-conf",
|
||||
"state": "healthy",
|
||||
"since": "2026-10-07T16:50:58.691388404Z"
|
||||
},
|
||||
{
|
||||
"part": "names",
|
||||
"state": "healthy",
|
||||
"since": "2026-10-07T16:50:58.691388404Z"
|
||||
},
|
||||
{
|
||||
"part": "bus",
|
||||
"state": "healthy",
|
||||
"since": "2026-10-07T16:48:58.691388404Z"
|
||||
},
|
||||
{
|
||||
"part": "route",
|
||||
"state": "healthy",
|
||||
"since": "2026-10-07T16:48:58.691388404Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -9,7 +9,7 @@ package catalogue
|
||||
// DeliverySeat is the seat mesh-delivery holds.
|
||||
const DeliverySeat = "mesh-delivery"
|
||||
|
||||
// deliveryVerbs are the delivery seat's tools: five that read, and the acts of a person and of healer H2.
|
||||
// deliveryVerbs are the delivery seat's tools: six that read, and the acts of a person and of healer H2.
|
||||
func deliveryVerbs() []Verb {
|
||||
return []Verb{
|
||||
{Name: "deliveries", Description: "Every delivery not final, and those that ended in the last day, one line " +
|
||||
@@ -32,6 +32,16 @@ func deliveryVerbs() []Verb {
|
||||
Input: schema(map[string]string{"repository": "owner/repository",
|
||||
"paths": "the files it changes, comma-separated, from the repository's root",
|
||||
"base": "the branch it merges into (default main)"}, []string{"repository", "paths"})},
|
||||
{Name: "checks", Description: "What the mesh's checks said of a pull request's head or of one commit: each " +
|
||||
"of the commit's mesh statuses (mesh/merge-gate, mesh/repo-check, mesh/delivery, …) with its state, " +
|
||||
"description and when it was set; the merge check's full verdict as the controller said it — each " +
|
||||
"layer's summary, the machine that ran it, when, its build id and its report; and whether the branch's " +
|
||||
"protection would let it merge, every required status being success.",
|
||||
Input: schema(map[string]string{"repository": "owner/repository",
|
||||
"number": "a pull request's number: its head is read",
|
||||
"commit": "a commit's sha, or the start of one, instead of a pull request"}, []string{"repository"}),
|
||||
// Added after the seat's first holder shipped: optional until mesh-delivery serves it everywhere.
|
||||
Optional: true},
|
||||
{Name: "table", Description: "The state table every delivery runs by: each transition with its guard, " +
|
||||
"each state's bound and what healer H2 may do once it has passed; and the machine steps' table.",
|
||||
Input: schema(map[string]string{}, nil)},
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The delivery seat answers "what did the mesh's checks say of this pull request?" as a verb of its own,
|
||||
// `checks` (novox/hq ADR 0239): read by repository and a pull request's number or a commit.
|
||||
func TestTheDeliverySeatPromisesChecks(t *testing.T) {
|
||||
seat, ok := SeatNamed(DeliverySeat)
|
||||
if !ok {
|
||||
t.Fatal("the delivery seat is not in the set")
|
||||
}
|
||||
var checks *Verb
|
||||
for i := range seat.Serves {
|
||||
if seat.Serves[i].Name == "checks" {
|
||||
checks = &seat.Serves[i]
|
||||
}
|
||||
}
|
||||
if checks == nil {
|
||||
t.Fatalf("the delivery seat promises %v and not checks", VerbNames(seat.Serves))
|
||||
}
|
||||
props, _ := checks.Input["properties"].(map[string]any)
|
||||
for _, arg := range []string{"repository", "number", "commit"} {
|
||||
if _, has := props[arg]; !has {
|
||||
t.Errorf("checks takes no %q", arg)
|
||||
}
|
||||
}
|
||||
if req, _ := checks.Input["required"].([]string); !reflect.DeepEqual(req, []string{"repository"}) {
|
||||
t.Errorf("checks requires %v, wanted only the repository", req)
|
||||
}
|
||||
|
||||
// Added after the seat's holder shipped, it is optional: the holder serving the ten verbs before it still
|
||||
// holds the seat, and one serving all eleven does too — so the controller and the catalogue can move in
|
||||
// either order, and no check of the catalogue fails on a module nobody touched between the two.
|
||||
if !checks.Optional {
|
||||
t.Fatal("checks is a condition of holding before its holder serves it")
|
||||
}
|
||||
var before []string
|
||||
for _, v := range VerbNames(seat.Serves) {
|
||||
if v != "checks" {
|
||||
before = append(before, v)
|
||||
}
|
||||
}
|
||||
m := Manifest{Module: "mesh-delivery", Claims: []Claim{{Name: DeliverySeat, Scope: ScopeMesh, Serves: before}}}
|
||||
if err := CanHold(m, seat); err != nil {
|
||||
t.Fatalf("a holder without checks yet: %v", err)
|
||||
}
|
||||
m.Claims[0].Serves = VerbNames(seat.Serves)
|
||||
if err := CanHold(m, seat); err != nil {
|
||||
t.Fatalf("a holder serving every verb: %v", err)
|
||||
}
|
||||
// Every other verb is still a condition of holding.
|
||||
m.Claims[0].Serves = append([]string{"checks"}, before[1:]...)
|
||||
if err := CanHold(m, seat); err == nil || !strings.Contains(err.Error(), before[0]) {
|
||||
t.Fatalf("a holder without %s: %v", before[0], err)
|
||||
}
|
||||
}
|
||||
@@ -426,6 +426,13 @@ type Manifest struct {
|
||||
// module claiming a seat answers what that seat's protocol promises (novox/hq ADR 0118).
|
||||
Tools []string `json:"tools,omitempty"`
|
||||
|
||||
// Replaces says, for a tool of this module's own, the shell commands it is the mesh's way to do —
|
||||
// `{"docker_logs": ["docker logs"]}` — as a seat's verb says it in its definition (novox/hq ADR
|
||||
// 0241). A seat's verb carries its own: what a role replaces is the role's, so a module never says it
|
||||
// for a verb it serves under a claim. The console's search, the agent's instructions and the guard on
|
||||
// its shell are built from both.
|
||||
Replaces map[string][]string `json:"replaces,omitempty"`
|
||||
|
||||
// Instances says whether this module's instances are the same anywhere — `interchangeable` —
|
||||
// so a call that names no machine may be answered by any of them (novox/hq ADR 0160). A fact
|
||||
// about the software, not about the bus: a stateless web tool says it; a database does not,
|
||||
@@ -1692,6 +1699,7 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
||||
}
|
||||
}
|
||||
problems = append(problems, invokeProblems(m)...)
|
||||
problems = append(problems, replacesProblems(m)...)
|
||||
problems = append(problems, endpointNameProblems(m)...)
|
||||
problems = append(problems, RouteProblems(m)...)
|
||||
for _, port := range m.Guards {
|
||||
@@ -2306,6 +2314,53 @@ func EndpointPort(m Manifest, name string) (int, bool) {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// MaxReplaced is the longest command a tool may say it replaces: a command's name and the words that
|
||||
// make it this command, never a script.
|
||||
const MaxReplaced = 80
|
||||
|
||||
// replacesProblems judges what a module says its tools replace (novox/hq ADR 0245): each key a tool it
|
||||
// declares, each entry a command line of one line, short, and said once.
|
||||
func replacesProblems(m Manifest) []string {
|
||||
var problems []string
|
||||
tools := map[string]bool{}
|
||||
for _, t := range m.Tools {
|
||||
tools[t] = true
|
||||
}
|
||||
keys := make([]string, 0, len(m.Replaces))
|
||||
for k := range m.Replaces {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, tool := range keys {
|
||||
if !tools[tool] {
|
||||
problems = append(problems, fmt.Sprintf("%s says what %q replaces, and declares no such tool: "+
|
||||
"a module says it for a tool in its `tools`; a seat's verb says it in the seat's definition "+
|
||||
"(novox/hq ADR 0245)", m.Module, tool))
|
||||
continue
|
||||
}
|
||||
if len(m.Replaces[tool]) == 0 {
|
||||
problems = append(problems, fmt.Sprintf("%s says %s replaces nothing: leave it out", m.Module, tool))
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, c := range m.Replaces[tool] {
|
||||
c = strings.TrimSpace(c)
|
||||
switch {
|
||||
case c == "":
|
||||
problems = append(problems, fmt.Sprintf("%s: %s replaces an empty command", m.Module, tool))
|
||||
case strings.ContainsAny(c, "\n\r"):
|
||||
problems = append(problems, fmt.Sprintf("%s: %s replaces %q, which is more than one line", m.Module, tool, c))
|
||||
case len(c) > MaxReplaced:
|
||||
problems = append(problems, fmt.Sprintf("%s: %s replaces %q, longer than %d characters: "+
|
||||
"the command's name and the words that make it this command", m.Module, tool, c, MaxReplaced))
|
||||
case seen[c]:
|
||||
problems = append(problems, fmt.Sprintf("%s: %s replaces %q twice", m.Module, tool, c))
|
||||
}
|
||||
seen[c] = true
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// invokeProblems judges what a module says it calls (novox/hq ADR 0152).
|
||||
//
|
||||
// Refused here, in the manifest's words, rather than at the next composition of the bus's user
|
||||
|
||||
@@ -42,7 +42,7 @@ func TestTheServiceManagerSeatServesTheUnitVerbs(t *testing.T) {
|
||||
if seat.Scope != ScopeNode {
|
||||
t.Fatalf("the service manager is a role each machine has once, and the seat is %s-scoped", seat.Scope)
|
||||
}
|
||||
want := []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal"}
|
||||
want := []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal", "failed"}
|
||||
var got []string
|
||||
for _, v := range seat.Serves {
|
||||
got = append(got, v.Name)
|
||||
@@ -58,3 +58,53 @@ func TestTheServiceManagerSeatServesTheUnitVerbs(t *testing.T) {
|
||||
t.Fatalf("the seat serves %v, not %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// **`failed` joins the seat optional** (the operator's direction 2026-10-07): the holder running today,
|
||||
// which does not serve it, still holds the seat; the holder that serves it is not refused; and a verb
|
||||
// the seat does not promise is still refused, and one it requires is still required.
|
||||
func TestFailedIsAnOptionalVerbOfTheServiceManager(t *testing.T) {
|
||||
seat, _ := SeatNamed(ServiceManagerSeat)
|
||||
holder := func(serves ...string) Manifest {
|
||||
return Manifest{Module: "systemd", Version: "1", Claims: []Claim{{Name: ServiceManagerSeat, Scope: ScopeNode, Serves: serves}}}
|
||||
}
|
||||
eight := []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal"}
|
||||
if err := CanHold(holder(eight...), seat); err != nil {
|
||||
t.Fatalf("today's holder, without failed, is refused: %v", err)
|
||||
}
|
||||
if err := CanHold(holder(append(eight, "failed")...), seat); err != nil {
|
||||
t.Fatalf("a holder serving failed is refused: %v", err)
|
||||
}
|
||||
if err := CanHold(holder(append(eight, "fail")...), seat); err == nil || !strings.Contains(err.Error(), "does not promise") {
|
||||
t.Fatalf("a verb the seat does not promise was accepted: %v", err)
|
||||
}
|
||||
if err := CanHold(holder(eight[1:]...), seat); err == nil || !strings.Contains(err.Error(), "does not serve units") {
|
||||
t.Fatalf("a holder missing a required verb was accepted: %v", err)
|
||||
}
|
||||
for _, v := range seat.Serves {
|
||||
if v.Optional != (v.Name == "failed") {
|
||||
t.Errorf("%s optional: %v", v.Name, v.Optional)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The optional mark is not stored, so a seat set read back from the store's rows takes it from the
|
||||
// compiled seat: otherwise `failed`, seeded into the row, would come back required.
|
||||
func TestAnOptionalVerbStaysOptionalInASetReadFromTheStore(t *testing.T) {
|
||||
defer UseSeats(DefaultSeats())
|
||||
var rows []Seat
|
||||
for _, s := range DefaultSeats() {
|
||||
row := s
|
||||
row.Serves = nil
|
||||
for _, v := range s.Serves {
|
||||
row.Serves = append(row.Serves, Verb{Name: v.Name, Description: v.Description, Input: v.Input})
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
UseSeats(rows)
|
||||
seat, _ := SeatNamed(ServiceManagerSeat)
|
||||
holder := Manifest{Module: "systemd", Version: "1", Claims: []Claim{{Name: ServiceManagerSeat, Scope: ScopeNode,
|
||||
Serves: []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal"}}}}
|
||||
if err := CanHold(holder, seat); err != nil {
|
||||
t.Fatalf("a set read from rows refuses today's holder: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// What a move does to a module's containers (novox/hq ADR 0245).
|
||||
//
|
||||
// A container whose declaration changes is recreated, whatever changed in it: an image, an environment
|
||||
// variable, a health check adopted. A module whose containers are recreated in one send is down while
|
||||
// they start again — on 2026-10-07 a catalogue change that only adopted the images' own health checks
|
||||
// recreated nine of mail's containers at once, and the operator's phone could not reach the mail. So a
|
||||
// send says, per module, how many of its containers it recreates and whether any image changed, before
|
||||
// it is sent and in the plan that sent it.
|
||||
|
||||
// Recreation is what moving a module from one build's manifest to another's does to its containers.
|
||||
type Recreation struct {
|
||||
// Containers is how many containers the new build declares.
|
||||
Containers int
|
||||
// Recreated are the ids of the containers whose declaration differs — changed, added or gone —
|
||||
// sorted.
|
||||
Recreated []string
|
||||
// Images are the ids among them whose image changed (or that are added or gone).
|
||||
Images []string
|
||||
}
|
||||
|
||||
// Recreates compares two builds of a module, container by container, by resource id.
|
||||
func Recreates(from, to Manifest) Recreation {
|
||||
before := containersByID(from)
|
||||
after := containersByID(to)
|
||||
var r Recreation
|
||||
r.Containers = len(after)
|
||||
for id, now := range after {
|
||||
was, held := before[id]
|
||||
switch {
|
||||
case !held:
|
||||
r.Recreated = append(r.Recreated, id)
|
||||
r.Images = append(r.Images, id)
|
||||
case !sameDeclaration(was, now):
|
||||
r.Recreated = append(r.Recreated, id)
|
||||
if !sameDeclaration(was["image"], now["image"]) {
|
||||
r.Images = append(r.Images, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
for id := range before {
|
||||
if _, kept := after[id]; !kept {
|
||||
r.Recreated = append(r.Recreated, id)
|
||||
r.Images = append(r.Images, id)
|
||||
}
|
||||
}
|
||||
sort.Strings(r.Recreated)
|
||||
sort.Strings(r.Images)
|
||||
return r
|
||||
}
|
||||
|
||||
// SpecOnly is whether the move recreates containers without any new image: a change to how they are
|
||||
// declared only — a health check adopted, a variable — which a person may well not expect to interrupt.
|
||||
func (r Recreation) SpecOnly() bool { return len(r.Recreated) > 0 && len(r.Images) == 0 }
|
||||
|
||||
// Say is the recreation in the mesh's words, for a module: empty when the move recreates nothing.
|
||||
func (r Recreation) Say(module string) string {
|
||||
if len(r.Recreated) == 0 {
|
||||
return ""
|
||||
}
|
||||
what := "with a new image"
|
||||
switch {
|
||||
case r.SpecOnly():
|
||||
what = "no new image, only their declaration"
|
||||
case len(r.Images) < len(r.Recreated):
|
||||
what = fmt.Sprintf("%d with a new image", len(r.Images))
|
||||
}
|
||||
whole := ""
|
||||
if len(r.Recreated) > 1 && len(r.Recreated) >= r.Containers {
|
||||
whole = ", every one at once: its service is interrupted until they are up again"
|
||||
} else if len(r.Recreated) > 1 {
|
||||
whole = ", at once: what they serve is interrupted until they are up again"
|
||||
}
|
||||
return fmt.Sprintf("recreates %d of %s's %d container(s) (%s: %s)%s", len(r.Recreated), module, r.Containers,
|
||||
what, strings.Join(r.Recreated, ", "), whole)
|
||||
}
|
||||
|
||||
func containersByID(m Manifest) map[string]map[string]any {
|
||||
out := map[string]map[string]any{}
|
||||
for _, r := range m.Resources {
|
||||
if t, _ := r["type"].(string); t != "container" {
|
||||
continue
|
||||
}
|
||||
id, _ := r["id"].(string)
|
||||
out[id] = r
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sameDeclaration compares two declarations as JSON, so a number read as an int and one read as a
|
||||
// float are one value.
|
||||
func sameDeclaration(a, b any) bool {
|
||||
ra, errA := json.Marshal(a)
|
||||
rb, errB := json.Marshal(b)
|
||||
if errA != nil || errB != nil {
|
||||
return reflect.DeepEqual(a, b)
|
||||
}
|
||||
var na, nb any
|
||||
_ = json.Unmarshal(ra, &na)
|
||||
_ = json.Unmarshal(rb, &nb)
|
||||
return reflect.DeepEqual(na, nb)
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The verbs an agent works around most say what they replace (novox/hq ADR 0245): the journal, a
|
||||
// restart, a unit's status, the hosts file — each the command the agent would otherwise type over ssh.
|
||||
func TestTheVerbsWorkedAroundSayWhatTheyReplace(t *testing.T) {
|
||||
want := map[string]string{
|
||||
ServiceManagerSeat + ".journal": "journalctl",
|
||||
ServiceManagerSeat + ".restart": "systemctl restart",
|
||||
ServiceManagerSeat + ".status": "systemctl status",
|
||||
"node-hostname.add": "edit /etc/hosts",
|
||||
ControllerSeatName + ".node": "hostnamectl",
|
||||
}
|
||||
for _, s := range DefaultSeats() {
|
||||
for _, v := range s.Serves {
|
||||
cmd, ok := want[s.Name+"."+v.Name]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
delete(want, s.Name+"."+v.Name)
|
||||
found := false
|
||||
for _, r := range v.Replaces {
|
||||
found = found || r == cmd
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("%s.%s does not say it replaces %q: %v", s.Name, v.Name, cmd, v.Replaces)
|
||||
}
|
||||
}
|
||||
}
|
||||
for verb := range want {
|
||||
t.Errorf("%s is not a verb of the compiled seats", verb)
|
||||
}
|
||||
}
|
||||
|
||||
// Every command a seat's verb replaces is one short line, said once — what the guard and the search match.
|
||||
func TestEveryReplacedCommandIsOneShortLine(t *testing.T) {
|
||||
for _, s := range DefaultSeats() {
|
||||
for _, v := range s.Serves {
|
||||
seen := map[string]bool{}
|
||||
for _, r := range v.Replaces {
|
||||
if strings.TrimSpace(r) == "" || strings.ContainsAny(r, "\n\r") || len(r) > MaxReplaced || seen[r] {
|
||||
t.Errorf("%s.%s replaces %q, which is not one short line said once", s.Name, v.Name, r)
|
||||
}
|
||||
seen[r] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A verb's definition in a manifest may say what it replaces, like the mesh's own.
|
||||
func TestAVerbDefinitionCarriesWhatItReplaces(t *testing.T) {
|
||||
m, err := ParseManifest([]byte(`{"module":"till","version":"1","tools":["refund"],` +
|
||||
`"seats":[{"name":"shop-till","serves":[{"name":"refund","replaces":["psql -c refund"]}]}],` +
|
||||
`"claims":[{"name":"shop-till","scope":"mesh"}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := m.DefinesSeats[0].Serves[0].Replaces; len(got) != 1 || got[0] != "psql -c refund" {
|
||||
t.Fatalf("replaces not read: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A module says what its own tools replace, and only for a tool it declares.
|
||||
func TestAModuleSaysWhatItsOwnToolsReplace(t *testing.T) {
|
||||
m, err := ParseManifest([]byte(`{"module":"docker","version":"1","tools":["docker_logs","docker_list"],` +
|
||||
`"replaces":{"docker_logs":["docker logs"],"docker_list":["docker ps"]}}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m.Replaces["docker_logs"][0] != "docker logs" || m.Replaces["docker_list"][0] != "docker ps" {
|
||||
t.Fatalf("replaces not read: %v", m.Replaces)
|
||||
}
|
||||
for _, c := range []struct{ manifest, says string }{
|
||||
{`{"module":"docker","version":"1","tools":["docker_logs"],"replaces":{"docker_lgos":["docker logs"]}}`,
|
||||
"declares no such tool"},
|
||||
{`{"module":"docker","version":"1","tools":["docker_logs"],"replaces":{"docker_logs":[]}}`, "replaces nothing"},
|
||||
{`{"module":"docker","version":"1","tools":["docker_logs"],"replaces":{"docker_logs":[" "]}}`, "empty command"},
|
||||
{`{"module":"docker","version":"1","tools":["docker_logs"],"replaces":{"docker_logs":["docker logs\nrm -rf /"]}}`,
|
||||
"more than one line"},
|
||||
{`{"module":"docker","version":"1","tools":["docker_logs"],"replaces":{"docker_logs":["docker logs","docker logs"]}}`,
|
||||
"twice"},
|
||||
{`{"module":"docker","version":"1","tools":["docker_logs"],"replaces":{"docker_logs":["` + strings.Repeat("x", MaxReplaced+1) + `"]}}`,
|
||||
"longer than"},
|
||||
} {
|
||||
if _, err := ParseManifest([]byte(c.manifest)); err == nil || !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("%s: want a refusal saying %q, got %v", c.manifest, c.says, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A seat row the store seeded before `replaces` existed carries none; the working set takes the compiled
|
||||
// one, so the `tools` answer says it on a mesh whose rows exist.
|
||||
func TestASeededRowTakesWhatItsVerbsReplaceFromTheCompiledSeat(t *testing.T) {
|
||||
was := Seats()
|
||||
t.Cleanup(func() { UseSeats(was) })
|
||||
var row Seat
|
||||
for _, s := range DefaultSeats() {
|
||||
if s.Name == ServiceManagerSeat {
|
||||
row = s
|
||||
}
|
||||
}
|
||||
stored := make([]Verb, len(row.Serves))
|
||||
for i, v := range row.Serves {
|
||||
v.Replaces = nil
|
||||
stored[i] = v
|
||||
}
|
||||
row.Serves = stored
|
||||
UseSeats([]Seat{row})
|
||||
got, _ := SeatNamed(ServiceManagerSeat)
|
||||
for _, v := range got.Serves {
|
||||
if v.Name == "journal" && (len(v.Replaces) == 0 || v.Replaces[0] != "journalctl") {
|
||||
t.Fatalf("the seeded journal verb says it replaces %v", v.Replaces)
|
||||
}
|
||||
}
|
||||
}
|
||||
+89
-25
@@ -102,10 +102,10 @@ var defaultSeats = append([]Seat{
|
||||
{Name: "mesh-store", Scope: ScopeMesh, Delivers: "postgres-database", Decision: "novox/hq ADR 0079",
|
||||
Serves: []Verb{
|
||||
{Name: "databases", Description: "Every database the store holds, with its on-disk size.",
|
||||
Input: schema(map[string]string{}, nil)},
|
||||
Input: schema(map[string]string{}, nil), Replaces: []string{"psql -l"}},
|
||||
{Name: "query", Description: "One read-only statement against one database the store holds.",
|
||||
Input: schema(map[string]string{"database": "the database to query", "sql": "the read-only statement"},
|
||||
[]string{"database", "sql"})},
|
||||
[]string{"database", "sql"}), Replaces: []string{"psql"}},
|
||||
}},
|
||||
// **Delivers the mesh's own bus, not `amqp`.** Those were the same word until
|
||||
// ADR 0127 separated them: `amqp` is a backing service a module may require, and this seat is
|
||||
@@ -174,15 +174,16 @@ var defaultSeats = append([]Seat{
|
||||
Serves: []Verb{
|
||||
{Name: "entries", Description: "Every line of this machine's /etc/hosts, each marked whose it is: " +
|
||||
"the operator's, or the block of the module or tool that writes it.",
|
||||
Input: schema(map[string]string{}, nil)},
|
||||
Input: schema(map[string]string{}, nil), Replaces: []string{"cat /etc/hosts", "getent hosts"}},
|
||||
{Name: "add", Description: "Add one address and its names to the operator's lines of this machine's " +
|
||||
"/etc/hosts — a name for this machine's own programs, not the mesh's.",
|
||||
Input: schema(map[string]string{"address": "the IPv4 or IPv6 address",
|
||||
"names": "the names for it, separated by spaces"}, []string{"address", "names"})},
|
||||
"names": "the names for it, separated by spaces"}, []string{"address", "names"}),
|
||||
Replaces: []string{"edit /etc/hosts", "HOSTALIASES"}},
|
||||
{Name: "remove", Description: "Remove one name, or every line of one address, from the operator's " +
|
||||
"lines of this machine's /etc/hosts. A line a module writes is refused, naming the module.",
|
||||
Input: schema(map[string]string{"name": "a host name, or an address to remove every line of"},
|
||||
[]string{"name"})},
|
||||
[]string{"name"}), Replaces: []string{"sed -i /etc/hosts"}},
|
||||
}},
|
||||
// The intrusion prevention's verbs (novox/hq ADR 0179): what a person asks a machine's ban list
|
||||
// whatever keeps it — who is banned and why, ban one address, let one go. Every holder serves all
|
||||
@@ -191,15 +192,17 @@ var defaultSeats = append([]Seat{
|
||||
Serves: []Verb{
|
||||
{Name: "status", Description: "Every jail on this machine with how many it is watching and " +
|
||||
"holding now, and the totals since the jail started; one jail's detail when named.",
|
||||
Input: schema(map[string]string{"jail": "one jail (optional)"}, nil)},
|
||||
Input: schema(map[string]string{"jail": "one jail (optional)"}, nil), Replaces: []string{"fail2ban-client status"}},
|
||||
{Name: "banned", Description: "Every address banned on this machine right now, with the jail " +
|
||||
"that holds it and when the ban ends.",
|
||||
Input: schema(map[string]string{"jail": "one jail (optional)"}, nil)},
|
||||
Input: schema(map[string]string{"jail": "one jail (optional)"}, nil), Replaces: []string{"fail2ban-client banned"}},
|
||||
{Name: "ban", Description: "Ban one address in one jail now, for the jail's ban time — an " +
|
||||
"operator's act on the live ban list, which the mesh never writes itself.",
|
||||
Input: schema(map[string]string{"ip": "the address", "jail": "the jail to hold it"}, []string{"ip", "jail"})},
|
||||
Input: schema(map[string]string{"ip": "the address", "jail": "the jail to hold it"}, []string{"ip", "jail"}),
|
||||
Replaces: []string{"fail2ban-client set banip"}},
|
||||
{Name: "unban", Description: "Let one address go, from one jail or from every jail when none is named.",
|
||||
Input: schema(map[string]string{"ip": "the address", "jail": "one jail (optional)"}, []string{"ip"})},
|
||||
Input: schema(map[string]string{"ip": "the address", "jail": "one jail (optional)"}, []string{"ip"}),
|
||||
Replaces: []string{"fail2ban-client unban", "fail2ban-client set unbanip"}},
|
||||
}},
|
||||
// The packet filter's verbs (novox/hq ADR 0170): what a person asks a machine's filter whatever
|
||||
// filter answers — the rules as enforced, reload the mesh's own, remove one thing the mesh did
|
||||
@@ -210,15 +213,17 @@ var defaultSeats = append([]Seat{
|
||||
"ruleset and, where the tool exists, the legacy filter's listings. Narrowed to one table or " +
|
||||
"chain when asked.",
|
||||
Input: schema(map[string]string{"table": "one nftables table, as `family name` (optional)",
|
||||
"chain": "one chain of that table (optional)"}, nil)},
|
||||
"chain": "one chain of that table (optional)"}, nil),
|
||||
Replaces: []string{"nft list ruleset", "iptables -L", "iptables-save"}},
|
||||
{Name: "reload", Description: "Load the mesh's own filter again from the file the mesh writes, " +
|
||||
"and answer with the mesh's table as loaded.",
|
||||
Input: schema(map[string]string{}, nil)},
|
||||
Input: schema(map[string]string{}, nil), Replaces: []string{"nft -f"}},
|
||||
{Name: "remove", Description: "Remove one rule set the mesh did not write, named exactly as the " +
|
||||
"host reports it (novox/hq ADR 0168) — `chain X (iptables-legacy)` or `table ip6 filter, chain " +
|
||||
"DOCKER-USER`. Refuses the mesh's tables, the runtime's own chains, a built-in chain and an " +
|
||||
"active found firewall's chains. An operator's act, by name, never a flush.",
|
||||
Input: schema(map[string]string{"where": "the rule set, as `node show` lists it"}, []string{"where"})},
|
||||
Input: schema(map[string]string{"where": "the rule set, as `node show` lists it"}, []string{"where"}),
|
||||
Replaces: []string{"nft delete", "iptables -X"}},
|
||||
}},
|
||||
// The machine's service manager (novox/hq ADR 0177). The host applies every declared unit,
|
||||
// system or user scope; the holder answers questions and operator acts about them, each verb
|
||||
@@ -338,12 +343,48 @@ func UseSeats(s []Seat) {
|
||||
if d, known := byName[row.Name]; known {
|
||||
row.Receives = d.Receives
|
||||
row.Replicated = d.Replicated
|
||||
row.Serves = optionalAsCompiled(row.Serves, d.Serves)
|
||||
}
|
||||
merged = append(merged, row)
|
||||
}
|
||||
seats = merged
|
||||
}
|
||||
|
||||
// optionalAsCompiled is a row's verbs with each one the compiled seat marks optional marked so. The mark
|
||||
// is never stored (Verb.Optional), and a row is what the working set holds once the store is read: a verb
|
||||
// the seeding added to the row — `failed`, `checks` — would otherwise come back required, and every holder
|
||||
// not serving it yet would be refused at registration and at handover, which the mark exists to prevent.
|
||||
//
|
||||
// **And what a verb replaces is the compiled one** (novox/hq ADR 0245): a row seeded before the field
|
||||
// existed carries none, and re-seeding widens a verb's arguments only, so read from the row alone the
|
||||
// `tools` answer — what the mesh MCP server's search and the agent's instructions are built from — would
|
||||
// say no verb replaces anything on a mesh whose rows exist. The mesh's statement of what a role replaces
|
||||
// is this binary's, like the optional mark.
|
||||
func optionalAsCompiled(row, compiled []Verb) []Verb {
|
||||
optional := map[string]bool{}
|
||||
replaces := map[string][]string{}
|
||||
for _, v := range compiled {
|
||||
if v.Optional {
|
||||
optional[v.Name] = true
|
||||
}
|
||||
if len(v.Replaces) > 0 {
|
||||
replaces[v.Name] = v.Replaces
|
||||
}
|
||||
}
|
||||
if len(optional) == 0 && len(replaces) == 0 {
|
||||
return row
|
||||
}
|
||||
out := make([]Verb, len(row))
|
||||
for i, v := range row {
|
||||
v.Optional = optional[v.Name]
|
||||
if r, ok := replaces[v.Name]; ok {
|
||||
v.Replaces = r
|
||||
}
|
||||
out[i] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// aliases maps a seat's former names to its current canonical name (novox/hq ADR 0122). Loaded from
|
||||
// the store alongside the set, so a reference to a name a seat used to have — a manifest's claim, a
|
||||
// held record — still resolves to it after a rename, and nothing downstream has to change.
|
||||
@@ -561,7 +602,8 @@ func SeatsWithAProtocol() []Seat {
|
||||
// serviceManagerVerbs is the contract every holder of node-service-manager serves (novox/hq ADR
|
||||
// 0177): the units on the machine in both scopes, read and acted on by name. Every verb takes an
|
||||
// optional scope — "system" when absent, "user" for the operator account's own manager — so a
|
||||
// caller asks for a user unit the way it asks for a system one.
|
||||
// caller asks for a user unit the way it asks for a system one; `failed` alone reads both managers
|
||||
// when none is named, and is optional (Verb.Optional).
|
||||
func serviceManagerVerbs() []Verb {
|
||||
scoped := func(more map[string]string, required []string) map[string]any {
|
||||
props := map[string]string{"scope": "\"system\" (the default) or \"user\": the operator account's own manager"}
|
||||
@@ -573,21 +615,43 @@ func serviceManagerVerbs() []Verb {
|
||||
unit := map[string]string{"unit": "the unit's name, as the service manager knows it"}
|
||||
return []Verb{
|
||||
{Name: "units", Description: "The units the service manager knows in a scope, each with its load, active and sub state; narrowed to a pattern when asked.",
|
||||
Input: scoped(map[string]string{"pattern": "a glob the unit's name must match (optional)"}, nil)},
|
||||
Input: scoped(map[string]string{"pattern": "a glob the unit's name must match (optional)"}, nil), Replaces: []string{"systemctl list-units"}},
|
||||
{Name: "status", Description: "One unit as the service manager sees it now: its states, whether it starts at boot, its main process, and whether the mesh declares it.",
|
||||
Input: scoped(unit, []string{"unit"})},
|
||||
Input: scoped(unit, []string{"unit"}), Replaces: []string{"systemctl status", "systemctl is-active"}},
|
||||
{Name: "start", Description: "Start one unit. For a unit the mesh declares, the answer says the host will restore what its declaration says at the next apply.",
|
||||
Input: scoped(unit, []string{"unit"})},
|
||||
Input: scoped(unit, []string{"unit"}), Replaces: []string{"systemctl start"}},
|
||||
{Name: "stop", Description: "Stop one unit; for a mesh-declared unit the answer says the host will restore its declared state.",
|
||||
Input: scoped(unit, []string{"unit"})},
|
||||
Input: scoped(unit, []string{"unit"}), Replaces: []string{"systemctl stop"}},
|
||||
{Name: "restart", Description: "Restart one unit.",
|
||||
Input: scoped(unit, []string{"unit"})},
|
||||
Input: scoped(unit, []string{"unit"}), Replaces: []string{"systemctl restart"}},
|
||||
{Name: "enable", Description: "Make one unit start at boot (or at the account's login, in user scope).",
|
||||
Input: scoped(unit, []string{"unit"})},
|
||||
Input: scoped(unit, []string{"unit"}), Replaces: []string{"systemctl enable"}},
|
||||
{Name: "disable", Description: "Stop one unit starting at boot (or at login, in user scope).",
|
||||
Input: scoped(unit, []string{"unit"})},
|
||||
{Name: "journal", Description: "The last lines of one unit's journal.",
|
||||
Input: scoped(map[string]string{"unit": unit["unit"], "lines": "how many lines from the end (default 100)"}, []string{"unit"})},
|
||||
Input: scoped(unit, []string{"unit"}), Replaces: []string{"systemctl disable"}},
|
||||
// **A window, not only a tail** (the operator's direction 2026-10-07): an incident is read for the
|
||||
// minutes it happened in, and with no window on the verb a person reached for a shell. Every
|
||||
// argument is the holder's to validate — passed to journalctl as one word of its own, never through
|
||||
// a shell — and what the unit printed of a secret is redacted before it is answered.
|
||||
{Name: "journal", Description: "The last lines of one unit's journal (at most 2000), in a time window and " +
|
||||
"narrowed to a priority and to lines holding a text when asked. A secret the unit printed is shown as " +
|
||||
"[redacted: <what it was>].",
|
||||
Input: scoped(map[string]string{
|
||||
"unit": unit["unit"],
|
||||
"lines": "how many lines from the end of what matches (default 100, at most 2000)",
|
||||
"since": "the window's start: an RFC 3339 time (2026-10-07T09:30:00Z) or relative to now (-30min, -2h, yesterday) (optional)",
|
||||
"until": "the window's end, in the same forms (optional; now when absent)",
|
||||
"match": "only the lines holding this text, as written — a fixed string, not a pattern (optional)",
|
||||
"priority": "only entries this severe or more: 0-7 or emerg, alert, crit, err, warning, notice, info, debug (optional)",
|
||||
}, []string{"unit"}),
|
||||
Replaces: []string{"journalctl"}},
|
||||
// What has failed, on the seat rather than as one holder's own tool: whatever holds the role answers
|
||||
// it, so a caller asks every machine the same way. **Optional while its holders catch up**: the
|
||||
// systemd module running today serves it as its own systemd_failed, and a required verb would
|
||||
// refuse it before the version serving `failed` could be delivered.
|
||||
{Name: "failed", Optional: true, Description: "Every failed unit on this machine, in the system manager and in the operator " +
|
||||
"account's; a manager that does not answer is reported with its error, never as nothing failed.",
|
||||
Input: schema(map[string]string{"scope": "\"system\" or \"user\": only that manager (both when absent)"}, nil),
|
||||
Replaces: []string{"systemctl --failed"}},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -600,10 +664,10 @@ func backupVerbs() []Verb {
|
||||
return []Verb{
|
||||
{Name: "backed-up", Description: "What this machine backs up: each module, what it declared, " +
|
||||
"its last good night, how many restore points are kept and the repository's size.",
|
||||
Input: schema(map[string]string{"module": "one module (optional)"}, nil)},
|
||||
Input: schema(map[string]string{"module": "one module (optional)"}, nil), Replaces: []string{"restic snapshots"}},
|
||||
{Name: "now", Description: "Take a backup now, of one module or of every module on this " +
|
||||
"machine — before a migration, a retirement or anything else that could go wrong.",
|
||||
Input: schema(map[string]string{"module": "one module (optional)"}, nil)},
|
||||
Input: schema(map[string]string{"module": "one module (optional)"}, nil), Replaces: []string{"restic backup"}},
|
||||
{Name: "restore", Description: "Restore one module's data from a restore point BESIDE the live " +
|
||||
"data, never over it: each directory as <path>.restored-<date>. Swapping it in is a " +
|
||||
"person's act. Lists the restore points when none is named.",
|
||||
@@ -611,7 +675,7 @@ func backupVerbs() []Verb {
|
||||
"module": "the module",
|
||||
"snapshot": "the restore point (from `backed-up`; the newest when omitted)",
|
||||
"path": "one of the module's directories (all of them when omitted)",
|
||||
}, []string{"module"})},
|
||||
}, []string{"module"}), Replaces: []string{"restic restore"}},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -22,6 +22,21 @@ type Verb struct {
|
||||
Description string `json:"description,omitempty"`
|
||||
Input map[string]any `json:"input,omitempty"`
|
||||
Output map[string]any `json:"output,omitempty"`
|
||||
// Optional marks a verb added to a mesh seat whose holder lives in another repository (design 33 §7,
|
||||
// additive within a version): a holder that serves it is accepted, and one that does not yet still
|
||||
// holds the seat. Without it the addition would be a deadlock — this controller refusing the holder that
|
||||
// does not serve the verb, the controller before it refusing the holder that does — and every check of
|
||||
// the catalogue between the two would fail on a module nobody touched. Once every holder serves it,
|
||||
// the mark is removed and the verb is a condition of holding like the rest. Never stored or said: the
|
||||
// seat set's protocol is the compiled one.
|
||||
Optional bool `json:"-"`
|
||||
// Replaces are the shell commands this verb is the mesh's way to do, each as it is typed —
|
||||
// `journalctl`, `systemctl restart` — so the agent finds the verb by the command it would have run
|
||||
// and is told to call it instead (novox/hq ADR 0245): the mesh MCP server's search matches them, the agent's
|
||||
// instructions list them, and the guard on its shell names the verb when it refuses a work-around.
|
||||
// A command line is matched by its words in order, the first being the command's name; `edit
|
||||
// /etc/hosts` and `HOSTALIASES` name the guard's two local work-arounds for a mesh name.
|
||||
Replaces []string `json:"replaces,omitempty"`
|
||||
}
|
||||
|
||||
func (v *Verb) UnmarshalJSON(raw []byte) error {
|
||||
@@ -86,7 +101,8 @@ var ControllerVerbs = []Verb{
|
||||
{Name: "nodes", Description: "Every machine the mesh knows, with whether it is converged or adopted.",
|
||||
Input: schema(nil, nil)},
|
||||
{Name: "node", Description: "What one machine reported it can do, what it is assigned, and why.",
|
||||
Input: schema(map[string]string{"node": "the machine's name"}, []string{"node"})},
|
||||
Input: schema(map[string]string{"node": "the machine's name"}, []string{"node"}),
|
||||
Replaces: []string{"hostnamectl", "uptime"}},
|
||||
{Name: "modules", Description: "Every module the mesh holds: version, the commit it was built from, " +
|
||||
"and which machines run it.",
|
||||
Input: schema(nil, nil)},
|
||||
@@ -442,7 +458,7 @@ func unservedVerbs(tools []string, promised []Verb) []string {
|
||||
}
|
||||
var missing []string
|
||||
for _, v := range promised {
|
||||
if !has[v.Name] {
|
||||
if !has[v.Name] && !v.Optional {
|
||||
missing = append(missing, v.Name)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -307,3 +307,38 @@ func (i *Inventory) BuildOf(ctx context.Context, module, commit string) (string,
|
||||
}
|
||||
return id, err
|
||||
}
|
||||
|
||||
// ManifestAt is the manifest a module was registered with at a commit (or a commit it abbreviates): the
|
||||
// newest successful build from it, with the artifacts of the build standing for it when its source was
|
||||
// unchanged (issue 280) — what a machine last sent that build runs. False when no such build, or none
|
||||
// with a manifest, is kept (novox/hq issue 295: a recorded module is composed at the build its machine
|
||||
// runs until a person's push moves it).
|
||||
func (i *Inventory) ManifestAt(ctx context.Context, module, commit string) (catalogue.Manifest, bool, error) {
|
||||
if commit == "" {
|
||||
return catalogue.Manifest{}, false, nil
|
||||
}
|
||||
var id string
|
||||
var raw []byte
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`select id, manifest from build
|
||||
where module = $1 and failed = '' and manifest is not null and manifest::text <> 'null'
|
||||
and (commit_hash = $2 or starts_with(commit_hash, $2))
|
||||
order by at desc limit 1`, module, commit).Scan(&id, &raw)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return catalogue.Manifest{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, false, err
|
||||
}
|
||||
if stands, same, err := i.StandingBuild(ctx, module, id); err != nil {
|
||||
return catalogue.Manifest{}, false, err
|
||||
} else if stands != "" && stands != id && len(same) > 0 {
|
||||
raw = same
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, false, fmt.Errorf("%s's build from %s is not a manifest the mesh can compose: %w",
|
||||
module, commit, err)
|
||||
}
|
||||
return m, true, nil
|
||||
}
|
||||
|
||||
@@ -40,6 +40,29 @@ type NodeHealth struct {
|
||||
Resources []ResourceHealth
|
||||
// Streaks is, per module, how many statements in a row said a resource of it was unhealthy.
|
||||
Streaks map[string]int
|
||||
// Network is the machine's own networking as its engine said it (novox/hq ADR 0241); nil from an
|
||||
// engine older than that judging.
|
||||
Network *NetworkHealth
|
||||
}
|
||||
|
||||
// NetworkHealth is a machine's networking as its engine said it (ADR 0241).
|
||||
type NetworkHealth struct {
|
||||
State string `json:"state"`
|
||||
Since time.Time `json:"since"`
|
||||
Parts []NetworkPart `json:"parts"`
|
||||
}
|
||||
|
||||
// NetworkPart is one part of it: resolv-conf, names, tunnel, bus or route.
|
||||
type NetworkPart struct {
|
||||
Part string `json:"part"`
|
||||
State string `json:"state"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
Said string `json:"said,omitempty"`
|
||||
Writer string `json:"writer,omitempty"`
|
||||
Owner string `json:"owner,omitempty"`
|
||||
Toward []string `json:"toward,omitempty"`
|
||||
Since time.Time `json:"since"`
|
||||
Streak int `json:"streak,omitempty"`
|
||||
}
|
||||
|
||||
// HealthOf is a machine's newest statement; false when its node-engine has never stated one.
|
||||
@@ -60,7 +83,7 @@ func (i *Inventory) Healths(ctx context.Context) (map[string]NodeHealth, error)
|
||||
|
||||
func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHealth, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select n.name, h.contract, h.said_at, h.heard_at, h.resources, h.streaks
|
||||
`select n.name, h.contract, h.said_at, h.heard_at, h.resources, h.streaks, h.network
|
||||
from node_health h join node n on n.id = h.node
|
||||
where $1 = '' or n.name = $1`, only)
|
||||
if err != nil {
|
||||
@@ -70,8 +93,8 @@ func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHe
|
||||
out := map[string]NodeHealth{}
|
||||
for rows.Next() {
|
||||
var h NodeHealth
|
||||
var resources, streaks []byte
|
||||
if err := rows.Scan(&h.Node, &h.Contract, &h.SaidAt, &h.HeardAt, &resources, &streaks); err != nil {
|
||||
var resources, streaks, network []byte
|
||||
if err := rows.Scan(&h.Node, &h.Contract, &h.SaidAt, &h.HeardAt, &resources, &streaks, &network); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := json.Unmarshal(resources, &h.Resources); err != nil {
|
||||
@@ -80,6 +103,11 @@ func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHe
|
||||
if err := json.Unmarshal(streaks, &h.Streaks); err != nil {
|
||||
return nil, fmt.Errorf("%s's health cannot be read: %w", h.Node, err)
|
||||
}
|
||||
if len(network) > 0 {
|
||||
if err := json.Unmarshal(network, &h.Network); err != nil {
|
||||
return nil, fmt.Errorf("%s's network health cannot be read: %w", h.Node, err)
|
||||
}
|
||||
}
|
||||
out[h.Node] = h
|
||||
}
|
||||
return out, rows.Err()
|
||||
@@ -102,18 +130,25 @@ func (i *Inventory) RecordHealth(ctx context.Context, h NodeHealth) (bool, error
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
var network []byte
|
||||
if h.Network != nil {
|
||||
if network, err = json.Marshal(h.Network); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
heard := h.HeardAt
|
||||
if heard.IsZero() {
|
||||
heard = time.Now()
|
||||
}
|
||||
var node string
|
||||
err = i.store.Pool().QueryRow(ctx,
|
||||
`insert into node_health (node, contract, said_at, heard_at, resources, streaks)
|
||||
select id, $2, $3, $4, $5, $6 from node where name = $1
|
||||
`insert into node_health (node, contract, said_at, heard_at, resources, streaks, network)
|
||||
select id, $2, $3, $4, $5, $6, $7 from node where name = $1
|
||||
on conflict (node) do update set contract = excluded.contract, said_at = excluded.said_at,
|
||||
heard_at = excluded.heard_at, resources = excluded.resources, streaks = excluded.streaks
|
||||
heard_at = excluded.heard_at, resources = excluded.resources, streaks = excluded.streaks,
|
||||
network = excluded.network
|
||||
where node_health.said_at <= excluded.said_at
|
||||
returning node`, h.Node, h.Contract, h.SaidAt, heard, resources, streaks).Scan(&node)
|
||||
returning node`, h.Node, h.Contract, h.SaidAt, heard, resources, streaks, network).Scan(&node)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
if _, nerr := i.NodeByName(ctx, h.Node); nerr != nil {
|
||||
return false, nerr
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
-- A machine says how its network is (novox/hq ADR 0241, which extends ADR 0240 from what a module runs to
|
||||
-- the machine it runs on).
|
||||
--
|
||||
-- Beside the state of every long-running resource, each machine's node-engine states its own networking:
|
||||
-- the resolver file the uplink holder declared and who rewrote it, the names through every resolver it
|
||||
-- lists, the tunnel's handshake with the hub, the bus and the default route — the worst of them, since
|
||||
-- when, and each part. Kept with the machine's newest statement, replaced with it, so `node show`, the
|
||||
-- gate and a controller started again read the same word. Null for a machine whose node-engine is older
|
||||
-- than this judging: its network is not known — never healthy, never a reason to raise anything.
|
||||
alter table node_health add column network jsonb;
|
||||
@@ -152,6 +152,10 @@ type CarriedMove struct {
|
||||
From string `json:"from,omitempty"`
|
||||
To string `json:"to"`
|
||||
Build string `json:"build,omitempty"`
|
||||
// Recreates is what the send does to the module's containers, in the mesh's words — how many it
|
||||
// recreates, and whether with a new image or only their declaration (novox/hq ADR 0245); empty when
|
||||
// it recreates none, or when the build the machine ran is not known.
|
||||
Recreates string `json:"recreates,omitempty"`
|
||||
}
|
||||
|
||||
// PlanRelease is a release plan's walk through the machines (novox/hq ADR 0236): every module build
|
||||
|
||||
@@ -126,6 +126,16 @@ func (i *Inventory) widenProtocol(ctx context.Context, s catalogue.Seat) error {
|
||||
if s.Name == catalogue.ControllerSeatName && !sameVerb(row.Serves[at], v) {
|
||||
row.Serves[at] = v
|
||||
changed = true
|
||||
continue
|
||||
}
|
||||
// **Any other seat's verb gains the arguments the binary names and the row lacks** — additive,
|
||||
// as the rest of the protocol is. The console refuses an argument the row does not name (issue
|
||||
// 244), so a holder taught a new argument (the service manager's journal window) would be
|
||||
// unreachable through it while the row kept the older schema. Nothing the row has is removed or
|
||||
// made required; the description is the binary's, since it describes the arguments added.
|
||||
if widened, ok := widenInput(row.Serves[at], v); ok {
|
||||
row.Serves[at] = widened
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
if !changed {
|
||||
@@ -296,6 +306,45 @@ func (i *Inventory) Holdings(ctx context.Context) ([]catalogue.Held, error) {
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// widenInput is the row's verb with every input property the binary's names and the row's lacks, and
|
||||
// the binary's description; and whether anything was added.
|
||||
func widenInput(row, binary catalogue.Verb) (catalogue.Verb, bool) {
|
||||
want, _ := binary.Input["properties"].(map[string]any)
|
||||
if len(want) == 0 {
|
||||
return row, false
|
||||
}
|
||||
have, _ := row.Input["properties"].(map[string]any)
|
||||
added := map[string]any{}
|
||||
for name, p := range want {
|
||||
if _, kept := have[name]; !kept {
|
||||
added[name] = p
|
||||
}
|
||||
}
|
||||
if len(added) == 0 {
|
||||
return row, false
|
||||
}
|
||||
input := map[string]any{}
|
||||
for k, v := range row.Input {
|
||||
input[k] = v
|
||||
}
|
||||
if input["type"] == nil {
|
||||
input["type"] = "object"
|
||||
}
|
||||
props := map[string]any{}
|
||||
for k, v := range have {
|
||||
props[k] = v
|
||||
}
|
||||
for k, v := range added {
|
||||
props[k] = v
|
||||
}
|
||||
input["properties"] = props
|
||||
row.Input = input
|
||||
if binary.Description != "" {
|
||||
row.Description = binary.Description
|
||||
}
|
||||
return row, true
|
||||
}
|
||||
|
||||
// sameVerb is whether two definitions of a verb say the same, read as the row stores them.
|
||||
func sameVerb(a, b catalogue.Verb) bool {
|
||||
ja, errA := json.Marshal(a)
|
||||
|
||||
@@ -156,3 +156,74 @@ func TestTheControllersVerbsInTheRowAreTheBinarys(t *testing.T) {
|
||||
t.Fatal("a verb this binary adds was not added")
|
||||
}
|
||||
}
|
||||
|
||||
// **A seat's verb gains the arguments a newer binary names** (the service manager's journal window,
|
||||
// 2026-10-07): the console refuses an argument the row does not name, so a row seeded before them would
|
||||
// keep the holder's new arguments unreachable. Added, never removed — an argument only the row has stays,
|
||||
// and nothing becomes required — and a verb the binary adds optional is optional in the working set read back.
|
||||
func TestASeatsVerbGainsTheArgumentsTheBinaryNames(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old := `[{"name":"journal","description":"The last lines of one unit's journal.","input":{"type":"object",
|
||||
"properties":{"unit":{"type":"string"},"lines":{"type":"string"},"scope":{"type":"string"},"kept":{"type":"string"}},
|
||||
"required":["unit"]}}]`
|
||||
if _, err := inv.store.Pool().Exec(ctx, `update seat set serves = $1 where name = $2`,
|
||||
[]byte(old), catalogue.ServiceManagerSeat); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seats, err := inv.Seats(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
verbs := map[string]catalogue.Verb{}
|
||||
for _, s := range seats {
|
||||
if s.Name == catalogue.ServiceManagerSeat {
|
||||
for _, v := range s.Serves {
|
||||
verbs[v.Name] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
props, _ := verbs["journal"].Input["properties"].(map[string]any)
|
||||
for _, arg := range []string{"since", "until", "match", "priority", "unit", "lines", "scope", "kept"} {
|
||||
if _, has := props[arg]; !has {
|
||||
t.Errorf("journal in the row does not take %s: %v", arg, props)
|
||||
}
|
||||
}
|
||||
if req, _ := verbs["journal"].Input["required"].([]any); len(req) != 1 || req[0] != "unit" {
|
||||
t.Errorf("what journal requires changed: %v", verbs["journal"].Input["required"])
|
||||
}
|
||||
if _, added := verbs["failed"]; !added {
|
||||
t.Fatal("failed was not added to the row")
|
||||
}
|
||||
// The optional mark is not stored; the working set read from the rows takes it from the compiled seat,
|
||||
// so today's holder, which does not serve failed, still holds the seat.
|
||||
catalogue.UseSeats(seats)
|
||||
defer catalogue.UseSeats(catalogue.DefaultSeats())
|
||||
live, _ := catalogue.SeatNamed(catalogue.ServiceManagerSeat)
|
||||
holder := catalogue.Manifest{Module: "systemd", Version: "1", Claims: []catalogue.Claim{{Name: catalogue.ServiceManagerSeat,
|
||||
Scope: catalogue.ScopeNode, Serves: []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal"}}}}
|
||||
if err := catalogue.CanHold(holder, live); err != nil {
|
||||
t.Fatalf("the seat read from the store refuses today's holder: %v", err)
|
||||
}
|
||||
// Seeding again changes nothing more.
|
||||
before := verbs["journal"]
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after, _ := inv.Seats(ctx)
|
||||
for _, s := range after {
|
||||
if s.Name == catalogue.ServiceManagerSeat {
|
||||
for _, v := range s.Serves {
|
||||
if v.Name == "journal" && !sameVerb(v, before) {
|
||||
t.Fatalf("re-seeding changed journal again: %+v", v)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -428,6 +428,45 @@ type Health struct {
|
||||
// At is when the engine looked, on the machine's clock: the order of its statements.
|
||||
At time.Time `json:"at"`
|
||||
Resources []ResourceHealth `json:"resources"`
|
||||
// Network is the machine's own networking, judged by its engine (novox/hq ADR 0241); nil from an engine
|
||||
// older than that judging, which is "not known", never healthy.
|
||||
Network *NetworkHealth `json:"network,omitempty"`
|
||||
}
|
||||
|
||||
// NetworkHealth is a machine's networking in one statement (ADR 0241): the worst of its parts, since
|
||||
// when, and each part. The node-engine's own (mesh-host internal/link NetworkHealth).
|
||||
type NetworkHealth struct {
|
||||
State string `json:"state"`
|
||||
Since time.Time `json:"since"`
|
||||
Parts []NetworkPart `json:"parts"`
|
||||
}
|
||||
|
||||
// The parts of a machine's networking its engine judges (ADR 0241).
|
||||
const (
|
||||
PartResolvConf = "resolv-conf"
|
||||
PartNames = "names"
|
||||
PartTunnel = "tunnel"
|
||||
PartBus = "bus"
|
||||
PartRoute = "route"
|
||||
// TowardHub is what a part failing toward the hub names in Toward.
|
||||
TowardHub = "hub"
|
||||
)
|
||||
|
||||
// NetworkPart is one part, as the engine judged it on its second look.
|
||||
type NetworkPart struct {
|
||||
Part string `json:"part"`
|
||||
State string `json:"state"`
|
||||
// Reason is in words with no address, path or domain; Said is the detail, kept as evidence.
|
||||
Reason string `json:"reason,omitempty"`
|
||||
Said string `json:"said,omitempty"`
|
||||
// Writer is the program that rewrote the resolver file, when the engine could name it; Owner the
|
||||
// module whose file it is.
|
||||
Writer string `json:"writer,omitempty"`
|
||||
Owner string `json:"owner,omitempty"`
|
||||
// Toward is what a failure points at: "hub", or each resolver's address that failed.
|
||||
Toward []string `json:"toward,omitempty"`
|
||||
Since time.Time `json:"since"`
|
||||
Streak int `json:"streak,omitempty"`
|
||||
}
|
||||
|
||||
// The states a resource is said in (ADR 0240 §4).
|
||||
|
||||
@@ -33,6 +33,13 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) {
|
||||
[]string{"module", "resource", "kind", "target", "state", "reason", "since", "streak", "restarts",
|
||||
"check", "needs"}},
|
||||
{HealthSaid{Node: "n"}, []string{"node", "health"}},
|
||||
// novox/hq ADR 0241: the machine's own networking, beside its resources.
|
||||
{Health{Contract: ReadinessContract, Resources: []ResourceHealth{}, Network: &NetworkHealth{State: "unhealthy",
|
||||
Parts: []NetworkPart{}}}, []string{"contract", "at", "resources", "network"}},
|
||||
{NetworkHealth{State: "unhealthy", Parts: []NetworkPart{}}, []string{"state", "since", "parts"}},
|
||||
{NetworkPart{Part: "resolv-conf", State: "unhealthy", Reason: "r", Said: "s", Writer: "w", Owner: "o",
|
||||
Toward: []string{"hub"}, Streak: 2}, []string{"part", "state", "reason", "said", "writer", "owner", "toward",
|
||||
"since", "streak"}},
|
||||
} {
|
||||
raw, err := json.Marshal(c.value)
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user