Merge feat/modules-name-no-node into bulk-b

This commit is contained in:
jochen
2026-10-22 21:31:14 +02:00
46 changed files with 1006 additions and 868 deletions
-3
View File
@@ -1,3 +0,0 @@
# The modules the check covers (novox/hq ADR 0507 §9, issue 1838), one a line; issue 1838 widens it.
pacman
kernel
+41 -16
View File
@@ -1,12 +1,14 @@
// Command particulars keeps this mesh's node names out of the modules it covers (novox/hq ADR 0507 §9,
// issue 1838): a module is for any mesh, so no manifest, file, code string, identifier, comment, test name or
// fixture of a covered module names a node of this one.
// fixture of any module names a node of this one.
//
// The names are in the file `particulars` at the catalogue's root, one a line, `#` comments — the one place
// they may be written. The modules covered are in `covered` at checks/particulars/covered, one a line. A name is matched
// whole and in any case; a hyphen, an underscore or a CamelCase boundary parts one, so a longer word holding a name is another word.
// The forge is not a node name: its host (git.novox.be) and the organisation path (novox/<repo>) are blanked
// before a line is read.
// they may be written. Every directory under modules/ is read, with no list of covered modules, so a new module
// is covered from its first commit. A name is matched whole and in any case; a hyphen, an underscore or a
// CamelCase boundary parts one, so a longer word holding a name is another word.
// Three names hold a node name and are not one, so they are blanked before a line is read: the forge's host
// (git.novox.be) and organisation path (novox/<repo>), the hardware model (asus-zephyrus-g14, ROG Zephyrus G14)
// and the software's own name (Novox Mesh, novox-mesh).
//
// go run . <catalogue root>
//
@@ -29,6 +31,11 @@ import (
// (testdata/novox/x, /srv/novox/data) is not blanked.
var forge = regexp.MustCompile(`(?i)git\.novox\.be(?:/novox/[\w.-]+)?|(?:^|[^/\w.-])novox/[\w.-]+`)
// ownNames are the hardware model's and the software's own names, blanked before a line is read: the model
// in any separator and case (asus-zephyrus-g14, ASUS_ZEPHYRUS_G14, ROG Zephyrus G14), the software as two
// words or hyphened, whole (Novox Mesh, novox mesh, novox-mesh), so novox-mesh-vault stays a node name joined.
var ownNames = regexp.MustCompile(`(?i)(?:asus[-_ ])?zephyrus[-_ ]g14|(?:^|[^\w-])novox[- ]mesh(?:$|[^\w-])`)
func main() {
root := "."
if len(os.Args) > 1 {
@@ -43,18 +50,18 @@ func check(root string) int {
if err == nil && len(names) == 0 {
err = fmt.Errorf("particulars lists no name — the check would pass on anything")
}
var covered []string
var modules []string
if err == nil {
covered, err = readList(filepath.Join(root, "checks/particulars/covered"))
if err == nil && len(covered) == 0 {
err = fmt.Errorf("covered lists no module — the check would pass on anything")
modules, err = moduleDirs(root)
if err == nil && len(modules) == 0 {
err = fmt.Errorf("modules/ holds no module — the check would pass on anything")
}
}
if err != nil {
fmt.Fprintln(os.Stderr, "particulars:", err)
return 2
}
findings, err := run(root, names, covered)
findings, err := run(root, names, modules)
if err != nil {
fmt.Fprintln(os.Stderr, "particulars:", err)
return 2
@@ -66,7 +73,7 @@ func check(root string) int {
fmt.Printf("particulars: %d node name(s) in a module for any mesh (novox/hq ADR 0507 §9) — say what it shows, not where it was seen\n", len(findings))
return 1
}
fmt.Printf("particulars: none of %d node names in the %d covered modules\n", len(names), len(covered))
fmt.Printf("particulars: none of %d node names in the %d modules\n", len(names), len(modules))
return 0
}
@@ -89,10 +96,25 @@ func readList(path string) ([]string, error) {
return out, sc.Err()
}
// run reads every text file of each covered module and returns the findings, sorted.
func run(root string, names, covered []string) ([]string, error) {
// moduleDirs returns every directory under the catalogue's modules/, sorted.
func moduleDirs(root string) ([]string, error) {
entries, err := os.ReadDir(filepath.Join(root, "modules"))
if err != nil {
return nil, err
}
var out []string
for _, m := range covered {
for _, e := range entries {
if e.IsDir() {
out = append(out, e.Name())
}
}
return out, nil
}
// run reads every text file of each module named and returns the findings, sorted.
func run(root string, names, modules []string) ([]string, error) {
var out []string
for _, m := range modules {
dir := filepath.Join(root, "modules", m)
err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
if err != nil {
@@ -143,9 +165,12 @@ func scan(file, src string, names []string) []string {
}
// hits returns the node names one line holds: as a word (a hyphen or underscore parts it), as a part of a
// CamelCase identifier (TestAce, novoxTree), the forge's host and organisation path blanked first.
// CamelCase identifier (TestAce, novoxTree), the forge's host and organisation path and the hardware's and the
// software's own names blanked first.
func hits(line string, names []string) []string {
line = forge.ReplaceAllStringFunc(line, func(s string) string { return strings.Repeat(" ", len(s)) })
blank := func(s string) string { return strings.Repeat(" ", len(s)) }
line = forge.ReplaceAllStringFunc(line, blank)
line = ownNames.ReplaceAllStringFunc(line, blank)
var out []string
for _, n := range names {
q := regexp.QuoteMeta(n)
+57 -2
View File
@@ -82,7 +82,6 @@ func TestCheckReturnsOneOnAFindingAndTwoOnAnUnreadableList(t *testing.T) {
os.WriteFile(p, []byte(s), 0o644)
}
write("particulars", "ace\n")
write("checks/particulars/covered", "kernel\n")
write("modules/kernel/a", "clean\n")
if got := check(root); got != 0 {
t.Errorf("clean = %d, want 0", got)
@@ -97,7 +96,7 @@ func TestCheckReturnsOneOnAFindingAndTwoOnAnUnreadableList(t *testing.T) {
}
}
func TestOnlyCoveredModulesAreRead(t *testing.T) {
func TestRunReadsOnlyTheModulesItIsGiven(t *testing.T) {
root := t.TempDir()
write := func(p, s string) {
p = filepath.Join(root, p)
@@ -114,3 +113,59 @@ func TestOnlyCoveredModulesAreRead(t *testing.T) {
t.Errorf("got %v", got)
}
}
// 1838/AC1: the check reads every directory under modules/ and no list of covered modules, so a module is
// covered from its first commit.
func TestEveryModuleIsReadWithoutAList(t *testing.T) {
root := t.TempDir()
write := func(p, s string) {
p = filepath.Join(root, p)
os.MkdirAll(filepath.Dir(p), 0o755)
os.WriteFile(p, []byte(s), 0o644)
}
write("particulars", "novox\n")
write("modules/kernel/files/a", "clean\n")
if got := check(root); got != 0 {
t.Errorf("clean, no covered list = %d, want 0", got)
}
write("modules/new-module/files/a", "on novox\n")
if got := check(root); got != 1 {
t.Errorf("a finding in a module no list names = %d, want 1", got)
}
}
// 1838/AC2: the hardware model's own name and the software's own name are not node names.
func TestTheHardwareModelAndTheSoftwareAreNotNodeNames(t *testing.T) {
for _, s := range []string{
"modules/asus-zephyrus-g14/README.md",
"the asus-zephyrus-g14 module",
"ASUS_ZEPHYRUS_G14=1",
"product_family: ROG Zephyrus G14",
"# This machine is a node of a Novox mesh",
"Novox Mesh turns a handful of personal machines into one system",
"<!-- novox mesh: this file was changed on two nodes",
`const sealInfo = "novox-mesh sealed box v1"`,
"module git.novox.be/novox/mesh-catalog/modules/kernel",
} {
if got := scan("f", s, names); len(got) != 0 {
t.Errorf("%q refused: %v", s, got)
}
}
}
// 1838/AC2: a node name standing alone is still one, beside the hardware's or the software's name too.
func TestANodeNameBesideTheHardwareOrTheSoftwareIsRefused(t *testing.T) {
cases := map[string]string{
"g14": "the asus-zephyrus-g14 module on g14",
"shanks": "Novox Mesh on shanks",
"novox": "resolves under novox.internal",
"ace": "ROG Zephyrus G14 beside ace",
"novox mesh-ish": "mesh.mod.gitea.call.x.novox-mesh-vault",
}
for want, s := range cases {
got := scan("modules/m/a.go", s, names)
if len(got) != 1 || !strings.HasPrefix(got[0], "modules/m/a.go:1: ") {
t.Errorf("%s: %q gave %v, want one finding naming file:line", want, s, got)
}
}
}
@@ -347,7 +347,7 @@ func TestAReportIsReadAsTheAgentModuleWritesIt(t *testing.T) {
func TestANodeReportingAnAdoptedAccountLaterIsBoundToIt(t *testing.T) {
mm := newMesh(t)
ctx := context.Background()
_, _ = mm.m.Consider(ctx, []Holdings{mm.login("novox", "rt-new", true, t0)})
_, _ = mm.m.Consider(ctx, []Holdings{mm.login("node-c", "rt-new", true, t0)})
late := mm.login("laptop", "rt-older", true, t0.Add(-24*time.Hour))
_, _ = mm.m.Consider(ctx, []Holdings{late})
if mm.state["laptop"].Licence != licence1 {
@@ -477,16 +477,16 @@ func TestEveryMoveSaysWhatWhyAndWho(t *testing.T) {
t.Fatal(err)
}
asked := WithCause(ctx, Cause{Why: "the switch verb", By: "g14.node-tools"})
asked := WithCause(ctx, Cause{Why: "the switch verb", By: "node-b.node-tools"})
if _, err := mm.m.Bind(asked, "laptop", "api"); err != nil {
t.Fatal(err)
}
if s := mm.state["laptop"]; s.Why != "the switch verb" || s.By != "g14.node-tools" {
if s := mm.state["laptop"]; s.Why != "the switch verb" || s.By != "node-b.node-tools" {
t.Fatalf("the node is told %+v", s)
}
said := strings.Join(lines, "\n")
if !strings.Contains(said, "switched laptop from "+licence1+" to api (generation ") ||
!strings.Contains(said, "the switch verb; asked by g14.node-tools") {
!strings.Contains(said, "the switch verb; asked by node-b.node-tools") {
t.Fatalf("the journal said:\n%s", said)
}
var switched string
@@ -495,18 +495,18 @@ func TestEveryMoveSaysWhatWhyAndWho(t *testing.T) {
switched = e
}
}
for _, want := range []string{`"by":"g14.node-tools"`, `"from":"` + licence1 + `"`, `"licence":"api"`, `"consumer":"laptop"`, `"generation":`} {
for _, want := range []string{`"by":"node-b.node-tools"`, `"from":"` + licence1 + `"`, `"licence":"api"`, `"consumer":"laptop"`, `"generation":`} {
if !strings.Contains(switched, want) {
t.Fatalf("the event %q lacks %s; events %v", switched, want, mm.events)
}
}
last := mm.store.Audits[len(mm.store.Audits)-1]
if raw, _ := json.Marshal(last); !strings.Contains(string(raw), `"by":"g14.node-tools"`) || !strings.Contains(string(raw), "switched") {
if raw, _ := json.Marshal(last); !strings.Contains(string(raw), `"by":"node-b.node-tools"`) || !strings.Contains(string(raw), "switched") {
t.Fatalf("the audit kept %s", raw)
}
_, _ = mm.m.Rotate(WithCause(ctx, Cause{Why: "the refresh verb", By: "novox.node-tools"}), licence1, true)
if !strings.Contains(strings.Join(mm.events, "|"), `licence.rotated {"by":"novox.node-tools"`) {
_, _ = mm.m.Rotate(WithCause(ctx, Cause{Why: "the refresh verb", By: "node-c.node-tools"}), licence1, true)
if !strings.Contains(strings.Join(mm.events, "|"), `licence.rotated {"by":"node-c.node-tools"`) {
t.Fatalf("a rotation asked for said no caller: %v", mm.events)
}
+4 -3
View File
@@ -38,7 +38,7 @@ the SDK, or the catalogue's bus, forge or build agent) resolves with `replay` or
needs a `reason`. A refused change changes nothing and says which rule refused it.
**Who works on it** (hq ADR 0284). A move to `diagnosing` or `fixing` names in `by` the issue's **worker**: who
works on it and its node, as `who@node` (such as `fix-399@g14` or `jochen@shanks`, kept in lower case). The issue
works on it and its node, as `who@node` (such as `fix-399@node-b` or `operator@node-d`, kept in lower case). The issue
keeps its worker (`worker`: `by`, `since`, `active`) while it stays at either status; any other status ends the
work. `get` shows it, with `worker-stale`, and each `list` row shows `worker`, `since` and `stale`. A move to either
status by someone else while the worker is current is refused, naming the worker; with `take-over` saying why, the
@@ -342,7 +342,7 @@ when. A change whose event the bus did not take is kept, and its answer says the
ADR 0275 §5 records the caller from the bus credential, never from an argument. **Who acted is the bus user
that called** (hq issue 365): a call on a tool's caller-named subject carries the caller's bus user as its last
token, the bus lets each user publish there with its own name alone, and the tool runner hands the module that
user from the subject the call arrived on (mesh-sdk `stdio.CallerOf`) — `shanks.node-tools` for an agent on
user from the subject the call arrived on (mesh-sdk `stdio.CallerOf`) — `node-d.node-tools` for an agent on
that machine's console, `person.<name>` for a person's credential. A call on the subject that names no caller
(served beside it for one release, hq issue 464), or through a runtime from before issue 365, is recorded as
by `unproven`. No verb takes an argument naming a caller, and none can be given one. The caller is only as sound
@@ -351,7 +351,8 @@ as the credential, which ADR 0266 keeps agents from reading (hq issue 333).
## Settings
`duplicate-check` (`on`; `off` stops the triage of new issues), `judge-model` (`sonnet`; `haiku` or `opus`) and
`forge-owner` (the forge's owner of the repositories `fixed-by` names, for asking a delivery) steer the triage;
`forge-owner` (the forge's owner of the repositories `fixed-by` names, for asking a delivery: required, with no
default, novox/hq ADR 0507 §9) steer the triage;
`stuck-after` (`2h`) is how long a merged fix may take to be delivered before it is said;
`delivered-not-resolved-after` (`1h`) how long an issue may stay in fixing once every fix is delivered before it is
said (hq issue 483).
@@ -427,9 +427,16 @@ func nonZero(t *time.Time) *time.Time {
// DeliveryState asks mesh-delivery for one pull request's delivery.
type DeliveryState func(repository string, number string) (Delivery, error)
// errNoForgeOwner is what asking the forge says when no forge owner is set: the forge is not asked under a
// name nobody gave (novox/hq ADR 0507 §9).
var errNoForgeOwner = errors.New("forge-owner is not set: the forge is not asked (set the module's forge-owner)")
// busDeliveryState asks mesh-delivery's checks over the bus.
func busDeliveryState(owner string) DeliveryState {
return func(repository, number string) (Delivery, error) {
if owner == "" {
return Delivery{}, errNoForgeOwner
}
raw, err := stdio.Ask("seat:mesh-delivery.checks", map[string]any{"repository": owner + "/" + repository, "number": number})
if err != nil {
return Delivery{}, err
@@ -362,3 +362,15 @@ func TestATransitionIsTakenUnderEveryNameTheTrackerHears(t *testing.T) {
})
}
}
// With no forge owner, the forge is not asked under a name nobody gave: the delivery and the approval each say
// the owner is not set (novox/hq ADR 0507 §9).
// 1838/AC4
func TestAnUnsetForgeOwnerAsksNoForgeAndSaysWhy(t *testing.T) {
if _, err := busDeliveryState("")("mesh-catalog", "5"); err == nil || !strings.Contains(err.Error(), "forge-owner is not set") {
t.Errorf("delivery with no owner: err %v, want it to say forge-owner is not set", err)
}
if _, _, err := busApproval(func() string { return "" })("mesh-catalog", 5); err == nil || !strings.Contains(err.Error(), "forge-owner is not set") {
t.Errorf("approval with no owner: err %v, want it to say forge-owner is not set", err)
}
}
@@ -638,7 +638,11 @@ func verdictApproves(body, commit string) bool {
// busApproval asks the forge's reviews of a pull request over the bus (gitea_pull_reviews).
func busApproval(owner func() string) Approval {
return func(repository string, number int) (time.Time, bool, error) {
raw, err := stdio.Ask("gitea.gitea_pull_reviews", map[string]any{"owner": owner(), "repo": repository, "number": number})
o := owner()
if o == "" {
return time.Time{}, false, errNoForgeOwner
}
raw, err := stdio.Ask("gitea.gitea_pull_reviews", map[string]any{"owner": o, "repo": repository, "number": number})
if err != nil {
return time.Time{}, false, err
}
+6 -5
View File
@@ -39,7 +39,8 @@ type Settings struct {
// JudgeModel is the model the duplicate check asks; DefaultJudgeModel when empty.
JudgeModel string `json:"judge-model,omitempty"`
// ForgeOwner is the owner on the forge of the repositories fixed-by names, for asking a pull request's
// delivery (ADR 0279 §3).
// delivery (ADR 0279 §3): the operator's, a required config value with no default (ADR 0507 §9); empty,
// the forge is not asked and each asking says so.
ForgeOwner string `json:"forge-owner,omitempty"`
// StuckAfter is how long a fix may be merged and not delivered before it is said stuck (ADR 0281), as a
// duration such as 2h; DefaultStuckAfter when empty.
@@ -99,7 +100,7 @@ func main() {
if s, err := t.Settings(); err == nil && strings.TrimSpace(s.ForgeOwner) != "" {
return strings.TrimSpace(s.ForgeOwner)
}
return "novox"
return ""
}
t.Deliveries = func(repository, number string) (Delivery, error) {
return busDeliveryState(forgeOwner())(repository, number)
@@ -533,7 +534,7 @@ func Tools(t *Tracker) []stdio.Tool {
{Name: Seat + ".move",
Description: "Change an issue's status — " + statuses + " — with what that status needs, under the " +
"development cycle's rules. Move an issue to diagnosing or fixing when you start work on it, with by: " +
"who works on it and its node, as who@node (such as fix-399@g14); the issue shows it until it moves on. " +
"who works on it and its node, as who@node (such as fix-399@node-b); the issue shows it until it moves on. " +
"Refused while someone else works on it, naming them, unless take-over says why (noted on the issue); a " +
"worker who did nothing on it for a day may be taken over from without a reason, and with no pull request open " +
"the answer says it was abandoned. located needs located-in, owners on " +
@@ -551,7 +552,7 @@ func Tools(t *Tracker) []stdio.Tool {
"replay": text("for a core issue resolved: the replay's id in mesh-lab's replays register, such as R361"),
"replay-none": text("for a core issue resolved: why no replay is possible"),
"reason": text("for wontfix: why it will not be fixed"),
"by": text("for diagnosing and fixing: who works on it and its node, as who@node, such as fix-399@g14 or jochen@shanks"),
"by": text("for diagnosing and fixing: who works on it and its node, as who@node, such as fix-399@node-b or operator@node-d"),
"take-over": text("for diagnosing and fixing, when someone else works on it: why you take it over"),
}),
Run: func(m map[string]any) (any, error) {
@@ -734,7 +735,7 @@ func Tools(t *Tracker) []stdio.Tool {
"operator's channel asks it to say in a condition's message which issue tracks it. Refused: a key not of a condition " +
"key's shape.",
Input: schema(nil, map[string]any{
"conditions": texts("only these condition keys, such as module.mounts.shanks.unhealthy; all when not given"),
"conditions": texts("only these condition keys, such as module.mounts.node-d.unhealthy; all when not given"),
}),
Run: func(m map[string]any) (any, error) {
a, err := readArgs("tracking", m, "conditions")
@@ -270,3 +270,40 @@ func TestEveryCodeModuleOfTheCatalogueIsAnOwner(t *testing.T) {
}
}
}
// The forge's owner differs between meshes, so it is a config value the operator gives, with no default naming
// this mesh, and the template the config merges into holds none either (novox/hq ADR 0507 §9).
// 1838/AC4
func TestTheForgeOwnerIsARequiredValueWithNoDefault(t *testing.T) {
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
if err != nil {
t.Fatal(err)
}
var m struct {
Config map[string]map[string]any `json:"config"`
}
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
owner, ok := m.Config["forge-owner"]
if !ok {
t.Fatal("no forge-owner config value")
}
if d, has := owner["default"]; has {
t.Errorf("forge-owner has the default %v: the forge's owner is the operator's, with no default", d)
}
if owner["required"] != true {
t.Errorf("forge-owner is not required: an unset owner would compose and ask no forge in silence")
}
tmpl, err := os.ReadFile(filepath.Join("..", "..", "files", "settings.json"))
if err != nil {
t.Fatal(err)
}
var s map[string]any
if err := json.Unmarshal(tmpl, &s); err != nil {
t.Fatal(err)
}
if v, has := s["forge-owner"]; has {
t.Errorf("files/settings.json holds forge-owner %v: the value comes from the config only", v)
}
}
@@ -393,7 +393,7 @@ func (t *Tracker) move(by string, a MoveAsk) (*Moved, error) {
if worked(to) {
if strings.TrimSpace(a.By) == "" {
return nil, fmt.Errorf("a move to %s needs by: who works on it and its node, as who@node, such as "+
"fix-399@g14 or jochen@shanks", to)
"fix-399@node-b or operator@node-d", to)
}
if worker, err = checkWorker(a.By); err != nil {
return nil, err
@@ -53,7 +53,7 @@ func checkWorkerField(field, s string) (string, error) {
s = strings.ToLower(s)
if !workerName.MatchString(s) {
return "", fmt.Errorf("%s %q is not who works on it and its node: give who@node, the agent or person and the "+
"machine it works on, such as fix-399@g14 or jochen@shanks", field, s)
"machine it works on, such as fix-399@node-b or operator@node-d", field, s)
}
return s, nil
}
-1
View File
@@ -161,7 +161,6 @@
],
"duplicate-check": "on",
"judge-model": "sonnet",
"forge-owner": "novox",
"stuck-after": "2h",
"worker-stale-after": "24h",
"delivered-not-resolved-after": "1h",
+2 -2
View File
@@ -267,9 +267,9 @@
},
"forge-owner": {
"type": "string",
"description": "The owner on the forge of the repositories an issue's fixed-by names, for asking after a pull request's delivery (novox/hq ADR 0279 §3); empty asks no forge.",
"description": "The owner on the forge of the repositories an issue's fixed-by names, for asking after a pull request's delivery (novox/hq ADR 0279 §3), such as the organisation that holds them: the operator's, with no default (novox/hq ADR 0507 §9).",
"trusted": true,
"default": "novox"
"required": true
},
"stuck-after": {
"type": "string",
@@ -30,8 +30,8 @@ func (b *conditionBed) controllerEvent(t *testing.T, event string, body any) {
func walkEvent(phase, outcome string, started, at time.Time) map[string]any {
b := map[string]any{"id": "plan-bus", "state": "walking", "walk": "plan-bus", "phase": phase, "budget_class": "leaf",
"bound_seconds": 600, "started_at": started.UTC().Format(time.RFC3339Nano),
"touches": []map[string]any{{"machine": "novox", "modules": []string{"gitea"}},
{"machine": "shanks", "modules": []string{"mesh-build-image"}, "declared": true}}}
"touches": []map[string]any{{"machine": "node-c", "modules": []string{"gitea"}},
{"machine": "node-d", "modules": []string{"mesh-build-image"}, "declared": true}}}
if phase == WalkEnded {
b["outcome"], b["ended_at"] = outcome, at.UTC().Format(time.RFC3339Nano)
}
@@ -64,23 +64,23 @@ func TestAWalksDeclaredMachineIsHeldUntilItSettledOverTheBus(t *testing.T) {
b := newConditionBedWith(t, `{"settle-seconds": 2}`)
started := time.Now()
b.controllerEvent(t, EventPlanMoved, walkEvent(WalkStarted, "", started, started))
b.controllerEvent(t, EventRaised, conditionEvent("machine.shanks.uncomposable", "machine", "shanks", "shanks",
"Shanks cannot get new instructions"))
b.controllerEvent(t, EventRaised, conditionEvent("machine.ace.silent", "machine", "ace", "ace", "Ace is silent"))
b.controllerEvent(t, EventRaised, conditionEvent("machine.node-d.uncomposable", "machine", "node-d", "node-d",
"Node-d cannot get new instructions"))
b.controllerEvent(t, EventRaised, conditionEvent("machine.node-a.silent", "machine", "node-a", "node-a", "Node-a is silent"))
b.r.until("the untouched machine's condition reaches the desk with its delivery line", 15*time.Second, func() bool {
m := b.toldWith("Ace is silent")
m := b.toldWith("Node-a is silent")
return m != nil && strings.HasSuffix(*m, "\nNo delivery running.")
})
if b.toldWith("Shanks cannot get new instructions") != nil {
if b.toldWith("Node-d cannot get new instructions") != nil {
t.Fatalf("the declared machine's condition was told during the walk")
}
b.controllerEvent(t, EventPlanMoved, walkEvent(WalkEnded, WalkDelivered, started, time.Now()))
time.Sleep(time.Second)
if b.toldWith("Shanks cannot get new instructions") != nil || b.toldWith("Delivered") != nil {
if b.toldWith("Node-d cannot get new instructions") != nil || b.toldWith("Delivered") != nil {
t.Fatalf("told before the settle passed: %+v", b.told())
}
b.r.until("the walk's end tells what it held once it settled", 30*time.Second, func() bool {
m := b.toldWith("Still open after the delivery of gitea to novox:")
return m != nil && strings.Contains(*m, "Shanks cannot get new instructions") && !strings.Contains(*m, "shanks,")
m := b.toldWith("Still open after the delivery of gitea to node-c:")
return m != nil && strings.Contains(*m, "Node-d cannot get new instructions") && !strings.Contains(*m, "node-d,")
})
}
+58 -58
View File
@@ -61,7 +61,7 @@ func controllerConditions(c *clock) []Condition {
// open, whatever name their keys give the controller and whatever host they name.
func TestTheControllersOwnConditionsAreHeldWhileAWalkMovesMeshController(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
walk(t, h, "plan-c", WalkStarted, "", c.now(), 1200, novox("mesh-controller"))
walk(t, h, "plan-c", WalkStarted, "", c.now(), 1200, nodeC("mesh-controller"))
for _, k := range controllerConditions(c) {
h.Condition(EventRaised, k)
}
@@ -79,7 +79,7 @@ func TestTheControllersOwnConditionsAreHeldWhileAWalkMovesMeshController(t *test
// ADR 0505 §2a: a walk that does not move mesh-controller holds none of them.
func TestTheControllersOwnConditionsAreNotHeldByAWalkOfAnotherModule(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
walk(t, h, "plan-g", WalkStarted, "", c.now(), 1200, Touch{Machine: "ace", Modules: []string{"gitea"}})
walk(t, h, "plan-g", WalkStarted, "", c.now(), 1200, Touch{Machine: "node-a", Modules: []string{"gitea"}})
for _, k := range controllerConditions(c) {
h.Condition(EventRaised, k)
}
@@ -91,32 +91,32 @@ func TestTheControllersOwnConditionsAreNotHeldByAWalkOfAnotherModule(t *testing.
}
}
// shanksDeclared is a touch as the controller says a machine whose declaration the walk's merges change and to which
// nodeDDeclared is a touch as the controller says a machine whose declaration the walk's merges change and to which
// it sends nothing (mesh-controller walk_said.go walkTouch, `declared`, ADR 0505 §2b).
var shanksDeclared = Touch{Machine: "shanks", Modules: []string{"mesh-build-image"}, Declared: true}
var nodeDDeclared = Touch{Machine: "node-d", Modules: []string{"mesh-build-image"}, Declared: true}
// ADR 0505 §2b: a machine whose declaration a walk's merges change is held like a machine it sends to, and is not
// named among the machines the walk reached.
func TestAMachineWhoseDeclarationTheWalkChangesIsHeldAndNotNamedAsReached(t *testing.T) {
ev, ok, err := DecodeWalk([]byte(`{"id":"plan-d","walk":"plan-d","phase":"started","budget_class":"leaf",` +
`"bound_seconds":600,"touches":[{"machine":"novox","modules":["gitea"]},` +
`{"machine":"shanks","modules":["mesh-build-image"],"declared":true}]}`))
`"bound_seconds":600,"touches":[{"machine":"node-c","modules":["gitea"]},` +
`{"machine":"node-d","modules":["mesh-build-image"],"declared":true}]}`))
if err != nil || !ok || len(ev.Touches) != 2 || !ev.Touches[1].Declared || ev.Touches[0].Declared {
t.Fatalf("a declared touch was not read: %+v %v %v", ev.Touches, ok, err)
}
h, tg, dt, c, _ := newHolder(t)
walk(t, h, "plan-d", WalkStarted, "", c.now(), 600, novox("gitea"), shanksDeclared)
k := cond("machine.shanks.uncomposable", Urgent, "shanks cannot get new instructions")
k.Machine, k.Raised = "shanks", c.now()
walk(t, h, "plan-d", WalkStarted, "", c.now(), 600, nodeC("gitea"), nodeDDeclared)
k := cond("machine.node-d.uncomposable", Urgent, "node-d cannot get new instructions")
k.Machine, k.Raised = "node-d", c.now()
h.Condition(EventRaised, k)
runFor(h, c, time.Minute)
if !has(outcomes(h, k.Key), "held") || heard(headlineOf(k.Key), tg, dt) {
t.Fatalf("a condition on the declared machine was not held: %v", outcomes(h, k.Key))
}
if got := touchWords(mergeTouches([]Touch{novox("gitea"), shanksDeclared}, nil)); got != "gitea on novox" {
t.Fatalf("what the walk reached reads %q, want only novox", got)
if got := touchWords(mergeTouches([]Touch{nodeC("gitea"), nodeDDeclared}, nil)); got != "gitea on node-c" {
t.Fatalf("what the walk reached reads %q, want only node-c", got)
}
if got := mergeTouches([]Touch{shanksDeclared}, []Touch{{Machine: "shanks", Modules: []string{"buildah"}}}); got[0].Declared {
if got := mergeTouches([]Touch{nodeDDeclared}, []Touch{{Machine: "node-d", Modules: []string{"buildah"}}}); got[0].Declared {
t.Fatalf("a machine the walk also sends to stayed declared: %+v", got)
}
}
@@ -127,12 +127,12 @@ func TestAMachineWhoseDeclarationTheWalkChangesIsHeldAndNotNamedAsReached(t *tes
func TestAConditionRaisedInTheSettleIsHeldAndToldAtItsEnd(t *testing.T) {
for _, clears := range []bool{false, true} {
h, tg, dt, c, _ := newHolder(t)
walk(t, h, "plan-s", WalkStarted, "", c.now(), 600, novox("gitea"))
walk(t, h, "plan-s", WalkStarted, "", c.now(), 600, nodeC("gitea"))
runFor(h, c, 3*time.Minute)
walk(t, h, "plan-s", WalkEnded, WalkDelivered, c.now(), 0, novox("gitea"))
walk(t, h, "plan-s", WalkEnded, WalkDelivered, c.now(), 0, nodeC("gitea"))
c.pass(30 * time.Second)
k := cond("machine.novox.uncomposable", Urgent, "novox cannot get new instructions")
k.Machine, k.Raised = "novox", c.now()
k := cond("machine.node-c.uncomposable", Urgent, "node-c cannot get new instructions")
k.Machine, k.Raised = "node-c", c.now()
h.Condition(EventRaised, k)
runFor(h, c, time.Minute)
if heard(headlineOf(k.Key), tg, dt) || !has(outcomes(h, k.Key), "held") {
@@ -146,7 +146,7 @@ func TestAConditionRaisedInTheSettleIsHeldAndToldAtItsEnd(t *testing.T) {
}
runFor(h, c, time.Minute)
switch {
case !clears && !heard("Still open after the delivery of gitea to novox:\n- Urgent: "+headlineOf(k.Key), tg, dt):
case !clears && !heard("Still open after the delivery of gitea to node-c:\n- Urgent: "+headlineOf(k.Key), tg, dt):
t.Fatalf("still open at the settle's end and not told under the delivery: %+v", dt.sends)
case clears && (heard(headlineOf(k.Key), tg, dt) || !heard("1 problem it raised resolved during it.", dt)):
t.Fatalf("cleared in the settle: told, or not counted: %+v", dt.sends)
@@ -158,7 +158,7 @@ func TestAConditionRaisedInTheSettleIsHeldAndToldAtItsEnd(t *testing.T) {
// any other message, and told at the settle's end only when the condition is still open.
func TestAReminderDuringAWalkIsHeld(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
k := cond("machine.novox.network", Urgent, "novox lost its default route")
k := cond("machine.node-c.network", Urgent, "node-c lost its default route")
k.Raised = c.now()
h.Condition(EventRaised, k)
runFor(h, c, time.Minute)
@@ -167,7 +167,7 @@ func TestAReminderDuringAWalkIsHeld(t *testing.T) {
t.Fatalf("the condition was not said before the walk")
}
runFor(h, c, RemindUrgent-3*time.Minute)
walk(t, h, "plan-r", WalkStarted, "", c.now(), 1200, novox("node-tools"))
walk(t, h, "plan-r", WalkStarted, "", c.now(), 1200, nodeC("node-tools"))
runFor(h, c, 5*time.Minute)
if saidOn(tg, dt) != first {
t.Fatalf("the reminder was said during the walk: %+v", dt.sends)
@@ -177,7 +177,7 @@ func TestAReminderDuringAWalkIsHeld(t *testing.T) {
}
walk(t, h, "plan-r", WalkEnded, WalkDelivered, c.now(), 0)
runFor(h, c, DefaultSettle+time.Minute)
if !heard("Still open after the delivery of node-tools to novox:", dt) {
if !heard("Still open after the delivery of node-tools to node-c:", dt) {
t.Fatalf("the held reminder was not told at the settle's end: %+v", dt.sends)
}
}
@@ -185,7 +185,7 @@ func TestAReminderDuringAWalkIsHeld(t *testing.T) {
// ADR 0505 §3: a router that starts again while a walk settles holds on and tells its end once the settle passed.
func TestAWalkThatSettlesSurvivesARouterRestart(t *testing.T) {
h, tg, dt, c, st := newHolder(t)
walk(t, h, "plan-x", WalkStarted, "", c.now(), 1200, novox("mesh-tools"))
walk(t, h, "plan-x", WalkStarted, "", c.now(), 1200, nodeC("mesh-tools"))
runFor(h, c, time.Minute)
walk(t, h, "plan-x", WalkEnded, WalkDelivered, c.now(), 0)
c.pass(10 * time.Second)
@@ -194,7 +194,7 @@ func TestAWalkThatSettlesSurvivesARouterRestart(t *testing.T) {
t.Fatal(err)
}
again.Start()
k := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent")
k := cond("machine.node-c.tools-silent", Urgent, "the tools on node-c are silent")
k.Raised = c.now()
again.Condition(EventRaised, k)
runFor(again, c, time.Minute)
@@ -202,10 +202,10 @@ func TestAWalkThatSettlesSurvivesARouterRestart(t *testing.T) {
t.Fatalf("not held by the settling walk read back: %v", outcomes(again, k.Key))
}
runFor(again, c, DefaultSettle)
if !heard("Still open after the delivery of mesh-tools to novox:\n- Urgent: "+headlineOf(k.Key), tg, dt) {
if !heard("Still open after the delivery of mesh-tools to node-c:\n- Urgent: "+headlineOf(k.Key), tg, dt) {
t.Fatalf("the settled walk's end was not told after the restart: %+v", dt.sends)
}
walk(t, again, "plan-x", WalkStarted, "", c.now().Add(-5*time.Minute), 1200, novox("mesh-tools"))
walk(t, again, "plan-x", WalkStarted, "", c.now().Add(-5*time.Minute), 1200, nodeC("mesh-tools"))
if len(again.walks) != 0 {
t.Fatalf("its late start opened it again: %+v", again.walks)
}
@@ -216,13 +216,13 @@ func TestAWalkThatSettlesSurvivesARouterRestart(t *testing.T) {
func TestAConditionJustAfterAFailedWalkIsSentAtOnceAndAfterADeliveredOneHeld(t *testing.T) {
for _, outcome := range []string{WalkFailed, WalkStopped, WalkDelivered} {
h, tg, dt, c, _ := newHolder(t)
walk(t, h, "plan-f", WalkStarted, "", c.now(), 600, novox("gitea"))
walk(t, h, "plan-o", WalkStarted, "", c.now(), 1200, novox("mesh-issues"))
walk(t, h, "plan-f", WalkStarted, "", c.now(), 600, nodeC("gitea"))
walk(t, h, "plan-o", WalkStarted, "", c.now(), 1200, nodeC("mesh-issues"))
runFor(h, c, 2*time.Minute)
walk(t, h, "plan-f", WalkEnded, outcome, c.now(), 0, novox("gitea"))
walk(t, h, "plan-f", WalkEnded, outcome, c.now(), 0, nodeC("gitea"))
runFor(h, c, 30*time.Second)
k := cond("machine.novox.uncomposable", Urgent, "novox cannot get new instructions")
k.Machine, k.Raised = "novox", c.now()
k := cond("machine.node-c.uncomposable", Urgent, "node-c cannot get new instructions")
k.Machine, k.Raised = "node-c", c.now()
h.Condition(EventRaised, k)
runFor(h, c, 30*time.Second)
sent := heard(headlineOf(k.Key), tg, dt)
@@ -238,12 +238,12 @@ func TestAConditionJustAfterAFailedWalkIsSentAtOnceAndAfterADeliveredOneHeld(t *
// ADR 0505 §6: an urgent condition on a machine no open or settling walk touches is sent at once during a walk.
func TestAnUrgentConditionOnAnUntouchedMachineIsSentAtOnceDuringAWalk(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
walk(t, h, "plan-u", WalkStarted, "", c.now(), 1200, novox("gitea"))
k := cond("machine.ace.silent", Urgent, "ace has not been heard")
k.Machine, k.Raised = "ace", c.now()
walk(t, h, "plan-u", WalkStarted, "", c.now(), 1200, nodeC("gitea"))
k := cond("machine.node-a.silent", Urgent, "node-a has not been heard")
k.Machine, k.Raised = "node-a", c.now()
h.Condition(EventRaised, k)
if !heard(headlineOf(k.Key), tg, dt) {
t.Fatalf("held, while no walk touches ace: %v", outcomes(h, k.Key))
t.Fatalf("held, while no walk touches node-a: %v", outcomes(h, k.Key))
}
}
@@ -263,30 +263,30 @@ func bodyOf(t *testing.T, h *Holder, key, what string) string {
// walk's settle, after a failed walk, and with no walk.
func TestEveryProblemMessageEndsWithWhetherADeliveryRan(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
k := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent")
k := cond("machine.node-c.tools-silent", Urgent, "the tools on node-c are silent")
k.Raised = c.now()
h.Condition(EventRaised, k)
if !strings.HasSuffix(tg.sends[0].Body, ".\nNo delivery running.") || !strings.Contains(tg.sends[0].Body, "\nSince ") {
t.Fatalf("no walk: %q", tg.sends[0].Body)
}
walk(t, h, "plan-l", WalkStarted, "", c.now(), 600, novox("gitea", "mesh-issues"), shanksDeclared)
if b := bodyOf(t, h, k.Key, "reminder"); !strings.HasSuffix(b, ".\nDuring the delivery of gitea, mesh-issues to novox.") {
walk(t, h, "plan-l", WalkStarted, "", c.now(), 600, nodeC("gitea", "mesh-issues"), nodeDDeclared)
if b := bodyOf(t, h, k.Key, "reminder"); !strings.HasSuffix(b, ".\nDuring the delivery of gitea, mesh-issues to node-c.") {
t.Fatalf("during a walk: %q", b)
}
runFor(h, c, time.Minute)
walk(t, h, "plan-l", WalkEnded, WalkDelivered, c.now(), 0)
c.pass(40 * time.Second)
if b := bodyOf(t, h, k.Key, "raised"); !strings.HasSuffix(b, "\n40 s after the delivery of gitea, mesh-issues to novox, which succeeded.") {
if b := bodyOf(t, h, k.Key, "raised"); !strings.HasSuffix(b, "\n40 s after the delivery of gitea, mesh-issues to node-c, which succeeded.") {
t.Fatalf("in the settle of a delivered walk: %q", b)
}
runFor(h, c, 3*time.Minute)
if b := bodyOf(t, h, k.Key, "reminder"); !strings.HasSuffix(b, "\nNo delivery running.") {
t.Fatalf("after the settle: %q", b)
}
walk(t, h, "plan-m", WalkStarted, "", c.now(), 600, novox("gitea"))
walk(t, h, "plan-m", WalkStarted, "", c.now(), 600, nodeC("gitea"))
walk(t, h, "plan-m", WalkEnded, WalkFailed, c.now(), 0)
c.pass(30 * time.Second)
if b := bodyOf(t, h, k.Key, "reminder"); !strings.HasSuffix(b, "\n30 s after the delivery of gitea to novox, which failed.") {
if b := bodyOf(t, h, k.Key, "reminder"); !strings.HasSuffix(b, "\n30 s after the delivery of gitea to node-c, which failed.") {
t.Fatalf("after a failed walk: %q", b)
}
keyless(t, dt.sends...)
@@ -303,13 +303,13 @@ func TestEveryProblemMessageEndsWithWhetherADeliveryRan(t *testing.T) {
// ADR 0505 §1: the line names at most six modules and counts the rest, and says no name the content rule refuses.
func TestTheDeliveryLineNamesAtMostSixModulesAndNothingTheRuleRefuses(t *testing.T) {
many := novox("a1", "a2", "a3", "a4", "a5", "a6", "a7", "a8")
if got := deliveryWords([]Touch{many}); got != "a1, a2, a3, a4, a5, a6 and 2 more to novox" {
many := nodeC("a1", "a2", "a3", "a4", "a5", "a6", "a7", "a8")
if got := deliveryWords([]Touch{many}); got != "a1, a2, a3, a4, a5, a6 and 2 more to node-c" {
t.Fatalf("eight modules: %q", got)
}
h, _, _, c, _ := newHolder(t)
walk(t, h, "plan-p", WalkStarted, "", c.now(), 600, novox("/var/lib/mesh-tools"))
k := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent")
walk(t, h, "plan-p", WalkStarted, "", c.now(), 600, nodeC("/var/lib/mesh-tools"))
k := cond("machine.node-c.tools-silent", Urgent, "the tools on node-c are silent")
k.Raised = c.now()
h.Condition(EventRaised, k)
if b := bodyOf(t, h, k.Key, "raised"); !strings.HasSuffix(b, "\nDuring a delivery.") {
@@ -323,18 +323,18 @@ func TestTheDeliveryLineNamesAtMostSixModulesAndNothingTheRuleRefuses(t *testing
func TestADeclaredMachinesConditionAcrossThreeWalksIsSaidAtTheFirstWalksLimit(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
t0 := c.now()
walk(t, h, "plan-1", WalkStarted, "", c.now(), 600, novox("gitea"), shanksDeclared) // limit: t0 + 10 + 2 min
k := cond("machine.shanks.uncomposable", Urgent, "shanks cannot get new instructions")
k.Machine, k.Raised = "shanks", c.now()
walk(t, h, "plan-1", WalkStarted, "", c.now(), 600, nodeC("gitea"), nodeDDeclared) // limit: t0 + 10 + 2 min
k := cond("machine.node-d.uncomposable", Urgent, "node-d cannot get new instructions")
k.Machine, k.Raised = "node-d", c.now()
h.Condition(EventRaised, k)
runFor(h, c, 5*time.Minute)
walk(t, h, "plan-1", WalkEnded, WalkDelivered, c.now(), 0, novox("gitea"), shanksDeclared)
walk(t, h, "plan-1", WalkEnded, WalkDelivered, c.now(), 0, nodeC("gitea"), nodeDDeclared)
runFor(h, c, time.Minute)
walk(t, h, "plan-2", WalkStarted, "", c.now(), 600, novox("mesh-issues"), shanksDeclared)
walk(t, h, "plan-2", WalkStarted, "", c.now(), 600, nodeC("mesh-issues"), nodeDDeclared)
runFor(h, c, 3*time.Minute)
walk(t, h, "plan-2", WalkEnded, WalkDelivered, c.now(), 0, novox("mesh-issues"), shanksDeclared)
walk(t, h, "plan-2", WalkEnded, WalkDelivered, c.now(), 0, nodeC("mesh-issues"), nodeDDeclared)
runFor(h, c, time.Minute)
walk(t, h, "plan-3", WalkStarted, "", c.now(), 1200, novox("gitea"), shanksDeclared)
walk(t, h, "plan-3", WalkStarted, "", c.now(), 1200, nodeC("gitea"), nodeDDeclared)
for c.now().Before(t0.Add(12*time.Minute - 10*time.Second)) {
runFor(h, c, 5*time.Second)
}
@@ -351,14 +351,14 @@ func TestADeclaredMachinesConditionAcrossThreeWalksIsSaidAtTheFirstWalksLimit(t
// once: one walk with both sayings' touches, and one end message.
func TestAWalkSaidAgainWithItsDeclaredMachinesActsOnce(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
walk(t, h, "plan-r", WalkStarted, "", c.now(), 600, novox("gitea"))
walk(t, h, "plan-r", WalkStarted, "", c.now(), 600, novox("gitea"), shanksDeclared)
walk(t, h, "plan-r", WalkStarted, "", c.now(), 600, nodeC("gitea"))
walk(t, h, "plan-r", WalkStarted, "", c.now(), 600, nodeC("gitea"), nodeDDeclared)
if len(h.walks) != 1 || len(h.walks["plan-r"].Touches) != 2 {
t.Fatalf("said again it is not one walk with both touches: %+v", h.walks)
}
runFor(h, c, time.Minute)
walk(t, h, "plan-r", WalkEnded, WalkFailed, c.now(), 0, novox("gitea"))
walk(t, h, "plan-r", WalkEnded, WalkFailed, c.now(), 0, novox("gitea"), shanksDeclared)
walk(t, h, "plan-r", WalkEnded, WalkFailed, c.now(), 0, nodeC("gitea"))
walk(t, h, "plan-r", WalkEnded, WalkFailed, c.now(), 0, nodeC("gitea"), nodeDDeclared)
runFor(h, c, 3*time.Minute)
n := 0
for _, m := range append(append([]Message(nil), dt.sends...), tg.sends...) {
@@ -376,9 +376,9 @@ func TestAWalkSaidAgainWithItsDeclaredMachinesActsOnce(t *testing.T) {
func TestAMessageTwoWalksFirstHeldIsSaidOnceAtTheSoonerLimit(t *testing.T) {
for _, sev := range []string{Warning, Urgent} {
h, tg, dt, c, _ := newHolder(t)
walk(t, h, "plan-a", WalkStarted, "", c.now(), 360, novox("gitea")) // its limit: 6 min and the settle
walk(t, h, "plan-b", WalkStarted, "", c.now(), 1200, novox("mesh-issues")) // its limit: 22 min
k := cond("machine.novox.network", sev, "novox lost its default route")
walk(t, h, "plan-a", WalkStarted, "", c.now(), 360, nodeC("gitea")) // its limit: 6 min and the settle
walk(t, h, "plan-b", WalkStarted, "", c.now(), 1200, nodeC("mesh-issues")) // its limit: 22 min
k := cond("machine.node-c.network", sev, "node-c lost its default route")
k.Raised = c.now()
h.Condition(EventRaised, k)
runFor(h, c, 7*time.Minute)
@@ -402,7 +402,7 @@ func TestAMessageTwoWalksFirstHeldIsSaidOnceAtTheSoonerLimit(t *testing.T) {
t.Fatalf("%s: said %d times on %s, want %d: %+v", sev, got, ch.name, n, ch.sends)
}
}
if !heard("Still open after the delivery of gitea to novox:", dt) {
if !heard("Still open after the delivery of gitea to node-c:", dt) {
t.Fatalf("%s: not told in the sooner walk's end message: %+v", sev, dt.sends)
}
}
@@ -426,7 +426,7 @@ func TestEveryAnswerThatIsNotTheOperatorsIsRefusedAndSaid(t *testing.T) {
"a sender the service did not say": {"telegram", "42", false, 2 * time.Hour, "Release", 0, "did not authenticate"},
"a link younger than an hour": {"telegram", "42", true, 59 * time.Minute, "Release", 0, "can answer this question from"},
"an ask that expired": {"telegram", "42", true, 2 * time.Hour, "Release", 3 * time.Hour, "expired"},
"a ticket shown on another kind": {"desktop", "g14", true, 2 * time.Hour, "Release", 0, "not offered here"},
"a ticket shown on another kind": {"desktop", "node-b", true, 2 * time.Hour, "Release", 0, "not offered here"},
} {
t.Run(name, func(t *testing.T) {
h, tg, _, c, store, w := newRouter(t)
@@ -532,13 +532,13 @@ func TestTheDeskAcknowledgesAndNeverApproves(t *testing.T) {
submit(t, h, "mesh-delivery", ackAsk(c, "d-1"))
settle(h, c)
silence := ticketFor(dt.sends[0], "Silence for a week")
ch := choiceOf(silence, "g14", c.now())
ch := choiceOf(silence, "node-b", c.now())
ch.Sender.Authenticated = false
if out, err := h.Chosen("desktop", ch); err != nil || out != "chosen" {
t.Fatalf("%s %v", out, err)
}
got := w.of("mesh-delivery")
if len(got) != 1 || got[0].By.Verified != "a desk click: whoever was at the operator's session on g14" || len(got[0].Proofs) != 0 {
if len(got) != 1 || got[0].By.Verified != "a desk click: whoever was at the operator's session on node-b" || len(got[0].Proofs) != 0 {
t.Errorf("%+v", got)
}
// An approve ticket forged onto the desk is refused: the kind cannot carry it.
@@ -553,7 +553,7 @@ func TestTheDeskAcknowledgesAndNeverApproves(t *testing.T) {
rec.Tickets[release] = tk
store2.asks["mesh-delivery.d-1"] = rec
h2.keepAsk("mesh-delivery.d-1", rec, store2.revs["mesh-delivery.d-1"])
if out, err := h2.Chosen("desktop", choiceOf(release, "g14", c2.now())); err == nil || out != "refused" ||
if out, err := h2.Chosen("desktop", choiceOf(release, "node-b", c2.now())); err == nil || out != "refused" ||
!strings.Contains(err.Error(), "cannot answer a question that asks for an approval") {
t.Errorf("the desk approved: %s %v", out, err)
}
@@ -564,7 +564,7 @@ func TestTheDeskAcknowledgesAndNeverApproves(t *testing.T) {
rec.Tickets[silence] = tk
store2.asks["mesh-delivery.d-1"] = rec
h2.keepAsk("mesh-delivery.d-1", rec, store2.revs["mesh-delivery.d-1"])
if out, _ := h2.Chosen("desktop", choiceOf(silence, "g14", c2.now())); out != "refused" ||
if out, _ := h2.Chosen("desktop", choiceOf(silence, "node-b", c2.now())); out != "refused" ||
store2.asks["mesh-delivery.d-1"].State != StateOpen {
t.Errorf("a desk click ended an ask that waits for an approval: %s", out)
}
@@ -681,7 +681,7 @@ func TestAnAskAboutAConditionIsAnsweredOnItsMessage(t *testing.T) {
t.Errorf("the ask is shown on %q", store.asks["mesh-controller.d-1"].Record)
}
silence := ticketFor(shown, "Silence for a week")
if out, err := h.Chosen("desktop", choiceOf(silence, "g14", c.now())); err != nil || out != "chosen" {
if out, err := h.Chosen("desktop", choiceOf(silence, "node-b", c.now())); err != nil || out != "chosen" {
t.Fatalf("%s %v", out, err)
}
if got := w.of("mesh-controller"); len(got) != 1 || got[0].About != "delivery.hq.held" {
@@ -781,7 +781,7 @@ func TestALinkCodeExpiresAndFiveWrongTriesEndIt(t *testing.T) {
t.Errorf("%+v", a)
}
// The desk cannot be linked: it proves nothing about who clicked.
h.Link("desktop", asks.LinkAsked{ID: "l3", At: c.now(), Sender: asks.Sender{Identity: "g14"}})
h.Link("desktop", asks.LinkAsked{ID: "l3", At: c.now(), Sender: asks.Sender{Identity: "node-b"}})
if _, pending := store.links["desktop"]; pending {
t.Error("the desk began a link")
}
@@ -1001,7 +1001,7 @@ func TestAWarningTheDeskCouldNotShowGoesAway(t *testing.T) {
if len(dt.sends) != 1 {
t.Fatalf("the desk was sent %d", len(dt.sends))
}
h.Failed("desktop", asks.Failed{Handle: "g14=1", Why: "no session answers", At: c.now()})
h.Failed("desktop", asks.Failed{Handle: "node-b=1", Why: "no session answers", At: c.now()})
c.pass(BurstWindow)
h.Flush()
if len(tg.sends) == 0 {
@@ -27,8 +27,8 @@ func TestTheEventsAreTheControllersSeat(t *testing.T) {
func TestAConditionAsTheStoreHoldsIt(t *testing.T) {
body := `{
"key": "machine.ace.silent", "kind": "silent",
"subject": {"scope": "machine", "id": "ace", "machine": "ace"},
"key": "machine.node-a.silent", "kind": "silent",
"subject": {"scope": "machine", "id": "node-a", "machine": "node-a"},
"severity": "urgent", "summary": "the home server has not been heard for 15 min",
"evidence": [{"at": "2026-10-06T12:00:00Z", "what": "last heartbeat"}],
"source": "S1", "raised": "2026-10-06T12:15:00Z", "last-observed": "2026-10-06T12:16:00Z",
@@ -40,7 +40,7 @@ func TestAConditionAsTheStoreHoldsIt(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if c.Key != "machine.ace.silent" || c.Kind != "silent" || c.Scope != "machine" || c.ID != "ace" ||
if c.Key != "machine.node-a.silent" || c.Kind != "silent" || c.Scope != "machine" || c.ID != "node-a" ||
c.Severity != Urgent || c.Source != "S1" || c.Observations != 3 || c.Resolver != "self" {
t.Fatalf("%+v", c)
}
@@ -50,7 +50,7 @@ func TestAConditionAsTheStoreHoldsIt(t *testing.T) {
if !c.SilencedAt(time.Date(2026, 10, 6, 13, 0, 0, 0, time.UTC)) || c.SilencedAt(time.Date(2026, 10, 6, 15, 0, 0, 0, time.UTC)) {
t.Fatalf("silenced: %v", c.SilencedTill)
}
if c.SubjectWords() != "machine ace" {
if c.SubjectWords() != "machine node-a" {
t.Fatalf("subject words: %q", c.SubjectWords())
}
}
@@ -65,7 +65,7 @@ func TestOtherSpellingsAndAKeyMadeFromItsParts(t *testing.T) {
if c.Key != "plan.41.stalled" || c.Source != "S3" || c.LastObserved.IsZero() || !c.SilencedTill.IsZero() {
t.Fatalf("%+v", c)
}
c, err = DecodeCondition(EventRaised, []byte(`{"key":"provider.keycloak.ace.gitea.failing","subject":"keycloak",
c, err = DecodeCondition(EventRaised, []byte(`{"key":"provider.keycloak.node-a.gitea.failing","subject":"keycloak",
"severity":"warning","summary":"the identity provider fails gitea"}`))
if err != nil || c.Scope != "provider" || c.Kind != "failing" || c.ID != "keycloak" {
t.Fatalf("%+v %v", c, err)
@@ -73,8 +73,8 @@ func TestOtherSpellingsAndAKeyMadeFromItsParts(t *testing.T) {
}
func TestAClearedEventNeedsOnlyItsKey(t *testing.T) {
c, err := DecodeCondition(EventCleared, []byte(`{"key":"machine.ace.silent","cleared":"2026-10-06T12:30:00Z"}`))
if err != nil || c.Key != "machine.ace.silent" || c.Cleared.IsZero() {
c, err := DecodeCondition(EventCleared, []byte(`{"key":"machine.node-a.silent","cleared":"2026-10-06T12:30:00Z"}`))
if err != nil || c.Key != "machine.node-a.silent" || c.Cleared.IsZero() {
t.Fatalf("%+v %v", c, err)
}
}
@@ -5,14 +5,14 @@ import "testing"
func TestRolesAndWordsMayLeave(t *testing.T) {
for _, s := range []string{
"URGENT: the home server has not been heard for 15 min",
"key: provider.keycloak.ace.gitea.failing",
"more: conditions show machine.ace.silent",
"key: provider.keycloak.node-a.gitea.failing",
"more: conditions show machine.node-a.silent",
"since: 2026-10-06 12:30 UTC",
"call call-1759752000123456789-12 has run past its bound of 10 min",
"ace/postgres.query answers no more",
"node-a/postgres.query answers no more",
"3/4 machines answered; plan 41 waits on build 7f3a9c1e",
"the bus advisory slow-consumer for ace_records",
"core.controller.novox.rolled-back",
"the bus advisory slow-consumer for node-a_records",
"core.controller.node-c.rolled-back",
"seat node-notifier has no live holder",
"HELD BACK: 5 message(s) over the cap of 20 an hour",
"STILL OPEN after 1.5 h: the controller's event loop takes no message",
@@ -32,7 +32,7 @@ func TestAnAddressAPathOrASecretMayNot(t *testing.T) {
"fe80::1 answered": "address",
"2001:db8:0:0:0:0:2:1 answered": "address",
"see https://git.example.org/x": "address",
"git.novox.internal does not answer": "address",
"git.node-c.internal does not answer": "address",
"the mail for admin@example.org bounced": "address",
"zurag.be is down": "address",
"localhost refused": "address",
@@ -38,7 +38,7 @@ func TestDetailsWorksOnEachOfThreeWarningsOnceEach(t *testing.T) {
var toks []string
if same {
// One problem said three times: raised, escalated, and reminded.
k := cond("machine.novox.silent", Warning, "the anchor is silent")
k := cond("machine.node-c.silent", Warning, "the anchor is silent")
k.Raised = c.now()
h.Condition(EventRaised, k)
settle(h, c)
@@ -49,7 +49,7 @@ func TestDetailsWorksOnEachOfThreeWarningsOnceEach(t *testing.T) {
h.Tick()
settle(h, c)
} else {
for _, key := range []string{"machine.ace.silent", "machine.shanks.units", "plan.41.stalled"} {
for _, key := range []string{"machine.node-a.silent", "machine.node-d.units", "plan.41.stalled"} {
k := cond(key, Warning, key+" needs a look")
k.Raised = c.now()
h.Condition(EventRaised, k)
@@ -97,7 +97,7 @@ func TestDetailsWorksOnEachOfThreeWarningsOnceEach(t *testing.T) {
func TestDetailsTicketsAreBoundedInNumberAndTime(t *testing.T) {
h, tg, _, c, _ := newHolder(t)
h.Desk = nil
k := cond("machine.ace.silent", Warning, "the home server is silent")
k := cond("machine.node-a.silent", Warning, "the home server is silent")
k.Raised = c.now()
h.Condition(EventRaised, k)
settle(h, c)
@@ -134,7 +134,7 @@ func TestDetailsTicketsAreBoundedInNumberAndTime(t *testing.T) {
func TestDetailsOutlivesAChangeOfTheActions(t *testing.T) {
h, tg, _, c, _ := newHolder(t)
h.Desk = nil
k := cond("machine.ace.silent", Warning, "the home server is silent")
k := cond("machine.node-a.silent", Warning, "the home server is silent")
k.Raised = c.now()
h.Condition(EventRaised, k)
settle(h, c)
@@ -165,7 +165,7 @@ func TestDetailsOnAMessageWithAnOpenAskNeverAnswersIt(t *testing.T) {
shown = m
}
for _, tok := range []string{older, detailsOf(t, shown)} {
if out, err := h.Chosen("desktop", choiceOf(tok, "g14", c.now())); err != nil || out != "shown" {
if out, err := h.Chosen("desktop", choiceOf(tok, "node-b", c.now())); err != nil || out != "shown" {
t.Fatalf("Details: %q %v", out, err)
}
}
@@ -182,7 +182,7 @@ func TestDetailsOnAMessageWithAnOpenAskNeverAnswersIt(t *testing.T) {
func TestAnUnreadyChannelPushesOutNoShownDetails(t *testing.T) {
h, tg, _, c, _ := newHolder(t)
h.Desk = nil
k := cond("machine.ace.silent", Warning, "the home server is silent")
k := cond("machine.node-a.silent", Warning, "the home server is silent")
k.Raised = c.now()
h.Condition(EventRaised, k)
settle(h, c)
@@ -23,7 +23,7 @@ func decisionAsk(c *clock) asks.Ask {
"• A deleted file is marked, and a newer change elsewhere outlives the deletion",
Who: asks.Operator, Expires: c.now().Add(23*time.Hour + 50*time.Minute), OnExpiry: "the record stays proposed; nothing of it is in force",
About: "knowledge.decision.343",
Details: "ADR 0343, proposed by fix-367@g14; the knowledge base's own review approved it. Fingerprint a59f 361b f60e cb5e d237 a837.\n" +
Details: "ADR 0343, proposed by fix-367@node-b; the knowledge base's own review approved it. Fingerprint a59f 361b f60e cb5e d237 a837.\n" +
"Read its 13 clauses with the knowledge base's get, number 343.",
Options: []asks.Option{
{ID: "approve", Label: "Approve", Does: "the record is accepted and its clauses are in force", Level: asks.Approve, Binds: bound("approve")},
@@ -50,7 +50,7 @@ func TestAnAsksMessageIsItsSummaryWithTheRestInItsNote(t *testing.T) {
"Asked by mesh-knowledge at " + asked + ". Unanswered by tomorrow " + a.Expires.In(time.UTC).Format("15:04") +
": the record stays proposed; nothing of it is in force.",
"Approve: the record is accepted and its clauses are in force. Decline: the record stays proposed; nothing of it is in force.",
"ADR 0343, proposed by fix-367@g14; the knowledge base's own review approved it. Fingerprint a59f 361b f60e cb5e d237 a837.",
"ADR 0343, proposed by fix-367@node-b; the knowledge base's own review approved it. Fingerprint a59f 361b f60e cb5e d237 a837.",
"Read its 13 clauses with the knowledge base's get, number 343.",
}
if strings.Join(note, "\n") != strings.Join(want, "\n") {
+15 -15
View File
@@ -184,7 +184,7 @@ func TestTodaysBacklogWithAnUrgentStillOpenIsOneMessage(t *testing.T) {
h, tg, dt, c := incidentHolder(t)
// Suppose the merge never got acted on, and three machines stayed behind: still open at 09:49.
stillOpen := map[string]bool{"merge.mesh-catalog.f2b51686.not-acted": true,
"machine.ace.core-behind": true, "machine.g14.core-behind": true, "machine.shanks.core-behind": true}
"machine.node-a.core-behind": true, "machine.node-b.core-behind": true, "machine.node-d.core-behind": true}
var kept []backlogEvent
for _, e := range backlog {
c, _ := DecodeCondition(e.event, e.body)
@@ -216,7 +216,7 @@ func TestTodaysBacklogWithAnUrgentStillOpenIsOneMessage(t *testing.T) {
func TestAnEventOlderThanTheReadingIsNotActedOn(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
h.Sync(nil, c.now())
k := cond("machine.ace.silent", Urgent, "the home server is silent")
k := cond("machine.node-a.silent", Urgent, "the home server is silent")
k.At = c.now().Add(-time.Minute)
k.Raised = k.At
h.Condition(EventRaised, k) // already cleared in the state read after it
@@ -228,7 +228,7 @@ func TestAnEventOlderThanTheReadingIsNotActedOn(t *testing.T) {
func TestAnOldEventIsStateNeverAMessage(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
k := cond("machine.ace.silent", Urgent, "the home server is silent")
k := cond("machine.node-a.silent", Urgent, "the home server is silent")
k.At = c.now().Add(-FreshFor - time.Second)
h.Condition(EventRaised, k)
runFor(h, c, 2*time.Minute)
@@ -247,10 +247,10 @@ func TestAnOldEventIsStateNeverAMessage(t *testing.T) {
t.Fatalf("a clearing of what was never said was said")
}
// The state, read now, says an urgent one is open that was never said: that is said, once.
h.Condition(EventRaised, Condition{Key: "machine.shanks.silent", Scope: "machine", ID: "shanks", Kind: "silent",
h.Condition(EventRaised, Condition{Key: "machine.node-d.silent", Scope: "machine", ID: "node-d", Kind: "silent",
Severity: Urgent, Summary: "the desktop is silent", At: c.now().Add(-time.Hour)})
h.Sync([]Condition{cond("machine.shanks.silent", Urgent, "the desktop is silent")}, c.now())
h.Sync([]Condition{cond("machine.shanks.silent", Urgent, "the desktop is silent")}, c.now().Add(time.Second))
h.Sync([]Condition{cond("machine.node-d.silent", Urgent, "the desktop is silent")}, c.now())
h.Sync([]Condition{cond("machine.node-d.silent", Urgent, "the desktop is silent")}, c.now().Add(time.Second))
if len(tg.sends) != 1 || len(dt.sends) != 1 {
t.Fatalf("telegram %d desktop %d", len(tg.sends), len(dt.sends))
}
@@ -320,7 +320,7 @@ func TestTheDesktopIsGentle(t *testing.T) {
runFor(h, c, 5*time.Minute)
h.Condition(EventRaised, cond("plan.4.stalled", Warning, "plan 4 waits"))
// Urgent is not held to the desktop's rhythm.
h.Condition(EventRaised, cond("machine.ace.silent", Urgent, "the home server is silent"))
h.Condition(EventRaised, cond("machine.node-a.silent", Urgent, "the home server is silent"))
h.Flush()
if len(dt.sends) != 2 || !strings.HasPrefix(dt.sends[1].Title, "Urgent: ") || len(tg.sends) != 1 {
t.Fatalf("urgent: desktop %d telegram %d", len(dt.sends), len(tg.sends))
@@ -346,7 +346,7 @@ func TestTheDesktopIsGentle(t *testing.T) {
func TestNothingIsSentInTheFirstMinuteButTheUrgentOpenAsOne(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
h.Start()
u1 := cond("machine.ace.silent", Urgent, "the home server is silent")
u1 := cond("machine.node-a.silent", Urgent, "the home server is silent")
u1.Raised = c.now().Add(-3 * time.Hour)
u2 := cond("bus.controller.refused", Urgent, "the bus refuses the controller")
u2.Raised = c.now().Add(-20 * time.Minute)
@@ -365,7 +365,7 @@ func TestNothingIsSentInTheFirstMinuteButTheUrgentOpenAsOne(t *testing.T) {
k := cond("plan.43.stalled", Warning, "plan 43 waits")
k.At = c.now()
h.Condition(EventRaised, k)
u3 := cond("machine.shanks.silent", Urgent, "the desktop is silent")
u3 := cond("machine.node-d.silent", Urgent, "the desktop is silent")
u3.At = c.now()
h.Condition(EventRaised, u3)
runFor(h, c, 45*time.Second)
@@ -378,14 +378,14 @@ func TestNothingIsSentInTheFirstMinuteButTheUrgentOpenAsOne(t *testing.T) {
t.Fatalf("after the grace: telegram %d desktop %d", len(tg.sends), len(dt.sends))
}
if d := dt.sends[1]; !strings.Contains(d.Body, "plan 42 stalled") || !strings.Contains(d.Body, "plan 43 stalled") ||
!strings.Contains(d.Body, "machine shanks silent") || strings.Contains(d.Body, "plan 41 stalled") {
!strings.Contains(d.Body, "machine node-d silent") || strings.Contains(d.Body, "plan 41 stalled") {
t.Fatalf("desktop digest: %q", d.Text())
}
}
func TestWhatEndedWhileAwayIsEndedQuietly(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
k := cond("machine.ace.silent", Urgent, "the home server is silent")
k := cond("machine.node-a.silent", Urgent, "the home server is silent")
k.At = c.now()
h.Condition(EventRaised, k)
if len(tg.sends) != 1 || len(dt.sends) != 1 {
@@ -406,7 +406,7 @@ func TestWhatEndedWhileAwayIsEndedQuietly(t *testing.T) {
func TestARestartSaysNothingAgain(t *testing.T) {
h, tg, dt, c, st := newHolder(t)
k := cond("machine.ace.silent", Urgent, "the home server is silent")
k := cond("machine.node-a.silent", Urgent, "the home server is silent")
k.At = c.now()
h.Condition(EventRaised, k)
w := cond("plan.41.stalled", Warning, "plan 41 waits")
@@ -426,7 +426,7 @@ func TestARestartSaysNothingAgain(t *testing.T) {
}
func TestTheOpenSetIsReadInEveryShape(t *testing.T) {
one := `{"key":"machine.ace.silent","kind":"silent","subject":{"scope":"machine","id":"ace"},"severity":"urgent","summary":"s","raised":"2026-10-06T09:00:00Z"}`
one := `{"key":"machine.node-a.silent","kind":"silent","subject":{"scope":"machine","id":"node-a"},"severity":"urgent","summary":"s","raised":"2026-10-06T09:00:00Z"}`
for _, raw := range []string{
`{"conditions":[` + one + `],"open":1}`,
`{"answer":{"conditions":[` + one + `]},"ok":true}`,
@@ -434,7 +434,7 @@ func TestTheOpenSetIsReadInEveryShape(t *testing.T) {
`{"content":[{"type":"text","text":"{\"conditions\":[` + strings.ReplaceAll(one, `"`, `\"`) + `]}"}]}`,
} {
got, err := DecodeOpen(json.RawMessage(raw))
if err != nil || len(got) != 1 || got[0].Key != "machine.ace.silent" || got[0].Severity != Urgent {
if err != nil || len(got) != 1 || got[0].Key != "machine.node-a.silent" || got[0].Severity != Urgent {
t.Fatalf("%s: %+v %v", raw, got, err)
}
}
@@ -449,7 +449,7 @@ func TestTheOpenSetIsReadInEveryShape(t *testing.T) {
}
func TestAnEventSaysItsOwnTime(t *testing.T) {
c, err := DecodeCondition(EventCleared, []byte(`{"key":"machine.ace.silent","at":"2026-10-06T08:36:50Z","raised":"2026-10-06T08:32:30Z"}`))
c, err := DecodeCondition(EventCleared, []byte(`{"key":"machine.node-a.silent","at":"2026-10-06T08:36:50Z","raised":"2026-10-06T08:32:30Z"}`))
if err != nil || !c.When(EventCleared).Equal(time.Date(2026, 10, 6, 8, 36, 50, 0, time.UTC)) {
t.Fatalf("%v %v", c.When(EventCleared), err)
}
+18 -18
View File
@@ -48,7 +48,7 @@ func (f *fakeChannel) Send(m Message) (string, error) {
f.sends = append(f.sends, m)
f.to = append(f.to, m.To)
if f.name == "desktop" {
return fmt.Sprintf("g14=%d", f.n), nil // as the desktop answers: the machine and the notification
return fmt.Sprintf("node-b=%d", f.n), nil // as the desktop answers: the machine and the notification
}
return fmt.Sprint(f.n), nil
}
@@ -157,7 +157,7 @@ func newHolder(t *testing.T) (*Holder, *fakeChannel, *fakeChannel, *clock, *memS
func TestARaisedConditionIsSentOnceByItsKey(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
k := cond("machine.ace.silent", Urgent, "the home server has not been heard for 15 min")
k := cond("machine.node-a.silent", Urgent, "the home server has not been heard for 15 min")
k.Raised = c.now()
h.Condition(EventRaised, k)
h.Condition(EventRaised, k) // a redelivery
@@ -167,7 +167,7 @@ func TestARaisedConditionIsSentOnceByItsKey(t *testing.T) {
t.Fatalf("urgent: telegram %d, desktop %d; want one each", len(tg.sends), len(dt.sends))
}
keyless(t, tg.sends...)
if tg.sends[0].Title != "Urgent: machine ace silent" || !strings.HasPrefix(tg.sends[0].Body, "Needs you: look at it. What it means: ") ||
if tg.sends[0].Title != "Urgent: machine node-a silent" || !strings.HasPrefix(tg.sends[0].Body, "Needs you: look at it. What it means: ") ||
!strings.Contains(tg.sends[0].Body, "\nSince ") {
t.Fatalf("message: %q", tg.sends[0].Text())
}
@@ -252,7 +252,7 @@ func TestAMessageCarryingAnAddressIsRefusedAndSaidWithItsWordsWithheld(t *testin
h, tg, _, c, _ := newHolder(t)
var emitted []string
h.Emit = func(e string, _ any) error { emitted = append(emitted, e); return nil }
h.Condition(EventRaised, cond("provider.keycloak.ace.failing", Urgent, "cannot reach 192.168.1.20:8443 with token=abc"))
h.Condition(EventRaised, cond("provider.keycloak.node-a.failing", Urgent, "cannot reach 192.168.1.20:8443 with token=abc"))
if len(tg.sends) != 1 {
t.Fatalf("sends: %d", len(tg.sends))
}
@@ -260,7 +260,7 @@ func TestAMessageCarryingAnAddressIsRefusedAndSaidWithItsWordsWithheld(t *testin
if strings.Contains(text, "192.168") || strings.Contains(text, "token=") {
t.Fatalf("the address left: %q", text)
}
if !strings.Contains(text, "withheld, because it carried an IPv4 address") || !strings.Contains(text, "provider keycloak ace failing") {
if !strings.Contains(text, "withheld, because it carried an IPv4 address") || !strings.Contains(text, "provider keycloak node-a failing") {
t.Fatalf("not said as refused, by key: %q", text)
}
hist := h.History(10)["refusals"].([]RefusalNote)
@@ -271,7 +271,7 @@ func TestAMessageCarryingAnAddressIsRefusedAndSaidWithItsWordsWithheld(t *testin
t.Fatalf("emitted: %v", emitted)
}
// Its clearing keeps the words withheld.
h.Condition(EventCleared, Condition{Key: "provider.keycloak.ace.failing"})
h.Condition(EventCleared, Condition{Key: "provider.keycloak.node-a.failing"})
c.pass(ClearSettle)
h.Tick()
for _, m := range tg.edits {
@@ -330,7 +330,7 @@ func TestStillOpenPastItsBoundItIsSaidOnceMore(t *testing.T) {
func TestClearedEditsTheFirstMessageAndReopenedWithinTenMinutesIsNotNew(t *testing.T) {
h, tg, dt, c, st := newHolder(t)
k := cond("machine.ace.silent", Urgent, "the home server is silent")
k := cond("machine.node-a.silent", Urgent, "the home server is silent")
k.Raised = c.now()
h.Condition(EventRaised, k)
c.pass(14 * time.Minute)
@@ -340,7 +340,7 @@ func TestClearedEditsTheFirstMessageAndReopenedWithinTenMinutesIsNotNew(t *testi
if len(tg.sends) != 1 || len(tg.edits) != 1 || len(dt.edits) != 1 {
t.Fatalf("telegram sends %d edits %d, desktop edits %d", len(tg.sends), len(tg.edits), len(dt.edits))
}
if m := tg.edits["1"]; m.Title != "Resolved: machine ace silent, after 14 min" || m.Body != "" {
if m := tg.edits["1"]; m.Title != "Resolved: machine node-a silent, after 14 min" || m.Body != "" {
t.Fatalf("edit: %q", m.Title)
}
if len(h.Open()) != 0 {
@@ -369,7 +369,7 @@ func TestClearedEditsTheFirstMessageAndReopenedWithinTenMinutesIsNotNew(t *testi
func TestASilencedConditionSendsNothing(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
k := cond("machine.ace.silent", Urgent, "silent")
k := cond("machine.node-a.silent", Urgent, "silent")
k.SilencedTill = c.now().Add(2 * time.Hour)
h.Condition(EventRaised, k)
c.pass(90 * time.Minute)
@@ -379,7 +379,7 @@ func TestASilencedConditionSendsNothing(t *testing.T) {
t.Fatalf("a silenced condition said something")
}
// Silenced after it was sent: no reminder.
k2 := cond("machine.shanks.silent", Urgent, "silent")
k2 := cond("machine.node-d.silent", Urgent, "silent")
k2.Raised = c.now()
h.Condition(EventRaised, k2)
k2.SilencedTill = c.now().Add(3 * time.Hour)
@@ -393,7 +393,7 @@ func TestASilencedConditionSendsNothing(t *testing.T) {
func TestEscalationIsSaidOnce(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
k := cond("machine.novox.silent", Warning, "the anchor is silent")
k := cond("machine.node-c.silent", Warning, "the anchor is silent")
h.Condition(EventRaised, k)
settle(h, c)
k.Severity = Urgent
@@ -401,12 +401,12 @@ func TestEscalationIsSaidOnce(t *testing.T) {
h.Condition(EventChanged, k)
settle(h, c)
// The desktop's warning is replaced by the urgent message (issue 373); Telegram, which had none, is sent one.
if len(dt.sends) != 1 || len(dt.replaced) != 1 || dt.replaced[0].id != "g14=1" ||
if len(dt.sends) != 1 || len(dt.replaced) != 1 || dt.replaced[0].id != "node-b=1" ||
!strings.HasPrefix(dt.replaced[0].m.Title, "Now urgent: ") || len(tg.sends) != 1 || !strings.HasPrefix(tg.sends[0].Title, "Now urgent: ") {
t.Fatalf("desktop %d sends %d replaced, telegram %d", len(dt.sends), len(dt.replaced), len(tg.sends))
}
// Escalated while the warning still waited its window: one message, the urgent one.
k2 := cond("machine.ace.silent", Warning, "the home server is silent")
k2 := cond("machine.node-a.silent", Warning, "the home server is silent")
h.Condition(EventRaised, k2)
k2.Severity = Urgent
h.Condition(EventChanged, k2)
@@ -434,7 +434,7 @@ func TestAChannelThatCannotSendSaysSoAndIsTriedAgain(t *testing.T) {
h, tg, _, c, _ := newHolder(t)
h.Desk = nil
tg.fail = errors.New("telegram sendMessage: cannot connect (dial)")
h.Condition(EventRaised, cond("machine.ace.silent", Urgent, "silent"))
h.Condition(EventRaised, cond("machine.node-a.silent", Urgent, "silent"))
st := h.Status("listening")
if !strings.HasPrefix(st.Verdict, "CANNOT SEND") || len(st.Unsent) != 1 || st.Channels[0].LastError == "" {
t.Fatalf("status: %+v", st)
@@ -464,7 +464,7 @@ func TestNothingConfiguredIsSaidInTheStatus(t *testing.T) {
func TestARestartForgetsNothing(t *testing.T) {
h, tg, _, c, st := newHolder(t)
h.Desk = nil
k := cond("machine.ace.silent", Urgent, "silent")
k := cond("machine.node-a.silent", Urgent, "silent")
k.Raised = c.now()
h.Condition(EventRaised, k)
again := &Holder{Away: tg, Store: st, Now: c.now, Logf: t.Logf}
@@ -507,12 +507,12 @@ func TestAnUnreadableEventIsToldOnceAnHour(t *testing.T) {
func TestNotifyIsKeptApartFromConditions(t *testing.T) {
h, tg, _, c, _ := newHolder(t)
h.Desk = nil
out, err := h.Notify("backup.ace.failed", Warning, "last night's backup of the home server failed", "backup", false)
out, err := h.Notify("backup.node-a.failed", Warning, "last night's backup of the home server failed", "backup", false)
if err != nil || out["held"] != true {
t.Fatalf("%v %v", out, err)
}
settle(h, c)
if err != nil || out["key"] != "notify.backup.ace.failed" || len(tg.sends) != 1 {
if err != nil || out["key"] != "notify.backup.node-a.failed" || len(tg.sends) != 1 {
t.Fatalf("%v %v %d", out, err, len(tg.sends))
}
if _, err := h.Notify("x", "", "s", "", false); err == nil {
@@ -529,7 +529,7 @@ func TestNotifyIsKeptApartFromConditions(t *testing.T) {
func TestATellReachesEveryChannelOnceAndOffersOnlyDetails(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
c.pass(14 * time.Hour) // well past midnight: it is not held for being "night" — there is no such hold
out, err := h.Tell("new mail arrived for ace", []string{"One message from the inbox you watch."}, "mail")
out, err := h.Tell("new mail arrived for node-a", []string{"One message from the inbox you watch."}, "mail")
if err != nil {
t.Fatal(err)
}
@@ -134,7 +134,7 @@ func TestTheToolsAgreeWithTheManifest(t *testing.T) {
if !reflect.DeepEqual(verbs, m.Claims[0].Serves) {
t.Fatalf("verbs %v, claim %v", verbs, m.Claims[0].Serves)
}
for _, never := range []string{"/home/", "jochen", "g14", "shanks", "novox", "zurag", "telegram"} {
for _, never := range []string{"/home/", "jochen", "node-b", "node-d", "node-c", "zurag", "telegram"} {
if strings.Contains(strings.ToLower(raw), never) {
t.Errorf("module.json names %q", never)
}
@@ -22,7 +22,7 @@ func saidOn(chs ...*fakeChannel) int {
// "again, 22nd time", no "Resolved" — and once it stays cleared, its clearing is said once.
func TestAFlappingConditionIsSaidOnceAndItsClearingOnceWhenItStops(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
k := cond("machine.shanks.units", Warning, "2 failed services on shanks")
k := cond("machine.node-d.units", Warning, "2 failed services on node-d")
k.Raised = c.now()
h.Condition(EventRaised, k)
settle(h, c)
@@ -58,7 +58,7 @@ func TestAFlappingConditionIsSaidOnceAndItsClearingOnceWhenItStops(t *testing.T)
t.Fatalf("open: %+v", open)
}
// A genuinely new problem meanwhile is said.
n := cond("machine.ace.disk-full", Urgent, "the home server's disk is full")
n := cond("machine.node-a.disk-full", Urgent, "the home server's disk is full")
n.Raised = c.now()
h.Condition(EventRaised, n)
if len(tg.sends) != 1 || len(dt.sends) != 2 {
@@ -67,7 +67,7 @@ func TestAFlappingConditionIsSaidOnceAndItsClearingOnceWhenItStops(t *testing.T)
// The unit stays mended: its clearing is said once, in place of its message.
h.Condition(EventCleared, Condition{Key: k.Key, At: c.now()})
runFor(h, c, 30*time.Minute)
if len(dt.edits) != 1 || !strings.HasPrefix(dt.edits["g14=1"].Title, "Resolved: ") || len(dt.sends) != 2 {
if len(dt.edits) != 1 || !strings.HasPrefix(dt.edits["node-b=1"].Title, "Resolved: ") || len(dt.sends) != 2 {
t.Fatalf("its clearing: edits %+v, sends %d", dt.edits, len(dt.sends))
}
}
@@ -75,7 +75,7 @@ func TestAFlappingConditionIsSaidOnceAndItsClearingOnceWhenItStops(t *testing.T)
// A clearing a few seconds long is never said, and the message continues; an urgent problem rang once.
func TestAnUrgentBlipRingsOnceAndItsBlipsSayNothing(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
k := cond("machine.ace.silent", Urgent, "the home server is silent")
k := cond("machine.node-a.silent", Urgent, "the home server is silent")
k.Raised = c.now()
h.Condition(EventRaised, k)
for i := 0; i < 5; i++ {
@@ -97,7 +97,7 @@ func TestAnUrgentBlipRingsOnceAndItsBlipsSayNothing(t *testing.T) {
// said, not swallowed as the same message.
func TestAConditionClearedBeforeItWasSaidAndBackIsSaid(t *testing.T) {
h, _, dt, c, _ := newHolder(t)
k := cond("machine.shanks.units", Warning, "2 failed services on shanks")
k := cond("machine.node-d.units", Warning, "2 failed services on node-d")
k.Raised = c.now()
h.Condition(EventRaised, k)
c.pass(10 * time.Second)
@@ -106,7 +106,7 @@ func TestAConditionClearedBeforeItWasSaidAndBackIsSaid(t *testing.T) {
k.At = c.now()
h.Condition(EventRaised, k)
runFor(h, c, 2*time.Minute)
if len(dt.sends) != 1 || !strings.Contains(dt.sends[0].Title, "machine shanks units") {
if len(dt.sends) != 1 || !strings.Contains(dt.sends[0].Title, "machine node-d units") {
t.Fatalf("desktop %+v", dt.sends)
}
}
@@ -169,7 +169,7 @@ func TestAnAsksExpiryIsSaidWithItsDay(t *testing.T) {
func TestAnEndedQuestionLeavesNoMessageOfferingItsAnswers(t *testing.T) {
h, tg, dt, c, store, _ := newRouter(t)
link(h, store, c, "42", c.now().Add(-2*time.Hour))
k := cond("machine.novox.secret-given", Urgent, "a secret was given at the terminal")
k := cond("machine.node-c.secret-given", Urgent, "a secret was given at the terminal")
k.Raised = c.now()
h.Condition(EventRaised, k)
settle(h, c)
@@ -191,7 +191,7 @@ func TestAnEndedQuestionLeavesNoMessageOfferingItsAnswers(t *testing.T) {
if len(ch.sends) != 1 {
t.Fatalf("%s: %d messages for one problem", ch.name, len(ch.sends))
}
last := ch.edits[map[string]string{"telegram": "1", "desktop": "g14=1"}[ch.name]]
last := ch.edits[map[string]string{"telegram": "1", "desktop": "node-b=1"}[ch.name]]
if len(ch.replaced) == 0 || len(last.Actions) > 1 || !strings.Contains(last.Body, "No longer answerable") {
t.Fatalf("%s: replaced %d, the message now: %q %v", ch.name, len(ch.replaced), last.Text(), labelsOf(last))
}
@@ -202,7 +202,7 @@ func TestAnEndedQuestionLeavesNoMessageOfferingItsAnswers(t *testing.T) {
// warning back as urgent rings, and a reminder waiting to go out when it cleared still goes out.
func TestABlipSwallowsNeitherAnEscalationNorAWaitingReminder(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
k := cond("machine.shanks.units", Warning, "2 failed services on shanks")
k := cond("machine.node-d.units", Warning, "2 failed services on node-d")
k.Raised = c.now()
h.Condition(EventRaised, k)
settle(h, c)
@@ -274,7 +274,7 @@ func TestAReminderInABurstReplacesItsMessageAndIsNoDigestLine(t *testing.T) {
func TestAClearingTheChannelCouldNotTakeIsSaidLater(t *testing.T) {
h, tg, _, c, _ := newHolder(t)
h.Desk = nil
k := cond("machine.ace.silent", Urgent, "the home server is silent")
k := cond("machine.node-a.silent", Urgent, "the home server is silent")
k.Raised = c.now()
h.Condition(EventRaised, k)
tg.fail = errorf("telegram is away")
@@ -297,7 +297,7 @@ func TestAClearingTheChannelCouldNotTakeIsSaidLater(t *testing.T) {
func TestASilenceIsSaidOnTheShownMessageAndTheMessageIsOpenAgainWhenItEnds(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
h.Zone = func() *time.Location { return time.UTC }
k := cond("machine.shanks.units", Warning, "2 failed services on shanks")
k := cond("machine.node-d.units", Warning, "2 failed services on node-d")
k.Raised = c.now()
h.Condition(EventRaised, k)
settle(h, c)
@@ -307,8 +307,8 @@ func TestASilenceIsSaidOnTheShownMessageAndTheMessageIsOpenAgainWhenItEnds(t *te
c.pass(time.Hour)
k.SilencedTill, k.SilencedWhy = c.now().Add(7*24*time.Hour), "the mounts are gone for good"
h.Condition(EventChanged, k)
m, edited := dt.edits["g14=1"]
if !edited || m.Title != "Silenced until Tue 13 Oct 13:00: machine shanks units" || !m.Quiet || m.Urgent ||
m, edited := dt.edits["node-b=1"]
if !edited || m.Title != "Silenced until Tue 13 Oct 13:00: machine node-d units" || !m.Quiet || m.Urgent ||
len(m.Actions) != 0 || !strings.Contains(m.Body, "Because: the mounts are gone for good.") {
t.Fatalf("silenced, and the desk's message says %q %q quiet=%v urgent=%v actions=%v", m.Title, m.Body, m.Quiet, m.Urgent, m.Actions)
}
@@ -324,7 +324,7 @@ func TestASilenceIsSaidOnTheShownMessageAndTheMessageIsOpenAgainWhenItEnds(t *te
// The silence runs out and the condition is still open: its message is open again, on the same notification.
c.pass(7 * 24 * time.Hour)
h.Tick()
if m := dt.edits["g14=1"]; m.Title != "machine shanks units" || m.Quiet || len(m.Actions) == 0 {
if m := dt.edits["node-b=1"]; m.Title != "machine node-d units" || m.Quiet || len(m.Actions) == 0 {
t.Fatalf("the silence ended, and the message says %q quiet=%v actions=%v", m.Title, m.Quiet, m.Actions)
}
if len(dt.sends) != 1 {
@@ -336,18 +336,18 @@ func TestASilenceIsSaidOnTheShownMessageAndTheMessageIsOpenAgainWhenItEnds(t *te
// desk's popup, which timed out, is not brought back.
func TestAConditionResolvedWhileSilencedSaysSoOnlyWhereAnEditNotifiesNobody(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
k := cond("machine.ace.silent", Urgent, "the home server is silent")
k := cond("machine.node-a.silent", Urgent, "the home server is silent")
k.Raised = c.now()
h.Condition(EventRaised, k)
k.SilencedTill, k.SilencedWhy = c.now().Add(24*time.Hour), "ace is moving house"
k.SilencedTill, k.SilencedWhy = c.now().Add(24*time.Hour), "node-a is moving house"
h.Condition(EventChanged, k)
if !strings.HasPrefix(tg.edits["1"].Title, "Silenced until ") || !strings.HasPrefix(dt.edits["g14=1"].Title, "Silenced until ") {
t.Fatalf("telegram %q desk %q", tg.edits["1"].Title, dt.edits["g14=1"].Title)
if !strings.HasPrefix(tg.edits["1"].Title, "Silenced until ") || !strings.HasPrefix(dt.edits["node-b=1"].Title, "Silenced until ") {
t.Fatalf("telegram %q desk %q", tg.edits["1"].Title, dt.edits["node-b=1"].Title)
}
h.Condition(EventCleared, Condition{Key: k.Key, At: c.now()})
runFor(h, c, ClearSettle+time.Minute)
if !strings.HasPrefix(tg.edits["1"].Title, "Resolved: ") || !strings.HasPrefix(dt.edits["g14=1"].Title, "Silenced until ") {
t.Fatalf("resolved while silenced: telegram %q desk %q", tg.edits["1"].Title, dt.edits["g14=1"].Title)
if !strings.HasPrefix(tg.edits["1"].Title, "Resolved: ") || !strings.HasPrefix(dt.edits["node-b=1"].Title, "Silenced until ") {
t.Fatalf("resolved while silenced: telegram %q desk %q", tg.edits["1"].Title, dt.edits["node-b=1"].Title)
}
if len(tg.sends)+len(dt.sends) != 2 || len(tg.replaced)+len(dt.replaced) != 0 {
t.Fatalf("a new message was made: sends %d %d", len(tg.sends), len(dt.sends))
@@ -358,7 +358,7 @@ func TestAConditionResolvedWhileSilencedSaysSoOnlyWhereAnEditNotifiesNobody(t *t
// same way, and the reading that says it ended makes the message open again.
func TestASilenceReadFromTheOpenSetIsSaidOnTheShownMessage(t *testing.T) {
h, _, dt, c, _ := newHolder(t)
k := cond("machine.shanks.units", Warning, "2 failed services on shanks")
k := cond("machine.node-d.units", Warning, "2 failed services on node-d")
k.Raised = c.now()
h.Condition(EventRaised, k)
settle(h, c)
@@ -366,12 +366,12 @@ func TestASilenceReadFromTheOpenSetIsSaidOnTheShownMessage(t *testing.T) {
s := k
s.SilencedTill, s.SilencedWhy = c.now().Add(7*24*time.Hour), "the mounts are gone for good"
h.Sync([]Condition{s}, c.now())
if m := dt.edits["g14=1"]; !strings.HasPrefix(m.Title, "Silenced until ") || !m.Quiet || len(m.Actions) != 0 {
if m := dt.edits["node-b=1"]; !strings.HasPrefix(m.Title, "Silenced until ") || !m.Quiet || len(m.Actions) != 0 {
t.Fatalf("read silenced, and the desk's message says %q quiet=%v actions=%v", m.Title, m.Quiet, m.Actions)
}
c.pass(10 * time.Minute)
h.Sync([]Condition{k}, c.now()) // the silence was lifted: the reading carries none
if m := dt.edits["g14=1"]; m.Title != "machine shanks units" || len(dt.sends) != 1 {
if m := dt.edits["node-b=1"]; m.Title != "machine node-d units" || len(dt.sends) != 1 {
t.Fatalf("read unsilenced, and the desk's message says %q (sends %d)", m.Title, len(dt.sends))
}
}
@@ -114,7 +114,7 @@ func TestAConditionsMessageIsReadWithoutBeingSent(t *testing.T) {
}
func TestAPreviewSaysTheWordsTheChannelWasGiven(t *testing.T) {
issues := []TrackedIssue{{Number: 386, Status: "fixing", Worker: "fix-386@g14", Since: at(21, 10)}}
issues := []TrackedIssue{{Number: 386, Status: "fixing", Worker: "fix-386@node-b", Since: at(21, 10)}}
h, tg, dt, c, st, k := previewHolder(t, issues)
sent := dt.sends[len(dt.sends)-1] // the message the desk was actually handed
was := mark(t, h, tg, dt, st, k.Key)
@@ -126,7 +126,7 @@ func TestAPreviewSaysTheWordsTheChannelWasGiven(t *testing.T) {
if out["title"] != sent.Title || out["body"] != sent.Body {
t.Fatalf("preview %q / %q; the channel was given %q / %q", out["title"], out["body"], sent.Title, sent.Body)
}
if !strings.Contains(sent.Body, "Tracked as issue 386: fix-386 on g14 is fixing it since ") {
if !strings.Contains(sent.Body, "Tracked as issue 386: fix-386 on node-b is fixing it since ") {
t.Fatalf("the message this is compared against carries no tracking line: %q", sent.Body)
}
was.check(t, h, tg, dt, st, c, k.Key)
@@ -187,11 +187,11 @@ func TestASilenceIsPreviewedOnlyWhileThereIsOne(t *testing.T) {
func TestAPreviewOfWhatThisRouterDoesNotHoldIsRefusedByName(t *testing.T) {
h, _, _, _, _, k := previewHolder(t, nil)
_, err := h.Preview("machine.ace.silent", "")
_, err := h.Preview("machine.node-a.silent", "")
if err == nil {
t.Fatal("a condition this router does not hold was previewed")
}
if !strings.Contains(err.Error(), "machine.ace.silent") || !strings.Contains(err.Error(), k.Key) {
if !strings.Contains(err.Error(), "machine.node-a.silent") || !strings.Contains(err.Error(), k.Key) {
t.Errorf("the refusal says neither what was asked nor what is held: %v", err)
}
@@ -248,7 +248,7 @@ func TestAPreviewSaysTheReducedWordsAndWhyWhenItsOwnMayNotLeave(t *testing.T) {
stated := 0
h.Emit = func(string, any) error { stated++; return nil }
// A warning that needs nothing is held, not sent, so this refusal is one no send has met yet.
k := cond("machine.ace.unreadable", Warning, "a file on it could not be read")
k := cond("machine.node-a.unreadable", Warning, "a file on it could not be read")
k.Explanation = "Nothing for you to do. A file at /home/someone/secrets.txt could not be read."
k.Needs = ""
k.Raised = c.now()
@@ -43,7 +43,7 @@ func TestAReplacedAsksMessageEndsWithIt(t *testing.T) {
t.Fatalf("open ask records: %v; want only the newer ask's", got)
}
for _, ch := range []*fakeChannel{tg, dt} {
id := map[string]string{"telegram": "1", "desktop": "g14=1"}[ch.name]
id := map[string]string{"telegram": "1", "desktop": "node-b=1"}[ch.name]
m, edited := ch.edits[id]
for _, r := range ch.replaced {
if r.id == id {
@@ -95,7 +95,7 @@ func TestAnAsksMessageLeftOpenAfterItsAskEndedIsEndedOnTheNextTick(t *testing.T)
// restarts never says a held message of something that already ended.
func TestAClearedRecordOwesNothing(t *testing.T) {
h, _, _, c, st := newHolder(t)
k := cond("machine.shanks.units", Warning, "2 failed services on shanks")
k := cond("machine.node-d.units", Warning, "2 failed services on node-d")
k.Raised = c.now()
h.Condition(EventRaised, k)
h.mu.Lock()
+13 -13
View File
@@ -20,7 +20,7 @@ func TestLostAnswersGetOnlyWhatIsOpenAndUnansweredSaidAgain(t *testing.T) {
h.Start()
c.pass(2 * time.Minute)
// Open, with a question the desk carries.
open := cond("machine.shanks.units", Warning, "2 failed services on shanks")
open := cond("machine.node-d.units", Warning, "2 failed services on node-d")
open.Raised = c.now()
h.Condition(EventRaised, open)
settle(h, c)
@@ -30,12 +30,12 @@ func TestLostAnswersGetOnlyWhatIsOpenAndUnansweredSaidAgain(t *testing.T) {
ask := ackAsk(c, "silence-units")
ask.About = open.Key
submit(t, h, ConditionOwner, ask)
if e := dt.edits["g14=1"]; ticketFor(e, "Silence for a week") == "" {
if e := dt.edits["node-b=1"]; ticketFor(e, "Silence for a week") == "" {
t.Fatalf("the question was not put on the open message: %+v", e)
}
// Silenced, with a question of its own, and its message says it is silenced.
runFor(h, c, 20*time.Minute)
sil := cond("machine.ace.agent-home", Warning, "Agent-home shrank")
sil := cond("machine.node-a.agent-home", Warning, "Agent-home shrank")
sil.Raised = c.now()
h.Condition(EventRaised, sil)
settle(h, c)
@@ -46,12 +46,12 @@ func TestLostAnswersGetOnlyWhatIsOpenAndUnansweredSaidAgain(t *testing.T) {
silAsk := ackAsk(c, "silence-home")
silAsk.About = sil.Key
submit(t, h, ConditionOwner, silAsk)
if e := dt.edits["g14=2"]; ticketFor(e, "Silence for a week") == "" {
if e := dt.edits["node-b=2"]; ticketFor(e, "Silence for a week") == "" {
t.Fatalf("the question was not put on the message to be silenced: %+v", e)
}
sil.SilencedTill, sil.SilencedWhy, sil.At = c.now().Add(7*24*time.Hour), "known", c.now()
h.Condition(EventChanged, sil)
if e := dt.edits["g14=2"]; !strings.HasPrefix(e.Title, "Silenced until") {
if e := dt.edits["node-b=2"]; !strings.HasPrefix(e.Title, "Silenced until") {
t.Fatalf("the silence was not said: %+v", e)
}
// Open, told, and asking nothing: not unanswered, so not said again.
@@ -74,10 +74,10 @@ func TestLostAnswersGetOnlyWhatIsOpenAndUnansweredSaidAgain(t *testing.T) {
}
h.Condition(EventCleared, Condition{Key: res.Key, At: c.now()})
runFor(h, c, 3*time.Minute)
if e := dt.edits["g14=4"]; !strings.HasPrefix(e.Title, "Resolved: ") {
if e := dt.edits["node-b=4"]; !strings.HasPrefix(e.Title, "Resolved: ") {
t.Fatalf("the clearing was not said: %+v", e)
}
delete(dt.edits, "g14=2") // so an edit said again would be seen as such
delete(dt.edits, "node-b=2") // so an edit said again would be seen as such
sends, edits, replaced, tgSaid := len(dt.sends), len(dt.edits), len(dt.replaced), saidOn(tg)
// The desk's notifier started again: its standing says when the answers were lost.
c.pass(time.Minute)
@@ -87,20 +87,20 @@ func TestLostAnswersGetOnlyWhatIsOpenAndUnansweredSaidAgain(t *testing.T) {
if len(dt.sends) != sends {
t.Fatalf("a new message was sent instead of one under its handle: %+v", dt.sends[sends:])
}
if len(dt.replaced) != replaced+1 || dt.replaced[replaced].id != "g14=1" {
if len(dt.replaced) != replaced+1 || dt.replaced[replaced].id != "node-b=1" {
t.Fatalf("the open question was not said again under its handle, once and alone: %+v", dt.replaced[replaced:])
}
again := dt.replaced[replaced].m
if ticketFor(again, "Silence for a week") == "" || !strings.Contains(again.Title, headlineOf(open.Key)) {
t.Fatalf("said again without its answer or its words: %+v", again)
}
if e, said := dt.edits["g14=2"]; said {
if e, said := dt.edits["node-b=2"]; said {
t.Fatalf("the silenced one was said again for the restart: %+v", e)
}
if len(dt.edits) != edits {
t.Fatalf("an edit reached the desk for the restart: %+v", dt.edits)
}
if e := dt.edits["g14=4"]; !strings.HasPrefix(e.Title, "Resolved: ") {
if e := dt.edits["node-b=4"]; !strings.HasPrefix(e.Title, "Resolved: ") {
t.Fatalf("the resolved one was changed: %+v", e)
}
if saidOn(tg) != tgSaid {
@@ -115,14 +115,14 @@ func TestLostAnswersGetOnlyWhatIsOpenAndUnansweredSaidAgain(t *testing.T) {
}
// Its ticket is one-time still: one warrant, however often it is tapped, and refused from another machine.
tk := ticketFor(again, "Silence for a week")
ch := choiceOf(tk, "g14", c.now())
ch := choiceOf(tk, "node-b", c.now())
ch.Sender.Authenticated = false
if got, err := h.Chosen("desktop", ch); got != "chosen" || err != nil {
t.Fatalf("the answer on the message said again: %s, %v", got, err)
}
ch.ID = "c-second"
_, _ = h.Chosen("desktop", ch) // the same tap again: said as recorded (issue 372), nothing more done
other := choiceOf(tk, "shanks", c.now())
other := choiceOf(tk, "node-d", c.now())
other.Sender.Authenticated = false
if got, _ := h.Chosen("desktop", other); got != "refused" {
t.Fatalf("the ticket taken from another machine after it was used: %s", got)
@@ -146,7 +146,7 @@ func TestLostAnswersAreJudgedPerMessageAndWaitForTheDesk(t *testing.T) {
h, tg, dt, c, _, _ := newRouter(t)
h.Start()
c.pass(2 * time.Minute)
open := cond("machine.shanks.units", Warning, "2 failed services on shanks")
open := cond("machine.node-d.units", Warning, "2 failed services on node-d")
open.Raised = c.now()
h.Condition(EventRaised, open)
settle(h, c)
@@ -297,7 +297,7 @@ func TestAnAccountInItsFirstHourIsUnlinkedFromTheDesk(t *testing.T) {
if labelsOf(asked) != "Unlink it" || !strings.Contains(asked.Body, "number 666") {
t.Fatalf("the desk was not asked: %+v", asked)
}
ch := choiceOf(ticketFor(asked, "Unlink it"), "g14", c.now())
ch := choiceOf(ticketFor(asked, "Unlink it"), "node-b", c.now())
ch.Sender.Authenticated = false
if out, err := h.Chosen("desktop", ch); err != nil || out != "chosen" {
t.Fatalf("%s %v", out, err)
@@ -409,7 +409,7 @@ func TestAnUnlinkFromTheDeskInTheFirstHourBarsNothing(t *testing.T) {
link(h, store, c, "42", c.now().Add(-10*time.Minute)) // the operator's own, linked just now
h.Link("telegram", asks.LinkAsked{ID: "l1", At: c.now(), Sender: asks.Sender{Identity: "666", Display: "X", Authenticated: true}})
asked := dt.sends[len(dt.sends)-1]
ch := choiceOf(ticketFor(asked, "Unlink it"), "g14", c.now()) // a click nobody can prove was the operator's
ch := choiceOf(ticketFor(asked, "Unlink it"), "node-b", c.now()) // a click nobody can prove was the operator's
ch.Sender.Authenticated = false
if out, err := h.Chosen("desktop", ch); err != nil || out != "chosen" {
t.Fatalf("%s %v", out, err)
File diff suppressed because it is too large Load Diff
@@ -44,15 +44,15 @@ func TestAConditionsMessageSaysWhereTheIssueThatTracksItStands(t *testing.T) {
"Whether an issue tracks this could not be checked.", "Nothing for you to do."},
{"nobody works on it", []TrackedIssue{{Number: 352, Status: "located"}}, true,
"Tracked as issue 352: nobody is working on it.", "A person repaired"},
{"an agent is fixing it", []TrackedIssue{{Number: 386, Status: "fixing", Worker: "fix-386@g14", Since: at(21, 10)}}, true,
"Tracked as issue 386: fix-386 on g14 is fixing it since 21:10.", "Nothing for you to do."},
{"an agent looks for its cause", []TrackedIssue{{Number: 386, Status: "diagnosing", Worker: "jochen@shanks", Since: at(18, 5)}}, true,
"Tracked as issue 386: jochen on shanks is looking for its cause since 18:05.", "Nothing for you to do."},
{"its worker went quiet", []TrackedIssue{{Number: 386, Status: "fixing", Worker: "fix-386@g14", Since: at(9, 0), Active: at(9, 30), Stale: true}}, true,
{"an agent is fixing it", []TrackedIssue{{Number: 386, Status: "fixing", Worker: "fix-386@node-b", Since: at(21, 10)}}, true,
"Tracked as issue 386: fix-386 on node-b is fixing it since 21:10.", "Nothing for you to do."},
{"an agent looks for its cause", []TrackedIssue{{Number: 386, Status: "diagnosing", Worker: "jochen@node-d", Since: at(18, 5)}}, true,
"Tracked as issue 386: jochen on node-d is looking for its cause since 18:05.", "Nothing for you to do."},
{"its worker went quiet", []TrackedIssue{{Number: 386, Status: "fixing", Worker: "fix-386@node-b", Since: at(9, 0), Active: at(9, 30), Stale: true}}, true,
"Tracked as issue 386: nobody has worked on it since 09:30.", "A person repaired"},
{"a fix waits to be merged", []TrackedIssue{{Number: 386, Status: "located", Fix: "not-merged"}}, true,
"Tracked as issue 386: a fix is written and waits to be merged.", "Nothing for you to do."},
{"its fix is on its way", []TrackedIssue{{Number: 386, Status: "fixing", Worker: "fix-386@g14", Since: at(9, 0), Fix: "merged"}}, true,
{"its fix is on its way", []TrackedIssue{{Number: 386, Status: "fixing", Worker: "fix-386@node-b", Since: at(9, 0), Fix: "merged"}}, true,
"Tracked as issue 386: its fix is merged and on its way.", "Nothing for you to do."},
{"the delivery was not heard yet", []TrackedIssue{{Number: 386, Status: "fixing", Fix: "unheard"}}, true,
"Tracked as issue 386: a fix is named, and nothing is heard yet of its delivery.", "Nothing for you to do."},
@@ -69,23 +69,23 @@ func TestAConditionsMessageSaysWhereTheIssueThatTracksItStands(t *testing.T) {
{"one not fixed after one open", []TrackedIssue{{Number: 100, Status: "wontfix"}, {Number: 200, Status: "open"}}, true,
"Tracked as issue 200: nobody is working on it. Also as issue 100: it will not be fixed.", "A person repaired"},
{"its worker went quiet on one that waits for a decision", []TrackedIssue{{Number: 400, Status: "fixing", Needs: "decision",
Worker: "fix-400@g14", Since: at(9, 0), Active: at(9, 30), Stale: true}}, true,
Worker: "fix-400@node-b", Since: at(9, 0), Active: at(9, 30), Stale: true}}, true,
"Tracked as issue 400: nobody has worked on it since 09:30.", "A person repaired"},
{"it waits for the operator", []TrackedIssue{{Number: 336, Status: "open", Needs: "operator"}}, true,
"Tracked as issue 336: it waits for your answer.", "Needs you: answer issue 336. A person repaired"},
{"it waits for a decision", []TrackedIssue{{Number: 400, Status: "open", Needs: "decision"}}, true,
"Tracked as issue 400: nobody is working on it; it waits for a decision first.", "A person repaired"},
{"two issues, one worked on", []TrackedIssue{
{Number: 386, Status: "fixing", Worker: "fix-386@g14", Since: at(21, 10)},
{Number: 386, Status: "fixing", Worker: "fix-386@node-b", Since: at(21, 10)},
{Number: 379, Status: "located"}}, true,
"Tracked as issue 379: nobody is working on it. Also as issue 386: fix-386 on g14 is fixing it since 21:10.",
"Tracked as issue 379: nobody is working on it. Also as issue 386: fix-386 on node-b is fixing it since 21:10.",
"Nothing for you to do."},
{"two issues, nobody on either", []TrackedIssue{{Number: 386, Status: "open"}, {Number: 379, Status: "located"}}, true,
"Tracked as issue 379: nobody is working on it. Also as issue 386: nobody is working on it.", "A person repaired"},
{"one of two waits for the operator", []TrackedIssue{
{Number: 386, Status: "fixing", Worker: "fix-386@g14", Since: at(21, 10)},
{Number: 386, Status: "fixing", Worker: "fix-386@node-b", Since: at(21, 10)},
{Number: 379, Status: "open", Needs: "operator"}}, true,
"Tracked as issue 379: it waits for your answer. Also as issue 386: fix-386 on g14 is fixing it since 21:10.",
"Tracked as issue 379: it waits for your answer. Also as issue 386: fix-386 on node-b is fixing it since 21:10.",
"Needs you: answer issue 379."},
{"more than three", []TrackedIssue{{Number: 1, Status: "open"}, {Number: 2, Status: "open"}, {Number: 3, Status: "open"},
{Number: 4, Status: "open"}, {Number: 5, Status: "resolved"}}, true,
@@ -116,9 +116,9 @@ func TestAConditionsMessageSaysWhereTheIssueThatTracksItStands(t *testing.T) {
func TestNothingForYouToDoIsNeverSaidWhileTheIssueWaitsOnTheOperatorOrNobodyWorksOnIt(t *testing.T) {
for _, issues := range [][]TrackedIssue{
{{Number: 336, Status: "open", Needs: "operator"}},
{{Number: 336, Status: "fixing", Worker: "fix-336@g14", Since: at(9, 0), Needs: "operator"}},
{{Number: 336, Status: "fixing", Worker: "fix-336@node-b", Since: at(9, 0), Needs: "operator"}},
{{Number: 352, Status: "located"}},
{{Number: 352, Status: "fixing", Worker: "fix-352@g14", Since: at(9, 0), Stale: true}},
{{Number: 352, Status: "fixing", Worker: "fix-352@node-b", Since: at(9, 0), Stale: true}},
} {
if m := trackedMessage(issues, true); strings.Contains(m.Body, "Nothing for you to do") {
t.Errorf("%+v: %q", issues, m.Body)
@@ -128,11 +128,11 @@ func TestNothingForYouToDoIsNeverSaidWhileTheIssueWaitsOnTheOperatorOrNobodyWork
// A verdict that already needs the operator is left as it is; the line is added below it.
func TestAVerdictThatNeedsTheOperatorStands(t *testing.T) {
r := &Record{Key: "module.mounts.shanks.unhealthy", Origin: "condition", Severity: Urgent, Headline: "mounts not working on shanks",
Explanation: "Needs you: give the share's login. mounts on shanks is not healthy.", Raised: trackNow.Add(-time.Hour)}
r := &Record{Key: "module.mounts.node-d.unhealthy", Origin: "condition", Severity: Urgent, Headline: "mounts not working on node-d",
Explanation: "Needs you: give the share's login. mounts on node-d is not healthy.", Raised: trackNow.Add(-time.Hour)}
m, _ := withTracking(compose(r, "raised", trackNow, time.UTC, 0), r, "raised",
[]TrackedIssue{{Number: 386, Status: "open", Needs: "operator"}}, true, trackNow, time.UTC)
want := "Needs you: give the share's login. mounts on shanks is not healthy.\nTracked as issue 386: it waits for your answer.\nSince 18:30."
want := "Needs you: give the share's login. mounts on node-d is not healthy.\nTracked as issue 386: it waits for your answer.\nSince 18:30."
if m.Body != want {
t.Fatalf("got %q\nwant %q", m.Body, want)
}
@@ -173,12 +173,12 @@ func TestTheRouterAsksTheTrackerAndSaysItsWordInTheMessage(t *testing.T) {
if fail {
return nil, errors.New("no responders")
}
return json.RawMessage(`{"conditions":{"module.mounts.shanks.unhealthy":[` +
return json.RawMessage(`{"conditions":{"module.mounts.node-d.unhealthy":[` +
`{"number":379,"status":"located"},` +
`{"number":386,"status":"fixing","worker":"fix-386@g14","since":"2026-10-06T11:10:00Z"}]}}`), nil
`{"number":386,"status":"fixing","worker":"fix-386@node-b","since":"2026-10-06T11:10:00Z"}]}}`), nil
}
k := cond("module.mounts.shanks.unhealthy", Urgent, "mounts on shanks is not healthy")
k.Explanation, k.Needs = "Nothing for you to do. mounts on shanks is not healthy.", ""
k := cond("module.mounts.node-d.unhealthy", Urgent, "mounts on node-d is not healthy")
k.Explanation, k.Needs = "Nothing for you to do. mounts on node-d is not healthy.", ""
k.Raised = c.now()
h.Condition(EventRaised, k)
settle(h, c)
@@ -186,7 +186,7 @@ func TestTheRouterAsksTheTrackerAndSaysItsWordInTheMessage(t *testing.T) {
t.Fatalf("urgent: telegram %d, desktop %d", len(tg.sends), len(dt.sends))
}
body := tg.sends[0].Body
if !strings.Contains(body, "\nTracked as issue 379: nobody is working on it. Also as issue 386: fix-386 on g14 is fixing it since 11:10.\n") {
if !strings.Contains(body, "\nTracked as issue 379: nobody is working on it. Also as issue 386: fix-386 on node-b is fixing it since 11:10.\n") {
t.Fatalf("body: %q", body)
}
if asked != 1 {
@@ -199,7 +199,7 @@ func TestTheRouterAsksTheTrackerAndSaysItsWordInTheMessage(t *testing.T) {
c.pass(TrackingKept + time.Minute)
h.Tick()
h.mu.Lock()
r := h.open["module.mounts.shanks.unhealthy"]
r := h.open["module.mounts.node-d.unhealthy"]
h.mu.Unlock()
if m := h.say(r, "reminder"); !strings.Contains(m.Body, "\nWhether an issue tracks this could not be checked.\n") {
t.Fatalf("an old word was used: %q", m.Body)
@@ -213,11 +213,11 @@ func TestTheRouterAsksTheTrackerAndSaysItsWordInTheMessage(t *testing.T) {
// built without it.
func TestARouterThatCannotAskAddsNothing(t *testing.T) {
h, _, _, c, _ := newHolder(t)
k := cond("machine.ace.silent", Urgent, "ace is silent")
k := cond("machine.node-a.silent", Urgent, "node-a is silent")
k.Raised = c.now()
h.Condition(EventRaised, k)
h.mu.Lock()
r := h.open["machine.ace.silent"]
r := h.open["machine.node-a.silent"]
h.mu.Unlock()
if m := h.say(r, "raised"); strings.Contains(m.Body, "issue") {
t.Fatalf("%q", m.Body)
+68 -68
View File
@@ -44,7 +44,7 @@ func walk(t *testing.T, h *Holder, id, phase, outcome string, at time.Time, boun
h.Walked(ev)
}
func novox(modules ...string) Touch { return Touch{Machine: "novox", Modules: modules} }
func nodeC(modules ...string) Touch { return Touch{Machine: "node-c", Modules: modules} }
// heard says whether any message on the channels carries the words: sent, edited or said again in place.
func heard(words string, chs ...*fakeChannel) bool {
@@ -94,11 +94,11 @@ func has(list []string, want string) bool {
// history says it was held — a module's condition, a machine's, and a module's notice alike.
func TestDuringAWalkWhatItTouchesIsHeldAndTheHistorySaysSo(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
walk(t, h, "plan-1", WalkStarted, "", c.now(), 600, novox("mesh-tools", "node-tools"))
mod := cond("module.node-tools.novox.unhealthy", Urgent, "the tool runner on novox does not answer")
mod.Machine, mod.Raised = "novox", c.now()
walk(t, h, "plan-1", WalkStarted, "", c.now(), 600, nodeC("mesh-tools", "node-tools"))
mod := cond("module.node-tools.node-c.unhealthy", Urgent, "the tool runner on node-c does not answer")
mod.Machine, mod.Raised = "node-c", c.now()
h.Condition(EventRaised, mod)
machine := cond("machine.novox.claude-code.agent-home.data-shrank", Urgent, "the agent's home shrank")
machine := cond("machine.node-c.claude-code.agent-home.data-shrank", Urgent, "the agent's home shrank")
machine.Raised = c.now()
h.Condition(EventRaised, machine)
if _, err := h.Notify("tools.slow", Urgent, "tools answer slowly", "node-tools", false, "tools slow",
@@ -123,24 +123,24 @@ func TestDuringAWalkWhatItTouchesIsHeldAndTheHistorySaysSo(t *testing.T) {
// nothing of it.
func TestAMessageTouchedByTwoOverlappingWalksIsHeldUntilTheSecondEnds(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
walk(t, h, "plan-a", WalkStarted, "", c.now(), 1200, novox("mesh-tools"))
walk(t, h, "plan-a", WalkStarted, "", c.now(), 1200, nodeC("mesh-tools"))
c.pass(time.Minute)
walk(t, h, "plan-b", WalkStarted, "", c.now(), 1200, novox("node-tools"), Touch{Machine: "ace", Modules: []string{"node-tools"}})
k := cond("machine.novox.network", Urgent, "novox lost its default route")
walk(t, h, "plan-b", WalkStarted, "", c.now(), 1200, nodeC("node-tools"), Touch{Machine: "node-a", Modules: []string{"node-tools"}})
k := cond("machine.node-c.network", Urgent, "node-c lost its default route")
k.Raised = c.now()
h.Condition(EventRaised, k)
runFor(h, c, time.Minute)
walk(t, h, "plan-a", WalkEnded, WalkDelivered, c.now(), 0, novox("mesh-tools"))
walk(t, h, "plan-a", WalkEnded, WalkDelivered, c.now(), 0, nodeC("mesh-tools"))
runFor(h, c, DefaultSettle+time.Minute) // its end is told once it settled (ADR 0505 §3)
if heard(headlineOf(k.Key), tg, dt) {
t.Fatalf("said at the first walk's end, while the second still holds it: %+v %+v", tg.sends, dt.sends)
}
if !heard("Delivered: mesh-tools on novox", tg, dt) {
if !heard("Delivered: mesh-tools on node-c", tg, dt) {
t.Fatalf("the first walk's end was not told: %+v", dt.sends)
}
walk(t, h, "plan-b", WalkEnded, WalkDelivered, c.now(), 0)
runFor(h, c, DefaultSettle+time.Minute)
if !heard("Still open after the delivery of node-tools to ace, novox:\n- Urgent: "+headlineOf(k.Key), tg, dt) {
if !heard("Still open after the delivery of node-tools to node-a, node-c:\n- Urgent: "+headlineOf(k.Key), tg, dt) {
t.Fatalf("the second walk's end did not tell it: telegram %+v, desk %+v", tg.sends, dt.sends)
}
}
@@ -149,18 +149,18 @@ func TestAMessageTouchedByTwoOverlappingWalksIsHeldUntilTheSecondEnds(t *testing
// sent at once, while a walk touches the machine they are on.
func TestWhatAWalkNeverHoldsIsSentAtOnce(t *testing.T) {
h, tg, dt, c, _, _ := newRouter(t)
walk(t, h, "plan-9", WalkStarted, "", c.now(), 1200, novox("telegram", "mesh-tools"))
walk(t, h, "plan-9", WalkStarted, "", c.now(), 1200, nodeC("telegram", "mesh-tools"))
for _, key := range []string{
"machine.novox.telegram.telegram-token.secret-given", // a secret given, on a touched machine
"machine.novox.mounts.smb.secret-replaced", // a secret replaced
"machine.ace.silent", // a machine no walk touches
"delivery.leaf.over-budget", // the walk's own delivery
"plan.plan-9.failed", // the walk itself
"machine.node-c.telegram.telegram-token.secret-given", // a secret given, on a touched machine
"machine.node-c.mounts.smb.secret-replaced", // a secret replaced
"machine.node-a.silent", // a machine no walk touches
"delivery.leaf.over-budget", // the walk's own delivery
"plan.plan-9.failed", // the walk itself
} {
k := cond(key, Urgent, "it happened")
k.Raised = c.now()
if !strings.HasPrefix(key, "machine.") {
k.Machine = "novox" // a walk's own failure, on a machine it touches
k.Machine = "node-c" // a walk's own failure, on a machine it touches
}
h.Condition(EventRaised, k)
runFor(h, c, time.Minute)
@@ -175,7 +175,7 @@ func TestWhatAWalkNeverHoldsIsSentAtOnce(t *testing.T) {
if !has(outcomes(h, "ask."+ConditionOwner+".own"), "sent") {
t.Errorf("an ask waiting for the operator was held: %v", outcomes(h, "ask."+ConditionOwner+".own"))
}
held := cond("machine.novox.units", Urgent, "2 failed services on novox")
held := cond("machine.node-c.units", Urgent, "2 failed services on node-c")
held.Raised = c.now()
h.Condition(EventRaised, held)
runFor(h, c, time.Minute)
@@ -195,15 +195,15 @@ func TestWhatAWalkNeverHoldsIsSentAtOnce(t *testing.T) {
// still open, one line each; the one that cleared is only counted, and nothing more is said of them after.
func TestAWalksEndIsToldOnceWithWhatIsStillOpen(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
walk(t, h, "plan-3", WalkStarted, "", c.now(), 600, novox("mesh-tools"))
still := cond("machine.novox.tools-silent", Warning, "the tools on novox are silent")
walk(t, h, "plan-3", WalkStarted, "", c.now(), 600, nodeC("mesh-tools"))
still := cond("machine.node-c.tools-silent", Warning, "the tools on node-c are silent")
still.Raised = c.now()
h.Condition(EventRaised, still)
gone := cond("module.mesh-tools.novox.unhealthy", Warning, "mesh-tools is unhealthy on novox")
gone := cond("module.mesh-tools.node-c.unhealthy", Warning, "mesh-tools is unhealthy on node-c")
gone.Raised = c.now()
h.Condition(EventRaised, gone)
c.pass(time.Second)
other := cond("machine.novox.core-behind", Urgent, "novox runs an older core")
other := cond("machine.node-c.core-behind", Urgent, "node-c runs an older core")
other.Raised = c.now()
h.Condition(EventRaised, other)
runFor(h, c, 2*time.Minute)
@@ -219,10 +219,10 @@ func TestAWalksEndIsToldOnceWithWhatIsStillOpen(t *testing.T) {
t.Fatalf("the end was not one message per kind: desk %+v, telegram %+v", dt.sends, tg.sends)
}
m := dt.sends[0]
if m.Title != "Delivery failed: mesh-tools on novox" || !strings.Contains(m.Body, "It took 7 min.") {
if m.Title != "Delivery failed: mesh-tools on node-c" || !strings.Contains(m.Body, "It took 7 min.") {
t.Fatalf("the end does not say its outcome, what it reached and how long: %q", m.Text())
}
want := "Still open after the delivery of mesh-tools to novox:\n- " + headlineOf(still.Key) + "\n- Urgent: " + headlineOf(other.Key) +
want := "Still open after the delivery of mesh-tools to node-c:\n- " + headlineOf(still.Key) + "\n- Urgent: " + headlineOf(other.Key) +
"\n1 problem it raised resolved during it."
if !strings.Contains(m.Body, want) || strings.Contains(m.Text(), headlineOf(gone.Key)) {
t.Fatalf("the end message %q does not say %q", m.Body, want)
@@ -238,18 +238,18 @@ func TestAWalksEndIsToldOnceWithWhatIsStillOpen(t *testing.T) {
// to say only what needs the operator.
func TestADeliveredWalkWithNothingHeldIsOneLineOrNothing(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
walk(t, h, "plan-4", WalkStarted, "", c.now(), 600, novox("mesh-tools"), Touch{Machine: "ace", Modules: []string{"mesh-tools"}})
walk(t, h, "plan-4", WalkStarted, "", c.now(), 600, nodeC("mesh-tools"), Touch{Machine: "node-a", Modules: []string{"mesh-tools"}})
runFor(h, c, 3*time.Minute)
walk(t, h, "plan-4", WalkEnded, WalkDelivered, c.now(), 0)
runFor(h, c, DefaultSettle+time.Minute)
if len(dt.sends) != 1 || len(tg.sends) != 0 {
t.Fatalf("desk %+v, telegram %+v", dt.sends, tg.sends)
}
if m := dt.sends[0]; m.Title != "Delivered: mesh-tools on ace, novox, in 3 min" || m.Body != "" {
if m := dt.sends[0]; m.Title != "Delivered: mesh-tools on node-a, node-c, in 3 min" || m.Body != "" {
t.Fatalf("not one line: %q", m.Text())
}
h.NeedsOnly = func(kind string) bool { return kind == "desktop" }
walk(t, h, "plan-5", WalkStarted, "", c.now(), 600, novox("mesh-tools"))
walk(t, h, "plan-5", WalkStarted, "", c.now(), 600, nodeC("mesh-tools"))
runFor(h, c, 3*time.Minute)
walk(t, h, "plan-5", WalkEnded, WalkDelivered, c.now(), 0)
runFor(h, c, DefaultSettle+time.Minute)
@@ -279,11 +279,11 @@ func TestAHoldEndsAtTheWalksBound(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
started := c.now()
walk(t, h, "plan-7", WalkStarted, "", started, tc.given, novox("node-tools"))
open := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent")
walk(t, h, "plan-7", WalkStarted, "", started, tc.given, nodeC("node-tools"))
open := cond("machine.node-c.tools-silent", Urgent, "the tools on node-c are silent")
open.Raised = c.now()
h.Condition(EventRaised, open)
gone := cond("module.node-tools.novox.unhealthy", Urgent, "node-tools is unhealthy")
gone := cond("module.node-tools.node-c.unhealthy", Urgent, "node-tools is unhealthy")
gone.Raised = c.now()
h.Condition(EventRaised, gone)
c.pass(30 * time.Second)
@@ -305,7 +305,7 @@ func TestAHoldEndsAtTheWalksBound(t *testing.T) {
}
}
// Past its bound the walk holds nothing: a new message about what it touches goes at once.
late := cond("machine.novox.disk", Urgent, "a disk fails on novox")
late := cond("machine.node-c.disk", Urgent, "a disk fails on node-c")
late.Raised = c.now()
h.Condition(EventRaised, late)
runFor(h, c, time.Minute)
@@ -319,8 +319,8 @@ func TestAHoldEndsAtTheWalksBound(t *testing.T) {
// §2, kept: what a walk holds survives the router's restart, and is told at the walk's end.
func TestHeldMessagesSurviveARouterRestart(t *testing.T) {
h, tg, dt, c, st := newHolder(t)
walk(t, h, "plan-8", WalkStarted, "", c.now(), 1200, novox("mesh-tools"))
k := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent")
walk(t, h, "plan-8", WalkStarted, "", c.now(), 1200, nodeC("mesh-tools"))
k := cond("machine.node-c.tools-silent", Urgent, "the tools on node-c are silent")
k.Raised = c.now()
h.Condition(EventRaised, k)
runFor(h, c, time.Minute)
@@ -345,8 +345,8 @@ func TestARestartSaysNothingItHasSaid(t *testing.T) {
h, tg, dt, c, _, _ := newRouter(t)
h.Start()
c.pass(2 * time.Minute)
keys := []string{"machine.novox.telegram.telegram-token.secret-given", "machine.novox.claude-code.agent-home.data-shrank",
"machine.shanks.mounts.smb-password-recalbox.secret-given", "machine.ace.nzbgeek.api-key.secret-given"}
keys := []string{"machine.node-c.telegram.telegram-token.secret-given", "machine.node-c.claude-code.agent-home.data-shrank",
"machine.node-d.mounts.smb-password-recalbox.secret-given", "machine.node-a.nzbgeek.api-key.secret-given"}
var silenced []Condition
for i, key := range keys {
k := cond(key, Urgent, "it happened")
@@ -359,7 +359,7 @@ func TestARestartSaysNothingItHasSaid(t *testing.T) {
stand(h, c)
silenced = append(silenced, k)
}
withheldOpen := cond("machine.novox.units", Urgent, "2 failed services")
withheldOpen := cond("machine.node-c.units", Urgent, "2 failed services")
withheldOpen.Explanation = "Needs you: read /etc/systemd/system/x.service."
withheldOpen.Raised = c.now()
h.Condition(EventRaised, withheldOpen)
@@ -415,14 +415,14 @@ func TestARestartSaysNothingItHasSaid(t *testing.T) {
// The controller's walk events are read as ADR 0308 §1 says them; any other plan-moved is not a walk's.
func TestTheWalkEventsAreReadAsTheControllerSaysThem(t *testing.T) {
at := time.Date(2026, 10, 12, 11, 12, 0, 0, time.UTC)
ev, ok, err := DecodeWalk(walkBody(t, "plan-1", WalkStarted, "", at, 1200, novox("mesh-tools", "node-tools")))
ev, ok, err := DecodeWalk(walkBody(t, "plan-1", WalkStarted, "", at, 1200, nodeC("mesh-tools", "node-tools")))
if err != nil || !ok || ev.Walk != "plan-1" || ev.BoundSeconds != 1200 || ev.Class != "leaf" || !ev.At.Equal(at) ||
len(ev.Touches) != 1 || ev.Touches[0].Machine != "novox" || len(ev.Touches[0].Modules) != 2 {
len(ev.Touches) != 1 || ev.Touches[0].Machine != "node-c" || len(ev.Touches[0].Modules) != 2 {
t.Fatalf("%+v %v %v", ev, ok, err)
}
// As mesh-controller's walk events (its pull request 247) say an end: ended_at, and carried beside modules.
end, ok, err := DecodeWalk([]byte(`{"id":"plan-1","walk":"plan-1","phase":"ended","outcome":"stopped",` +
`"touches":[{"machine":"novox","modules":["node-tools","messenger"],"carried":["messenger"]}],` +
`"touches":[{"machine":"node-c","modules":["node-tools","messenger"],"carried":["messenger"]}],` +
`"started_at":"2026-10-12T11:12:00Z","ended_at":"2026-10-12T11:19:00Z","updated":"2026-10-12T11:19:01Z"}`))
if err != nil || !ok || end.Outcome != WalkStopped || !end.At.Equal(at.Add(7*time.Minute)) || len(end.Touches[0].Modules) != 2 {
t.Fatalf("%+v %v %v", end, ok, err)
@@ -439,7 +439,7 @@ func TestTheWalkEventsAreReadAsTheControllerSaysThem(t *testing.T) {
}
// A start older than its bound is history: nothing is held.
h, _, _, c, _ := newHolder(t)
walk(t, h, "plan-old", WalkStarted, "", c.now().Add(-time.Hour), 1200, novox("mesh-tools"))
walk(t, h, "plan-old", WalkStarted, "", c.now().Add(-time.Hour), 1200, nodeC("mesh-tools"))
if len(h.walks) != 0 {
t.Fatalf("an old start opened a walk: %+v", h.walks)
}
@@ -452,8 +452,8 @@ func TestWhatAWalkHeldIsSaidOnceAChannelCanSendAgain(t *testing.T) {
for _, at := range []string{"end", "bound"} {
t.Run(at, func(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
walk(t, h, "plan-c", WalkStarted, "", c.now(), 600, novox("mesh-tools"))
k := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent")
walk(t, h, "plan-c", WalkStarted, "", c.now(), 600, nodeC("mesh-tools"))
k := cond("machine.node-c.tools-silent", Urgent, "the tools on node-c are silent")
k.Raised = c.now()
h.Condition(EventRaised, k)
runFor(h, c, 2*time.Minute)
@@ -485,12 +485,12 @@ func TestWhatAWalkHeldIsSaidOnceAChannelCanSendAgain(t *testing.T) {
// limit (its start, bound and settle) has passed, while B still runs, with B's delivery line.
func TestAMessageIsHeldOnByALaterWalkThatTouchesIt(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
walk(t, h, "plan-a", WalkStarted, "", c.now(), 360, novox("mesh-tools")) // its limit: 6 min and the settle
k := cond("machine.novox.network", Urgent, "novox lost its default route")
walk(t, h, "plan-a", WalkStarted, "", c.now(), 360, nodeC("mesh-tools")) // its limit: 6 min and the settle
k := cond("machine.node-c.network", Urgent, "node-c lost its default route")
k.Raised = c.now()
h.Condition(EventRaised, k)
runFor(h, c, time.Minute)
walk(t, h, "plan-b", WalkStarted, "", c.now(), 1200, novox("node-tools"))
walk(t, h, "plan-b", WalkStarted, "", c.now(), 1200, nodeC("node-tools"))
runFor(h, c, time.Minute)
walk(t, h, "plan-a", WalkEnded, WalkDelivered, c.now(), 0)
runFor(h, c, DefaultSettle+time.Minute)
@@ -501,7 +501,7 @@ func TestAMessageIsHeldOnByALaterWalkThatTouchesIt(t *testing.T) {
t.Fatalf("the second walk does not hold it on: %v", got)
}
runFor(h, c, 4*time.Minute) // past the first walk's limit, 8 minutes after it was held
if !heard("During the delivery of node-tools to novox.", tg, dt) || !heard(headlineOf(k.Key), tg, dt) {
if !heard("During the delivery of node-tools to node-c.", tg, dt) || !heard(headlineOf(k.Key), tg, dt) {
t.Fatalf("held past its first hold's limit: %+v %+v", tg.sends, dt.sends)
}
}
@@ -510,8 +510,8 @@ func TestAMessageIsHeldOnByALaterWalkThatTouchesIt(t *testing.T) {
// what it held is said, with the still-running line, before the walk is forgotten.
func TestAWalkReadBackADayLateSaysWhatItHeldBeforeItIsForgotten(t *testing.T) {
h, tg, dt, c, st := newHolder(t)
walk(t, h, "plan-d", WalkStarted, "", c.now(), 600, novox("mesh-tools"))
k := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent")
walk(t, h, "plan-d", WalkStarted, "", c.now(), 600, nodeC("mesh-tools"))
k := cond("machine.node-c.tools-silent", Urgent, "the tools on node-c are silent")
k.Raised = c.now()
h.Condition(EventRaised, k)
runFor(h, c, time.Minute)
@@ -540,14 +540,14 @@ func TestAStartHeardAfterItsEndOpensNothing(t *testing.T) {
h, tg, dt, c, st := newHolder(t)
started := c.now()
c.pass(20 * time.Second)
end, _, err := DecodeWalk([]byte(`{"walk":"plan-e","phase":"ended","outcome":"delivered","touches":[{"machine":"novox","modules":["mesh-tools"]}],` +
end, _, err := DecodeWalk([]byte(`{"walk":"plan-e","phase":"ended","outcome":"delivered","touches":[{"machine":"node-c","modules":["mesh-tools"]}],` +
`"started_at":"` + started.Format(time.RFC3339Nano) + `","ended_at":"` + c.now().Format(time.RFC3339Nano) + `"}`))
if err != nil {
t.Fatal(err)
}
h.Walked(end)
c.pass(time.Second)
walk(t, h, "plan-e", WalkStarted, "", started, 600, novox("mesh-tools"))
walk(t, h, "plan-e", WalkStarted, "", started, 600, nodeC("mesh-tools"))
if len(h.walks) != 0 {
t.Fatalf("a start heard after its end opened a walk: %+v", h.walks)
}
@@ -556,11 +556,11 @@ func TestAStartHeardAfterItsEndOpensNothing(t *testing.T) {
t.Fatal(err)
}
again.Start()
walk(t, again, "plan-e", WalkStarted, "", started, 600, novox("mesh-tools"))
walk(t, again, "plan-e", WalkStarted, "", started, 600, nodeC("mesh-tools"))
if len(again.walks) != 0 {
t.Fatalf("after a restart, a start heard after its end opened a walk: %+v", again.walks)
}
k := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent")
k := cond("machine.node-c.tools-silent", Urgent, "the tools on node-c are silent")
k.Raised = c.now()
again.Condition(EventRaised, k)
runFor(again, c, time.Minute)
@@ -568,7 +568,7 @@ func TestAStartHeardAfterItsEndOpensNothing(t *testing.T) {
t.Fatalf("held by a walk that had ended: %v", outcomes(again, k.Key))
}
// Retried: the same id walks again, started after the end.
walk(t, again, "plan-e", WalkStarted, "", c.now(), 600, novox("mesh-tools"))
walk(t, again, "plan-e", WalkStarted, "", c.now(), 600, nodeC("mesh-tools"))
if len(again.walks) != 1 {
t.Fatalf("a walk started after the last end did not open: %+v", again.walks)
}
@@ -581,8 +581,8 @@ func TestARestartSaysAgainAnAskedMessageAWalkTookBeforeItsAsk(t *testing.T) {
h, _, dt, c, _, _ := newRouter(t)
h.Start()
stand(h, c)
walk(t, h, "plan-f", WalkStarted, "", c.now(), 1200, novox("mesh-tools"))
k := cond("machine.novox.units", Urgent, "2 failed services on novox")
walk(t, h, "plan-f", WalkStarted, "", c.now(), 1200, nodeC("mesh-tools"))
k := cond("machine.node-c.units", Urgent, "2 failed services on node-c")
k.Raised = c.now()
h.Condition(EventRaised, k)
runFor(h, c, time.Minute)
@@ -607,8 +607,8 @@ func TestARestartSaysAgainAnAskedMessageAWalkTookBeforeItsAsk(t *testing.T) {
// file path) is said in its plain form, and what it held, still open, is said on its own, not marked told.
func TestWhatAWalksPlainEndCannotCarryIsSaidOnItsOwn(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
walk(t, h, "plan-g", WalkStarted, "", c.now(), 600, novox("/var/lib/mesh-tools"))
k := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent")
walk(t, h, "plan-g", WalkStarted, "", c.now(), 600, nodeC("/var/lib/mesh-tools"))
k := cond("machine.node-c.tools-silent", Urgent, "the tools on node-c are silent")
k.Raised = c.now()
h.Condition(EventRaised, k)
runFor(h, c, time.Minute)
@@ -634,8 +634,8 @@ func TestAHeldMessageIsToldOnlyOnceTheWalksMessageWasSent(t *testing.T) {
if then == "restart warning" {
sev = Warning
}
walk(t, h, "plan-h", WalkStarted, "", c.now(), 1200, novox("mesh-tools"))
k := cond("machine.novox.tools-silent", sev, "the tools on novox are silent")
walk(t, h, "plan-h", WalkStarted, "", c.now(), 1200, nodeC("mesh-tools"))
k := cond("machine.node-c.tools-silent", sev, "the tools on node-c are silent")
k.Raised = c.now()
h.Condition(EventRaised, k)
runFor(h, c, time.Minute)
@@ -652,7 +652,7 @@ func TestAHeldMessageIsToldOnlyOnceTheWalksMessageWasSent(t *testing.T) {
tg.fail, dt.fail = nil, nil
if then == "sent" {
runFor(h, c, 2*time.Minute)
if !heard("Still open after the delivery of mesh-tools to novox:\n- Urgent: "+headlineOf(k.Key), tg, dt) || saidOn(tg, dt) != 2 {
if !heard("Still open after the delivery of mesh-tools to node-c:\n- Urgent: "+headlineOf(k.Key), tg, dt) || saidOn(tg, dt) != 2 {
t.Fatalf("not told once in the walk's message: telegram %+v, desk %+v", tg.sends, dt.sends)
}
if r := h.open[k.Key]; r == nil || !r.told() || r.Owed != "" || r.Sent["telegram"] != "" || r.Sent["desktop"] != "" {
@@ -694,11 +694,11 @@ func TestAWalksEndHandsOnOnlyWhatALaterWalkWouldHold(t *testing.T) {
h, tg, dt, c, _, _ := newRouter(t)
h.Start()
stand(h, c)
walk(t, h, "plan-a", WalkStarted, "", c.now(), 1200, novox("mesh-tools"))
quiet := cond("machine.novox.network", Urgent, "novox lost its default route")
walk(t, h, "plan-a", WalkStarted, "", c.now(), 1200, nodeC("mesh-tools"))
quiet := cond("machine.node-c.network", Urgent, "node-c lost its default route")
quiet.Raised = c.now()
h.Condition(EventRaised, quiet)
asked := cond("machine.novox.units", Urgent, "2 failed services on novox")
asked := cond("machine.node-c.units", Urgent, "2 failed services on node-c")
asked.Raised = c.now()
h.Condition(EventRaised, asked)
runFor(h, c, time.Minute)
@@ -713,7 +713,7 @@ func TestAWalksEndHandsOnOnlyWhatALaterWalkWouldHold(t *testing.T) {
submit(t, h, ConditionOwner, ask)
runFor(h, c, time.Minute)
stand(h, c)
walk(t, h, "plan-b", WalkStarted, "", c.now(), 1200, novox("node-tools"))
walk(t, h, "plan-b", WalkStarted, "", c.now(), 1200, nodeC("node-tools"))
runFor(h, c, time.Minute)
stand(h, c)
helds := func(key string) int {
@@ -747,7 +747,7 @@ func TestAStartHeardAfterItsEndOpensNothingWhenTheControllersClockIsAhead(t *tes
h, _, _, c, _ := newHolder(t)
ended := c.now().Add(10 * time.Minute) // by the controller's clock
started := ended.Add(-20 * time.Second)
body := `{"walk":"plan-k","phase":"ended","outcome":"delivered","touches":[{"machine":"novox","modules":["mesh-tools"]}],` +
body := `{"walk":"plan-k","phase":"ended","outcome":"delivered","touches":[{"machine":"node-c","modules":["mesh-tools"]}],` +
`"ended_at":"` + ended.Format(time.RFC3339Nano) + `"`
if said == "both" {
body += `,"started_at":"` + started.Format(time.RFC3339Nano) + `"`
@@ -758,11 +758,11 @@ func TestAStartHeardAfterItsEndOpensNothingWhenTheControllersClockIsAhead(t *tes
}
h.Walked(end)
c.pass(time.Second)
walk(t, h, "plan-k", WalkStarted, "", started, 600, novox("mesh-tools"))
walk(t, h, "plan-k", WalkStarted, "", started, 600, nodeC("mesh-tools"))
if len(h.walks) != 0 {
t.Fatalf("a start heard after its end opened a walk: %+v", h.walks)
}
walk(t, h, "plan-k", WalkStarted, "", ended.Add(time.Minute), 600, novox("mesh-tools"))
walk(t, h, "plan-k", WalkStarted, "", ended.Add(time.Minute), 600, nodeC("mesh-tools"))
if len(h.walks) != 1 {
t.Fatalf("the next walk under its id, started after the end, did not open: %+v", h.walks)
}
@@ -16,7 +16,7 @@ import (
// with them whole, and Details with the fingerprint and where to read it, which the note shows.
func proposalAsk(c *clock, id string) asks.Ask {
a := heldAsk(c, id)
a.Headline = "Set mounts on shanks?"
a.Headline = "Set mounts on node-d?"
a.Options = a.Options[:2]
a.Explanation = "+ shares: media=‹path›\n+ sources: recalbox=‹address›@‹path›\nProposed by an agent at 15:11."
a.Whole = "+ shares: media=/storage/media\n+ sources: recalbox=smb://nas.lan/recalbox@/mnt/recalbox\nProposed by an agent at 15:11."
@@ -46,7 +46,7 @@ func TestAnAsksWholeWordsAreShownWhereTheSenderIsProvenAndMaskedElsewhere(t *tes
!strings.Contains(phone.Note, "Read it whole: settings proposals s-1 at the controller's terminal.") {
t.Errorf("the phone's note lacks the Details:\n%s", phone.Note)
}
if phone.Title != "Set mounts on shanks?" || ticketFor(phone, "Release") == "" || ticketFor(phone, DetailsLabel) != "" {
if phone.Title != "Set mounts on node-d?" || ticketFor(phone, "Release") == "" || ticketFor(phone, DetailsLabel) != "" {
t.Errorf("the phone's message: %q offering %s", phone.Title, labelsOf(phone))
}
// The desk proves nothing: the same record composed for it shows the explanation, its values masked.
@@ -69,7 +69,7 @@ func TestAnAsksWholeWordsAreShownWhereTheSenderIsProvenAndMaskedElsewhere(t *tes
if !ok {
t.Fatalf("the phone's message was not edited: %+v", tg.edits)
}
if edited.Title != "Set mounts on shanks?" || edited.Body == "" ||
if edited.Title != "Set mounts on node-d?" || edited.Body == "" ||
!strings.HasPrefix(edited.Note, "You chose Release at ") || len(edited.Actions) != 0 {
t.Errorf("the edited message: %q\n%s\nactions %+v", edited.Title, edited.Body, edited.Actions)
}
@@ -176,7 +176,7 @@ func TestEveryKeyAndItsWholeValueIsInTheNoteWhereTheSenderIsProven(t *testing.T)
return strings.Join(out, "\n")
}
a := proposalAsk(c, "s-9")
a.Explanation = "Changes 5 setting(s) of mounts on shanks.\n+ a: 1\n+ b: 2\nand 3 more, each whole below: c, d, shares"
a.Explanation = "Changes 5 setting(s) of mounts on node-d.\n+ a: 1\n+ b: 2\nand 3 more, each whole below: c, d, shares"
a.Whole = ""
a.Details = "Fingerprint 1234 5678 9abc def0 1234 5678.\n" + lines(masked)
a.WholeDetails = "Fingerprint 1234 5678 9abc def0 1234 5678.\n" + lines(keys)
+37 -37
View File
@@ -64,58 +64,58 @@ var examples = []example{{
cleared: "Resolved: openrazer delivery no longer waiting, after 5.0 h", openFor: 5 * time.Hour,
}, {
name: "a module unhealthy",
before: "WARNING: openrazer on g14 is not healthy: its unit openrazer-daemon.service failed in the account's own " +
"service manager (exit-code)\nabout: module openrazer.g14 on g14 (module-unhealthy)\nsince: 2026-10-08 " +
"10:40 UTC\nkey: module.openrazer.g14.unhealthy\nmore: conditions show module.openrazer.g14.unhealthy",
key: "module.openrazer.g14.unhealthy", kind: "module-unhealthy", severity: Warning,
subject: map[string]string{"scope": "module", "id": "openrazer.g14", "machine": "g14"},
summary: "openrazer on g14 is not healthy: its unit openrazer-daemon.service failed in the account's own " +
before: "WARNING: openrazer on node-b is not healthy: its unit openrazer-daemon.service failed in the account's own " +
"service manager (exit-code)\nabout: module openrazer.node-b on node-b (module-unhealthy)\nsince: 2026-10-08 " +
"10:40 UTC\nkey: module.openrazer.node-b.unhealthy\nmore: conditions show module.openrazer.node-b.unhealthy",
key: "module.openrazer.node-b.unhealthy", kind: "module-unhealthy", severity: Warning,
subject: map[string]string{"scope": "module", "id": "openrazer.node-b", "machine": "node-b"},
summary: "openrazer on node-b is not healthy: its unit openrazer-daemon.service failed in the account's own " +
"service manager (exit-code)",
headline: "openrazer not working on g14",
needs: "restart its service openrazer-daemon on g14; if it fails again, the details say why.",
explanation: "Needs you: restart its service openrazer-daemon on g14; if it fails again, the details say why. " +
"openrazer on g14 is not healthy: its service openrazer-daemon stopped with an error. It clears as soon as " +
headline: "openrazer not working on node-b",
needs: "restart its service openrazer-daemon on node-b; if it fails again, the details say why.",
explanation: "Needs you: restart its service openrazer-daemon on node-b; if it fails again, the details say why. " +
"openrazer on node-b is not healthy: its service openrazer-daemon stopped with an error. It clears as soon as " +
"it runs again.",
resolved: "openrazer works again on g14", popup: true,
title: "openrazer not working on g14",
body: "Needs you: restart its service openrazer-daemon on g14; if it fails again, the details say why. openrazer " +
"on g14 is not healthy: its service openrazer-daemon stopped with an error. It clears as soon as it runs " +
resolved: "openrazer works again on node-b", popup: true,
title: "openrazer not working on node-b",
body: "Needs you: restart its service openrazer-daemon on node-b; if it fails again, the details say why. openrazer " +
"on node-b is not healthy: its service openrazer-daemon stopped with an error. It clears as soon as it runs " +
"again.\nNo issue tracks this yet.\nSince 12:40.\nNo delivery running.",
offers: "Details", choose: "Details",
cleared: "Resolved: openrazer works again on g14, after 2.0 h", openFor: 2 * time.Hour,
cleared: "Resolved: openrazer works again on node-b, after 2.0 h", openFor: 2 * time.Hour,
}, {
name: "failed units on a machine",
before: "WARNING: shanks's service manager is degraded: 3 failed unit(s) no module places — mnt-recalbox.mount, " +
before: "WARNING: node-d's service manager is degraded: 3 failed unit(s) no module places — mnt-recalbox.mount, " +
"storage-media.mount, greenclip.service (the account's own manager). Each is the machine's own: mend or " +
"remove it there, or have a module place it\nabout: machine shanks on shanks (machine-units)\nsince: " +
"2026-10-08 10:40 UTC\nkey: machine.shanks.units\nmore: conditions show machine.shanks.units",
key: "machine.shanks.units", kind: "machine-units", severity: Warning,
subject: map[string]string{"scope": "machine", "id": "shanks", "machine": "shanks"},
summary: "shanks's service manager is degraded: 3 failed unit(s) no module places — mnt-recalbox.mount, " +
"remove it there, or have a module place it\nabout: machine node-d on node-d (machine-units)\nsince: " +
"2026-10-08 10:40 UTC\nkey: machine.node-d.units\nmore: conditions show machine.node-d.units",
key: "machine.node-d.units", kind: "machine-units", severity: Warning,
subject: map[string]string{"scope": "machine", "id": "node-d", "machine": "node-d"},
summary: "node-d's service manager is degraded: 3 failed unit(s) no module places — mnt-recalbox.mount, " +
"storage-media.mount, greenclip.service (the account's own manager). Each is the machine's own: mend or " +
"remove it there, or have a module place it",
headline: "3 failed services on shanks", needs: "mend or remove them on shanks, or silence this if they do not matter.",
explanation: "Needs you: mend or remove them on shanks, or silence this if they do not matter. On shanks, " +
headline: "3 failed services on node-d", needs: "mend or remove them on node-d, or silence this if they do not matter.",
explanation: "Needs you: mend or remove them on node-d, or silence this if they do not matter. On node-d, " +
"mnt-recalbox (a mount), storage-media (a mount) and greenclip failed. No module manages them, so the mesh " +
"does not repair them.",
actions: []Action{{Label: "Silence for a week", Verb: "mesh-controller.conditions",
Arguments: map[string]string{"silence": "machine.shanks.units", "for": "7d", "why": "", "cause": "operator-answer"}}},
resolved: "No failed services on shanks any more", popup: true,
title: "3 failed services on shanks",
body: "Needs you: mend or remove them on shanks, or silence this if they do not matter. On shanks, mnt-recalbox " +
Arguments: map[string]string{"silence": "machine.node-d.units", "for": "7d", "why": "", "cause": "operator-answer"}}},
resolved: "No failed services on node-d any more", popup: true,
title: "3 failed services on node-d",
body: "Needs you: mend or remove them on node-d, or silence this if they do not matter. On node-d, mnt-recalbox " +
"(a mount), storage-media (a mount) and greenclip failed. No module manages them, so the mesh does not repair " +
"them.\nNo issue tracks this yet.\nSince 12:40.\nNo delivery running.",
offers: "Details", choose: "Details",
cleared: "Resolved: no failed services on shanks any more, after 45 min", openFor: 45 * time.Minute,
cleared: "Resolved: no failed services on node-d any more, after 45 min", openFor: 45 * time.Minute,
}, {
name: "a healer wanted",
before: "WARNING: \"push\" was repaired by hand 35 times in 14 days, the last by g14.node-tools, through the " +
before: "WARNING: \"push\" was repaired by hand 35 times in 14 days, the last by node-b.node-tools, through the " +
"mesh-controller seat: a healer is wanted for it\nabout: mesh hand-acts.push (healer-wanted)\nsince: " +
"2026-10-08 10:40 UTC\nkey: mesh.hand-acts.push.healer-wanted\nmore: conditions show " +
"mesh.hand-acts.push.healer-wanted",
key: "mesh.hand-acts.push.healer-wanted", kind: "healer-wanted", severity: Warning,
subject: map[string]string{"scope": "mesh", "id": "hand-acts.push"},
summary: "\"push\" was repaired by hand 35 times in 14 days, the last by g14.node-tools, through the " +
summary: "\"push\" was repaired by hand 35 times in 14 days, the last by node-b.node-tools, through the " +
"mesh-controller seat: a healer is wanted for it",
headline: "Push keeps being fixed by hand",
explanation: "Nothing for you to do. A person repaired push by hand 35 times in 14 days, so an automatic repair " +
@@ -217,7 +217,7 @@ func TestRealNotificationsReadInOneGlanceAndAreAnsweredFromTheNotification(t *te
}
// The router performs nothing: Details is shown where it was chosen, and the condition's own actions
// are never called from here (novox/hq ADR 0259) — the controller asks for them.
outcome, err := h.Chosen("desktop", choiceOf(tok, "g14", c.now()))
outcome, err := h.Chosen("desktop", choiceOf(tok, "node-b", c.now()))
done := dt.sends[len(dt.sends)-1]
if err != nil || outcome != "shown" || len(asked) != 0 || done.Title != "Details: "+e.headline {
t.Fatalf("details: %q %v %v %q", outcome, err, asked, done.Title)
@@ -230,7 +230,7 @@ func TestRealNotificationsReadInOneGlanceAndAreAnsweredFromTheNotification(t *te
h.Condition(EventCleared, cleared)
c.pass(ClearSettle)
h.Tick()
end := dt.edits["g14=1"]
end := dt.edits["node-b=1"]
if end.Title != e.cleared || end.Body != "" || len(end.Actions) != 0 {
t.Errorf("clearance %q / %q / %v, want one line %q", end.Title, end.Body, end.Actions, e.cleared)
}
@@ -275,7 +275,7 @@ func TestDetailsAreShownOnTheMachineThatAsked(t *testing.T) {
if details.Label != DetailsLabel {
t.Fatalf("%+v", m.Actions)
}
if _, err := h.Chosen("desktop", choiceOf(details.Token, "g14", c.now())); err != nil {
if _, err := h.Chosen("desktop", choiceOf(details.Token, "node-b", c.now())); err != nil {
t.Fatal(err)
}
shown := dt.sends[len(dt.sends)-1]
@@ -283,7 +283,7 @@ func TestDetailsAreShownOnTheMachineThatAsked(t *testing.T) {
!strings.Contains(shown.Body, "Since: 8 Oct 10:40") {
t.Errorf("details: %q / %q", shown.Title, shown.Body)
}
if _, err := h.Chosen("desktop", choiceOf("not-a-token", "g14", c.now())); err == nil {
if _, err := h.Chosen("desktop", choiceOf("not-a-token", "node-b", c.now())); err == nil {
t.Error("an unknown answer was taken")
}
}
@@ -292,11 +292,11 @@ func TestDetailsAreShownOnTheMachineThatAsked(t *testing.T) {
func TestAConditionWithoutPlainWordsIsSaidFromItsSummary(t *testing.T) {
h, tg, _, c, _ := newHolder(t)
h.Zone = func() *time.Location { return time.UTC }
k := Condition{Key: "machine.ace.silent", Scope: "machine", ID: "ace", Kind: "silent", Severity: Urgent,
Summary: "ace has not been heard from: `push ace` sends it again", Raised: c.now()}
k := Condition{Key: "machine.node-a.silent", Scope: "machine", ID: "node-a", Kind: "silent", Severity: Urgent,
Summary: "node-a has not been heard from: `push node-a` sends it again", Raised: c.now()}
h.Condition(EventRaised, k)
m := tg.sends[0]
if m.Title != "Urgent: Silent on ace" || m.Body != "Ace has not been heard from: a command sends it again\nSince 12:00.\nNo delivery running." {
if m.Title != "Urgent: Silent on node-a" || m.Body != "Node-a has not been heard from: a command sends it again\nSince 12:00.\nNo delivery running." {
t.Errorf("%q / %q", m.Title, m.Body)
}
keyless(t, m)
+8 -8
View File
@@ -10,7 +10,7 @@ import (
"time"
)
// respond answers one query: novox.internal A 10.10.0.1 via a compressed name, anything else rcode.
// respond answers one query: node-c.internal A 10.10.0.1 via a compressed name, anything else rcode.
func respond(q []byte, rcode byte) []byte {
end := 12
for q[end] != 0 {
@@ -20,7 +20,7 @@ func respond(q []byte, rcode byte) []byte {
name := strings.ToLower(string(q[13:end]))
r := append([]byte{}, q[:2]...)
r = append(r, 0x81, 0x80|rcode, 0, 1)
answer := rcode == 0 && strings.Contains(name, "novox") && binary.BigEndian.Uint16(q[end+1:]) == 1
answer := rcode == 0 && strings.Contains(name, "node-c") && binary.BigEndian.Uint16(q[end+1:]) == 1
if answer {
r = append(r, 0, 1, 0, 0, 0, 0)
} else {
@@ -80,11 +80,11 @@ func tcpResolver(t *testing.T) string {
// Issue 348: the question that could not be asked during the outage — does the resolver at one address answer?
func TestDNSAsksANamedResolverOverUDPAndTCPAndSaysItsResponseCode(t *testing.T) {
got, err := AskDNS("novox.internal", "A", udpResolver(t, 0), false, time.Second)
got, err := AskDNS("node-c.internal", "A", udpResolver(t, 0), false, time.Second)
if err != nil || got.Rcode != "NOERROR" || got.Protocol != "udp" || len(got.Answers) != 1 || got.Answers[0] != "10.10.0.1" || got.Error != "" {
t.Fatalf("udp: %+v, %v", got, err)
}
got, err = AskDNS("novox.internal", "A", tcpResolver(t), true, time.Second)
got, err = AskDNS("node-c.internal", "A", tcpResolver(t), true, time.Second)
if err != nil || got.Rcode != "NOERROR" || got.Protocol != "tcp" || len(got.Answers) != 1 || got.Answers[0] != "10.10.0.1" {
t.Fatalf("tcp: %+v, %v", got, err)
}
@@ -92,7 +92,7 @@ func TestDNSAsksANamedResolverOverUDPAndTCPAndSaysItsResponseCode(t *testing.T)
if got.Rcode != "NXDOMAIN" || got.Error == "" || len(got.Answers) != 0 {
t.Fatalf("no such name is a result with its code: %+v", got)
}
got, _ = AskDNS("novox.internal", "A", udpResolver(t, 5), false, time.Second)
got, _ = AskDNS("node-c.internal", "A", udpResolver(t, 5), false, time.Second)
if got.Rcode != "REFUSED" || got.Error == "" {
t.Fatalf("a refusal: %+v", got)
}
@@ -102,14 +102,14 @@ func TestDNSSaysARefusedPortAndRefusesANameAsServer(t *testing.T) {
l, _ := net.Listen("tcp", "127.0.0.1:0")
addr := l.Addr().String()
l.Close()
got, err := AskDNS("novox.internal", "A", addr, true, time.Second)
got, err := AskDNS("node-c.internal", "A", addr, true, time.Second)
if err != nil || !strings.Contains(got.Error, "connection refused") {
t.Fatalf("nothing listening is a result naming it: %+v, %v", got, err)
}
if _, err := AskDNS("novox.internal", "A", "resolver.example", false, time.Second); err == nil {
if _, err := AskDNS("node-c.internal", "A", "resolver.example", false, time.Second); err == nil {
t.Fatal("a server given by name was accepted")
}
if _, err := AskDNS("novox.internal", "A", "", true, time.Second); err == nil {
if _, err := AskDNS("node-c.internal", "A", "", true, time.Second); err == nil {
t.Fatal("tcp without a server was accepted")
}
if a, _ := serverAddress("10.10.0.1"); a != "10.10.0.1:53" {
+7 -7
View File
@@ -13,8 +13,8 @@ func dump(now time.Time) string {
fresh := now.Add(-40 * time.Second).Unix()
old := now.Add(-10 * time.Minute).Unix()
return privateKey + "\tPUBkeyOfTheHubAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\t51820\toff\n" +
"PUBkeyAceAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\t(none)\t84.193.151.6:40122\t10.10.0.2/32\t" + itoa(old) + "\t1000\t2000\t25\n" +
"PUBkeyShanksAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\t(none)\t84.193.151.6:51820\t10.10.0.3/32\t" + itoa(fresh) + "\t3000\t4000\toff\n" +
"PUBkeyNodeAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\t(none)\t84.193.151.6:40122\t10.10.0.2/32\t" + itoa(old) + "\t1000\t2000\t25\n" +
"PUBkeyNodeDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\t(none)\t84.193.151.6:51820\t10.10.0.3/32\t" + itoa(fresh) + "\t3000\t4000\toff\n" +
"PUBkeyNewAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\t(none)\t(none)\t10.10.0.9/32\t0\t0\t0\toff\n"
}
@@ -33,12 +33,12 @@ func TestTheDumpIsReadAndNoKeyIsSaid(t *testing.T) {
if tun.ListenPort != 51820 || len(tun.Peers) != 3 {
t.Fatalf("got %+v", tun)
}
ace, shanks, fresh := tun.Peers[0], tun.Peers[1], tun.Peers[2]
if ace.Endpoint != "84.193.151.6:40122" || ace.HandshakeAge != 600 || !ace.Stale || ace.Keepalive != "25s" || ace.Sent != 2000 {
t.Errorf("ace %+v", ace)
nodeA, nodeD, fresh := tun.Peers[0], tun.Peers[1], tun.Peers[2]
if nodeA.Endpoint != "84.193.151.6:40122" || nodeA.HandshakeAge != 600 || !nodeA.Stale || nodeA.Keepalive != "25s" || nodeA.Sent != 2000 {
t.Errorf("nodeA %+v", nodeA)
}
if shanks.Stale || shanks.HandshakeAge != 40 || shanks.Keepalive != "" {
t.Errorf("shanks %+v", shanks)
if nodeD.Stale || nodeD.HandshakeAge != 40 || nodeD.Keepalive != "" {
t.Errorf("nodeD %+v", nodeD)
}
if fresh.Endpoint != "" || fresh.HandshakeAge != -1 || !fresh.Stale || fresh.Handshake != "" {
t.Errorf("a peer never heard from %+v", fresh)
+3 -3
View File
@@ -71,7 +71,7 @@ through an injected runner and a fake root in the tests.
## What changes when it is assigned
| | g14 | shanks |
| | node-b | node-d |
|---|---|---|
| package | none: `nextcloud-client` 34.0.4 is installed, explicitly, from `extra` | the same |
| start | none: dex starts it from the client's own entry (`--background`, in the login session's scope) | none on disk. **The client running now came from the predecessor's window-manager line** (`nextcloud`, a child of i3, since the session of 2026-10-04 16:00). That session began before the `i3` module dropped the line and installed `dex`, so the next login is the first that starts it from its entry |
@@ -83,11 +83,11 @@ The workstations are already in the state this module describes.
Nothing is required on either machine.
- **shanks:** log out and in once, or run `nextcloud_restart`, and the client runs from its one start.
- **node-d:** log out and in once, or run `nextcloud_restart`, and the client runs from its one start.
`nextcloud_check` then answers `ok`.
- **Optional, both:** the client has kept a `nextcloud.cfg.backup_<date>_<version>` from every upgrade
since 2023 (about twenty on each machine). It also keeps a sync log that it no longer writes, at
`~/.config/Nextcloud/Nextcloud_sync.log`, from 2024 on g14 and 2023 on shanks. They are the
`~/.config/Nextcloud/Nextcloud_sync.log`, from 2024 on node-b and 2023 on node-d. They are the
operator's to delete. The module leaves them.
## Leaves as found
@@ -81,16 +81,16 @@ func TestWithoutTheBackupListTheExportsStayAndItIsSaid(t *testing.T) {
}
func TestBackupFromTheSeatsAndTheData(t *testing.T) {
seats := json.RawMessage(`{"answer":{"seats":[{"seat":"node-backup","scope":"node","holders":[{"module":"restic","node":"ace"},{"module":"borg","node":"shanks"}]}]},"ok":true}`)
data := json.RawMessage(`{"answer":{"data":[{"machine":"ace","module":"restic","item":"repository","path":"/storage/restic"},{"machine":"ace","module":"restic","item":"state","path":"/var/lib/restic"},{"machine":"ace","module":"plex","item":"data","path":"/mnt/plex/data"}]},"ok":true}`)
holder, paths, err := BackupFrom("ace", seats, data)
seats := json.RawMessage(`{"answer":{"seats":[{"seat":"node-backup","scope":"node","holders":[{"module":"restic","node":"node-a"},{"module":"borg","node":"node-d"}]}]},"ok":true}`)
data := json.RawMessage(`{"answer":{"data":[{"machine":"node-a","module":"restic","item":"repository","path":"/storage/restic"},{"machine":"node-a","module":"restic","item":"state","path":"/var/lib/restic"},{"machine":"node-a","module":"plex","item":"data","path":"/mnt/plex/data"}]},"ok":true}`)
holder, paths, err := BackupFrom("node-a", seats, data)
if err != nil || holder != "restic" || strings.Join(paths, " ") != "/storage/restic /var/lib/restic" {
t.Errorf("got %q %v %v", holder, paths, err)
}
if _, _, err := BackupFrom("g14", seats, data); err == nil {
if _, _, err := BackupFrom("node-b", seats, data); err == nil {
t.Errorf("a machine whose backup seat nobody holds is not said")
}
if _, _, err := BackupFrom("ace", seats, json.RawMessage(`{"ok":true}`)); err == nil {
if _, _, err := BackupFrom("node-a", seats, json.RawMessage(`{"ok":true}`)); err == nil {
t.Errorf("an answer with no data list reads as an empty list")
}
}
@@ -99,11 +99,11 @@ func TestBackupFromTheSeatsAndTheData(t *testing.T) {
func TestAFailedAskKeepsTheLastBackupPaths(t *testing.T) {
f := newFake(t)
m := f.machine()
m.TakeBackupWith("ace", func(string, any) (json.RawMessage, error) {
m.TakeBackupWith("node-a", func(string, any) (json.RawMessage, error) {
return nil, errors.New("no responders")
})
f.backup("restic", "/storage/restic")
m.TakeBackupWith("ace", func(string, any) (json.RawMessage, error) {
m.TakeBackupWith("node-a", func(string, any) (json.RawMessage, error) {
return nil, errors.New("timeout")
})
var b BackupFromBus
@@ -64,9 +64,9 @@ func TestAFailedReloadIsTriedAgainWhenNothingElseAsks(t *testing.T) {
}
func TestBackupFromRefusesAnAnswerWithoutADataList(t *testing.T) {
seats := json.RawMessage(`{"answer":{"seats":[{"seat":"node-backup","holders":[{"module":"restic","node":"ace"}]}]}}`)
seats := json.RawMessage(`{"answer":{"seats":[{"seat":"node-backup","holders":[{"module":"restic","node":"node-a"}]}]}}`)
for _, data := range []string{`{"answer":{}}`, `{"answer":{"data":null}}`} {
if _, paths, err := BackupFrom("ace", seats, json.RawMessage(data)); err == nil {
if _, paths, err := BackupFrom("node-a", seats, json.RawMessage(data)); err == nil {
t.Errorf("%s was read as the empty list %v: never \"nothing to fear\"", data, paths)
}
}
+1 -1
View File
@@ -8,7 +8,7 @@
//
// **Which broker nodes are the mesh's.** Never guessed: a flow may talk to a broker that has nothing
// to do with this mesh. The step owns the node it creates itself (id `mesh-mqtt-topic`, "mesh:
// mqtt-topic") and the ones an assignment names in settings (`mqtt.brokers`: node ids — how ace's
// mqtt-topic") and the ones an assignment names in settings (`mqtt.brokers`: node ids — how node-a's
// existing broker node, which every one of its MQTT flows uses, is handed over). With none named and
// none made yet, it makes one, so a fresh Node-RED has a broker the flows can pick.
//
+15 -15
View File
@@ -4,7 +4,7 @@
// differs; a node that is not named is never touched; a loopback broker address is refused.
//
// Node-RED and the broker are fakes answering as the real ones do (admin API v2 of nodered/node-red
// 5.0.7, the build ace runs).
// 5.0.7, the build node-a runs).
import { test } from "node:test";
import assert from "node:assert/strict";
@@ -13,10 +13,10 @@ import { MESH_BROKER_ID, namedBrokers, reconcileBrokers, type Binding, type Flow
import type { Probe } from "../mqtt/probe.ts";
const MINTED = "mesh-minted-password";
const binding = (at = "ace.internal"): Binding => ({ provision: "mqtt-topic", from: "ace", at, as: "mesh_ace_nodered", serves: { scheme: "mqtt", port: 1883 } });
const binding = (at = "node-a.internal"): Binding => ({ provision: "mqtt-topic", from: "node-a", at, as: "mesh_node_a_nodered", serves: { scheme: "mqtt", port: 1883 } });
/** ace's flows, reduced: its one broker node (dead, zurag.be:1884) and a node that uses it. */
function aceFlows(): FlowNode[] {
/** node-a's flows, reduced: its one broker node (dead, zurag.be:1884) and a node that uses it. */
function nodeAFlows(): FlowNode[] {
return [
{ id: "2b0aece9c5f3b307", type: "mqtt-broker", name: "MQTT Broker", broker: "zurag.be", port: "1884", clientid: "", usetls: false, protocolVersion: "4", keepalive: "60" },
{ id: "fe0cae96f1e3ae4d", type: "mqtt in", topic: "stat/sonoff_office_light_switch/RESULT", broker: "2b0aece9c5f3b307", z: "t" },
@@ -52,17 +52,17 @@ const marks = (): Marks & { store: Map<string, string> } => {
const store = new Map<string, string>();
return { store, get: async (k) => store.get(k), set: async (k, v) => void store.set(k, v) };
};
const takes: Probe = async (_h, _p, user, pass) => ({ connack: user === "mesh_ace_nodered" && pass === MINTED ? 0 : 5, suback: 0 });
const takes: Probe = async (_h, _p, user, pass) => ({ connack: user === "mesh_node_a_nodered" && pass === MINTED ? 0 : 5, suback: 0 });
test("ace: the named broker node is moved to the bound broker and login; the other broker is not touched", async () => {
const f = fakeNodeRed(aceFlows(), { "2b0aece9c5f3b307": { user: "luffy", password: "old" }, "other-broker": { user: "x", password: "y" } });
test("node-a: the named broker node is moved to the bound broker and login; the other broker is not touched", async () => {
const f = fakeNodeRed(nodeAFlows(), { "2b0aece9c5f3b307": { user: "luffy", password: "old" }, "other-broker": { user: "x", password: "y" } });
const m = marks();
const out = await reconcileBrokers({ nodered: f.nodered, probe: takes, marks: m }, binding(), `${MINTED}\n`, ["2b0aece9c5f3b307"]);
assert.deepEqual(out, [{ what: "broker 2b0aece9c5f3b307", result: "written", fields: ["broker", "port", "user", "password"] }]);
const node = f.flows.find((n) => n.id === "2b0aece9c5f3b307");
assert.deepEqual(node, { ...aceFlows()[0], broker: "ace.internal", port: "1883", usetls: false });
assert.deepEqual(f.creds["2b0aece9c5f3b307"], { user: "mesh_ace_nodered", password: MINTED });
assert.deepEqual(f.flows.find((n) => n.id === "other-broker"), aceFlows()[2]);
assert.deepEqual(node, { ...nodeAFlows()[0], broker: "node-a.internal", port: "1883", usetls: false });
assert.deepEqual(f.creds["2b0aece9c5f3b307"], { user: "mesh_node_a_nodered", password: MINTED });
assert.deepEqual(f.flows.find((n) => n.id === "other-broker"), nodeAFlows()[2]);
assert.deepEqual(f.creds["other-broker"], { user: "x", password: "y" });
// Only the changed node carried credentials in the deploy.
assert.deepEqual(f.deploys[0].filter((n) => n.credentials).map((n) => n.id), ["2b0aece9c5f3b307"]);
@@ -78,19 +78,19 @@ test("fresh: with nothing named, the step makes its own broker node", async () =
const out = await reconcileBrokers({ nodered: f.nodered, probe: takes, marks: marks() }, binding(), MINTED, []);
assert.deepEqual(out, [{ what: `broker ${MESH_BROKER_ID}`, result: "written", fields: ["node"] }]);
assert.equal(f.flows[0].type, "mqtt-broker");
assert.equal(f.flows[0].broker, "ace.internal");
assert.deepEqual(f.creds[MESH_BROKER_ID], { user: "mesh_ace_nodered", password: MINTED });
assert.equal(f.flows[0].broker, "node-a.internal");
assert.deepEqual(f.creds[MESH_BROKER_ID], { user: "mesh_node_a_nodered", password: MINTED });
});
test("a login the broker does not take is never written", async () => {
const f = fakeNodeRed(aceFlows());
const f = fakeNodeRed(nodeAFlows());
const out = await reconcileBrokers({ nodered: f.nodered, probe: async () => ({ connack: 5 }), marks: marks() }, binding(), MINTED, ["2b0aece9c5f3b307"]);
assert.equal(out[0].result, "refused");
assert.equal(f.deploys.length, 0);
});
test("a named node that is not there, or a loopback broker, is refused", async () => {
const f = fakeNodeRed(aceFlows());
const f = fakeNodeRed(nodeAFlows());
const out = await reconcileBrokers({ nodered: f.nodered, probe: takes, marks: marks() }, binding(), MINTED, ["gone"]);
assert.match((out[0] as { problem: string }).problem, /no such node/);
const lo = await reconcileBrokers({ nodered: f.nodered, probe: takes, marks: marks() }, binding("127.0.0.1"), MINTED, []);
@@ -99,7 +99,7 @@ test("a named node that is not there, or a loopback broker, is refused", async (
});
test("a deploy that raced another is read again once", async () => {
const f = fakeNodeRed(aceFlows());
const f = fakeNodeRed(nodeAFlows());
let first = true;
const racing: NodeRed = {
...f.nodered,
+1 -1
View File
@@ -43,7 +43,7 @@ type SecretAnswer struct {
var aName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`)
// aCaller is who asked, as the controller names the bus's caller (novox/hq ADR 0277): a principal such as
// `g14/claude-code`, or an account at a shell, in a name's characters and spaces, at most 80. Nothing else — no line
// `node-b/claude-code`, or an account at a shell, in a name's characters and spaces, at most 80. Nothing else — no line
// break, no markup, no punctuation a sentence is written with — so the prompt's words stay this holder's.
var aCaller = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/@ -]{0,79}$`)
+6 -6
View File
@@ -113,11 +113,11 @@ func TestThePromptIsWrittenHereFromNamesAndSaysTheControllerAsks(t *testing.T) {
t.Errorf("prompt %q, message %q", q.Prompt, q.Message)
}
// Asked by the vault's own account, as the runtime proved it (issue 723): the prompt says the vault asks.
if q, err := secretQuestion("novox.mesh-vault", base); err != nil ||
if q, err := secretQuestion("node-c.mesh-vault", base); err != nil ||
!strings.HasPrefix(q.Message, "The mesh's vault asks for telegram-token") {
t.Errorf("asked by the vault: %q %v", q.Message, err)
}
if q, _ := secretQuestion("novox.mesh-vault-not", base); !strings.HasPrefix(q.Message, "The mesh's controller") {
if q, _ := secretQuestion("node-c.mesh-vault-not", base); !strings.HasPrefix(q.Message, "The mesh's controller") {
t.Errorf("a caller that is not the vault's account is said to be the vault: %q", q.Message)
}
// And who asked the controller, when it says so (novox/hq ADR 0277): a caller's name, in a name's characters.
@@ -125,13 +125,13 @@ func TestThePromptIsWrittenHereFromNamesAndSaysTheControllerAsks(t *testing.T) {
for k, v := range base {
withAsker[k] = v
}
withAsker["asked_by"] = "g14/claude-code"
withAsker["asked_by"] = "node-b/claude-code"
q, err = secretQuestion("unproven", withAsker)
if err != nil || !strings.Contains(q.Message, "It was asked for by g14/claude-code.") {
if err != nil || !strings.Contains(q.Message, "It was asked for by node-b/claude-code.") {
t.Errorf("asked_by: %q %v", q.Message, err)
}
withAsker["asked_by"] = "jochen at a shell on novox"
if q, err = secretQuestion("unproven", withAsker); err != nil || !strings.Contains(q.Message, "It was asked for by jochen at a shell on novox.") {
withAsker["asked_by"] = "operator at a shell on node-c"
if q, err = secretQuestion("unproven", withAsker); err != nil || !strings.Contains(q.Message, "It was asked for by operator at a shell on node-c.") {
t.Errorf("asked_by at a shell: %q %v", q.Message, err)
}
for key, value := range map[string]any{
@@ -45,15 +45,15 @@ func (b *bucketBus) Delete(_ context.Context, bucket, key string) error {
// the entry's values are the controller's composition, copied unchanged, and no vault is asked.
func TestAConsumerOfARouteIsToldItsNamesFromItsPlanEntry(t *testing.T) {
bus := &bucketBus{buckets: map[string]map[string][]byte{}}
plan, _ := json.Marshal(map[string]any{"node": "g14", "module": "gitea", "provisioning": map[string]any{
"provisions": []any{map[string]any{"provision": ProvisionName, "provider": "novox",
"provider-module": ModuleName, "at": "10.7.0.1", "from": "novox", "as": "mesh_g14_gitea",
"names": map[string]any{"name": "git.example.org", "internal-name": "git.novox.internal",
plan, _ := json.Marshal(map[string]any{"node": "node-b", "module": "gitea", "provisioning": map[string]any{
"provisions": []any{map[string]any{"provision": ProvisionName, "provider": "node-c",
"provider-module": ModuleName, "at": "10.7.0.1", "from": "node-c", "as": "mesh_node_b_gitea",
"names": map[string]any{"name": "git.example.org", "internal-name": "git.node-c.internal",
"name-web": "git.example.org"}}}}})
if err := bus.Put(context.Background(), provider.AssignmentsBucket, "g14.gitea", plan); err != nil {
if err := bus.Put(context.Background(), provider.AssignmentsBucket, "node-b.gitea", plan); err != nil {
t.Fatal(err)
}
p := newProvider("novox", bus, t.Logf)
p := newProvider("node-c", bus, t.Logf)
out, err := p.Pass(context.Background())
if err != nil {
t.Fatal(err)
@@ -62,11 +62,11 @@ func TestAConsumerOfARouteIsToldItsNamesFromItsPlanEntry(t *testing.T) {
t.Fatalf("the pass did %+v", out)
}
var e provider.Entry
if err := json.Unmarshal(bus.buckets[provider.BucketOf(ModuleName)]["g14.gitea.route"], &e); err != nil {
if err := json.Unmarshal(bus.buckets[provider.BucketOf(ModuleName)]["node-b.gitea.route"], &e); err != nil {
t.Fatal(err)
}
if e.Values["name-web"] != "git.example.org" || e.Values["name"] != "git.example.org" ||
e.Values["internal-name"] != "git.novox.internal" || e.VaultName != "" || e.User != "" || e.Refused != "" {
e.Values["internal-name"] != "git.node-c.internal" || e.VaultName != "" || e.User != "" || e.Refused != "" {
t.Fatalf("the entry is %+v", e)
}
}
+11 -11
View File
@@ -217,7 +217,7 @@ func TestAQuestionIsShownWithItsTicketsAndRingsAndAWarningIsSilent(t *testing.T)
if f.of("sendMessage")[1].Body["disable_notification"] != true {
t.Error("a clearing rang (D4)")
}
subject, body = work(asks.Send, asks.Message{Handle: "m3", Title: "Urgent: ace silent", Urgent: true, To: "42"})
subject, body = work(asks.Send, asks.Message{Handle: "m3", Title: "Urgent: node-a silent", Urgent: true, To: "42"})
_ = c.Take(subject, body)
if f.of("sendMessage")[2].Body["disable_notification"] != false {
t.Error("an urgent message was silent (D4)")
@@ -243,7 +243,7 @@ func TestALongMessageIsCutAndTheCutSaid(t *testing.T) {
// nor injects. The tap on it is still compared by the plain words, which is what Telegram gives back.
func TestAMessageIsRenderedInHTMLWithItsMarkupEscaped(t *testing.T) {
c, f, s, _ := newChannel(t, goodToken)
subject, body := work(asks.Show, asks.Message{Handle: "q1", Title: "Set mounts on <shanks> & co?",
subject, body := work(asks.Show, asks.Message{Handle: "q1", Title: "Set mounts on <node-d> & co?",
Body: "Changes two settings.\n+ shares: media=/storage/media\n~ sources: a=b (was: c)",
Note: "Asked by mesh-controller at 15:11.\nFingerprint 1234 5678 9abc.",
To: "42", Buttons: []asks.Button{{Label: "Approve", Ticket: "tkt"}}})
@@ -251,7 +251,7 @@ func TestAMessageIsRenderedInHTMLWithItsMarkupEscaped(t *testing.T) {
t.Fatal(err)
}
sent := f.of("sendMessage")
want := "<b>Set mounts on &lt;shanks&gt; &amp; co?</b>\nChanges two settings.\n<code>+ shares: media=/storage/media</code>\n" +
want := "<b>Set mounts on &lt;node-d&gt; &amp; co?</b>\nChanges two settings.\n<code>+ shares: media=/storage/media</code>\n" +
"<code>~ sources: a=b (was: c)</code>\n\n<blockquote expandable>Asked by mesh-controller at 15:11.\nFingerprint 1234 5678 9abc.</blockquote>"
if len(sent) != 1 || sent[0].Body["text"] != want || sent[0].Body["parse_mode"] != "HTML" {
t.Fatalf("sent %v", sent)
@@ -262,7 +262,7 @@ func TestAMessageIsRenderedInHTMLWithItsMarkupEscaped(t *testing.T) {
// Telegram gives the words back plain, the space around the blockquote as it pleases: the tap on them is taken.
c.Handle(Update{UpdateID: 1, Callback: &Callback{ID: "cb1", Data: "tkt", From: User{ID: 42, FirstName: "Jo"},
Message: &Msg{MessageID: shown, Chat: Chat{ID: 42, Type: "private"},
Text: "Set mounts on <shanks> & co?\nChanges two settings.\n+ shares: media=/storage/media\n~ sources: a=b (was: c)\n" +
Text: "Set mounts on <node-d> & co?\nChanges two settings.\n+ shares: media=/storage/media\n~ sources: a=b (was: c)\n" +
"Asked by mesh-controller at 15:11.\nFingerprint 1234 5678 9abc."}}})
if got := s.of(asks.IntakeSubject(asks.Chosen, Kind)); len(got) != 1 {
t.Fatalf("the tap on the rendered message was not taken: %s", got)
@@ -270,20 +270,20 @@ func TestAMessageIsRenderedInHTMLWithItsMarkupEscaped(t *testing.T) {
// Words changed by another holder of the token are still told apart.
c.Handle(Update{UpdateID: 2, Callback: &Callback{ID: "cb2", Data: "tkt", From: User{ID: 42, FirstName: "Jo"},
Message: &Msg{MessageID: shown, Chat: Chat{ID: 42, Type: "private"},
Text: "Set mounts on <shanks> & co?\nChanges three settings.\n+ shares: media=/storage/media\n~ sources: a=b (was: c)\n" +
Text: "Set mounts on <node-d> & co?\nChanges three settings.\n+ shares: media=/storage/media\n~ sources: a=b (was: c)\n" +
"Asked by mesh-controller at 15:11.\nFingerprint 1234 5678 9abc."}}})
if got := s.of(asks.IntakeSubject(asks.Chosen, Kind)); len(got) != 1 {
t.Fatalf("a tap on changed words was taken: %s", got)
}
// The edit says what was chosen, one line with the headline, and is in the same mode.
subject, body = work(asks.Edit, asks.Message{Handle: "q1", Title: "You chose Approve at 15:06 on Telegram · Set mounts on <shanks> & co?",
subject, body = work(asks.Edit, asks.Message{Handle: "q1", Title: "You chose Approve at 15:06 on Telegram · Set mounts on <node-d> & co?",
Note: "Told to mesh-controller, which acts on it itself.", To: "42"})
if err := c.Take(subject, body); err != nil {
t.Fatal(err)
}
ed := f.of("editMessageText")
if len(ed) != 1 || ed[0].Body["parse_mode"] != "HTML" ||
ed[0].Body["text"] != "<b>You chose Approve at 15:06 on Telegram · Set mounts on &lt;shanks&gt; &amp; co?</b>\n\n<blockquote expandable>Told to mesh-controller, which acts on it itself.</blockquote>" {
ed[0].Body["text"] != "<b>You chose Approve at 15:06 on Telegram · Set mounts on &lt;node-d&gt; &amp; co?</b>\n\n<blockquote expandable>Told to mesh-controller, which acts on it itself.</blockquote>" {
t.Errorf("edited %v", ed)
}
// A plain answer in the chat is escaped too.
@@ -657,7 +657,7 @@ func TestAToastIsCutAtTelegramsBound(t *testing.T) {
// loses its buttons. A new handle is a new message, and nothing is deleted.
func TestAMessageSaidAgainUnderItsHandleReplacesTheEarlierOne(t *testing.T) {
c, f, _, st := newChannel(t, goodToken)
subject, body := work(asks.Show, asks.Message{Handle: "m1", Title: "Urgent: ace silent", Urgent: true, To: "42",
subject, body := work(asks.Show, asks.Message{Handle: "m1", Title: "Urgent: node-a silent", Urgent: true, To: "42",
Buttons: []asks.Button{{Label: "Silence for a week", Ticket: "0123456789abcdef0123456789abcdef"}}})
if err := c.Take(subject, body); err != nil {
t.Fatal(err)
@@ -668,7 +668,7 @@ func TestAMessageSaidAgainUnderItsHandleReplacesTheEarlierOne(t *testing.T) {
if len(f.of("deleteMessage")) != 0 {
t.Fatal("a new handle deleted a message")
}
subject, body = work(asks.Show, asks.Message{Handle: "m1", Title: "Still open: ace silent", Urgent: true, To: "42",
subject, body = work(asks.Show, asks.Message{Handle: "m1", Title: "Still open: node-a silent", Urgent: true, To: "42",
Buttons: []asks.Button{{Label: "Silence for a week", Ticket: "fedcba9876543210fedcba9876543210"}}})
if err := c.Take(subject, body); err != nil {
t.Fatal(err)
@@ -686,7 +686,7 @@ func TestAMessageSaidAgainUnderItsHandleReplacesTheEarlierOne(t *testing.T) {
return 400, `{"ok":false,"error_code":400,"description":"Bad Request: message can't be deleted"}`
}
second := st.placed["m1"].ID
subject, body = work(asks.Send, asks.Message{Handle: "m1", Title: "Now urgent: ace silent", Urgent: true, To: "42"})
subject, body = work(asks.Send, asks.Message{Handle: "m1", Title: "Now urgent: node-a silent", Urgent: true, To: "42"})
_ = c.Take(subject, body)
ed := f.of("editMessageText")
if len(ed) != 1 || ed[0].Body["message_id"] != float64(second) || !strings.Contains(ed[0].Body["text"].(string), "Replaced") {
@@ -705,7 +705,7 @@ func TestAReplacementThatCannotBeSentLeavesTheEarlierMessage(t *testing.T) {
f.answer["sendMessage"] = func(map[string]any) (int, string) {
return 400, `{"ok":false,"error_code":400,"description":"Bad Request: chat not found"}`
}
subject, body := work(asks.Send, asks.Message{Handle: "m1", Title: "Still open: ace silent", Urgent: true, To: "42"})
subject, body := work(asks.Send, asks.Message{Handle: "m1", Title: "Still open: node-a silent", Urgent: true, To: "42"})
_ = c.Take(subject, body)
if len(f.of("deleteMessage")) != 0 || st.placed["m1"].ID != 7 {
t.Fatalf("the earlier message was taken away though its replacement was not sent: %+v", f.of("deleteMessage"))
+1 -1
View File
@@ -20,7 +20,7 @@ var adr0343 = asks.Message{Handle: "h-343", To: "42",
"• A deleted file is marked, and a newer change elsewhere outlives the deletion",
Note: "Asked by mesh-knowledge at 16:05. Unanswered by tomorrow 15:55: the record stays proposed; nothing of it is in force.\n" +
"Approve: the record is accepted and its clauses are in force. Decline: the record stays proposed; nothing of it is in force.\n" +
"ADR 0343, proposed by fix-367@g14; the knowledge base's own review approved it. Fingerprint a59f 361b f60e cb5e d237 a837.\n" +
"ADR 0343, proposed by fix-367@node-b; the knowledge base's own review approved it. Fingerprint a59f 361b f60e cb5e d237 a837.\n" +
"Read its 13 clauses with the knowledge base's get, number 343.",
Buttons: []asks.Button{{Label: "Approve", Ticket: "t-approve"}, {Label: "Decline", Ticket: "t-decline"}}}