Compare commits
14
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
863ebd4277 | ||
|
|
2799e95035 | ||
|
|
582f4a082a | ||
|
|
6c7af5c63f | ||
|
|
9d15f3a39e | ||
|
|
725fcd977e | ||
|
|
1cc6a2d759 | ||
|
|
5d3e52219b | ||
|
|
099c176fa9 | ||
|
|
ec3769a8a6 | ||
|
|
8b2abd08cd | ||
|
|
858b4672dd | ||
|
|
3d7ccc8aeb | ||
|
|
b39eaa485a |
@@ -17,7 +17,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// `check-here` is a pull request's merge check run on the machine at hand **exactly as the build seat
|
// `check-here` is a pull request's merge check run on the machine at hand **exactly as the build seat
|
||||||
// runs it** (novox/hq issue 286): the same code (builder.Check), the same ask the controller would make of
|
// runs it** (novox/hq issue 283): the same code (builder.Check), the same ask the controller would make of
|
||||||
// the seat — the gate's modules and judge by the planner's own answer over the facts snapshot, the
|
// the seat — the gate's modules and judge by the planner's own answer over the facts snapshot, the
|
||||||
// repositories beside it at the refs the snapshot says the seat clones them at — in the toolchain image
|
// repositories beside it at the refs the snapshot says the seat clones them at — in the toolchain image
|
||||||
// the mesh holds, as the user the build seat runs as, against a throwaway store and bus of the versions
|
// the mesh holds, as the user the build seat runs as, against a throwaway store and bus of the versions
|
||||||
@@ -126,7 +126,7 @@ func checkHereCommand(ctx context.Context, args []string) error {
|
|||||||
}
|
}
|
||||||
if len(toolchains) == 0 {
|
if len(toolchains) == 0 {
|
||||||
return errors.New("the facts snapshot names no toolchain: it was taken by a controller from before " +
|
return errors.New("the facts snapshot names no toolchain: it was taken by a controller from before " +
|
||||||
"issue 286, and the seat's toolchain cannot be known here")
|
"issue 283, and the seat's toolchain cannot be known here")
|
||||||
}
|
}
|
||||||
beside := map[string]builder.Beside{}
|
beside := map[string]builder.Beside{}
|
||||||
for d, ref := range f.Beside {
|
for d, ref := range f.Beside {
|
||||||
|
|||||||
@@ -406,11 +406,11 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return snapshot.Facts{}, err
|
return snapshot.Facts{}, err
|
||||||
}
|
}
|
||||||
mod := snapshot.Module{Name: e.Manifest.Module, Repository: e.Source.Repository, Path: e.Source.Path,
|
mod := snapshot.Module{Name: e.Manifest.Module, Repository: snapshot.RepositoryName(e.Source.Repository), Path: e.Source.Path,
|
||||||
Commit: e.Source.BuiltFrom, Provided: e.Provided, RollOut: current[e.Manifest.Module].RollOut,
|
Commit: e.Source.BuiltFrom, Provided: e.Provided, RollOut: current[e.Manifest.Module].RollOut,
|
||||||
Manifest: raw}
|
Manifest: raw}
|
||||||
for _, r := range read[e.Manifest.Module] {
|
for _, r := range read[e.Manifest.Module] {
|
||||||
mod.Reads = append(mod.Reads, r.Repository)
|
mod.Reads = append(mod.Reads, snapshot.RepositoryName(r.Repository))
|
||||||
}
|
}
|
||||||
f.Modules = append(f.Modules, mod)
|
f.Modules = append(f.Modules, mod)
|
||||||
if e.Provided || e.Source.Repository == "" {
|
if e.Provided || e.Source.Repository == "" {
|
||||||
@@ -426,7 +426,8 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot
|
|||||||
if commit == "" {
|
if commit == "" {
|
||||||
commit = s.BuiltFrom
|
commit = s.BuiltFrom
|
||||||
}
|
}
|
||||||
f.Sources = append(f.Sources, snapshot.Source{Repository: repository, Commit: commit, Modules: count[repository]})
|
f.Sources = append(f.Sources, snapshot.Source{Repository: snapshot.RepositoryName(repository), Commit: commit,
|
||||||
|
Modules: count[repository]})
|
||||||
}
|
}
|
||||||
for _, e := range edges {
|
for _, e := range edges {
|
||||||
f.Edges = append(f.Edges, snapshot.Edge{From: e.From, To: e.To, Kind: e.Kind})
|
f.Edges = append(f.Edges, snapshot.Edge{From: e.From, To: e.To, Kind: e.Kind})
|
||||||
|
|||||||
@@ -105,6 +105,10 @@ type gateFacts struct {
|
|||||||
// holder is who holds the controller lease, for judging the controller.
|
// holder is who holds the controller lease, for judging the controller.
|
||||||
holder *lease.Holder
|
holder *lease.Holder
|
||||||
holderErr error
|
holderErr error
|
||||||
|
// health is each machine's newest health statement (ADR 0240); a machine absent never stated one.
|
||||||
|
// healthErr is why they could not be read.
|
||||||
|
health map[string]inventory.NodeHealth
|
||||||
|
healthErr error
|
||||||
}
|
}
|
||||||
|
|
||||||
// gatherGateFacts reads what a judging needs, from the store, the bus and this controller's memory. A
|
// gatherGateFacts reads what a judging needs, from the store, the bus and this controller's memory. A
|
||||||
@@ -127,6 +131,9 @@ var gatherGateFacts = func(ctx context.Context, open *stores, component string)
|
|||||||
for _, n := range nodes {
|
for _, n := range nodes {
|
||||||
f.engines[n.Name] = n.HostVersion
|
f.engines[n.Name] = n.HostVersion
|
||||||
}
|
}
|
||||||
|
// What each machine says of its long-running resources (ADR 0240): unreadable is said, never read as
|
||||||
|
// healthy.
|
||||||
|
f.health, f.healthErr = inv.Healths(ctx)
|
||||||
if d := doctorFrom; d != nil {
|
if d := doctorFrom; d != nil {
|
||||||
if d.keeper != nil {
|
if d.keeper != nil {
|
||||||
f.judged = true
|
f.judged = true
|
||||||
@@ -193,7 +200,9 @@ func judgeHealth(module, component string, m catalogue.Manifest, machine string,
|
|||||||
if !onIt {
|
if !onIt {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if c.Subject.Scope == conditions.ScopeMachine || slices.Contains(strings.Split(c.Subject.ID, "."), module) {
|
// A module's own health condition names it whole, its name's dots and all (ADR 0240).
|
||||||
|
ownHealth := c.Subject.Scope == conditions.ScopeModule && c.Subject.ID == module+"."+machine
|
||||||
|
if c.Subject.Scope == conditions.ScopeMachine || ownHealth || slices.Contains(strings.Split(c.Subject.ID, "."), module) {
|
||||||
return healthNotYet, fmt.Sprintf("raised since it was sent: %s — %s", c.Key, c.Summary)
|
return healthNotYet, fmt.Sprintf("raised since it was sent: %s — %s", c.Key, c.Summary)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -241,6 +250,11 @@ func judgeHealth(module, component string, m catalogue.Manifest, machine string,
|
|||||||
return healthNotYet, fmt.Sprintf("the node tools on %s do not serve %s's tools", machine, module)
|
return healthNotYet, fmt.Sprintf("the node tools on %s do not serve %s's tools", machine, module)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// **And what it runs is stated healthy** (ADR 0240 §4): every long-running resource of it on that
|
||||||
|
// machine, in a statement heard since the send. A resource still starting makes the judging wait.
|
||||||
|
if h, why := moduleHealthWord(module, machine, since, f); h != healthGood {
|
||||||
|
return h, why
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return healthGood, ""
|
return healthGood, ""
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -346,6 +346,48 @@ func TestTheHealthDefinitions(t *testing.T) {
|
|||||||
!strings.Contains(why, "first run") {
|
!strings.Contains(why, "first run") {
|
||||||
t.Errorf("a controller not ready is %v (%s)", h, why)
|
t.Errorf("a controller not ready is %v (%s)", h, why)
|
||||||
}
|
}
|
||||||
|
// What the module runs (novox/hq ADR 0240 §4): a judging passes only when every long-running
|
||||||
|
// resource of it on that machine is stated healthy since the send; starting and unhealthy wait.
|
||||||
|
f = applied("anchor")
|
||||||
|
stated := func(state, reason string, heard time.Time) {
|
||||||
|
f.health = map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Contract: 1, SaidAt: heard, HeardAt: heard,
|
||||||
|
Resources: []inventory.ResourceHealth{
|
||||||
|
{Module: "app", Resource: "app.server", Kind: "container", Target: "app-server", State: state, Reason: reason},
|
||||||
|
{Module: "other", Resource: "other.server", Kind: "container", State: link.StateUnhealthy, Reason: "down"}}}}
|
||||||
|
}
|
||||||
|
plain := catalogue.Manifest{Module: "app"}
|
||||||
|
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthGood {
|
||||||
|
t.Errorf("a machine whose engine states no health is judged as before, not %v (%s)", h, why)
|
||||||
|
}
|
||||||
|
stated(link.StateStarting, "", now)
|
||||||
|
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "starting") {
|
||||||
|
t.Errorf("a resource still starting is not yet a pass: %v (%s)", h, why)
|
||||||
|
}
|
||||||
|
stated(link.StateUnhealthy, "restarting", now)
|
||||||
|
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "restarting") {
|
||||||
|
t.Errorf("a resource unhealthy fails the judging: %v (%s)", h, why)
|
||||||
|
}
|
||||||
|
stated(link.StateHealthy, "", since.Add(-time.Minute))
|
||||||
|
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet {
|
||||||
|
t.Errorf("a statement from before the send says nothing of the new build: %v (%s)", h, why)
|
||||||
|
}
|
||||||
|
stated(link.StateHealthy, "", now)
|
||||||
|
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthGood {
|
||||||
|
t.Errorf("every resource of it healthy since the send — another module's state is not its — is %v (%s)", h, why)
|
||||||
|
}
|
||||||
|
f.healthErr = errors.New("the store is away")
|
||||||
|
if h, _ := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet {
|
||||||
|
t.Errorf("health that cannot be read is never read as healthy: %v", h)
|
||||||
|
}
|
||||||
|
// And the condition it raises after the send holds it, as every condition about it does.
|
||||||
|
f.healthErr = nil
|
||||||
|
f.judged = true
|
||||||
|
f.open = []conditions.Condition{{Key: "module.app.anchor.unhealthy", Kind: kindModuleUnhealthy, Subject: conditions.Subject{
|
||||||
|
Scope: conditions.ScopeModule, ID: "app.anchor", Machine: "anchor"}, Raised: now, Summary: "app on anchor is not healthy"}}
|
||||||
|
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "module.app.anchor.unhealthy") {
|
||||||
|
t.Errorf("a module held on its own unhealthy condition is %v (%s)", h, why)
|
||||||
|
}
|
||||||
|
|
||||||
// A machine that refused what it was sent: broken.
|
// A machine that refused what it was sent: broken.
|
||||||
f = applied("anchor")
|
f = applied("anchor")
|
||||||
r := f.reports["anchor"]
|
r := f.reports["anchor"]
|
||||||
|
|||||||
@@ -41,6 +41,11 @@ type handActVerb struct {
|
|||||||
// causeLeakedInLogs is the cause a rotation after a value was printed into a log gives.
|
// causeLeakedInLogs is the cause a rotation after a value was printed into a log gives.
|
||||||
const causeLeakedInLogs = "leaked-in-logs"
|
const causeLeakedInLogs = "leaked-in-logs"
|
||||||
|
|
||||||
|
// causeDrill is the cause of every act `hand-act drill` records, and the one cause `hand-act record`
|
||||||
|
// refuses: a drill has its own verb, so whether an act was a drill is said by the verb a person chose,
|
||||||
|
// never by a word typed into a repair's cause (novox/hq issue 292).
|
||||||
|
const causeDrill = "drill"
|
||||||
|
|
||||||
// handActVerbs is every verb that writes the hand-act log (novox/hq to-be 45 §7). **S15 reads it**:
|
// handActVerbs is every verb that writes the hand-act log (novox/hq to-be 45 §7). **S15 reads it**:
|
||||||
// an act recorded by a verb whose entry names a decision is the mesh working as decided, never a
|
// an act recorded by a verb whose entry names a decision is the mesh working as decided, never a
|
||||||
// repair, and does not count toward `healer-wanted` — whatever cause it gives. A verb not listed, or
|
// repair, and does not count toward `healer-wanted` — whatever cause it gives. A verb not listed, or
|
||||||
@@ -57,8 +62,14 @@ var handActVerbs = []handActVerb{
|
|||||||
{Verb: "broker consumer-reset"},
|
{Verb: "broker consumer-reset"},
|
||||||
// Silencing the same condition twice says the condition, or what it watches, wants mending.
|
// Silencing the same condition twice says the condition, or what it watches, wants mending.
|
||||||
{Verb: "conditions silence"},
|
{Verb: "conditions silence"},
|
||||||
// An act done outside the mesh: the mesh cannot tell a repair from a decision there, so it counts.
|
// An act done outside the mesh: the mesh cannot tell a repair from a decision there, so it counts —
|
||||||
{Verb: "hand-act record"},
|
// except a drill recorded through it before `hand-act drill` existed (2026-10-07). It refuses the
|
||||||
|
// cause since, so no act recorded through it now carries it.
|
||||||
|
{Verb: "hand-act record", Decision: "a drill recorded before `hand-act drill` existed: a person's " +
|
||||||
|
"deliberate test, never a repair", DecidedFor: []string{causeDrill}},
|
||||||
|
// A drill: something broken on purpose to see the mesh raise and clear it. A person's test, never a
|
||||||
|
// repair, however often it is run.
|
||||||
|
{Verb: "hand-act drill", Decision: "a drill is a person's deliberate test of the mesh, never a repair"},
|
||||||
// A person's decisions by design.
|
// A person's decisions by design.
|
||||||
{Verb: "retire approve", Decision: "nothing is retired past its bound without a person (ADR 0230)"},
|
{Verb: "retire approve", Decision: "nothing is retired past its bound without a person (ADR 0230)"},
|
||||||
{Verb: "retire reject", Decision: "keeping a consumer active is a person's word (ADR 0230)"},
|
{Verb: "retire reject", Decision: "keeping a consumer active is a person's word (ADR 0230)"},
|
||||||
@@ -174,6 +185,10 @@ func handActCommand(ctx context.Context, args []string) error {
|
|||||||
return errors.New("hand-act record says the cause too: --cause <word>, the word a second " +
|
return errors.New("hand-act record says the cause too: --cause <word>, the word a second " +
|
||||||
"act for the same reason will use — it is how a repair done twice is found")
|
"act for the same reason will use — it is how a repair done twice is found")
|
||||||
}
|
}
|
||||||
|
if strings.EqualFold(strings.TrimSpace(*f.cause), causeDrill) {
|
||||||
|
return errors.New("a drill is not recorded as a repair: hand-act drill <what was done> --why <text> " +
|
||||||
|
"records it as the person's deliberate test it is, which no healer is wanted for. Nothing was recorded")
|
||||||
|
}
|
||||||
act := link.HandAct{Verb: "hand-act record", Args: []string{what}, Why: strings.TrimSpace(*f.why),
|
act := link.HandAct{Verb: "hand-act record", Args: []string{what}, Why: strings.TrimSpace(*f.why),
|
||||||
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
|
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
|
||||||
return onTheBus(func(conn *nats.Conn) error {
|
return onTheBus(func(conn *nats.Conn) error {
|
||||||
@@ -186,8 +201,12 @@ func handActCommand(ctx context.Context, args []string) error {
|
|||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
if len(args) > 0 && args[0] == "drill" {
|
||||||
|
return handActDrill(ctx, args[1:])
|
||||||
|
}
|
||||||
if len(args) > 0 && args[0] != "list" && !strings.HasPrefix(args[0], "-") {
|
if len(args) > 0 && args[0] != "list" && !strings.HasPrefix(args[0], "-") {
|
||||||
return errors.New("hand-act record <what> --why <text> --cause <word> | hand-acts [--days N] [--json]")
|
return errors.New("hand-act record <what> --why <text> --cause <word> | hand-act drill <what> --why <text> " +
|
||||||
|
"| hand-acts [--days N] [--json]")
|
||||||
}
|
}
|
||||||
if len(args) > 0 && args[0] == "list" {
|
if len(args) > 0 && args[0] == "list" {
|
||||||
args = args[1:]
|
args = args[1:]
|
||||||
@@ -239,6 +258,38 @@ func handActCommand(ctx context.Context, args []string) error {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// handActDrill is `hand-act drill`: an act done on purpose to test the mesh — a module stopped, a
|
||||||
|
// process killed — recorded so the conditions it raises are read as the drill they are. Its cause is
|
||||||
|
// always causeDrill and S15 never counts it (handActVerbs).
|
||||||
|
func handActDrill(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("hand-act drill", flag.ContinueOnError)
|
||||||
|
why := set.String("why", "", "what the drill tests — recorded in the hand-act log (novox/hq to-be 45 §7)")
|
||||||
|
condition := set.String("condition", "", "the key of the condition the drill is meant to raise, if any")
|
||||||
|
positionals, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
what := strings.TrimSpace(strings.Join(positionals, " "))
|
||||||
|
if what == "" {
|
||||||
|
return errors.New("hand-act drill <what was done on purpose> --why <what it tests> [--condition <key>]")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(*why) == "" {
|
||||||
|
return errors.New("a drill says what it tests: --why <text> (recorded in the hand-act log, novox/hq " +
|
||||||
|
"to-be 45 §7). Nothing was recorded")
|
||||||
|
}
|
||||||
|
act := link.HandAct{Verb: "hand-act drill", Args: []string{what}, Why: strings.TrimSpace(*why),
|
||||||
|
Cause: causeDrill, Condition: strings.TrimSpace(*condition)}
|
||||||
|
return onTheBus(func(conn *nats.Conn) error {
|
||||||
|
written, err := link.RecordHandAct(ctx, conn, act)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the drill could not be recorded: %w", err)
|
||||||
|
}
|
||||||
|
fmt.Printf("recorded as %s: %s drilled %q, because %q — a drill, which no healer is wanted for\n",
|
||||||
|
written.ID, written.By, what, written.Why)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
// repairs are the acts that are not a person's decision by design: what S15 counts.
|
// repairs are the acts that are not a person's decision by design: what S15 counts.
|
||||||
func repairs(acts []link.HandAct) []link.HandAct {
|
func repairs(acts []link.HandAct) []link.HandAct {
|
||||||
out := make([]link.HandAct, 0, len(acts))
|
out := make([]link.HandAct, 0, len(acts))
|
||||||
|
|||||||
@@ -92,3 +92,32 @@ func TestDurationsAreSummarisedPerSubject(t *testing.T) {
|
|||||||
t.Fatalf("a minute between words suggests %q", got[1].Suggests)
|
t.Fatalf("a minute between words suggests %q", got[1].Suggests)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **A drill has its own verb** — `hand-act drill`, the seat's `drill` — and `hand-act record` refuses
|
||||||
|
// the cause, so a repair cannot pass for a drill by the word it gives.
|
||||||
|
func TestADrillIsRecordedThroughItsOwnVerb(t *testing.T) {
|
||||||
|
err := handActCommand(context.Background(), []string{"record", "stopped searxng", "--why", "a test", "--cause", "drill"})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "hand-act drill") {
|
||||||
|
t.Errorf("hand-act record --cause drill was not sent to the drill verb: %v", err)
|
||||||
|
}
|
||||||
|
if err := handActCommand(context.Background(), []string{"drill", "stopped searxng"}); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "--why") {
|
||||||
|
t.Errorf("a drill without what it tests: %v", err)
|
||||||
|
}
|
||||||
|
if err := handActCommand(context.Background(), []string{"drill", "--why", "a test"}); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "hand-act drill <what") {
|
||||||
|
t.Errorf("a drill without what was done: %v", err)
|
||||||
|
}
|
||||||
|
argv, err := argvFor("drill", map[string]any{"what": "stopped searxng", "why": "ADR 0240 phase A",
|
||||||
|
"condition": "machine.ace.module.searxng.unhealthy"})
|
||||||
|
if want := "hand-act drill stopped searxng --why ADR 0240 phase A --condition machine.ace.module.searxng.unhealthy"; err != nil ||
|
||||||
|
strings.Join(argv, " ") != want {
|
||||||
|
t.Errorf("the seat's drill: %v %v, want %q", argv, err, want)
|
||||||
|
}
|
||||||
|
if _, err := argvFor("drill", map[string]any{"what": "stopped searxng"}); err == nil {
|
||||||
|
t.Error("the seat's drill without why was not refused")
|
||||||
|
}
|
||||||
|
if repairingCommand([]string{"hand-act", "drill", "x"}) != "hand-act drill" {
|
||||||
|
t.Error("a drill through `command` is not held to why")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -123,7 +123,7 @@ func run() error {
|
|||||||
// The merge gate: every machine of the snapshot composed with a change (novox/hq to-be 45 §9).
|
// The merge gate: every machine of the snapshot composed with a change (novox/hq to-be 45 §9).
|
||||||
case "merge-gate":
|
case "merge-gate":
|
||||||
return mergeGateCommand(ctx, args[1:])
|
return mergeGateCommand(ctx, args[1:])
|
||||||
// A pull request's merge check run here exactly as the build seat runs it (novox/hq issue 286).
|
// A pull request's merge check run here exactly as the build seat runs it (novox/hq issue 283).
|
||||||
case "check-here":
|
case "check-here":
|
||||||
return checkHereCommand(ctx, args[1:])
|
return checkHereCommand(ctx, args[1:])
|
||||||
case "upgrade":
|
case "upgrade":
|
||||||
|
|||||||
@@ -108,7 +108,7 @@ type mergeVerdict struct {
|
|||||||
Facts time.Time `json:"facts"`
|
Facts time.Time `json:"facts"`
|
||||||
Failures []string `json:"failures,omitempty"`
|
Failures []string `json:"failures,omitempty"`
|
||||||
// Errors are what kept the gate from judging: a machine the mesh composes that the gate could not
|
// Errors are what kept the gate from judging: a machine the mesh composes that the gate could not
|
||||||
// raise as it is. Any one makes the verdict an error — never a pass (novox/hq issue 285).
|
// raise as it is. Any one makes the verdict an error — never a pass (novox/hq issue 282).
|
||||||
Errors []string `json:"errors,omitempty"`
|
Errors []string `json:"errors,omitempty"`
|
||||||
Warnings []string `json:"warnings,omitempty"`
|
Warnings []string `json:"warnings,omitempty"`
|
||||||
Notes []string `json:"notes,omitempty"`
|
Notes []string `json:"notes,omitempty"`
|
||||||
@@ -222,7 +222,7 @@ func mergeGateCommand(ctx context.Context, args []string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// errMergeGateCouldNotJudge is the gate's own error: the mesh as it is could not be raised, so nothing
|
// errMergeGateCouldNotJudge is the gate's own error: the mesh as it is could not be raised, so nothing
|
||||||
// the change does to it can be seen. Never a pass (novox/hq issue 285).
|
// the change does to it can be seen. Never a pass (novox/hq issue 282).
|
||||||
var errMergeGateCouldNotJudge = errors.New("the merge gate could not judge the change")
|
var errMergeGateCouldNotJudge = errors.New("the merge gate could not judge the change")
|
||||||
|
|
||||||
// errMergeGateFailed is the gate's own failure: the verdict says why, so the error says nothing more.
|
// errMergeGateFailed is the gate's own failure: the verdict says why, so the error says nothing more.
|
||||||
@@ -351,7 +351,7 @@ func judgeChange(ctx context.Context, in mergeCheckInput) (mergeVerdict, error)
|
|||||||
v.Failures = append(v.Failures, fmt.Sprintf("%s: nothing could be sent to it with this change — %s",
|
v.Failures = append(v.Failures, fmt.Sprintf("%s: nothing could be sent to it with this change — %s",
|
||||||
gm.Described, firstOr(newOnly(gm.Change.Problems, gm.Base.Problems), gm.Change.Problems)))
|
gm.Described, firstOr(newOnly(gm.Change.Problems, gm.Base.Problems), gm.Change.Problems)))
|
||||||
case !gm.Base.Composes && m.Declaration.Composes:
|
case !gm.Base.Composes && m.Declaration.Composes:
|
||||||
// **The mesh composes it and the gate could not raise it as it is** (novox/hq issue 285): a fact
|
// **The mesh composes it and the gate could not raise it as it is** (novox/hq issue 282): a fact
|
||||||
// the snapshot does not carry, or one the gate does not raise. Then the change is judged against a
|
// the snapshot does not carry, or one the gate does not raise. Then the change is judged against a
|
||||||
// machine that is not the mesh's — broken against broken, which passes whatever the change does —
|
// machine that is not the mesh's — broken against broken, which passes whatever the change does —
|
||||||
// so the gate cannot judge, and says so: an error, never a pass.
|
// so the gate cannot judge, and says so: an error, never a pass.
|
||||||
|
|||||||
@@ -323,11 +323,11 @@ func busMesh(t *testing.T) snapshot.Facts {
|
|||||||
return f
|
return f
|
||||||
}
|
}
|
||||||
|
|
||||||
// **Issue 285**: every machine running a module on the bus failed to compose in the gate's store, with
|
// **Issue 282**: every machine running a module on the bus failed to compose in the gate's store, with
|
||||||
// the change and without — the store held the module's credential and no account for it, which
|
// the change and without — the store held the module's credential and no account for it, which
|
||||||
// composition refuses (issue 203) — and the gate passed every change, "0 of 4 compose". The account the
|
// composition refuses (issue 203) — and the gate passed every change, "0 of 4 compose". The account the
|
||||||
// mesh issued is raised with its credential, so the machine composes in the gate as on the mesh.
|
// mesh issued is raised with its credential, so the machine composes in the gate as on the mesh.
|
||||||
func TestIssue285AModuleOnTheBusComposesInTheGate(t *testing.T) {
|
func TestIssue282AModuleOnTheBusComposesInTheGate(t *testing.T) {
|
||||||
f := busMesh(t)
|
f := busMesh(t)
|
||||||
v := gateJudged(t, f, "")
|
v := gateJudged(t, f, "")
|
||||||
if v.Verdict != "pass" {
|
if v.Verdict != "pass" {
|
||||||
@@ -343,10 +343,10 @@ func TestIssue285AModuleOnTheBusComposesInTheGate(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// **Issue 285**: a machine the mesh composes that the gate cannot raise as it is leaves the change judged
|
// **Issue 282**: a machine the mesh composes that the gate cannot raise as it is leaves the change judged
|
||||||
// against a machine that is not the mesh's — broken against broken, which passes whatever the change does.
|
// against a machine that is not the mesh's — broken against broken, which passes whatever the change does.
|
||||||
// That is an error, never a pass.
|
// That is an error, never a pass.
|
||||||
func TestIssue285AMachineTheGateCannotRaiseIsAnErrorNeverAPass(t *testing.T) {
|
func TestIssue282AMachineTheGateCannotRaiseIsAnErrorNeverAPass(t *testing.T) {
|
||||||
f := busMesh(t)
|
f := busMesh(t)
|
||||||
for i := range f.Machines {
|
for i := range f.Machines {
|
||||||
// A fact the snapshot does not carry: the module's credential, gone from the laptop's.
|
// A fact the snapshot does not carry: the module's credential, gone from the laptop's.
|
||||||
@@ -386,7 +386,7 @@ func TestAWithheldPathIsStoodInForByAPath(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// **Issue 286**: a check run by hand clones beside a change what the seat clones — one rule, read by the
|
// **Issue 283**: a check run by hand clones beside a change what the seat clones — one rule, read by the
|
||||||
// controller's ask and by the facts — and finds the repository it checks from its origin.
|
// controller's ask and by the facts — and finds the repository it checks from its origin.
|
||||||
func TestACheckByHandClonesWhatTheSeatClones(t *testing.T) {
|
func TestACheckByHandClonesWhatTheSeatClones(t *testing.T) {
|
||||||
for dir, refs := range map[string]map[string]string{
|
for dir, refs := range map[string]map[string]string{
|
||||||
|
|||||||
@@ -0,0 +1,249 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A module says how it is healthy, and the node-engine judges it (novox/hq ADR 0240, to-be 48 §4 and §5,
|
||||||
|
// Phase A).
|
||||||
|
//
|
||||||
|
// **The node-engine owns every verdict; the controller keeps the last word and raises the condition.** A
|
||||||
|
// machine states, in every report and as an event between reports, the state of every long-running
|
||||||
|
// resource it runs for a module. The controller keeps the newest statement per machine (node_health), and
|
||||||
|
// raises `module.<module>.<machine>.unhealthy` when two statements in a row say a resource of the module
|
||||||
|
// is unhealthy — one statement is listed as unconfirmed, as the self-check does a finding one look can be
|
||||||
|
// wrong about (to-be 45 §4, issue 277) — and clears it on the first that does not. The release gate reads
|
||||||
|
// the stated health: a judging passes a module only when every long-running resource of it on that
|
||||||
|
// machine is stated healthy, so a resource still starting is not yet a pass.
|
||||||
|
//
|
||||||
|
// **An engine older than the judging states nothing**, and its machine's health is not known: never
|
||||||
|
// healthy, never a reason to raise anything, and the gate judges it as it did before.
|
||||||
|
|
||||||
|
// The condition a module's health raises.
|
||||||
|
const (
|
||||||
|
kindModuleUnhealthy = "module-unhealthy"
|
||||||
|
// sourceHealth is what raised it: the machine's own statement.
|
||||||
|
sourceHealth = "health"
|
||||||
|
// moduleUnhealthyUrgentAfter is how long it stands before it is urgent (to-be 48 §4).
|
||||||
|
moduleUnhealthyUrgentAfter = 4 * time.Hour
|
||||||
|
// moduleUnhealthyAfter is how many statements in a row raise it.
|
||||||
|
moduleUnhealthyAfter = 2
|
||||||
|
)
|
||||||
|
|
||||||
|
// healthRefused counts the statements refused as older than the one kept, for the log and a test.
|
||||||
|
var healthRefused atomic.Int64
|
||||||
|
|
||||||
|
// moduleHealth keeps what the machines state, for the link (link.Healths).
|
||||||
|
type moduleHealth struct {
|
||||||
|
inv *inventory.Inventory
|
||||||
|
keeper func() *conditions.Keeper
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m moduleHealth) Stated(ctx context.Context, node string, h link.Health) error {
|
||||||
|
return stateHealth(ctx, m.inv, m.keeper(), node, h, time.Now())
|
||||||
|
}
|
||||||
|
|
||||||
|
// stateHealth keeps one machine's statement and raises or clears its modules' conditions from it. An
|
||||||
|
// older statement than the one kept is refused, by when the engine looked.
|
||||||
|
func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, node string, h link.Health,
|
||||||
|
now time.Time) error {
|
||||||
|
if h.Contract == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
prev, had, err := inv.HealthOf(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if had && h.At.Before(prev.SaidAt) {
|
||||||
|
healthRefused.Add(1)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
unhealthy := map[string][]inventory.ResourceHealth{}
|
||||||
|
resources := make([]inventory.ResourceHealth, 0, len(h.Resources))
|
||||||
|
for _, r := range h.Resources {
|
||||||
|
kept := inventory.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target,
|
||||||
|
State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts}
|
||||||
|
resources = append(resources, kept)
|
||||||
|
if r.State == link.StateUnhealthy && r.Module != "" {
|
||||||
|
unhealthy[r.Module] = append(unhealthy[r.Module], kept)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
streaks := map[string]int{}
|
||||||
|
for module := range unhealthy {
|
||||||
|
streaks[module] = prev.Streaks[module] + 1
|
||||||
|
}
|
||||||
|
stored, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: node, Contract: h.Contract, SaidAt: h.At,
|
||||||
|
HeardAt: now, Resources: resources, Streaks: streaks})
|
||||||
|
if err != nil || !stored {
|
||||||
|
if err == nil {
|
||||||
|
healthRefused.Add(1)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if k == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return judgeModuleHealth(ctx, k, node, unhealthy, streaks, now)
|
||||||
|
}
|
||||||
|
|
||||||
|
// judgeModuleHealth raises a module's condition on a machine on the second statement in a row that says a
|
||||||
|
// resource of it is unhealthy — or on the first while it is already open — and clears every one this
|
||||||
|
// statement no longer says.
|
||||||
|
func judgeModuleHealth(ctx context.Context, k *conditions.Keeper, node string,
|
||||||
|
unhealthy map[string][]inventory.ResourceHealth, streaks map[string]int, now time.Time) error {
|
||||||
|
open, err := k.Open(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
standing := map[string]conditions.Condition{}
|
||||||
|
for _, c := range open {
|
||||||
|
if c.Kind == kindModuleUnhealthy && c.Subject.Machine == node {
|
||||||
|
standing[c.Key] = c
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var problems []string
|
||||||
|
modules := make([]string, 0, len(unhealthy))
|
||||||
|
for m := range unhealthy {
|
||||||
|
modules = append(modules, m)
|
||||||
|
}
|
||||||
|
sort.Strings(modules)
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for _, m := range modules {
|
||||||
|
o := moduleUnhealthyObservation(m, node, unhealthy[m])
|
||||||
|
seen[o.Key()] = true
|
||||||
|
c, isOpen := standing[o.Key()]
|
||||||
|
if streaks[m] < moduleUnhealthyAfter && !isOpen {
|
||||||
|
continue // unconfirmed: one statement can be wrong; `node show` lists it
|
||||||
|
}
|
||||||
|
if isOpen && now.Sub(c.Raised) >= moduleUnhealthyUrgentAfter {
|
||||||
|
o.Severity = conditions.Urgent
|
||||||
|
}
|
||||||
|
if _, err := k.Observe(ctx, o); err != nil {
|
||||||
|
problems = append(problems, err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for key, c := range standing {
|
||||||
|
if seen[key] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
module := strings.TrimSuffix(c.Subject.ID, "."+node)
|
||||||
|
if _, err := k.Clear(ctx, key, fmt.Sprintf("%s says no resource of %s is unhealthy", node, module)); err != nil {
|
||||||
|
problems = append(problems, err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(problems) > 0 {
|
||||||
|
return fmt.Errorf("%s", strings.Join(problems, "; "))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module,
|
||||||
|
// the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence.
|
||||||
|
func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {
|
||||||
|
var words, said []string
|
||||||
|
for _, r := range rs {
|
||||||
|
words = append(words, fmt.Sprintf("its %s %s %s", r.Kind, r.Resource, reasonWords(r)))
|
||||||
|
said = append(said, fmt.Sprintf("%s (%s %s): %s, %d look(s) in a row, %d restart(s) counted, since %s",
|
||||||
|
r.Resource, r.Kind, r.Target, orNotSaid(r.Reason), r.Streak, r.Restarts,
|
||||||
|
r.Since.UTC().Format("2006-01-02 15:04:05 MST")))
|
||||||
|
}
|
||||||
|
return conditions.Observation{Scope: conditions.ScopeModule, ID: module + "." + node, Token: "unhealthy",
|
||||||
|
Kind: kindModuleUnhealthy, Machine: node, Severity: conditions.Warning, Source: sourceHealth,
|
||||||
|
Summary: fmt.Sprintf("%s on %s is not healthy: %s", module, node, strings.Join(words, "; ")),
|
||||||
|
Said: strings.Join(said, "; ")}
|
||||||
|
}
|
||||||
|
|
||||||
|
// reasonWords is why a resource is unhealthy, as a person reads it.
|
||||||
|
func reasonWords(r inventory.ResourceHealth) string {
|
||||||
|
switch r.Reason {
|
||||||
|
case "restarting":
|
||||||
|
return fmt.Sprintf("keeps restarting (%d restart(s) counted)", r.Restarts)
|
||||||
|
case "down":
|
||||||
|
return "is not running"
|
||||||
|
case "":
|
||||||
|
return "is unhealthy"
|
||||||
|
}
|
||||||
|
return "is unhealthy: " + r.Reason
|
||||||
|
}
|
||||||
|
|
||||||
|
func orNotSaid(s string) string {
|
||||||
|
if s == "" {
|
||||||
|
return "no reason said"
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// moduleHealthWord is the gate's reading of a module's stated health on a machine (ADR 0240 §4, ADR 0236
|
||||||
|
// §2 as amended): good when every long-running resource of it is stated healthy in a statement heard since
|
||||||
|
// the send; not yet otherwise, saying which. A machine that never stated health is judged as before.
|
||||||
|
func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (health, string) {
|
||||||
|
if f.healthErr != nil {
|
||||||
|
return healthNotYet, "what " + machine + " says of its resources' health cannot be read: " + firstLine(f.healthErr.Error())
|
||||||
|
}
|
||||||
|
h, states := f.health[machine]
|
||||||
|
if !states {
|
||||||
|
return healthGood, ""
|
||||||
|
}
|
||||||
|
if h.HeardAt.Before(since) {
|
||||||
|
return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module)
|
||||||
|
}
|
||||||
|
for _, r := range h.Resources {
|
||||||
|
if r.Module != module {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
switch r.State {
|
||||||
|
case link.StateHealthy:
|
||||||
|
case link.StateStarting:
|
||||||
|
return healthNotYet, fmt.Sprintf("its %s %s on %s is still starting", r.Kind, r.Resource, machine)
|
||||||
|
case link.StateUnhealthy:
|
||||||
|
return healthNotYet, fmt.Sprintf("its %s %s on %s %s", r.Kind, r.Resource, machine, reasonWords(r))
|
||||||
|
default:
|
||||||
|
return healthNotYet, fmt.Sprintf("its %s %s on %s is %s%s", r.Kind, r.Resource, machine, r.State,
|
||||||
|
reasonAfter(r.Reason))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return healthGood, ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func reasonAfter(s string) string {
|
||||||
|
if s == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return ": " + s
|
||||||
|
}
|
||||||
|
|
||||||
|
// healthLines is what `node show` says of a machine's long-running resources: each with its state and
|
||||||
|
// since when, an unhealthy one said once marked unconfirmed.
|
||||||
|
func healthLines(h inventory.NodeHealth, had bool, now time.Time) []string {
|
||||||
|
if !had {
|
||||||
|
return []string{" its node-engine does not say how what it runs is — it is older than the judging (ADR 0240)"}
|
||||||
|
}
|
||||||
|
if len(h.Resources) == 0 {
|
||||||
|
return []string{fmt.Sprintf(" it runs nothing long-lived for a module (said %s ago)", roughly(now.Sub(h.HeardAt)))}
|
||||||
|
}
|
||||||
|
out := []string{fmt.Sprintf(" what it runs, as it said %s ago:", roughly(now.Sub(h.HeardAt)))}
|
||||||
|
for _, r := range h.Resources {
|
||||||
|
line := fmt.Sprintf(" %-10s %-34s %s %s, since %s", r.State, r.Resource, r.Kind, r.Target,
|
||||||
|
r.Since.Local().Format("2006-01-02 15:04"))
|
||||||
|
if r.Reason != "" {
|
||||||
|
line += " — " + r.Reason
|
||||||
|
}
|
||||||
|
if r.Restarts > 0 {
|
||||||
|
line += fmt.Sprintf(", %d restart(s) counted", r.Restarts)
|
||||||
|
}
|
||||||
|
if r.State == link.StateUnhealthy && h.Streaks[r.Module] < moduleUnhealthyAfter {
|
||||||
|
line += " (unconfirmed: said once)"
|
||||||
|
}
|
||||||
|
out = append(out, line)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,163 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A module's stated health, as the controller keeps it and raises from it (novox/hq ADR 0240, "how it is
|
||||||
|
// checked", rule 4): one unhealthy statement raises nothing and is listed unconfirmed; two raise; a
|
||||||
|
// healthy one clears; a condition from before the send clears at the new build's start; an older
|
||||||
|
// statement is refused; and an engine that states nothing raises nothing.
|
||||||
|
|
||||||
|
var h0 = time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC)
|
||||||
|
|
||||||
|
func aStatement(at time.Time, states ...string) link.Health {
|
||||||
|
h := link.Health{Contract: link.LivenessContract, At: at}
|
||||||
|
for i, s := range states {
|
||||||
|
r := link.ResourceHealth{Module: "letta", Resource: "letta.server", Kind: "container", Target: "letta-server",
|
||||||
|
State: s, Since: at}
|
||||||
|
if i > 0 {
|
||||||
|
r.Module, r.Resource, r.Target = "mqtt", "mqtt.broker", "mosquitto.service"
|
||||||
|
}
|
||||||
|
if s == link.StateUnhealthy {
|
||||||
|
r.Reason, r.Restarts, r.Streak = "restarting", 4, 2
|
||||||
|
}
|
||||||
|
h.Resources = append(h.Resources, r)
|
||||||
|
}
|
||||||
|
return h
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTwoUnhealthyStatementsRaiseTheModulesConditionAndAHealthyOneClearsIt(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
k := conditionsFrom
|
||||||
|
const key = "module.letta.anchor.unhealthy"
|
||||||
|
openKeys := func() []string {
|
||||||
|
t.Helper()
|
||||||
|
list, err := k.Open(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var keys []string
|
||||||
|
for _, c := range list {
|
||||||
|
keys = append(keys, c.Key)
|
||||||
|
}
|
||||||
|
return keys
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0, link.StateHealthy, link.StateHealthy), h0); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// One statement: nothing raised, and `node show` lists it as unconfirmed.
|
||||||
|
if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(time.Minute), link.StateUnhealthy, link.StateHealthy),
|
||||||
|
h0.Add(time.Minute)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if keys := openKeys(); len(keys) != 0 {
|
||||||
|
t.Fatalf("one statement raised %v", keys)
|
||||||
|
}
|
||||||
|
kept, had, err := inv.HealthOf(ctx, "anchor")
|
||||||
|
if err != nil || !had {
|
||||||
|
t.Fatalf("the statement was not kept: %v %v", had, err)
|
||||||
|
}
|
||||||
|
if lines := strings.Join(healthLines(kept, had, h0.Add(time.Minute)), "\n"); !strings.Contains(lines, "unconfirmed") ||
|
||||||
|
!strings.Contains(lines, "letta.server") {
|
||||||
|
t.Fatalf("node show does not list the first statement as unconfirmed:\n%s", lines)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The second in a row raises it — the module's own, never the other module's on the machine.
|
||||||
|
if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(2*time.Minute), link.StateUnhealthy, link.StateHealthy),
|
||||||
|
h0.Add(2*time.Minute)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if keys := openKeys(); len(keys) != 1 || keys[0] != key {
|
||||||
|
t.Fatalf("two statements raised %v, not %s", keys, key)
|
||||||
|
}
|
||||||
|
c, _, _ := k.Get(ctx, key)
|
||||||
|
if c.Severity != conditions.Warning || c.Resolver != conditions.ResolverSelf || c.Subject.Machine != "anchor" ||
|
||||||
|
!strings.Contains(c.Summary, "letta on anchor") || !strings.Contains(c.Summary, "keeps restarting") ||
|
||||||
|
!strings.Contains(c.Evidence[0].Said, "letta-server") {
|
||||||
|
t.Fatalf("the condition does not say it in words with its evidence: %+v", c)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Standing four hours, it is urgent.
|
||||||
|
if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(5*time.Hour), link.StateUnhealthy, link.StateHealthy),
|
||||||
|
time.Now().Add(5*time.Hour)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if c, _, _ := k.Get(ctx, key); c.Severity != conditions.Urgent {
|
||||||
|
t.Fatalf("unhealthy for four hours is still %s", c.Severity)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A new build's start — every start begins in `starting` — clears it: what follows is the new build's.
|
||||||
|
if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(6*time.Hour), link.StateStarting, link.StateHealthy),
|
||||||
|
h0.Add(6*time.Hour)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if keys := openKeys(); len(keys) != 0 {
|
||||||
|
t.Fatalf("the first statement that says no resource is unhealthy did not clear it: %v", keys)
|
||||||
|
}
|
||||||
|
// And the streak starts again: one unhealthy statement after it raises nothing.
|
||||||
|
if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(7*time.Hour), link.StateUnhealthy), h0.Add(7*time.Hour)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if keys := openKeys(); len(keys) != 0 {
|
||||||
|
t.Fatalf("one statement after a clearing raised %v", keys)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnOlderHealthStatementIsRefused(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
before := healthRefused.Load()
|
||||||
|
if err := stateHealth(ctx, inv, nil, "anchor", aStatement(h0.Add(time.Minute), link.StateHealthy), h0); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// An event said before the report that overtook it arrives late.
|
||||||
|
if err := stateHealth(ctx, inv, nil, "anchor", aStatement(h0, link.StateUnhealthy), h0); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
kept, _, err := inv.HealthOf(ctx, "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !kept.SaidAt.Equal(h0.Add(time.Minute)) || kept.Resources[0].State != link.StateHealthy {
|
||||||
|
t.Fatalf("the older statement replaced the newer: %+v", kept)
|
||||||
|
}
|
||||||
|
if healthRefused.Load() != before+1 {
|
||||||
|
t.Fatalf("the refusal was not counted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **An engine older than the judging states nothing**: its reports raise nothing, keep nothing, and the
|
||||||
|
// gate judges its machine as before — never healthy for having said nothing, never unhealthy.
|
||||||
|
func TestAReportWithNoHealthRaisesAndKeepsNothing(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
l := nudgingListener{Enrolment: link.Enrolment{Inventory: open.inventory}}
|
||||||
|
if _, err := l.Heard(ctx, link.Report{Node: "anchor", Declared: "d1", Applied: []string{"letta.server"}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, had, err := open.inventory.HealthOf(ctx, "anchor"); err != nil || had {
|
||||||
|
t.Fatalf("health kept for an engine that said none: %v %v", had, err)
|
||||||
|
}
|
||||||
|
if lines := healthLines(inventory.NodeHealth{}, false, time.Now()); !strings.Contains(lines[0], "older than the judging") {
|
||||||
|
t.Fatalf("node show: %v", lines)
|
||||||
|
}
|
||||||
|
// And one that does, through the report, is kept.
|
||||||
|
h := aStatement(time.Now().UTC(), link.StateHealthy)
|
||||||
|
if _, err := l.Heard(ctx, link.Report{Node: "anchor", Declared: "d1", Health: &h}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if kept, had, err := open.inventory.HealthOf(ctx, "anchor"); err != nil || !had || len(kept.Resources) != 1 {
|
||||||
|
t.Fatalf("the report's health was not kept: %+v %v %v", kept, had, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -537,6 +537,17 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// What it runs for its modules, and how each is (novox/hq ADR 0240): "is it working" answered without
|
||||||
|
// a terminal on the machine.
|
||||||
|
if h, had, err := inv.HealthOf(ctx, name); err != nil {
|
||||||
|
fmt.Printf("\n what it says of what it runs could NOT be read: %v\n", err)
|
||||||
|
} else {
|
||||||
|
fmt.Println()
|
||||||
|
for _, line := range healthLines(h, had, time.Now()) {
|
||||||
|
fmt.Println(line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
held, err := inv.Profile(ctx, name)
|
held, err := inv.Profile(ctx, name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -128,6 +128,10 @@ func serve(ctx context.Context) (err error) {
|
|||||||
stopActing()
|
stopActing()
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
}
|
}
|
||||||
|
// Stopping, whichever way: a verb arriving from here is refused as a handover, so its caller
|
||||||
|
// asks the controller after this one rather than have a command started and killed with this
|
||||||
|
// process (novox/hq issue 289).
|
||||||
|
handingOver.Store(true)
|
||||||
}()
|
}()
|
||||||
defer func() {
|
defer func() {
|
||||||
select {
|
select {
|
||||||
@@ -182,6 +186,9 @@ func serve(ctx context.Context) (err error) {
|
|||||||
if err := server.Watches(standings{keeper: func() *conditions.Keeper { return conditionsFrom }}); err != nil {
|
if err := server.Watches(standings{keeper: func() *conditions.Keeper { return conditionsFrom }}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
// And what each machine says of what it runs between its reports, kept and raised from (novox/hq ADR
|
||||||
|
// 0240): the gate, `node show` and the conditions read it.
|
||||||
|
server.Hears(moduleHealth{inv: inv, keeper: func() *conditions.Keeper { return conditionsFrom }})
|
||||||
// And what they say about consumers the mesh stopped asking for: one waiting for a person is an
|
// And what they say about consumers the mesh stopped asking for: one waiting for a person is an
|
||||||
// urgent condition, and an act asked of a provider some other way is recorded by hand (ADR 0230).
|
// urgent condition, and an act asked of a provider some other way is recorded by hand (ADR 0230).
|
||||||
if err := server.KeepsRetirements(retirements{keeper: func() *conditions.Keeper { return conditionsFrom },
|
if err := server.KeepsRetirements(retirements{keeper: func() *conditions.Keeper { return conditionsFrom },
|
||||||
|
|||||||
@@ -1,9 +1,17 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"slices"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The replays of the controller's incidents (novox/hq to-be 45 §9, M9): each a scripted replay of what
|
// The replays of the controller's incidents (novox/hq to-be 45 §9, M9): each a scripted replay of what
|
||||||
@@ -111,3 +119,170 @@ func TestReplay273AConsumerBesideItsStoreStaysBoundToIt(t *testing.T) {
|
|||||||
t.Fatalf("the resolver was bound to %q; its seat is held on anchor (issue 258)", network)
|
t.Fatalf("the resolver was bound to %q; its seat is held on anchor (issue 258)", network)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **R-crashloop — a module that crash-loops after it applied fails its gate on its first machine** (novox/hq
|
||||||
|
// ADR 0240; research 032 §6). On the home server the agent server restarted about a hundred times while the
|
||||||
|
// mesh read it applied, its tools served and no condition raised — the gate judged a module by what the
|
||||||
|
// mesh saw from outside, and nothing looked at what it ran. The outcome asserted: a build whose container
|
||||||
|
// exits at start never passes its gate on the first machine, is put back there at the bound, and never
|
||||||
|
// reaches the second.
|
||||||
|
//
|
||||||
|
// The machine is heard as a node-engine says it: its report of the apply, then the same account said again
|
||||||
|
// later, each carrying what the engine states of what it runs — `starting` as the apply ends, then the
|
||||||
|
// crash loop. What it states of the crash loop is MESH_REPLAY_STATEMENT when the lab's replay ran the
|
||||||
|
// engine against a real container (mesh-lab replays, R-crashloop), and otherwise what the engine said of
|
||||||
|
// one, kept below. Heard as bytes, so an older controller reads them as it reads any report — this file is
|
||||||
|
// written only with what the controller had before the judging, for the prover to lay over that commit.
|
||||||
|
func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
|
||||||
|
crashLoop := []byte(`{"contract":1,"at":"2026-10-07T00:00:00Z","resources":[{"module":"app","resource":"app.server",` +
|
||||||
|
`"kind":"container","target":"app-server","state":"unhealthy","reason":"restarting","since":"2026-10-07T00:00:00Z",` +
|
||||||
|
`"streak":5,"restarts":2}]}`)
|
||||||
|
if path := os.Getenv("MESH_REPLAY_STATEMENT"); path != "" {
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("the engine's statement of the crash loop: %v", err)
|
||||||
|
}
|
||||||
|
crashLoop = raw
|
||||||
|
}
|
||||||
|
// `null` is an engine that states nothing — older than the judging — and its reports carry no health.
|
||||||
|
var stated map[string]any
|
||||||
|
if err := json.Unmarshal(crashLoop, &stated); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, b := range []inventory.Build{
|
||||||
|
{ID: "build-1", Module: "app", Commit: "c1", Repository: "novox/mesh-catalog", Path: "modules/app",
|
||||||
|
Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)},
|
||||||
|
{ID: "build-2", Module: "app", Commit: "c2", Repository: "novox/mesh-catalog", Path: "modules/app",
|
||||||
|
Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)},
|
||||||
|
} {
|
||||||
|
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: b.Commit})
|
||||||
|
b.Manifest = manifest
|
||||||
|
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
registerAt := func(commit string, asked time.Time) {
|
||||||
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: commit},
|
||||||
|
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: commit,
|
||||||
|
Head: commit, Asked: asked}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
registerAt("c1", time.Now().Add(-2*time.Hour))
|
||||||
|
for _, n := range []string{"anchor", "laptop"} {
|
||||||
|
if _, err := inv.Assign(ctx, n, "app"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"app": "c1"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
registerAt("c2", time.Now().Add(-time.Minute))
|
||||||
|
|
||||||
|
// The machine: what it is sent is applied, and its report says what runs is starting.
|
||||||
|
listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}}
|
||||||
|
sequence := int64(0)
|
||||||
|
say := func(node, digest, state string) {
|
||||||
|
t.Helper()
|
||||||
|
sequence++
|
||||||
|
health := map[string]any{}
|
||||||
|
for k, v := range stated {
|
||||||
|
health[k] = v
|
||||||
|
}
|
||||||
|
health["at"] = time.Now().UTC().Format(time.RFC3339Nano)
|
||||||
|
if state != "" && stated != nil {
|
||||||
|
resources := []any{}
|
||||||
|
for _, r := range stated["resources"].([]any) {
|
||||||
|
kept := map[string]any{}
|
||||||
|
for k, v := range r.(map[string]any) {
|
||||||
|
kept[k] = v
|
||||||
|
}
|
||||||
|
kept["state"], kept["reason"] = state, ""
|
||||||
|
resources = append(resources, kept)
|
||||||
|
}
|
||||||
|
health["resources"] = resources
|
||||||
|
}
|
||||||
|
said := map[string]any{"node": node, "applied": []string{"app.server"}, "declared": digest,
|
||||||
|
"report_sequence": sequence}
|
||||||
|
if stated != nil {
|
||||||
|
said["health"] = health
|
||||||
|
}
|
||||||
|
body, _ := json.Marshal(said)
|
||||||
|
var report link.Report
|
||||||
|
if err := json.Unmarshal(body, &report); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := listener.Heard(ctx, report); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var sent [][]string
|
||||||
|
last := map[string]string{}
|
||||||
|
n := 0
|
||||||
|
wasSend := sendRollout
|
||||||
|
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||||
|
sent = append(sent, append([]string(nil), names...))
|
||||||
|
current, err := open.inventory.CurrentBuilds(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, node := range names {
|
||||||
|
n++
|
||||||
|
digest := fmt.Sprintf("d-%s-%d", node, n)
|
||||||
|
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, map[string]string{"app": current["app"].Commit}); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
last[node] = digest
|
||||||
|
say(node, digest, "starting")
|
||||||
|
}
|
||||||
|
return names, nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { sendRollout = wasSend })
|
||||||
|
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
||||||
|
gateSettle, gateEvery = 0, 0
|
||||||
|
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
|
||||||
|
|
||||||
|
built := time.Now().UTC()
|
||||||
|
plan := inventory.Plan{ID: "plan-crashloop", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
|
||||||
|
Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
|
||||||
|
Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-2"}}}
|
||||||
|
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
advancePlans(ctx, open) // the first machine is sent the new build, and starts it
|
||||||
|
if len(sent) == 0 || !slices.Equal(sent[0], []string{"anchor"}) {
|
||||||
|
t.Fatalf("sent %v, not the first machine first", sent)
|
||||||
|
}
|
||||||
|
advancePlans(ctx, open) // judged while it starts
|
||||||
|
// Its container exits at start, and the runtime restarts it: the machine says so, again and again.
|
||||||
|
for i := 0; i < 6 && len(sent) == 1; i++ {
|
||||||
|
say("anchor", last["anchor"], "")
|
||||||
|
advancePlans(ctx, open)
|
||||||
|
}
|
||||||
|
gateBound = -time.Second // and the bound passes
|
||||||
|
advancePlans(ctx, open)
|
||||||
|
|
||||||
|
p, err := inv.PlanByID(ctx, "plan-crashloop")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
gate := p.Modules["app"].Gate
|
||||||
|
for _, names := range sent {
|
||||||
|
if slices.Contains(names, "laptop") {
|
||||||
|
t.Fatalf("the crash loop passed its gate on anchor and was sent to laptop: sent %v, the gate %+v", sent, gate)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if gate == nil || gate.Verdict != inventory.GateFailed || p.State != inventory.PlanFailed {
|
||||||
|
t.Fatalf("a crash-looping build did not fail its gate on the first machine: the plan is %s (%s), its gate %+v",
|
||||||
|
p.State, p.Note, gate)
|
||||||
|
}
|
||||||
|
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" {
|
||||||
|
t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -456,6 +456,15 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
|||||||
argv = append(argv, "--condition", c)
|
argv = append(argv, "--condition", c)
|
||||||
}
|
}
|
||||||
return argv, nil
|
return argv, nil
|
||||||
|
case "drill":
|
||||||
|
if err := need("what", "why"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
argv := []string{"hand-act", "drill", str("what"), "--why", str("why")}
|
||||||
|
if c := str("condition"); c != "" {
|
||||||
|
argv = append(argv, "--condition", c)
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
case "hand-acts":
|
case "hand-acts":
|
||||||
argv := []string{"hand-acts", "--json"}
|
argv := []string{"hand-acts", "--json"}
|
||||||
if d := str("days"); d != "" {
|
if d := str("days"); d != "" {
|
||||||
@@ -723,6 +732,8 @@ func repairingCommand(argv []string) string {
|
|||||||
return "plans " + argv[1]
|
return "plans " + argv[1]
|
||||||
case argv[0] == "broker" && len(argv) > 1 && argv[1] == "consumer-reset":
|
case argv[0] == "broker" && len(argv) > 1 && argv[1] == "consumer-reset":
|
||||||
return "broker consumer-reset"
|
return "broker consumer-reset"
|
||||||
|
case argv[0] == "hand-act" && len(argv) > 1 && argv[1] == "drill":
|
||||||
|
return "hand-act drill"
|
||||||
case argv[0] == "hand-act":
|
case argv[0] == "hand-act":
|
||||||
return "hand-act record"
|
return "hand-act record"
|
||||||
case argv[0] == "conditions" && len(argv) > 1 && argv[1] == "silence":
|
case argv[0] == "conditions" && len(argv) > 1 && argv[1] == "silence":
|
||||||
@@ -740,12 +751,18 @@ func isWhyFlag(word string) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// runVerb runs this binary with the given command line and gathers what it said.
|
// runVerb runs this binary with the given command line and gathers what it said.
|
||||||
|
//
|
||||||
|
// **This binary is the image this process runs, never the file at the path it started from**
|
||||||
|
// (novox/hq issue 289): the node-engine's witness moves a running build aside when it places the next
|
||||||
|
// one, into a directory only it may enter, and deletes it once the next is proved — while this process
|
||||||
|
// may still be serving. A verb run from the path then failed with "permission denied" for the seconds
|
||||||
|
// the old controller still answered. selfCommand runs what this process is running, wherever its file
|
||||||
|
// went; a verb it still cannot start is refused as a handover, which the caller asks again.
|
||||||
func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
||||||
self, err := os.Executable()
|
if handingOver.Load() {
|
||||||
if err != nil {
|
return verbAnswer{}, fmt.Errorf("%w: this controller is stopping and runs no new command", link.ErrHandingOver)
|
||||||
return verbAnswer{}, err
|
|
||||||
}
|
}
|
||||||
cmd := exec.CommandContext(ctx, self, argv...)
|
cmd := selfCommand(ctx, argv)
|
||||||
// The same environment: the stores' credentials, the bus, the broker — everything a command run
|
// The same environment: the stores' credentials, the bus, the broker — everything a command run
|
||||||
// from a shell in this container would have, because it is that.
|
// from a shell in this container would have, because it is that.
|
||||||
cmd.Env = os.Environ()
|
cmd.Env = os.Environ()
|
||||||
@@ -773,6 +790,12 @@ func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
|||||||
var exit *exec.ExitError
|
var exit *exec.ExitError
|
||||||
if runErr != nil && !errors.As(runErr, &exit) {
|
if runErr != nil && !errors.As(runErr, &exit) {
|
||||||
// Not the command refusing — the command not running at all, which is this process's fault.
|
// Not the command refusing — the command not running at all, which is this process's fault.
|
||||||
|
if errors.Is(runErr, os.ErrPermission) || errors.Is(runErr, os.ErrNotExist) {
|
||||||
|
// Its own image unreachable: a build replaced under a process that has not yet stopped.
|
||||||
|
// Refused as a handover, so the caller asks the controller that follows.
|
||||||
|
return answer, fmt.Errorf("%w: could not run %s from this controller's own build: %v",
|
||||||
|
link.ErrHandingOver, strings.Join(argv, " "), runErr)
|
||||||
|
}
|
||||||
return answer, fmt.Errorf("could not run %s: %w", strings.Join(argv, " "), runErr)
|
return answer, fmt.Errorf("could not run %s: %w", strings.Join(argv, " "), runErr)
|
||||||
}
|
}
|
||||||
return answer, nil
|
return answer, nil
|
||||||
|
|||||||
@@ -0,0 +1,46 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"runtime"
|
||||||
|
"sync/atomic"
|
||||||
|
)
|
||||||
|
|
||||||
|
// startedFrom is the path this process's executable had when it started: what a verb's command line
|
||||||
|
// is named in a process listing, and what is run where the image cannot be named otherwise.
|
||||||
|
var startedFrom, _ = os.Executable()
|
||||||
|
|
||||||
|
// ownImage is how a process names the executable it is running, as long as it runs, wherever the file
|
||||||
|
// has gone since. On Linux the kernel keeps it: /proc/self/exe is the running image itself, not a path,
|
||||||
|
// so it is valid after the file is renamed into a directory this process may not enter, or deleted —
|
||||||
|
// which is what the node-engine's witness does to a build it replaces (novox/hq issue 289). Read in the
|
||||||
|
// child, it names the child's image, which until the exec is this process's.
|
||||||
|
//
|
||||||
|
// os.Executable reads the same link and returns the path it points at *now*: correct at start and
|
||||||
|
// wrong the moment the file moves, which is the fault. A variable so a test can name another.
|
||||||
|
var ownImage = func() string {
|
||||||
|
if runtime.GOOS == "linux" {
|
||||||
|
if _, err := os.Stat("/proc/self/exe"); err == nil {
|
||||||
|
return "/proc/self/exe"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return startedFrom
|
||||||
|
}
|
||||||
|
|
||||||
|
// selfCommand is this binary run with a command line: the image this process runs, named in a process
|
||||||
|
// listing as the path it started from.
|
||||||
|
func selfCommand(ctx context.Context, argv []string) *exec.Cmd {
|
||||||
|
cmd := exec.CommandContext(ctx, ownImage(), argv...)
|
||||||
|
if startedFrom != "" {
|
||||||
|
cmd.Args[0] = startedFrom
|
||||||
|
}
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
// handingOver is set when the serving controller begins to stop — a signal from its supervisor, a lease
|
||||||
|
// lost. From then a verb that would run a command is refused as a handover rather than started and
|
||||||
|
// killed with this process: the caller asks again, and the controller after this one answers
|
||||||
|
// (novox/hq issue 289).
|
||||||
|
var handingOver atomic.Bool
|
||||||
@@ -0,0 +1,177 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The roles a copy of this test binary plays in TestAVerbRunsWhileItsBuildIsMovedOrDeleted.
|
||||||
|
const selfExecRole = "MESH_CONTROLLER_SELFEXEC_ROLE"
|
||||||
|
|
||||||
|
// TestSelfExecServer is a controller standing in, when run as one: it waits until its build has been
|
||||||
|
// moved, then runs a verb as the seat runs one, and prints what came of it as JSON.
|
||||||
|
func TestSelfExecServer(t *testing.T) {
|
||||||
|
if os.Getenv(selfExecRole) != "server" {
|
||||||
|
t.Skip("run by TestAVerbRunsWhileItsBuildIsMovedOrDeleted")
|
||||||
|
}
|
||||||
|
line, _ := bufio.NewReader(os.Stdin).ReadString('\n')
|
||||||
|
if strings.TrimSpace(line) != "go" {
|
||||||
|
t.Fatalf("told %q", line)
|
||||||
|
}
|
||||||
|
if err := os.Setenv(selfExecRole, "verb"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
answer, err := runVerb(t.Context(), []string{"-test.run", "^TestSelfExecVerb$", "-test.v"})
|
||||||
|
said := map[string]any{"ok": answer.OK, "output": answer.Output}
|
||||||
|
if err != nil {
|
||||||
|
said["error"] = err.Error()
|
||||||
|
}
|
||||||
|
body, _ := json.Marshal(said)
|
||||||
|
fmt.Println("ANSWER " + string(body))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSelfExecVerb is the verb, when run as one.
|
||||||
|
func TestSelfExecVerb(t *testing.T) {
|
||||||
|
if os.Getenv(selfExecRole) != "verb" {
|
||||||
|
t.Skip("run by TestSelfExecServer")
|
||||||
|
}
|
||||||
|
fmt.Println("the verb ran")
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A verb runs while the build it was started from is moved where its user may not go, or deleted**
|
||||||
|
// (novox/hq issue 289). The node-engine's witness moves a running controller's build into a directory
|
||||||
|
// only it may enter as it places the next, and deletes it once the next is proved; the controller
|
||||||
|
// still serving in between ran its verbs from the path and answered "permission denied".
|
||||||
|
//
|
||||||
|
// A copy of this test binary is the controller: started from one place, moved into a directory closed
|
||||||
|
// to everyone (or deleted), and only then asked to run a verb.
|
||||||
|
func TestAVerbRunsWhileItsBuildIsMovedOrDeleted(t *testing.T) {
|
||||||
|
if runtime.GOOS != "linux" {
|
||||||
|
t.Skip("the image is named through /proc on Linux only")
|
||||||
|
}
|
||||||
|
self, err := os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, how := range []string{"moved into a closed directory", "deleted"} {
|
||||||
|
t.Run(how, func(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
placed := filepath.Join(root, "mesh-controller", "mesh-controller")
|
||||||
|
if err := os.MkdirAll(filepath.Dir(placed), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
copyFile(t, self, placed)
|
||||||
|
|
||||||
|
server := exec.Command(placed, "-test.run", "^TestSelfExecServer$", "-test.v")
|
||||||
|
server.Env = append(os.Environ(), selfExecRole+"=server")
|
||||||
|
stdin, err := server.StdinPipe()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
stdout, err := server.StdoutPipe()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
server.Stderr = os.Stderr
|
||||||
|
if err := server.Start(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = server.Process.Kill(); _ = server.Wait() })
|
||||||
|
|
||||||
|
// What the witness does to a running build, once the process runs.
|
||||||
|
switch how {
|
||||||
|
case "deleted":
|
||||||
|
if err := os.RemoveAll(filepath.Dir(placed)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
kept := filepath.Join(root, ".witness", "mesh-controller", "previous")
|
||||||
|
if err := os.MkdirAll(filepath.Dir(kept), 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Rename(filepath.Dir(placed), kept); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Chmod(filepath.Join(root, ".witness"), 0); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = os.Chmod(filepath.Join(root, ".witness"), 0o700) })
|
||||||
|
}
|
||||||
|
if _, err := io.WriteString(stdin, "go\n"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
out, _ := io.ReadAll(stdout)
|
||||||
|
_ = server.Wait()
|
||||||
|
var said struct {
|
||||||
|
OK bool `json:"ok"`
|
||||||
|
Output string `json:"output"`
|
||||||
|
Error string `json:"error"`
|
||||||
|
}
|
||||||
|
found := false
|
||||||
|
for _, line := range strings.Split(string(out), "\n") {
|
||||||
|
if rest, ok := strings.CutPrefix(line, "ANSWER "); ok {
|
||||||
|
found = json.Unmarshal([]byte(rest), &said) == nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatalf("the controller standing in answered nothing:\n%s", out)
|
||||||
|
}
|
||||||
|
if said.Error != "" || !said.OK || !strings.Contains(said.Output, "the verb ran") {
|
||||||
|
t.Fatalf("the verb did not run from the controller's own image after its build was %s: %+v", how, said)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A verb the controller cannot start from its own build is refused as a handover — marked so its
|
||||||
|
// caller asks again — never a permission error; and so is one arriving once the controller is stopping.
|
||||||
|
func TestAVerbThatCannotRunIsRefusedAsAHandover(t *testing.T) {
|
||||||
|
closed := filepath.Join(t.TempDir(), "closed")
|
||||||
|
if err := os.MkdirAll(closed, 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
was := ownImage
|
||||||
|
ownImage = func() string { return filepath.Join(closed, "gone", "mesh-controller") }
|
||||||
|
t.Cleanup(func() { ownImage = was })
|
||||||
|
_, err := runVerb(t.Context(), []string{"status"})
|
||||||
|
if !errors.Is(err, link.ErrHandingOver) {
|
||||||
|
t.Fatalf("a build that is not there was answered %v, not a handover", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ownImage = was
|
||||||
|
handingOver.Store(true)
|
||||||
|
t.Cleanup(func() { handingOver.Store(false) })
|
||||||
|
if _, err := runVerb(t.Context(), []string{"-test.run", "^$"}); !errors.Is(err, link.ErrHandingOver) {
|
||||||
|
t.Fatalf("a controller stopping ran a verb: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyFile(t *testing.T, from, to string) {
|
||||||
|
t.Helper()
|
||||||
|
in, err := os.Open(from)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer in.Close()
|
||||||
|
out, err := os.OpenFile(to, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o755)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := io.Copy(out, in); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := out.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -550,3 +550,47 @@ func TestAWalkWaitingFourHoursIsUrgentAndNamesTheWayOn(t *testing.T) {
|
|||||||
t.Fatalf("it does not say how on: %q", got[0].Summary)
|
t.Fatalf("it does not say how on: %q", got[0].Summary)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **A drill is a person's deliberate test, never a repair** — the log of 2026-10-07: two drills of ADR
|
||||||
|
// 0240 recorded through `hand-act record --cause drill` before `hand-act drill` existed raised
|
||||||
|
// `mesh.hand-acts.drill.healer-wanted`. A drill through its own verb never counts, the two recorded
|
||||||
|
// before it clear on the next tick, and the cause word alone on any other verb still counts — whether
|
||||||
|
// an act is a drill is said by the verb chosen, not by a word typed into a repair's cause.
|
||||||
|
func TestADrillIsNoRepairAndItsHealerWantedClears(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC)
|
||||||
|
f := calm(now)
|
||||||
|
f.handActs = []link.HandAct{actOf(now.Add(-26*time.Hour), "hand-act drill", causeDrill),
|
||||||
|
actOf(now.Add(-time.Hour), "hand-act drill", causeDrill), actOf(now.Add(-time.Minute), "hand-act drill", causeDrill)}
|
||||||
|
if got := watchHandActs(f); len(got) != 0 {
|
||||||
|
t.Errorf("drills repeated asked for a healer: %+v", got)
|
||||||
|
}
|
||||||
|
for _, verb := range []string{"push", "conditions silence", "plans close", "a verb nobody listed"} {
|
||||||
|
f := calm(now)
|
||||||
|
f.handActs = []link.HandAct{actOf(now.Add(-26*time.Hour), verb, causeDrill), actOf(now.Add(-time.Hour), verb, causeDrill)}
|
||||||
|
if got := watchHandActs(f); len(got) != 1 {
|
||||||
|
t.Errorf("%s with the cause %q passed for a drill: %+v", verb, causeDrill, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
store := conditions.NewInMemory()
|
||||||
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||||
|
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
||||||
|
defer k.Close(context.Background())
|
||||||
|
before := []conditions.Observation{{Scope: conditions.ScopeMesh, ID: "hand-acts." + causeDrill,
|
||||||
|
Token: "healer-wanted", Kind: "healer-wanted", Severity: conditions.Warning,
|
||||||
|
Summary: "\"drill\" was repaired by hand 2 times in 14 days"}}
|
||||||
|
if err := k.Reconcile(t.Context(), "S15", before); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if open, _ := k.Open(t.Context()); len(open) != 1 {
|
||||||
|
t.Fatalf("the condition of the build before was not open: %+v", open)
|
||||||
|
}
|
||||||
|
f = calm(now)
|
||||||
|
f.handActs = []link.HandAct{actOf(now.Add(-11*time.Hour), "hand-act record", causeDrill),
|
||||||
|
actOf(now.Add(-30*time.Minute), "hand-act record", causeDrill)}
|
||||||
|
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
||||||
|
w.see(t.Context(), f)
|
||||||
|
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||||
|
t.Fatalf("a healer-wanted for two drills stayed open: %+v", open)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"os"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -204,6 +205,15 @@ func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, e
|
|||||||
if news {
|
if news {
|
||||||
l.summary.nudge()
|
l.summary.nudge()
|
||||||
}
|
}
|
||||||
|
// What it says of its long-running resources (novox/hq ADR 0240): kept, and its modules' conditions
|
||||||
|
// raised or cleared from it. Only from an account of the machine the store took.
|
||||||
|
if err == nil && report.Health != nil && report.Superseded == "" && report.Rekey == nil && report.Node != "" &&
|
||||||
|
l.Enrolment.Inventory != nil {
|
||||||
|
if herr := stateHealth(ctx, l.Enrolment.Inventory, conditionsFrom, report.Node, *report.Health, now); herr != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "mesh-controller: could not keep what %s says of its resources' health: %v\n",
|
||||||
|
report.Node, herr)
|
||||||
|
}
|
||||||
|
}
|
||||||
if err == nil && l.open != nil && startedWell(report) {
|
if err == nil && l.open != nil && startedWell(report) {
|
||||||
// Off the report's path: replacing a given value sends the machine, and a report waits for
|
// Off the report's path: replacing a given value sends the machine, and a report waits for
|
||||||
// nothing it caused (novox/hq ADR 0228).
|
// nothing it caused (novox/hq ADR 0228).
|
||||||
|
|||||||
@@ -86,7 +86,9 @@ var WritersTable = []WriterRow{
|
|||||||
Others: "read", Subjects: []string{"mesh.node.*.declare"}, Writes: isController},
|
Others: "read", Subjects: []string{"mesh.node.*.declare"}, Writes: isController},
|
||||||
{State: "a machine's applied state and its report", Writer: "the node-engine's apply queue",
|
{State: "a machine's applied state and its report", Writer: "the node-engine's apply queue",
|
||||||
KeptIn: "the machine; the report on the bus", Others: "the reconcile and a delivery enqueue, never apply",
|
KeptIn: "the machine; the report on the bus", Others: "the reconcile and a delivery enqueue, never apply",
|
||||||
Subjects: []string{"mesh.control.*.report"}, Writes: ownMachine},
|
// And its health statement between reports (novox/hq ADR 0240): the same writer stating the same
|
||||||
|
// machine, inside the grant it already had (`mesh.control.<its own>.>`).
|
||||||
|
Subjects: []string{"mesh.control.*.report", "mesh.control.*.health"}, Writes: ownMachine},
|
||||||
{State: "the controller lease", Writer: "the controller instance holding it", KeptIn: "key-value " + LeaseBucket,
|
{State: "the controller lease", Writer: "the controller instance holding it", KeptIn: "key-value " + LeaseBucket,
|
||||||
Others: "a candidate waits", Subjects: kvOf(LeaseBucket), Writes: isController},
|
Others: "a candidate waits", Subjects: kvOf(LeaseBucket), Writes: isController},
|
||||||
{State: "plans and their tiers", Writer: "controller (lease holder), compare-and-set on the plan's revision",
|
{State: "plans and their tiers", Writer: "controller (lease holder), compare-and-set on the plan's revision",
|
||||||
|
|||||||
@@ -298,6 +298,13 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
|||||||
say("check", "the facts of %s (%s): %d machine(s), the bus at %s, the store at %s", f.Taken.Format(time.RFC3339),
|
say("check", "the facts of %s (%s): %d machine(s), the bus at %s, the store at %s", f.Taken.Format(time.RFC3339),
|
||||||
short(strings.TrimPrefix(digest, "sha256:")), len(f.Machines), f.Versions.Bus, f.Versions.Store)
|
short(strings.TrimPrefix(digest, "sha256:")), len(f.Machines), f.Versions.Bus, f.Versions.Store)
|
||||||
|
|
||||||
|
// **What an earlier delivery of this same ask left is removed first** (novox/hq issue 285): an ask is
|
||||||
|
// redelivered when the holder that took it stopped mid-check — the build agent itself updated by the
|
||||||
|
// rollout it is checking — and its containers, named by the ask's id, are still there. Raising the
|
||||||
|
// store again under that name was refused, and the check said it could not run.
|
||||||
|
if n, err := RemoveContainersOf(ctx, run, spec.ID); err == nil && n > 0 {
|
||||||
|
say("check", "removed %d throwaway container(s) an earlier delivery of this check left", n)
|
||||||
|
}
|
||||||
// The throwaway store and bus, of the versions the mesh runs, removed whatever happens.
|
// The throwaway store and bus, of the versions the mesh runs, removed whatever happens.
|
||||||
defer func() {
|
defer func() {
|
||||||
removing, done := context.WithTimeout(context.Background(), time.Minute)
|
removing, done := context.WithTimeout(context.Background(), time.Minute)
|
||||||
@@ -519,21 +526,24 @@ func gateLayer(ctx context.Context, spec CheckSpec, tree, root, gate, verdictFil
|
|||||||
case "fail":
|
case "fail":
|
||||||
return "fail", said.Summary
|
return "fail", said.Summary
|
||||||
case "error":
|
case "error":
|
||||||
// The gate could not raise the mesh as it is (novox/hq issue 285): said in its own words.
|
// The gate could not raise the mesh as it is (novox/hq issue 282): said in its own words.
|
||||||
return "error", said.Summary
|
return "error", said.Summary
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// The gate could not judge: not the change's fault, and never a pass.
|
// The gate could not judge: not the change's fault, and never a pass.
|
||||||
return "error", "the merge gate could not judge the change: " + lastLine(out.String())
|
return "error", "the merge gate could not judge the change: " + lastLine(out.String())
|
||||||
}
|
}
|
||||||
var said struct {
|
|
||||||
Verdict string `json:"verdict"`
|
|
||||||
Summary string `json:"summary"`
|
|
||||||
}
|
|
||||||
raw, err := os.ReadFile(verdictFile)
|
raw, err := os.ReadFile(verdictFile)
|
||||||
if err != nil || json.Unmarshal(raw, &said) != nil || said.Verdict == "" {
|
if err != nil {
|
||||||
return "error", "the merge gate said no verdict"
|
return "error", "the merge gate said no verdict"
|
||||||
}
|
}
|
||||||
|
said, ok := readGateVerdict(raw)
|
||||||
|
if !ok {
|
||||||
|
return "error", "the merge gate said no verdict"
|
||||||
|
}
|
||||||
|
if verdict, summary, raised := gateRaisedTheMesh(said); !raised {
|
||||||
|
return verdict, summary
|
||||||
|
}
|
||||||
|
|
||||||
// 3. **The replays of what the mesh runs** (to-be 45 §9, M9): mesh-lab's, from its main — reviewed code,
|
// 3. **The replays of what the mesh runs** (to-be 45 §9, M9): mesh-lab's, from its main — reviewed code,
|
||||||
// so given the container runtime the resolver replay raises containers with — against the bus of the
|
// so given the container runtime the resolver replay raises containers with — against the bus of the
|
||||||
@@ -683,6 +693,58 @@ func newProblems(change, base string) []string {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// gateVerdict is what of the gate's verdict the seat reads: the verdict, and every machine — whether it
|
||||||
|
// composes on the mesh and whether it composed in the gate's store without the change.
|
||||||
|
type gateVerdict struct {
|
||||||
|
Verdict string `json:"verdict"`
|
||||||
|
Summary string `json:"summary"`
|
||||||
|
Machines []struct {
|
||||||
|
Described string `json:"described"`
|
||||||
|
Live bool `json:"live-composes"`
|
||||||
|
Base struct {
|
||||||
|
Composes bool `json:"composes"`
|
||||||
|
} `json:"base"`
|
||||||
|
} `json:"machines"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// readGateVerdict reads the verdict the gate wrote, from its first line that opens a JSON document: a
|
||||||
|
// judge from before the verdict was alone on its output printed what composition said ahead of it.
|
||||||
|
func readGateVerdict(raw []byte) (gateVerdict, bool) {
|
||||||
|
var v gateVerdict
|
||||||
|
text := string(raw)
|
||||||
|
if i := strings.Index(text, "\n{"); i >= 0 && !strings.HasPrefix(strings.TrimSpace(text), "{") {
|
||||||
|
text = text[i+1:]
|
||||||
|
}
|
||||||
|
if json.Unmarshal([]byte(text), &v) != nil || v.Verdict == "" {
|
||||||
|
return gateVerdict{}, false
|
||||||
|
}
|
||||||
|
return v, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// gateRaisedTheMesh is the seat's own reading of a verdict that passes (novox/hq issue 285): **a machine
|
||||||
|
// the mesh composes that did not compose in the gate's store without the change makes the gate an error,
|
||||||
|
// never a pass** — the change was judged against a machine that is not the mesh's, broken against broken.
|
||||||
|
// The judge says so itself since issue 285, but the judge is the controller the mesh runs, and one from
|
||||||
|
// before it passed such a verdict; read here too, the rule holds whatever judged. It answers false, with
|
||||||
|
// the verdict to report, when the gate did not raise the mesh.
|
||||||
|
func gateRaisedTheMesh(v gateVerdict) (string, string, bool) {
|
||||||
|
if v.Verdict != "pass" && v.Verdict != "warning" {
|
||||||
|
return "", "", true
|
||||||
|
}
|
||||||
|
var unraised []string
|
||||||
|
for _, m := range v.Machines {
|
||||||
|
if m.Live && !m.Base.Composes {
|
||||||
|
unraised = append(unraised, m.Described)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(unraised) == 0 {
|
||||||
|
return "", "", true
|
||||||
|
}
|
||||||
|
return "error", fmt.Sprintf("the mesh as it is could not be raised, so the change cannot be judged against it: "+
|
||||||
|
"%d of %d machines compose on the mesh and not in the gate (the first: %s) — novox/hq issue 285",
|
||||||
|
len(unraised), len(v.Machines), unraised[0]), false
|
||||||
|
}
|
||||||
|
|
||||||
// gitShow is a file as a ref has it.
|
// gitShow is a file as a ref has it.
|
||||||
func gitShow(ctx context.Context, dir, ref, file string) ([]byte, error) {
|
func gitShow(ctx context.Context, dir, ref, file string) ([]byte, error) {
|
||||||
cmd := exec.CommandContext(ctx, "git", "show", ref+":"+filepath.ToSlash(file))
|
cmd := exec.CommandContext(ctx, "git", "show", ref+":"+filepath.ToSlash(file))
|
||||||
@@ -835,7 +897,7 @@ func (t *tail) String() string {
|
|||||||
|
|
||||||
// whatFailed is the line of a failed script's output that says what failed, for the status a pull request
|
// whatFailed is the line of a failed script's output that says what failed, for the status a pull request
|
||||||
// shows: the first failing test, the first failing package, the files not formatted — a bare "FAIL" or a
|
// shows: the first failing test, the first failing package, the files not formatted — a bare "FAIL" or a
|
||||||
// file's name said nothing a reader could act on (novox/hq issue 286) — and the last line otherwise.
|
// file's name said nothing a reader could act on (novox/hq issue 283) — and the last line otherwise.
|
||||||
func whatFailed(s string) string {
|
func whatFailed(s string) string {
|
||||||
lines := strings.Split(strings.TrimSpace(s), "\n")
|
lines := strings.Split(strings.TrimSpace(s), "\n")
|
||||||
for i, line := range lines {
|
for i, line := range lines {
|
||||||
|
|||||||
@@ -415,7 +415,7 @@ func TestABuildSaysWhetherItsCommitIsOnTheTrunk(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// **Issue 286**: a failed merge-check.sh was said by its last line — a bare "FAIL", or the name of a file
|
// **Issue 283**: a failed merge-check.sh was said by its last line — a bare "FAIL", or the name of a file
|
||||||
// gofmt listed — which named nothing a reader could act on. The status says what failed.
|
// gofmt listed — which named nothing a reader could act on. The status says what failed.
|
||||||
func TestAFailedScriptIsSaidByWhatFailed(t *testing.T) {
|
func TestAFailedScriptIsSaidByWhatFailed(t *testing.T) {
|
||||||
for out, want := range map[string]string{
|
for out, want := range map[string]string{
|
||||||
@@ -429,3 +429,54 @@ func TestAFailedScriptIsSaidByWhatFailed(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **Issue 285**: a judge from before the rule passed "every machine composes with the change as it did
|
||||||
|
// without (0 of 4 compose)". The seat reads the machines itself: a machine the mesh composes that did not
|
||||||
|
// compose in the gate's store makes the gate an error, whatever judged it.
|
||||||
|
func TestTheSeatCallsAGateThatRaisedNoMachineAnError(t *testing.T) {
|
||||||
|
old := "bus users without a credential: controller\n" + `{"verdict":"pass","summary":"every machine composes with the change as it did without (0 of 2 compose)",
|
||||||
|
"machines":[{"described":"the hub","live-composes":true,"base":{"composes":false}},
|
||||||
|
{"described":"a machine","live-composes":true,"base":{"composes":false}}]}`
|
||||||
|
v, ok := readGateVerdict([]byte(old))
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("a verdict after a line of composition's was not read")
|
||||||
|
}
|
||||||
|
verdict, summary, raised := gateRaisedTheMesh(v)
|
||||||
|
if raised || verdict != "error" || !strings.Contains(summary, "2 of 2 machines") || !strings.Contains(summary, "the hub") {
|
||||||
|
t.Errorf("0 of 2 composing is %q %q", verdict, summary)
|
||||||
|
}
|
||||||
|
good := `{"verdict":"pass","summary":"(2 of 2 compose)","machines":[{"described":"the hub","live-composes":true,"base":{"composes":true}},
|
||||||
|
{"described":"a laptop","live-composes":false,"base":{"composes":false}}]}`
|
||||||
|
v, _ = readGateVerdict([]byte(good))
|
||||||
|
if _, _, raised := gateRaisedTheMesh(v); !raised {
|
||||||
|
t.Error("a machine that does not compose on the mesh either was read as the gate's failure")
|
||||||
|
}
|
||||||
|
failed := `{"verdict":"fail","summary":"x","machines":[{"described":"the hub","live-composes":true,"base":{"composes":false}}]}`
|
||||||
|
v, _ = readGateVerdict([]byte(failed))
|
||||||
|
if _, _, raised := gateRaisedTheMesh(v); !raised {
|
||||||
|
t.Error("a failing verdict is the change's, and stands")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Issue 285**: an ask redelivered after its holder stopped mid-check found its own throwaway store still
|
||||||
|
// there under its name, and the check said it could not run. What an earlier delivery left goes first.
|
||||||
|
func TestARedeliveredCheckRemovesWhatItsEarlierDeliveryLeft(t *testing.T) {
|
||||||
|
registry := checkEnvironment(t)
|
||||||
|
repo, head := aCheckedRepository(t, map[string]string{CheckScript: "echo checked\n"})
|
||||||
|
id := fmt.Sprintf("check-again-%d", time.Now().UnixNano())
|
||||||
|
if out, err := exec.Command("docker", "run", "-d", "--rm", "--label", BuildLabel+"="+id, "--name", id+"-store",
|
||||||
|
"postgres:17-alpine", "sleep", "300").CombinedOutput(); err != nil {
|
||||||
|
t.Skipf("no container for the earlier delivery: %v %s", err, out)
|
||||||
|
}
|
||||||
|
v, err := Check(t.Context(), Command, CheckSpec{ID: id, Repository: repo, Ref: head, Owner: "novox",
|
||||||
|
Repo: "hq", Number: 7, Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("a redelivered check could not run: %v", err)
|
||||||
|
}
|
||||||
|
if v.Repo == nil || v.Repo.Verdict != "pass" {
|
||||||
|
t.Errorf("the repository's check answered %+v", v.Repo)
|
||||||
|
}
|
||||||
|
if left := labelled(id); len(left) > 0 {
|
||||||
|
t.Errorf("the check left %d container(s) behind", len(left))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -262,6 +262,14 @@ var ControllerVerbs = []Verb{
|
|||||||
"cause": "the cause in a word, or a condition's kind — the word a second act for the same reason uses",
|
"cause": "the cause in a word, or a condition's kind — the word a second act for the same reason uses",
|
||||||
"condition": "the key of the condition it addressed, if any (optional)",
|
"condition": "the key of the condition it addressed, if any (optional)",
|
||||||
}, []string{"what", "why", "cause"})},
|
}, []string{"what", "why", "cause"})},
|
||||||
|
{Name: "drill", Description: "Record a drill — something broken on purpose to see the mesh raise and clear it: " +
|
||||||
|
"a module stopped, a process killed — with what it tests, in the hand-act log. A drill is a person's " +
|
||||||
|
"deliberate test, never a repair: no healer is wanted for it however often it is run (S15).",
|
||||||
|
Input: schema(map[string]string{
|
||||||
|
"what": "what was done on purpose, in a line",
|
||||||
|
"why": "what the drill tests",
|
||||||
|
"condition": "the key of the condition the drill is meant to raise, if any (optional)",
|
||||||
|
}, []string{"what", "why"})},
|
||||||
{Name: "hand-acts", Description: "What was done by hand lately — pushes, plans ended, consumers re-made, acts " +
|
{Name: "hand-acts", Description: "What was done by hand lately — pushes, plans ended, consumers re-made, acts " +
|
||||||
"recorded — who, why and the cause of each, and which causes repeat: each repeat is a healer the mesh lacks.",
|
"recorded — who, why and the cause of each, and which causes repeat: each repeat is a healer the mesh lacks.",
|
||||||
Input: schema(map[string]string{"days": "how many days back (default 14)"}, nil)},
|
Input: schema(map[string]string{"days": "how many days back (default 14)"}, nil)},
|
||||||
|
|||||||
@@ -46,11 +46,14 @@ const (
|
|||||||
ScopeMesh = "mesh"
|
ScopeMesh = "mesh"
|
||||||
// ScopeDelivery is a delivery mesh-delivery owns, by its id (novox/hq ADR 0239).
|
// ScopeDelivery is a delivery mesh-delivery owns, by its id (novox/hq ADR 0239).
|
||||||
ScopeDelivery = "delivery"
|
ScopeDelivery = "delivery"
|
||||||
|
// ScopeModule is a module on a machine, by `<module>.<machine>`: what it runs is not healthy there
|
||||||
|
// (novox/hq ADR 0240).
|
||||||
|
ScopeModule = "module"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Scopes is every scope, in the order a person reads them.
|
// Scopes is every scope, in the order a person reads them.
|
||||||
var Scopes = []string{ScopeMachine, ScopePlan, ScopeCall, ScopeBuild, ScopeMerge, ScopeProvider,
|
var Scopes = []string{ScopeMachine, ScopePlan, ScopeCall, ScopeBuild, ScopeMerge, ScopeProvider,
|
||||||
ScopeSeat, ScopeBus, ScopeCore, ScopeProbe, ScopeMesh, ScopeDelivery}
|
ScopeSeat, ScopeBus, ScopeCore, ScopeProbe, ScopeMesh, ScopeDelivery, ScopeModule}
|
||||||
|
|
||||||
// Who resolves a condition.
|
// Who resolves a condition.
|
||||||
const (
|
const (
|
||||||
|
|||||||
@@ -64,7 +64,7 @@ type Facts struct {
|
|||||||
Edges []Edge `json:"edges,omitempty"`
|
Edges []Edge `json:"edges,omitempty"`
|
||||||
// Beside is the ref each repository is cloned at beside a merge check, by the directory it is found
|
// Beside is the ref each repository is cloned at beside a merge check, by the directory it is found
|
||||||
// under — the commits the mesh runs, the catalogue's main — so a check run by hand reads the siblings
|
// under — the commits the mesh runs, the catalogue's main — so a check run by hand reads the siblings
|
||||||
// the build seat reads (novox/hq issue 286).
|
// the build seat reads (novox/hq issue 283).
|
||||||
Beside map[string]string `json:"beside,omitempty"`
|
Beside map[string]string `json:"beside,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -85,7 +85,7 @@ type Versions struct {
|
|||||||
// Toolchains are the toolchain images a merge check runs in, by language, as the mesh holds them —
|
// Toolchains are the toolchain images a merge check runs in, by language, as the mesh holds them —
|
||||||
// with no address: the artifact store the snapshot is read from is where they are pulled from. What
|
// with no address: the artifact store the snapshot is read from is where they are pulled from. What
|
||||||
// a repository's merge-check.sh runs in on the build seat, and so what it must run in anywhere else
|
// a repository's merge-check.sh runs in on the build seat, and so what it must run in anywhere else
|
||||||
// (novox/hq issue 286): two releases of one compiler disagree, down to how gofmt lays out a file.
|
// (novox/hq issue 283): two releases of one compiler disagree, down to how gofmt lays out a file.
|
||||||
Toolchains map[string]string `json:"toolchains,omitempty"`
|
Toolchains map[string]string `json:"toolchains,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -172,3 +172,18 @@ func TestAWithheldPathStaysAPath(t *testing.T) {
|
|||||||
t.Errorf("a key became %q", got)
|
t.Errorf("a key became %q", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **Issue 288**: a repository was kept as the URL it was cloned from, the forge's address with it.
|
||||||
|
func TestARepositoryIsNamedWithoutTheForge(t *testing.T) {
|
||||||
|
for in, want := range map[string]string{
|
||||||
|
"http://forge.internal:3000/owner/repo.git": "owner/repo",
|
||||||
|
"ssh://git@forge.internal:222/owner/repo": "owner/repo",
|
||||||
|
"git@forge.internal:owner/repo.git": "owner/repo",
|
||||||
|
"owner/repo": "owner/repo",
|
||||||
|
"": "",
|
||||||
|
} {
|
||||||
|
if got := RepositoryName(in); got != want {
|
||||||
|
t.Errorf("%q is named %q, not %q", in, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -321,3 +321,31 @@ func standIn(run string) string {
|
|||||||
}
|
}
|
||||||
return run
|
return run
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RepositoryName is a repository as `owner/repository`, without the forge's address it was cloned from:
|
||||||
|
// http://forge.internal:3000/owner/repo.git → owner/repo (novox/hq issue 288). What a check matches a
|
||||||
|
// pull request's repository by is its owner and name, never the forge's address, so nothing is lost.
|
||||||
|
func RepositoryName(repository string) string {
|
||||||
|
r := strings.TrimSuffix(strings.TrimSuffix(strings.TrimSpace(repository), "/"), ".git")
|
||||||
|
if r == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
if _, rest, found := strings.Cut(r, "://"); found {
|
||||||
|
r = rest
|
||||||
|
if _, path, found := strings.Cut(r, "/"); found {
|
||||||
|
r = path
|
||||||
|
} else {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
} else if at := strings.Index(r, "@"); at >= 0 {
|
||||||
|
// scp-like: git@forge:owner/repo
|
||||||
|
if _, path, found := strings.Cut(r[at+1:], ":"); found {
|
||||||
|
r = path
|
||||||
|
}
|
||||||
|
}
|
||||||
|
parts := strings.Split(strings.Trim(r, "/"), "/")
|
||||||
|
if len(parts) >= 2 {
|
||||||
|
return parts[len(parts)-2] + "/" + parts[len(parts)-1]
|
||||||
|
}
|
||||||
|
return parts[len(parts)-1]
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,120 @@
|
|||||||
|
package inventory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What each machine says of its long-running resources (novox/hq ADR 0240, to-be 48 §4): the newest
|
||||||
|
// statement per machine, and per module how many statements in a row said a resource of it was unhealthy.
|
||||||
|
|
||||||
|
// ResourceHealth is one long-running resource's state as the node-engine said it.
|
||||||
|
type ResourceHealth struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
Resource string `json:"resource"`
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Target string `json:"target"`
|
||||||
|
State string `json:"state"`
|
||||||
|
Reason string `json:"reason,omitempty"`
|
||||||
|
Since time.Time `json:"since"`
|
||||||
|
Streak int `json:"streak,omitempty"`
|
||||||
|
Restarts int `json:"restarts,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// NodeHealth is a machine's newest statement, as kept.
|
||||||
|
type NodeHealth struct {
|
||||||
|
Node string
|
||||||
|
Contract int
|
||||||
|
// SaidAt is when the engine looked (the machine's clock); HeardAt when this controller heard it.
|
||||||
|
SaidAt time.Time
|
||||||
|
HeardAt time.Time
|
||||||
|
Resources []ResourceHealth
|
||||||
|
// Streaks is, per module, how many statements in a row said a resource of it was unhealthy.
|
||||||
|
Streaks map[string]int
|
||||||
|
}
|
||||||
|
|
||||||
|
// HealthOf is a machine's newest statement; false when its node-engine has never stated one.
|
||||||
|
func (i *Inventory) HealthOf(ctx context.Context, nodeName string) (NodeHealth, bool, error) {
|
||||||
|
all, err := i.healths(ctx, nodeName)
|
||||||
|
if err != nil {
|
||||||
|
return NodeHealth{}, false, err
|
||||||
|
}
|
||||||
|
h, ok := all[nodeName]
|
||||||
|
return h, ok, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Healths is every machine's newest statement, by machine. A machine absent never stated one: its
|
||||||
|
// node-engine is older than the judging, and its health is not known.
|
||||||
|
func (i *Inventory) Healths(ctx context.Context) (map[string]NodeHealth, error) {
|
||||||
|
return i.healths(ctx, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHealth, error) {
|
||||||
|
rows, err := i.store.Pool().Query(ctx,
|
||||||
|
`select n.name, h.contract, h.said_at, h.heard_at, h.resources, h.streaks
|
||||||
|
from node_health h join node n on n.id = h.node
|
||||||
|
where $1 = '' or n.name = $1`, only)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
out := map[string]NodeHealth{}
|
||||||
|
for rows.Next() {
|
||||||
|
var h NodeHealth
|
||||||
|
var resources, streaks []byte
|
||||||
|
if err := rows.Scan(&h.Node, &h.Contract, &h.SaidAt, &h.HeardAt, &resources, &streaks); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(resources, &h.Resources); err != nil {
|
||||||
|
return nil, fmt.Errorf("%s's health cannot be read: %w", h.Node, err)
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(streaks, &h.Streaks); err != nil {
|
||||||
|
return nil, fmt.Errorf("%s's health cannot be read: %w", h.Node, err)
|
||||||
|
}
|
||||||
|
out[h.Node] = h
|
||||||
|
}
|
||||||
|
return out, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordHealth keeps a machine's statement in place of the one kept — unless the one kept is newer, by
|
||||||
|
// when the engine looked: then nothing is written, and false says it was refused as older.
|
||||||
|
func (i *Inventory) RecordHealth(ctx context.Context, h NodeHealth) (bool, error) {
|
||||||
|
if h.Resources == nil {
|
||||||
|
h.Resources = []ResourceHealth{}
|
||||||
|
}
|
||||||
|
if h.Streaks == nil {
|
||||||
|
h.Streaks = map[string]int{}
|
||||||
|
}
|
||||||
|
resources, err := json.Marshal(h.Resources)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
streaks, err := json.Marshal(h.Streaks)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
heard := h.HeardAt
|
||||||
|
if heard.IsZero() {
|
||||||
|
heard = time.Now()
|
||||||
|
}
|
||||||
|
var node string
|
||||||
|
err = i.store.Pool().QueryRow(ctx,
|
||||||
|
`insert into node_health (node, contract, said_at, heard_at, resources, streaks)
|
||||||
|
select id, $2, $3, $4, $5, $6 from node where name = $1
|
||||||
|
on conflict (node) do update set contract = excluded.contract, said_at = excluded.said_at,
|
||||||
|
heard_at = excluded.heard_at, resources = excluded.resources, streaks = excluded.streaks
|
||||||
|
where node_health.said_at <= excluded.said_at
|
||||||
|
returning node`, h.Node, h.Contract, h.SaidAt, heard, resources, streaks).Scan(&node)
|
||||||
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
if _, nerr := i.NodeByName(ctx, h.Node); nerr != nil {
|
||||||
|
return false, nerr
|
||||||
|
}
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
return err == nil, err
|
||||||
|
}
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
-- A module says how it is healthy, and the node-engine judges it (novox/hq ADR 0240, to-be 48 Phase A).
|
||||||
|
--
|
||||||
|
-- Every machine's node-engine states the health of every long-running resource it runs for a module — a
|
||||||
|
-- container that stays up, a process that stays up, a service stated running — in every report and as an
|
||||||
|
-- event between reports. The controller keeps the newest statement per machine, here, so the release
|
||||||
|
-- gate, `node show` and a controller started again all read the same word; and with it, per module, how
|
||||||
|
-- many statements in a row said a resource of it was unhealthy — `module.<module>.<machine>.unhealthy` is
|
||||||
|
-- raised on the second (ADR 0240 §4).
|
||||||
|
--
|
||||||
|
-- One row per machine, replaced, as node_report is: the question is the machine's state now. A machine
|
||||||
|
-- whose node-engine is older than the judging has no row, and its health is not known — never healthy,
|
||||||
|
-- never unhealthy.
|
||||||
|
create table node_health (
|
||||||
|
node uuid primary key references node(id) on delete cascade,
|
||||||
|
-- The statement's version (the engine's liveness contract).
|
||||||
|
contract int not null,
|
||||||
|
-- When the node-engine looked, on the machine's clock: the order of its statements. An older one
|
||||||
|
-- than this is refused.
|
||||||
|
said_at timestamptz not null,
|
||||||
|
-- When this controller heard it, on its own: what "since the send" is judged by.
|
||||||
|
heard_at timestamptz not null default now(),
|
||||||
|
-- [{module, resource, kind, target, state, reason, since, streak, restarts}], as the engine said them.
|
||||||
|
resources jsonb not null default '[]',
|
||||||
|
-- {module: statements in a row saying a resource of it is unhealthy}.
|
||||||
|
streaks jsonb not null default '{}'
|
||||||
|
);
|
||||||
@@ -75,8 +75,16 @@ const (
|
|||||||
// ToolsAliveSubjects is every machine's node tools saying they are there (novox/hq to-be 45 §3,
|
// ToolsAliveSubjects is every machine's node tools saying they are there (novox/hq to-be 45 §3,
|
||||||
// S11): core NATS like the host's, for the same reason.
|
// S11): core NATS like the host's, for the same reason.
|
||||||
ToolsAliveSubjects = "mesh.control.*.tools-alive"
|
ToolsAliveSubjects = "mesh.control.*.tools-alive"
|
||||||
|
|
||||||
|
// HealthSubjects is every machine's health statement between its reports (novox/hq ADR 0240): core
|
||||||
|
// NATS like the heartbeat, because a statement lost is said again within a minute while anything is
|
||||||
|
// not healthy, and the next report carries it whatever happens.
|
||||||
|
HealthSubjects = "mesh.control.*.health"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// HealthSubject is one machine's health statement.
|
||||||
|
func HealthSubject(node string) string { return "mesh.control." + node + ".health" }
|
||||||
|
|
||||||
// ReportSubject is where one node says what it did. On the CONTROL stream, because it is the
|
// ReportSubject is where one node says what it did. On the CONTROL stream, because it is the
|
||||||
// message the store-window guarantee is about (ADR 0083).
|
// message the store-window guarantee is about (ADR 0083).
|
||||||
func ReportSubject(node string) string { return "mesh.control." + node + ".report" }
|
func ReportSubject(node string) string { return "mesh.control." + node + ".report" }
|
||||||
|
|||||||
+13
-1
@@ -29,6 +29,15 @@ import (
|
|||||||
// then and with "still running as call <id>" when it does not; and every call is kept here, with
|
// then and with "still running as call <id>" when it does not; and every call is kept here, with
|
||||||
// what came of it, including an answer the bus refused, so `calls` can say it.
|
// what came of it, including an answer the bus refused, so `calls` can say it.
|
||||||
|
|
||||||
|
// ErrHandingOver is a call refused because the controller answering it is being replaced: stopping, or
|
||||||
|
// unable to run its own build because the node-engine has moved it aside (novox/hq issue 289). Nothing
|
||||||
|
// was done, and the controller after it answers the same call — so the answer carries
|
||||||
|
// `"retry": RetryHandingOver`, and a caller asks once more.
|
||||||
|
var ErrHandingOver = errors.New("the controller is handing over to the next one; nothing was done, ask again")
|
||||||
|
|
||||||
|
// RetryHandingOver is the `retry` mark of an answer refused by ErrHandingOver.
|
||||||
|
const RetryHandingOver = "handing-over"
|
||||||
|
|
||||||
// AnswerWithin is how long a call runs before its caller is answered that it is still running. Well
|
// AnswerWithin is how long a call runs before its caller is answered that it is still running. Well
|
||||||
// inside the shortest wait of a caller the mesh ships (the console's thirty seconds) and the bus's
|
// inside the shortest wait of a caller the mesh ships (the console's thirty seconds) and the bus's
|
||||||
// own window for an answer (broker.ResponseTTL), so the one answer a call has is never late for
|
// own window for an answer (broker.ResponseTTL), so the one answer a call has is never late for
|
||||||
@@ -524,7 +533,10 @@ func (l *CallLog) serveCall(seat, verb string, args json.RawMessage, reply strin
|
|||||||
var body []byte
|
var body []byte
|
||||||
result, err := handle(ctx, args)
|
result, err := handle(ctx, args)
|
||||||
failed := err != nil
|
failed := err != nil
|
||||||
if err != nil {
|
if errors.Is(err, ErrHandingOver) {
|
||||||
|
// Marked as well as said, so a caller asks again without reading the words (issue 289).
|
||||||
|
body, _ = json.Marshal(map[string]any{"error": err.Error(), "retry": RetryHandingOver})
|
||||||
|
} else if err != nil {
|
||||||
body, _ = json.Marshal(map[string]any{"error": err.Error()})
|
body, _ = json.Marshal(map[string]any{"error": err.Error()})
|
||||||
} else if body, err = json.Marshal(map[string]any{"result": result}); err != nil {
|
} else if body, err = json.Marshal(map[string]any{"result": result}); err != nil {
|
||||||
failed = true
|
failed = true
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -177,3 +178,24 @@ func recentOf(l *CallLog) []Call {
|
|||||||
out, _ := l.Recent()
|
out, _ := l.Recent()
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A call refused because its controller is handing over says so in a mark as well as in words, so the
|
||||||
|
// caller asks once more without parsing a sentence (novox/hq issue 289).
|
||||||
|
func TestAHandoverRefusalIsMarkedRetryable(t *testing.T) {
|
||||||
|
l, a := NewCallLog(), newAnswers(t)
|
||||||
|
l.serveCall("mesh-controller", "rotate", nil, "_INBOX.x.9", func(context.Context, json.RawMessage) (any, error) {
|
||||||
|
return nil, fmt.Errorf("%w: stopping", ErrHandingOver)
|
||||||
|
}, a.respond, nil)
|
||||||
|
got := a.only()
|
||||||
|
if got["retry"] != RetryHandingOver || !strings.Contains(fmt.Sprint(got["error"]), "handing over") {
|
||||||
|
t.Fatalf("answered %v", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
l, a = NewCallLog(), newAnswers(t)
|
||||||
|
l.serveCall("mesh-controller", "rotate", nil, "_INBOX.x.10", func(context.Context, json.RawMessage) (any, error) {
|
||||||
|
return nil, errors.New("refused for its own reason")
|
||||||
|
}, a.respond, nil)
|
||||||
|
if _, marked := a.only()["retry"]; marked {
|
||||||
|
t.Fatal("an ordinary refusal was marked to be asked again")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -35,6 +35,10 @@ var Contracts = map[string]Contract{
|
|||||||
KindHeartbeat: {Unordered: "a word that the machine is there: the newest heard is the newest said, and one " +
|
KindHeartbeat: {Unordered: "a word that the machine is there: the newest heard is the newest said, and one " +
|
||||||
"lost is the next one"},
|
"lost is the next one"},
|
||||||
KindToolsHeartbeat: {Unordered: "a word that the node tools are there, as a machine's heartbeat"},
|
KindToolsHeartbeat: {Unordered: "a word that the node tools are there, as a machine's heartbeat"},
|
||||||
|
KindHealth: {Ordered: "by the time the node-engine looked (`at`, on the machine's clock): a statement older " +
|
||||||
|
"than the one kept for that machine is refused, so an event arriving after a newer report does not undo it " +
|
||||||
|
"(novox/hq ADR 0240)",
|
||||||
|
Tests: []string{"TestAnOlderHealthStatementIsRefused"}},
|
||||||
KindEnrolment: {Unordered: "a request answered once, under a token spent once: a second presentation is " +
|
KindEnrolment: {Unordered: "a request answered once, under a token spent once: a second presentation is " +
|
||||||
"refused by the token, not by an order (issue 083)",
|
"refused by the token, not by an order (issue 083)",
|
||||||
Tests: []string{"TestAnEnrolmentMetByAHeldTokenIsAskedToTryAgain"}},
|
Tests: []string{"TestAnEnrolmentMetByAHeldTokenIsAskedToTryAgain"}},
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ import (
|
|||||||
func TestEveryConsumedKindHasAContract(t *testing.T) {
|
func TestEveryConsumedKindHasAContract(t *testing.T) {
|
||||||
// What the controller can be handed, from the subjects it is granted and the ones it derives.
|
// What the controller can be handed, from the subjects it is granted and the ones it derives.
|
||||||
subjects := []string{EnrolSubject, BuiltSubject, ReportSubject("anchor"), AliveSubject("anchor"),
|
subjects := []string{EnrolSubject, BuiltSubject, ReportSubject("anchor"), AliveSubject("anchor"),
|
||||||
ToolsAliveSubject("anchor"), "mesh.mod.postgres.event.provisioner.failing"}
|
ToolsAliveSubject("anchor"), HealthSubject("anchor"), "mesh.mod.postgres.event.provisioner.failing"}
|
||||||
subjects = append(subjects, broker.ControllerFollows...)
|
subjects = append(subjects, broker.ControllerFollows...)
|
||||||
kinds := map[string]bool{}
|
kinds := map[string]bool{}
|
||||||
for _, s := range subjects {
|
for _, s := range subjects {
|
||||||
|
|||||||
@@ -0,0 +1,56 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A machine's health statement (novox/hq ADR 0240, to-be 48 §4).
|
||||||
|
//
|
||||||
|
// **The node-engine owns every verdict; the controller keeps the last word per machine.** A statement
|
||||||
|
// arrives in every report and as an event between reports — on each change, and again every minute while
|
||||||
|
// a resource is not healthy. What the controller does with it — keep it, refuse an older one, raise
|
||||||
|
// `module.<module>.<machine>.unhealthy` on the second statement that says so — is the Healths it is given.
|
||||||
|
|
||||||
|
// Healths keeps what machines state of their long-running resources.
|
||||||
|
type Healths interface {
|
||||||
|
// Stated keeps one machine's statement, from a report or the event. An older statement than the one
|
||||||
|
// kept is refused there, by its time on the machine.
|
||||||
|
Stated(ctx context.Context, node string, h Health) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Hears says where the machines' health statements are kept. Their subscription is the heartbeats':
|
||||||
|
// core, and always made, so nothing is asked of the bus here.
|
||||||
|
func (s *Server) Hears(h Healths) { s.healths = h }
|
||||||
|
|
||||||
|
// healthSaid acts on one health event. Core NATS, so there is nothing to hold: a statement that could not
|
||||||
|
// be kept is said in the log, and the next one — a minute away while anything is not healthy — is kept.
|
||||||
|
func (s *Server) healthSaid(ctx context.Context, m Control) {
|
||||||
|
defer func() { _ = m.Took() }()
|
||||||
|
var said HealthSaid
|
||||||
|
if err := json.Unmarshal(m.Body(), &said); err != nil || said.Health.Contract == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// The machine is the one in the subject the bus let it publish on, never the body's.
|
||||||
|
node, ok := nodeOfHealth(m.Subject())
|
||||||
|
if !ok || s.healths == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := s.healths.Stated(ctx, node, said.Health); err != nil {
|
||||||
|
s.log.Printf("could not keep what %s says of its resources' health: %v", node, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// nodeOfHealth is the machine a health statement names in its subject.
|
||||||
|
func nodeOfHealth(subject string) (string, bool) {
|
||||||
|
rest, ok := strings.CutPrefix(subject, "mesh.control.")
|
||||||
|
if !ok {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
node, kind, ok := strings.Cut(rest, ".")
|
||||||
|
if !ok || kind != "health" || node == "" || strings.Contains(node, ".") {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
return node, true
|
||||||
|
}
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A health event is kept as the machine its subject names says it, whatever its body claims, and taken
|
||||||
|
// (novox/hq ADR 0240): core NATS, nothing to hold.
|
||||||
|
|
||||||
|
type keptHealths struct{ by map[string]Health }
|
||||||
|
|
||||||
|
func (k *keptHealths) Stated(_ context.Context, node string, h Health) error {
|
||||||
|
k.by[node] = h
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAHealthEventIsKeptAsTheMachineInItsSubject(t *testing.T) {
|
||||||
|
s, in := serving()
|
||||||
|
kept := &keptHealths{by: map[string]Health{}}
|
||||||
|
s.Hears(kept)
|
||||||
|
to := &settled{}
|
||||||
|
m := in.sends(t, to, KindHealth, HealthSaid{Node: "laptop", Health: Health{Contract: LivenessContract, At: time.Now(),
|
||||||
|
Resources: []ResourceHealth{{Module: "letta", Resource: "letta.server", State: StateUnhealthy}}}}).(*fakeControl)
|
||||||
|
m.subject = HealthSubject("anchor")
|
||||||
|
s.act(t.Context(), m)
|
||||||
|
if !to.acked {
|
||||||
|
t.Fatal("a health event was not taken")
|
||||||
|
}
|
||||||
|
if _, lied := kept.by["laptop"]; lied || len(kept.by["anchor"].Resources) != 1 {
|
||||||
|
t.Fatalf("kept %+v: the machine is the subject's, never the body's", kept.by)
|
||||||
|
}
|
||||||
|
if kind, ok := kindOfSubject(HealthSubject("anchor")); !ok || kind != KindHealth {
|
||||||
|
t.Fatalf("%s is not read as a health statement", HealthSubject("anchor"))
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -262,6 +262,13 @@ type Report struct {
|
|||||||
// `witness` and `not-reversible` are sent only to a machine whose report carries it.
|
// `witness` and `not-reversible` are sent only to a machine whose report carries it.
|
||||||
Witness int `json:"witness,omitempty"`
|
Witness int `json:"witness,omitempty"`
|
||||||
|
|
||||||
|
// Health is the machine's word on every long-running resource it runs for a module (novox/hq ADR
|
||||||
|
// 0240, to-be 48 §4; mesh-host's internal/liveness): its state, since when, its failing streak and
|
||||||
|
// the restarts its node-engine counted. **Absent from an engine older than the judging**, which is
|
||||||
|
// read as "not known" — never as healthy, never as a reason to raise anything; present with no
|
||||||
|
// resources from a machine that runs nothing long-lived.
|
||||||
|
Health *Health `json:"health,omitempty"`
|
||||||
|
|
||||||
// Rekey is a node taking a found tunnel's key as its overlay key after enrolment (novox/hq
|
// Rekey is a node taking a found tunnel's key as its overlay key after enrolment (novox/hq
|
||||||
// ADR 0105). A report carrying one is not an account of the machine: it moves the node's
|
// ADR 0105). A report carrying one is not an account of the machine: it moves the node's
|
||||||
// overlay key and tunnel and nothing else.
|
// overlay key and tunnel and nothing else.
|
||||||
@@ -404,3 +411,45 @@ func EnrolProof(secret string, public []byte, overlay, sealing, serving string)
|
|||||||
return []byte("novox-mesh-enrol\x00" + secret + "\x00" + base64.StdEncoding.EncodeToString(public) +
|
return []byte("novox-mesh-enrol\x00" + secret + "\x00" + base64.StdEncoding.EncodeToString(public) +
|
||||||
"\x00" + overlay + "\x00" + sealing + "\x00" + serving)
|
"\x00" + overlay + "\x00" + sealing + "\x00" + serving)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// LivenessContract is the version of the health statement this controller reads (ADR 0240 Phase A).
|
||||||
|
const LivenessContract = 1
|
||||||
|
|
||||||
|
// Health is one statement of a machine's long-running resources (to-be 48 §4): in every report, as the
|
||||||
|
// event HealthSubject between reports on each change, and again every minute while one is not healthy.
|
||||||
|
// The node-engine's own (mesh-host internal/link Health); a test on each side holds the field names.
|
||||||
|
type Health struct {
|
||||||
|
Contract int `json:"contract"`
|
||||||
|
// At is when the engine looked, on the machine's clock: the order of its statements.
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
Resources []ResourceHealth `json:"resources"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// The states a resource is said in (ADR 0240 §4).
|
||||||
|
const (
|
||||||
|
StateHealthy = "healthy"
|
||||||
|
StateUnhealthy = "unhealthy"
|
||||||
|
StateStarting = "starting"
|
||||||
|
StateHeld = "held"
|
||||||
|
StateUnknown = "unknown"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ResourceHealth is one long-running resource's state.
|
||||||
|
type ResourceHealth struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
Resource string `json:"resource"`
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Target string `json:"target"`
|
||||||
|
State string `json:"state"`
|
||||||
|
Reason string `json:"reason,omitempty"`
|
||||||
|
Since time.Time `json:"since"`
|
||||||
|
Streak int `json:"streak,omitempty"`
|
||||||
|
Restarts int `json:"restarts,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// HealthSaid is the health event's body: the machine and its statement. The machine is read from the
|
||||||
|
// subject the bus let it publish on, never from here.
|
||||||
|
type HealthSaid struct {
|
||||||
|
Node string `json:"node"`
|
||||||
|
Health Health `json:"health"`
|
||||||
|
}
|
||||||
|
|||||||
@@ -25,6 +25,13 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) {
|
|||||||
[]string{"protocol", "address", "port", "by", "published", "container-port"}},
|
[]string{"protocol", "address", "port", "by", "published", "container-port"}},
|
||||||
{EnrolRequest{Node: "n", Secret: "s", PublicKey: []byte("k")},
|
{EnrolRequest{Node: "n", Secret: "s", PublicKey: []byte("k")},
|
||||||
[]string{"node", "secret", "public_key"}},
|
[]string{"node", "secret", "public_key"}},
|
||||||
|
// novox/hq ADR 0240: every long-running resource's health, in every report and in its own event.
|
||||||
|
{Report{Node: "n", Health: &Health{Contract: LivenessContract}}, []string{"node", "health"}},
|
||||||
|
{Health{Contract: LivenessContract, Resources: []ResourceHealth{}}, []string{"contract", "at", "resources"}},
|
||||||
|
{ResourceHealth{Module: "m", Resource: "m.r", Kind: "container", Target: "t", State: StateUnhealthy,
|
||||||
|
Reason: "restarting", Streak: 2, Restarts: 3},
|
||||||
|
[]string{"module", "resource", "kind", "target", "state", "reason", "since", "streak", "restarts"}},
|
||||||
|
{HealthSaid{Node: "n"}, []string{"node", "health"}},
|
||||||
} {
|
} {
|
||||||
raw, err := json.Marshal(c.value)
|
raw, err := json.Marshal(c.value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -30,8 +30,10 @@ const (
|
|||||||
KindHeartbeat = "heartbeat"
|
KindHeartbeat = "heartbeat"
|
||||||
// KindToolsHeartbeat is a machine's node tools saying they are there (novox/hq to-be 45 S11).
|
// KindToolsHeartbeat is a machine's node tools saying they are there (novox/hq to-be 45 S11).
|
||||||
KindToolsHeartbeat = "tools-heartbeat"
|
KindToolsHeartbeat = "tools-heartbeat"
|
||||||
KindBuilt = "built"
|
// KindHealth is a machine's statement of its long-running resources' health (novox/hq ADR 0240).
|
||||||
KindModuleMoved = "module-moved"
|
KindHealth = "health"
|
||||||
|
KindBuilt = "built"
|
||||||
|
KindModuleMoved = "module-moved"
|
||||||
// KindSourceMoved is the forge announcing a merge: a source moved, and what it produces is
|
// KindSourceMoved is the forge announcing a merge: a source moved, and what it produces is
|
||||||
// built without anybody telling the mesh (novox/hq 04-ISSUES/131).
|
// built without anybody telling the mesh (novox/hq 04-ISSUES/131).
|
||||||
KindSourceMoved = "source-moved"
|
KindSourceMoved = "source-moved"
|
||||||
|
|||||||
@@ -108,6 +108,13 @@ func (n *natsInbound) Receive(ctx context.Context, act func(context.Context, Con
|
|||||||
return fmt.Errorf("subscribing to the node tools' heartbeats: %w", err)
|
return fmt.Errorf("subscribing to the node tools' heartbeats: %w", err)
|
||||||
}
|
}
|
||||||
defer func() { _ = toolsAlive.Unsubscribe() }()
|
defer func() { _ = toolsAlive.Unsubscribe() }()
|
||||||
|
// And what each machine says of its long-running resources between reports (novox/hq ADR 0240): core
|
||||||
|
// too, and said again while it matters.
|
||||||
|
health, err := conn.ChanSubscribe(HealthSubjects, beats)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("subscribing to the machines' health: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = health.Unsubscribe() }()
|
||||||
|
|
||||||
// The events the controller follows, when something is listening for them.
|
// The events the controller follows, when something is listening for them.
|
||||||
var events chan *nats.Msg
|
var events chan *nats.Msg
|
||||||
@@ -239,6 +246,8 @@ func kindOfSubject(subject string) (string, bool) {
|
|||||||
return KindHeartbeat, true
|
return KindHeartbeat, true
|
||||||
case "tools-alive":
|
case "tools-alive":
|
||||||
return KindToolsHeartbeat, true
|
return KindToolsHeartbeat, true
|
||||||
|
case "health":
|
||||||
|
return KindHealth, true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
switch subject {
|
switch subject {
|
||||||
|
|||||||
@@ -94,6 +94,8 @@ type Server struct {
|
|||||||
retirements Retirements
|
retirements Retirements
|
||||||
// checker asks for a pull request's merge check (novox/hq to-be 45 §9).
|
// checker asks for a pull request's merge check (novox/hq to-be 45 §9).
|
||||||
checker Checker
|
checker Checker
|
||||||
|
// healths keeps what machines say of their long-running resources (novox/hq ADR 0240).
|
||||||
|
healths Healths
|
||||||
|
|
||||||
log *log.Logger
|
log *log.Logger
|
||||||
// giveUp is how long one message is held for the store; zero means GiveUpAfter.
|
// giveUp is how long one message is held for the store; zero means GiveUpAfter.
|
||||||
@@ -194,6 +196,8 @@ func (s *Server) act(ctx context.Context, m Control) {
|
|||||||
s.heartbeat(m)
|
s.heartbeat(m)
|
||||||
case KindToolsHeartbeat:
|
case KindToolsHeartbeat:
|
||||||
s.toolsHeartbeat(m)
|
s.toolsHeartbeat(m)
|
||||||
|
case KindHealth:
|
||||||
|
s.healthSaid(ctx, m)
|
||||||
case KindBuilt:
|
case KindBuilt:
|
||||||
s.wasBuilt(ctx, m)
|
s.wasBuilt(ctx, m)
|
||||||
case KindModuleMoved:
|
case KindModuleMoved:
|
||||||
|
|||||||
@@ -60,6 +60,7 @@
|
|||||||
"pause",
|
"pause",
|
||||||
"resume",
|
"resume",
|
||||||
"hand-act",
|
"hand-act",
|
||||||
|
"drill",
|
||||||
"hand-acts",
|
"hand-acts",
|
||||||
"durations",
|
"durations",
|
||||||
"conditions",
|
"conditions",
|
||||||
|
|||||||
Reference in New Issue
Block a user