Keep calls and hand acts on the bus, answer status at once, record durations (hq to-be 45 Phase 0)
A controller restart lost every call's outcome, `status` composed the mesh while its caller waited (18.6s live on 2026-10-06, past the 10s window), a repair by hand left no trace, and the core's bounds had nothing measured to be set from. - calls: kept in the controller's bucket mesh-controller_calls (last 1000 or 14 days, answers bounded to 64 KiB), read by id across a restart; a controller starting marks a stopped one's running calls abandoned; each call names its caller from the inbox its answer goes to. - status: the serving controller composes it at start, after news from a machine, a build or an acting verb, and every minute; the verb answers the last composition at once with when and how long it took. Composing resolves each machine once instead of twice. - hand-act log in mesh-controller_hand-acts: push (required through the seat), plans stop/close, broker consumer-reset and the new hand-act record take --why/--cause/--condition; `hand-acts` lists them and repeated causes; status counts the week's. - durations (migration 0066): apply (send to first report), heartbeat gap, plan tier and build, recorded as heard; `durations` summarises them. - the controller's seat row takes this binary's definition of its own verbs, so the console no longer judges calls against an older build's schema. - the controller is granted its two buckets' subjects.
This commit is contained in:
@@ -683,6 +683,10 @@ type answers struct {
|
||||
// (novox/hq ADR 0225): its provider leaves it out of the grants and composes everything else, so
|
||||
// this is the one place it is said across the mesh. Not well while there is any.
|
||||
overflowing []catalogue.Overflow
|
||||
// handActs is how many acts were done by hand in the last seven days (novox/hq to-be 45 §7), nil
|
||||
// where the log is not on hand; handActsUnread why it could not be read when it could not.
|
||||
handActs *int
|
||||
handActsUnread string
|
||||
}
|
||||
|
||||
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
|
||||
|
||||
@@ -35,10 +35,10 @@ func TestAPushAnswersBeforeItSends(t *testing.T) {
|
||||
args map[string]any
|
||||
want bool
|
||||
}{
|
||||
{"push", map[string]any{"node": "anchor"}, true},
|
||||
{"push", map[string]any{}, true},
|
||||
{"command", map[string]any{"command": "push anchor"}, true},
|
||||
{"command", map[string]any{"command": "push --behind"}, true},
|
||||
{"push", map[string]any{"node": "anchor", "why": "w"}, true},
|
||||
{"push", map[string]any{"why": "w"}, true},
|
||||
{"command", map[string]any{"command": "push anchor --why w"}, true},
|
||||
{"command", map[string]any{"command": "push --behind --why=w"}, true},
|
||||
{"command", map[string]any{"command": "builds"}, false},
|
||||
{"status", map[string]any{}, false},
|
||||
{"assign", map[string]any{"node": "anchor", "module": "m"}, false},
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// What the core's bounds are set from (novox/hq to-be 45 Phase 0).
|
||||
//
|
||||
// **A bound is set from what was measured, not from what seemed reasonable.** Phase 1 puts a watchdog
|
||||
// on each row of the signals table, and each has a bound: S1 three heartbeat intervals, S2 three times
|
||||
// a machine's last apply, S3 a tier's build and apply time, S6 a build's timeout. Marked provisional
|
||||
// in the design until a fortnight of these says what the mesh actually takes. Recorded by the serving
|
||||
// controller as it hears each — a send's first report, a machine's next word, a plan leaving a tier, a
|
||||
// build's outcome — and summarised here per machine, repository or module.
|
||||
|
||||
// recordBuildDuration measures one build from its ask to its outcome heard.
|
||||
func recordBuildDuration(ctx context.Context, inv *inventory.Inventory, result link.BuildResult, asked time.Time) {
|
||||
if asked.IsZero() || result.ID == "" {
|
||||
return
|
||||
}
|
||||
subject := result.Module
|
||||
if subject == "" {
|
||||
subject = result.Repository
|
||||
}
|
||||
detail := "built"
|
||||
if result.Failed != "" {
|
||||
detail = "failed: " + firstLine(result.Failed)
|
||||
}
|
||||
if err := inv.RecordDuration(ctx, inventory.Duration{Kind: inventory.DurationBuild, Subject: subject,
|
||||
Node: result.On, Ref: result.ID, Started: asked, Took: time.Since(asked), Detail: detail}); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: how long it took could not be recorded: %v\n", result.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// durationSummary is one subject's measurements of one kind.
|
||||
type durationSummary struct {
|
||||
Kind string `json:"kind"`
|
||||
Subject string `json:"subject"`
|
||||
Count int `json:"count"`
|
||||
Median string `json:"median"`
|
||||
P90 string `json:"p90"`
|
||||
Max string `json:"max"`
|
||||
// Bound is what to-be 45's rule would make of these, where the rule is a multiple of a measured
|
||||
// time: three times the slowest apply (S2), three times the median word interval (S1).
|
||||
Suggests string `json:"suggests,omitempty"`
|
||||
}
|
||||
|
||||
func summarise(ds []inventory.Duration) []durationSummary {
|
||||
type key struct{ kind, subject string }
|
||||
by := map[key][]time.Duration{}
|
||||
for _, d := range ds {
|
||||
k := key{d.Kind, d.Subject}
|
||||
by[k] = append(by[k], d.Took)
|
||||
}
|
||||
var out []durationSummary
|
||||
for k, took := range by {
|
||||
slices.Sort(took)
|
||||
at := func(q float64) time.Duration { return took[int(q*float64(len(took)-1))] }
|
||||
s := durationSummary{Kind: k.kind, Subject: k.subject, Count: len(took),
|
||||
Median: round(at(0.5)), P90: round(at(0.9)), Max: round(took[len(took)-1])}
|
||||
switch k.kind {
|
||||
case inventory.DurationApply:
|
||||
s.Suggests = "S2 bound max(2m, 3×last apply) ≈ " + round(max(2*time.Minute, 3*at(0.9))) + " at the p90"
|
||||
case inventory.DurationHeartbeatGap:
|
||||
s.Suggests = "S1 bound 3×interval ≈ " + round(3*at(0.5))
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
ki, kj := slices.Index(inventory.DurationKinds, out[i].Kind), slices.Index(inventory.DurationKinds, out[j].Kind)
|
||||
if ki != kj {
|
||||
return ki < kj
|
||||
}
|
||||
return out[i].Subject < out[j].Subject
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func round(d time.Duration) string {
|
||||
switch {
|
||||
case d < time.Second:
|
||||
return d.Round(time.Millisecond).String()
|
||||
case d < time.Minute:
|
||||
return d.Round(100 * time.Millisecond).String()
|
||||
default:
|
||||
return d.Round(time.Second).String()
|
||||
}
|
||||
}
|
||||
|
||||
// durationsCommand is `durations`: the summary per kind and subject, or every measurement as data.
|
||||
func durationsCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("durations", flag.ContinueOnError)
|
||||
kind := set.String("kind", "", "one kind: "+strings.Join(inventory.DurationKinds, ", "))
|
||||
days := set.Int("days", 14, "how many days back")
|
||||
asJSON := set.Bool("json", false, "the summary as data")
|
||||
all := set.Bool("all", false, "every measurement rather than the summary")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
}
|
||||
if *kind != "" && !slices.Contains(inventory.DurationKinds, *kind) {
|
||||
return fmt.Errorf("%q is not a kind of duration: %s", *kind, strings.Join(inventory.DurationKinds, ", "))
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
ds, err := open.inventory.Durations(ctx, *kind, time.Now().Add(-time.Duration(*days)*24*time.Hour))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *all {
|
||||
body, err := json.MarshalIndent(ds, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
summary := summarise(ds)
|
||||
if *asJSON {
|
||||
body, err := json.MarshalIndent(map[string]any{"days": *days, "durations": summary}, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
if len(summary) == 0 {
|
||||
fmt.Printf("nothing measured in the last %d day(s): the serving controller records apply, heartbeat-gap, "+
|
||||
"plan-tier and build durations as it hears them\n", *days)
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("durations over the last %d day(s) — what the core's bounds are set from (to-be 45 Phase 0)\n\n", *days)
|
||||
fmt.Printf(" %-14s %-28s %6s %10s %10s %10s\n", "kind", "of", "count", "median", "p90", "max")
|
||||
for _, s := range summary {
|
||||
fmt.Printf(" %-14s %-28s %6d %10s %10s %10s\n", s.Kind, s.Subject, s.Count, s.Median, s.P90, s.Max)
|
||||
if s.Suggests != "" {
|
||||
fmt.Printf(" %-14s %-28s %s\n", "", "", s.Suggests)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// forgettingOldDurations removes what is older than a month, at start and daily after.
|
||||
func forgettingOldDurations(ctx context.Context, inv *inventory.Inventory) {
|
||||
for {
|
||||
if n, err := inv.ForgetOldDurations(ctx); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "durations older than %s could not be removed: %v\n", inventory.DurationsKeptFor, err)
|
||||
} else if n > 0 {
|
||||
fmt.Printf("removed %d duration(s) older than %s\n", n, inventory.DurationsKeptFor)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(24 * time.Hour):
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// Acts done by hand, and why (novox/hq to-be 45 §7).
|
||||
//
|
||||
// **Which verbs ask why.** `plans close` and `plans stop`, `broker consumer-reset` and `hand-act
|
||||
// record` refuse without it everywhere: nothing automated runs them, so a call without a reason is a
|
||||
// person who has not given one. A named `push` asks for it through the mesh-controller seat, which is
|
||||
// how a person or an agent acts by hand on the mesh; at a shell `--why` is recorded when given and not
|
||||
// required, because the installer and the lab push by command line as a step of what they do, and a
|
||||
// step of a procedure is not a repair. `conditions silence` joins them when the condition store does
|
||||
// (Phase 1).
|
||||
|
||||
// handActFlags are the flags every repairing verb takes.
|
||||
type handActFlags struct {
|
||||
why, cause, condition *string
|
||||
}
|
||||
|
||||
func addHandActFlags(set *flag.FlagSet) handActFlags {
|
||||
return handActFlags{
|
||||
why: set.String("why", "", "why this is done by hand — recorded in the hand-act log (novox/hq to-be 45 §7)"),
|
||||
cause: set.String("cause", "", "the cause, in a word or a condition's kind; the verb's own name when not given"),
|
||||
condition: set.String("condition", "", "the key of the condition this act addresses, if any"),
|
||||
}
|
||||
}
|
||||
|
||||
// given is whether a reason was given.
|
||||
func (f handActFlags) given() bool { return strings.TrimSpace(*f.why) != "" }
|
||||
|
||||
// require refuses an act without a reason, before anything is done.
|
||||
func (f handActFlags) require(verb string) error {
|
||||
if f.given() {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%s is a repair done by hand, and says why: --why <text> (recorded in the hand-act "+
|
||||
"log, novox/hq to-be 45 §7). Nothing was done", verb)
|
||||
}
|
||||
|
||||
// handActConn is the serving controller's connection, for what it reads of the log itself; a
|
||||
// command dials its own.
|
||||
var handActConn *nats.Conn
|
||||
|
||||
// onTheBus runs f with a connection to the bus: the serving controller's, or one of its own.
|
||||
func onTheBus(f func(*nats.Conn) error) error {
|
||||
if handActConn != nil {
|
||||
return f(handActConn)
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot reach the bus: %w", err)
|
||||
}
|
||||
defer js.Close()
|
||||
return f(js.Conn())
|
||||
}
|
||||
|
||||
// record writes the entry for an act about to be done. **Before the act, and never instead of it**:
|
||||
// a log that cannot be written is said loudly, and the repair it was about still happens — a mesh
|
||||
// whose bus is down is exactly the mesh somebody is repairing by hand.
|
||||
func (f handActFlags) record(ctx context.Context, verb string, args []string) {
|
||||
if !f.given() {
|
||||
return
|
||||
}
|
||||
act := link.HandAct{Verb: verb, Args: args, Why: strings.TrimSpace(*f.why),
|
||||
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
written, err := link.RecordHandAct(ctx, conn, act)
|
||||
act = written
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "this act by hand could NOT be recorded in the hand-act log, and is done anyway: %v\n", err)
|
||||
return
|
||||
}
|
||||
fmt.Printf("recorded as %s in the hand-act log: %s, because %q (cause: %s)\n", act.ID, act.By, act.Why, act.Cause)
|
||||
}
|
||||
|
||||
// handActCommand is `hand-act record` and `hand-acts`.
|
||||
func handActCommand(ctx context.Context, args []string) error {
|
||||
if len(args) > 0 && args[0] == "record" {
|
||||
set := flag.NewFlagSet("hand-act record", flag.ContinueOnError)
|
||||
f := addHandActFlags(set)
|
||||
positionals, err := parseAround(set, args[1:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
what := strings.TrimSpace(strings.Join(positionals, " "))
|
||||
if what == "" {
|
||||
return errors.New("hand-act record <what was done> --why <text> [--cause <word>] [--condition <key>]")
|
||||
}
|
||||
if err := f.require("hand-act record"); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(*f.cause) == "" {
|
||||
return errors.New("hand-act record says the cause too: --cause <word>, the word a second " +
|
||||
"act for the same reason will use — it is how a repair done twice is found")
|
||||
}
|
||||
act := link.HandAct{Verb: "hand-act record", Args: []string{what}, Why: strings.TrimSpace(*f.why),
|
||||
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
written, err := link.RecordHandAct(ctx, conn, act)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the act could not be recorded: %w", err)
|
||||
}
|
||||
fmt.Printf("recorded as %s: %s did %q, because %q (cause: %s)\n", written.ID, written.By, what,
|
||||
written.Why, written.Cause)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if len(args) > 0 && args[0] != "list" && !strings.HasPrefix(args[0], "-") {
|
||||
return errors.New("hand-act record <what> --why <text> --cause <word> | hand-acts [--days N] [--json]")
|
||||
}
|
||||
if len(args) > 0 && args[0] == "list" {
|
||||
args = args[1:]
|
||||
}
|
||||
set := flag.NewFlagSet("hand-acts", flag.ContinueOnError)
|
||||
days := set.Int("days", 14, "how many days back")
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
}
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
now := time.Now()
|
||||
acts, err := link.HandActs(ctx, conn, now.Add(-time.Duration(*days)*24*time.Hour))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
repeated := link.RepeatedCauses(acts, now)
|
||||
if *asJSON {
|
||||
body, err := json.MarshalIndent(map[string]any{"acts": acts, "repeated": repeated}, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
if len(acts) == 0 {
|
||||
fmt.Printf("nothing was done by hand in the last %d day(s)\n", *days)
|
||||
return nil
|
||||
}
|
||||
for i := len(acts) - 1; i >= 0; i-- {
|
||||
a := acts[i]
|
||||
fmt.Printf("%s %s %s %s\n by %s — %s (cause: %s", a.At.Local().Format("2006-01-02 15:04"), a.ID,
|
||||
a.Verb, strings.Join(a.Args, " "), a.By, a.Why, a.Cause)
|
||||
if a.Condition != "" {
|
||||
fmt.Printf(", condition %s", a.Condition)
|
||||
}
|
||||
fmt.Println(")")
|
||||
}
|
||||
if len(repeated) > 0 {
|
||||
causes := make([]string, 0, len(repeated))
|
||||
for c, n := range repeated {
|
||||
causes = append(causes, fmt.Sprintf("%s ×%d", c, n))
|
||||
}
|
||||
sort.Strings(causes)
|
||||
fmt.Printf("\ndone by hand more than once in a fortnight — a healer is wanted (to-be 45 S15): %s\n",
|
||||
strings.Join(causes, ", "))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// handActsThisWeek is how many acts were done by hand in the last seven days, for `status`; -1 when
|
||||
// the log could not be read, which status says rather than reading as none.
|
||||
func handActsThisWeek(ctx context.Context) (int, string) {
|
||||
n := -1
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
reading, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
acts, err := link.HandActs(reading, conn, time.Now().Add(-7*24*time.Hour))
|
||||
n = len(acts)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return -1, err.Error()
|
||||
}
|
||||
return n, ""
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// **Every verb that repairs by hand takes a required why** (novox/hq to-be 45 §7): refused before
|
||||
// anything is done, through the seat, through `command`, and at a shell where nothing automated runs
|
||||
// the verb.
|
||||
func TestARepairByHandWithoutAReasonIsRefused(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
}{
|
||||
{"push", map[string]any{"node": "anchor"}},
|
||||
{"push", map[string]any{}},
|
||||
{"plans", map[string]any{"close": "plan-1"}},
|
||||
{"plans", map[string]any{"stop": "plan-1"}},
|
||||
{"hand-act", map[string]any{"what": "restarted the proxy", "cause": "proxy-stuck"}},
|
||||
{"command", map[string]any{"command": "push anchor"}},
|
||||
{"command", map[string]any{"command": "plans close plan-1"}},
|
||||
{"command", map[string]any{"command": "broker consumer-reset EVENTS controller"}},
|
||||
{"command", map[string]any{"command": "hand-act record restarted --cause x"}},
|
||||
} {
|
||||
argv, err := argvFor(c.verb, c.args)
|
||||
if c.verb == "plans" && err == nil {
|
||||
// The seat composes the command line; the command refuses it, before opening anything.
|
||||
err = plansCommand(context.Background(), argv[1:])
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), "why") {
|
||||
t.Errorf("%s %v was not refused for want of why: %v %v", c.verb, c.args, argv, err)
|
||||
}
|
||||
}
|
||||
for _, args := range [][]string{{"EVENTS", "controller"}} {
|
||||
if err := consumerReset(context.Background(), args); err == nil || !strings.Contains(err.Error(), "--why") {
|
||||
t.Errorf("consumer-reset without why: %v", err)
|
||||
}
|
||||
}
|
||||
if err := handActCommand(context.Background(), []string{"record", "restarted the proxy", "--cause", "x"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "--why") {
|
||||
t.Errorf("hand-act record without why: %v", err)
|
||||
}
|
||||
if err := handActCommand(context.Background(), []string{"record", "restarted the proxy", "--why", "it hung"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "--cause") {
|
||||
t.Errorf("hand-act record without a cause: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// With a reason, the seat passes it to the command, and a verb that only reads is not held to one.
|
||||
func TestARepairByHandCarriesItsReason(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
want string
|
||||
}{
|
||||
{"push", map[string]any{"node": "anchor", "why": "stuck", "cause": "sent-not-reported"},
|
||||
"push anchor --wait 0 --why stuck --cause sent-not-reported"},
|
||||
{"plans", map[string]any{"close": "plan-1", "why": "the report will not come"},
|
||||
"plans close plan-1 --why the report will not come"},
|
||||
{"plans", map[string]any{"retry": "plan-1"}, "plans retry plan-1"},
|
||||
{"hand-act", map[string]any{"what": "restarted", "why": "hung", "cause": "proxy", "condition": "machine.a.silent"},
|
||||
"hand-act record restarted --why hung --cause proxy --condition machine.a.silent"},
|
||||
{"command", map[string]any{"command": "push anchor --why stuck"}, "push anchor --why stuck"},
|
||||
{"command", map[string]any{"command": "plans plan-1"}, "plans plan-1"},
|
||||
} {
|
||||
argv, err := argvFor(c.verb, c.args)
|
||||
if err != nil || strings.Join(argv, " ") != c.want {
|
||||
t.Errorf("%s %v: %v %v, want %q", c.verb, c.args, argv, err, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The summary of durations says, per kind and subject, what a bound would be set from.
|
||||
func TestDurationsAreSummarisedPerSubject(t *testing.T) {
|
||||
var ds []inventory.Duration
|
||||
for i := 1; i <= 10; i++ {
|
||||
ds = append(ds, inventory.Duration{Kind: inventory.DurationApply, Subject: "anchor",
|
||||
Took: time.Duration(i) * time.Second})
|
||||
}
|
||||
ds = append(ds, inventory.Duration{Kind: inventory.DurationHeartbeatGap, Subject: "anchor", Took: time.Minute})
|
||||
got := summarise(ds)
|
||||
if len(got) != 2 || got[0].Kind != inventory.DurationApply || got[0].Count != 10 ||
|
||||
got[0].Max != "10s" || got[0].Median != "5s" || got[0].P90 != "9s" {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
if !strings.Contains(got[1].Suggests, "3m0s") {
|
||||
t.Fatalf("a minute between words suggests %q", got[1].Suggests)
|
||||
}
|
||||
}
|
||||
@@ -145,6 +145,14 @@ func run() error {
|
||||
return seatCommand(ctx, args[1:])
|
||||
case "status":
|
||||
return statusCommand(ctx, args[1:])
|
||||
// Acts done by hand, and why (novox/hq to-be 45 §7).
|
||||
case "hand-act":
|
||||
return handActCommand(ctx, args[1:])
|
||||
case "hand-acts":
|
||||
return handActCommand(ctx, append([]string{"list"}, args[1:]...))
|
||||
// What the mesh's bounds will be set from (novox/hq to-be 45 Phase 0).
|
||||
case "durations":
|
||||
return durationsCommand(ctx, args[1:])
|
||||
case "version":
|
||||
fmt.Println(version)
|
||||
return nil
|
||||
@@ -226,6 +234,12 @@ func usage() {
|
||||
kill <id> end a build where it runs; recorded failed, killed by hand
|
||||
pause [<node>] / resume [<node>] the build seat's holder there, or every holder, takes nothing new / again
|
||||
plans retry <id> ask a failed plan's failed builds again, and carry the plan on
|
||||
plans stop|close <id> --why <text> end a plan by hand; recorded in the hand-act log
|
||||
hand-act record <what> --why <text> --cause <word> [--condition <key>]
|
||||
record an act done by hand outside the mesh (to-be 45 §7)
|
||||
hand-acts [--days N] [--json] what was done by hand lately, why, and which causes repeat
|
||||
durations [--kind K] [--days N] [--json]
|
||||
apply, heartbeat, plan-tier and build durations, per machine or module
|
||||
collection [--json] kept archives held/unheld by a manifest, and what the sweep may let go
|
||||
builder issue <name> a broker account for a build machine, scoped to build work,
|
||||
delivered as the builder module's broker secret (module add it first)
|
||||
@@ -238,7 +252,8 @@ func usage() {
|
||||
which provider this one gets a provision from: the module, and its node
|
||||
unpin <node> <provision> put that question back
|
||||
plan <node> [--files|--json] what that node would run, and why
|
||||
push [<node>] [--behind] send a node everything it should be, or only those that need it
|
||||
push [<node>] [--behind] [--why <text>] send a node everything it should be, or only those
|
||||
that need it; --why records it in the hand-act log
|
||||
version what this binary is
|
||||
|
||||
Each context reaches its own store through its own credential (novox/hq ADR 0008), named
|
||||
@@ -291,6 +306,9 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
||||
// When it was asked, so a plan takes as its outcome only a build asked for it or after it
|
||||
// (novox/hq 04-ISSUES/219). Zero when the id does not say.
|
||||
asked, _ := link.BuildAskedAt(result.ID)
|
||||
// And how long it took, asked to heard, which a build's bound will be set from (novox/hq to-be 45
|
||||
// Phase 0). Said if lost; never a reason not to take the build in.
|
||||
recordBuildDuration(ctx, b.inv, result, asked)
|
||||
switch {
|
||||
case err != nil && result.Failed != "":
|
||||
fmt.Printf("%s: %v\n", result.ID, err)
|
||||
@@ -317,5 +335,7 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
||||
result.ID, manifest.Module, manifest.Version, result.On, short(result.Commit))
|
||||
saysWhenThePolicyActs(ctx, b.inv, manifest.Module)
|
||||
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
|
||||
// A module registered may be one a machine is now behind: `status` is composed again.
|
||||
statusFrom.nudge()
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -387,7 +387,7 @@ func brokerCommand(ctx context.Context, args []string) error {
|
||||
return busAccounts(ctx, args[1:])
|
||||
}
|
||||
if len(args) > 0 && args[0] == "consumer-reset" {
|
||||
return consumerReset(args[1:])
|
||||
return consumerReset(ctx, args[1:])
|
||||
}
|
||||
if len(args) == 0 || args[0] != "show" {
|
||||
return errors.New("broker show | broker certificate [--check] --into <directory> | broker accounts --into <file> | " +
|
||||
@@ -414,10 +414,21 @@ func brokerCommand(ctx context.Context, args []string) error {
|
||||
// consumerReset re-makes one consumer on a stream that keeps history to start from now (novox/hq issue
|
||||
// 248): the way out of a consumer replaying a week of announcements, said rather than done by hand. A
|
||||
// person's act — what was pending is dropped — so it is a command, and nothing calls it on its own.
|
||||
func consumerReset(args []string) error {
|
||||
if len(args) != 2 {
|
||||
return errors.New("broker consumer-reset <stream> <consumer>, e.g. broker consumer-reset EVENTS controller")
|
||||
func consumerReset(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("broker consumer-reset", flag.ContinueOnError)
|
||||
// A repair by hand, which says why (novox/hq to-be 45 §7).
|
||||
why := addHandActFlags(set)
|
||||
args, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(args) != 2 {
|
||||
return errors.New("broker consumer-reset <stream> <consumer> --why <text>, e.g. broker consumer-reset EVENTS controller --why ...")
|
||||
}
|
||||
if err := why.require("broker consumer-reset"); err != nil {
|
||||
return err
|
||||
}
|
||||
why.record(ctx, "broker consumer-reset", args)
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -105,7 +105,10 @@ func serve(ctx context.Context) error {
|
||||
|
||||
work := link.Enrolment{Inventory: inv, Identity: ident, Broker: known,
|
||||
OnNATS: true}
|
||||
server, err := connectLink(ctx, inv, work, work)
|
||||
// `status` from a summary kept current here (novox/hq to-be 45 Phase 0): a machine saying
|
||||
// something new is one thing that moves it, so the listener nudges it.
|
||||
statusFrom = newStatusSummary(composeStatus(open))
|
||||
server, err := connectLink(ctx, inv, work, nudgingListener{work, statusFrom})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -121,6 +124,8 @@ func serve(ctx context.Context) error {
|
||||
// Open plans move on a timer as well as on outcomes (novox/hq ADR 0162): a tier waiting for
|
||||
// machines to report moves when they have, and a plan left by a replaced controller resumes.
|
||||
go planTicker(ctx, open)
|
||||
// The durations the core's bounds are set from are kept a month (novox/hq to-be 45 Phase 0).
|
||||
go forgettingOldDurations(ctx, inv)
|
||||
// And what the catalogue decided a build meant. The builder's own result is already handled
|
||||
// above; this is the other half — the control plane is the only one of the three that knows
|
||||
// which machines run the thing, so it is the one that acts (novox/hq ADR 0072).
|
||||
@@ -158,8 +163,21 @@ func serve(ctx context.Context) error {
|
||||
if !isNATS {
|
||||
return errors.New("the mesh's verbs are served over the bus, and this control plane is not on it")
|
||||
}
|
||||
// The hand-act log is counted for `status` on this connection rather than a new one a minute.
|
||||
handActConn = bus.Conn
|
||||
// Composed now and kept current, before the verb that answers from it is served.
|
||||
go statusFrom.keep(ctx)
|
||||
// A call that outlasts its caller's patience is followed by `calls` (novox/hq issue 265).
|
||||
link.Calls.Follow = catalogue.ControllerSeatName + ".calls"
|
||||
// And every call is kept on the bus, so a restart of this process keeps what came of each
|
||||
// (novox/hq to-be 45 §6). A bus without the bucket is said and served from memory, as before:
|
||||
// answering no calls at all would be worse than answering them without the record.
|
||||
said := log.New(os.Stdout, "", log.LstdFlags)
|
||||
if keeper, err := link.CallsOnTheBus(ctx, bus.Conn); err != nil {
|
||||
fmt.Printf("calls are kept in memory only, and lost when this controller stops: %v\n", err)
|
||||
} else if err := link.Calls.Durably(ctx, keeper, controllerProcess(), said); err != nil {
|
||||
fmt.Printf("calls are kept on the bus from now on; the ones kept before could not be read: %v\n", err)
|
||||
}
|
||||
stopServing, err := bus.ServeSeatTools(catalogue.ControllerSeatName, handlers, log.New(os.Stdout, "", log.LstdFlags))
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -260,6 +278,9 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
// (novox/hq ADR 0010). 0 waits for nothing, which is the old fire-and-forget.
|
||||
wait := set.Duration("wait", 0,
|
||||
"for a named node, how long to wait for it to report applying what it was sent (0: do not wait)")
|
||||
// A push by hand is a repair, and says why (novox/hq to-be 45 §7): required through the seat,
|
||||
// recorded when given at a shell — see handacts.go for why a shell is not refused.
|
||||
why := addHandActFlags(set)
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -274,6 +295,11 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
return errors.New("push <node> or push --behind, not both: one names a machine and the " +
|
||||
"other asks which machines need one")
|
||||
}
|
||||
recorded := append([]string(nil), args...)
|
||||
if *behind {
|
||||
recorded = append(recorded, "--behind")
|
||||
}
|
||||
why.record(ctx, "push", recorded)
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1039,6 +1065,20 @@ func digestOf(body []byte) string {
|
||||
// be worked out" is a different problem with a different remedy, and `plan` is where it is said.
|
||||
func wouldSend(ctx context.Context, open *stores,
|
||||
nodes []inventory.Node) (map[string]string, error) {
|
||||
return wouldSendFrom(ctx, open, nodes, nil)
|
||||
}
|
||||
|
||||
// planned is one machine's plan as planFor answered it, for a caller that already asked.
|
||||
type planned struct {
|
||||
plan catalogue.Resolution
|
||||
settings catalogue.SettingsBy
|
||||
}
|
||||
|
||||
// wouldSendFrom is wouldSend reusing the plans a caller worked out a moment before: resolving a
|
||||
// machine is most of what `status` costs, and it used to resolve every machine twice (novox/hq
|
||||
// to-be 45 Phase 0). A machine absent from plans is worked out here.
|
||||
func wouldSendFrom(ctx context.Context, open *stores,
|
||||
nodes []inventory.Node, plans map[string]planned) (map[string]string, error) {
|
||||
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
@@ -1046,9 +1086,12 @@ func wouldSend(ctx context.Context, open *stores,
|
||||
}
|
||||
out := map[string]string{}
|
||||
for _, n := range nodes {
|
||||
plan, settings, err := planFor(ctx, open, n.Name)
|
||||
if err != nil {
|
||||
continue
|
||||
known, have := plans[n.Name]
|
||||
plan, settings := known.plan, known.settings
|
||||
if !have {
|
||||
if plan, settings, err = planFor(ctx, open, n.Name); err != nil {
|
||||
continue
|
||||
}
|
||||
}
|
||||
declared, err := declarationWith(ctx, open, n.Name, plan, settings, gens, Reading)
|
||||
if err != nil {
|
||||
@@ -1136,6 +1179,11 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string)
|
||||
if err := broker.RaiseCancelledSets(js, inventory.MeshSeats()); err != nil {
|
||||
return err
|
||||
}
|
||||
// And the controller's own buckets (novox/hq to-be 45 §1): the calls it serves and the acts done
|
||||
// by hand, kept where a restart of this process does not take them.
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
return err
|
||||
}
|
||||
// Every module's state (novox/hq ADR 0201), from the catalogue: a bucket exists from
|
||||
// registration, so a module reading one may watch it before its owner runs anywhere. One that
|
||||
// nothing declares any more is said and kept — what it holds is data.
|
||||
@@ -1271,3 +1319,11 @@ func reportUnheldPushed(w io.Writer, named bool, asked []string, unheld map[stri
|
||||
fmt.Fprintf(w, "%s: %d unmet seat dependenc(ies) — see `status`\n", node, len(lines))
|
||||
}
|
||||
}
|
||||
|
||||
// controllerProcess names this serving process among controllers: the machine, the process and when
|
||||
// it started — what a call kept on the bus carries, so the next controller can tell a call this one
|
||||
// left running from one it is running itself (novox/hq to-be 45 §6).
|
||||
func controllerProcess() string {
|
||||
host, _ := os.Hostname()
|
||||
return fmt.Sprintf("controller@%s pid %d since %s", host, os.Getpid(), time.Now().UTC().Format(time.RFC3339))
|
||||
}
|
||||
|
||||
@@ -78,6 +78,11 @@ type meshStatus struct {
|
||||
// that machine holds, with the modules that could hold it (novox/hq ADR 0207). Absent when every
|
||||
// dependency is met. Reported, not refused, until the switch.
|
||||
Unheld []catalogue.Unheld `json:"unheld,omitempty"`
|
||||
// HandActsThisWeek is how many acts were done by hand in the last seven days (novox/hq to-be 45
|
||||
// §7): every one is a repair a healer could have made. Absent where the log is not on hand;
|
||||
// HandActsUnread says why when it could not be read, rather than reading as none.
|
||||
HandActsThisWeek *int `json:"handActsThisWeek,omitempty"`
|
||||
HandActsUnread string `json:"handActsUnread,omitempty"`
|
||||
// Failing is every consumer a provider says it keeps failing, with the class of error, since
|
||||
// when, and when it was last said (novox/hq ADR 0224). Absent when no provider says so. A
|
||||
// document without this called the mesh well while the identity provider refused every consumer
|
||||
@@ -218,6 +223,7 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
||||
}
|
||||
}
|
||||
out.Unheld = asked.unheld
|
||||
out.HandActsThisWeek, out.HandActsUnread = asked.handActs, asked.handActsUnread
|
||||
out.Failing = asked.failing
|
||||
out.Overflowing = asked.overflowing
|
||||
for name := range asked.refused {
|
||||
|
||||
@@ -986,10 +986,18 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
whatIf := set.String("what-if", "", "owner/repository: the plan a merge there would produce, saving nothing — with --paths or --modules")
|
||||
paths := set.String("paths", "", "the files the merge would change, comma-separated, from the repository's root")
|
||||
modules := set.String("modules", "", "or the modules it would change, comma-separated")
|
||||
// Ending a plan by hand is a repair, and says why (novox/hq to-be 45 §7).
|
||||
why := addHandActFlags(set)
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) == 2 && (positionals[0] == "stop" || positionals[0] == "close") {
|
||||
// Refused before anything is opened: a repair by hand says why.
|
||||
if err := why.require("plans " + positionals[0]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1061,8 +1069,9 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
if !p.Open() {
|
||||
return fmt.Errorf("%s is already %s", p.ID, p.State)
|
||||
}
|
||||
why.record(ctx, "plans "+positionals[0], positionals[1:])
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = how + " by hand at tier " + fmt.Sprint(p.Tier)
|
||||
p.Note = how + " by hand at tier " + fmt.Sprint(p.Tier) + ": " + strings.TrimSpace(*why.why)
|
||||
sayUnsent(&p, func(m string) bool {
|
||||
u, err := inv.UpgradeOf(ctx, m)
|
||||
return err == nil && u.RollOut
|
||||
|
||||
@@ -9,9 +9,11 @@ import (
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
"os"
|
||||
"os/exec"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
@@ -239,6 +241,12 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
if len(argv) == 0 {
|
||||
return nil, errors.New("command names no command")
|
||||
}
|
||||
// The generic verb is no way round the hand-act log (novox/hq to-be 45 §7): a repair through
|
||||
// it says why, as it would through its own verb.
|
||||
if repair := repairingCommand(argv); repair != "" && !slices.ContainsFunc(argv, isWhyFlag) {
|
||||
return nil, fmt.Errorf("%s is a repair done by hand, and says why: add --why <text> to the command "+
|
||||
"line (recorded in the hand-act log). Nothing was done", repair)
|
||||
}
|
||||
return argv, nil
|
||||
case "tools":
|
||||
return nil, errors.New("tools is answered from the records, not by a command")
|
||||
@@ -280,7 +288,18 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
}
|
||||
for _, act := range []string{"stop", "close", "retry"} {
|
||||
if id := str(act); id != "" {
|
||||
return []string{"plans", act, id}, nil
|
||||
argv := []string{"plans", act, id}
|
||||
if act == "retry" {
|
||||
return argv, nil
|
||||
}
|
||||
// Ending a plan by hand says why (novox/hq to-be 45 §7); the command refuses it without.
|
||||
if w := str("why"); w != "" {
|
||||
argv = append(argv, "--why", w)
|
||||
}
|
||||
if c := str("cause"); c != "" {
|
||||
argv = append(argv, "--cause", c)
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
}
|
||||
if id := str("id"); id != "" {
|
||||
@@ -355,16 +374,48 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
// Sent and not waited for: the asker reads `status` for what the machine did, which is
|
||||
// what a person at a shell does too. A tool call that blocked for a push's whole apply would
|
||||
// time out on every machine that takes a minute, and say nothing about the ones that did not.
|
||||
// A push through the seat is a push by hand, and says why (novox/hq to-be 45 §7).
|
||||
if err := need("why"); err != nil {
|
||||
return nil, fmt.Errorf("%w: a push by hand is a repair, recorded in the hand-act log with why", err)
|
||||
}
|
||||
why := []string{"--why", str("why")}
|
||||
if c := str("cause"); c != "" {
|
||||
why = append(why, "--cause", c)
|
||||
}
|
||||
if n := str("node"); n != "" {
|
||||
// behind is not read here: given with a machine, it is refused as passed over — naming
|
||||
// a machine and asking for every machine behind are two requests, and guessing one
|
||||
// would push a machine nobody named, or not push one somebody did.
|
||||
return []string{"push", n, "--wait", "0"}, nil
|
||||
return append([]string{"push", n, "--wait", "0"}, why...), nil
|
||||
}
|
||||
// No machine: the whole mesh, whether or not behind said so. The command's answer says it
|
||||
// first, so a caller who meant one machine reads that it was not one.
|
||||
on("behind")
|
||||
return []string{"push", "--behind", "--wait", "0"}, nil
|
||||
return append([]string{"push", "--behind", "--wait", "0"}, why...), nil
|
||||
case "hand-act":
|
||||
if err := need("what", "why", "cause"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argv := []string{"hand-act", "record", str("what"), "--why", str("why"), "--cause", str("cause")}
|
||||
if c := str("condition"); c != "" {
|
||||
argv = append(argv, "--condition", c)
|
||||
}
|
||||
return argv, nil
|
||||
case "hand-acts":
|
||||
argv := []string{"hand-acts", "--json"}
|
||||
if d := str("days"); d != "" {
|
||||
argv = append(argv, "--days", d)
|
||||
}
|
||||
return argv, nil
|
||||
case "durations":
|
||||
argv := []string{"durations", "--json"}
|
||||
if k := str("kind"); k != "" {
|
||||
argv = append(argv, "--kind", k)
|
||||
}
|
||||
if d := str("days"); d != "" {
|
||||
argv = append(argv, "--days", d)
|
||||
}
|
||||
return argv, nil
|
||||
case "rotate":
|
||||
if p := str("provision"); p != "" {
|
||||
argv := []string{"rotate", p}
|
||||
@@ -442,7 +493,28 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
}
|
||||
|
||||
// jsonVerbs are the verbs whose command speaks JSON, so the answer carries it as data as well.
|
||||
var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "collection": true}
|
||||
var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "collection": true,
|
||||
"hand-acts": true, "durations": true}
|
||||
|
||||
// repairingCommand names a command line that repairs by hand, and so says why: a push, a plan stopped
|
||||
// or closed, a consumer re-made (novox/hq to-be 45 §7). Empty for any other.
|
||||
func repairingCommand(argv []string) string {
|
||||
switch {
|
||||
case argv[0] == "push":
|
||||
return "push"
|
||||
case argv[0] == "plans" && len(argv) > 1 && (argv[1] == "stop" || argv[1] == "close"):
|
||||
return "plans " + argv[1]
|
||||
case argv[0] == "broker" && len(argv) > 1 && argv[1] == "consumer-reset":
|
||||
return "broker consumer-reset"
|
||||
case argv[0] == "hand-act":
|
||||
return "hand-act record"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func isWhyFlag(word string) bool {
|
||||
return word == "--why" || word == "-why" || strings.HasPrefix(word, "--why=") || strings.HasPrefix(word, "-why=")
|
||||
}
|
||||
|
||||
// runVerb runs this binary with the given command line and gathers what it said.
|
||||
func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
||||
@@ -454,6 +526,12 @@ func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
||||
// The same environment: the stores' credentials, the bus, the broker — everything a command run
|
||||
// from a shell in this container would have, because it is that.
|
||||
cmd.Env = os.Environ()
|
||||
// And who asked, so an act it does by hand is recorded as theirs (novox/hq to-be 45 §7).
|
||||
caller := link.CallerIn(ctx)
|
||||
if caller == "" {
|
||||
caller = "a seat call whose caller the bus did not name"
|
||||
}
|
||||
cmd.Env = append(cmd.Env, link.CallerVar+"="+caller+", through the "+catalogue.ControllerSeatName+" seat")
|
||||
// Two buffers, one answer. What the command *says* is both streams, in the order a person at
|
||||
// a shell would read them; what it *answers as data* is standard output alone — `status --json`
|
||||
// prints its warnings beside the document, and a JSON parsed from the two together parsed
|
||||
@@ -544,6 +622,14 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if verb == "status" && statusFrom != nil {
|
||||
// At once, from the summary the serving controller keeps (novox/hq to-be 45 Phase 0).
|
||||
return statusFrom.answer(ctx)
|
||||
}
|
||||
if !readingVerbs[verb] && !(verb == "plans" && !actsOnAPlan(args)) {
|
||||
// Whatever it did, `status` is composed again once it has.
|
||||
defer statusFrom.nudge()
|
||||
}
|
||||
if answersFirst(argv) {
|
||||
// Before anything is sent: a push sends the bus's own machine first, and a broker
|
||||
// reloading its user list forgets the answer it was about to permit (novox/hq issue 265).
|
||||
@@ -555,6 +641,16 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
||||
return handlers, behind, nil
|
||||
}
|
||||
|
||||
// actsOnAPlan is `plans` asked to stop, close or retry one rather than to show them.
|
||||
func actsOnAPlan(args map[string]any) bool {
|
||||
for _, act := range []string{"stop", "close", "retry"} {
|
||||
if v, _ := args[act].(string); strings.TrimSpace(v) != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// inProcess are the verbs answered by this process rather than by a command it runs: `tools` from
|
||||
// the records, `calls` from what this process served.
|
||||
var inProcess = map[string]bool{"tools": true, "calls": true}
|
||||
@@ -568,22 +664,41 @@ func answersFirst(argv []string) bool {
|
||||
}
|
||||
|
||||
// callsAnswer is what `calls` answers: the kept calls, newest first, without their answers — or
|
||||
// one call whole.
|
||||
// one call whole. Kept on the bus, so a call a controller before this one served is answered too
|
||||
// (novox/hq to-be 45 §6); where the bus cannot be read, what this process served is answered and
|
||||
// the reason said beside it.
|
||||
func callsAnswer(log *link.CallLog, id string) (any, error) {
|
||||
if id != "" {
|
||||
c, ok := log.Get(id)
|
||||
c, ok, err := log.Get(id)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("call %s is not in this controller's memory, and the calls kept on the "+
|
||||
"bus could not be read: %w", id, err)
|
||||
}
|
||||
if !ok {
|
||||
if log.IsDurable() {
|
||||
return nil, fmt.Errorf("no call %s is kept: the bus keeps the last %d calls, or %s, and this "+
|
||||
"is not among them — `calls` lists them", id, broker.KeptCallsDurably, broker.CallsKeptFor)
|
||||
}
|
||||
return nil, fmt.Errorf("no call %s is kept here: calls are kept by the controller that "+
|
||||
"answered them, the last %d, and not across a restart — `calls` lists them", id, link.KeptCalls)
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
recent := log.Recent()
|
||||
recent, err := log.Recent()
|
||||
for i := range recent {
|
||||
recent[i].Answer = nil
|
||||
}
|
||||
return map[string]any{"calls": recent, "kept": link.KeptCalls,
|
||||
"note": "newest first; `calls` with a call's id gives its whole answer"}, nil
|
||||
answer := map[string]any{"calls": recent, "note": "newest first; `calls` with a call's id gives its whole answer"}
|
||||
if log.IsDurable() {
|
||||
answer["kept"] = fmt.Sprintf("the last %d calls, or %s, on the bus — across a restart of the controller",
|
||||
broker.KeptCallsDurably, broker.CallsKeptFor)
|
||||
} else {
|
||||
answer["kept"] = fmt.Sprintf("the last %d calls this controller served, in its memory only", link.KeptCalls)
|
||||
}
|
||||
if err != nil {
|
||||
answer["unread"] = err.Error()
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// seatTools is what `tools` answers: every seat with a protocol, and the tools each serves, from the
|
||||
|
||||
@@ -145,17 +145,17 @@ func TestAnArgumentAVerbDoesNotDeclareIsRefused(t *testing.T) {
|
||||
// The push that was the cause: a machine named is that machine; none named is the whole mesh, and
|
||||
// naming one beside behind is refused rather than one of the two guessed.
|
||||
func TestAPushIsOneMachineOrSaysItIsTheWholeMesh(t *testing.T) {
|
||||
argv, err := argvFor("push", map[string]any{"node": "g1"})
|
||||
if err != nil || strings.Join(argv, " ") != "push g1 --wait 0" {
|
||||
argv, err := argvFor("push", map[string]any{"node": "g1", "why": "w"})
|
||||
if err != nil || strings.Join(argv, " ") != "push g1 --wait 0 --why w" {
|
||||
t.Fatalf("a named push: %v %v", argv, err)
|
||||
}
|
||||
for _, args := range []map[string]any{{}, {"behind": "true"}} {
|
||||
for _, args := range []map[string]any{{"why": "w"}, {"behind": "true", "why": "w"}} {
|
||||
argv, err := argvFor("push", args)
|
||||
if err != nil || strings.Join(argv, " ") != "push --behind --wait 0" {
|
||||
if err != nil || strings.Join(argv, " ") != "push --behind --wait 0 --why w" {
|
||||
t.Fatalf("a push of the whole mesh %v: %v %v", args, argv, err)
|
||||
}
|
||||
}
|
||||
if _, err := argvFor("push", map[string]any{"node": "g1", "behind": "true"}); err == nil ||
|
||||
if _, err := argvFor("push", map[string]any{"node": "g1", "behind": "true", "why": "w"}); err == nil ||
|
||||
!strings.Contains(err.Error(), `"behind"`) {
|
||||
t.Fatalf("a named push with behind was taken: %v", err)
|
||||
}
|
||||
@@ -268,6 +268,11 @@ var accountedFlags = map[string]map[string]string{
|
||||
},
|
||||
"builds": {"n": "=limit"},
|
||||
"plans": {"n": "=limit", "what-if": "=repository"},
|
||||
"durations": {
|
||||
"json": "set by the verb: the answer is data",
|
||||
"all": "withheld: every measurement of a fortnight is more than a call should carry; `command` reaches it",
|
||||
},
|
||||
"hand-acts": {"json": "set by the verb: the answer is data"},
|
||||
}
|
||||
|
||||
// **Every flag of the command a verb runs is in the verb's schema, or accounted for here.** Derived
|
||||
|
||||
@@ -107,8 +107,8 @@ func TestAVerbMissingWhatItNeedsIsRefused(t *testing.T) {
|
||||
// A push and a build are sent, not waited for: the asker reads status, or the build's log by its
|
||||
// id, for what happened. A repository given as a forge path is said to be one (issue 176).
|
||||
func TestActsDoNotBlockTheCall(t *testing.T) {
|
||||
argv, _ := argvFor("push", map[string]any{"node": "one"})
|
||||
if strings.Join(argv, " ") != "push one --wait 0" {
|
||||
argv, _ := argvFor("push", map[string]any{"node": "one", "why": "w"})
|
||||
if strings.Join(argv, " ") != "push one --wait 0 --why w" {
|
||||
t.Fatalf("push waits: %v", argv)
|
||||
}
|
||||
argv, _ = argvFor("build", map[string]any{"repository": "novox/x", "path": "modules/x"})
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
@@ -315,6 +316,16 @@ func printStatus(asked answers) error {
|
||||
fmt.Printf("\n a shorter `slug` in the module's definition fits it; `module check` refuses one before merge\n\n")
|
||||
}
|
||||
|
||||
// Repairs done by hand this week (novox/hq to-be 45 §7). Not a fault, so it does not break "all
|
||||
// well"; each is a healer the mesh does not have yet, and the count is how that is watched.
|
||||
switch {
|
||||
case asked.handActsUnread != "":
|
||||
fmt.Printf("the hand-act log could not be read, so how much was done by hand this week is not known: %s\n\n",
|
||||
asked.handActsUnread)
|
||||
case asked.handActs != nil && *asked.handActs > 0:
|
||||
fmt.Printf("%d act(s) done by hand in the last seven days — `hand-acts` lists them, and why\n\n", *asked.handActs)
|
||||
}
|
||||
|
||||
if adopted := adoptedNodes(nodes); len(adopted) > 0 {
|
||||
// Said, because nothing forces the flip: a node left adopted is visible here rather than
|
||||
// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
|
||||
@@ -423,14 +434,16 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
||||
// ADR 0207). Each machine resolved again rather than threaded through whoResolves, whose answer
|
||||
// the private network is built from and should say nothing else; a machine that does not
|
||||
// resolve is already in refused, and is passed over here.
|
||||
plans := map[string]planned{}
|
||||
for _, n := range out.nodes {
|
||||
plan, _, err := planFor(ctx, open, n.Name)
|
||||
plan, settings, err := planFor(ctx, open, n.Name)
|
||||
if err != nil {
|
||||
if unresolvable(err) {
|
||||
continue
|
||||
}
|
||||
return answers{}, err
|
||||
}
|
||||
plans[n.Name] = planned{plan, settings}
|
||||
out.unheld = append(out.unheld, plan.Unheld...)
|
||||
// And which of its modules a provider leaves out of its grants, for an identity too long
|
||||
// for what the provision keeps (novox/hq ADR 0225) — judged from the consumer's own
|
||||
@@ -449,6 +462,16 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
||||
}
|
||||
// Whether the build seat takes work, for a plan waiting on it (novox/hq ADR 0219).
|
||||
out.paused = buildSeatPause(ctx, inv, out.plans)
|
||||
// And how many repairs were done by hand this week (novox/hq to-be 45 §7) — where there is a bus
|
||||
// to read the log from; a process with none has no log to count.
|
||||
if _, onBus := broker.BusAddress(); onBus == nil {
|
||||
n, unread := handActsThisWeek(ctx)
|
||||
if unread != "" {
|
||||
out.handActsUnread = unread
|
||||
} else {
|
||||
out.handActs = &n
|
||||
}
|
||||
}
|
||||
|
||||
// And which machines are not running what the mesh would send them. The same question as a
|
||||
// module being behind its source, one level down: that one says the catalogue is out of date,
|
||||
@@ -460,7 +483,7 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
||||
// said nothing at all and `status --json` emitted prose to stderr and no JSON anywhere. The
|
||||
// reason is kept and reported as data; every question that does not depend on it is still
|
||||
// answered.
|
||||
would, err := wouldSend(ctx, open, out.nodes)
|
||||
would, err := wouldSendFrom(ctx, open, out.nodes, plans)
|
||||
if err != nil {
|
||||
out.network = err.Error()
|
||||
would = map[string]string{}
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// `status` answered from a summary the serving controller keeps current (novox/hq to-be 45 Phase 0,
|
||||
// §4's D9 and §8's health of the controller).
|
||||
//
|
||||
// **Asked, it was composed: every machine resolved, twice, while its caller waited.** On 2026-10-06
|
||||
// the verb took eighteen seconds on a mesh of four machines, so its caller read "still running" and
|
||||
// had to ask `calls` for the answer to "is the mesh alright" — the one question that must answer at
|
||||
// once, and the one a self-check and a rollout gate will ask every few minutes. So the serving
|
||||
// controller composes it in the background — at its start, after anything that changes what it says
|
||||
// (a machine's report, a build, a verb that acts), and every minute regardless — and the verb answers
|
||||
// the last composition at once, saying when it was composed and how long that took. A caller who
|
||||
// needs it newer than that reads the time and asks again; nothing is answered as current that is not.
|
||||
|
||||
// statusEvery is how often the summary is composed with nothing having nudged it; statusSettle how
|
||||
// long a nudge waits for the next, so a push answered by four machines is composed once.
|
||||
var (
|
||||
statusEvery = time.Minute
|
||||
statusSettle = 2 * time.Second
|
||||
// statusComposeWithin bounds one composition, so a store that hangs cannot stop the summary for
|
||||
// good; the attempt is said as failed, and the last summary stands with its age.
|
||||
statusComposeWithin = 2 * time.Minute
|
||||
)
|
||||
|
||||
// statusSummary is the last composed `status --json`, and when.
|
||||
type statusSummary struct {
|
||||
compose func(context.Context) ([]byte, error)
|
||||
// every, settle and within are the clocks above, read once when it is made.
|
||||
every, settle, within time.Duration
|
||||
|
||||
mu sync.Mutex
|
||||
body []byte
|
||||
composedAt time.Time
|
||||
took time.Duration
|
||||
failed string
|
||||
failedAt time.Time
|
||||
started time.Time
|
||||
first chan struct{} // closed when the first attempt ends, either way
|
||||
|
||||
nudged chan struct{}
|
||||
}
|
||||
|
||||
func newStatusSummary(compose func(context.Context) ([]byte, error)) *statusSummary {
|
||||
return &statusSummary{compose: compose, started: time.Now(), first: make(chan struct{}),
|
||||
nudged: make(chan struct{}, 1), every: statusEvery, settle: statusSettle, within: statusComposeWithin}
|
||||
}
|
||||
|
||||
// statusFrom is the serving controller's summary; nil in any other process, where `status` is
|
||||
// composed when asked, as at a shell.
|
||||
var statusFrom *statusSummary
|
||||
|
||||
// nudge asks for a composition soon. Never blocks: one pending is as good as many.
|
||||
func (s *statusSummary) nudge() {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case s.nudged <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// keep composes until ctx ends: now, on a nudge once things settle, and every statusEvery.
|
||||
func (s *statusSummary) keep(ctx context.Context) {
|
||||
once := sync.Once{}
|
||||
for {
|
||||
s.composeOnce(ctx)
|
||||
once.Do(func() { close(s.first) })
|
||||
timer := time.NewTimer(s.every)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return
|
||||
case <-timer.C:
|
||||
case <-s.nudged:
|
||||
timer.Stop()
|
||||
// Let what else is arriving arrive, then compose once for all of it.
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(s.settle):
|
||||
}
|
||||
select {
|
||||
case <-s.nudged:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *statusSummary) composeOnce(ctx context.Context) {
|
||||
start := time.Now()
|
||||
asking, cancel := context.WithTimeout(ctx, s.within)
|
||||
body, err := s.compose(asking)
|
||||
cancel()
|
||||
took := time.Since(start)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if err != nil {
|
||||
s.failed, s.failedAt = err.Error(), time.Now()
|
||||
fmt.Printf("status could not be composed (after %s): %v — `status` answers the last summary, "+
|
||||
"with its age\n", took.Round(time.Millisecond), err)
|
||||
return
|
||||
}
|
||||
s.body, s.composedAt, s.took, s.failed = body, start, took, ""
|
||||
}
|
||||
|
||||
// answer is what the `status` verb answers: the last summary at once, the same document `status
|
||||
// --json` prints, with when it was composed. Before the first composition has ended it waits for it,
|
||||
// but never past the caller's window; a controller that has none says so and why, rather than
|
||||
// answering an empty mesh as a well one.
|
||||
func (s *statusSummary) answer(ctx context.Context) (any, error) {
|
||||
wait := time.NewTimer(link.AnswerWithin - time.Second)
|
||||
defer wait.Stop()
|
||||
select {
|
||||
case <-s.first:
|
||||
case <-wait.C:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.body == nil {
|
||||
why := "its first composition has not finished"
|
||||
if s.failed != "" {
|
||||
why = "it could not be composed: " + s.failed
|
||||
}
|
||||
return nil, fmt.Errorf("this controller started %s ago and has no status to answer yet — %s. "+
|
||||
"Ask again shortly", time.Since(s.started).Round(time.Second), why)
|
||||
}
|
||||
var parsed any
|
||||
_ = json.Unmarshal(s.body, &parsed)
|
||||
out := map[string]any{
|
||||
"output": string(s.body), "ok": true, "answer": parsed,
|
||||
"composed": s.composedAt.UTC().Format(time.RFC3339),
|
||||
"age": time.Since(s.composedAt).Round(time.Second).String(),
|
||||
"composedIn": s.took.Round(time.Millisecond).String(),
|
||||
"note": "composed by the serving controller at its start, after each report, build or act, and " +
|
||||
"every minute; answered at once from the last composition",
|
||||
}
|
||||
if s.failed != "" && s.failedAt.After(s.composedAt) {
|
||||
out["lastAttemptFailed"] = fmt.Sprintf("%s: %s — this summary is the last that could be composed",
|
||||
s.failedAt.UTC().Format(time.RFC3339), s.failed)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// composeStatus is `status --json`, composed in this process against its stores.
|
||||
func composeStatus(open *stores) func(context.Context) ([]byte, error) {
|
||||
return func(ctx context.Context) ([]byte, error) {
|
||||
asked, err := theThreeQuestions(ctx, open)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return statusAsJSON(asked)
|
||||
}
|
||||
}
|
||||
|
||||
// readingVerbs are the verbs that only read; after any other, what `status` says may have changed, so the summary is
|
||||
// composed again; a verb that only reads leaves it alone, or a console polling `nodes` would keep the
|
||||
// controller composing for ever.
|
||||
var readingVerbs = map[string]bool{
|
||||
"tools": true, "calls": true, "status": true, "nodes": true, "node": true, "modules": true,
|
||||
"seats": true, "builds": true, "plan": true, "queue": true, "durations": true, "hand-acts": true,
|
||||
}
|
||||
|
||||
// nudgingListener is the enrolment, nudging the summary when a machine said something new.
|
||||
type nudgingListener struct {
|
||||
link.Enrolment
|
||||
summary *statusSummary
|
||||
}
|
||||
|
||||
func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, error) {
|
||||
news, err := l.Enrolment.Heard(ctx, report)
|
||||
if news {
|
||||
l.summary.nudge()
|
||||
}
|
||||
return news, err
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// quickly shortens the summary's clocks for one test.
|
||||
func quickly(t *testing.T) {
|
||||
t.Helper()
|
||||
every, settle := statusEvery, statusSettle
|
||||
statusEvery, statusSettle = time.Hour, 10*time.Millisecond
|
||||
t.Cleanup(func() { statusEvery, statusSettle = every, settle })
|
||||
}
|
||||
|
||||
// **`status` answers in full within ten seconds, five times in a row** (novox/hq to-be 45 Phase 0,
|
||||
// D9) — however long composing it takes. On 2026-10-06 composing took eighteen seconds and the
|
||||
// verb answered "still running"; from the summary it answers at once, in full, saying when.
|
||||
func TestStatusAnswersAtOnceHoweverLongComposingTakes(t *testing.T) {
|
||||
quickly(t)
|
||||
var composed atomic.Int32
|
||||
slow := make(chan struct{})
|
||||
s := newStatusSummary(func(ctx context.Context) ([]byte, error) {
|
||||
if composed.Add(1) > 1 {
|
||||
<-slow // every composition after the first outlasts any caller
|
||||
}
|
||||
return []byte(`{"wrong":[],"machines":4}`), nil
|
||||
})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
defer close(slow)
|
||||
go s.keep(ctx)
|
||||
for i := 0; i < 5; i++ {
|
||||
s.nudge() // a composition is under way and does not finish
|
||||
start := time.Now()
|
||||
got, err := s.answer(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if took := time.Since(start); took > time.Second {
|
||||
t.Fatalf("answer %d took %s", i+1, took)
|
||||
}
|
||||
m := got.(map[string]any)
|
||||
if m["answer"].(map[string]any)["machines"] != float64(4) || m["composed"] == "" || m["ok"] != true {
|
||||
t.Fatalf("answer %d was not in full: %v", i+1, m)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The first composition is waited for, never past the caller's window; a controller with none yet
|
||||
// says so rather than answering an empty mesh as a well one.
|
||||
func TestStatusBeforeItsFirstCompositionSaysSo(t *testing.T) {
|
||||
quickly(t)
|
||||
was := link.AnswerWithin
|
||||
link.AnswerWithin = 1100 * time.Millisecond
|
||||
t.Cleanup(func() { link.AnswerWithin = was })
|
||||
never := make(chan struct{})
|
||||
defer close(never)
|
||||
s := newStatusSummary(func(context.Context) ([]byte, error) { <-never; return nil, nil })
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go s.keep(ctx)
|
||||
start := time.Now()
|
||||
_, err := s.answer(ctx)
|
||||
if err == nil || !strings.Contains(err.Error(), "has no status to answer yet") {
|
||||
t.Fatalf("answered %v", err)
|
||||
}
|
||||
if took := time.Since(start); took > link.AnswerWithin {
|
||||
t.Fatalf("waited %s, past the caller's window", took)
|
||||
}
|
||||
}
|
||||
|
||||
// A nudge composes it again, once for several close together; a failed composition leaves the last
|
||||
// summary standing and says it is the last that could be composed.
|
||||
func TestANudgeComposesAgainAndAFailureKeepsTheLastSummary(t *testing.T) {
|
||||
quickly(t)
|
||||
var composed atomic.Int32
|
||||
fail := atomic.Bool{}
|
||||
s := newStatusSummary(func(context.Context) ([]byte, error) {
|
||||
n := composed.Add(1)
|
||||
if fail.Load() {
|
||||
return nil, errors.New("the store did not answer")
|
||||
}
|
||||
body, _ := json.Marshal(map[string]any{"n": n})
|
||||
return body, nil
|
||||
})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go s.keep(ctx)
|
||||
if _, err := s.answer(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.nudge()
|
||||
s.nudge()
|
||||
s.nudge()
|
||||
waitFor(t, func() bool { return composed.Load() == 2 })
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if n := composed.Load(); n != 2 {
|
||||
t.Fatalf("three nudges together composed %d times after the first", n-1)
|
||||
}
|
||||
fail.Store(true)
|
||||
s.nudge()
|
||||
waitFor(t, func() bool { return composed.Load() == 3 })
|
||||
waitFor(t, func() bool {
|
||||
got, err := s.answer(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := got.(map[string]any)
|
||||
return m["lastAttemptFailed"] != nil && m["answer"].(map[string]any)["n"] == float64(2)
|
||||
})
|
||||
}
|
||||
|
||||
// The seat's `status` answers from the summary when this process keeps one.
|
||||
func TestTheStatusVerbAnswersFromTheSummary(t *testing.T) {
|
||||
quickly(t)
|
||||
s := newStatusSummary(func(context.Context) ([]byte, error) { return []byte(`{"from":"summary"}`), nil })
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go s.keep(ctx)
|
||||
statusFrom = s
|
||||
t.Cleanup(func() { statusFrom = nil })
|
||||
handlers, _, err := seatToolHandlers()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := handlers["status"](ctx, json.RawMessage(`{}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.(map[string]any)["answer"].(map[string]any)["from"] != "summary" {
|
||||
t.Fatalf("answered %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func waitFor(t *testing.T, ok func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for !ok() {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("never happened")
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
@@ -75,21 +75,25 @@ func TestAPersonClosesAStuckPlan(t *testing.T) {
|
||||
if err := open.inventory.SavePlan(ctx, stuck); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := plansCommand(ctx, []string{"close", stuck.ID}); err != nil {
|
||||
if err := plansCommand(ctx, []string{"close", stuck.ID}); err == nil || !strings.Contains(err.Error(), "--why") {
|
||||
t.Fatalf("a plan was closed by hand without saying why: %v", err)
|
||||
}
|
||||
if err := plansCommand(ctx, []string{"close", stuck.ID, "--why", "its report will not come"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
closed, err := open.inventory.PlanByID(ctx, stuck.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if closed.State != inventory.PlanFailed || !strings.Contains(closed.Note, "closed by hand") {
|
||||
if closed.State != inventory.PlanFailed || !strings.Contains(closed.Note, "closed by hand") ||
|
||||
!strings.Contains(closed.Note, "its report will not come") {
|
||||
t.Fatalf("the plan was left %s: %q", closed.State, closed.Note)
|
||||
}
|
||||
if err := plansCommand(ctx, []string{"close", stuck.ID}); err == nil {
|
||||
if err := plansCommand(ctx, []string{"close", stuck.ID, "--why", "again"}); err == nil {
|
||||
t.Fatal("a plan already closed was closed again")
|
||||
}
|
||||
if argv, err := argvFor("plans", map[string]any{"close": stuck.ID}); err != nil ||
|
||||
!reflect.DeepEqual(argv, []string{"plans", "close", stuck.ID}) {
|
||||
if argv, err := argvFor("plans", map[string]any{"close": stuck.ID, "why": "w"}); err != nil ||
|
||||
!reflect.DeepEqual(argv, []string{"plans", "close", stuck.ID, "--why", "w"}) {
|
||||
t.Fatalf("the seat's verb does not close a plan: %v %v", argv, err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user