Archived
Compare commits
189
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
81f3c2d01b | ||
|
|
c58516f819 | ||
|
|
54b04a7604 | ||
|
|
af025562c7 | ||
|
|
51c8c7ec52 | ||
|
|
63e85b25e6 | ||
|
|
93c6ca5432 | ||
|
|
14ab2ddd9b | ||
|
|
510c91f144 | ||
|
|
33dc85d850 | ||
|
|
ea1d3ed96d | ||
|
|
eca6390d6f | ||
|
|
5438d7ff75 | ||
|
|
eb7a158086 | ||
|
|
03dac87d3a | ||
|
|
0c47f48537 | ||
|
|
2e1a9abbf7 | ||
|
|
95e7a7120a | ||
|
|
d15eee61bb | ||
|
|
1b780eae3a | ||
|
|
a5e8baf6da | ||
|
|
e168b60159 | ||
|
|
d951f22c04 | ||
|
|
b1897a3498 | ||
|
|
926dbd7a97 | ||
|
|
db702312af | ||
|
|
d7fe5b609b | ||
|
|
b4dff64ae0 | ||
|
|
2b8a28adab | ||
|
|
20d4f1ae71 | ||
|
|
528951319c | ||
|
|
dcbbf4487a | ||
|
|
fcfbf7e69e | ||
|
|
4d60628f37 | ||
|
|
b13a0f71a4 | ||
|
|
822e52123c | ||
|
|
9a7ae131cb | ||
|
|
55d8b43f30 | ||
|
|
4354d9d7c7 | ||
|
|
1fdc68a8aa | ||
|
|
36008e0642 | ||
|
|
f476494173 | ||
|
|
155819f307 | ||
|
|
eb5f9d2f53 | ||
|
|
0f58602c74 | ||
|
|
c3ae3f3e09 | ||
|
|
2a9697cf70 | ||
|
|
e4d2868679 | ||
|
|
fe857ba081 | ||
|
|
b9fc09c375 | ||
|
|
ec7b8bcd58 | ||
|
|
cec797e996 | ||
|
|
0e0ba93f6c | ||
|
|
0f1e1b09fd | ||
|
|
1b502a37e0 | ||
|
|
522f2253e7 | ||
|
|
2073bfe2e6 | ||
|
|
f5824b31c6 | ||
|
|
d059311c0f | ||
|
|
1a28cb3357 | ||
|
|
758537dd4e | ||
|
|
add807f034 | ||
|
|
8ca4b04321 | ||
|
|
8170fc58a3 | ||
|
|
efcdd5dd7d | ||
|
|
5d7d8ee2d6 | ||
|
|
5b7e6ff453 | ||
|
|
cc7fb99f29 | ||
|
|
1e04670052 | ||
|
|
81e5458cbf | ||
|
|
fb74e24c9e | ||
|
|
ca09a07fdf | ||
|
|
9b028b4212 | ||
|
|
d7fab82a89 | ||
|
|
7f65e62743 | ||
|
|
2b01f8786e | ||
|
|
909062e729 | ||
|
|
826dcb91b1 | ||
|
|
193168e086 | ||
|
|
59fdffb979 | ||
|
|
5d47e0bfd6 | ||
|
|
e3ec15f707 | ||
|
|
ac91357a53 | ||
|
|
b5f2c3b961 | ||
|
|
af63b233db | ||
|
|
0cf5a3a5ba | ||
|
|
f5680ba8da | ||
|
|
76babaea52 | ||
|
|
e41b78cd77 | ||
|
|
1acce7132e | ||
|
|
3f68a495f1 | ||
|
|
298ec06ae0 | ||
|
|
a5f53ef9eb | ||
|
|
8adb7f1a05 | ||
|
|
e33da2dc1c | ||
|
|
249d97d1c8 | ||
|
|
7d63d2e68c | ||
|
|
fe00fec52c | ||
|
|
056414bc06 | ||
|
|
59620fdacb | ||
|
|
b74268fb08 | ||
|
|
a44dc01c65 | ||
|
|
c6e372896b | ||
|
|
c5663aa18b | ||
|
|
9907df6530 | ||
|
|
41d6019fa0 | ||
|
|
58e143bbc0 | ||
|
|
d9588b4f13 | ||
|
|
34611c8e38 | ||
|
|
a63939160e | ||
|
|
7ad9dbcb5d | ||
|
|
363898ec8a | ||
|
|
8984c3437f | ||
|
|
9766338368 | ||
|
|
b1acb3d9de | ||
|
|
a759ac65a5 | ||
|
|
2e6a9b1efc | ||
|
|
45ae1d0112 | ||
|
|
76cfbac7a1 | ||
|
|
28712eaea1 | ||
|
|
51d7bbfdc8 | ||
|
|
e3b5c224e8 | ||
|
|
6df30b6d98 | ||
|
|
1a4305213d | ||
|
|
7df72edd2b | ||
|
|
1fce541023 | ||
|
|
950562afe1 | ||
|
|
a8f60f1f69 | ||
|
|
38928c4e1f | ||
|
|
175b28ee42 | ||
|
|
557b23d43f | ||
|
|
c43277f9c6 | ||
|
|
4d337ff897 | ||
|
|
6c98e7ea51 | ||
|
|
6910417090 | ||
|
|
d3d7fc6873 | ||
|
|
1358861275 | ||
|
|
14d36e813f | ||
|
|
cb0327de7f | ||
|
|
1a50d6e5ab | ||
|
|
c5a2edf04a | ||
|
|
df653e9af0 | ||
|
|
85b2a1855b | ||
|
|
c714d07739 | ||
|
|
59c448713f | ||
|
|
0563389057 | ||
|
|
e35c0e69b6 | ||
|
|
8bbfdb53db | ||
|
|
913095d291 | ||
|
|
b05ac4f4b2 | ||
|
|
174e06ed08 | ||
|
|
4499e85476 | ||
|
|
c724ef456f | ||
|
|
56b206fe04 | ||
|
|
4d9894138e | ||
|
|
6ea457df74 | ||
|
|
6faf701656 | ||
|
|
80f9d26e6d | ||
|
|
5ddc59cd32 | ||
|
|
cdbbd9ec4b | ||
|
|
e92a3fe237 | ||
|
|
98ef7bac6e | ||
|
|
db953e7da8 | ||
|
|
7597294ff8 | ||
|
|
9a7ac3af46 | ||
|
|
e7dca6c280 | ||
|
|
c37d7742ba | ||
|
|
ea09f074db | ||
|
|
6a0d84b3f6 | ||
|
|
dd668ec887 | ||
|
|
7dbe80ad63 | ||
|
|
697395af32 | ||
|
|
036b6cc873 | ||
|
|
96c34c52dc | ||
|
|
3808634a9f | ||
|
|
4f80d87750 | ||
|
|
ea92af2a7d | ||
|
|
4469cab7f4 | ||
|
|
875a4e5758 | ||
|
|
bf11a8baac | ||
|
|
65610f2ea2 | ||
|
|
85d664438f | ||
|
|
13b6fc6d97 | ||
|
|
d851573793 | ||
|
|
f6aea4bfbb | ||
|
|
1fdff00794 | ||
|
|
40e42606cf | ||
|
|
b8bbf9c79f | ||
|
|
8bcf787258 |
@@ -274,6 +274,22 @@ func (h *holder) removeContainers(id string) (int, error) {
|
||||
return builder.RemoveContainersOf(cleanup, h.remove, id)
|
||||
}
|
||||
|
||||
// removeLeftBehind removes, once, what earlier holders on this machine left — called before anything is
|
||||
// taken, so none of it is this holder's — and says what it did. A runtime that cannot be asked is said
|
||||
// and the holder takes work all the same: what is left is waste, not a reason to stop building.
|
||||
func (h *holder) removeLeftBehind() int {
|
||||
cleanup, stop := context.WithTimeout(context.Background(), killRemoves)
|
||||
defer stop()
|
||||
n, err := builder.RemoveLeftBehind(cleanup, h.remove)
|
||||
switch {
|
||||
case err != nil:
|
||||
fmt.Fprintf(os.Stderr, "could not look for containers earlier builds left on %s: %v\n", h.on, err)
|
||||
case n > 0:
|
||||
fmt.Fprintf(os.Stderr, "removed %d container(s) earlier builds left on %s\n", n, h.on)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// returned records that the build's work ended, and says whether a kill came first — only then is
|
||||
// the build killed; an error it ended with on its own is its own outcome.
|
||||
func (h *holder) returned(r *running) bool {
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -146,3 +147,26 @@ func TestAKillArrivingAfterTheBuildEndedIsRefusedAndAnUnannouncedKillSaysSo(t *t
|
||||
t.Fatalf("kill said %q (%v)", said, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **Issue 306**: a holder starting removes the containers earlier builds left on its machine, through
|
||||
// the runner a kill uses, and a runtime that cannot be asked does not stop it.
|
||||
func TestAHolderStartingRemovesWhatEarlierBuildsLeftHere(t *testing.T) {
|
||||
h := newHolder("novox", link.TheBuildMachine, t.TempDir(), nil)
|
||||
var removed []string
|
||||
h.remove = func(_ context.Context, _ string, name string, args ...string) (string, error) {
|
||||
removed = append(removed, name+" "+strings.Join(args, " "))
|
||||
if args[0] == "ps" {
|
||||
return "c1 build-1\nc2 build-1\nc3 check-here-2\n", nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
if n := h.removeLeftBehind(); n != 2 || len(removed) != 2 || removed[1] != "docker rm -f c1 c2" {
|
||||
t.Fatalf("removed %d: %v", n, removed)
|
||||
}
|
||||
h.remove = func(context.Context, string, string, ...string) (string, error) {
|
||||
return "", errors.New("no runtime")
|
||||
}
|
||||
if n := h.removeLeftBehind(); n != 0 {
|
||||
t.Errorf("a runtime that cannot be asked removed %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,6 +27,8 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
@@ -129,6 +131,9 @@ func run() error {
|
||||
if h.Paused() {
|
||||
fmt.Fprintf(os.Stderr, "paused (kept in %s): taking no build until resumed\n", pausedFile(workspace))
|
||||
}
|
||||
// **What an earlier holder here left goes before anything is taken** (novox/hq issue 306): a holder
|
||||
// recreated mid-check left the check's store and bus running, and the ask went to another machine.
|
||||
h.removeLeftBehind()
|
||||
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused})
|
||||
defer machine.Close()
|
||||
|
||||
@@ -172,7 +177,9 @@ func dialFor(credential Credential) (*broker.JetStream, string, error) {
|
||||
if !credential.onTheNewBus() {
|
||||
return nil, "", fmt.Errorf("the credential at hand names %q, which is not the mesh's bus", credential.URL)
|
||||
}
|
||||
js, err := broker.DialPinned(credential.natsURL(), credential.Fingerprint)
|
||||
// Named for what it is, not the controller whose code dials it (novox/hq issue 327).
|
||||
host, _ := os.Hostname()
|
||||
js, err := broker.DialPinned(credential.natsURL(), credential.Fingerprint, nats.Name("build agent on "+host))
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
@@ -261,6 +268,8 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
|
||||
forgeFrom(), say, request.Seats)
|
||||
}
|
||||
// What it copied into the store, whatever became of the build (novox/hq ADR 0257).
|
||||
result.Mirrored = built.Mirrored
|
||||
// Only a build the kill ended: the kill came before its work did. One that finished — built, or
|
||||
// failed on its own — in the moment the kill arrived says what it did, and the kill is refused.
|
||||
killed := false
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
@@ -187,7 +188,8 @@ func (a *actor) serveUnderTheLease(ctx context.Context, inv *inventory.Inventory
|
||||
a.mu.Lock()
|
||||
a.serving = true
|
||||
a.mu.Unlock()
|
||||
js, err := broker.Dial(address)
|
||||
// Its own connection, held as long as the lease, and named so (novox/hq issue 327).
|
||||
js, err := broker.Dial(address, nats.Name(broker.ConnectionName+" lease"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the mesh is on the bus at %s and this control plane cannot reach it to take the "+
|
||||
"lease: %w", broker.BareAddress(address), err)
|
||||
|
||||
@@ -7,8 +7,10 @@ import (
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The things the mesh can be asked to do, separated from how it was asked.
|
||||
@@ -41,9 +43,17 @@ import (
|
||||
// It costs a resolution per machine. Assignment is a person typing a command, and being told which
|
||||
// machines this just blocked is worth more than the milliseconds.
|
||||
func assign(ctx context.Context, open *stores, node string, modules ...string) (string, error) {
|
||||
return assignWith(ctx, open, node, assignOptions{}, modules...)
|
||||
}
|
||||
|
||||
// assignWith is assign asked with its options: build, for a module known and not built (novox/hq issue 325).
|
||||
func assignWith(ctx context.Context, open *stores, node string, opts assignOptions, modules ...string) (string, error) {
|
||||
if len(modules) == 0 {
|
||||
return "", fmt.Errorf("assign %s names no module", node)
|
||||
}
|
||||
if err := refusedMovingTheController(modules); err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Held while it is recorded, so it cannot land between a converge's preview and its flip and
|
||||
// be taken without ever having been previewed (novox/hq ADR 0100).
|
||||
ctx, release, err := holdNodes(ctx, open, []string{node})
|
||||
@@ -51,6 +61,29 @@ func assign(ctx context.Context, open *stores, node string, modules ...string) (
|
||||
return "", err
|
||||
}
|
||||
defer release()
|
||||
// **A module the catalogue does not hold is looked for further before it is refused** (novox/hq issue
|
||||
// 325): a build in flight keeps the assignment pending, a module known and not built is said with how to
|
||||
// build it, and only a name the mesh never heard of is refused as unknown. Several modules in one act
|
||||
// are judged together (ADR 0207), so an act naming one not registered is not made in part.
|
||||
if missing, err := notInCatalogue(ctx, open, modules); err != nil {
|
||||
return "", err
|
||||
} else if len(missing) > 0 {
|
||||
if len(modules) == 1 {
|
||||
return notRegistered(ctx, open, node, modules[0], opts)
|
||||
}
|
||||
var lines []string
|
||||
for _, m := range missing {
|
||||
where, err := whereIs(ctx, open.inventory, m, time.Now())
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
lines = append(lines, where.said)
|
||||
}
|
||||
return "", fmt.Errorf("%w: nothing was assigned, since modules assigned together are judged together "+
|
||||
"(ADR 0207) and %s not registered:\n %s\n assign them together once each is registered, or each "+
|
||||
"alone to keep it pending on its build", inventory.ErrNoSuchModule, strings.Join(missing, ", "),
|
||||
strings.Join(lines, "\n "))
|
||||
}
|
||||
// **The one assignment refused for what the node lacks** (novox/hq ADR 0207). Everything else
|
||||
// an assignment leaves unresolved is kept, because assignment is not an ordering; a module whose
|
||||
// resources are applied through a seat nothing on the node holds is refused, because that order
|
||||
@@ -179,6 +212,9 @@ func seatDependenciesOnAssign(ctx context.Context, open *stores, node string, mo
|
||||
// (novox/hq ADR 0207) — the other side of refusing that module's assignment without one. Several
|
||||
// modules in one act are judged together, so a holder and its dependents come off in one command.
|
||||
func unassign(ctx context.Context, open *stores, node string, modules ...string) (string, error) {
|
||||
if err := refusedMovingTheController(modules); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(modules) == 0 {
|
||||
return "", fmt.Errorf("unassign %s names no module", node)
|
||||
}
|
||||
@@ -200,6 +236,17 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string)
|
||||
for _, a := range assigned {
|
||||
runs[a] = true
|
||||
}
|
||||
// A module only pending on the node (novox/hq issue 325) is withdrawn, when it is the act's one module:
|
||||
// that is the undo of an assignment kept while its build runs.
|
||||
if len(modules) == 1 && !runs[modules[0]] {
|
||||
said, withdrawn, err := withdrawPending(ctx, open.inventory, node, modules[0])
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if withdrawn {
|
||||
return said, nil
|
||||
}
|
||||
}
|
||||
for _, module := range modules {
|
||||
if !runs[module] {
|
||||
return "", fmt.Errorf("%s is not assigned to %s", module, node)
|
||||
@@ -319,3 +366,17 @@ func issueOnAssign(ctx context.Context, open *stores, node, module string) strin
|
||||
}
|
||||
return fmt.Sprintf("its bus credential is issued and sealed to %s, and arrives with the push", node)
|
||||
}
|
||||
|
||||
// refusedMovingTheController refuses assigning or unassigning the controller's module through a verb (review of
|
||||
// novox/hq ADR 0272): the node it is assigned to is the control-node, and the control-node's operator account is
|
||||
// the controller's terminal, so whoever moves the module chooses the terminal. At the terminal alone, as every
|
||||
// change that says who may change what.
|
||||
func refusedMovingTheController(modules []string) error {
|
||||
verb, through := throughAVerb()
|
||||
if !through || !slices.Contains(modules, controllerModule) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%s is assigned and unassigned at the controller's terminal only (mesh-cli on the control-node), "+
|
||||
"never through a verb (this came through %q): the node it runs on is the control-node, whose operator is the "+
|
||||
"terminal (novox/hq ADR 0272). Nothing was assigned", controllerModule, verb)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,250 @@
|
||||
package main
|
||||
|
||||
// The account agents run as (novox/hq ADR 0266).
|
||||
//
|
||||
// On the control node every agent session ran as the operator's account, which may become root without a
|
||||
// password — so any agent there could become root without a person, and ADR 0259 §8 (an answer from the
|
||||
// operator's phone authorises an act) rests on that being false where the router and its channels run. The
|
||||
// decision: a node may name an account its agents run as, of their own and without sudo; the operator's
|
||||
// account keeps its sudo.
|
||||
//
|
||||
// - **Named at the controller's terminal only** (`node agent-account`): not a verb, not a setting, so no
|
||||
// agent can name itself another account. Empty is a real state: agents run as the operator there.
|
||||
// - **Composed** as `${machine:agent-account}`, `${machine:agent-home}` and `${machine:agent-root}` for the
|
||||
// agent's module, which declares the account with `root: never`, and as MESH_AGENT_ACCOUNT and
|
||||
// MESH_AGENT_HOME for its tools (catalogue/machine_into_files.go, runtime.go).
|
||||
// - **Judged by the machine itself.** The node-engine reads, on every look, whether an account declared
|
||||
// `root: never` can become root without a person — uid 0, a group that grants root, a sudo rule, a
|
||||
// secret of the mesh it may read — and says it as the declaring module's account verdict, marked
|
||||
// Root "never". agentConfined reads that verdict; the self-check (probe DA) raises
|
||||
// `agent-can-become-root` while it does not hold, and `node show` says it.
|
||||
//
|
||||
// A verdict not given is never a pass: an engine older than the judging, an account not yet declared, a
|
||||
// statement that says nothing of it — each is "not judged", and fails.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// kindAgentCanBecomeRoot is the condition raised while a machine's agent account can become root without
|
||||
// a person, or is not judged (ADR 0266).
|
||||
const kindAgentCanBecomeRoot = "agent-can-become-root"
|
||||
|
||||
// agentAccountProbe is the self-check's probe of it.
|
||||
const agentAccountProbe = "DA"
|
||||
|
||||
// agentConfined says whether the agents of a machine that names an agent account are confined: the
|
||||
// machine's newest statement holds a healthy account verdict, judged for root, on that account. named is
|
||||
// false for a machine that names none — agents run as the operator account there, which this does not
|
||||
// judge. why is said either way, in the mesh's words; err is a store that could not be read.
|
||||
//
|
||||
// The one judgement: `node show`, the self-check, and ADR 0259's router honouring a verified sender read
|
||||
// it here.
|
||||
func agentConfined(ctx context.Context, inv *inventory.Inventory, node string) (named, confined bool, why string, err error) {
|
||||
n, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
if n.AgentAccount == "" {
|
||||
return false, false, fmt.Sprintf("%s names no agent account: agents run as the operator account (%s)",
|
||||
node, orNoneKnown(n.Account)), nil
|
||||
}
|
||||
h, had, err := inv.HealthOf(ctx, node)
|
||||
if err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
confined, why = judgedConfined(n.AgentAccount, h, had, time.Now())
|
||||
return true, confined, why, nil
|
||||
}
|
||||
|
||||
// verdictFreshFor is how old the statement holding the verdict may be, by this controller's clock. A
|
||||
// node-engine states its health on every change and at least every five minutes (mesh-host's sayAnyway), so
|
||||
// three statements missed is a node-engine stopped, or a machine away. **A stale verdict is not a pass**: an
|
||||
// agent that stopped the node-engine must not leave "cannot become root" standing from before.
|
||||
const verdictFreshFor = 15 * time.Minute
|
||||
|
||||
// judgedConfined is the judgement over one statement, without the store, at now.
|
||||
func judgedConfined(agent string, h inventory.NodeHealth, had bool, now time.Time) (bool, string) {
|
||||
if !had {
|
||||
return false, fmt.Sprintf("the agent account %s is not judged: the machine's node-engine has stated "+
|
||||
"nothing of what it runs", agent)
|
||||
}
|
||||
if age := now.Sub(h.HeardAt); age > verdictFreshFor {
|
||||
return false, fmt.Sprintf("the agent account %s is not judged: the machine's newest statement was heard at "+
|
||||
"%s, more than %d minutes ago, and a verdict that old is not a verdict on now", agent,
|
||||
h.HeardAt.Local().Format("2006-01-02 15:04"), int(verdictFreshFor.Minutes()))
|
||||
}
|
||||
if h.Contract < link.RootContract {
|
||||
return false, fmt.Sprintf("the agent account %s is not judged: the machine's node-engine is older than "+
|
||||
"the judging of an account's root (its statement's contract is %d, the judging is %d)",
|
||||
agent, h.Contract, link.RootContract)
|
||||
}
|
||||
var verdicts []inventory.ResourceHealth
|
||||
for _, r := range h.Resources {
|
||||
if r.Kind == link.KindAccount && r.Target == agent && r.Root == link.RootNever {
|
||||
verdicts = append(verdicts, r)
|
||||
}
|
||||
}
|
||||
if len(verdicts) == 0 {
|
||||
return false, fmt.Sprintf("the agent account %s is not judged: the machine's newest statement holds no "+
|
||||
"verdict on it — no module there declares it never to become root, or the declaration naming it "+
|
||||
"has not been applied", agent)
|
||||
}
|
||||
sort.Slice(verdicts, func(i, j int) bool {
|
||||
return verdicts[i].Module+verdicts[i].Resource < verdicts[j].Module+verdicts[j].Resource
|
||||
})
|
||||
for _, v := range verdicts {
|
||||
switch v.State {
|
||||
case link.StateHealthy:
|
||||
case link.StateUnhealthy:
|
||||
// The engine's own words, which start with link.ReasonRoot when it found a way to root.
|
||||
return false, fmt.Sprintf("the agent account %s %s (said by %s's %s)", agent,
|
||||
orNoneKnown(v.Reason), v.Module, v.Resource)
|
||||
default:
|
||||
return false, fmt.Sprintf("the agent account %s is not judged: %s (%s's %s, %s)", agent,
|
||||
orNoneKnown(v.Reason), v.Module, v.Resource, v.State)
|
||||
}
|
||||
}
|
||||
return true, fmt.Sprintf("the agent account %s cannot become root without a person (judged %s)", agent,
|
||||
h.SaidAt.Local().Format("2006-01-02 15:04"))
|
||||
}
|
||||
|
||||
// searchQuietFor is how long the controller lets an agent account's verdict wait for the node-engine's setuid
|
||||
// search before that is itself the urgent condition: the engine's bound on one search (link.RootSearchBound, the
|
||||
// engine's own value), counted from when this controller first saw it waiting, never from the engine's start.
|
||||
const searchQuietFor = link.RootSearchBound
|
||||
|
||||
// The kinds of an agent account's verdict, for the quiet a search earns.
|
||||
const (
|
||||
verdictOther = iota // anything else: a stale statement, an older engine, no verdict, another unknown
|
||||
verdictPending // waiting for the search, and otherwise healthy
|
||||
verdictComplete // judged: healthy, or a way to root found
|
||||
)
|
||||
|
||||
// rootVerdictKind reads a statement for the agent account (novox/hq ADR 0266): pending when every verdict on it
|
||||
// is healthy or not judged yet because the engine's setuid search runs, at least one of them that; complete when
|
||||
// every verdict is healthy or one found a way to root. The engine's own "since" is not read: it starts again at
|
||||
// every restart of the engine.
|
||||
func rootVerdictKind(agent string, h inventory.NodeHealth, had bool, now time.Time) int {
|
||||
if !had || now.Sub(h.HeardAt) > verdictFreshFor || h.Contract < link.RootContract {
|
||||
return verdictOther
|
||||
}
|
||||
pending, any := false, false
|
||||
for _, r := range h.Resources {
|
||||
if r.Kind != link.KindAccount || r.Target != agent || r.Root != link.RootNever {
|
||||
continue
|
||||
}
|
||||
any = true
|
||||
switch {
|
||||
case r.State == link.StateHealthy:
|
||||
case r.State == link.StateUnhealthy:
|
||||
return verdictComplete
|
||||
case r.State == link.StateUnknown && strings.HasPrefix(r.Reason, link.ReasonRootPending):
|
||||
pending = true
|
||||
default:
|
||||
return verdictOther
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case !any:
|
||||
return verdictOther
|
||||
case pending:
|
||||
return verdictPending
|
||||
}
|
||||
return verdictComplete
|
||||
}
|
||||
|
||||
// searchStillRunning says the one thing keeping an agent account from being judged is the node-engine's setuid
|
||||
// search, and that this controller first saw it waiting less than searchQuietFor ago — kept in the store, so a
|
||||
// node-engine restarted in a loop does not keep it quiet. A complete verdict forgets when it began.
|
||||
func searchStillRunning(ctx context.Context, inv *inventory.Inventory, node, agent string, h inventory.NodeHealth,
|
||||
had bool, now time.Time) (bool, error) {
|
||||
switch rootVerdictKind(agent, h, had, now) {
|
||||
case verdictComplete:
|
||||
return false, inv.RootSearchJudged(ctx, node)
|
||||
case verdictPending:
|
||||
since, err := inv.RootSearchPending(ctx, node, now)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return now.Sub(since) <= searchQuietFor, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// probeAgentAccounts is DA: every machine that names an agent account has it judged, on its node-engine's
|
||||
// newest statement, unable to become root without a person (ADR 0266).
|
||||
func probeAgentAccounts(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
inv := d.open.inventory
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []conditions.Observation
|
||||
for _, n := range nodes {
|
||||
if n.AgentAccount == "" {
|
||||
continue
|
||||
}
|
||||
h, had, err := inv.HealthOf(ctx, n.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := time.Now()
|
||||
quiet, err := searchStillRunning(ctx, inv, n.Name, n.AgentAccount, h, had, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
confined, why := judgedConfined(n.AgentAccount, h, had, now)
|
||||
if confined {
|
||||
continue
|
||||
}
|
||||
// Not judged yet only because the first search since the node-engine started is still running: not the
|
||||
// urgent condition after every restart. The agent is still not confined — ADR 0259's router reads
|
||||
// agentConfined, not this — and `node show` still says not judged. Loud again once the search fails,
|
||||
// runs out its bound, or the statement goes stale.
|
||||
if quiet {
|
||||
continue
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "agent-root",
|
||||
Machine: n.Name, Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("on %s, %s (ADR 0266): an agent there may become root without a person, and "+
|
||||
"no answer from a channel authorises an act there (ADR 0259 §8)", n.Name, why),
|
||||
Said: why})
|
||||
}
|
||||
return sortedFound(out), nil
|
||||
}
|
||||
|
||||
// agentAccountLines is what `node show` says of the account agents run as.
|
||||
func agentAccountLines(ctx context.Context, inv *inventory.Inventory, n inventory.Node) []string {
|
||||
if n.AgentAccount == "" {
|
||||
return []string{fmt.Sprintf(" agents run as the operator account (%s); no agent account is named",
|
||||
orNoneKnown(n.Account))}
|
||||
}
|
||||
_, confined, why, err := agentConfined(ctx, inv, n.Name)
|
||||
if err != nil {
|
||||
return []string{fmt.Sprintf(" agents run as %s (home %s); whether it can become root could NOT be read: %v",
|
||||
n.AgentAccount, n.AgentHome(), err)}
|
||||
}
|
||||
verdict := "CAN become root, or is not judged: " + why
|
||||
if confined {
|
||||
verdict = why
|
||||
}
|
||||
return []string{fmt.Sprintf(" agents run as %s (home %s)", n.AgentAccount, n.AgentHome()),
|
||||
" " + verdict}
|
||||
}
|
||||
|
||||
// orNoneKnown is a value, or that none is known.
|
||||
func orNoneKnown(s string) string {
|
||||
if strings.TrimSpace(s) == "" {
|
||||
return "none known"
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"github.com/novox/mesh-host/rootsearch"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
snapshot "github.com/novox/mesh-controller/internal/facts"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The agent account's judgement (novox/hq ADR 0266): confined only on a healthy account verdict judged for
|
||||
// root, on the very account; every verdict not given — no statement, an older engine, no verdict on it, a
|
||||
// verdict of unknown — is "not judged" and fails, never a pass.
|
||||
func TestAnAgentAccountIsConfinedOnlyOnAHealthyVerdictJudgedForRoot(t *testing.T) {
|
||||
at := time.Date(2026, 10, 8, 19, 21, 0, 0, time.UTC)
|
||||
verdict := func(target, root, state, reason string) inventory.ResourceHealth {
|
||||
return inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account",
|
||||
Kind: link.KindAccount, Target: target, State: state, Reason: reason, Root: root, Account: target}
|
||||
}
|
||||
statement := func(contract int, rs ...inventory.ResourceHealth) inventory.NodeHealth {
|
||||
return inventory.NodeHealth{Node: "anchor", Contract: contract, SaidAt: at, HeardAt: at, Resources: rs}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
h inventory.NodeHealth
|
||||
had bool
|
||||
confined bool
|
||||
says string
|
||||
}{
|
||||
{"judged and unable", statement(link.RootContract, verdict("agent", link.RootNever, link.StateHealthy, "")),
|
||||
true, true, "cannot become root without a person"},
|
||||
{"judged and able", statement(link.RootContract, verdict("agent", link.RootNever, link.StateUnhealthy,
|
||||
link.ReasonRoot+": in the group docker, which grants root")), true, false, "in the group docker"},
|
||||
{"a verdict of unknown", statement(link.RootContract, verdict("agent", link.RootNever, link.StateUnknown,
|
||||
"sudo could not be read")), true, false, "not judged"},
|
||||
{"no statement", inventory.NodeHealth{}, false, false, "not judged"},
|
||||
{"an older engine", statement(link.ReadinessContract, verdict("agent", "", link.StateHealthy, "")),
|
||||
true, false, "older than the judging"},
|
||||
{"a verdict on groups only", statement(link.RootContract, verdict("agent", "", link.StateHealthy, "")),
|
||||
true, false, "no verdict on it"},
|
||||
{"a verdict on another account", statement(link.RootContract, verdict("ops", link.RootNever, link.StateHealthy, "")),
|
||||
true, false, "no verdict on it"},
|
||||
} {
|
||||
confined, why := judgedConfined("agent", c.h, c.had, at.Add(time.Minute))
|
||||
if confined != c.confined || !strings.Contains(why, c.says) {
|
||||
t.Errorf("%s: confined %v, %q; want %v saying %q", c.name, confined, why, c.confined, c.says)
|
||||
}
|
||||
}
|
||||
// A verdict heard longer ago than the bound is no verdict: an agent that stopped the node-engine must not
|
||||
// leave "healthy" standing.
|
||||
fresh := statement(link.RootContract, verdict("agent", link.RootNever, link.StateHealthy, ""))
|
||||
if ok, _ := judgedConfined("agent", fresh, true, at.Add(verdictFreshFor)); !ok {
|
||||
t.Error("a verdict exactly at the bound is still one")
|
||||
}
|
||||
if ok, why := judgedConfined("agent", fresh, true, at.Add(verdictFreshFor+time.Second)); ok ||
|
||||
!strings.Contains(why, "not judged") {
|
||||
t.Errorf("a stale healthy verdict passed: %q", why)
|
||||
}
|
||||
}
|
||||
|
||||
// DA raises an urgent condition, with plain words, on a machine whose agent account is not judged unable to
|
||||
// become root; a machine that names none is not its to judge.
|
||||
func TestTheSelfCheckSaysAnAgentAccountThatCanBecomeRoot(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.NodeByName(ctx, n); err != nil {
|
||||
if _, err := inv.AddNode(ctx, n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.SetAccount(ctx, n, "ops", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.SetAgentAccount(ctx, "anchor", "agent", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &doctor{open: open}
|
||||
found, err := probeAgentAccounts(ctx, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found = onlyMachine(found, "anchor")
|
||||
if len(found) != 1 || found[0].Machine != "anchor" || found[0].Severity != conditions.Urgent ||
|
||||
!strings.Contains(found[0].Said, "not judged") {
|
||||
t.Fatalf("a named agent account with no verdict: %+v", found)
|
||||
}
|
||||
w := plainWordings[kindAgentCanBecomeRoot](conditions.Observation{Kind: kindAgentCanBecomeRoot, Machine: "anchor"})
|
||||
if w.Headline == "" || w.Needs == "" || w.Resolved == "" {
|
||||
t.Errorf("the condition has no plain words: %+v", w)
|
||||
}
|
||||
|
||||
healthy := inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account",
|
||||
Kind: link.KindAccount, Target: "agent", State: link.StateHealthy, Root: link.RootNever, Account: "agent"}
|
||||
if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.RootContract,
|
||||
SaidAt: time.Now(), HeardAt: time.Now(), Resources: []inventory.ResourceHealth{healthy}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if found, err = probeAgentAccounts(ctx, d); err != nil || len(onlyMachine(found, "anchor")) != 0 {
|
||||
t.Fatalf("a judged agent account still fails: %+v %v", found, err)
|
||||
}
|
||||
if named, confined, why, err := agentConfined(ctx, inv, "anchor"); err != nil || !named || !confined {
|
||||
t.Fatalf("agentConfined on anchor: %v %v %q %v", named, confined, why, err)
|
||||
}
|
||||
if named, _, why, err := agentConfined(ctx, inv, "laptop"); err != nil || named ||
|
||||
!strings.Contains(why, "operator account") {
|
||||
t.Fatalf("agentConfined on a machine naming none: %v %q %v", named, why, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheAgentRootWordsArePlain(t *testing.T) {
|
||||
w := plainWordings[kindAgentCanBecomeRoot](conditions.Observation{Kind: kindAgentCanBecomeRoot, Machine: "anchor"})
|
||||
if why, ok := conditions.PlainWords(w, "anchor"); !ok {
|
||||
t.Fatalf("not plain: %s: %+v", why, w)
|
||||
}
|
||||
}
|
||||
|
||||
func onlyMachine(obs []conditions.Observation, machine string) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, o := range obs {
|
||||
if o.Machine == machine {
|
||||
out = append(out, o)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// The snapshot a merge check composes from carries the agent account as a pseudonym (novox/hq ADR 0266),
|
||||
// so a change is judged against machines that name one, and the name never leaves.
|
||||
func TestTheFactsCarryTheAgentAccountAsAPseudonym(t *testing.T) {
|
||||
open, _ := aMeshWithSecrets(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.SetAccount(ctx, "anchor", "keeper", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetAgentAccount(ctx, "anchor", "warden", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := gatherFacts(ctx, open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, _ := f.Encode()
|
||||
if strings.Contains(string(body), "warden") {
|
||||
t.Error("the agent account's name is in the snapshot")
|
||||
}
|
||||
m, ok := f.Machine(snapshot.Pseudonym("machine", "anchor"))
|
||||
if !ok || m.AgentAccount != snapshot.Pseudonym("account", "warden") || m.Account == m.AgentAccount {
|
||||
t.Fatalf("the anchor's agent account reads as %q (operator %q)", m.AgentAccount, m.Account)
|
||||
}
|
||||
}
|
||||
|
||||
// No verb runs a `node` command that sets something: through the generic `command` verb, `node account`,
|
||||
// `node agent-account` and every other `node` subcommand but list and show are refused, naming the terminal.
|
||||
func TestNoVerbSetsANodesAccounts(t *testing.T) {
|
||||
for _, line := range []string{
|
||||
"node agent-account novox --clear",
|
||||
"node agent-account novox ops",
|
||||
"node account novox agent",
|
||||
"node account novox",
|
||||
"node add intruder",
|
||||
"node public-domain novox --clear",
|
||||
"node",
|
||||
"node frobnicate",
|
||||
} {
|
||||
argv, err := argvFor("command", map[string]any{"command": line})
|
||||
if err == nil || !strings.Contains(err.Error(), "controller's terminal") ||
|
||||
!strings.Contains(err.Error(), "ADR 0266") {
|
||||
t.Errorf("%q ran as %v (%v); want a refusal naming the terminal", line, argv, err)
|
||||
}
|
||||
}
|
||||
for _, line := range []string{"node show novox", "node list --json", "status --json"} {
|
||||
if _, err := argvFor("command", map[string]any{"command": line}); err != nil {
|
||||
t.Errorf("%q, a read, was refused: %v", line, err)
|
||||
}
|
||||
}
|
||||
if err := terminalOnly([]string{"node", "account", "a", "b"}); err == nil {
|
||||
t.Error("the refusal is not only the command verb's")
|
||||
}
|
||||
}
|
||||
|
||||
// Naming the agent account is the controller's terminal's alone: the `node` verb only shows.
|
||||
func TestTheNodeVerbOnlyShows(t *testing.T) {
|
||||
argv, err := argvFor("node", map[string]any{"node": "anchor"})
|
||||
if err != nil || strings.Join(argv, " ") != "node show anchor" {
|
||||
t.Fatalf("the node verb runs %v (%v)", argv, err)
|
||||
}
|
||||
for _, v := range catalogue.ControllerVerbs {
|
||||
if strings.Contains(v.Name, "agent") {
|
||||
t.Errorf("a verb %q may name the agent account", v.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// After every node-engine restart its search for setuid programs runs for up to its bound, and the agent account
|
||||
// is not judged until it ends. DA does not raise that as urgent while the search is within its bound, counted from
|
||||
// when this controller first saw it waiting — never from the engine's own "since", which a restart resets, so an
|
||||
// engine restarted in a loop does not keep it quiet. The account is still not confined, and `node show` still
|
||||
// says not judged; a search that failed, or a way to root found, is urgent at once.
|
||||
func TestASearchStillRunningAfterARestartIsNotUrgent(t *testing.T) {
|
||||
if searchQuietFor != rootsearch.Bound {
|
||||
t.Fatalf("the quiet is %s and the node-engine's bound %s: they are one value", searchQuietFor, rootsearch.Bound)
|
||||
}
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if _, err := inv.NodeByName(ctx, "anchor"); err != nil {
|
||||
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.SetAgentAccount(ctx, "anchor", "agent", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &doctor{open: open}
|
||||
say := func(state, reason string) []conditions.Observation {
|
||||
t.Helper()
|
||||
// The engine's since is always now: it was just restarted.
|
||||
v := inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account",
|
||||
Kind: link.KindAccount, Target: "agent", State: state, Reason: reason, Root: link.RootNever,
|
||||
Account: "agent", Since: time.Now()}
|
||||
if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.RootContract,
|
||||
SaidAt: time.Now(), HeardAt: time.Now(), Resources: []inventory.ResourceHealth{v}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found, err := probeAgentAccounts(ctx, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return onlyMachine(found, "anchor")
|
||||
}
|
||||
running := link.ReasonRootPending + ": the search for setuid programs, started at 19:21, has not finished yet"
|
||||
healthy := func() {
|
||||
t.Helper()
|
||||
if found := say(link.StateHealthy, ""); len(found) != 0 {
|
||||
t.Fatalf("a healthy verdict raised: %+v", found)
|
||||
}
|
||||
}
|
||||
if found := say(link.StateUnknown, running); len(found) != 0 {
|
||||
t.Fatalf("a search first seen now was raised: %+v", found)
|
||||
}
|
||||
if _, confined, why, _ := agentConfined(ctx, inv, "anchor"); confined || !strings.Contains(why, "not judged") {
|
||||
t.Fatalf("an account whose search runs was read as confined: %q", why)
|
||||
}
|
||||
// The engine restarted again and again, each statement's own since fresh: the controller's clock runs on.
|
||||
if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
healthy() // a complete verdict forgets when the waiting began …
|
||||
if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil {
|
||||
t.Fatal(err) // … and this restart loop began past the bound
|
||||
}
|
||||
if found := say(link.StateUnknown, running); len(found) != 1 || found[0].Severity != conditions.Urgent {
|
||||
t.Fatalf("a search pending past the bound, by the controller's clock, was not urgent: %+v", found)
|
||||
}
|
||||
healthy()
|
||||
incomplete := rootsearch.ReasonIncomplete + ": the search for setuid programs did not finish (last tried at " +
|
||||
"19:23: timeout); it is tried again later"
|
||||
if found := say(link.StateUnknown, incomplete); len(found) != 1 || found[0].Severity != conditions.Urgent {
|
||||
t.Fatalf("a search that did not finish was not urgent: %+v", found)
|
||||
}
|
||||
if found := say(link.StateUnhealthy, link.ReasonRoot+": in the group docker; "+running); len(found) != 1 ||
|
||||
found[0].Severity != conditions.Urgent {
|
||||
t.Fatalf("a way to root found while the search runs was not urgent: %+v", found)
|
||||
}
|
||||
}
|
||||
@@ -160,6 +160,9 @@ func buildFrom(result link.BuildResult) inventory.Build {
|
||||
for _, ref := range result.Against {
|
||||
kept.Against = append(kept.Against, catalogue.Recorded(ref))
|
||||
}
|
||||
for _, ref := range result.Mirrored {
|
||||
kept.Mirrored = append(kept.Mirrored, catalogue.Recorded(ref))
|
||||
}
|
||||
for _, r := range result.Read {
|
||||
kept.Read = append(kept.Read, inventory.ReadRepository{Repository: r.Repository, Ref: r.Ref})
|
||||
}
|
||||
@@ -396,15 +399,49 @@ func buildBehind(ctx context.Context, wait time.Duration) error {
|
||||
//
|
||||
// Separated from the command so `--behind` can walk a list without a second path to the same act.
|
||||
func buildOne(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error {
|
||||
_, err := buildOneAsked(ctx, source, path, ref, wait, false)
|
||||
_, err := buildOneAsked(ctx, source, path, ref, wait, false, "build")
|
||||
return err
|
||||
}
|
||||
|
||||
// recordAsked keeps a build request once it is asked (novox/hq issue 325), with who asked it, in a store
|
||||
// opened for the purpose: the asks reach here from processes that hold none.
|
||||
func recordAsked(ctx context.Context, r inventory.BuildRequest) {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "build %s is asked and not kept as a build request: %v\n", r.ID, err)
|
||||
return
|
||||
}
|
||||
defer open.Close()
|
||||
recordBuildRequest(ctx, open.inventory, r)
|
||||
}
|
||||
|
||||
// markWaitFailed says what became of a kept build request whose waited ask failed: never handed over, so
|
||||
// nothing runs; or handed over and no longer waited for, so asked with its outcome unknown — still read as in
|
||||
// flight until its outcome or its bound (novox/hq issue 325).
|
||||
func markWaitFailed(ctx context.Context, id string, why error) {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "build %s could not be marked: %v\n", id, err)
|
||||
return
|
||||
}
|
||||
defer open.Close()
|
||||
mark := open.inventory.MarkOutcomeUnknown
|
||||
if errors.Is(why, link.ErrNotHandedOver) {
|
||||
mark = open.inventory.MarkNotAsked
|
||||
}
|
||||
if err := mark(ctx, id, why.Error()); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "build %s could not be marked: %v\n", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
// buildOneAsked is buildOne answering the id it asked with — what a plan keeps to match the outcome
|
||||
// by (novox/hq ADR 0219) — and, for an ask not waited for, optionally a dry run: built and looked
|
||||
// at, never taken in (issue 240), which is what `replay` asks unless told to register.
|
||||
//
|
||||
// asker is who asked, for the build request kept once the ask is made (novox/hq issue 325); empty for an
|
||||
// asker that keeps the request itself, with its own name, once it knows the ask was made.
|
||||
func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wait time.Duration,
|
||||
dryRun bool) (string, error) {
|
||||
dryRun bool, asker string) (string, error) {
|
||||
if dryRun && wait != 0 {
|
||||
return "", errors.New("a dry run waited for is `build --dry-run`")
|
||||
}
|
||||
@@ -414,6 +451,10 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Through a verb, only a repository the catalogue builds from (novox/hq ADR 0266).
|
||||
if err := verbMayAsk(ctx, source, repository); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
ident, err := openIdentity(ctx)
|
||||
if err != nil {
|
||||
@@ -459,6 +500,14 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
|
||||
}
|
||||
defer ask.Close()
|
||||
fmt.Printf(" of %s\n", seat)
|
||||
// Kept once it is asked, so `assign` knows a module is coming while its build runs (novox/hq issue 325);
|
||||
// never before, so an ask that failed never reads as a build in flight. A dry run registers nothing, and
|
||||
// is not kept.
|
||||
keep := !dryRun && asker != ""
|
||||
// Asked at the terminal is what lets its outcome register a module from a repository the catalogue does
|
||||
// not build it from (novox/hq ADR 0266); never through a verb.
|
||||
asked := inventory.BuildRequest{ID: request.ID, Repository: source.Repository, Seat: source.Seat, Path: path,
|
||||
Ref: ref, For: asker, AtTerminal: startedAtTheTerminal()}
|
||||
|
||||
if wait == 0 {
|
||||
// Asked and not waited for (novox/hq issue 176): the outcome is the role's event, and the
|
||||
@@ -468,6 +517,9 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
|
||||
if err := ask.Ask(ctx, request); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if keep {
|
||||
recordAsked(ctx, asked)
|
||||
}
|
||||
if dryRun {
|
||||
fmt.Printf("asked as a dry run, not waited for: `builds --log %s` follows it as it runs; "+
|
||||
"its outcome is not taken in\n", request.ID)
|
||||
@@ -478,8 +530,16 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
|
||||
return request.ID, nil
|
||||
}
|
||||
|
||||
// Waited for: kept while it is waited for, since that can take minutes, and marked when the hand-over
|
||||
// failed (not asked) or the wait did (asked, outcome unknown) — an outcome heard later is the last word.
|
||||
if keep {
|
||||
recordAsked(ctx, asked)
|
||||
}
|
||||
result, err := ask.Submit(ctx, request, wait)
|
||||
if err != nil {
|
||||
if keep {
|
||||
markWaitFailed(ctx, request.ID, err)
|
||||
}
|
||||
return request.ID, err
|
||||
}
|
||||
|
||||
@@ -489,6 +549,10 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
|
||||
}
|
||||
defer open.Close()
|
||||
manifest, kept, err := takeIn(ctx, open.inventory, result)
|
||||
// The assignments pending on this build, made or ended (novox/hq issue 325).
|
||||
for _, line := range settlePendingOnBuild(ctx, open, result, manifest.Module, err) {
|
||||
fmt.Printf(" %s\n", line)
|
||||
}
|
||||
if err != nil {
|
||||
return request.ID, err
|
||||
}
|
||||
@@ -573,6 +637,21 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), and the mesh does not register it: %w",
|
||||
result.On, result.Repository, short(result.Commit), err)
|
||||
}
|
||||
// **Only from the repository the catalogue builds the module from** (novox/hq ADR 0266): else the trunk
|
||||
// below is the trunk of whatever repository was built, which may be one an agent made — and a module named
|
||||
// `sudo` from it would be what the next push sends. Another repository is the operator's, at the terminal.
|
||||
trunk := result.Trunk
|
||||
if was, err := inv.SourceOf(ctx, manifest.Module); err == nil && followedBranch(was.Ref) != "" {
|
||||
trunk = followedBranch(was.Ref)
|
||||
}
|
||||
if trunk == "" {
|
||||
trunk = "main"
|
||||
}
|
||||
repoID, err := mayRegisterFrom(ctx, inv, manifest.Module, recorded, result.ID, result.Path, trunk)
|
||||
if err != nil {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On,
|
||||
manifest.Module, short(result.Commit), err)
|
||||
}
|
||||
// **Only a commit on the trunk is published** (novox/hq ADR 0238): a commit off its repository's
|
||||
// default branch — a pull request's head, a feature branch built by hand, a `rebuild` or `replay
|
||||
// --register` of one — is for checking, and is never a module's version; nothing could then send it.
|
||||
@@ -620,6 +699,10 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
||||
}
|
||||
return manifest, kept, err
|
||||
}
|
||||
// Which repository it is registered from, by the forge's own id (novox/hq ADR 0266).
|
||||
if err := inv.SetSourceIdentity(ctx, manifest.Module, repoID); err != nil {
|
||||
return manifest, kept, err
|
||||
}
|
||||
// The keep set just moved, and new bytes just landed (novox/hq ADR 0189). Asked here rather
|
||||
// than on a timer of its own: this is the only moment either is true. Never fatal — the build
|
||||
// worked and the module is registered.
|
||||
@@ -661,6 +744,9 @@ func buildAndShow(ctx context.Context, source buildSource, path, ref string, wai
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := verbMayAsk(ctx, source, repository); err != nil {
|
||||
return err
|
||||
}
|
||||
ident, err := openIdentity(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -720,6 +806,8 @@ type answers struct {
|
||||
plans []inventory.Plan
|
||||
// paused is whether the build seat takes work, which a plan waiting on it says (ADR 0219).
|
||||
paused pauseView
|
||||
// tierBounds is how long each repository's plan tier may take before it is late (novox/hq issue 296).
|
||||
tierBounds tierBounds
|
||||
// refused is why a machine cannot be worked out at all, by name. A different thing from every
|
||||
// other answer here: those are about a machine that was told something, and this is about one
|
||||
// that cannot be told anything — it never reaches waiting, because nothing was computed for it
|
||||
@@ -766,6 +854,11 @@ type answers struct {
|
||||
// handed to the operator; healsUnread why it could not be read.
|
||||
heals *healsCount
|
||||
healsUnread string
|
||||
// pending is every assignment waiting for its module's build, and every one ended in the last day
|
||||
// (novox/hq issue 325); pendingUnread why they could not be read. One waiting, or one ended other than
|
||||
// applied, is not well: an assignment somebody made is not made yet, or will not be.
|
||||
pending []inventory.PendingAssignment
|
||||
pendingUnread string
|
||||
}
|
||||
|
||||
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
|
||||
@@ -814,6 +907,10 @@ func heldBy(ctx context.Context) map[string]string {
|
||||
// the mesh runs on today this needs the controller's own connection, so it is handed one; on the bus
|
||||
// being built it dials, because a build request is a one-shot and holds nothing else.
|
||||
func askOverOn(seat string) (link.Builders, error) {
|
||||
// The serving controller asks on its own connection (novox/hq issue 327).
|
||||
if serving := servingBus.Load(); serving != nil {
|
||||
return link.BuildsOn(serving, seat), nil
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -872,11 +969,7 @@ func buildSeatAmong(entries []inventory.Entry) string {
|
||||
// with a consumer of its own that is gone when this returns, so nothing accumulates in the server
|
||||
// for the reading, and filtered by subject, so one build's lines are all that travel.
|
||||
func buildLog(ctx context.Context, id string) error {
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot reach the bus to read a build's log: %w", err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,458 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// Where a build may register a module from (novox/hq ADR 0266).
|
||||
//
|
||||
// A build's outcome registers its module, and a registered module is what the next push sends. Before this,
|
||||
// an outcome registered whatever its manifest named, from whichever repository it was built from: an agent
|
||||
// that made a repository of its own, committed `modules/sudo/module.json` granting itself a rule without a
|
||||
// password, and asked the `build` verb for it had its repository registered as the module `sudo` — whose next
|
||||
// push made the agent root on every node. A fork of the node-engine did the same everywhere. The trunk rule
|
||||
// (ADR 0238) did not stop it: the trunk it checked was the trunk of the repository built, which was the
|
||||
// agent's own.
|
||||
//
|
||||
// So **an outcome registers a module only from the repository the catalogue already builds that module
|
||||
// from**; and a module new to the catalogue only from a repository the catalogue already builds another
|
||||
// module from — whose trunk takes a reviewed, approved merge, which is how a merge adds a module (novox/hq
|
||||
// issue 300). Anything else — a module moved to another repository, a module from a repository the
|
||||
// catalogue has never built — is the operator's, at the controller's terminal: allowed only when the build
|
||||
// request was kept as asked there. Judged at the take-in, which every outcome reaches whoever hears it and
|
||||
// whichever verb asked it (`build`, `rebuild`, `replay --register`, `assign` with build), and before the ask
|
||||
// for a call through a verb, so an agent cannot have a build node run a repository the catalogue does not
|
||||
// build from at all.
|
||||
|
||||
// errNotItsSource is an outcome refused for where it was built from.
|
||||
var errNotItsSource = errors.New("not built from the repository the catalogue builds it from")
|
||||
|
||||
// servedVar marks every process the serving controller starts — each verb's command, each child — and the serving
|
||||
// process itself, so none of them can read as the operator at the terminal (novox/hq ADR 0266). Set by serve
|
||||
// before it answers anything, inherited by every child through os.Environ.
|
||||
const servedVar = "MESH_SERVED_BY_THE_CONTROLLER"
|
||||
|
||||
// startedAtTheTerminal says this process was started at the controller's terminal: not the serving controller,
|
||||
// not anything it started, not a verb's command, not a seat call's. The serving controller marks its own
|
||||
// environment (servedVar), so a build asked in it, or by any process it starts, never reads as the terminal's;
|
||||
// runVerb also names the verb and the caller.
|
||||
//
|
||||
// **And a line mesh-cli asked as the controller's terminal** (novox/hq ADR 0272 §4): the serving controller runs it
|
||||
// without the served mark and without a verb, names its caller, and marks it with cliTerminalVar — a mark only the
|
||||
// mesh-cli path sets and every other command line the controller runs is stripped of (commandEnvironment).
|
||||
func startedAtTheTerminal() bool {
|
||||
if os.Getenv(servedVar) != "" || os.Getenv(verbVar) != "" {
|
||||
return false
|
||||
}
|
||||
return os.Getenv(link.CallerVar) == "" || os.Getenv(cliTerminalVar) != ""
|
||||
}
|
||||
|
||||
// cliTerminalVar marks a command line the serving controller runs as the controller's terminal for mesh-cli.
|
||||
const cliTerminalVar = "MESH_CLI_TERMINAL"
|
||||
|
||||
// markServed marks this process, and so everything it starts, as the serving controller's.
|
||||
func markServed() {
|
||||
if err := os.Setenv(servedVar, "1"); err != nil {
|
||||
panic("the serving controller could not mark its environment: " + err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// sourceForms compares sources however each is spelled: a path on a seat's holder (ADR 0111) or a URL — a
|
||||
// build asked of a seat's path is registered with the URL composed from it when its outcome does not echo
|
||||
// the seat. A seat's path is composed into its URL where the seat's holder is known, so both spellings of
|
||||
// one repository are one; where it is not, a seat's path matches only the same seat's same path.
|
||||
type sourceForms struct {
|
||||
bases map[string]string // the seat's clone base, `scheme://host:port`, by seat; "" where it is not known
|
||||
base func(seat string) string
|
||||
}
|
||||
|
||||
// newSourceForms reads the seats' bases from the mesh once, when first needed.
|
||||
func newSourceForms(ctx context.Context, inv *inventory.Inventory) *sourceForms {
|
||||
f := &sourceForms{bases: map[string]string{}}
|
||||
var world *catalogue.World
|
||||
f.base = func(seat string) string {
|
||||
if b, known := f.bases[seat]; known {
|
||||
return b
|
||||
}
|
||||
if world == nil {
|
||||
w := catalogue.World{}
|
||||
if shelf, err := inv.Catalogue(ctx); err == nil {
|
||||
if read, err := theRestOfTheMesh(ctx, inv, shelf, ""); err == nil {
|
||||
w = read
|
||||
}
|
||||
}
|
||||
world = &w
|
||||
}
|
||||
b, err := seatBase(*world, seat)
|
||||
if err != nil {
|
||||
b = ""
|
||||
}
|
||||
f.bases[seat] = b
|
||||
return b
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// canonical is one spelling of a repository: lower case, no `.git`, no trailing slash, and a seat's path as
|
||||
// the URL its holder serves it at where that is known.
|
||||
func (f *sourceForms) canonical(repository, seat string) string {
|
||||
trim := func(s string) string {
|
||||
s = strings.TrimSpace(strings.ToLower(s))
|
||||
s = strings.TrimRight(s, "/")
|
||||
return strings.TrimRight(strings.TrimSuffix(s, ".git"), "/")
|
||||
}
|
||||
if seat == "" {
|
||||
return trim(repository)
|
||||
}
|
||||
if b := f.base(seat); b != "" {
|
||||
return trim(b + "/" + strings.Trim(repository, "/"))
|
||||
}
|
||||
return "seat:" + seat + ":" + trim(strings.Trim(repository, "/"))
|
||||
}
|
||||
|
||||
// same says two sources are one repository.
|
||||
func (f *sourceForms) same(aRepository, aSeat, bRepository, bSeat string) bool {
|
||||
if aRepository == "" || bRepository == "" {
|
||||
return false
|
||||
}
|
||||
return f.canonical(aRepository, aSeat) == f.canonical(bRepository, bSeat)
|
||||
}
|
||||
|
||||
// buildsFrom says the catalogue builds some module from this repository.
|
||||
func (f *sourceForms) buildsFrom(entries []inventory.Entry, repository, seat string) bool {
|
||||
for _, e := range entries {
|
||||
if e.Provided || e.Source.Repository == "" {
|
||||
continue
|
||||
}
|
||||
if f.same(e.Source.Repository, e.Source.Seat, repository, seat) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// forgeFacts is what the mesh's forge says of a repository a module is registered from (novox/hq ADR 0266).
|
||||
type forgeFacts struct {
|
||||
// ID is the forge's own id of the repository: what tells it from one deleted and made again by its name.
|
||||
ID int64
|
||||
// Guarded is whether the branch is protected as a module's trunk must be: no direct push, at least one
|
||||
// required status, and no administrator merging past one. Why says what is missing when it is not.
|
||||
Guarded bool
|
||||
Why string
|
||||
}
|
||||
|
||||
// askTheForge asks the forge, through its module's tools on the bus, for a repository's id and its branch's
|
||||
// protection. A variable so a test needs no forge.
|
||||
var askTheForge = func(ctx context.Context, owner, repo, branch string) (forgeFacts, error) {
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
return forgeFacts{}, err
|
||||
}
|
||||
defer js.Close()
|
||||
bus := link.OverNATS{Conn: js.Conn()}
|
||||
ask := func(tool string, args map[string]any, into any) error {
|
||||
raw, _ := json.Marshal(args)
|
||||
answer, err := link.Ask(ctx, bus, "gitea", tool, raw, 30*time.Second)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return fmt.Errorf("gitea.%s: %s", tool, answer.Error)
|
||||
}
|
||||
return json.Unmarshal(answer.Result, into)
|
||||
}
|
||||
var found struct {
|
||||
Result struct {
|
||||
ID int64 `json:"id"`
|
||||
FullName string `json:"full_name"`
|
||||
} `json:"result"`
|
||||
}
|
||||
if err := ask("gitea_api", map[string]any{"path": "/repos/" + owner + "/" + repo}, &found); err != nil {
|
||||
return forgeFacts{}, err
|
||||
}
|
||||
if found.Result.ID == 0 {
|
||||
return forgeFacts{}, fmt.Errorf("the forge named no id for %s/%s", owner, repo)
|
||||
}
|
||||
var rules struct {
|
||||
Rules []protectionRule `json:"rules"`
|
||||
}
|
||||
if err := ask("gitea_branch_protection_get", map[string]any{"owner": owner, "repo": repo}, &rules); err != nil {
|
||||
return forgeFacts{}, err
|
||||
}
|
||||
facts := judgedRule(rules.Rules, branch)
|
||||
facts.ID = found.Result.ID
|
||||
return facts, nil
|
||||
}
|
||||
|
||||
// protectionRule is a branch protection rule as the forge's tool summarises it.
|
||||
type protectionRule struct {
|
||||
Rule string `json:"rule"`
|
||||
Push bool `json:"push"`
|
||||
RequiredStatuses []string `json:"required_statuses"`
|
||||
AdminMayOverride bool `json:"admin_may_override"`
|
||||
}
|
||||
|
||||
// judgedRule judges the one rule the forge applies to a branch, as the forge picks it: the rule named for the
|
||||
// branch, else the first glob rule, in the forge's order, that covers it (gitea's first matching rule). Never a
|
||||
// later rule that happens to be stronger: the forge does not read it.
|
||||
func judgedRule(rules []protectionRule, branch string) forgeFacts {
|
||||
var applied *protectionRule
|
||||
for i := range rules {
|
||||
if rules[i].Rule == branch {
|
||||
applied = &rules[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if applied == nil {
|
||||
for i := range rules {
|
||||
if ruleCovers(rules[i].Rule, branch) {
|
||||
applied = &rules[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
switch r := applied; {
|
||||
case r == nil:
|
||||
return forgeFacts{Why: fmt.Sprintf("no protection rule covers %s", branch)}
|
||||
case r.Push:
|
||||
return forgeFacts{Why: fmt.Sprintf("the rule %s lets a person push to %s directly", r.Rule, branch)}
|
||||
case len(r.RequiredStatuses) == 0:
|
||||
return forgeFacts{Why: fmt.Sprintf("the rule %s requires no status before a merge into %s", r.Rule, branch)}
|
||||
case r.AdminMayOverride:
|
||||
return forgeFacts{Why: fmt.Sprintf("the rule %s lets an administrator merge into %s past a status", r.Rule, branch)}
|
||||
}
|
||||
return forgeFacts{Guarded: true}
|
||||
}
|
||||
|
||||
// ruleCovers says a protection rule's name — a branch, or a glob of them — covers a branch, as the forge reads it.
|
||||
func ruleCovers(rule, branch string) bool {
|
||||
if rule == branch {
|
||||
return true
|
||||
}
|
||||
if !strings.ContainsAny(rule, "*?[") {
|
||||
return false
|
||||
}
|
||||
var re strings.Builder
|
||||
re.WriteString("^")
|
||||
for i := 0; i < len(rule); i++ {
|
||||
switch c := rule[i]; {
|
||||
case c == '*' && i+1 < len(rule) && rule[i+1] == '*':
|
||||
re.WriteString(".*")
|
||||
i++
|
||||
case c == '*':
|
||||
re.WriteString("[^/]*")
|
||||
case c == '?':
|
||||
re.WriteString("[^/]")
|
||||
default:
|
||||
re.WriteString(regexp.QuoteMeta(string(c)))
|
||||
}
|
||||
}
|
||||
re.WriteString("$")
|
||||
ok, _ := regexp.MatchString(re.String(), branch)
|
||||
return ok
|
||||
}
|
||||
|
||||
// onTheForge is a source's owner and name on the mesh's own forge (the git seat's holder), and whether it is
|
||||
// there at all.
|
||||
func (f *sourceForms) onTheForge(repository, seat string) (owner, name string, ok bool) {
|
||||
var rest string
|
||||
switch {
|
||||
case seat == gitSeat:
|
||||
rest = strings.Trim(repository, "/")
|
||||
case seat == "":
|
||||
base := f.base(gitSeat)
|
||||
if base == "" {
|
||||
return "", "", false
|
||||
}
|
||||
url, prefix := f.canonical(repository, ""), f.canonical(base, "")+"/"
|
||||
if !strings.HasPrefix(url, prefix) {
|
||||
return "", "", false
|
||||
}
|
||||
// The case the forge spells it with: the URL as given, past the base.
|
||||
rest = strings.TrimSuffix(strings.Trim(repository[len(prefix):], "/"), ".git")
|
||||
default:
|
||||
return "", "", false
|
||||
}
|
||||
parts := strings.Split(rest, "/")
|
||||
if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
|
||||
return "", "", false
|
||||
}
|
||||
return parts[0], parts[1], true
|
||||
}
|
||||
|
||||
// mayRegisterFrom says whether a build's outcome may register module from the source it was built from, and the
|
||||
// forge's id of that repository to record (novox/hq ADR 0266). Through any verb, only:
|
||||
//
|
||||
// - from the module's registered repository — the same repository by the forge's own id, not only its name; or,
|
||||
// for a module new to the catalogue, from a repository the catalogue builds another module from;
|
||||
// - and from a repository on the mesh's forge whose trunk is protected as a trunk must be: no direct push, at
|
||||
// least one required status, no administrator merging past one.
|
||||
//
|
||||
// Anything else only when the build request was kept as asked at the controller's terminal, for this very
|
||||
// repository and path. path is the module's directory as built; branch the trunk it is registered from.
|
||||
func mayRegisterFrom(ctx context.Context, inv *inventory.Inventory, module string, built inventory.Source,
|
||||
buildID, path, branch string) (int64, error) {
|
||||
forms := newSourceForms(ctx, inv)
|
||||
was, err := inv.SourceOf(ctx, module)
|
||||
isNew := errors.Is(err, inventory.ErrNoSuchModule)
|
||||
if err != nil && !isNew {
|
||||
return 0, err
|
||||
}
|
||||
terminal, err := askedHereFor(ctx, inv, forms, buildID, built, path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
refuse := func(why string) (int64, error) {
|
||||
return 0, fmt.Errorf("%w: %s. A module is registered from such a source only by a build asked at the "+
|
||||
"controller's terminal, never through a verb: a registered module is what the next push sends, and whoever "+
|
||||
"may call a verb includes agents (novox/hq ADR 0266)", errNotItsSource, why)
|
||||
}
|
||||
|
||||
var why string
|
||||
var alongside []inventory.Entry // the modules a new one's repository already builds
|
||||
switch {
|
||||
case !isNew && forms.same(was.Repository, was.Seat, built.Repository, built.Seat):
|
||||
case !isNew:
|
||||
registered := was.Repository
|
||||
if registered == "" {
|
||||
registered = "no repository (it was handed over by hand)"
|
||||
}
|
||||
why = fmt.Sprintf("%s is built from %s, and this build is of %s", module, sourceWords(registered, was.Seat),
|
||||
sourceWords(built.Repository, built.Seat))
|
||||
default:
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if !e.Provided && e.Source.Repository != "" &&
|
||||
forms.same(e.Source.Repository, e.Source.Seat, built.Repository, built.Seat) {
|
||||
alongside = append(alongside, e)
|
||||
}
|
||||
}
|
||||
if len(alongside) == 0 {
|
||||
why = fmt.Sprintf("%s is new to the catalogue, and %s is no repository the catalogue builds a module from",
|
||||
module, sourceWords(built.Repository, built.Seat))
|
||||
}
|
||||
}
|
||||
if why != "" && !terminal {
|
||||
return refuse(why)
|
||||
}
|
||||
|
||||
owner, name, onForge := forms.onTheForge(built.Repository, built.Seat)
|
||||
if !onForge {
|
||||
if terminal {
|
||||
fmt.Printf("%s: %s is not on the mesh's forge — registered, as asked at the controller's terminal\n",
|
||||
buildID, sourceWords(built.Repository, built.Seat))
|
||||
return 0, nil
|
||||
}
|
||||
return refuse(fmt.Sprintf("%s is not on the mesh's forge, so whether its trunk is protected cannot be read",
|
||||
sourceWords(built.Repository, built.Seat)))
|
||||
}
|
||||
facts, err := askTheForge(ctx, owner, name, branch)
|
||||
if err != nil {
|
||||
if terminal {
|
||||
fmt.Printf("%s: the forge could not be asked about %s/%s (%v) — registered, as asked at the controller's "+
|
||||
"terminal\n", buildID, owner, name, err)
|
||||
return 0, nil
|
||||
}
|
||||
return refuse(fmt.Sprintf("the forge could not say whether %s/%s's %s is protected: %v", owner, name, branch, err))
|
||||
}
|
||||
if terminal {
|
||||
if why != "" || !facts.Guarded {
|
||||
fmt.Printf("%s: %s — registered, as asked at the controller's terminal\n", buildID,
|
||||
strings.Trim(why+"; "+facts.Why, "; "))
|
||||
}
|
||||
return facts.ID, nil
|
||||
}
|
||||
if !facts.Guarded {
|
||||
return refuse(fmt.Sprintf("%s/%s's %s is not protected as a module's trunk must be: %s", owner, name, branch,
|
||||
facts.Why))
|
||||
}
|
||||
// The same repository by the forge's id, not only its name: one deleted and made again is another.
|
||||
recorded := map[string]int64{}
|
||||
if !isNew {
|
||||
id, err := inv.SourceIdentity(ctx, module)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
recorded[module] = id
|
||||
}
|
||||
for _, e := range alongside {
|
||||
id, err := inv.SourceIdentity(ctx, e.Manifest.Module)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
recorded[e.Manifest.Module] = id
|
||||
}
|
||||
for m, id := range recorded {
|
||||
if id != 0 && id != facts.ID {
|
||||
return refuse(fmt.Sprintf("%s/%s is not the repository %s was registered from: the forge knows it as "+
|
||||
"repository %d, and %s was registered from repository %d — one of that name deleted and made again",
|
||||
owner, name, m, facts.ID, m, id))
|
||||
}
|
||||
}
|
||||
return facts.ID, nil
|
||||
}
|
||||
|
||||
// askedHereFor says the build was asked at the controller's terminal, for this repository and this path: a kept
|
||||
// request marked so, whose source is the outcome's (novox/hq ADR 0266).
|
||||
func askedHereFor(ctx context.Context, inv *inventory.Inventory, forms *sourceForms, buildID string,
|
||||
built inventory.Source, path string) (bool, error) {
|
||||
r, found, err := inv.BuildRequestByID(ctx, buildID)
|
||||
if err != nil || !found || !r.AtTerminal {
|
||||
return false, err
|
||||
}
|
||||
if !forms.same(r.Repository, r.Seat, built.Repository, built.Seat) ||
|
||||
strings.Trim(r.Path, "/") != strings.Trim(path, "/") {
|
||||
fmt.Printf("%s was asked at the terminal of %s at %q, and its outcome is of %s at %q: not the terminal's\n",
|
||||
buildID, sourceWords(r.Repository, r.Seat), r.Path, sourceWords(built.Repository, built.Seat), path)
|
||||
return false, nil
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// sourceWords is a source as a person reads it.
|
||||
func sourceWords(repository, seat string) string {
|
||||
if seat == "" {
|
||||
return repository
|
||||
}
|
||||
return buildSource{Repository: repository, Seat: seat}.String()
|
||||
}
|
||||
|
||||
// verbMayAsk refuses, for a call through a verb, a build of a repository the catalogue builds no module from
|
||||
// (novox/hq ADR 0266): the build node would run what an agent wrote, and its outcome could never be
|
||||
// registered anyway. url is the repository as it is cloned. At the terminal anything may be asked.
|
||||
func verbMayAsk(ctx context.Context, source buildSource, url string) error {
|
||||
if startedAtTheTerminal() {
|
||||
return nil
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
entries, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
forms := newSourceForms(ctx, open.inventory)
|
||||
if forms.buildsFrom(entries, source.Repository, source.Seat) || forms.buildsFrom(entries, url, "") {
|
||||
return nil
|
||||
}
|
||||
return terminalRefusal("%s is no repository the catalogue builds a module from, and a build of any other is "+
|
||||
"asked at the controller's terminal only, never through a verb: a build node runs what the repository "+
|
||||
"says, and its outcome would register a module the next push sends — whoever may call a verb includes "+
|
||||
"agents (novox/hq ADR 0266). Nothing was asked", source)
|
||||
}
|
||||
@@ -0,0 +1,380 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"hash/fnv"
|
||||
"os"
|
||||
"os/exec"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The routes to root a review of ADR 0266 found (novox/hq ADR 0266 §7): an agent makes a repository of its
|
||||
// own — or forks one the mesh builds from — commits a module.json naming a module the mesh runs everywhere
|
||||
// (`sudo`, granting itself a rule without a password; `mesh-host`, the node-engine), and asks the `build` verb
|
||||
// for it. Its outcome was registered under that name from the agent's repository, and the next push sent it.
|
||||
|
||||
// onTrunk is an outcome of a commit on its repository's trunk, as the build seat says it.
|
||||
func onTrunk(id, repository, seat, path string, manifest map[string]any) link.BuildResult {
|
||||
raw, _ := json.Marshal(manifest)
|
||||
r := link.BuildResult{ID: id, Repository: "http://forge.internal:20000/" + repository + ".git", Path: path,
|
||||
Ref: "main", On: "anchor", Commit: "c0ffee0123456789", Manifest: raw,
|
||||
Trunk: "main", OnTrunk: true, Branches: []string{"main"}}
|
||||
if seat != "" {
|
||||
r.Source = &link.SourceOnSeat{Seat: seat, Repository: repository}
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// keptAsked keeps a build request as the asker would: through a verb, or at the terminal.
|
||||
func keptAsked(t *testing.T, inv *inventory.Inventory, id, repository, path string, atTerminal bool) {
|
||||
t.Helper()
|
||||
if err := inv.RecordBuildRequest(t.Context(), inventory.BuildRequest{ID: id, Repository: repository, Seat: "git",
|
||||
Path: path, For: "build", AtTerminal: atTerminal}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// theCatalogue is a mesh whose sudo is built from the catalogue repository and whose node-engine from its own.
|
||||
func theCatalogue(t *testing.T) *stores {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
for _, b := range []link.BuildResult{
|
||||
onTrunk("build-sudo", "novox/mesh-catalog", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "1"}),
|
||||
onTrunk("build-host", "novox/mesh-host", "git", "", map[string]any{"module": "mesh-host", "version": "1"}),
|
||||
} {
|
||||
keptAsked(t, open.inventory, b.ID, b.Source.Repository, b.Path, true)
|
||||
if _, _, err := takeIn(ctx, open.inventory, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return open
|
||||
}
|
||||
|
||||
func TestABuildFromAnAgentsRepositoryIsNotRegisteredAsAModuleTheMeshHolds(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
for _, c := range []struct {
|
||||
name, repository, path, module string
|
||||
}{
|
||||
{"its own repository naming sudo", "agent/sudo", "modules/sudo", "sudo"},
|
||||
{"a fork of the catalogue", "agent/mesh-catalog", "modules/sudo", "sudo"},
|
||||
{"a fork of the node-engine", "agent/mesh-host", "", "mesh-host"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
id := "build-" + strings.ReplaceAll(c.repository, "/", "-")
|
||||
keptAsked(t, open.inventory, id, c.repository, c.path, false) // through the build verb
|
||||
evil := onTrunk(id, c.repository, "git", c.path, map[string]any{"module": c.module, "version": "evil"})
|
||||
_, _, err := takeIn(ctx, open.inventory, evil)
|
||||
if !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("a build of %s was taken in as %s: %v", c.repository, c.module, err)
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := shelf[c.module].Version; got != "1" {
|
||||
t.Fatalf("%s is now %q, from %s", c.module, got, c.repository)
|
||||
}
|
||||
if _, found, _ := open.inventory.BuildByID(ctx, id); !found {
|
||||
t.Errorf("the refused build %s is not recorded", id)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestANewModuleFromARepositoryTheCatalogueDoesNotBuildFromIsNotRegistered(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
keptAsked(t, open.inventory, "build-new", "agent/tools", "", false)
|
||||
_, _, err := takeIn(ctx, open.inventory, onTrunk("build-new", "agent/tools", "git", "",
|
||||
map[string]any{"module": "agent-tools", "version": "1"}))
|
||||
if !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("a new module from an agent's repository was taken in: %v", err)
|
||||
}
|
||||
// And one asked of nobody here — an outcome on the bus no request was kept for — the same.
|
||||
_, _, err = takeIn(ctx, open.inventory, onTrunk("build-unasked", "agent/tools", "git", "",
|
||||
map[string]any{"module": "agent-tools", "version": "1"}))
|
||||
if !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("an outcome nobody asked for was taken in: %v", err)
|
||||
}
|
||||
if shelf, _ := open.inventory.Catalogue(ctx); shelf["agent-tools"].Module != "" {
|
||||
t.Fatal("the refused module is in the catalogue")
|
||||
}
|
||||
}
|
||||
|
||||
// The operator at the terminal may still move a module, or add one from a new repository.
|
||||
func TestAtTheTerminalAnotherRepositoryIsRegistered(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
keptAsked(t, open.inventory, "build-moved", "novox/sudo", "", true)
|
||||
if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-moved", "novox/sudo", "git", "",
|
||||
map[string]any{"module": "sudo", "version": "2"})); err != nil {
|
||||
t.Fatalf("a move the operator asked for at the terminal was refused: %v", err)
|
||||
}
|
||||
if src, _ := open.inventory.SourceOf(ctx, "sudo"); src.Repository != "novox/sudo" {
|
||||
t.Fatalf("sudo is built from %q", src.Repository)
|
||||
}
|
||||
if err := open.inventory.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-external",
|
||||
Repository: "http://forge.internal:20000/someone/app.git", For: "build", AtTerminal: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-external", "someone/app", "", "",
|
||||
map[string]any{"module": "app", "version": "1"})); err != nil {
|
||||
t.Fatalf("a new module the operator asked for at the terminal was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The delivery's flow is untouched: a merge's rebuild of a module from its own repository, and a merge adding
|
||||
// a module to a repository the catalogue builds from (novox/hq issue 300), are registered with no terminal.
|
||||
func TestADeliveryFromTheRegisteredRepositoryIsRegistered(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
rebuilt := onTrunk("build-plan", "novox/mesh-catalog", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "2"})
|
||||
if err := open.inventory.RecordBuildRequest(ctx, inventory.BuildRequest{ID: rebuilt.ID,
|
||||
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/sudo", For: "plan"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, rebuilt); err != nil {
|
||||
t.Fatalf("a plan's build of the module's own repository was refused: %v", err)
|
||||
}
|
||||
added := onTrunk("build-merge", "novox/mesh-catalog", "git", "modules/zram", map[string]any{"module": "zram", "version": "1"})
|
||||
if _, _, err := takeIn(ctx, open.inventory, added); err != nil {
|
||||
t.Fatalf("a module a merge added to the catalogue repository was refused: %v", err)
|
||||
}
|
||||
shelf, _ := open.inventory.Catalogue(ctx)
|
||||
if shelf["sudo"].Version != "2" || shelf["zram"].Module == "" {
|
||||
t.Fatalf("not registered: sudo %q, zram %q", shelf["sudo"].Version, shelf["zram"].Module)
|
||||
}
|
||||
}
|
||||
|
||||
// Through a verb, a build of a repository the catalogue builds nothing from is not even asked: the build node
|
||||
// would run what the agent wrote.
|
||||
func TestAVerbAsksNoBuildOfARepositoryTheCatalogueDoesNotBuildFrom(t *testing.T) {
|
||||
theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
t.Setenv(verbVar, "build")
|
||||
t.Setenv(link.CallerVar, "node-tools.anchor, through the mesh-controller seat")
|
||||
err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, "http://forge.internal:20000/agent/sudo.git")
|
||||
var policy *heldAtTheTerminal
|
||||
if !errors.As(err, &policy) {
|
||||
t.Fatalf("a verb's build of an agent's repository was asked: %v", err)
|
||||
}
|
||||
if err := verbMayAsk(ctx, buildSource{Repository: "novox/mesh-catalog", Seat: "git"},
|
||||
"http://forge.internal:20000/novox/mesh-catalog.git"); err != nil {
|
||||
t.Fatalf("a verb's build of the catalogue repository was refused: %v", err)
|
||||
}
|
||||
t.Setenv(verbVar, "")
|
||||
t.Setenv(link.CallerVar, "")
|
||||
if err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, ""); err != nil {
|
||||
t.Fatalf("the terminal was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// asTheOperator keeps a build as asked at the controller's terminal, as the operator's first build of a module
|
||||
// from a repository the catalogue does not yet build from is (novox/hq ADR 0266), and hands it back.
|
||||
func asTheOperator(t *testing.T, inv *inventory.Inventory, b link.BuildResult) link.BuildResult {
|
||||
t.Helper()
|
||||
repository, seat := b.Repository, ""
|
||||
if b.Source != nil {
|
||||
repository, seat = b.Source.Repository, b.Source.Seat
|
||||
}
|
||||
if err := inv.RecordBuildRequest(t.Context(), inventory.BuildRequest{ID: b.ID, Repository: repository, Seat: seat,
|
||||
Path: b.Path, For: "build", AtTerminal: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// A rollback puts back only a build of the module's own repository: an agent's build of the module's name,
|
||||
// recorded and refused, at the very commit the machine ran before (a fork carries it), is never registered by
|
||||
// the back door of a failed gate.
|
||||
func TestARollbackNeverPutsBackABuildFromAnotherRepository(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
fork := onTrunk("build-1791500000000000000", "agent/mesh-catalog", "git", "modules/sudo",
|
||||
map[string]any{"module": "sudo", "version": "evil"})
|
||||
fork.Commit = "c0ffee0123456789" // the commit sudo was registered at
|
||||
keptAsked(t, inv, fork.ID, "agent/mesh-catalog", "modules/sudo", false)
|
||||
if _, _, err := takeIn(ctx, inv, fork); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("the fork's build was taken in: %v", err)
|
||||
}
|
||||
failed := onTrunk("build-1791600000000000000", "novox/mesh-catalog", "git", "modules/sudo",
|
||||
map[string]any{"module": "sudo", "version": "2"})
|
||||
failed.Commit = "badbadbad0123456"
|
||||
if _, _, err := takeIn(ctx, inv, failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, _, err := inv.BuildByID(ctx, failed.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
previous, found, err := inv.PreviousBuild(ctx, "sudo", "c0ffee0123456789", record)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if found && previous.ID == fork.ID {
|
||||
t.Fatalf("a rollback would put back the fork's build %s", previous.ID)
|
||||
}
|
||||
if !found || previous.ID != "build-sudo" {
|
||||
t.Fatalf("a rollback puts back %q (found %v), want the registered build-sudo", previous.ID, found)
|
||||
}
|
||||
}
|
||||
|
||||
// theForge is what the forge says in a test, by owner/name: a repository not named here is protected as a
|
||||
// trunk must be, with an id of its own.
|
||||
var theForge sync.Map
|
||||
|
||||
func init() {
|
||||
askTheForge = func(_ context.Context, owner, repo, _ string) (forgeFacts, error) {
|
||||
if said, ok := theForge.Load(owner + "/" + repo); ok {
|
||||
switch f := said.(type) {
|
||||
case error:
|
||||
return forgeFacts{}, f
|
||||
case forgeFacts:
|
||||
return f, nil
|
||||
}
|
||||
}
|
||||
h := fnv.New32a()
|
||||
_, _ = h.Write([]byte(owner + "/" + repo))
|
||||
return forgeFacts{ID: int64(h.Sum32()), Guarded: true}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// A module's trunk the forge does not protect — direct pushes, no required status — or a forge that cannot say,
|
||||
// registers nothing through a verb: the trunk rule means nothing on a branch anyone pushes to.
|
||||
func TestATrunkTheForgeDoesNotProtectRegistersNothing(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
theForge.Store("novox/unguarded", forgeFacts{ID: 7, Why: "the rule main lets a person push to main directly"})
|
||||
t.Cleanup(func() { theForge.Delete("novox/unguarded") })
|
||||
first := onTrunk("build-unguarded-1", "novox/unguarded", "git", "", map[string]any{"module": "unguarded", "version": "1"})
|
||||
keptAsked(t, open.inventory, first.ID, "novox/unguarded", "", true)
|
||||
if _, _, err := takeIn(ctx, open.inventory, first); err != nil {
|
||||
t.Fatalf("at the terminal: %v", err)
|
||||
}
|
||||
again := onTrunk("build-unguarded-2", "novox/unguarded", "git", "", map[string]any{"module": "unguarded", "version": "2"})
|
||||
if _, _, err := takeIn(ctx, open.inventory, again); !errors.Is(err, errNotItsSource) ||
|
||||
!strings.Contains(err.Error(), "push to main directly") {
|
||||
t.Fatalf("a rebuild from an unprotected trunk was taken in: %v", err)
|
||||
}
|
||||
theForge.Store("novox/unguarded", errors.New("nothing serves gitea.gitea_api"))
|
||||
if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-unguarded-3", "novox/unguarded", "git", "",
|
||||
map[string]any{"module": "unguarded", "version": "3"})); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("a forge that could not say was read as a protected trunk: %v", err)
|
||||
}
|
||||
if shelf, _ := open.inventory.Catalogue(ctx); shelf["unguarded"].Version != "1" {
|
||||
t.Fatalf("unguarded is %q", shelf["unguarded"].Version)
|
||||
}
|
||||
}
|
||||
|
||||
// A repository deleted and made again under the module's repository's name is another repository: the forge's
|
||||
// id, recorded at registration, tells them apart.
|
||||
func TestARepositoryMadeAgainUnderItsNameIsNotTheModulesSource(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
theForge.Store("novox/remade", forgeFacts{ID: 100, Guarded: true})
|
||||
t.Cleanup(func() { theForge.Delete("novox/remade") })
|
||||
first := onTrunk("build-remade-1", "novox/remade", "git", "", map[string]any{"module": "remade", "version": "1"})
|
||||
keptAsked(t, open.inventory, first.ID, "novox/remade", "", true)
|
||||
if _, _, err := takeIn(ctx, open.inventory, first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if id, _ := open.inventory.SourceIdentity(ctx, "remade"); id != 100 {
|
||||
t.Fatalf("the forge's id was not recorded: %d", id)
|
||||
}
|
||||
theForge.Store("novox/remade", forgeFacts{ID: 101, Guarded: true}) // deleted, and made again by an agent
|
||||
if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-remade-2", "novox/remade", "git", "",
|
||||
map[string]any{"module": "remade", "version": "evil"})); !errors.Is(err, errNotItsSource) ||
|
||||
!strings.Contains(err.Error(), "made again") {
|
||||
t.Fatalf("a repository made again under the name was taken in: %v", err)
|
||||
}
|
||||
// And a new module from it, beside the one registered from the first, the same.
|
||||
if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-remade-3", "novox/remade", "git", "modules/other",
|
||||
map[string]any{"module": "other", "version": "1"})); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("a new module from a repository made again was taken in: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The terminal's mark is the operator's for the repository and path they asked: an outcome of another, under that
|
||||
// build's id, is not theirs.
|
||||
func TestATerminalRequestCoversOnlyWhatItAsked(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-asked", Repository: "novox/app",
|
||||
Seat: "git", Path: "modules/app", For: "build", AtTerminal: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
other := onTrunk("build-asked", "agent/sudo", "git", "modules/app", map[string]any{"module": "sudo", "version": "evil"})
|
||||
if _, _, err := takeIn(ctx, open.inventory, other); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("an outcome of another repository under a terminal request's id was taken in: %v", err)
|
||||
}
|
||||
elsewhere := onTrunk("build-asked", "novox/app", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "evil"})
|
||||
if _, _, err := takeIn(ctx, open.inventory, elsewhere); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("an outcome of another path under a terminal request's id was taken in: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The serving controller, and everything it starts, is never the terminal: a build asked in it reads as asked
|
||||
// through the mesh even when no verb and no caller is named.
|
||||
func TestTheServingControllerIsNeverTheTerminal(t *testing.T) {
|
||||
t.Setenv(verbVar, "")
|
||||
t.Setenv(link.CallerVar, "")
|
||||
t.Setenv(servedVar, "")
|
||||
if !startedAtTheTerminal() {
|
||||
t.Fatal("a process started by hand is not the terminal")
|
||||
}
|
||||
markServed()
|
||||
if startedAtTheTerminal() {
|
||||
t.Fatal("the serving controller reads as the terminal")
|
||||
}
|
||||
child := exec.Command(os.Args[0], "-test.run=^$")
|
||||
child.Env = os.Environ()
|
||||
if !slices.Contains(child.Env, servedVar+"=1") {
|
||||
t.Fatal("what the serving controller starts does not carry its mark")
|
||||
}
|
||||
}
|
||||
|
||||
// The forge applies one rule to a branch: the rule named for it, else the first glob that covers it. A stronger
|
||||
// rule later in the list is not what guards the branch, and is not read as if it were (the confirmation review).
|
||||
func TestTheRuleJudgedIsTheOneTheForgeApplies(t *testing.T) {
|
||||
guarded := protectionRule{Rule: "*", RequiredStatuses: []string{"mesh/merge-gate"}}
|
||||
open := protectionRule{Rule: "main", Push: true, RequiredStatuses: []string{"mesh/merge-gate"}}
|
||||
if f := judgedRule([]protectionRule{guarded, open}, "main"); f.Guarded {
|
||||
t.Error("an exact rule letting pushes was passed over for a glob that guards")
|
||||
}
|
||||
if f := judgedRule([]protectionRule{{Rule: "ma*", RequiredStatuses: nil}, {Rule: "*", RequiredStatuses: []string{"x"}}},
|
||||
"main"); f.Guarded || !strings.Contains(f.Why, "ma*") {
|
||||
t.Errorf("the first glob covering the branch was not the one judged: %+v", f)
|
||||
}
|
||||
if f := judgedRule([]protectionRule{{Rule: "release/*"}, {Rule: "main", RequiredStatuses: []string{"x"}}}, "main"); !f.Guarded {
|
||||
t.Errorf("the rule named for the branch was not judged: %+v", f)
|
||||
}
|
||||
if f := judgedRule(nil, "main"); f.Guarded {
|
||||
t.Error("no rule is no protection")
|
||||
}
|
||||
}
|
||||
|
||||
// An id the forge could not give keeps the id already recorded.
|
||||
func TestAnUnknownIdentityKeepsTheRecordedOne(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.SetSourceIdentity(ctx, "sudo", 100); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetSourceIdentity(ctx, "sudo", 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if id, _ := open.inventory.SourceIdentity(ctx, "sudo"); id != 100 {
|
||||
t.Fatalf("the recorded id became %d", id)
|
||||
}
|
||||
}
|
||||
@@ -19,11 +19,11 @@ func TestABuildHeardIsRecordedAndRegistered(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
manifest, _ := json.Marshal(map[string]any{"module": "shop", "version": "3"})
|
||||
m, _, err := takeIn(ctx, open.inventory, link.BuildResult{
|
||||
m, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, link.BuildResult{
|
||||
ID: "b-1", Repository: "http://forge.internal:20000/novox/shop.git", Path: "modules/shop",
|
||||
Ref: "main", On: "anchor", Commit: "abcdef0123", Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/shop"},
|
||||
})
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -78,7 +78,7 @@ func TestABuildAtACommitKeepsTheBranchTheModuleFollows(t *testing.T) {
|
||||
Path: "modules/unifi", Ref: ref, On: "anchor", Commit: commit, Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, result("b-1", "main", "1111111aaaa")); err != nil {
|
||||
if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, result("b-1", "main", "1111111aaaa"))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, result("b-2", "9c97a8a", "9c97a8a1d2c3")); err != nil {
|
||||
@@ -117,7 +117,7 @@ func TestAnOlderBuildHeardLaterDoesNotReplaceTheNewer(t *testing.T) {
|
||||
Path: "modules/postgres", Ref: "main", On: "anchor", Commit: "efff5415", Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, result(newer, "4bcd5f73")); err != nil {
|
||||
if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, result(newer, "4bcd5f73"))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, err := takeIn(ctx, open.inventory, result(older, "0ab07fa9"))
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A send held for the bus's planned step (novox/hq issue 336).
|
||||
//
|
||||
// **A wait only a person can end is said to that person at once.** No send replaces the bus but its planned
|
||||
// step (sendToEach, ADR 0236), and the step is a person's `bus upgrade`. So while a new bus build waits, every
|
||||
// walk whose tier sends to the bus's machine is refused, and tries again each tick. The refusal was kept only
|
||||
// as the walk's note; nothing was raised, and the operator found the hold by asking why a walk did not move.
|
||||
// Row S17 raises it on the first tick after the first refusal, in the bus's scope, for the operator: what
|
||||
// waits, behind which bus build, since when, and the verb. A walk held only by it is not late, so S3 leaves it
|
||||
// out, as it leaves out a walk under a paused build seat. It clears once the bus's machine has been sent the
|
||||
// build the mesh holds — the step was taken, and no send is refused for the bus any more — whatever the walks'
|
||||
// notes still say. Whether the new bus came up healthy is the step's own condition (probe DB).
|
||||
|
||||
// kindBusStepWaiting is S17's kind: bus.<module>.step-waiting.
|
||||
const kindBusStepWaiting = "bus-step-waiting"
|
||||
|
||||
// busFacts is a new bus build waiting for its planned step, and the sends held for it.
|
||||
type busFacts struct {
|
||||
module, to string
|
||||
// from is the build each machine of the bus runs; machines those whose bus the step would replace.
|
||||
from map[string]string
|
||||
machines []string
|
||||
waits []busWaitFacts
|
||||
}
|
||||
|
||||
// busWaitFacts is one walk whose send was refused because it would replace the bus.
|
||||
type busWaitFacts struct {
|
||||
plan, repository, commit string
|
||||
// modules are what its tier sends: what waits.
|
||||
modules []string
|
||||
// since is the first refusal: as this controller saw it, or, read back, the save that kept the refusal.
|
||||
since time.Time
|
||||
}
|
||||
|
||||
// busRefusedFirst is when this controller first saw each walk refused for the bus's step. The walk's note
|
||||
// keeps the refusal across a restart; this keeps its moment more exactly than the walk's last save.
|
||||
var busRefusedFirst = &firstSeen{at: map[string]time.Time{}}
|
||||
|
||||
type firstSeen struct {
|
||||
mu sync.Mutex
|
||||
at map[string]time.Time
|
||||
}
|
||||
|
||||
// mark keeps the first moment an id was seen.
|
||||
func (s *firstSeen) mark(id string, at time.Time) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if _, seen := s.at[id]; !seen {
|
||||
s.at[id] = at
|
||||
}
|
||||
}
|
||||
|
||||
// of is when an id was first seen; zero when it was not.
|
||||
func (s *firstSeen) of(id string) time.Time {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.at[id]
|
||||
}
|
||||
|
||||
// keepOnly forgets every id not given: a walk no longer held is not held since then.
|
||||
func (s *firstSeen) keepOnly(ids map[string]bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for id := range s.at {
|
||||
if !ids[id] {
|
||||
delete(s.at, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// refusedForTheBus is whether an error is the refusal of a send for the bus's planned step.
|
||||
func refusedForTheBus(err error) bool {
|
||||
return err != nil && strings.Contains(err.Error(), errBusWaits.Error())
|
||||
}
|
||||
|
||||
// busWaitsOf is the sends held for the bus's step, from the open walks: each walk not waiting for its
|
||||
// delivery's word whose note keeps a refusal for this very bus build, while the build would still replace
|
||||
// the bus on a machine. first is when this controller first saw a walk refused, zero when it did not.
|
||||
func busWaitsOf(plans []inventory.Plan, b busPending, first func(string) time.Time) busFacts {
|
||||
f := busFacts{module: b.module, to: b.to, from: b.from}
|
||||
for _, n := range b.machines {
|
||||
if b.moves(n) {
|
||||
f.machines = append(f.machines, n)
|
||||
}
|
||||
}
|
||||
if len(f.machines) == 0 {
|
||||
return f
|
||||
}
|
||||
for _, p := range plans {
|
||||
if !p.Open() || p.Waiting() || !strings.Contains(p.Note, errBusWaits.Error()) || !strings.Contains(p.Note, short(b.to)) {
|
||||
continue
|
||||
}
|
||||
since := p.Updated
|
||||
if seen := first(p.ID); !seen.IsZero() && (since.IsZero() || seen.Before(since)) {
|
||||
since = seen
|
||||
}
|
||||
var modules []string
|
||||
if p.Tier >= 0 && p.Tier < len(p.Tiers) {
|
||||
modules = append(modules, p.Tiers[p.Tier]...)
|
||||
} else {
|
||||
modules = planModules(p)
|
||||
}
|
||||
sort.Strings(modules)
|
||||
f.waits = append(f.waits, busWaitFacts{plan: p.ID, repository: p.Repository, commit: p.Commit, modules: modules,
|
||||
since: since})
|
||||
}
|
||||
sort.Slice(f.waits, func(i, j int) bool { return f.waits[i].since.Before(f.waits[j].since) })
|
||||
return f
|
||||
}
|
||||
|
||||
// heldByTheBus is the walks of a bus's facts, by id: what S3 leaves out.
|
||||
func (b busFacts) heldByTheBus() map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, w := range b.waits {
|
||||
out[w.plan] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// busUpgradeVerb is the call that ends the wait, as the summary names it: through the mesh MCP server, with
|
||||
// why, saying whether the new version can be undone ("reversible": "true") or not ("irreversible": "true").
|
||||
const busUpgradeVerb = "`mesh_call mesh-controller.bus {\"upgrade\": \"true\", \"why\": \"…\", \"reversible\" or \"irreversible\": \"true\"}`"
|
||||
|
||||
// watchBusWaits is S17: a send held for the bus's planned step, said at once to the operator.
|
||||
func watchBusWaits(f *signalFacts) []conditions.Observation {
|
||||
b := f.bus
|
||||
if len(b.waits) == 0 || len(b.machines) == 0 {
|
||||
return nil
|
||||
}
|
||||
since := b.waits[0].since
|
||||
var walks, what []string
|
||||
for _, w := range b.waits {
|
||||
walks = append(walks, fmt.Sprintf("%s (%s %s, tier: %s)", w.plan, w.repository, short(w.commit),
|
||||
strings.Join(w.modules, ", ")))
|
||||
what = append(what, deliveryWhat(w.modules, w.repository))
|
||||
}
|
||||
var from []string
|
||||
for _, n := range b.machines {
|
||||
from = append(from, short(orNotKnown(b.from[n])))
|
||||
}
|
||||
machines := namesWords(b.machines, 3)
|
||||
in := f.now.Sub(since)
|
||||
return []conditions.Observation{{Scope: conditions.ScopeBus, ID: b.module, Token: "step-waiting",
|
||||
Kind: kindBusStepWaiting, Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
|
||||
Machine: b.machines[0], Also: b.machines[1:],
|
||||
Summary: fmt.Sprintf("sends to %s wait for the bus's planned step: a new bus build (%s %s → %s) would replace "+
|
||||
"the bus there, which only a person's %s does; waiting since %s: %s", strings.Join(b.machines, ", "),
|
||||
b.module, strings.Join(sortedUnique(from), ", "), short(b.to), busUpgradeVerb,
|
||||
since.UTC().Format(time.RFC3339), strings.Join(walks, "; ")),
|
||||
Said: fmt.Sprintf("%d walk(s) refused since %s", len(b.waits), since.UTC().Format(time.RFC3339)),
|
||||
Headline: clipWords("Sends to "+machines+" wait for a bus upgrade", conditions.HeadlineMax),
|
||||
Explanation: clipWords(fmt.Sprintf("A new version of the mesh's message system is built, and only a person "+
|
||||
"installs it. Until then nothing else is sent to %s: %s waits, for %s so far.", machines,
|
||||
namesWords(sortedUnique(what), 3), humanDuration(in)), conditions.ExplanationMax),
|
||||
Needs: "start the bus upgrade " + FromMeshMCPServer,
|
||||
Resolved: "Resolved: the bus upgrade started, and sends go on"}}
|
||||
}
|
||||
|
||||
// sortedUnique is a list sorted, each once.
|
||||
func sortedUnique(xs []string) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, x := range xs {
|
||||
if !seen[x] {
|
||||
seen[x] = true
|
||||
out = append(out, x)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// clipWords keeps a plain sentence within a bound, at a word.
|
||||
func clipWords(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
cut := strings.LastIndex(s[:n-1], " ")
|
||||
if cut <= 0 {
|
||||
cut = n - 1
|
||||
}
|
||||
return s[:cut] + "…"
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq issue 336: a send held because it would replace the bus outside its planned step waits for a
|
||||
// person, so a person is told at once — what waits, behind which bus build, since when, and the verb that
|
||||
// ends it — and the wait is not read as a walk running late.
|
||||
|
||||
// aBusOnAnchor is a new bus build waiting for its step on anchor: nats 88135ad0 there, 32307bd1 held.
|
||||
func aBusOnAnchor() busPending {
|
||||
return busPending{module: "nats", machines: []string{"anchor"}, from: map[string]string{"anchor": "88135ad0aaaa"},
|
||||
to: "32307bd1bbbb", same: map[string]bool{}}
|
||||
}
|
||||
|
||||
// refusedNote is the note a walk keeps when its send was refused for the bus, as advanceHeld writes it.
|
||||
func refusedNote(b busPending, tier int) string {
|
||||
held := "sending anchor would replace the bus (nats " + short(b.from["anchor"]) + " → " + short(b.to) +
|
||||
"), which is a planned step: `bus upgrade --why …` snapshots its streams first and checks them after (novox/hq ADR 0236)"
|
||||
return "tier " + string(rune('0'+tier)) + ": " + errBusWaits.Error() + ": " + held + " — tried again"
|
||||
}
|
||||
|
||||
func TestASendHeldForTheBusStepIsFoundFromTheWalksItHolds(t *testing.T) {
|
||||
now := time.Date(2026, 10, 8, 18, 30, 0, 0, time.UTC)
|
||||
b := aBusOnAnchor()
|
||||
walk := func(id, repo, note string, updated time.Time) inventory.Plan {
|
||||
return inventory.Plan{ID: id, Repository: repo, Commit: "c0ffee001122", State: inventory.PlanRolling, Tier: 0,
|
||||
Tiers: [][]string{{"mesh-host"}}, Modules: map[string]*inventory.PlanModule{"mesh-host": {}}, Note: note,
|
||||
Updated: updated}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
bus busPending
|
||||
plans []inventory.Plan
|
||||
first map[string]time.Time
|
||||
want []string
|
||||
since time.Time
|
||||
}{
|
||||
{"a walk refused for the bus", b,
|
||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now.Add(-27*time.Minute))}, nil,
|
||||
[]string{"plan-1"}, now.Add(-27 * time.Minute)},
|
||||
{"refused earlier than its last save, as this controller saw it", b,
|
||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now.Add(-5*time.Minute))},
|
||||
map[string]time.Time{"plan-1": now.Add(-28 * time.Minute)}, []string{"plan-1"}, now.Add(-28 * time.Minute)},
|
||||
{"a walk refused for another reason", b,
|
||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", "tier 0: the build seat is paused — tried again", now)}, nil,
|
||||
nil, time.Time{}},
|
||||
{"refused for an older bus build than the one held now", func() busPending { o := b; o.to = "99999999cccc"; return o }(),
|
||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)}, nil, nil, time.Time{}},
|
||||
{"the bus already runs the build held: its step started", func() busPending {
|
||||
o := aBusOnAnchor()
|
||||
o.from = map[string]string{"anchor": o.to}
|
||||
return o
|
||||
}(), []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)}, nil, nil, time.Time{}},
|
||||
{"a walk waiting for its delivery's word asks no send", b, func() []inventory.Plan {
|
||||
p := walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)
|
||||
p.Delivery = &inventory.PlanDelivery{Awaits: "mesh-delivery"}
|
||||
return []inventory.Plan{p}
|
||||
}(), nil, nil, time.Time{}},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
got := busWaitsOf(c.plans, c.bus, func(id string) time.Time { return c.first[id] })
|
||||
var ids []string
|
||||
for _, w := range got.waits {
|
||||
ids = append(ids, w.plan)
|
||||
}
|
||||
if strings.Join(ids, ",") != strings.Join(c.want, ",") {
|
||||
t.Fatalf("held for the bus: %v, want %v", ids, c.want)
|
||||
}
|
||||
if len(c.want) > 0 {
|
||||
if !got.waits[0].since.Equal(c.since) {
|
||||
t.Errorf("waiting since %s, want %s", got.waits[0].since, c.since)
|
||||
}
|
||||
if strings.Join(got.machines, ",") != "anchor" || got.to != b.to || got.module != "nats" {
|
||||
t.Errorf("behind %+v", got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// **Said at once, not as lateness, and cleared when the step starts**: the walk held only for the bus raises
|
||||
// the bus's condition on the first tick, before any tier bound, naming what waits, the bus build from and to,
|
||||
// since when and `bus upgrade`, for the operator; S3 says nothing of it even past its bound; and the
|
||||
// condition clears once the bus's machine runs the new build.
|
||||
func TestASendHeldForTheBusStepIsSaidAtOnceAndNotAsLateness(t *testing.T) {
|
||||
now := time.Date(2026, 10, 8, 18, 30, 0, 0, time.UTC)
|
||||
b := aBusOnAnchor()
|
||||
held := func(entered time.Duration) *signalFacts {
|
||||
f := calm(now)
|
||||
f.plans = []planFacts{{id: "plan-1", repository: "novox/mesh-host", commit: "c0ffee00", tier: 0, tiers: 1,
|
||||
entered: now.Add(-entered), bound: 30 * time.Minute, waiting: refusedNote(b, 0), bus: true}}
|
||||
f.bus = busFacts{module: "nats", to: b.to, from: b.from, machines: []string{"anchor"},
|
||||
waits: []busWaitFacts{{plan: "plan-1", repository: "novox/mesh-host", commit: "c0ffee001122",
|
||||
modules: []string{"mesh-host"}, since: now.Add(-time.Minute)}}}
|
||||
return f
|
||||
}
|
||||
|
||||
f := held(time.Minute)
|
||||
got := watchBusWaits(f)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("a send held for the bus a minute ago raised %+v", got)
|
||||
}
|
||||
o := got[0]
|
||||
if o.Key() != "bus.nats.step-waiting" || o.Kind != kindBusStepWaiting || o.Resolver != conditions.ResolverOperator {
|
||||
t.Fatalf("raised %s (%s), resolver %q", o.Key(), o.Kind, o.Resolver)
|
||||
}
|
||||
for _, want := range []string{"anchor", "mesh-host", "88135ad0", "32307bd1", "mesh_call mesh-controller.bus",
|
||||
`"upgrade": "true"`, `"reversible"`, `"irreversible"`, "2026-10-08T18:29:00Z", "plan-1"} {
|
||||
if !strings.Contains(o.Summary, want) {
|
||||
t.Errorf("its summary does not say %q: %s", want, o.Summary)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(o.Explanation, "mesh-host") || !strings.Contains(o.Headline, "bus upgrade") || o.Needs == "" {
|
||||
t.Errorf("its words do not say what waits and what the operator does: %+v", o)
|
||||
}
|
||||
if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Explanation: o.Explanation,
|
||||
Resolved: o.Resolved, Needs: o.Needs}, "anchor"); !ok {
|
||||
t.Errorf("its words are not plain: %s", why)
|
||||
}
|
||||
|
||||
// Past S3's bound: still the bus's wait, never a stalled walk.
|
||||
late := held(45 * time.Minute)
|
||||
if s3 := watchPlans(late); len(s3) != 0 {
|
||||
t.Fatalf("a walk held only by the bus step was said stalled: %+v", s3)
|
||||
}
|
||||
// A walk held for something else past its bound is still stalled.
|
||||
other := held(45 * time.Minute)
|
||||
other.plans[0].bus = false
|
||||
if s3 := watchPlans(other); len(s3) != 1 {
|
||||
t.Fatalf("a walk held for something else past its bound raised %+v", s3)
|
||||
}
|
||||
|
||||
// Through the keeper: open at the first tick, cleared when the step started.
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: &conditions.Told{},
|
||||
Now: func() time.Time { return now }})
|
||||
defer k.Close(context.Background())
|
||||
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
||||
w.see(t.Context(), held(time.Minute))
|
||||
open, err := k.Open(t.Context())
|
||||
if err != nil || len(open) != 1 || open[0].Key != "bus.nats.step-waiting" {
|
||||
t.Fatalf("after the first tick, open: %+v (%v)", open, err)
|
||||
}
|
||||
started := held(time.Minute)
|
||||
started.bus = busFacts{module: "nats", to: b.to}
|
||||
started.plans[0].bus = false
|
||||
w.see(t.Context(), started)
|
||||
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||
t.Fatalf("the step started, and open: %+v", open)
|
||||
}
|
||||
}
|
||||
@@ -37,8 +37,6 @@ func TestAPushAnswersBeforeItSends(t *testing.T) {
|
||||
}{
|
||||
{"push", map[string]any{"node": "anchor", "why": "w"}, true},
|
||||
{"push", map[string]any{"why": "w"}, true},
|
||||
{"command", map[string]any{"command": "push anchor --why w"}, true},
|
||||
{"command", map[string]any{"command": "push --behind --why=w"}, true},
|
||||
{"command", map[string]any{"command": "builds"}, false},
|
||||
{"status", map[string]any{}, false},
|
||||
{"assign", map[string]any{"node": "anchor", "module": "m"}, false},
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
@@ -50,7 +51,13 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case name == "nats":
|
||||
case (name == "nats" || catalogue.ProvidesBus(e.Manifest)) && len(e.On) > 0:
|
||||
// Said before the merge (novox/hq issue 336): a new bus build holds every send to the bus's machine
|
||||
// until a person takes the step, so merging it is a promise to take it.
|
||||
p.Steps = append(p.Steps, fmt.Sprintf("%s: %s is never sent by an ordinary send, and until %s runs, "+
|
||||
"nothing else is sent to %s either — unless the new build turns out the same as the one running there "+
|
||||
"(issue 280)", busUpgradeNeeded, name, busUpgradeVerb, strings.Join(e.On, ", ")))
|
||||
case name == "nats" || catalogue.ProvidesBus(e.Manifest):
|
||||
p.Steps = append(p.Steps, "a planned bus step: the bus is upgraded by `bus upgrade`, never by an ordinary send")
|
||||
case waits && len(e.On) > 0:
|
||||
p.Steps = append(p.Steps, fmt.Sprintf("%s waits for a person: its policy records (%s)", name,
|
||||
@@ -81,6 +88,9 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
|
||||
return p
|
||||
}
|
||||
|
||||
// busUpgradeNeeded is how a change plan says that merging it holds the bus's machine for a person's step.
|
||||
const busUpgradeNeeded = "merging this needs a person's bus upgrade"
|
||||
|
||||
// summaryOf is a change plan in one line: what it builds, where it goes, and whether the bus moves.
|
||||
func summaryOf(p link.ChangePlan) string {
|
||||
if len(p.Moved) == 0 && len(p.New) == 0 {
|
||||
@@ -110,7 +120,10 @@ func summaryOf(p link.ChangePlan) string {
|
||||
}
|
||||
bus := "no bus step"
|
||||
for _, s := range p.Steps {
|
||||
if strings.HasPrefix(s, "a planned bus step") {
|
||||
switch {
|
||||
case strings.HasPrefix(s, busUpgradeNeeded):
|
||||
bus = busUpgradeNeeded
|
||||
case strings.HasPrefix(s, "a planned bus step") && bus == "no bus step":
|
||||
bus = "a bus step"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -45,7 +45,7 @@ func TestAChangePlanSaysWhatEachMachineReceives(t *testing.T) {
|
||||
}
|
||||
|
||||
p = plan("modules/nats/Dockerfile", "modules/photos/x.js")
|
||||
if !strings.Contains(p.Summary, "a bus step") || !strings.Contains(p.Summary, "2 wait(s) for a person") ||
|
||||
if !strings.Contains(p.Summary, "needs a person's bus upgrade") || !strings.Contains(p.Summary, "2 wait(s) for a person") ||
|
||||
!strings.Contains(p.Summary, "(+1 dependent(s))") {
|
||||
t.Errorf("the bus and a held module read %q", p.Summary)
|
||||
}
|
||||
@@ -58,7 +58,11 @@ func TestAChangePlanSaysWhatEachMachineReceives(t *testing.T) {
|
||||
t.Errorf("the deploy plan reads %v", got)
|
||||
}
|
||||
text := strings.Join(p.Steps, "\n")
|
||||
for _, want := range []string{"a planned bus step", "photos waits for a person", "nats provides mesh-bus"} {
|
||||
// Said before the merge (novox/hq issue 336): merging it holds every send to the bus's machine until a
|
||||
// person runs the bus's step, and the verb that does.
|
||||
for _, want := range []string{"merging this needs a person's bus upgrade", "nothing else is sent to anchor",
|
||||
"mesh_call mesh-controller.bus", "the same as the one running there", "photos waits for a person",
|
||||
"nats provides mesh-bus"} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Errorf("the steps do not say %q:\n%s", want, text)
|
||||
}
|
||||
|
||||
@@ -65,6 +65,13 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
}
|
||||
// A definition names no installation (novox/hq ADR 0112, ADR 0155): judged here, in the
|
||||
// catalogue-wide test, and at registration, which refuses in the same words.
|
||||
if wrong := catalogue.TrustProblems(m); len(wrong) > 0 {
|
||||
for _, p := range wrong {
|
||||
fmt.Fprintf(out, "%s: %s\n", path, p)
|
||||
}
|
||||
failed += len(wrong)
|
||||
faulted[m.Module] = true
|
||||
}
|
||||
if named := catalogue.InstallationProblems(m); len(named) > 0 {
|
||||
for _, p := range named {
|
||||
fmt.Fprintf(out, "%s: %s\n", path, p)
|
||||
@@ -130,6 +137,13 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
}
|
||||
}
|
||||
|
||||
// **A file that asks for a setting and does not say whether it is trusted counts as trusted** (novox/hq issue
|
||||
// 339): listed and counted, never refused, so an author can opt out a file nothing trusts.
|
||||
unsaid := 0
|
||||
for _, name := range names {
|
||||
unsaid += len(catalogue.UnsaidTrust(shelf[name]))
|
||||
}
|
||||
|
||||
for _, name := range names {
|
||||
m := shelf[name]
|
||||
if faulted[name] {
|
||||
@@ -171,6 +185,11 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
if len(checks) > 0 {
|
||||
fmt.Fprintf(out, ", ready: %s", strings.Join(checks, "; "))
|
||||
}
|
||||
if missing := catalogue.UnsaidTrust(m); len(missing) > 0 {
|
||||
fmt.Fprintf(out, "; WARNING: %s ask(s) for a setting and do(es) not say whether it is trusted, so it counts as "+
|
||||
"trusted: set at the terminal alone; say %q false where nothing trusts it (novox/hq issue 339)",
|
||||
strings.Join(missing, ", "), catalogue.TrustedField)
|
||||
}
|
||||
if missing := catalogue.Undeclared(m); len(missing) > 0 {
|
||||
fmt.Fprintf(out, "; WARNING: %s stay(s) up and say(s) not how it is ready — judged by liveness alone, "+
|
||||
"refused from %s (ADR 0240 rule 8)", strings.Join(missing, ", "), catalogue.HealthRequiredFrom.Format("2006-01-02"))
|
||||
@@ -179,6 +198,7 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
}
|
||||
// The count the catalogue keeps (ADR 0240 rule 8), in a line its merge check reads.
|
||||
fmt.Fprintf(out, "%s %d\n", UndeclaredHealthLine, undeclared)
|
||||
fmt.Fprintf(out, "%s %d\n", UnsaidTrustLine, unsaid)
|
||||
if failed > 0 {
|
||||
return fmt.Errorf("%d problem(s) in %d manifest(s)", failed, len(paths))
|
||||
}
|
||||
@@ -193,6 +213,10 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
// `health` in, over the manifests given: the catalogue's merge check compares it with the number it keeps.
|
||||
const UndeclaredHealthLine = "long-running resources without health:"
|
||||
|
||||
// UnsaidTrustLine starts the line `module check` says the count of files that ask for a setting and do not say
|
||||
// whether it is trusted, and so count as trusted (novox/hq issue 339).
|
||||
const UnsaidTrustLine = "files asking for a setting without saying whether it is trusted:"
|
||||
|
||||
// checkNow is the clock `module check` judges the date by; a test sets it.
|
||||
var checkNow = time.Now
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ import (
|
||||
)
|
||||
|
||||
// `check-here` is a pull request's merge check run on the machine at hand **exactly as the build seat
|
||||
// runs it** (novox/hq issue 283): the same code (builder.Check), the same ask the controller would make of
|
||||
// runs it** (novox/hq issue 286): the same code (builder.Check), the same ask the controller would make of
|
||||
// the seat — the gate's modules and judge by the planner's own answer over the facts snapshot, the
|
||||
// repositories beside it at the refs the snapshot says the seat clones them at — in the toolchain image
|
||||
// the mesh holds, as the user the build seat runs as, against a throwaway store and bus of the versions
|
||||
@@ -126,7 +126,7 @@ func checkHereCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
if len(toolchains) == 0 {
|
||||
return errors.New("the facts snapshot names no toolchain: it was taken by a controller from before " +
|
||||
"issue 283, and the seat's toolchain cannot be known here")
|
||||
"issue 286, and the seat's toolchain cannot be known here")
|
||||
}
|
||||
beside := map[string]builder.Beside{}
|
||||
for d, ref := range f.Beside {
|
||||
|
||||
+176
-75
@@ -17,14 +17,173 @@ import (
|
||||
// moment the keep set moved, so it is the moment to say what may go — and it needs no timer of
|
||||
// its own. Reclaiming the bytes is the store's own nightly step; this only decides.
|
||||
//
|
||||
// **And run when a person asks** (novox/hq ADR 0251): the `collect` verb is the same sweep, with
|
||||
// bounds of its own. One implementation, so what a person runs is exactly what a build runs.
|
||||
//
|
||||
// Never fatal to a build. The build succeeded, the module is registered, and a store that could
|
||||
// not be reached is a thing to say rather than a reason to undo any of that. The next build asks
|
||||
// again, and the references it could not collect are still uncollected, so nothing is lost by
|
||||
// having failed.
|
||||
|
||||
// sweepBounds is how much one sweep may ask of the store.
|
||||
type sweepBounds struct {
|
||||
// most is how many artifacts it lets go of at most.
|
||||
most int
|
||||
// budget is the longest it keeps its caller waiting.
|
||||
budget time.Duration
|
||||
// platforms is whether an index is let go of with its own platform manifests (novox/hq ADR 0257).
|
||||
// Only a sweep a person confirmed, after its dry run listed what stays and names a platform: the
|
||||
// records cannot see an unrecorded index, or a copy in progress, naming the same platform, and the
|
||||
// store's tools can.
|
||||
platforms bool
|
||||
}
|
||||
|
||||
// afterBuild are the bounds of the sweep inside somebody's build.
|
||||
//
|
||||
// **Bounded, because this runs inside somebody's build.** The first sweep of a mesh that has never
|
||||
// collected has the whole history to get through, and a person waiting on `build` should not pay for
|
||||
// it. What is left over is simply offered again next time — builds are frequent, and the point is
|
||||
// that the store stops growing, not that it empties tonight.
|
||||
var afterBuild = sweepBounds{most: mostPerSweep, budget: sweepBudget}
|
||||
|
||||
// sweepResult is what one sweep did.
|
||||
type sweepResult struct {
|
||||
// HoldersWritten is how many kept archives the store now holds by a manifest it did not before.
|
||||
HoldersWritten int
|
||||
// Missing is how many kept archives the store does not have at all.
|
||||
Missing int
|
||||
// Held is whether every kept archive could be asked to be held; false stops the sweep before it
|
||||
// lets anything go.
|
||||
Held bool
|
||||
// LetGo is what the store let go of, and the record now says collected.
|
||||
LetGo []string
|
||||
// Skipped is how many references the sweep will not address (novox/hq issue 226).
|
||||
Skipped int
|
||||
// Indexes is how many eligible indexes a sweep a person did not confirm left for one that is: an
|
||||
// index goes only with its platform manifests (novox/hq ADR 0257 §4).
|
||||
Indexes int
|
||||
// Left is how many eligible references were not asked about this time.
|
||||
Left int
|
||||
// Bounded is whether it stopped at its bounds rather than at a refusal.
|
||||
Bounded bool
|
||||
// Stopped says why the sweep stopped before the end; empty when it did not.
|
||||
Stopped string
|
||||
}
|
||||
|
||||
// sweep holds every kept archive, then asks the store to let go of each reference in order, records
|
||||
// what it let go of, and says what it did.
|
||||
func sweep(ctx context.Context, inv *inventory.Inventory, store artifacts.Store, references, kept []string,
|
||||
bounds sweepBounds) sweepResult {
|
||||
within, stop := context.WithTimeout(ctx, bounds.budget)
|
||||
defer stop()
|
||||
var r sweepResult
|
||||
|
||||
// **Hold before letting go** (novox/hq issue 253). The store's collector keeps only what a
|
||||
// manifest names, and archives were published as bare blobs, so every kept archive is first
|
||||
// held by its manifest — which backfills the ones published before holders, a few at a time
|
||||
// as builds come, and is two HEADs each once done. A kept archive that could not be held stops
|
||||
// the sweep before it deletes anything: "everything kept is held" is the precondition the
|
||||
// collector's safety rests on, and a store refusing a hold would refuse the deletes too.
|
||||
// **An index goes with its platform manifests only when a person confirmed it** (novox/hq ADR
|
||||
// 0257), and a kept index keeps its own: which platform manifests the kept indexes name is read
|
||||
// from the store for every repository the sweep will touch, before the first delete. A single
|
||||
// read that fails stops the sweep before it deletes anything, as a kept archive that cannot be
|
||||
// held does. The sweep after a build leaves an eligible index for such a collect, below.
|
||||
store.Spare = nil
|
||||
if bounds.platforms {
|
||||
if inv == nil {
|
||||
r.Stopped = "no records to read which platform manifests a kept index names, so nothing was let go"
|
||||
r.Left = len(references)
|
||||
return r
|
||||
}
|
||||
images, err := inv.KeptImages(ctx)
|
||||
if err == nil {
|
||||
store.Spare, err = store.SpareKeptIndexes(within, images, references)
|
||||
}
|
||||
if err != nil {
|
||||
r.Stopped = fmt.Sprintf("which platform manifests a kept index names could not be read, so nothing was let go: %v", err)
|
||||
r.Left = len(references)
|
||||
return r
|
||||
}
|
||||
}
|
||||
|
||||
wrote, missing, err := holdKept(within, store, kept)
|
||||
r.HoldersWritten, r.Missing = wrote, missing
|
||||
if err != nil {
|
||||
r.Stopped = fmt.Sprintf("not every kept archive could be held, so nothing was let go: %v", err)
|
||||
r.Left = len(references)
|
||||
return r
|
||||
}
|
||||
r.Held = true
|
||||
|
||||
for i, reference := range references {
|
||||
if i >= bounds.most || within.Err() != nil {
|
||||
r.Left = len(references) - i
|
||||
r.Bounded = true
|
||||
if i >= bounds.most {
|
||||
r.Stopped = fmt.Sprintf("asked about %d, the most one sweep asks", bounds.most)
|
||||
} else {
|
||||
r.Stopped = fmt.Sprintf("ran out of its %s", bounds.budget)
|
||||
}
|
||||
break
|
||||
}
|
||||
if !bounds.platforms {
|
||||
// **An index waits for a confirmed collect** (novox/hq ADR 0257 §4). Let go of alone,
|
||||
// its platform manifests would stay in the store and the record would say collected, so
|
||||
// no later sweep would offer it again and they would stay for ever. Left eligible, the
|
||||
// next collect a person confirms takes it with them.
|
||||
index, err := store.IsIndex(within, reference)
|
||||
if err != nil && !errors.Is(err, artifacts.Gone) && !errors.Is(err, artifacts.ErrNotOurs) {
|
||||
r.Stopped = fmt.Sprintf("the artifact store could not say whether %s is an index, so nothing more was asked of it: %v", reference, err)
|
||||
r.Left = len(references) - i
|
||||
break
|
||||
}
|
||||
if index {
|
||||
r.Indexes++
|
||||
continue
|
||||
}
|
||||
}
|
||||
err := store.LetGo(within, reference)
|
||||
if err == nil || errors.Is(err, artifacts.Gone) {
|
||||
// Gone is the outcome wanted, already true. Recorded so the next sweep does not ask
|
||||
// again for ever.
|
||||
r.LetGo = append(r.LetGo, reference)
|
||||
continue
|
||||
}
|
||||
if errors.Is(err, artifacts.ErrNotOurs) {
|
||||
// **A fact about this record, so this record is skipped** (novox/hq issue 226). Not
|
||||
// marked collected — the mesh did not remove it and should not claim to — and not a
|
||||
// reason to stop, because the store was never asked. One of these at the front of
|
||||
// the oldest-first order ended every sweep until this.
|
||||
r.Skipped++
|
||||
if r.Skipped == 1 {
|
||||
fmt.Fprintf(os.Stderr, "the sweep will not address %s and went on: %v\n", reference, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
// **Stopped at the first refusal by the STORE, not pushed through.** A store that refuses
|
||||
// one refuses all of them — deletion disabled, the store down, the network gone — so
|
||||
// going on would be a hundred identical failures and a hundred identical log lines in
|
||||
// front of whoever was building something.
|
||||
r.Stopped = fmt.Sprintf("the artifact store kept %s, so nothing more was asked of it: %v", reference, err)
|
||||
r.Left = len(references) - i
|
||||
break
|
||||
}
|
||||
|
||||
if len(r.LetGo) > 0 {
|
||||
// Recorded outside `within`: the deletions happened, and losing the record of them because
|
||||
// the sweep ran out of budget would mean asking about them again for ever.
|
||||
if err := inv.MarkCollected(ctx, r.LetGo); err != nil {
|
||||
r.Stopped = fmt.Sprintf("the store let go of %d artifact(s) and the record of it did not keep: %v",
|
||||
len(r.LetGo), err)
|
||||
}
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// collect asks the store to let go of everything the mesh made and no longer keeps, and records
|
||||
// what it let go of. Says what it did and what it could not; returns nothing, because nothing
|
||||
// upstream should branch on it.
|
||||
// what it let go of — after a build. Says what it did and what it could not; returns nothing,
|
||||
// because nothing upstream should branch on it.
|
||||
func collect(ctx context.Context, inv *inventory.Inventory) {
|
||||
references, err := inv.ToCollect(ctx)
|
||||
if err != nil {
|
||||
@@ -55,86 +214,28 @@ func collect(ctx context.Context, inv *inventory.Inventory) {
|
||||
return
|
||||
}
|
||||
|
||||
// **Bounded, because this runs inside somebody's build.** The first sweep of a mesh that has
|
||||
// never collected has the whole history to get through, and a person waiting on `build` should
|
||||
// not pay for it. Two bounds, and what is left over is simply offered again next time —
|
||||
// builds are frequent, and the point is that the store stops growing, not that it empties
|
||||
// tonight.
|
||||
within, stop := context.WithTimeout(ctx, sweepBudget)
|
||||
defer stop()
|
||||
store := artifacts.Store{Address: address}
|
||||
|
||||
// **Hold before letting go** (novox/hq issue 253). The store's collector keeps only what a
|
||||
// manifest names, and archives were published as bare blobs, so every kept archive is first
|
||||
// held by its manifest — which backfills the ones published before holders, a few at a time
|
||||
// as builds come, and is two HEADs each once done. A kept archive that could not be held stops
|
||||
// the sweep before it deletes anything: "everything kept is held" is the precondition the
|
||||
// collector's safety rests on, and a store refusing a hold would refuse the deletes too.
|
||||
wrote, missing, err := holdKept(within, store, kept)
|
||||
if wrote > 0 {
|
||||
fmt.Fprintf(os.Stderr, "the artifact store now holds %d more kept archive(s) by a manifest\n", wrote)
|
||||
r := sweep(ctx, inv, artifacts.Store{Address: address}, references, kept, afterBuild)
|
||||
if r.HoldersWritten > 0 {
|
||||
fmt.Fprintf(os.Stderr, "the artifact store now holds %d more kept archive(s) by a manifest\n", r.HoldersWritten)
|
||||
}
|
||||
if missing > 0 {
|
||||
if r.Missing > 0 {
|
||||
fmt.Fprintf(os.Stderr, "%d archive(s) the mesh keeps are not in the artifact store at all; "+
|
||||
"`collection` lists them\n", missing)
|
||||
"`collection` lists them\n", r.Missing)
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "not every kept archive could be held, so nothing was let go: %v\n", err)
|
||||
return
|
||||
if len(r.LetGo) > 0 {
|
||||
fmt.Fprintf(os.Stderr, "the artifact store let go of %d artifact(s) the mesh no longer keeps\n", len(r.LetGo))
|
||||
}
|
||||
|
||||
var done []string
|
||||
var left, skipped int
|
||||
for i, reference := range references {
|
||||
if i >= mostPerSweep || within.Err() != nil {
|
||||
left = len(references) - i
|
||||
break
|
||||
}
|
||||
err := store.LetGo(within, reference)
|
||||
if err == nil || errors.Is(err, artifacts.Gone) {
|
||||
// Gone is the outcome wanted, already true. Recorded so the next sweep does not ask
|
||||
// again for ever.
|
||||
done = append(done, reference)
|
||||
continue
|
||||
}
|
||||
if errors.Is(err, artifacts.ErrNotOurs) {
|
||||
// **A fact about this record, so this record is skipped** (novox/hq issue 226). Not
|
||||
// marked collected — the mesh did not remove it and should not claim to — and not a
|
||||
// reason to stop, because the store was never asked. One of these at the front of
|
||||
// the oldest-first order ended every sweep until this.
|
||||
skipped++
|
||||
if skipped == 1 {
|
||||
fmt.Fprintf(os.Stderr,
|
||||
"the sweep will not address %s and went on: %v\n", reference, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
// **Stopped at the first refusal by the STORE, not pushed through.** A store that refuses
|
||||
// one refuses all of them — deletion disabled, the store down, the network gone — so
|
||||
// going on would be a hundred identical failures and a hundred identical log lines in
|
||||
// front of whoever was building something.
|
||||
fmt.Fprintf(os.Stderr, "the artifact store kept %s, so nothing more was asked of it: %v\n",
|
||||
reference, err)
|
||||
left = len(references) - i
|
||||
break
|
||||
if r.Stopped != "" && !r.Bounded {
|
||||
fmt.Fprintln(os.Stderr, r.Stopped)
|
||||
}
|
||||
|
||||
if len(done) > 0 {
|
||||
// Recorded outside `within`: the deletions happened, and losing the record of them because
|
||||
// the sweep ran out of budget would mean asking about them again for ever.
|
||||
if err := inv.MarkCollected(ctx, done); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "the store let go of %d artifact(s) and the record of it did not keep: %v\n",
|
||||
len(done), err)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "the artifact store let go of %d artifact(s) the mesh no longer keeps\n",
|
||||
len(done))
|
||||
if r.Held && r.Left > 0 {
|
||||
fmt.Fprintf(os.Stderr, "%d more to collect; the next build asks again\n", r.Left)
|
||||
}
|
||||
if left > 0 {
|
||||
fmt.Fprintf(os.Stderr, "%d more to collect; the next build asks again\n", left)
|
||||
if r.Skipped > 0 {
|
||||
fmt.Fprintf(os.Stderr, "%d artifact(s) the sweep will not address were skipped\n", r.Skipped)
|
||||
}
|
||||
if skipped > 0 {
|
||||
fmt.Fprintf(os.Stderr, "%d artifact(s) the sweep will not address were skipped\n", skipped)
|
||||
if r.Indexes > 0 {
|
||||
fmt.Fprintf(os.Stderr, "%d eligible index(es) wait for a collect a person confirms, which takes their platforms too\n", r.Indexes)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
@@ -107,19 +108,20 @@ func conditionsCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
switch sub {
|
||||
case "list":
|
||||
return listConditions(ctx, args)
|
||||
return listConditions(ctx, args, os.Stdout)
|
||||
case "show":
|
||||
return showCondition(ctx, args)
|
||||
return showCondition(ctx, args, os.Stdout)
|
||||
case "silence":
|
||||
return silenceCondition(ctx, args)
|
||||
case "history":
|
||||
return conditionHistory(ctx, args)
|
||||
return conditionHistory(ctx, args, os.Stdout)
|
||||
}
|
||||
return errors.New(conditionsUsage)
|
||||
}
|
||||
|
||||
func listConditions(ctx context.Context, args []string) error {
|
||||
func listConditions(ctx context.Context, args []string, w io.Writer) error {
|
||||
set := flag.NewFlagSet("conditions", flag.ContinueOnError)
|
||||
usageTo(set, w)
|
||||
scope := set.String("scope", "", "only this scope: "+strings.Join(conditions.Scopes, ", "))
|
||||
severity := set.String("severity", "", "only urgent, or only warning")
|
||||
machine := set.String("machine", "", "only those about this machine")
|
||||
@@ -144,26 +146,64 @@ func listConditions(ctx context.Context, args []string) error {
|
||||
}
|
||||
}
|
||||
if *asJSON {
|
||||
if out == nil {
|
||||
out = []conditions.Condition{}
|
||||
}
|
||||
return printJSON(map[string]any{"conditions": out, "open": len(open),
|
||||
"note": "urgent first, then oldest first; a condition clears when observation says so, never by hand"})
|
||||
return printJSONTo(w, map[string]any{"conditions": inBrief(out), "open": len(open), "counted": counted(out),
|
||||
"note": "urgent first, then oldest first; a condition clears when observation says so, never by hand; " +
|
||||
"each with its newest evidence — `conditions key=<key>` gives one whole"})
|
||||
}
|
||||
if len(out) == 0 {
|
||||
if len(open) == 0 {
|
||||
fmt.Println("no open conditions")
|
||||
fmt.Fprintln(w, "no open conditions")
|
||||
} else {
|
||||
fmt.Printf("none of the %d open condition(s) is about that\n", len(open))
|
||||
fmt.Fprintf(w, "none of the %d open condition(s) is about that\n", len(open))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for _, line := range conditionLines(out, time.Now()) {
|
||||
fmt.Println(line)
|
||||
fmt.Fprintln(w, line)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// briefEvidence is how many observations a condition carries in a list of them: its newest. The store
|
||||
// keeps ten, and a list of hundreds of conditions with ten each was the larger half of an answer more
|
||||
// than the bus carries in one message (novox/hq issue 314). One condition whole is `conditions key=`.
|
||||
const briefEvidence = 1
|
||||
|
||||
// conditionInBrief is a condition as a list carries it: its newest evidence, with how much more there is
|
||||
// and where to read it. Everything else of it, so a reader of the list — the operator's channel among
|
||||
// them — reads the same condition it always did.
|
||||
type conditionInBrief struct {
|
||||
conditions.Condition
|
||||
// EvidenceKept is how many observations the condition keeps, when the list shows fewer.
|
||||
EvidenceKept int `json:"evidence-kept,omitempty"`
|
||||
More string `json:"more,omitempty"`
|
||||
}
|
||||
|
||||
// inBrief is a list of conditions in brief; never null.
|
||||
func inBrief(list []conditions.Condition) []conditionInBrief {
|
||||
out := make([]conditionInBrief, 0, len(list))
|
||||
for _, c := range list {
|
||||
b := conditionInBrief{Condition: c}
|
||||
if len(c.Evidence) > briefEvidence {
|
||||
b.EvidenceKept = len(c.Evidence)
|
||||
b.Evidence = c.Evidence[:briefEvidence:briefEvidence]
|
||||
b.More = c.Show()
|
||||
}
|
||||
out = append(out, b)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// counted is how many of a list each source raised of each kind — `D14 stalled: 689` — so a list of
|
||||
// hundreds says its shape before its lines.
|
||||
func counted(list []conditions.Condition) map[string]int {
|
||||
out := map[string]int{}
|
||||
for _, c := range list {
|
||||
out[c.Source+" "+c.Kind]++
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// concerns says whether a condition is about a machine: it names it, or its key does.
|
||||
func concerns(c conditions.Condition, machine string) bool {
|
||||
if c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) {
|
||||
@@ -195,8 +235,9 @@ func conditionLines(list []conditions.Condition, now time.Time) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
func showCondition(ctx context.Context, args []string) error {
|
||||
func showCondition(ctx context.Context, args []string, w io.Writer) error {
|
||||
set := flag.NewFlagSet("conditions show", flag.ContinueOnError)
|
||||
usageTo(set, w)
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
rest, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
@@ -228,34 +269,34 @@ func showCondition(ctx context.Context, args []string) error {
|
||||
"history --key %s` what became of it", key, key)
|
||||
}
|
||||
if *asJSON {
|
||||
return printJSON(c)
|
||||
return printJSONTo(w, c)
|
||||
}
|
||||
now := time.Now()
|
||||
fmt.Printf("%s %s\n %s\n\n", strings.ToUpper(string(c.Severity)), c.Key, c.Summary)
|
||||
fmt.Printf(" kind %s\n about %s %s", c.Kind, c.Subject.Scope, c.Subject.ID)
|
||||
fmt.Fprintf(w, "%s %s\n %s\n\n", strings.ToUpper(string(c.Severity)), c.Key, c.Summary)
|
||||
fmt.Fprintf(w, " kind %s\n about %s %s", c.Kind, c.Subject.Scope, c.Subject.ID)
|
||||
if c.Subject.Machine != "" {
|
||||
fmt.Printf(", on %s", c.Subject.Machine)
|
||||
fmt.Fprintf(w, ", on %s", c.Subject.Machine)
|
||||
}
|
||||
fmt.Printf("\n raised by %s\n since %s (%s ago), observed %d time(s), last %s ago\n",
|
||||
fmt.Fprintf(w, "\n raised by %s\n since %s (%s ago), observed %d time(s), last %s ago\n",
|
||||
c.Source, c.Raised.Local().Format("2006-01-02 15:04:05"), roughly(now.Sub(c.Raised)), c.Observations,
|
||||
now.Sub(c.LastObserved).Round(time.Second))
|
||||
if c.Count > 1 {
|
||||
fmt.Printf(" raised %d times, each within ten minutes of clearing\n", c.Count)
|
||||
fmt.Fprintf(w, " raised %d times, each within ten minutes of clearing\n", c.Count)
|
||||
}
|
||||
fmt.Printf(" resolved by %s\n", resolverWords(c.Resolver))
|
||||
fmt.Fprintf(w, " resolved by %s\n", resolverWords(c.Resolver))
|
||||
if c.Silenced != nil {
|
||||
fmt.Printf(" silenced until %s by %s: %s\n", c.Silenced.Until.Local().Format("2006-01-02 15:04"),
|
||||
fmt.Fprintf(w, " silenced until %s by %s: %s\n", c.Silenced.Until.Local().Format("2006-01-02 15:04"),
|
||||
c.Silenced.By, c.Silenced.Why)
|
||||
}
|
||||
if len(c.Tried) > 0 {
|
||||
fmt.Println("\n tried:")
|
||||
fmt.Fprintln(w, "\n tried:")
|
||||
for _, t := range c.Tried {
|
||||
fmt.Printf(" %s %s — %s: %s\n", t.At.Local().Format("2006-01-02 15:04"), orHealer(t.By), t.What, t.Outcome)
|
||||
fmt.Fprintf(w, " %s %s — %s: %s\n", t.At.Local().Format("2006-01-02 15:04"), orHealer(t.By), t.What, t.Outcome)
|
||||
}
|
||||
}
|
||||
fmt.Println("\n evidence, newest first:")
|
||||
fmt.Fprintln(w, "\n evidence, newest first:")
|
||||
for _, e := range c.Evidence {
|
||||
fmt.Printf(" %s %s\n", e.At.Local().Format("2006-01-02 15:04:05"), e.Said)
|
||||
fmt.Fprintf(w, " %s %s\n", e.At.Local().Format("2006-01-02 15:04:05"), e.Said)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -340,8 +381,9 @@ func parseFor(s string) (time.Duration, error) {
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func conditionHistory(ctx context.Context, args []string) error {
|
||||
func conditionHistory(ctx context.Context, args []string, w io.Writer) error {
|
||||
set := flag.NewFlagSet("conditions history", flag.ContinueOnError)
|
||||
usageTo(set, w)
|
||||
days := set.Int("days", 7, "how many days back, at most 90")
|
||||
key := set.String("key", "", "only this condition")
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
@@ -382,10 +424,10 @@ func conditionHistory(ctx context.Context, args []string) error {
|
||||
if out == nil {
|
||||
out = []conditions.Event{}
|
||||
}
|
||||
return printJSON(map[string]any{"history": out, "days": *days})
|
||||
return printJSONTo(w, map[string]any{"history": out, "days": *days})
|
||||
}
|
||||
if len(out) == 0 {
|
||||
fmt.Printf("nothing was raised, changed or cleared in the last %d day(s)\n", *days)
|
||||
fmt.Fprintf(w, "nothing was raised, changed or cleared in the last %d day(s)\n", *days)
|
||||
return nil
|
||||
}
|
||||
for _, e := range out {
|
||||
@@ -402,7 +444,7 @@ func conditionHistory(ctx context.Context, args []string) error {
|
||||
line += " — " + e.Why
|
||||
}
|
||||
}
|
||||
fmt.Println(line)
|
||||
fmt.Fprintln(w, line)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// stalledConditions are n open conditions like those of 2026-10-08: one per delivery held past its bound,
|
||||
// each keeping all its evidence.
|
||||
func stalledConditions(n int) []conditions.Condition {
|
||||
at := time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC)
|
||||
var out []conditions.Condition
|
||||
for i := 0; i < n; i++ {
|
||||
id := fmt.Sprintf("d-%012x", 0x51a11ed+i)
|
||||
lines := stalledObservations([]stalledLine{{ID: id, State: "checking", For: "13h2m", Bound: "45m",
|
||||
H2: "close: recheck", Says: "the check was asked and nothing has answered"}})
|
||||
o := lines[0]
|
||||
c := conditions.Condition{Key: o.Key(), Kind: o.Kind, Subject: conditions.Subject{Scope: o.Scope, ID: o.ID},
|
||||
Severity: o.Severity, Summary: o.Summary, Source: probeDeliveriesID, Raised: at, LastObserved: at,
|
||||
Observations: 160, Count: 1, Resolver: conditions.ResolverSelf}
|
||||
for e := 0; e < conditions.KeptEvidence; e++ {
|
||||
c.Evidence = append(c.Evidence, conditions.Evidence{At: at.Add(time.Duration(e) * 5 * time.Minute),
|
||||
Said: "checking for " + (13*time.Hour + time.Duration(e)*time.Minute).String()})
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// novox/hq issue 314: a list of hundreds of open conditions carries each one's newest evidence, with
|
||||
// how much more it keeps and where to read it, and the verb's answer says it once — as data, not again
|
||||
// as the text the command printed. On 2026-10-08, 689 stalled deliveries made that answer more than the
|
||||
// bus carries in one message; now it is less than half of one.
|
||||
func TestAListOfConditionsCarriesEachInBrief(t *testing.T) {
|
||||
list := stalledConditions(689)
|
||||
printed, _ := json.MarshalIndent(map[string]any{"conditions": inBrief(list), "open": len(list),
|
||||
"counted": counted(list)}, "", " ")
|
||||
var parsed any
|
||||
if err := json.Unmarshal(printed, &parsed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wire, _ := json.Marshal(map[string]any{"result": verbAnswer{OK: true, Answer: parsed,
|
||||
Output: "its answer, as data, is `answer`\n"}})
|
||||
before, _ := json.MarshalIndent(map[string]any{"conditions": list}, "", " ")
|
||||
t.Logf("689 stalled deliveries: %d bytes on the wire, where the whole list printed was %d and was sent twice",
|
||||
len(wire), len(before))
|
||||
if len(wire) >= 512<<10 {
|
||||
t.Fatalf("the answer of 689 conditions is %d bytes, not under half of one message of the bus", len(wire))
|
||||
}
|
||||
var doc struct {
|
||||
Conditions []conditionInBrief `json:"conditions"`
|
||||
Counted map[string]int `json:"counted"`
|
||||
}
|
||||
if err := json.Unmarshal(printed, &doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c := doc.Conditions[0]
|
||||
if len(doc.Conditions) != 689 || len(c.Evidence) != 1 || c.EvidenceKept != conditions.KeptEvidence ||
|
||||
c.Key != list[0].Key || c.Summary != list[0].Summary || !strings.Contains(c.More, "key="+c.Key) {
|
||||
t.Fatalf("a condition in brief reads %+v", c)
|
||||
}
|
||||
if doc.Counted["D14 stalled"] != 689 {
|
||||
t.Fatalf("counted %v", doc.Counted)
|
||||
}
|
||||
if one := inBrief(list[:1])[0]; len(list[0].Evidence) != conditions.KeptEvidence || one.Evidence[0] != list[0].Evidence[0] {
|
||||
t.Fatal("the brief changed the condition it was made from, or kept other than the newest")
|
||||
}
|
||||
}
|
||||
|
||||
// The self-check's verdict carries at most overviewFindings of one probe's findings, with their count and
|
||||
// how to read them all; `doctor probe=` gives that probe whole (novox/hq issue 314).
|
||||
func TestAVerdictCarriesAProbesFindingsInBrief(t *testing.T) {
|
||||
var found []string
|
||||
for i := 0; i < 689; i++ {
|
||||
found = append(found, fmt.Sprintf("delivery d-%012x has been checking for 13h, past its bound of 45m", i))
|
||||
}
|
||||
run := doctorRun{Run: "run-1", Probes: []probeVerdict{{ID: "D14", Verdict: "failed", Found: found},
|
||||
{ID: "D1", Verdict: "passed"}}}
|
||||
answer, err := verdictAnswer(run, time.Now(), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
brief := answer["run"].(doctorRun).Probes[0]
|
||||
if len(brief.Found) != overviewFindings || brief.FoundCount != 689 || !strings.Contains(brief.More, "probe=D14") {
|
||||
t.Fatalf("the verdict carries %d findings of D14, count %d, more %q", len(brief.Found), brief.FoundCount, brief.More)
|
||||
}
|
||||
if len(run.Probes[0].Found) != 689 {
|
||||
t.Fatal("the overview cut the run it was made from")
|
||||
}
|
||||
whole, err := verdictAnswer(run, time.Now(), "d14")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p := whole["run"].(doctorRun).Probes; len(p) != 1 || len(p[0].Found) != 689 || p[0].More != "" {
|
||||
t.Fatalf("doctor probe=D14 answered %+v", p)
|
||||
}
|
||||
if _, err := verdictAnswer(run, time.Now(), "D99"); err == nil {
|
||||
t.Fatal("a probe the run does not have was answered")
|
||||
}
|
||||
if !strings.Contains(doctorText(answer), "probe=D14` gives them all") {
|
||||
t.Fatalf("the text does not say how to read them all:\n%s", doctorText(answer))
|
||||
}
|
||||
}
|
||||
@@ -86,7 +86,7 @@ func TestASilenceThroughTheVerbHoldsAndTheConditionStaysOpen(t *testing.T) {
|
||||
func TestUnreadableConditionsAreNotAWellMesh(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
_, store := withConditionsInMemory(t)
|
||||
store.Fail = errors.New("the bus is away")
|
||||
store.SetFail(errors.New("the bus is away"))
|
||||
asked, err := theThreeQuestions(t.Context(), open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -98,7 +98,7 @@ func TestUnreadableConditionsAreNotAWellMesh(t *testing.T) {
|
||||
if !strings.HasPrefix(said, "the open conditions could NOT be read") || strings.Contains(said, "no open conditions") {
|
||||
t.Fatalf("%s", said)
|
||||
}
|
||||
store.Fail = nil
|
||||
store.SetFail(nil)
|
||||
asked, _ = theThreeQuestions(t.Context(), open)
|
||||
said = printed(t, func() error { return printStatus(asked) })
|
||||
if asked.well() && !strings.Contains(said, "no open conditions;") {
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// What a consumer gave up on, answered by the serving controller (novox/hq issue 330).
|
||||
//
|
||||
// **In this process, on its own connection**: DEAD_LETTERS is read and changed on the bus, and the
|
||||
// serving controller is already on it. A verb run as a fresh process would open a connection of its own
|
||||
// for each call (novox/hq issue 327).
|
||||
|
||||
// busHandles are the serving controller's connection and JetStream handle.
|
||||
type busHandles struct {
|
||||
conn *nats.Conn
|
||||
js nats.JetStreamContext
|
||||
}
|
||||
|
||||
// defaultDeadLetters is how many the list says when not asked for more.
|
||||
const defaultDeadLetters = 50
|
||||
|
||||
// causeDeadLetter is the cause a delivery or drop gives when the caller gives none.
|
||||
const causeDeadLetter = "dead-letter"
|
||||
|
||||
// deadLettersAnswer is what `dead-letters` answers: the list, one whole, or what came of delivering one
|
||||
// again or dropping it.
|
||||
func deadLettersAnswer(ctx context.Context, a *verbArguments) (any, error) {
|
||||
serving := servingBus.Load()
|
||||
if serving == nil {
|
||||
return nil, errors.New("this controller is not serving, so it does not read DEAD_LETTERS: ask again, " +
|
||||
"and the serving controller answers")
|
||||
}
|
||||
on := &busHandles{conn: serving.Conn(), js: serving.Context()}
|
||||
// The shape first, and every argument it reads; one given beside it is refused before anything is
|
||||
// done, as every verb refuses what it would pass over (novox/hq issue 244).
|
||||
var deliver, drop, why, cause, idText, consumer, limit string
|
||||
switch {
|
||||
case a.given["deliver"] != "" || a.given["drop"] != "":
|
||||
deliver, drop, why, cause = a.str("deliver"), a.str("drop"), a.str("why"), a.str("cause")
|
||||
case a.given["id"] != "":
|
||||
idText = a.str("id")
|
||||
default:
|
||||
consumer, limit = a.str("consumer"), a.str("limit")
|
||||
}
|
||||
if unused := a.unused(); len(unused) > 0 {
|
||||
return nil, fmt.Errorf("dead-letters did not use %s together with %s, and an argument a verb would pass "+
|
||||
"over is refused: nothing was done", quoteAll(unused), quoteAll(a.usedGiven()))
|
||||
}
|
||||
switch {
|
||||
case deliver != "" && drop != "":
|
||||
return nil, errors.New("dead-letters delivers one again or drops one, not both. Nothing was done")
|
||||
case deliver != "" || drop != "":
|
||||
act, text := "deliver", deliver
|
||||
if drop != "" {
|
||||
act, text = "drop", drop
|
||||
}
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return nil, fmt.Errorf("dead-letters %s is a hand act, and says why: why is required and recorded in "+
|
||||
"the hand-act log (novox/hq to-be 45 §7). Nothing was done", act)
|
||||
}
|
||||
id, err := deadLetterID(text)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return actOnDeadLetter(ctx, on, act, id, why, cause)
|
||||
case idText != "":
|
||||
id, err := deadLetterID(idText)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return link.DeadLetterNamed(on.js, id)
|
||||
}
|
||||
most := defaultDeadLetters
|
||||
if limit != "" {
|
||||
n, err := strconv.Atoi(limit)
|
||||
if err != nil || n <= 0 {
|
||||
return nil, fmt.Errorf("limit is a number of dead letters, not %q", limit)
|
||||
}
|
||||
most = n
|
||||
}
|
||||
held, total, err := link.DeadLetters(on.js, consumer, most)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
answer := map[string]any{"dead_letters": held, "held": total,
|
||||
"note": "newest first; with id, one whole; deliver or drop one with why"}
|
||||
if total == 0 {
|
||||
answer["note"] = "no consumer gave up on a message that is still kept"
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// deadLetterID is a dead letter's id as a caller wrote it.
|
||||
func deadLetterID(text string) (uint64, error) {
|
||||
id, err := strconv.ParseUint(strings.TrimSpace(text), 10, 64)
|
||||
if err != nil || id == 0 {
|
||||
return 0, fmt.Errorf("a dead letter's id is its number in %s, as dead-letters lists it, not %q",
|
||||
broker.DeadLettersStream, text)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// actOnDeadLetter delivers one again or drops it, recorded in the hand-act log before it is done. A log
|
||||
// that cannot be written is said, and the act still happens: the log is never the reason a person's act
|
||||
// is refused (handacts.go).
|
||||
func actOnDeadLetter(ctx context.Context, on *busHandles, act string, id uint64, why, cause string) (any, error) {
|
||||
d, err := link.DeadLetterNamed(on.js, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if act == "deliver" {
|
||||
// Refused before it is recorded: an act that cannot be done is not an act.
|
||||
if _, err := link.AgainTo(on.js, d); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if cause == "" {
|
||||
cause = causeDeadLetter
|
||||
}
|
||||
by := link.CallerIn(ctx)
|
||||
if by == "" {
|
||||
by = "a seat call whose caller the bus did not name"
|
||||
}
|
||||
answer := map[string]any{"dead_letter": d.ID, "consumer": d.Who, "subject": d.Subject}
|
||||
recorded, logErr := link.RecordHandAct(ctx, on.conn, link.HandAct{Verb: "dead-letters " + act,
|
||||
Args: []string{strconv.FormatUint(id, 10), d.Stream + "." + d.Consumer}, Why: why, Cause: cause,
|
||||
Condition: conditions.Key(conditions.ScopeBus, d.Stream+"."+d.Consumer, link.AdvisoryMaxDeliveries),
|
||||
By: by + ", through the " + catalogue.ControllerSeatName + " seat"})
|
||||
if logErr != nil {
|
||||
answer["unrecorded"] = "the hand-act log could not be written, and the act was done all the same: " + logErr.Error()
|
||||
} else {
|
||||
answer["recorded"] = recorded.ID
|
||||
}
|
||||
switch act {
|
||||
case "deliver":
|
||||
_, to, err := link.DeliverAgain(on.js, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
answer["delivered_on"] = to
|
||||
answer["done"] = fmt.Sprintf("dead letter %d was delivered again to %s, and nobody else; it is no longer kept",
|
||||
id, consumerWho(d.Stream, d.Consumer))
|
||||
case "drop":
|
||||
if _, err := link.DropDeadLetter(on.js, id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
answer["done"] = fmt.Sprintf("dead letter %d, which %s gave up on, was dropped for good", id,
|
||||
consumerWho(d.Stream, d.Consumer))
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// The serving controller's bus, with the mesh's streams, for the verb to read and act on.
|
||||
func servingDeadLetters(t *testing.T) *broker.JetStream {
|
||||
t.Helper()
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := servingBus.Load()
|
||||
servingBus.Store(js)
|
||||
t.Cleanup(func() { servingBus.Store(before) })
|
||||
return js
|
||||
}
|
||||
|
||||
func askDeadLetters(t *testing.T, args map[string]any) (any, error) {
|
||||
t.Helper()
|
||||
a, err := readArguments("dead-letters", args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return deadLettersAnswer(context.Background(), a)
|
||||
}
|
||||
|
||||
func TestDeadLettersListsDropsAndRecordsWhy(t *testing.T) {
|
||||
js := servingDeadLetters(t)
|
||||
if _, err := js.Context().Publish("mesh.mod.gitea.event.pull.merged", []byte(`{"n":1}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, err := link.KeepDeadLetter(js.Context(), []byte(`{"stream":"EVENTS","consumer":"media_sonarr","stream_seq":1,"deliveries":5}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
answer, err := askDeadLetters(t, map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
listed := answer.(map[string]any)
|
||||
if listed["held"] != 1 || len(listed["dead_letters"].([]link.DeadLetter)) != 1 {
|
||||
t.Fatalf("listed %v", listed)
|
||||
}
|
||||
|
||||
// Refused before anything is done: an act without why, an argument the shape passes over, both acts.
|
||||
for _, args := range []map[string]any{
|
||||
{"drop": "1"},
|
||||
{"drop": "1", "why": "x", "limit": "3"},
|
||||
{"drop": "1", "deliver": "1", "why": "x"},
|
||||
{"why": "x"},
|
||||
{"id": "nought"},
|
||||
} {
|
||||
if _, err := askDeadLetters(t, args); err == nil {
|
||||
t.Errorf("%v was done", args)
|
||||
}
|
||||
}
|
||||
if _, total, _ := link.DeadLetters(js.Context(), "", 0); total != 1 {
|
||||
t.Fatalf("a refused call changed what is kept: %d left", total)
|
||||
}
|
||||
|
||||
done, err := askDeadLetters(t, map[string]any{"drop": "1", "why": "the media server took the download in by hand"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if said := done.(map[string]any); said["recorded"] == nil || !strings.Contains(said["done"].(string), "dropped") {
|
||||
t.Fatalf("answered %v", said)
|
||||
}
|
||||
acts, err := link.HandActs(context.Background(), js.Conn(), time.Now().Add(-time.Minute))
|
||||
if err != nil || len(acts) != 1 || acts[0].Verb != "dead-letters drop" || acts[0].Cause != causeDeadLetter ||
|
||||
!strings.Contains(acts[0].Condition, "media_sonarr") {
|
||||
t.Fatalf("recorded %+v (%v)", acts, err)
|
||||
}
|
||||
if !personsDecision(acts[0]) {
|
||||
t.Error("dropping a dead letter is counted as a repair, so S15 would want a healer for it")
|
||||
}
|
||||
if _, err := askDeadLetters(t, map[string]any{"id": "1"}); err == nil {
|
||||
t.Errorf("dead letter %d is still answered after it was dropped", d.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// Open while DEAD_LETTERS holds a message for the consumer, in words the operator reads in one pass:
|
||||
// what is held, and where to act.
|
||||
func TestAConsumersDeadLettersAreSaidUntilActedOn(t *testing.T) {
|
||||
f := &signalFacts{now: time.Now(), deadLetters: map[string]int{"EVENTS.media_sonarr": 4, "EVENTS.controller": 1}}
|
||||
found := watchDeadLetters(f)
|
||||
if len(found) != 2 {
|
||||
t.Fatalf("said %d conditions", len(found))
|
||||
}
|
||||
for _, o := range found {
|
||||
if o.Kind != "max-deliveries" || o.Severity != conditions.Warning || o.Needs == "" {
|
||||
t.Errorf("%+v", o)
|
||||
}
|
||||
if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Needs: o.Needs,
|
||||
Explanation: o.Explanation, Resolved: o.Resolved}, "media"); !ok {
|
||||
t.Errorf("%q is not plain: %s", o.Headline, why)
|
||||
}
|
||||
if !strings.Contains(o.Needs, "mesh MCP server") {
|
||||
t.Errorf("does not say where to act: %q", o.Needs)
|
||||
}
|
||||
}
|
||||
sonarr := found[1]
|
||||
if sonarr.ID != "EVENTS.media_sonarr" || sonarr.Machine != "media" ||
|
||||
sonarr.Headline != "Sonarr on media could not handle 4 messages" ||
|
||||
!strings.Contains(sonarr.Summary, "DEAD_LETTERS") {
|
||||
t.Errorf("%+v", sonarr)
|
||||
}
|
||||
if found[0].Headline != "The controller could not handle a message" {
|
||||
t.Errorf("%q", found[0].Headline)
|
||||
}
|
||||
// None held, none said: it clears when they are delivered again or dropped.
|
||||
if left := watchDeadLetters(&signalFacts{now: time.Now()}); len(left) != 0 {
|
||||
t.Fatalf("%v", left)
|
||||
}
|
||||
}
|
||||
+129
-12
@@ -168,11 +168,22 @@ func (m orderMember) pull() link.PullUpdated {
|
||||
ModuleDirs: m.ModuleDirs, ModuleDirsSaid: m.ModuleDirsSaid}
|
||||
}
|
||||
|
||||
// orderPair is one "before" among a group's members, and why.
|
||||
// orderPair is one "before" among a group's members, why, and the rules it won over.
|
||||
type orderPair struct {
|
||||
Before string `json:"before"`
|
||||
After string `json:"after"`
|
||||
Why string `json:"why"`
|
||||
// Over is the rules that ordered the pair the other way and lost to Why, by precedence (novox/hq ADR
|
||||
// 0249): said, so a person reading the order sees which rule won.
|
||||
Over []string `json:"over,omitempty"`
|
||||
}
|
||||
|
||||
// orderContradiction is two members that rules of one rank order both ways: no precedence resolves it, so
|
||||
// the group is refused, the rules named, and Said says how a person declares the order.
|
||||
type orderContradiction struct {
|
||||
Members []string `json:"members"`
|
||||
Rules []orderPair `json:"rules"`
|
||||
Said string `json:"said"`
|
||||
}
|
||||
|
||||
// orderReach is what a member moves, as the planner says.
|
||||
@@ -183,12 +194,14 @@ type orderReach struct {
|
||||
Manifests []string `json:"manifests,omitempty"`
|
||||
}
|
||||
|
||||
// groupOrder is a group's order: the members in it, every pair and why, and the cycle when there is one.
|
||||
// groupOrder is a group's order: the members in it, every pair and why, and the cycle when there is one —
|
||||
// with each contradiction no precedence resolves.
|
||||
type groupOrder struct {
|
||||
Order []string `json:"order"`
|
||||
Pairs []orderPair `json:"pairs,omitempty"`
|
||||
Cycle []string `json:"cycle,omitempty"`
|
||||
Reach map[string]orderReach `json:"reach,omitempty"`
|
||||
Order []string `json:"order"`
|
||||
Pairs []orderPair `json:"pairs,omitempty"`
|
||||
Cycle []string `json:"cycle,omitempty"`
|
||||
Contradictions []orderContradiction `json:"contradictions,omitempty"`
|
||||
Reach map[string]orderReach `json:"reach,omitempty"`
|
||||
}
|
||||
|
||||
// The reasons a pair is ordered, in the words the delivery plan shows.
|
||||
@@ -199,7 +212,23 @@ const (
|
||||
orderEngineFirst = "engine before controller"
|
||||
)
|
||||
|
||||
// orderOf is a group's order (novox/hq ADR 0239 decision 4), pure. A member goes before another when:
|
||||
// orderRank is a rule's precedence when two rules order one pair both ways (novox/hq ADR 0249), the lower
|
||||
// winning: what a person declared; then what the graph and the change say (built by, version skew); then
|
||||
// the rollout default, which infers a direction from the modules' names alone.
|
||||
func orderRank(why string) int {
|
||||
switch why {
|
||||
case orderDeclared:
|
||||
return 0
|
||||
case orderBuiltBy, orderVersionSkew:
|
||||
return 1
|
||||
case orderEngineFirst:
|
||||
return 2
|
||||
}
|
||||
return 3
|
||||
}
|
||||
|
||||
// orderOf is a group's order (novox/hq ADR 0239 decision 4, ADR 0249), pure. A member goes before another
|
||||
// when:
|
||||
//
|
||||
// - its pull request is named in the other's `after:` lines (declared);
|
||||
// - it moves a module the other's moved modules are built by, stand on, package or declare (built by);
|
||||
@@ -209,24 +238,28 @@ const (
|
||||
// does not yet grant, and a controller sends nothing an older engine would refuse — ADR 0236's rollout
|
||||
// order).
|
||||
//
|
||||
// Otherwise, by repository then id, so every reading gives one order. A pair both ways is a cycle: the
|
||||
// members in it are named, and none of them is ordered.
|
||||
// When rules order one pair both ways, the one of higher precedence wins (orderRank) and the pair says
|
||||
// what it won over. Rules of one rank both ways are a contradiction: both are kept as pairs, the members
|
||||
// are the cycle, and the contradiction says how to declare the order. Otherwise, by repository then id, so
|
||||
// every reading gives one order. A cycle through three members or more is named by its members, and none
|
||||
// of them is ordered.
|
||||
func orderOf(members []orderMember, reach map[string]orderReach, edges []inventory.Edge) groupOrder {
|
||||
out := groupOrder{Reach: reach}
|
||||
byID := map[string]orderMember{}
|
||||
for _, m := range members {
|
||||
byID[m.ID] = m
|
||||
}
|
||||
var claims []orderPair
|
||||
add := func(before, after, why string) {
|
||||
if before == after {
|
||||
return
|
||||
}
|
||||
for _, p := range out.Pairs {
|
||||
if p.Before == before && p.After == after {
|
||||
for _, p := range claims {
|
||||
if p.Before == before && p.After == after && p.Why == why {
|
||||
return
|
||||
}
|
||||
}
|
||||
out.Pairs = append(out.Pairs, orderPair{Before: before, After: after, Why: why})
|
||||
claims = append(claims, orderPair{Before: before, After: after, Why: why})
|
||||
}
|
||||
named := func(said, repository string) bool {
|
||||
said = strings.TrimSuffix(strings.TrimSpace(said), ".git")
|
||||
@@ -263,6 +296,7 @@ func orderOf(members []orderMember, reach map[string]orderReach, edges []invento
|
||||
}
|
||||
}
|
||||
}
|
||||
out.Pairs, out.Contradictions = resolveClaims(claims, byID)
|
||||
sort.Slice(out.Pairs, func(i, j int) bool {
|
||||
if out.Pairs[i].Before != out.Pairs[j].Before {
|
||||
return out.Pairs[i].Before < out.Pairs[j].Before
|
||||
@@ -312,6 +346,89 @@ func orderOf(members []orderMember, reach map[string]orderReach, edges []invento
|
||||
return out
|
||||
}
|
||||
|
||||
// orderRules is the rules in the order a pair names them when more than one says the same.
|
||||
var orderRules = []string{orderDeclared, orderBuiltBy, orderVersionSkew, orderEngineFirst}
|
||||
|
||||
// resolveClaims is the pairs the rules' claims come to (novox/hq ADR 0249): for each two members, the claims
|
||||
// of the highest precedence decide, and the claims they overrule are said on the pair. Claims of one rank
|
||||
// both ways are a contradiction: both pairs are kept, so the walk finds the cycle, and the contradiction
|
||||
// says how a person declares the order.
|
||||
func resolveClaims(claims []orderPair, byID map[string]orderMember) ([]orderPair, []orderContradiction) {
|
||||
byTwo := map[[2]string][]orderPair{}
|
||||
var keys [][2]string
|
||||
for _, c := range claims {
|
||||
k := [2]string{min(c.Before, c.After), max(c.Before, c.After)}
|
||||
if _, ok := byTwo[k]; !ok {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
byTwo[k] = append(byTwo[k], c)
|
||||
}
|
||||
slices.SortFunc(keys, func(a, b [2]string) int {
|
||||
if c := strings.Compare(a[0], b[0]); c != 0 {
|
||||
return c
|
||||
}
|
||||
return strings.Compare(a[1], b[1])
|
||||
})
|
||||
ruleIndex := func(why string) int { return slices.Index(orderRules, why) }
|
||||
var pairs []orderPair
|
||||
var contradictions []orderContradiction
|
||||
for _, k := range keys {
|
||||
cs := byTwo[k]
|
||||
slices.SortStableFunc(cs, func(a, b orderPair) int { return ruleIndex(a.Why) - ruleIndex(b.Why) })
|
||||
top := orderRank(cs[0].Why)
|
||||
var won []orderPair
|
||||
ways := map[string]bool{}
|
||||
for _, c := range cs {
|
||||
if orderRank(c.Why) == top {
|
||||
won = append(won, c)
|
||||
ways[c.Before] = true
|
||||
}
|
||||
}
|
||||
if len(ways) == 1 {
|
||||
p := orderPair{Before: won[0].Before, After: won[0].After, Why: won[0].Why}
|
||||
for _, c := range cs {
|
||||
if c.Before != p.Before && !slices.Contains(p.Over, c.Why) {
|
||||
p.Over = append(p.Over, c.Why)
|
||||
}
|
||||
}
|
||||
pairs = append(pairs, p)
|
||||
continue
|
||||
}
|
||||
// One rank both ways: no precedence resolves it.
|
||||
var first, second *orderPair
|
||||
var said []string
|
||||
for i := range won {
|
||||
c := won[i]
|
||||
if c.Before == k[0] && first == nil {
|
||||
first = &won[i]
|
||||
}
|
||||
if c.Before == k[1] && second == nil {
|
||||
second = &won[i]
|
||||
}
|
||||
said = append(said, fmt.Sprintf("%s puts %s first", c.Why, c.Before))
|
||||
}
|
||||
pairs = append(pairs, orderPair{Before: first.Before, After: first.After, Why: first.Why},
|
||||
orderPair{Before: second.Before, After: second.After, Why: second.Why})
|
||||
a, b := byID[k[0]], byID[k[1]]
|
||||
how := fmt.Sprintf("neither outranks the other: declare the order with a line `after: %s` in %s's "+
|
||||
"description, or `after: %s` in %s's", a.Repository, pullName(b), b.Repository, pullName(a))
|
||||
if top == orderRank(orderDeclared) {
|
||||
how = "each pull request declares it goes after the other: remove one of the `after:` lines"
|
||||
}
|
||||
contradictions = append(contradictions, orderContradiction{Members: []string{k[0], k[1]}, Rules: won,
|
||||
Said: fmt.Sprintf("%s and %s: %s; %s", k[0], k[1], strings.Join(said, ", "), how)})
|
||||
}
|
||||
return pairs, contradictions
|
||||
}
|
||||
|
||||
// pullName is a member as a person finds it: its pull request, or its id when no number was said.
|
||||
func pullName(m orderMember) string {
|
||||
if m.Number > 0 {
|
||||
return fmt.Sprintf("%s#%d", m.Repository, m.Number)
|
||||
}
|
||||
return m.ID
|
||||
}
|
||||
|
||||
// reachOfMembers is each member's reach, as the planner says it.
|
||||
func reachOfMembers(members []orderMember, entries []inventory.Entry, read map[string][]inventory.ReadRepository,
|
||||
edges []inventory.Edge) map[string]orderReach {
|
||||
|
||||
@@ -33,12 +33,14 @@ var deliveryOwnerWithin = 10 * time.Second
|
||||
|
||||
// stalledLine is one delivery past its bound, as mesh-delivery's `stalled` says it.
|
||||
type stalledLine struct {
|
||||
ID string `json:"id"`
|
||||
State string `json:"state"`
|
||||
For string `json:"for"`
|
||||
Bound string `json:"bound"`
|
||||
H2 string `json:"h2"`
|
||||
Says string `json:"says"`
|
||||
ID string `json:"id"`
|
||||
// Number is the pull request's, for a line of a head the forge never announced (novox/hq issue 347).
|
||||
Number int `json:"number,omitempty"`
|
||||
State string `json:"state"`
|
||||
For string `json:"for"`
|
||||
Bound string `json:"bound"`
|
||||
H2 string `json:"h2"`
|
||||
Says string `json:"says"`
|
||||
}
|
||||
|
||||
// operatorsOnly is whether the table leaves H2 nothing to do for the line: the state is the operator's.
|
||||
@@ -113,6 +115,7 @@ func stalledObservations(lines []stalledLine) []conditions.Observation {
|
||||
if l.operatorsOnly() {
|
||||
o.Resolver = conditions.ResolverOperator
|
||||
}
|
||||
o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions = stalledWords(l, o)
|
||||
out = append(out, o)
|
||||
}
|
||||
return out
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -63,11 +64,82 @@ func TestAGroupIsOrderedByTheGraphAndWhatItsPullRequestsSay(t *testing.T) {
|
||||
t.Fatalf("another reading ordered %v", again.Order)
|
||||
}
|
||||
|
||||
// A declared order against an inferred one is a cycle: named, and nothing ordered.
|
||||
// A declared order against an inferred one wins (novox/hq ADR 0249), and the pair says what it won over.
|
||||
members[1].After = []string{"mesh-catalog"}
|
||||
got = orderOf(members, reach, edges)
|
||||
if !reflect.DeepEqual(got.Cycle, []string{"cat", "ctl"}) {
|
||||
t.Fatalf("the cycle is %v, ordered %v", got.Cycle, got.Order)
|
||||
if len(got.Cycle) > 0 || len(got.Contradictions) > 0 {
|
||||
t.Fatalf("a declared order did not win: cycle %v, %+v", got.Cycle, got.Contradictions)
|
||||
}
|
||||
if want := []string{"agent", "lab", "app", "cat", "host", "ctl"}; !reflect.DeepEqual(got.Order, want) {
|
||||
t.Fatalf("ordered %v, wanted %v", got.Order, want)
|
||||
}
|
||||
if !slices.ContainsFunc(got.Pairs, func(p orderPair) bool {
|
||||
return p.Before == "cat" && p.After == "ctl" && p.Why == orderDeclared && reflect.DeepEqual(p.Over, []string{orderVersionSkew})
|
||||
}) || slices.ContainsFunc(got.Pairs, func(p orderPair) bool { return p.Before == "ctl" && p.After == "cat" }) {
|
||||
t.Fatalf("the declared pair does not say it won over version skew: %+v", got.Pairs)
|
||||
}
|
||||
|
||||
// Declared both ways is a contradiction no precedence resolves: a cycle, and how to end it.
|
||||
members[0].After = []string{"novox/mesh-controller"}
|
||||
got = orderOf(members, reach, edges)
|
||||
if !reflect.DeepEqual(got.Cycle, []string{"cat", "ctl"}) || len(got.Contradictions) != 1 {
|
||||
t.Fatalf("the cycle is %v, ordered %v, contradictions %+v", got.Cycle, got.Order, got.Contradictions)
|
||||
}
|
||||
if c := got.Contradictions[0]; !reflect.DeepEqual(c.Members, []string{"cat", "ctl"}) ||
|
||||
!strings.Contains(c.Said, "remove one of the `after:` lines") {
|
||||
t.Fatalf("the contradiction is said %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0249: when two rules order one pair both ways, the rule of higher precedence wins and the
|
||||
// pair says what it overruled; rules of one rank both ways are refused, naming both and how to declare.
|
||||
func TestAnOrderRuleYieldsToAStrongerOne(t *testing.T) {
|
||||
members := []orderMember{
|
||||
{ID: "ctl", Repository: "novox/mesh-controller", Number: 7},
|
||||
{ID: "host", Repository: "novox/mesh-host", Number: 9},
|
||||
}
|
||||
reach := map[string]orderReach{
|
||||
"ctl": {Moved: []string{"mesh-controller", "build-agent"}},
|
||||
"host": {Moved: []string{"mesh-host"}},
|
||||
}
|
||||
builtBy := []inventory.Edge{{From: "mesh-host", To: "build-agent", Kind: inventory.EdgeBuiltBy}}
|
||||
|
||||
// Built by outranks engine before controller.
|
||||
got := orderOf(members, reach, builtBy)
|
||||
if !reflect.DeepEqual(got.Order, []string{"ctl", "host"}) || len(got.Cycle) > 0 {
|
||||
t.Fatalf("ordered %v, cycle %v", got.Order, got.Cycle)
|
||||
}
|
||||
if want := []orderPair{{Before: "ctl", After: "host", Why: orderBuiltBy, Over: []string{orderEngineFirst}}}; !reflect.DeepEqual(got.Pairs, want) {
|
||||
t.Fatalf("pairs %+v, wanted %+v", got.Pairs, want)
|
||||
}
|
||||
|
||||
// A declared line outranks both.
|
||||
members[0].After = []string{"mesh-host"}
|
||||
got = orderOf(members, reach, builtBy)
|
||||
if !reflect.DeepEqual(got.Order, []string{"host", "ctl"}) || len(got.Cycle) > 0 ||
|
||||
!reflect.DeepEqual(got.Pairs[0].Over, []string{orderBuiltBy}) || got.Pairs[0].Why != orderDeclared {
|
||||
t.Fatalf("a declared order did not win: %v %+v", got.Order, got.Pairs)
|
||||
}
|
||||
|
||||
// Without the build agent, engine before controller stands alone.
|
||||
members[0].After = nil
|
||||
got = orderOf(members, map[string]orderReach{"ctl": {Moved: []string{"mesh-controller"}}, "host": reach["host"]}, builtBy)
|
||||
if !reflect.DeepEqual(got.Order, []string{"host", "ctl"}) || got.Pairs[0].Why != orderEngineFirst || len(got.Pairs[0].Over) > 0 {
|
||||
t.Fatalf("engine before controller alone: %v %+v", got.Order, got.Pairs)
|
||||
}
|
||||
|
||||
// Built by both ways is one rank against itself: refused, both rules named, and how to declare.
|
||||
both := append(builtBy, inventory.Edge{From: "build-agent", To: "mesh-host", Kind: inventory.EdgeStandsOn})
|
||||
got = orderOf(members, reach, both)
|
||||
if !reflect.DeepEqual(got.Cycle, []string{"ctl", "host"}) || len(got.Order) > 0 || len(got.Contradictions) != 1 {
|
||||
t.Fatalf("a contradiction was ordered: %v, cycle %v, %+v", got.Order, got.Cycle, got.Contradictions)
|
||||
}
|
||||
said := got.Contradictions[0].Said
|
||||
for _, w := range []string{"built by puts ctl first", "built by puts host first",
|
||||
"`after: novox/mesh-controller` in novox/mesh-host#9's description", "`after: novox/mesh-host` in novox/mesh-controller#7's"} {
|
||||
if !strings.Contains(said, w) {
|
||||
t.Errorf("the contradiction does not say %q: %s", w, said)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -116,6 +116,16 @@ var probeRegistry = []probe{
|
||||
{ID: probeDeliveriesID, Asserts: "no delivery is held past its state's bound unsaid: mesh-delivery's " +
|
||||
"`stalled`, each with the transition its table lets healer H2 take", From: "ADR 0239",
|
||||
Kind: kindDeliveryStalled, Phase: 3, run: probeDeliveries},
|
||||
// A client of the bus reconnecting in a loop (novox/hq issue 327), from the server's record of closed
|
||||
// connections, which the bus's own module reads.
|
||||
{ID: probeReconnectsID, Asserts: "no user of the bus had its connection dropped more than twelve times in the " +
|
||||
"last hour: the bus module's nats_closed_connections", From: "issue 327", Kind: kindBusReconnects,
|
||||
Phase: 1, run: probeReconnects},
|
||||
// The account agents run as (novox/hq ADR 0266): where a machine names one, its node-engine has judged it
|
||||
// unable to become root without a person — what ADR 0259 §8 rests an authorised answer on.
|
||||
{ID: agentAccountProbe, Asserts: "every machine that names an agent account has it judged, on its node-engine's " +
|
||||
"newest statement, unable to become root without a person", From: "ADR 0266, ADR 0259 §8",
|
||||
Kind: kindAgentCanBecomeRoot, Phase: 1, run: probeAgentAccounts},
|
||||
{ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals",
|
||||
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs},
|
||||
// The core's health definitions (novox/hq to-be 45 §8, ADR 0236): what a core component's new build is
|
||||
@@ -151,6 +161,50 @@ type probeVerdict struct {
|
||||
Unconfirmed []string `json:"unconfirmed,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Took string `json:"took,omitempty"`
|
||||
// FoundCount and More are an overview's: how many findings the probe had when the overview shows
|
||||
// fewer, and how to read them all (novox/hq issue 314). Never in a heartbeat.
|
||||
FoundCount int `json:"found-count,omitempty"`
|
||||
More string `json:"more,omitempty"`
|
||||
}
|
||||
|
||||
// overviewFindings is how many findings of one probe an overview carries. On 2026-10-08 D14 found 689
|
||||
// stalled deliveries, and an overview carrying every one of them, beside the conditions they raised,
|
||||
// was more than the bus carries in one message (novox/hq issue 314); one probe's verdict whole is
|
||||
// `doctor probe=<id>`.
|
||||
const overviewFindings = 20
|
||||
|
||||
// inOverview is a run as an overview carries it: each probe's findings at most overviewFindings, with
|
||||
// their count and how to read them all. The run itself is not changed.
|
||||
func inOverview(run doctorRun) doctorRun {
|
||||
probes := make([]probeVerdict, len(run.Probes))
|
||||
for i, p := range run.Probes {
|
||||
total := len(p.Found) + len(p.Unconfirmed)
|
||||
if total > overviewFindings {
|
||||
p.FoundCount = len(p.Found)
|
||||
if len(p.Found) > overviewFindings {
|
||||
p.Found = p.Found[:overviewFindings:overviewFindings]
|
||||
}
|
||||
if room := overviewFindings - len(p.Found); len(p.Unconfirmed) > room {
|
||||
p.Unconfirmed = p.Unconfirmed[:room:room]
|
||||
}
|
||||
p.More = fmt.Sprintf("%d found and %d unconfirmed, %d of them shown — `doctor probe=%s` gives them all",
|
||||
p.FoundCount, total-p.FoundCount, len(p.Found)+len(p.Unconfirmed), p.ID)
|
||||
}
|
||||
probes[i] = p
|
||||
}
|
||||
run.Probes = probes
|
||||
return run
|
||||
}
|
||||
|
||||
// oneProbe is a run with only one probe's verdict, whole.
|
||||
func oneProbe(run doctorRun, id string) (doctorRun, error) {
|
||||
for _, p := range run.Probes {
|
||||
if strings.EqualFold(p.ID, id) {
|
||||
run.Probes = []probeVerdict{p}
|
||||
return run, nil
|
||||
}
|
||||
}
|
||||
return doctorRun{}, fmt.Errorf("the run %s has no probe %q — `doctor probes=true` lists them", run.Run, id)
|
||||
}
|
||||
|
||||
// doctorCounts are a run's verdicts, counted.
|
||||
@@ -386,12 +440,13 @@ func doctorCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
set := flag.NewFlagSet("doctor", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
probe := set.String("probe", "", "one probe's verdict, with every finding")
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New("doctor [run|probes|signals] [--json]")
|
||||
return errors.New("doctor [run|probes|signals] [--probe <id>] [--json]")
|
||||
}
|
||||
answer, err := doctorAnswer(ctx, sub)
|
||||
answer, err := doctorAnswer(ctx, sub, *probe)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -403,12 +458,18 @@ func doctorCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
|
||||
// doctorAnswer is what the verb answers, as data.
|
||||
func doctorAnswer(ctx context.Context, sub string) (any, error) {
|
||||
//
|
||||
// A verdict is an overview — each probe's findings at most overviewFindings — unless probe names one,
|
||||
// which is then answered alone and whole (novox/hq issue 314).
|
||||
func doctorAnswer(ctx context.Context, sub, probe string) (any, error) {
|
||||
if probe != "" && sub != "" && sub != "run" {
|
||||
return nil, fmt.Errorf("probe names one probe of a verdict; %s has none", sub)
|
||||
}
|
||||
switch sub {
|
||||
case "":
|
||||
if doctorFrom != nil {
|
||||
if run := doctorFrom.lastRun(); run != nil {
|
||||
return verdictAnswer(*run, time.Now()), nil
|
||||
return verdictAnswer(*run, time.Now(), probe)
|
||||
}
|
||||
return nil, fmt.Errorf("the self-check has not finished its first run yet: it runs %s after the "+
|
||||
"controller starts, then every %s — `doctor run` runs it now", doctorFirstAfter, doctorEvery)
|
||||
@@ -417,7 +478,7 @@ func doctorAnswer(ctx context.Context, sub string) (any, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return verdictAnswer(run, time.Now()), nil
|
||||
return verdictAnswer(run, time.Now(), probe)
|
||||
case "run":
|
||||
d := doctorFrom
|
||||
if d == nil {
|
||||
@@ -428,7 +489,7 @@ func doctorAnswer(ctx context.Context, sub string) (any, error) {
|
||||
defer closeIt()
|
||||
d = local
|
||||
}
|
||||
return verdictAnswer(d.runOnce(ctx, "asked by "+link.Caller()), time.Now()), nil
|
||||
return verdictAnswer(d.runOnce(ctx, "asked by "+link.Caller()), time.Now(), probe)
|
||||
case "probes":
|
||||
return probesAnswer(), nil
|
||||
case "signals":
|
||||
@@ -442,11 +503,20 @@ func doctorAnswer(ctx context.Context, sub string) (any, error) {
|
||||
}
|
||||
|
||||
// verdictAnswer is a run as the verb answers it, with its age.
|
||||
func verdictAnswer(run doctorRun, now time.Time) map[string]any {
|
||||
func verdictAnswer(run doctorRun, now time.Time, probe string) (map[string]any, error) {
|
||||
if probe != "" {
|
||||
one, err := oneProbe(run, probe)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
run = one
|
||||
} else {
|
||||
run = inOverview(run)
|
||||
}
|
||||
return map[string]any{"run": run, "age": now.Sub(run.At).Round(time.Second).String(),
|
||||
"note": "a probe that could not run is never a pass; each failure is an open condition until a run passes it, " +
|
||||
"and one a single look can be wrong about — an unanswered question, a slow answer — is raised when two " +
|
||||
"runs in a row see it"}
|
||||
"runs in a row see it"}, nil
|
||||
}
|
||||
|
||||
// probesAnswer is the registry.
|
||||
@@ -515,6 +585,9 @@ func doctorText(answer any) string {
|
||||
for _, f := range p.Unconfirmed {
|
||||
fmt.Fprintf(&b, " unconfirmed, raised if the next run sees it too: %s\n", f)
|
||||
}
|
||||
if p.More != "" {
|
||||
fmt.Fprintf(&b, " … %s\n", p.More)
|
||||
}
|
||||
if p.Error != "" {
|
||||
fmt.Fprintf(&b, " %s\n", p.Error)
|
||||
}
|
||||
|
||||
@@ -45,7 +45,7 @@ func TestAMachineIsSentTheEpochOnlyOnceItSaysItReadsOne(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &bodiesDelivery{recordedDelivery: recordedDelivery{inv: inv}, bodies: map[string][]byte{}}
|
||||
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, composeForPush(open, gens), d, ""); err != nil {
|
||||
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, composeForPush(open, gens), d, "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
carried := func(node string) (epoch float64, has bool) {
|
||||
@@ -92,7 +92,7 @@ func TestNothingIsComposedOrSentWithoutTheLease(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &bodiesDelivery{recordedDelivery: recordedDelivery{inv: open.inventory}, bodies: map[string][]byte{}}
|
||||
refused, err := sendRound(ctx, open, []string{"anchor"}, composeForPush(open, gens), d, "")
|
||||
refused, err := sendRound(ctx, open, []string{"anchor"}, composeForPush(open, gens), d, "", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -269,7 +269,8 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot
|
||||
engines := map[string]bool{}
|
||||
for _, n := range nodes {
|
||||
m := snapshot.Machine{Name: scrub.Machine(n.Name), Length: len(n.Name), Adopted: n.Adopted,
|
||||
AccountHome: scrub.Text(n.AccountHome), NodeEngine: n.HostVersion, PublicDomain: domains[n.Name]}
|
||||
AccountHome: scrub.Text(n.AccountHome), NodeEngine: n.HostVersion, PublicDomain: domains[n.Name],
|
||||
AgentAccount: scrub.Account(n.AgentAccount), AgentAccountHome: scrub.Text(n.AgentAccountHome)}
|
||||
switch n.Account {
|
||||
case "", "root":
|
||||
m.Account = n.Account
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A directory the node-engine uses as found (novox/hq issue 339) waits for a person to hand it over at the
|
||||
// machine. Found before this send, it is no fault of the build: the gate passes with the wait carried, so an
|
||||
// urgent fix of that module still goes through. Found by this send, the send brought it, and the gate holds.
|
||||
func foundDirectory(module string, since time.Time) inventory.ResourceHealth {
|
||||
return inventory.ResourceHealth{Module: module, Resource: module + ".data", Kind: link.KindDirectory,
|
||||
Target: "/srv/" + module, State: link.StateUnhealthy, Since: since,
|
||||
Reason: link.ReasonUsedAsFound + " owned by 1000:1000, mode 700, as found; root, mode 755 was declared and " +
|
||||
"not given it — `mesh-host hand-over` at the machine hands it to the mesh"}
|
||||
}
|
||||
|
||||
func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) {
|
||||
now := time.Now()
|
||||
sent := now.Add(-time.Minute)
|
||||
f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now,
|
||||
Resources: []inventory.ResourceHealth{foundDirectory("notes", sent.Add(-24*time.Hour))}}}}
|
||||
h, why := moduleHealthWord("notes", "laptop", sent, f)
|
||||
if h != healthPerson || !strings.Contains(why, "notes.data") || !strings.Contains(why, "hand-over") {
|
||||
t.Fatalf("a directory found before the send reads %v %q; want a wait for a person", h, why)
|
||||
}
|
||||
// Found by this very send: the send brought it, and it is not passed.
|
||||
f.health["laptop"] = inventory.NodeHealth{Node: "laptop", HeardAt: now,
|
||||
Resources: []inventory.ResourceHealth{foundDirectory("notes", sent.Add(time.Second))}}
|
||||
if h, why := moduleHealthWord("notes", "laptop", sent, f); h != healthNotYet {
|
||||
t.Fatalf("a directory this send found reads %v %q; want not yet", h, why)
|
||||
}
|
||||
// A container down beside the old wait is a fault, as before.
|
||||
f.health["laptop"] = inventory.NodeHealth{Node: "laptop", HeardAt: now, Resources: []inventory.ResourceHealth{
|
||||
foundDirectory("notes", sent.Add(-time.Hour)),
|
||||
{Module: "notes", Resource: "notes.web", Kind: "container", Target: "notes", State: link.StateUnhealthy, Reason: "down"}}}
|
||||
if h, why := moduleHealthWord("notes", "laptop", sent, f); h != healthNotYet {
|
||||
t.Fatalf("a container down beside the wait reads %v %q; want not yet", h, why)
|
||||
}
|
||||
}
|
||||
|
||||
// The whole walk: a module whose directory was used as found long before still gets its fix to every machine,
|
||||
// its pass kept and the wait said; a directory this very send found holds it and puts it back.
|
||||
func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
found time.Duration // when the directory was found, against now
|
||||
passes bool
|
||||
}{
|
||||
{"found a day before the send", -24 * time.Hour, true},
|
||||
{"found by this send", time.Hour, false},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
|
||||
return map[string]bool{"anchor": true, "laptop": true}, nil
|
||||
}
|
||||
backlogFacts := gatherGateFacts
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
f, err := backlogFacts(ctx, open, component)
|
||||
// The used-as-found condition, raised after the send (its second statement): its own kind, never a
|
||||
// fault the gate reads as the build's.
|
||||
f.judged, f.openErr = true, nil
|
||||
f.open = append(f.open, conditions.Condition{Key: usedAsFoundKey("app", "anchor"), Kind: kindUsedAsFound,
|
||||
Subject: conditions.Subject{Scope: conditions.ScopeModule, ID: "app.anchor", Machine: "anchor"},
|
||||
Raised: time.Now()})
|
||||
f.health = map[string]inventory.NodeHealth{}
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
f.health[n] = inventory.NodeHealth{Node: n, HeardAt: time.Now(), Resources: []inventory.ResourceHealth{
|
||||
{Module: "app", Resource: "app.web", Kind: "container", Target: "app", State: link.StateHealthy},
|
||||
foundDirectory("app", time.Now().Add(c.found)),
|
||||
{Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateHealthy}}}
|
||||
}
|
||||
return f, err
|
||||
}
|
||||
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
||||
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
|
||||
gateSettle, gateEvery, gateBound = 0, 0, 300*time.Millisecond
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
advancePlans(ctx, b.open)
|
||||
if p := b.release(t); p.State != inventory.PlanRolling {
|
||||
break
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
p := b.release(t)
|
||||
v, found, err := inv.GateOf(ctx, "build-app-c2")
|
||||
if c.passes {
|
||||
if p.State != inventory.PlanDone || err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("the walk is %s (%s); app's verdict %+v: want the fix through", p.State, p.Note, v)
|
||||
}
|
||||
if !strings.Contains(v.Why+p.Note, "hand it over") {
|
||||
t.Errorf("the wait is not carried: verdict %q, walk %q", v.Why, p.Note)
|
||||
}
|
||||
return
|
||||
}
|
||||
if p.State == inventory.PlanDone {
|
||||
t.Fatalf("a directory this send found let the walk through: %s", p.Note)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The condition says the wait in its own kind: the operator's, never urgent, and cleared once handed over.
|
||||
func TestADirectoryUsedAsFoundIsItsOwnCondition(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
ctx := t.Context()
|
||||
rs := map[string][]inventory.ResourceHealth{"notes": {foundDirectory("notes", time.Now().Add(-time.Hour))}}
|
||||
for i := 0; i < 2; i++ {
|
||||
if err := judgeModuleHealth(ctx, nil, k, "laptop", rs, map[string]int{"notes": i + 1}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
open, _ := k.Open(ctx)
|
||||
var got *conditions.Condition
|
||||
for i, c := range open {
|
||||
if c.Key == usedAsFoundKey("notes", "laptop") {
|
||||
got = &open[i]
|
||||
}
|
||||
if c.Kind == kindModuleUnhealthy {
|
||||
t.Fatalf("raised as a fault: %+v", c)
|
||||
}
|
||||
}
|
||||
if got == nil || got.Resolver != conditions.ResolverOperator || got.Severity == conditions.Urgent ||
|
||||
!strings.Contains(got.Summary, "notes.data") {
|
||||
t.Fatalf("the condition: %+v", got)
|
||||
}
|
||||
// Long open is still not urgent: only a person can hand it over, and nothing is broken by the wait.
|
||||
if err := judgeModuleHealth(ctx, nil, k, "laptop", rs, map[string]int{"notes": 3}, time.Now().Add(48*time.Hour)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open, _ = k.Open(ctx)
|
||||
for _, c := range open {
|
||||
if c.Key == usedAsFoundKey("notes", "laptop") && c.Severity == conditions.Urgent {
|
||||
t.Fatal("a directory used as found became urgent")
|
||||
}
|
||||
}
|
||||
if err := judgeModuleHealth(ctx, nil, k, "laptop", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open, _ = k.Open(ctx)
|
||||
for _, c := range open {
|
||||
if c.Key == usedAsFoundKey("notes", "laptop") {
|
||||
t.Fatal("not cleared once handed over")
|
||||
}
|
||||
}
|
||||
}
|
||||
+202
-18
@@ -77,6 +77,10 @@ type health int
|
||||
|
||||
const (
|
||||
healthGood health = iota
|
||||
// healthPerson is a wait for a person (novox/hq ADR 0254): everything unhealthy of the module waits for
|
||||
// one person's new login (personWait). The build did what it should; no bound a machine can meet ends
|
||||
// the wait. It counts as a pass, and the gate carries the reading along in its verdict.
|
||||
healthPerson
|
||||
// healthWaiting is not a pass and not a fault: what the module's checks find waits on an unhealthy
|
||||
// provider (ADR 0240 rule 5), so the judging waits — past the bound too — rather than putting back a
|
||||
// build for something it did not do.
|
||||
@@ -115,6 +119,9 @@ type gateFacts struct {
|
||||
healthErr error
|
||||
// heldOn is, per "<module>@<machine>", the provider its findings are held under (ADR 0240 rule 5).
|
||||
heldOn map[string]string
|
||||
// groupsAdded is, per module, whether the move judged puts an account in a group its previous build did
|
||||
// not (issue 318 review): the only move whose wait for a new login is excused.
|
||||
groupsAdded map[string]bool
|
||||
}
|
||||
|
||||
// gatherGateFacts reads what a judging needs, from the store, the bus and this controller's memory. A
|
||||
@@ -209,7 +216,10 @@ func judgeHealth(module, component string, m catalogue.Manifest, machine string,
|
||||
firstLine(f.openErr.Error())
|
||||
}
|
||||
for _, c := range f.open {
|
||||
if c.Source == gateProbe || c.Raised.Before(since) {
|
||||
// A wait for a person's new login, or for a directory used as found to be handed over, is the module's
|
||||
// reading, not a fault raised since the send: the gate reads it from the statement below (ADR 0254,
|
||||
// novox/hq issue 339).
|
||||
if c.Source == gateProbe || c.Raised.Before(since) || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound {
|
||||
continue
|
||||
}
|
||||
onIt := c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) ||
|
||||
@@ -283,6 +293,8 @@ type machineWord struct {
|
||||
facts gateFacts
|
||||
on map[string]string
|
||||
whole string
|
||||
// waiting is what holds the machine on something shown to be another's (ADR 0241): the judging waits.
|
||||
waiting string
|
||||
}
|
||||
|
||||
// kindCoreBehind is D10's kind: a machine runs core components older than the mesh holds, or has not
|
||||
@@ -318,7 +330,8 @@ func aboutTheMachine(machine string, moved []string, since time.Time, f gateFact
|
||||
for _, c := range f.open {
|
||||
aboutIt := c.Subject.Scope == conditions.ScopeMachine && (c.Subject.ID == machine || c.Subject.Machine == machine ||
|
||||
slices.Contains(c.Subject.Also, machine))
|
||||
if !aboutIt || c.Source == gateProbe || c.Raised.Before(since) {
|
||||
// A directory used as found waits for a person, whatever the send did (novox/hq issue 339).
|
||||
if !aboutIt || c.Source == gateProbe || c.Raised.Before(since) || c.Kind == kindUsedAsFound {
|
||||
kept = append(kept, c)
|
||||
continue
|
||||
}
|
||||
@@ -346,6 +359,17 @@ func aboutTheMachine(machine string, moved []string, since time.Time, f gateFact
|
||||
}
|
||||
case coreBehind:
|
||||
// Not what the send moved: said nowhere against it.
|
||||
case c.Kind == kindMachineUnits:
|
||||
// **The machine's own failed units** (issue 315): no module places any of them, so nothing a
|
||||
// send moved is among them — a module's failed unit is that module's own condition. Said
|
||||
// nowhere against the send.
|
||||
case c.Kind == kindNetworkRewritten || c.Kind == kindNetworkUnreachable && c.Subject.ID != machine:
|
||||
// **Shown to be somebody else's** (ADR 0241): another program rewrote the resolver file the send
|
||||
// did not move, or the machine cannot reach another that is down. Nothing the send did; the
|
||||
// judging waits for it rather than putting back a build at the bound.
|
||||
if w.waiting == "" {
|
||||
w.waiting = machine + "'s network: " + said
|
||||
}
|
||||
default:
|
||||
if w.whole == "" {
|
||||
w.whole = machine + " as a whole: " + said
|
||||
@@ -411,6 +435,7 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
facts.groupsAdded = movesAddingGroups(ctx, open.inventory, g, pairs, shelf)
|
||||
// **What is wrong with a machine itself is the machine's** (novox/hq issue 281): read once for each
|
||||
// machine judged, apart from what is wrong with a module there, and never pinned on the module the
|
||||
// gate happens to be kept on.
|
||||
@@ -422,56 +447,209 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
for node, moved := range byMachine {
|
||||
words[node] = aboutTheMachine(node, moved, *g.Since, facts)
|
||||
}
|
||||
// **Each module is judged on its own** (novox/hq ADR 0254, issue 318): its reading is the worst of its
|
||||
// machines', its passes are counted apart, and the send's verdict is still one. **Every module of a send
|
||||
// leaves it with a verdict** (issue 318 review): one healthy for the passes the gate asks, after the
|
||||
// settle time, keeps a pass when another beside it fails; one found broken holds the send's verdict until
|
||||
// the modules beside it have theirs, or the bound; and at the verdict, every module that did not pass is
|
||||
// put back with what failed, so none is left on the machine unjudged for other walks to wait on.
|
||||
worst, why := healthGood, ""
|
||||
var failing []string
|
||||
broken := map[string]bool{}
|
||||
reading := map[string]health{}
|
||||
waits := map[string]string{}
|
||||
var modules []string
|
||||
for _, j := range pairs {
|
||||
w := words[j.node]
|
||||
h, said := judgeHealth(j.module, coreComponent(j.module), shelf[j.module], j.node, *g.Since, w.facts)
|
||||
if h == healthGood {
|
||||
if h == healthGood || h == healthPerson {
|
||||
if on, named := w.on[j.module]; named {
|
||||
h, said = healthNotYet, on
|
||||
} else if w.whole != "" {
|
||||
h, said = healthNotYet, w.whole
|
||||
} else if w.waiting != "" {
|
||||
h, said = healthWaiting, w.waiting
|
||||
}
|
||||
}
|
||||
if h != healthGood && !slices.Contains(failing, j.module) {
|
||||
failing = append(failing, j.module)
|
||||
if _, seen := reading[j.module]; !seen {
|
||||
modules = append(modules, j.module)
|
||||
}
|
||||
if h == healthBroken {
|
||||
broken[j.module] = true
|
||||
if h == healthBroken && !slices.Contains(g.Broken, j.module) {
|
||||
g.Broken = append(g.Broken, j.module)
|
||||
if g.BrokenWhy == "" {
|
||||
g.BrokenWhy = said
|
||||
}
|
||||
}
|
||||
if slices.Contains(g.Broken, j.module) {
|
||||
h = healthBroken // found broken once, broken for the rest of the judging
|
||||
}
|
||||
if h > reading[j.module] {
|
||||
reading[j.module] = h
|
||||
}
|
||||
if h == healthPerson {
|
||||
waits[j.module] = joinSaid(waits[j.module], said)
|
||||
}
|
||||
if h > worst {
|
||||
worst, why = h, said
|
||||
} else if h == worst && h != healthGood && why == "" {
|
||||
} else if h == worst && h > healthPerson && why == "" {
|
||||
why = said
|
||||
}
|
||||
}
|
||||
if g.Healthy == nil {
|
||||
g.Healthy = map[string]int{}
|
||||
}
|
||||
if g.HealthyAt == nil {
|
||||
g.HealthyAt = map[string]time.Time{}
|
||||
}
|
||||
g.Waits = nil
|
||||
var failing []string
|
||||
for _, m := range modules {
|
||||
if reading[m] > healthPerson {
|
||||
failing = append(failing, m)
|
||||
g.Healthy[m] = 0
|
||||
delete(g.HealthyAt, m)
|
||||
continue
|
||||
}
|
||||
// **A pass is counted only gateEvery after the one before** (issue 318 review): a send judged more
|
||||
// often while another module beside it is not yet healthy counts no faster.
|
||||
if at, counted := g.HealthyAt[m]; !counted || now.Sub(at) >= gateEvery {
|
||||
g.Healthy[m]++
|
||||
g.HealthyAt[m] = now
|
||||
}
|
||||
if w := waits[m]; w != "" {
|
||||
if g.Waits == nil {
|
||||
g.Waits = map[string]string{}
|
||||
}
|
||||
g.Waits[m] = w
|
||||
}
|
||||
}
|
||||
settled := now.Sub(*g.Since) >= gateSettle
|
||||
passedAlone := func(m string) bool {
|
||||
return settled && reading[m] <= healthPerson && g.Healthy[m] >= gatePasses
|
||||
}
|
||||
// fail decides the send failed: what passed on its own keeps its pass, everything else is put back.
|
||||
fail := func(why string) {
|
||||
g.Passing, g.Failing = nil, nil
|
||||
for _, m := range modules {
|
||||
if passedAlone(m) {
|
||||
g.Passing = append(g.Passing, m)
|
||||
} else {
|
||||
g.Failing = append(g.Failing, m)
|
||||
}
|
||||
}
|
||||
decide(g, inventory.GateFailed, why, now)
|
||||
}
|
||||
pastBound := now.Sub(*g.Since) > gateBound
|
||||
switch {
|
||||
case worst == healthBroken:
|
||||
// What broke is put back; what was only not yet healthy beside it is too — they moved together.
|
||||
g.Failing = failing
|
||||
decide(g, inventory.GateFailed, why, now)
|
||||
var judging []string
|
||||
for _, m := range modules {
|
||||
if reading[m] != healthBroken && !passedAlone(m) {
|
||||
judging = append(judging, m)
|
||||
}
|
||||
}
|
||||
if len(judging) == 0 || pastBound {
|
||||
fail(g.BrokenWhy)
|
||||
break
|
||||
}
|
||||
// What broke fails the send, and is put back at once by the caller (putBackBroken); what is beside
|
||||
// it is judged to its own verdict first, within the bound.
|
||||
g.Passes, g.LastPass, g.Failing = 0, nil, failing
|
||||
g.Last = fmt.Sprintf("%s; %s judged to its own verdict before the send's", g.BrokenWhy, strings.Join(judging, ", "))
|
||||
case worst == healthWaiting:
|
||||
// Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either —
|
||||
// the provider's own condition says what is wrong (ADR 0240 rule 5).
|
||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
||||
case worst == healthNotYet:
|
||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
||||
if now.Sub(*g.Since) > gateBound {
|
||||
decide(g, inventory.GateFailed, fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why), now)
|
||||
if pastBound {
|
||||
fail(fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why))
|
||||
}
|
||||
default:
|
||||
// Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict.
|
||||
g.Passes++
|
||||
g.LastPass, g.Last, g.Failing = &now, "", nil
|
||||
if g.Passes >= gatePasses && now.Sub(*g.Since) >= gateSettle {
|
||||
if g.Passes >= gatePasses && settled {
|
||||
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
|
||||
now.Sub(*g.Since).Round(time.Second)), now)
|
||||
now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now)
|
||||
}
|
||||
}
|
||||
return g.Verdict, nil
|
||||
}
|
||||
|
||||
// whyFor is a passing gate's why as one module's verdict says it: the send's, and that module's own wait
|
||||
// for a person, never another's (issue 318 review).
|
||||
func whyFor(g *inventory.PlanGate, module string) string {
|
||||
why, _, _ := strings.Cut(g.Why, waitsPrefix)
|
||||
if w := g.Waits[module]; w != "" {
|
||||
why += waitsPrefix + w
|
||||
}
|
||||
return why
|
||||
}
|
||||
|
||||
const waitsPrefix = "; and it waits for a person: "
|
||||
|
||||
// waitsSaid is the waits for a person a passing gate carries, as its verdict says them.
|
||||
func waitsSaid(waits map[string]string) string {
|
||||
if len(waits) == 0 {
|
||||
return ""
|
||||
}
|
||||
modules := make([]string, 0, len(waits))
|
||||
for m := range waits {
|
||||
modules = append(modules, m)
|
||||
}
|
||||
sort.Strings(modules)
|
||||
var said []string
|
||||
for _, m := range modules {
|
||||
said = append(said, waits[m])
|
||||
}
|
||||
return waitsPrefix + strings.Join(said, "; ")
|
||||
}
|
||||
|
||||
func joinSaid(a, b string) string {
|
||||
switch {
|
||||
case a == "":
|
||||
return b
|
||||
case b == "" || strings.Contains(a, b):
|
||||
return a
|
||||
}
|
||||
return a + "; " + b
|
||||
}
|
||||
|
||||
// movesAddingGroups is, per module a gate judges, whether its move puts an account in a group the build it
|
||||
// moved from did not: read from the builds' manifests. A module whose move is not known adds none.
|
||||
func movesAddingGroups(ctx context.Context, inv *inventory.Inventory, g *inventory.PlanGate, pairs []judged,
|
||||
shelf map[string]catalogue.Manifest) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, j := range pairs {
|
||||
if _, done := out[j.module]; done {
|
||||
continue
|
||||
}
|
||||
from, to := g.From, g.To
|
||||
for _, c := range g.Carried {
|
||||
if c.Module == j.module {
|
||||
from, to = c.From, c.To
|
||||
break
|
||||
}
|
||||
}
|
||||
target, found, err := inv.ManifestAt(ctx, j.module, to)
|
||||
if err != nil || !found {
|
||||
target = shelf[j.module]
|
||||
}
|
||||
// What it moved from is not known — no build named (a plan's own module whose previous build was not
|
||||
// recorded), or no manifest kept for it: no wait is excused, rather than one the move did not bring.
|
||||
if from == "" {
|
||||
out[j.module] = false
|
||||
continue
|
||||
}
|
||||
before, had, err := inv.ManifestAt(ctx, j.module, from)
|
||||
if err != nil || !had {
|
||||
out[j.module] = false
|
||||
continue
|
||||
}
|
||||
out[j.module] = addsAccountGroups(before, had, target)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// decide sets a gate's verdict.
|
||||
func decide(g *inventory.PlanGate, verdict, why string, now time.Time) {
|
||||
g.Verdict, g.Why, g.JudgedAt = verdict, why, &now
|
||||
@@ -483,7 +661,7 @@ func gatePassed(ctx context.Context, open *stores, p *inventory.Plan, module str
|
||||
g := state.Gate
|
||||
err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: state.Build, Module: module,
|
||||
Commit: state.Commit, Previous: state.Previous, Plan: p.ID, Machines: g.Machines,
|
||||
Verdict: inventory.GatePassed, Why: g.Why, Component: g.Component, JudgingFrom: g.Since})
|
||||
Verdict: inventory.GatePassed, Why: whyFor(g, module), Component: g.Component, JudgingFrom: g.Since})
|
||||
if err != nil && state.Build != "" {
|
||||
fmt.Printf("%s: %s passed its gate, and the verdict could not be kept: %v\n", p.ID, module, err)
|
||||
}
|
||||
@@ -944,6 +1122,12 @@ func gateLine(g *inventory.PlanGate) string {
|
||||
} else if g.Verdict == "" {
|
||||
line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gatePasses)
|
||||
}
|
||||
if g.Verdict == "" && len(g.Waits) > 0 {
|
||||
line += waitsSaid(g.Waits)
|
||||
}
|
||||
if len(g.Passing) > 0 {
|
||||
line += "; passed on their own and kept: " + strings.Join(g.Passing, ", ")
|
||||
}
|
||||
if g.Rollback != "" {
|
||||
line += "; " + g.Rollback
|
||||
}
|
||||
|
||||
@@ -0,0 +1,453 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// What the review of issue 318's fix found (novox/hq ADR 0254): every module of a send leaves it with a
|
||||
// verdict, a pass is counted no faster than the gate's spacing, a carried build's verdict carries only its own
|
||||
// wait, and a provider waiting for a login says who waits on it.
|
||||
|
||||
// withGateBounds sets the gate's bounds for one test.
|
||||
func withGateBounds(t *testing.T, settle, every, bound time.Duration) {
|
||||
t.Helper()
|
||||
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
||||
gateSettle, gateEvery, gateBound = settle, every, bound
|
||||
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
|
||||
}
|
||||
|
||||
// factsOn is a judging's facts for one machine that applied its send: the node tools answer, and what it says
|
||||
// of its modules' resources.
|
||||
func factsOn(t *testing.T, machine string, since time.Time, rs ...inventory.ResourceHealth) func() gateFacts {
|
||||
return func() gateFacts {
|
||||
now := time.Now()
|
||||
at := now
|
||||
return gateFacts{now: now,
|
||||
reports: map[string]inventory.Reported{machine: {Node: machine, Outcome: inventory.OutcomeApplied, At: &at, Current: true}},
|
||||
engines: map[string]string{},
|
||||
rolledBack: map[string][]lease.Rollback{},
|
||||
served: map[string]served{machine: {runtime: true, tools: map[string]bool{}}},
|
||||
health: map[string]inventory.NodeHealth{machine: {Node: machine, HeardAt: now, Resources: rs}},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func healthyContainer(module string) inventory.ResourceHealth {
|
||||
return inventory.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module,
|
||||
State: link.StateHealthy}
|
||||
}
|
||||
|
||||
// **A fault decided before the settle time does not strand the module beside it** (review (a)): the node
|
||||
// tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass,
|
||||
// counted over the gate's spacing — before it says its own. The broken one alone is put back.
|
||||
func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
withGateBounds(t, 150*time.Millisecond, 20*time.Millisecond, 10*time.Second)
|
||||
since := time.Now().Add(-time.Second)
|
||||
base := factsOn(t, "laptop", since, healthyContainer("app"))
|
||||
wasGather := gatherGateFacts
|
||||
t.Cleanup(func() { gatherGateFacts = wasGather })
|
||||
gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) {
|
||||
f := base()
|
||||
f.rolledBack["laptop"] = []lease.Rollback{{Component: lease.ComponentNodeTools, Outcome: lease.OutcomeRolledBack,
|
||||
At: since.Add(100 * time.Millisecond), Why: "the node tools did not answer"}}
|
||||
return f, nil
|
||||
}
|
||||
g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since}
|
||||
pairs := []judged{{module: broker_runtime, node: "laptop"}, {module: "app", node: "laptop"}}
|
||||
judgings := 0
|
||||
for deadline := time.Now().Add(5 * time.Second); g.Verdict == "" && time.Now().Before(deadline); {
|
||||
if _, err := judgeMoves(ctx, open, g, pairs, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
judgings++
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
}
|
||||
if g.Verdict != inventory.GateFailed || judgings < gatePasses {
|
||||
t.Fatalf("verdict %q after %d judging(s): %+v; want failed, after the module beside it was judged", g.Verdict,
|
||||
judgings, g)
|
||||
}
|
||||
if !reflect.DeepEqual(g.Passing, []string{"app"}) || !reflect.DeepEqual(g.Failing, []string{broker_runtime}) {
|
||||
t.Fatalf("passing %v, failing %v; want app kept and the node tools put back", g.Passing, g.Failing)
|
||||
}
|
||||
if !strings.Contains(g.Why, "witness") {
|
||||
t.Errorf("the verdict does not say what broke: %q", g.Why)
|
||||
}
|
||||
}
|
||||
|
||||
// broker_runtime is the node tools' module name, a core component a witness judges.
|
||||
const broker_runtime = "node-tools"
|
||||
|
||||
// **A pass is counted only the gate's spacing after the one before** (review (c)): a send judged every tick
|
||||
// while a module beside it is not yet healthy counts the healthy one once.
|
||||
func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
withGateBounds(t, 0, time.Hour, time.Hour)
|
||||
since := time.Now().Add(-time.Minute)
|
||||
base := factsOn(t, "laptop", since, healthyContainer("app"), inventory.ResourceHealth{Module: "late",
|
||||
Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy, Reason: "down"})
|
||||
wasGather := gatherGateFacts
|
||||
t.Cleanup(func() { gatherGateFacts = wasGather })
|
||||
gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) { return base(), nil }
|
||||
g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since}
|
||||
pairs := []judged{{module: "app", node: "laptop"}, {module: "late", node: "laptop"}}
|
||||
now := time.Now()
|
||||
for i := 0; i < 3; i++ {
|
||||
if _, err := judgeMoves(ctx, open, g, pairs, now.Add(time.Duration(i)*time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if g.Healthy["app"] != 1 || g.Healthy["late"] != 0 {
|
||||
t.Fatalf("counted %v in three judgings within a second; want app once and late never", g.Healthy)
|
||||
}
|
||||
}
|
||||
|
||||
// **A carried build's pass carries its own wait, never another's** (review (e)).
|
||||
func TestACarriedPassCarriesOnlyItsOwnWait(t *testing.T) {
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
since := time.Now().Add(-time.Minute)
|
||||
g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since, Verdict: inventory.GatePassed,
|
||||
Why: "healthy 3 times over 2m0s" + waitsSaid(map[string]string{"late": "relogin needed on laptop: late waits"}),
|
||||
Waits: map[string]string{"late": "relogin needed on laptop: late waits"},
|
||||
Carried: []inventory.CarriedMove{{Module: "app", Node: "laptop", From: "c1", To: "c2", Build: "build-app-c2"},
|
||||
{Module: "late", Node: "laptop", From: "c1", To: "c2", Build: "build-late-c2"}}}
|
||||
passCarried(ctx, b.open, &inventory.Plan{ID: "release-test"}, g, "")
|
||||
app, _, _ := inv.GateOf(ctx, "build-app-c2")
|
||||
late, _, _ := inv.GateOf(ctx, "build-late-c2")
|
||||
if strings.Contains(app.Why, "waits for a person") || app.Verdict != inventory.GatePassed {
|
||||
t.Errorf("app's pass: %+v; want it without late's wait", app)
|
||||
}
|
||||
if !strings.Contains(late.Why, "relogin needed on laptop") {
|
||||
t.Errorf("late's pass: %+v; want its own wait", late)
|
||||
}
|
||||
}
|
||||
|
||||
// **The tier path keeps the pass of the module the gate is kept on** (review (b)): a plan's first send
|
||||
// carried its own module, healthy, and a build waiting on that machine that never became healthy. At the
|
||||
// bound the waiting one is put back and marked; the plan's own module keeps its pass, so no walk waits on it.
|
||||
func TestThePlansOwnModuleKeepsItsPassWhenItsSendFails(t *testing.T) {
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := g.open.inventory
|
||||
// `late` waits on anchor for a gate: c1 sent, c2 registered and built.
|
||||
for _, b := range []inventory.Build{
|
||||
{ID: "build-late-1", Module: "late", Commit: "l1", Repository: "novox/mesh-catalog", Path: "modules/late",
|
||||
Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)},
|
||||
{ID: "build-late-2", Module: "late", Commit: "l2", Repository: "novox/mesh-catalog", Path: "modules/late",
|
||||
Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)},
|
||||
} {
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: "late", Version: b.Commit})
|
||||
b.Manifest = manifest
|
||||
b.Made = []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + b.Commit}}
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "late", Version: b.Commit}, inventory.Source{
|
||||
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/late", BuiltFrom: b.Commit, Head: b.Commit,
|
||||
Asked: b.Asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b.Commit == "l1" {
|
||||
if _, err := inv.Assign(ctx, "anchor", "late"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, g.open, "anchor"), "d-anchor-late", map[string]string{"app": "c1",
|
||||
"late": "l1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
wasMoves := machineMoves
|
||||
t.Cleanup(func() { machineMoves = wasMoves })
|
||||
machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
|
||||
modules, err := open.inventory.Assigned(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sent, known, err := open.inventory.SentBuilds(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f.moves(node, modules, sent, known, all), nil
|
||||
}
|
||||
gather := gatherGateFacts
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
f, err := gather(ctx, open, component)
|
||||
f.health = map[string]inventory.NodeHealth{"anchor": {Node: "anchor", HeardAt: time.Now(), Resources: []inventory.ResourceHealth{
|
||||
healthyContainer("app"), {Module: "late", Resource: "late.web", Kind: "container", Target: "late",
|
||||
State: link.StateUnhealthy, Reason: "down"}}}}
|
||||
return f, err
|
||||
}
|
||||
gateEvery, gateBound = 0, 400*time.Millisecond
|
||||
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); {
|
||||
advancePlans(ctx, g.open)
|
||||
if p := g.plan(t); p.State == inventory.PlanFailed || p.State == inventory.PlanDone {
|
||||
break
|
||||
}
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
}
|
||||
p := g.plan(t)
|
||||
if p.State != inventory.PlanFailed {
|
||||
t.Fatalf("the plan is %s: %s; want it failed on late", p.State, p.Note)
|
||||
}
|
||||
if v, found, err := inv.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed ||
|
||||
!strings.Contains(v.Why, "on its own") {
|
||||
t.Fatalf("app's verdict: %+v (found %v, %v); want its own pass kept", v, found, err)
|
||||
}
|
||||
if failed, _ := inv.GateFailed(ctx, "build-late-2"); !failed {
|
||||
t.Fatal("late's build is not marked failed at its gate")
|
||||
}
|
||||
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" || current["late"].Commit != "l1" {
|
||||
t.Fatalf("registered app %s, late %s; want app kept at c2 and late put back to l1", current["app"].Commit,
|
||||
current["late"].Commit)
|
||||
}
|
||||
}
|
||||
|
||||
// **A provider waiting for a login says who waits on it** (review (g)): its consumer, failing a check that
|
||||
// needs it, is held under it, and the provider's relogin-needed condition lists it and is urgent.
|
||||
func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
k := conditionsFrom
|
||||
register(t, open, catalogue.Manifest{Module: "db", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}})
|
||||
register(t, open, catalogue.Manifest{Module: "shop", Version: "1", Requires: []string{"postgres-database"}})
|
||||
for _, a := range [][2]string{{"anchor", "db"}, {"laptop", "shop"}} {
|
||||
if _, err := inv.Assign(ctx, a[0], a[1]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.RecordBindings(ctx, "laptop", []inventory.Binding{{Machine: "laptop", Consumer: "shop",
|
||||
Provision: "postgres-database", Provider: catalogue.Chosen{Node: "anchor", Module: "db"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
at := h0
|
||||
say := func(machine string, rs ...link.ResourceHealth) {
|
||||
t.Helper()
|
||||
at = at.Add(time.Second)
|
||||
for i := range rs {
|
||||
rs[i].Since = at
|
||||
}
|
||||
if err := stateHealth(ctx, inv, k, machine, link.Health{Contract: link.ReadinessContract, At: at, Resources: rs}, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
account := link.ResourceHealth{Module: "db", Resource: "db.operator", Kind: link.KindAccount, Target: "operator",
|
||||
Account: "operator", State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": operator is in the group db"}
|
||||
unit := link.ResourceHealth{Module: "db", Resource: "db.server", Kind: link.KindUnit, Target: "db.service",
|
||||
Account: "operator", State: link.StateUnhealthy, Reason: "failed in the account's own service manager (exit-code)"}
|
||||
shop := link.ResourceHealth{Module: "shop", Resource: "shop.web", Kind: "container", Target: "shop",
|
||||
State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"}
|
||||
for look := 0; look < 2; look++ {
|
||||
say("anchor", account, unit)
|
||||
say("laptop", shop)
|
||||
}
|
||||
list, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keys []string
|
||||
var c conditions.Condition
|
||||
for _, x := range list {
|
||||
keys = append(keys, x.Key)
|
||||
if x.Key == "module.db.anchor.relogin-needed" {
|
||||
c = x
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(keys, []string{"module.db.anchor.relogin-needed"}) {
|
||||
t.Fatalf("open %v; want the provider's wait alone, its consumer held under it", keys)
|
||||
}
|
||||
if c.Severity != conditions.Urgent || !strings.Contains(c.Evidence[0].Said, "shop on laptop") {
|
||||
t.Fatalf("the provider's wait: %s, %q; want it urgent and naming its consumer", c.Severity, c.Evidence[0].Said)
|
||||
}
|
||||
}
|
||||
|
||||
// **A broken module is put back at once** (issue 318 review): the laptop's witness put the node tools back
|
||||
// within a minute of a send that also moved `app`, and `app` reads not yet healthy because of it. The node
|
||||
// tools are marked and put back in the very judging that found them broken — their registered build is the
|
||||
// one before, and the laptop is sent it — while `app` goes on being judged, recovers, and keeps its own pass.
|
||||
func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
withConditionsInMemory(t)
|
||||
was := doctorFrom
|
||||
doctorFrom = nil
|
||||
t.Cleanup(func() { doctorFrom = was })
|
||||
old, recent := time.Now().Add(-3*time.Hour), time.Now().Add(-time.Minute)
|
||||
build := func(module, commit string, asked time.Time) {
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: module, Version: "1"})
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + module + "-" + commit, Module: module, Commit: commit,
|
||||
Repository: "novox/mesh-catalog", Path: "modules/" + module, Manifest: manifest, Asked: asked, At: asked,
|
||||
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + module + commit}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: module, Version: "1"}, inventory.Source{
|
||||
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + module, BuiltFrom: commit, Head: commit,
|
||||
Asked: asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, m := range []string{"app", broker_runtime} {
|
||||
build(m, "c1", old)
|
||||
if _, err := inv.Assign(ctx, "laptop", m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop", map[string]string{"app": "c1", broker_runtime: "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
build("app", "c2", recent)
|
||||
build(broker_runtime, "c2", recent)
|
||||
|
||||
wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts
|
||||
t.Cleanup(func() {
|
||||
machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
|
||||
})
|
||||
machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
|
||||
modules, _ := open.inventory.Assigned(ctx, node)
|
||||
sent, known, err := open.inventory.SentBuilds(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f.moves(node, modules, sent, known, all), nil
|
||||
}
|
||||
releaseHeard = func(context.Context, *stores) (map[string]bool, error) { return map[string]bool{"laptop": true}, nil }
|
||||
var sends []string
|
||||
n := 0
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, node := range names {
|
||||
n++
|
||||
carried := map[string]string{"app": current["app"].Commit, broker_runtime: current[broker_runtime].Commit}
|
||||
sends = append(sends, node+":"+carried[broker_runtime])
|
||||
digest := "d-" + node + "-" + time.Now().Format("150405.000000") + string(rune('a'+n))
|
||||
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
|
||||
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
var seen []string // the node tools' registered build at each judging
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
now := time.Now()
|
||||
f := gateFacts{now: now, reports: map[string]inventory.Reported{}, engines: map[string]string{},
|
||||
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
|
||||
reports, _ := open.inventory.LastReports(ctx)
|
||||
for _, r := range reports {
|
||||
f.reports[r.Node] = r
|
||||
}
|
||||
current, _ := open.inventory.CurrentBuilds(ctx)
|
||||
seen = append(seen, current[broker_runtime].Commit)
|
||||
app := healthyContainer("app")
|
||||
if current[broker_runtime].Commit == "c2" {
|
||||
// The witness reverted the node tools; app cannot reach them yet.
|
||||
f.rolledBack["laptop"] = []lease.Rollback{{Component: lease.ComponentNodeTools, Outcome: lease.OutcomeRolledBack,
|
||||
From: "c2", To: "c1", At: now, Why: "the node tools did not answer"}}
|
||||
app.State, app.Reason = link.StateUnhealthy, "down"
|
||||
}
|
||||
f.served["laptop"] = served{runtime: current[broker_runtime].Commit == "c1", tools: map[string]bool{}}
|
||||
f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now, Resources: []inventory.ResourceHealth{app}}}
|
||||
return f, nil
|
||||
}
|
||||
withGateBounds(t, 100*time.Millisecond, 0, 10*time.Second)
|
||||
|
||||
release := func() inventory.Plan {
|
||||
t.Helper()
|
||||
plans, _ := inv.RecentPlans(ctx, 10)
|
||||
for _, p := range plans {
|
||||
if p.Release != nil {
|
||||
return p
|
||||
}
|
||||
}
|
||||
t.Fatal("no walk")
|
||||
return inventory.Plan{}
|
||||
}
|
||||
// Judged until the first judging has found the node tools broken, and one more.
|
||||
for i := 0; i < 20 && len(seen) < 2; i++ {
|
||||
advancePlans(ctx, open)
|
||||
}
|
||||
if len(seen) < 2 || seen[0] != "c2" || seen[1] != "c1" {
|
||||
t.Fatalf("the node tools' registered build at each judging: %v; want c2, then c1 at the very next judging", seen)
|
||||
}
|
||||
if failed, _ := inv.GateFailed(ctx, "build-"+broker_runtime+"-c2"); !failed {
|
||||
t.Fatal("the node tools' build is not marked failed at once")
|
||||
}
|
||||
if p := release(); p.State != inventory.PlanRolling || p.Release.Gate == nil || p.Release.Gate.Verdict != "" {
|
||||
t.Fatalf("the walk is %s with gate %+v; want it still judging app", p.State, p.Release.Gate)
|
||||
}
|
||||
if !slices.Contains(sends, "laptop:c1") {
|
||||
t.Fatalf("sends %v; want the laptop sent the node tools' earlier build at once", sends)
|
||||
}
|
||||
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); {
|
||||
advancePlans(ctx, open)
|
||||
if release().State != inventory.PlanRolling {
|
||||
break
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
p := release()
|
||||
if p.State != inventory.PlanFailed {
|
||||
t.Fatalf("the walk is %s: %s; want failed, for the node tools", p.State, p.Note)
|
||||
}
|
||||
if v, found, _ := inv.GateOf(ctx, "build-app-c2"); !found || v.Verdict != inventory.GatePassed || !strings.Contains(v.Why, "on its own") {
|
||||
t.Fatalf("app's verdict: %+v; want its own pass", v)
|
||||
}
|
||||
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" {
|
||||
t.Fatalf("app is registered at %s; want it kept at c2", current["app"].Commit)
|
||||
}
|
||||
}
|
||||
|
||||
// **A move from a build not known excuses no wait** (issue 318 review): a plan's own module whose previous
|
||||
// build was not recorded, or whose previous manifest is not kept, cannot be shown to have added the group.
|
||||
func TestAMoveFromAnUnknownBuildExcusesNoWait(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
record := func(commit string, m catalogue.Manifest) {
|
||||
raw, _ := json.Marshal(m)
|
||||
at := time.Now().Add(-time.Hour)
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-lights-" + commit, Module: "lights", Commit: commit,
|
||||
Repository: "novox/mesh-catalog", Path: "modules/lights", Manifest: raw, Asked: at, At: at}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
record("c1", catalogue.Manifest{Module: "lights"})
|
||||
record("c2", catalogue.Manifest{Module: "lights", Resources: []map[string]any{{"id": "operator", "type": "user",
|
||||
"name": "operator", "groups": []any{"lights"}}}})
|
||||
pairs := []judged{{module: "lights", node: "laptop"}}
|
||||
shelf := map[string]catalogue.Manifest{}
|
||||
for _, c := range []struct {
|
||||
from string
|
||||
want bool
|
||||
}{{"c1", true}, {"", false}, {"c0-never-built", false}} {
|
||||
g := &inventory.PlanGate{From: c.from, To: "c2"}
|
||||
if got := movesAddingGroups(ctx, inv, g, pairs, shelf)["lights"]; got != c.want {
|
||||
t.Errorf("a move from %q adds a group: %v; want %v", c.from, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -221,7 +221,8 @@ func TestABuildThatFailsItsGateIsRolledBackOnItsFirstMachineAndGoesNoFurther(t *
|
||||
t.Fatalf("sent again after the rollback: %v", g.sent)
|
||||
}
|
||||
_, _, err = takeIn(ctx, inv, link.BuildResult{ID: "build-2", Repository: "novox/mesh-catalog", Path: "modules/app",
|
||||
Commit: "c2", Manifest: mustJSON(t, catalogue.Manifest{Module: "app", Version: "c2"})})
|
||||
Commit: "c2", Manifest: mustJSON(t, catalogue.Manifest{Module: "app", Version: "c2"}),
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}})
|
||||
if err == nil || !strings.Contains(err.Error(), "failed its gate") {
|
||||
t.Fatalf("the failed build was registered again: %v", err)
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
@@ -14,7 +15,7 @@ import (
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
@@ -36,6 +37,9 @@ type handActVerb struct {
|
||||
Decision string
|
||||
// DecidedFor limits Decision to these causes; empty, it holds for every act of the verb.
|
||||
DecidedFor []string
|
||||
// DecidedWhen limits Decision to the acts it answers true for: what the controller read the act to
|
||||
// be from what it did (a push's kind), never a word the person gave.
|
||||
DecidedWhen func(link.HandAct) bool
|
||||
}
|
||||
|
||||
// causeLeakedInLogs is the cause a rotation after a value was printed into a log gives.
|
||||
@@ -52,16 +56,20 @@ const causeDrill = "drill"
|
||||
// listed without a decision, counts, so a new verb is a repair until its entry says otherwise.
|
||||
var handActVerbs = []handActVerb{
|
||||
// Repairs: each repeated is a healer the mesh lacks. A push by hand is exactly what roll-out by
|
||||
// default (ADR 0236) exists to end.
|
||||
{Verb: "push"},
|
||||
// default (ADR 0236) exists to end — except a push that only moved builds a `record` policy held for
|
||||
// a person's word (ADR 0242), which the push itself reads from what it carried (recorded_push.go).
|
||||
{Verb: "push", Decision: "a recorded build moves only by a person's push: that push is the word its " +
|
||||
"upgrade policy asks for (ADR 0242)", DecidedWhen: pushedRecorded},
|
||||
{Verb: "plans stop"},
|
||||
{Verb: "plans close"},
|
||||
// A walk started by a person instead of its delivery's owner (novox/hq ADR 0239): the owner down, or
|
||||
// not trusted with it — either is a repair the owner should have made.
|
||||
{Verb: "plans go"},
|
||||
{Verb: "broker consumer-reset"},
|
||||
// Silencing the same condition twice says the condition, or what it watches, wants mending.
|
||||
{Verb: "conditions silence"},
|
||||
// Silencing the same condition twice says the condition, or what it watches, wants mending — unless
|
||||
// it is the operator's answer on a notification: a decision to live with it (novox/hq ADR 0258).
|
||||
{Verb: "conditions silence", Decision: "the operator's answer on a notification is their decision, " +
|
||||
"not a repair (ADR 0258)", DecidedFor: []string{conditions.CauseOperatorAnswer}},
|
||||
// An act done outside the mesh: the mesh cannot tell a repair from a decision there, so it counts —
|
||||
// except a drill recorded through it before `hand-act drill` existed (2026-10-07). It refuses the
|
||||
// cause since, so no act recorded through it now carries it.
|
||||
@@ -74,6 +82,19 @@ var handActVerbs = []handActVerb{
|
||||
{Verb: "retire approve", Decision: "nothing is retired past its bound without a person (ADR 0230)"},
|
||||
{Verb: "retire reject", Decision: "keeping a consumer active is a person's word (ADR 0230)"},
|
||||
{Verb: "cleanup delete", Decision: "nothing retired is deleted without a person (ADR 0230)"},
|
||||
// What becomes of a message a consumer gave up on (novox/hq issue 330): kept until a person says.
|
||||
{Verb: "dead-letters deliver", Decision: "a message a consumer gave up on is delivered again only on a " +
|
||||
"person's word (issue 330)"},
|
||||
{Verb: "dead-letters drop", Decision: "a message a consumer gave up on is let go only on a person's word " +
|
||||
"(issue 330)"},
|
||||
// The sweep run on a person's word rather than after a build: the same decision the records make, at
|
||||
// a moment the person chose (ADR 0251) — never a repair.
|
||||
{Verb: "collect", Decision: "letting the store go of what the records keep for no reason, now rather " +
|
||||
"than at the next build, is a person's word (ADR 0251)"},
|
||||
// Recording a copy no record names as the mesh's: a person's reading of the store, never a repair
|
||||
// (ADR 0257).
|
||||
{Verb: "mirrors", Decision: "which copies in the store no record names are the mesh's is a person's " +
|
||||
"word (ADR 0257)"},
|
||||
{Verb: "bus upgrade", Decision: "the bus is never rolled by the mesh: replacing it is a planned step a " +
|
||||
"person starts (ADR 0236)"},
|
||||
{Verb: "upgrade release-backlog", Decision: "after a release plan failed, the next opens only when a " +
|
||||
@@ -93,7 +114,8 @@ func personsDecision(a link.HandAct) bool {
|
||||
if v.Verb != a.Verb {
|
||||
continue
|
||||
}
|
||||
return v.Decision != "" && (len(v.DecidedFor) == 0 || slices.Contains(v.DecidedFor, a.Cause))
|
||||
return v.Decision != "" && (len(v.DecidedFor) == 0 || slices.Contains(v.DecidedFor, a.Cause)) &&
|
||||
(v.DecidedWhen == nil || v.DecidedWhen(a))
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -132,14 +154,10 @@ func onTheBus(f func(*nats.Conn) error) error {
|
||||
if handActConn != nil {
|
||||
return f(handActConn)
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot reach the bus: %w", err)
|
||||
}
|
||||
defer js.Close()
|
||||
return f(js.Conn())
|
||||
}
|
||||
@@ -153,6 +171,19 @@ func (f handActFlags) record(ctx context.Context, verb string, args []string) {
|
||||
}
|
||||
act := link.HandAct{Verb: verb, Args: args, Why: strings.TrimSpace(*f.why),
|
||||
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
|
||||
// A push naming one machine says whether it only moves recorded builds (novox/hq issue 301):
|
||||
// read from what it carries, before it is sent.
|
||||
if verb == "push" && len(args) == 1 && !strings.HasPrefix(args[0], "-") {
|
||||
switch carried, why, err := recordedPushOf(ctx, args[0]); {
|
||||
case err != nil:
|
||||
fmt.Fprintf(os.Stderr, "whether this push only moves recorded builds could not be read, so it is "+
|
||||
"recorded as a push by hand: %v\n", err)
|
||||
case len(carried) > 0:
|
||||
act.Kind, act.Carried = link.KindRecordedBuilds, carried
|
||||
default:
|
||||
fmt.Printf("a push by hand, not of recorded builds only: %s\n", why)
|
||||
}
|
||||
}
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
written, err := link.RecordHandAct(ctx, conn, act)
|
||||
act = written
|
||||
@@ -162,6 +193,12 @@ func (f handActFlags) record(ctx context.Context, verb string, args []string) {
|
||||
fmt.Fprintf(os.Stderr, "this act by hand could NOT be recorded in the hand-act log, and is done anyway: %v\n", err)
|
||||
return
|
||||
}
|
||||
if act.Kind == link.KindRecordedBuilds {
|
||||
fmt.Printf("recorded as %s in the hand-act log: a push of recorded builds (%s) by %s, because %q "+
|
||||
"— the person's word their upgrade policy asks for, which no healer is wanted for\n", act.ID,
|
||||
strings.Join(act.Carried, "; "), act.By, act.Why)
|
||||
return
|
||||
}
|
||||
fmt.Printf("recorded as %s in the hand-act log: %s, because %q (cause: %s)\n", act.ID, act.By, act.Why, act.Cause)
|
||||
}
|
||||
|
||||
@@ -211,7 +248,13 @@ func handActCommand(ctx context.Context, args []string) error {
|
||||
if len(args) > 0 && args[0] == "list" {
|
||||
args = args[1:]
|
||||
}
|
||||
return listHandActs(ctx, args, os.Stdout)
|
||||
}
|
||||
|
||||
// listHandActs is `hand-acts`: what was done by hand lately, and the causes done more than once.
|
||||
func listHandActs(ctx context.Context, args []string, w io.Writer) error {
|
||||
set := flag.NewFlagSet("hand-acts", flag.ContinueOnError)
|
||||
usageTo(set, w)
|
||||
days := set.Int("days", 14, "how many days back")
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
@@ -229,21 +272,28 @@ func handActCommand(ctx context.Context, args []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
fmt.Fprintln(w, string(body))
|
||||
return nil
|
||||
}
|
||||
if len(acts) == 0 {
|
||||
fmt.Printf("nothing was done by hand in the last %d day(s)\n", *days)
|
||||
fmt.Fprintf(w, "nothing was done by hand in the last %d day(s)\n", *days)
|
||||
return nil
|
||||
}
|
||||
for i := len(acts) - 1; i >= 0; i-- {
|
||||
a := acts[i]
|
||||
fmt.Printf("%s %s %s %s\n by %s — %s (cause: %s", a.At.Local().Format("2006-01-02 15:04"), a.ID,
|
||||
fmt.Fprintf(w, "%s %s %s %s\n by %s — %s (cause: %s", a.At.Local().Format("2006-01-02 15:04"), a.ID,
|
||||
a.Verb, strings.Join(a.Args, " "), a.By, a.Why, a.Cause)
|
||||
if a.Condition != "" {
|
||||
fmt.Printf(", condition %s", a.Condition)
|
||||
fmt.Fprintf(w, ", condition %s", a.Condition)
|
||||
}
|
||||
fmt.Fprintln(w, ")")
|
||||
if pushedRecorded(a) {
|
||||
carried := strings.Join(a.Carried, "; ")
|
||||
if carried == "" {
|
||||
carried = recordedBefore[a.ID]
|
||||
}
|
||||
fmt.Fprintf(w, " a push of recorded builds, no repair: %s\n", carried)
|
||||
}
|
||||
fmt.Println(")")
|
||||
}
|
||||
if len(repeated) > 0 {
|
||||
causes := make([]string, 0, len(repeated))
|
||||
@@ -251,7 +301,7 @@ func handActCommand(ctx context.Context, args []string) error {
|
||||
causes = append(causes, fmt.Sprintf("%s ×%d", c, n))
|
||||
}
|
||||
sort.Strings(causes)
|
||||
fmt.Printf("\ndone by hand more than once in a fortnight — a healer is wanted (to-be 45 S15): %s\n",
|
||||
fmt.Fprintf(w, "\ndone by hand more than once in a fortnight — a healer is wanted (to-be 45 S15): %s\n",
|
||||
strings.Join(causes, ", "))
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -22,10 +22,6 @@ func TestARepairByHandWithoutAReasonIsRefused(t *testing.T) {
|
||||
{"plans", map[string]any{"close": "plan-1"}},
|
||||
{"plans", map[string]any{"stop": "plan-1"}},
|
||||
{"hand-act", map[string]any{"what": "restarted the proxy", "cause": "proxy-stuck"}},
|
||||
{"command", map[string]any{"command": "push anchor"}},
|
||||
{"command", map[string]any{"command": "plans close plan-1"}},
|
||||
{"command", map[string]any{"command": "broker consumer-reset EVENTS controller"}},
|
||||
{"command", map[string]any{"command": "hand-act record restarted --cause x"}},
|
||||
} {
|
||||
argv, err := argvFor(c.verb, c.args)
|
||||
if c.verb == "plans" && err == nil {
|
||||
@@ -65,7 +61,6 @@ func TestARepairByHandCarriesItsReason(t *testing.T) {
|
||||
{"plans", map[string]any{"retry": "plan-1"}, "plans retry plan-1"},
|
||||
{"hand-act", map[string]any{"what": "restarted", "why": "hung", "cause": "proxy", "condition": "machine.a.silent"},
|
||||
"hand-act record restarted --why hung --cause proxy --condition machine.a.silent"},
|
||||
{"command", map[string]any{"command": "push anchor --why stuck"}, "push anchor --why stuck"},
|
||||
{"command", map[string]any{"command": "plans plan-1"}, "plans plan-1"},
|
||||
} {
|
||||
argv, err := argvFor(c.verb, c.args)
|
||||
|
||||
@@ -45,3 +45,28 @@ func TestModuleCheckCountsTheUndeclaredAndRefusesThemFromTheDate(t *testing.T) {
|
||||
t.Errorf("the refusal does not name the resource:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A file that asks for a setting without saying whether it is trusted counts as trusted (novox/hq issue 339):
|
||||
// `module check` lists and counts it, and never refuses it — there is nothing unsafe to refuse.
|
||||
func TestModuleCheckListsUnmarkedFilesAndNeverRefusesThem(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "module.json")
|
||||
os.WriteFile(path, []byte(`{"module":"power","resources":[
|
||||
{"id":"logind","type":"file","path":"/etc/systemd/logind.conf.d/power.conf","mode":"0644","trusted":true,
|
||||
"content":"HandleLidSwitch=${setting:lid}\n"},
|
||||
{"id":"note","type":"file","path":"/var/lib/power/note","mode":"0644","content":"${setting:greeting}\n"}]}`), 0o600)
|
||||
defer func() { checkNow = time.Now }()
|
||||
for _, at := range []time.Time{time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC), time.Date(2036, 1, 1, 0, 0, 0, 0, time.UTC)} {
|
||||
checkNow = func() time.Time { return at }
|
||||
var out bytes.Buffer
|
||||
if err := moduleCheck([]string{path}, &out); err != nil {
|
||||
t.Fatalf("refused at %v: %v\n%s", at, err, out.String())
|
||||
}
|
||||
for _, want := range []string{"note ask(s) for a setting and do(es) not say whether it is trusted, so it counts as trusted",
|
||||
UnsaidTrustLine + " 1"} {
|
||||
if !strings.Contains(out.String(), want) {
|
||||
t.Errorf("the check does not say %q:\n%s", want, out.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -179,7 +179,7 @@ func sayHeld(w io.Writer, node string, why []string) {
|
||||
// machines that could not be composed, as refusals.
|
||||
func flushBehind(ctx context.Context, open *stores, nodes []inventory.Node, handled map[string]bool,
|
||||
compose func(held context.Context, node string) (sendable, error), d delivery, holder string,
|
||||
w io.Writer) ([]string, error) {
|
||||
keep keeping, w io.Writer) ([]string, error) {
|
||||
inv := open.inventory
|
||||
var refusals []string
|
||||
// Bounded by the node count: a node is marked handled the round it is considered and is never
|
||||
@@ -237,7 +237,7 @@ func flushBehind(ctx context.Context, open *stores, nodes []inventory.Node, hand
|
||||
// a refusal and reported at the end, and the others are still sent (novox/hq ADR 0066).
|
||||
// Held for this round only, and after the last round's were given back, so two pushes
|
||||
// cascading into each other's machines never each wait on the other.
|
||||
refused, err := sendRound(ctx, open, sending, compose, d, holder)
|
||||
refused, err := sendRound(ctx, open, sending, compose, d, holder, keep)
|
||||
refusals = append(refusals, refused...)
|
||||
if err != nil {
|
||||
return refusals, err
|
||||
|
||||
@@ -203,7 +203,7 @@ func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
|
||||
}
|
||||
compose := composeForPush(open, gens)
|
||||
d := &recordedDelivery{inv: inv}
|
||||
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, ""); err != nil {
|
||||
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if builds, known, err := inv.SentBuilds(ctx, "laptop"); err != nil || !known || builds["resolver"] != "c1c1c1c1c1" {
|
||||
@@ -221,13 +221,13 @@ func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
|
||||
// The change merges; the policy is record. `push anchor` sends the anchor...
|
||||
aResolver(t, open, "c2c2c2c2c2", asked.Add(time.Minute))
|
||||
d.declared = nil
|
||||
if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, ""); err != nil {
|
||||
if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// ...and its cascade leaves the laptop, saying so.
|
||||
var said bytes.Buffer
|
||||
d.declared = nil
|
||||
refused, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true}, compose, d, "", &said)
|
||||
refused, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true}, compose, d, "", nil, &said)
|
||||
if err != nil || len(refused) != 0 {
|
||||
t.Fatalf("the cascade failed: %v %v", refused, err)
|
||||
}
|
||||
@@ -248,13 +248,13 @@ func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
|
||||
// is held, and that what else it is owed waits with it.
|
||||
aThirdMachine(t, open)
|
||||
d.declared = nil
|
||||
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, ""); err != nil {
|
||||
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.declared = nil
|
||||
said.Reset()
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
|
||||
compose, d, "", &said); err != nil {
|
||||
compose, d, "", nil, &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(d.declared) != 0 || digestOfLaptop() != before {
|
||||
@@ -271,7 +271,7 @@ func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
|
||||
}
|
||||
said.Reset()
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
|
||||
compose, d, "", &said); err != nil {
|
||||
compose, d, "", nil, &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(d.declared) != 0 || !strings.Contains(said.String(), "has passed no gate yet") {
|
||||
@@ -284,7 +284,7 @@ func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
|
||||
}
|
||||
said.Reset()
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
|
||||
compose, d, "", &said); err != nil {
|
||||
compose, d, "", nil, &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(d.declared, []string{"laptop"}) || digestOfLaptop() == before {
|
||||
@@ -311,18 +311,18 @@ func TestANamedPushStillSendsAConsequenceNothingHolds(t *testing.T) {
|
||||
}
|
||||
compose := composeForPush(open, gens)
|
||||
d := &recordedDelivery{inv: inv}
|
||||
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, ""); err != nil {
|
||||
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// `push spare`, the machine just placed: the others' peers change with it.
|
||||
aThirdMachine(t, open)
|
||||
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, ""); err != nil {
|
||||
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.declared = nil
|
||||
var said bytes.Buffer
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", &said); err != nil {
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(d.declared, []string{"anchor", "laptop"}) {
|
||||
@@ -342,7 +342,7 @@ func TestANamedPushStillSendsAConsequenceNothingHolds(t *testing.T) {
|
||||
}
|
||||
d.declared = nil
|
||||
said.Reset()
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", &said); err != nil {
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The anchor, the hub, may still be settling from the machine placed above; the laptop is the
|
||||
|
||||
@@ -23,11 +23,11 @@ func TestASendRoundGivesItsHoldBackOnEveryWayOut(t *testing.T) {
|
||||
|
||||
for name, round := range map[string]func() error{
|
||||
"a body that cannot be marshalled": func() error {
|
||||
_, err := sendRound(ctx, open, []string{"anchor"}, unmarshallable, fine, "")
|
||||
_, err := sendRound(ctx, open, []string{"anchor"}, unmarshallable, fine, "", nil)
|
||||
return err
|
||||
},
|
||||
"a send that fails": func() error {
|
||||
_, err := sendRound(ctx, open, []string{"anchor"}, plain, failing, "")
|
||||
_, err := sendRound(ctx, open, []string{"anchor"}, plain, failing, "", nil)
|
||||
return err
|
||||
},
|
||||
} {
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// The live case of 2026-10-07 (novox/hq issue 299), replayed: the node-uplink seat gained its first
|
||||
// verbs, `resolvers` and `links`, both optional while its holders catch up, and the holders serve
|
||||
// neither. The working set is read from the store, whose rows carry no optional mark, so the seat comes
|
||||
// back through UseSeats as it does live. Its holder on every machine answers nothing for it on the bus,
|
||||
// and must not be judged silent: a seat whose verbs are all optional asks no holder to answer.
|
||||
func TestAHolderOfASeatWhoseVerbsAreAllOptionalIsNotSilent(t *testing.T) {
|
||||
defer catalogue.UseSeats(catalogue.DefaultSeats())
|
||||
var rows []catalogue.Seat
|
||||
for _, s := range catalogue.DefaultSeats() {
|
||||
stored := s
|
||||
stored.Serves = nil
|
||||
for _, v := range s.Serves {
|
||||
v.Optional = false // the store never keeps the mark
|
||||
stored.Serves = append(stored.Serves, v)
|
||||
}
|
||||
rows = append(rows, stored)
|
||||
}
|
||||
catalogue.UseSeats(rows)
|
||||
|
||||
uplink, ok := catalogue.SeatNamed("node-uplink")
|
||||
if !ok || len(uplink.Serves) == 0 {
|
||||
t.Fatalf("node-uplink must serve verbs for this replay: %+v", uplink)
|
||||
}
|
||||
for _, v := range uplink.Serves {
|
||||
if !v.Optional {
|
||||
t.Fatalf("node-uplink's %s is required; this replay is of a seat whose verbs are all optional", v.Name)
|
||||
}
|
||||
}
|
||||
|
||||
nodes := []string{"anchor", "home-server", "workstation", "laptop"}
|
||||
heard := map[string]bool{}
|
||||
var recorded []catalogue.Held
|
||||
for _, n := range nodes {
|
||||
heard[n] = true
|
||||
recorded = append(recorded, catalogue.Held{Claim: "node-uplink", Scope: catalogue.ScopeNode, Node: n,
|
||||
Module: "networkmanager"})
|
||||
}
|
||||
expected := holdersToHear(catalogue.SeatsWithAProtocol(), recorded, nil, heard, nil, time.Now())
|
||||
if _, asked := expected["node-uplink"]; asked {
|
||||
t.Fatalf("node-uplink's holders are expected to answer though every verb of the seat is optional: %v",
|
||||
expected["node-uplink"])
|
||||
}
|
||||
// Nothing answers for the seat, as live: no silence is said about it.
|
||||
for _, o := range silentHolders(expected, map[string]map[string]bool{}) {
|
||||
if o.ID == "node-uplink."+o.Machine {
|
||||
t.Errorf("a holder serving none of a seat's optional verbs was judged silent: %s", o.Summary)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Silence is still judged on a required verb: a seat with one required and one optional verb, held on a
|
||||
// machine that is heard from, whose holder answers nothing — silent.
|
||||
func TestAHolderServingNoRequiredVerbIsStillSilent(t *testing.T) {
|
||||
seat := catalogue.Seat{Name: "example-seat", Scope: catalogue.ScopeNode, Serves: []catalogue.Verb{
|
||||
{Name: "state"}, {Name: "later", Optional: true}}}
|
||||
recorded := []catalogue.Held{{Claim: "example-seat", Scope: catalogue.ScopeNode, Node: "laptop", Module: "m"}}
|
||||
expected := holdersToHear([]catalogue.Seat{seat}, recorded, nil, map[string]bool{"laptop": true}, nil, time.Now())
|
||||
if !expected["example-seat"]["laptop"] {
|
||||
t.Fatalf("a holder of a seat with a required verb is not expected to answer: %v", expected)
|
||||
}
|
||||
out := silentHolders(expected, map[string]map[string]bool{})
|
||||
if len(out) != 1 || out[0].ID != "example-seat.laptop" || out[0].Token != "silent" {
|
||||
t.Fatalf("a holder serving no required verb is not said to be silent: %+v", out)
|
||||
}
|
||||
if got := silentHolders(expected, map[string]map[string]bool{"example-seat": {"laptop": true}}); len(got) != 0 {
|
||||
t.Fatalf("a holder that answers is said to be silent: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHoldingNeedsAnAnswer(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
serves []catalogue.Verb
|
||||
want bool
|
||||
}{
|
||||
{"no verb", nil, false},
|
||||
{"only optional verbs", []catalogue.Verb{{Name: "a", Optional: true}, {Name: "b", Optional: true}}, false},
|
||||
{"a required verb among optional ones", []catalogue.Verb{{Name: "a", Optional: true}, {Name: "b"}}, true},
|
||||
{"only required verbs", []catalogue.Verb{{Name: "a"}}, true},
|
||||
} {
|
||||
if got := holdingNeedsAnAnswer(catalogue.Seat{Name: "s", Serves: c.serves}); got != c.want {
|
||||
t.Errorf("%s: holdingNeedsAnAnswer = %v, want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The control-node withholds the login shell's `execute` (novox/hq ADR 0268): its holder there serves
|
||||
// nothing on the seat, and must not be judged silent for it, as the store's rows read it back.
|
||||
func TestALoginShellWithholdingExecuteIsNotSilent(t *testing.T) {
|
||||
defer catalogue.UseSeats(catalogue.DefaultSeats())
|
||||
var rows []catalogue.Seat
|
||||
for _, s := range catalogue.DefaultSeats() {
|
||||
stored := s
|
||||
stored.Serves = nil
|
||||
for _, v := range s.Serves {
|
||||
v.Optional = false // the store never keeps the mark
|
||||
stored.Serves = append(stored.Serves, v)
|
||||
}
|
||||
rows = append(rows, stored)
|
||||
}
|
||||
catalogue.UseSeats(rows)
|
||||
recorded := []catalogue.Held{{Claim: catalogue.LoginShellSeat, Scope: catalogue.ScopeNode, Node: "anchor", Module: "zsh"}}
|
||||
expected := holdersToHear(catalogue.SeatsWithAProtocol(), recorded, nil, map[string]bool{"anchor": true}, nil, time.Now())
|
||||
if _, asked := expected[catalogue.LoginShellSeat]; asked {
|
||||
t.Fatalf("the login shell's holder is expected to answer, so withholding execute would be said silent: %v",
|
||||
expected[catalogue.LoginShellSeat])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// aMachineJoining is a machine with a live token issued for a tunnel key, given its address — what
|
||||
// `token issue --overlay-key` records before it pushes the hub (novox/hq ADR 0169).
|
||||
func aMachineJoining(t *testing.T, ctx context.Context, inv *inventory.Inventory, name string,
|
||||
validFor time.Duration) string {
|
||||
t.Helper()
|
||||
record, err := inv.AddNode(ctx, name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.IssueToken(ctx, name, validFor); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := aPublicKey(t)
|
||||
if err := inv.RecordOverlayKey(ctx, record.ID, key); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.BindTokenToKey(ctx, record.ID, key); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cidr, err := overlayRange(ctx, inv)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.AssignAddress(ctx, record.ID, cidr); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
// hubPeers are the keys the hub's composed tunnel carries.
|
||||
func hubPeers(t *testing.T, ctx context.Context, inv *inventory.Inventory) map[string]bool {
|
||||
t.Helper()
|
||||
g, err := network(ctx, inv, map[string]bool{"anchor": true, "laptop": true}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := map[string]bool{}
|
||||
for _, p := range g.Graph()["anchor"] {
|
||||
out[p.Key] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// **A machine joining is a peer of the hub while its token can be used, and not after** (novox/hq
|
||||
// ADR 0169): the hub's composed tunnel carries a machine whose token was issued for its key, so the
|
||||
// tunnel answers the first time it knocks; a token that expired unused takes the peer with it at the
|
||||
// hub's next composition.
|
||||
func TestAMachineJoiningIsAPeerOfTheHubUntilItsTokenExpires(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
|
||||
joining := aMachineJoining(t, ctx, inv, "joiner", time.Hour)
|
||||
expired := aMachineJoining(t, ctx, inv, "late", 2*time.Second)
|
||||
time.Sleep(2100 * time.Millisecond)
|
||||
|
||||
peers := hubPeers(t, ctx, inv)
|
||||
if !peers[joining] {
|
||||
t.Fatalf("the hub does not carry the machine its live token was issued for: %v", peers)
|
||||
}
|
||||
if peers[expired] {
|
||||
t.Fatalf("the hub still carries a machine whose token expired unused: %v", peers)
|
||||
}
|
||||
|
||||
// A token issued without a key gives the hub nothing to carry: there is no key to carry.
|
||||
if _, err := inv.AddNode(ctx, "keyless"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.IssueToken(ctx, "keyless", time.Hour); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if after := hubPeers(t, ctx, inv); len(after) != len(peers) {
|
||||
t.Fatalf("a token issued without a key changed the hub's peers: %v, was %v", after, peers)
|
||||
}
|
||||
}
|
||||
|
||||
// **A tunnel key that is not one is refused before anything is recorded** (novox/hq ADR 0169): a
|
||||
// token issued for it would be a tunnel the hub could never answer.
|
||||
func TestATokenIsNotIssuedForSomethingThatIsNotATunnelKey(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
record, err := open.inventory.AddNode(ctx, "joiner")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := throughTheTunnel(ctx, open, record, "not-a-key", "10.77.0.1:4222"); err == nil {
|
||||
t.Fatal("a token was issued for something that is not a tunnel key")
|
||||
}
|
||||
if held := placementOf(t, ctx, open.inventory, "joiner"); held.Key != "" || held.Address != "" {
|
||||
t.Fatalf("a refused key left a record behind: %+v", held)
|
||||
}
|
||||
}
|
||||
@@ -232,7 +232,7 @@ func licenceKey(ctx context.Context, args []string) error {
|
||||
// and printing the value here would put the one copy that matters on a terminal.
|
||||
fmt.Printf("sealed to %d holder(s). The mesh has discarded the key and cannot read it back\n",
|
||||
sealed)
|
||||
fmt.Printf(" run `push` to deliver it\n")
|
||||
fmt.Printf(" run `push --behind` to deliver it\n")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -340,7 +340,7 @@ func licenceSetGrant(ctx context.Context, args []string) error {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s now holds a refresh token for %s, sealed to that node's key and readable by it "+
|
||||
"alone.\n the control plane stored the box without opening it; run `push` to deliver it\n",
|
||||
"alone.\n the control plane stored the box without opening it; run `push --behind` to deliver it\n",
|
||||
"the manager", name)
|
||||
return nil
|
||||
}
|
||||
@@ -423,7 +423,7 @@ func licenceSubmitRefresh(ctx context.Context, args []string) error {
|
||||
"manager node alone"
|
||||
}
|
||||
fmt.Printf("submitted a refresh for %s: a new access token sealed to %d holder(s), and %s.\n"+
|
||||
" run `push` to deliver it\n", name, sealed, rotatedNote)
|
||||
" run `push --behind` to deliver it\n", name, sealed, rotatedNote)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -456,7 +456,7 @@ func licenceRefresh(ctx context.Context, args []string) error {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("refreshed %s: a new access token sealed to %d holder(s), and the refresh token left "+
|
||||
"with its manager.\n run `push` to deliver it\n", name, sealed)
|
||||
"with its manager.\n run `push --behind` to deliver it\n", name, sealed)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,359 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A machine says how its network is (novox/hq ADR 0241, which extends ADR 0240 from what a module runs
|
||||
// to the machine it runs on).
|
||||
//
|
||||
// **Every machine's node-engine judges its own networking** — the resolver file the uplink holder
|
||||
// declared, the names through every resolver it lists, the tunnel's handshake with the hub, the bus, the
|
||||
// default route — on the two-look rule, and states it beside its resources. The controller keeps the
|
||||
// newest statement per machine and raises from all of them together:
|
||||
//
|
||||
// - **an outside writer of the resolver file is its own finding**, `machine.<m>.<owner>.rewritten`:
|
||||
// the file the uplink holder declares was rewritten by another program, named where the engine could
|
||||
// name it. The names failing through what that program wrote are that finding's consequence, said in
|
||||
// it — never a second condition;
|
||||
// - **what is the machine's own** — its route, its tunnel, its resolvers answering wrong, a resolver
|
||||
// that is no mesh machine — is `machine.<m>.network`;
|
||||
// - **what points at another machine is said once, there** (the provider hold of ADR 0240 rule 5, for
|
||||
// the network): a failure toward the hub or toward a mesh resolver is held under that machine when it
|
||||
// is down on the record — silent, or its own network unhealthy — or when a second machine finds the
|
||||
// same; then `machine.<x>.unreachable` names every machine that cannot reach it, and none of them
|
||||
// raises anything of its own for it. One machine alone failing toward a healthy one is its own.
|
||||
//
|
||||
// Each is a warning; urgent on the control node, or when the bus cannot be reached, or for the hub.
|
||||
// Cleared on the first statement that no longer says it. An engine older than this judging says nothing
|
||||
// of its network, and nothing is raised for it.
|
||||
|
||||
// The conditions a machine's network raises.
|
||||
const (
|
||||
kindMachineNetwork = "machine-network"
|
||||
kindNetworkRewritten = "network-rewritten"
|
||||
kindNetworkUnreachable = "network-unreachable"
|
||||
sourceNetwork = "network"
|
||||
)
|
||||
|
||||
// networkKinds are the kinds this judging owns: every open one it no longer says, it clears.
|
||||
var networkKinds = []string{kindMachineNetwork, kindNetworkRewritten, kindNetworkUnreachable}
|
||||
|
||||
// networkFacts is what one judging of every machine's network reads.
|
||||
type networkFacts struct {
|
||||
healths map[string]inventory.NodeHealth
|
||||
// byAddress is each machine's address on the private network; hub the hub's name; control the
|
||||
// control node's.
|
||||
byAddress map[string]string
|
||||
hub string
|
||||
control string
|
||||
// silent is every machine whose silence is an open condition.
|
||||
silent map[string]bool
|
||||
}
|
||||
|
||||
// judgeNetworks raises and clears every machine's network conditions from every machine's newest
|
||||
// statement, after one machine's statement was kept.
|
||||
func judgeNetworks(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, now time.Time) error {
|
||||
healths, err := inv.Healths(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
overlays, err := inv.Overlays(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
open, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f := networkFacts{healths: healths, byAddress: map[string]string{}, control: controlHost(ctx, inv),
|
||||
silent: map[string]bool{}}
|
||||
for _, o := range overlays {
|
||||
if o.Address != "" {
|
||||
f.byAddress[o.Address] = o.Name
|
||||
}
|
||||
if o.Hub {
|
||||
f.hub = o.Name
|
||||
}
|
||||
}
|
||||
for _, c := range open {
|
||||
if c.Subject.Scope == conditions.ScopeMachine && c.Kind == "silent" {
|
||||
f.silent[c.Subject.ID] = true
|
||||
}
|
||||
}
|
||||
var problems []string
|
||||
said := map[string]bool{}
|
||||
for _, o := range networkObservations(f) {
|
||||
said[o.Key()] = true
|
||||
if _, err := k.Observe(ctx, o); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
}
|
||||
for _, c := range open {
|
||||
if !slices.Contains(networkKinds, c.Kind) || said[c.Key] {
|
||||
continue
|
||||
}
|
||||
why := "no machine says it any more"
|
||||
if c.Subject.Machine != "" {
|
||||
why = c.Subject.Machine + "'s network no longer says it"
|
||||
}
|
||||
if _, err := k.Clear(ctx, c.Key, why); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
return fmt.Errorf("%s", strings.Join(problems, "; "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// pointed is one machine's failing part that points at another machine.
|
||||
type pointed struct {
|
||||
from string
|
||||
part inventory.NetworkPart
|
||||
}
|
||||
|
||||
// networkObservations is every network condition the statements say now. Pure.
|
||||
func networkObservations(f networkFacts) []conditions.Observation {
|
||||
machines := make([]string, 0, len(f.healths))
|
||||
for m := range f.healths {
|
||||
machines = append(machines, m)
|
||||
}
|
||||
sort.Strings(machines)
|
||||
|
||||
unhealthy := func(m string) []inventory.NetworkPart {
|
||||
h := f.healths[m]
|
||||
if h.Network == nil {
|
||||
return nil
|
||||
}
|
||||
var out []inventory.NetworkPart
|
||||
for _, p := range h.Network.Parts {
|
||||
if p.State == link.StateUnhealthy {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
// The machines a part points at, other than its own: the hub, and each mesh resolver by its address.
|
||||
// An address that is no mesh machine's — the resolver a VPN client wrote in — is the machine's own.
|
||||
targets := func(m string, p inventory.NetworkPart) ([]string, bool) {
|
||||
if len(p.Toward) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
var out []string
|
||||
for _, t := range p.Toward {
|
||||
x := f.byAddress[t]
|
||||
if t == link.TowardHub {
|
||||
x = f.hub
|
||||
}
|
||||
if x == "" || x == m {
|
||||
return nil, false
|
||||
}
|
||||
if !slices.Contains(out, x) {
|
||||
out = append(out, x)
|
||||
}
|
||||
}
|
||||
return out, true
|
||||
}
|
||||
|
||||
// First pass: what points at whom.
|
||||
pointing := map[string][]pointed{}
|
||||
for _, m := range machines {
|
||||
for _, p := range unhealthy(m) {
|
||||
if xs, ok := targets(m, p); ok {
|
||||
for _, x := range xs {
|
||||
pointing[x] = append(pointing[x], pointed{from: m, part: p})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
from := func(x string) []string {
|
||||
var out []string
|
||||
for _, pt := range pointing[x] {
|
||||
if !slices.Contains(out, pt.from) {
|
||||
out = append(out, pt.from)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
// A machine is down on the record when its silence is open or its own network is unhealthy, or when
|
||||
// two machines find it unreachable: then what points at it is held there.
|
||||
down := func(x string) bool {
|
||||
return f.silent[x] || len(unhealthy(x)) > 0 || len(from(x)) >= 2
|
||||
}
|
||||
|
||||
var out []conditions.Observation
|
||||
saysOwn := map[string]bool{}
|
||||
for _, m := range machines {
|
||||
parts := unhealthy(m)
|
||||
if len(parts) == 0 {
|
||||
continue
|
||||
}
|
||||
var own []inventory.NetworkPart
|
||||
var rewritten *inventory.NetworkPart
|
||||
for i, p := range parts {
|
||||
if p.Part == link.PartResolvConf {
|
||||
rewritten = &parts[i]
|
||||
continue
|
||||
}
|
||||
if xs, ok := targets(m, p); ok && allDown(xs, down) {
|
||||
continue // held at the machines it points at
|
||||
}
|
||||
own = append(own, p)
|
||||
}
|
||||
if rewritten != nil {
|
||||
out = append(out, rewrittenObservation(m, *rewritten, parts, f))
|
||||
// The names failing through what another program wrote are that finding's, said in it.
|
||||
kept := own[:0]
|
||||
for _, p := range own {
|
||||
if p.Part != link.PartNames {
|
||||
kept = append(kept, p)
|
||||
}
|
||||
}
|
||||
own = kept
|
||||
}
|
||||
if len(own) > 0 {
|
||||
out = append(out, machineNetworkObservation(m, own, f, from(m)))
|
||||
saysOwn[m] = true
|
||||
}
|
||||
}
|
||||
// Said once, at the machine everybody points at — unless its own network condition already says it
|
||||
// (listed there), or its silence does.
|
||||
targetsSorted := make([]string, 0, len(pointing))
|
||||
for x := range pointing {
|
||||
targetsSorted = append(targetsSorted, x)
|
||||
}
|
||||
sort.Strings(targetsSorted)
|
||||
for _, x := range targetsSorted {
|
||||
if !down(x) || saysOwn[x] || f.silent[x] {
|
||||
continue
|
||||
}
|
||||
out = append(out, unreachableObservation(x, pointing[x], from(x), f))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func allDown(xs []string, down func(string) bool) bool {
|
||||
for _, x := range xs {
|
||||
if !down(x) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(xs) > 0
|
||||
}
|
||||
|
||||
// rewrittenObservation is the resolver file rewritten by another program: its own finding, naming the
|
||||
// writer where the engine could, and what it costs the machine.
|
||||
func rewrittenObservation(m string, p inventory.NetworkPart, all []inventory.NetworkPart, f networkFacts) conditions.Observation {
|
||||
writer := ""
|
||||
if p.Writer != "" {
|
||||
writer = " (" + p.Writer + ")"
|
||||
}
|
||||
cost := "the names through it are not yet judged"
|
||||
said := []string{p.Part + ": " + p.Said}
|
||||
for _, q := range all {
|
||||
if q.Part == link.PartNames {
|
||||
cost = strings.TrimSuffix(q.Reason, ".")
|
||||
said = append(said, q.Part+": "+q.Said)
|
||||
}
|
||||
}
|
||||
if cost == "the names through it are not yet judged" {
|
||||
cost = "the names still resolve through what it wrote"
|
||||
}
|
||||
id := m
|
||||
if p.Owner != "" {
|
||||
// Named by the module whose file it is: a send that moved that module is what the gate
|
||||
// holds it on (issue 281's rule — what names a moved module is that module's).
|
||||
id = m + "." + p.Owner
|
||||
}
|
||||
severity := conditions.Warning
|
||||
if m == f.control {
|
||||
severity = conditions.Urgent
|
||||
}
|
||||
summary := fmt.Sprintf("the resolver file on %s was rewritten by another program%s — %s until the "+
|
||||
"node-engine writes it back at its next reconcile, or that program gives it back", m, writer, cost)
|
||||
if p.Owner != "" {
|
||||
summary = fmt.Sprintf("the resolver file %s writes on %s was rewritten by another program%s — %s until "+
|
||||
"the node-engine writes it back at its next reconcile, or that program gives it back", p.Owner, m, writer, cost)
|
||||
}
|
||||
return conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: "rewritten", Kind: kindNetworkRewritten,
|
||||
Machine: m, Severity: severity, Source: sourceNetwork, Summary: summary,
|
||||
Said: fmt.Sprintf("since %s: %s", p.Since.UTC().Format("2006-01-02 15:04:05 MST"), strings.Join(said, " | "))}
|
||||
}
|
||||
|
||||
// machineNetworkObservation is what is wrong with a machine's own networking.
|
||||
func machineNetworkObservation(m string, parts []inventory.NetworkPart, f networkFacts, waiting []string) conditions.Observation {
|
||||
var words, said []string
|
||||
severity := conditions.Warning
|
||||
for _, p := range parts {
|
||||
words = append(words, p.Reason)
|
||||
said = append(said, fmt.Sprintf("%s since %s: %s", p.Part, p.Since.UTC().Format("2006-01-02 15:04:05 MST"), p.Said))
|
||||
if p.Part == link.PartBus {
|
||||
severity = conditions.Urgent
|
||||
}
|
||||
}
|
||||
if m == f.control || m == f.hub {
|
||||
severity = conditions.Urgent
|
||||
}
|
||||
summary := fmt.Sprintf("%s's network is not healthy: %s", m, strings.Join(words, "; "))
|
||||
if len(waiting) > 0 {
|
||||
severity = conditions.Urgent
|
||||
summary += fmt.Sprintf("; %s cannot reach it", strings.Join(waiting, ", "))
|
||||
}
|
||||
return conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Token: "network", Kind: kindMachineNetwork,
|
||||
Machine: m, Severity: severity, Source: sourceNetwork, Summary: summary, Said: strings.Join(said, " | ")}
|
||||
}
|
||||
|
||||
// unreachableObservation is one machine others cannot reach, said once there.
|
||||
func unreachableObservation(x string, pts []pointed, from []string, f networkFacts) conditions.Observation {
|
||||
var what []string
|
||||
var said []string
|
||||
for _, pt := range pts {
|
||||
w := map[string]string{link.PartTunnel: "the tunnel to it", link.PartBus: "the bus on it",
|
||||
link.PartNames: "its resolver"}[pt.part.Part]
|
||||
if w == "" {
|
||||
w = pt.part.Part
|
||||
}
|
||||
if !slices.Contains(what, w) {
|
||||
what = append(what, w)
|
||||
}
|
||||
said = append(said, fmt.Sprintf("%s: %s: %s", pt.from, pt.part.Part, pt.part.Said))
|
||||
}
|
||||
severity := conditions.Warning
|
||||
if x == f.hub || x == f.control || slices.Contains(what, "the bus on it") {
|
||||
severity = conditions.Urgent
|
||||
}
|
||||
return conditions.Observation{Scope: conditions.ScopeMachine, ID: x, Token: "unreachable", Kind: kindNetworkUnreachable,
|
||||
Machine: x, Also: from, Severity: severity, Source: sourceNetwork,
|
||||
Summary: fmt.Sprintf("%s cannot be reached from %s: %s", x, strings.Join(from, ", "), strings.Join(what, ", ")),
|
||||
Said: strings.Join(said, " | ")}
|
||||
}
|
||||
|
||||
// networkLines is what `node show` says of a machine's networking.
|
||||
func networkLines(h inventory.NodeHealth, had bool, now time.Time) []string {
|
||||
if !had || h.Network == nil {
|
||||
return []string{" its node-engine does not say how its network is — it is older than that judging (ADR 0241)"}
|
||||
}
|
||||
out := []string{fmt.Sprintf(" its network: %s since %s", h.Network.State, h.Network.Since.Local().Format("2006-01-02 15:04"))}
|
||||
for _, p := range h.Network.Parts {
|
||||
line := fmt.Sprintf(" %-10s %s", p.State, p.Part)
|
||||
if p.Reason != "" {
|
||||
line += " — " + p.Reason
|
||||
}
|
||||
if p.Writer != "" {
|
||||
line += " (" + p.Writer + ")"
|
||||
}
|
||||
out = append(out, line)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,306 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A machine says how its network is (novox/hq ADR 0241, "how it is checked"): the resolver file rewritten
|
||||
// by another program is one finding, naming the writer, with the names it costs said in it; what is the
|
||||
// machine's own is `machine.<m>.network`; what points at another machine that is down is said once, there;
|
||||
// one machine alone failing toward a healthy one is its own; the control node, the hub and the bus are
|
||||
// urgent; an engine that says nothing of its network raises nothing; and the gate waits on what is shown to
|
||||
// be another's.
|
||||
|
||||
func aNetwork(state string, parts ...inventory.NetworkPart) *inventory.NetworkHealth {
|
||||
for i := range parts {
|
||||
if parts[i].State == "" {
|
||||
parts[i].State = link.StateUnhealthy
|
||||
}
|
||||
parts[i].Since = h0
|
||||
}
|
||||
return &inventory.NetworkHealth{State: state, Since: h0, Parts: parts}
|
||||
}
|
||||
|
||||
func netFacts(healths map[string]*inventory.NetworkHealth) networkFacts {
|
||||
f := networkFacts{healths: map[string]inventory.NodeHealth{}, hub: "anchor", control: "anchor",
|
||||
byAddress: map[string]string{"10.77.0.1": "anchor", "10.77.0.2": "laptop", "10.77.0.3": "spare"},
|
||||
silent: map[string]bool{}}
|
||||
for m, n := range healths {
|
||||
f.healths[m] = inventory.NodeHealth{Node: m, Network: n}
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
var (
|
||||
rewrittenByVPN = inventory.NetworkPart{Part: link.PartResolvConf, Reason: "the resolver file was rewritten by another program",
|
||||
Said: "/etc/resolv.conf lists 172.16.5.5 where 10.77.0.1 is declared", Writer: "FortiClient", Owner: "networkmanager"}
|
||||
namesThroughVPN = inventory.NetworkPart{Part: link.PartNames, Reason: "mesh names do not resolve",
|
||||
Said: "172.16.5.5 — anchor.internal (IPv4): says no such name", Toward: []string{"172.16.5.5"}}
|
||||
tunnelDown = inventory.NetworkPart{Part: link.PartTunnel, Reason: "the tunnel to the hub has not handshaken for over five minutes",
|
||||
Said: "mesh0's newest handshake with the hub was 9m0s ago", Toward: []string{link.TowardHub}}
|
||||
noRoute = inventory.NetworkPart{Part: link.PartRoute, Reason: "the machine has no default route", Said: "no default route"}
|
||||
)
|
||||
|
||||
func keysOf(obs []conditions.Observation) []string {
|
||||
var keys []string
|
||||
for _, o := range obs {
|
||||
keys = append(keys, o.Key())
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
func TestAResolverFileRewrittenIsOneFindingNamingItsWriterAndWhatItCosts(t *testing.T) {
|
||||
obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
|
||||
"anchor": aNetwork(link.StateHealthy),
|
||||
"laptop": aNetwork(link.StateUnhealthy, rewrittenByVPN, namesThroughVPN),
|
||||
}))
|
||||
if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" {
|
||||
t.Fatalf("want one finding, the rewrite, got %v", keys)
|
||||
}
|
||||
o := obs[0]
|
||||
for _, want := range []string{"laptop", "rewritten by another program (FortiClient)", "mesh names do not resolve",
|
||||
"next reconcile"} {
|
||||
if !strings.Contains(o.Summary, want) {
|
||||
t.Errorf("the summary does not say %q: %s", want, o.Summary)
|
||||
}
|
||||
}
|
||||
if strings.Contains(o.Summary, "172.16.") || strings.Contains(o.Summary, "/etc/") {
|
||||
t.Errorf("an address or a path reached the summary: %s", o.Summary)
|
||||
}
|
||||
if !strings.Contains(o.Said, "172.16.5.5") || o.Severity != conditions.Warning || o.Machine != "laptop" {
|
||||
t.Errorf("the evidence or the severity is wrong: %+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOneMachineFailingTowardAHealthyHubIsItsOwn(t *testing.T) {
|
||||
obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
|
||||
"anchor": aNetwork(link.StateHealthy),
|
||||
"laptop": aNetwork(link.StateUnhealthy, tunnelDown),
|
||||
"spare": aNetwork(link.StateHealthy),
|
||||
}))
|
||||
if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" {
|
||||
t.Fatalf("want the laptop's own, got %v", keys)
|
||||
}
|
||||
if obs[0].Severity != conditions.Warning {
|
||||
t.Fatalf("a laptop's own tunnel is a warning, got %s", obs[0].Severity)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTwoMachinesThatCannotReachTheHubAreSaidOnceAtTheHub(t *testing.T) {
|
||||
obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
|
||||
"anchor": aNetwork(link.StateHealthy),
|
||||
"laptop": aNetwork(link.StateUnhealthy, tunnelDown),
|
||||
"spare": aNetwork(link.StateUnhealthy, tunnelDown),
|
||||
}))
|
||||
if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.unreachable" {
|
||||
t.Fatalf("want one condition at the hub, got %v", keys)
|
||||
}
|
||||
o := obs[0]
|
||||
if o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop, spare") || len(o.Also) != 2 {
|
||||
t.Fatalf("the hub's condition is %+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhatPointsAtASilentHubIsHeldUnderItsSilence(t *testing.T) {
|
||||
f := netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, tunnelDown)})
|
||||
f.silent["anchor"] = true
|
||||
if obs := networkObservations(f); len(obs) != 0 {
|
||||
t.Fatalf("the hub's silence says it; got %v", keysOf(obs))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAHubWhoseOwnNetworkIsUnhealthyListsWhoCannotReachIt(t *testing.T) {
|
||||
obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
|
||||
"anchor": aNetwork(link.StateUnhealthy, noRoute),
|
||||
"laptop": aNetwork(link.StateUnhealthy, tunnelDown),
|
||||
}))
|
||||
if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.network" {
|
||||
t.Fatalf("want the hub's own, holding the laptop's, got %v", keys)
|
||||
}
|
||||
if o := obs[0]; o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop cannot reach it") {
|
||||
t.Fatalf("the hub's condition is %+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOneMachineFailingAHealthyMeshResolverIsItsOwnAndTwoAreTheResolvers(t *testing.T) {
|
||||
silentResolver := inventory.NetworkPart{Part: link.PartNames, Reason: "1 of its 2 resolvers do not answer as the mesh's do",
|
||||
Said: "10.77.0.3 — no answer within 1s", Toward: []string{"10.77.0.3"}}
|
||||
obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
|
||||
"anchor": aNetwork(link.StateHealthy), "spare": aNetwork(link.StateHealthy),
|
||||
"laptop": aNetwork(link.StateUnhealthy, silentResolver),
|
||||
}))
|
||||
if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" {
|
||||
t.Fatalf("one machine alone: want its own, got %v", keys)
|
||||
}
|
||||
obs = networkObservations(netFacts(map[string]*inventory.NetworkHealth{
|
||||
"anchor": aNetwork(link.StateUnhealthy, silentResolver), "spare": aNetwork(link.StateHealthy),
|
||||
"laptop": aNetwork(link.StateUnhealthy, silentResolver),
|
||||
}))
|
||||
if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.spare.unreachable" {
|
||||
t.Fatalf("two machines: want it said once at the resolver's machine, got %v", keys)
|
||||
}
|
||||
if !strings.Contains(obs[0].Summary, "its resolver") {
|
||||
t.Fatalf("the resolver's machine is said %s", obs[0].Summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheControlNodeAndTheBusAreUrgent(t *testing.T) {
|
||||
f := netFacts(map[string]*inventory.NetworkHealth{"anchor": aNetwork(link.StateUnhealthy, rewrittenByVPN)})
|
||||
if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent {
|
||||
t.Fatalf("the control node's rewritten file: %+v", obs)
|
||||
}
|
||||
bus := inventory.NetworkPart{Part: link.PartBus, Reason: "the bus cannot be reached", Said: "no link"}
|
||||
f = netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, bus, noRoute)})
|
||||
if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent {
|
||||
t.Fatalf("the bus unreachable from the laptop: %+v", obs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnEngineThatSaysNothingOfItsNetworkRaisesNothing(t *testing.T) {
|
||||
f := netFacts(map[string]*inventory.NetworkHealth{"laptop": nil, "anchor": aNetwork(link.StateHealthy)})
|
||||
if obs := networkObservations(f); len(obs) != 0 {
|
||||
t.Fatalf("got %v", keysOf(obs))
|
||||
}
|
||||
}
|
||||
|
||||
// Through the store and the keeper: the statement kept, the rewrite raised from it, cleared when the file
|
||||
// is written back, and node show saying it.
|
||||
func TestARewrittenResolverFileIsRaisedFromTheStatementAndClearedWhenWrittenBack(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv, k := open.inventory, conditionsFrom
|
||||
say := func(at time.Time, n *link.NetworkHealth) {
|
||||
t.Helper()
|
||||
if err := stateHealth(ctx, inv, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
openKeys := func() []string {
|
||||
t.Helper()
|
||||
list, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keys []string
|
||||
for _, c := range list {
|
||||
keys = append(keys, c.Key)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
healthy := &link.NetworkHealth{State: link.StateHealthy, Since: h0, Parts: []link.NetworkPart{
|
||||
{Part: link.PartResolvConf, State: link.StateHealthy, Since: h0}}}
|
||||
rewritten := &link.NetworkHealth{State: link.StateUnhealthy, Since: h0.Add(time.Minute), Parts: []link.NetworkPart{
|
||||
{Part: link.PartResolvConf, State: link.StateUnhealthy, Reason: rewrittenByVPN.Reason, Said: rewrittenByVPN.Said,
|
||||
Writer: "FortiClient", Owner: "networkmanager", Since: h0.Add(time.Minute)},
|
||||
{Part: link.PartNames, State: link.StateUnhealthy, Reason: namesThroughVPN.Reason, Said: namesThroughVPN.Said,
|
||||
Toward: namesThroughVPN.Toward, Since: h0.Add(time.Minute)}}}
|
||||
|
||||
say(h0, healthy)
|
||||
if keys := openKeys(); len(keys) != 0 {
|
||||
t.Fatalf("a healthy network raised %v", keys)
|
||||
}
|
||||
say(h0.Add(time.Minute), rewritten)
|
||||
if keys := openKeys(); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" {
|
||||
t.Fatalf("the rewrite raised %v", keys)
|
||||
}
|
||||
kept, had, err := inv.HealthOf(ctx, "laptop")
|
||||
if err != nil || !had || kept.Network == nil || kept.Network.Parts[0].Writer != "FortiClient" {
|
||||
t.Fatalf("the statement's network was not kept: %+v %v", kept.Network, err)
|
||||
}
|
||||
if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "FortiClient") ||
|
||||
!strings.Contains(lines, "unhealthy resolv-conf") {
|
||||
t.Fatalf("node show says:\n%s", lines)
|
||||
}
|
||||
say(h0.Add(2*time.Minute), healthy)
|
||||
if keys := openKeys(); len(keys) != 0 {
|
||||
t.Fatalf("written back, still open: %v", keys)
|
||||
}
|
||||
// An engine older than the judging says no network: nothing raised, and node show says it is not known.
|
||||
say(h0.Add(3*time.Minute), nil)
|
||||
kept, had, _ = inv.HealthOf(ctx, "laptop")
|
||||
if keys := openKeys(); len(keys) != 0 || kept.Network != nil {
|
||||
t.Fatalf("an older engine: %v %+v", keys, kept.Network)
|
||||
}
|
||||
if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "older than that judging") {
|
||||
t.Fatalf("node show says:\n%s", lines)
|
||||
}
|
||||
}
|
||||
|
||||
// The gate: a resolver file another program rewrote waits the judging rather than failing it at the
|
||||
// bound; the same, when the send moved the module whose file it is, is that module's; a machine's own
|
||||
// network fault holds the machine as a whole, as before.
|
||||
func TestTheGateWaitsOnARewriteItDidNotMakeAndHoldsTheOwnerOnOneItMoved(t *testing.T) {
|
||||
since := h0
|
||||
rewrite := conditions.Condition{Key: "machine.laptop.networkmanager.rewritten", Kind: kindNetworkRewritten,
|
||||
Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop.networkmanager", Machine: "laptop"},
|
||||
Summary: "the resolver file was rewritten", Raised: since.Add(time.Minute), Source: sourceNetwork}
|
||||
f := gateFacts{judged: true, open: []conditions.Condition{rewrite}}
|
||||
if w := aboutTheMachine("laptop", []string{"letta"}, since, f); w.waiting == "" || w.whole != "" || len(w.on) != 0 {
|
||||
t.Fatalf("a rewrite the send did not make: %+v", w)
|
||||
}
|
||||
if w := aboutTheMachine("laptop", []string{"networkmanager", "letta"}, since, f); w.on["networkmanager"] == "" ||
|
||||
w.on["letta"] != "" || w.waiting != "" {
|
||||
t.Fatalf("a rewrite of the file a moved module owns: %+v", w)
|
||||
}
|
||||
own := conditions.Condition{Key: "machine.laptop.network", Kind: kindMachineNetwork,
|
||||
Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop", Machine: "laptop"},
|
||||
Summary: "laptop's network is not healthy", Raised: since.Add(time.Minute), Source: sourceNetwork}
|
||||
if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true,
|
||||
open: []conditions.Condition{own}}); w.whole == "" || w.waiting != "" {
|
||||
t.Fatalf("the machine's own network: %+v", w)
|
||||
}
|
||||
unreachable := conditions.Condition{Key: "machine.anchor.unreachable", Kind: kindNetworkUnreachable,
|
||||
Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor", Also: []string{"laptop", "spare"}},
|
||||
Summary: "anchor cannot be reached", Raised: since.Add(time.Minute), Source: sourceNetwork}
|
||||
if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true,
|
||||
open: []conditions.Condition{unreachable}}); w.waiting == "" || w.whole != "" {
|
||||
t.Fatalf("a machine that cannot reach the hub: %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTheDrillsStatementsRaiseAndClearTheRewrite replays the drill of ADR 0241 (mesh-host
|
||||
// internal/network TestDrill…): what a node-engine said in a throwaway container while its resolver file
|
||||
// was declared, rewritten as a VPN client rewrites it, and written back — recorded, with the mesh's names
|
||||
// and addresses replaced by this test mesh's. Healthy raises nothing; the rewrite, on its second look, is
|
||||
// raised as one finding naming the writer; written back, it clears.
|
||||
func TestTheDrillsStatementsRaiseAndClearTheRewrite(t *testing.T) {
|
||||
raw, err := os.ReadFile("testdata/network-drill.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var said []link.Health
|
||||
if err := json.Unmarshal(raw, &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
k := conditionsFrom
|
||||
var raisedAt []int
|
||||
for i, h := range said {
|
||||
if err := stateHealth(ctx, open.inventory, k, "laptop", h, h.At); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
list, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range list {
|
||||
if c.Key != "machine.laptop.networkmanager.rewritten" || !strings.Contains(c.Summary, "(FortiClient)") {
|
||||
t.Fatalf("statement %d raised %s: %s", i, c.Key, c.Summary)
|
||||
}
|
||||
raisedAt = append(raisedAt, i)
|
||||
}
|
||||
}
|
||||
// Five statements: healthy, healthy, one failing look (still healthy), unhealthy, written back.
|
||||
if len(raisedAt) != 1 || raisedAt[0] != 3 {
|
||||
t.Fatalf("the rewrite was open after statements %v; want after the fourth alone", raisedAt)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A failed unit is never silent (novox/hq issue 315).
|
||||
//
|
||||
// **A module's failed unit was never raised, and neither was the machine's.** Liveness judges what a
|
||||
// module runs long-lived; a module whose daemon is a package's unit, started by D-Bus activation, states
|
||||
// no service, and its unit failed at every start while the machine read healthy. The profile's
|
||||
// `service-manager` said `degraded`, and nothing raised that.
|
||||
//
|
||||
// The node-engine now reads every failed unit in the managers the mesh places units in, and says whose
|
||||
// each is:
|
||||
//
|
||||
// - **a module's** — the declaration states it, writes its file, or installs the package its file
|
||||
// belongs to — is among the module's resources, unhealthy, of kind `unit`: the module's own
|
||||
// `module.<module>.<machine>.unhealthy`, naming the unit, raised on the second statement and held by
|
||||
// the gate as any unhealthy resource is (ADR 0240 §4);
|
||||
// - **the machine's** — no module places it: a mount of the machine's own table, a unit a removed
|
||||
// package left behind — is one finding for the machine, `machine.<m>.units`, listing them, so a
|
||||
// degraded service manager is never silent. A warning; cleared on the first statement that lists none.
|
||||
//
|
||||
// The engine applies the two-look rule itself (ADR 0241 §2), so the machine's finding is raised on the
|
||||
// statement that first says it. It is nothing a send did — no module places what it lists — so the gate
|
||||
// never holds a send on it. An engine older than this reading says nothing of its units, and nothing is
|
||||
// raised or cleared for it.
|
||||
|
||||
// The condition a machine's own failed units raise.
|
||||
const (
|
||||
kindMachineUnits = "machine-units"
|
||||
sourceUnits = "units"
|
||||
)
|
||||
|
||||
// unitsKept is a statement's units as the inventory keeps them.
|
||||
func unitsKept(u *link.UnitsHealth) *inventory.UnitsHealth {
|
||||
if u == nil {
|
||||
return nil
|
||||
}
|
||||
out := &inventory.UnitsHealth{State: u.State, Failed: []inventory.FailedUnit{}, Unread: u.Unread}
|
||||
for _, f := range u.Failed {
|
||||
out.Failed = append(out.Failed, inventory.FailedUnit{Unit: f.Unit, Scope: f.Scope, Load: f.Load, Result: f.Result,
|
||||
Resource: f.Resource, Since: f.Since})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// judgeUnits raises the machine's own failed units as one finding, or clears it when the statement lists
|
||||
// none. A statement that says nothing of units — an older engine — leaves it as it is.
|
||||
func judgeUnits(ctx context.Context, k *conditions.Keeper, node string, u *link.UnitsHealth) error {
|
||||
if k == nil || u == nil {
|
||||
return nil
|
||||
}
|
||||
o, raise := machineUnitsObservation(node, u)
|
||||
if raise {
|
||||
_, err := k.Observe(ctx, o)
|
||||
return err
|
||||
}
|
||||
open, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, c := range open {
|
||||
if c.Kind == kindMachineUnits && c.Key == o.Key() {
|
||||
why := fmt.Sprintf("%s's service managers list no failed unit the mesh does not place", node)
|
||||
if u.State == link.UnitsRunning {
|
||||
why = fmt.Sprintf("%s's service managers are running, no unit failed", node)
|
||||
}
|
||||
_, err := k.Clear(ctx, c.Key, why)
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// machineUnitsObservation is the machine's own failed units in one finding, and whether there are any.
|
||||
// The summary names each unit and its manager, which is what a person looks for; how each failed and
|
||||
// since when is evidence. Pure.
|
||||
func machineUnitsObservation(node string, u *link.UnitsHealth) (conditions.Observation, bool) {
|
||||
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Token: "units", Kind: kindMachineUnits,
|
||||
Machine: node, Severity: conditions.Warning, Source: sourceUnits}
|
||||
if len(u.Failed) == 0 {
|
||||
return o, false
|
||||
}
|
||||
var names, said, plain []string
|
||||
for _, f := range u.Failed {
|
||||
plain = append(plain, unitPlainWords(f.Unit))
|
||||
where := ""
|
||||
if f.Scope == "user" {
|
||||
where = " (the account's own manager)"
|
||||
}
|
||||
names = append(names, f.Unit+where)
|
||||
line := fmt.Sprintf("%s in the %s manager: %s", f.Unit, orSystem(f.Scope), orNotSaid(f.Result))
|
||||
if f.Load != "" && f.Load != "loaded" {
|
||||
line += ", its unit " + f.Load
|
||||
}
|
||||
if f.Resource != "" {
|
||||
line += ", named by the mesh's own " + f.Resource
|
||||
}
|
||||
line += ", since " + f.Since.UTC().Format("2006-01-02 15:04:05 MST")
|
||||
said = append(said, line)
|
||||
}
|
||||
o.Summary = fmt.Sprintf("%s's service manager is degraded: %d failed unit(s) no module places — %s. "+
|
||||
"Each is the machine's own: mend or remove it there, or have a module place it",
|
||||
node, len(u.Failed), strings.Join(names, ", "))
|
||||
o.Said = strings.Join(said, "; ")
|
||||
failed := "1 failed service"
|
||||
if len(u.Failed) > 1 {
|
||||
failed = fmt.Sprintf("%d failed services", len(u.Failed))
|
||||
}
|
||||
o.Headline = fmt.Sprintf("%s on %s", conditions.Capital(failed), node)
|
||||
o.Needs = fmt.Sprintf("mend or remove them on %s, or silence this if they do not matter.", node)
|
||||
o.Explanation = fmt.Sprintf("On %s, %s failed. No module manages them, so the mesh does not repair them.",
|
||||
node, namesWords(plain, 3))
|
||||
o.Actions = []conditions.Action{conditions.SilenceAction(o.Key())}
|
||||
o.Resolved = "No failed services on " + node + " any more"
|
||||
return o, true
|
||||
}
|
||||
|
||||
func orSystem(scope string) string {
|
||||
if scope == "" {
|
||||
return "system"
|
||||
}
|
||||
return scope
|
||||
}
|
||||
|
||||
// unitsLines is what `node show` says of a machine's service managers.
|
||||
func unitsLines(h inventory.NodeHealth, had bool, _ time.Time) []string {
|
||||
if !had || h.Units == nil {
|
||||
return []string{" its node-engine does not say which units failed — it is older than that reading (issue 315)"}
|
||||
}
|
||||
out := []string{" its service managers: " + h.Units.State}
|
||||
for _, f := range h.Units.Failed {
|
||||
line := fmt.Sprintf(" failed %s (%s) — no module places it", f.Unit, orSystem(f.Scope))
|
||||
if f.Result != "" {
|
||||
line += ", " + f.Result
|
||||
}
|
||||
out = append(out, line)
|
||||
}
|
||||
for _, r := range h.Resources {
|
||||
if r.Kind == link.KindUnit {
|
||||
out = append(out, fmt.Sprintf(" failed %s — %s's, %s", r.Target, r.Module, r.Reason))
|
||||
}
|
||||
}
|
||||
for _, w := range h.Units.Unread {
|
||||
out = append(out, " unread "+w)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A failed unit is never silent (novox/hq issue 315, "how it is checked"): a module's failed unit raises
|
||||
// the module's condition on the second statement, naming the unit; the machine's own failed units raise
|
||||
// one finding for the machine, listing them, cleared when none is listed; an engine that says nothing of
|
||||
// its units raises and clears nothing; and the gate never holds a send on the machine's own.
|
||||
|
||||
// desktop is the workstation as found on 2026-10-08, with the mesh's names: the Razer daemon's packaged
|
||||
// user unit, the module's; two mounts of the machine's own table and a unit a removed package left
|
||||
// behind, the machine's.
|
||||
func desktop(at time.Time) link.Health {
|
||||
return link.Health{Contract: link.ReadinessContract, At: at,
|
||||
Resources: []link.ResourceHealth{{Module: "openrazer", Resource: "openrazer.daemon", Kind: link.KindUnit,
|
||||
Target: "openrazer-daemon.service", State: link.StateUnhealthy,
|
||||
Reason: "failed in the account's own service manager (exit-code)", Since: h0, Streak: 2}},
|
||||
Units: &link.UnitsHealth{State: link.UnitsDegraded, Failed: []link.FailedUnit{
|
||||
{Unit: "mnt-recalbox.mount", Scope: "system", Load: "loaded", Result: "exit-code", Since: h0},
|
||||
{Unit: "storage-media.mount", Scope: "system", Load: "loaded", Result: "timeout", Since: h0},
|
||||
{Unit: "greenclip.service", Scope: "user", Load: "not-found", Result: "start-limit-hit", Since: h0},
|
||||
}}}
|
||||
}
|
||||
|
||||
func TestAFailedUnitIsTheModulesConditionAndTheMachinesOwnAreOneFinding(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv, k := open.inventory, conditionsFrom
|
||||
openConditions := func() map[string]conditions.Condition {
|
||||
t.Helper()
|
||||
list, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := map[string]conditions.Condition{}
|
||||
for _, c := range list {
|
||||
out[c.Key] = c
|
||||
}
|
||||
return out
|
||||
}
|
||||
say := func(at time.Time, h link.Health) {
|
||||
t.Helper()
|
||||
if err := stateHealth(ctx, inv, k, "laptop", h, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
say(h0, desktop(h0))
|
||||
got := openConditions()
|
||||
machine, raised := got["machine.laptop.units"]
|
||||
if !raised || len(got) != 1 {
|
||||
t.Fatalf("the machine's own failed units are one finding at once (the engine looked twice): %v", got)
|
||||
}
|
||||
for _, unit := range []string{"mnt-recalbox.mount", "storage-media.mount", "greenclip.service"} {
|
||||
if !strings.Contains(machine.Summary, unit) {
|
||||
t.Errorf("the finding names %s: %s", unit, machine.Summary)
|
||||
}
|
||||
}
|
||||
if strings.Contains(machine.Summary, "openrazer") || machine.Severity != conditions.Warning {
|
||||
t.Errorf("the module's unit is the module's, and the machine's a warning: %+v", machine)
|
||||
}
|
||||
|
||||
say(h0.Add(time.Minute), desktop(h0.Add(time.Minute)))
|
||||
got = openConditions()
|
||||
module, raised := got["module.openrazer.laptop.unhealthy"]
|
||||
if !raised || !strings.Contains(module.Summary, "openrazer-daemon.service") ||
|
||||
!strings.Contains(module.Summary, "account's own service manager") {
|
||||
t.Fatalf("the module's failed unit raises its condition on the second statement, naming the unit: %+v", got)
|
||||
}
|
||||
|
||||
kept, had, err := inv.HealthOf(ctx, "laptop")
|
||||
if err != nil || !had || kept.Units == nil || len(kept.Units.Failed) != 3 {
|
||||
t.Fatalf("the statement's units were not kept: %+v %v", kept.Units, err)
|
||||
}
|
||||
lines := strings.Join(unitsLines(kept, had, h0), "\n")
|
||||
for _, want := range []string{"degraded", "storage-media.mount (system) — no module places it",
|
||||
"openrazer-daemon.service — openrazer's"} {
|
||||
if !strings.Contains(lines, want) {
|
||||
t.Errorf("node show does not say %q:\n%s", want, lines)
|
||||
}
|
||||
}
|
||||
|
||||
// An older engine says nothing of its units: nothing cleared on its word.
|
||||
older := desktop(h0.Add(2 * time.Minute))
|
||||
older.Units = nil
|
||||
say(h0.Add(2*time.Minute), older)
|
||||
if _, still := openConditions()["machine.laptop.units"]; !still {
|
||||
t.Fatal("a statement saying nothing of units cleared the machine's finding")
|
||||
}
|
||||
|
||||
// The operator mends the mounts and removes the leftover link; the module's unit still fails.
|
||||
mended := desktop(h0.Add(3 * time.Minute))
|
||||
mended.Units.Failed = []link.FailedUnit{}
|
||||
say(h0.Add(3*time.Minute), mended)
|
||||
got = openConditions()
|
||||
if _, still := got["machine.laptop.units"]; still {
|
||||
t.Fatalf("no failed unit of the machine's own, still open: %v", got)
|
||||
}
|
||||
if _, still := got["module.openrazer.laptop.unhealthy"]; !still {
|
||||
t.Fatalf("the module's unit still fails, and its condition stands: %v", got)
|
||||
}
|
||||
|
||||
// And the module's unit mended: running, nothing open.
|
||||
well := link.Health{Contract: link.ReadinessContract, At: h0.Add(4 * time.Minute),
|
||||
Units: &link.UnitsHealth{State: link.UnitsRunning, Failed: []link.FailedUnit{}}}
|
||||
say(h0.Add(4*time.Minute), well)
|
||||
if got = openConditions(); len(got) != 0 {
|
||||
t.Fatalf("a running service manager leaves nothing open: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheGateNeverHoldsASendOnTheMachinesOwnUnits(t *testing.T) {
|
||||
since := h0
|
||||
units := conditions.Condition{Key: "machine.laptop.units", Kind: kindMachineUnits,
|
||||
Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop", Machine: "laptop"},
|
||||
Summary: "laptop's service manager is degraded", Raised: since.Add(time.Minute), Source: sourceUnits}
|
||||
if w := aboutTheMachine("laptop", []string{"openrazer"}, since, gateFacts{judged: true,
|
||||
open: []conditions.Condition{units}}); w.whole != "" || w.waiting != "" || len(w.on) != 0 {
|
||||
t.Fatalf("the machine's own failed units held a send: %+v", w)
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/identity"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/licences"
|
||||
@@ -53,6 +54,9 @@ func run() error {
|
||||
return fmt.Errorf("no command given")
|
||||
}
|
||||
|
||||
// Every connection this process dials says what it is (novox/hq issue 327).
|
||||
broker.ConnectionName = connectionName(args[0], os.Getenv(verbVar))
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
// Whatever this process holds of the controller's lease is given back as it ends (novox/hq to-be
|
||||
@@ -93,6 +97,15 @@ func run() error {
|
||||
return pauseCommand(ctx, args[0], args[1:])
|
||||
case "collection":
|
||||
return collectionCommand(ctx, args[1:])
|
||||
// What the records say the registries may keep, asked on demand (novox/hq ADR 0251).
|
||||
case "artifacts":
|
||||
return artifactsCommand(ctx, args[1:])
|
||||
case "collect":
|
||||
return collectCommand(ctx, args[1:])
|
||||
case "images":
|
||||
return imagesCommand(ctx, args[1:])
|
||||
case "mirrors":
|
||||
return mirrorsCommand(ctx, args[1:])
|
||||
case "plans":
|
||||
return plansCommand(ctx, args[1:])
|
||||
case "delivery":
|
||||
@@ -123,7 +136,7 @@ func run() error {
|
||||
// The merge gate: every machine of the snapshot composed with a change (novox/hq to-be 45 §9).
|
||||
case "merge-gate":
|
||||
return mergeGateCommand(ctx, args[1:])
|
||||
// A pull request's merge check run here exactly as the build seat runs it (novox/hq issue 283).
|
||||
// A pull request's merge check run here exactly as the build seat runs it (novox/hq issue 286).
|
||||
case "check-here":
|
||||
return checkHereCommand(ctx, args[1:])
|
||||
case "upgrade":
|
||||
@@ -203,6 +216,10 @@ func run() error {
|
||||
func usage() {
|
||||
fmt.Fprint(os.Stderr, `mesh-controller — the control plane
|
||||
|
||||
On a node the operator types these as 'mesh-cli <command>' (in zsh, 'nox <command>'): asked
|
||||
through the node's engine, and run as the controller's terminal only on the control-node
|
||||
(novox/hq ADR 0272).
|
||||
|
||||
migrate bring each context's schema up to date
|
||||
prepare the same, asked the way the mesh asks any module (ADR 0135)
|
||||
node add <name> [--adopted] create a node record; --adopted: the machine is in use
|
||||
@@ -240,6 +257,10 @@ func usage() {
|
||||
cleanup [list] [--json] every retired consumer per provider: age, size, why
|
||||
cleanup delete <node> <module> <consumer> --why <text> the provider deletes that one retired consumer
|
||||
cleanup delete --older-than <days> --why <text> [--confirm] list those older; delete only with --confirm
|
||||
artifacts [--json] [--repository <r>] [--collected] every artifact the mesh made: kept and why, eligible, collected (ADR 0251)
|
||||
collect [--json] [--most <n>] what the store's sweep would let go of, now; nothing is done
|
||||
collect --confirm --why <text> [--most <n>] run the sweep now: hold every kept archive, let go of the eligible
|
||||
images <node> [--json] every container image the machine's declaration names, now and as last sent
|
||||
seats [--json] every seat this mesh defines, what it delivers, and who holds it
|
||||
seat rename <from> <to> rename a seat; its former name still resolves (ADR 0122)
|
||||
seat <name> --to <node>/<module> hand a seat to that assignment as one act; never empty in between (ADR 0131)
|
||||
@@ -358,6 +379,15 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
||||
return nil
|
||||
}
|
||||
manifest, _, err := takeIn(ctx, b.inv, result)
|
||||
// The assignments pending on this build, made or ended (novox/hq issue 325), said in the daemon's log
|
||||
// after what became of the build.
|
||||
if b.open != nil {
|
||||
defer func() {
|
||||
for _, line := range settlePendingOnBuild(ctx, b.open, result, manifest.Module, err) {
|
||||
fmt.Printf("%s: %s\n", result.ID, line)
|
||||
}
|
||||
}()
|
||||
}
|
||||
// When it was asked, so a plan takes as its outcome only a build asked for it or after it
|
||||
// (novox/hq 04-ISSUES/219). Zero when the id does not say.
|
||||
asked, _ := link.BuildAskedAt(result.ID)
|
||||
@@ -394,3 +424,18 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
||||
statusFrom.nudge()
|
||||
return nil
|
||||
}
|
||||
|
||||
// verbVar carries the seat verb a command runs for, from the serving controller to the process it starts.
|
||||
const verbVar = "MESH_VERB"
|
||||
|
||||
// connectionName is what this process's connections say they are in the bus's list (novox/hq issue 327):
|
||||
// the serving controller, a verb's own process and which verb, or a command run at a shell and which.
|
||||
func connectionName(command, verb string) string {
|
||||
switch {
|
||||
case command == "serve":
|
||||
return "mesh-controller serving"
|
||||
case verb != "":
|
||||
return "mesh-controller verb " + verb
|
||||
}
|
||||
return "mesh-controller command " + command
|
||||
}
|
||||
|
||||
@@ -108,7 +108,7 @@ type mergeVerdict struct {
|
||||
Facts time.Time `json:"facts"`
|
||||
Failures []string `json:"failures,omitempty"`
|
||||
// Errors are what kept the gate from judging: a machine the mesh composes that the gate could not
|
||||
// raise as it is. Any one makes the verdict an error — never a pass (novox/hq issue 282).
|
||||
// raise as it is. Any one makes the verdict an error — never a pass (novox/hq issue 285).
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
Warnings []string `json:"warnings,omitempty"`
|
||||
Notes []string `json:"notes,omitempty"`
|
||||
@@ -135,6 +135,7 @@ type mergeComposed struct {
|
||||
Problems []string `json:"problems,omitempty"`
|
||||
Withheld []string `json:"withheld,omitempty"`
|
||||
Unbound []string `json:"unbound,omitempty"`
|
||||
Unplaced []string `json:"unplaced,omitempty"`
|
||||
LeftOut map[string]string `json:"left-out,omitempty"`
|
||||
Resources []string `json:"resources,omitempty"`
|
||||
}
|
||||
@@ -222,7 +223,7 @@ func mergeGateCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
|
||||
// errMergeGateCouldNotJudge is the gate's own error: the mesh as it is could not be raised, so nothing
|
||||
// the change does to it can be seen. Never a pass (novox/hq issue 282).
|
||||
// the change does to it can be seen. Never a pass (novox/hq issue 285).
|
||||
var errMergeGateCouldNotJudge = errors.New("the merge gate could not judge the change")
|
||||
|
||||
// errMergeGateFailed is the gate's own failure: the verdict says why, so the error says nothing more.
|
||||
@@ -351,7 +352,7 @@ func judgeChange(ctx context.Context, in mergeCheckInput) (mergeVerdict, error)
|
||||
v.Failures = append(v.Failures, fmt.Sprintf("%s: nothing could be sent to it with this change — %s",
|
||||
gm.Described, firstOr(newOnly(gm.Change.Problems, gm.Base.Problems), gm.Change.Problems)))
|
||||
case !gm.Base.Composes && m.Declaration.Composes:
|
||||
// **The mesh composes it and the gate could not raise it as it is** (novox/hq issue 282): a fact
|
||||
// **The mesh composes it and the gate could not raise it as it is** (novox/hq issue 285): a fact
|
||||
// the snapshot does not carry, or one the gate does not raise. Then the change is judged against a
|
||||
// machine that is not the mesh's — broken against broken, which passes whatever the change does —
|
||||
// so the gate cannot judge, and says so: an error, never a pass.
|
||||
@@ -372,6 +373,12 @@ func judgeChange(ctx context.Context, in mergeCheckInput) (mergeVerdict, error)
|
||||
v.Failures = append(v.Failures, fmt.Sprintf("%s: the change leaves a credential bound elsewhere — %s",
|
||||
gm.Described, u))
|
||||
}
|
||||
// A contribution its holder's template renders nothing for (novox/hq ADR 0255): the machine
|
||||
// would be sent without it, so a change that adds one is refused, naming it.
|
||||
for _, u := range newOnly(gm.Change.Unplaced, gm.Base.Unplaced) {
|
||||
v.Failures = append(v.Failures, fmt.Sprintf("%s: the change leaves a contribution unplaced — %s",
|
||||
gm.Described, u))
|
||||
}
|
||||
for module, why := range gm.Change.LeftOut {
|
||||
if _, was := gm.Base.LeftOut[module]; !was {
|
||||
v.Failures = append(v.Failures, fmt.Sprintf("%s: the change leaves %s out of its declaration — %s",
|
||||
@@ -780,6 +787,7 @@ func composeEveryMachine(ctx context.Context, in mergeCheckInput, shelf map[stri
|
||||
for _, u := range declared.unbound {
|
||||
c.Unbound = append(c.Unbound, u.String())
|
||||
}
|
||||
c.Unplaced = declared.unplaced
|
||||
sort.Strings(c.Withheld)
|
||||
sort.Strings(c.Unbound)
|
||||
out[m.Name] = c
|
||||
@@ -1005,6 +1013,11 @@ func raiseFromFacts(ctx context.Context, open *stores, f snapshot.Facts, shelf m
|
||||
return notes, err
|
||||
}
|
||||
}
|
||||
if m.AgentAccount != "" {
|
||||
if err := inv.SetAgentAccount(ctx, m.Name, m.AgentAccount, m.AgentAccountHome); err != nil {
|
||||
return notes, err
|
||||
}
|
||||
}
|
||||
if m.PublicDomain != "" {
|
||||
if err := inv.SetPublicDomain(ctx, m.Name, m.PublicDomain); err != nil {
|
||||
return notes, err
|
||||
|
||||
@@ -323,11 +323,11 @@ func busMesh(t *testing.T) snapshot.Facts {
|
||||
return f
|
||||
}
|
||||
|
||||
// **Issue 282**: every machine running a module on the bus failed to compose in the gate's store, with
|
||||
// **Issue 285**: every machine running a module on the bus failed to compose in the gate's store, with
|
||||
// the change and without — the store held the module's credential and no account for it, which
|
||||
// composition refuses (issue 203) — and the gate passed every change, "0 of 4 compose". The account the
|
||||
// mesh issued is raised with its credential, so the machine composes in the gate as on the mesh.
|
||||
func TestIssue282AModuleOnTheBusComposesInTheGate(t *testing.T) {
|
||||
func TestIssue285AModuleOnTheBusComposesInTheGate(t *testing.T) {
|
||||
f := busMesh(t)
|
||||
v := gateJudged(t, f, "")
|
||||
if v.Verdict != "pass" {
|
||||
@@ -343,10 +343,10 @@ func TestIssue282AModuleOnTheBusComposesInTheGate(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// **Issue 282**: a machine the mesh composes that the gate cannot raise as it is leaves the change judged
|
||||
// **Issue 285**: a machine the mesh composes that the gate cannot raise as it is leaves the change judged
|
||||
// against a machine that is not the mesh's — broken against broken, which passes whatever the change does.
|
||||
// That is an error, never a pass.
|
||||
func TestIssue282AMachineTheGateCannotRaiseIsAnErrorNeverAPass(t *testing.T) {
|
||||
func TestIssue285AMachineTheGateCannotRaiseIsAnErrorNeverAPass(t *testing.T) {
|
||||
f := busMesh(t)
|
||||
for i := range f.Machines {
|
||||
// A fact the snapshot does not carry: the module's credential, gone from the laptop's.
|
||||
@@ -386,7 +386,7 @@ func TestAWithheldPathIsStoodInForByAPath(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// **Issue 283**: a check run by hand clones beside a change what the seat clones — one rule, read by the
|
||||
// **Issue 286**: a check run by hand clones beside a change what the seat clones — one rule, read by the
|
||||
// controller's ask and by the facts — and finds the repository it checks from its origin.
|
||||
func TestACheckByHandClonesWhatTheSeatClones(t *testing.T) {
|
||||
for dir, refs := range map[string]map[string]string{
|
||||
|
||||
@@ -0,0 +1,261 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The operator's command, `mesh-cli` (alias `nox`), answered here (novox/hq ADR 0272).
|
||||
//
|
||||
// mesh-cli asks the node-engine on its own machine; the engine reads the asking account from the kernel and asks
|
||||
// this controller on its own node's subject. Here it is judged — the controller's terminal, an ordinary call, or
|
||||
// nothing — and run as every verb's line is: a fresh process of this binary, with this process's environment.
|
||||
//
|
||||
// **The terminal** is a line from a node's operator account, on the control-node (§4). Phase 2 adds a node whose
|
||||
// agents run under an account of their own, judged unable to become root (ADR 0266, not built yet); until then no
|
||||
// other node is the terminal, because agents there run as its operator. **An ordinary call** is a line from that
|
||||
// account elsewhere: it meets every refusal of the generic `command` verb and runs with `MESH_VERB=mesh-cli`, so a
|
||||
// terminal-only change is refused with its reason. **Any other account is refused**, root included: those two
|
||||
// accounts already reach the mesh's verbs through the mesh MCP server, and no other account gains anything here.
|
||||
|
||||
// cliVerb is what a line from mesh-cli that is not the terminal runs as: the verb its process names, so every
|
||||
// terminal-only refusal applies and says it came through mesh-cli.
|
||||
const cliVerb = "mesh-cli"
|
||||
|
||||
// cliServers are commands that serve until stopped. Run for mesh-cli they would hold a second server under this
|
||||
// one until the call's bound killed it.
|
||||
var cliServers = map[string]bool{"serve": true, "api": true, "board": true}
|
||||
|
||||
// cliVerdict is how a line is run, or why it is not.
|
||||
type cliVerdict struct {
|
||||
terminal bool
|
||||
// why says why the line is or is not the terminal, in words the operator reads under the answer.
|
||||
why string
|
||||
// refused says why nothing runs.
|
||||
refused string
|
||||
}
|
||||
|
||||
// judgeCLI decides how a line from mesh-cli runs (ADR 0272 §4). nodes is every node the mesh knows; control is
|
||||
// every node the controller's module is assigned to, which is exactly one in a mesh that is well.
|
||||
func judgeCLI(node string, asked link.CLIAsked, nodes []inventory.Node, control []string) cliVerdict {
|
||||
var record *inventory.Node
|
||||
for i := range nodes {
|
||||
if nodes[i].Name == node {
|
||||
record = &nodes[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case record == nil:
|
||||
return cliVerdict{refused: fmt.Sprintf("%s is not a node this mesh knows, so nothing ran", node)}
|
||||
case asked.UID == 0 || asked.Account == "root":
|
||||
return cliVerdict{refused: "mesh-cli answers the operator's own account, never root: run it as yourself, " +
|
||||
"not under sudo. Nothing ran"}
|
||||
case record.Account == "":
|
||||
return cliVerdict{refused: fmt.Sprintf("the mesh does not know %s's operator account (`node account <node> <login>`), "+
|
||||
"so no account there is answered. Nothing ran", node)}
|
||||
case asked.Account != record.Account:
|
||||
return cliVerdict{refused: fmt.Sprintf("mesh-cli answers %s's operator account (%s) only, and this line came "+
|
||||
"from %s. Nothing ran", node, record.Account, asked.Account)}
|
||||
}
|
||||
if len(control) != 1 {
|
||||
return cliVerdict{why: fmt.Sprintf("not the controller's terminal: the mesh names %d control-nodes, and the "+
|
||||
"terminal is the one control-node's operator account", len(control))}
|
||||
}
|
||||
if control[0] == node {
|
||||
// **A person at a terminal, not a service of the operator's** (review of ADR 0272): the tool runner and the
|
||||
// account's user units run as the operator too, and only a login session is the terminal.
|
||||
if asked.Session == "" {
|
||||
return cliVerdict{why: fmt.Sprintf("not the controller's terminal: %s on %s asked from no login session — a "+
|
||||
"service or a user unit running as the operator, not a person at a terminal", asked.Account, node)}
|
||||
}
|
||||
return cliVerdict{terminal: true, why: fmt.Sprintf("the controller's terminal: %s on the control-node %s, "+
|
||||
"login session %s", asked.Account, node, asked.Session)}
|
||||
}
|
||||
return cliVerdict{why: fmt.Sprintf("not the controller's terminal: agents on %s may run as %s, so a "+
|
||||
"terminal-only change is made from the control-node %s (novox/hq ADR 0272)", node, asked.Account, control[0])}
|
||||
}
|
||||
|
||||
// controlNodes is every node the controller's module is assigned to.
|
||||
func controlNodes(ctx context.Context, inv *inventory.Inventory, nodes []inventory.Node) ([]string, error) {
|
||||
var out []string
|
||||
for _, n := range nodes {
|
||||
modules, err := inv.Assigned(ctx, n.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if slices.Contains(modules, controllerModule) {
|
||||
out = append(out, n.Name)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// controllerModule is the module that runs the controller, and so marks the control-node.
|
||||
const controllerModule = "mesh-controller"
|
||||
|
||||
// answerMeshCLI is the serving controller's answer to mesh-cli.
|
||||
func answerMeshCLI(inv *inventory.Inventory) link.CLIHandler {
|
||||
return func(ctx context.Context, node string, asked link.CLIAsked) link.CLIAnswer {
|
||||
if handingOver.Load() {
|
||||
return link.CLIRefusal("this controller is stopping and runs no new command; ask again in a moment. Nothing ran")
|
||||
}
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return link.CLIRefusal("the mesh's nodes cannot be read, so nothing ran: " + err.Error())
|
||||
}
|
||||
control, err := controlNodes(ctx, inv, nodes)
|
||||
if err != nil {
|
||||
return link.CLIRefusal("which node is the control-node cannot be read, so nothing ran: " + err.Error())
|
||||
}
|
||||
return runForMeshCLI(ctx, node, asked, judgeCLI(node, asked, nodes, control))
|
||||
}
|
||||
}
|
||||
|
||||
// cliJournal says one line in the controller's journal (its standard output); a variable so a test can read it.
|
||||
var cliJournal = func(line string) { fmt.Println(line) }
|
||||
|
||||
// runForMeshCLI runs a judged line and answers what it said. Every line is said in the journal first: its call,
|
||||
// who asked on which node, its command word only, and how it runs (review of ADR 0272).
|
||||
func runForMeshCLI(ctx context.Context, node string, asked link.CLIAsked, v cliVerdict) link.CLIAnswer {
|
||||
how := "as an ordinary call"
|
||||
switch {
|
||||
case v.refused != "":
|
||||
how = "refused: " + v.refused
|
||||
case v.terminal:
|
||||
how = "as the controller's terminal"
|
||||
}
|
||||
call := link.CallIDIn(ctx)
|
||||
if call == "" {
|
||||
call = "(no call)"
|
||||
}
|
||||
cliJournal(fmt.Sprintf("mesh-cli %s: %s on %s asked %q, %s", call, asked.Account, node, asked.Line[0], how))
|
||||
if v.refused != "" {
|
||||
return link.CLIRefusal(v.refused)
|
||||
}
|
||||
if cliServers[asked.Line[0]] {
|
||||
return link.CLIAnswer{Exit: 1, Why: v.why, Refused: fmt.Sprintf("%s serves until stopped, and is not a "+
|
||||
"command line mesh-cli runs. Nothing ran", asked.Line[0])}
|
||||
}
|
||||
verb, line := "", asked.Line
|
||||
if !v.terminal {
|
||||
composed, err := ordinaryLine(asked.Line)
|
||||
if err != nil {
|
||||
return link.CLIAnswer{Exit: 1, Why: v.why, Refused: err.Error()}
|
||||
}
|
||||
verb, line = cliVerb, composed
|
||||
}
|
||||
cmd := selfCommand(ctx, line)
|
||||
cmd.Env = commandEnvironment(fmt.Sprintf("%s through mesh-cli on %s", asked.Account, node), verb, v.terminal)
|
||||
// No standard input: a command that reads one gets nothing, and fails saying so (ADR 0272 §5).
|
||||
cmd.Stdin = nil
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout, cmd.Stderr = &stdout, &stderr
|
||||
err := cmd.Run()
|
||||
answer := link.CLIAnswer{Stdout: stdout.Bytes(), Stderr: stderr.Bytes(), Terminal: v.terminal, Why: v.why}
|
||||
var exit *exec.ExitError
|
||||
switch {
|
||||
case err == nil:
|
||||
case errors.As(err, &exit):
|
||||
answer.Exit = exit.ExitCode()
|
||||
if answer.Exit < 0 {
|
||||
// Killed: by the call's bound, or by this controller stopping.
|
||||
answer.Exit = 1
|
||||
answer.Stderr = append(answer.Stderr, []byte(fmt.Sprintf("\nmesh-cli: the command was stopped (%v)\n",
|
||||
exit))...)
|
||||
}
|
||||
default:
|
||||
answer.Exit = 1
|
||||
answer.Refused = fmt.Sprintf("could not run %s from this controller's own build: %v", strings.Join(asked.Line, " "), err)
|
||||
}
|
||||
return answer
|
||||
}
|
||||
|
||||
// settingsForms is what an ordinary `settings` line may say: the settings verb's own forms.
|
||||
const settingsForms = "settings set <module> <values> [--replace] [--node <node>], settings clear <module> " +
|
||||
"[--node <node>], settings show <module> [--history] [--node <node>], or settings preferences [<module>] " +
|
||||
"[--node <node>]"
|
||||
|
||||
// ordinaryLine is the command line an ordinary call runs (ADR 0272 §4): a `settings` line composed exactly as the
|
||||
// settings verb composes its own, so its refusals — the terminal-only keys among them — are that verb's (review of
|
||||
// ADR 0272); any other line as the generic `command` verb takes it, with its refusals.
|
||||
func ordinaryLine(argv []string) ([]string, error) {
|
||||
if len(argv) == 0 || argv[0] != "settings" {
|
||||
// What the generic verb runs, and nothing it would refuse: its reading forms only, and never a command that
|
||||
// is the terminal's alone (novox/hq ADR 0266) — the two checks argvFor makes of the `command` verb's line,
|
||||
// made of the words as given rather than re-split from one string.
|
||||
if err := refusedAsTheGenericCommand(argv); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := terminalOnly(argv); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
args := map[string]any{}
|
||||
var words []string
|
||||
rest := argv[1:]
|
||||
for i := 0; i < len(rest); i++ {
|
||||
w := rest[i]
|
||||
switch {
|
||||
case w == "--replace" || w == "-replace":
|
||||
args["replace"] = "true"
|
||||
case w == "--history" || w == "-history":
|
||||
args["history"] = "true"
|
||||
case w == "--node" || w == "-node":
|
||||
if i+1 >= len(rest) {
|
||||
return nil, fmt.Errorf("--node names no node; settings takes %s. Nothing ran", settingsForms)
|
||||
}
|
||||
i++
|
||||
args["node"] = rest[i]
|
||||
case strings.HasPrefix(w, "--node=") || strings.HasPrefix(w, "-node="):
|
||||
_, args["node"], _ = strings.Cut(w, "=")
|
||||
case strings.HasPrefix(w, "-"):
|
||||
return nil, fmt.Errorf("settings takes no %s through mesh-cli outside the terminal; it takes %s. "+
|
||||
"Nothing ran", w, settingsForms)
|
||||
default:
|
||||
words = append(words, w)
|
||||
}
|
||||
}
|
||||
wrong := fmt.Errorf("through mesh-cli outside the terminal, settings takes the settings verb's forms: %s. "+
|
||||
"Nothing ran", settingsForms)
|
||||
if len(words) == 0 {
|
||||
return nil, wrong
|
||||
}
|
||||
switch words[0] {
|
||||
case "set":
|
||||
if len(words) != 3 {
|
||||
return nil, wrong
|
||||
}
|
||||
args["module"], args["values"] = words[1], words[2]
|
||||
case "clear":
|
||||
if len(words) != 2 {
|
||||
return nil, wrong
|
||||
}
|
||||
args["module"], args["clear"] = words[1], "true"
|
||||
case "show":
|
||||
if len(words) != 2 {
|
||||
return nil, wrong
|
||||
}
|
||||
args["module"] = words[1]
|
||||
case "preferences":
|
||||
if len(words) > 2 {
|
||||
return nil, wrong
|
||||
}
|
||||
args["list"] = "preferences"
|
||||
if len(words) == 2 {
|
||||
args["module"] = words[1]
|
||||
}
|
||||
default:
|
||||
return nil, wrong
|
||||
}
|
||||
return argvFor("settings", args)
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// echoEnvironment makes the test binary, run as a command line, say the verb and caller it was given (TestMain).
|
||||
const echoEnvironment = "MESH_TEST_ECHO_ENVIRONMENT"
|
||||
|
||||
var cliNodes = []inventory.Node{
|
||||
{Name: "control", Account: "operator"},
|
||||
{Name: "laptop", Account: "operator"},
|
||||
{Name: "unnamed"},
|
||||
}
|
||||
|
||||
func asked(account string, uid uint32, line ...string) link.CLIAsked {
|
||||
return link.CLIAsked{Line: line, Account: account, UID: uid, Session: "session-1.scope"}
|
||||
}
|
||||
|
||||
// Who is the controller's terminal, and who is an ordinary call or refused (novox/hq ADR 0272 §4).
|
||||
func TestMeshCLIIsTheTerminalOnlyForTheControlNodesOperator(t *testing.T) {
|
||||
control := []string{"control"}
|
||||
cases := []struct {
|
||||
name, node string
|
||||
asked link.CLIAsked
|
||||
control []string
|
||||
terminal bool
|
||||
refused string
|
||||
why string
|
||||
}{
|
||||
{"the control-node's operator", "control", asked("operator", 1000, "status"), control, true, "", "the controller's terminal"},
|
||||
{"another node's operator", "laptop", asked("operator", 1000, "status"), control, false, "", "agents on laptop may run as operator"},
|
||||
{"another account", "control", asked("agent", 1001, "status"), control, false, "operator account (operator) only", ""},
|
||||
{"root", "control", asked("root", 0, "status"), control, false, "never root", ""},
|
||||
{"a node with no operator account", "unnamed", asked("operator", 1000, "status"), control, false, "does not know unnamed's operator account", ""},
|
||||
{"a node the mesh does not know", "elsewhere", asked("operator", 1000, "status"), control, false, "not a node this mesh knows", ""},
|
||||
{"two control-nodes", "control", asked("operator", 1000, "status"), []string{"control", "laptop"}, false, "", "2 control-nodes"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
v := judgeCLI(c.node, c.asked, cliNodes, c.control)
|
||||
if v.terminal != c.terminal {
|
||||
t.Errorf("%s: terminal %v, want %v (%+v)", c.name, v.terminal, c.terminal, v)
|
||||
}
|
||||
if c.refused == "" && v.refused != "" || c.refused != "" && !strings.Contains(v.refused, c.refused) {
|
||||
t.Errorf("%s: refused %q, want %q", c.name, v.refused, c.refused)
|
||||
}
|
||||
if c.why != "" && !strings.Contains(v.why, c.why) {
|
||||
t.Errorf("%s: why %q, want %q", c.name, v.why, c.why)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The terminal runs its line without MESH_VERB, even where this process carries one; an ordinary call names
|
||||
// mesh-cli; both record who asked through mesh-cli.
|
||||
func TestTheTerminalRunsWithoutAVerbAndAnOrdinaryCallNamesMeshCLI(t *testing.T) {
|
||||
t.Setenv(echoEnvironment, "1")
|
||||
t.Setenv("MESH_VERB", "leaked-from-the-serving-process")
|
||||
// The serving controller marks itself (ADR 0266); its terminal line for mesh-cli is still the terminal's.
|
||||
t.Setenv(servedVar, "1")
|
||||
ctx := context.Background()
|
||||
|
||||
a := runForMeshCLI(ctx, "control", asked("operator", 1000, "status"), cliVerdict{terminal: true, why: "the terminal"})
|
||||
if a.Exit != 0 || a.Refused != "" || !a.Terminal {
|
||||
t.Fatalf("the terminal's line did not run: %+v", a)
|
||||
}
|
||||
if got := string(a.Stdout); !strings.Contains(got, `verb=""`) || !strings.Contains(got, "operator through mesh-cli on control") ||
|
||||
!strings.Contains(got, "terminal=true") {
|
||||
t.Fatalf("the terminal's line ran with %s", got)
|
||||
}
|
||||
|
||||
a = runForMeshCLI(ctx, "laptop", asked("operator", 1000, "status"), cliVerdict{why: "not the terminal"})
|
||||
if a.Exit != 0 || a.Terminal || a.Why != "not the terminal" {
|
||||
t.Fatalf("an ordinary line did not run as one: %+v", a)
|
||||
}
|
||||
if got := string(a.Stdout); !strings.Contains(got, `verb="mesh-cli"`) || !strings.Contains(got, "terminal=false") {
|
||||
t.Fatalf("an ordinary line ran with %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// An ordinary call meets every refusal of the generic command verb, and nothing runs; a server is never run.
|
||||
func TestAnOrdinaryCallMeetsTheCommandVerbsRefusals(t *testing.T) {
|
||||
t.Setenv(echoEnvironment, "1")
|
||||
ctx := context.Background()
|
||||
ordinary := cliVerdict{why: "not the terminal"}
|
||||
a := runForMeshCLI(ctx, "laptop", asked("operator", 1000, "cleanup", "delete", "x"), ordinary)
|
||||
if a.Refused == "" || len(a.Stdout) != 0 || a.Exit != 1 || a.Why != "not the terminal" {
|
||||
t.Fatalf("a repair without --why ran as an ordinary call: %+v", a)
|
||||
}
|
||||
a = runForMeshCLI(ctx, "laptop", asked("operator", 1000, "settings", "set", "claude-code", "{}"), ordinary)
|
||||
if a.Refused != "" || !strings.Contains(string(a.Stdout), `verb="mesh-cli"`) {
|
||||
t.Fatalf("an ordinary settings set did not run through the settings verb's path with MESH_VERB set: %+v", a)
|
||||
}
|
||||
for _, server := range []string{"serve", "api", "board"} {
|
||||
a := runForMeshCLI(ctx, "control", asked("operator", 1000, server), cliVerdict{terminal: true})
|
||||
if a.Refused == "" || len(a.Stdout) != 0 {
|
||||
t.Fatalf("%s was run for mesh-cli: %+v", server, a)
|
||||
}
|
||||
}
|
||||
a = runForMeshCLI(ctx, "control", asked("agent", 1001, "status"), cliVerdict{refused: "agent is not answered"})
|
||||
if a.Refused != "agent is not answered" || len(a.Stdout) != 0 {
|
||||
t.Fatalf("a refused line ran: %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
// **Which node is the terminal does not follow a verb** (review of ADR 0272): the controller's module is assigned
|
||||
// and unassigned at the terminal alone, so no caller of `assign` can move the terminal to a node of its choosing.
|
||||
// Asked through the acts themselves, as the verbs ask them; refused before any store is touched (none is given).
|
||||
func TestTheControllersModuleIsMovedAtTheTerminalAlone(t *testing.T) {
|
||||
t.Setenv("MESH_VERB", "assign")
|
||||
ctx := context.Background()
|
||||
if _, err := assignWith(ctx, nil, "laptop", assignOptions{}, "zsh", "mesh-controller"); err == nil ||
|
||||
!strings.Contains(err.Error(), "terminal") {
|
||||
t.Fatalf("assigning the controller through a verb was not refused: %v", err)
|
||||
}
|
||||
if _, err := assign(ctx, nil, "laptop", "mesh-controller"); err == nil || !strings.Contains(err.Error(), "terminal") {
|
||||
t.Fatalf("assigning the controller through a verb was not refused: %v", err)
|
||||
}
|
||||
t.Setenv("MESH_VERB", "unassign")
|
||||
if _, err := unassign(ctx, nil, "control", "mesh-controller"); err == nil || !strings.Contains(err.Error(), "terminal") {
|
||||
t.Fatalf("unassigning the controller through a verb was not refused: %v", err)
|
||||
}
|
||||
// Another module through a verb, and the controller's at the terminal, are not refused for it.
|
||||
if err := refusedMovingTheController([]string{"zsh"}); err != nil {
|
||||
t.Fatalf("another module was refused: %v", err)
|
||||
}
|
||||
t.Setenv("MESH_VERB", "")
|
||||
if err := refusedMovingTheController([]string{"mesh-controller"}); err != nil {
|
||||
t.Fatalf("the terminal was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// An ordinary `settings set|clear` goes down the settings verb's own path: composed as that verb composes it, and
|
||||
// run with MESH_VERB set, so its refusals — the terminal-only keys among them — are the settings command's own,
|
||||
// not a blanket refusal of the generic command (review of ADR 0272).
|
||||
func TestAnOrdinarySettingsLineTakesTheSettingsVerbsPath(t *testing.T) {
|
||||
cases := []struct {
|
||||
line []string
|
||||
want string
|
||||
err string
|
||||
}{
|
||||
{[]string{"settings", "set", "zsh", `{"execute":"withhold"}`, "--node", "laptop"}, `settings set zsh {"execute":"withhold"} --node laptop`, ""},
|
||||
{[]string{"settings", "set", "zsh", "{}", "--replace"}, "settings set zsh {} --replace", ""},
|
||||
{[]string{"settings", "clear", "zsh", "--node", "laptop"}, "settings clear zsh --node laptop", ""},
|
||||
{[]string{"settings", "show", "zsh", "--history"}, "settings show zsh --history", ""},
|
||||
{[]string{"settings", "preferences"}, "settings preferences", ""},
|
||||
{[]string{"settings", "set", "zsh"}, "", "settings"},
|
||||
{[]string{"settings", "set", "zsh", "{}", "--sideways"}, "", "--sideways"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
argv, err := ordinaryLine(c.line)
|
||||
if c.err != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), c.err) {
|
||||
t.Errorf("%q: refused with %v, want %q", c.line, err, c.err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil || strings.Join(argv, " ") != c.want {
|
||||
t.Errorf("%q: composed %q (%v), want %q", c.line, argv, err, c.want)
|
||||
}
|
||||
}
|
||||
// Anything else still meets the generic command verb's refusals.
|
||||
if _, err := ordinaryLine([]string{"retire", "delete", "x"}); err == nil {
|
||||
t.Error("a repair composed as an ordinary line")
|
||||
}
|
||||
}
|
||||
|
||||
// Every line is said in the controller's journal, with its call, who asked where and how it ran — its command word
|
||||
// only, never the rest of the line (review of ADR 0272).
|
||||
func TestEveryMeshCLILineIsSaidInTheJournal(t *testing.T) {
|
||||
t.Setenv(echoEnvironment, "1")
|
||||
var said []string
|
||||
was := cliJournal
|
||||
cliJournal = func(line string) { said = append(said, line) }
|
||||
t.Cleanup(func() { cliJournal = was })
|
||||
ctx := link.WithCallID(context.Background(), "call-1")
|
||||
runForMeshCLI(ctx, "control", asked("operator", 1000, "settings", "set", "x", `{"password":"s3cret"}`),
|
||||
cliVerdict{terminal: true, why: "the terminal"})
|
||||
runForMeshCLI(ctx, "control", asked("agent", 1001, "status"), cliVerdict{refused: "agent is not answered"})
|
||||
all := strings.Join(said, "\n")
|
||||
if len(said) != 2 || !strings.Contains(all, "call-1") || !strings.Contains(all, "operator on control") ||
|
||||
!strings.Contains(all, "as the controller's terminal") || !strings.Contains(all, "refused") {
|
||||
t.Fatalf("the journal said %q", said)
|
||||
}
|
||||
if strings.Contains(all, "s3cret") {
|
||||
t.Fatalf("the journal carries the line's values: %q", said)
|
||||
}
|
||||
}
|
||||
|
||||
// **An ordinary line runs only what the generic command verb would** (review of ADR 0272, ADR 0266): its reading
|
||||
// forms, and never a command that is the terminal's alone. Each of these, from another node's operator, is refused
|
||||
// and runs nothing.
|
||||
func TestAnOrdinaryLineRunsNothingTheCommandVerbWouldRefuse(t *testing.T) {
|
||||
t.Setenv(echoEnvironment, "1")
|
||||
for _, line := range [][]string{
|
||||
{"node", "account", "control", "x"},
|
||||
{"node", "agent-account", "control", "x", "--clear"},
|
||||
{"token", "issue", "laptop"},
|
||||
{"secret", "export", "x"},
|
||||
{"operator", "key", "set", "x"},
|
||||
{"assign", "laptop", "zsh"},
|
||||
} {
|
||||
if _, err := ordinaryLine(line); err == nil {
|
||||
t.Errorf("%q composed as an ordinary line", line)
|
||||
}
|
||||
a := runForMeshCLI(context.Background(), "laptop", asked("operator", 1000, line...), cliVerdict{why: "not the terminal"})
|
||||
if a.Refused == "" || len(a.Stdout) != 0 {
|
||||
t.Errorf("%q ran as an ordinary line: %+v", line, a)
|
||||
}
|
||||
}
|
||||
if argv, err := ordinaryLine([]string{"status"}); err != nil || argv[0] != "status" {
|
||||
t.Fatalf("a read was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// **`calls` never shows a mesh-cli line's answer** (review of ADR 0272): the verb's own answer is read for a line
|
||||
// served over a bus, and neither the answer's text nor the base64 JSON writes its bytes in is there.
|
||||
func TestTheCallsVerbNeverShowsAMeshCLILinesAnswer(t *testing.T) {
|
||||
conn, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
stop, err := link.OverNATS{Conn: conn}.ServeCLI(func(context.Context, string, link.CLIAsked) link.CLIAnswer {
|
||||
return link.CLIAnswer{Stdout: []byte("s3cret-join")}
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
body, _ := json.Marshal(link.CLIAsked{Line: []string{"token", "issue", "x"}, Account: "operator"})
|
||||
msg, err := conn.Request(link.CLISubject("control"), body, 5*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(msg.Data), base64.StdEncoding.EncodeToString([]byte("s3cret-join"))) {
|
||||
t.Fatalf("the asker was not given its answer: %s", msg.Data)
|
||||
}
|
||||
recent, _ := link.Calls.Recent()
|
||||
var id string
|
||||
for _, c := range recent {
|
||||
if c.Seat == link.CLISeat {
|
||||
id = c.ID
|
||||
break
|
||||
}
|
||||
}
|
||||
shown, err := callsAnswer(link.Calls, id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said, _ := json.Marshal(shown)
|
||||
if strings.Contains(string(said), "s3cret-join") ||
|
||||
strings.Contains(string(said), base64.StdEncoding.EncodeToString([]byte("s3cret-join"))) {
|
||||
t.Fatalf("calls showed a mesh-cli line's answer: %s", said)
|
||||
}
|
||||
}
|
||||
|
||||
// **Only the terminal's line carries the terminal's mark** (review of ADR 0272): a mark the serving controller's
|
||||
// environment holds — leaked, or set by anything — is stripped from every other command line it runs, a verb's and
|
||||
// an ordinary mesh-cli line alike, so neither reads as the terminal.
|
||||
func TestTheTerminalsMarkIsStrippedFromEveryOtherLine(t *testing.T) {
|
||||
t.Setenv(echoEnvironment, "1")
|
||||
t.Setenv(cliTerminalVar, "1")
|
||||
t.Setenv(servedVar, "1")
|
||||
for _, env := range [][]string{commandEnvironment("someone", "status", false)} {
|
||||
for _, kv := range env {
|
||||
if strings.HasPrefix(kv, cliTerminalVar+"=") {
|
||||
t.Fatalf("a verb's line carries the terminal's mark: %s", kv)
|
||||
}
|
||||
}
|
||||
}
|
||||
a := runForMeshCLI(context.Background(), "laptop", asked("operator", 1000, "status"), cliVerdict{why: "not the terminal"})
|
||||
if got := string(a.Stdout); !strings.Contains(got, "terminal=false") || !strings.Contains(got, `verb="mesh-cli"`) {
|
||||
t.Fatalf("an ordinary line with the mark in the serving environment ran as %s", got)
|
||||
}
|
||||
a = runForMeshCLI(context.Background(), "control", asked("operator", 1000, "status"), cliVerdict{terminal: true})
|
||||
if got := string(a.Stdout); !strings.Contains(got, "terminal=true") {
|
||||
t.Fatalf("the terminal's line ran as %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **Only a login session is the terminal** (review of ADR 0272): the operator's account on the control-node, asking
|
||||
// from a service — the tool runner, a user unit — and not a login session, is an ordinary call.
|
||||
func TestOnlyALoginSessionIsTheTerminal(t *testing.T) {
|
||||
control := []string{"control"}
|
||||
v := judgeCLI("control", link.CLIAsked{Line: []string{"status"}, Account: "operator", UID: 1000}, cliNodes, control)
|
||||
if v.terminal || v.refused != "" || !strings.Contains(v.why, "login session") {
|
||||
t.Fatalf("a line from no login session reads %+v", v)
|
||||
}
|
||||
v = judgeCLI("control", link.CLIAsked{Line: []string{"status"}, Account: "operator", UID: 1000, Session: "session-3.scope"},
|
||||
cliNodes, control)
|
||||
if !v.terminal {
|
||||
t.Fatalf("a line from a login session on the control-node reads %+v", v)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The bases the mesh copied into its artifact store, and the copies no record names (novox/hq ADR 0257).
|
||||
//
|
||||
// A build that stands on an image published elsewhere copies it in first (ADR 0096, 0097). Since ADR 0257
|
||||
// each copy is recorded where it is made, and a copy is kept while a kept build stood on it. Copies made
|
||||
// before then by a build that failed — or before the records kept what a build stood on — are in the store
|
||||
// with no record naming them, and ADR 0189 §3 keeps the sweep away from anything no record names. This
|
||||
// verb is how a person says such a copy is the mesh's: it records it, and the sweep then decides as it
|
||||
// does for every other copy. It deletes nothing.
|
||||
|
||||
// mirrorRepository is a repository only the mirror writes: one image's repository, or a module's own
|
||||
// repository of a base from before ADR 0257.
|
||||
var mirrorRepository = regexp.MustCompile(`^(upstream/[a-z0-9][a-z0-9._/-]*|[a-z0-9][a-z0-9._-]*/on-[a-z0-9_]+)$`)
|
||||
|
||||
// mirrorReference reads a reference a person gave into its recorded form, refusing anything that is not
|
||||
// a manifest in a repository the mirror writes.
|
||||
func mirrorReference(given string) (string, error) {
|
||||
reference := catalogue.Recorded(strings.TrimSpace(given))
|
||||
path, ours := catalogue.InArtifactStore(reference)
|
||||
if !ours {
|
||||
return "", fmt.Errorf("%q is not a reference into the artifact store (artifact-store://<repository>@sha256:<hex>)", given)
|
||||
}
|
||||
repository, digest, ok := strings.Cut(path, "@sha256:")
|
||||
if !ok || len(digest) != 64 || strings.Trim(digest, "0123456789abcdef") != "" {
|
||||
return "", fmt.Errorf("%q names no manifest by digest", given)
|
||||
}
|
||||
if !mirrorRepository.MatchString(repository) {
|
||||
return "", fmt.Errorf("%q is in %s, which is not a repository the mirror writes "+
|
||||
"(upstream/<host>/<path>, or <module>/on-<argument>)", given, repository)
|
||||
}
|
||||
return reference, nil
|
||||
}
|
||||
|
||||
type mirrorEntry struct {
|
||||
Reference string `json:"reference"`
|
||||
State string `json:"state"`
|
||||
Why []string `json:"why,omitempty"`
|
||||
}
|
||||
|
||||
type mirrorsAnswer struct {
|
||||
DryRun bool `json:"dry_run,omitempty"`
|
||||
// Mirrors is every copy the records hold that is not yet let go of.
|
||||
Mirrors []mirrorEntry `json:"mirrors"`
|
||||
Counts struct {
|
||||
Kept int `json:"kept"`
|
||||
Eligible int `json:"eligible"`
|
||||
Collected int `json:"collected"`
|
||||
} `json:"counts"`
|
||||
// Recorded, AlreadyRecorded and Refused answer a record: what was (or, a dry run, would be)
|
||||
// recorded, what the records already held, and what was refused, with why.
|
||||
Recorded []string `json:"recorded,omitempty"`
|
||||
// Then says what recording does: a recorded copy is eligible, and the next sweep lets it go.
|
||||
Then string `json:"then,omitempty"`
|
||||
AlreadyRecorded []string `json:"already_recorded,omitempty"`
|
||||
Refused map[string]string `json:"refused,omitempty"`
|
||||
}
|
||||
|
||||
// mirrorsOf is the copies among the recorded states.
|
||||
func mirrorsOf(states []inventory.ArtifactState, mirrored map[string]bool) mirrorsAnswer {
|
||||
a := mirrorsAnswer{Mirrors: []mirrorEntry{}}
|
||||
for _, s := range states {
|
||||
if !mirrored[s.Reference] {
|
||||
continue
|
||||
}
|
||||
switch s.State {
|
||||
case inventory.ArtifactKept:
|
||||
a.Counts.Kept++
|
||||
case inventory.ArtifactEligible:
|
||||
a.Counts.Eligible++
|
||||
case inventory.ArtifactCollected:
|
||||
a.Counts.Collected++
|
||||
continue
|
||||
}
|
||||
a.Mirrors = append(a.Mirrors, mirrorEntry{Reference: s.Reference, State: s.State, Why: s.Why})
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// splitReferences reads references separated by spaces or commas.
|
||||
func splitReferences(given string) []string {
|
||||
return strings.FieldsFunc(given, func(r rune) bool { return r == ',' || r == ' ' || r == '\n' || r == '\t' })
|
||||
}
|
||||
|
||||
// recordMirrors decides, for each reference given, whether it may be recorded: in a repository the
|
||||
// mirror writes, not already recorded, and held by the store. A real run records those.
|
||||
func recordMirrors(ctx context.Context, inv *inventory.Inventory, store artifacts.Store, given []string,
|
||||
known map[string]bool, why string, real bool) (mirrorsAnswer, error) {
|
||||
a := mirrorsAnswer{DryRun: !real, Mirrors: []mirrorEntry{}, Refused: map[string]string{}}
|
||||
var record []string
|
||||
seen := map[string]bool{}
|
||||
for _, g := range given {
|
||||
reference, err := mirrorReference(g)
|
||||
if err != nil {
|
||||
a.Refused[g] = err.Error()
|
||||
continue
|
||||
}
|
||||
if seen[reference] {
|
||||
continue
|
||||
}
|
||||
seen[reference] = true
|
||||
if known[reference] {
|
||||
a.AlreadyRecorded = append(a.AlreadyRecorded, reference)
|
||||
continue
|
||||
}
|
||||
if store.Address == "" {
|
||||
a.Refused[g] = "this mesh has no artifact store on its network to ask whether it holds this"
|
||||
continue
|
||||
}
|
||||
held, err := store.HoldsManifest(ctx, reference)
|
||||
switch {
|
||||
case err != nil:
|
||||
a.Refused[g] = err.Error()
|
||||
continue
|
||||
case !held:
|
||||
a.Refused[g] = "the artifact store does not hold it"
|
||||
continue
|
||||
}
|
||||
record = append(record, reference)
|
||||
}
|
||||
a.Recorded = record
|
||||
if len(record) > 0 {
|
||||
verb := "would become"
|
||||
if real {
|
||||
verb = "became"
|
||||
}
|
||||
a.Then = fmt.Sprintf("%d cop%s %s eligible: the next sweep (after any build, or collect) lets each go "+
|
||||
"unless one of the five kept builds of a module the mesh holds stood on it", len(record),
|
||||
map[bool]string{true: "y", false: "ies"}[len(record) == 1], verb)
|
||||
}
|
||||
if real && len(record) > 0 {
|
||||
if err := inv.RecordMirrored(ctx, "", why, record); err != nil {
|
||||
return a, fmt.Errorf("recording %d copies: %w", len(record), err)
|
||||
}
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
func mirrorsCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("mirrors", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "answer as JSON")
|
||||
record := set.String("record", "", "references of copies no record names, separated by spaces or commas, to record as the mesh's")
|
||||
confirm := set.Bool("confirm", false, "record them, rather than only say what would be recorded")
|
||||
f := addHandActFlags(set)
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 0 {
|
||||
return errors.New("mirrors [--json] [--record <references> [--confirm --why <text>]]")
|
||||
}
|
||||
if *confirm {
|
||||
if strings.TrimSpace(*record) == "" {
|
||||
return errors.New("mirrors: --confirm records what --record names, and it names nothing. Nothing was done")
|
||||
}
|
||||
if err := f.require("mirrors"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
states, err := inv.Artifacts(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
known, err := inv.Mirrored(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var answer mirrorsAnswer
|
||||
if strings.TrimSpace(*record) == "" {
|
||||
answer = mirrorsOf(states, known)
|
||||
} else {
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
address, err := artifactStoreAddress(ctx, inv, shelf, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
answer, err = recordMirrors(ctx, inv, artifacts.Store{Address: address}, splitReferences(*record), known,
|
||||
strings.TrimSpace(*f.why), *confirm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The hand act is logged once the records say what it did, never before: an act that
|
||||
// failed half-way is not logged as done.
|
||||
if *confirm && len(answer.Recorded) > 0 {
|
||||
f.record(ctx, "mirrors", append([]string{"--record"}, answer.Recorded...))
|
||||
}
|
||||
}
|
||||
if *asJSON {
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
encoder.SetIndent("", " ")
|
||||
return encoder.Encode(answer)
|
||||
}
|
||||
if answer.DryRun && len(answer.Recorded) > 0 {
|
||||
fmt.Println("a dry run: nothing was recorded (--confirm --why <text> to record)")
|
||||
}
|
||||
if answer.Then != "" {
|
||||
fmt.Println(answer.Then)
|
||||
}
|
||||
for _, r := range answer.Recorded {
|
||||
fmt.Printf(" record %s\n", r)
|
||||
}
|
||||
for _, r := range answer.AlreadyRecorded {
|
||||
fmt.Printf(" already %s\n", r)
|
||||
}
|
||||
for g, why := range answer.Refused {
|
||||
fmt.Printf(" refused %s: %s\n", g, why)
|
||||
}
|
||||
for _, m := range answer.Mirrors {
|
||||
fmt.Printf(" %-9s %s %s\n", m.State, m.Reference, strings.Join(m.Why, ", "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,298 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The bases the mesh copied in, and recording a copy no record names (novox/hq ADR 0257).
|
||||
|
||||
func TestTheMirrorsVerbComposesItsCommandAndRefusesWhatItDoesNotTake(t *testing.T) {
|
||||
r := ref("route-proxy", "on-go_base", 1)
|
||||
for _, c := range []struct {
|
||||
args map[string]any
|
||||
want string
|
||||
}{
|
||||
{map[string]any{}, "mirrors --json"},
|
||||
{map[string]any{"record": r}, "mirrors --json --record " + r},
|
||||
{map[string]any{"record": r, "confirm": "true", "why": "copied before copies were recorded"},
|
||||
"mirrors --json --record " + r + " --confirm --why copied before copies were recorded"},
|
||||
} {
|
||||
argv, err := argvFor("mirrors", c.args)
|
||||
if err != nil || strings.Join(argv, " ") != c.want {
|
||||
t.Errorf("mirrors %v: %q %v, want %q", c.args, argv, err, c.want)
|
||||
}
|
||||
}
|
||||
for _, args := range []map[string]any{
|
||||
{"record": r, "confirm": "true"}, // recorded without a why
|
||||
{"record": r, "why": "x"}, // a why for a dry run
|
||||
{"confirm": "true", "why": "x"}, // nothing to record
|
||||
{"record": r, "confirm": "yes", "why": "x"}, // a switch is true or false
|
||||
{"delete": "true"},
|
||||
} {
|
||||
if argv, err := argvFor("mirrors", args); err == nil {
|
||||
t.Errorf("mirrors %v was composed as %q", args, argv)
|
||||
}
|
||||
}
|
||||
if repairingCommand([]string{"mirrors", "--json", "--record", r, "--confirm", "--why", "x"}) != "mirrors" {
|
||||
t.Error("recording a copy through the generic verb would go unrecorded")
|
||||
}
|
||||
if repairingCommand([]string{"mirrors", "--json", "--record", r}) != "" {
|
||||
t.Error("a dry run was taken for an act by hand")
|
||||
}
|
||||
if !personsDecision(link.HandAct{Verb: "mirrors"}) {
|
||||
t.Error("recording a copy would count toward a healer the mesh lacks")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyACopyInARepositoryTheMirrorWritesIsTaken(t *testing.T) {
|
||||
for given, ok := range map[string]bool{
|
||||
ref("route-proxy", "on-go_base", 1): true,
|
||||
catalogue.ArtifactStoreScheme + "upstream/docker.io/library/golang@sha256:" + strings.Repeat("a", 64): true,
|
||||
ref("route-proxy", "server", 1): false, // a module's own image
|
||||
catalogue.ArtifactStoreScheme + "novox/invoicing-api@sha256:" + strings.Repeat("a", 64): false,
|
||||
archiveRef("web", "on-tools", 1): false, // a blob
|
||||
catalogue.ArtifactStoreScheme + "web/on-x@sha256:abc": false, // not a whole digest
|
||||
"docker.io/library/golang@sha256:" + strings.Repeat("a", 64): false,
|
||||
} {
|
||||
if _, err := mirrorReference(given); (err == nil) != ok {
|
||||
t.Errorf("%s: taken %v, want %v (%v)", given, err == nil, ok, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// heldStore answers a manifest HEAD with 200 for the digests it holds, and records every request.
|
||||
func heldStore(t *testing.T, holds ...string) (artifacts.Store, *[]string) {
|
||||
t.Helper()
|
||||
var asked []string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
asked = append(asked, r.Method+" "+r.URL.Path)
|
||||
if r.Method == http.MethodHead && slices.ContainsFunc(holds, func(d string) bool { return strings.HasSuffix(r.URL.Path, d) }) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
return artifacts.Store{Address: strings.TrimPrefix(server.URL, "http://")}, &asked
|
||||
}
|
||||
|
||||
func TestRecordingACopyTakesWhatTheStoreHoldsAndDeletesNothing(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
held := ref("route-proxy", "on-go_base", 1)
|
||||
absent := ref("route-proxy", "on-go_base", 2)
|
||||
known := ref("nats", "on-nats_base", 3)
|
||||
notACopy := ref("route-proxy", "server", 4)
|
||||
if err := inv.RecordMirrored(t.Context(), "b1", "", []string{known}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store, asked := heldStore(t, fmt.Sprintf("%064x", 1))
|
||||
given := []string{held, absent, known, notACopy}
|
||||
mirrored, err := inv.Mirrored(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A dry run says, and records nothing.
|
||||
dry, err := recordMirrors(t.Context(), inv, store, given, mirrored, "", false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !dry.DryRun || !slices.Equal(dry.Recorded, []string{held}) || !slices.Equal(dry.AlreadyRecorded, []string{known}) ||
|
||||
len(dry.Refused) != 2 || dry.Refused[absent] == "" || dry.Refused[notACopy] == "" {
|
||||
t.Fatalf("a dry run answered %+v", dry)
|
||||
}
|
||||
if !strings.Contains(dry.Then, "would become eligible") || !strings.Contains(dry.Then, "next sweep") {
|
||||
t.Fatalf("a dry run did not say what recording does: %q", dry.Then)
|
||||
}
|
||||
if again, _ := inv.Mirrored(t.Context()); again[held] {
|
||||
t.Fatal("a dry run recorded a copy")
|
||||
}
|
||||
|
||||
// A real one records what the store holds, with the person's why, and the sweep may now decide.
|
||||
real, err := recordMirrors(t.Context(), inv, store, given, mirrored, "copied before copies were recorded", true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Equal(real.Recorded, []string{held}) {
|
||||
t.Fatalf("recorded %v", real.Recorded)
|
||||
}
|
||||
left, err := inv.ToCollect(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Contains(left, held) {
|
||||
t.Fatalf("a recorded copy no kept build stood on is not offered to collect: %v", left)
|
||||
}
|
||||
for _, r := range *asked {
|
||||
if !strings.HasPrefix(r, "HEAD ") {
|
||||
t.Fatalf("recording a copy asked the store %s", r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// indexRegistry holds indexes and platforms of one image's repository, answers GETs with their
|
||||
// documents, refuses GETs for the digests in fail, and records every delete.
|
||||
type indexRegistry struct {
|
||||
mu sync.Mutex
|
||||
indexes map[string][]string
|
||||
held map[string]bool
|
||||
fail map[string]bool
|
||||
deleted []string
|
||||
}
|
||||
|
||||
func (f *indexRegistry) serve(t *testing.T) artifacts.Store {
|
||||
t.Helper()
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
digest := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:]
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
if f.fail[digest] {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if children, ok := f.indexes[digest]; ok && f.held[digest] {
|
||||
var named []string
|
||||
for _, c := range children {
|
||||
named = append(named, `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"`+c+`"}`)
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[` +
|
||||
strings.Join(named, ",") + `]}`))
|
||||
return
|
||||
}
|
||||
if f.held[digest] {
|
||||
_, _ = w.Write([]byte(`{"schemaVersion":2,"layers":[]}`))
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
case http.MethodHead:
|
||||
if f.held[digest] {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
case http.MethodDelete:
|
||||
if !f.held[digest] {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
delete(f.held, digest)
|
||||
f.deleted = append(f.deleted, digest)
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
default:
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
return artifacts.Store{Address: strings.TrimPrefix(server.URL, "http://")}
|
||||
}
|
||||
|
||||
func copyDigest(n int) string { return fmt.Sprintf("sha256:%064x", n) }
|
||||
|
||||
// twoCopies records, in one image's repository, a kept copy (stood on by a held module's build) naming
|
||||
// platforms 11 and 12, and an eligible one (a failed build's) naming 12 and 13.
|
||||
func twoCopies(t *testing.T, inv *inventory.Inventory) (kept, eligible string, reg *indexRegistry) {
|
||||
t.Helper()
|
||||
const repository = "upstream/docker.io/library/golang"
|
||||
kept = catalogue.ArtifactStoreScheme + repository + "@" + copyDigest(1)
|
||||
eligible = catalogue.ArtifactStoreScheme + repository + "@" + copyDigest(2)
|
||||
if err := inv.RegisterModule(t.Context(), catalogue.Manifest{Module: "proxy", Version: "1"},
|
||||
inventory.Source{Repository: "https://forge.invalid/proxy.git"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failed := inventory.Build{ID: "f1", Repository: "https://forge.invalid/proxy.git", On: "a-build-machine",
|
||||
Failed: "a recipe refused", Mirrored: []string{eligible}}
|
||||
if err := inv.RecordBuild(t.Context(), failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ok := inventory.Build{ID: "b1", Repository: "https://forge.invalid/proxy.git", Module: "proxy", On: "a-build-machine",
|
||||
Commit: "c0ffee", Made: []inventory.Artifact{{Name: "app", Kind: "image", Reference: ref("proxy", "app", 7)}},
|
||||
Against: []string{kept}, Mirrored: []string{kept}}
|
||||
if err := inv.RecordBuild(t.Context(), ok); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reg = &indexRegistry{
|
||||
indexes: map[string][]string{copyDigest(1): {copyDigest(11), copyDigest(12)}, copyDigest(2): {copyDigest(12), copyDigest(13)}},
|
||||
held: map[string]bool{copyDigest(1): true, copyDigest(2): true, copyDigest(11): true, copyDigest(12): true, copyDigest(13): true},
|
||||
fail: map[string]bool{},
|
||||
}
|
||||
return kept, eligible, reg
|
||||
}
|
||||
|
||||
// Through the records: a confirmed collect lets the eligible index go with the platform only it names,
|
||||
// and leaves the platform the kept index names.
|
||||
func TestAConfirmedCollectLetsAnIndexGoWithItsOwnPlatformsOnly(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
_, eligible, reg := twoCopies(t, inv)
|
||||
store := reg.serve(t)
|
||||
references, err := inv.ToCollect(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Equal(references, []string{eligible}) {
|
||||
t.Fatalf("offered %v, want the failed build's copy", references)
|
||||
}
|
||||
a := runCollect(t.Context(), inv, store, references, nil, true,
|
||||
sweepBounds{most: 10, budget: 5 * time.Second, platforms: true})
|
||||
if !slices.Equal(a.LetGo, []string{eligible}) || a.Stopped != "" {
|
||||
t.Fatalf("let go %v, stopped %q", a.LetGo, a.Stopped)
|
||||
}
|
||||
if !slices.Equal(reg.deleted, []string{copyDigest(13), copyDigest(2)}) {
|
||||
t.Fatalf("deleted %v; want its own platform, then the index", reg.deleted)
|
||||
}
|
||||
}
|
||||
|
||||
// The sweep after a build leaves an eligible index in place and eligible: let go of alone, its
|
||||
// platforms would stay for ever under a record that says collected. A later collect a person confirms
|
||||
// takes it with the platform only it names. An image the same sweep reaches is let go of as before.
|
||||
func TestTheSweepAfterABuildLeavesAnIndexForAConfirmedCollect(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
_, eligible, reg := twoCopies(t, inv)
|
||||
reg.held[copyDigest(5)] = true
|
||||
image := catalogue.ArtifactStoreScheme + "upstream/docker.io/library/golang@" + copyDigest(5)
|
||||
store := reg.serve(t)
|
||||
r := sweep(t.Context(), inv, store, []string{eligible, image}, nil, afterBuild)
|
||||
if r.Indexes != 1 || !slices.Equal(r.LetGo, []string{image}) || !slices.Equal(reg.deleted, []string{copyDigest(5)}) {
|
||||
t.Fatalf("after a build: %d indexes left, let go %v, deleted %v; want the index left and the image gone",
|
||||
r.Indexes, r.LetGo, reg.deleted)
|
||||
}
|
||||
left, err := inv.ToCollect(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Equal(left, []string{eligible}) {
|
||||
t.Fatalf("after the sweep the records offer %v; want the index still eligible", left)
|
||||
}
|
||||
a := runCollect(t.Context(), inv, store, left, nil, true,
|
||||
sweepBounds{most: 10, budget: 5 * time.Second, platforms: true})
|
||||
if !slices.Equal(a.LetGo, []string{eligible}) ||
|
||||
!slices.Equal(reg.deleted, []string{copyDigest(5), copyDigest(13), copyDigest(2)}) {
|
||||
t.Fatalf("a confirmed collect let go %v, deleted %v; want the index with its own platform", a.LetGo, reg.deleted)
|
||||
}
|
||||
}
|
||||
|
||||
// A kept index the store will not answer for stops a confirmed collect before its first delete.
|
||||
func TestASpareListThatCannotBeReadStopsTheSweepBeforeAnyDelete(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
_, eligible, reg := twoCopies(t, inv)
|
||||
reg.fail[copyDigest(1)] = true
|
||||
store := reg.serve(t)
|
||||
a := runCollect(t.Context(), inv, store, []string{eligible}, nil, true,
|
||||
sweepBounds{most: 10, budget: 5 * time.Second, platforms: true})
|
||||
if len(a.LetGo) != 0 || len(reg.deleted) != 0 || !strings.Contains(a.Stopped, "nothing was let go") {
|
||||
t.Fatalf("let go %v, deleted %v, stopped %q", a.LetGo, reg.deleted, a.Stopped)
|
||||
}
|
||||
}
|
||||
@@ -73,7 +73,7 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke
|
||||
for _, r := range h.Resources {
|
||||
kept := inventory.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target,
|
||||
State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts,
|
||||
Check: r.Check, Needs: r.Needs}
|
||||
Check: r.Check, Needs: r.Needs, Account: r.Account, Root: r.Root}
|
||||
resources = append(resources, kept)
|
||||
if r.State == link.StateUnhealthy && r.Module != "" {
|
||||
unhealthy[r.Module] = append(unhealthy[r.Module], kept)
|
||||
@@ -83,8 +83,16 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke
|
||||
for module := range unhealthy {
|
||||
streaks[module] = prev.Streaks[module] + 1
|
||||
}
|
||||
var network *inventory.NetworkHealth
|
||||
if h.Network != nil {
|
||||
network = &inventory.NetworkHealth{State: h.Network.State, Since: h.Network.Since, Parts: []inventory.NetworkPart{}}
|
||||
for _, p := range h.Network.Parts {
|
||||
network.Parts = append(network.Parts, inventory.NetworkPart{Part: p.Part, State: p.State, Reason: p.Reason,
|
||||
Said: p.Said, Writer: p.Writer, Owner: p.Owner, Toward: p.Toward, Since: p.Since, Streak: p.Streak})
|
||||
}
|
||||
}
|
||||
stored, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: node, Contract: h.Contract, SaidAt: h.At,
|
||||
HeardAt: now, Resources: resources, Streaks: streaks})
|
||||
HeardAt: now, Resources: resources, Streaks: streaks, Network: network, Units: unitsKept(h.Units)})
|
||||
if err != nil || !stored {
|
||||
if err == nil {
|
||||
healthRefused.Add(1)
|
||||
@@ -94,7 +102,24 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke
|
||||
if k == nil {
|
||||
return nil
|
||||
}
|
||||
return judgeModuleHealth(ctx, inv, k, node, unhealthy, streaks, now)
|
||||
err = judgeModuleHealth(ctx, inv, k, node, unhealthy, streaks, now)
|
||||
// And the machine's own failed units (issue 315): what no module places, one finding for the machine.
|
||||
if uerr := judgeUnits(ctx, k, node, h.Units); uerr != nil {
|
||||
if err == nil {
|
||||
err = uerr
|
||||
} else {
|
||||
err = fmt.Errorf("%w; %v", err, uerr)
|
||||
}
|
||||
}
|
||||
// And every machine's network, from every machine's newest statement (ADR 0241): a statement about one
|
||||
// machine can hold another's finding, or release it.
|
||||
if nerr := judgeNetworks(ctx, inv, k, now); nerr != nil {
|
||||
if err == nil {
|
||||
return nerr
|
||||
}
|
||||
return fmt.Errorf("%w; %v", err, nerr)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// judgeModuleHealth raises a module's condition on a machine on the second statement in a row that says a
|
||||
@@ -110,7 +135,8 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
}
|
||||
standing := map[string]conditions.Condition{}
|
||||
for _, c := range open {
|
||||
if c.Kind == kindModuleUnhealthy && c.Subject.Machine == node {
|
||||
if (c.Kind == kindModuleUnhealthy || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound) &&
|
||||
c.Subject.Machine == node {
|
||||
standing[c.Key] = c
|
||||
}
|
||||
}
|
||||
@@ -127,9 +153,46 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
}
|
||||
sort.Strings(modules)
|
||||
seen := map[string]bool{}
|
||||
// became is, per module, the kind of condition this statement says of it: what a standing one of the
|
||||
// other kind turned into.
|
||||
became := map[string]string{}
|
||||
heldOn := map[string]string{}
|
||||
providers := map[catalogue.Chosen]bool{}
|
||||
for _, m := range modules {
|
||||
// **A directory used as found is said as that** (novox/hq issue 339): the operator's to hand over at the
|
||||
// machine, never urgent — nothing is broken by the wait that a person was not told of — and its own kind,
|
||||
// so the gate never reads it as a fault of the build that happened to be sent beside it.
|
||||
if said, waits := foundWait(m, node, unhealthy[m]); waits {
|
||||
o := usedAsFoundObservation(m, node, said, unhealthy[m])
|
||||
seen[o.Key()] = true
|
||||
became[m] = kindUsedAsFound
|
||||
if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen {
|
||||
continue
|
||||
}
|
||||
if _, err := k.Observe(ctx, o); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
continue
|
||||
}
|
||||
// **A wait for a person's new login is said as that** (novox/hq ADR 0254): one plain sentence to the
|
||||
// operator, never urgent, cleared on the first statement that no longer says it.
|
||||
if said, waits := personWait(m, node, unhealthy[m]); waits {
|
||||
o := reloginObservation(m, node, said, operatorOn(ctx, inv, node), unhealthy[m])
|
||||
// **A provider waiting for a login says who waits on it** (novox/hq issue 318 review): its
|
||||
// consumers are held under it, and its own condition is where they are said.
|
||||
if hold != nil {
|
||||
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
|
||||
}
|
||||
seen[o.Key()] = true
|
||||
became[m] = kindReloginNeeded
|
||||
if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen {
|
||||
continue
|
||||
}
|
||||
if _, err := k.Observe(ctx, o); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
continue
|
||||
}
|
||||
o := moduleUnhealthyObservation(m, node, unhealthy[m])
|
||||
if hold != nil {
|
||||
if p, held := hold.heldUnder(node, m, unhealthy[m]); held {
|
||||
@@ -138,13 +201,10 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
providers[p] = true
|
||||
continue
|
||||
}
|
||||
if waiters := hold.waitersOn(catalogue.Chosen{Node: node, Module: m}); len(waiters) > 0 {
|
||||
o.Severity = conditions.Urgent
|
||||
o.Said += "; " + waitingWords(waiters)
|
||||
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
|
||||
}
|
||||
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
|
||||
}
|
||||
seen[o.Key()] = true
|
||||
became[m] = kindModuleUnhealthy
|
||||
c, isOpen := standing[o.Key()]
|
||||
if streaks[m] < moduleUnhealthyAfter && !isOpen {
|
||||
continue // unconfirmed: one statement can be wrong; `node show` lists it
|
||||
@@ -162,10 +222,27 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
}
|
||||
module := strings.TrimSuffix(c.Subject.ID, "."+node)
|
||||
why := fmt.Sprintf("%s says no resource of %s is unhealthy", node, module)
|
||||
if c.Kind == kindReloginNeeded {
|
||||
why = fmt.Sprintf("%s says %s no longer waits for a new login", node, module)
|
||||
}
|
||||
if c.Kind == kindUsedAsFound {
|
||||
why = fmt.Sprintf("%s says no directory of %s is used as found any more", node, module)
|
||||
}
|
||||
if on, held := heldOn[key]; held {
|
||||
why = fmt.Sprintf("what %s finds on %s waits on %s, which is unhealthy: held under its condition", module, node, on)
|
||||
}
|
||||
if _, err := k.Clear(ctx, key, why); err != nil {
|
||||
// **A condition that became the other kind** is not "working again" (issue 318 review): its clearing
|
||||
// line says what it became.
|
||||
resolved := ""
|
||||
switch {
|
||||
case c.Kind == kindModuleUnhealthy && became[module] == kindReloginNeeded:
|
||||
why = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
|
||||
resolved = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
|
||||
case c.Kind == kindReloginNeeded && became[module] == kindModuleUnhealthy:
|
||||
why = fmt.Sprintf("%s on %s no longer waits for a new login, and is not healthy", module, node)
|
||||
resolved = fmt.Sprintf("The new login on %s is done, and %s still does not work", node, module)
|
||||
}
|
||||
if _, err := k.ClearSaying(ctx, key, why, resolved); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
}
|
||||
@@ -187,7 +264,7 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p catalogue.Chosen, now time.Time) error {
|
||||
var raisedAt *conditions.Condition
|
||||
for i, c := range hold.open {
|
||||
if c.Key == moduleUnhealthyKey(p.Module, p.Node) {
|
||||
if c.Key == moduleUnhealthyKey(p.Module, p.Node) || c.Key == reloginKey(p.Module, p.Node) {
|
||||
raisedAt = &hold.open[i]
|
||||
}
|
||||
}
|
||||
@@ -204,20 +281,114 @@ func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p cata
|
||||
return nil
|
||||
}
|
||||
o := moduleUnhealthyObservation(p.Module, p.Node, rs)
|
||||
if waiters := hold.waitersOn(p); len(waiters) > 0 {
|
||||
o.Severity = conditions.Urgent
|
||||
o.Said += "; " + waitingWords(waiters)
|
||||
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
|
||||
if said, waits := personWait(p.Module, p.Node, rs); waits {
|
||||
o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rs)
|
||||
}
|
||||
if o.Key() != raisedAt.Key {
|
||||
return nil // its own statement says it next
|
||||
}
|
||||
sayWaitingOn(&o, hold.waitersOn(p))
|
||||
_, err := k.Observe(ctx, o)
|
||||
return err
|
||||
}
|
||||
|
||||
// reloginKey is a module's relogin-needed condition on a machine.
|
||||
func reloginKey(module, node string) string {
|
||||
return conditions.Key(conditions.ScopeModule, module+"."+node, kindReloginNeeded)
|
||||
}
|
||||
|
||||
// operatorOn is the operator's account on a machine, or "" when it is not known (to-be 29).
|
||||
func operatorOn(ctx context.Context, inv *inventory.Inventory, node string) string {
|
||||
if inv == nil {
|
||||
return ""
|
||||
}
|
||||
n, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return n.Account
|
||||
}
|
||||
|
||||
// reloginObservation is a module waiting for a person's new login on a machine (novox/hq ADR 0254): the
|
||||
// operator's, a warning, its summary the one sentence that says what to do; the resources are evidence. Its
|
||||
// plain words (ADR 0253) are the kind's: it needs the operator, and offers no answer — no verb can log a
|
||||
// person in again, and a restart of the module's service would start it in the same session. operator is
|
||||
// the machine's operator account, when known: the words say "your account" only for it.
|
||||
func reloginObservation(module, node, said, operator string, rs []inventory.ResourceHealth) conditions.Observation {
|
||||
o := moduleUnhealthyObservation(module, node, rs)
|
||||
o.Token, o.Kind, o.Resolver, o.Summary = kindReloginNeeded, kindReloginNeeded, conditions.ResolverOperator, said
|
||||
accounts := waitingAccounts(module, rs)
|
||||
yours := operator != "" && len(accounts) > 0
|
||||
for _, a := range accounts {
|
||||
yours = yours && a == operator
|
||||
}
|
||||
w := reloginWords(module, node, yours)
|
||||
o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions = w.Headline, w.Explanation, w.Resolved, w.Needs, nil
|
||||
return o
|
||||
}
|
||||
|
||||
// reloginWords is what the operator reads of a module waiting for a new login on a machine (ADR 0253,
|
||||
// ADR 0254): never quiet, since only a person can do it, and no button, since nothing else can. yours says
|
||||
// the account waiting is the operator's own on that machine; otherwise the words do not claim it is.
|
||||
func reloginWords(module, node string, yours bool) words {
|
||||
if yours {
|
||||
return words{Headline: fmt.Sprintf("%s waits for a new login on %s", module, node),
|
||||
Needs: fmt.Sprintf("log out of %s completely and log in again, or restart it.", node),
|
||||
Explanation: fmt.Sprintf("%s put your account in a group it needs. You logged in before that, so %s "+
|
||||
"cannot run until you log in again. Its update is in place and nothing was undone.",
|
||||
conditions.Capital(module), module),
|
||||
Resolved: fmt.Sprintf("%s runs on %s after your new login", module, node)}
|
||||
}
|
||||
return words{Headline: fmt.Sprintf("%s waits for a new login on %s", module, node),
|
||||
Needs: fmt.Sprintf("have the account it names log out of %s completely and log in again, or restart %s.", node, node),
|
||||
Explanation: fmt.Sprintf("%s put an account on %s in a group it needs. That account logged in before that, "+
|
||||
"so %s cannot run until it logs in again. Its update is in place and nothing was undone.",
|
||||
conditions.Capital(module), node, module),
|
||||
Resolved: fmt.Sprintf("%s runs on %s after the new login", module, node)}
|
||||
}
|
||||
|
||||
// waitingAccounts is every account of a module whose resource says it waits for a new login, sorted.
|
||||
func waitingAccounts(module string, rs []inventory.ResourceHealth) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, r := range rs {
|
||||
if r.Module == module && r.Kind == link.KindAccount && r.State == link.StateUnhealthy &&
|
||||
strings.HasPrefix(r.Reason, link.ReasonRelogin) && accountOf(r) != "" && !seen[accountOf(r)] {
|
||||
seen[accountOf(r)] = true
|
||||
out = append(out, accountOf(r))
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// sayWaitingOn adds to a module's condition the consumers held under it (to-be 48 §6): urgent while anyone
|
||||
// waits on it, whether it is not working or waits for a new login.
|
||||
func sayWaitingOn(o *conditions.Observation, waiters []string) {
|
||||
if len(waiters) == 0 {
|
||||
return
|
||||
}
|
||||
o.Severity = conditions.Urgent
|
||||
o.Said += "; " + waitingWords(waiters)
|
||||
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
|
||||
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
|
||||
}
|
||||
|
||||
// moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module,
|
||||
// the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence.
|
||||
func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {
|
||||
var words, said []string
|
||||
var words, said, plain []string
|
||||
needs := moduleNeeds(node, rs)
|
||||
for _, r := range rs {
|
||||
plain = append(plain, resourcePlainWords(r))
|
||||
if r.Kind == link.KindUnit {
|
||||
// A failed unit is named by the unit, which is what a person looks for (issue 315); the
|
||||
// resource that places it — a package, a file — is evidence.
|
||||
words = append(words, fmt.Sprintf("its unit %s %s", r.Target, r.Reason))
|
||||
said = append(said, fmt.Sprintf("%s (unit %s, placed by %s): %s, since %s", r.Target, r.Target, r.Resource,
|
||||
orNotSaid(r.Reason), r.Since.UTC().Format("2006-01-02 15:04:05 MST")))
|
||||
continue
|
||||
}
|
||||
words = append(words, fmt.Sprintf("its %s %s %s", r.Kind, r.Resource, reasonWords(r)))
|
||||
said = append(said, fmt.Sprintf("%s (%s %s): %s, %d look(s) in a row, %d restart(s) counted, since %s",
|
||||
r.Resource, r.Kind, r.Target, orNotSaid(r.Reason), r.Streak, r.Restarts,
|
||||
@@ -225,12 +396,21 @@ func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHeal
|
||||
}
|
||||
return conditions.Observation{Scope: conditions.ScopeModule, ID: module + "." + node, Token: "unhealthy",
|
||||
Kind: kindModuleUnhealthy, Machine: node, Severity: conditions.Warning, Source: sourceHealth,
|
||||
Summary: fmt.Sprintf("%s on %s is not healthy: %s", module, node, strings.Join(words, "; ")),
|
||||
Said: strings.Join(said, "; ")}
|
||||
Summary: fmt.Sprintf("%s on %s is not healthy: %s", module, node, strings.Join(words, "; ")),
|
||||
Said: strings.Join(said, "; "),
|
||||
Headline: fmt.Sprintf("%s not working on %s", module, node),
|
||||
Explanation: fmt.Sprintf("%s on %s is not healthy: %s. It clears as soon as it runs again.", module, node,
|
||||
namesWords(plain, 3)),
|
||||
Needs: needs,
|
||||
Resolved: fmt.Sprintf("%s works again on %s", module, node)}
|
||||
}
|
||||
|
||||
// reasonWords is why a resource is unhealthy, as a person reads it.
|
||||
func reasonWords(r inventory.ResourceHealth) string {
|
||||
// An account waiting for a new login (ADR 0252) is said in the mesh's words, not the engine's.
|
||||
if r.Kind == link.KindAccount && strings.HasPrefix(r.Reason, link.ReasonRelogin) {
|
||||
return fmt.Sprintf("waits for a new login of %s, which is in the group and logged in before it was", accountOf(r))
|
||||
}
|
||||
switch r.Reason {
|
||||
case "restarting":
|
||||
return fmt.Sprintf("keeps restarting (%d restart(s) counted)", r.Restarts)
|
||||
@@ -254,9 +434,71 @@ func orNotSaid(s string) string {
|
||||
return s
|
||||
}
|
||||
|
||||
// kindReloginNeeded is a module's condition while it waits for a person's new login on a machine (novox/hq
|
||||
// ADR 0254): its own kind, so that neither the gate nor anyone reading the conditions takes it for a fault.
|
||||
const kindReloginNeeded = "relogin-needed"
|
||||
|
||||
// personWait is whether everything unhealthy of a module on a machine waits for one person's new login, and
|
||||
// that wait in one plain sentence (novox/hq ADR 0254, issue 318). Narrow on purpose, so that a fault is never
|
||||
// excused:
|
||||
//
|
||||
// - at least one of the module's resources is its account (kind account), unhealthy with a reason that
|
||||
// starts "relogin needed" (ADR 0252): the database lists the account in the group, and the session
|
||||
// running began before;
|
||||
// - every other unhealthy resource of the module runs in the own service manager of one of those very
|
||||
// accounts (Account names it): the manager that began before the group and does not hold it.
|
||||
//
|
||||
// Anything else unhealthy of the module — a container, a unit of the machine's own manager, a unit in
|
||||
// another account's manager, a resource whose engine does not say whose manager it is in, an account
|
||||
// not in its group or that could not be read — is not a wait, and the module is judged as before. A
|
||||
// resource still starting is not unhealthy and does not make a wait either. Pure.
|
||||
func personWait(module, machine string, rs []inventory.ResourceHealth) (string, bool) {
|
||||
accounts := waitingAccounts(module, rs)
|
||||
if len(accounts) == 0 {
|
||||
return "", false
|
||||
}
|
||||
waiting := map[string]bool{}
|
||||
for _, a := range accounts {
|
||||
waiting[a] = true
|
||||
}
|
||||
var units []string
|
||||
for _, r := range rs {
|
||||
if r.Module != module || r.State != link.StateUnhealthy {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case r.Kind == link.KindAccount && strings.HasPrefix(r.Reason, link.ReasonRelogin) && waiting[accountOf(r)]:
|
||||
case r.Kind != link.KindAccount && r.Account != "" && waiting[r.Account]:
|
||||
units = append(units, r.Target)
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
said := fmt.Sprintf("relogin needed on %s: %s waits for a new login of %s, which is in its group and whose "+
|
||||
"login began before it was; log out of %[1]s completely and log in again, or restart it", machine, module,
|
||||
strings.Join(accounts, ", "))
|
||||
if len(units) > 0 {
|
||||
sort.Strings(units)
|
||||
said += fmt.Sprintf(" (until then %s cannot run)", strings.Join(units, ", "))
|
||||
}
|
||||
return said, true
|
||||
}
|
||||
|
||||
// accountOf is the account a resource of kind account is: named by the engine, or its target.
|
||||
func accountOf(r inventory.ResourceHealth) string {
|
||||
if r.Account != "" {
|
||||
return r.Account
|
||||
}
|
||||
return r.Target
|
||||
}
|
||||
|
||||
// moduleHealthWord is the gate's reading of a module's stated health on a machine (ADR 0240 §4, ADR 0236
|
||||
// §2 as amended): good when every long-running resource of it is stated healthy in a statement heard since
|
||||
// the send; not yet otherwise, saying which. A machine that never stated health is judged as before.
|
||||
//
|
||||
// **A wait for a person is its own reading** (novox/hq ADR 0254): when everything unhealthy of the module
|
||||
// waits for one person's new login (personWait), the reading is healthPerson — not a fault of the build,
|
||||
// and not something a machine can meet within a bound.
|
||||
func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (health, string) {
|
||||
if f.healthErr != nil {
|
||||
return healthNotYet, "what " + machine + " says of its resources' health cannot be read: " + firstLine(f.healthErr.Error())
|
||||
@@ -268,10 +510,30 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
|
||||
if h.HeardAt.Before(since) {
|
||||
return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module)
|
||||
}
|
||||
wait, waits := personWait(module, machine, h.Resources)
|
||||
// **Only a build whose own send put the account in a new group is excused** (issue 318 review): read from
|
||||
// what the controller sent, never from when the machine says the wait began — that time is the engine's
|
||||
// memory, reset by its restart and moved by a change of words. A build that adds no account group cannot
|
||||
// have brought a wait, so a later build sent while the login is still owed is judged as before.
|
||||
if waits && !f.groupsAdded[module] {
|
||||
waits = false
|
||||
}
|
||||
var found []string
|
||||
for _, r := range h.Resources {
|
||||
if r.Module != module {
|
||||
continue
|
||||
}
|
||||
if waits && r.State == link.StateUnhealthy {
|
||||
continue
|
||||
}
|
||||
// **A directory used as found before this send waits for a person** (novox/hq issue 339): the node-engine
|
||||
// left its owner and mode, and only someone at the machine can hand it over. It is no fault of this
|
||||
// build, so it does not hold the module's walk — an urgent fix still goes through — and the verdict
|
||||
// carries the wait. Found by this very send, the send brought it, and it is judged as unhealthy.
|
||||
if usedAsFound(r) && r.Since.Before(since) {
|
||||
found = append(found, r.Resource)
|
||||
continue
|
||||
}
|
||||
switch r.State {
|
||||
case link.StateHealthy:
|
||||
case link.StateStarting:
|
||||
@@ -287,9 +549,25 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
|
||||
reasonAfter(r.Reason))
|
||||
}
|
||||
}
|
||||
if waits || len(found) > 0 {
|
||||
var said []string
|
||||
if waits {
|
||||
said = append(said, wait)
|
||||
}
|
||||
if len(found) > 0 {
|
||||
said = append(said, fmt.Sprintf("on %s, %s uses %s as found and waits for a person to hand it over "+
|
||||
"(`mesh-host hand-over <directory>` at the machine)", machine, module, strings.Join(found, ", ")))
|
||||
}
|
||||
return healthPerson, strings.Join(said, "; ")
|
||||
}
|
||||
return healthGood, ""
|
||||
}
|
||||
|
||||
// usedAsFound is a directory the node-engine states it uses as found (novox/hq issue 339).
|
||||
func usedAsFound(r inventory.ResourceHealth) bool {
|
||||
return r.Kind == link.KindDirectory && r.State == link.StateUnhealthy && strings.HasPrefix(r.Reason, link.ReasonUsedAsFound)
|
||||
}
|
||||
|
||||
func reasonAfter(s string) string {
|
||||
if s == "" {
|
||||
return ""
|
||||
@@ -323,3 +601,84 @@ func healthLines(h inventory.NodeHealth, had bool, now time.Time) []string {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// accountGroups is every account group a manifest declares, as "<account>/<group>": a user resource's
|
||||
// groups (ADR 0252).
|
||||
func accountGroups(m catalogue.Manifest) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, r := range m.Resources {
|
||||
if t, _ := r["type"].(string); t != "user" {
|
||||
continue
|
||||
}
|
||||
name, _ := r["name"].(string)
|
||||
groups, _ := r["groups"].([]any)
|
||||
for _, g := range groups {
|
||||
if group, ok := g.(string); ok && group != "" {
|
||||
out[name+"/"+group] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// addsAccountGroups is whether a move from one manifest of a module to another puts an account in a group the
|
||||
// earlier one did not (issue 318 review): the only send that can bring a wait for a new login. A module new to
|
||||
// the machine (no earlier manifest) adds every group it declares.
|
||||
func addsAccountGroups(from catalogue.Manifest, hadFrom bool, to catalogue.Manifest) bool {
|
||||
before := map[string]bool{}
|
||||
if hadFrom {
|
||||
before = accountGroups(from)
|
||||
}
|
||||
for g := range accountGroups(to) {
|
||||
if !before[g] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// kindUsedAsFound is a module's condition while the node-engine uses one of its directories as found (novox/hq
|
||||
// issue 339): its own kind, the operator's, never urgent, and never read by the gate as a fault of a build.
|
||||
const kindUsedAsFound = "directory-used-as-found"
|
||||
|
||||
// usedAsFoundKey is a module's used-as-found condition on a machine.
|
||||
func usedAsFoundKey(module, node string) string {
|
||||
return conditions.Key(conditions.ScopeModule, module+"."+node, kindUsedAsFound)
|
||||
}
|
||||
|
||||
// foundWait is whether everything unhealthy of a module on a machine is a directory used as found, and that in
|
||||
// one sentence. Anything else unhealthy beside it is judged as a fault, with the directory among its resources.
|
||||
func foundWait(module, node string, rs []inventory.ResourceHealth) (string, bool) {
|
||||
var ids, why []string
|
||||
for _, r := range rs {
|
||||
if r.Module != module || r.State != link.StateUnhealthy {
|
||||
continue
|
||||
}
|
||||
if !usedAsFound(r) {
|
||||
return "", false
|
||||
}
|
||||
ids = append(ids, r.Resource)
|
||||
why = append(why, strings.TrimSpace(strings.TrimPrefix(r.Reason, link.ReasonUsedAsFound)))
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
return "", false
|
||||
}
|
||||
return fmt.Sprintf("%s on %s uses %s as found: %s", module, node, strings.Join(ids, ", "),
|
||||
strings.Join(why, "; ")), true
|
||||
}
|
||||
|
||||
// usedAsFoundObservation is a module whose directory the node-engine uses as found, in words: the operator's, a
|
||||
// warning however long it stays, its summary naming the directories and their owners; the paths are evidence.
|
||||
func usedAsFoundObservation(module, node, said string, rs []inventory.ResourceHealth) conditions.Observation {
|
||||
o := moduleUnhealthyObservation(module, node, rs)
|
||||
o.Token, o.Kind, o.Resolver, o.Severity, o.Summary = kindUsedAsFound, kindUsedAsFound, conditions.ResolverOperator,
|
||||
conditions.Warning, said
|
||||
o.Headline = fmt.Sprintf("%s waits for a directory on %s", module, node)
|
||||
o.Explanation = fmt.Sprintf("A directory of %s was already on %s, with another owner or mode than %s declares. "+
|
||||
"The mesh left it as it was rather than hand it to an account, so %s may not be able to use it.",
|
||||
module, node, module, module)
|
||||
o.Needs = fmt.Sprintf("on %s, run mesh-host hand-over with the directory's path as root.", node)
|
||||
o.Resolved = fmt.Sprintf("%s's directory on %s is the mesh's", module, node)
|
||||
o.Actions = nil
|
||||
return o
|
||||
}
|
||||
|
||||
+334
-12
@@ -182,6 +182,9 @@ func moduleCommand(ctx context.Context, args []string) error {
|
||||
Requires []string `json:"requires,omitempty"`
|
||||
Claims []string `json:"claims,omitempty"`
|
||||
Capabilities []string `json:"capabilities,omitempty"`
|
||||
// Replaces is what each of its own tools replaces (novox/hq ADR 0245), for the console's
|
||||
// search and the agent's instructions.
|
||||
Replaces map[string][]string `json:"replaces,omitempty"`
|
||||
}
|
||||
out := make([]listed, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
@@ -189,7 +192,7 @@ func moduleCommand(ctx context.Context, args []string) error {
|
||||
l := listed{Module: m.Module, Version: m.Version, Built: e.Source.BuiltFrom, Head: e.Source.Head,
|
||||
Current: e.Provided || e.Source.Repository == "" || e.Source.Current(), Provided: e.Provided,
|
||||
On: append([]string{}, e.On...), Provides: m.Offers(), Requires: m.Requires,
|
||||
Capabilities: m.Capabilities, Tools: declaresTools(m)}
|
||||
Capabilities: m.Capabilities, Tools: declaresTools(m), Replaces: m.Replaces}
|
||||
for _, c := range m.Claims {
|
||||
l.Claims = append(l.Claims, c.At()+"/"+c.Name)
|
||||
}
|
||||
@@ -352,9 +355,20 @@ func moduleCommand(ctx context.Context, args []string) error {
|
||||
func assignCommand(ctx context.Context, verb string, args []string) error {
|
||||
// Several modules in one act (novox/hq ADR 0207): holders that depend on each other — the
|
||||
// service manager and the package manager — can only go on, or come off, together.
|
||||
set := flag.NewFlagSet(verb, flag.ContinueOnError)
|
||||
// A module known and not built: ask for its build, and keep the assignment pending on it (novox/hq
|
||||
// issue 325). Only assign reads it.
|
||||
build := set.Bool("build", false, "for a module known and not built: ask for its build, and assign it when it registers")
|
||||
args, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(args) < 2 {
|
||||
return fmt.Errorf("%s <node> <module> [<module>…]", verb)
|
||||
}
|
||||
if *build && verb == "unassign" {
|
||||
return errors.New("unassign takes no --build")
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -363,7 +377,9 @@ func assignCommand(ctx context.Context, verb string, args []string) error {
|
||||
|
||||
// The act itself is in acts.go, so the command API refuses exactly what this refuses
|
||||
// (novox/hq ADR 0035). What differs between the surfaces is how the answer is printed.
|
||||
act := assign
|
||||
act := func(ctx context.Context, open *stores, node string, modules ...string) (string, error) {
|
||||
return assignWith(ctx, open, node, assignOptions{Build: *build}, modules...)
|
||||
}
|
||||
if verb == "unassign" {
|
||||
act = unassign
|
||||
}
|
||||
@@ -380,7 +396,9 @@ func assignCommand(ctx context.Context, verb string, args []string) error {
|
||||
|
||||
func settingsCommand(ctx context.Context, args []string) error {
|
||||
if len(args) == 0 {
|
||||
return errors.New("settings set <module> <file> [--node <node>], or settings clear <module> [--node <node>]")
|
||||
return errors.New("settings show <module> [--node <node>] [--history], settings set <module> <file> " +
|
||||
"[--node <node>] [--replace], settings clear <module> [--node <node>], or settings preferences " +
|
||||
"[<module>] [--node <node>]")
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
@@ -391,6 +409,11 @@ func settingsCommand(ctx context.Context, args []string) error {
|
||||
|
||||
set := flag.NewFlagSet("settings", flag.ContinueOnError)
|
||||
node := set.String("node", "", "one machine, rather than the whole mesh")
|
||||
// **What a set removes is refused unless meant** (novox/hq ADR 0217). A layer is replaced whole,
|
||||
// and on 2026-10-05 setting one placement dropped a machine's whole layer for a module without a
|
||||
// word (novox/hq issue 304). Adding and changing keys needs nothing; removing one needs this.
|
||||
replace := set.Bool("replace", false, "for set: remove the keys the new layer does not name")
|
||||
history := set.Bool("history", false, "for show: the layers this one replaced, the latest first")
|
||||
positionals, err := parseAround(set, args[1:])
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -418,23 +441,171 @@ func settingsCommand(ctx context.Context, args []string) error {
|
||||
if err := json.Unmarshal(raw, &values); err != nil {
|
||||
return fmt.Errorf("%s is not a settings file: %w", positionals[1], err)
|
||||
}
|
||||
before, _, err := inv.Layer(ctx, *node, positionals[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := refuseTerminalSettingsThroughAVerb(ctx, inv, before, values, positionals[0], where); err != nil {
|
||||
return err
|
||||
}
|
||||
added, changed, removed := settingsChange(before, values)
|
||||
if len(removed) > 0 && !*replace {
|
||||
return fmt.Errorf("%s on %s: this layer would no longer set %s. A layer is replaced whole; "+
|
||||
"read it with `settings show %s%s` and include what should stay, or add --replace if the "+
|
||||
"removal is meant (novox/hq ADR 0217). Nothing was changed",
|
||||
positionals[0], where, strings.Join(removed, ", "), positionals[0], nodeFlag(*node))
|
||||
}
|
||||
if err := inv.SetSettings(ctx, *node, positionals[0], values); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var keys []string
|
||||
for k := range values {
|
||||
keys = append(keys, k)
|
||||
fmt.Printf("%s on %s:\n", positionals[0], where)
|
||||
if len(added)+len(changed)+len(removed) == 0 {
|
||||
fmt.Println(" nothing changed")
|
||||
}
|
||||
sort.Strings(keys)
|
||||
fmt.Printf("%s on %s: %s\n", positionals[0], where, strings.Join(keys, ", "))
|
||||
fmt.Println(" run `push` to send it")
|
||||
for _, p := range added {
|
||||
fmt.Printf(" + %s\n", p)
|
||||
}
|
||||
for _, p := range changed {
|
||||
fmt.Printf(" ~ %s\n", p)
|
||||
}
|
||||
for _, p := range removed {
|
||||
fmt.Printf(" - %s\n", p)
|
||||
}
|
||||
if before != nil {
|
||||
fmt.Printf(" the layer it replaced is kept: settings show %s%s --history\n", positionals[0], nodeFlag(*node))
|
||||
}
|
||||
if *node != "" {
|
||||
fmt.Printf(" run `plan %s --diff` to see what it changes, `push %s` to send it\n", *node, *node)
|
||||
} else {
|
||||
fmt.Println(" run `push --behind` to send it to the machines running it")
|
||||
}
|
||||
return nil
|
||||
|
||||
case "show":
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("settings show <module> [--node <node>] [--history]")
|
||||
}
|
||||
if *history {
|
||||
past, err := inv.SettingsHistory(ctx, *node, positionals[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(past) == 0 {
|
||||
fmt.Printf("%s on %s: no layer has been replaced\n", positionals[0], where)
|
||||
return nil
|
||||
}
|
||||
for _, p := range past {
|
||||
shown, _ := json.MarshalIndent(p.Values, " ", " ")
|
||||
fmt.Printf("%s on %s, until %s (%s):\n %s\n", positionals[0], where,
|
||||
p.ReplacedAt.Local().Format("2006-01-02 15:04:05"), p.ReplacedBy, shown)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
values, has, err := inv.Layer(ctx, *node, positionals[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !has {
|
||||
fmt.Printf("%s on %s: no layer — the module's definition says\n", positionals[0], where)
|
||||
} else {
|
||||
shown, err := json.MarshalIndent(values, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(shown))
|
||||
}
|
||||
// Every value the module gives a default or a layer sets, and where it came from (novox/hq
|
||||
// ADR 0262): the default, the mesh's layer, or this node's. Said after the layer, which stays
|
||||
// the first thing printed because a caller reads it before replacing it (ADR 0217).
|
||||
known, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m, ok := known[positionals[0]]
|
||||
if !ok || len(m.Settings) == 0 {
|
||||
return nil
|
||||
}
|
||||
var layers []catalogue.Layer
|
||||
if *node != "" {
|
||||
if mesh, has, err := inv.Layer(ctx, "", positionals[0]); err != nil {
|
||||
return err
|
||||
} else if has {
|
||||
layers = append(layers, catalogue.Layer{From: catalogue.MeshWideLayer, Values: mesh})
|
||||
}
|
||||
if has {
|
||||
layers = append(layers, catalogue.Layer{From: *node, Values: values})
|
||||
}
|
||||
} else if has {
|
||||
layers = append(layers, catalogue.Layer{From: catalogue.MeshWideLayer, Values: values})
|
||||
}
|
||||
fmt.Print(describeEffective(positionals[0], where, catalogue.Effective(m, layers)))
|
||||
return nil
|
||||
|
||||
case "preferences":
|
||||
// Every module's preferences, and each machine's value with its source (novox/hq ADR 0262).
|
||||
if len(positionals) > 1 {
|
||||
return errors.New("settings preferences [<module>] [--node <node>]")
|
||||
}
|
||||
only := ""
|
||||
if len(positionals) == 1 {
|
||||
only = positionals[0]
|
||||
}
|
||||
if *node != "" {
|
||||
// A machine the mesh does not know is refused, never answered with an empty listing.
|
||||
if _, err := inv.NodeByName(ctx, *node); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var listed []preferencesOf
|
||||
for _, e := range entries {
|
||||
m := e.Manifest
|
||||
if len(m.Settings) == 0 || (only != "" && m.Module != only) {
|
||||
continue
|
||||
}
|
||||
if *node != "" && !containsString(e.On, *node) {
|
||||
// Asked for one machine: a module not on it has no value there to say.
|
||||
continue
|
||||
}
|
||||
p := preferencesOf{Manifest: m, On: map[string][]catalogue.SettingSource{}}
|
||||
for _, n := range e.On {
|
||||
if *node != "" && n != *node {
|
||||
continue
|
||||
}
|
||||
p.Nodes = append(p.Nodes, n)
|
||||
layers, err := inv.SettingsFor(ctx, n, m.Module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p.On[n] = catalogue.Effective(m, layers)
|
||||
}
|
||||
listed = append(listed, p)
|
||||
}
|
||||
if only != "" && len(listed) == 0 {
|
||||
fmt.Printf("%s declares no preferences%s\n", only, onNode(*node))
|
||||
return nil
|
||||
}
|
||||
if len(listed) == 0 && *node != "" {
|
||||
fmt.Printf("no module on %s declares a preference\n", *node)
|
||||
return nil
|
||||
}
|
||||
fmt.Print(describePreferences(listed))
|
||||
return nil
|
||||
|
||||
case "clear":
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("settings clear <module> [--node <node>]")
|
||||
}
|
||||
before, _, err := inv.Layer(ctx, *node, positionals[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := refuseTerminalSettingsThroughAVerb(ctx, inv, before, nil, positionals[0], where); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inv.ClearSettings(ctx, *node, positionals[0]); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -442,10 +613,95 @@ func settingsCommand(ctx context.Context, args []string) error {
|
||||
return nil
|
||||
|
||||
default:
|
||||
return fmt.Errorf("settings has no %q; it has set and clear", args[0])
|
||||
return fmt.Errorf("settings has no %q; it has show, set, clear and preferences", args[0])
|
||||
}
|
||||
}
|
||||
|
||||
// describeEffective says each setting's value on a machine or the whole mesh, where it came from, and
|
||||
// the module's default when a layer overrides it.
|
||||
func describeEffective(module, where string, values []catalogue.SettingSource) string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "%s on %s, every value and where it comes from:\n", module, where)
|
||||
for _, v := range values {
|
||||
value, _ := json.Marshal(v.Value)
|
||||
fmt.Fprintf(&b, " %s = %s (%s", v.Key, value, v.From)
|
||||
if v.HasDefault && !v.FromDefault {
|
||||
d, _ := json.Marshal(v.Default)
|
||||
fmt.Fprintf(&b, "; the default is %s", d)
|
||||
}
|
||||
b.WriteString(")\n")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// onNode is ` on <node>` for one machine, nothing for the whole mesh.
|
||||
func onNode(node string) string {
|
||||
if node == "" {
|
||||
return ""
|
||||
}
|
||||
return " on " + node
|
||||
}
|
||||
|
||||
// preferencesOf is one module's preferences and its value on each machine it is assigned to.
|
||||
type preferencesOf struct {
|
||||
Manifest catalogue.Manifest
|
||||
Nodes []string
|
||||
On map[string][]catalogue.SettingSource
|
||||
}
|
||||
|
||||
// describePreferences lists each module's preferences — key, default and why — and, per machine it is
|
||||
// assigned to, the value and where it comes from (novox/hq ADR 0262).
|
||||
func describePreferences(modules []preferencesOf) string {
|
||||
if len(modules) == 0 {
|
||||
return "no module declares a preference\n"
|
||||
}
|
||||
var b strings.Builder
|
||||
for i, p := range modules {
|
||||
if i > 0 {
|
||||
b.WriteString("\n")
|
||||
}
|
||||
on := "assigned nowhere"
|
||||
if len(p.Nodes) > 0 {
|
||||
on = "on " + strings.Join(p.Nodes, ", ")
|
||||
}
|
||||
fmt.Fprintf(&b, "%s (%s)\n", p.Manifest.Module, on)
|
||||
keys := make([]string, 0, len(p.Manifest.Settings))
|
||||
for k := range p.Manifest.Settings {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
d := p.Manifest.Settings[k]
|
||||
def, _ := json.Marshal(d.Default)
|
||||
fmt.Fprintf(&b, " %s, default %s: %s\n", k, def, d.Why)
|
||||
for _, n := range p.Nodes {
|
||||
for _, s := range p.On[n] {
|
||||
if s.Key != k {
|
||||
continue
|
||||
}
|
||||
v, _ := json.Marshal(s.Value)
|
||||
from := s.From
|
||||
if s.FromDefault {
|
||||
from = catalogue.DefaultLayer
|
||||
} else if s.From != catalogue.MeshWideLayer {
|
||||
from = "the node"
|
||||
}
|
||||
fmt.Fprintf(&b, " %s: %s (%s)\n", n, v, from)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// nodeFlag is ` --node <node>` for a machine's layer, nothing for the whole mesh's.
|
||||
func nodeFlag(node string) string {
|
||||
if node == "" {
|
||||
return ""
|
||||
}
|
||||
return " --node " + node
|
||||
}
|
||||
|
||||
// describeOffers says what a module provides, and marks the ones answered from anywhere in the
|
||||
// mesh — because "provides a database" and "provides a shell" are read the same way and mean
|
||||
// entirely different things about where the answer has to be.
|
||||
@@ -680,7 +936,7 @@ func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manife
|
||||
Emits: m.EmitsAll(), Consumes: m.Consumes, Serves: m.Tools,
|
||||
}); needed {
|
||||
if busAddress == "" {
|
||||
fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push`, then "+
|
||||
fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push --behind`, then "+
|
||||
"`rollout mint` again is harmless)\n", m.Module)
|
||||
} else {
|
||||
js, err := broker.Dial(busAddress)
|
||||
@@ -742,6 +998,9 @@ func whereItComesFrom(repository, ref, commit, path string, self bool) (inventor
|
||||
// definition that got past the check — written elsewhere, or checked by nobody — is refused here
|
||||
// in the same words. A name meant on purpose is declared with its reason and passes.
|
||||
func namesNoInstallation(m catalogue.Manifest) error {
|
||||
if problems := catalogue.TrustProblems(m); len(problems) > 0 {
|
||||
return fmt.Errorf("%s", strings.Join(problems, "; "))
|
||||
}
|
||||
named := catalogue.InstallationProblems(m)
|
||||
if len(named) == 0 {
|
||||
return nil
|
||||
@@ -805,3 +1064,66 @@ func declaresTools(m catalogue.Manifest) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// The keys no verb may change are catalogue.TerminalKeys (novox/hq issue 339). `places` says where the
|
||||
// node-engine creates and, as root, owns a module's directories, with an owner the setting names; `accesses` says
|
||||
// which of the machine's paths are mounted into a module's container; a provider's trust anchors say what every
|
||||
// consumer trusts. Set through a verb, any of them lets any caller of the mesh's verbs — an agent among them —
|
||||
// have root hand it a directory, mount one of the machine's into a container it reaches, or have the mesh trust
|
||||
// an authority of its own. They are the operator's, typed at the controller's terminal.
|
||||
|
||||
// throughAVerb says whether this process runs a seat verb's command line: the serving controller names the
|
||||
// verb in the environment of every command it runs for one (runVerb), and a person at the terminal runs none.
|
||||
// Every verb route reaches a command through runVerb — the named verbs and the generic `command` alike — so
|
||||
// this is the one place that knows, whatever line the verb composed.
|
||||
func throughAVerb() (string, bool) {
|
||||
verb := os.Getenv(verbVar)
|
||||
return verb, verb != ""
|
||||
}
|
||||
|
||||
// sameLayer says whether two layers hold the same values, an absent layer and an empty one alike.
|
||||
func sameLayer(a, b map[string]any) bool {
|
||||
if len(a) == 0 && len(b) == 0 {
|
||||
return true
|
||||
}
|
||||
ra, _ := json.Marshal(a)
|
||||
rb, _ := json.Marshal(b)
|
||||
return string(ra) == string(rb)
|
||||
}
|
||||
|
||||
// refuseTerminalSettingsThroughAVerb refuses a layer change through a verb that would add, change or remove
|
||||
// places or accesses; a change that leaves both as they were is not refused.
|
||||
func refuseTerminalSettingsThroughAVerb(ctx context.Context, inv *inventory.Inventory, before, after map[string]any,
|
||||
module, where string) error {
|
||||
verb, through := throughAVerb()
|
||||
if !through {
|
||||
return nil
|
||||
}
|
||||
// Judged against the module's definition as the catalogue holds it: what it serves and which of its files
|
||||
// are trusted. A catalogue that cannot be read refuses rather than judging against nothing.
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("which settings of %s are the terminal's cannot be read, so nothing was changed: %w", module, err)
|
||||
}
|
||||
// A trusted mergeable file takes any key, so its module's whole layer is the terminal's (novox/hq issue 340).
|
||||
if files := catalogue.TrustedMergeable(shelf[module]); len(files) > 0 && !sameLayer(before, after) {
|
||||
return fmt.Errorf("the settings of %s on %s are set at the controller's terminal only (`mesh-cli` on the control-node), never through a verb (this "+
|
||||
"line came through %q): %s merges whatever key a layer sets into a file root or a consumer trusts, so "+
|
||||
"any key could point the module at a listener of the caller's, and whoever may call a verb includes "+
|
||||
"agents (novox/hq issue 340; a file nothing trusts says \"trusted\": false). Nothing was changed",
|
||||
module, where, verb, strings.Join(files, ", "))
|
||||
}
|
||||
for _, key := range catalogue.TerminalKeys(shelf[module]) {
|
||||
was, _ := json.Marshal(before[key])
|
||||
now, _ := json.Marshal(after[key])
|
||||
if string(was) == string(now) {
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf("%s of %s on %s is set at the controller's terminal only (`mesh-cli` on the control-node), never through a verb (this "+
|
||||
"line came through %q): it says where root creates and owns a module's directories, which of "+
|
||||
"the machine's paths are mounted into its container, what the mesh's consumers trust, or what a file "+
|
||||
"root or a person's session obeys takes, and whoever may call a verb includes agents (novox/hq issue 339; "+
|
||||
"issue 340 for a mergeable file's own keys). Nothing was changed", key, module, where, verb)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -232,9 +232,22 @@ func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool,
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// **A machine joining is on the network before it is anything else** (novox/hq ADR 0169). Its
|
||||
// token was issued for its tunnel key and gave it an address, so while that token can still be
|
||||
// used the hub carries it as a peer: it brings its tunnel up from the token and enrols over it.
|
||||
// When the token is spent the machine is on the network by what it runs, as every other is; when
|
||||
// it expires unused, the peer goes with it at the hub's next composition.
|
||||
joining, err := inv.NodesWithALiveToken(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
isJoining := map[string]bool{}
|
||||
for _, name := range joining {
|
||||
isJoining[name] = true
|
||||
}
|
||||
nodes := make([]overlay.Node, 0, len(places))
|
||||
for _, p := range places {
|
||||
if !on[p.Name] {
|
||||
if !on[p.Name] && !(isJoining[p.Name] && p.Key != "" && p.Address != "") {
|
||||
continue
|
||||
}
|
||||
n := overlay.Node{
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// asksWithPaths records every build a merge asks for, as repository, path and ref.
|
||||
func asksWithPaths(t *testing.T) *[][3]string {
|
||||
t.Helper()
|
||||
var asked [][3]string
|
||||
was := askABuild
|
||||
askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) {
|
||||
asked = append(asked, [3]string{source.Repository, path, ref})
|
||||
return "b-" + path, nil
|
||||
}
|
||||
t.Cleanup(func() { askABuild = was })
|
||||
return &asked
|
||||
}
|
||||
|
||||
// The live case of 2026-10-07 (novox/hq issue 300), replayed: a merge of the catalogue adds one module's
|
||||
// directory and touches nothing else the mesh holds. Its delivery plan said "builds new: …, sent nowhere";
|
||||
// the merge said it changed nothing any module the mesh holds is built from, and built nothing, so the
|
||||
// module was never registered and `assign` refused it. The merge asks for its build, at the branch merged
|
||||
// into, from the repository as the mesh spells it — and opens no plan, since nothing it holds moved.
|
||||
func TestAMergeBuildsTheNewModuleItAdds(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asked := asksWithPaths(t)
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "networkmanager", Version: "1"},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/networkmanager", Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "3da80a4b00",
|
||||
Paths: []string{"modules/systemd-resolved/module.json", "modules/systemd-resolved/cmd/main.go"},
|
||||
ModuleDirs: []string{"modules/systemd-resolved"}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := [3]string{"novox/mesh-catalog", "modules/systemd-resolved", "main"}
|
||||
if len(*asked) != 1 || (*asked)[0] != want {
|
||||
t.Fatalf("the merge adding modules/systemd-resolved asked for %v, not %v", *asked, want)
|
||||
}
|
||||
if plans, err := inv.OpenPlans(ctx); err != nil || len(plans) != 0 {
|
||||
t.Fatalf("a merge moving nothing the mesh holds opened a plan: %+v %v", plans, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge that changes a held module and adds a new one does both: the held one walks its plan, the new
|
||||
// one is asked for beside it. A merge adding nothing builds nothing new.
|
||||
func TestAMergeBuildsItsNewModuleBesideItsPlan(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asked := asksWithPaths(t)
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1aaaaaaaa",
|
||||
Paths: []string{"modules/app/index.ts", "modules/fresh/module.json"},
|
||||
ModuleDirs: []string{"modules/app", "modules/fresh"}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
paths := map[string]bool{}
|
||||
for _, a := range *asked {
|
||||
paths[a[1]] = true
|
||||
}
|
||||
if !paths["modules/fresh"] || !paths["modules/app"] || len(*asked) != 2 {
|
||||
t.Fatalf("asked %v; want the new module and the plan's first tier", *asked)
|
||||
}
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil || len(plans) != 1 || plans[0].Modules["fresh"] != nil || plans[0].Modules["app"] == nil {
|
||||
t.Fatalf("the plan should walk app alone: %+v %v", plans, err)
|
||||
}
|
||||
|
||||
*asked = nil
|
||||
m2 := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c2aaaaaaaa",
|
||||
Paths: []string{"README.md"}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(*asked) != 0 {
|
||||
t.Fatalf("a merge adding no module asked for %v", *asked)
|
||||
}
|
||||
}
|
||||
@@ -2,15 +2,20 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/token"
|
||||
)
|
||||
@@ -95,6 +100,12 @@ func nodeCommand(ctx context.Context, args []string) error {
|
||||
"containers reaching outward. The machine reports which of its links face outside; see " +
|
||||
"`node show <name>`")
|
||||
|
||||
case "agent-account":
|
||||
// The account agents run as on this machine, when it is not the operator's (novox/hq ADR 0266). Here,
|
||||
// at the controller's terminal, and nowhere else: no verb and no setting names it, so no agent can
|
||||
// name itself another account.
|
||||
return nodeAgentAccount(ctx, inv, args[1:])
|
||||
|
||||
case "account":
|
||||
// The operator's login on this machine (novox/hq to-be 29): what a home-scoped file is
|
||||
// owned by and which account `ssh <node>` uses. Reports with no argument; sets with one;
|
||||
@@ -102,7 +113,7 @@ func nodeCommand(ctx context.Context, args []string) error {
|
||||
return nodeAccount(ctx, inv, args[1:])
|
||||
|
||||
default:
|
||||
return fmt.Errorf("node has no %q; it has add, list, show, public-domain and account", args[0])
|
||||
return fmt.Errorf("node has no %q; it has add, list, show, public-domain, account and agent-account", args[0])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -173,6 +184,57 @@ func nodeAccount(ctx context.Context, inv *inventory.Inventory, positionals []st
|
||||
return nil
|
||||
}
|
||||
|
||||
const agentAccountUsage = "node agent-account <name> — what it is now; " +
|
||||
"<name> <account> [home] to name the account agents run as (home defaults to /home/<account>); " +
|
||||
"<name> --clear to have them run as the operator account again"
|
||||
|
||||
// nodeAgentAccount reports, names or clears the account agents run as on a node (novox/hq ADR 0266). Read-
|
||||
// shaped with no account, like public-domain; clearing is asked for by name.
|
||||
func nodeAgentAccount(ctx context.Context, inv *inventory.Inventory, args []string) error {
|
||||
set := flag.NewFlagSet("node agent-account", flag.ContinueOnError)
|
||||
clear := set.Bool("clear", false, "agents run as the operator account again")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) == 0 || len(positionals) > 3 {
|
||||
return errors.New(agentAccountUsage)
|
||||
}
|
||||
node := positionals[0]
|
||||
switch {
|
||||
case *clear && len(positionals) > 1:
|
||||
return fmt.Errorf("name an agent account for %s or --clear, not both", node)
|
||||
case *clear:
|
||||
if err := inv.SetAgentAccount(ctx, node, "", ""); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("agents on %s run as the operator account again\n", node)
|
||||
fmt.Printf(" run `push %s` to send it; the agent account itself is kept (the mesh never deletes a login)\n", node)
|
||||
return nil
|
||||
case len(positionals) >= 2:
|
||||
home := ""
|
||||
if len(positionals) == 3 {
|
||||
home = positionals[2]
|
||||
}
|
||||
if err := inv.SetAgentAccount(ctx, node, positionals[1], home); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("agents on %s run as %s\n", node, positionals[1])
|
||||
fmt.Printf(" run `push %s` to send it; the self-check says once its node-engine has judged it "+
|
||||
"unable to become root\n", node)
|
||||
return nil
|
||||
default:
|
||||
n, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, line := range agentAccountLines(ctx, inv, n) {
|
||||
fmt.Println(strings.TrimPrefix(line, " "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
const publicDomainUsage = "node public-domain <name> — what it is now; " +
|
||||
"<name> <domain> to set it; <name> --clear to take it away"
|
||||
|
||||
@@ -247,6 +309,8 @@ func tokenCommand(ctx context.Context, args []string) error {
|
||||
validFor := set.Duration("for", time.Hour, "how long the token may be used")
|
||||
adopted := set.Bool("adopted", false,
|
||||
"the machine joining is in use: it is adopted, and keeps what is found on it")
|
||||
tunnelKey := set.String("overlay-key", "",
|
||||
"the public half of the tunnel key the machine made (`nox-mesh-host key`): it joins through the tunnel")
|
||||
if err := set.Parse(args[1:]); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -300,6 +364,14 @@ func tokenCommand(ctx context.Context, args []string) error {
|
||||
default:
|
||||
return err
|
||||
}
|
||||
// **Through the tunnel** (novox/hq ADR 0169): the machine's key recorded, its address given, the
|
||||
// hub sent it as a peer — all before the token is shown, so the tunnel answers the first time the
|
||||
// machine knocks. The bus is then reached at its address on the private network.
|
||||
if *tunnelKey != "" {
|
||||
if made.Tunnel, made.Broker, err = throughTheTunnel(ctx, open, issued.Node, *tunnelKey, made.Broker); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
encoded, err := made.Encode()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -324,6 +396,78 @@ func tokenCommand(ctx context.Context, args []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// throughTheTunnel makes a machine a peer of the hub for its token, and says what the token carries
|
||||
// for it: its first tunnel, and the bus at its address on the private network (novox/hq ADR 0169).
|
||||
//
|
||||
// The hub is pushed here, before the token is shown. A token shown before the hub knew the key is a
|
||||
// tunnel that does not answer, and a machine that cannot tell that from a bus that is down.
|
||||
func throughTheTunnel(ctx context.Context, open *stores, node inventory.Node, key, busAt string) (
|
||||
*token.Tunnel, string, error) {
|
||||
inv := open.inventory
|
||||
key = strings.TrimSpace(key)
|
||||
if raw, err := base64.StdEncoding.DecodeString(key); err != nil || len(raw) != 32 {
|
||||
return nil, "", fmt.Errorf("%q is not a tunnel public key: it is 32 bytes in base64, as "+
|
||||
"`nox-mesh-host key` prints it", key)
|
||||
}
|
||||
// The bus on the private network is its holder's address at the bus's own port, so the port must
|
||||
// be known before anything is recorded.
|
||||
_, port, err := net.SplitHostPort(busAt)
|
||||
if err != nil || port == "" {
|
||||
return nil, "", fmt.Errorf("the bus's address %q has no port to reach it on", busAt)
|
||||
}
|
||||
places, err := inv.Overlays(ctx)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
var hub *inventory.Overlay
|
||||
for i := range places {
|
||||
if places[i].Hub {
|
||||
hub = &places[i]
|
||||
}
|
||||
}
|
||||
if hub == nil || hub.Key == "" || hub.Endpoint == "" || hub.Address == "" {
|
||||
return nil, "", errors.New("this mesh has no hub with a key, an address and an endpoint to " +
|
||||
"dial, so there is no tunnel to join through: place one (`overlay place <node> --hub " +
|
||||
"--endpoint <host>:<port>`), or issue the token without --overlay-key")
|
||||
}
|
||||
if err := inv.RecordOverlayKey(ctx, node.ID, key); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if err := inv.BindTokenToKey(ctx, node.ID, key); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
cidr, err := overlayRange(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
address, err := inv.AssignAddress(ctx, node.ID, cidr)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
// The bus at its holder's address on the private network, reached through the hub like the rest
|
||||
// of the range; the hub's own when no machine is recorded as holding it yet.
|
||||
busAddress := hub.Address
|
||||
if holders, err := seatHolders(ctx, inv); err != nil {
|
||||
return nil, "", err
|
||||
} else if h, held := holders[catalogue.BrokerSeat]; held {
|
||||
for _, p := range places {
|
||||
if p.Name == h.Node && p.Address != "" {
|
||||
busAddress = p.Address
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := sendTo(ctx, open, []string{hub.Name}); err != nil {
|
||||
return nil, "", fmt.Errorf("%s was made a peer of the hub, and the hub could not be sent "+
|
||||
"it, so the tunnel would not answer — the token is not shown; issue it again once %s "+
|
||||
"can be pushed: %w", node.Name, hub.Name, err)
|
||||
}
|
||||
// An address, not a name — nothing resolves before the machine has joined (novox/hq ADR 0004).
|
||||
return &token.Tunnel{
|
||||
Key: key, Address: address + "/32", Range: cidr,
|
||||
HubKey: hub.Key, HubEndpoint: hub.Endpoint,
|
||||
}, net.JoinHostPort(busAddress, port), nil
|
||||
}
|
||||
|
||||
// issueFor is the inventory's half of issuing a token: the record, made when it is new, adopted
|
||||
// when the operator says so, and the one-time secret for it. The node in what it returns carries
|
||||
// its mode, which is what the token says.
|
||||
@@ -505,6 +649,10 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
|
||||
if err := showMode(ctx, inv, node); err != nil {
|
||||
return err
|
||||
}
|
||||
// Whom agents run as here, and whether that account can become root without a person (ADR 0266).
|
||||
for _, line := range agentAccountLines(ctx, inv, node) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
|
||||
// The domain its routed names are composed under, when it has one (novox/hq ADR 0066). Shown
|
||||
// only when set: a machine that serves nothing to the outside has no domain, and saying so of
|
||||
@@ -546,6 +694,12 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
|
||||
for _, line := range healthLines(h, had, time.Now()) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
for _, line := range networkLines(h, had, time.Now()) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
for _, line := range unitsLines(h, had, time.Now()) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
}
|
||||
|
||||
held, err := inv.Profile(ctx, name)
|
||||
@@ -593,11 +747,14 @@ func orNotReported(s string) string {
|
||||
}
|
||||
|
||||
// printJSON prints a value as indented JSON, the shape every `--json` answers in.
|
||||
func printJSON(v any) error {
|
||||
func printJSON(v any) error { return printJSONTo(os.Stdout, v) }
|
||||
|
||||
// printJSONTo is printJSON to a writer of the caller's.
|
||||
func printJSONTo(w io.Writer, v any) error {
|
||||
body, err := json.MarshalIndent(v, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
fmt.Fprintln(w, string(body))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,752 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A module not registered yet, and the assignment that waits for it (novox/hq issue 325, ADR 0261).
|
||||
//
|
||||
// On 2026-10-08 the catalogue's pull request adding `sensors` merged; a minute later `assign g14 sensors`
|
||||
// answered "no module of that name: sensors", and a few minutes later the same call worked. The merge had
|
||||
// asked for the build (issue 300) and the build had not finished. The answer read as "you forgot to register
|
||||
// it", and nothing in it said a build was on its way.
|
||||
//
|
||||
// So an assignment of a module the catalogue does not hold looks further before it refuses, and says which
|
||||
// of three cases it is in:
|
||||
//
|
||||
// - **a build is in flight**: a build request for the module's directory has no outcome yet, or its build
|
||||
// succeeded a moment ago and is being registered. The assignment is kept as a **pending assignment**,
|
||||
// which the controller makes when the build registers the module (ADR 0261).
|
||||
// - **known, not built**: the controller asked for the directory before, and the build failed, was not
|
||||
// registered, said nothing within its bound, or could not be asked. Said, with `build` and assign's
|
||||
// own build argument, which asks for the build and keeps the assignment pending on it.
|
||||
// - **unknown**: no module registered and no build request kept by that name. Refused as before, with the
|
||||
// closest names.
|
||||
//
|
||||
// **Pending by default while a build is in flight** (ADR 0261), without an argument to ask for it: an
|
||||
// assignment is what a person meant and is kept even when its machine does not resolve (acts.go), and
|
||||
// `unassign` withdraws it. Asking for a second act once the build lands is the two acts by hand issue 300
|
||||
// removed. A build is never asked for by default: it runs on another machine, and a directory whose last
|
||||
// build failed is a fault to look at before it is a thing to retry.
|
||||
//
|
||||
// **Settling is the controller's tick, never a read** (settlingPending): `status`, the board and the summary
|
||||
// only read the rows.
|
||||
|
||||
// buildRequestBound is how long a build request may go without an outcome before it is no longer read as in
|
||||
// flight, and a pending assignment waiting for it expires. Generous: a build waits behind others on the
|
||||
// build seat, and a pending assignment that waits a little longer costs nothing.
|
||||
const buildRequestBound = 2 * time.Hour
|
||||
|
||||
// registerGrace is how long a build heard as built is read as still in flight while it is not registered:
|
||||
// the outcome is recorded first and registered after it, in the same take-in. Past it, the build was
|
||||
// recorded and not registered, and says so.
|
||||
const registerGrace = 5 * time.Minute
|
||||
|
||||
// pendingShownFor is how long an ended pending assignment is listed in `status`.
|
||||
const pendingShownFor = 24 * time.Hour
|
||||
|
||||
// settleEvery is how often the controller settles the pending assignments.
|
||||
const settleEvery = time.Minute
|
||||
|
||||
// claimStaleAfter is how long a pending assignment may be held as applying before the tick reads the claim
|
||||
// as left by a controller that stopped, and settles it from what the mesh holds.
|
||||
const claimStaleAfter = 5 * time.Minute
|
||||
|
||||
// kindPendingEnded is a pending assignment that ended without being made: expired or refused.
|
||||
const kindPendingEnded = "assignment-not-made"
|
||||
|
||||
// assignOptions are what an assignment was asked with beside its machine and modules.
|
||||
type assignOptions struct {
|
||||
// Build asks for the build of a module known and not built, and keeps the assignment pending on it.
|
||||
Build bool
|
||||
}
|
||||
|
||||
// recordBuildRequest keeps a build request so `assign` can find it, once it is asked. Said, never fatal: the
|
||||
// build was asked.
|
||||
func recordBuildRequest(ctx context.Context, inv *inventory.Inventory, r inventory.BuildRequest) {
|
||||
if err := inv.RecordBuildRequest(ctx, r); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "build %s was asked, and could not be kept as a build request, so `assign` "+
|
||||
"will not know it is coming: %v\n", r.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The cases of a module the catalogue does not hold (novox/hq issue 325).
|
||||
const (
|
||||
unknownModule = iota
|
||||
buildInFlight
|
||||
knownNotBuilt
|
||||
)
|
||||
|
||||
// notHeld is where a module the catalogue does not hold stands: its case, the build request it was read
|
||||
// from, and the sentence that says it.
|
||||
type notHeld struct {
|
||||
kind int
|
||||
request inventory.RequestOutcome
|
||||
said string
|
||||
}
|
||||
|
||||
// whereIs looks for a module the catalogue does not hold among the build requests the controller keeps.
|
||||
func whereIs(ctx context.Context, inv *inventory.Inventory, module string, now time.Time) (notHeld, error) {
|
||||
requests, err := inv.RequestsNamed(ctx, module)
|
||||
if err != nil {
|
||||
return notHeld{}, err
|
||||
}
|
||||
if r, ok := inFlight(requests, now); ok {
|
||||
being := "being built"
|
||||
if r.Heard {
|
||||
being = "built and being registered"
|
||||
}
|
||||
return notHeld{kind: buildInFlight, request: r, said: fmt.Sprintf("%s is not registered yet: it is %s "+
|
||||
"from %s since %s, as build %s; it can be assigned once that build registers it", module, being,
|
||||
requestSource(r), clock(r.At), r.ID)}, nil
|
||||
}
|
||||
if len(requests) > 0 {
|
||||
return notHeld{kind: knownNotBuilt, request: requests[0], said: fmt.Sprintf("%s is known and not "+
|
||||
"registered: %s", module, whyNotBuilt(requests[0], now))}, nil
|
||||
}
|
||||
// Only what was looked at is claimed: the catalogue, and the build requests the controller keeps.
|
||||
said := fmt.Sprintf("%v: %s — the catalogue holds no module of that name, and no build request the "+
|
||||
"controller kept (the last 30 days) is for a directory of that name", inventory.ErrNoSuchModule, module)
|
||||
if near := closestNames(ctx, inv, module); len(near) > 0 {
|
||||
said += "; the closest names it holds: " + strings.Join(near, ", ")
|
||||
}
|
||||
return notHeld{kind: unknownModule, said: said}, nil
|
||||
}
|
||||
|
||||
// notInCatalogue is the modules named that the catalogue does not hold.
|
||||
func notInCatalogue(ctx context.Context, open *stores, modules []string) ([]string, error) {
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var missing []string
|
||||
for _, m := range modules {
|
||||
if _, ok := shelf[m]; !ok {
|
||||
missing = append(missing, m)
|
||||
}
|
||||
}
|
||||
return missing, nil
|
||||
}
|
||||
|
||||
// openPending is the open pending assignment of a module to a machine, if there is one.
|
||||
func openPending(ctx context.Context, inv *inventory.Inventory, node, module string) (*inventory.PendingAssignment, error) {
|
||||
rows, err := inv.PendingFor(ctx, node, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range rows {
|
||||
if rows[i].Open() {
|
||||
return &rows[i], nil
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// notRegistered answers an assignment of a module the catalogue does not hold: pending on a build in flight,
|
||||
// known and not built, or unknown (novox/hq issue 325). An answer with no error is a pending assignment kept.
|
||||
func notRegistered(ctx context.Context, open *stores, node, module string, opts assignOptions) (string, error) {
|
||||
inv := open.inventory
|
||||
if _, err := inv.NodeByName(ctx, node); err != nil {
|
||||
return "", err
|
||||
}
|
||||
now := time.Now()
|
||||
where, err := whereIs(ctx, inv, module, now)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
waiting, err := openPending(ctx, inv, node, module)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
switch where.kind {
|
||||
case buildInFlight:
|
||||
lead := where.said
|
||||
if opts.Build {
|
||||
lead += "\n no second build was asked: this one is running"
|
||||
}
|
||||
return keepPending(ctx, open, node, module, where.request, waiting, lead)
|
||||
case knownNotBuilt:
|
||||
last := where.request
|
||||
if !opts.Build {
|
||||
also := ""
|
||||
if waiting != nil {
|
||||
also = fmt.Sprintf("\n %s already has a pending assignment of %s, waiting for build %s; build "+
|
||||
"\"true\" asks for a new build and makes it wait for that one", node, module, waiting.Build)
|
||||
}
|
||||
return "", fmt.Errorf("%w: %s%s\n assign it again with build \"true\" to ask for its build and keep this "+
|
||||
"assignment until the build registers it; or `build` it (repository %s, path %s) and assign it "+
|
||||
"once it is registered", inventory.ErrNoSuchModule, where.said, also, last.Repository, orRoot(last.Path))
|
||||
}
|
||||
if waiting != nil && waiting.State == inventory.PendingApplying {
|
||||
return "", fmt.Errorf("%s is being assigned %s now; nothing was asked", node, module)
|
||||
}
|
||||
source := buildSource{Repository: last.Repository, Seat: last.Seat}
|
||||
id, err := askABuild(ctx, source, last.Path, last.Ref)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s\n its build could not be asked for: %w; nothing was assigned", where.said, err)
|
||||
}
|
||||
asked := inventory.RequestOutcome{BuildRequest: inventory.BuildRequest{ID: id, Repository: last.Repository,
|
||||
Seat: last.Seat, Path: last.Path, Ref: last.Ref, For: "assign", At: now.UTC()}}
|
||||
recordBuildRequest(ctx, inv, asked.BuildRequest)
|
||||
lead := fmt.Sprintf("%s\n build %s of %s is asked for now; %s can be assigned once it registers it",
|
||||
where.said, id, requestSource(asked), module)
|
||||
return keepPending(ctx, open, node, module, asked, waiting, lead)
|
||||
}
|
||||
answer := strings.TrimPrefix(where.said, inventory.ErrNoSuchModule.Error()) +
|
||||
"\n a module in a repository is built with `build` (its repository and path), then assigned"
|
||||
if opts.Build {
|
||||
answer += "\n build \"true\" asks for nothing here: the controller has no build request saying where a " +
|
||||
"module of that name is"
|
||||
}
|
||||
return "", fmt.Errorf("%w%s", inventory.ErrNoSuchModule, answer)
|
||||
}
|
||||
|
||||
// keepPending records the assignment as pending on a build request — or, where one is already open, makes it
|
||||
// wait for that build — and says so.
|
||||
func keepPending(ctx context.Context, open *stores, node, module string, r inventory.RequestOutcome,
|
||||
waiting *inventory.PendingAssignment, lead string) (string, error) {
|
||||
inv := open.inventory
|
||||
if waiting != nil {
|
||||
if waiting.Build == r.ID {
|
||||
return lead + fmt.Sprintf("\n %s already waits for %s on this build: nothing changed", node, module), nil
|
||||
}
|
||||
moved, err := inv.RepointPending(ctx, waiting.ID, r.ID, r.Repository, r.Path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !moved {
|
||||
return lead + fmt.Sprintf("\n the pending assignment of %s to %s ended while this was asked: `status` "+
|
||||
"says how", module, node), nil
|
||||
}
|
||||
return lead + fmt.Sprintf("\n the pending assignment of %s to %s, which waited for build %s, now waits "+
|
||||
"for build %s", module, node, waiting.Build, r.ID), nil
|
||||
}
|
||||
_, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: node, Module: module, Build: r.ID,
|
||||
Repository: r.Repository, Path: r.Path})
|
||||
if errors.Is(err, inventory.ErrAlreadyPending) {
|
||||
return lead + fmt.Sprintf("\n %s already waits for %s: nothing changed", node, module), nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// The controller makes it, not this act (ADR 0261): when the build registers the module, or at its next
|
||||
// tick if the module was registered between the look and the record.
|
||||
return lead + fmt.Sprintf("\n the assignment is kept as pending: the controller assigns %s to %s when the "+
|
||||
"build registers it, and `status` lists it until then. If the build fails it expires, says why and "+
|
||||
"raises a condition; `unassign %s %s` withdraws it", module, node, node, module), nil
|
||||
}
|
||||
|
||||
// inFlight is the newest build request for the module still to register it: no outcome yet within the
|
||||
// bound, or a successful outcome heard a moment ago and being registered.
|
||||
func inFlight(requests []inventory.RequestOutcome, now time.Time) (inventory.RequestOutcome, bool) {
|
||||
for _, r := range requests {
|
||||
if stillComing(r, now) {
|
||||
return r, true
|
||||
}
|
||||
}
|
||||
return inventory.RequestOutcome{}, false
|
||||
}
|
||||
|
||||
func stillComing(r inventory.RequestOutcome, now time.Time) bool {
|
||||
if r.Heard {
|
||||
return r.Failed == "" && now.Sub(r.HeardAt) < registerGrace
|
||||
}
|
||||
return r.NotAsked == "" && now.Sub(r.At) < buildRequestBound
|
||||
}
|
||||
|
||||
// whyNotBuilt says what came of a module's last build request. What was heard comes first: an outcome is the
|
||||
// last word whatever its asker said.
|
||||
func whyNotBuilt(r inventory.RequestOutcome, now time.Time) string {
|
||||
where := fmt.Sprintf("%s in %s", orRoot(r.Path), r.Repository)
|
||||
switch {
|
||||
case r.Heard && r.Failed != "":
|
||||
return fmt.Sprintf("%s; its last build, %s at %s, failed: %s", where, r.ID, clock(r.HeardAt),
|
||||
firstLine(r.Failed))
|
||||
case r.Heard && r.Module != "" && r.Module != r.Name():
|
||||
return fmt.Sprintf("%s; its last build, %s, built it as %s", where, r.ID, r.Module)
|
||||
case r.Heard:
|
||||
return fmt.Sprintf("%s; its last build, %s at %s, was recorded and not registered — `builds` says why",
|
||||
where, r.ID, clock(r.HeardAt))
|
||||
case r.NotAsked != "" && r.For == "merge":
|
||||
return fmt.Sprintf("%s; the merge that added it (%s) could not ask for its build: %s", where,
|
||||
short(r.Commit), firstLine(r.NotAsked))
|
||||
case r.NotAsked != "":
|
||||
return fmt.Sprintf("%s; build %s, asked by %s, was not handed over: %s", where, r.ID,
|
||||
askerOr(r.For), firstLine(r.NotAsked))
|
||||
case r.OutcomeUnknown != "":
|
||||
return fmt.Sprintf("%s; build %s was asked at %s, its asker stopped waiting (%s), and no outcome has "+
|
||||
"been heard in %s", where, r.ID, clock(r.At), firstLine(r.OutcomeUnknown), now.Sub(r.At).Round(time.Minute))
|
||||
default:
|
||||
return fmt.Sprintf("%s; build %s was asked at %s, and no outcome has been heard in %s", where, r.ID,
|
||||
clock(r.At), now.Sub(r.At).Round(time.Minute))
|
||||
}
|
||||
}
|
||||
|
||||
func askerOr(who string) string {
|
||||
if who == "" {
|
||||
return "the controller"
|
||||
}
|
||||
return who
|
||||
}
|
||||
|
||||
// requestSource is a build request's source as a person reads it: repository@commit (directory).
|
||||
func requestSource(r inventory.RequestOutcome) string {
|
||||
at := short(r.Commit)
|
||||
if at == "" {
|
||||
at = r.Ref
|
||||
}
|
||||
if at == "" {
|
||||
at = "its default branch"
|
||||
}
|
||||
return fmt.Sprintf("%s@%s (%s)", r.Repository, at, orRoot(r.Path))
|
||||
}
|
||||
|
||||
// clock is a moment as a person reads it, in the controller's local time.
|
||||
func clock(t time.Time) string { return t.Local().Format("2006-01-02 15:04:05 MST") }
|
||||
|
||||
// closestNames is up to three names near the one asked: registered modules and directories the controller
|
||||
// asked to build.
|
||||
func closestNames(ctx context.Context, inv *inventory.Inventory, name string) []string {
|
||||
var candidates []string
|
||||
if shelf, err := inv.Catalogue(ctx); err == nil {
|
||||
for m := range shelf {
|
||||
candidates = append(candidates, m)
|
||||
}
|
||||
}
|
||||
if asked, err := inv.RequestedNames(ctx); err == nil {
|
||||
candidates = append(candidates, asked...)
|
||||
}
|
||||
type scored struct {
|
||||
name string
|
||||
distance int
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var near []scored
|
||||
limit := max(2, len(name)/3)
|
||||
for _, c := range candidates {
|
||||
if seen[c] || c == name {
|
||||
continue
|
||||
}
|
||||
seen[c] = true
|
||||
d := editDistance(name, c)
|
||||
if d <= limit || strings.Contains(c, name) || strings.Contains(name, c) {
|
||||
near = append(near, scored{c, d})
|
||||
}
|
||||
}
|
||||
sort.Slice(near, func(i, j int) bool {
|
||||
if near[i].distance != near[j].distance {
|
||||
return near[i].distance < near[j].distance
|
||||
}
|
||||
return near[i].name < near[j].name
|
||||
})
|
||||
var out []string
|
||||
for i := 0; i < len(near) && i < 3; i++ {
|
||||
out = append(out, near[i].name)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// editDistance is the Levenshtein distance between two names.
|
||||
func editDistance(a, b string) int {
|
||||
ra, rb := []rune(a), []rune(b)
|
||||
prev := make([]int, len(rb)+1)
|
||||
for j := range prev {
|
||||
prev[j] = j
|
||||
}
|
||||
for i := 1; i <= len(ra); i++ {
|
||||
cur := make([]int, len(rb)+1)
|
||||
cur[0] = i
|
||||
for j := 1; j <= len(rb); j++ {
|
||||
cost := 1
|
||||
if ra[i-1] == rb[j-1] {
|
||||
cost = 0
|
||||
}
|
||||
cur[j] = min(prev[j]+1, cur[j-1]+1, prev[j-1]+cost)
|
||||
}
|
||||
prev = cur
|
||||
}
|
||||
return prev[len(rb)]
|
||||
}
|
||||
|
||||
// applyPending makes a pending assignment now that its module is registered (ADR 0261). Under the machine's
|
||||
// hold it claims the row (waiting → applying), so a withdrawal cannot land between the look and the act,
|
||||
// then assigns by the same act a person's assign is, then says what came of it: applied, or refused with the
|
||||
// refusal. Returns what it said, or nothing when the row no longer waited.
|
||||
func applyPending(ctx context.Context, open *stores, p inventory.PendingAssignment, by string) string {
|
||||
inv := open.inventory
|
||||
ctx, release, err := holdNodes(ctx, open, []string{p.Node})
|
||||
if err != nil {
|
||||
// Left waiting: the next tick tries again.
|
||||
return fmt.Sprintf("the pending assignment of %s to %s waits: %s could not be held: %v", p.Module, p.Node,
|
||||
p.Node, err)
|
||||
}
|
||||
defer release()
|
||||
claimed, err := inv.ClaimPending(ctx, p.ID)
|
||||
if err != nil {
|
||||
return fmt.Sprintf("the pending assignment of %s to %s could not be taken for making: %v", p.Module, p.Node, err)
|
||||
}
|
||||
if !claimed {
|
||||
return ""
|
||||
}
|
||||
answer, err := assign(ctx, open, p.Node, p.Module)
|
||||
state, note := inventory.PendingApplied, ""
|
||||
switch {
|
||||
case err != nil:
|
||||
state = inventory.PendingRefused
|
||||
note = fmt.Sprintf("the pending assignment of %s to %s was refused when build %s registered it: %s",
|
||||
p.Module, p.Node, by, firstLine(err.Error()))
|
||||
case strings.Contains(answer, "already runs"):
|
||||
note = fmt.Sprintf("%s already runs %s: the pending assignment is met", p.Node, p.Module)
|
||||
default:
|
||||
// The same as a person's assign: nothing is sent by this act.
|
||||
note = fmt.Sprintf("%s is assigned %s: build %s registered it (pending since %s); `push %s` sends it",
|
||||
p.Node, p.Module, by, clock(p.Since), p.Node)
|
||||
}
|
||||
settled, serr := inv.SettlePending(ctx, p.ID, inventory.PendingApplying, state, note)
|
||||
if serr != nil {
|
||||
return fmt.Sprintf("%s — and it could not be recorded as settled: %v", note, serr)
|
||||
}
|
||||
if !settled {
|
||||
return ""
|
||||
}
|
||||
return note
|
||||
}
|
||||
|
||||
// expirePending ends a waiting pending assignment whose build will not register its module, with why.
|
||||
func expirePending(ctx context.Context, inv *inventory.Inventory, p inventory.PendingAssignment, why string) string {
|
||||
note := fmt.Sprintf("the pending assignment of %s to %s expired: %s; nothing was assigned. Assign it again with "+
|
||||
"build \"true\" to ask for the build again", p.Module, p.Node, why)
|
||||
settled, err := inv.SettlePending(ctx, p.ID, inventory.PendingWaiting, inventory.PendingExpired, note)
|
||||
if err != nil {
|
||||
return fmt.Sprintf("%s — and it could not be recorded: %v", note, err)
|
||||
}
|
||||
if !settled {
|
||||
return ""
|
||||
}
|
||||
return note
|
||||
}
|
||||
|
||||
// settlePendingOnBuild is a build's outcome reaching the assignments waiting for it, wherever the outcome is
|
||||
// taken in (novox/hq issue 325): a module registered makes every assignment pending on it; a build of a
|
||||
// pending assignment that failed, or was not registered, ends it with why. Returns what was said.
|
||||
func settlePendingOnBuild(ctx context.Context, open *stores, result link.BuildResult, module string, takeErr error) []string {
|
||||
inv := open.inventory
|
||||
waiting, err := inv.Pending(ctx, time.Time{})
|
||||
if err != nil {
|
||||
return []string{fmt.Sprintf("the pending assignments could not be read after build %s: %v", result.ID, err)}
|
||||
}
|
||||
registered := module != "" && (takeErr == nil || errors.Is(takeErr, inventory.ErrSuperseded))
|
||||
var said []string
|
||||
for _, p := range waiting {
|
||||
switch {
|
||||
case registered && p.Module == module:
|
||||
if line := applyPending(ctx, open, p, result.ID); line != "" {
|
||||
said = append(said, line)
|
||||
}
|
||||
case !registered && p.Build == result.ID:
|
||||
why := firstLine(result.Failed)
|
||||
if why == "" && takeErr != nil {
|
||||
why = firstLine(takeErr.Error())
|
||||
}
|
||||
what := fmt.Sprintf("build %s of %s failed: %s", result.ID, orRoot(p.Path), why)
|
||||
if result.Failed == "" {
|
||||
what = fmt.Sprintf("build %s of %s was recorded and not registered: %s", result.ID, orRoot(p.Path), why)
|
||||
}
|
||||
if line := expirePending(ctx, inv, p, what); line != "" {
|
||||
said = append(said, line)
|
||||
}
|
||||
}
|
||||
}
|
||||
return said
|
||||
}
|
||||
|
||||
// settlingPending is the controller's tick over the pending assignments (ADR 0261), every settleEvery until
|
||||
// the context ends. What it does is printed to the controller's log, kept in each row's note (which `status`
|
||||
// lists for a day), and raised as a condition for an assignment that was not made.
|
||||
func settlingPending(ctx context.Context, open *stores) {
|
||||
for {
|
||||
for _, line := range settlePending(ctx, open, time.Now()) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
for _, line := range raiseUnknownFields(ctx, open.inventory) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(settleEvery):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// settlePending is one tick: claims left by a controller that stopped are settled from what the mesh holds;
|
||||
// a module registered meanwhile is assigned; a build heard and not registered, or silent past its bound,
|
||||
// ends its assignment; ended ones are raised as conditions and cleared once answered; ended rows older than
|
||||
// KeptFor are deleted. Returns what it did.
|
||||
func settlePending(ctx context.Context, open *stores, now time.Time) []string {
|
||||
inv := open.inventory
|
||||
var said []string
|
||||
say := func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }
|
||||
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
say("the pending assignments are not settled this time: the catalogue could not be read: %v", err)
|
||||
return said
|
||||
}
|
||||
stuck, err := inv.Stuck(ctx, now.Add(-claimStaleAfter))
|
||||
if err != nil {
|
||||
say("claims left by a stopped controller could not be read: %v", err)
|
||||
}
|
||||
for _, p := range stuck {
|
||||
assigned, err := inv.Assigned(ctx, p.Node)
|
||||
if err != nil {
|
||||
say("the claim on %s for %s could not be settled: %v", p.Node, p.Module, err)
|
||||
continue
|
||||
}
|
||||
if containsString(assigned, p.Module) {
|
||||
if _, err := inv.SettlePending(ctx, p.ID, inventory.PendingApplying, inventory.PendingApplied,
|
||||
fmt.Sprintf("%s is assigned %s (settled after the controller that made it stopped)", p.Node, p.Module)); err != nil {
|
||||
say("the claim on %s for %s could not be settled: %v", p.Node, p.Module, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := inv.ReleaseClaim(ctx, p.ID); err != nil {
|
||||
say("the claim on %s for %s could not be released: %v", p.Node, p.Module, err)
|
||||
}
|
||||
}
|
||||
|
||||
waiting, err := inv.Pending(ctx, time.Time{})
|
||||
if err != nil {
|
||||
say("the pending assignments could not be read to settle them: %v", err)
|
||||
}
|
||||
for _, p := range waiting {
|
||||
if _, ok := shelf[p.Module]; ok {
|
||||
if line := applyPending(ctx, open, p, p.Build); line != "" {
|
||||
said = append(said, line)
|
||||
}
|
||||
continue
|
||||
}
|
||||
requests, err := inv.RequestsNamed(ctx, p.Module)
|
||||
if err != nil {
|
||||
say("the build requests for %s could not be read: %v", p.Module, err)
|
||||
continue
|
||||
}
|
||||
var r *inventory.RequestOutcome
|
||||
for i := range requests {
|
||||
if requests[i].ID == p.Build {
|
||||
r = &requests[i]
|
||||
}
|
||||
}
|
||||
why := ""
|
||||
switch {
|
||||
case r == nil && now.Sub(p.Since) > buildRequestBound:
|
||||
why = fmt.Sprintf("build %s is no longer on record", p.Build)
|
||||
case r != nil && !stillComing(*r, now) && (r.Heard || r.NotAsked != "" || r.OutcomeUnknown != ""):
|
||||
why = whyNotBuilt(*r, now)
|
||||
case r != nil && !stillComing(*r, now):
|
||||
why = fmt.Sprintf("no outcome of build %s was heard within %s of asking", p.Build, buildRequestBound)
|
||||
}
|
||||
if why == "" {
|
||||
continue
|
||||
}
|
||||
if line := expirePending(ctx, inv, p, why); line != "" {
|
||||
said = append(said, line)
|
||||
}
|
||||
}
|
||||
|
||||
said = append(said, raisePendingEnded(ctx, inv)...)
|
||||
if n, err := inv.ForgetEndedPending(ctx, now); err != nil {
|
||||
say("ended pending assignments could not be deleted: %v", err)
|
||||
} else if n > 0 {
|
||||
say("deleted %d pending assignment(s) ended more than %s ago", n, inventory.KeptFor)
|
||||
}
|
||||
return said
|
||||
}
|
||||
|
||||
func containsString(xs []string, x string) bool {
|
||||
for _, y := range xs {
|
||||
if y == x {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// pendingObservation is the condition for a pending assignment that was not made: one per pending
|
||||
// assignment, its row's id the last part of its id, so one row's condition is never another's.
|
||||
func pendingObservation(p inventory.PendingAssignment) conditions.Observation {
|
||||
return conditions.Observation{Scope: conditions.ScopeMachine, ID: fmt.Sprintf("%s.%s.%d", p.Node, p.Module, p.ID),
|
||||
Token: kindPendingEnded, Kind: kindPendingEnded, Machine: p.Node, Severity: conditions.Warning,
|
||||
Resolver: conditions.ResolverOperator, Source: "pending assignments",
|
||||
Summary: p.Note}
|
||||
}
|
||||
|
||||
// raisePendingEnded raises a condition for each pending assignment that expired or was refused, once, and
|
||||
// clears it when it is answered: the module was assigned to that machine since, a newer pending assignment of
|
||||
// it is open, or a person took it back with `unassign`. Where this process keeps no conditions, nothing is
|
||||
// stamped, and the serving controller's tick raises it.
|
||||
func raisePendingEnded(ctx context.Context, inv *inventory.Inventory) []string {
|
||||
keeper := conditionsFrom
|
||||
if keeper == nil {
|
||||
return nil
|
||||
}
|
||||
var said []string
|
||||
toRaise, err := inv.ToRaise(ctx)
|
||||
if err != nil {
|
||||
return []string{fmt.Sprintf("pending assignments not made could not be read to raise them: %v", err)}
|
||||
}
|
||||
for _, p := range toRaise {
|
||||
if _, err := keeper.Observe(ctx, pendingObservation(p)); err != nil {
|
||||
said = append(said, fmt.Sprintf("the condition for %s on %s could not be raised: %v", p.Module, p.Node, err))
|
||||
continue
|
||||
}
|
||||
if err := inv.MarkPending(ctx, p.ID, "raised"); err != nil {
|
||||
said = append(said, fmt.Sprintf("the condition for %s on %s was raised and not recorded: %v", p.Module, p.Node, err))
|
||||
}
|
||||
}
|
||||
toClear, err := inv.ToClear(ctx)
|
||||
if err != nil {
|
||||
return append(said, fmt.Sprintf("pending assignments raised could not be read to clear them: %v", err))
|
||||
}
|
||||
for _, p := range toClear {
|
||||
why, answered, err := pendingAnswered(ctx, inv, p)
|
||||
if err != nil {
|
||||
said = append(said, fmt.Sprintf("whether %s on %s is answered could not be read: %v", p.Module, p.Node, err))
|
||||
continue
|
||||
}
|
||||
if !answered {
|
||||
continue
|
||||
}
|
||||
if _, err := keeper.Clear(ctx, pendingObservation(p).Key(), why); err != nil {
|
||||
said = append(said, fmt.Sprintf("the condition for %s on %s could not be cleared: %v", p.Module, p.Node, err))
|
||||
continue
|
||||
}
|
||||
if err := inv.MarkPending(ctx, p.ID, "cleared"); err != nil {
|
||||
said = append(said, fmt.Sprintf("the condition for %s on %s was cleared and not recorded: %v", p.Module, p.Node, err))
|
||||
}
|
||||
}
|
||||
return append(said, clearOrphaned(ctx, keeper, inv)...)
|
||||
}
|
||||
|
||||
// pendingAnswered says whether a pending assignment that was not made has been answered since, and how.
|
||||
func pendingAnswered(ctx context.Context, inv *inventory.Inventory, p inventory.PendingAssignment) (string, bool, error) {
|
||||
if p.Acknowledged != nil {
|
||||
return "taken back with unassign", true, nil
|
||||
}
|
||||
assigned, err := inv.Assigned(ctx, p.Node)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
if containsString(assigned, p.Module) {
|
||||
return p.Module + " is assigned to " + p.Node + " since", true, nil
|
||||
}
|
||||
rows, err := inv.PendingFor(ctx, p.Node, p.Module)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
// A newer pending assignment answers it only while it is open or once it was made: one that itself
|
||||
// ended unmade is its own condition, and answers nothing.
|
||||
for _, r := range rows {
|
||||
if r.ID != p.ID && r.Since.After(p.Since) && (r.Open() || r.State == inventory.PendingApplied) {
|
||||
return "assigned again, pending on another build", true, nil
|
||||
}
|
||||
}
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
// clearOrphaned clears every open condition of an assignment not made whose pending assignment is no longer
|
||||
// on record: its machine was removed, which takes its pending assignments with it.
|
||||
func clearOrphaned(ctx context.Context, keeper *conditions.Keeper, inv *inventory.Inventory) []string {
|
||||
open, err := keeper.Open(ctx)
|
||||
if err != nil {
|
||||
return []string{fmt.Sprintf("the open conditions could not be read to clear orphaned ones: %v", err)}
|
||||
}
|
||||
byID := map[int64]string{}
|
||||
var ids []int64
|
||||
for _, c := range open {
|
||||
if c.Kind != kindPendingEnded {
|
||||
continue
|
||||
}
|
||||
// `<scope>.<node>.<module>.<row>.<kind>`: the row is the part before the kind.
|
||||
parts := strings.Split(c.Key, ".")
|
||||
if len(parts) < 2 {
|
||||
continue
|
||||
}
|
||||
var id int64
|
||||
if _, err := fmt.Sscan(parts[len(parts)-2], &id); err != nil {
|
||||
continue
|
||||
}
|
||||
byID[id] = c.Key
|
||||
ids = append(ids, id)
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
known, err := inv.PendingKnown(ctx, ids)
|
||||
if err != nil {
|
||||
return []string{fmt.Sprintf("the pending assignments could not be read to clear orphaned conditions: %v", err)}
|
||||
}
|
||||
var said []string
|
||||
for id, key := range byID {
|
||||
if known[id] {
|
||||
continue
|
||||
}
|
||||
if _, err := keeper.Clear(ctx, key, "its pending assignment is no longer on record: its machine was removed"); err != nil {
|
||||
said = append(said, fmt.Sprintf("the condition %s could not be cleared: %v", key, err))
|
||||
}
|
||||
}
|
||||
return said
|
||||
}
|
||||
|
||||
// withdrawPending is `unassign` of a module only pending on a machine, under the machine's hold: a waiting
|
||||
// pending assignment is withdrawn; one being made now is refused, never overridden; one that expired or was
|
||||
// refused is taken back, which answers its condition. False when there is none.
|
||||
func withdrawPending(ctx context.Context, inv *inventory.Inventory, node, module string) (string, bool, error) {
|
||||
rows, err := inv.PendingFor(ctx, node, module)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
var takenBack []string
|
||||
for _, p := range rows {
|
||||
switch p.State {
|
||||
case inventory.PendingApplying:
|
||||
return "", true, fmt.Errorf("%s is being assigned %s now, as build %s registered it: nothing was "+
|
||||
"withdrawn; `unassign %s %s` again once it is assigned takes it off", node, module, p.Build, node, module)
|
||||
case inventory.PendingWaiting:
|
||||
note := fmt.Sprintf("the pending assignment of %s to %s is withdrawn; build %s goes on, and registers "+
|
||||
"%s assigned nowhere", module, node, p.Build, module)
|
||||
settled, err := inv.SettlePending(ctx, p.ID, inventory.PendingWaiting, inventory.PendingWithdrawn, note)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
if !settled {
|
||||
return "", true, fmt.Errorf("the pending assignment of %s to %s changed while it was withdrawn: "+
|
||||
"`status` says how; nothing was withdrawn", module, node)
|
||||
}
|
||||
return note, true, nil
|
||||
case inventory.PendingExpired, inventory.PendingRefused:
|
||||
if p.Cleared != nil || p.Acknowledged != nil {
|
||||
continue
|
||||
}
|
||||
if err := inv.MarkPending(ctx, p.ID, "acknowledged"); err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
takenBack = append(takenBack, fmt.Sprintf("build %s (%s)", p.Build, p.State))
|
||||
}
|
||||
}
|
||||
if len(takenBack) > 0 {
|
||||
return fmt.Sprintf("the pending assignment(s) of %s to %s that were not made are taken back: %s; their "+
|
||||
"conditions clear", module, node, strings.Join(takenBack, ", ")), true, nil
|
||||
}
|
||||
return "", false, nil
|
||||
}
|
||||
@@ -0,0 +1,864 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// failedBuild settles the merge's build of modules/sensors as failed.
|
||||
func failedBuild(t *testing.T, open *stores, id string) {
|
||||
t.Helper()
|
||||
if err := (builds{open.inventory, open}).Built(t.Context(), outcome(id, "modules/sensors", "3da80a4b00aa",
|
||||
"", "boom")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func conditionOpen(t *testing.T, p inventory.PendingAssignment) bool {
|
||||
t.Helper()
|
||||
_, found, err := conditionsFrom.Get(t.Context(), pendingObservation(p).Key())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return found
|
||||
}
|
||||
|
||||
func rowOf(t *testing.T, open *stores, id int64) inventory.PendingAssignment {
|
||||
t.Helper()
|
||||
rows, err := open.inventory.Pending(t.Context(), time.Unix(0, 0))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range rows {
|
||||
if r.ID == id {
|
||||
return r
|
||||
}
|
||||
}
|
||||
t.Fatalf("pending assignment %d is not on record", id)
|
||||
return inventory.PendingAssignment{}
|
||||
}
|
||||
|
||||
// novox/hq issue 325: an assignment of a module the catalogue does not hold says which case it is in — a
|
||||
// build in flight (kept pending), known and not built (said, with build), or unknown (refused, with the
|
||||
// closest names) — and a pending assignment is made when its build registers the module, or ends with why.
|
||||
|
||||
// aCatalogueMesh is aMesh with one module held from the catalogue, so a merge of it is acted on.
|
||||
func aCatalogueMesh(t *testing.T) *stores {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: "networkmanager", Version: "1"},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/networkmanager", Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return open
|
||||
}
|
||||
|
||||
// mergeAdding is the catalogue's merge adding one module's directory, acted on as the bus hands it over.
|
||||
func mergeAdding(t *testing.T, open *stores, dir, commit string) {
|
||||
t.Helper()
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit,
|
||||
Paths: []string{dir + "/module.json"}, ModuleDirs: []string{dir}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(t.Context(), m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// outcome is a build's outcome as the build seat announces it: registered as module, or failed.
|
||||
func outcome(id, dir, commit, module, failed string) link.BuildResult {
|
||||
r := link.BuildResult{ID: id, Repository: "novox/mesh-catalog", Path: dir, Ref: "main", Commit: commit,
|
||||
On: "anchor", Failed: failed, Source: &link.SourceOnSeat{Repository: "novox/mesh-catalog", Seat: "git"}}
|
||||
if failed == "" {
|
||||
r.Module = module
|
||||
r.Manifest, _ = json.Marshal(catalogue.Manifest{Module: module, Version: "1"})
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func assignedTo(t *testing.T, open *stores, node string) []string {
|
||||
t.Helper()
|
||||
got, err := open.inventory.Assigned(t.Context(), node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
func pendingOf(t *testing.T, open *stores) []inventory.PendingAssignment {
|
||||
t.Helper()
|
||||
got, err := open.inventory.Pending(t.Context(), time.Now().Add(-time.Hour))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
// A build in flight: the merge asked for it, the request is kept as the merge's, the assignment is kept
|
||||
// pending and said so with the build, status reads it without settling anything, and the build's outcome
|
||||
// makes it — saying that a push sends it, and nothing more.
|
||||
func TestAnAssignmentWaitsForTheBuildInFlight(t *testing.T) {
|
||||
open := aCatalogueMesh(t)
|
||||
ctx := t.Context()
|
||||
asksWithPaths(t)
|
||||
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
|
||||
|
||||
requests, err := open.inventory.RequestsNamed(ctx, "sensors")
|
||||
if err != nil || len(requests) != 1 || requests[0].For != "merge" || requests[0].Commit != "3da80a4b00aa" {
|
||||
t.Fatalf("the merge's build request: %+v %v", requests, err)
|
||||
}
|
||||
|
||||
said, err := assign(ctx, open, "laptop", "sensors")
|
||||
if err != nil {
|
||||
t.Fatalf("an assignment while its build runs was refused: %v", err)
|
||||
}
|
||||
t.Logf("assign laptop sensors, while its build runs:\n%s", said)
|
||||
for _, want := range []string{"being built from novox/mesh-catalog@3da80a4b (modules/sensors)",
|
||||
"build b-modules/sensors", "kept as pending", "the controller assigns sensors to laptop", "`unassign laptop sensors`"} {
|
||||
if !strings.Contains(said, want) {
|
||||
t.Fatalf("the answer does not say %q:\n%s", want, said)
|
||||
}
|
||||
}
|
||||
if slices.Contains(assignedTo(t, open, "laptop"), "sensors") {
|
||||
t.Fatal("a module not registered was assigned")
|
||||
}
|
||||
pending := pendingOf(t, open)
|
||||
if len(pending) != 1 || pending[0].State != inventory.PendingWaiting || pending[0].Build != "b-modules/sensors" {
|
||||
t.Fatalf("pending: %+v", pending)
|
||||
}
|
||||
if again, err := assign(ctx, open, "laptop", "sensors"); err != nil || !strings.Contains(again, "already waits") {
|
||||
t.Fatalf("a second assignment: %q %v", again, err)
|
||||
}
|
||||
|
||||
asked, err := theThreeQuestions(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if asked.well() {
|
||||
t.Fatal("status called the mesh well while an assignment waits")
|
||||
}
|
||||
body, err := statusAsJSON(asked)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var doc meshStatus
|
||||
if err := json.Unmarshal(body, &doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(doc.Pending) != 1 || doc.Pending[0].State != "waiting" || doc.Pending[0].Module != "sensors" {
|
||||
t.Fatalf("status says pending %+v", doc.Pending)
|
||||
}
|
||||
|
||||
if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors",
|
||||
"3da80a4b00aa", "sensors", "")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") {
|
||||
t.Fatal("the build registered sensors and the pending assignment was not made")
|
||||
}
|
||||
pending = pendingOf(t, open)
|
||||
if len(pending) != 1 || pending[0].State != inventory.PendingApplied ||
|
||||
!strings.HasSuffix(pending[0].Note, "`push laptop` sends it") {
|
||||
t.Fatalf("pending after the build: %+v", pending)
|
||||
}
|
||||
}
|
||||
|
||||
// **A read settles nothing** (review of #150, point 6): status — and so the board, the summary and the
|
||||
// probe, which compose from the same reading — leaves a pending assignment whose module is registered as it
|
||||
// is; the controller's tick makes it.
|
||||
func TestAStatusReadSettlesNothing(t *testing.T) {
|
||||
open := aCatalogueMesh(t)
|
||||
ctx := t.Context()
|
||||
asksWithPaths(t)
|
||||
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
|
||||
if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"})
|
||||
for range 2 {
|
||||
if _, err := theThreeQuestions(ctx, open); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if slices.Contains(assignedTo(t, open, "laptop"), "sensors") || pendingOf(t, open)[0].State != inventory.PendingWaiting {
|
||||
t.Fatalf("a status read settled a pending assignment: %+v", pendingOf(t, open))
|
||||
}
|
||||
said := settlePending(ctx, open, time.Now())
|
||||
if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") || pendingOf(t, open)[0].State != inventory.PendingApplied {
|
||||
t.Fatalf("the tick did not make it: %v %+v", said, pendingOf(t, open))
|
||||
}
|
||||
}
|
||||
|
||||
// The build of a pending assignment fails: it expires with the build's words and nothing is assigned; the
|
||||
// tick raises it as a condition once; it then reads as known and not built; and `unassign` takes it back,
|
||||
// after which the tick clears the condition. Status is not called well while the condition is open, and is
|
||||
// again once it clears — no fixed day of "not well" (review point 6).
|
||||
func TestAPendingAssignmentExpiresWhenItsBuildFails(t *testing.T) {
|
||||
open := aCatalogueMesh(t)
|
||||
ctx := t.Context()
|
||||
asksWithPaths(t)
|
||||
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
|
||||
if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors",
|
||||
"3da80a4b00aa", "", "go build: undefined: sensorsRead")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pending := pendingOf(t, open)
|
||||
if len(pending) != 1 || pending[0].State != inventory.PendingExpired {
|
||||
t.Fatalf("pending after a failed build: %+v", pending)
|
||||
}
|
||||
for _, want := range []string{"expired", "build b-modules/sensors of modules/sensors failed",
|
||||
"undefined: sensorsRead", "nothing was assigned", "build \"true\""} {
|
||||
if !strings.Contains(pending[0].Note, want) {
|
||||
t.Fatalf("the expiry does not say %q: %s", want, pending[0].Note)
|
||||
}
|
||||
}
|
||||
if slices.Contains(assignedTo(t, open, "laptop"), "sensors") {
|
||||
t.Fatal("a failed build's module was assigned")
|
||||
}
|
||||
|
||||
settlePending(ctx, open, time.Now())
|
||||
key := pendingObservation(pending[0]).Key()
|
||||
c, found, err := conditionsFrom.Get(ctx, key)
|
||||
if err != nil || !found || c.Kind != kindPendingEnded {
|
||||
t.Fatalf("no condition %s for the assignment not made: %+v %v %v", key, c, found, err)
|
||||
}
|
||||
if asked, err := theThreeQuestions(ctx, open); err != nil || asked.well() || len(asked.conditions) == 0 {
|
||||
t.Fatalf("status does not hold the open condition: %v", err)
|
||||
}
|
||||
|
||||
_, err = assign(ctx, open, "laptop", "sensors")
|
||||
if !errors.Is(err, inventory.ErrNoSuchModule) {
|
||||
t.Fatalf("a module whose build failed: %v", err)
|
||||
}
|
||||
for _, want := range []string{"sensors is known and not registered", "its last build, b-modules/sensors",
|
||||
"failed: go build: undefined: sensorsRead", "build \"true\"", "repository novox/mesh-catalog, path modules/sensors"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("the refusal does not say %q:\n%v", want, err)
|
||||
}
|
||||
}
|
||||
|
||||
said, err := unassign(ctx, open, "laptop", "sensors")
|
||||
if err != nil || !strings.Contains(said, "taken back") {
|
||||
t.Fatalf("unassign of an expired pending assignment: %q %v", said, err)
|
||||
}
|
||||
settlePending(ctx, open, time.Now())
|
||||
if _, found, _ := conditionsFrom.Get(ctx, key); found {
|
||||
t.Fatal("the condition stayed open after the assignment was taken back")
|
||||
}
|
||||
// Nothing of it keeps status from being well any more: no open pending assignment, no open condition of it.
|
||||
asked, err := theThreeQuestions(ctx, open)
|
||||
if err != nil || pendingOpen(asked.pending) {
|
||||
t.Fatalf("status still counts the pending assignment: %+v %v", asked.pending, err)
|
||||
}
|
||||
for _, c := range asked.conditions {
|
||||
if c.Kind == kindPendingEnded {
|
||||
t.Fatalf("status still holds the condition: %+v", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The condition also clears when the module is later assigned to the machine.
|
||||
func TestTheConditionClearsWhenTheModuleIsAssignedLater(t *testing.T) {
|
||||
open := aCatalogueMesh(t)
|
||||
ctx := t.Context()
|
||||
asksWithPaths(t)
|
||||
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
|
||||
if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors",
|
||||
"3da80a4b00aa", "", "boom")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settlePending(ctx, open, time.Now())
|
||||
key := pendingObservation(pendingOf(t, open)[0]).Key()
|
||||
register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"})
|
||||
if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settlePending(ctx, open, time.Now())
|
||||
if _, found, _ := conditionsFrom.Get(ctx, key); found {
|
||||
t.Fatal("the condition stayed open after the module was assigned")
|
||||
}
|
||||
}
|
||||
|
||||
// Known and not built, asked with build: the build is asked from where the last one was, the request kept as
|
||||
// assign's, and the assignment kept pending on the new build.
|
||||
func TestAssignWithBuildAsksForAKnownModule(t *testing.T) {
|
||||
open := aCatalogueMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-old", Repository: "novox/mesh-catalog",
|
||||
Seat: "git", Path: "modules/sensors", Ref: "main", For: "build", At: time.Now().Add(-time.Hour)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-old", Repository: "novox/mesh-catalog", Ref: "main",
|
||||
Path: "modules/sensors", On: "anchor", Failed: "no space left on device"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asked := asksWithPaths(t)
|
||||
said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(*asked) != 1 || (*asked)[0] != [3]string{"novox/mesh-catalog", "modules/sensors", "main"} {
|
||||
t.Fatalf("asked %v", *asked)
|
||||
}
|
||||
for _, want := range []string{"known and not registered", "no space left on device",
|
||||
"build b-modules/sensors of novox/mesh-catalog@main (modules/sensors) is asked for now", "kept as pending"} {
|
||||
if !strings.Contains(said, want) {
|
||||
t.Fatalf("the answer does not say %q:\n%s", want, said)
|
||||
}
|
||||
}
|
||||
pending := pendingOf(t, open)
|
||||
if len(pending) != 1 || pending[0].Build != "b-modules/sensors" {
|
||||
t.Fatalf("pending: %+v", pending)
|
||||
}
|
||||
requests, _ := inv.RequestsNamed(ctx, "sensors")
|
||||
if len(requests) != 2 || requests[0].ID != "b-modules/sensors" || requests[0].For != "assign" {
|
||||
t.Fatalf("the request is not kept as assign's: %+v", requests)
|
||||
}
|
||||
}
|
||||
|
||||
// **build "true" with a pending assignment already waiting** (review point 5): the waiting one is made to
|
||||
// wait for the new build, and no build is asked that nothing waits on.
|
||||
func TestAssignWithBuildRepointsTheWaitingAssignment(t *testing.T) {
|
||||
open := aCatalogueMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-old", Repository: "novox/mesh-catalog",
|
||||
Seat: "git", Path: "modules/sensors", Ref: "main", For: "merge", At: time.Now().Add(-time.Hour)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-old", Repository: "novox/mesh-catalog", Ref: "main",
|
||||
Path: "modules/sensors", On: "anchor", Failed: "boom"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Waiting on the failed build, its expiry not yet settled.
|
||||
if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "laptop", Module: "sensors",
|
||||
Build: "build-old", Repository: "novox/mesh-catalog", Path: "modules/sensors"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := assign(ctx, open, "laptop", "sensors")
|
||||
if err == nil || !strings.Contains(err.Error(), "already has a pending assignment of sensors, waiting for build build-old") {
|
||||
t.Fatalf("known and not built with a pending assignment waiting: %v", err)
|
||||
}
|
||||
asked := asksWithPaths(t)
|
||||
said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors")
|
||||
if err != nil || !strings.Contains(said, "which waited for build build-old, now waits for build b-modules/sensors") {
|
||||
t.Fatalf("assign with build: %q %v", said, err)
|
||||
}
|
||||
if len(*asked) != 1 {
|
||||
t.Fatalf("asked %v", *asked)
|
||||
}
|
||||
pending := pendingOf(t, open)
|
||||
if len(pending) != 1 || pending[0].Build != "b-modules/sensors" || pending[0].State != inventory.PendingWaiting {
|
||||
t.Fatalf("pending: %+v", pending)
|
||||
}
|
||||
// The new build in flight now: build "true" again asks nothing.
|
||||
if said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors"); err != nil ||
|
||||
!strings.Contains(said, "no second build was asked") || len(*asked) != 1 {
|
||||
t.Fatalf("a second build true while the build runs: %q %v, asked %v", said, err, *asked)
|
||||
}
|
||||
}
|
||||
|
||||
// **A failed ask never reads as in flight** (review point 2): a merge whose ask could not be made keeps the
|
||||
// request as not asked, and assign says the merge could not ask; a request whose asker could not hand it over
|
||||
// or stopped waiting reads the same, unless its outcome was heard.
|
||||
func TestAFailedAskIsNotABuildInFlight(t *testing.T) {
|
||||
open := aCatalogueMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
was := askABuild
|
||||
askABuild = func(context.Context, buildSource, string, string) (string, error) {
|
||||
return "", errors.New("cannot submit a build: nats: timeout")
|
||||
}
|
||||
t.Cleanup(func() { askABuild = was })
|
||||
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
|
||||
_, err := assign(ctx, open, "laptop", "sensors")
|
||||
if err == nil || !strings.Contains(err.Error(), "the merge that added it (3da80a4b) could not ask for its build: cannot submit") {
|
||||
t.Fatalf("a merge that could not ask: %v", err)
|
||||
}
|
||||
if len(pendingOf(t, open)) != 0 {
|
||||
t.Fatal("a pending assignment waits on a build never asked")
|
||||
}
|
||||
|
||||
if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-waited", Repository: "novox/mesh-catalog",
|
||||
Seat: "git", Path: "modules/gauges", Ref: "main", For: "build"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.MarkNotAsked(ctx, "build-waited", "cannot submit a build: no responders"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = assign(ctx, open, "laptop", "gauges")
|
||||
if err == nil || !strings.Contains(err.Error(), "build build-waited, asked by build, was not handed over") {
|
||||
t.Fatalf("a build not handed over: %v", err)
|
||||
}
|
||||
// Heard first, the outcome stands: marking it afterwards changes nothing.
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-waited", Repository: "novox/mesh-catalog", Ref: "main",
|
||||
Path: "modules/gauges", On: "anchor", Failed: "the real failure"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = assign(ctx, open, "laptop", "gauges")
|
||||
if err == nil || !strings.Contains(err.Error(), "failed: the real failure") {
|
||||
t.Fatalf("an outcome heard after a failed wait: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A plan's tier keeps its requests as the plan's (review point 3).
|
||||
func TestAPlansBuildRequestIsThePlans(t *testing.T) {
|
||||
open := aCatalogueMesh(t)
|
||||
ctx := t.Context()
|
||||
asksWithPaths(t)
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1aaaaaaaa",
|
||||
Paths: []string{"modules/app/index.ts"}, ModuleDirs: []string{"modules/app"}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requests, err := open.inventory.RequestsNamed(ctx, "app")
|
||||
if err != nil || len(requests) != 1 || requests[0].For != "plan" {
|
||||
t.Fatalf("the plan's request: %+v %v", requests, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **A build heard as built and not registered yet is in flight** (review point 4): the outcome is recorded
|
||||
// before the module is registered; for that moment assign keeps the assignment pending. Past the grace it is
|
||||
// recorded and not registered, and the tick ends the pending assignment with that.
|
||||
func TestABuildBeingRegisteredIsInFlight(t *testing.T) {
|
||||
open := aCatalogueMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-done", Repository: "novox/mesh-catalog",
|
||||
Seat: "git", Path: "modules/sensors", Ref: "main", For: "merge"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-done", Repository: "novox/mesh-catalog", Ref: "main",
|
||||
Path: "modules/sensors", Module: "sensors", On: "anchor"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said, err := assign(ctx, open, "laptop", "sensors")
|
||||
if err != nil || !strings.Contains(said, "built and being registered") || !strings.Contains(said, "kept as pending") {
|
||||
t.Fatalf("a build being registered: %q %v", said, err)
|
||||
}
|
||||
settlePending(ctx, open, time.Now().Add(registerGrace+time.Minute))
|
||||
p := pendingOf(t, open)
|
||||
if len(p) != 1 || p[0].State != inventory.PendingExpired || !strings.Contains(p[0].Note, "recorded and not registered") {
|
||||
t.Fatalf("past the grace: %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
// Unknown: refused as no module of that name, claiming only what was looked at, with the closest names; build
|
||||
// asks for nothing.
|
||||
func TestAnUnknownModuleIsRefusedWithTheClosestNames(t *testing.T) {
|
||||
open := aCatalogueMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "sensord", Version: "1"})
|
||||
asked := asksWithPaths(t)
|
||||
_, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors")
|
||||
if !errors.Is(err, inventory.ErrNoSuchModule) {
|
||||
t.Fatalf("an unknown module: %v", err)
|
||||
}
|
||||
for _, want := range []string{"no module of that name: sensors", "the catalogue holds no module of that name",
|
||||
"no build request the controller kept (the last 30 days)", "the closest names it holds: sensord",
|
||||
"asks for nothing here"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("the refusal does not say %q:\n%v", want, err)
|
||||
}
|
||||
}
|
||||
if strings.Contains(err.Error(), "merge") {
|
||||
t.Fatalf("the refusal claims a merge it never looked at: %v", err)
|
||||
}
|
||||
if len(*asked) != 0 || len(pendingOf(t, open)) != 0 {
|
||||
t.Fatalf("an unknown module asked %v, pending %+v", *asked, pendingOf(t, open))
|
||||
}
|
||||
}
|
||||
|
||||
// Several modules in one act, one of them not registered: nothing is assigned and nothing kept pending, and
|
||||
// each missing one is said.
|
||||
func TestAnActWithAModuleNotRegisteredIsRefusedWhole(t *testing.T) {
|
||||
open := aCatalogueMesh(t)
|
||||
ctx := t.Context()
|
||||
asksWithPaths(t)
|
||||
register(t, open, catalogue.Manifest{Module: "known", Version: "1"})
|
||||
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
|
||||
_, err := assign(ctx, open, "laptop", "known", "sensors")
|
||||
if err == nil || !strings.Contains(err.Error(), "nothing was assigned") || !strings.Contains(err.Error(), "being built") {
|
||||
t.Fatalf("an act naming a module in flight: %v", err)
|
||||
}
|
||||
if slices.Contains(assignedTo(t, open, "laptop"), "known") || len(pendingOf(t, open)) != 0 {
|
||||
t.Fatal("an act refused whole was made in part")
|
||||
}
|
||||
}
|
||||
|
||||
// unassign withdraws a waiting pending assignment; the build goes on and registers the module assigned
|
||||
// nowhere: a withdrawn row is never claimed.
|
||||
func TestUnassignWithdrawsAPendingAssignment(t *testing.T) {
|
||||
open := aCatalogueMesh(t)
|
||||
ctx := t.Context()
|
||||
asksWithPaths(t)
|
||||
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
|
||||
if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said, err := unassign(ctx, open, "laptop", "sensors")
|
||||
if err != nil || !strings.Contains(said, "withdrawn") {
|
||||
t.Fatalf("unassign of a pending assignment: %q %v", said, err)
|
||||
}
|
||||
if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors",
|
||||
"3da80a4b00aa", "sensors", "")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settlePending(ctx, open, time.Now())
|
||||
if slices.Contains(assignedTo(t, open, "laptop"), "sensors") {
|
||||
t.Fatal("a withdrawn assignment was made")
|
||||
}
|
||||
}
|
||||
|
||||
// **A claim is never overridden** (review point 1): a pending assignment being made is claimed
|
||||
// (waiting → applying); an unassign meanwhile is refused and says so, and leaves the claim; making a row
|
||||
// already withdrawn makes nothing. A claim left by a controller that stopped is settled by the tick from what
|
||||
// the mesh holds: back to waiting when the module is not assigned, applied when it is.
|
||||
func TestAWithdrawalNeverOverridesAClaim(t *testing.T) {
|
||||
open := aCatalogueMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asksWithPaths(t)
|
||||
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
|
||||
if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := pendingOf(t, open)[0]
|
||||
if ok, err := inv.ClaimPending(ctx, p.ID); err != nil || !ok {
|
||||
t.Fatalf("claim: %v %v", ok, err)
|
||||
}
|
||||
_, err := unassign(ctx, open, "laptop", "sensors")
|
||||
if err == nil || !strings.Contains(err.Error(), "is being assigned sensors now") {
|
||||
t.Fatalf("unassign of a claimed pending assignment: %v", err)
|
||||
}
|
||||
if got := pendingOf(t, open)[0]; got.State != inventory.PendingApplying {
|
||||
t.Fatalf("the claim was overridden: %+v", got)
|
||||
}
|
||||
// Pending(zero) is the waiting ones alone (review point 7).
|
||||
if waiting, err := inv.Pending(ctx, time.Time{}); err != nil || len(waiting) != 0 {
|
||||
t.Fatalf("Pending(zero) read a claimed row: %+v %v", waiting, err)
|
||||
}
|
||||
|
||||
// Left by a controller that stopped: back to waiting, since sensors is not assigned.
|
||||
settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute))
|
||||
if got := pendingOf(t, open)[0]; got.State != inventory.PendingWaiting {
|
||||
t.Fatalf("a stale claim was not released: %+v", got)
|
||||
}
|
||||
|
||||
// Withdrawn, then the build registers it: applyPending finds nothing to claim.
|
||||
if _, err := unassign(ctx, open, "laptop", "sensors"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"})
|
||||
if line := applyPending(ctx, open, p, "b-modules/sensors"); line != "" {
|
||||
t.Fatalf("a withdrawn pending assignment was made: %s", line)
|
||||
}
|
||||
if slices.Contains(assignedTo(t, open, "laptop"), "sensors") {
|
||||
t.Fatal("a withdrawn pending assignment was assigned")
|
||||
}
|
||||
|
||||
// A stale claim whose module was assigned is applied.
|
||||
q, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "sensors", Build: "b-x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ok, _ := inv.ClaimPending(ctx, q.ID); !ok {
|
||||
t.Fatal("claim")
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "anchor", "sensors"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute))
|
||||
rows, _ := inv.PendingFor(ctx, "anchor", "sensors")
|
||||
if len(rows) != 1 || rows[0].State != inventory.PendingApplied {
|
||||
t.Fatalf("a stale claim of an assigned module: %+v", rows)
|
||||
}
|
||||
}
|
||||
|
||||
// A build that says nothing within the bound: the pending assignment expires at the next tick, saying so,
|
||||
// rather than waiting for ever; and a module registered by a process that kept no pending assignment is
|
||||
// assigned at the next tick.
|
||||
func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) {
|
||||
open := aCatalogueMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
for _, r := range []inventory.BuildRequest{
|
||||
{ID: "build-lost", Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/lost", Ref: "main",
|
||||
For: "merge", At: time.Now().Add(-buildRequestBound / 2)},
|
||||
{ID: "build-heard", Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/heard", Ref: "main",
|
||||
For: "merge", At: time.Now()},
|
||||
} {
|
||||
if err := inv.RecordBuildRequest(ctx, r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, m := range []string{"lost", "heard"} {
|
||||
if _, err := assign(ctx, open, "laptop", m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "heard", Version: "1"})
|
||||
|
||||
settlePending(ctx, open, time.Now().Add(buildRequestBound))
|
||||
if !slices.Contains(assignedTo(t, open, "laptop"), "heard") {
|
||||
t.Fatal("a module registered meanwhile was not assigned at the next tick")
|
||||
}
|
||||
byModule := map[string]inventory.PendingAssignment{}
|
||||
for _, p := range pendingOf(t, open) {
|
||||
byModule[p.Module] = p
|
||||
}
|
||||
if p := byModule["lost"]; p.State != inventory.PendingExpired || !strings.Contains(p.Note, "no outcome of build build-lost") {
|
||||
t.Fatalf("a build that said nothing: %+v", p)
|
||||
}
|
||||
if p := byModule["heard"]; p.State != inventory.PendingApplied {
|
||||
t.Fatalf("a module registered meanwhile: %+v", p)
|
||||
}
|
||||
|
||||
// Ended rows are deleted after KeptFor (review point 7); open ones never, nor one whose condition is
|
||||
// still open (second review, bug B).
|
||||
if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "lost", Build: "build-lost"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said := settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour))
|
||||
rows, err := inv.Pending(ctx, time.Unix(0, 0))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range rows {
|
||||
if !r.Open() && (r.Raised == nil || r.Cleared != nil) {
|
||||
t.Fatalf("an ended pending assignment with no open condition outlived %s: %+v (%v)", inventory.KeptFor, r, said)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(strings.Join(said, "\n"), "deleted 1 pending assignment(s)") {
|
||||
t.Fatalf("the applied row was not deleted: %v", said)
|
||||
}
|
||||
}
|
||||
|
||||
// The seat's assign passes build on; unassign takes none.
|
||||
func TestTheSeatsAssignPassesBuild(t *testing.T) {
|
||||
argv, err := argvFor("assign", map[string]any{"node": "g14", "module": "sensors", "build": "true"})
|
||||
if err != nil || !slices.Equal(argv, []string{"assign", "g14", "sensors", "--build"}) {
|
||||
t.Fatalf("assign with build: %v %v", argv, err)
|
||||
}
|
||||
if _, err := argvFor("unassign", map[string]any{"node": "g14", "module": "sensors", "build": "true"}); err == nil {
|
||||
t.Fatal("unassign took build")
|
||||
}
|
||||
}
|
||||
|
||||
// The condition's words are plain.
|
||||
func TestAnAssignmentNotMadeIsSaidPlainly(t *testing.T) {
|
||||
o := pendingObservation(inventory.PendingAssignment{Node: "g14", Module: "sensors",
|
||||
Note: "the pending assignment of sensors to g14 expired: build b-1 of modules/sensors failed: boom"})
|
||||
w := plainWordings[kindPendingEnded](o)
|
||||
if why, ok := conditions.PlainWords(w, "g14"); !ok {
|
||||
t.Fatalf("not plain: %s %+v", why, w)
|
||||
}
|
||||
if w.Headline != "sensors was not put on g14" {
|
||||
t.Fatalf("headline %q", w.Headline)
|
||||
}
|
||||
}
|
||||
|
||||
// **One row's condition is never another's** (second review, bug A). The sequence: the first pending
|
||||
// assignment's build fails and the tick raises it; the person assigns again with build "true", a second
|
||||
// pending assignment; its build fails too before the next tick. That tick raises the second, and must not
|
||||
// clear it by judging the first answered by the second: each row has its own condition, and a newer row that
|
||||
// itself ended unmade answers nothing.
|
||||
func TestASecondAssignmentNotMadeKeepsItsCondition(t *testing.T) {
|
||||
open := aCatalogueMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := 0
|
||||
was := askABuild
|
||||
askABuild = func(context.Context, buildSource, string, string) (string, error) {
|
||||
asked++
|
||||
return fmt.Sprintf("b-%d", asked), nil
|
||||
}
|
||||
t.Cleanup(func() { askABuild = was })
|
||||
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
|
||||
if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failedBuild(t, open, "b-1")
|
||||
settlePending(ctx, open, time.Now())
|
||||
first := pendingOf(t, open)[0]
|
||||
if !conditionOpen(t, first) {
|
||||
t.Fatal("the first assignment not made raised nothing")
|
||||
}
|
||||
|
||||
if _, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failedBuild(t, open, "b-2")
|
||||
settlePending(ctx, open, time.Now())
|
||||
|
||||
var second inventory.PendingAssignment
|
||||
for _, p := range pendingOf(t, open) {
|
||||
if p.Build == "b-2" {
|
||||
second = p
|
||||
}
|
||||
}
|
||||
if second.State != inventory.PendingExpired {
|
||||
t.Fatalf("the second: %+v", second)
|
||||
}
|
||||
if !conditionOpen(t, second) {
|
||||
t.Fatal("the second assignment's condition was cleared in the tick that raised it")
|
||||
}
|
||||
if !conditionOpen(t, rowOf(t, open, first.ID)) {
|
||||
t.Fatal("the first was judged answered by a second that itself was not made")
|
||||
}
|
||||
// unassign takes both back, and the next tick clears both.
|
||||
if said, err := unassign(ctx, open, "laptop", "sensors"); err != nil || !strings.Contains(said, "b-1") ||
|
||||
!strings.Contains(said, "b-2") {
|
||||
t.Fatalf("unassign: %q %v", said, err)
|
||||
}
|
||||
settlePending(ctx, open, time.Now())
|
||||
if conditionOpen(t, first) || conditionOpen(t, second) {
|
||||
t.Fatal("a condition stayed open after both were taken back")
|
||||
}
|
||||
}
|
||||
|
||||
// **A raised row outlives the pruning until its condition clears** (second review, bug B); and a machine's
|
||||
// removal, which takes its rows with it, clears their conditions at the next tick.
|
||||
func TestARaisedRowIsKeptUntilItsConditionClears(t *testing.T) {
|
||||
open := aCatalogueMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asksWithPaths(t)
|
||||
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
|
||||
if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failedBuild(t, open, "b-modules/sensors")
|
||||
settlePending(ctx, open, time.Now())
|
||||
row := pendingOf(t, open)[0]
|
||||
|
||||
settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour))
|
||||
if rowOf(t, open, row.ID).Raised == nil || !conditionOpen(t, row) {
|
||||
t.Fatal("a raised row was pruned, or its condition closed, while the condition was open")
|
||||
}
|
||||
if _, err := unassign(ctx, open, "laptop", "sensors"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settlePending(ctx, open, time.Now())
|
||||
settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour))
|
||||
if known, err := inv.PendingKnown(ctx, []int64{row.ID}); err != nil || known[row.ID] {
|
||||
t.Fatalf("a cleared row outlived %s: %v", inventory.KeptFor, err)
|
||||
}
|
||||
|
||||
// On anchor, then anchor removed: the row goes, and its condition with it at the next tick.
|
||||
if _, err := assign(ctx, open, "anchor", "sensors"); err == nil {
|
||||
t.Fatal("known and not built was not refused")
|
||||
}
|
||||
q, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "sensors", Build: "b-x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.SettlePending(ctx, q.ID, inventory.PendingWaiting, inventory.PendingExpired, "boom"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settlePending(ctx, open, time.Now())
|
||||
q.Node = "anchor"
|
||||
if !conditionOpen(t, q) {
|
||||
t.Fatal("not raised")
|
||||
}
|
||||
if _, err := inv.RemoveNodeForTest(ctx, "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settlePending(ctx, open, time.Now())
|
||||
if conditionOpen(t, q) {
|
||||
t.Fatal("a removed machine's condition stayed open")
|
||||
}
|
||||
}
|
||||
|
||||
// **A claim left by a controller that stopped** (second review): the tick settles it from what the mesh
|
||||
// holds — back to waiting when the module is not assigned there, applied when it is — and leaves a fresh
|
||||
// claim alone.
|
||||
func TestAStaleClaimIsSettledFromWhatTheMeshHolds(t *testing.T) {
|
||||
open := aCatalogueMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
claimed := func(node string) inventory.PendingAssignment {
|
||||
p, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: node, Module: "sensors", Build: "b-x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ok, err := inv.ClaimPending(ctx, p.ID); err != nil || !ok {
|
||||
t.Fatalf("claim: %v %v", ok, err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
notAssigned, assigned := claimed("laptop"), claimed("anchor")
|
||||
register(t, open, catalogue.Manifest{Module: "unrelated", Version: "1"})
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "sensors", Version: "1"}, inventory.Source{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "anchor", "sensors"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
settlePending(ctx, open, time.Now())
|
||||
if rowOf(t, open, notAssigned.ID).State != inventory.PendingApplying || rowOf(t, open, assigned.ID).State != inventory.PendingApplying {
|
||||
t.Fatal("a fresh claim was settled")
|
||||
}
|
||||
settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute))
|
||||
if got := rowOf(t, open, assigned.ID); got.State != inventory.PendingApplied {
|
||||
t.Fatalf("a stale claim of an assigned module: %+v", got)
|
||||
}
|
||||
// Released to waiting, and in the same tick made, since sensors is registered now.
|
||||
if got := rowOf(t, open, notAssigned.ID); got.State != inventory.PendingApplied ||
|
||||
!slices.Contains(assignedTo(t, open, "laptop"), "sensors") {
|
||||
t.Fatalf("a stale claim of a module not assigned: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **A waited build whose asker stopped waiting is asked, outcome unknown** (second review): it may still run,
|
||||
// so it reads as in flight until its outcome or its bound, never as not asked; a build never handed over
|
||||
// is not asked.
|
||||
func TestABuildNoLongerWaitedForIsStillInFlight(t *testing.T) {
|
||||
open := aCatalogueMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
for _, id := range []string{"build-timeout", "build-nothandedover"} {
|
||||
dir := "modules/" + strings.TrimPrefix(id, "build-")
|
||||
if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: id, Repository: "novox/mesh-catalog",
|
||||
Seat: "git", Path: dir, Ref: "main", For: "build"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
markWaitFailed(ctx, "build-timeout", errors.New("no build machine answered within 10m0s"))
|
||||
markWaitFailed(ctx, "build-nothandedover", fmt.Errorf("%w: cannot submit a build: nats: timeout", link.ErrNotHandedOver))
|
||||
|
||||
said, err := assign(ctx, open, "laptop", "timeout")
|
||||
if err != nil || !strings.Contains(said, "being built") || !strings.Contains(said, "kept as pending") {
|
||||
t.Fatalf("a build no longer waited for: %q %v", said, err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "nothandedover"); err == nil || !strings.Contains(err.Error(), "was not handed over") {
|
||||
t.Fatalf("a build never handed over: %v", err)
|
||||
}
|
||||
settlePending(ctx, open, time.Now().Add(buildRequestBound+time.Minute))
|
||||
rows, _ := inv.PendingFor(ctx, "laptop", "timeout")
|
||||
if len(rows) != 1 || rows[0].State != inventory.PendingExpired || !strings.Contains(rows[0].Note, "its asker stopped waiting") {
|
||||
t.Fatalf("past the bound: %+v", rows)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A wait for a person is not a failure, and one module's verdict is not every module's (novox/hq ADR 0254,
|
||||
// issue 318).
|
||||
|
||||
func relogin(module, account string) inventory.ResourceHealth {
|
||||
return inventory.ResourceHealth{Module: module, Resource: module + "." + account, Kind: link.KindAccount, Target: account,
|
||||
Account: account, State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": " + account + " is in the group " + module +
|
||||
", and its running session began before it was; log out of every session and in again, or reboot"}
|
||||
}
|
||||
|
||||
func userUnit(module, account string) inventory.ResourceHealth {
|
||||
return inventory.ResourceHealth{Module: module, Resource: module + ".daemon", Kind: link.KindUnit,
|
||||
Target: module + "-daemon.service", Account: account, State: link.StateUnhealthy,
|
||||
Reason: "failed in the account's own service manager (exit-code)"}
|
||||
}
|
||||
|
||||
// Only what depends on the new login alone is a wait; anything else unhealthy of the module is judged as
|
||||
// before, so that a fault is never excused.
|
||||
func TestOnlyWhatDependsOnTheNewLoginIsAWait(t *testing.T) {
|
||||
notInGroup := relogin("lights", "operator")
|
||||
notInGroup.Reason = "not in the group lights: the apply has not put operator there; its outcome says why"
|
||||
systemUnit := userUnit("lights", "operator")
|
||||
systemUnit.Account = ""
|
||||
otherAccount := userUnit("lights", "guest")
|
||||
container := inventory.ResourceHealth{Module: "lights", Resource: "lights.web", Kind: "container", Target: "lights",
|
||||
State: link.StateUnhealthy, Reason: "down"}
|
||||
otherModule := userUnit("sound", "operator")
|
||||
starting := userUnit("lights", "operator")
|
||||
starting.State = link.StateStarting
|
||||
byTarget := relogin("lights", "operator")
|
||||
byTarget.Account = "" // an account said by its target alone is still that account
|
||||
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
rs []inventory.ResourceHealth
|
||||
waits bool
|
||||
}{
|
||||
{"the account alone", []inventory.ResourceHealth{relogin("lights", "operator")}, true},
|
||||
{"the account and its unit in that account's manager", []inventory.ResourceHealth{relogin("lights", "operator"),
|
||||
userUnit("lights", "operator")}, true},
|
||||
{"an account named by its target", []inventory.ResourceHealth{byTarget, userUnit("lights", "operator")}, true},
|
||||
{"another module's unit is not this module's", []inventory.ResourceHealth{relogin("lights", "operator"), otherModule}, true},
|
||||
{"a unit still starting is no fault", []inventory.ResourceHealth{relogin("lights", "operator"), starting}, true},
|
||||
{"a unit and no account", []inventory.ResourceHealth{userUnit("lights", "operator")}, false},
|
||||
{"an account not in its group", []inventory.ResourceHealth{notInGroup, userUnit("lights", "operator")}, false},
|
||||
{"a unit whose manager is not said", []inventory.ResourceHealth{relogin("lights", "operator"), systemUnit}, false},
|
||||
{"a unit in another account's manager", []inventory.ResourceHealth{relogin("lights", "operator"), otherAccount}, false},
|
||||
{"a container beside the wait", []inventory.ResourceHealth{relogin("lights", "operator"), container}, false},
|
||||
} {
|
||||
said, waits := personWait("lights", "laptop", c.rs)
|
||||
if waits != c.waits {
|
||||
t.Errorf("%s: waits %v; want %v", c.name, waits, c.waits)
|
||||
continue
|
||||
}
|
||||
if waits && !strings.HasPrefix(said, "relogin needed on laptop: lights waits for a new login of operator") {
|
||||
t.Errorf("%s: said %q", c.name, said)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The gate reads a wait for a person as its own reading, never a fault, and passes with it; the same module
|
||||
// with a container down beside it is not yet healthy, as before.
|
||||
func TestAWaitForAPersonIsAPassCarriedAlong(t *testing.T) {
|
||||
now := time.Now()
|
||||
since := now.Add(-time.Minute)
|
||||
account := relogin("lights", "operator")
|
||||
f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now,
|
||||
Resources: []inventory.ResourceHealth{account, userUnit("lights", "operator")}}},
|
||||
groupsAdded: map[string]bool{"lights": true}}
|
||||
if h, why := moduleHealthWord("lights", "laptop", since, f); h != healthPerson || !strings.Contains(why, "relogin needed on laptop") {
|
||||
t.Fatalf("a wait for a new login reads %v %q; want a wait for a person", h, why)
|
||||
}
|
||||
// **Only a move that put the account in a new group is excused** (issue 318 review): a later build that
|
||||
// adds no group did not bring the wait, and is judged as before.
|
||||
f.groupsAdded["lights"] = false
|
||||
if h, why := moduleHealthWord("lights", "laptop", since, f); h != healthNotYet {
|
||||
t.Fatalf("a wait the move did not bring reads %v %q; want not yet", h, why)
|
||||
}
|
||||
f.groupsAdded["lights"] = true
|
||||
h := f.health["laptop"]
|
||||
h.Resources = append(h.Resources, inventory.ResourceHealth{Module: "lights", Resource: "lights.web", Kind: "container",
|
||||
Target: "lights", State: link.StateUnhealthy, Reason: "down"})
|
||||
f.health["laptop"] = h
|
||||
if got, why := moduleHealthWord("lights", "laptop", since, f); got != healthNotYet {
|
||||
t.Fatalf("a container down beside the wait reads %v %q; want not yet", got, why)
|
||||
}
|
||||
}
|
||||
|
||||
// The module's condition says the wait in one sentence for a person, and clears on the first statement that
|
||||
// no longer says it — said as the module's own fault when its unit still fails after the new login.
|
||||
func TestTheReloginConditionSaysTheWaitAndClearsAfterTheLogin(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
k, _ := withConditionsInMemory(t)
|
||||
at := h0
|
||||
say := func(rs ...link.ResourceHealth) {
|
||||
t.Helper()
|
||||
at = at.Add(time.Second)
|
||||
for i := range rs {
|
||||
rs[i].Since = at
|
||||
}
|
||||
if err := stateHealth(ctx, open.inventory, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at,
|
||||
Resources: rs}, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
asLink := func(r inventory.ResourceHealth) link.ResourceHealth {
|
||||
return link.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target, State: r.State,
|
||||
Reason: r.Reason, Account: r.Account}
|
||||
}
|
||||
account, unit := asLink(relogin("lights", "operator")), asLink(userUnit("lights", "operator"))
|
||||
openNow := func() []conditions.Condition {
|
||||
t.Helper()
|
||||
list, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return list
|
||||
}
|
||||
say(account, unit)
|
||||
say(account, unit)
|
||||
list := openNow()
|
||||
if len(list) != 1 || list[0].Key != "module.lights.laptop.relogin-needed" {
|
||||
t.Fatalf("raised %+v; want lights' relogin-needed alone", list)
|
||||
}
|
||||
c := list[0]
|
||||
if c.Severity != conditions.Warning || c.Resolver != conditions.ResolverOperator ||
|
||||
!strings.HasPrefix(c.Summary, "relogin needed on laptop:") {
|
||||
t.Fatalf("the condition is %s %s %q", c.Severity, c.Resolver, c.Summary)
|
||||
}
|
||||
// After the new login the account is healthy, and the unit, still failing, is the module's own fault.
|
||||
account.State, account.Reason = link.StateHealthy, ""
|
||||
say(account, unit)
|
||||
list = openNow()
|
||||
if len(list) != 1 || list[0].Key != "module.lights.laptop.unhealthy" {
|
||||
t.Fatalf("after the login: %+v; want lights' own unhealthy condition alone", list)
|
||||
}
|
||||
unit.State, unit.Reason = link.StateHealthy, ""
|
||||
say(account, unit)
|
||||
if list = openNow(); len(list) != 0 {
|
||||
t.Fatalf("after the unit runs: %+v; want nothing open", list)
|
||||
}
|
||||
}
|
||||
|
||||
// A module that is healthy on its own keeps its pass when another module of the same send fails at the
|
||||
// bound: it is neither put back nor left without a verdict for every other walk to wait on.
|
||||
func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) {
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
|
||||
return map[string]bool{"laptop": true}, nil
|
||||
}
|
||||
backlogFacts := gatherGateFacts
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
f, err := backlogFacts(ctx, open, component)
|
||||
f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: time.Now(),
|
||||
Resources: []inventory.ResourceHealth{
|
||||
{Module: "app", Resource: "app.web", Kind: "container", Target: "app", State: link.StateHealthy},
|
||||
{Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy,
|
||||
Reason: "down"}}}}
|
||||
return f, err
|
||||
}
|
||||
gateEvery, gateBound = 0, 300*time.Millisecond
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
advancePlans(ctx, b.open)
|
||||
if p := b.release(t); p.State != inventory.PlanRolling {
|
||||
break
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
p := b.release(t)
|
||||
if p.State != inventory.PlanFailed {
|
||||
t.Fatalf("the walk is %s: %s; want it failed on late", p.State, p.Note)
|
||||
}
|
||||
if v, found, err := inv.GateOf(ctx, "build-app-c2"); err != nil || !found || v.Verdict != inventory.GatePassed ||
|
||||
!strings.Contains(v.Why, "on its own") {
|
||||
t.Fatalf("app's verdict: %+v (found %v, %v); want its own pass kept", v, found, err)
|
||||
}
|
||||
if failed, _ := inv.GateFailed(ctx, "build-late-c2"); !failed {
|
||||
t.Fatal("late's build is not marked failed at its gate")
|
||||
}
|
||||
current, _ := inv.CurrentBuilds(ctx)
|
||||
if current["app"].Commit != "c2" || current["late"].Commit != "c1" {
|
||||
t.Fatalf("registered: app %s, late %s; want app kept at c2 and late put back to c1", current["app"].Commit,
|
||||
current["late"].Commit)
|
||||
}
|
||||
if !strings.Contains(p.Note, "passed on their own and kept: app") {
|
||||
t.Errorf("the walk does not say what kept its pass: %s", p.Note)
|
||||
}
|
||||
}
|
||||
|
||||
// What the operator reads of the wait (ADR 0253, ADR 0254): it needs them, so it is never quiet, and it offers
|
||||
// no button, since nothing but their own new login can do it.
|
||||
func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) {
|
||||
o := reloginObservation("openrazer", "g14", "relogin needed on g14: …", "operator", []inventory.ResourceHealth{
|
||||
relogin("openrazer", "operator"), userUnit("openrazer", "operator")})
|
||||
plainExample(t, o, "openrazer waits for a new login on g14",
|
||||
"Needs you: log out of g14 completely and log in again, or restart it. Openrazer put your account in a "+
|
||||
"group it needs. You logged in before that, so openrazer cannot run until you log in again. Its "+
|
||||
"update is in place and nothing was undone.")
|
||||
if len(o.Actions) != 0 || o.Needs == "" {
|
||||
t.Errorf("relogin: needs %q, actions %+v; want needs and no button", o.Needs, o.Actions)
|
||||
}
|
||||
// **Not "your account" when it is not the operator's** (issue 318 review).
|
||||
other := reloginObservation("openrazer", "g14", "relogin needed on g14: …", "operator", []inventory.ResourceHealth{
|
||||
relogin("openrazer", "guest"), userUnit("openrazer", "guest")})
|
||||
plainExample(t, other, "openrazer waits for a new login on g14",
|
||||
"Needs you: have the account it names log out of g14 completely and log in again, or restart g14. Openrazer "+
|
||||
"put an account on g14 in a group it needs. That account logged in before that, so openrazer cannot run "+
|
||||
"until it logs in again. Its update is in place and nothing was undone.")
|
||||
if unknown := reloginObservation("openrazer", "g14", "…", "", []inventory.ResourceHealth{relogin("openrazer", "operator")}); strings.Contains(unknown.Explanation, "your account") {
|
||||
t.Errorf("an operator not known is still told it is their account: %q", unknown.Explanation)
|
||||
}
|
||||
// The kind's own wording, for a condition raised without words, names the module whole, dots and all,
|
||||
// and never claims the account is the operator's.
|
||||
w := plainWordings[kindReloginNeeded](conditions.Observation{Scope: conditions.ScopeModule, ID: "razer.lights.g14", Machine: "g14"})
|
||||
if w.Headline != "razer.lights waits for a new login on g14" || w.Needs == "" || len(w.Actions) != 0 ||
|
||||
strings.Contains(w.Explanation, "your account") {
|
||||
t.Errorf("the kind's wording: %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
// When a module not working turns into one waiting for a new login, and back, the clearing line says what it
|
||||
// became, never that it works again (issue 318 review).
|
||||
func TestAConditionThatBecameTheOtherKindSaysSoWhenItClears(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
store := conditions.NewInMemory()
|
||||
told := &conditions.Told{}
|
||||
k := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store, Teller: told})
|
||||
t.Cleanup(func() { k.Close(context.Background()) })
|
||||
at := h0
|
||||
say := func(rs ...link.ResourceHealth) {
|
||||
t.Helper()
|
||||
at = at.Add(time.Second)
|
||||
for i := range rs {
|
||||
rs[i].Since = at
|
||||
}
|
||||
if err := stateHealth(ctx, open.inventory, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at,
|
||||
Resources: rs}, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
unit := link.ResourceHealth{Module: "lights", Resource: "lights.daemon", Kind: link.KindUnit, Target: "lights.service",
|
||||
Account: "operator", State: link.StateUnhealthy, Reason: "failed in the account's own service manager (exit-code)"}
|
||||
account := link.ResourceHealth{Module: "lights", Resource: "lights.operator", Kind: link.KindAccount, Target: "operator",
|
||||
Account: "operator", State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": operator is in the group lights"}
|
||||
say(unit)
|
||||
say(unit) // lights not working
|
||||
say(account, unit)
|
||||
say(account, unit) // now it waits for a login
|
||||
var resolved []string
|
||||
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline) && len(resolved) == 0; {
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
for _, e := range told.Said() {
|
||||
if e.Change == conditions.ChangeCleared {
|
||||
resolved = append(resolved, e.Condition.Key+": "+e.Condition.Resolved)
|
||||
// The clearing line is held to the plain rule too (ADR 0253).
|
||||
w := conditions.Words{Headline: e.Condition.Headline, Explanation: e.Condition.Explanation,
|
||||
Resolved: e.Condition.Resolved, Needs: e.Condition.Needs}
|
||||
if why, ok := conditions.PlainWords(w, "laptop"); !ok {
|
||||
t.Errorf("%s: its clearing words are not plain: %s", e.Condition.Key, why)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(resolved) != 1 || !strings.Contains(resolved[0], "module.lights.laptop.unhealthy: lights on laptop now waits only for a new login") {
|
||||
t.Fatalf("cleared %v; want the not-working condition cleared as now waiting for a login", resolved)
|
||||
}
|
||||
}
|
||||
|
||||
// Which moves add an account group, read from the builds' manifests (issue 318 review).
|
||||
func TestOnlyAMoveThatAddsAnAccountGroupCanBringAWait(t *testing.T) {
|
||||
user := func(groups ...any) catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "lights", Resources: []map[string]any{{"id": "operator", "type": "user",
|
||||
"name": "operator", "groups": groups}}}
|
||||
}
|
||||
none := catalogue.Manifest{Module: "lights"}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
from catalogue.Manifest
|
||||
had bool
|
||||
to catalogue.Manifest
|
||||
addsSome bool
|
||||
}{
|
||||
{"a group added", none, true, user("lights"), true},
|
||||
{"the same group kept", user("lights"), true, user("lights"), false},
|
||||
{"a second group added", user("lights"), true, user("lights", "video"), true},
|
||||
{"a group taken away", user("lights", "video"), true, user("lights"), false},
|
||||
{"new to the machine, with a group", none, false, user("lights"), true},
|
||||
{"new to the machine, no group", none, false, none, false},
|
||||
} {
|
||||
if got := addsAccountGroups(c.from, c.had, c.to); got != c.addsSome {
|
||||
t.Errorf("%s: adds %v; want %v", c.name, got, c.addsSome)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,854 @@
|
||||
package main
|
||||
|
||||
// The plain words of every condition kind the controller raises (novox/hq ADR 0253): what the operator
|
||||
// reads — a headline, one or two sentences on what it means and whether to act, and the line said when it
|
||||
// clears. The summary beside them keeps the ids, commits and verbs for whoever looks closer.
|
||||
//
|
||||
// A kind whose words need more than the condition's subject (a walk's modules, a unit's name, a hand-act's
|
||||
// cause) is worded where it is raised, and its kind here is the fallback. Every kind raised in the test
|
||||
// suite is held to having words, and to their being plain (plain_words_test.go).
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
type words = conditions.Words
|
||||
|
||||
// worded is a wording that needs only the condition's subject.
|
||||
func worded(fn func(o conditions.Observation) words) func(conditions.Observation) words { return fn }
|
||||
|
||||
func init() {
|
||||
for kind, fn := range plainWordings {
|
||||
conditions.Wording(kind, fn)
|
||||
}
|
||||
}
|
||||
|
||||
// machineOr is the machine a condition concerns, or what to say when none is named.
|
||||
func machineOr(o conditions.Observation, none string) string {
|
||||
if o.Machine != "" {
|
||||
return o.Machine
|
||||
}
|
||||
return none
|
||||
}
|
||||
|
||||
// idPart is the n-th dotted part of a condition's id, or "".
|
||||
func idPart(o conditions.Observation, n int) string {
|
||||
parts := strings.Split(o.ID, ".")
|
||||
if n < len(parts) {
|
||||
return parts[n]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// dataWords is "<item> of <module>" for a data condition keyed by machine, module and item.
|
||||
func dataWords(o conditions.Observation) (what, module string) {
|
||||
module, item := idPart(o, 1), strings.Join(strings.Split(o.ID, ".")[min(2, len(strings.Split(o.ID, "."))):], " ")
|
||||
switch {
|
||||
case module != "" && item != "":
|
||||
return item + " of " + module, module
|
||||
case module != "":
|
||||
return "data of " + module, module
|
||||
}
|
||||
return "data", "a module"
|
||||
}
|
||||
|
||||
var plainWordings = map[string]func(conditions.Observation) words{
|
||||
// Machines and their node-engine.
|
||||
"silent": worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
w := words{Headline: m + " is not answering",
|
||||
Explanation: fmt.Sprintf("The mesh has not heard from %s for several minutes. A machine asleep or away is "+
|
||||
"normal.", m),
|
||||
Resolved: m + " answers again"}
|
||||
if o.Severity == conditions.Urgent {
|
||||
w.Needs = fmt.Sprintf("check that %s is on and online.", m)
|
||||
w.Explanation = fmt.Sprintf("The mesh has not heard from %s for half an hour, and the mesh is run from it: "+
|
||||
"nothing changes anywhere until it is back.", m)
|
||||
}
|
||||
return w
|
||||
}),
|
||||
"sent-not-reported": worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + " has not confirmed a change",
|
||||
Explanation: fmt.Sprintf("The mesh sent %s new instructions and %s has not said it applied them yet. "+
|
||||
"It usually catches up by itself.", m, m),
|
||||
Resolved: m + " applied the change"}
|
||||
}),
|
||||
"tools-silent": worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "Tools on " + m + " are not answering",
|
||||
Explanation: fmt.Sprintf("The tool runner on %s has not checked in, so nothing can be done on %s through "+
|
||||
"the mesh until it does. A machine asleep is normal.", m, m),
|
||||
Resolved: "Tools on " + m + " answer again"}
|
||||
}),
|
||||
kindAwaitingPush: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + " waits for new instructions",
|
||||
Explanation: fmt.Sprintf("%s's new instructions go out with the next push. Nothing is broken meanwhile.",
|
||||
conditions.Capital(m)),
|
||||
Resolved: m + " got its new instructions"}
|
||||
}),
|
||||
"declaration-refused": worded(func(o conditions.Observation) words {
|
||||
if o.Scope == conditions.ScopeMesh {
|
||||
return words{Headline: "No machine can get new instructions",
|
||||
Needs: "decide whether to undo the last catalogue change; the details say what fails.",
|
||||
Explanation: "The mesh cannot work out its private network, so no machine's instructions can be " +
|
||||
"made. Machines keep running what they have.",
|
||||
Resolved: "Machines can get new instructions again"}
|
||||
}
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + " cannot get new instructions",
|
||||
Needs: "decide whether to undo the last catalogue change; the details say what fails.",
|
||||
Explanation: fmt.Sprintf("What the mesh would send %s is wrong and would be refused, so nothing new "+
|
||||
"reaches it. It keeps running what it has.", m),
|
||||
Resolved: m + " can get new instructions again"}
|
||||
}),
|
||||
kindAgentCanBecomeRoot: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "Sessions on " + m + " could become root",
|
||||
Needs: "take the sessions' own account out of every group and rule that grants root; the details say which.",
|
||||
Explanation: fmt.Sprintf("Assistant sessions on %s run under an account of their own, so that none "+
|
||||
"can take over the machine without you. The machine cannot show that this holds now, so an answer "+
|
||||
"from your phone authorises nothing there until it does.", m),
|
||||
Resolved: "Sessions on " + m + " cannot become root again"}
|
||||
}),
|
||||
"own-address-banned": worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + " has banned the mesh",
|
||||
Needs: fmt.Sprintf("lift the ban on %s, or wait for it to expire.", m),
|
||||
Explanation: fmt.Sprintf("The intrusion protection on %s banned one of the mesh's own addresses, so the "+
|
||||
"mesh is locked out of it.", m),
|
||||
Resolved: m + " no longer bans the mesh"}
|
||||
}),
|
||||
"core-behind": worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + " runs an old node-engine",
|
||||
Explanation: fmt.Sprintf("%s runs an older build of the mesh's own software than the mesh holds. It "+
|
||||
"works, and gets the newer one with a later delivery.", conditions.Capital(m)),
|
||||
Resolved: m + " runs the current node-engine"}
|
||||
}),
|
||||
kindMachineUnits: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "Failed services on " + m,
|
||||
Needs: fmt.Sprintf("mend or remove them on %s, or silence this if they do not matter.", m),
|
||||
Explanation: "No module manages them, so the mesh does not repair them.",
|
||||
Resolved: "No failed services on " + m + " any more", Actions: []conditions.Action{conditions.SilenceAction(o.Key())}}
|
||||
}),
|
||||
kindMachineNetwork: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + " has a network problem",
|
||||
Explanation: fmt.Sprintf("%s's own network check failed: its name lookups, tunnel, bus connection or "+
|
||||
"route. It often passes again once a network change settles.", conditions.Capital(m)),
|
||||
Resolved: m + "'s network is fine again"}
|
||||
}),
|
||||
kindNetworkRewritten: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + "'s network settings were changed",
|
||||
Explanation: fmt.Sprintf("Something other than the mesh rewrote %s's network settings, often a VPN "+
|
||||
"client. Mesh names may not resolve there until it is undone.", m),
|
||||
Resolved: m + "'s network settings are the mesh's again"}
|
||||
}),
|
||||
kindNetworkUnreachable: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + " cannot be reached",
|
||||
Explanation: fmt.Sprintf("Other machines cannot reach %s over the mesh's network. A machine away or "+
|
||||
"asleep is normal.", m),
|
||||
Resolved: m + " can be reached again"}
|
||||
}),
|
||||
// A pending assignment that was not made (novox/hq issue 325, ADR 0261).
|
||||
kindPendingEnded: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
module := idPart(o, 1)
|
||||
if module == "" {
|
||||
module = "a module"
|
||||
}
|
||||
return words{Headline: module + " was not put on " + m,
|
||||
Needs: "decide whether to build it again or take the assignment back; the details say why.",
|
||||
Explanation: fmt.Sprintf("An assignment of %s to %s waited for its build, and the build did not "+
|
||||
"register it, so it was not made.", module, m),
|
||||
Resolved: "Resolved: " + module + " on " + m + " is settled"}
|
||||
}),
|
||||
kindBindingKept: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "A module's data source is held on " + m,
|
||||
Needs: "decide where its data lives; the details name both places.",
|
||||
Explanation: fmt.Sprintf("A module on %s should switch to another provider of its data, and the mesh "+
|
||||
"kept the old one because switching would leave the data behind.", m),
|
||||
Resolved: "Resolved: the data source on " + m + " is decided"}
|
||||
}),
|
||||
kindBindingMoving: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "A module's data source moves on " + m,
|
||||
Needs: "move its data to the new provider before the next push.",
|
||||
Explanation: fmt.Sprintf("At the next push a module on %s switches to another provider of its data.", m),
|
||||
Resolved: "Resolved: the data source move on " + m + " is settled"}
|
||||
}),
|
||||
kindBindingMoved: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "A module's data source moved on " + m,
|
||||
Needs: "check that it found its data.",
|
||||
Explanation: fmt.Sprintf("A module on %s switched to another provider, and its data may still be with the old one.", m),
|
||||
Resolved: "Resolved: the data source move on " + m + " is settled"}
|
||||
}),
|
||||
|
||||
// Modules and providers.
|
||||
kindModuleUnhealthy: worded(func(o conditions.Observation) words {
|
||||
thing := conditions.ThingWords(o)
|
||||
return words{Headline: conditions.Capital(thing) + " is not working",
|
||||
Explanation: fmt.Sprintf("%s is not healthy. It clears as soon as it runs again.", conditions.Capital(thing)),
|
||||
Resolved: conditions.Capital(thing) + " works again"}
|
||||
}),
|
||||
kindReloginNeeded: worded(func(o conditions.Observation) words {
|
||||
// A module's name may hold dots: it is the id without its machine.
|
||||
module := ""
|
||||
if o.Scope == conditions.ScopeModule && o.Machine != "" {
|
||||
module = strings.TrimSuffix(o.ID, "."+o.Machine)
|
||||
}
|
||||
return reloginWords(orModule(module), machineOr(o, "a machine"), false)
|
||||
}),
|
||||
kindUsedAsFound: worded(func(o conditions.Observation) words {
|
||||
module := ""
|
||||
if o.Scope == conditions.ScopeModule && o.Machine != "" {
|
||||
module = strings.TrimSuffix(o.ID, "."+o.Machine)
|
||||
}
|
||||
w := usedAsFoundObservation(orModule(module), machineOr(o, "a machine"), o.Summary, nil)
|
||||
return words{Headline: w.Headline, Explanation: w.Explanation, Needs: w.Needs, Resolved: w.Resolved}
|
||||
}),
|
||||
kindProviderFailing: worded(func(o conditions.Observation) words {
|
||||
thing, consumer := conditions.ThingWords(o), idPart(o, 2)
|
||||
if consumer == "" {
|
||||
consumer = "a module"
|
||||
}
|
||||
return words{Headline: conditions.Capital(thing) + " keeps failing " + consumer,
|
||||
Explanation: fmt.Sprintf("%s provides something %s needs, such as a database, and keeps failing to set "+
|
||||
"it up. It tries again by itself.", conditions.Capital(thing), consumer),
|
||||
Resolved: conditions.Capital(thing) + " serves " + consumer + " again"}
|
||||
}),
|
||||
kindProviderSilent: worded(func(o conditions.Observation) words {
|
||||
thing := conditions.ThingWords(o)
|
||||
return words{Headline: conditions.Capital(thing) + " went quiet",
|
||||
Explanation: fmt.Sprintf("%s said it keeps failing a module that depends on it, and then stopped "+
|
||||
"reporting.", conditions.Capital(thing)),
|
||||
Resolved: conditions.Capital(thing) + " reports again"}
|
||||
}),
|
||||
kindRetireWaiting: worded(func(o conditions.Observation) words {
|
||||
thing := conditions.ThingWords(o)
|
||||
return words{Headline: "Retiring data on " + thing + " waits for you",
|
||||
Needs: "approve or reject retiring it; the details list what would go.",
|
||||
Explanation: fmt.Sprintf("%s keeps data the mesh no longer asks for, too much to retire on its own.",
|
||||
conditions.Capital(thing)),
|
||||
Resolved: "Resolved: the retirement on " + thing + " is decided"}
|
||||
}),
|
||||
kindRetireRejected: worded(func(o conditions.Observation) words {
|
||||
thing := conditions.ThingWords(o)
|
||||
return words{Headline: "Unused data kept on " + thing,
|
||||
Explanation: fmt.Sprintf("You chose to keep data on %s that the mesh no longer asks for; it is still there.", thing),
|
||||
Resolved: "Resolved: the unused data on " + thing + " is gone"}
|
||||
}),
|
||||
kindCleanupWaiting: worded(func(o conditions.Observation) words {
|
||||
var where string
|
||||
if o.Scope == conditions.ScopeProvider {
|
||||
where = conditions.ThingWords(o)
|
||||
} else {
|
||||
what, _ := dataWords(o)
|
||||
where = what + " on " + machineOr(o, "a machine")
|
||||
}
|
||||
return words{Headline: "Retired data waits for cleanup",
|
||||
Needs: "delete it or keep it; nothing is deleted without your word.",
|
||||
Explanation: fmt.Sprintf("Retired data of %s has been kept for over thirty days.", where),
|
||||
Resolved: "Resolved: the retired data is dealt with"}
|
||||
}),
|
||||
|
||||
// Data.
|
||||
kindDataUnmeasured: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "Data on " + m + " could not be measured",
|
||||
Explanation: fmt.Sprintf("The backup holder on %s did not say what it measured, so nothing is known about "+
|
||||
"the data there. The next measurement may settle it.", m),
|
||||
Resolved: "Data on " + m + " is measured again"}
|
||||
}),
|
||||
kindProtectionMissing: worded(func(o conditions.Observation) words {
|
||||
what, _ := dataWords(o)
|
||||
return words{Headline: conditions.Capital(what) + " is unprotected",
|
||||
Needs: "check the storage it is on.",
|
||||
Explanation: fmt.Sprintf("%s on %s is said to be protected by redundant storage, and it is not on any "+
|
||||
"the mesh can see.", conditions.Capital(what), machineOr(o, "its machine")),
|
||||
Resolved: conditions.Capital(what) + " is protected again"}
|
||||
}),
|
||||
kindDataMissing: worded(func(o conditions.Observation) words {
|
||||
what, _ := dataWords(o)
|
||||
return words{Headline: conditions.Capital(what) + " is gone",
|
||||
// No silence from a click: data loss is never waved away from a notification (ADR 0258).
|
||||
Needs: "restore it from a backup, or silence this " + FromMeshMCPServer,
|
||||
Explanation: fmt.Sprintf("Where %s is kept on %s, nothing exists any more.", what, machineOr(o, "its machine")),
|
||||
Resolved: conditions.Capital(what) + " is back"}
|
||||
}),
|
||||
kindDataShrank: worded(func(o conditions.Observation) words {
|
||||
what, _ := dataWords(o)
|
||||
m := machineOr(o, "its machine")
|
||||
if idPart(o, 1) == "dataset" {
|
||||
what = "a dataset"
|
||||
}
|
||||
return words{Headline: conditions.Capital(what) + " shrank on " + m,
|
||||
// No silence from a click: data loss is never waved away from a notification (ADR 0258).
|
||||
Needs: "restore the last good copy from a backup, or silence this " + FromMeshMCPServer,
|
||||
Explanation: "More than half of what it held is gone within a week.",
|
||||
Resolved: "Resolved: the shrinking on " + m + " is explained"}
|
||||
}),
|
||||
kindDataQuiet: worded(func(o conditions.Observation) words {
|
||||
what, _ := dataWords(o)
|
||||
return words{Headline: conditions.Capital(what) + " stopped changing",
|
||||
Needs: "check that what writes it still runs.",
|
||||
Explanation: fmt.Sprintf("%s on %s is normally written all the time and has not been for longer than "+
|
||||
"usual.", conditions.Capital(what), machineOr(o, "its machine")),
|
||||
Resolved: conditions.Capital(what) + " is written again"}
|
||||
}),
|
||||
kindBackupStale: worded(func(o conditions.Observation) words {
|
||||
what, _ := dataWords(o)
|
||||
m := machineOr(o, "its machine")
|
||||
return words{Headline: "No recent backup of " + what,
|
||||
Needs: fmt.Sprintf("check that backups run on %s.", m),
|
||||
Explanation: fmt.Sprintf("%s on %s has no good backup recent enough. If the machine failed now, what "+
|
||||
"changed since would be lost.", conditions.Capital(what), m),
|
||||
Resolved: conditions.Capital(what) + " is backed up again"}
|
||||
}),
|
||||
kindArrayDegraded: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "Storage on " + m + " is degraded",
|
||||
Needs: fmt.Sprintf("check the disks of %s.", m),
|
||||
Explanation: "Its redundant storage lost a disk or is rebuilding; the data on it is less protected until it is whole.",
|
||||
Resolved: "Storage on " + m + " is whole again"}
|
||||
}),
|
||||
kindEmptyReplacement: worded(func(o conditions.Observation) words {
|
||||
module := idPart(o, 1)
|
||||
if o.Scope == conditions.ScopeProvider {
|
||||
module = idPart(o, 2)
|
||||
}
|
||||
if module == "" {
|
||||
module = "a module"
|
||||
}
|
||||
return words{Headline: conditions.Capital(module) + " runs on empty data",
|
||||
Needs: "decide where its data lives: move the data, or move it back.",
|
||||
Explanation: fmt.Sprintf("%s on %s is using an empty copy of its data while the full copy is kept "+
|
||||
"elsewhere.", conditions.Capital(module), machineOr(o, "its machine")),
|
||||
Resolved: conditions.Capital(module) + " has its data again"}
|
||||
}),
|
||||
kindDataHeldTwice: worded(func(o conditions.Observation) words {
|
||||
consumer := idPart(o, 1)
|
||||
if consumer == "" {
|
||||
consumer = "a module"
|
||||
}
|
||||
return words{Headline: conditions.Capital(consumer) + "'s data is on several machines",
|
||||
Needs: "decide which copy to keep.",
|
||||
Explanation: fmt.Sprintf("%s has live data on more than one machine and uses only one.", conditions.Capital(consumer)),
|
||||
Resolved: conditions.Capital(consumer) + "'s data is in one place again"}
|
||||
}),
|
||||
|
||||
// Deliveries and walks.
|
||||
"stalled": worded(func(o conditions.Observation) words {
|
||||
if o.Scope == conditions.ScopeDelivery {
|
||||
return words{Headline: "A delivery is held too long",
|
||||
Explanation: "A delivery has been held past its time limit.", Needs: conditions.ResolverNeeds(o),
|
||||
Resolved: "Resolved: the delivery moves again"}
|
||||
}
|
||||
return words{Headline: "A delivery is stuck halfway",
|
||||
Explanation: "Its walk across the machines has not moved for longer than usual. Nothing is lost.",
|
||||
Resolved: "Resolved: the delivery moves again"}
|
||||
}),
|
||||
kindWalkWaiting: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A delivery is waiting to start",
|
||||
Explanation: "A merged change is built, and mesh-delivery (the module that decides when a delivery goes " +
|
||||
"out) has not let it start yet.",
|
||||
Resolved: "Resolved: the delivery is no longer waiting"}
|
||||
}),
|
||||
"merge-not-acted": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A merge was never picked up",
|
||||
Explanation: "A pull request was merged, and the controller never heard of it, so nothing was built or " +
|
||||
"sent. The mesh catches up on missed merges by itself.",
|
||||
Resolved: "Resolved: the merge was picked up"}
|
||||
}),
|
||||
"ask-lost": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A build has no result",
|
||||
Explanation: "A build was asked for and never answered. It is asked again with the next delivery.",
|
||||
Resolved: "Resolved: the build has its result"}
|
||||
}),
|
||||
kindRolledBack: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
module := idPart(o, 0)
|
||||
if o.Scope != conditions.ScopeBuild && o.Scope != conditions.ScopeCore {
|
||||
module = "a module"
|
||||
}
|
||||
if o.Scope == conditions.ScopeCore {
|
||||
return words{Headline: "A core update was put back on " + m,
|
||||
Explanation: fmt.Sprintf("A new build of the mesh's own %s did not become healthy on %s, so the build "+
|
||||
"before was put back and runs.", module, m),
|
||||
Resolved: "Resolved: the core on " + m + " is healthy"}
|
||||
}
|
||||
return words{Headline: conditions.Capital(module) + " update put back on " + m,
|
||||
Explanation: fmt.Sprintf("The new build of %s did not become healthy on %s, so the mesh put back the one "+
|
||||
"before, which runs. The change is not delivered until it is fixed.", module, m),
|
||||
Resolved: "Resolved: " + module + " on " + m + " is settled"}
|
||||
}),
|
||||
kindRollbackFailed: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "Putting back an update failed on " + m,
|
||||
Needs: fmt.Sprintf("check what runs on %s; the details say which build.", m),
|
||||
Explanation: fmt.Sprintf("A build that failed on %s could not be put back to the one before.", m),
|
||||
Resolved: "Resolved: " + m + " runs a good build again"}
|
||||
}),
|
||||
"release-held": worded(func(o conditions.Observation) words {
|
||||
return releaseHeldWords(nil, o.Also)
|
||||
}),
|
||||
"facts-stale": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Merge checks use outdated facts",
|
||||
Explanation: "The facts a merge check judges a change against are out of date; they are gathered again " +
|
||||
"by themselves.",
|
||||
Resolved: "Merge checks use current facts again"}
|
||||
}),
|
||||
|
||||
// The controller and the core.
|
||||
"controller-deaf": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The controller stopped listening",
|
||||
Needs: "restart the controller if this stays, " + FromMeshMCPServer,
|
||||
Explanation: "The controller, which coordinates the mesh, has taken no messages for minutes while some " +
|
||||
"wait. Changes and repairs do not happen until it recovers.",
|
||||
Resolved: "The controller listens again"}
|
||||
}),
|
||||
"self-check-silent": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The mesh's self-check stopped",
|
||||
Needs: "restart the controller if this stays, " + FromMeshMCPServer,
|
||||
Explanation: "The self-check, which looks over the whole mesh every few minutes, has not finished a run. " +
|
||||
"Problems may go unnoticed until it runs again.",
|
||||
Resolved: "The self-check runs again"}
|
||||
}),
|
||||
"watchdogs-silent": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The mesh's watchdogs stopped",
|
||||
Needs: "restart the controller if this stays, " + FromMeshMCPServer,
|
||||
Explanation: "The watchdogs, which notice when something expected does not happen, have not run, so " +
|
||||
"missed signals are not noticed.",
|
||||
Resolved: "The watchdogs run again"}
|
||||
}),
|
||||
"lease-lost": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The controller lost its lease",
|
||||
Needs: "make sure only one controller runs; the details say which machines claim it.",
|
||||
Explanation: "The controller holds a lease so that only one copy of it acts at a time. It lost or could " +
|
||||
"not renew it, so two could act at once, or none.",
|
||||
Resolved: "The controller holds its lease again"}
|
||||
}),
|
||||
"lease-split": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Two controllers may be acting",
|
||||
Needs: "make sure only one controller runs; the details say which machines claim it.",
|
||||
Explanation: "The lease that lets one controller act at a time does not agree with its record.",
|
||||
Resolved: "One controller acts again"}
|
||||
}),
|
||||
"stale-writer": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Outdated instructions are being sent",
|
||||
Explanation: "Something kept sending instructions older than what the machines already hold, and they " +
|
||||
"were refused, which is harmless.",
|
||||
Resolved: "Resolved: no more outdated instructions"}
|
||||
}),
|
||||
"status-slow": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The mesh answers slowly",
|
||||
Explanation: "Asking the mesh for its status took longer than ten seconds. It still works, only slowly.",
|
||||
Resolved: "The mesh answers quickly again"}
|
||||
}),
|
||||
kindCoreUnhealthy: worded(func(o conditions.Observation) words {
|
||||
if o.Scope == conditions.ScopeBus {
|
||||
return words{Headline: "The bus is not healthy",
|
||||
Needs: "check the machine the bus runs on; the details say what fails.",
|
||||
Explanation: "The bus, which carries every message in the mesh, is not healthy. Instructions, builds " +
|
||||
"and tool calls may fail until it recovers.",
|
||||
Resolved: "The bus is healthy again"}
|
||||
}
|
||||
m := machineOr(o, "its machine")
|
||||
return words{Headline: "The mesh's own software fails on " + m,
|
||||
Explanation: fmt.Sprintf("A part of the mesh's own software on %s is not healthy; it is restarted by itself.", m),
|
||||
Resolved: "The mesh's own software on " + m + " is healthy again"}
|
||||
}),
|
||||
"call-hung": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A tool call is hanging",
|
||||
Explanation: "A call to one of the mesh's tools has run far past its time limit; it is ended by itself.",
|
||||
Resolved: "Resolved: the tool call ended"}
|
||||
}),
|
||||
"probe-failed": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A check of the mesh could not run",
|
||||
Explanation: "One of the mesh's own checks could not run, so what it watches is unknown for now. It runs " +
|
||||
"again every few minutes.",
|
||||
Resolved: "The check runs again"}
|
||||
}),
|
||||
kindHealersBraked: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "All automatic repairs stopped",
|
||||
Needs: "read the details and decide whether the repairs may run again.",
|
||||
Explanation: "The healers (the mesh's automatic repairs) acted more often in an hour than allowed, which " +
|
||||
"looks like a loop, so all of them stopped.",
|
||||
Resolved: "Automatic repairs run again"}
|
||||
}),
|
||||
"healer-wanted": worded(func(o conditions.Observation) words {
|
||||
cause := strings.TrimPrefix(o.ID, "hand-acts.")
|
||||
if o.Scope != conditions.ScopeMesh {
|
||||
cause = "something"
|
||||
}
|
||||
return words{Headline: conditions.Capital(strings.ReplaceAll(cause, ".", " ")) + " keeps being fixed by hand",
|
||||
Explanation: "A person keeps repairing this by hand, and an automatic repair is wanted for it. Nothing is " +
|
||||
"broken now.",
|
||||
Resolved: "Resolved: no more hand repairs of " + strings.ReplaceAll(cause, ".", " ")}
|
||||
}),
|
||||
|
||||
// The bus and seats.
|
||||
"resolver-wrong": worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "Name lookups fail on " + m,
|
||||
Needs: fmt.Sprintf("check that %s is on and online.", m),
|
||||
Explanation: fmt.Sprintf("The mesh's name resolver on %s does not answer machine names correctly, so "+
|
||||
"machines that rely on it may not find each other.", m),
|
||||
Resolved: "Name lookups work on " + m + " again"}
|
||||
}),
|
||||
"holder-silent": worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
seat := idPart(o, 0)
|
||||
if o.Scope != conditions.ScopeSeat {
|
||||
seat = "a seat"
|
||||
}
|
||||
return words{Headline: conditions.Capital(seat) + " on " + m + " is not answering",
|
||||
Explanation: fmt.Sprintf("Whatever serves %s on %s does not answer, so its tools do nothing there. A "+
|
||||
"machine asleep is normal.", seat, m),
|
||||
Resolved: conditions.Capital(seat) + " on " + m + " answers again"}
|
||||
}),
|
||||
kindBusObjectsUnasserted: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The bus could not be fully set up",
|
||||
Explanation: "When instructions were sent, the bus's queues could not all be checked; they are checked " +
|
||||
"again with the next send.",
|
||||
Resolved: "The bus is fully set up again"}
|
||||
}),
|
||||
kindBusMaintenance: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The bus is being upgraded",
|
||||
Explanation: "The mesh's message system is being upgraded; some things pause until it is done.",
|
||||
Resolved: "The bus upgrade is done"}
|
||||
}),
|
||||
kindBusStepWaiting: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Sends wait for a bus upgrade",
|
||||
Needs: "start the bus upgrade " + FromMeshMCPServer,
|
||||
Explanation: "A new version of the mesh's message system is built, and only a person installs it. Until " +
|
||||
"then nothing else is sent to the machine that runs it.",
|
||||
Resolved: "Resolved: the bus upgrade started, and sends go on"}
|
||||
}),
|
||||
kindBusUpgradeFailed: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The bus upgrade failed",
|
||||
Needs: "decide whether to put the bus back to the version before; the details say how.",
|
||||
Explanation: "The upgrade of the mesh's message system did not end healthy in time.",
|
||||
Resolved: "Resolved: the bus is healthy after its upgrade"}
|
||||
}),
|
||||
kindConsumerBehind: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Messages pile up for a listener",
|
||||
Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A " +
|
||||
"healer restarts it if it stays.",
|
||||
Resolved: "The listener caught up"}
|
||||
}),
|
||||
"consumer-wrong": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A listener on the bus is set up wrong",
|
||||
Explanation: "One of the mesh's listeners on the bus is missing or not as defined; the next send sets " +
|
||||
"it up again.",
|
||||
Resolved: "The listener is set up right again"}
|
||||
}),
|
||||
"consumer-lost": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A listener on the bus is gone",
|
||||
Explanation: "One of the mesh's listeners is no longer on the bus; the next send sets it up again.",
|
||||
Resolved: "The listener is back"}
|
||||
}),
|
||||
"slow-consumer": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A listener on the bus is too slow",
|
||||
Explanation: "The bus reports a listener too slow to keep up, so messages to it are late.",
|
||||
Resolved: "The listener keeps up again"}
|
||||
}),
|
||||
kindBusReconnects: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A client keeps losing the bus",
|
||||
Explanation: "One of the mesh's clients lost its connection to the bus again and again in the last hour. " +
|
||||
"While it reconnects, what it says and what it is asked waits.",
|
||||
Resolved: "Resolved: the client stays connected"}
|
||||
}),
|
||||
"max-deliveries": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A listener gave up on messages",
|
||||
Needs: "deliver them again or drop them, from the mesh MCP server.",
|
||||
Explanation: "A listener on the bus could not handle messages after several tries, so what they asked " +
|
||||
"for was not done. The mesh keeps them until you deliver them again or drop them.",
|
||||
Resolved: "Resolved: the messages given up on were delivered again or dropped"}
|
||||
}),
|
||||
"refused": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The bus refuses some messages",
|
||||
Explanation: "The bus refused messages from a part of the mesh, so what they carried did not happen.",
|
||||
Resolved: "Resolved: the bus takes the messages again"}
|
||||
}),
|
||||
"stream-wrong": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Part of the bus's storage is wrong",
|
||||
Needs: "check the machine the bus runs on; the details say what is missing.",
|
||||
Explanation: "A store the controller keeps on the bus is missing or not as defined, so what it keeps there may be lost.",
|
||||
Resolved: "The bus's storage is right again"}
|
||||
}),
|
||||
"archives-unheld": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Kept builds are about to be cleaned up",
|
||||
Explanation: "Some kept builds are not claimed by any module, so the store's cleanup would delete them.",
|
||||
Resolved: "Resolved: every kept build is claimed"}
|
||||
}),
|
||||
"archives-missing": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Kept builds are missing",
|
||||
Explanation: "Some builds the mesh relies on are no longer in the store; they are built again when needed.",
|
||||
Resolved: "Resolved: the kept builds are back"}
|
||||
}),
|
||||
}
|
||||
|
||||
// humanDuration is a duration as a person says it: "56 minutes", "36 hours", "3 days".
|
||||
func humanDuration(d time.Duration) string {
|
||||
switch {
|
||||
case d < 2*time.Minute:
|
||||
return "a minute"
|
||||
case d < 2*time.Hour:
|
||||
return fmt.Sprintf("%d minutes", int(d.Minutes()))
|
||||
case d < 48*time.Hour:
|
||||
return fmt.Sprintf("%d hours", int(d.Hours()))
|
||||
}
|
||||
return fmt.Sprintf("%d days", int(d.Hours()/24))
|
||||
}
|
||||
|
||||
// repoName is a repository as a person says it: its name without its owner.
|
||||
func repoName(repository string) string {
|
||||
if i := strings.LastIndex(repository, "/"); i >= 0 {
|
||||
return repository[i+1:]
|
||||
}
|
||||
return repository
|
||||
}
|
||||
|
||||
// namesWords lists names as a person says them: "a", "a and b", "a, b and 2 more".
|
||||
func namesWords(names []string, most int) string {
|
||||
switch {
|
||||
case len(names) == 0:
|
||||
return ""
|
||||
case len(names) == 1:
|
||||
return names[0]
|
||||
case len(names) <= most:
|
||||
return strings.Join(names[:len(names)-1], ", ") + " and " + names[len(names)-1]
|
||||
}
|
||||
return strings.Join(names[:most], ", ") + fmt.Sprintf(" and %d more", len(names)-most)
|
||||
}
|
||||
|
||||
// planModules are the modules a walk moves, sorted.
|
||||
func planModules(p inventory.Plan) []string {
|
||||
out := make([]string, 0, len(p.Modules))
|
||||
for m := range p.Modules {
|
||||
out = append(out, m)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// deliveryWhat is what a delivery changes, as a person names it: its modules when they are few, else
|
||||
// their count and the repository.
|
||||
func deliveryWhat(modules []string, repository string) string {
|
||||
if len(modules) > 0 && len(modules) <= 2 {
|
||||
return namesWords(modules, 2)
|
||||
}
|
||||
if len(modules) > 2 {
|
||||
return fmt.Sprintf("%d modules of %s", len(modules), repoName(repository))
|
||||
}
|
||||
return repoName(repository)
|
||||
}
|
||||
|
||||
// deliveryName is a delivery's name in a headline: "openrazer delivery", "mesh-catalog delivery".
|
||||
func deliveryName(modules []string, repository string) string {
|
||||
if len(modules) > 0 && len(modules) <= 2 {
|
||||
return namesWords(modules, 2) + " delivery"
|
||||
}
|
||||
return repoName(repository) + " delivery"
|
||||
}
|
||||
|
||||
// walkWaitingWords explains a walk waiting for its delivery's word.
|
||||
func walkWaitingWords(w waitFacts, in time.Duration, severity conditions.Severity) string {
|
||||
what := deliveryWhat(w.modules, w.repository)
|
||||
if severity == conditions.Urgent {
|
||||
return fmt.Sprintf("The change to %s is merged and built, and mesh-delivery (the module that decides when a "+
|
||||
"delivery goes out) has not let it start for %s, so mesh-delivery may be stuck.", what, humanDuration(in))
|
||||
}
|
||||
return fmt.Sprintf("The change to %s is merged and built, and has waited %s for mesh-delivery (the module that "+
|
||||
"decides when a delivery goes out) to let it start. It becomes a question for you if it still waits after %s.",
|
||||
what, humanDuration(in), humanDuration(waitUrgentAfter))
|
||||
}
|
||||
|
||||
// waitingNeeds is what the operator does about a walk waiting past its urgent bound: nothing before it.
|
||||
func waitingNeeds(severity conditions.Severity) string {
|
||||
if severity == conditions.Urgent {
|
||||
return "start it, or stop it, " + FromMeshMCPServer
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// moduleNeeds is what the operator can do about a module unhealthy on a machine: log in again where its
|
||||
// account's groups wait for it (ADR 0252), restart a failed service, or nothing where the mesh restarts it.
|
||||
// No answer is offered for a restart: a desk click performs only an acknowledgement (ADR 0258).
|
||||
func moduleNeeds(node string, rs []inventory.ResourceHealth) string {
|
||||
unit := ""
|
||||
for _, r := range rs {
|
||||
if strings.Contains(r.Reason, "relogin needed") {
|
||||
return reloginNeeds(node)
|
||||
}
|
||||
if r.Kind == link.KindUnit && unit == "" {
|
||||
unit = unitPlainWords(r.Target)
|
||||
}
|
||||
}
|
||||
if unit != "" {
|
||||
return fmt.Sprintf("restart its service %s on %s %s", unit, node, FromMeshMCPServer)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// FromMeshMCPServer ends what the operator needs when no notification can do it (ADR 0258), naming the mesh MCP
|
||||
// server (the glossary's word; "console" is retired): the answer is not an
|
||||
// acknowledgement, so it is given where the operator is known to be the one asking, until answers are
|
||||
// authorised (to-be 46 phases 5 and 6).
|
||||
const FromMeshMCPServer = "from the mesh MCP server; this notification cannot do it."
|
||||
|
||||
// releaseHeldWords are the plain words of updates held after a walk failed: which modules wait,
|
||||
// on which machines, and where the operator releases them (ADR 0258: a release is not an acknowledgement, so
|
||||
// no notification gives it). Raised with the modules and machines (backlogObservation); the kind's fallback
|
||||
// knows neither.
|
||||
func releaseHeldWords(modules, machines []string) words {
|
||||
what := "Some module updates"
|
||||
headline := "Updates wait for your release"
|
||||
if len(modules) > 0 {
|
||||
what = "Updates of " + namesWords(modules, 3)
|
||||
if h := what + " wait for your release"; len(h) <= conditions.HeadlineMax {
|
||||
headline = h
|
||||
} else if h := fmt.Sprintf("%d module updates wait for your release", len(modules)); len(h) <= conditions.HeadlineMax {
|
||||
headline = h
|
||||
}
|
||||
}
|
||||
where := ""
|
||||
if len(machines) > 0 {
|
||||
where = " on " + namesWords(machines, 4)
|
||||
}
|
||||
return words{Headline: headline,
|
||||
Needs: "release them " + FromMeshMCPServer,
|
||||
Explanation: fmt.Sprintf("%s%s wait for a person to release them, because the last walk failed."+
|
||||
" They are not delivered until then, and stay held if you leave them.", what, where),
|
||||
Resolved: "Resolved: the held updates are released"}
|
||||
}
|
||||
|
||||
// reloginNeeds is what an account waiting for its groups needs (ADR 0252).
|
||||
func reloginNeeds(node string) string {
|
||||
return fmt.Sprintf("log out of %s completely and log in again, or restart it.", node)
|
||||
}
|
||||
|
||||
// stalledWords are the plain words of a delivery held past its bound, as mesh-delivery says it.
|
||||
func stalledWords(l stalledLine, o conditions.Observation) (headline, explanation, resolved, needs string,
|
||||
actions []conditions.Action) {
|
||||
repository, _, _ := strings.Cut(l.ID, "@")
|
||||
name := repoName(repository)
|
||||
if l.State == "unannounced" {
|
||||
// A pull request the forge never announced (novox/hq issue 347): no delivery exists, so there is nothing to
|
||||
// stop, release or close; a new commit on its branch is announced and checked.
|
||||
long := "for too long"
|
||||
if d, err := time.ParseDuration(l.For); err == nil {
|
||||
long = "for " + humanDuration(d)
|
||||
}
|
||||
if o.Resolver == conditions.ResolverOperator {
|
||||
needs = "push a new commit to its branch; the forge announces it and the mesh checks it."
|
||||
}
|
||||
pull := "A pull request of " + name
|
||||
if l.Number > 0 {
|
||||
pull = fmt.Sprintf("Pull request %s #%d", name, l.Number)
|
||||
}
|
||||
return fmt.Sprintf("%s has had no merge check %s", pull, long),
|
||||
fmt.Sprintf("%s has been open %s on a branch that requires the merge check, and the mesh was never asked "+
|
||||
"to check it: the forge never announced it. It cannot merge until it is checked.", pull, long),
|
||||
fmt.Sprintf("%s has a merge check now, or is closed", pull), needs, nil
|
||||
}
|
||||
held := l.State
|
||||
if held == "" {
|
||||
held = "held"
|
||||
}
|
||||
long := "too long"
|
||||
if d, err := time.ParseDuration(l.For); err == nil {
|
||||
long = "for " + humanDuration(d)
|
||||
}
|
||||
if o.Resolver == conditions.ResolverOperator {
|
||||
// Words only: releasing or stopping a delivery is not an acknowledgement, so no desk click
|
||||
// performs it (ADR 0258).
|
||||
switch held {
|
||||
case "held":
|
||||
needs = "release it, or stop it, " + FromMeshMCPServer
|
||||
case "ready", "checked":
|
||||
needs = "merge its pull request, or close it."
|
||||
default:
|
||||
needs = "stop it " + FromMeshMCPServer
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("Delivery of %s %s %s", name, held, long),
|
||||
fmt.Sprintf("A delivery of %s has been %s %s, past its limit.", name, held, long),
|
||||
fmt.Sprintf("Delivery of %s is no longer %s", name, held), needs, actions
|
||||
}
|
||||
|
||||
// causeWords is a hand-act's cause as a person says it.
|
||||
func causeWords(cause string) string {
|
||||
return strings.NewReplacer(".", " ", "_", " ").Replace(cause)
|
||||
}
|
||||
|
||||
// unitPlainWords is a unit as a person names it: "greenclip", "storage-media (a mount)".
|
||||
func unitPlainWords(unit string) string {
|
||||
for suffix, what := range map[string]string{".service": "", ".mount": "a mount", ".timer": "a timer",
|
||||
".socket": "a socket", ".path": "a path unit", ".automount": "an automount", ".scope": "a scope"} {
|
||||
if name, ok := strings.CutSuffix(unit, suffix); ok {
|
||||
if what == "" {
|
||||
return strings.ReplaceAll(name, ".", " ")
|
||||
}
|
||||
return strings.ReplaceAll(name, ".", " ") + " (" + what + ")"
|
||||
}
|
||||
}
|
||||
return strings.ReplaceAll(unit, ".", " ")
|
||||
}
|
||||
|
||||
// resourcePlainWords is what is wrong with one of a module's resources, as a person says it.
|
||||
func resourcePlainWords(r inventory.ResourceHealth) string {
|
||||
if r.Kind == link.KindUnit {
|
||||
return "its service " + unitPlainWords(r.Target) + " " + unitReasonWords(r.Reason)
|
||||
}
|
||||
what := "its " + strings.ReplaceAll(r.Kind, "-", " ")
|
||||
switch r.Reason {
|
||||
case "restarting":
|
||||
return what + " keeps restarting"
|
||||
case "down":
|
||||
return what + " is not running"
|
||||
}
|
||||
if r.Check != "" {
|
||||
return what + " fails its " + r.Check + " check"
|
||||
}
|
||||
return what + " is not healthy"
|
||||
}
|
||||
|
||||
// unitReasonWords is how a unit failed, as a person says it, from the node-engine's words (which may be
|
||||
// systemd's result, such as exit-code, or a sentence carrying it).
|
||||
func unitReasonWords(reason string) string {
|
||||
switch {
|
||||
case strings.Contains(reason, "start-limit"):
|
||||
return "failed too often to be started again"
|
||||
case strings.Contains(reason, "exit-code"):
|
||||
return "stopped with an error"
|
||||
case strings.Contains(reason, "core-dump"), strings.Contains(reason, "signal"):
|
||||
return "crashed"
|
||||
case strings.Contains(reason, "timeout"):
|
||||
return "timed out"
|
||||
}
|
||||
return "failed"
|
||||
}
|
||||
|
||||
// orModule is a module's name, or "a module" when none is known.
|
||||
func orModule(m string) string {
|
||||
if m == "" {
|
||||
return "a module"
|
||||
}
|
||||
return m
|
||||
}
|
||||
@@ -0,0 +1,332 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The conditions the operator read on 2026-10-08, raised again from the same facts (novox/hq ADR 0253):
|
||||
// each keeps its summary for whoever looks closer, and now carries a headline, an explanation and a
|
||||
// resolved line the operator reads at a glance — no plan id, commit, key, verb or clock time in them.
|
||||
|
||||
// plainExample checks one finding's words as the keeper will keep them: plain, within their bounds,
|
||||
// opened by the verdict, and saying what the operator needs; and its actions by label.
|
||||
func plainExample(t *testing.T, o conditions.Observation, headline, explanation string, actions ...string) {
|
||||
t.Helper()
|
||||
machines := append([]string{o.Machine}, o.Also...)
|
||||
w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs,
|
||||
Actions: o.Actions}
|
||||
if why, ok := conditions.PlainWords(w, machines...); !ok {
|
||||
t.Errorf("%s: not plain: %s\n %+v", o.Key(), why, w)
|
||||
}
|
||||
said := conditions.Verdict(o.Needs, o.Explanation)
|
||||
if o.Headline != headline || said != explanation {
|
||||
t.Errorf("%s: says\n %q\n %q\nwant\n %q\n %q", o.Key(), o.Headline, said, headline, explanation)
|
||||
}
|
||||
var labels []string
|
||||
for _, a := range o.Actions {
|
||||
labels = append(labels, a.Label)
|
||||
}
|
||||
if strings.Join(labels, ",") != strings.Join(actions, ",") {
|
||||
t.Errorf("%s: actions %v, want %v", o.Key(), labels, actions)
|
||||
}
|
||||
quiet := ""
|
||||
if o.Needs == "" && o.Severity == conditions.Warning {
|
||||
quiet = " (needs nothing and is a warning: no popup; it stays in conditions and the history)"
|
||||
}
|
||||
t.Logf("\nBEFORE: %s\nAFTER: %s%s\n %s\n actions %v\nCLEARS: %s", o.Summary, o.Headline, quiet,
|
||||
said, labels, o.Resolved)
|
||||
}
|
||||
|
||||
// **A delivery waiting**: the popup read "the walk of novox/mesh-catalog a6385479 has waited 56m0s for
|
||||
// mesh-delivery's word to start: `mesh-delivery.show` for the delivery that landed as a6385479 says why;
|
||||
// `plans go plan-1791454185265004861 --why …` starts it by hand". Under four hours it needs nothing — no
|
||||
// popup; past them, the operator starts or stops it from the notification.
|
||||
func TestADeliveryWaitingNeedsNothingUntilItsBoundThenOffersStartAndStop(t *testing.T) {
|
||||
now := time.Date(2026, 10, 8, 11, 36, 0, 0, time.UTC)
|
||||
f := calm(now)
|
||||
f.waits = []waitFacts{{id: "plan-1791454185265004861", repository: "novox/mesh-catalog", commit: "a6385479c0ffee",
|
||||
awaits: "mesh-delivery", since: now.Add(-56 * time.Minute), modules: []string{"openrazer"}}}
|
||||
got := watchWaits(f)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
plainExample(t, got[0], "openrazer delivery waiting to start",
|
||||
"Nothing for you to do. The change to openrazer is merged and built, and has waited 56 minutes for "+
|
||||
"mesh-delivery (the module that decides when a delivery goes out) to let it start. It becomes a question "+
|
||||
"for you if it still waits after 4 hours.")
|
||||
if !strings.Contains(got[0].Summary, "plans go plan-1791454185265004861") {
|
||||
t.Errorf("the summary lost the way on for whoever looks closer: %q", got[0].Summary)
|
||||
}
|
||||
|
||||
// Past four hours it is urgent, and offers the controller's own answers.
|
||||
f.waits[0].since = now.Add(-5 * time.Hour)
|
||||
got = watchWaits(f)
|
||||
plainExample(t, got[0], "openrazer delivery waiting to start",
|
||||
"Needs you: start it, or stop it, from the mesh MCP server; this notification cannot do it. The change to openrazer is merged and built, and mesh-delivery (the "+
|
||||
"module that decides when a delivery goes out) has not let it start for 5 hours, so mesh-delivery may "+
|
||||
"be stuck.")
|
||||
|
||||
// Many modules are counted, not listed in the headline.
|
||||
f.waits[0].modules = []string{"a", "b", "c", "d"}
|
||||
got = watchWaits(f)
|
||||
if got[0].Headline != "mesh-catalog delivery waiting to start" || !strings.Contains(got[0].Explanation, "4 modules of mesh-catalog") {
|
||||
t.Errorf("%+v", got[0])
|
||||
}
|
||||
}
|
||||
|
||||
// **A module unhealthy**: "openrazer on g14 is not healthy: its unit openrazer-daemon.service failed in the
|
||||
// account's own service manager (exit-code)". Restarting is not an acknowledgement, so it is said in words
|
||||
// and offered as no answer (ADR 0258).
|
||||
func TestAModuleUnhealthyAsksForARestartInWords(t *testing.T) {
|
||||
o := moduleUnhealthyObservation("openrazer", "g14", []inventory.ResourceHealth{{Kind: link.KindUnit,
|
||||
Resource: "openrazer-daemon", Target: "openrazer-daemon.service",
|
||||
Reason: "failed in the account's own service manager (exit-code)", Since: time.Now()}})
|
||||
plainExample(t, o, "openrazer not working on g14",
|
||||
"Needs you: restart its service openrazer-daemon on g14 from the mesh MCP server; this notification cannot do it. "+
|
||||
"openrazer on g14 is not healthy: its service openrazer-daemon stopped with an error. It clears as soon "+
|
||||
"as it runs again.")
|
||||
// An account waiting for a new login (ADR 0252) asks for the login, held to the plain rule.
|
||||
o = moduleUnhealthyObservation("openrazer", "g14", []inventory.ResourceHealth{{Kind: "account",
|
||||
Resource: "operator-in-group", Target: "jochen", Reason: "relogin needed: the account is in the group"}})
|
||||
if o.Needs != "log out of g14 completely and log in again, or restart it." || len(o.Actions) != 0 {
|
||||
t.Errorf("relogin: %q %+v", o.Needs, o.Actions)
|
||||
}
|
||||
w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs}
|
||||
if why, ok := conditions.PlainWords(w, "g14"); !ok {
|
||||
t.Errorf("the relogin words are not plain: %s", why)
|
||||
}
|
||||
// And the kind issue 318 raises for it (ADR 0254).
|
||||
rw := plainWordings["relogin-needed"](conditions.Observation{Scope: conditions.ScopeModule, ID: "openrazer.g14",
|
||||
Machine: "g14", Kind: "relogin-needed", Severity: conditions.Warning})
|
||||
if why, ok := conditions.PlainWords(rw, "g14"); !ok || rw.Needs == "" || len(rw.Actions) != 0 {
|
||||
t.Errorf("relogin-needed: %s %+v", why, rw)
|
||||
}
|
||||
if strings.Contains(o.Summary, "session") || !strings.Contains(o.Summary, "waits for a new login of jochen") {
|
||||
t.Errorf("relogin summary: %q", o.Summary)
|
||||
}
|
||||
}
|
||||
|
||||
// **Failed units on a machine**: "shanks's service manager is degraded: 3 failed unit(s) no module places —
|
||||
// mnt-recalbox.mount, storage-media.mount, greenclip.service (the account's own manager). …"
|
||||
func TestFailedUnitsOnAMachineAreNamedWithoutTheirSuffixes(t *testing.T) {
|
||||
o, raise := machineUnitsObservation("shanks", &link.UnitsHealth{State: "degraded", Failed: []link.FailedUnit{
|
||||
{Unit: "mnt-recalbox.mount", Result: "exit-code"}, {Unit: "storage-media.mount", Result: "exit-code"},
|
||||
{Unit: "greenclip.service", Scope: "user", Result: "exit-code"}}})
|
||||
if !raise {
|
||||
t.Fatal("not raised")
|
||||
}
|
||||
plainExample(t, o, "3 failed services on shanks",
|
||||
"Needs you: mend or remove them on shanks, or silence this if they do not matter. On shanks, "+
|
||||
"mnt-recalbox (a mount), storage-media (a mount) and greenclip failed. No module manages them, so the "+
|
||||
"mesh does not repair them.", "Silence for a week")
|
||||
}
|
||||
|
||||
// **A healer wanted**: "\"push\" was repaired by hand 35 times in 14 days, the last by g14.node-tools, through
|
||||
// the mesh-controller seat: a healer is wanted for it". Nothing is broken: no popup.
|
||||
func TestAHealerWantedNeedsNothingFromTheOperator(t *testing.T) {
|
||||
now := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)
|
||||
f := calm(now)
|
||||
for i := 0; i < 35; i++ {
|
||||
a := actByHand(now.Add(-time.Duration(i+1)*time.Hour), "push")
|
||||
a.Verb, a.By = "push", "g14.node-tools, through the mesh-controller seat"
|
||||
f.handActs = append(f.handActs, a)
|
||||
}
|
||||
got := watchHandActs(f)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
plainExample(t, got[0], "Push keeps being fixed by hand",
|
||||
"Nothing for you to do. A person repaired push by hand 35 times in 14 days, so an automatic repair is "+
|
||||
"wanted for it. Nothing is broken now.")
|
||||
}
|
||||
|
||||
// **A delivery held past its bound**, as mesh-delivery says it: "the delivery novox/hq@055550802096 has been
|
||||
// held for 36h2m6s, past its bound of 24h0m0s (it waits for the operator): healer H2 may none: …".
|
||||
func TestADeliveryHeldAsksForReleaseOrStopInWords(t *testing.T) {
|
||||
got := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s",
|
||||
Bound: "24h0m0s", H2: "none: the state is the operator's", Says: "it waits for the operator"}})
|
||||
plainExample(t, got[0], "Delivery of hq held for 36 hours",
|
||||
"Needs you: release it, or stop it, from the mesh MCP server; this notification cannot do it. A delivery of hq has been held for 36 hours, past its limit.",
|
||||
)
|
||||
}
|
||||
|
||||
// **Every kind the controller raises has plain words**, and its words are plain for a subject of every
|
||||
// shape it is raised with.
|
||||
func TestEveryWordingIsPlain(t *testing.T) {
|
||||
subjects := []conditions.Observation{
|
||||
{Scope: conditions.ScopeMachine, ID: "ace", Machine: "ace"},
|
||||
{Scope: conditions.ScopeMachine, ID: "ace.immich.library", Machine: "ace"},
|
||||
{Scope: conditions.ScopeModule, ID: "openrazer.g14", Machine: "g14"},
|
||||
{Scope: conditions.ScopeProvider, ID: "postgres.anchor.app_db", Machine: "anchor"},
|
||||
{Scope: conditions.ScopePlan, ID: "plan-1791454185265004861"},
|
||||
{Scope: conditions.ScopeDelivery, ID: "novox/hq@055550802096"},
|
||||
{Scope: conditions.ScopeCore, ID: "controller.anchor", Machine: "anchor"},
|
||||
{Scope: conditions.ScopeBus, ID: "mesh"},
|
||||
{Scope: conditions.ScopeSeat, ID: "node-backup.ace", Machine: "ace"},
|
||||
{Scope: conditions.ScopeMesh, ID: "hand-acts.disk-load"},
|
||||
{Scope: conditions.ScopeBuild, ID: "openrazer.g14", Machine: "g14"},
|
||||
}
|
||||
for kind, fn := range plainWordings {
|
||||
if !conditions.Worded(kind) {
|
||||
t.Errorf("%s is not registered", kind)
|
||||
}
|
||||
for _, s := range subjects {
|
||||
s.Kind = kind
|
||||
w := fn(s)
|
||||
w.Explanation = conditions.Verdict(w.Needs, w.Explanation)
|
||||
if why, ok := conditions.PlainWords(w, s.Machine); !ok {
|
||||
t.Errorf("%s about %s %s: %s\n %+v", kind, s.Scope, s.ID, why, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **An answer on a notification is no repair** (novox/hq ADR 0258): silencing chosen by the operator on the
|
||||
// desk does not count toward a healer wanted; the same silence by hand for another cause still does.
|
||||
func TestAnOperatorsAnswerIsNoHandRepair(t *testing.T) {
|
||||
now := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)
|
||||
f := calm(now)
|
||||
for i := 0; i < 3; i++ {
|
||||
a := actByHand(now.Add(-time.Duration(i+1)*time.Hour), conditions.CauseOperatorAnswer)
|
||||
a.Verb = "conditions silence"
|
||||
f.handActs = append(f.handActs, a)
|
||||
}
|
||||
if got := watchHandActs(f); len(got) != 0 {
|
||||
t.Fatalf("an answer counted as a repair: %+v", got)
|
||||
}
|
||||
for i := range f.handActs {
|
||||
f.handActs[i].Cause = "machine-units"
|
||||
}
|
||||
if got := watchHandActs(f); len(got) != 1 {
|
||||
t.Fatalf("a silence by hand stopped counting: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **Data loss is never silenced from a notification** (ADR 0258): data missing or shrunk offers no answer,
|
||||
// and says where it is silenced.
|
||||
func TestDataLossOffersNoSilence(t *testing.T) {
|
||||
for _, kind := range []string{kindDataMissing, kindDataShrank} {
|
||||
w := plainWordings[kind](conditions.Observation{Scope: conditions.ScopeMachine, ID: "ace.immich.library",
|
||||
Machine: "ace", Kind: kind, Severity: conditions.Urgent})
|
||||
if len(w.Actions) != 0 || !strings.HasSuffix(w.Needs, FromMeshMCPServer) {
|
||||
t.Errorf("%s: %+v", kind, w)
|
||||
}
|
||||
if why, ok := conditions.PlainWords(w, "ace"); !ok {
|
||||
t.Errorf("%s: %s", kind, why)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **Updates held after a failed release** (2026-10-08): the popup read "Needs you: release them, or leave
|
||||
// them held." — naming neither what waits nor where it is released. It names the modules and machines, and
|
||||
// the mesh MCP server.
|
||||
func TestUpdatesHeldNameWhatWaitsAndWhereItIsReleased(t *testing.T) {
|
||||
saved := backlogNow
|
||||
t.Cleanup(func() { backlogNow = saved })
|
||||
backlogNow.held = "release-1791457717307061152 failed (failed its gate on g14); what waits is released again by a person"
|
||||
backlogNow.waiting = map[string][]inventory.CarriedMove{
|
||||
"shanks": {{Module: "openrazer"}},
|
||||
"g14": {{Module: "openrazer"}, {Module: "sensors"}},
|
||||
}
|
||||
got := backlogObservation()
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("raised %d", len(got))
|
||||
}
|
||||
plainExample(t, got[0], "Updates of openrazer and sensors wait for your release",
|
||||
"Needs you: release them from the mesh MCP server; this notification cannot do it. Updates of openrazer and "+
|
||||
"sensors on g14 and shanks wait for a person to release them, because the last walk failed. "+
|
||||
"They are not delivered until then, and stay held if you leave them.")
|
||||
}
|
||||
|
||||
// **What held them is said in the glossary's words** (2026-10-08 review): the backlog is held after any
|
||||
// walk failed, not a release, and a "check" is a pull request's status. So the words say the walk failed,
|
||||
// and never that a release failed or a check did — with the modules and machines named or not.
|
||||
func TestUpdatesHeldSayTheWalkFailed(t *testing.T) {
|
||||
for _, w := range []words{
|
||||
releaseHeldWords([]string{"openrazer"}, []string{"g14"}),
|
||||
releaseHeldWords(nil, nil),
|
||||
plainWordings["release-held"](conditions.Observation{Scope: conditions.ScopeMesh, ID: "release"}),
|
||||
} {
|
||||
if !strings.Contains(w.Explanation, "because the last walk failed.") {
|
||||
t.Errorf("does not say the walk failed: %q", w.Explanation)
|
||||
}
|
||||
for _, wrong := range []string{"release failed", "check"} {
|
||||
if strings.Contains(w.Explanation, wrong) {
|
||||
t.Errorf("says %q: %q", wrong, w.Explanation)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// mcpVerb is a need that opens with a verb only the mesh MCP server performs (ADR 0258 §1): release, stop,
|
||||
// start, restart, and a restore.
|
||||
var mcpVerb = regexp.MustCompile(`^(release|stop|start|restart|restore)\b`)
|
||||
|
||||
// **A need no notification can answer says where it is answered** (ADR 0258 §1): every wording whose need
|
||||
// opens with a verb the mesh MCP server performs, and offers no action, ends with FromMeshMCPServer — the
|
||||
// kinds worded here for every subject shape, and those worded where they are raised. A new kind that misses
|
||||
// it fails here; release-held did (2026-10-08).
|
||||
func TestANeedNoNotificationAnswersNamesTheMeshMCPServer(t *testing.T) {
|
||||
check := func(what string, w words) {
|
||||
t.Helper()
|
||||
if w.Needs == "" || len(w.Actions) > 0 || !mcpVerb.MatchString(w.Needs) {
|
||||
return
|
||||
}
|
||||
if !strings.HasSuffix(w.Needs, FromMeshMCPServer) {
|
||||
t.Errorf("%s needs %q without %q", what, w.Needs, FromMeshMCPServer)
|
||||
}
|
||||
}
|
||||
subjects := []conditions.Observation{
|
||||
{Scope: conditions.ScopeMachine, ID: "ace", Machine: "ace"},
|
||||
{Scope: conditions.ScopeMachine, ID: "ace.immich.library", Machine: "ace"},
|
||||
{Scope: conditions.ScopeModule, ID: "openrazer.g14", Machine: "g14"},
|
||||
{Scope: conditions.ScopeDelivery, ID: "novox/hq@055550802096"},
|
||||
{Scope: conditions.ScopeCore, ID: "controller.anchor", Machine: "anchor"},
|
||||
{Scope: conditions.ScopeMesh, ID: "release", Also: []string{"g14"}},
|
||||
}
|
||||
for kind, fn := range plainWordings {
|
||||
for _, s := range subjects {
|
||||
for _, sev := range []conditions.Severity{conditions.Warning, conditions.Urgent} {
|
||||
s.Kind, s.Severity, s.Resolver = kind, sev, conditions.ResolverOperator
|
||||
check(kind+" about "+s.ID, fn(s))
|
||||
}
|
||||
}
|
||||
}
|
||||
check("a walk waiting", words{Needs: waitingNeeds(conditions.Urgent)})
|
||||
check("a module's failed service", words{Needs: moduleNeeds("g14",
|
||||
[]inventory.ResourceHealth{{Kind: link.KindUnit, Target: "openrazer-daemon.service"}})})
|
||||
for _, state := range []string{"held", "ready", "failing"} {
|
||||
_, _, _, needs, actions := stalledWords(stalledLine{ID: "novox/hq@055550802096", State: state, For: "36h"},
|
||||
conditions.Observation{Resolver: conditions.ResolverOperator})
|
||||
check("a delivery "+state, words{Needs: needs, Actions: actions})
|
||||
}
|
||||
check("updates held", releaseHeldWords([]string{"openrazer"}, []string{"g14"}))
|
||||
}
|
||||
|
||||
// **A pull request the forge never announced says what to do about it** (novox/hq issue 347): mesh-delivery says one
|
||||
// as a stalled line in state `unannounced` — no delivery exists, so there is nothing to stop, release or close —
|
||||
// and the operator's one act is a new commit on its branch, which the forge announces.
|
||||
func TestAnUnannouncedPullRequestSaysToPushANewCommit(t *testing.T) {
|
||||
l := stalledLine{ID: "novox/hq@bfe82315f42c", Number: 243, State: "unannounced", For: "11m0s", Bound: "10m0s",
|
||||
H2: "none: the forge never announced it, so there is no delivery to close — the operator's",
|
||||
Says: "novox/hq#243 is open on main, which requires the merge check, and its head has had no merge check"}
|
||||
obs := stalledObservations([]stalledLine{l})
|
||||
if len(obs) != 1 || obs[0].Resolver != conditions.ResolverOperator {
|
||||
t.Fatalf("an unannounced pull request is not the operator's: %+v", obs)
|
||||
}
|
||||
o := obs[0]
|
||||
if strings.Contains(o.Needs, "stop") || strings.Contains(o.Needs, "release") || !strings.Contains(o.Needs, "commit") {
|
||||
t.Fatalf("it says to %q", o.Needs)
|
||||
}
|
||||
if !strings.Contains(o.Headline, "no merge check") || !strings.Contains(o.Headline, "#243") ||
|
||||
!strings.Contains(o.Explanation, "never") {
|
||||
t.Fatalf("it reads %q / %q", o.Headline, o.Explanation)
|
||||
}
|
||||
}
|
||||
@@ -128,7 +128,7 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
|
||||
}
|
||||
|
||||
// A recorded module is composed at the build this machine runs, on any send but a person's push
|
||||
// (novox/hq issue 295, ADR 0242).
|
||||
// (novox/hq issue 295, ADR 0245).
|
||||
if _, err := keepRecorded(ctx, open, nodeName, shelf); err != nil {
|
||||
return catalogue.Resolution{}, nil, err
|
||||
}
|
||||
@@ -136,7 +136,8 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
|
||||
resolved, err := catalogue.Resolve(shelf, assigned,
|
||||
catalogue.Node{Name: nodeName, Site: site, Capabilities: capabilities,
|
||||
At: onNetwork[nodeName], PublicDomain: publicDomain,
|
||||
Account: who.Account, AccountHome: who.AccountHome}, world)
|
||||
Account: who.Account, AccountHome: who.AccountHome,
|
||||
AgentAccount: who.AgentAccount, AgentAccountHome: who.AgentAccountHome}, world)
|
||||
if err != nil {
|
||||
// The node's own set does not compose. Marked, because this is the only failure here that
|
||||
// a mesh-wide gatherer may pass over — see notResolvable.
|
||||
@@ -419,7 +420,7 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
||||
out := sendable{Resources: composed.Resources, Adoption: adoption,
|
||||
Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
|
||||
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld,
|
||||
unbound: with.Unbound, foreseen: composed.Foreseen}
|
||||
unbound: with.Unbound, foreseen: composed.Foreseen, unplaced: composed.Unplaced}
|
||||
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
|
||||
// 0221). Read only on the send path: a question about what would be sent records nothing.
|
||||
if choosing == Allocating {
|
||||
@@ -439,7 +440,7 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
||||
names = append(names, m.Module)
|
||||
}
|
||||
out.Builds = carriedBuilds(names, composed.LeftOut, current, before)
|
||||
// A recorded module kept at the build the machine runs is recorded as carrying that one (ADR 0242).
|
||||
// A recorded module kept at the build the machine runs is recorded as carrying that one (ADR 0245).
|
||||
kept, err := recordedKept(ctx, open, node)
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
@@ -511,13 +512,21 @@ func sortedKeysOf(m map[string]string) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// sayPlanDiff is `plan --diff`: what the declaration leaves out, then the diff. A module left out is not in
|
||||
// the body, so the diff alone would say "nothing would change" for a module just assigned whose settings
|
||||
// cannot compose — success-shaped silence. Said first, with why, as push and the plain plan say it
|
||||
// (novox/hq ADR 0163, rule 6).
|
||||
func sayPlanDiff(node string, declared sendable, diff func() error) error {
|
||||
reportLeftOut(node, declared)
|
||||
return diff()
|
||||
}
|
||||
|
||||
// reportLeftOut says which of a machine's modules its declaration leaves out and why (novox/hq ADR
|
||||
// 0163, rule 6), one line each: the machine is told everything else, and is told it was left out.
|
||||
func reportLeftOut(node string, declared sendable) {
|
||||
for _, m := range declared.LeftOut {
|
||||
fmt.Printf("%s: %s left out — a setting stored for it cannot compose with its definition; "+
|
||||
"what the machine holds for it is kept and its containers are untouched. %s\n",
|
||||
node, m, declared.leftOutWhy[m])
|
||||
fmt.Printf("%s: %s left out — what the machine holds for it is kept and its containers are "+
|
||||
"untouched. %s\n", node, m, declared.leftOutWhy[m])
|
||||
}
|
||||
// And whom it serves nothing, because their identity overflows what the provision keeps (ADR
|
||||
// 0225): the machine is sent everything else, and the consumer is named.
|
||||
@@ -528,6 +537,11 @@ func reportLeftOut(node string, declared sendable) {
|
||||
for _, u := range declared.unbound {
|
||||
fmt.Printf("%s: %s\n", node, u)
|
||||
}
|
||||
// And every contribution its holder could not render, which the machine is sent without (novox/hq
|
||||
// ADR 0255).
|
||||
for _, u := range declared.unplaced {
|
||||
fmt.Printf("%s: %s\n", node, u)
|
||||
}
|
||||
}
|
||||
|
||||
// busCredentialIssued refuses an own secret called `broker` whose bus account nobody issued.
|
||||
@@ -881,8 +895,13 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
judgesRoot, err := engineJudgesRoot(ctx, inv, node)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
return catalogue.Rendering{
|
||||
ReadsHealth: readsHealth,
|
||||
JudgesRoot: judgesRoot,
|
||||
BusMembership: memberships[node],
|
||||
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Foreseen: foreseen, Ports: ports,
|
||||
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
||||
@@ -905,6 +924,16 @@ func engineReadsHealth(ctx context.Context, inv *inventory.Inventory, node strin
|
||||
return had && stated.Contract >= link.ReadinessContract, nil
|
||||
}
|
||||
|
||||
// engineJudgesRoot says whether a machine's node-engine judges a user's declared `root` (novox/hq ADR 0266),
|
||||
// by its own newest statement, for the same reason as engineReadsHealth: an older engine parses strictly.
|
||||
func engineJudgesRoot(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) {
|
||||
stated, had, err := inv.HealthOf(ctx, node)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return had && stated.Contract >= link.RootContract, nil
|
||||
}
|
||||
|
||||
// zonesInTheMesh is every zone a module in the mesh declares, where the mesh placed it (novox/hq ADR
|
||||
// 0199): the zone settled from that node's settings, the node's private address, the port the
|
||||
// answering listen is published on there.
|
||||
@@ -1185,12 +1214,15 @@ func planCommand(ctx context.Context, args []string) error {
|
||||
// checking it against the host's own parser, most usefully, which is the only way to know
|
||||
// that what the control plane emits is what the host accepts.
|
||||
asJSON := set.Bool("json", false, "print the declaration this node would be sent")
|
||||
// What a push would change, against what the machine was last sent (novox/hq ADR 0217): read
|
||||
// before sending, so a change that removes, moves or recreates something is seen first.
|
||||
asDiff := set.Bool("diff", false, "what a push would change on this node, against what it was last sent")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("plan <node> [--files] [--json]")
|
||||
return errors.New("plan <node> [--files] [--json] [--diff]")
|
||||
}
|
||||
args = positionals
|
||||
open, err := openStores(ctx)
|
||||
@@ -1207,6 +1239,19 @@ func planCommand(ctx context.Context, args []string) error {
|
||||
fmt.Printf("%s is assigned nothing\n", args[0])
|
||||
return nil
|
||||
}
|
||||
if *asDiff {
|
||||
declared, err := declarationFor(ctx, open, args[0], plan, settings)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return sayPlanDiff(args[0], declared, func() error {
|
||||
return writePlanDiff(ctx, open.inventory, args[0], body)
|
||||
})
|
||||
}
|
||||
if *asJSON {
|
||||
declared, err := declarationFor(ctx, open, args[0], plan, settings)
|
||||
if err != nil {
|
||||
@@ -1406,7 +1451,7 @@ func servedOnNode(ctx context.Context, inv *inventory.Inventory, node string,
|
||||
}
|
||||
serves := catalogue.ServedOn(m, provision, ports)
|
||||
if len(serves) > 0 {
|
||||
serves, err = catalogue.Settle(serves, layers)
|
||||
serves, err = catalogue.Settle(serves, catalogue.WithDefaults(m, layers))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// novox/hq issue 296: a plan saved a few seconds ago that has stood in its tier for forty minutes says
|
||||
// forty minutes, and LATE — the line counted from the last save, which every advance made.
|
||||
func TestAPlansLineCountsFromItsTierNotFromItsLastSave(t *testing.T) {
|
||||
now := time.Date(2026, 10, 7, 20, 30, 0, 0, time.UTC)
|
||||
asked := now.Add(-40 * time.Minute)
|
||||
p := inventory.Plan{ID: "plan-t", Repository: "novox/mesh-catalog", Commit: "3ad22356", State: inventory.PlanBuilding,
|
||||
Created: now.Add(-40 * time.Minute), Updated: now.Add(-3 * time.Second), TierEntered: now.Add(-40 * time.Minute),
|
||||
Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked}}}
|
||||
|
||||
line := planLine(p, now)
|
||||
if !strings.Contains(line, "building for 40m0s") || !strings.Contains(line, "LATE") {
|
||||
t.Fatalf("a plan in its tier for 40m, saved 3s ago, reads %q", line)
|
||||
}
|
||||
st := planStatuses([]inventory.Plan{p}, now, pauseView{}, nil)[0]
|
||||
if !st.Late || !st.Since.Equal(p.TierEntered) {
|
||||
t.Errorf("status --json says %+v", st)
|
||||
}
|
||||
if _, late := openPlans([]inventory.Plan{p}, now, pauseView{}, nil); late != 1 {
|
||||
t.Errorf("status counted %d late", late)
|
||||
}
|
||||
|
||||
// LATE is the stalled condition said on the line: the same start and the same bound, measured or not.
|
||||
bounds := boundsOf([]inventory.Duration{{Subject: p.Repository, Took: 20 * time.Minute}})
|
||||
if got := bounds.of(p.Repository); got != time.Hour {
|
||||
t.Fatalf("a repository whose tiers took 20m is given %s", got)
|
||||
}
|
||||
if got := bounds.of("novox/other"); got != tierAtLeast {
|
||||
t.Fatalf("a repository with nothing measured is given %s", got)
|
||||
}
|
||||
for _, in := range []time.Duration{40 * time.Minute, 70 * time.Minute} {
|
||||
q := p
|
||||
q.TierEntered = now.Add(-in)
|
||||
bound := bounds.of(q.Repository)
|
||||
late := strings.Contains(planLineWith(q, now, pauseView{}, bound), "LATE")
|
||||
stalled := len(watchPlans(&signalFacts{now: now, plans: []planFacts{{id: q.ID, repository: q.Repository,
|
||||
commit: q.Commit, tiers: 1, entered: inTierSince(q), bound: bound}}})) == 1
|
||||
if late != stalled || late != (in > bound) {
|
||||
t.Errorf("in its tier %s with a bound of %s: LATE %v, stalled %v", in, bound, late, stalled)
|
||||
}
|
||||
}
|
||||
|
||||
// A plan read without its tier's stamp counts from its last save, the only time it has.
|
||||
old := p
|
||||
old.TierEntered = time.Time{}
|
||||
if line := planLine(old, now); !strings.Contains(line, "for 3s") || strings.Contains(line, "LATE") {
|
||||
t.Errorf("a plan without its tier's stamp reads %q", line)
|
||||
}
|
||||
|
||||
// A walk that waited for its delivery's word counts its wait from when it was opened, and its tier
|
||||
// from the word.
|
||||
waiting := p
|
||||
waiting.Delivery = &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}
|
||||
waiting.Modules = map[string]*inventory.PlanModule{}
|
||||
if line := planLine(waiting, now); !strings.Contains(line, "for 40m0s") || strings.Contains(line, "LATE") {
|
||||
t.Errorf("a walk waiting 40m reads %q", line)
|
||||
}
|
||||
if st := planStatuses([]inventory.Plan{waiting}, now, pauseView{}, nil)[0]; st.Late {
|
||||
t.Errorf("a waiting walk is called late: %+v", st)
|
||||
}
|
||||
let := now.Add(-5 * time.Minute)
|
||||
gone := p
|
||||
gone.Delivery = &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat, Go: &let}
|
||||
if line := planLine(gone, now); !strings.Contains(line, "building for 5m0s") || strings.Contains(line, "LATE") {
|
||||
t.Errorf("a walk let go 5m ago, opened 40m ago, reads %q", line)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq issue 296: a plan whose tier stands still is not written again by every advance — its revision,
|
||||
// its last save and its `plan-moved` stay where its last change left them.
|
||||
func TestAPlanStandingStillIsNotSavedAgain(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asked := asksRecorded(t)
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1aaaaaaaa",
|
||||
Paths: []string{"modules/app/index.ts"}, ModuleDirs: []string{"modules/app"}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
recent, err := inv.RecentPlans(ctx, 1)
|
||||
if err != nil || len(recent) != 1 || len(*asked) != 1 {
|
||||
t.Fatalf("no plan building: %v %v, asked %v", recent, err, *asked)
|
||||
}
|
||||
before := recent[0]
|
||||
if before.State != inventory.PlanBuilding || before.Modules["app"] == nil || before.Modules["app"].State != "asked" {
|
||||
t.Fatalf("the plan is not waiting on its build: %+v", before)
|
||||
}
|
||||
|
||||
for range 3 {
|
||||
advanceHeld(ctx, open)
|
||||
}
|
||||
after, err := inv.PlanByID(ctx, before.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if after.Revision != before.Revision || !after.Updated.Equal(before.Updated) {
|
||||
t.Fatalf("a plan that did not move was saved again: revision %d → %d, saved %s → %s",
|
||||
before.Revision, after.Revision, before.Updated, after.Updated)
|
||||
}
|
||||
if len(*asked) != 1 {
|
||||
t.Fatalf("advancing asked again: %v", *asked)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// 2026-10-09: nfs-server was assigned to the home server, its file asked for a setting nothing set, and
|
||||
// `plan --diff` said "nothing would change". The diff now says what is left out, and why, before the diff.
|
||||
func TestPlanDiffSaysAModuleLeftOutBeforeTheDiff(t *testing.T) {
|
||||
declared := sendable{LeftOut: []string{"nfs-server"},
|
||||
leftOutWhy: map[string]string{"nfs-server": `nothing sets "shares" for it`}}
|
||||
said := printed(t, func() error {
|
||||
return sayPlanDiff("home", declared, func() error {
|
||||
writeDiff(os.Stdout, "home", sentDiff{}, nil)
|
||||
return nil
|
||||
})
|
||||
})
|
||||
left := strings.Index(said, "home: nfs-server left out")
|
||||
nothing := strings.Index(said, "home: nothing would change")
|
||||
if left < 0 || !strings.Contains(said, `nothing sets "shares" for it`) || nothing < left {
|
||||
t.Errorf("the left-out module and why, then the diff:\n%s", said)
|
||||
}
|
||||
}
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
@@ -98,11 +97,9 @@ func buildSeatPause(ctx context.Context, inv *inventory.Inventory, plans []inven
|
||||
if len(holders) == 0 {
|
||||
return pauseView{}
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return pauseView{}
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
// On the serving controller's own connection when this is it: a watchdog tick while a walk waits for a
|
||||
// build dialled one every 30 seconds (novox/hq issue 327).
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not reach the bus to read whether the build seat is paused: %v\n", err)
|
||||
return pauseView{}
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// **The planner over edges the store derives**, not edges written by hand: modules registered, builds
|
||||
// recorded with what they stood on and which repositories they read, the relation answered by
|
||||
// inventory.Dependencies (dependenciesOf over the records), and the merge planned by reachOfMerge — the
|
||||
// path a real merge takes, short of the bus.
|
||||
//
|
||||
// **The repository rows are CURRENT BEHAVIOUR, documented — not the rule the operator states**
|
||||
// (novox/hq issue 338, and the decision pending on it): a build that read a repository gives its module a
|
||||
// packages edge to every module built from that repository, and mergeCandidates moves it on any merge to
|
||||
// that repository, whatever the files. So a change to C alone, or to a README, moves the module that
|
||||
// packages C's repository. ADR 0238 §3 records exactly that today ("a repository a recipe names"); the
|
||||
// expectations marked 338 change with that decision.
|
||||
func TestASharedRepositoryIsPlannedFromTheRecordsAsItIsToday(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
ctx := t.Context()
|
||||
asked := time.Now().Add(-time.Hour)
|
||||
register := func(m catalogue.Manifest, repository, path string, against []string, read []inventory.ReadRepository) {
|
||||
t.Helper()
|
||||
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: repository, Seat: "git", Path: path,
|
||||
Ref: "main", BuiltFrom: "old", Asked: asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + m.Module, Repository: repository, Ref: "main",
|
||||
Module: m.Module, Commit: "old", On: "builder", Path: path, Against: against, Read: read, Asked: asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
controllerRead := []inventory.ReadRepository{{Repository: "novox/mesh-controller", Ref: "main"}}
|
||||
agent := catalogue.Manifest{Module: "build-agent", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}}
|
||||
|
||||
// The shape of issue 338.
|
||||
register(catalogue.Manifest{Module: "mesh-controller", Version: "1"}, "novox/mesh-controller", "", nil, nil)
|
||||
register(agent, "novox/mesh-catalog", "modules/build-agent", nil, controllerRead)
|
||||
register(catalogue.Manifest{Module: "route-proxy", Version: "1"}, "novox/mesh-catalog", "modules/route-proxy", nil, controllerRead)
|
||||
register(catalogue.Manifest{Module: "gitea", Version: "1"}, "novox/mesh-catalog", "modules/gitea", nil, nil)
|
||||
// A, B and C in one repository; D built against A's artifact, E declaring B, P packaging the repository.
|
||||
for _, n := range []string{"a", "b", "c"} {
|
||||
register(catalogue.Manifest{Module: n, Version: "1"}, "novox/one", "modules/"+n, nil, nil)
|
||||
}
|
||||
register(catalogue.Manifest{Module: "d", Version: "1"}, "novox/two", "d",
|
||||
[]string{catalogue.ArtifactStoreScheme + "a/runtime@sha256:" + strings.Repeat("0", 64)}, nil)
|
||||
register(catalogue.Manifest{Module: "e", Version: "1", Build: &catalogue.Build{
|
||||
On: []catalogue.BuildsOn{{Arg: "BASE", Module: "b", Artifact: "runtime"}}}}, "novox/two", "e", nil, nil)
|
||||
register(catalogue.Manifest{Module: "p", Version: "1"}, "novox/two", "p", nil,
|
||||
[]inventory.ReadRepository{{Repository: "novox/one", Ref: "main"}})
|
||||
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
read, err := inv.ReadRepositories(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
edges, err := inv.Dependencies(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The edges the hand-written rows of TestASharedRepositoryMovesWhatPackagesItAsItDoesToday use are
|
||||
// the ones derived here.
|
||||
var shared []inventory.Edge
|
||||
in338 := map[string]bool{"mesh-controller": true, "build-agent": true, "route-proxy": true, "gitea": true}
|
||||
for _, e := range edges {
|
||||
if in338[e.From] && in338[e.To] {
|
||||
shared = append(shared, e)
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(shared, sharedRepositoryEdges) {
|
||||
t.Errorf("derived %v\nthe hand-written rows use %v", shared, sharedRepositoryEdges)
|
||||
}
|
||||
// Each kind derived from its record: built against (stands-on), build.on (declared), read (packages).
|
||||
for _, want := range []inventory.Edge{
|
||||
dep("d", inventory.EdgeStandsOn, "a"),
|
||||
dep("e", inventory.EdgeDeclared, "b"),
|
||||
dep("p", inventory.EdgePackages, "a"),
|
||||
dep("p", inventory.EdgePackages, "b"),
|
||||
dep("p", inventory.EdgePackages, "c"),
|
||||
dep("d", inventory.EdgeBuiltBy, "build-agent"),
|
||||
} {
|
||||
found := false
|
||||
for _, e := range edges {
|
||||
found = found || e == want
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("no %s %s %s derived: %v", want.From, want.Kind, want.To, edges)
|
||||
}
|
||||
}
|
||||
|
||||
for _, c := range []struct {
|
||||
what, repo string
|
||||
paths []string
|
||||
want string
|
||||
issue338 bool
|
||||
}{
|
||||
{"A and B changed, C untouched: D after A, E after B; P packages their repository", "one",
|
||||
[]string{"modules/a/x.go", "modules/b/x.go"}, "a,b,p | d,e", false},
|
||||
{"C alone: C, and P, which packages C's repository", "one",
|
||||
[]string{"modules/c/x.go"}, "c,p", true},
|
||||
{"a README of the repository P packages: P moves, nothing built from it does", "one",
|
||||
[]string{"README.md"}, "p", true},
|
||||
{"the dependent's repository: D alone", "two", []string{"d/main.go"}, "d", false},
|
||||
{"a README of the controller's repository: all three, three tiers", "mesh-controller",
|
||||
[]string{"README.md"}, "mesh-controller | build-agent | route-proxy", true},
|
||||
{"the route proxy's directory in the catalogue: it alone", "mesh-catalog",
|
||||
[]string{"modules/route-proxy/module.json"}, "route-proxy", false},
|
||||
{"the build agent's directory: it alone, nothing it builds", "mesh-catalog",
|
||||
[]string{"modules/build-agent/module.json"}, "build-agent", false},
|
||||
} {
|
||||
_, got := planMerge(t, c.repo, c.paths, entries, read, edges)
|
||||
if got != c.want {
|
||||
tag := ""
|
||||
if c.issue338 {
|
||||
tag = " (current behaviour, issue 338)"
|
||||
}
|
||||
t.Errorf("%s: planned %q, wanted %q%s", c.what, got, c.want, tag)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,447 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The delivery planner's rules, as recorded (novox/hq ADR 0162 §1, ADR 0238 §3), held by one table and
|
||||
// one property over reachOfMerge — the planner's one answer to "what does this merge move, and in which
|
||||
// order". A row that fails here on main is a planner that breaks a recorded rule: the row stays, the
|
||||
// expectation is not bent to the code.
|
||||
//
|
||||
// The kinds of edge, as the code reads them (release_plan.go):
|
||||
//
|
||||
// kind widens the plan orders the tiers
|
||||
// stands-on yes yes, after its base is built
|
||||
// declared yes yes, after its base is built
|
||||
// packages yes no, the same tier (a code dependency)
|
||||
// built-by no yes, after the build machine — except for what the build machine stands
|
||||
// on, and for the controller whose worker it binds
|
||||
// worker-of no yes, the build seat's holder after the controller (hq issue 206)
|
||||
|
||||
const (
|
||||
repoOne = "http://forge.internal:20000/novox/one.git"
|
||||
repoTwo = "http://forge.internal:20000/novox/two.git"
|
||||
)
|
||||
|
||||
// dep is one edge of the catalogue's relation: from depends on to, in the way kind says.
|
||||
func dep(from, kind, to string) inventory.Edge {
|
||||
return inventory.Edge{From: from, To: to, Kind: kind}
|
||||
}
|
||||
|
||||
// tiered is a plan's tiers as one line: a tier's modules by comma, tiers by " | ". Empty for no plan.
|
||||
func tiered(tiers [][]string) string {
|
||||
var out []string
|
||||
for _, t := range tiers {
|
||||
out = append(out, strings.Join(t, ","))
|
||||
}
|
||||
return strings.Join(out, " | ")
|
||||
}
|
||||
|
||||
// planMerge is what reachOfMerge plans for a merge of these files into a repository's main: its tiers as
|
||||
// one line, and the files no build reads. It also holds the plan to its own shape: every module it builds
|
||||
// is in exactly one tier.
|
||||
func planMerge(t *testing.T, repo string, paths []string, entries []inventory.Entry,
|
||||
read map[string][]inventory.ReadRepository, edges []inventory.Edge) (mergeReach, string) {
|
||||
t.Helper()
|
||||
m := link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Commit: "head", Paths: paths}
|
||||
r := reachOfMerge(m, entries, read, edges)
|
||||
seen := map[string]int{}
|
||||
for _, tier := range r.Plan.Tiers {
|
||||
for _, name := range tier {
|
||||
seen[name]++
|
||||
}
|
||||
}
|
||||
for name := range r.Plan.Modules {
|
||||
if seen[name] != 1 {
|
||||
t.Errorf("%s/%v: %s is in %d tiers of %v", repo, paths, name, seen[name], r.Plan.Tiers)
|
||||
}
|
||||
}
|
||||
if len(seen) != len(r.Plan.Modules) {
|
||||
t.Errorf("%s/%v: the tiers %v hold modules the plan does not (%d)", repo, paths, r.Plan.Tiers, len(r.Plan.Modules))
|
||||
}
|
||||
return r, tiered(r.Plan.Tiers)
|
||||
}
|
||||
|
||||
// **A merge moves the modules whose directory it changed, and everything built on them; nothing else.**
|
||||
// Each row is a catalogue (modules in directories of one or two repositories), its dependencies with
|
||||
// their kinds, the files one commit changed, and the exact plan: the moved set and its tier order.
|
||||
func TestAPlanIsWhatTheChangeTouchedAndWhatIsBuiltOnIt(t *testing.T) {
|
||||
in := func(repo, dir string, names ...string) []inventory.Entry {
|
||||
var out []inventory.Entry
|
||||
for _, n := range names {
|
||||
d := dir + "/" + n
|
||||
if dir == "" {
|
||||
d = n
|
||||
}
|
||||
out = append(out, fromRepo(n, repo, d))
|
||||
}
|
||||
return out
|
||||
}
|
||||
// Modules a to f, x and z in novox/one under modules/; g in novox/two at g/.
|
||||
entries := append(in(repoOne, "modules", "a", "b", "c", "d", "e", "f", "x", "z"), in(repoTwo, "", "g")...)
|
||||
const (
|
||||
standsOn = inventory.EdgeStandsOn
|
||||
declared = inventory.EdgeDeclared
|
||||
packages = inventory.EdgePackages
|
||||
builtBy = inventory.EdgeBuiltBy
|
||||
workerOf = inventory.EdgeWorkerOf
|
||||
)
|
||||
// The two real cycles the kinds resolve themselves (ADR 0162 §1, hq issue 206).
|
||||
runtime := append(in(repoOne, "modules", "runtime", "builder"), fromRepo("controller", repoTwo, ""))
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
entries []inventory.Entry
|
||||
edges []inventory.Edge
|
||||
repo string
|
||||
paths []string
|
||||
want string // the tiers, " | " between them
|
||||
unread string
|
||||
cycle bool
|
||||
}{
|
||||
// The operator's case: two modules changed, a third beside them in the same repository untouched,
|
||||
// each changed one with a dependent.
|
||||
{what: "A and B changed, C untouched beside them, D on A and E on B",
|
||||
edges: []inventory.Edge{dep("d", standsOn, "a"), dep("e", declared, "b")},
|
||||
repo: "one", paths: []string{"modules/a/main.go", "modules/b/module.json"}, want: "a,b | d,e"},
|
||||
{what: "only A's directory: A and what stands on it",
|
||||
edges: []inventory.Edge{dep("d", standsOn, "a"), dep("e", declared, "b")},
|
||||
repo: "one", paths: []string{"modules/a/main.go"}, want: "a | d"},
|
||||
{what: "only C's directory: C alone, nothing is built on it",
|
||||
edges: []inventory.Edge{dep("d", standsOn, "a"), dep("e", declared, "b")},
|
||||
repo: "one", paths: []string{"modules/c/Dockerfile"}, want: "c"},
|
||||
{what: "only the dependent changed: its base does not move",
|
||||
edges: []inventory.Edge{dep("d", standsOn, "a")},
|
||||
repo: "one", paths: []string{"modules/d/main.go"}, want: "d"},
|
||||
{what: "transitive: F on D on A, A changed",
|
||||
edges: []inventory.Edge{dep("f", standsOn, "d"), dep("d", standsOn, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x"}, want: "a | d | f"},
|
||||
{what: "transitive across kinds: F declared on D, D packages A",
|
||||
edges: []inventory.Edge{dep("f", declared, "d"), dep("d", packages, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x"}, want: "a,d | f"},
|
||||
|
||||
// Each kind alone: X depends on A, A changed (widening), then both changed (ordering).
|
||||
{what: "stands-on (built against A's artifact) widens", edges: []inventory.Edge{dep("x", standsOn, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x"}, want: "a | x"},
|
||||
{what: "stands-on orders", edges: []inventory.Edge{dep("x", standsOn, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a | x"},
|
||||
{what: "declared (build.on) widens", edges: []inventory.Edge{dep("x", declared, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x"}, want: "a | x"},
|
||||
{what: "declared orders", edges: []inventory.Edge{dep("x", declared, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a | x"},
|
||||
{what: "packages widens, into the same tier", edges: []inventory.Edge{dep("x", packages, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x"}, want: "a,x"},
|
||||
{what: "packages does not order", edges: []inventory.Edge{dep("x", packages, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a,x"},
|
||||
{what: "built-by never widens", edges: []inventory.Edge{dep("x", builtBy, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x"}, want: "a"},
|
||||
{what: "built-by orders", edges: []inventory.Edge{dep("x", builtBy, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a | x"},
|
||||
{what: "worker-of never widens", edges: []inventory.Edge{dep("x", workerOf, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x"}, want: "a"},
|
||||
{what: "worker-of orders", edges: []inventory.Edge{dep("x", workerOf, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a | x"},
|
||||
{what: "a change to the base alone does not move what it builds", edges: []inventory.Edge{dep("x", builtBy, "a"),
|
||||
dep("d", builtBy, "a"), dep("e", standsOn, "a")},
|
||||
repo: "one", paths: []string{"modules/a/module.json"}, want: "a | e"},
|
||||
|
||||
// The cycles the kinds resolve: the build machine stands on the runtime image the runtime image is
|
||||
// built by; the build seat's holder follows the controller that is built by it.
|
||||
{what: "the build machine's base comes first, built by the build machine that runs", entries: runtime,
|
||||
edges: []inventory.Edge{dep("runtime", builtBy, "builder"), dep("builder", standsOn, "runtime")},
|
||||
repo: "one", paths: []string{"modules/runtime/Dockerfile", "modules/builder/main.go"}, want: "runtime | builder"},
|
||||
{what: "the runtime image alone takes the build machine on it along", entries: runtime,
|
||||
edges: []inventory.Edge{dep("runtime", builtBy, "builder"), dep("builder", standsOn, "runtime")},
|
||||
repo: "one", paths: []string{"modules/runtime/Dockerfile"}, want: "runtime | builder"},
|
||||
{what: "the build machine alone moves alone", entries: runtime,
|
||||
edges: []inventory.Edge{dep("runtime", builtBy, "builder"), dep("builder", standsOn, "runtime")},
|
||||
repo: "one", paths: []string{"modules/builder/main.go"}, want: "builder"},
|
||||
{what: "the build seat's holder follows the controller it binds the worker of", entries: runtime,
|
||||
edges: []inventory.Edge{dep("controller", builtBy, "builder"), dep("builder", workerOf, "controller")},
|
||||
repo: "two", paths: []string{"cmd/main.go"}, want: "controller"},
|
||||
|
||||
// Two repositories.
|
||||
{what: "a dependent in another repository follows its base",
|
||||
edges: []inventory.Edge{dep("g", standsOn, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x"}, want: "a | g"},
|
||||
{what: "a directory of the same name in another repository is not this one's",
|
||||
edges: []inventory.Edge{dep("g", standsOn, "a")},
|
||||
repo: "two", paths: []string{"modules/a/x"}, want: "", unread: "modules/a/x"},
|
||||
{what: "the dependent's own repository moves the dependent alone",
|
||||
edges: []inventory.Edge{dep("g", standsOn, "a")},
|
||||
repo: "two", paths: []string{"g/main.go"}, want: "g"},
|
||||
|
||||
// A diamond.
|
||||
{what: "a diamond, one side changed", edges: []inventory.Edge{dep("d", standsOn, "a"), dep("d", standsOn, "b")},
|
||||
repo: "one", paths: []string{"modules/a/x"}, want: "a | d"},
|
||||
{what: "a diamond, both sides changed", edges: []inventory.Edge{dep("d", standsOn, "a"), dep("d", standsOn, "b")},
|
||||
repo: "one", paths: []string{"modules/a/x", "modules/b/x"}, want: "a,b | d"},
|
||||
{what: "a diamond on one base", edges: []inventory.Edge{dep("d", standsOn, "a"), dep("d", declared, "b"),
|
||||
dep("a", standsOn, "z"), dep("b", standsOn, "z")},
|
||||
repo: "one", paths: []string{"modules/z/x"}, want: "z | a,b | d"},
|
||||
{what: "a diamond of mixed kinds orders on the ordering side only",
|
||||
edges: []inventory.Edge{dep("d", standsOn, "a"), dep("d", packages, "b")},
|
||||
repo: "one", paths: []string{"modules/b/x"}, want: "b,d"},
|
||||
|
||||
// A cycle the catalogue should never produce: what remains is one last tier, and said.
|
||||
{what: "a cycle is one last tier, not lost", edges: []inventory.Edge{dep("a", standsOn, "b"), dep("b", standsOn, "a"),
|
||||
dep("f", standsOn, "c")},
|
||||
repo: "one", paths: []string{"modules/a/x", "modules/c/x"}, want: "c | f | a,b", cycle: true},
|
||||
|
||||
// Files no build reads.
|
||||
{what: "a README at the root of a repository whose modules all live below it",
|
||||
edges: []inventory.Edge{dep("d", standsOn, "a")},
|
||||
repo: "one", paths: []string{"README.md"}, want: "", unread: "README.md"},
|
||||
{what: "a directory no module lives in", edges: []inventory.Edge{dep("d", standsOn, "a")},
|
||||
repo: "one", paths: []string{"modules/lib/x.go", "modules/README.md"}, want: "",
|
||||
unread: "modules/lib/x.go,modules/README.md"},
|
||||
{what: "a module's directory beside a root file", edges: []inventory.Edge{dep("d", standsOn, "a")},
|
||||
repo: "one", paths: []string{"merge-check.sh", "modules/a/x"}, want: "a | d", unread: "merge-check.sh"},
|
||||
{what: "a directory whose name begins with a module's", edges: []inventory.Edge{dep("d", standsOn, "a")},
|
||||
repo: "one", paths: []string{"modules/ab/x"}, want: "", unread: "modules/ab/x"},
|
||||
} {
|
||||
e := entries
|
||||
if c.entries != nil {
|
||||
e = c.entries
|
||||
}
|
||||
r, got := planMerge(t, c.repo, c.paths, e, nil, c.edges)
|
||||
if got != c.want {
|
||||
t.Errorf("%s: planned %q, wanted %q", c.what, got, c.want)
|
||||
}
|
||||
if u := strings.Join(r.Unread, ","); u != c.unread {
|
||||
t.Errorf("%s: unread %q, wanted %q", c.what, u, c.unread)
|
||||
}
|
||||
// A packages edge in the last tier is no cycle; hasCycle said one on main at 8170fc5.
|
||||
if hasCycle(r.Plan.Tiers, c.edges) != c.cycle {
|
||||
t.Errorf("%s: a cycle said %v, wanted %v (%v)", c.what, !c.cycle, c.cycle, r.Plan.Tiers)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **CURRENT BEHAVIOUR, documented — not the rule the operator states.** novox/hq issue 338 (a module
|
||||
// built from a shared repository moves on every merge to it) and the decision pending on it would change
|
||||
// every row here. Today:
|
||||
//
|
||||
// - mesh-controller is built from its repository's root, so every file of that repository touches it;
|
||||
// - route-proxy and build-agent package the whole of that repository (a build context), so the build
|
||||
// record's `read` makes them move on any merge to it, whatever the files, and dependenciesOf gives
|
||||
// each a packages edge to every module built from it;
|
||||
// - built-by (route-proxy on build-agent) and worker-of (build-agent on the controller) make it three
|
||||
// tiers.
|
||||
//
|
||||
// These follow ADR 0238 §3 as written ("the whole repository for a module built from its root, and a
|
||||
// repository a recipe names"), so they are not failures; when the decision on issue 338 lands, these
|
||||
// expectations change with it. The edges are the ones dependenciesOf derives from this catalogue — held
|
||||
// to that by TestASharedRepositoryIsPlannedFromTheRecordsAsItIsToday, which derives them from the store.
|
||||
func TestASharedRepositoryMovesWhatPackagesItAsItDoesToday(t *testing.T) {
|
||||
const catalogueRepo = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||
const controllerRepo = "http://forge.internal:20000/novox/mesh-controller.git"
|
||||
entries := []inventory.Entry{
|
||||
fromRepo("mesh-controller", controllerRepo, ""),
|
||||
fromRepo("build-agent", catalogueRepo, "modules/build-agent"),
|
||||
fromRepo("route-proxy", catalogueRepo, "modules/route-proxy"),
|
||||
fromRepo("gitea", catalogueRepo, "modules/gitea"),
|
||||
}
|
||||
read := map[string][]inventory.ReadRepository{
|
||||
"build-agent": {{Repository: "novox/mesh-controller", Ref: "main"}},
|
||||
"route-proxy": {{Repository: "novox/mesh-controller", Ref: "main"}},
|
||||
}
|
||||
edges := sharedRepositoryEdges
|
||||
for _, c := range []struct {
|
||||
what, repo string
|
||||
paths []string
|
||||
want string
|
||||
}{
|
||||
// The live three-tier plan of 2026-10-08 (issue 338), in the worker-of order (issue 206) that
|
||||
// TestAMergeIsPlannedInTiersAlongTheThreeKindsOfDependency's controller case holds too.
|
||||
{"a README of the controller's repository moves all three, in three tiers", "mesh-controller",
|
||||
[]string{"README.md"}, "mesh-controller | build-agent | route-proxy"},
|
||||
{"the controller's own code: the same", "mesh-controller",
|
||||
[]string{"cmd/mesh-controller/main.go"}, "mesh-controller | build-agent | route-proxy"},
|
||||
{"the route proxy's program alone: the same, the controller with it", "mesh-controller",
|
||||
[]string{"examples/route-proxy/main.go"}, "mesh-controller | build-agent | route-proxy"},
|
||||
// In the catalogue, where they live, the rule is path-precise.
|
||||
{"the route proxy's directory in the catalogue: it alone", "mesh-catalog",
|
||||
[]string{"modules/route-proxy/module.json"}, "route-proxy"},
|
||||
{"the build agent's directory: it alone, nothing it builds", "mesh-catalog",
|
||||
[]string{"modules/build-agent/module.json"}, "build-agent"},
|
||||
{"another module of the catalogue: neither", "mesh-catalog",
|
||||
[]string{"modules/gitea/index.ts"}, "gitea"},
|
||||
} {
|
||||
r, got := planMerge(t, c.repo, c.paths, entries, read, edges)
|
||||
if got != c.want {
|
||||
t.Errorf("%s: planned %q, wanted %q (as today; issue 338)", c.what, got, c.want)
|
||||
}
|
||||
if c.repo == "mesh-controller" && strings.Join(r.Unread, ",") != "" {
|
||||
t.Errorf("%s: a root-built module reads every file, and %v were said unread", c.what, r.Unread)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sharedRepositoryEdges is what dependenciesOf derives for the catalogue of the test above, sorted as it
|
||||
// sorts them.
|
||||
var sharedRepositoryEdges = []inventory.Edge{
|
||||
dep("build-agent", inventory.EdgePackages, "mesh-controller"),
|
||||
dep("build-agent", inventory.EdgeWorkerOf, "mesh-controller"),
|
||||
dep("gitea", inventory.EdgeBuiltBy, "build-agent"),
|
||||
dep("mesh-controller", inventory.EdgeBuiltBy, "build-agent"),
|
||||
dep("route-proxy", inventory.EdgeBuiltBy, "build-agent"),
|
||||
dep("route-proxy", inventory.EdgePackages, "mesh-controller"),
|
||||
}
|
||||
|
||||
// **The planner's invariant, over random catalogues.** For any catalogue whose dependencies form no cycle
|
||||
// and any set of changed files in one repository:
|
||||
//
|
||||
// - the plan is exactly the modules of that repository whose directory holds a changed file (every file,
|
||||
// for a module built from the root), and everything reachable from them along stands-on, declared and
|
||||
// packages — never along built-by or worker-of;
|
||||
// - every stands-on, declared, built-by and worker-of edge with both ends in the plan has the module
|
||||
// depended on in an earlier tier;
|
||||
// - no cycle is said.
|
||||
//
|
||||
// Seeded, so a failure is replayed by its seed and case.
|
||||
func TestAPlanIsTheTouchedModulesAndWhatIsReachableAlongTheWideningEdges(t *testing.T) {
|
||||
kinds := []string{inventory.EdgeStandsOn, inventory.EdgeDeclared, inventory.EdgePackages,
|
||||
inventory.EdgeBuiltBy, inventory.EdgeWorkerOf}
|
||||
widens := map[string]bool{inventory.EdgeStandsOn: true, inventory.EdgeDeclared: true, inventory.EdgePackages: true}
|
||||
orders := map[string]bool{inventory.EdgeStandsOn: true, inventory.EdgeDeclared: true,
|
||||
inventory.EdgeBuiltBy: true, inventory.EdgeWorkerOf: true}
|
||||
// Directory names drawn from one pool, so two repositories hold directories of the same name, and one
|
||||
// is a prefix of another.
|
||||
dirs := []string{"a", "ab", "b", "c", "lib/x", "lib/y", "modules/a", "modules/a/sub"}
|
||||
files := []string{"README.md", "merge-check.sh", "lib/z.go", "docs/x.md", "modules/README.md"}
|
||||
|
||||
falseCycles, firstFalseCycle := 0, ""
|
||||
for _, seed := range []uint64{1, 2, 3, 0x338, 0x162} {
|
||||
rng := rand.New(rand.NewPCG(seed, seed^0x9e3779b97f4a7c15))
|
||||
for n := 0; n < 100; n++ {
|
||||
repos := 1 + rng.IntN(3)
|
||||
repoName := func(i int) string { return fmt.Sprintf("r%d", i) }
|
||||
count := 1 + rng.IntN(12)
|
||||
var entries []inventory.Entry
|
||||
repoOf, dirOf := map[string]int{}, map[string]string{}
|
||||
for i := 0; i < count; i++ {
|
||||
name := fmt.Sprintf("m%02d", i)
|
||||
repo := rng.IntN(repos)
|
||||
dir := dirs[rng.IntN(len(dirs))]
|
||||
if rng.IntN(12) == 0 {
|
||||
dir = "" // built from the repository's root
|
||||
}
|
||||
repoOf[name], dirOf[name] = repo, dir
|
||||
entries = append(entries, fromRepo(name, "http://forge.internal:20000/novox/"+repoName(repo)+".git", dir))
|
||||
}
|
||||
// A graph with no cycle: a module depends only on modules made before it.
|
||||
var edges []inventory.Edge
|
||||
for i := 1; i < count; i++ {
|
||||
for j := 0; j < i; j++ {
|
||||
if rng.IntN(4) == 0 {
|
||||
edges = append(edges, dep(fmt.Sprintf("m%02d", i), kinds[rng.IntN(len(kinds))], fmt.Sprintf("m%02d", j)))
|
||||
}
|
||||
}
|
||||
}
|
||||
merged := rng.IntN(repos)
|
||||
var paths []string
|
||||
for k := 1 + rng.IntN(4); k > 0; k-- {
|
||||
if rng.IntN(3) == 0 {
|
||||
paths = append(paths, files[rng.IntN(len(files))])
|
||||
} else {
|
||||
paths = append(paths, dirs[rng.IntN(len(dirs))]+"/f.go")
|
||||
}
|
||||
}
|
||||
|
||||
// What the rules say.
|
||||
want := map[string]bool{}
|
||||
for _, e := range entries {
|
||||
name := e.Manifest.Module
|
||||
if repoOf[name] != merged {
|
||||
continue
|
||||
}
|
||||
for _, p := range paths {
|
||||
if d := dirOf[name]; d == "" || p == d || strings.HasPrefix(p, d+"/") {
|
||||
want[name] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
for grew := true; grew; {
|
||||
grew = false
|
||||
for _, e := range edges {
|
||||
if widens[e.Kind] && want[e.To] && !want[e.From] {
|
||||
want[e.From], grew = true, true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
r, _ := planMerge(t, repoName(merged), paths, entries, nil, edges)
|
||||
got := map[string]bool{}
|
||||
tierOf := map[string]int{}
|
||||
for i, tier := range r.Plan.Tiers {
|
||||
for _, name := range tier {
|
||||
got[name], tierOf[name] = true, i
|
||||
}
|
||||
}
|
||||
replay := func() string {
|
||||
return fmt.Sprintf("seed %#x case %d: repository %s, files %v\n modules %v\n edges %v\n tiers %v",
|
||||
seed, n, repoName(merged), paths, describe(entries), edges, r.Plan.Tiers)
|
||||
}
|
||||
if !sameSet(got, want) {
|
||||
t.Fatalf("planned %v, wanted %v\n%s", keys(got), keys(want), replay())
|
||||
}
|
||||
for _, e := range edges {
|
||||
if orders[e.Kind] && got[e.From] && got[e.To] && tierOf[e.To] >= tierOf[e.From] {
|
||||
t.Fatalf("%s %s %s, and %s is in tier %d, not before %s's %d\n%s", e.From, e.Kind, e.To,
|
||||
e.To, tierOf[e.To], e.From, tierOf[e.From], replay())
|
||||
}
|
||||
}
|
||||
if hasCycle(r.Plan.Tiers, edges) {
|
||||
falseCycles++
|
||||
if firstFalseCycle == "" {
|
||||
firstFalseCycle = replay()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Said once, after the other invariants have run over every case, so it hides none of them (hasCycle
|
||||
// counted a packages edge on main at 8170fc5: 19 of these 500 cases).
|
||||
if falseCycles > 0 {
|
||||
t.Errorf("a cycle said of a graph with none in %d of 500 cases; "+
|
||||
"the first:\n%s", falseCycles, firstFalseCycle)
|
||||
}
|
||||
}
|
||||
|
||||
func sameSet(a, b map[string]bool) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for k := range a {
|
||||
if !b[k] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func keys(m map[string]bool) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func describe(entries []inventory.Entry) []string {
|
||||
var out []string
|
||||
for _, e := range entries {
|
||||
out = append(out, fmt.Sprintf("%s@%s:%q", e.Manifest.Module,
|
||||
strings.TrimSuffix(strings.TrimPrefix(e.Source.Repository, "http://forge.internal:20000/novox/"), ".git"),
|
||||
e.Source.Path))
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -419,8 +419,9 @@ func askResolver(ctx context.Context, at, name string, kind dnsmessage.Type) ([]
|
||||
return addresses, answer.RCode, nil
|
||||
}
|
||||
|
||||
// probeHolders is D3: every seat that serves verbs has, on every machine that holds it and is heard
|
||||
// from, a holder answering the bus's discovery for that seat. A machine past its heartbeat's bound is
|
||||
// probeHolders is D3: every seat with a verb its holder must serve has, on every machine that holds it
|
||||
// and is heard from, a holder answering the bus's discovery for that seat. A seat whose verbs are all
|
||||
// still optional is not asked about (holdingNeedsAnAnswer). A machine past its heartbeat's bound is
|
||||
// S1's, and is not asked about here.
|
||||
func probeHolders(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
entries, err := d.open.inventory.Catalogued(ctx)
|
||||
@@ -436,44 +437,7 @@ func probeHolders(ctx context.Context, d *doctor) ([]conditions.Observation, err
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := time.Now()
|
||||
expected := map[string]map[string]bool{} // seat → machine
|
||||
add := func(seat, node string) {
|
||||
if expected[seat] == nil {
|
||||
expected[seat] = map[string]bool{}
|
||||
}
|
||||
expected[seat][node] = true
|
||||
}
|
||||
for _, s := range catalogue.SeatsWithAProtocol() {
|
||||
if len(s.Serves) == 0 || s.Name == catalogue.ControllerSeatName {
|
||||
continue // a seat with no verb has nothing to answer with; this controller is answering now
|
||||
}
|
||||
onRecord := false
|
||||
for _, h := range recorded {
|
||||
if h.Claim == s.Name && heard[h.Node] {
|
||||
add(s.Name, h.Node)
|
||||
onRecord = true
|
||||
}
|
||||
}
|
||||
if onRecord && s.Scope == catalogue.ScopeMesh {
|
||||
continue
|
||||
}
|
||||
for _, e := range entries {
|
||||
for _, c := range e.Manifest.Claims {
|
||||
if c.Name != s.Name {
|
||||
continue
|
||||
}
|
||||
for _, node := range e.On {
|
||||
// Assigned is not there yet: a holder is expected once its machine was sent it and
|
||||
// had time to report, never between `assign` and `push` (novox/hq issue 275).
|
||||
if heard[node] && (s.Scope == catalogue.ScopeNode || !onRecord) &&
|
||||
delivered[node].settled(e.Manifest.Module, now) {
|
||||
add(s.Name, node)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
expected := holdersToHear(catalogue.SeatsWithAProtocol(), recorded, entries, heard, delivered, time.Now())
|
||||
if len(expected) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -509,6 +473,11 @@ func probeHolders(ctx context.Context, d *doctor) ([]conditions.Observation, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return silentHolders(expected, answering), nil
|
||||
}
|
||||
|
||||
// silentHolders is every expected holder the discovery did not hear, as D3's observations.
|
||||
func silentHolders(expected, answering map[string]map[string]bool) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for seat, nodes := range expected {
|
||||
for node := range nodes {
|
||||
@@ -521,7 +490,71 @@ func probeHolders(ctx context.Context, d *doctor) ([]conditions.Observation, err
|
||||
Said: fmt.Sprintf("no answer for %s from %s to the bus's discovery", seat, node)})
|
||||
}
|
||||
}
|
||||
return sortedFound(out), nil
|
||||
return sortedFound(out)
|
||||
}
|
||||
|
||||
// holdersToHear is D3's expectation: seat → every machine whose holder of it must answer the bus's
|
||||
// discovery. A seat is listed only if holding it requires serving a verb (holdingNeedsAnAnswer); a holder
|
||||
// is expected on a machine that is heard from and that holds it on record or — for a node seat, or a mesh
|
||||
// seat nobody holds on record — was sent the module that claims it and had time to report.
|
||||
func holdersToHear(seats []catalogue.Seat, recorded []catalogue.Held, entries []inventory.Entry,
|
||||
heard map[string]bool, delivered map[string]lastSend, now time.Time) map[string]map[string]bool {
|
||||
expected := map[string]map[string]bool{} // seat → machine
|
||||
add := func(seat, node string) {
|
||||
if expected[seat] == nil {
|
||||
expected[seat] = map[string]bool{}
|
||||
}
|
||||
expected[seat][node] = true
|
||||
}
|
||||
for _, s := range seats {
|
||||
if s.Name == catalogue.ControllerSeatName {
|
||||
continue // this controller is answering now
|
||||
}
|
||||
if !holdingNeedsAnAnswer(s) {
|
||||
continue // nothing its holder must serve, so nothing its silence would say (novox/hq issue 299)
|
||||
}
|
||||
onRecord := false
|
||||
for _, h := range recorded {
|
||||
if h.Claim == s.Name && heard[h.Node] {
|
||||
add(s.Name, h.Node)
|
||||
onRecord = true
|
||||
}
|
||||
}
|
||||
if onRecord && s.Scope == catalogue.ScopeMesh {
|
||||
continue
|
||||
}
|
||||
for _, e := range entries {
|
||||
for _, c := range e.Manifest.Claims {
|
||||
if c.Name != s.Name {
|
||||
continue
|
||||
}
|
||||
for _, node := range e.On {
|
||||
// Assigned is not there yet: a holder is expected once its machine was sent it and
|
||||
// had time to report, never between `assign` and `push` (novox/hq issue 275).
|
||||
if heard[node] && (s.Scope == catalogue.ScopeNode || !onRecord) &&
|
||||
delivered[node].settled(e.Manifest.Module, now) {
|
||||
add(s.Name, node)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return expected
|
||||
}
|
||||
|
||||
// holdingNeedsAnAnswer says whether a seat's holder is judged silent when the bus's discovery hears
|
||||
// nothing from it: only when the seat has a verb its holder must serve. **Silence is judged on required
|
||||
// verbs alone** (novox/hq issue 299). An optional verb is one the seat's holders are still catching up
|
||||
// to (Verb.Optional; design 33 §7, novox/hq ADR 0246): a holder that does not serve it yet holds the seat
|
||||
// rightly, and a holder serving nothing because every verb is still optional is exactly that holder, not
|
||||
// a silent one. A seat with no verb at all has nothing to answer with either.
|
||||
func holdingNeedsAnAnswer(s catalogue.Seat) bool {
|
||||
for _, v := range s.Serves {
|
||||
if !v.Optional {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// reportGrace is how long a machine is given, after it was sent a declaration, to apply it and report
|
||||
@@ -842,6 +875,16 @@ func streamDiffers(want broker.Stream, have nats.StreamConfig) string {
|
||||
if perSubject != 0 && have.MaxMsgsPerSubject != perSubject {
|
||||
differs = append(differs, fmt.Sprintf("keeps %d per subject, defined %d", have.MaxMsgsPerSubject, perSubject))
|
||||
}
|
||||
if want.MaxBytes > 0 && have.MaxBytes != want.MaxBytes {
|
||||
differs = append(differs, fmt.Sprintf("holds up to %d bytes, defined %d", have.MaxBytes, want.MaxBytes))
|
||||
}
|
||||
if want.DuplicatesSeconds > 0 && have.Duplicates != time.Duration(want.DuplicatesSeconds)*time.Second {
|
||||
differs = append(differs, fmt.Sprintf("keeps one of a message id for %s, defined %s", have.Duplicates,
|
||||
time.Duration(want.DuplicatesSeconds)*time.Second))
|
||||
}
|
||||
if want.DiscardNew && have.Discard != nats.DiscardNew {
|
||||
differs = append(differs, "drops what it holds when full, defined to refuse what comes next")
|
||||
}
|
||||
return strings.Join(differs, "; ")
|
||||
}
|
||||
|
||||
|
||||
+110
-8
@@ -65,6 +65,8 @@ func connectLink(ctx context.Context, inv *inventory.Inventory, enroller link.En
|
||||
}
|
||||
|
||||
func serve(ctx context.Context) (err error) {
|
||||
// Nothing this process does, or starts, is the operator at the terminal (novox/hq ADR 0266).
|
||||
markServed()
|
||||
// The one process whose log is read over time, so the one that says each change to a node's
|
||||
// unmet seat dependencies once (novox/hq ADR 0207).
|
||||
logUnheldChanges = true
|
||||
@@ -165,6 +167,10 @@ func serve(ctx context.Context) (err error) {
|
||||
// Open plans move on a timer as well as on outcomes (novox/hq ADR 0162): a tier waiting for
|
||||
// machines to report moves when they have, and a plan left by a replaced controller resumes.
|
||||
go planTicker(ctx, open)
|
||||
// And the pending assignments settled on a tick of their own (novox/hq ADR 0261): made once their module
|
||||
// is registered, ended with why when its build will not register it, raised and cleared as conditions.
|
||||
// Never by a read.
|
||||
go settlingPending(ctx, open)
|
||||
// The durations the core's bounds are set from are kept a month (novox/hq to-be 45 Phase 0).
|
||||
go forgettingOldDurations(ctx, inv)
|
||||
// And what the catalogue decided a build meant. The builder's own result is already handled
|
||||
@@ -215,6 +221,10 @@ func serve(ctx context.Context) (err error) {
|
||||
}
|
||||
// The hand-act log is counted for `status` on this connection rather than a new one a minute.
|
||||
handActConn = bus.Conn
|
||||
// And everything else this controller does on the bus for a moment (novox/hq issue 327).
|
||||
servingBus.Store(server.JetStream())
|
||||
// No longer serving: nothing is lent, and dead-letters says it is not read here (novox/hq issue 330).
|
||||
defer servingBus.Store(nil)
|
||||
// And says when it replaced a value given by hand (novox/hq ADR 0228).
|
||||
givenEvents = bus
|
||||
// And a pull request's merge check, asked when the forge announces its head and said when judged
|
||||
@@ -256,6 +266,12 @@ func serve(ctx context.Context) (err error) {
|
||||
return err
|
||||
}
|
||||
defer stopServing()
|
||||
// And the operator's command on every node, asked through each node's engine (novox/hq ADR 0272).
|
||||
stopCLI, err := bus.ServeCLI(answerMeshCLI(open.inventory), log.New(os.Stdout, "", log.LstdFlags))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stopCLI()
|
||||
// And says so on the bus (novox/hq ADR 0197): what it serves, as the NATS services protocol asks.
|
||||
stopAnnouncing, err := bus.Announce(seatAnnouncement(handlers), log.New(os.Stdout, "", log.LstdFlags))
|
||||
if err != nil {
|
||||
@@ -359,14 +375,31 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
// A push by hand is a repair, and says why (novox/hq to-be 45 §7): required through the seat,
|
||||
// recorded when given at a shell — see handacts.go for why a shell is not refused.
|
||||
why := addHandActFlags(set)
|
||||
// Every machine at once is said, not defaulted to (novox/hq ADR 0217): a bare `push` sent the
|
||||
// whole mesh on 2026-10-05 when its usage was wanted.
|
||||
all := set.Bool("all", false, "every machine")
|
||||
// A running module's data moving is sent only when said, per module (novox/hq ADR 0217).
|
||||
move := set.String("move", "", "modules whose data may move with this push, comma-separated")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
args = positionals
|
||||
if len(args) > 1 {
|
||||
return errors.New("push [<node>] [--behind] — one node, or all of them")
|
||||
return errors.New("push <node> | push --behind | push --all — one machine, the ones behind, or all of them")
|
||||
}
|
||||
if len(args) == 0 && !*behind && !*all {
|
||||
return errors.New("push names a machine, or says --behind (the ones not running what they " +
|
||||
"should) or --all (every machine) — a push to the whole mesh is not the default (novox/hq ADR 0217)")
|
||||
}
|
||||
if *all && (*behind || len(args) == 1) {
|
||||
return errors.New("push --all is every machine; it takes no machine and no --behind")
|
||||
}
|
||||
movable := map[string]bool{}
|
||||
for _, m := range splitModules(*move) {
|
||||
movable[m] = true
|
||||
}
|
||||
var heldBack []string
|
||||
if len(args) == 1 && *behind {
|
||||
// Naming a machine and asking for the ones that need it are two different requests, and
|
||||
// guessing which was meant would sometimes push to a machine somebody did not name.
|
||||
@@ -377,6 +410,12 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
if *behind {
|
||||
recorded = append(recorded, "--behind")
|
||||
}
|
||||
if *all {
|
||||
recorded = append(recorded, "--all")
|
||||
}
|
||||
if *move != "" {
|
||||
recorded = append(recorded, "--move", *move)
|
||||
}
|
||||
why.record(ctx, "push", recorded)
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
@@ -587,19 +626,28 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
})
|
||||
|
||||
defer release()
|
||||
// A machine whose declaration would move a running module's data is held, and only it (novox/hq
|
||||
// ADR 0217): said, with the command that sends it, and left out of everything below.
|
||||
toSend, err := holdingBack(ctx, inv, sending, movable, &heldBack)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// **What it recreates, said before it is sent** (novox/hq ADR 0245): a person's push moves every
|
||||
// module whose build changed, and recreates what changed in it — a gated send said so, a push did not.
|
||||
sayWhatAPushRecreates(ctx, open, toSend, os.Stdout)
|
||||
// Each machine's memberships first, then the declarations (novox/hq issue 249, ADR 0160): a push
|
||||
// is the one most operators run, and on 2026-10-01 it was the one path that issued none.
|
||||
bus := overTheBus{open: open, server: server, signer: ident}
|
||||
sentDigest, err := deliver(ctx, bus, holder, sending)
|
||||
sentDigest, err := deliver(ctx, bus, holder, toSend)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
release()
|
||||
told := make([]string, 0, len(sending))
|
||||
for _, r := range sending {
|
||||
told := make([]string, 0, len(toSend))
|
||||
for _, r := range toSend {
|
||||
told = append(told, r.node)
|
||||
}
|
||||
fmt.Printf("\n%d node(s) told: %s\n", len(sending), strings.Join(told, ", "))
|
||||
fmt.Printf("\n%d node(s) told: %s\n", len(toSend), strings.Join(told, ", "))
|
||||
reportUnheldPushed(os.Stdout, len(args) == 1, asked, unheld)
|
||||
|
||||
// **A named push leaves the mesh consistent, not just the machine it named** (novox/hq
|
||||
@@ -621,7 +669,12 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
for n := range saidHeld {
|
||||
handled[n] = true
|
||||
}
|
||||
refused, err := flushBehind(ctx, open, nodes, handled, composeForPush(open, gens), bus, holder, os.Stdout)
|
||||
// The cascade is a push too, and a machine in it whose data would move is held the same way
|
||||
// (novox/hq ADR 0217).
|
||||
keep := keeping(func(held context.Context, sending []readyNode) ([]readyNode, error) {
|
||||
return holdingBack(held, inv, sending, movable, &heldBack)
|
||||
})
|
||||
refused, err := flushBehind(ctx, open, nodes, handled, composeForPush(open, gens), bus, holder, keep, os.Stdout)
|
||||
refusals = append(refusals, refused...)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -636,6 +689,11 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(heldBack) > 0 {
|
||||
sort.Strings(heldBack)
|
||||
return fmt.Errorf("held %s: a running module's data would move — see above; nothing was sent "+
|
||||
"there (novox/hq ADR 0217)", strings.Join(heldBack, ", "))
|
||||
}
|
||||
return couldNotBeResolved(refusals, len(sending))
|
||||
}
|
||||
|
||||
@@ -744,7 +802,7 @@ func composeEach(names []string, allot func(node string) (order, error),
|
||||
// still sent. Their memberships go before their declarations, as every send's do (issue 249).
|
||||
func sendRound(ctx context.Context, open *stores, names []string,
|
||||
compose func(held context.Context, node string) (sendable, error),
|
||||
d delivery, holder string) ([]string, error) {
|
||||
d delivery, holder string, keep keeping) ([]string, error) {
|
||||
held, release, err := holdNodes(ctx, open, names)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -753,12 +811,46 @@ func sendRound(ctx context.Context, open *stores, names []string,
|
||||
sending, refused := composeEach(names, allotting(held, open.inventory), func(node string) (sendable, error) {
|
||||
return compose(held, node)
|
||||
})
|
||||
if keep != nil {
|
||||
if sending, err = keep(held, sending); err != nil {
|
||||
return refused, err
|
||||
}
|
||||
}
|
||||
if _, err := deliver(held, d, holder, sending); err != nil {
|
||||
return refused, err
|
||||
}
|
||||
return refused, nil
|
||||
}
|
||||
|
||||
// keeping is what a round of sends keeps of the machines it composed, before any is delivered: nil
|
||||
// keeps them all. A push holds back the machines whose running modules' data would move (holdingBack).
|
||||
type keeping func(held context.Context, sending []readyNode) ([]readyNode, error)
|
||||
|
||||
// holdingBack leaves out each machine whose declaration would move a running module's data, says so,
|
||||
// and names it among those held back (novox/hq ADR 0217); the rest go on to be delivered, grants first
|
||||
// (issue 249). A declaration's body is the same bytes however often it is read.
|
||||
func holdingBack(ctx context.Context, inv *inventory.Inventory, sending []readyNode, movable map[string]bool,
|
||||
heldBack *[]string) ([]readyNode, error) {
|
||||
var out []readyNode
|
||||
for _, s := range sending {
|
||||
body, err := s.declared.Body()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
moves, err := heldMoves(ctx, inv, s.node, body, movable)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(moves) > 0 {
|
||||
sayDataHeld(os.Stdout, s.node, moves)
|
||||
*heldBack = append(*heldBack, s.node)
|
||||
continue
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// delivery is the two acts of sending machines what they should be, apart, so the order between
|
||||
// them is one function's and can be read and tested there (novox/hq issue 249).
|
||||
type delivery interface {
|
||||
@@ -1082,7 +1174,7 @@ func sendToEach(ctx context.Context, open *stores, names []string) ([]string, er
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// **A recorded build moves only by a person's push** (novox/hq issue 295, ADR 0242): this send — a
|
||||
// **A recorded build moves only by a person's push** (novox/hq issue 295, ADR 0245): this send — a
|
||||
// plan's, a release plan's, a rollback's, a healer's, a rotation's — composes every recorded module at
|
||||
// the build its machine runs. The bus step is a person's word for the bus alone.
|
||||
if ctx, err = sendKeeps(ctx, inv); err != nil {
|
||||
@@ -1470,6 +1562,16 @@ func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body
|
||||
if err := inv.RecordSentUnder(kept, record.ID, digest, builds, epoch); err != nil {
|
||||
return "", err
|
||||
}
|
||||
// And what it was, summarised, so the next push can be compared with it (novox/hq ADR 0217).
|
||||
// Never a reason to fail a send that is already away: a summary that cannot be kept means the
|
||||
// next comparison has nothing to hold against, which is how every machine starts.
|
||||
if summary, err := summarize(body); err == nil {
|
||||
if raw, err := json.Marshal(summary); err == nil {
|
||||
if err := inv.RecordSentSummary(kept, record.ID, raw); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: what it was sent is not kept for comparison: %v\n", node, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return digest, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The live case of 2026-10-07, 20:27 UTC: a person's `push` of one machine moved mail to a new build whose
|
||||
// automx container had a new image, and recreated it; the push's answer said nothing of it. The push now
|
||||
// says, per machine, every module it moves and what that recreates — before it is sent. A module whose
|
||||
// build did not change, and one new to the machine, are not moves.
|
||||
func TestAPushSaysWhatItRecreates(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
start := time.Now().Add(-time.Hour)
|
||||
image := func(c string) string { return "registry.invalid:5000/mailu/x@sha256:" + strings.Repeat(c, 64) }
|
||||
for _, b := range []link.BuildResult{
|
||||
aContainerBuild(t, "mailu", "c74f407a", catalogue.PolicyRecord, start,
|
||||
map[string][2]string{"smtp": {image("a"), ""}, "automx": {image("c"), ""}}),
|
||||
aContainerBuild(t, "postgres", "c1111111", "", start.Add(time.Second),
|
||||
map[string][2]string{"server": {image("e"), ""}}),
|
||||
} {
|
||||
if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, m := range []string{"mailu", "postgres"} {
|
||||
if _, err := inv.Assign(ctx, "anchor", m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor",
|
||||
map[string]string{"mailu": "c74f407a", "postgres": "c1111111"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Mail's new build: automx's image changes, smtp's does not.
|
||||
if _, _, err := takeIn(ctx, inv, aContainerBuild(t, "mailu", "1ab84e69", catalogue.PolicyRecord,
|
||||
start.Add(time.Minute), map[string][2]string{"smtp": {image("a"), ""}, "automx": {image("d"), ""}})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Composed as a person's push composes it.
|
||||
plan, settings, err := planFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationWith(ctx, open, "anchor", plan, settings, gens, Allocating)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if declared.Builds["mailu"] != "1ab84e69" {
|
||||
t.Fatalf("a person's push carries mail at %q", declared.Builds["mailu"])
|
||||
}
|
||||
|
||||
var said bytes.Buffer
|
||||
sayWhatAPushRecreates(ctx, open, []readyNode{{node: "anchor", declared: declared}}, &said)
|
||||
out := said.String()
|
||||
for _, want := range []string{"anchor moves 1 module(s)", "mailu", "c74f407a → 1ab84e69",
|
||||
"recreates 1 of mailu's 2 container(s) (with a new image: automx)"} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Fatalf("the push says %q; it does not say %q", out, want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(out, "postgres") {
|
||||
t.Fatalf("a module whose build did not change is said to move: %q", out)
|
||||
}
|
||||
|
||||
// A machine never sent anything before has no moves to say.
|
||||
said.Reset()
|
||||
sayWhatAPushRecreates(ctx, open, []readyNode{{node: "laptop", declared: declared}}, &said)
|
||||
if said.Len() != 0 {
|
||||
t.Fatalf("a machine with no last send is said to move: %q", said.String())
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -37,22 +38,21 @@ const (
|
||||
killAnswer = 75 * time.Second
|
||||
)
|
||||
|
||||
// dialTheBus opens the controller's own connection, for a command that reads or changes the queue.
|
||||
// dialTheBus is the controller's connection, for a command that reads or changes the queue: the serving
|
||||
// controller's own, lent, when this process is it (novox/hq issue 327).
|
||||
func dialTheBus() (*broker.JetStream, error) {
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot reach the bus: %w", err)
|
||||
}
|
||||
return js, nil
|
||||
return aBus()
|
||||
}
|
||||
|
||||
// queueCommand prints every ask in the build seat's work queue.
|
||||
func queueCommand(ctx context.Context, args []string) error {
|
||||
return listQueue(ctx, args, os.Stdout)
|
||||
}
|
||||
|
||||
// listQueue is `queue`: the build queue, as a person reads it or as JSON.
|
||||
func listQueue(ctx context.Context, args []string, w io.Writer) error {
|
||||
set := flag.NewFlagSet("queue", flag.ContinueOnError)
|
||||
usageTo(set, w)
|
||||
asJSON := set.Bool("json", false, "the queue as JSON")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
@@ -72,10 +72,10 @@ func queueCommand(ctx context.Context, args []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
fmt.Fprintln(w, string(body))
|
||||
return nil
|
||||
}
|
||||
fmt.Print(queueText(q, time.Now()))
|
||||
fmt.Fprint(w, queueText(q, time.Now()))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -406,6 +406,8 @@ func rebuildCommand(ctx context.Context, args []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
recordAsked(ctx, inventory.BuildRequest{ID: id, Repository: source.Repository, Seat: source.Seat, Path: path,
|
||||
Ref: ref, For: "rebuild"})
|
||||
fmt.Printf("rebuild asked as %s\n", id)
|
||||
return nil
|
||||
}
|
||||
@@ -490,7 +492,11 @@ func replayCommand(ctx context.Context, args []string) error {
|
||||
if err := replayRefusal(b, module, history, *register, *older, outstanding); err != nil {
|
||||
return err
|
||||
}
|
||||
id, err := buildOneAsked(ctx, source, b.Path, b.Commit, 0, !*register)
|
||||
asker := ""
|
||||
if *register {
|
||||
asker = "replay"
|
||||
}
|
||||
id, err := buildOneAsked(ctx, source, b.Path, b.Commit, 0, !*register, asker)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -276,21 +276,21 @@ func TestAPlanWaitingOnAPausedSeatSaysSoAndIsNotLate(t *testing.T) {
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked}}}
|
||||
|
||||
all := pauseOf([]string{"g14", "ace"}, map[string]link.HolderState{"ace": {Paused: true}, "g14": {Paused: true}})
|
||||
line := planLineWith(p, now, all)
|
||||
line := planLineWith(p, now, all, tierAtLeast)
|
||||
if !strings.Contains(line, "waiting: the build seat is paused on ace, g14 (asked 12m0s ago)") || strings.Contains(line, "LATE") {
|
||||
t.Errorf("a plan on a paused seat reads %q", line)
|
||||
}
|
||||
if st := planStatuses([]inventory.Plan{p}, now, all)[0]; st.Late || !strings.Contains(st.Waiting, "paused on ace, g14") {
|
||||
if st := planStatuses([]inventory.Plan{p}, now, all, nil)[0]; st.Late || !strings.Contains(st.Waiting, "paused on ace, g14") {
|
||||
t.Errorf("status --json says %+v", st)
|
||||
}
|
||||
if _, late := openPlans([]inventory.Plan{p}, all); late != 0 {
|
||||
if _, late := openPlans([]inventory.Plan{p}, now, all, nil); late != 0 {
|
||||
t.Errorf("a plan waiting on a paused seat counted late")
|
||||
}
|
||||
|
||||
longAgo := now.Add(-25 * time.Hour)
|
||||
forgotten := p
|
||||
forgotten.Modules = map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &longAgo}}
|
||||
if line := planLineWith(forgotten, now, all); !strings.Contains(line, "PAUSED OVER A DAY") || strings.Contains(line, "LATE") {
|
||||
if line := planLineWith(forgotten, now, all, tierAtLeast); !strings.Contains(line, "PAUSED OVER A DAY") || strings.Contains(line, "LATE") {
|
||||
t.Errorf("a plan paused over a day reads %q", line)
|
||||
}
|
||||
|
||||
@@ -298,10 +298,10 @@ func TestAPlanWaitingOnAPausedSeatSaysSoAndIsNotLate(t *testing.T) {
|
||||
if some.All || !reflect.DeepEqual(some.Nodes, []string{"ace"}) {
|
||||
t.Fatalf("%+v", some)
|
||||
}
|
||||
if line := planLineWith(p, now, some); !strings.Contains(line, "LATE") {
|
||||
if line := planLineWith(p, now, some, tierAtLeast); !strings.Contains(line, "LATE") {
|
||||
t.Errorf("a seat paused on one holder of two made the plan not late: %q", line)
|
||||
}
|
||||
if st := planStatuses([]inventory.Plan{p}, now, some)[0]; !st.Late {
|
||||
if st := planStatuses([]inventory.Plan{p}, now, some, nil)[0]; !st.Late {
|
||||
t.Errorf("status --json: %+v", st)
|
||||
}
|
||||
// A plan rolling out, or with nothing asked, is not waiting on the seat.
|
||||
|
||||
@@ -91,8 +91,8 @@ type meshStatus struct {
|
||||
// Conditions is every open condition, urgent first and then oldest first (novox/hq to-be 45 §2):
|
||||
// what is wrong, as the watchdogs, the self-check and the providers say it. Always present — an
|
||||
// empty list is "none open" — unless they could not be read, which ConditionsUnread says.
|
||||
Conditions []conditions.Condition `json:"conditions"`
|
||||
ConditionsUnread string `json:"conditionsUnread,omitempty"`
|
||||
Conditions []conditionInBrief `json:"conditions"`
|
||||
ConditionsUnread string `json:"conditionsUnread,omitempty"`
|
||||
// Failing is every consumer a provider says it keeps failing (novox/hq ADR 0224): the open
|
||||
// conditions of that kind, carried here as well because ADR 0224 names this field. Absent when no
|
||||
// provider says so. A document without it called the mesh well while the identity provider
|
||||
@@ -101,6 +101,24 @@ type meshStatus struct {
|
||||
// Overflowing is every module whose identity overflows the bound of a provision it requires, and
|
||||
// so is left out of its provider's grants (novox/hq ADR 0225). Absent when every identity fits.
|
||||
Overflowing []catalogue.Overflow `json:"overflowing,omitempty"`
|
||||
// Pending is every assignment waiting for its module's build to register it, and every one ended in
|
||||
// the last day with what ended it (novox/hq issue 325). Absent when there is none; PendingUnread says
|
||||
// why they could not be read.
|
||||
Pending []pendingStatus `json:"pending,omitempty"`
|
||||
PendingUnread string `json:"pendingUnread,omitempty"`
|
||||
}
|
||||
|
||||
// pendingStatus is one pending assignment as status says it.
|
||||
type pendingStatus struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
State string `json:"state"`
|
||||
Build string `json:"build"`
|
||||
Repository string `json:"repository,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Since time.Time `json:"since"`
|
||||
Settled *time.Time `json:"settled,omitempty"`
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
// machineFiltered is one rule set on a converged machine that the mesh did not write and that
|
||||
@@ -199,7 +217,7 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
||||
out := meshStatus{Machines: len(nodes), Wrong: []machineDoing{},
|
||||
Quiet: []machineQuiet{}, Behind: []moduleBehind{}, Waiting: []machineWaiting{},
|
||||
Reported: []machineReported{}, Unresolved: []machineUnresolved{},
|
||||
Network: asked.network, Adopted: adoptedNodes(nodes), Plans: planStatuses(asked.plans, time.Now(), asked.paused)}
|
||||
Network: asked.network, Adopted: adoptedNodes(nodes), Plans: planStatuses(asked.plans, time.Now(), asked.paused, asked.tierBounds)}
|
||||
// In a stated order, so two readings of an unchanged mesh are the same document.
|
||||
untakenNodes := make([]string, 0, len(asked.untaken))
|
||||
for name := range asked.untaken {
|
||||
@@ -235,12 +253,16 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
||||
out.Unheld = asked.unheld
|
||||
out.HandActsThisWeek, out.HandActsUnread = asked.handActs, asked.handActsUnread
|
||||
out.HealsThisWeek, out.HealsUnread = asked.heals, asked.healsUnread
|
||||
out.Conditions, out.ConditionsUnread = asked.conditions, asked.conditionsUnread
|
||||
if out.Conditions == nil {
|
||||
out.Conditions = []conditions.Condition{}
|
||||
}
|
||||
// In brief, as `conditions` lists them: status leads with every open condition, and their whole
|
||||
// evidence is each one's own (novox/hq issue 314).
|
||||
out.Conditions, out.ConditionsUnread = inBrief(asked.conditions), asked.conditionsUnread
|
||||
out.Failing = providerStandings(asked.conditions)
|
||||
out.Overflowing = asked.overflowing
|
||||
out.PendingUnread = asked.pendingUnread
|
||||
for _, p := range asked.pending {
|
||||
out.Pending = append(out.Pending, pendingStatus{Node: p.Node, Module: p.Module, State: p.State,
|
||||
Build: p.Build, Repository: p.Repository, Path: p.Path, Since: p.Since, Settled: p.Settled, Note: p.Note})
|
||||
}
|
||||
for name := range asked.refused {
|
||||
out.Unresolved = append(out.Unresolved, machineUnresolved{
|
||||
Node: name, Problem: asked.refused[name]})
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// D15: no client of the bus reconnects in a loop (novox/hq issue 327).
|
||||
//
|
||||
// The server's connection total was the one number that would show a client reconnecting, and every verb
|
||||
// the controller served opened and closed a connection of its own, hundreds an hour, so a loop was
|
||||
// invisible in it. The bus's own module reads the server's record of closed connections
|
||||
// (`nats_closed_connections`); this asks it every run, and says each user whose connections were dropped —
|
||||
// closed by anything but the client itself: a read or write error, a stale connection, a slow consumer, a
|
||||
// refused login — more often than reconnectBound in the last hour.
|
||||
|
||||
const (
|
||||
probeReconnectsID = "D15"
|
||||
kindBusReconnects = "bus-reconnects"
|
||||
// reconnectBound is how many dropped connections in an hour one user may have before it is said:
|
||||
// a client that loses its connection every five minutes. Provisional.
|
||||
reconnectBound = 12
|
||||
// busModule is the module that is the bus, and closedTool its tool that reads closed connections.
|
||||
busModule = "nats"
|
||||
closedTool = "nats_closed_connections"
|
||||
closedAsk = 20 * time.Second
|
||||
)
|
||||
|
||||
// closedConnections is what the bus's module answers.
|
||||
type closedConnections struct {
|
||||
Hours float64 `json:"hours"`
|
||||
Reaches bool `json:"reaches"`
|
||||
Users []struct {
|
||||
User string `json:"user"`
|
||||
Closed int `json:"closed"`
|
||||
Dropped int `json:"dropped"`
|
||||
DroppedPerHour float64 `json:"dropped_per_hour"`
|
||||
Names []struct {
|
||||
Name string `json:"name"`
|
||||
Closed int `json:"closed"`
|
||||
Dropped int `json:"dropped"`
|
||||
Reasons map[string]int `json:"reasons"`
|
||||
} `json:"names"`
|
||||
} `json:"users"`
|
||||
}
|
||||
|
||||
// probeReconnects is D15.
|
||||
func probeReconnects(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
if d.js == nil {
|
||||
return nil, fmt.Errorf("no bus to ask the bus's module over")
|
||||
}
|
||||
on, err := d.open.inventory.Running(ctx, busModule)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(on) == 0 {
|
||||
return nil, nil // no bus module assigned: a mesh whose bus is not the mesh's module
|
||||
}
|
||||
return reconnectsOn(ctx, d.js.Conn(), on[0])
|
||||
}
|
||||
|
||||
// reconnectsOn asks the bus module on its machine and says who reconnects in a loop.
|
||||
func reconnectsOn(ctx context.Context, conn *nats.Conn, node string) ([]conditions.Observation, error) {
|
||||
read, err := askClosed(ctx, conn, node)
|
||||
if isNothingServes(err) {
|
||||
// A bus module older than its tool has nothing it can say, which is not a failure of the probe.
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return reconnecting(read), nil
|
||||
}
|
||||
|
||||
// askClosed asks the bus's module, on its machine, who closed connections in the last hour.
|
||||
func askClosed(ctx context.Context, conn *nats.Conn, node string) (closedConnections, error) {
|
||||
var read closedConnections
|
||||
answer, err := link.AskModuleToolOn(ctx, conn, busModule, closedTool, node, map[string]any{"hours": 1}, closedAsk)
|
||||
if err != nil {
|
||||
return read, err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return read, fmt.Errorf("%s on %s answered %s with an error: %s", busModule, node, closedTool, answer.Error)
|
||||
}
|
||||
if err := unmarshalAnswer(answer, &read); err != nil {
|
||||
return read, fmt.Errorf("%s on %s answered %s with something unreadable: %w", busModule, node, closedTool, err)
|
||||
}
|
||||
return read, nil
|
||||
}
|
||||
|
||||
// reconnecting is one observation per user whose connections were dropped more than reconnectBound times
|
||||
// an hour.
|
||||
func reconnecting(read closedConnections) []conditions.Observation {
|
||||
hours := read.Hours
|
||||
if hours <= 0 {
|
||||
hours = 1
|
||||
}
|
||||
var out []conditions.Observation
|
||||
for _, u := range read.Users {
|
||||
if u.User == "" || strings.HasPrefix(u.User, "(") {
|
||||
// Refused before it logged in: no client of the mesh's, so nobody's reconnect loop. A login
|
||||
// refused again and again is a question of its own, not this probe's.
|
||||
continue
|
||||
}
|
||||
perHour := float64(u.Dropped) / hours
|
||||
if perHour <= reconnectBound {
|
||||
continue
|
||||
}
|
||||
reasons := map[string]int{}
|
||||
var names []string
|
||||
for _, n := range u.Names {
|
||||
if n.Dropped == 0 {
|
||||
continue
|
||||
}
|
||||
names = append(names, fmt.Sprintf("%q ×%d", n.Name, n.Dropped))
|
||||
for r, c := range n.Reasons {
|
||||
if r != "Client Closed" {
|
||||
reasons[r] += c
|
||||
}
|
||||
}
|
||||
}
|
||||
var why []string
|
||||
for r, c := range reasons {
|
||||
why = append(why, fmt.Sprintf("%s ×%d", r, c))
|
||||
}
|
||||
sort.Strings(why)
|
||||
partial := ""
|
||||
if !read.Reaches {
|
||||
partial = " (at least: the server's record of closed connections does not reach back the whole hour)"
|
||||
}
|
||||
who, machine := busUserWords(u.User)
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: u.User, Kind: kindBusReconnects,
|
||||
Machine: machine, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the bus dropped %s's connection %d times in the last hour%s, more than %d: a "+
|
||||
"client reconnecting in a loop", u.User, u.Dropped, partial, reconnectBound),
|
||||
Said: fmt.Sprintf("%d of %d closed connections dropped in %.0f h; by name %s; why %s", u.Dropped,
|
||||
u.Closed, hours, strings.Join(names, ", "), strings.Join(why, ", ")),
|
||||
Headline: clip(conditions.Capital(who)+" keeps losing the bus", 60),
|
||||
Explanation: conditions.Capital(fmt.Sprintf("%s lost its connection to the bus %d times in the last hour "+
|
||||
"and connected again each time. While it reconnects, what it says and what it is asked waits.", who,
|
||||
u.Dropped)),
|
||||
Resolved: "Resolved: " + who + " stays connected"})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// busUserWords is a bus user as the operator says it, and the machine it is on: `node.<machine>` the
|
||||
// node-engine, `<machine>.node-tools` the tool runner, `controller` the controller, `<machine>.<module>` a
|
||||
// module.
|
||||
func busUserWords(user string) (string, string) {
|
||||
switch {
|
||||
case user == "controller":
|
||||
return "the controller", ""
|
||||
case strings.HasPrefix(user, "node."):
|
||||
m := strings.TrimPrefix(user, "node.")
|
||||
return "the node-engine on " + m, m
|
||||
}
|
||||
if m, module, ok := strings.Cut(user, "."); ok {
|
||||
if module == "node-tools" {
|
||||
return "the tool runner on " + m, m
|
||||
}
|
||||
return module + " on " + m, m
|
||||
}
|
||||
return "a client of the bus", ""
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// The serving controller's own connection serves what it does for a moment: no connection is opened,
|
||||
// and closing what it was lent leaves the serving one open (novox/hq issue 327).
|
||||
func TestTheServingControllerLendsItsOwnConnection(t *testing.T) {
|
||||
bus := testbus.Start(t)
|
||||
serving, err := broker.Dial(bus.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer serving.Close()
|
||||
if err := serving.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
was := servingBus.Load()
|
||||
servingBus.Store(serving)
|
||||
defer servingBus.Store(was)
|
||||
t.Setenv(broker.NATSVar, bus.ClientURL())
|
||||
|
||||
before, _ := bus.Varz(nil)
|
||||
lent, err := aBus()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lent.Close()
|
||||
if !serving.Conn().IsConnected() {
|
||||
t.Fatal("closing a lent connection closed the serving controller's")
|
||||
}
|
||||
if err := onTheBus(func(*nats.Conn) error { return nil }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handlers, _, err := seatToolHandlers()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
answer, err := handlers["hand-acts"](context.Background(), json.RawMessage(`{}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a := answer.(verbAnswer); !a.OK || a.Answer == nil {
|
||||
t.Fatalf("hand-acts answered %+v", a)
|
||||
}
|
||||
_, _ = handlers["queue"](context.Background(), json.RawMessage(`{}`))
|
||||
after, _ := bus.Varz(nil)
|
||||
if opened := after.TotalConnections - before.TotalConnections; opened != 0 {
|
||||
t.Fatalf("the serving controller opened %d connection(s) of its own", opened)
|
||||
}
|
||||
}
|
||||
|
||||
// A verb that still runs as a process of its own says which in the bus's list of connections.
|
||||
func TestAVerbsOwnProcessNamesItsConnection(t *testing.T) {
|
||||
bus := testbus.Start(t)
|
||||
setup, err := broker.Dial(bus.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := setup.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
setup.Close()
|
||||
asAProcess(t, bus.ClientURL())
|
||||
handlers, _, err := seatToolHandlers()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A silence acts, so it is still its own process; it dials, and finds no such condition.
|
||||
_, _ = handlers["conditions"](context.Background(),
|
||||
json.RawMessage(`{"silence":"bus.nothing.here","for":"1h","why":"a test"}`))
|
||||
names := closedNames(t, bus)
|
||||
found := false
|
||||
for _, n := range names {
|
||||
found = found || n == "mesh-controller verb conditions"
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("the verb's connection was named %q", names)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEachProcessNamesItsConnectionsForWhatItIs(t *testing.T) {
|
||||
for _, c := range []struct{ command, verb, want string }{
|
||||
{"serve", "", "mesh-controller serving"},
|
||||
{"conditions", "conditions", "mesh-controller verb conditions"},
|
||||
{"delivery", "delivery-check", "mesh-controller verb delivery-check"},
|
||||
{"push", "", "mesh-controller command push"},
|
||||
} {
|
||||
if got := connectionName(c.command, c.verb); got != c.want {
|
||||
t.Errorf("%s/%s: %q", c.command, c.verb, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// D15: a user whose connections are dropped more than twelve times an hour is said, in plain words; one
|
||||
// whose client closes its own short connections is not.
|
||||
func TestAClientReconnectingInALoopIsSaid(t *testing.T) {
|
||||
var read closedConnections
|
||||
if err := json.Unmarshal([]byte(`{"hours":1,"reaches":true,"users":[
|
||||
{"user":"ace.node-tools","closed":40,"dropped":40,"dropped_per_hour":40,"names":[
|
||||
{"name":"ace.node-tools","closed":40,"dropped":40,"reasons":{"Stale Connection":30,"Read Error":10}}]},
|
||||
{"user":"controller","closed":300,"dropped":0,"names":[
|
||||
{"name":"mesh-controller verb delivery-check","closed":300,"dropped":0,"reasons":{"Client Closed":300}}]},
|
||||
{"user":"(no user: refused before it logged in)","closed":90,"dropped":90,"names":[{"name":"","closed":90,
|
||||
"dropped":90,"reasons":{"Authentication Failure":90}}]},
|
||||
{"user":"node.anchor","closed":5,"dropped":5,"names":[{"name":"mesh-host/anchor","closed":5,"dropped":5,
|
||||
"reasons":{"Read Error":5}}]}]}`), &read); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found := reconnecting(read)
|
||||
if len(found) != 1 {
|
||||
t.Fatalf("%+v", found)
|
||||
}
|
||||
o := found[0]
|
||||
if o.ID != "ace.node-tools" || o.Machine != "ace" || o.Kind != kindBusReconnects ||
|
||||
o.Headline != "The tool runner on ace keeps losing the bus" || !strings.Contains(o.Said, "Stale Connection ×30") {
|
||||
t.Fatalf("%+v", o)
|
||||
}
|
||||
if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Explanation: o.Explanation,
|
||||
Resolved: o.Resolved}, "ace"); !ok {
|
||||
t.Fatalf("not plain: %s", why)
|
||||
}
|
||||
}
|
||||
|
||||
// The bus's module is asked on its machine, over the bus.
|
||||
func TestTheBusModuleIsAskedWhoClosedConnections(t *testing.T) {
|
||||
bus := testbus.Start(t)
|
||||
conn, err := nats.Connect(bus.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
sub, err := conn.Subscribe(link.ModuleToolOn("nats", "nats_closed_connections", "anchor"), func(m *nats.Msg) {
|
||||
_ = m.Respond([]byte(`{"result":{"hours":1,"reaches":true,"users":[{"user":"controller","closed":2,"dropped":0}]}}`))
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
read, err := askClosed(context.Background(), conn, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(read.Users) != 1 || read.Users[0].Closed != 2 {
|
||||
t.Fatalf("%+v", read)
|
||||
}
|
||||
}
|
||||
|
||||
// A bus module older than the tool answers nothing to the question: the probe passes over it quietly.
|
||||
func TestAnOlderBusModuleIsPassedOverQuietly(t *testing.T) {
|
||||
bus := testbus.Start(t)
|
||||
conn, err := nats.Connect(bus.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
found, err := reconnectsOn(context.Background(), conn, "anchor")
|
||||
if err != nil || len(found) != 0 {
|
||||
t.Fatalf("a bus module without the tool: %v %v", found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A verb answered in the serving controller says its flag errors in its answer, not in the controller's log.
|
||||
func TestAFlagErrorIsSaidInTheAnswer(t *testing.T) {
|
||||
bus := testbus.Start(t)
|
||||
serving, err := broker.Dial(bus.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer serving.Close()
|
||||
was := servingBus.Load()
|
||||
servingBus.Store(serving)
|
||||
defer servingBus.Store(was)
|
||||
answer, read := readHere(context.Background(), []string{"hand-acts", "--bogus"})
|
||||
if !read || answer.OK || !strings.Contains(answer.Output, "flag provided but not defined") {
|
||||
t.Fatalf("answered %+v", answer)
|
||||
}
|
||||
}
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A recorded build reaches a machine only by a person's push (novox/hq issue 295, ADR 0242).
|
||||
// A recorded build reaches a machine only by a person's push (novox/hq issue 295, ADR 0245).
|
||||
//
|
||||
// **A send carries the machine's whole declaration** (ADR 0221), composed from the build the mesh holds
|
||||
// of every module on it. A module whose upgrade policy records — postgres, mongodb, keycloak, the
|
||||
@@ -121,7 +121,7 @@ func keepRecorded(ctx context.Context, open *stores, node string, shelf map[stri
|
||||
if !found {
|
||||
return nil, fmt.Errorf("%s records rather than rolls out, and %s runs its build %s, which the build "+
|
||||
"records no longer hold: this send cannot keep it and does not move it — `push %s` sends the new "+
|
||||
"one on a person's word (novox/hq ADR 0242)", m, node, short(kept[m]), node)
|
||||
"one on a person's word (novox/hq ADR 0245)", m, node, short(kept[m]), node)
|
||||
}
|
||||
shelf[m] = ran
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A recorded build reaches a machine only by a person's push (novox/hq issue 295, ADR 0242).
|
||||
// A recorded build reaches a machine only by a person's push (novox/hq issue 295, ADR 0245).
|
||||
|
||||
// aContainerBuild is a build outcome of a module of containers, each named by id with the image and the
|
||||
// health it is given; a policy when one is said.
|
||||
@@ -71,7 +71,7 @@ func TestARecordedBuildIsCarriedOnlyByAPersonsPush(t *testing.T) {
|
||||
aContainerBuild(t, "mailu", "c1111111", "", start.Add(time.Second),
|
||||
map[string][2]string{"smtp": {mailImage, ""}, "imap": {mailImage, ""}}),
|
||||
} {
|
||||
if _, _, err := takeIn(ctx, inv, b); err != nil {
|
||||
if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -189,7 +189,7 @@ func TestARecordedBuildIsCarriedOnlyByAPersonsPush(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The send says what it recreates (ADR 0242): mail's health checks adopted recreate both its
|
||||
// The send says what it recreates (ADR 0245): mail's health checks adopted recreate both its
|
||||
// containers, with no new image, every one at once.
|
||||
func TestASendSaysWhatItRecreates(t *testing.T) {
|
||||
image := "registry.invalid:5000/mailu/smtp@sha256:" + strings.Repeat("c", 64)
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A push of recorded builds is no repair (novox/hq issue 301, ADR 0242, to-be 45 §7).
|
||||
//
|
||||
// **A recorded build moves only by a person's push** (ADR 0242): a module whose upgrade policy is
|
||||
// `record` — the network path, the providers whose restart costs, mail — has its new build registered
|
||||
// at its merge and sent by nothing but `push <node>`. That push is the person's word the policy asks
|
||||
// for: the mesh working as decided. The hand-act log counted it as a repair all the same, because a push
|
||||
// by hand is the repair roll-out by default exists to end. On 2026-10-07 the operator approved, node by
|
||||
// node, new builds of the resolver, the FortiClient adapter, the network managers and the packet filter;
|
||||
// ten pushes later S15 wanted a healer for `split-dns`, `words` and `uplink-verbs`.
|
||||
//
|
||||
// **The push says what it was, from what it carried.** Never from a word the person gives — that is
|
||||
// how a repair could pass for anything (issue 292). Before it is recorded, a push that names one machine
|
||||
// reads what that machine was last sent against what the mesh holds. It is a **push of recorded builds**
|
||||
// when:
|
||||
//
|
||||
// - what the machine was last sent is known;
|
||||
// - its last report is no failure or refusal, so the push does not re-send to mend one;
|
||||
// - it moves at least one module's build; and
|
||||
// - every module it moves records rather than rolls out: a held new build of one the machine runs, or
|
||||
// the first build of one newly assigned there (whose `assign` says "run `push <node>` to send it").
|
||||
//
|
||||
// Anything else — a build that rolls out, a module taken off, nothing moved at all — is a push by hand
|
||||
// as before, and counts. The act is written with `kind: recorded-builds` and what it carried; the table
|
||||
// of verbs (handActVerbs) reads a push of that kind as a person's decision, and S15 never counts it.
|
||||
//
|
||||
// What is not compared: settings and grants. A push of a recorded build carries whatever else
|
||||
// changed in the machine's declaration with it, as any push does.
|
||||
|
||||
// recordedPush answers what a push to one machine carries, one "module from → to" each, when every
|
||||
// build it moves is a recorded one held for a person's word; otherwise nil and why it is not.
|
||||
//
|
||||
// `modules` is the machine's set; `sent` and `known` what it was last sent (Inventory.SentBuilds);
|
||||
// `current` what the mesh holds; `identical` whether two builds of a module put the same thing on a
|
||||
// machine (moveFacts.identical); `failing` the machine's last report when it was a failure or a refusal.
|
||||
func recordedPush(modules []string, sent map[string]string, known bool, current map[string]inventory.CurrentBuild,
|
||||
identical func(module, a, b string) bool, failing string) ([]string, string) {
|
||||
if !known {
|
||||
return nil, "what the machine was last sent is not known"
|
||||
}
|
||||
if failing != "" {
|
||||
return nil, "its last report was " + failing + ": the push sends again what failed"
|
||||
}
|
||||
in := map[string]bool{}
|
||||
var carried []string
|
||||
for _, m := range modules {
|
||||
in[m] = true
|
||||
// A module the mesh holds no build of, or holds without a source, has no build to move.
|
||||
now, held := current[m]
|
||||
if !held || now.Commit == "" {
|
||||
continue
|
||||
}
|
||||
was, ran := sent[m]
|
||||
if ran && identical(m, was, now.Commit) {
|
||||
continue
|
||||
}
|
||||
if now.RollOut {
|
||||
return nil, fmt.Sprintf("%s would move %sto %s, and it rolls out: its build is a plan's to send", m,
|
||||
fromBuild(was, ran), buildName(now.Commit))
|
||||
}
|
||||
carried = append(carried, fmt.Sprintf("%s %s→ %s", m, fromBuild(was, ran), buildName(now.Commit)))
|
||||
}
|
||||
for m := range sent {
|
||||
if !in[m] {
|
||||
return nil, m + " is taken off the machine"
|
||||
}
|
||||
}
|
||||
if len(carried) == 0 {
|
||||
return nil, "it moves no build: a send again"
|
||||
}
|
||||
sort.Strings(carried)
|
||||
return carried, ""
|
||||
}
|
||||
|
||||
// recordedPushOf reads, for a push about to name one machine, whether it is a push of recorded builds: what
|
||||
// it carries,
|
||||
// or nil and why not. An error is that it could not be read; the push is then recorded as a push by hand,
|
||||
// as every push was before, and says why.
|
||||
func recordedPushOf(ctx context.Context, node string) ([]string, string, error) {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
plan, _, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("its set cannot be worked out: %w", err)
|
||||
}
|
||||
modules := make([]string, 0, len(plan.Modules))
|
||||
for _, m := range plan.Modules {
|
||||
modules = append(modules, m.Module)
|
||||
}
|
||||
sent, known, err := inv.SentBuilds(ctx, node)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
f, err := readMoveFacts(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
doing, said, err := inv.DoingOf(ctx, node)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
failing := ""
|
||||
if said && (doing.Outcome == inventory.OutcomeFailed || doing.Outcome == inventory.OutcomeRefused) {
|
||||
failing = doing.Outcome
|
||||
}
|
||||
carried, why := recordedPush(modules, sent, known, f.current, f.identical, failing)
|
||||
return carried, why, nil
|
||||
}
|
||||
|
||||
// recordedBefore are the pushes recorded before a push said its kind, each a push of recorded builds by
|
||||
// what it carried (novox/hq issue 301): the operator's word, machine by machine, for new builds of modules
|
||||
// whose upgrade policy records — systemd-resolved, forticlient, networkmanager, systemd-networkd,
|
||||
// nftables, mailu. The log did not keep what a push carried then, so the record names them; no push
|
||||
// recorded since needs it, because every push now says its kind. S15 reads these as it reads a push of
|
||||
// that kind, and the three `healer-wanted` they opened clear on the next tick.
|
||||
var recordedBefore = map[string]string{
|
||||
"act-1791404241010309198-1": "systemd-resolved, first build on its machine (ADR 0247)",
|
||||
"act-1791404587689907257-1": "systemd-resolved, first build; nftables, held build (catalogue #111)",
|
||||
"act-1791404809925218992-1": "nftables, held build (catalogue #111)",
|
||||
"act-1791404844504887009-1": "mailu and nftables, held builds (catalogue #106, #111)",
|
||||
"act-1791408072745552019-1": "forticlient, held build (catalogue #114)",
|
||||
"act-1791408101286318350-1": "forticlient, held build (catalogue #114)",
|
||||
"act-1791409209593713522-1": "networkmanager, held build (catalogue #107)",
|
||||
"act-1791409280125022264-1": "networkmanager, held build (catalogue #107)",
|
||||
"act-1791409336586475835-1": "networkmanager, held build (catalogue #107)",
|
||||
"act-1791409393494198352-1": "systemd-networkd, held build (catalogue #107)",
|
||||
}
|
||||
|
||||
// pushedRecorded is whether an act in the log is a push of recorded builds.
|
||||
func pushedRecorded(a link.HandAct) bool {
|
||||
if a.Verb != "push" {
|
||||
return false
|
||||
}
|
||||
if a.Kind == link.KindRecordedBuilds {
|
||||
return true
|
||||
}
|
||||
_, before := recordedBefore[a.ID]
|
||||
return before && a.Kind == ""
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A push says whether it pushed recorded builds only, from what it carries** (novox/hq issue 301):
|
||||
// every module it moves records rather than rolls out, the machine's last send is known and its last
|
||||
// report no failure. Anything else is a push by hand, as before.
|
||||
func TestAPushIsOfRecordedBuildsOnlyWhenEveryBuildItMovesWasHeldForAPersonsWord(t *testing.T) {
|
||||
current := map[string]inventory.CurrentBuild{
|
||||
"resolver": {Commit: "r2222222"}, "nftables": {Commit: "n2222222"}, "mail": {Commit: "m2222222"},
|
||||
"web": {Commit: "w2222222", RollOut: true}, "proxy": {Commit: "p1111111", RollOut: true},
|
||||
}
|
||||
same := func(_, a, b string) bool { return a == b }
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
modules []string
|
||||
sent map[string]string
|
||||
known bool
|
||||
failing string
|
||||
carried string // what it carried, joined; empty when it is not a push of recorded builds
|
||||
why string
|
||||
}{
|
||||
{name: "one held build", modules: []string{"resolver", "proxy"},
|
||||
sent: map[string]string{"resolver": "r1111111", "proxy": "p1111111"}, known: true,
|
||||
carried: "resolver from r1111111 → r2222222"},
|
||||
{name: "two held builds", modules: []string{"mail", "nftables", "proxy"},
|
||||
sent: map[string]string{"mail": "m1111111", "nftables": "n1111111", "proxy": "p1111111"}, known: true,
|
||||
carried: "mail from m1111111 → m2222222; nftables from n1111111 → n2222222"},
|
||||
{name: "a recorded module's first build, newly assigned", modules: []string{"resolver", "proxy"},
|
||||
sent: map[string]string{"proxy": "p1111111"}, known: true,
|
||||
carried: "resolver (never sent it) → r2222222"},
|
||||
{name: "a build that rolls out moves too", modules: []string{"resolver", "web"},
|
||||
sent: map[string]string{"resolver": "r1111111", "web": "w1111111"}, known: true, why: "rolls out"},
|
||||
{name: "nothing moves", modules: []string{"resolver", "proxy"},
|
||||
sent: map[string]string{"resolver": "r2222222", "proxy": "p1111111"}, known: true, why: "a send again"},
|
||||
{name: "a module taken off", modules: []string{"resolver"},
|
||||
sent: map[string]string{"resolver": "r1111111", "proxy": "p1111111"}, known: true, why: "taken off"},
|
||||
{name: "what it was last sent is not known", modules: []string{"resolver"}, why: "not known"},
|
||||
{name: "its last report failed", modules: []string{"resolver"},
|
||||
sent: map[string]string{"resolver": "r1111111"}, known: true, failing: "failed", why: "failed"},
|
||||
} {
|
||||
carried, why := recordedPush(c.modules, c.sent, c.known, current, same, c.failing)
|
||||
if got := strings.Join(carried, "; "); got != c.carried {
|
||||
t.Errorf("%s: carried %q, want %q (%s)", c.name, got, c.carried, why)
|
||||
}
|
||||
if c.why != "" && !strings.Contains(why, c.why) {
|
||||
t.Errorf("%s: not of recorded builds because %q, want it to say %q", c.name, why, c.why)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **The live case, as the log holds it** (novox/hq issue 301). The ten pushes of 2026-10-07 recorded
|
||||
// before a push said its kind pushed recorded builds only; the three `healer-wanted` they opened clear.
|
||||
// The five pushes for the settings beside them, and any push that says no kind, still count; and a push
|
||||
// recorded as one of recorded builds since, repeated, never does.
|
||||
func TestThePushesOfRecordedBuildsOf20261007WantNoHealer(t *testing.T) {
|
||||
now := time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC)
|
||||
pushed := func(id, cause string, at time.Time) link.HandAct {
|
||||
return link.HandAct{ID: id, At: at, By: "operator", Verb: "push", Args: []string{"laptop"},
|
||||
Why: "operator's yes", Cause: cause}
|
||||
}
|
||||
var acts []link.HandAct
|
||||
at := now.Add(-12 * time.Hour)
|
||||
causes := map[string]string{
|
||||
"act-1791404241010309198-1": "split-dns", "act-1791404587689907257-1": "split-dns",
|
||||
"act-1791408072745552019-1": "split-dns", "act-1791408101286318350-1": "split-dns",
|
||||
"act-1791404809925218992-1": "words", "act-1791404844504887009-1": "words",
|
||||
"act-1791409209593713522-1": "uplink-verbs", "act-1791409280125022264-1": "uplink-verbs",
|
||||
"act-1791409336586475835-1": "uplink-verbs", "act-1791409393494198352-1": "uplink-verbs",
|
||||
}
|
||||
if len(causes) != len(recordedBefore) {
|
||||
t.Fatalf("the live case has %d acts and the record names %d", len(causes), len(recordedBefore))
|
||||
}
|
||||
for id, cause := range causes {
|
||||
if _, named := recordedBefore[id]; !named {
|
||||
t.Fatalf("%s is not named a push of recorded builds", id)
|
||||
}
|
||||
acts = append(acts, pushed(id, cause, at))
|
||||
at = at.Add(time.Minute)
|
||||
}
|
||||
for i := range 2 {
|
||||
acts = append(acts, pushed("act-settings-"+string(rune('a'+i)), "push", at))
|
||||
at = at.Add(time.Minute)
|
||||
}
|
||||
f := calm(now)
|
||||
f.handActs = acts
|
||||
got := watchHandActs(f)
|
||||
if len(got) != 1 || got[0].ID != "hand-acts.push" {
|
||||
t.Fatalf("S15 saw %+v; want only the settings pushes, counted as before", got)
|
||||
}
|
||||
|
||||
// A push that says it carried recorded builds, repeated, wants no healer; one recorded with the same word and no
|
||||
// kind still counts, as does an act of another verb whose ID such a push once had.
|
||||
ofRecorded := func(at time.Time) link.HandAct {
|
||||
a := pushed("", "split-dns", at)
|
||||
a.Kind, a.Carried = link.KindRecordedBuilds, []string{"resolver from r1111111 → r2222222"}
|
||||
return a
|
||||
}
|
||||
f = calm(now)
|
||||
f.handActs = []link.HandAct{ofRecorded(now.Add(-26 * time.Hour)), ofRecorded(now.Add(-time.Hour)),
|
||||
ofRecorded(now.Add(-time.Minute))}
|
||||
if got := watchHandActs(f); len(got) != 0 {
|
||||
t.Errorf("pushes of recorded builds repeated asked for a healer: %+v", got)
|
||||
}
|
||||
f = calm(now)
|
||||
f.handActs = []link.HandAct{pushed("a", "split-dns", now.Add(-time.Hour)), pushed("b", "split-dns", now.Add(-time.Minute))}
|
||||
if got := watchHandActs(f); len(got) != 1 {
|
||||
t.Errorf("pushes that said no kind were not counted: %+v", got)
|
||||
}
|
||||
other := actOf(now.Add(-time.Hour), "plans close", "split-dns")
|
||||
other.ID = "act-1791404241010309198-1"
|
||||
f = calm(now)
|
||||
f.handActs = []link.HandAct{other, actOf(now.Add(-time.Minute), "plans close", "split-dns")}
|
||||
if got := watchHandActs(f); len(got) != 1 {
|
||||
t.Errorf("another verb under such a push's ID passed for one: %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,443 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// What the records say the registries may keep, asked on demand (novox/hq ADR 0251, to-be 51).
|
||||
//
|
||||
// Three commands, each a verb of the mesh-controller seat:
|
||||
//
|
||||
// - `artifacts` — every reference the mesh recorded making, with its state: what the store's own
|
||||
// tools set beside what the store holds;
|
||||
// - `collect` — the sweep that runs after a build, run when a person asks, a dry run without
|
||||
// --confirm;
|
||||
// - `images` — every container image a machine's declaration names, now and as last sent: what
|
||||
// a machine keeps when it prunes its images.
|
||||
|
||||
// artifactEntry is one recorded reference as `artifacts` answers it. Compact: the answer carries every
|
||||
// kept and eligible reference the mesh has, in one bus message.
|
||||
type artifactEntry struct {
|
||||
Reference string `json:"reference"`
|
||||
Repository string `json:"repository"`
|
||||
Digest string `json:"digest"`
|
||||
Kind string `json:"kind"`
|
||||
State string `json:"state"`
|
||||
Why []string `json:"why,omitempty"`
|
||||
}
|
||||
|
||||
type artifactCounts struct {
|
||||
Kept int `json:"kept"`
|
||||
Eligible int `json:"eligible"`
|
||||
Collected int `json:"collected"`
|
||||
}
|
||||
|
||||
type artifactsAnswer struct {
|
||||
Store string `json:"store"`
|
||||
KeptBuilds int `json:"kept_builds"`
|
||||
Counts artifactCounts `json:"counts"`
|
||||
References []artifactEntry `json:"references"`
|
||||
}
|
||||
|
||||
// artifactEntryOf reads a recorded reference into its repository, digest and kind; false for one that
|
||||
// is not into the mesh's artifact store or names nothing by digest there.
|
||||
func artifactEntryOf(s inventory.ArtifactState) (artifactEntry, bool) {
|
||||
path, ours := catalogue.InArtifactStore(s.Reference)
|
||||
if !ours {
|
||||
return artifactEntry{}, false
|
||||
}
|
||||
e := artifactEntry{Reference: s.Reference, State: s.State, Why: s.Why}
|
||||
if repository, digest, ok := strings.Cut(path, "@sha256:"); ok {
|
||||
e.Repository, e.Digest, e.Kind = repository, "sha256:"+digest, "image"
|
||||
return e, true
|
||||
}
|
||||
if repository, digest, ok := strings.Cut(path, "/blobs/sha256:"); ok {
|
||||
e.Repository, e.Digest, e.Kind = repository, "sha256:"+digest, "archive"
|
||||
return e, true
|
||||
}
|
||||
return artifactEntry{}, false
|
||||
}
|
||||
|
||||
// artifactsOf is the answer for these states: counted whole, listed narrowed.
|
||||
//
|
||||
// **Collected references are listed only when asked for** (novox/hq ADR 0251, issue 314). They are
|
||||
// history: every artifact the mesh ever let go stays collected for ever, so their number only grows,
|
||||
// and a verb's answer is one bus message — an answer that outgrows it is lost on its way back. Kept
|
||||
// and eligible are bounded by what the mesh holds now; the collected are counted, always.
|
||||
func artifactsOf(states []inventory.ArtifactState, repository string, withCollected bool) artifactsAnswer {
|
||||
a := artifactsAnswer{KeptBuilds: inventory.KeptBuilds, References: []artifactEntry{}}
|
||||
for _, s := range states {
|
||||
e, ours := artifactEntryOf(s)
|
||||
if !ours {
|
||||
continue
|
||||
}
|
||||
switch s.State {
|
||||
case inventory.ArtifactKept:
|
||||
a.Counts.Kept++
|
||||
case inventory.ArtifactEligible:
|
||||
a.Counts.Eligible++
|
||||
case inventory.ArtifactCollected:
|
||||
a.Counts.Collected++
|
||||
}
|
||||
if repository != "" && e.Repository != repository {
|
||||
continue
|
||||
}
|
||||
if s.State == inventory.ArtifactCollected && !withCollected {
|
||||
continue
|
||||
}
|
||||
a.References = append(a.References, e)
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
func artifactsCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("artifacts", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "answer as JSON")
|
||||
repository := set.String("repository", "", "only this repository's references (<module>/<artifact>); the counts stay whole")
|
||||
collected := set.Bool("collected", false, "list the collected references too")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 0 {
|
||||
return errors.New("artifacts [--json] [--repository <module>/<artifact>] [--collected]")
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
states, err := open.inventory.Artifacts(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
answer := artifactsOf(states, strings.TrimSpace(*repository), *collected)
|
||||
if shelf, err := open.inventory.Catalogue(ctx); err == nil {
|
||||
answer.Store, _ = artifactStoreAddress(ctx, open.inventory, shelf, "")
|
||||
}
|
||||
if *asJSON {
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
encoder.SetIndent("", " ")
|
||||
return encoder.Encode(answer)
|
||||
}
|
||||
fmt.Printf("artifact store %s\nkept %d\neligible %d\ncollected %d\n",
|
||||
storeOrNone(answer.Store), answer.Counts.Kept, answer.Counts.Eligible, answer.Counts.Collected)
|
||||
for _, e := range answer.References {
|
||||
why := ""
|
||||
if len(e.Why) > 0 {
|
||||
why = " (" + strings.Join(e.Why, ", ") + ")"
|
||||
}
|
||||
fmt.Printf(" %-9s %s%s\n", e.State, e.Reference, why)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func storeOrNone(s string) string {
|
||||
if s == "" {
|
||||
return "(not on the network)"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// collectAnswer is what `collect` did, or — a dry run — would do.
|
||||
type collectAnswer struct {
|
||||
DryRun bool `json:"dry_run"`
|
||||
Store string `json:"store"`
|
||||
KeptArchives int `json:"kept_archives"`
|
||||
// Held is how many kept archives the store holds by a manifest: asked with HEADs in a dry run,
|
||||
// after the holding in a real one.
|
||||
Held int `json:"held"`
|
||||
Unheld int `json:"unheld"`
|
||||
HoldersWritten int `json:"holders_written"`
|
||||
Missing int `json:"missing"`
|
||||
Eligible int `json:"eligible"`
|
||||
// WouldLetGo is the eligible references, oldest first, at most mostListed of them.
|
||||
WouldLetGo []string `json:"would_let_go"`
|
||||
NotListed int `json:"not_listed,omitempty"`
|
||||
LetGo []string `json:"let_go"`
|
||||
Skipped int `json:"skipped"`
|
||||
Left int `json:"left"`
|
||||
Stopped string `json:"stopped,omitempty"`
|
||||
}
|
||||
|
||||
// mostListed is how many references `collect` lists by name; the rest are counted.
|
||||
const mostListed = 1000
|
||||
|
||||
// onDemand are the bounds of a sweep a person asks for: more than after a build, inside a module's
|
||||
// sixty-second ask (novox/hq ADR 0251).
|
||||
const (
|
||||
collectMostDefault = 500
|
||||
collectMostAtMost = 5000
|
||||
collectBudget = 45 * time.Second
|
||||
)
|
||||
|
||||
// causeCollect is the cause `collect` records when none is given.
|
||||
const causeCollect = "collect-on-demand"
|
||||
|
||||
func collectCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("collect", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "answer as JSON")
|
||||
confirm := set.Bool("confirm", false, "let go of what is eligible, rather than only say what would go")
|
||||
most := set.Int("most", collectMostDefault, "let go of at most this many")
|
||||
f := addHandActFlags(set)
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 0 {
|
||||
return errors.New("collect [--json] [--confirm --why <text>] [--most <n>]")
|
||||
}
|
||||
if *confirm {
|
||||
// A deletion a person asks for says why, before anything is asked of the store.
|
||||
if err := f.require("collect"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if *most < 1 {
|
||||
return fmt.Errorf("collect: --most is at least 1, not %d", *most)
|
||||
}
|
||||
bounds := sweepBounds{most: min(*most, collectMostAtMost), budget: collectBudget, platforms: true}
|
||||
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
references, err := inv.ToCollect(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
kept, err := inv.KeptArchives(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
address, err := artifactStoreAddress(ctx, inv, shelf, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *confirm {
|
||||
if strings.TrimSpace(*f.cause) == "" {
|
||||
*f.cause = causeCollect
|
||||
}
|
||||
f.record(ctx, "collect", []string{"--most", fmt.Sprint(bounds.most)})
|
||||
}
|
||||
answer := runCollect(ctx, inv, artifacts.Store{Address: address}, references, kept, *confirm, bounds)
|
||||
if *asJSON {
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
encoder.SetIndent("", " ")
|
||||
return encoder.Encode(answer)
|
||||
}
|
||||
printCollect(answer)
|
||||
return nil
|
||||
}
|
||||
|
||||
// runCollect is `collect` against a store: a dry run asks only HEADs, a real one is the sweep.
|
||||
func runCollect(ctx context.Context, inv *inventory.Inventory, store artifacts.Store, references, kept []string,
|
||||
real bool, bounds sweepBounds) collectAnswer {
|
||||
a := collectAnswer{DryRun: !real, Store: store.Address, KeptArchives: len(kept), Eligible: len(references),
|
||||
WouldLetGo: []string{}, LetGo: []string{}}
|
||||
listed := references
|
||||
if len(listed) > mostListed {
|
||||
listed, a.NotListed = listed[:mostListed], len(listed)-mostListed
|
||||
}
|
||||
a.WouldLetGo = append(a.WouldLetGo, listed...)
|
||||
if store.Address == "" {
|
||||
a.Stopped = "this mesh has no artifact store on its network: nothing was asked of it"
|
||||
a.Left = len(references)
|
||||
return a
|
||||
}
|
||||
if !real {
|
||||
// Reads only: each kept archive asked about with HEADs, as `collection` does.
|
||||
report := collectionReport{Unheld: []string{}, Missing: []string{}}
|
||||
unasked, stopped := askHeld(ctx, store, kept, &report)
|
||||
a.Held, a.Unheld, a.Missing = report.Held, len(report.Unheld), len(report.Missing)
|
||||
if unasked > 0 {
|
||||
a.Stopped = fmt.Sprintf("%d kept archive(s) not asked about: %s", unasked, stopped)
|
||||
}
|
||||
return a
|
||||
}
|
||||
r := sweep(ctx, inv, store, references, kept, bounds)
|
||||
a.HoldersWritten, a.Missing, a.Skipped, a.Left, a.Stopped = r.HoldersWritten, r.Missing, r.Skipped, r.Left, r.Stopped
|
||||
if r.Held {
|
||||
a.Held = len(kept) - r.Missing
|
||||
}
|
||||
a.LetGo = append(a.LetGo, r.LetGo...)
|
||||
return a
|
||||
}
|
||||
|
||||
func printCollect(a collectAnswer) {
|
||||
if a.DryRun {
|
||||
fmt.Println("a dry run: nothing was held and nothing was let go (--confirm --why <text> to collect)")
|
||||
}
|
||||
fmt.Printf("artifact store %s\n", storeOrNone(a.Store))
|
||||
fmt.Printf("kept archives %d (held %d, unheld %d, missing %d)\n", a.KeptArchives, a.Held, a.Unheld, a.Missing)
|
||||
if a.HoldersWritten > 0 {
|
||||
fmt.Printf("holders written %d\n", a.HoldersWritten)
|
||||
}
|
||||
fmt.Printf("eligible to let go %d\n", a.Eligible)
|
||||
if !a.DryRun {
|
||||
fmt.Printf("let go %d (skipped %d, left %d)\n", len(a.LetGo), a.Skipped, a.Left)
|
||||
}
|
||||
if a.Stopped != "" {
|
||||
fmt.Printf("stopped: %s\n", a.Stopped)
|
||||
}
|
||||
for _, reference := range a.WouldLetGo {
|
||||
fmt.Printf(" %s\n", reference)
|
||||
}
|
||||
if a.NotListed > 0 {
|
||||
fmt.Printf(" … and %d more not listed\n", a.NotListed)
|
||||
}
|
||||
}
|
||||
|
||||
// declaredImage is one image a machine's declaration names.
|
||||
type declaredImage struct {
|
||||
Image string `json:"image"`
|
||||
Resources []string `json:"resources"`
|
||||
// In says which declarations name it: "declaration" (what the mesh would send now), "sent" (what
|
||||
// the machine was last sent).
|
||||
In []string `json:"in"`
|
||||
}
|
||||
|
||||
type imagesAnswer struct {
|
||||
Node string `json:"node"`
|
||||
Images []declaredImage `json:"images"`
|
||||
// SentKnown is whether what the machine was last sent is kept with its images; false when it is
|
||||
// not kept, or was kept before the images were.
|
||||
SentKnown bool `json:"sent_known"`
|
||||
}
|
||||
|
||||
// imagesOf is every container image of the declaration about to be sent and of the summary of the one
|
||||
// last sent.
|
||||
func imagesOf(node string, body []byte, sent []sentResource, sentKept bool) (imagesAnswer, error) {
|
||||
a := imagesAnswer{Node: node, Images: []declaredImage{}, SentKnown: sentKept}
|
||||
byImage := map[string]*declaredImage{}
|
||||
add := func(image, resource, in string) {
|
||||
if image == "" {
|
||||
return
|
||||
}
|
||||
d := byImage[image]
|
||||
if d == nil {
|
||||
d = &declaredImage{Image: image}
|
||||
byImage[image] = d
|
||||
}
|
||||
if !slices.Contains(d.Resources, resource) {
|
||||
d.Resources = append(d.Resources, resource)
|
||||
}
|
||||
if !slices.Contains(d.In, in) {
|
||||
d.In = append(d.In, in)
|
||||
}
|
||||
}
|
||||
if body != nil {
|
||||
var envelope struct {
|
||||
Resources []map[string]any `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &envelope); err != nil {
|
||||
return a, fmt.Errorf("a declaration that is not one: %w", err)
|
||||
}
|
||||
for _, r := range envelope.Resources {
|
||||
if r["type"] != "container" {
|
||||
continue
|
||||
}
|
||||
image, _ := r["image"].(string)
|
||||
add(image, fmt.Sprint(r["id"]), "declaration")
|
||||
}
|
||||
}
|
||||
for _, s := range sent {
|
||||
if s.Type != "container" {
|
||||
continue
|
||||
}
|
||||
if s.Image == "" {
|
||||
// Kept before the summary carried images: what was sent cannot be said.
|
||||
a.SentKnown = false
|
||||
continue
|
||||
}
|
||||
add(s.Image, s.ID, "sent")
|
||||
}
|
||||
for _, d := range byImage {
|
||||
sort.Strings(d.Resources)
|
||||
a.Images = append(a.Images, *d)
|
||||
}
|
||||
sort.Slice(a.Images, func(i, j int) bool { return a.Images[i].Image < a.Images[j].Image })
|
||||
return a, nil
|
||||
}
|
||||
|
||||
func imagesCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("images", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "answer as JSON")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("images <node> [--json]")
|
||||
}
|
||||
node := positionals[0]
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
if _, err := open.inventory.NodeByName(ctx, node); err != nil {
|
||||
return fmt.Errorf("%s: %w", node, err)
|
||||
}
|
||||
// What the mesh would send now, composed as `plan` composes it.
|
||||
var body []byte
|
||||
plan, settings, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(plan.Modules) > 0 {
|
||||
declared, err := declarationFor(ctx, open, node, plan, settings)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if body, err = declared.Body(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// And what it was last sent, as `plan --diff` compares against.
|
||||
prev, err := open.inventory.SentSummary(ctx, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var sent []sentResource
|
||||
if len(prev) > 0 {
|
||||
if err := json.Unmarshal(prev, &sent); err != nil {
|
||||
return fmt.Errorf("%s: what it was last sent is kept in a form this version does not read: %w", node, err)
|
||||
}
|
||||
}
|
||||
answer, err := imagesOf(node, body, sent, len(prev) > 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
encoder.SetIndent("", " ")
|
||||
return encoder.Encode(answer)
|
||||
}
|
||||
for _, d := range answer.Images {
|
||||
fmt.Printf("%s (%s; %s)\n", d.Image, strings.Join(d.In, ", "), strings.Join(d.Resources, ", "))
|
||||
}
|
||||
if !answer.SentKnown {
|
||||
fmt.Printf("%s: which images it was last sent is not kept — the next push keeps it\n", node)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,303 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// What the records say the registries may keep, asked on demand (novox/hq ADR 0251).
|
||||
|
||||
func TestTheRegistryVerbsComposeTheirCommandsAndRefuseWhatTheyDoNotTake(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
want string
|
||||
}{
|
||||
{"artifacts", map[string]any{}, "artifacts --json"},
|
||||
{"artifacts", map[string]any{"repository": "web/app", "collected": "true"},
|
||||
"artifacts --json --repository web/app --collected"},
|
||||
{"artifacts", map[string]any{"collected": "false"}, "artifacts --json"},
|
||||
{"collect", map[string]any{}, "collect --json"},
|
||||
{"collect", map[string]any{"most": "50"}, "collect --json --most 50"},
|
||||
{"collect", map[string]any{"confirm": "true", "why": "the disk is full"},
|
||||
"collect --json --confirm --why the disk is full"},
|
||||
{"collect", map[string]any{"confirm": true, "why": "x", "most": "10"}, "collect --json --most 10 --confirm --why x"},
|
||||
{"images", map[string]any{"node": "anchor"}, "images anchor --json"},
|
||||
} {
|
||||
argv, err := argvFor(c.verb, c.args)
|
||||
if err != nil || strings.Join(argv, " ") != c.want {
|
||||
t.Errorf("%s %v: %q %v, want %q", c.verb, c.args, argv, err, c.want)
|
||||
}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
}{
|
||||
{"collect", map[string]any{"confirm": "true"}}, // a deletion without a why
|
||||
{"collect", map[string]any{"why": "tidy"}}, // a why for a dry run, recorded nowhere
|
||||
{"collect", map[string]any{"confirm": "yes", "why": "x"}}, // a switch is true or false
|
||||
{"collect", map[string]any{"force": "true"}},
|
||||
{"artifacts", map[string]any{"node": "anchor"}},
|
||||
{"images", map[string]any{}},
|
||||
{"images", map[string]any{"node": "anchor", "all": "true"}},
|
||||
} {
|
||||
if argv, err := argvFor(c.verb, c.args); err == nil {
|
||||
t.Errorf("%s %v was composed as %q", c.verb, c.args, argv)
|
||||
}
|
||||
}
|
||||
if repairingCommand([]string{"collect", "--json", "--confirm", "--why", "x"}) != "collect" {
|
||||
t.Error("a real collect through the generic verb would go unrecorded")
|
||||
}
|
||||
if repairingCommand([]string{"collect", "--json"}) != "" {
|
||||
t.Error("a dry run was taken for an act by hand")
|
||||
}
|
||||
if !personsDecision(link.HandAct{Verb: "collect"}) {
|
||||
t.Error("a collect a person asked for would count toward a healer the mesh lacks")
|
||||
}
|
||||
}
|
||||
|
||||
func ref(module, artifact string, n int) string {
|
||||
return fmt.Sprintf("%s%s/%s@sha256:%064x", catalogue.ArtifactStoreScheme, module, artifact, n)
|
||||
}
|
||||
|
||||
func archiveRef(module, artifact string, n int) string {
|
||||
return fmt.Sprintf("%s%s/%s/blobs/sha256:%064x", catalogue.ArtifactStoreScheme, module, artifact, n)
|
||||
}
|
||||
|
||||
func TestArtifactsCountsEverythingAndListsTheCollectedOnlyWhenAsked(t *testing.T) {
|
||||
states := []inventory.ArtifactState{
|
||||
{Reference: ref("web", "app", 1), State: inventory.ArtifactCollected},
|
||||
{Reference: ref("web", "app", 2), State: inventory.ArtifactEligible},
|
||||
{Reference: archiveRef("web", "tools", 3), State: inventory.ArtifactKept,
|
||||
Why: []string{inventory.KeptByRecentBuild, inventory.KeptByDefinition}},
|
||||
{Reference: ref("db", "server", 4), State: inventory.ArtifactKept, Why: []string{inventory.KeptByDefinition}},
|
||||
// Not the mesh's own store: never listed, never counted.
|
||||
{Reference: "registry.invalid/x@sha256:" + strings.Repeat("a", 64), State: inventory.ArtifactKept,
|
||||
Why: []string{inventory.KeptUnaddressable}},
|
||||
}
|
||||
a := artifactsOf(states, "", false)
|
||||
if a.Counts != (artifactCounts{Kept: 2, Eligible: 1, Collected: 1}) {
|
||||
t.Errorf("counts %+v", a.Counts)
|
||||
}
|
||||
if len(a.References) != 3 {
|
||||
t.Fatalf("listed %d, want the kept and the eligible: %+v", len(a.References), a.References)
|
||||
}
|
||||
archive := a.References[1]
|
||||
if archive.Kind != "archive" || archive.Repository != "web/tools" || archive.Digest != fmt.Sprintf("sha256:%064x", 3) ||
|
||||
!slices.Equal(archive.Why, []string{"recent-build", "definition"}) {
|
||||
t.Errorf("an archive read as %+v", archive)
|
||||
}
|
||||
if image := a.References[0]; image.Kind != "image" || image.Repository != "web/app" || image.State != "eligible" {
|
||||
t.Errorf("an image read as %+v", image)
|
||||
}
|
||||
narrowed := artifactsOf(states, "web/app", true)
|
||||
if narrowed.Counts != a.Counts || len(narrowed.References) != 2 {
|
||||
t.Errorf("narrowed to one repository: %+v", narrowed)
|
||||
}
|
||||
raw, _ := json.Marshal(a.References[0])
|
||||
if strings.Contains(string(raw), `"why"`) {
|
||||
t.Errorf("an empty why is carried: %s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeStore answers as a registry does for the sweep's questions, and records every request.
|
||||
type fakeStore struct {
|
||||
mu sync.Mutex
|
||||
requests []string
|
||||
refuse string // a DELETE whose path contains this is refused
|
||||
}
|
||||
|
||||
func (f *fakeStore) serve(t *testing.T) artifacts.Store {
|
||||
t.Helper()
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
f.mu.Lock()
|
||||
f.requests = append(f.requests, r.Method+" "+r.URL.Path)
|
||||
refuse := f.refuse
|
||||
f.mu.Unlock()
|
||||
switch {
|
||||
case r.Method == http.MethodHead && strings.Contains(r.URL.Path, "/blobs/"):
|
||||
w.Header().Set("Content-Length", "10")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case r.Method == http.MethodHead:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/manifests/"):
|
||||
// An image, not an index: it names no platform manifests.
|
||||
w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
|
||||
_, _ = w.Write([]byte(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","layers":[]}`))
|
||||
case r.Method == http.MethodPost:
|
||||
w.Header().Set("Location", "/upload/x?state=1")
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
case r.Method == http.MethodPut:
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
case r.Method == http.MethodDelete && refuse != "" && strings.Contains(r.URL.Path, refuse):
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
case r.Method == http.MethodDelete:
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
default:
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
return artifacts.Store{Address: strings.TrimPrefix(server.URL, "http://")}
|
||||
}
|
||||
|
||||
func (f *fakeStore) seen() []string {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return slices.Clone(f.requests)
|
||||
}
|
||||
|
||||
func TestADryRunCollectAsksOnlyAndChangesNothing(t *testing.T) {
|
||||
f := &fakeStore{}
|
||||
store := f.serve(t)
|
||||
kept := []string{archiveRef("web", "tools", 1)}
|
||||
eligible := []string{ref("web", "app", 2), archiveRef("web", "tools", 3)}
|
||||
a := runCollect(context.Background(), nil, store, eligible, kept, false,
|
||||
sweepBounds{most: 10, budget: 5 * time.Second})
|
||||
if !a.DryRun || a.Eligible != 2 || !slices.Equal(a.WouldLetGo, eligible) || len(a.LetGo) != 0 {
|
||||
t.Errorf("a dry run answered %+v", a)
|
||||
}
|
||||
if a.KeptArchives != 1 || a.Held != 0 || a.Unheld != 1 {
|
||||
t.Errorf("the kept archive is unheld in the fake store, and was said as %+v", a)
|
||||
}
|
||||
for _, r := range f.seen() {
|
||||
if !strings.HasPrefix(r, "HEAD ") {
|
||||
t.Errorf("a dry run asked the store %s", r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectSaysWhatItDoesNotListAndWithNoStoreAsksNothing(t *testing.T) {
|
||||
var many []string
|
||||
for i := range mostListed + 5 {
|
||||
many = append(many, ref("web", "app", i))
|
||||
}
|
||||
a := runCollect(context.Background(), nil, artifacts.Store{}, many, nil, false, sweepBounds{most: 1, budget: time.Second})
|
||||
if len(a.WouldLetGo) != mostListed || a.NotListed != 5 || a.Left != len(many) || a.Stopped == "" {
|
||||
t.Errorf("with no store and %d eligible: %d listed, %d not, left %d, stopped %q",
|
||||
len(many), len(a.WouldLetGo), a.NotListed, a.Left, a.Stopped)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARealCollectHoldsEveryKeptArchiveBeforeItLetsAnythingGo(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
f := &fakeStore{}
|
||||
store := f.serve(t)
|
||||
kept := []string{archiveRef("web", "tools", 1)}
|
||||
eligible := []string{ref("web", "app", 2), ref("web", "app", 3), ref("web", "app", 4)}
|
||||
madeBy(t, inv, eligible)
|
||||
a := runCollect(t.Context(), inv, store, eligible, kept, true, sweepBounds{most: 2, budget: 5 * time.Second})
|
||||
if a.DryRun || a.HoldersWritten != 1 || a.Held != 1 {
|
||||
t.Errorf("the kept archive was not held first: %+v", a)
|
||||
}
|
||||
if !slices.Equal(a.LetGo, eligible[:2]) || a.Left != 1 || a.Stopped == "" {
|
||||
t.Errorf("bounded at two: let go %v, left %d, stopped %q", a.LetGo, a.Left, a.Stopped)
|
||||
}
|
||||
requests := f.seen()
|
||||
firstPut := slices.IndexFunc(requests, func(r string) bool { return strings.HasPrefix(r, "PUT ") && strings.Contains(r, "/manifests/") })
|
||||
firstDelete := slices.IndexFunc(requests, func(r string) bool { return strings.HasPrefix(r, "DELETE ") })
|
||||
if firstPut < 0 || firstDelete < 0 || firstPut > firstDelete {
|
||||
t.Errorf("the holder was not put before the first delete: %v", requests)
|
||||
}
|
||||
// What was let go is recorded collected: the sweep no longer offers it.
|
||||
left, err := inv.ToCollect(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Equal(left, eligible[2:]) {
|
||||
t.Errorf("after letting go of two, the records offer %v", left)
|
||||
}
|
||||
}
|
||||
|
||||
// madeBy records one successful build, of a module the mesh does not hold, that made these images:
|
||||
// eligible, every one.
|
||||
func madeBy(t *testing.T, inv *inventory.Inventory, references []string) {
|
||||
t.Helper()
|
||||
b := inventory.Build{ID: "b1", Repository: "https://forge.invalid/web.git", Module: "web", On: "a-build-machine",
|
||||
Commit: "c0ffee"}
|
||||
for i, r := range references {
|
||||
b.Made = append(b.Made, inventory.Artifact{Name: fmt.Sprintf("app%d", i), Kind: "image", Reference: r})
|
||||
}
|
||||
if err := inv.RecordBuild(t.Context(), b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARealCollectStopsAtTheStoresFirstRefusal(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
f := &fakeStore{refuse: fmt.Sprintf("%064x", 3)}
|
||||
store := f.serve(t)
|
||||
eligible := []string{ref("web", "app", 2), ref("web", "app", 3), ref("web", "app", 4)}
|
||||
a := runCollect(t.Context(), inv, store, eligible, nil, true, sweepBounds{most: 10, budget: 5 * time.Second})
|
||||
if !slices.Equal(a.LetGo, eligible[:1]) || a.Left != 2 || !strings.Contains(a.Stopped, "kept") {
|
||||
t.Errorf("after a refusal: let go %v, left %d, stopped %q", a.LetGo, a.Left, a.Stopped)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImagesAreEveryContainerImageOfTheDeclarationAndOfWhatWasSent(t *testing.T) {
|
||||
body := []byte(`{"declaration":1,"resources":[
|
||||
{"id":"web.server","type":"container","image":"store.invalid/web/server@sha256:aa"},
|
||||
{"id":"web.collect","type":"container","image":"store.invalid/web/server@sha256:aa","schedule":"30 3 * * *"},
|
||||
{"id":"db.server","type":"container","image":"postgres@sha256:bb"},
|
||||
{"id":"web.bundle-tools","type":"archive","source":"http://store.invalid/v2/web/tools/blobs/sha256:cc"}]}`)
|
||||
sent := []sentResource{
|
||||
{ID: "web.server", Type: "container", Image: "store.invalid/web/server@sha256:99"},
|
||||
{ID: "db.server", Type: "container", Image: "postgres@sha256:bb"},
|
||||
{ID: "web.state", Type: "directory"},
|
||||
}
|
||||
a, err := imagesOf("anchor", body, sent, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !a.SentKnown || len(a.Images) != 3 {
|
||||
t.Fatalf("answered %+v", a)
|
||||
}
|
||||
byImage := map[string]declaredImage{}
|
||||
for _, d := range a.Images {
|
||||
byImage[d.Image] = d
|
||||
}
|
||||
if d := byImage["store.invalid/web/server@sha256:aa"]; !slices.Equal(d.Resources, []string{"web.collect", "web.server"}) ||
|
||||
!slices.Equal(d.In, []string{"declaration"}) {
|
||||
t.Errorf("a scheduled step's image and its server's are one: %+v", d)
|
||||
}
|
||||
if d := byImage["store.invalid/web/server@sha256:99"]; !slices.Equal(d.In, []string{"sent"}) {
|
||||
t.Errorf("the image last sent is kept beside the one to send: %+v", d)
|
||||
}
|
||||
if d := byImage["postgres@sha256:bb"]; !slices.Equal(d.In, []string{"declaration", "sent"}) {
|
||||
t.Errorf("an image in both: %+v", d)
|
||||
}
|
||||
|
||||
// A summary kept before it carried images cannot say what was sent.
|
||||
old, err := imagesOf("anchor", body, []sentResource{{ID: "web.server", Type: "container"}}, true)
|
||||
if err != nil || old.SentKnown {
|
||||
t.Errorf("an old summary was read as knowing what was sent: %+v %v", old, err)
|
||||
}
|
||||
none, err := imagesOf("anchor", body, nil, false)
|
||||
if err != nil || none.SentKnown || len(none.Images) != 2 {
|
||||
t.Errorf("with nothing sent yet: %+v %v", none, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSentSummaryKeepsAContainersImage(t *testing.T) {
|
||||
summary, err := summarize([]byte(`{"resources":[{"id":"web.server","type":"container","image":"x@sha256:aa"},
|
||||
{"id":"web.state","type":"directory","path":"/srv"}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if summary[0].Image != "x@sha256:aa" || summary[1].Image != "" {
|
||||
t.Errorf("summarized as %+v", summary)
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -246,7 +247,7 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
|
||||
}
|
||||
}
|
||||
if len(refused) > 0 {
|
||||
return nil, fmt.Errorf("%w: %s — a release plan sends them, one machine at a time, each judged (novox/hq ADR "+
|
||||
return nil, fmt.Errorf("%w: %s — a walk sends them, one machine at a time, each judged (novox/hq ADR "+
|
||||
"0236); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; "))
|
||||
}
|
||||
return kept, nil
|
||||
@@ -318,7 +319,7 @@ func passCarried(ctx context.Context, open *stores, p *inventory.Plan, g *invent
|
||||
}
|
||||
seen[c.Build] = true
|
||||
if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: c.Build, Module: c.Module, Commit: c.To,
|
||||
Previous: c.From, Plan: p.ID, Machines: []string{c.Node}, Verdict: inventory.GatePassed, Why: g.Why,
|
||||
Previous: c.From, Plan: p.ID, Machines: []string{c.Node}, Verdict: inventory.GatePassed, Why: whyFor(g, c.Module),
|
||||
Component: coreComponent(c.Module), JudgingFrom: g.Since}); err != nil {
|
||||
fmt.Printf("%s: %s passed its gate on %s, and the verdict could not be kept: %v\n", p.ID, c.Module, c.Node, err)
|
||||
}
|
||||
@@ -333,6 +334,18 @@ func failCarried(ctx context.Context, open *stores, p *inventory.Plan, g *invent
|
||||
notes = append(notes, p.Note)
|
||||
}
|
||||
done := map[string]bool{except: true}
|
||||
for _, m := range g.Returned {
|
||||
done[m] = true // put back at once when it broke
|
||||
}
|
||||
// **What passed on its own keeps its pass** (novox/hq ADR 0254, issue 318): healthy for the passes the
|
||||
// gate asks while another module of the send failed, it is neither put back nor left unjudged — a build
|
||||
// left without a verdict is one more move every other walk would wait for.
|
||||
if kept := keepPassing(ctx, open, p, g, except); len(kept) > 0 {
|
||||
notes = append(notes, "passed on their own and kept: "+strings.Join(kept, ", "))
|
||||
for _, m := range kept {
|
||||
done[m] = true
|
||||
}
|
||||
}
|
||||
for _, c := range g.Carried {
|
||||
if done[c.Module] || (len(g.Failing) > 0 && !slices.Contains(g.Failing, c.Module)) {
|
||||
continue
|
||||
@@ -355,6 +368,100 @@ func failCarried(ctx context.Context, open *stores, p *inventory.Plan, g *invent
|
||||
p.Note = strings.Join(notes, "; ")
|
||||
}
|
||||
|
||||
// keepPassing keeps a pass as the verdict of every build the gate carried that passed on its own when the
|
||||
// send failed (ADR 0254), except the plan's own module; answers the modules kept.
|
||||
func keepPassing(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) []string {
|
||||
var kept []string
|
||||
seen := map[string]bool{}
|
||||
for _, c := range g.Carried {
|
||||
if c.Module == except || c.Build == "" || seen[c.Build] || !slices.Contains(g.Passing, c.Module) {
|
||||
continue
|
||||
}
|
||||
seen[c.Build] = true
|
||||
why := passedAloneWhy(g, c.Module)
|
||||
if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: c.Build, Module: c.Module, Commit: c.To,
|
||||
Previous: c.From, Plan: p.ID, Machines: []string{c.Node}, Verdict: inventory.GatePassed, Why: why,
|
||||
Component: coreComponent(c.Module), JudgingFrom: g.Since}); err != nil {
|
||||
fmt.Printf("%s: %s passed its gate on %s on its own, and the verdict could not be kept: %v\n", p.ID, c.Module,
|
||||
c.Node, err)
|
||||
continue
|
||||
}
|
||||
if !slices.Contains(kept, c.Module) {
|
||||
kept = append(kept, c.Module)
|
||||
}
|
||||
}
|
||||
return kept
|
||||
}
|
||||
|
||||
// putBackBroken puts back, at once, every module a gate found broken and has not put back yet (issue 318
|
||||
// review): a witness that reverted a core component, or a machine that refused or failed its send, is not
|
||||
// left registered at the build that broke for as long as the modules beside it take to be judged — a resend
|
||||
// meanwhile would declare it again. The send goes on judging the others to their own verdicts; its own
|
||||
// verdict is still failed. lead and leadState are the module a plan's gate is kept on, when it is a plan's.
|
||||
// Answers whether anything was put back.
|
||||
func putBackBroken(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, lead string,
|
||||
leadState *inventory.PlanModule) bool {
|
||||
var todo []string
|
||||
for _, m := range g.Broken {
|
||||
if !slices.Contains(g.Returned, m) {
|
||||
todo = append(todo, m)
|
||||
}
|
||||
}
|
||||
if len(todo) == 0 || g.Verdict != "" {
|
||||
return false
|
||||
}
|
||||
stateWas, noteWas := p.State, p.Note
|
||||
batched, back := batchingRollbacks(ctx)
|
||||
var said []string
|
||||
for _, m := range todo {
|
||||
machines := g.Machines
|
||||
var state *inventory.PlanModule
|
||||
if m == lead && leadState != nil {
|
||||
judged := *leadState
|
||||
judged.Gate = nil // its own record of the failure; the send's gate goes on judging
|
||||
state = &judged
|
||||
} else {
|
||||
i := slices.IndexFunc(g.Carried, func(c inventory.CarriedMove) bool { return c.Module == m })
|
||||
if i < 0 {
|
||||
continue
|
||||
}
|
||||
c := g.Carried[i]
|
||||
state = &inventory.PlanModule{Build: c.Build, Previous: c.From, Commit: c.To}
|
||||
if sent, err := sentTheBuild(ctx, open, m, c.To); err == nil && len(sent) > 0 {
|
||||
machines = sent
|
||||
} else {
|
||||
machines = []string{c.Node}
|
||||
}
|
||||
}
|
||||
// Counted as put back only once there is something to put back (a carried move or the plan's own).
|
||||
g.Returned = append(g.Returned, m)
|
||||
p.Note = ""
|
||||
gateFailed(batched, open, p, m, state, machines, g.BrokenWhy)
|
||||
if m == lead && leadState != nil {
|
||||
leadState.Why = "put back at once, its send still judged: " + g.BrokenWhy
|
||||
}
|
||||
said = append(said, m)
|
||||
}
|
||||
sendRollbacks(ctx, open, p, back)
|
||||
p.State = stateWas
|
||||
p.Note = noteWas
|
||||
if len(said) > 0 {
|
||||
p.Note = fmt.Sprintf("put back at once: %s (%s); the rest of the send judged to their own verdicts",
|
||||
strings.Join(said, ", "), g.BrokenWhy)
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// passedAloneWhy is the verdict of a module that passed on its own while its send failed, with its own wait.
|
||||
func passedAloneWhy(g *inventory.PlanGate, module string) string {
|
||||
why := fmt.Sprintf("healthy %d times on its own while the send failed: %s", g.Healthy[module], g.Why)
|
||||
if w := g.Waits[module]; w != "" {
|
||||
why += waitsPrefix + w
|
||||
}
|
||||
return why
|
||||
}
|
||||
|
||||
// sentTheBuild is every machine running a module that was last sent this build of it.
|
||||
func sentTheBuild(ctx context.Context, open *stores, module, commit string) ([]string, error) {
|
||||
running, err := open.inventory.Running(ctx, module)
|
||||
@@ -573,6 +680,9 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
||||
}
|
||||
switch verdict {
|
||||
case "":
|
||||
if putBackBroken(ctx, open, p, g, "", nil) {
|
||||
return true, nil
|
||||
}
|
||||
note := fmt.Sprintf("judging %s: %s", strings.Join(g.Machines, ", "), gateLine(g))
|
||||
changed := p.Note != note
|
||||
p.Note = note
|
||||
@@ -600,15 +710,27 @@ func backlogObservation() []conditions.Observation {
|
||||
}
|
||||
n := 0
|
||||
var machines []string
|
||||
seen := map[string]bool{}
|
||||
var modules []string
|
||||
for node, moves := range backlogNow.waiting {
|
||||
n += len(moves)
|
||||
machines = append(machines, node)
|
||||
for _, mv := range moves {
|
||||
if !seen[mv.Module] {
|
||||
seen[mv.Module] = true
|
||||
modules = append(modules, mv.Module)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(machines)
|
||||
return []conditions.Observation{{Scope: conditions.ScopeMesh, ID: "release", Token: "held", Kind: "release-held",
|
||||
Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
|
||||
sort.Strings(modules)
|
||||
o := conditions.Observation{Scope: conditions.ScopeMesh, ID: "release", Token: "held", Kind: "release-held",
|
||||
Severity: conditions.Warning, Resolver: conditions.ResolverOperator, Also: machines,
|
||||
Summary: fmt.Sprintf("%d build move(s) on %s wait for a gate and are not released: %s — `upgrade backlog` lists "+
|
||||
"them, `upgrade release-backlog --why …` releases them", n, strings.Join(machines, ", "), backlogNow.held)}}
|
||||
"them, `upgrade release-backlog --why …` releases them", n, strings.Join(machines, ", "), backlogNow.held)}
|
||||
w := releaseHeldWords(modules, machines)
|
||||
o.Headline, o.Explanation, o.Resolved, o.Needs = w.Headline, w.Explanation, w.Resolved, w.Needs
|
||||
return []conditions.Observation{o}
|
||||
}
|
||||
|
||||
// backlogCommand is `upgrade backlog`, read-only, and `upgrade release-backlog --why`.
|
||||
@@ -671,7 +793,7 @@ func backlogCommand(ctx context.Context, sub string, args []string) error {
|
||||
}
|
||||
|
||||
// sayRecreations says, on each move a send carries, what it does to the module's containers (novox/hq
|
||||
// ADR 0242): the build the machine ran against the one it is sent, container by container. Left unsaid
|
||||
// ADR 0245): the build the machine ran against the one it is sent, container by container. Left unsaid
|
||||
// for a move whose earlier build is not in the records.
|
||||
func sayRecreations(ctx context.Context, open *stores, moves []inventory.CarriedMove) {
|
||||
if len(moves) == 0 {
|
||||
@@ -694,6 +816,56 @@ func sayRecreations(ctx context.Context, open *stores, moves []inventory.Carried
|
||||
}
|
||||
}
|
||||
|
||||
// sayWhatAPushRecreates says, per machine a push is about to send, every module the send moves and what
|
||||
// the move recreates (novox/hq ADR 0245) — the same words a gated send keeps on its plan. A person's push
|
||||
// carries every module's new build, whatever its policy and whether or not a gate saw it, so it is the
|
||||
// send that most needs to say so: on 2026-10-07 a `push` of one machine moved mail to a new build and
|
||||
// recreated one of its containers with a new image, and the answer said nothing about it. A machine whose
|
||||
// last send's builds are not known is not said, and neither is a failure to read: the push goes on.
|
||||
func sayWhatAPushRecreates(ctx context.Context, open *stores, sending []readyNode, w io.Writer) {
|
||||
if len(sending) == 0 {
|
||||
return
|
||||
}
|
||||
f, err := readMoveFacts(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, r := range sending {
|
||||
sent, known, err := open.inventory.SentBuilds(ctx, r.node)
|
||||
if err != nil || !known {
|
||||
continue
|
||||
}
|
||||
moves := pushMoves(f, r.node, r.declared.Builds, sent)
|
||||
if len(moves) == 0 {
|
||||
continue
|
||||
}
|
||||
sayRecreations(ctx, open, moves)
|
||||
fmt.Fprintf(w, "%s moves %d module(s):\n", r.node, len(moves))
|
||||
for _, mv := range moves {
|
||||
said := mv.Recreates
|
||||
if said == "" {
|
||||
said = "recreates none of its containers, or what it ran is not in the records"
|
||||
}
|
||||
fmt.Fprintf(w, " %-28s %s → %s: %s\n", mv.Module, short(mv.From), short(mv.To), said)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// pushMoves is what a send carrying these builds moves on a machine last sent `sent`: every module it ran
|
||||
// before at a build not identical to the one it is now sent. A module new to the machine is not a move.
|
||||
func pushMoves(f moveFacts, node string, carries, sent map[string]string) []inventory.CarriedMove {
|
||||
var out []inventory.CarriedMove
|
||||
for m, to := range carries {
|
||||
was, ran := sent[m]
|
||||
if !ran || was == "" || to == "" || f.identical(m, was, to) {
|
||||
continue
|
||||
}
|
||||
out = append(out, inventory.CarriedMove{Module: m, Node: node, From: was, To: to})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module })
|
||||
return out
|
||||
}
|
||||
|
||||
// recreationsSaid is every recreation a send's moves say, joined: what a plan's note carries.
|
||||
func recreationsSaid(moves []inventory.CarriedMove) string {
|
||||
var said []string
|
||||
|
||||
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
@@ -156,7 +157,9 @@ func reachableFrom(moved []string, edges []inventory.Edge) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// hasCycle says whether the tiers' last tier holds modules that still depend on each other.
|
||||
// hasCycle says whether the tiers' last tier holds modules that still depend on each other. A packages
|
||||
// edge orders nothing (tiersOf), so a module and what packages its source share a tier by rule: that is
|
||||
// no cycle, and saying one was is a false report in every such plan's log.
|
||||
func hasCycle(tiers [][]string, edges []inventory.Edge) bool {
|
||||
if len(tiers) == 0 {
|
||||
return false
|
||||
@@ -166,6 +169,9 @@ func hasCycle(tiers [][]string, edges []inventory.Edge) bool {
|
||||
last[m] = true
|
||||
}
|
||||
for _, e := range edges {
|
||||
if e.Kind == inventory.EdgePackages {
|
||||
continue
|
||||
}
|
||||
if last[e.From] && last[e.To] {
|
||||
return true
|
||||
}
|
||||
@@ -336,8 +342,16 @@ func askTier(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan) e
|
||||
|
||||
// askABuild is how a plan asks for one build, not waited for, and learns the id it asked under. A
|
||||
// variable so a test of what a plan does around an ask needs no build machine.
|
||||
var askABuild = func(ctx context.Context, source buildSource, path, ref string) (string, error) {
|
||||
return buildOneAsked(ctx, source, path, ref, 0, false)
|
||||
//
|
||||
// Set in init, not where it is declared: an assignment pending on a build asks for one too (novox/hq issue
|
||||
// 325), and a build's outcome makes pending assignments, so the two refer to each other.
|
||||
var askABuild func(ctx context.Context, source buildSource, path, ref string) (string, error)
|
||||
|
||||
func init() {
|
||||
askABuild = func(ctx context.Context, source buildSource, path, ref string) (string, error) {
|
||||
// Kept by each asker with its own name once the ask is made (novox/hq issue 325).
|
||||
return buildOneAsked(ctx, source, path, ref, 0, false, "")
|
||||
}
|
||||
}
|
||||
|
||||
// askModule asks the build machine for one module of a plan and marks it asked, with the id it was
|
||||
@@ -368,6 +382,8 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
|
||||
p.Note = fmt.Sprintf("%s could not be asked for: %v", name, err)
|
||||
return
|
||||
}
|
||||
recordAsked(ctx, inventory.BuildRequest{ID: id, Repository: e.Source.Repository, Seat: e.Source.Seat,
|
||||
Path: e.Source.Path, Ref: followedBranch(e.Source.Ref), Commit: p.Commit, For: "plan"})
|
||||
state.State = "asked"
|
||||
state.AskedAt = &now
|
||||
state.Build = id
|
||||
@@ -506,23 +522,37 @@ func advanceHeld(ctx context.Context, open *stores) {
|
||||
for i := range plans {
|
||||
p := &plans[i]
|
||||
for p.Open() {
|
||||
// **Saved only when the step changed it** (novox/hq issue 296). Every step was saved, and
|
||||
// the steps run on every tick and after every build outcome on the mesh: a plan whose tier
|
||||
// stood still for half an hour was written every few seconds, its revision moved, `plan-moved`
|
||||
// said on the bus, and its last save read as when it began building.
|
||||
before := planSnapshot(*p)
|
||||
moved, err := advanceOnce(ctx, open, p, edges, rollsOut)
|
||||
if err != nil {
|
||||
fmt.Printf("%s: %v\n", p.ID, err)
|
||||
// Kept in the plan, so `plans` says why it has not moved rather than the log alone;
|
||||
// the state is left as it was and the step is tried again on the next tick.
|
||||
// the state is left as it was and the step is tried again on the next tick. Said and
|
||||
// kept when it is new: the same refusal on every tick is one fact, not one per tick.
|
||||
p.Note = "tier " + fmt.Sprint(p.Tier) + ": " + err.Error() + " — tried again"
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
|
||||
// A refusal for the bus's planned step is a person's to end: S17 says it from its first moment.
|
||||
if refusedForTheBus(err) {
|
||||
busRefusedFirst.mark(p.ID, time.Now())
|
||||
}
|
||||
if planSnapshot(*p) != before {
|
||||
fmt.Printf("%s: %v\n", p.ID, err)
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
if p.State == inventory.PlanFailed {
|
||||
sayUnsent(p, rollsOut)
|
||||
}
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
|
||||
break
|
||||
if planSnapshot(*p) != before {
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
|
||||
break
|
||||
}
|
||||
}
|
||||
if !moved {
|
||||
break
|
||||
@@ -531,6 +561,18 @@ func advanceHeld(ctx context.Context, open *stores) {
|
||||
}
|
||||
}
|
||||
|
||||
// planSnapshot is a plan as a save would write it, to compare before and after a step: what the store
|
||||
// stamps itself (when it was saved, its revision and epoch, when its tier was entered) is left out.
|
||||
func planSnapshot(p inventory.Plan) string {
|
||||
p.Updated, p.Revision, p.Epoch, p.TierEntered = time.Time{}, 0, 0, time.Time{}
|
||||
b, err := json.Marshal(p)
|
||||
if err != nil {
|
||||
// Unreadable is never the same: the plan is saved, as every step was before.
|
||||
return fmt.Sprintf("unmarshallable %p %v", &p, err)
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// advanceOnce takes one step of one plan and says whether anything changed.
|
||||
func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
edges []inventory.Edge, rollsOut func(string) bool) (bool, error) {
|
||||
@@ -668,7 +710,6 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
}
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
step := nextRollout(*state, running, policy.Together, reports, now, planWaitBound)
|
||||
// **Sent with others, judged with them** (issue 281): the gate of the send that carried it is its
|
||||
// verdict on its first machine. A failure there stopped the plan already.
|
||||
if state.GatedBy != "" {
|
||||
@@ -680,8 +721,11 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
if lead.Gate.Verdict != inventory.GatePassed {
|
||||
continue
|
||||
}
|
||||
passedWith(state, state.GatedBy, lead.Gate)
|
||||
passedWith(m, state, state.GatedBy, lead.Gate)
|
||||
}
|
||||
// Read after its pass is taken over from the send that carried it, so a passed gate is never judged
|
||||
// again from the first machine's later reports (novox/hq issue 335).
|
||||
step := nextRollout(*state, running, policy.Together, reports, now, planWaitBound)
|
||||
switch {
|
||||
case step.failed != "":
|
||||
// The first machine refused or failed what it was sent, or never said: the gate failed, and
|
||||
@@ -706,6 +750,9 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
}
|
||||
switch verdict {
|
||||
case "":
|
||||
if putBackBroken(ctx, open, p, state.Gate, m, state) {
|
||||
return true, nil
|
||||
}
|
||||
pending = append(pending, fmt.Sprintf("%s judged on %s: %s", m,
|
||||
strings.Join(state.Gate.Machines, ", "), gateLine(state.Gate)))
|
||||
continue
|
||||
@@ -904,7 +951,7 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
||||
strings.Join(sent, ", "))
|
||||
fmt.Printf("%s: tier %d built; sent %d module(s) to %s first in one send (%s), the rest once its gate passes\n",
|
||||
p.ID, p.Tier, len(modules), strings.Join(sent, ", "), strings.Join(modules, ", "))
|
||||
// What the send recreates, said with it (novox/hq ADR 0242).
|
||||
// What the send recreates, said with it (novox/hq ADR 0245).
|
||||
if said := recreationsSaid(carried); said != "" {
|
||||
p.Note += "; " + said
|
||||
fmt.Printf("%s: %s\n", p.ID, said)
|
||||
@@ -914,14 +961,14 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
||||
|
||||
// passedWith keeps, on a module sent with others, the verdict of the gate that judged the send: the gate
|
||||
// on the first of them passed, and its pass was kept for every build it carried (passCarried).
|
||||
func passedWith(s *inventory.PlanModule, lead string, g *inventory.PlanGate) {
|
||||
func passedWith(module string, s *inventory.PlanModule, lead string, g *inventory.PlanGate) {
|
||||
if s.Gate == nil {
|
||||
s.Gate = &inventory.PlanGate{Machines: g.Machines, From: s.Previous, To: s.Commit, Since: g.Since}
|
||||
}
|
||||
if s.Gate.Verdict != "" {
|
||||
return
|
||||
}
|
||||
s.Gate.Verdict, s.Gate.Why, s.Gate.JudgedAt, s.Gate.Took = g.Verdict, "judged with "+lead+": "+g.Why, g.JudgedAt, g.Took
|
||||
s.Gate.Verdict, s.Gate.Why, s.Gate.JudgedAt, s.Gate.Took = g.Verdict, "judged with "+lead+": "+whyFor(g, module), g.JudgedAt, g.Took
|
||||
s.Gate.Passes, s.Gate.Kept = g.Passes, true
|
||||
}
|
||||
|
||||
@@ -938,8 +985,39 @@ func failFirstSend(ctx context.Context, open *stores, p *inventory.Plan, module
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
}()
|
||||
g := state.Gate
|
||||
if g == nil || g.Verdict == "" || len(g.Failing) == 0 || slices.Contains(g.Failing, module) {
|
||||
if g != nil && g.Verdict == inventory.GatePassed {
|
||||
// **A guard: a build that passed its gate is never put back for what came after** (novox/hq issue 335).
|
||||
// Not reached while nextRollout answers a passed gate with the rest to send, and advanceOnce reads it
|
||||
// after a carried module takes over its lead's pass; it is here so that a path added later cannot
|
||||
// overturn a verdict. If it is reached, the plan stops and says why, and the build is not marked failed.
|
||||
// The module is left a stopped rollout (its Why said, sent first and not to the rest), which
|
||||
// `plans retry` takes: it sends the first machine again, and the passed gate then sends the rest.
|
||||
state.Why = "passed its gate; its walk then stopped: " + why
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = fmt.Sprintf("%s passed its gate on %s in tier %d (%s) and is kept; its walk stopped after: %s",
|
||||
module, strings.Join(g.Machines, ", "), p.Tier, g.Why, why)
|
||||
return
|
||||
}
|
||||
if g != nil && slices.Contains(g.Returned, module) {
|
||||
// Put back at once when it broke: its rollback was made then, and is not made again.
|
||||
state.Why = "put back when it broke; its send failed: " + g.Why
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = fmt.Sprintf("the send to %s in tier %d failed its gate: %s; %s was put back when it broke",
|
||||
strings.Join(g.Machines, ", "), p.Tier, g.Why, module)
|
||||
} else if g == nil || g.Verdict == "" || len(g.Failing) == 0 || slices.Contains(g.Failing, module) {
|
||||
gateFailed(batched, open, p, module, state, machines, why)
|
||||
} else if slices.Contains(g.Passing, module) && state.Build != "" {
|
||||
// **The module the gate is kept on keeps its own pass too** (issue 318 review): healthy for the passes
|
||||
// asked while another of its send failed, its build has a verdict, and other walks do not wait on it.
|
||||
if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: state.Build, Module: module,
|
||||
Commit: state.Commit, Previous: state.Previous, Plan: p.ID, Machines: g.Machines, Verdict: inventory.GatePassed,
|
||||
Why: passedAloneWhy(g, module), Component: g.Component, JudgingFrom: g.Since}); err != nil {
|
||||
fmt.Printf("%s: %s passed its gate on its own, and the verdict could not be kept: %v\n", p.ID, module, err)
|
||||
}
|
||||
state.Why = "passed on its own; stopped with the send that carried it: " + g.Why
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = fmt.Sprintf("the send to %s in tier %d failed its gate: %s; %s passed on its own and is kept",
|
||||
strings.Join(g.Machines, ", "), p.Tier, g.Why, module)
|
||||
} else {
|
||||
state.Why = "not found wanting; stopped with the send that carried it: " + g.Why
|
||||
p.State = inventory.PlanFailed
|
||||
@@ -1007,6 +1085,15 @@ func nextRollout(s inventory.PlanModule, running []string, together bool, report
|
||||
rest = append(rest, n)
|
||||
}
|
||||
}
|
||||
// **A passed gate is the first machine's verdict, and its later reports are not** (novox/hq issue 335).
|
||||
// Once the build passed there, what that machine reports next is about whatever it was sent after —
|
||||
// another walk's send, a push — and says nothing of this build. On 2026-10-08 a build passed on the
|
||||
// laptop, its send to the rest waited on another walk, that walk sent the laptop a new declaration it did
|
||||
// not report for half an hour, and the plan read the silence as the first machine never applying the
|
||||
// passed build: it marked it failed at its gate and put it back. The rest are sent, as the pass said.
|
||||
if s.Gate != nil && s.Gate.Verdict == inventory.GatePassed {
|
||||
return rolloutStep{send: rest}
|
||||
}
|
||||
var waiting, failed []string
|
||||
for _, n := range s.First {
|
||||
r, said := byNode[n]
|
||||
@@ -1070,12 +1157,37 @@ func planTicker(ctx context.Context, open *stores) {
|
||||
}
|
||||
}
|
||||
|
||||
// planLine is one plan as `status` says it.
|
||||
func planLine(p inventory.Plan, now time.Time) string { return planLineWith(p, now, pauseView{}) }
|
||||
// planLine is one plan as `status` says it, given the least bound of a tier.
|
||||
func planLine(p inventory.Plan, now time.Time) string {
|
||||
return planLineWith(p, now, pauseView{}, tierAtLeast)
|
||||
}
|
||||
|
||||
// inTierSince is when a plan began waiting on what it waits on now, which `plans`, `status` and the
|
||||
// watchdog of a plan's progress (S3) all count from (novox/hq issue 296): a walk waiting for its
|
||||
// delivery's word from when it was opened, as S16 counts; any other plan from when it entered its tier,
|
||||
// or, when the delivery let it go after that, from the go. Never from the plan's last save: a plan is
|
||||
// saved for many reasons while its tier stands still, and a clock reset by each save said a build queued
|
||||
// for minutes had been building for a few seconds. A plan read without the stamp (one from before it
|
||||
// was kept) counts from its last save, the only time it has.
|
||||
func inTierSince(p inventory.Plan) time.Time {
|
||||
if p.Waiting() {
|
||||
return p.Created
|
||||
}
|
||||
since := p.TierEntered
|
||||
if since.IsZero() {
|
||||
since = p.Updated
|
||||
}
|
||||
if d := p.Delivery; d != nil && d.Go != nil && d.Go.After(since) {
|
||||
since = *d.Go
|
||||
}
|
||||
return since
|
||||
}
|
||||
|
||||
// planLineWith is planLine knowing whether the build seat is paused (novox/hq ADR 0219): a plan
|
||||
// waiting on builds nobody will take until a person resumes the seat says so, and is not late.
|
||||
func planLineWith(p inventory.Plan, now time.Time, pause pauseView) string {
|
||||
// waiting on builds nobody will take until a person resumes the seat says so, and is not late. bound is
|
||||
// the plan's tier bound, the one its stalled condition is raised at (tierBounds): LATE is that condition
|
||||
// said on the line (novox/hq issue 296).
|
||||
func planLineWith(p inventory.Plan, now time.Time, pause pauseView, bound time.Duration) string {
|
||||
where := fmt.Sprintf("tier %d of %d", min(p.Tier+1, len(p.Tiers)), len(p.Tiers))
|
||||
switch p.State {
|
||||
case inventory.PlanDone:
|
||||
@@ -1085,7 +1197,7 @@ func planLineWith(p inventory.Plan, now time.Time, pause pauseView) string {
|
||||
case inventory.PlanSuperseded:
|
||||
return fmt.Sprintf("%s %s %s", p.Repository, short(p.Commit), p.Note)
|
||||
}
|
||||
since := now.Sub(p.Updated).Round(time.Second)
|
||||
since := now.Sub(inTierSince(p)).Round(time.Second)
|
||||
if p.Waiting() {
|
||||
// Waiting for its delivery's word is no lateness of the walk's (novox/hq ADR 0239).
|
||||
return fmt.Sprintf("%s %s %s, %s, for %s", p.Repository, short(p.Commit), where, waitingNote(p), since)
|
||||
@@ -1094,7 +1206,7 @@ func planLineWith(p inventory.Plan, now time.Time, pause pauseView) string {
|
||||
return fmt.Sprintf("%s %s %s, %s", p.Repository, short(p.Commit), where, waiting)
|
||||
}
|
||||
late := ""
|
||||
if since > planWaitBound {
|
||||
if since > bound {
|
||||
late = " — LATE"
|
||||
}
|
||||
what := "building"
|
||||
@@ -1167,17 +1279,19 @@ type planStatus struct {
|
||||
Late bool `json:"late"`
|
||||
}
|
||||
|
||||
func planStatuses(plans []inventory.Plan, now time.Time, pause pauseView) []planStatus {
|
||||
func planStatuses(plans []inventory.Plan, now time.Time, pause pauseView, bounds tierBounds) []planStatus {
|
||||
out := make([]planStatus, 0, len(plans))
|
||||
for _, p := range plans {
|
||||
ps := planStatus{ID: p.ID, Repository: p.Repository, Commit: p.Commit, State: p.State,
|
||||
Tier: p.Tier, Tiers: len(p.Tiers), Since: p.Updated}
|
||||
if p.Open() {
|
||||
ps.Since = inTierSince(p)
|
||||
ps.Waiting = p.Note
|
||||
if ps.Waiting == "" {
|
||||
ps.Waiting = "builds of tier " + fmt.Sprint(p.Tier)
|
||||
}
|
||||
ps.Late = now.Sub(p.Updated) > planWaitBound
|
||||
// A walk waiting for its delivery's word is no tier late (ADR 0239): S16 is its watchdog.
|
||||
ps.Late = !p.Waiting() && now.Sub(ps.Since) > bounds.of(p.Repository)
|
||||
// Paused is a person's decision, not lateness (novox/hq ADR 0219).
|
||||
if waiting, paused := pausedWaiting(p, pause, now); paused {
|
||||
ps.Waiting, ps.Late = waiting, false
|
||||
@@ -1189,16 +1303,16 @@ func planStatuses(plans []inventory.Plan, now time.Time, pause pauseView) []plan
|
||||
}
|
||||
|
||||
// openPlans is the open plans among the recent ones, and how many have waited past the bound.
|
||||
func openPlans(plans []inventory.Plan, pause pauseView) ([]inventory.Plan, int) {
|
||||
func openPlans(plans []inventory.Plan, now time.Time, pause pauseView, bounds tierBounds) ([]inventory.Plan, int) {
|
||||
var open []inventory.Plan
|
||||
late := 0
|
||||
for _, p := range plans {
|
||||
if p.Open() {
|
||||
open = append(open, p)
|
||||
if _, paused := pausedWaiting(p, pause, time.Now()); paused {
|
||||
if _, paused := pausedWaiting(p, pause, now); paused || p.Waiting() {
|
||||
continue
|
||||
}
|
||||
if time.Since(p.Updated) > planWaitBound {
|
||||
if now.Sub(inTierSince(p)) > bounds.of(p.Repository) {
|
||||
late++
|
||||
}
|
||||
}
|
||||
@@ -1240,7 +1354,9 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s — %s\n", p.ID, planLineWith(p, now, buildSeatPause(ctx, inv, []inventory.Plan{p})))
|
||||
bounds := readTierBounds(ctx, inv, now)
|
||||
fmt.Printf("%s — %s\n", p.ID, planLineWith(p, now, buildSeatPause(ctx, inv, []inventory.Plan{p}),
|
||||
bounds.of(p.Repository)))
|
||||
if r := p.Release; r != nil {
|
||||
// A release plan's walk (ADR 0236): machines done, the one judged, those to come.
|
||||
fmt.Printf(" machines in order: %s; done: %s; skipped: %s\n", strings.Join(r.Order, ", "),
|
||||
@@ -1287,7 +1403,7 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
case s != nil && s.Gate != nil:
|
||||
fmt.Printf(" %-22s %s\n", "", gateLine(s.Gate))
|
||||
}
|
||||
// What the send to its first machine did to its containers (ADR 0242).
|
||||
// What the send to its first machine did to its containers (ADR 0245).
|
||||
if s != nil && s.Gate != nil {
|
||||
for _, c := range s.Gate.Carried {
|
||||
if c.Recreates != "" {
|
||||
@@ -1369,8 +1485,9 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
return nil
|
||||
}
|
||||
pause := buildSeatPause(ctx, inv, plans)
|
||||
bounds := readTierBounds(ctx, inv, now)
|
||||
for _, p := range plans {
|
||||
fmt.Printf("%-28s %s\n", p.ID, planLineWith(p, now, pause))
|
||||
fmt.Printf("%-28s %s\n", p.ID, planLineWith(p, now, pause, bounds.of(p.Repository)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -27,6 +27,8 @@ func TestAMergeIsPlannedInTiersAlongTheThreeKindsOfDependency(t *testing.T) {
|
||||
{From: "mesh-controller", To: "builder", Kind: inventory.EdgeBuiltBy},
|
||||
{From: "mesh-tools", To: "builder", Kind: inventory.EdgeBuiltBy},
|
||||
{From: "builder", To: "mesh-tools", Kind: inventory.EdgeStandsOn},
|
||||
// the build seat's holder follows the controller that defines its worker (hq issue 206)
|
||||
{From: "builder", To: "mesh-controller", Kind: inventory.EdgeWorkerOf},
|
||||
{From: "unrelated", To: "alpine", Kind: inventory.EdgeStandsOn},
|
||||
}
|
||||
// The runtime image moved: everything on it, and what is built by what is on it.
|
||||
@@ -62,12 +64,15 @@ func TestAMergeIsPlannedInTiersAlongTheThreeKindsOfDependency(t *testing.T) {
|
||||
if len(small) != 3 {
|
||||
t.Fatalf("a controller merge rebuilds the controller and what packages it: %v", small)
|
||||
}
|
||||
// The builder packages the controller's source (same tier by that edge) and the controller is
|
||||
// built by the builder (next tier by that one): the builder first, then the controller and the
|
||||
// proxy together — a code dependency in one tier, a runtime dependency across tiers.
|
||||
// The builder and the proxy package the controller's source, which orders nothing. The builder holds
|
||||
// the build seat, whose worker the controller defines, so it follows the controller (worker-of,
|
||||
// novox/hq issue 206), and the controller's built-by edge to it yields: the controller is built by the
|
||||
// build machine that is running. The proxy is built by the new builder: the controller, the builder,
|
||||
// the proxy — the live plan of every controller merge. (This read "the builder, then the controller
|
||||
// and the proxy together" before issue 206, and the fixture had no worker-of edge.)
|
||||
smallTiers := tiersOf(small, edges)
|
||||
if len(smallTiers) != 2 || smallTiers[0][0] != "builder" || len(smallTiers[1]) != 2 {
|
||||
t.Fatalf("the builder, then the controller and the proxy together: %v", smallTiers)
|
||||
if got := tiered(smallTiers); got != "mesh-controller | builder | route-proxy" {
|
||||
t.Fatalf("the controller, then the builder, then the proxy: %v", smallTiers)
|
||||
}
|
||||
// The builder alone moved: the builder, and nothing it builds.
|
||||
if only := reachableFrom([]string{"builder"}, edges); len(only) != 1 {
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **R292 — a drill is never counted as a repair** (novox/hq issue 292). On 2026-10-07 two deliberate
|
||||
// drills of ADR 0240's health check — a module stopped on purpose, then one made to crash, each with the
|
||||
// operator's word — were recorded through `hand-act record --cause drill`, there being no verb for a drill,
|
||||
// and S15 raised `mesh.hand-acts.drill.healer-wanted`: "drill" repaired by hand twice, a healer wanted. The
|
||||
// outcome asserted: those two acts, as the log holds them, want no healer.
|
||||
//
|
||||
// In a file of its own, written only with what the controller had before the fix (watchHandActs, calm, the
|
||||
// hand-act log's row), so the prover can lay it over that commit; replays_test.go holds replays of later
|
||||
// commits that do not build there.
|
||||
func TestReplay292TwoDrillsRecordedByHandWantNoHealer(t *testing.T) {
|
||||
now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC)
|
||||
drill := func(at time.Time, what, why string) link.HandAct {
|
||||
return link.HandAct{ID: "act-" + at.Format("150405"), At: at, By: "the operator at a shell on the laptop",
|
||||
Verb: "hand-act record", Args: []string{what}, Why: why, Cause: "drill"}
|
||||
}
|
||||
f := calm(now)
|
||||
f.handActs = []link.HandAct{
|
||||
drill(now.Add(-11*time.Hour), "stopped a module on purpose", "ADR 0240: an exited container is said"),
|
||||
drill(now.Add(-30*time.Minute), "made a module crash on purpose", "ADR 0240: a crash loop is said"),
|
||||
}
|
||||
if got := watchHandActs(f); len(got) != 0 {
|
||||
t.Fatalf("two drills the operator approved were counted as repairs: %s", got[0].Summary)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq issue 309, replayed with only what orderOf had before its fix, so it can be laid over the older
|
||||
// commit. The group `feat/a-machine-joins-through-the-tunnel` (2026-10-08): the controller's pull request
|
||||
// moved the controller and the build agent, the node-engine's moved the node-engine, the lab's moved
|
||||
// nothing. The build agent builds the node-engine (built by: the controller's member first), and the
|
||||
// node-engine goes before the controller (engine before controller: the node-engine's member first). Both
|
||||
// rules applied to one pair in opposite directions, and the group was rejected as a cycle; the person merged
|
||||
// the controller first by hand. A rule read from the graph outranks the rollout default, so the group is
|
||||
// ordered, the controller's member first.
|
||||
func TestReplay309(t *testing.T) {
|
||||
members := []orderMember{
|
||||
{ID: "novox/mesh-controller@8bbfdb53db2b", Repository: "novox/mesh-controller", Number: 132},
|
||||
{ID: "novox/mesh-host@86cbebd10f8a", Repository: "novox/mesh-host", Number: 52},
|
||||
{ID: "novox/mesh-lab@e80a4b1642bf", Repository: "novox/mesh-lab", Number: 61},
|
||||
}
|
||||
reach := map[string]orderReach{
|
||||
"novox/mesh-controller@8bbfdb53db2b": {Moved: []string{"build-agent", "mesh-controller", "route-proxy"},
|
||||
Manifests: []string{"module.json"}},
|
||||
"novox/mesh-host@86cbebd10f8a": {Moved: []string{"mesh-host"}},
|
||||
"novox/mesh-lab@e80a4b1642bf": {},
|
||||
}
|
||||
// Every source-built module is built by the build seat's holder; the holder follows the controller.
|
||||
edges := []inventory.Edge{
|
||||
{From: "build-agent", To: "mesh-controller", Kind: inventory.EdgeWorkerOf},
|
||||
{From: "mesh-controller", To: "build-agent", Kind: inventory.EdgeBuiltBy},
|
||||
{From: "mesh-host", To: "build-agent", Kind: inventory.EdgeBuiltBy},
|
||||
{From: "route-proxy", To: "build-agent", Kind: inventory.EdgeBuiltBy},
|
||||
}
|
||||
got := orderOf(members, reach, edges)
|
||||
if len(got.Cycle) > 0 {
|
||||
t.Fatalf("the group is refused as a cycle %v; pairs %+v", got.Cycle, got.Pairs)
|
||||
}
|
||||
want := []string{"novox/mesh-controller@8bbfdb53db2b", "novox/mesh-host@86cbebd10f8a", "novox/mesh-lab@e80a4b1642bf"}
|
||||
if !reflect.DeepEqual(got.Order, want) {
|
||||
t.Fatalf("ordered %v, wanted %v; pairs %+v", got.Order, want, got.Pairs)
|
||||
}
|
||||
for _, p := range got.Pairs {
|
||||
if p.Before == want[1] && p.After == want[0] {
|
||||
t.Fatalf("a pair that lost still stands: %+v", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,250 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// novox/hq issue 318, replayed with only what the controller had before its fix, so it can be laid over the
|
||||
// older commit: the statements are read from the node-engine's JSON, as the controller hears them.
|
||||
//
|
||||
// 2026-10-08, 11:08 UTC: the walk of the builds waiting for a gate sent the laptop three moves in one send —
|
||||
// docker ba6058ab → c78b5fc9, node-tools 9a4140b4 → d4845f7c, openrazer 88135ad0 → c78b5fc9 — and judged
|
||||
// them together. openrazer's build put the operator's account in the group `openrazer` (ADR 0252); the
|
||||
// laptop said the account "relogin needed", and the daemon's unit failed in the account's own service
|
||||
// manager, which began before the group. The gate read "not yet healthy" for ten minutes, failed at its
|
||||
// bound, put openrazer back (which by ADR 0252 also took the group back), and the walk failed: docker and
|
||||
// node-tools, healthy at every judging, never reached the workstation, and every other walk was refused there.
|
||||
//
|
||||
// The rule's outcome: a wait for a person is not a failure. The send passes with the wait carried along,
|
||||
// nothing is put back, the walk goes on to the workstation, and openrazer's condition says, in one sentence
|
||||
// for the operator, that a new login is needed on the laptop.
|
||||
func TestReplay318(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
keeper, _ := withConditionsInMemory(t)
|
||||
was := doctorFrom
|
||||
doctorFrom = nil
|
||||
t.Cleanup(func() { doctorFrom = was })
|
||||
|
||||
build := func(module, repository, commit string, asked time.Time) {
|
||||
m := catalogue.Manifest{Module: module, Version: "1"}
|
||||
if module == "openrazer" && commit == "c78b5fc9" {
|
||||
// The build that put the operator's account in the group `openrazer` (ADR 0252).
|
||||
m.Resources = []map[string]any{{"id": "operator", "type": "user", "name": "operator",
|
||||
"groups": []any{"openrazer"}}}
|
||||
}
|
||||
manifest, _ := json.Marshal(m)
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + module + "-" + commit, Module: module, Commit: commit,
|
||||
Repository: repository, Path: "modules/" + module, Manifest: manifest, Asked: asked, At: asked,
|
||||
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + module + "-" + commit}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, m, inventory.Source{
|
||||
Repository: repository, Seat: "git", Path: "modules/" + module, BuiltFrom: commit, Head: commit,
|
||||
Asked: asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
type move struct{ module, repository, from, to string }
|
||||
moves := []move{
|
||||
{"docker", "novox/mesh-catalog", "ba6058ab", "c78b5fc9"},
|
||||
{"node-tools", "novox/mesh-tools", "9a4140b4", "d4845f7c"},
|
||||
{"openrazer", "novox/mesh-catalog", "88135ad0", "c78b5fc9"},
|
||||
}
|
||||
machines := []string{"laptop", "workstation"}
|
||||
if _, err := inv.AddNode(ctx, "workstation"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old, recent := time.Now().Add(-3*time.Hour), time.Now().Add(-time.Minute)
|
||||
for _, mv := range moves {
|
||||
build(mv.module, mv.repository, mv.from, old)
|
||||
for _, n := range machines {
|
||||
if _, err := inv.Assign(ctx, n, mv.module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, n := range machines {
|
||||
sent := map[string]string{}
|
||||
for _, mv := range moves {
|
||||
sent[mv.module] = mv.from
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n, sent); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, mv := range moves {
|
||||
build(mv.module, mv.repository, mv.to, recent)
|
||||
}
|
||||
|
||||
// What each machine says, as its node-engine words it: openrazer's account waits for a new login, and the
|
||||
// daemon's unit failed in that account's own manager; docker's container is healthy.
|
||||
statement := func(at time.Time) link.Health {
|
||||
raw := fmt.Sprintf(`{"contract": %d, "at": %q, "resources": [
|
||||
{"module": "docker", "resource": "docker.engine", "kind": "service", "target": "docker.service",
|
||||
"state": "healthy", "since": %q},
|
||||
{"module": "openrazer", "resource": "openrazer.operator", "kind": "account", "target": "operator",
|
||||
"account": "operator", "state": "unhealthy", "since": %q, "streak": 3,
|
||||
"reason": "relogin needed: operator is in the group openrazer, and its running session began before it was; log out of every session and in again, or reboot"},
|
||||
{"module": "openrazer", "resource": "openrazer.daemon", "kind": "unit", "target": "openrazer-daemon.service",
|
||||
"account": "operator", "state": "unhealthy", "since": %q, "streak": 3,
|
||||
"reason": "failed in the account's own service manager (exit-code)"}]}`,
|
||||
link.ReadinessContract, at.Format(time.RFC3339Nano), at.Format(time.RFC3339Nano), at.Format(time.RFC3339Nano),
|
||||
at.Format(time.RFC3339Nano))
|
||||
var h link.Health
|
||||
if err := json.Unmarshal([]byte(raw), &h); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts
|
||||
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
||||
t.Cleanup(func() {
|
||||
machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
|
||||
gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound
|
||||
})
|
||||
machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
|
||||
modules, err := open.inventory.Assigned(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sent, known, err := open.inventory.SentBuilds(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f.moves(node, modules, sent, known, all), nil
|
||||
}
|
||||
releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
|
||||
return map[string]bool{"laptop": true, "workstation": true}, nil
|
||||
}
|
||||
var sends [][]string
|
||||
n := 0
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
sends = append(sends, append([]string(nil), names...))
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, node := range names {
|
||||
modules, _ := open.inventory.Assigned(ctx, node)
|
||||
carried := map[string]string{}
|
||||
for _, m := range modules {
|
||||
carried[m] = current[m].Commit
|
||||
}
|
||||
n++
|
||||
digest := fmt.Sprintf("d-%s-%d", node, n)
|
||||
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
|
||||
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
// Every judging hears each machine's newest statement first, as the controller does between judgings:
|
||||
// the account's wait and the failed unit are raised as openrazer's condition, after the send.
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
now := time.Now()
|
||||
f := gateFacts{now: now, reports: map[string]inventory.Reported{}, engines: map[string]string{},
|
||||
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}, health: map[string]inventory.NodeHealth{},
|
||||
judged: true}
|
||||
for _, m := range machines {
|
||||
if err := stateHealth(ctx, open.inventory, keeper, m, statement(now), now); err != nil {
|
||||
return f, err
|
||||
}
|
||||
f.served[m] = served{runtime: true, tools: map[string]bool{}}
|
||||
}
|
||||
reports, err := open.inventory.LastReports(ctx)
|
||||
if err != nil {
|
||||
return f, err
|
||||
}
|
||||
for _, r := range reports {
|
||||
f.reports[r.Node] = r
|
||||
}
|
||||
if f.health, err = open.inventory.Healths(ctx); err != nil {
|
||||
return f, err
|
||||
}
|
||||
f.open, f.openErr = keeper.Open(ctx)
|
||||
return f, nil
|
||||
}
|
||||
// The bound is reached at the first judging that is not a pass: what happened at 11:21 happens at once.
|
||||
gateSettle, gateEvery, gateBound = 0, 0, 0
|
||||
|
||||
for i := 0; i < 12; i++ {
|
||||
advancePlans(ctx, open)
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
|
||||
var p inventory.Plan
|
||||
plans, err := inv.RecentPlans(ctx, 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, q := range plans {
|
||||
if q.Release != nil {
|
||||
p = q
|
||||
break
|
||||
}
|
||||
}
|
||||
if p.Release == nil {
|
||||
t.Fatal("no walk of the builds waiting for a gate was opened")
|
||||
}
|
||||
if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Done, machines) {
|
||||
t.Fatalf("the walk is %s on %v: %s; want it done on the laptop and then the workstation", p.State,
|
||||
p.Release.Done, p.Note)
|
||||
}
|
||||
if !reflect.DeepEqual(sends, [][]string{{"laptop"}, {"workstation"}}) {
|
||||
t.Fatalf("sent %v; want the laptop, then the workstation, and nothing put back", sends)
|
||||
}
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if current["openrazer"].Commit != "c78b5fc9" {
|
||||
t.Fatalf("openrazer is registered at %s: its build was put back for a wait for a person", current["openrazer"].Commit)
|
||||
}
|
||||
for _, mv := range moves {
|
||||
v, found, err := inv.GateOf(ctx, "build-"+mv.module+"-"+mv.to)
|
||||
if err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("%s's build %s: verdict %+v (found %v, %v); want a pass", mv.module, mv.to, v, found, err)
|
||||
}
|
||||
if mv.module == "openrazer" && !strings.Contains(v.Why, "relogin needed on ") {
|
||||
t.Errorf("openrazer's pass does not carry its wait for a person: %q", v.Why)
|
||||
}
|
||||
}
|
||||
// What the operator reads: one condition per machine for openrazer, saying a new login is needed there,
|
||||
// a warning for a person, and no fault of the build.
|
||||
list, err := keeper.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var said []string
|
||||
for _, c := range list {
|
||||
said = append(said, c.Key+": "+c.Summary)
|
||||
if c.Subject.ID == "openrazer.laptop" {
|
||||
if !strings.HasPrefix(c.Summary, "relogin needed on laptop") || c.Severity == conditions.Urgent ||
|
||||
c.Resolver != conditions.ResolverOperator {
|
||||
t.Errorf("openrazer's condition on the laptop: %s %s %s %q", c.Key, c.Severity, c.Resolver, c.Summary)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !strings.Contains(strings.Join(said, "\n"), "relogin needed on laptop") {
|
||||
t.Errorf("nothing says a new login is needed on the laptop:\n%s", strings.Join(said, "\n"))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// novox/hq issue 325, replayed with only what the controller had before its fix, so it can be laid over the
|
||||
// older commit. On 2026-10-08 the catalogue's pull request adding `sensors` merged, and the merge asked for
|
||||
// its build (issue 300). About a minute later `assign g14 sensors` answered "no module of that name:
|
||||
// sensors"; a few minutes later the same call worked, because the build had registered the module. The
|
||||
// answer read as "nobody registered it". An assignment made while the build runs says the build — where it
|
||||
// is from, since when, its id — is kept, and is made when the build registers the module.
|
||||
func TestReplay325(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "networkmanager", Version: "1"},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/networkmanager", Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asksWithPaths(t)
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "5e450125aa",
|
||||
Paths: []string{"modules/sensors/module.json", "modules/sensors/cmd/sensors/main.go"},
|
||||
ModuleDirs: []string{"modules/sensors"}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
said, err := assign(ctx, open, "laptop", "sensors")
|
||||
if err != nil {
|
||||
t.Fatalf("assign while the merge's build runs was refused: %v", err)
|
||||
}
|
||||
for _, want := range []string{"being built", "novox/mesh-catalog", "modules/sensors", "b-modules/sensors"} {
|
||||
if !strings.Contains(said, want) {
|
||||
t.Fatalf("the answer does not say %q:\n%s", want, said)
|
||||
}
|
||||
}
|
||||
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: "sensors", Version: "1"})
|
||||
if err := (builds{open.inventory, open}).Built(ctx, link.BuildResult{ID: "b-modules/sensors",
|
||||
Repository: "novox/mesh-catalog", Path: "modules/sensors", Ref: "main", Commit: "5e450125aa", On: "anchor",
|
||||
Module: "sensors", Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Repository: "novox/mesh-catalog", Seat: "git"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assigned, err := open.inventory.Assigned(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Contains(assigned, "sensors") {
|
||||
t.Fatalf("the build registered sensors and laptop runs %v", assigned)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// novox/hq issue 327, replayed with only what the controller had before its fix, so it can be laid over
|
||||
// the older commit. On 2026-10-08 the bus's connection total rose by 5.2 a minute while the same 16
|
||||
// connections stayed open, and three calls of `conditions` in one second added three: each verb ran as a
|
||||
// process of the controller's own binary, which dialled the bus, logged in and left. The operator's
|
||||
// channel reads `conditions` at least once a minute. A verb that only reads, served by the serving
|
||||
// controller, opens no connection of its own.
|
||||
func TestReplay327(t *testing.T) {
|
||||
bus := testbus.Start(t)
|
||||
serving, err := broker.Dial(bus.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer serving.Close()
|
||||
if err := serving.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
keeper, err := keeperOn(context.Background(), serving.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The serving controller: its keeper and its connection, as serve sets them.
|
||||
keptBefore, connBefore := conditionsFrom, handActConn
|
||||
conditionsFrom, handActConn = keeper, serving.Conn()
|
||||
defer func() { conditionsFrom, handActConn = keptBefore, connBefore }()
|
||||
// A verb that runs as a process of its own runs this controller's binary, on this bus.
|
||||
asAProcess(t, bus.ClientURL())
|
||||
|
||||
handlers, _, err := seatToolHandlers()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
accepted := func() uint64 {
|
||||
v, err := bus.Varz(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return v.TotalConnections
|
||||
}
|
||||
before := accepted()
|
||||
for i := 0; i < 3; i++ {
|
||||
answer, err := handlers["conditions"](context.Background(), json.RawMessage(`{}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a, ok := answer.(verbAnswer); !ok || !a.OK {
|
||||
t.Fatalf("conditions answered %+v", answer)
|
||||
}
|
||||
}
|
||||
if opened := accepted() - before; opened != 0 {
|
||||
t.Fatalf("three conditions calls opened %d connection(s) to the bus; the serving controller is on it already",
|
||||
opened)
|
||||
}
|
||||
}
|
||||
|
||||
// asAProcess makes a verb that runs as a process of its own run this package's binary, on the bus at url:
|
||||
// built into the test's own directory, which goes with the test.
|
||||
func asAProcess(t *testing.T, url string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "mesh-controller")
|
||||
if out, err := exec.Command("go", "build", "-o", path, ".").CombinedOutput(); err != nil {
|
||||
t.Fatalf("the controller could not be built to run a verb as its own process: %v: %s", err, out)
|
||||
}
|
||||
was := ownImage
|
||||
ownImage = func() string { return path }
|
||||
t.Cleanup(func() { ownImage = was })
|
||||
t.Setenv(broker.NATSVar, url)
|
||||
t.Setenv(broker.CertificateVar, "")
|
||||
}
|
||||
|
||||
// closedNames are the names of the connections the bus saw closed.
|
||||
func closedNames(t *testing.T, bus *server.Server) []string {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
var names []string
|
||||
for time.Now().Before(deadline) {
|
||||
connz, err := bus.Connz(&server.ConnzOptions{State: server.ConnClosed})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
names = names[:0]
|
||||
for _, c := range connz.Conns {
|
||||
names = append(names, c.Name)
|
||||
}
|
||||
if len(names) > 0 {
|
||||
return names
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
return names
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// TestReplay335 replays novox/hq issue 335 (2026-10-08): dunst's new build passed its gate on the laptop
|
||||
// (healthy 3 times over 2m5s); its send to the rest was refused while another walk's build waited on the
|
||||
// workstation; that walk then sent the laptop a declaration the laptop did not report on; and thirty minutes
|
||||
// after the first send the plan read the laptop's silence as the passed build never applied, marked it failed
|
||||
// at its gate with the pass's own words, and put it back. Written with only what the controller had before
|
||||
// its fix, so it is laid over the commit before.
|
||||
func TestReplay335(t *testing.T) {
|
||||
t.Run("the first machine's later reports", testAPassedGateIsNotJudgedAgainFromTheFirstMachinesLaterReports)
|
||||
t.Run("a walk stopped after the pass", testAWalkStoppedAfterItsGatePassedKeepsThePass)
|
||||
}
|
||||
|
||||
// novox/hq issue 335: a build that passed its gate on its first machine is not judged again from that
|
||||
// machine's later reports. On 2026-10-08 the send to the rest waited on another walk, that walk sent the
|
||||
// first machine a declaration it did not report for half an hour, and the plan failed the passed build at
|
||||
// the wait's bound and put it back.
|
||||
func testAPassedGateIsNotJudgedAgainFromTheFirstMachinesLaterReports(t *testing.T) {
|
||||
sentAt := time.Date(2026, 10, 8, 16, 44, 45, 0, time.UTC)
|
||||
judged := sentAt.Add(2 * time.Minute)
|
||||
later := sentAt.Add(5 * time.Minute)
|
||||
state := inventory.PlanModule{First: []string{"laptop"}, FirstAt: &sentAt,
|
||||
Gate: &inventory.PlanGate{Machines: []string{"laptop"}, Verdict: inventory.GatePassed,
|
||||
Why: "healthy 3 times over 2m5s", JudgedAt: &judged, Kept: true}}
|
||||
running := []string{"laptop", "workstation"}
|
||||
now := sentAt.Add(30*time.Minute + 9*time.Second)
|
||||
|
||||
for what, reports := range map[string][]inventory.Reported{
|
||||
"no report about what it was sent since": {{Node: "laptop", At: &later, Outcome: inventory.OutcomeApplied, Current: false}},
|
||||
"another send failed there since": {{Node: "laptop", At: &later, Outcome: inventory.OutcomeFailed, Current: true}},
|
||||
"no report at all": nil,
|
||||
} {
|
||||
step := nextRollout(state, running, false, reports, now, 30*time.Minute)
|
||||
if step.failed != "" || step.waiting != "" || !reflect.DeepEqual(step.send, []string{"workstation"}) {
|
||||
t.Errorf("%s: %+v, want the rest sent as the pass said", what, step)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq issue 335: whatever stops a walk after its gate passed, the passed build is not marked failed at
|
||||
// its gate, nor put back.
|
||||
func testAWalkStoppedAfterItsGatePassedKeepsThePass(t *testing.T) {
|
||||
sentAt := time.Date(2026, 10, 8, 16, 44, 45, 0, time.UTC)
|
||||
g := &inventory.PlanGate{Machines: []string{"laptop"}, Verdict: inventory.GatePassed, Why: "healthy 3 times over 2m5s"}
|
||||
state := &inventory.PlanModule{Build: "build-1", First: []string{"laptop"}, FirstAt: &sentAt, Gate: g}
|
||||
p := &inventory.Plan{ID: "plan-1", Modules: map[string]*inventory.PlanModule{"dunst": state}}
|
||||
// No stores: a walk that keeps the pass touches none, and one that reaches for them is putting it back.
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
t.Fatalf("the passed build was taken to be failed and put back: %v", r)
|
||||
}
|
||||
}()
|
||||
failFirstSend(t.Context(), nil, p, "dunst", state, []string{"laptop"}, "laptop did not report it applied within 30m0s",
|
||||
[]string{"workstation"})
|
||||
if g.Verdict != inventory.GatePassed || g.Rollback != "" {
|
||||
t.Fatalf("the passed gate became %q, rollback %q", g.Verdict, g.Rollback)
|
||||
}
|
||||
if strings.Contains(p.Note, "failed its gate") || strings.Contains(p.Note, "put back") ||
|
||||
!strings.Contains(p.Note, "did not report it applied") {
|
||||
t.Fatalf("the note reads %q", p.Note)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq issue 335 review: **a module carried by its lead's send takes over the lead's pass before its own
|
||||
// step is read.** The state is the one a step leaves when the lead's gate passed and the step ended before the
|
||||
// carried module's turn (an error read before it, kept with the plan): the lead passed, the carried module has
|
||||
// no gate of its own yet. Since then another send reached the first machine and it has not reported on it, and
|
||||
// the first send is older than the wait for a first machine's report. Read before the pass is taken over, the
|
||||
// carried module's step said the first machine never applied it, and the passed build was put back.
|
||||
func TestACarriedModuleTakesItsLeadsPassBeforeItsStepIsRead(t *testing.T) {
|
||||
tm := aTierMesh(t, "m01", "m02")
|
||||
ctx := t.Context()
|
||||
inv := tm.open.inventory
|
||||
|
||||
advancePlans(ctx, tm.open)
|
||||
if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) {
|
||||
t.Fatalf("sent %v: the first machine once, for the tier", tm.sent)
|
||||
}
|
||||
p := tm.plan(t)
|
||||
lead, carried := p.Modules["m01"], p.Modules["m02"]
|
||||
if lead.Gate == nil || carried.GatedBy != "m01" {
|
||||
t.Fatalf("m02 is not carried by m01's send: lead %+v, carried %+v", lead.Gate, carried)
|
||||
}
|
||||
// The lead passed; the carried module's turn did not come. The first send is past the wait's bound.
|
||||
sent := time.Now().UTC().Add(-planWaitBound - time.Minute)
|
||||
judged := sent.Add(2 * time.Minute)
|
||||
lead.FirstAt, carried.FirstAt, lead.Gate.Since = &sent, &sent, &sent
|
||||
lead.Gate.Verdict, lead.Gate.Why, lead.Gate.JudgedAt, lead.Gate.Kept = inventory.GatePassed,
|
||||
"healthy 3 times over 2m5s", &judged, true
|
||||
carried.Gate = nil
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Another walk's send reached anchor, which has not reported on it.
|
||||
if err := inv.RecordSent(ctx, nodeID(t, tm.open, "anchor"), "d-anchor-elsewhere",
|
||||
map[string]string{"m01": "c2", "m02": "c2"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
advancePlans(ctx, tm.open)
|
||||
p = tm.plan(t)
|
||||
if p.State == inventory.PlanFailed {
|
||||
t.Fatalf("the plan failed after its gate passed: %s", p.Note)
|
||||
}
|
||||
if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"laptop"}}) {
|
||||
t.Fatalf("sent %v: the rest once, as the pass said", tm.sent)
|
||||
}
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, m := range []string{"m01", "m02"} {
|
||||
if current[m].Commit != "c2" {
|
||||
t.Errorf("%s was put back to %s after its gate passed", m, current[m].Commit)
|
||||
}
|
||||
if failed, _ := inv.GateFailed(ctx, "build-"+m+"-2"); failed {
|
||||
t.Errorf("%s's build was marked failed at its gate after the gate passed", m)
|
||||
}
|
||||
}
|
||||
if g := p.Modules["m02"].Gate; g == nil || g.Verdict != inventory.GatePassed {
|
||||
t.Errorf("m02 did not take over m01's pass: %+v", g)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// TestReplay336 replays novox/hq issue 336 (2026-10-08): a catalogue merge built a new bus build for the
|
||||
// control-node; from then on every send to that machine was refused for the bus's planned step, which only a
|
||||
// person starts, and for 28 minutes nothing but the walks' notes said so. The outcome asserted: the first
|
||||
// watchdog tick after the first refusal opens a condition for the operator that names what waits and the verb
|
||||
// that ends it, and it clears once the bus's machine runs the new build. Written with only what the controller
|
||||
// had before its fix — the stores, register, assign, a plan saved and advanced, the watchdogs' gathering and
|
||||
// seeing — so it is laid over the commit before.
|
||||
func TestReplay336(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
bus := catalogue.Manifest{Module: "nats", Version: "2", Provides: []catalogue.Offer{{Name: "mesh-bus"}}}
|
||||
if err := inv.RegisterModule(ctx, bus, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
|
||||
Path: "modules/nats", BuiltFrom: "32307bd1", Head: "32307bd1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "engine", Version: "1"})
|
||||
for _, m := range []string{"nats", "engine"} {
|
||||
if _, err := inv.Assign(ctx, "anchor", m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// The control-node runs the bus build it was last sent; the mesh holds a newer one, which waits for its step.
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"nats": "88135ad0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A walk of the engine, built, whose tier sends to the control-node.
|
||||
now := time.Now().UTC()
|
||||
plan := inventory.Plan{ID: "plan-engine", Repository: "novox/mesh-host", Commit: "e1e1e1e1", Created: now,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"engine"}},
|
||||
Modules: map[string]*inventory.PlanModule{"engine": {State: "built", BuiltAt: &now, Commit: "e1e1e1e1", Build: "b"}}}
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, open)
|
||||
p, err := inv.PlanByID(ctx, "plan-engine")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(p.Note, errBusWaits.Error()) {
|
||||
t.Fatalf("the walk's send to the bus's machine was not refused for the bus: %s %q", p.State, p.Note)
|
||||
}
|
||||
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store, Teller: &conditions.Told{}})
|
||||
defer k.Close(context.Background())
|
||||
w := &watchdogs{open: open, keeper: k, started: now.Add(-time.Hour)}
|
||||
waiting := func() []conditions.Condition {
|
||||
t.Helper()
|
||||
w.see(ctx, w.gather(ctx))
|
||||
all, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out []conditions.Condition
|
||||
for _, c := range all {
|
||||
if strings.HasPrefix(c.Key, "bus.") && c.Resolver == conditions.ResolverOperator {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
got := waiting()
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("the first tick after the refusal told the operator nothing about the bus's step: %d condition(s)", len(got))
|
||||
}
|
||||
for _, want := range []string{"anchor", "engine", "mesh-controller.bus", "upgrade", "32307bd1"} {
|
||||
if !strings.Contains(got[0].Summary, want) {
|
||||
t.Errorf("the condition does not say %q: %s", want, got[0].Summary)
|
||||
}
|
||||
}
|
||||
// The step taken: the control-node is sent the new bus build, and the wait is over.
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-2", map[string]string{"nats": "32307bd1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := waiting(); len(got) != 0 {
|
||||
t.Fatalf("the bus's machine runs the new build, and the operator is still asked: %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -3,16 +3,20 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// The replays of the controller's incidents (novox/hq to-be 45 §9, M9): each a scripted replay of what
|
||||
@@ -341,3 +345,241 @@ func TestReplaySilentWebAppIsRaisedWithinTwoLooks(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **R296 — a plan's clock counts from its tier, not from its last save.** On 2026-10-07 a plan for a
|
||||
// merge to the catalogue, its one module's build asked at 19:48:41 and queued at the build seat for
|
||||
// minutes, read "tier 1 of 1, building for 1s", then 4s, then 9s: the line counted from the plan's last
|
||||
// save, and every advance — on the 30-second tick and after every build outcome on the mesh — saved it
|
||||
// whether or not anything moved. A clock reset by each save is never LATE, so `plans` and `status` could
|
||||
// not say a plan was stuck. The outcome asserted: a plan in its tier for forty minutes, saved seconds ago,
|
||||
// says forty minutes and LATE.
|
||||
func TestReplay296APlansLineCountsFromWhenItEnteredItsTier(t *testing.T) {
|
||||
now := time.Date(2026, 10, 7, 20, 30, 0, 0, time.UTC)
|
||||
entered := time.Date(2026, 10, 7, 19, 48, 41, 0, time.UTC)
|
||||
asked := entered
|
||||
p := inventory.Plan{ID: "plan-296", Repository: "novox/mesh-catalog", Branch: "main", Commit: "3ad22356",
|
||||
State: inventory.PlanBuilding, Created: entered, TierEntered: entered, Updated: now.Add(-4 * time.Second),
|
||||
Tiers: [][]string{{"app"}}, Modules: map[string]*inventory.PlanModule{"app": {State: "asked", AskedAt: &asked}}}
|
||||
line := planLine(p, now)
|
||||
if !strings.Contains(line, "for 41m19s") || !strings.Contains(line, "LATE") {
|
||||
t.Fatalf("a plan in its tier since 19:48:41, read at 20:30:00 and saved 4s before, reads %q", line)
|
||||
}
|
||||
}
|
||||
|
||||
// **R296, the save.** The same live case from the store's side: a plan whose only build was asked and
|
||||
// still waits is advanced as the tick and the mesh's build outcomes advance it, and is not written again —
|
||||
// its revision and its last save stay where its last change left them. Before the fix each advance saved
|
||||
// it, and its revision went from 3 to 6.
|
||||
func TestReplay296APlanStandingStillInItsTierIsNotSavedAgain(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asked := asksRecorded(t)
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "3ad22356aa",
|
||||
Paths: []string{"modules/app/index.ts"}, ModuleDirs: []string{"modules/app"}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
recent, err := inv.RecentPlans(ctx, 1)
|
||||
if err != nil || len(recent) != 1 || len(*asked) != 1 {
|
||||
t.Fatalf("no plan waiting on its build: %v %v, asked %v", recent, err, *asked)
|
||||
}
|
||||
before := recent[0]
|
||||
if before.State != inventory.PlanBuilding || before.Modules["app"] == nil || before.Modules["app"].State != "asked" {
|
||||
t.Fatalf("the plan is not waiting on its build: %+v", before)
|
||||
}
|
||||
for range 3 {
|
||||
advanceHeld(ctx, open)
|
||||
}
|
||||
after, err := inv.PlanByID(ctx, before.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if after.Revision != before.Revision || !after.Updated.Equal(before.Updated) {
|
||||
t.Fatalf("a plan that did not move was saved again: revision %d → %d, saved %s → %s",
|
||||
before.Revision, after.Revision, before.Updated, after.Updated)
|
||||
}
|
||||
}
|
||||
|
||||
// **R299 — a holder of a seat whose verbs are all optional is not silent.** On 2026-10-07 the
|
||||
// node-uplink seat gained its first verbs, `resolvers` and `links`, both optional while its holders catch
|
||||
// up (ADR 0246, step 1); its holders served neither, so they registered nothing on the bus's discovery for
|
||||
// it, and the self-check's D3 raised `seat.node-uplink.<machine>.silent` on every machine of the mesh,
|
||||
// which the healer then acted on. The outcome asserted: through D3 itself — the seats read back from the
|
||||
// store as the controller reads them, a holder on record on every machine heard from, and nothing
|
||||
// answering the discovery — no holder of a seat whose verbs are all optional is said silent.
|
||||
//
|
||||
// The live seat is replayed while its verbs are all optional; beside it, a seat whose verbs are all
|
||||
// optional by definition, so the outcome is still held once node-uplink's verbs are required (step 3).
|
||||
func TestReplay299AHolderOfASeatWhoseVerbsAreAllOptionalIsNotSilent(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
catchingUp := catalogue.Seat{Name: "node-catching-up", Scope: catalogue.ScopeNode, Decision: "novox/hq issue 299",
|
||||
Serves: []catalogue.Verb{{Name: "first", Optional: true}, {Name: "second", Optional: true}}}
|
||||
if _, err := inv.SeedSeats(ctx, append(catalogue.DefaultSeats(), catchingUp)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rows, err := inv.Seats(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The working set as the serving controller holds it: the store's rows, the compiled seats' marks.
|
||||
// A seat the compiled set does not know keeps its mark only as defined here.
|
||||
for i := range rows {
|
||||
if rows[i].Name == catchingUp.Name {
|
||||
rows[i].Serves = catchingUp.Serves
|
||||
}
|
||||
}
|
||||
catalogue.UseSeats(rows)
|
||||
t.Cleanup(func() { catalogue.UseSeats(catalogue.DefaultSeats()) })
|
||||
|
||||
seats := []string{catchingUp.Name}
|
||||
if uplink, ok := catalogue.SeatNamed("node-uplink"); ok && len(uplink.Serves) > 0 {
|
||||
allOptional := true
|
||||
for _, v := range uplink.Serves {
|
||||
allOptional = allOptional && v.Optional
|
||||
}
|
||||
if allOptional {
|
||||
seats = append(seats, "node-uplink")
|
||||
} else {
|
||||
t.Logf("node-uplink's verbs are required at this commit; the live seat is not replayed, its kind is")
|
||||
}
|
||||
}
|
||||
|
||||
register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1"})
|
||||
now := time.Now()
|
||||
var heard []machineFacts
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.Assign(ctx, node, "networkmanager"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, seat := range seats {
|
||||
if err := inv.HoldSeat(ctx, seat, catalogue.ScopeNode, node, "networkmanager"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
heard = append(heard, machineFacts{name: node, lastHeard: now})
|
||||
}
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
d := &doctor{open: open, js: js, watchdogs: &watchdogs{last: &signalFacts{now: now, machines: heard}}}
|
||||
|
||||
found, err := probeHolders(ctx, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, o := range found {
|
||||
for _, seat := range seats {
|
||||
if strings.HasPrefix(o.ID, seat+".") {
|
||||
t.Errorf("a holder serving none of a seat's optional verbs was said silent: %s (%s)", o.Summary, o.Said)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **R300 — a merge that adds a module builds it.** On 2026-10-07 a pull request to the catalogue added
|
||||
// one module, systemd-resolved; its delivery plan said "builds new: modules/systemd-resolved, sent
|
||||
// nowhere", and at the merge the controller said it "changed nothing any module the mesh holds is built
|
||||
// from" and built nothing, so the module was never registered and `assign` refused it until a person
|
||||
// built it by hand. The outcome asserted: the merge asks the build seat for the new module's directory, at
|
||||
// the branch merged into, from the repository as the mesh spells it, and opens no plan for it.
|
||||
func TestReplay300AMergeAddingAModuleAsksForItsBuild(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
var asked [][3]string
|
||||
was := askABuild
|
||||
askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) {
|
||||
asked = append(asked, [3]string{source.Repository, path, ref})
|
||||
return "build-" + path, nil
|
||||
}
|
||||
t.Cleanup(func() { askABuild = was })
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "networkmanager", Version: "1"},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/networkmanager", Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "3da80a4b00",
|
||||
Paths: []string{"modules/systemd-resolved/module.json", "modules/systemd-resolved/cmd/main.go"},
|
||||
ModuleDirs: []string{"modules/systemd-resolved"}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := [3]string{"novox/mesh-catalog", "modules/systemd-resolved", "main"}
|
||||
if len(asked) != 1 || asked[0] != want {
|
||||
t.Fatalf("the merge adding modules/systemd-resolved asked the build seat for %v, not %v", asked, want)
|
||||
}
|
||||
if plans, err := inv.OpenPlans(ctx); err != nil || len(plans) != 0 {
|
||||
t.Fatalf("a merge moving nothing the mesh holds opened a plan: %+v %v", plans, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **R301 — a person's push of a held recorded build is no repair.** On 2026-10-07 the operator approved,
|
||||
// machine by machine, new builds of modules whose upgrade policy records rather than rolls out — the
|
||||
// resolver, the FortiClient adapter, the network managers, the packet filter, mail. A recorded build
|
||||
// moves only by a person's push (ADR 0242), so each was `push <machine>` through the seat with the
|
||||
// operator's yes as its why and a word for the change as its cause. The hand-act log counted every one
|
||||
// as a repair, and S15 raised `mesh.hand-acts.split-dns.healer-wanted` ("repaired by hand 4 times in 14
|
||||
// days … a healer is wanted"), and the same for `words` and `uplink-verbs`. The outcome asserted: a push
|
||||
// that carries only held recorded builds, recorded as the push records it, wants no healer however
|
||||
// often the person gives that word — through S15 itself, reading the log as the watchdog reads it.
|
||||
func TestReplay301APersonsPushOfAHeldRecordedBuildIsNoRepair(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
start := time.Now().Add(-time.Hour)
|
||||
image := "registry.invalid:5000/resolver/server@sha256:" + strings.Repeat("e", 64)
|
||||
if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, aContainerBuild(t, "resolver", "c1111111", catalogue.PolicyRecord, start,
|
||||
map[string][2]string{"server": {image, ""}}))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.Assign(ctx, node, "resolver"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, node), "d-"+node, map[string]string{"resolver": "c1111111"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// The merge: the resolver's new build, registered and held for a person's word.
|
||||
if _, _, err := takeIn(ctx, inv, aContainerBuild(t, "resolver", "c2222222", catalogue.PolicyRecord,
|
||||
start.Add(time.Minute), map[string][2]string{"server": {image, "runtime"}})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
conn := onATestBus(t)
|
||||
// The operator's yes, machine by machine, as the seat's push records it before it sends.
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
set := flag.NewFlagSet("push", flag.ContinueOnError)
|
||||
why := addHandActFlags(set)
|
||||
if err := set.Parse([]string{"--why", "operator's yes: the resolver's new build on " + node,
|
||||
"--cause", "split-dns"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
why.record(ctx, "push", []string{node})
|
||||
}
|
||||
acts, err := link.HandActs(ctx, conn, start)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(acts) != 2 {
|
||||
t.Fatalf("the two pushes were recorded as %d act(s): %+v", len(acts), acts)
|
||||
}
|
||||
f := calm(time.Now())
|
||||
f.handActs = acts
|
||||
for _, o := range watchHandActs(f) {
|
||||
if strings.Contains(o.ID, "split-dns") {
|
||||
t.Fatalf("a person's push of a held recorded build was counted as a repair: %s — %s", o.ID, o.Summary)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -131,7 +131,10 @@ func readinessOf(ctx context.Context, inv *inventory.Inventory) (broker.Readines
|
||||
// **Dialled the way the mesh dials it** — credential and pin — because a bare connect to a
|
||||
// bus that requires TLS and a user fails at the handshake, and the check then reported a
|
||||
// standing server as absent (seen live, 2026-09-28).
|
||||
if js, err := broker.Dial(address, nats.Timeout(5*time.Second)); err == nil {
|
||||
// The serving controller's own connection answers it without a second (novox/hq issue 327).
|
||||
if serving := servingBus.Load(); serving != nil && serving.Conn().IsConnected() {
|
||||
state.ServerStanding = true
|
||||
} else if js, err := broker.Dial(address, nats.Timeout(5*time.Second)); err == nil {
|
||||
state.ServerStanding = true
|
||||
js.Close()
|
||||
}
|
||||
|
||||
@@ -54,7 +54,7 @@ func TestARebuildOfAnUnchangedSourceKeepsItsArtifacts(t *testing.T) {
|
||||
// Another module's merge rebuilt it: a new commit, a new image digest, the same source.
|
||||
anImageBuild(t, "app", "", "c2bbbbbb", strings.Repeat("b", 64), "src1:same", start.Add(time.Minute)),
|
||||
} {
|
||||
if _, _, err := takeIn(ctx, inv, b); err != nil {
|
||||
if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil {
|
||||
t.Fatalf("build %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
@@ -114,7 +114,7 @@ func TestABusRebuiltFromAnUnchangedSourceDemandsNoBusStep(t *testing.T) {
|
||||
b.Manifest = manifest
|
||||
return b
|
||||
}
|
||||
if _, _, err := takeIn(ctx, inv, bus("", "n1111111", strings.Repeat("a", 64), "src1:bus", start)); err != nil {
|
||||
if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, bus("", "n1111111", strings.Repeat("a", 64), "src1:bus", start))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "anchor", "nats"); err != nil {
|
||||
|
||||
@@ -26,10 +26,29 @@ import (
|
||||
// shape fails the suite, naming its source. The keeper would say such a summary in words at run time;
|
||||
// this is where the producer is made to say it rightly in the first place.
|
||||
func TestMain(m *testing.M) {
|
||||
// The process a mesh-cli test runs as a command line: it says the verb and the caller it was given, and
|
||||
// ends (meshcli_test.go).
|
||||
if os.Getenv(echoEnvironment) != "" {
|
||||
fmt.Printf("verb=%q caller=%q terminal=%v\n", os.Getenv("MESH_VERB"), os.Getenv("MESH_CALLER"), startedAtTheTerminal())
|
||||
os.Exit(0)
|
||||
}
|
||||
conditions.Unsayable = func(o conditions.Observation, field string, r outward.Refusal) {
|
||||
unsaid.note(o, field, r)
|
||||
}
|
||||
conditions.Unworded = func(o conditions.Observation, why string) {
|
||||
if o.Source != "test" { // a kind a test makes up to exercise the keeper
|
||||
unworded.add(o, why)
|
||||
}
|
||||
}
|
||||
code := m.Run()
|
||||
if said := unworded.all(); len(said) > 0 {
|
||||
fmt.Fprintf(os.Stderr, "FAIL: %d condition(s) raised in these tests have no plain words of their own, or "+
|
||||
"words that are not plain (novox/hq ADR 0253: a headline, an explanation and a resolved line the operator "+
|
||||
"reads at a glance — plain_words.go):\n %s\n", len(said), strings.Join(said, "\n "))
|
||||
if code == 0 {
|
||||
code = 1
|
||||
}
|
||||
}
|
||||
if said := unsaid.all(); len(said) > 0 {
|
||||
fmt.Fprintf(os.Stderr, "FAIL: %d condition(s) raised in these tests say what the operator's channel withholds "+
|
||||
"(an address, a domain, a path or a secret's shape belongs in the evidence, not the summary):\n %s\n",
|
||||
@@ -74,6 +93,34 @@ func (u *unsayable) all() []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// unworded collects, across the suite, every finding said in borrowed words (ADR 0253).
|
||||
var unworded wordless
|
||||
|
||||
type wordless struct {
|
||||
mu sync.Mutex
|
||||
seen map[string]bool
|
||||
}
|
||||
|
||||
func (u *wordless) add(o conditions.Observation, why string) {
|
||||
u.mu.Lock()
|
||||
defer u.mu.Unlock()
|
||||
if u.seen == nil {
|
||||
u.seen = map[string]bool{}
|
||||
}
|
||||
u.seen[fmt.Sprintf("kind %s (scope %s, source %s): %s", o.Kind, o.Scope, o.Source, why)] = true
|
||||
}
|
||||
|
||||
func (u *wordless) all() []string {
|
||||
u.mu.Lock()
|
||||
defer u.mu.Unlock()
|
||||
var out []string
|
||||
for s := range u.seen {
|
||||
out = append(out, s)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// linted is a producer's findings, held to the content rule as a keeper would hold them: for a test
|
||||
// that reads a producer's findings without raising them.
|
||||
func linted(obs []conditions.Observation) []conditions.Observation {
|
||||
@@ -85,6 +132,18 @@ func linted(obs []conditions.Observation) []conditions.Observation {
|
||||
if r, ok := outward.Check(o.Summary, machines...); !ok {
|
||||
unsaid.note(o, "summary", r)
|
||||
}
|
||||
switch {
|
||||
case o.Headline != "":
|
||||
w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved}
|
||||
if w.Resolved == "" {
|
||||
w.Resolved = "Resolved"
|
||||
}
|
||||
if why, ok := conditions.PlainWords(w, machines...); !ok {
|
||||
unworded.add(o, "its source's words are not plain: "+why)
|
||||
}
|
||||
case o.Kind != "" && !conditions.Worded(o.Kind):
|
||||
unworded.add(o, "the kind "+o.Kind+" has no plain words")
|
||||
}
|
||||
}
|
||||
return obs
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"slices"
|
||||
@@ -28,6 +29,9 @@ import (
|
||||
// answer. It also means a refusal is the same refusal in the same words, because it is the same
|
||||
// output.
|
||||
|
||||
// verbKey carries the verb a call is for, to the process it runs.
|
||||
type verbKey struct{}
|
||||
|
||||
// verbAnswer is what a verb answers: what the command printed, whether it succeeded, and — where the
|
||||
// command speaks JSON — the same as data.
|
||||
type verbAnswer struct {
|
||||
@@ -51,6 +55,9 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||
return nil, err
|
||||
}
|
||||
argv, err := a.commandLine()
|
||||
if err == nil {
|
||||
err = terminalOnly(argv)
|
||||
}
|
||||
if len(a.misread) > 0 {
|
||||
// The table and the command line disagree: the verb reads an argument no caller can see
|
||||
// in its schema, so no caller could ever pass it.
|
||||
@@ -222,6 +229,35 @@ func quoteAll(xs []string) string {
|
||||
return strings.Join(q, ", ")
|
||||
}
|
||||
|
||||
// refusedAsTheGenericCommand is what the generic `command` verb refuses of a command line, and so what every
|
||||
// line asked through a verb's route refuses: the `command` verb's, and an ordinary line from mesh-cli (novox/hq ADR
|
||||
// 0272 §4). One function, so the two routes cannot drift apart.
|
||||
func refusedAsTheGenericCommand(argv []string) error {
|
||||
if len(argv) == 0 {
|
||||
return errors.New("command names no command")
|
||||
}
|
||||
// A layer is written through the settings verb, never the generic one (novox/hq issue 339): the
|
||||
// settings verb is where what a verb may not set is refused, and one route is one set of words.
|
||||
// The command refuses places and accesses through any verb as well; this says so before it runs.
|
||||
if argv[0] == "settings" && slices.ContainsFunc(argv[1:], func(w string) bool { return w == "set" || w == "clear" }) {
|
||||
return &heldAtTheTerminal{msg: "settings are set and cleared through the settings verb, not the " +
|
||||
"generic command; and places and accesses only at the controller's terminal (novox/hq issue 339). " +
|
||||
"Nothing was done"}
|
||||
}
|
||||
// The generic verb only reads (novox/hq ADR 0266): what writes has a named verb that composes its own
|
||||
// line, or is the operator's at the controller's terminal.
|
||||
if err := commandReads(argv); err != nil {
|
||||
return err
|
||||
}
|
||||
// The generic verb is no way round the hand-act log (novox/hq to-be 45 §7): a repair through
|
||||
// it says why, as it would through its own verb.
|
||||
if repair := repairingCommand(argv); repair != "" && !slices.ContainsFunc(argv, isWhyFlag) {
|
||||
return fmt.Errorf("%s is a repair done by hand, and says why: add --why <text> to the command "+
|
||||
"line (recorded in the hand-act log). Nothing was done", repair)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// commandLine composes the command. Every argument it reads is one it uses: a branch that reads an
|
||||
// argument and then drops it would pass it over, which is what the check after it exists to refuse.
|
||||
func (a *verbArguments) commandLine() ([]string, error) {
|
||||
@@ -238,18 +274,14 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(argv) == 0 {
|
||||
return nil, errors.New("command names no command")
|
||||
}
|
||||
// The generic verb is no way round the hand-act log (novox/hq to-be 45 §7): a repair through
|
||||
// it says why, as it would through its own verb.
|
||||
if repair := repairingCommand(argv); repair != "" && !slices.ContainsFunc(argv, isWhyFlag) {
|
||||
return nil, fmt.Errorf("%s is a repair done by hand, and says why: add --why <text> to the command "+
|
||||
"line (recorded in the hand-act log). Nothing was done", repair)
|
||||
if err := refusedAsTheGenericCommand(argv); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return argv, nil
|
||||
case "tools":
|
||||
return nil, errors.New("tools is answered from the records, not by a command")
|
||||
case "dead-letters":
|
||||
return nil, errors.New("dead-letters is answered by the serving controller, on its own connection, not by a command")
|
||||
case "status":
|
||||
return []string{"status", "--json"}, nil
|
||||
case "nodes":
|
||||
@@ -407,6 +439,10 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
if on("files") {
|
||||
return []string{"plan", str("node"), "--files"}, nil
|
||||
}
|
||||
// What a push would change there (novox/hq ADR 0217); diff with files is passed over, and so refused.
|
||||
if on("diff") {
|
||||
return []string{"plan", str("node"), "--diff"}, nil
|
||||
}
|
||||
return []string{"plan", str("node"), "--json"}, nil
|
||||
case "assign", "unassign":
|
||||
if err := need("node", "module"); err != nil {
|
||||
@@ -414,7 +450,13 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
}
|
||||
// Several modules comma-separated, judged as one act (novox/hq ADR 0207): the holders of
|
||||
// the seats that apply resources depend on each other and go on together.
|
||||
return append([]string{verb, str("node")}, splitModules(str("module"))...), nil
|
||||
argv := append([]string{verb, str("node")}, splitModules(str("module"))...)
|
||||
// A module known and not built: its build asked for, the assignment pending on it (novox/hq
|
||||
// issue 325). unassign declares no build, so a call giving it is refused before this.
|
||||
if verb == "assign" && on("build") {
|
||||
argv = append(argv, "--build")
|
||||
}
|
||||
return argv, nil
|
||||
case "pin":
|
||||
if err := need("node", "provision", "from", "module"); err != nil {
|
||||
return nil, err
|
||||
@@ -441,10 +483,16 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
// behind is not read here: given with a machine, it is refused as passed over — naming
|
||||
// a machine and asking for every machine behind are two requests, and guessing one
|
||||
// would push a machine nobody named, or not push one somebody did.
|
||||
return append([]string{"push", n, "--wait", "0"}, why...), nil
|
||||
argv := []string{"push", n, "--wait", "0"}
|
||||
// A running module's data moving is sent only when said, per module (novox/hq ADR 0217).
|
||||
if m := str("move"); m != "" {
|
||||
argv = append(argv, "--move", m)
|
||||
}
|
||||
return append(argv, why...), nil
|
||||
}
|
||||
// No machine: the whole mesh, whether or not behind said so. The command's answer says it
|
||||
// first, so a caller who meant one machine reads that it was not one.
|
||||
// first, so a caller who meant one machine reads that it was not one. move is not read: a
|
||||
// machine whose data would move is held and named, and sent by a push that names it.
|
||||
on("behind")
|
||||
return append([]string{"push", "--behind", "--wait", "0"}, why...), nil
|
||||
case "hand-act":
|
||||
@@ -563,6 +611,62 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
return argv, nil
|
||||
}
|
||||
return []string{"cleanup", "list", "--json"}, nil
|
||||
// novox/hq ADR 0251.
|
||||
case "artifacts":
|
||||
argv := []string{"artifacts", "--json"}
|
||||
if r := str("repository"); r != "" {
|
||||
argv = append(argv, "--repository", r)
|
||||
}
|
||||
if on("collected") {
|
||||
argv = append(argv, "--collected")
|
||||
}
|
||||
return argv, nil
|
||||
case "collect":
|
||||
argv := []string{"collect", "--json"}
|
||||
if m := str("most"); m != "" {
|
||||
argv = append(argv, "--most", m)
|
||||
}
|
||||
why := str("why")
|
||||
if !on("confirm") {
|
||||
if why != "" {
|
||||
return nil, errors.New("collect takes why only with confirm: without confirm it is a dry run, " +
|
||||
"and a reason for nothing would be recorded nowhere. Nothing was done")
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
if err := need("why"); err != nil {
|
||||
return nil, fmt.Errorf("%w: letting go of artifacts is a hand act, which says why. Nothing was done", err)
|
||||
}
|
||||
return append(argv, "--confirm", "--why", why), nil
|
||||
case "images":
|
||||
if err := need("node"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []string{"images", str("node"), "--json"}, nil
|
||||
case "mirrors":
|
||||
// novox/hq ADR 0257.
|
||||
argv := []string{"mirrors", "--json"}
|
||||
record := str("record")
|
||||
why := str("why")
|
||||
if record == "" {
|
||||
if on("confirm") || why != "" {
|
||||
return nil, errors.New("mirrors takes confirm and why only with record: there is nothing " +
|
||||
"else it records. Nothing was done")
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
argv = append(argv, "--record", record)
|
||||
if !on("confirm") {
|
||||
if why != "" {
|
||||
return nil, errors.New("mirrors takes why only with confirm: without confirm it is a dry run, " +
|
||||
"and a reason for nothing would be recorded nowhere. Nothing was done")
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
if err := need("why"); err != nil {
|
||||
return nil, fmt.Errorf("%w: recording a copy as the mesh's is a hand act, which says why. Nothing was done", err)
|
||||
}
|
||||
return append(argv, "--confirm", "--why", why), nil
|
||||
case "data":
|
||||
argv := []string{"data", "--json"}
|
||||
if m := str("machine"); m != "" {
|
||||
@@ -631,6 +735,9 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
if which > 1 {
|
||||
return nil, errors.New("doctor answers one of run, probes or signals at a time")
|
||||
}
|
||||
if p := str("probe"); p != "" {
|
||||
argv = append(argv, "--probe", p)
|
||||
}
|
||||
return append(argv, "--json"), nil
|
||||
case "rotate":
|
||||
if p := str("provision"); p != "" {
|
||||
@@ -665,20 +772,70 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
// Neither shape: the command says its usage, which names both, and that is the answer the
|
||||
// caller needs.
|
||||
return []string{"rotate"}, nil
|
||||
case "token":
|
||||
// `token issue` at a shell (novox/hq ADR 0169). Exactly one of node or new; the command
|
||||
// refuses both or neither in its own words.
|
||||
argv := []string{"token", "issue"}
|
||||
if n := str("node"); n != "" {
|
||||
argv = append(argv, "--node", n)
|
||||
}
|
||||
if n := str("new"); n != "" {
|
||||
argv = append(argv, "--new", n)
|
||||
}
|
||||
if k := str("overlay_key"); k != "" {
|
||||
argv = append(argv, "--overlay-key", k)
|
||||
}
|
||||
if d := str("for"); d != "" {
|
||||
argv = append(argv, "--for", d)
|
||||
}
|
||||
if str("adopted") == "true" {
|
||||
argv = append(argv, "--adopted")
|
||||
}
|
||||
return argv, nil
|
||||
case "settings":
|
||||
// Every module's preferences, their defaults and each machine's value (novox/hq ADR 0262):
|
||||
// the one interface for them, so no module builds a settings tool of its own. Asked for by
|
||||
// name, or by naming no module, since a layer is always some module's.
|
||||
if str("module") == "" && str("values") != "" {
|
||||
return nil, errors.New("settings: a module is needed to set values; name it with module")
|
||||
}
|
||||
if str("module") == "" && on("clear") {
|
||||
return nil, errors.New("settings: a module is needed to clear a layer; name it with module")
|
||||
}
|
||||
if list := str("list"); list != "" || str("module") == "" {
|
||||
if list != "" && list != "preferences" {
|
||||
return nil, fmt.Errorf("settings lists %q only; %q is not a listing", "preferences", list)
|
||||
}
|
||||
argv := []string{"settings", "preferences"}
|
||||
if m := str("module"); m != "" {
|
||||
argv = append(argv, m)
|
||||
}
|
||||
if n := str("node"); n != "" {
|
||||
argv = append(argv, "--node", n)
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
// `settings set|clear` at a shell (novox/hq issue 198). The values travel as an argument
|
||||
// because a tool has no file to hand the command; the command reads either.
|
||||
if err := need("module"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var argv []string
|
||||
if on("clear") {
|
||||
switch {
|
||||
case on("clear"):
|
||||
argv = []string{"settings", "clear", str("module")}
|
||||
} else {
|
||||
argv = []string{"settings", "set", str("module")}
|
||||
// Neither values nor clear: the command says its usage, which names both.
|
||||
if v := str("values"); v != "" {
|
||||
argv = append(argv, v)
|
||||
case str("values") != "":
|
||||
argv = []string{"settings", "set", str("module"), str("values")}
|
||||
// What a set removes is refused unless meant (novox/hq ADR 0217).
|
||||
if on("replace") {
|
||||
argv = append(argv, "--replace")
|
||||
}
|
||||
default:
|
||||
// Neither values nor clear: the layer as it stands, which is what a caller reads before
|
||||
// replacing it (novox/hq ADR 0217) — and with history, the layers it replaced.
|
||||
argv = []string{"settings", "show", str("module")}
|
||||
if on("history") {
|
||||
argv = append(argv, "--history")
|
||||
}
|
||||
}
|
||||
if n := str("node"); n != "" {
|
||||
@@ -694,8 +851,19 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
}
|
||||
return []string{"module", "issue", str("module"), "--node", str("node")}, nil
|
||||
case "build":
|
||||
if err := need("repository"); err != nil {
|
||||
return nil, err
|
||||
// The command's three shapes (`build --on`, `build --behind`, `build <repository>`), one per
|
||||
// call: each branch reads only its own argument, so another given beside it is refused as
|
||||
// passed over rather than dropped. Asked and not waited for, the same as a single build.
|
||||
if b := str("on"); b != "" {
|
||||
return []string{"build", "--on", b, "--wait", "0"}, nil
|
||||
}
|
||||
if on("behind") {
|
||||
return []string{"build", "--behind", "--wait", "0"}, nil
|
||||
}
|
||||
if a.given["repository"] == "" {
|
||||
// No shape named: the command says its usage, which names all three — the answer the
|
||||
// caller needs, and the same as `rotate` given neither of its shapes.
|
||||
return []string{"build"}, nil
|
||||
}
|
||||
// Not waited for: a tool call cannot hold a connection for the minutes a build takes; the
|
||||
// daemon takes the outcome in when it comes and the id follows the build (issue 176). A
|
||||
@@ -719,9 +887,16 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
// jsonVerbs are the verbs whose command speaks JSON, so the answer carries it as data as well.
|
||||
var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "collection": true,
|
||||
"hand-acts": true, "durations": true, "conditions": true, "doctor": true, "retire": true, "cleanup": true, "data": true,
|
||||
// What the records say the registries may keep (novox/hq ADR 0251).
|
||||
"artifacts": true, "collect": true, "images": true, "mirrors": true,
|
||||
// The delivery's owner's verbs answer JSON where they read (plan, order, check, walks) — novox/hq ADR 0239.
|
||||
"delivery": true}
|
||||
|
||||
// overviewVerbs are the JSON verbs whose answer is said once, as data, and not again as the text the
|
||||
// command printed: the overviews, which grow with the mesh (novox/hq issue 314). Every reader of theirs
|
||||
// reads `answer` first.
|
||||
var overviewVerbs = map[string]bool{"status": true, "conditions": true, "doctor": true}
|
||||
|
||||
// repairingCommand names a command line that repairs by hand, and so says why: a push, a plan stopped
|
||||
// or closed, a consumer re-made (novox/hq to-be 45 §7). Empty for any other.
|
||||
func repairingCommand(argv []string) string {
|
||||
@@ -742,6 +917,10 @@ func repairingCommand(argv []string) string {
|
||||
return "retire " + argv[1]
|
||||
case argv[0] == "cleanup" && len(argv) > 1 && argv[1] == "delete":
|
||||
return "cleanup delete"
|
||||
case argv[0] == "collect" && slices.Contains(argv, "--confirm"):
|
||||
return "collect"
|
||||
case argv[0] == "mirrors" && slices.Contains(argv, "--confirm"):
|
||||
return "mirrors"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -750,6 +929,30 @@ func isWhyFlag(word string) bool {
|
||||
return word == "--why" || word == "-why" || strings.HasPrefix(word, "--why=") || strings.HasPrefix(word, "-why=")
|
||||
}
|
||||
|
||||
// commandEnvironment is the environment of a command line this controller runs for someone: its own — the
|
||||
// stores' credentials, the bus, the broker, everything a command run from a shell beside it would have, because it
|
||||
// is that — with who asked, and how it came.
|
||||
//
|
||||
// **terminal** is said, never inferred (novox/hq ADR 0272): only a line mesh-cli asked as the controller's terminal
|
||||
// passes true. Its line has no `MESH_VERB`, not the served mark ADR 0266 puts on everything the serving controller
|
||||
// starts, and the terminal's mark (cliTerminalVar), so startedAtTheTerminal reads yes. Every other line names its
|
||||
// verb, and is stripped of the terminal's mark whatever this process's environment holds.
|
||||
func commandEnvironment(caller, verb string, terminal bool) []string {
|
||||
env := make([]string, 0, len(os.Environ())+3)
|
||||
for _, kv := range os.Environ() {
|
||||
if strings.HasPrefix(kv, verbVar+"=") || strings.HasPrefix(kv, link.CallerVar+"=") ||
|
||||
strings.HasPrefix(kv, cliTerminalVar+"=") || (terminal && strings.HasPrefix(kv, servedVar+"=")) {
|
||||
continue
|
||||
}
|
||||
env = append(env, kv)
|
||||
}
|
||||
env = append(env, link.CallerVar+"="+caller)
|
||||
if terminal {
|
||||
return append(env, cliTerminalVar+"=1")
|
||||
}
|
||||
return append(env, verbVar+"="+verb)
|
||||
}
|
||||
|
||||
// runVerb runs this binary with the given command line and gathers what it said.
|
||||
//
|
||||
// **This binary is the image this process runs, never the file at the path it started from**
|
||||
@@ -763,15 +966,17 @@ func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
||||
return verbAnswer{}, fmt.Errorf("%w: this controller is stopping and runs no new command", link.ErrHandingOver)
|
||||
}
|
||||
cmd := selfCommand(ctx, argv)
|
||||
// The same environment: the stores' credentials, the bus, the broker — everything a command run
|
||||
// from a shell in this container would have, because it is that.
|
||||
cmd.Env = os.Environ()
|
||||
// And who asked, so an act it does by hand is recorded as theirs (novox/hq to-be 45 §7).
|
||||
caller := link.CallerIn(ctx)
|
||||
if caller == "" {
|
||||
caller = "a seat call whose caller the bus did not name"
|
||||
}
|
||||
cmd.Env = append(cmd.Env, link.CallerVar+"="+caller+", through the "+catalogue.ControllerSeatName+" seat")
|
||||
// And which verb, so the connection it dials says so in the bus's list (novox/hq issue 327).
|
||||
verb, _ := ctx.Value(verbKey{}).(string)
|
||||
if verb == "" {
|
||||
verb = argv[0]
|
||||
}
|
||||
cmd.Env = commandEnvironment(caller+", through the "+catalogue.ControllerSeatName+" seat", verb, false)
|
||||
// Two buffers, one answer. What the command *says* is both streams, in the order a person at
|
||||
// a shell would read them; what it *answers as data* is standard output alone — `status --json`
|
||||
// prints its warnings beside the document, and a JSON parsed from the two together parsed
|
||||
@@ -780,13 +985,7 @@ func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
runErr := cmd.Run()
|
||||
answer := verbAnswer{Output: stdout.String() + stderr.String(), OK: runErr == nil}
|
||||
if jsonVerbs[argv[0]] && runErr == nil {
|
||||
var parsed any
|
||||
if json.Unmarshal(bytes.TrimSpace(stdout.Bytes()), &parsed) == nil {
|
||||
answer.Answer = parsed
|
||||
}
|
||||
}
|
||||
answer := answerOf(argv, stdout.Bytes(), stderr.String(), runErr == nil)
|
||||
var exit *exec.ExitError
|
||||
if runErr != nil && !errors.As(runErr, &exit) {
|
||||
// Not the command refusing — the command not running at all, which is this process's fault.
|
||||
@@ -801,6 +1000,66 @@ func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// answerOf is what a command said, as a verb answers it: both streams as text, and standard output as data
|
||||
// where the command speaks JSON.
|
||||
func answerOf(argv []string, stdout []byte, stderr string, ok bool) verbAnswer {
|
||||
answer := verbAnswer{Output: string(stdout) + stderr, OK: ok}
|
||||
if jsonVerbs[argv[0]] && ok {
|
||||
var parsed any
|
||||
if json.Unmarshal(bytes.TrimSpace(stdout), &parsed) == nil {
|
||||
answer.Answer = parsed
|
||||
if overviewVerbs[argv[0]] {
|
||||
// Once, as data: the same document again as text doubled an answer that already
|
||||
// outgrew one message of the bus (novox/hq issue 314).
|
||||
answer.Output = stderr + "its answer, as data, is `answer`\n"
|
||||
}
|
||||
}
|
||||
}
|
||||
return answer
|
||||
}
|
||||
|
||||
// readHere answers a verb that only reads the bus in the serving controller itself, on its own connection
|
||||
// and keeper (novox/hq issue 327): the same command, writing to the answer rather than to a process's
|
||||
// output, so the answer is the one the command prints. False for any other command line, which runs as a
|
||||
// command of its own. Every `conditions` call — the operator's channel reads it at least once a minute —
|
||||
// was a process that dialled the bus, logged in and left.
|
||||
func readHere(ctx context.Context, argv []string) (verbAnswer, bool) {
|
||||
var read func(context.Context, []string, io.Writer) error
|
||||
args := argv[1:]
|
||||
// Each where this process holds what it reads: the serving keeper, the hand-act log's connection, the
|
||||
// serving connection.
|
||||
switch argv[0] {
|
||||
case "conditions":
|
||||
sub := "list"
|
||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
sub, args = args[0], args[1:]
|
||||
}
|
||||
if conditionsFrom != nil {
|
||||
read = map[string]func(context.Context, []string, io.Writer) error{
|
||||
"list": listConditions, "show": showCondition, "history": conditionHistory}[sub]
|
||||
}
|
||||
case "hand-acts":
|
||||
if handActConn != nil || servingBus.Load() != nil {
|
||||
read = listHandActs
|
||||
}
|
||||
case "queue":
|
||||
if servingBus.Load() != nil {
|
||||
read = listQueue
|
||||
}
|
||||
}
|
||||
if read == nil {
|
||||
return verbAnswer{}, false
|
||||
}
|
||||
var out bytes.Buffer
|
||||
stderr := ""
|
||||
err := read(ctx, args, &out)
|
||||
if err != nil {
|
||||
// As the command says it when it fails (main).
|
||||
stderr = "mesh-controller: " + err.Error() + "\n"
|
||||
}
|
||||
return answerOf(argv, out.Bytes(), stderr, err == nil), true
|
||||
}
|
||||
|
||||
// seatToolHandlers are the handlers for every verb the mesh-controller seat declares, from the
|
||||
// store's row, so a verb the row does not carry is not served. A verb it carries that this binary
|
||||
// cannot run is named at start and answers the reason when called — never a refusal to serve, which
|
||||
@@ -830,6 +1089,9 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
||||
if verb == "calls" {
|
||||
return callsAnswer(link.Calls, a.given["call"])
|
||||
}
|
||||
if verb == "dead-letters" {
|
||||
return deadLettersAnswer(ctx, a)
|
||||
}
|
||||
if verb == "doctor" {
|
||||
// From the serving controller, which runs the self-check and hears the signals
|
||||
// (novox/hq to-be 45 §4): the last verdict at once, or a run now.
|
||||
@@ -838,16 +1100,17 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
||||
return nil, err
|
||||
}
|
||||
sub := ""
|
||||
if len(argv) > 2 {
|
||||
if len(argv) > 2 && !strings.HasPrefix(argv[1], "-") {
|
||||
sub = argv[1]
|
||||
}
|
||||
return doctorAnswer(ctx, sub)
|
||||
return doctorAnswer(ctx, sub, a.given["probe"])
|
||||
}
|
||||
return seatTools(), nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
if _, err := argvFor(verb, sampleArguments(v)); err != nil {
|
||||
var policy *heldAtTheTerminal
|
||||
if _, err := argvFor(verb, sampleArguments(v)); err != nil && !errors.As(err, &policy) {
|
||||
// **A row ahead of this binary is not a reason to go silent.**
|
||||
//
|
||||
// The row is the store's and a control plane follows it (novox/hq ADR 0154), so a verb
|
||||
@@ -881,6 +1144,7 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx = context.WithValue(ctx, verbKey{}, verb)
|
||||
if verb == "status" && statusFrom != nil {
|
||||
// At once, from the summary the serving controller keeps (novox/hq to-be 45 Phase 0).
|
||||
return statusFrom.answer(ctx)
|
||||
@@ -889,6 +1153,9 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
||||
// Whatever it did, `status` is composed again once it has.
|
||||
defer statusFrom.nudge()
|
||||
}
|
||||
if answer, read := readHere(ctx, argv); read {
|
||||
return answer, nil
|
||||
}
|
||||
if answersFirst(argv) {
|
||||
// Before anything is sent: a push sends the bus's own machine first, and a broker
|
||||
// reloading its user list forgets the answer it was about to permit (novox/hq issue 265).
|
||||
@@ -912,7 +1179,10 @@ func actsOnAPlan(args map[string]any) bool {
|
||||
|
||||
// inProcess are the verbs answered by this process rather than by a command it runs: `tools` from
|
||||
// the records, `calls` from what this process served.
|
||||
var inProcess = map[string]bool{"tools": true, "calls": true, "doctor": true}
|
||||
var inProcess = map[string]bool{"tools": true, "calls": true, "doctor": true,
|
||||
// What a consumer gave up on, read and changed on the serving controller's own connection (novox/hq
|
||||
// issue 330).
|
||||
"dead-letters": true}
|
||||
|
||||
// answersFirst is a command line whose caller is answered before it runs: a push, by its verb or
|
||||
// through `command`. A push sends the machine holding the bus first when its user list changed, the
|
||||
@@ -928,7 +1198,7 @@ func answersFirst(argv []string) bool {
|
||||
// the reason said beside it.
|
||||
func callsAnswer(log *link.CallLog, id string) (any, error) {
|
||||
if id != "" {
|
||||
c, ok, err := log.Get(id)
|
||||
c, ok, err := log.Shown(id)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("call %s is not in this controller's memory, and the calls kept on the "+
|
||||
"bus could not be read: %w", id, err)
|
||||
@@ -970,9 +1240,15 @@ func seatTools() map[string]any {
|
||||
}
|
||||
var tools []map[string]any
|
||||
for _, v := range s.Serves {
|
||||
tools = append(tools, map[string]any{
|
||||
tool := map[string]any{
|
||||
"name": v.Name, "description": v.Description, "input": v.Input, "output": v.Output,
|
||||
})
|
||||
}
|
||||
// What the verb is the mesh's way to do (novox/hq ADR 0245): read by the console's search and
|
||||
// the agent's instructions.
|
||||
if len(v.Replaces) > 0 {
|
||||
tool["replaces"] = v.Replaces
|
||||
}
|
||||
tools = append(tools, tool)
|
||||
}
|
||||
seats = append(seats, map[string]any{"seat": s.Name, "scope": s.Scope, "tools": tools})
|
||||
}
|
||||
@@ -1095,3 +1371,131 @@ func seatAnnouncement(handlers map[string]link.ToolHandler) micro.Info {
|
||||
Endpoints: endpoints,
|
||||
}
|
||||
}
|
||||
|
||||
// nodeReads are the `node` subcommands a verb may run: the ones that only read.
|
||||
var nodeReads = map[string]bool{"list": true, "show": true}
|
||||
|
||||
// flagsOnly says a command line's rest names no subcommand: empty, or beginning with a flag. For a command
|
||||
// with no subcommands every word is a flag, its value or a name it reads.
|
||||
func flagsOnly(rest []string) bool { return len(rest) == 0 || strings.HasPrefix(rest[0], "-") }
|
||||
|
||||
// subIn says the rest begins with one of these subcommands.
|
||||
func subIn(rest []string, subs ...string) bool {
|
||||
return len(rest) > 0 && slices.Contains(subs, rest[0])
|
||||
}
|
||||
|
||||
// commandReadForms are the command lines the generic `command` verb may run (novox/hq ADR 0266): **an allow
|
||||
// list of the ones that only read**, judged command by command. Anything else — every command that writes a
|
||||
// record, sends, builds, issues an account or a token, sets a key, accepts, rotates, recovers or exports a
|
||||
// secret — is refused, and a command added later is refused until it is judged a read. Writing has its named
|
||||
// verbs, which compose their own lines and are judged by terminalOnly; the rest is the operator's at the
|
||||
// controller's terminal.
|
||||
var commandReadForms = map[string]func(rest []string) bool{
|
||||
"status": flagsOnly, "version": flagsOnly, "help": flagsOnly, "seats": flagsOnly, "healers": flagsOnly,
|
||||
"hand-acts": flagsOnly, "durations": flagsOnly, "collection": flagsOnly, "images": flagsOnly,
|
||||
"artifacts": flagsOnly, "data": flagsOnly, "builds": flagsOnly, "queue": flagsOnly,
|
||||
// `plan <node>` previews a node's declaration; it sends nothing.
|
||||
"plan": func([]string) bool { return true },
|
||||
// `plans` lists and `plans <id>` shows one; `plans stop|close|go` acts.
|
||||
// Judged on every word, not the first: a flag before the subcommand (`plans --json go <id>`) still acts.
|
||||
"plans": func(r []string) bool {
|
||||
return !slices.ContainsFunc(r, func(w string) bool { return slices.Contains(plansActs, w) })
|
||||
},
|
||||
// `doctor` answers the last run, `probes` and `signals` describe; `doctor run` runs.
|
||||
"doctor": func(r []string) bool { return flagsOnly(r) || subIn(r, "probes", "signals") },
|
||||
"conditions": func(r []string) bool { return flagsOnly(r) || subIn(r, "list", "show", "history") },
|
||||
"node": func(r []string) bool { return subIn(r, "list", "show") },
|
||||
"module": func(r []string) bool { return subIn(r, "list") },
|
||||
"settings": func(r []string) bool { return subIn(r, "show", "preferences") },
|
||||
"retire": func(r []string) bool { return subIn(r, "list") },
|
||||
"cleanup": func(r []string) bool { return subIn(r, "list") },
|
||||
"delivery": func(r []string) bool { return subIn(r, "plan", "walks") },
|
||||
// `bus` alone says the bus's step; `bus upgrade` takes one.
|
||||
"bus": func(r []string) bool { return len(r) == 0 },
|
||||
// `mirrors` lists; --record and --confirm keep a mirror.
|
||||
"mirrors": func(r []string) bool {
|
||||
return flagsOnly(r) && !slices.ContainsFunc(r, func(w string) bool {
|
||||
return w == "--record" || w == "-record" || strings.HasPrefix(w, "--record=") || strings.HasPrefix(w, "-record=") ||
|
||||
w == "--confirm" || w == "-confirm" || strings.HasPrefix(w, "--confirm=")
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
// plansActs are the `plans` subcommands that act on a walk; no other word of a plans line is one of them.
|
||||
var plansActs = []string{"go", "stop", "close", "retry"}
|
||||
|
||||
// heldAtTheTerminal is a refusal of policy (novox/hq ADR 0266): the verb is known and served, and this line is
|
||||
// the operator's at the controller's terminal. Never read as a verb this binary is behind on.
|
||||
type heldAtTheTerminal struct{ msg string }
|
||||
|
||||
func (e *heldAtTheTerminal) Error() string { return e.msg }
|
||||
|
||||
func terminalRefusal(format string, args ...any) error {
|
||||
return &heldAtTheTerminal{fmt.Sprintf(format, args...)}
|
||||
}
|
||||
|
||||
// commandReads refuses a line the generic verb may not run, saying what it may.
|
||||
func commandReads(argv []string) error {
|
||||
if read, ok := commandReadForms[argv[0]]; ok && read(argv[1:]) {
|
||||
return nil
|
||||
}
|
||||
return terminalRefusal("%q is not a reading command, and the generic command verb only reads (novox/hq ADR 0266): "+
|
||||
"whoever may call a verb includes agents, and a line that writes, issues, sets a key or reveals a secret "+
|
||||
"would be theirs to run. Use the named verb for it, or run it at the controller's terminal. The verb may "+
|
||||
"run: %s. Nothing was done", strings.Join(argv, " "), commandReadNames())
|
||||
}
|
||||
|
||||
func commandReadNames() string {
|
||||
names := make([]string, 0, len(commandReadForms))
|
||||
for n := range commandReadForms {
|
||||
names = append(names, n)
|
||||
}
|
||||
sort.Strings(names)
|
||||
return strings.Join(names, ", ") + " (each in its reading forms)"
|
||||
}
|
||||
|
||||
// terminalOnlyCommands are the commands no verb runs, whatever composed them (novox/hq ADR 0266): they set
|
||||
// the operator's key, issue a credential or a token that is answered to the caller, or accept, recover or
|
||||
// export a secret. Their answers or effects hand whoever calls them what the runtime's account holds.
|
||||
var terminalOnlyCommands = map[string]string{
|
||||
"operator": "the operator's key and credential",
|
||||
"identity": "the mesh's identity keys",
|
||||
"token": "a token a machine joins with, answered to the caller",
|
||||
"broker": "the bus's accounts",
|
||||
"api": "the controller's API keys",
|
||||
"licence": "the licences' secrets",
|
||||
}
|
||||
|
||||
// terminalOnly refuses, through any verb, a command that is the operator's at the controller's terminal
|
||||
// alone (novox/hq ADR 0266). **Every `node` subcommand that is not a read**: `node account` and
|
||||
// `node agent-account` above all. Whoever may call a verb includes agents, and an agent that named itself
|
||||
// the operator account, or cleared the agent account, would have the next send grant it root through the
|
||||
// sudo module's rule. An allow list, so a subcommand added later is refused until it is judged a read.
|
||||
func terminalOnly(argv []string) error {
|
||||
if len(argv) == 0 {
|
||||
return nil
|
||||
}
|
||||
if what, kept := terminalOnlyCommands[argv[0]]; kept {
|
||||
return terminalRefusal("%s is run at the controller's terminal only, never through a verb: it holds %s, and "+
|
||||
"whoever may call a verb includes agents (novox/hq ADR 0266). Nothing was done", argv[0], what)
|
||||
}
|
||||
// Of a secret's commands only rotation, which seals the new value to the machine that uses it.
|
||||
if argv[0] == "secret" && (len(argv) < 2 || argv[1] != "rotate") {
|
||||
return terminalRefusal("secret %s is run at the controller's terminal only, never through a verb: accepting, "+
|
||||
"recovering or exporting a secret hands it to whoever asks, and that includes agents (novox/hq ADR "+
|
||||
"0266). Nothing was done", strings.Join(argv[1:], " "))
|
||||
}
|
||||
if argv[0] != "node" {
|
||||
return nil
|
||||
}
|
||||
if len(argv) > 1 && nodeReads[argv[1]] {
|
||||
return nil
|
||||
}
|
||||
sub := "node"
|
||||
if len(argv) > 1 {
|
||||
sub += " " + argv[1]
|
||||
}
|
||||
return terminalRefusal("%s is run at the controller's terminal only, never through a verb: a node's accounts "+
|
||||
"decide who may become root on it (novox/hq ADR 0266). A verb may run node list and node show. "+
|
||||
"Nothing was done", sub)
|
||||
}
|
||||
|
||||
@@ -258,16 +258,19 @@ var accountedFlags = map[string]map[string]string{
|
||||
"seats": {"json": "set by the verb: the answer is data"},
|
||||
"queue": {"json": "not set: the verb answers the table a person reads"},
|
||||
"plan": {"json": "set by the verb unless files is asked"},
|
||||
"push": {"wait": "set by the verb to 0: a tool call cannot hold a connection for a whole apply"},
|
||||
"push": {
|
||||
"wait": "set by the verb to 0: a tool call cannot hold a connection for a whole apply",
|
||||
"all": "withheld: the verb naming no machine is --behind, and a push of every machine is said at a " +
|
||||
"shell (novox/hq ADR 0217); `command` reaches it",
|
||||
},
|
||||
"build": {
|
||||
"wait": "set by the verb to 0: the id follows the build (issue 176)",
|
||||
"self": "set by the verb from the repository's form: a path on the forge, or a URL",
|
||||
"dry-run": "withheld: a dry run answers only when the build ends, which a call cannot wait for; `command` reaches it",
|
||||
"behind": "withheld: the named verb builds one named repository; `command` reaches the rest",
|
||||
"on": "withheld: the named verb builds one named repository; `command` reaches the rest",
|
||||
},
|
||||
"builds": {"n": "=limit"},
|
||||
"plans": {"n": "=limit", "what-if": "=repository"},
|
||||
// The machine's tunnel key, named as the verb's other arguments are (novox/hq ADR 0169).
|
||||
"durations": {
|
||||
"json": "set by the verb: the answer is data",
|
||||
"all": "withheld: every measurement of a fortnight is more than a call should carry; `command` reaches it",
|
||||
@@ -277,6 +280,10 @@ var accountedFlags = map[string]map[string]string{
|
||||
"retire": {"json": "set by the verb: the answer is data"},
|
||||
"cleanup": {"json": "set by the verb: the answer is data"},
|
||||
"data": {"json": "set by the verb: the answer is data"},
|
||||
"artifacts": {"json": "set by the verb: the answer is data"},
|
||||
"collect": {"json": "set by the verb: the answer is data"},
|
||||
"images": {"json": "set by the verb: the answer is data"},
|
||||
"mirrors": {"json": "set by the verb: the answer is data"},
|
||||
"conditions history": {"json": "set by the verb: the answer is data"},
|
||||
"conditions show": {"json": "set by the verb: the answer is data"},
|
||||
"healers": {"json": "set by the verb: the answer is data"},
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user