Archived
Compare commits
359
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
dc5a8208a2 | ||
|
|
a5a355aeec | ||
|
|
4d30b5de13 | ||
|
|
c97942d591 | ||
|
|
9c69b9da17 | ||
|
|
00065dbdaf | ||
|
|
16362e1bbd | ||
|
|
3496b58f66 | ||
|
|
d1cc9b4e20 | ||
|
|
cc11de2513 | ||
|
|
04fcce2fcc | ||
|
|
b9ceeace41 | ||
|
|
2e0a7d1cbd | ||
|
|
3ced96fc6f | ||
|
|
d3b4549611 | ||
|
|
065cfa0d1d | ||
|
|
0e0e143055 | ||
|
|
2887691414 | ||
|
|
e1499d4196 | ||
|
|
96fc4209d3 | ||
|
|
8a53532d89 | ||
|
|
d0161fac52 | ||
|
|
887de9b5f2 | ||
|
|
1560c498b6 | ||
|
|
df6d72aec2 | ||
|
|
a6bc0936e1 | ||
|
|
513ebd0577 | ||
|
|
88e84a4dd7 | ||
|
|
7b5c063cd3 | ||
|
|
abd3078491 | ||
|
|
0e5aed1253 | ||
|
|
6d3523ff10 | ||
|
|
bd35b1c06c | ||
|
|
150f038ff8 | ||
|
|
6c616838a5 | ||
|
|
de55c63e12 | ||
|
|
abe5f7dc17 | ||
|
|
4d1b81b6cb | ||
|
|
9517f590ac | ||
|
|
9cef820117 | ||
|
|
272ca2a578 | ||
|
|
7160d95323 | ||
|
|
9551bc2380 | ||
|
|
cdaba36eca | ||
|
|
eaf5195998 | ||
|
|
a6f831a633 | ||
|
|
a138c045bb | ||
|
|
988e501ccc | ||
|
|
19eefb66c6 | ||
|
|
081d9244d6 | ||
|
|
b55a38ca9f | ||
|
|
747734687e | ||
|
|
9006c82393 | ||
|
|
0c8c9ffae9 | ||
|
|
1c7c385839 | ||
|
|
65ff6159ad | ||
|
|
e6e1e3bc89 | ||
|
|
07e59c535e | ||
|
|
ba97297f66 | ||
|
|
e6b00e2e51 | ||
|
|
fa19a2d718 | ||
|
|
3e5086a2f6 | ||
|
|
ed331eb972 | ||
|
|
be59f29f46 | ||
|
|
5689553406 | ||
|
|
0559b80887 | ||
|
|
74d35600a6 | ||
|
|
d19c9ed5b7 | ||
|
|
799eec0a5a | ||
|
|
ad406e81b8 | ||
|
|
646c5e53db | ||
|
|
2190e2c664 | ||
|
|
0909d7b125 | ||
|
|
744b0b9162 | ||
|
|
8de4dc7951 | ||
|
|
2913c54c29 | ||
|
|
ef26d4cb0f | ||
|
|
d9a730307c | ||
|
|
9ca7952d5e | ||
|
|
58cb586c37 | ||
|
|
c3c56a69e2 | ||
|
|
63b87b6f7b | ||
|
|
997a4925b0 | ||
|
|
077ddf0eb8 | ||
|
|
c58516f819 | ||
|
|
54b04a7604 | ||
|
|
af025562c7 | ||
|
|
c544c2a17b | ||
|
|
5866b2db94 | ||
|
|
983bf65141 | ||
|
|
0e46b8302d | ||
|
|
4c375dafed | ||
|
|
e2a45b18ba | ||
|
|
5fa8e40667 | ||
|
|
b3fd360ddb | ||
|
|
9372e80cec | ||
|
|
c9be75b13e | ||
|
|
f5f315cc95 | ||
|
|
51c8c7ec52 | ||
|
|
63e85b25e6 | ||
|
|
93c6ca5432 | ||
|
|
14ab2ddd9b | ||
|
|
510c91f144 | ||
|
|
33dc85d850 | ||
|
|
ea1d3ed96d | ||
|
|
eca6390d6f | ||
|
|
5438d7ff75 | ||
|
|
eb7a158086 | ||
|
|
03dac87d3a | ||
|
|
0c47f48537 | ||
|
|
2e1a9abbf7 | ||
|
|
95e7a7120a | ||
|
|
d15eee61bb | ||
|
|
1b780eae3a | ||
|
|
a5e8baf6da | ||
|
|
e168b60159 | ||
|
|
d951f22c04 | ||
|
|
b1897a3498 | ||
|
|
926dbd7a97 | ||
|
|
db702312af | ||
|
|
d7fe5b609b | ||
|
|
b4dff64ae0 | ||
|
|
2b8a28adab | ||
|
|
20d4f1ae71 | ||
|
|
528951319c | ||
|
|
dcbbf4487a | ||
|
|
fcfbf7e69e | ||
|
|
4d60628f37 | ||
|
|
b13a0f71a4 | ||
|
|
822e52123c | ||
|
|
9a7ae131cb | ||
|
|
55d8b43f30 | ||
|
|
4354d9d7c7 | ||
|
|
1fdc68a8aa | ||
|
|
36008e0642 | ||
|
|
f476494173 | ||
|
|
155819f307 | ||
|
|
eb5f9d2f53 | ||
|
|
0f58602c74 | ||
|
|
c3ae3f3e09 | ||
|
|
2a9697cf70 | ||
|
|
e4d2868679 | ||
|
|
fe857ba081 | ||
|
|
b9fc09c375 | ||
|
|
ec7b8bcd58 | ||
|
|
cec797e996 | ||
|
|
0e0ba93f6c | ||
|
|
0f1e1b09fd | ||
|
|
1b502a37e0 | ||
|
|
522f2253e7 | ||
|
|
2073bfe2e6 | ||
|
|
f5824b31c6 | ||
|
|
d059311c0f | ||
|
|
1a28cb3357 | ||
|
|
758537dd4e | ||
|
|
add807f034 | ||
|
|
8ca4b04321 | ||
|
|
8170fc58a3 | ||
|
|
efcdd5dd7d | ||
|
|
5d7d8ee2d6 | ||
|
|
5b7e6ff453 | ||
|
|
cc7fb99f29 | ||
|
|
1e04670052 | ||
|
|
81e5458cbf | ||
|
|
fb74e24c9e | ||
|
|
ca09a07fdf | ||
|
|
9b028b4212 | ||
|
|
d7fab82a89 | ||
|
|
7f65e62743 | ||
|
|
2b01f8786e | ||
|
|
909062e729 | ||
|
|
826dcb91b1 | ||
|
|
193168e086 | ||
|
|
59fdffb979 | ||
|
|
5d47e0bfd6 | ||
|
|
e3ec15f707 | ||
|
|
ac91357a53 | ||
|
|
b5f2c3b961 | ||
|
|
af63b233db | ||
|
|
0cf5a3a5ba | ||
|
|
f5680ba8da | ||
|
|
76babaea52 | ||
|
|
e41b78cd77 | ||
|
|
1acce7132e | ||
|
|
3f68a495f1 | ||
|
|
298ec06ae0 | ||
|
|
a5f53ef9eb | ||
|
|
8adb7f1a05 | ||
|
|
e33da2dc1c | ||
|
|
249d97d1c8 | ||
|
|
7d63d2e68c | ||
|
|
fe00fec52c | ||
|
|
056414bc06 | ||
|
|
59620fdacb | ||
|
|
b74268fb08 | ||
|
|
a44dc01c65 | ||
|
|
c6e372896b | ||
|
|
c5663aa18b | ||
|
|
9907df6530 | ||
|
|
41d6019fa0 | ||
|
|
58e143bbc0 | ||
|
|
d9588b4f13 | ||
|
|
34611c8e38 | ||
|
|
a63939160e | ||
|
|
7ad9dbcb5d | ||
|
|
363898ec8a | ||
|
|
8984c3437f | ||
|
|
9766338368 | ||
|
|
b1acb3d9de | ||
|
|
a759ac65a5 | ||
|
|
2e6a9b1efc | ||
|
|
45ae1d0112 | ||
|
|
76cfbac7a1 | ||
|
|
28712eaea1 | ||
|
|
51d7bbfdc8 | ||
|
|
e3b5c224e8 | ||
|
|
6df30b6d98 | ||
|
|
1a4305213d | ||
|
|
7df72edd2b | ||
|
|
1fce541023 | ||
|
|
950562afe1 | ||
|
|
a8f60f1f69 | ||
|
|
38928c4e1f | ||
|
|
175b28ee42 | ||
|
|
557b23d43f | ||
|
|
c43277f9c6 | ||
|
|
4d337ff897 | ||
|
|
6c98e7ea51 | ||
|
|
6910417090 | ||
|
|
d3d7fc6873 | ||
|
|
1358861275 | ||
|
|
14d36e813f | ||
|
|
cb0327de7f | ||
|
|
1a50d6e5ab | ||
|
|
c5a2edf04a | ||
|
|
df653e9af0 | ||
|
|
85b2a1855b | ||
|
|
c714d07739 | ||
|
|
59c448713f | ||
|
|
0563389057 | ||
|
|
e35c0e69b6 | ||
|
|
8bbfdb53db | ||
|
|
913095d291 | ||
|
|
b05ac4f4b2 | ||
|
|
174e06ed08 | ||
|
|
4499e85476 | ||
|
|
c724ef456f | ||
|
|
56b206fe04 | ||
|
|
4d9894138e | ||
|
|
6ea457df74 | ||
|
|
6faf701656 | ||
|
|
80f9d26e6d | ||
|
|
5ddc59cd32 | ||
|
|
cdbbd9ec4b | ||
|
|
e92a3fe237 | ||
|
|
98ef7bac6e | ||
|
|
db953e7da8 | ||
|
|
7597294ff8 | ||
|
|
9a7ac3af46 | ||
|
|
e7dca6c280 | ||
|
|
c37d7742ba | ||
|
|
ea09f074db | ||
|
|
6a0d84b3f6 | ||
|
|
dd668ec887 | ||
|
|
7dbe80ad63 | ||
|
|
697395af32 | ||
|
|
036b6cc873 | ||
|
|
96c34c52dc | ||
|
|
3808634a9f | ||
|
|
4f80d87750 | ||
|
|
ea92af2a7d | ||
|
|
4469cab7f4 | ||
|
|
875a4e5758 | ||
|
|
bf11a8baac | ||
|
|
65610f2ea2 | ||
|
|
85d664438f | ||
|
|
13b6fc6d97 | ||
|
|
d851573793 | ||
|
|
f6aea4bfbb | ||
|
|
e550c95543 | ||
|
|
1fdff00794 | ||
|
|
5efe999733 | ||
|
|
cdf30349a7 | ||
|
|
40e42606cf | ||
|
|
b8bbf9c79f | ||
|
|
8bcf787258 | ||
|
|
a5a132ac15 | ||
|
|
7f25666cee | ||
|
|
4291fee68e | ||
|
|
b98fd0f396 | ||
|
|
863ebd4277 | ||
|
|
2799e95035 | ||
|
|
582f4a082a | ||
|
|
6c7af5c63f | ||
|
|
9d15f3a39e | ||
|
|
725fcd977e | ||
|
|
1cc6a2d759 | ||
|
|
5d3e52219b | ||
|
|
099c176fa9 | ||
|
|
ec3769a8a6 | ||
|
|
8b2abd08cd | ||
|
|
858b4672dd | ||
|
|
3d7ccc8aeb | ||
|
|
b39eaa485a | ||
|
|
0d2fd2c5bb | ||
|
|
a011743c69 | ||
|
|
72d7802415 | ||
|
|
b7d9f44936 | ||
|
|
75213b9091 | ||
|
|
487aa040de | ||
|
|
ba26ba2772 | ||
|
|
3d05d74400 | ||
|
|
58ebe590a5 | ||
|
|
b24bb030ec | ||
|
|
327654e57b | ||
|
|
14127d4878 | ||
|
|
d0580a17e5 | ||
|
|
d6e0a8250a | ||
|
|
9b6b0c5686 | ||
|
|
792352dfad | ||
|
|
b9e0cd34c3 | ||
|
|
1c26235bc1 | ||
|
|
bbd442cdf4 | ||
|
|
042874e0ce | ||
|
|
be92762969 | ||
|
|
9c714f00d6 | ||
|
|
cbce8f96ef | ||
|
|
068283137b | ||
|
|
0090bf6af7 | ||
|
|
58924dc920 | ||
|
|
b1016e5c66 | ||
|
|
b1e02aca1b | ||
|
|
4635341a9d | ||
|
|
dcec6a4db3 | ||
|
|
2bfa6ae4a0 | ||
|
|
41f7b2c152 | ||
|
|
37229b4db5 | ||
|
|
d7bf1bae83 | ||
|
|
c6f3d8cdfa | ||
|
|
d9289ef6d4 | ||
|
|
81f497e5dd | ||
|
|
8e8712e352 | ||
|
|
7aa98e64ce | ||
|
|
48581af35c | ||
|
|
4d05385819 | ||
|
|
dcee8cb5bf | ||
|
|
2b5060789f | ||
|
|
abf9125689 | ||
|
|
52af210e47 | ||
|
|
4b25af2aa3 | ||
|
|
fc65215c25 | ||
|
|
c988d6d7be | ||
|
|
8bfaf1523e | ||
|
|
b037c73fd5 | ||
|
|
18da8b37e9 | ||
|
|
4f4d365360 | ||
|
|
08e11ad761 | ||
|
|
3b2adda1c8 | ||
|
|
68009b16fe |
@@ -102,18 +102,18 @@ proxy-image:
|
||||
# The whole gate. Raises a database, runs everything against it, and takes it down again --
|
||||
# including when the tests fail, which is why the teardown is not conditional.
|
||||
#
|
||||
# **One package at a time (-p 1), and it is not about speed.** The live tests reach one bus, and on
|
||||
# it they assert, read and remove the mesh's own objects -- streams and consumers with fixed names,
|
||||
# because those names are the mesh's and a test cannot choose others. Two packages doing that at once
|
||||
# is one deleting a consumer the other is reading through, and the failure lands in whichever test
|
||||
# was reading, as "no response from stream". That reads as a bug in the code under test.
|
||||
# **Packages in parallel, under the race detector, each test on a bus of its own** (internal/testbus).
|
||||
# It was one package at a time against one shared bus, because the live tests assert, read and remove
|
||||
# the mesh's own objects by their fixed names, and two packages at once deleted what the other read; the
|
||||
# suite was red run as Go runs it and read as noise. A bus per test, of the release the mesh runs, made
|
||||
# it the same in any order. The timeout bounds a hang to a failure with a stack, never a stalled gate.
|
||||
check: fmt vet postgres
|
||||
@go test -p 1 ./... ; status=$$? ; $(MAKE) postgres-stop ; exit $$status
|
||||
@go test -race -timeout 15m ./... ; status=$$? ; $(MAKE) postgres-stop ; exit $$status
|
||||
|
||||
# Without a database the live tests skip rather than fail, so this is the honest subset and not
|
||||
# the gate. Serialised for the same reason check is: a bus may be configured even when a store is not.
|
||||
# Without a database the store's tests skip rather than fail, so this is the honest subset and not
|
||||
# the gate.
|
||||
test:
|
||||
go test -p 1 ./...
|
||||
go test -timeout 15m ./...
|
||||
|
||||
vet:
|
||||
go vet ./...
|
||||
|
||||
@@ -274,6 +274,22 @@ func (h *holder) removeContainers(id string) (int, error) {
|
||||
return builder.RemoveContainersOf(cleanup, h.remove, id)
|
||||
}
|
||||
|
||||
// removeLeftBehind removes, once, what earlier holders on this machine left — called before anything is
|
||||
// taken, so none of it is this holder's — and says what it did. A runtime that cannot be asked is said
|
||||
// and the holder takes work all the same: what is left is waste, not a reason to stop building.
|
||||
func (h *holder) removeLeftBehind() int {
|
||||
cleanup, stop := context.WithTimeout(context.Background(), killRemoves)
|
||||
defer stop()
|
||||
n, err := builder.RemoveLeftBehind(cleanup, h.remove)
|
||||
switch {
|
||||
case err != nil:
|
||||
fmt.Fprintf(os.Stderr, "could not look for containers earlier builds left on %s: %v\n", h.on, err)
|
||||
case n > 0:
|
||||
fmt.Fprintf(os.Stderr, "removed %d container(s) earlier builds left on %s\n", n, h.on)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// returned records that the build's work ended, and says whether a kill came first — only then is
|
||||
// the build killed; an error it ended with on its own is its own outcome.
|
||||
func (h *holder) returned(r *running) bool {
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -146,3 +147,26 @@ func TestAKillArrivingAfterTheBuildEndedIsRefusedAndAnUnannouncedKillSaysSo(t *t
|
||||
t.Fatalf("kill said %q (%v)", said, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **Issue 306**: a holder starting removes the containers earlier builds left on its machine, through
|
||||
// the runner a kill uses, and a runtime that cannot be asked does not stop it.
|
||||
func TestAHolderStartingRemovesWhatEarlierBuildsLeftHere(t *testing.T) {
|
||||
h := newHolder("novox", link.TheBuildMachine, t.TempDir(), nil)
|
||||
var removed []string
|
||||
h.remove = func(_ context.Context, _ string, name string, args ...string) (string, error) {
|
||||
removed = append(removed, name+" "+strings.Join(args, " "))
|
||||
if args[0] == "ps" {
|
||||
return "c1 build-1\nc2 build-1\nc3 check-here-2\n", nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
if n := h.removeLeftBehind(); n != 2 || len(removed) != 2 || removed[1] != "docker rm -f c1 c2" {
|
||||
t.Fatalf("removed %d: %v", n, removed)
|
||||
}
|
||||
h.remove = func(context.Context, string, string, ...string) (string, error) {
|
||||
return "", errors.New("no runtime")
|
||||
}
|
||||
if n := h.removeLeftBehind(); n != 0 {
|
||||
t.Errorf("a runtime that cannot be asked removed %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,6 +27,8 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
@@ -129,6 +131,9 @@ func run() error {
|
||||
if h.Paused() {
|
||||
fmt.Fprintf(os.Stderr, "paused (kept in %s): taking no build until resumed\n", pausedFile(workspace))
|
||||
}
|
||||
// **What an earlier holder here left goes before anything is taken** (novox/hq issue 306): a holder
|
||||
// recreated mid-check left the check's store and bus running, and the ask went to another machine.
|
||||
h.removeLeftBehind()
|
||||
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused})
|
||||
defer machine.Close()
|
||||
|
||||
@@ -172,7 +177,9 @@ func dialFor(credential Credential) (*broker.JetStream, string, error) {
|
||||
if !credential.onTheNewBus() {
|
||||
return nil, "", fmt.Errorf("the credential at hand names %q, which is not the mesh's bus", credential.URL)
|
||||
}
|
||||
js, err := broker.DialPinned(credential.natsURL(), credential.Fingerprint)
|
||||
// Named for what it is, not the controller whose code dials it (novox/hq issue 327).
|
||||
host, _ := os.Hostname()
|
||||
js, err := broker.DialPinned(credential.natsURL(), credential.Fingerprint, nats.Name("build agent on "+host))
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
@@ -237,7 +244,21 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
|
||||
npmrc, err := packagesFrom()
|
||||
var built builder.Result
|
||||
if err == nil {
|
||||
if request.Check != nil {
|
||||
// **A pull request's merge check, not a build** (novox/hq to-be 45 §9): nothing is built,
|
||||
// published or registered; the verdict is the outcome.
|
||||
result.Checked = request.Check
|
||||
registry := ""
|
||||
if r, ok := publisher.(builder.Registry); ok {
|
||||
registry = r.Address
|
||||
}
|
||||
var v builder.CheckVerdict
|
||||
v, err = builder.Check(building, builder.Command, checkSpecOf(request), workspace, registry, forgeFrom(), say)
|
||||
if err == nil {
|
||||
result.Check = &link.CheckOutcome{Verdict: v.Verdict, Summary: v.Summary, Report: v.Report,
|
||||
Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo)}
|
||||
}
|
||||
} else if err == nil {
|
||||
// The package-registry credential is a build input, so it is resolved before the clone: a
|
||||
// build that could not have resolved its dependencies is refused in front of the reason, not
|
||||
// after a clone that then fails at npm ci.
|
||||
@@ -247,6 +268,8 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
|
||||
forgeFrom(), say, request.Seats)
|
||||
}
|
||||
// What it copied into the store, whatever became of the build (novox/hq ADR 0257).
|
||||
result.Mirrored = built.Mirrored
|
||||
// Only a build the kill ended: the kill came before its work did. One that finished — built, or
|
||||
// failed on its own — in the moment the kill arrived says what it did, and the kill is refused.
|
||||
killed := false
|
||||
@@ -263,6 +286,8 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
// builder that is not running, and those want completely different responses.
|
||||
result.Failed = err.Error()
|
||||
say("failed", err.Error())
|
||||
} else if request.Check != nil {
|
||||
say("checked", result.Check.Verdict+": "+result.Check.Summary)
|
||||
} else {
|
||||
manifest, marshalErr := json.Marshal(built.Manifest)
|
||||
if marshalErr != nil {
|
||||
@@ -276,9 +301,14 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
})
|
||||
}
|
||||
result.Against = built.Against
|
||||
result.SourceFingerprint = built.Source
|
||||
result.Trunk, result.OnTrunk, result.Branches = built.Trunk, built.OnTrunk, built.Branches
|
||||
for _, r := range built.Read {
|
||||
result.Read = append(result.Read, link.ReadRepository{Repository: r.Repository, Ref: r.Ref})
|
||||
}
|
||||
for _, s := range built.Sources {
|
||||
result.Sources = append(result.Sources, link.BuildSource{Repository: s.Repository, Ref: s.Ref, Paths: s.Paths})
|
||||
}
|
||||
say("built", built.Manifest.Module+" from "+short(built.Commit))
|
||||
}
|
||||
}
|
||||
@@ -312,6 +342,31 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
}
|
||||
}
|
||||
|
||||
// checkSpecOf is a check request as the builder runs it.
|
||||
func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
|
||||
c := request.Check
|
||||
spec := builder.CheckSpec{ID: request.ID, Repository: request.Repository, Ref: request.Ref,
|
||||
Owner: c.Owner, Repo: c.Repo, Number: c.Number, Paths: c.Paths, Beside: map[string]builder.Beside{},
|
||||
Modules: c.Modules, New: c.New, Manifests: c.Manifests, Judge: c.Judge, Base: c.Base,
|
||||
Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held)}
|
||||
for dir, b := range c.Beside {
|
||||
spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref}
|
||||
}
|
||||
for _, m := range c.Members {
|
||||
spec.Group = append(spec.Group, builder.GroupHead{Owner: m.Owner, Repo: m.Repo, Repository: m.Repository,
|
||||
Ref: m.Ref, Paths: m.Paths})
|
||||
}
|
||||
return spec
|
||||
}
|
||||
|
||||
// layerOf is one layer of a check as the outcome carries it.
|
||||
func layerOf(l *builder.Layer) *link.CheckLayer {
|
||||
if l == nil {
|
||||
return nil
|
||||
}
|
||||
return &link.CheckLayer{Verdict: l.Verdict, Summary: l.Summary, Modules: l.Modules}
|
||||
}
|
||||
|
||||
// packagesFrom is where a build resolves the mesh's own published packages — the SDK above all
|
||||
// (novox/hq ADR 0076, issue 053).
|
||||
//
|
||||
|
||||
@@ -105,6 +105,7 @@ func buildOnce(ctx context.Context, args []string) error {
|
||||
Ref: *ref,
|
||||
Manifest: built.Manifest,
|
||||
Against: built.Against,
|
||||
Source: built.Source,
|
||||
}
|
||||
for _, r := range built.Read {
|
||||
out.Read = append(out.Read, readRepository{Repository: r.Repository, Ref: r.Ref})
|
||||
@@ -135,6 +136,7 @@ type onceResult struct {
|
||||
Made []madeArtifact `json:"made"`
|
||||
Against []string `json:"against,omitempty"`
|
||||
Read []readRepository `json:"read,omitempty"`
|
||||
Source string `json:"source-fingerprint,omitempty"`
|
||||
}
|
||||
|
||||
// readRepository is a repository this build read source from besides the module's own.
|
||||
|
||||
@@ -5,9 +5,11 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
@@ -178,7 +180,21 @@ func (a *actor) release() {
|
||||
// holderOf is this process as the lease's holder.
|
||||
func holderOf(instance string) lease.Holder {
|
||||
host, _ := os.Hostname()
|
||||
return lease.Holder{Instance: instance, Host: host, Build: version}
|
||||
build := runningBuild()
|
||||
if build == "" {
|
||||
build = version
|
||||
}
|
||||
return lease.Holder{Instance: instance, Host: host, Build: build}
|
||||
}
|
||||
|
||||
// RunningBuildVar is where the declaration tells this process which build it is (module.json, the
|
||||
// controller process's env): the version its bundle is delivered as, `${version}` composed by the
|
||||
// catalogue from the bundle's digest. Empty for a process placed by hand.
|
||||
const RunningBuildVar = "MESH_CONTROLLER_VERSION"
|
||||
|
||||
// runningBuild is the version of the build this process is, or empty when the declaration did not say.
|
||||
func runningBuild() string {
|
||||
return strings.TrimSpace(os.Getenv(RunningBuildVar))
|
||||
}
|
||||
|
||||
// serveUnderTheLease takes the lease for the serving controller, waiting while another holds it, and
|
||||
@@ -187,7 +203,8 @@ func (a *actor) serveUnderTheLease(ctx context.Context, inv *inventory.Inventory
|
||||
a.mu.Lock()
|
||||
a.serving = true
|
||||
a.mu.Unlock()
|
||||
js, err := broker.Dial(address)
|
||||
// Its own connection, held as long as the lease, and named so (novox/hq issue 327).
|
||||
js, err := broker.Dial(address, nats.Name(broker.ConnectionName+" lease"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the mesh is on the bus at %s and this control plane cannot reach it to take the "+
|
||||
"lease: %w", broker.BareAddress(address), err)
|
||||
@@ -206,7 +223,7 @@ func (a *actor) serveUnderTheLease(ctx context.Context, inv *inventory.Inventory
|
||||
}
|
||||
say := func(format string, args ...any) { fmt.Printf(format+"\n", args...) }
|
||||
l, err := lease.Open(ctx, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance),
|
||||
Floor: inv.HighestEpoch, Say: say, Moved: func(was, floor uint64) {
|
||||
Floor: inv.HighestEpoch, Say: say, Health: controllerHealth, Moved: func(was, floor uint64) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
a.reset = time.Now()
|
||||
|
||||
@@ -7,8 +7,10 @@ import (
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The things the mesh can be asked to do, separated from how it was asked.
|
||||
@@ -41,9 +43,17 @@ import (
|
||||
// It costs a resolution per machine. Assignment is a person typing a command, and being told which
|
||||
// machines this just blocked is worth more than the milliseconds.
|
||||
func assign(ctx context.Context, open *stores, node string, modules ...string) (string, error) {
|
||||
return assignWith(ctx, open, node, assignOptions{}, modules...)
|
||||
}
|
||||
|
||||
// assignWith is assign asked with its options: build, for a module known and not built (novox/hq issue 325).
|
||||
func assignWith(ctx context.Context, open *stores, node string, opts assignOptions, modules ...string) (string, error) {
|
||||
if len(modules) == 0 {
|
||||
return "", fmt.Errorf("assign %s names no module", node)
|
||||
}
|
||||
if err := refusedMovingTheController(modules); err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Held while it is recorded, so it cannot land between a converge's preview and its flip and
|
||||
// be taken without ever having been previewed (novox/hq ADR 0100).
|
||||
ctx, release, err := holdNodes(ctx, open, []string{node})
|
||||
@@ -51,6 +61,29 @@ func assign(ctx context.Context, open *stores, node string, modules ...string) (
|
||||
return "", err
|
||||
}
|
||||
defer release()
|
||||
// **A module the catalogue does not hold is looked for further before it is refused** (novox/hq issue
|
||||
// 325): a build in flight keeps the assignment pending, a module known and not built is said with how to
|
||||
// build it, and only a name the mesh never heard of is refused as unknown. Several modules in one act
|
||||
// are judged together (ADR 0207), so an act naming one not registered is not made in part.
|
||||
if missing, err := notInCatalogue(ctx, open, modules); err != nil {
|
||||
return "", err
|
||||
} else if len(missing) > 0 {
|
||||
if len(modules) == 1 {
|
||||
return notRegistered(ctx, open, node, modules[0], opts)
|
||||
}
|
||||
var lines []string
|
||||
for _, m := range missing {
|
||||
where, err := whereIs(ctx, open.inventory, m, time.Now())
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
lines = append(lines, where.said)
|
||||
}
|
||||
return "", fmt.Errorf("%w: nothing was assigned, since modules assigned together are judged together "+
|
||||
"(ADR 0207) and %s not registered:\n %s\n assign them together once each is registered, or each "+
|
||||
"alone to keep it pending on its build", inventory.ErrNoSuchModule, strings.Join(missing, ", "),
|
||||
strings.Join(lines, "\n "))
|
||||
}
|
||||
// **The one assignment refused for what the node lacks** (novox/hq ADR 0207). Everything else
|
||||
// an assignment leaves unresolved is kept, because assignment is not an ordering; a module whose
|
||||
// resources are applied through a seat nothing on the node holds is refused, because that order
|
||||
@@ -179,6 +212,9 @@ func seatDependenciesOnAssign(ctx context.Context, open *stores, node string, mo
|
||||
// (novox/hq ADR 0207) — the other side of refusing that module's assignment without one. Several
|
||||
// modules in one act are judged together, so a holder and its dependents come off in one command.
|
||||
func unassign(ctx context.Context, open *stores, node string, modules ...string) (string, error) {
|
||||
if err := refusedMovingTheController(modules); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(modules) == 0 {
|
||||
return "", fmt.Errorf("unassign %s names no module", node)
|
||||
}
|
||||
@@ -200,6 +236,17 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string)
|
||||
for _, a := range assigned {
|
||||
runs[a] = true
|
||||
}
|
||||
// A module only pending on the node (novox/hq issue 325) is withdrawn, when it is the act's one module:
|
||||
// that is the undo of an assignment kept while its build runs.
|
||||
if len(modules) == 1 && !runs[modules[0]] {
|
||||
said, withdrawn, err := withdrawPending(ctx, open.inventory, node, modules[0])
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if withdrawn {
|
||||
return said, nil
|
||||
}
|
||||
}
|
||||
for _, module := range modules {
|
||||
if !runs[module] {
|
||||
return "", fmt.Errorf("%s is not assigned to %s", module, node)
|
||||
@@ -224,6 +271,10 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string)
|
||||
for _, line := range unheldChange(shelf, node, assigned, left) {
|
||||
answer += "\n " + line
|
||||
}
|
||||
// What it leaves behind that is irreplaceable is kept and retired, never removed (novox/hq ADR 0233).
|
||||
for _, line := range keptOnUnassign(ctx, open.inventory, node, modules) {
|
||||
answer += "\n " + line
|
||||
}
|
||||
return answer + blockedElsewhere(ctx, open, node), nil
|
||||
}
|
||||
|
||||
@@ -315,3 +366,17 @@ func issueOnAssign(ctx context.Context, open *stores, node, module string) strin
|
||||
}
|
||||
return fmt.Sprintf("its bus credential is issued and sealed to %s, and arrives with the push", node)
|
||||
}
|
||||
|
||||
// refusedMovingTheController refuses assigning or unassigning the controller's module through a verb (review of
|
||||
// novox/hq ADR 0272): the node it is assigned to is the control-node, and the control-node's operator account is
|
||||
// the controller's terminal, so whoever moves the module chooses the terminal. At the terminal alone, as every
|
||||
// change that says who may change what.
|
||||
func refusedMovingTheController(modules []string) error {
|
||||
verb, through := throughAVerb()
|
||||
if !through || !slices.Contains(modules, controllerModule) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%s is assigned and unassigned at the controller's terminal only (mesh-cli on the control-node), "+
|
||||
"never through a verb (this came through %q): the node it runs on is the control-node, whose operator is the "+
|
||||
"terminal (novox/hq ADR 0272). Nothing was assigned", controllerModule, verb)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,253 @@
|
||||
package main
|
||||
|
||||
// The account agents run as (novox/hq ADR 0266).
|
||||
//
|
||||
// On the control node every agent session ran as the operator's account, which may become root without a
|
||||
// password — so any agent there could become root without a person, and ADR 0259 §8 (an answer from the
|
||||
// operator's phone authorises an act) rests on that being false where the router and its channels run. The
|
||||
// decision: a node may name an account its agents run as, of their own and without sudo; the operator's
|
||||
// account keeps its sudo.
|
||||
//
|
||||
// - **Named at the controller's terminal only** (`node agent-account`): not a verb, not a setting, so no
|
||||
// agent can name itself another account. Empty is a real state: agents run as the operator there.
|
||||
// - **Composed** as `${machine:agent-account}`, `${machine:agent-home}` and `${machine:agent-root}` for the
|
||||
// agent's module, which declares the account with `root: never`, and as MESH_AGENT_ACCOUNT and
|
||||
// MESH_AGENT_HOME for its tools (catalogue/machine_into_files.go, runtime.go).
|
||||
// - **Judged by the machine itself.** The node-engine reads, on every look, whether an account declared
|
||||
// `root: never` can become root without a person — uid 0, a group that grants root, a sudo rule, a
|
||||
// secret of the mesh it may read — and says it as the declaring module's account verdict, marked
|
||||
// Root "never". agentConfined reads that verdict; the self-check (probe DA) raises
|
||||
// `agent-can-become-root` while it does not hold, and `node show` says it.
|
||||
//
|
||||
// A verdict not given is never a pass: an engine older than the judging, an account not yet declared, a
|
||||
// statement that says nothing of it — each is "not judged", and fails.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// kindAgentCanBecomeRoot is the condition raised while a machine's agent account can become root without
|
||||
// a person, or is not judged (ADR 0266).
|
||||
const kindAgentCanBecomeRoot = "agent-can-become-root"
|
||||
|
||||
// agentAccountProbe is the self-check's probe of it.
|
||||
const agentAccountProbe = "DA"
|
||||
|
||||
// agentConfined says whether the agents of a machine that names an agent account are confined: the
|
||||
// machine's newest statement holds a healthy account verdict, judged for root, on that account. named is
|
||||
// false for a machine that names none — agents run as the operator account there, which this does not
|
||||
// judge. why is said either way, in the mesh's words; err is a store that could not be read.
|
||||
//
|
||||
// The one judgement: `node show`, the self-check, and ADR 0259's router honouring a verified sender read
|
||||
// it here.
|
||||
// now is the judging clock, threaded so a caller judging several things at one instant judges them all at it.
|
||||
func agentConfined(ctx context.Context, inv *inventory.Inventory, node string, now time.Time) (named, confined bool, why string, err error) {
|
||||
n, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
if n.AgentAccount == "" {
|
||||
return false, false, fmt.Sprintf("%s names no agent account: agents run as the operator account (%s)",
|
||||
node, orNoneKnown(n.Account)), nil
|
||||
}
|
||||
h, had, err := inv.HealthOf(ctx, node)
|
||||
if err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
confined, why = judgedConfined(n.AgentAccount, h, had, now)
|
||||
return true, confined, why, nil
|
||||
}
|
||||
|
||||
// verdictFreshFor is how old the statement holding the verdict may be, by this controller's clock. A
|
||||
// node-engine states its health on every change and at least every five minutes (mesh-host's sayAnyway), so
|
||||
// three statements missed is a node-engine stopped, or a machine away. **A stale verdict is not a pass**: an
|
||||
// agent that stopped the node-engine must not leave "cannot become root" standing from before.
|
||||
const verdictFreshFor = 15 * time.Minute
|
||||
|
||||
// judgedConfined is the judgement over one statement, without the store, at now.
|
||||
func judgedConfined(agent string, h inventory.NodeHealth, had bool, now time.Time) (bool, string) {
|
||||
if !had {
|
||||
return false, fmt.Sprintf("the agent account %s is not judged: the machine's node-engine has stated "+
|
||||
"nothing of what it runs", agent)
|
||||
}
|
||||
if age := now.Sub(h.HeardAt); age > verdictFreshFor {
|
||||
return false, fmt.Sprintf("the agent account %s is not judged: the machine's newest statement was heard at "+
|
||||
"%s, more than %d minutes ago, and a verdict that old is not a verdict on now", agent,
|
||||
h.HeardAt.Local().Format("2006-01-02 15:04"), int(verdictFreshFor.Minutes()))
|
||||
}
|
||||
if h.Contract < link.RootContract {
|
||||
return false, fmt.Sprintf("the agent account %s is not judged: the machine's node-engine is older than "+
|
||||
"the judging of an account's root (its statement's contract is %d, the judging is %d)",
|
||||
agent, h.Contract, link.RootContract)
|
||||
}
|
||||
var verdicts []inventory.ResourceHealth
|
||||
for _, r := range h.Resources {
|
||||
if r.Kind == link.KindAccount && r.Target == agent && r.Root == link.RootNever {
|
||||
verdicts = append(verdicts, r)
|
||||
}
|
||||
}
|
||||
if len(verdicts) == 0 {
|
||||
return false, fmt.Sprintf("the agent account %s is not judged: the machine's newest statement holds no "+
|
||||
"verdict on it — no module there declares it never to become root, or the declaration naming it "+
|
||||
"has not been applied", agent)
|
||||
}
|
||||
sort.Slice(verdicts, func(i, j int) bool {
|
||||
return verdicts[i].Module+verdicts[i].Resource < verdicts[j].Module+verdicts[j].Resource
|
||||
})
|
||||
for _, v := range verdicts {
|
||||
switch v.State {
|
||||
case link.StateHealthy:
|
||||
case link.StateUnhealthy:
|
||||
// The engine's own words, which start with link.ReasonRoot when it found a way to root.
|
||||
return false, fmt.Sprintf("the agent account %s %s (said by %s's %s)", agent,
|
||||
orNoneKnown(v.Reason), v.Module, v.Resource)
|
||||
default:
|
||||
return false, fmt.Sprintf("the agent account %s is not judged: %s (%s's %s, %s)", agent,
|
||||
orNoneKnown(v.Reason), v.Module, v.Resource, v.State)
|
||||
}
|
||||
}
|
||||
return true, fmt.Sprintf("the agent account %s cannot become root without a person (judged %s)", agent,
|
||||
h.SaidAt.Local().Format("2006-01-02 15:04"))
|
||||
}
|
||||
|
||||
// searchQuietFor is how long the controller lets an agent account's verdict wait for the node-engine's setuid
|
||||
// search before that is itself the urgent condition: the longest a search is expected to take (link.RootSearchQuiet,
|
||||
// the engine's own value; novox/hq issue 361 — the search runs to completion, with no bound of its own), counted
|
||||
// from when this controller first saw it waiting, never from the engine's start. Quiet raises nothing; it never
|
||||
// makes the agent account confined, which only a healthy verdict from a complete, fresh search does.
|
||||
const searchQuietFor = link.RootSearchQuiet
|
||||
|
||||
// The kinds of an agent account's verdict, for the quiet a search earns.
|
||||
const (
|
||||
verdictOther = iota // anything else: a stale statement, an older engine, no verdict, another unknown
|
||||
verdictPending // waiting for the search, and otherwise healthy
|
||||
verdictComplete // judged: healthy, or a way to root found
|
||||
)
|
||||
|
||||
// rootVerdictKind reads a statement for the agent account (novox/hq ADR 0266): pending when every verdict on it
|
||||
// is healthy or not judged yet because the engine's setuid search runs, at least one of them that; complete when
|
||||
// every verdict is healthy or one found a way to root. The engine's own "since" is not read: it starts again at
|
||||
// every restart of the engine.
|
||||
func rootVerdictKind(agent string, h inventory.NodeHealth, had bool, now time.Time) int {
|
||||
if !had || now.Sub(h.HeardAt) > verdictFreshFor || h.Contract < link.RootContract {
|
||||
return verdictOther
|
||||
}
|
||||
pending, any := false, false
|
||||
for _, r := range h.Resources {
|
||||
if r.Kind != link.KindAccount || r.Target != agent || r.Root != link.RootNever {
|
||||
continue
|
||||
}
|
||||
any = true
|
||||
switch {
|
||||
case r.State == link.StateHealthy:
|
||||
case r.State == link.StateUnhealthy:
|
||||
return verdictComplete
|
||||
case r.State == link.StateUnknown && strings.HasPrefix(r.Reason, link.ReasonRootPending):
|
||||
pending = true
|
||||
default:
|
||||
return verdictOther
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case !any:
|
||||
return verdictOther
|
||||
case pending:
|
||||
return verdictPending
|
||||
}
|
||||
return verdictComplete
|
||||
}
|
||||
|
||||
// searchStillRunning says the one thing keeping an agent account from being judged is the node-engine's setuid
|
||||
// search, and that this controller first saw it waiting less than searchQuietFor ago — kept in the store, so a
|
||||
// node-engine restarted in a loop does not keep it quiet. A complete verdict forgets when it began.
|
||||
func searchStillRunning(ctx context.Context, inv *inventory.Inventory, node, agent string, h inventory.NodeHealth,
|
||||
had bool, now time.Time) (bool, error) {
|
||||
switch rootVerdictKind(agent, h, had, now) {
|
||||
case verdictComplete:
|
||||
return false, inv.RootSearchJudged(ctx, node)
|
||||
case verdictPending:
|
||||
since, err := inv.RootSearchPending(ctx, node, now)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return now.Sub(since) <= searchQuietFor, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// probeAgentAccounts is DA: every machine that names an agent account has it judged, on its node-engine's
|
||||
// newest statement, unable to become root without a person (ADR 0266).
|
||||
func probeAgentAccounts(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
inv := d.open.inventory
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []conditions.Observation
|
||||
for _, n := range nodes {
|
||||
if n.AgentAccount == "" {
|
||||
continue
|
||||
}
|
||||
h, had, err := inv.HealthOf(ctx, n.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := time.Now()
|
||||
quiet, err := searchStillRunning(ctx, inv, n.Name, n.AgentAccount, h, had, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
confined, why := judgedConfined(n.AgentAccount, h, had, now)
|
||||
if confined {
|
||||
continue
|
||||
}
|
||||
// Not judged yet only because the node-engine's search runs and no complete, fresh one judges: not the
|
||||
// urgent condition after every restart. The agent is still not confined — ADR 0259's router reads
|
||||
// agentConfined, not this — and `node show` still says not judged. Loud again once the search fails,
|
||||
// waits past searchQuietFor, or the statement goes stale.
|
||||
if quiet {
|
||||
continue
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "agent-root",
|
||||
Machine: n.Name, Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("on %s, %s (ADR 0266): an agent there may become root without a person, and "+
|
||||
"no answer from a channel authorises an act there (ADR 0259 §8)", n.Name, why),
|
||||
Said: why})
|
||||
}
|
||||
return sortedFound(out), nil
|
||||
}
|
||||
|
||||
// agentAccountLines is what `node show` says of the account agents run as.
|
||||
func agentAccountLines(ctx context.Context, inv *inventory.Inventory, n inventory.Node) []string {
|
||||
if n.AgentAccount == "" {
|
||||
return []string{fmt.Sprintf(" agents run as the operator account (%s); no agent account is named",
|
||||
orNoneKnown(n.Account))}
|
||||
}
|
||||
_, confined, why, err := agentConfined(ctx, inv, n.Name, time.Now())
|
||||
if err != nil {
|
||||
return []string{fmt.Sprintf(" agents run as %s (home %s); whether it can become root could NOT be read: %v",
|
||||
n.AgentAccount, n.AgentHome(), err)}
|
||||
}
|
||||
verdict := "CAN become root, or is not judged: " + why
|
||||
if confined {
|
||||
verdict = why
|
||||
}
|
||||
return []string{fmt.Sprintf(" agents run as %s (home %s)", n.AgentAccount, n.AgentHome()),
|
||||
" " + verdict}
|
||||
}
|
||||
|
||||
// orNoneKnown is a value, or that none is known.
|
||||
func orNoneKnown(s string) string {
|
||||
if strings.TrimSpace(s) == "" {
|
||||
return "none known"
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"github.com/novox/mesh-host/rootsearch"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
snapshot "github.com/novox/mesh-controller/internal/facts"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The agent account's judgement (novox/hq ADR 0266): confined only on a healthy account verdict judged for
|
||||
// root, on the very account; every verdict not given — no statement, an older engine, no verdict on it, a
|
||||
// verdict of unknown — is "not judged" and fails, never a pass.
|
||||
func TestAnAgentAccountIsConfinedOnlyOnAHealthyVerdictJudgedForRoot(t *testing.T) {
|
||||
at := time.Date(2026, 10, 8, 19, 21, 0, 0, time.UTC)
|
||||
verdict := func(target, root, state, reason string) inventory.ResourceHealth {
|
||||
return inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account",
|
||||
Kind: link.KindAccount, Target: target, State: state, Reason: reason, Root: root, Account: target}
|
||||
}
|
||||
statement := func(contract int, rs ...inventory.ResourceHealth) inventory.NodeHealth {
|
||||
return inventory.NodeHealth{Node: "anchor", Contract: contract, SaidAt: at, HeardAt: at, Resources: rs}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
h inventory.NodeHealth
|
||||
had bool
|
||||
confined bool
|
||||
says string
|
||||
}{
|
||||
{"judged and unable", statement(link.RootContract, verdict("agent", link.RootNever, link.StateHealthy, "")),
|
||||
true, true, "cannot become root without a person"},
|
||||
{"judged and able", statement(link.RootContract, verdict("agent", link.RootNever, link.StateUnhealthy,
|
||||
link.ReasonRoot+": in the group docker, which grants root")), true, false, "in the group docker"},
|
||||
{"a verdict of unknown", statement(link.RootContract, verdict("agent", link.RootNever, link.StateUnknown,
|
||||
"sudo could not be read")), true, false, "not judged"},
|
||||
{"no statement", inventory.NodeHealth{}, false, false, "not judged"},
|
||||
{"an older engine", statement(link.ReadinessContract, verdict("agent", "", link.StateHealthy, "")),
|
||||
true, false, "older than the judging"},
|
||||
{"a verdict on groups only", statement(link.RootContract, verdict("agent", "", link.StateHealthy, "")),
|
||||
true, false, "no verdict on it"},
|
||||
{"a verdict on another account", statement(link.RootContract, verdict("ops", link.RootNever, link.StateHealthy, "")),
|
||||
true, false, "no verdict on it"},
|
||||
} {
|
||||
confined, why := judgedConfined("agent", c.h, c.had, at.Add(time.Minute))
|
||||
if confined != c.confined || !strings.Contains(why, c.says) {
|
||||
t.Errorf("%s: confined %v, %q; want %v saying %q", c.name, confined, why, c.confined, c.says)
|
||||
}
|
||||
}
|
||||
// A verdict heard longer ago than the bound is no verdict: an agent that stopped the node-engine must not
|
||||
// leave "healthy" standing.
|
||||
fresh := statement(link.RootContract, verdict("agent", link.RootNever, link.StateHealthy, ""))
|
||||
if ok, _ := judgedConfined("agent", fresh, true, at.Add(verdictFreshFor)); !ok {
|
||||
t.Error("a verdict exactly at the bound is still one")
|
||||
}
|
||||
if ok, why := judgedConfined("agent", fresh, true, at.Add(verdictFreshFor+time.Second)); ok ||
|
||||
!strings.Contains(why, "not judged") {
|
||||
t.Errorf("a stale healthy verdict passed: %q", why)
|
||||
}
|
||||
}
|
||||
|
||||
// DA raises an urgent condition, with plain words, on a machine whose agent account is not judged unable to
|
||||
// become root; a machine that names none is not its to judge.
|
||||
func TestTheSelfCheckSaysAnAgentAccountThatCanBecomeRoot(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.NodeByName(ctx, n); err != nil {
|
||||
if _, err := inv.AddNode(ctx, n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.SetAccount(ctx, n, "ops", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.SetAgentAccount(ctx, "anchor", "agent", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &doctor{open: open}
|
||||
found, err := probeAgentAccounts(ctx, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found = onlyMachine(found, "anchor")
|
||||
if len(found) != 1 || found[0].Machine != "anchor" || found[0].Severity != conditions.Urgent ||
|
||||
!strings.Contains(found[0].Said, "not judged") {
|
||||
t.Fatalf("a named agent account with no verdict: %+v", found)
|
||||
}
|
||||
w := plainWordings[kindAgentCanBecomeRoot](conditions.Observation{Kind: kindAgentCanBecomeRoot, Machine: "anchor"})
|
||||
if w.Headline == "" || w.Needs == "" || w.Resolved == "" {
|
||||
t.Errorf("the condition has no plain words: %+v", w)
|
||||
}
|
||||
|
||||
healthy := inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account",
|
||||
Kind: link.KindAccount, Target: "agent", State: link.StateHealthy, Root: link.RootNever, Account: "agent"}
|
||||
if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.RootContract,
|
||||
SaidAt: time.Now(), HeardAt: time.Now(), Resources: []inventory.ResourceHealth{healthy}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if found, err = probeAgentAccounts(ctx, d); err != nil || len(onlyMachine(found, "anchor")) != 0 {
|
||||
t.Fatalf("a judged agent account still fails: %+v %v", found, err)
|
||||
}
|
||||
if named, confined, why, err := agentConfined(ctx, inv, "anchor", time.Now()); err != nil || !named || !confined {
|
||||
t.Fatalf("agentConfined on anchor: %v %v %q %v", named, confined, why, err)
|
||||
}
|
||||
if named, _, why, err := agentConfined(ctx, inv, "laptop", time.Now()); err != nil || named ||
|
||||
!strings.Contains(why, "operator account") {
|
||||
t.Fatalf("agentConfined on a machine naming none: %v %q %v", named, why, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheAgentRootWordsArePlain(t *testing.T) {
|
||||
w := plainWordings[kindAgentCanBecomeRoot](conditions.Observation{Kind: kindAgentCanBecomeRoot, Machine: "anchor"})
|
||||
if why, ok := conditions.PlainWords(w, "anchor"); !ok {
|
||||
t.Fatalf("not plain: %s: %+v", why, w)
|
||||
}
|
||||
}
|
||||
|
||||
func onlyMachine(obs []conditions.Observation, machine string) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, o := range obs {
|
||||
if o.Machine == machine {
|
||||
out = append(out, o)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// The snapshot a merge check composes from carries the agent account as a pseudonym (novox/hq ADR 0266),
|
||||
// so a change is judged against machines that name one, and the name never leaves.
|
||||
func TestTheFactsCarryTheAgentAccountAsAPseudonym(t *testing.T) {
|
||||
open, _ := aMeshWithSecrets(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.SetAccount(ctx, "anchor", "keeper", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetAgentAccount(ctx, "anchor", "warden", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := gatherFacts(ctx, open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, _ := f.Encode()
|
||||
if strings.Contains(string(body), "warden") {
|
||||
t.Error("the agent account's name is in the snapshot")
|
||||
}
|
||||
m, ok := f.Machine(snapshot.Pseudonym("machine", "anchor"))
|
||||
if !ok || m.AgentAccount != snapshot.Pseudonym("account", "warden") || m.Account == m.AgentAccount {
|
||||
t.Fatalf("the anchor's agent account reads as %q (operator %q)", m.AgentAccount, m.Account)
|
||||
}
|
||||
}
|
||||
|
||||
// No verb runs a `node` command that sets something: through the generic `command` verb, `node account`,
|
||||
// `node agent-account` and every other `node` subcommand but list and show are refused, naming the terminal.
|
||||
func TestNoVerbSetsANodesAccounts(t *testing.T) {
|
||||
for _, line := range []string{
|
||||
"node agent-account novox --clear",
|
||||
"node agent-account novox ops",
|
||||
"node account novox agent",
|
||||
"node account novox",
|
||||
"node add intruder",
|
||||
"node public-domain novox --clear",
|
||||
"node",
|
||||
"node frobnicate",
|
||||
} {
|
||||
argv, err := argvFor("command", map[string]any{"command": line})
|
||||
if err == nil || !strings.Contains(err.Error(), "controller's terminal") ||
|
||||
!strings.Contains(err.Error(), "ADR 0266") {
|
||||
t.Errorf("%q ran as %v (%v); want a refusal naming the terminal", line, argv, err)
|
||||
}
|
||||
}
|
||||
for _, line := range []string{"node show novox", "node list --json", "status --json"} {
|
||||
if _, err := argvFor("command", map[string]any{"command": line}); err != nil {
|
||||
t.Errorf("%q, a read, was refused: %v", line, err)
|
||||
}
|
||||
}
|
||||
if err := terminalOnly([]string{"node", "account", "a", "b"}); err == nil {
|
||||
t.Error("the refusal is not only the command verb's")
|
||||
}
|
||||
}
|
||||
|
||||
// Naming the agent account is the controller's terminal's alone: the `node` verb only shows.
|
||||
func TestTheNodeVerbOnlyShows(t *testing.T) {
|
||||
argv, err := argvFor("node", map[string]any{"node": "anchor"})
|
||||
if err != nil || strings.Join(argv, " ") != "node show anchor" {
|
||||
t.Fatalf("the node verb runs %v (%v)", argv, err)
|
||||
}
|
||||
for _, v := range catalogue.ControllerVerbs {
|
||||
if strings.Contains(v.Name, "agent") {
|
||||
t.Errorf("a verb %q may name the agent account", v.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Until a complete search for setuid programs judges — none since the node-engine's state was kept, or the last
|
||||
// older than the engine lets one judge — the agent account is not judged, and the search runs to its end with no
|
||||
// bound (novox/hq issue 361). DA does not raise that as urgent within searchQuietFor, counted from when this
|
||||
// controller first saw it waiting — never from the engine's own "since", which a restart resets, so an engine
|
||||
// restarted in a loop does not keep it quiet. The account is still not confined, and `node show` still says not
|
||||
// judged; a search that failed, or a way to root found, is urgent at once.
|
||||
func TestASearchStillRunningAfterARestartIsNotUrgent(t *testing.T) {
|
||||
if searchQuietFor != rootsearch.Quiet {
|
||||
t.Fatalf("the quiet is %s and the node-engine's %s: they are one value", searchQuietFor, rootsearch.Quiet)
|
||||
}
|
||||
// A daily search that finishes within the quiet never leaves the account unjudged between two of them.
|
||||
if rootsearch.FreshFor < rootsearch.Every+rootsearch.Quiet {
|
||||
t.Fatalf("a complete search judges for %s, less than a day's search (%s) and the quiet (%s)",
|
||||
rootsearch.FreshFor, rootsearch.Every, rootsearch.Quiet)
|
||||
}
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if _, err := inv.NodeByName(ctx, "anchor"); err != nil {
|
||||
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.SetAgentAccount(ctx, "anchor", "agent", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &doctor{open: open}
|
||||
say := func(state, reason string) []conditions.Observation {
|
||||
t.Helper()
|
||||
// The engine's since is always now: it was just restarted.
|
||||
v := inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account",
|
||||
Kind: link.KindAccount, Target: "agent", State: state, Reason: reason, Root: link.RootNever,
|
||||
Account: "agent", Since: time.Now()}
|
||||
if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.RootContract,
|
||||
SaidAt: time.Now(), HeardAt: time.Now(), Resources: []inventory.ResourceHealth{v}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found, err := probeAgentAccounts(ctx, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return onlyMachine(found, "anchor")
|
||||
}
|
||||
running := link.ReasonRootPending + ": the search for setuid programs, started at 19:21, has not finished yet"
|
||||
healthy := func() {
|
||||
t.Helper()
|
||||
if found := say(link.StateHealthy, ""); len(found) != 0 {
|
||||
t.Fatalf("a healthy verdict raised: %+v", found)
|
||||
}
|
||||
}
|
||||
if found := say(link.StateUnknown, running); len(found) != 0 {
|
||||
t.Fatalf("a search first seen now was raised: %+v", found)
|
||||
}
|
||||
if _, confined, why, _ := agentConfined(ctx, inv, "anchor", time.Now()); confined || !strings.Contains(why, "not judged") {
|
||||
t.Fatalf("an account whose search runs was read as confined: %q", why)
|
||||
}
|
||||
// The engine restarted again and again, each statement's own since fresh: the controller's clock runs on.
|
||||
if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
healthy() // a complete verdict forgets when the waiting began …
|
||||
if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil {
|
||||
t.Fatal(err) // … and this restart loop began past the bound
|
||||
}
|
||||
if found := say(link.StateUnknown, running); len(found) != 1 || found[0].Severity != conditions.Urgent {
|
||||
t.Fatalf("a search pending past the bound, by the controller's clock, was not urgent: %+v", found)
|
||||
}
|
||||
healthy()
|
||||
incomplete := rootsearch.ReasonIncomplete + ": the search for setuid programs did not finish (last tried at " +
|
||||
"19:23: timeout); it is tried again later"
|
||||
if found := say(link.StateUnknown, incomplete); len(found) != 1 || found[0].Severity != conditions.Urgent {
|
||||
t.Fatalf("a search that did not finish was not urgent: %+v", found)
|
||||
}
|
||||
if found := say(link.StateUnhealthy, link.ReasonRoot+": in the group docker; "+running); len(found) != 1 ||
|
||||
found[0].Severity != conditions.Urgent {
|
||||
t.Fatalf("a way to root found while the search runs was not urgent: %+v", found)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,937 @@
|
||||
package main
|
||||
|
||||
// The controller asks, and acts on the operator's warrant (novox/hq ADR 0259 §6). It holds no channel, no
|
||||
// identity and no factor: it asks the router like any other module, and performs the answer chosen with its
|
||||
// own grant.
|
||||
//
|
||||
// - **For every open, unsilenced condition that needs the operator and names its answers**, one ask is
|
||||
// published on the `operator-channel` seat under the controller's own name: the condition's words, its
|
||||
// actions as options at their levels (Silence acknowledges; Release, Stop, Start and Restart approve),
|
||||
// answered by the operator, expiring after a day (a week when every option only acknowledges). A
|
||||
// condition that clears, is silenced, or changes its answers has its ask cancelled; an ask that expired
|
||||
// unanswered is asked again while the condition lasts; one the router refused is asked again after a wait
|
||||
// that grows with each refusal in a row, at most half an hour, so a refusal is never believed longer. Each ask is kept in the controller's bucket
|
||||
// `asked`, so a restart neither asks twice nor forgets.
|
||||
// - **On a warrant**, heard on the seat's event under the controller's own name (which only the router may
|
||||
// say), the controller acts once per ask: only for an ask it holds, only for the option it offered at
|
||||
// that option's level, and only while the condition is still open. It performs the action as itself —
|
||||
// a silence through its own conditions, any other through the verb the action names — with the warrant's
|
||||
// words as its why, and records it in the hand-act log as the operator's decision, naming the channel,
|
||||
// the ask and the proofs. An ask that ended without a choice is recorded and nothing is done.
|
||||
// - **A warrant it missed** while away is read from the router's record of its asks, under its own name.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The asker's name on the seat: the controller's module.
|
||||
const askerName = broker.ControllerSeat
|
||||
|
||||
// How long an ask lasts: a day when an answer approves, a week when every answer only acknowledges.
|
||||
const (
|
||||
// askApproveFor is a day less a margin, so an ask is never refused at the router for lasting a day and
|
||||
// a moment (the SDK's bound is a day).
|
||||
askApproveFor = 24*time.Hour - 10*time.Minute
|
||||
askAcknowledgeFor = 7 * 24 * time.Hour
|
||||
// askEvery is how often what is open is asked about again, beside every change.
|
||||
askEvery = time.Minute
|
||||
// askCatchUpAfter is how old an open ask is before the router's record of it is read: a warrant heard
|
||||
// on the event needs no reading.
|
||||
askCatchUpAfter = 2 * time.Minute
|
||||
// askAgainAfterAnswer is how long a condition the operator answered is not asked about again with the
|
||||
// same answers: what was chosen takes a while to clear it, and asking again at once would ask twice.
|
||||
askAgainAfterAnswer = time.Hour
|
||||
// askMostOpen is how many asks the controller holds open at once (the router refuses a fourth): the
|
||||
// most urgent conditions first, then the oldest.
|
||||
askMostOpen = asks.MostOpen
|
||||
// askRefusedRetryMost is the longest a refusal is believed without asking again (novox/hq issue 369): the
|
||||
// router refused while its channels had not yet said they could send, the channels could send twenty
|
||||
// minutes later, and the controller repeated that refusal for eleven hours. A refused ask is asked again
|
||||
// after askEvery, then twice as long after each refusal in a row, never longer than this — and at once when
|
||||
// the channels change.
|
||||
askRefusedRetryMost = 30 * time.Minute
|
||||
// askRefusedCounted is how far back refusals in a row are counted for that wait.
|
||||
askRefusedCounted = 6 * time.Hour
|
||||
// askVerdictWait is how long an ask made again after a refusal waits for the router's word before it is
|
||||
// taken as taken: the router refuses an ask as it reads it, so a refusal comes within seconds. Meanwhile
|
||||
// the condition saying it was not delivered stands as it was, neither cleared nor raised again.
|
||||
askVerdictWait = 2 * time.Minute
|
||||
)
|
||||
|
||||
// refusedRetryAfter is how long a part refused n times in a row waits before it is asked again.
|
||||
func refusedRetryAfter(n int) time.Duration {
|
||||
wait := askEvery
|
||||
for i := 1; i < n && wait < askRefusedRetryMost; i++ {
|
||||
wait *= 2
|
||||
}
|
||||
return min(wait, askRefusedRetryMost)
|
||||
}
|
||||
|
||||
// What became of an ask, as the controller keeps it.
|
||||
const (
|
||||
askOpen = "open"
|
||||
askCancelled = "cancelled"
|
||||
)
|
||||
|
||||
// asked is one ask the controller made, as it keeps it.
|
||||
type asked struct {
|
||||
ID string `json:"id"`
|
||||
Condition string `json:"condition"`
|
||||
// Channels is what the channels were when it was asked (asker.channels): an ask the router refused is asked
|
||||
// again at once when the condition's answers or the channels change, and otherwise after a wait that grows
|
||||
// with each refusal in a row (refusedRetryAfter, novox/hq issue 369).
|
||||
Channels string `json:"channels,omitempty"`
|
||||
Ask asks.Ask `json:"ask"`
|
||||
Actions []conditions.Action `json:"actions"`
|
||||
// Options are the actions by option id.
|
||||
Options map[string]int `json:"options"`
|
||||
State string `json:"state"`
|
||||
Opened time.Time `json:"opened"`
|
||||
Ended time.Time `json:"ended,omitempty"`
|
||||
Warrant *asks.Warrant `json:"warrant,omitempty"`
|
||||
// Acted is what the controller did on the warrant: empty before it did anything, "acting" while it acts,
|
||||
// then "done", "failed: …" or "nothing: …". Anything but empty is never acted on again.
|
||||
Acted string `json:"acted,omitempty"`
|
||||
// Part is which ask of its condition this is (askPart): empty for the one that carries the condition's
|
||||
// answers, or the authorising ones where it has both; "acknowledge" for its acknowledging answers asked
|
||||
// apart (the review of 2026-10-09, M1).
|
||||
Part string `json:"part,omitempty"`
|
||||
// Rehearsal is an ask started at the controller's terminal (rehearse.go): about no condition, its answers
|
||||
// perform nothing, and the reconciling of conditions leaves it alone.
|
||||
Rehearsal bool `json:"rehearsal,omitempty"`
|
||||
// Proposal is a settings layer proposed through a verb (proposals.go, novox/hq ADR 0277): about no
|
||||
// condition, set on Approve by the serving controller itself, and left alone by the reconciling of conditions.
|
||||
Proposal *settingsProposal `json:"proposal,omitempty"`
|
||||
}
|
||||
|
||||
// ofACondition says an ask is one of a condition's: not a rehearsal, not a proposal.
|
||||
func (r asked) ofACondition() bool { return !r.Rehearsal && r.Proposal == nil }
|
||||
|
||||
// partKey is an ask's place among what is asked: its condition and its part.
|
||||
func partKey(condition, part string) string { return condition + "#" + part }
|
||||
|
||||
// partAcknowledge is the part of a condition asked apart for its acknowledging answers.
|
||||
const partAcknowledge = "acknowledge"
|
||||
|
||||
// askPart is one ask a condition is asked with: its part, what it is about, and its answers.
|
||||
type askPart struct {
|
||||
name string
|
||||
about string
|
||||
actions []conditions.Action
|
||||
}
|
||||
|
||||
// levelOf is an action's level as an option offers it: one that says none is never taken for less than
|
||||
// approve.
|
||||
func levelOf(act conditions.Action) asks.Level {
|
||||
if act.Level == "" {
|
||||
return asks.Approve
|
||||
}
|
||||
return asks.Level(act.Level)
|
||||
}
|
||||
|
||||
// partsOf is the asks a condition is asked with (the review of 2026-10-09, M1): one, when its answers are all
|
||||
// of one kind; else its authorising answers (Release, Stop, Restart) in one ask, about the condition, and its
|
||||
// acknowledging ones (Silence) in another. **An acknowledgement never shares an ask with an approval**: a
|
||||
// channel that only acknowledges would otherwise answer the ask, and end the approval with it.
|
||||
func partsOf(c conditions.Condition) []askPart {
|
||||
var ack, auth []conditions.Action
|
||||
for _, act := range c.Actions {
|
||||
if levelOf(act) == asks.Acknowledge {
|
||||
ack = append(ack, act)
|
||||
} else {
|
||||
auth = append(auth, act)
|
||||
}
|
||||
}
|
||||
if len(ack) == 0 || len(auth) == 0 {
|
||||
return []askPart{{about: c.Key, actions: c.Actions}}
|
||||
}
|
||||
return []askPart{{about: c.Key, actions: auth},
|
||||
{name: partAcknowledge, about: c.Key + "." + partAcknowledge, actions: ack}}
|
||||
}
|
||||
|
||||
// askedStore keeps the asks (broker.AskedBucket). **Every write after the first is a compare-and-set** (the
|
||||
// review of 2026-10-09, L2): an ask is created once, and changed only over the revision it was read at, the
|
||||
// change decided again on what is read — so two controllers, or two deliveries of one warrant, never write
|
||||
// over each other, and of two that would act only the one whose write stands does.
|
||||
type askedStore interface {
|
||||
Get(ctx context.Context, id string) (*asked, error)
|
||||
// Create keeps a new ask, and refuses one already kept under its id.
|
||||
Create(ctx context.Context, a asked) error
|
||||
// Change applies change to the ask kept under id, by compare-and-set, and says whether its write stood.
|
||||
// change says whether to write at all; on a write that came between, it is asked again on what is read.
|
||||
Change(ctx context.Context, id string, change func(*asked) bool) (bool, error)
|
||||
All(ctx context.Context) ([]asked, error)
|
||||
}
|
||||
|
||||
// askChangeTries is how often a change is read and tried again when another write came between.
|
||||
const askChangeTries = 5
|
||||
|
||||
// asker is the controller asking the operator and acting on the answer.
|
||||
type asker struct {
|
||||
open func(ctx context.Context) ([]conditions.Condition, error)
|
||||
silence func(ctx context.Context, key string, d time.Duration, by, why string) error
|
||||
store askedStore
|
||||
// publish puts a message on a subject's stream, de-duplicated by id.
|
||||
publish func(ctx context.Context, subject string, body []byte, id string) error
|
||||
// call performs an action's verb with its arguments, as the controller.
|
||||
call func(ctx context.Context, a conditions.Action, args map[string]string) error
|
||||
// setLayer sets a proposed settings layer on the operator's warrant (novox/hq ADR 0277); nil cannot.
|
||||
setLayer setOnWarrant
|
||||
// record writes the hand-act log.
|
||||
record func(ctx context.Context, act link.HandAct) error
|
||||
// routerRecord reads the router's record of an ask for a warrant missed; nil reads nothing.
|
||||
routerRecord func(ctx context.Context, id string) (*asks.Warrant, error)
|
||||
// routerHere says whether a router holds the seat and takes asks under the asker's name; nil is yes.
|
||||
routerHere func(ctx context.Context) (bool, error)
|
||||
// grantHeld says whether the bus holds the controller's grant to ask: the user list the bus's machine was
|
||||
// last sent is the one the mesh composes now (novox/hq issue 353). The grant is composed from the router's
|
||||
// assignment and reaches the bus only when that machine is next pushed, so between `assign` and `push`
|
||||
// the record says a router is here and the bus refuses every ask. why says what to do; nil is yes.
|
||||
grantHeld func(ctx context.Context) (held bool, why string, err error)
|
||||
// channels is what the channels are now, as a fingerprint: who holds which kind, promising what.
|
||||
channels func(ctx context.Context) string
|
||||
// raise keeps the asker's own condition (sourceAsker): which conditions needing the operator could not be
|
||||
// asked, and why. Nil raises nothing (a test that does not look).
|
||||
raise func(ctx context.Context, obs []conditions.Observation) error
|
||||
now func() time.Time
|
||||
logf func(string, ...any)
|
||||
|
||||
saidNoRouter bool
|
||||
saidNoGrant bool
|
||||
|
||||
mu sync.Mutex
|
||||
nudged chan struct{}
|
||||
}
|
||||
|
||||
func (a *asker) nudge() {
|
||||
if a == nil {
|
||||
return
|
||||
}
|
||||
a.mu.Lock()
|
||||
if a.nudged == nil {
|
||||
a.nudged = make(chan struct{}, 1)
|
||||
}
|
||||
ch := a.nudged
|
||||
a.mu.Unlock()
|
||||
select {
|
||||
case ch <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// keep asks until ctx ends: now, on every change of a condition, and every askEvery.
|
||||
func (a *asker) keep(ctx context.Context) {
|
||||
a.nudge()
|
||||
tick := time.NewTicker(askEvery)
|
||||
defer tick.Stop()
|
||||
a.mu.Lock()
|
||||
nudged := a.nudged
|
||||
a.mu.Unlock()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
case <-nudged:
|
||||
}
|
||||
if err := a.reconcile(ctx); err != nil {
|
||||
a.logf("what the operator is asked could not be brought up to date: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// wants says whether a condition is one to ask about now.
|
||||
func wants(c conditions.Condition, now time.Time) bool {
|
||||
return len(c.Actions) > 0 && c.Needs != "" && !c.SilencedAt(now)
|
||||
}
|
||||
|
||||
func sameAsked(a []conditions.Action, b []conditions.Action) bool {
|
||||
x, _ := json.Marshal(a)
|
||||
y, _ := json.Marshal(b)
|
||||
return string(x) == string(y)
|
||||
}
|
||||
|
||||
// reconcile brings what is asked in line with what is open.
|
||||
func (a *asker) reconcile(ctx context.Context) error {
|
||||
now := a.now()
|
||||
if a.routerHere != nil {
|
||||
here, err := a.routerHere(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !here {
|
||||
if !a.saidNoRouter {
|
||||
a.logf("no router takes asks under the controller's name (a module declaring %s with its ask "+
|
||||
"named by its caller, assigned): the operator is asked nothing until one is", broker.AsksSeat)
|
||||
a.saidNoRouter = true
|
||||
}
|
||||
open, err := a.open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var unasked []conditions.Condition
|
||||
for _, c := range open {
|
||||
if wants(c, now) {
|
||||
unasked = append(unasked, c)
|
||||
}
|
||||
}
|
||||
return a.sayUnasked(ctx, unasked, "no router takes the controller's asks: no module holding "+
|
||||
broker.AsksSeat+" that takes an ask under its asker's name is assigned")
|
||||
}
|
||||
a.saidNoRouter = false
|
||||
}
|
||||
if a.grantHeld != nil {
|
||||
held, why, err := a.grantHeld(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !held {
|
||||
if !a.saidNoGrant {
|
||||
a.logf("the bus does not hold the controller's grant to ask yet: %s; the operator is asked nothing until it does", why)
|
||||
a.saidNoGrant = true
|
||||
}
|
||||
open, err := a.open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var unasked []conditions.Condition
|
||||
for _, c := range open {
|
||||
if wants(c, now) {
|
||||
unasked = append(unasked, c)
|
||||
}
|
||||
}
|
||||
return a.sayUnasked(ctx, unasked, "the bus does not hold the controller's grant to ask yet: "+why)
|
||||
}
|
||||
a.saidNoGrant = false
|
||||
}
|
||||
channels := ""
|
||||
if a.channels != nil {
|
||||
channels = a.channels(ctx)
|
||||
}
|
||||
open, err := a.open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
all, err := a.store.All(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
byCondition := map[string]asked{} // by partKey
|
||||
// What is open and not a condition's — a rehearsal, a proposal — still counts toward what the router holds
|
||||
// open for the controller (askMostOpen); expired unanswered, it is kept so, as the router says it too.
|
||||
otherOpen := 0
|
||||
for _, r := range all {
|
||||
if r.State == askOpen && !r.ofACondition() && !now.Before(r.Ask.Expires) {
|
||||
if _, err := a.store.Change(ctx, r.ID, func(x *asked) bool {
|
||||
if x.State != askOpen || x.Acted != "" {
|
||||
return false
|
||||
}
|
||||
x.State, x.Ended, x.Acted = string(asks.OutcomeExpired), now, "nothing: the ask expired unanswered"
|
||||
return true
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if r.State == askOpen && !r.ofACondition() {
|
||||
otherOpen++
|
||||
}
|
||||
if r.State == askOpen && r.ofACondition() {
|
||||
k := partKey(r.Condition, r.Part)
|
||||
if prior, held := byCondition[k]; !held || r.Opened.After(prior.Opened) {
|
||||
byCondition[k] = r
|
||||
}
|
||||
}
|
||||
}
|
||||
// A warrant missed while away, read from the router's record: for a condition's ask, and for a proposal's or a
|
||||
// rehearsal's alike.
|
||||
if a.routerRecord != nil {
|
||||
var lookedUp []asked
|
||||
for _, r := range byCondition {
|
||||
lookedUp = append(lookedUp, r)
|
||||
}
|
||||
for _, r := range all {
|
||||
if r.State == askOpen && !r.ofACondition() {
|
||||
lookedUp = append(lookedUp, r)
|
||||
}
|
||||
}
|
||||
for _, r := range lookedUp {
|
||||
if now.Sub(r.Opened) < askCatchUpAfter {
|
||||
continue
|
||||
}
|
||||
if w, err := a.routerRecord(ctx, r.ID); err == nil && w != nil {
|
||||
body, _ := json.Marshal(w)
|
||||
if err := a.Decided(ctx, body); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if all, err = a.store.All(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
byCondition = map[string]asked{}
|
||||
for _, r := range all {
|
||||
if r.State == askOpen && r.ofACondition() {
|
||||
byCondition[partKey(r.Condition, r.Part)] = r
|
||||
}
|
||||
}
|
||||
}
|
||||
// What the operator answered lately, by condition: not asked again at once; and what the router refused,
|
||||
// newest first: not asked again until the answers or the channels change.
|
||||
answered, refused := map[string]asked{}, map[string]asked{}
|
||||
for _, r := range all {
|
||||
k := partKey(r.Condition, r.Part)
|
||||
if r.State == string(asks.OutcomeChosen) && now.Sub(r.Ended) < askAgainAfterAnswer {
|
||||
answered[k] = r
|
||||
}
|
||||
if r.State == string(asks.OutcomeRefused) {
|
||||
if prior, has := refused[k]; !has || r.Opened.After(prior.Opened) {
|
||||
refused[k] = r
|
||||
}
|
||||
}
|
||||
}
|
||||
// Refusals in a row, by part: those since the last ask that was not refused, within askRefusedCounted.
|
||||
// superseded: a part asked since its last refusal, by an ask the router did not refuse — open, answered,
|
||||
// expired or cancelled. Its refusal is history then, never said again (the review of PR 198: an answered
|
||||
// retry brought the refusal back).
|
||||
inRow, superseded := map[string]int{}, map[string]asked{}
|
||||
for k, last := range refused {
|
||||
for _, r := range all {
|
||||
if partKey(r.Condition, r.Part) == k && r.State != string(asks.OutcomeRefused) && r.State != askUnsent &&
|
||||
r.Opened.After(last.Opened) && r.Opened.After(superseded[k].Opened) {
|
||||
superseded[k] = r
|
||||
}
|
||||
}
|
||||
var since time.Time
|
||||
for _, r := range all {
|
||||
if partKey(r.Condition, r.Part) == k && r.State != string(asks.OutcomeRefused) && r.State != askUnsent &&
|
||||
!r.Opened.After(last.Opened) && r.Opened.After(since) {
|
||||
since = r.Opened
|
||||
}
|
||||
}
|
||||
for _, r := range all {
|
||||
if partKey(r.Condition, r.Part) == k && r.State == string(asks.OutcomeRefused) && r.Opened.After(since) &&
|
||||
now.Sub(r.Opened) < askRefusedCounted {
|
||||
inRow[k]++
|
||||
}
|
||||
}
|
||||
}
|
||||
wanted := map[string]bool{}
|
||||
var unasked []conditions.Condition // refused by the router, and nothing it was refused for changed
|
||||
var refusedWords []string
|
||||
// The most urgent first, then the oldest: those are asked when no more than askMostOpen may be.
|
||||
sort.SliceStable(open, func(i, j int) bool {
|
||||
ui, uj := open[i].Severity == conditions.Urgent, open[j].Severity == conditions.Urgent
|
||||
if ui != uj {
|
||||
return ui
|
||||
}
|
||||
if !open[i].Raised.Equal(open[j].Raised) {
|
||||
return open[i].Raised.Before(open[j].Raised)
|
||||
}
|
||||
return open[i].Key < open[j].Key
|
||||
})
|
||||
openNow := otherOpen
|
||||
for _, c := range open {
|
||||
if !wants(c, now) {
|
||||
continue
|
||||
}
|
||||
for _, p := range partsOf(c) {
|
||||
if r, held := byCondition[partKey(c.Key, p.name)]; held && sameAsked(r.Actions, p.actions) && now.Before(r.Ask.Expires) {
|
||||
openNow++
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, c := range open {
|
||||
if !wants(c, now) {
|
||||
continue
|
||||
}
|
||||
saidUnasked := false
|
||||
for _, p := range partsOf(c) {
|
||||
key := partKey(c.Key, p.name)
|
||||
wanted[key] = true
|
||||
if r, was := refused[key]; was && sameAsked(r.Actions, p.actions) {
|
||||
cur, held := byCondition[key]
|
||||
ended := r.Ended
|
||||
if ended.IsZero() {
|
||||
ended = r.Opened
|
||||
}
|
||||
later, asked := superseded[key]
|
||||
waiting := !held && !asked && r.Channels == channels && now.Sub(ended) < refusedRetryAfter(inRow[key])
|
||||
// Asked again now (the wait over), or lately and the router's word not in yet: said as it was until
|
||||
// that word, so the condition neither clears nor is raised again at each try.
|
||||
retrying := !held && !asked && !waiting
|
||||
verdictDue := held && asked && later.ID == cur.ID && now.Sub(cur.Opened) < askVerdictWait
|
||||
if waiting || retrying || verdictDue {
|
||||
if !saidUnasked {
|
||||
unasked, saidUnasked = append(unasked, c), true
|
||||
}
|
||||
if r.Warrant != nil && r.Warrant.Words != "" {
|
||||
refusedWords = append(refusedWords, r.Warrant.Words)
|
||||
}
|
||||
}
|
||||
if waiting {
|
||||
continue // refused lately, and nothing it was refused for has changed: asked again after the wait
|
||||
}
|
||||
}
|
||||
if r, done := answered[key]; done && sameAsked(r.Actions, p.actions) {
|
||||
if _, held := byCondition[key]; !held {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if r, held := byCondition[key]; held {
|
||||
switch {
|
||||
case !sameAsked(r.Actions, p.actions):
|
||||
if err := a.cancel(ctx, r, "its answers changed"); err != nil {
|
||||
return err
|
||||
}
|
||||
case !now.Before(r.Ask.Expires):
|
||||
// Expired unanswered: the router says so too; asked again below while it lasts.
|
||||
if _, err := a.store.Change(ctx, r.ID, func(x *asked) bool {
|
||||
if x.State != askOpen {
|
||||
return false
|
||||
}
|
||||
x.State, x.Ended = string(asks.OutcomeExpired), now
|
||||
return true
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
openNow--
|
||||
default:
|
||||
continue
|
||||
}
|
||||
}
|
||||
if openNow >= askMostOpen {
|
||||
continue // asked when one of the open ones ends, most urgent first
|
||||
}
|
||||
if err := a.ask(ctx, c, p, channels); err != nil {
|
||||
a.logf("the operator could not be asked about %s: %v", c.Key, err)
|
||||
continue
|
||||
}
|
||||
openNow++
|
||||
}
|
||||
}
|
||||
stillOpen := map[string]conditions.Condition{}
|
||||
for _, c := range open {
|
||||
stillOpen[c.Key] = c
|
||||
}
|
||||
for key, r := range byCondition {
|
||||
if wanted[key] {
|
||||
continue
|
||||
}
|
||||
// **A silence never takes an approval back** (the confirmation review of 2026-10-09, M1). Silence is an
|
||||
// acknowledgement — anyone at the desk may give it — so a condition silenced while its approval is asked
|
||||
// keeps that ask open, unchanged, until it is answered on a channel that proves who answered, or expires.
|
||||
// It is not asked again once it ends, while the silence lasts.
|
||||
if c, open := stillOpen[r.Condition]; open && c.SilencedAt(now) && r.Ask.Highest() != asks.Acknowledge &&
|
||||
now.Before(r.Ask.Expires) && keepsItsAnswers(c, r) {
|
||||
continue
|
||||
}
|
||||
if err := a.cancel(ctx, r, "the condition ended, was silenced or needs nothing now"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
why := "the router refused the ask"
|
||||
if len(refusedWords) > 0 {
|
||||
why += ": " + refusedWords[0]
|
||||
}
|
||||
return a.sayUnasked(ctx, unasked, why)
|
||||
}
|
||||
|
||||
// keepsItsAnswers says a condition still offers the answers an ask kept was asked with.
|
||||
func keepsItsAnswers(c conditions.Condition, r asked) bool {
|
||||
for _, p := range partsOf(c) {
|
||||
if partKey(c.Key, p.name) == partKey(r.Condition, r.Part) {
|
||||
return sameAsked(r.Actions, p.actions)
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// sourceAsker raises the asker's own condition.
|
||||
const sourceAsker = "asker"
|
||||
|
||||
// sayUnasked keeps the asker's one condition: while a condition that needs the operator could not be asked
|
||||
// on any channel, said loudly (failure must be loud), cleared when every one could be.
|
||||
func (a *asker) sayUnasked(ctx context.Context, unasked []conditions.Condition, why string) error {
|
||||
if a.raise == nil {
|
||||
return nil
|
||||
}
|
||||
var obs []conditions.Observation
|
||||
if len(unasked) > 0 {
|
||||
keys := make([]string, 0, len(unasked))
|
||||
severity := conditions.Warning
|
||||
for _, c := range unasked {
|
||||
keys = append(keys, c.Key)
|
||||
if c.Severity == conditions.Urgent {
|
||||
severity = conditions.Urgent
|
||||
}
|
||||
}
|
||||
sort.Strings(keys)
|
||||
obs = append(obs, conditions.Observation{Scope: conditions.ScopeSeat, ID: broker.AsksSeat, Token: "unasked",
|
||||
Kind: "asks-undelivered", Severity: severity, Source: sourceAsker,
|
||||
Summary: fmt.Sprintf("%d condition(s) that need the operator could not be asked on any channel: %s; %s",
|
||||
len(keys), strings.Join(keys, ", "), why),
|
||||
Headline: "Questions for you not delivered",
|
||||
Explanation: "The mesh could not send you its questions on any channel.",
|
||||
Needs: "answer them from the mesh MCP server, and check why no channel carries them.",
|
||||
Resolved: "The mesh can ask you again"})
|
||||
}
|
||||
if err := a.raise(ctx, obs); err != nil {
|
||||
a.logf("whether the operator could be asked could not be kept as a condition: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// optionID is an action's label as an option's id: "Silence for a week" is silence-for-a-week.
|
||||
func optionID(label string) string {
|
||||
var b strings.Builder
|
||||
dash := false
|
||||
for _, r := range strings.ToLower(label) {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
|
||||
b.WriteRune(r)
|
||||
dash = false
|
||||
case !dash && b.Len() > 0:
|
||||
b.WriteByte('-')
|
||||
dash = true
|
||||
}
|
||||
}
|
||||
return strings.TrimSuffix(b.String(), "-")
|
||||
}
|
||||
|
||||
// doesWords is what an action does, in the words an option says it with.
|
||||
func doesWords(act conditions.Action) string {
|
||||
switch {
|
||||
case act.Arguments["silence"] != "":
|
||||
return "nothing more is said of it for a week"
|
||||
case act.Verb == "mesh-delivery.release":
|
||||
return "the delivery goes on"
|
||||
case act.Verb == "mesh-delivery.stop":
|
||||
return "the delivery ends"
|
||||
case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["go"] != "":
|
||||
return "the delivery starts"
|
||||
case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["stop"] != "":
|
||||
return "the delivery is stopped"
|
||||
case strings.HasSuffix(act.Verb, ".restart"):
|
||||
return "its service is restarted on " + act.Machine
|
||||
}
|
||||
return strings.ToLower(act.Label)
|
||||
}
|
||||
|
||||
// askText is a condition's words as an ask says them: without where an answer is given when no channel can
|
||||
// give it (FromMeshMCPServer), since the ask is answered on a channel and the router says where else.
|
||||
func askText(s string) string {
|
||||
for _, with := range []string{", " + FromMeshMCPServer, " " + FromMeshMCPServer} {
|
||||
s = strings.ReplaceAll(s, with, ".")
|
||||
}
|
||||
return strings.ReplaceAll(s, "..", ".")
|
||||
}
|
||||
|
||||
// askOf is the ask one part of a condition is asked with.
|
||||
func askOf(id string, c conditions.Condition, p askPart, now time.Time) (asks.Ask, map[string]int) {
|
||||
q := asks.Ask{ID: id, Headline: c.Headline, Explanation: askText(c.Explanation), Who: asks.Operator,
|
||||
OnExpiry: "nothing is done, and you are asked again while it lasts", About: p.about,
|
||||
Urgent: c.Severity == conditions.Urgent}
|
||||
options := map[string]int{}
|
||||
approves := false
|
||||
for i, act := range p.actions {
|
||||
level := levelOf(act) // an action that says nothing of its level is never taken for less than approve
|
||||
approves = approves || level != asks.Acknowledge
|
||||
oid := optionID(act.Label)
|
||||
options[oid] = i
|
||||
// Every option binds the exact act it stands for (novox/hq ADR 0259 §6): the verb, the machine and
|
||||
// every argument. The warrant then authorises that act and no other.
|
||||
binds, _ := asks.ActDigest(boundAct(act))
|
||||
q.Options = append(q.Options, asks.Option{ID: oid, Label: act.Label, Does: doesWords(act), Level: level,
|
||||
Binds: binds})
|
||||
}
|
||||
q.Expires = now.Add(askAcknowledgeFor)
|
||||
if approves {
|
||||
q.Expires = now.Add(askApproveFor)
|
||||
}
|
||||
return q, options
|
||||
}
|
||||
|
||||
// boundAct is what an option's Binds digests: the act exactly as the controller will perform it — its verb,
|
||||
// machine, level, and each argument as "arg.<name>" — and never its label or words.
|
||||
func boundAct(act conditions.Action) asks.Act {
|
||||
out := asks.Act{"verb": act.Verb, "machine": act.Machine, "level": act.Level}
|
||||
for k, v := range act.Arguments {
|
||||
out["arg."+k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func newAskID() string {
|
||||
var b [8]byte
|
||||
_, _ = rand.Read(b[:])
|
||||
return "c" + hex.EncodeToString(b[:])
|
||||
}
|
||||
|
||||
// askUnsent is an ask kept and never published: asked again at the next look.
|
||||
const askUnsent = "unsent"
|
||||
|
||||
// ask publishes one ask about a part of a condition, kept before it is published (the review of 2026-10-09,
|
||||
// L3): a warrant for it then always finds it, and one whose publishing failed is marked so and asked again.
|
||||
func (a *asker) ask(ctx context.Context, c conditions.Condition, p askPart, channels string) error {
|
||||
now := a.now()
|
||||
id := newAskID()
|
||||
q, options := askOf(id, c, p, now)
|
||||
if err := q.Check(now); err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := json.Marshal(q)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := a.store.Create(ctx, asked{ID: id, Condition: c.Key, Part: p.name, Ask: q, Actions: p.actions,
|
||||
Options: options, State: askOpen, Opened: now, Channels: channels}); err != nil {
|
||||
return fmt.Errorf("the ask could not be kept, so it was not asked: %w", err)
|
||||
}
|
||||
if err := a.publish(ctx, asks.AskSubject(askerName), body, "ask."+id); err != nil {
|
||||
if _, cerr := a.store.Change(ctx, id, func(x *asked) bool {
|
||||
if x.State != askOpen || x.Acted != "" {
|
||||
return false
|
||||
}
|
||||
x.State, x.Ended, x.Acted = askUnsent, a.now(), "nothing: it could not be published: "+err.Error()
|
||||
return true
|
||||
}); cerr != nil {
|
||||
a.logf("the ask %s could not be published, and could not be marked so: %v", id, cerr)
|
||||
}
|
||||
return err
|
||||
}
|
||||
a.logf("asked the operator about %s (%s): %d answer(s)", c.Key, id, len(q.Options))
|
||||
return nil
|
||||
}
|
||||
|
||||
// cancel takes an ask back: kept cancelled first, so a warrant that comes after is refused, then said to the
|
||||
// router; a cancel the router did not hear leaves the ask to expire there, and nothing is done on it here.
|
||||
func (a *asker) cancel(ctx context.Context, r asked, why string) error {
|
||||
stood, err := a.store.Change(ctx, r.ID, func(x *asked) bool {
|
||||
if x.State != askOpen {
|
||||
return false
|
||||
}
|
||||
x.State, x.Ended = askCancelled, a.now()
|
||||
return true
|
||||
})
|
||||
if err != nil || !stood {
|
||||
return err
|
||||
}
|
||||
body, _ := json.Marshal(map[string]string{"id": r.ID})
|
||||
if err := a.publish(ctx, asks.CancelSubject(askerName), body, "cancel."+r.ID); err != nil {
|
||||
a.logf("the ask %s about %s is taken back here, and the router could not be told (%v): it expires there, "+
|
||||
"and no answer to it is acted on", r.ID, r.Condition, err)
|
||||
return nil
|
||||
}
|
||||
a.logf("took back the ask %s about %s: %s", r.ID, r.Condition, why)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Decided takes the router's word on one of the controller's asks (link.Decider). An error is returned only
|
||||
// when what was decided could not be kept, so the word is held and heard again.
|
||||
func (a *asker) Decided(ctx context.Context, body []byte) error {
|
||||
var w asks.Warrant
|
||||
if err := json.Unmarshal(body, &w); err != nil {
|
||||
a.logf("the router's word on an ask could not be read; ignored: %v", err)
|
||||
return nil
|
||||
}
|
||||
if w.Asker != askerName {
|
||||
a.logf("REFUSED a warrant for %s's ask %s: the controller acts only on its own", w.Asker, w.Ask)
|
||||
return nil
|
||||
}
|
||||
r, err := a.store.Get(ctx, w.Ask)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if r == nil {
|
||||
a.logf("REFUSED a warrant for the ask %s, which the controller does not hold", w.Ask)
|
||||
return nil
|
||||
}
|
||||
if r.Acted != "" {
|
||||
return nil // heard again: acted on once
|
||||
}
|
||||
now := a.now()
|
||||
if w.Outcome != asks.OutcomeChosen {
|
||||
acted := "nothing: the ask " + string(w.Outcome)
|
||||
if w.Words != "" {
|
||||
acted += ": " + w.Words
|
||||
}
|
||||
if _, err := a.store.Change(ctx, r.ID, func(x *asked) bool {
|
||||
if x.Acted != "" {
|
||||
return false
|
||||
}
|
||||
x.State, x.Ended, x.Warrant, x.Acted = string(w.Outcome), now, &w, acted
|
||||
return true
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
a.logf("the ask %s about %s ended %s; nothing is done", r.ID, r.Condition, w.Outcome)
|
||||
return nil
|
||||
}
|
||||
if r.State != askOpen {
|
||||
// Cancelled, replaced or expired in the controller's own record: no answer to it is acted on.
|
||||
a.logf("REFUSED a warrant for the ask %s, which is %s in the controller's own record", r.ID, r.State)
|
||||
return nil
|
||||
}
|
||||
option, err := w.For(askerName, r.Ask)
|
||||
if err != nil {
|
||||
a.logf("REFUSED a warrant for the ask %s: %v", r.ID, err)
|
||||
return nil
|
||||
}
|
||||
index, offered := r.Options[option.ID]
|
||||
if !offered || index >= len(r.Actions) {
|
||||
a.logf("REFUSED a warrant for the ask %s: it chose %s, which no action stands for", r.ID, option.ID)
|
||||
return nil
|
||||
}
|
||||
act := r.Actions[index]
|
||||
// The act about to be performed is the one the option bound when the controller asked: a record changed
|
||||
// since is refused, never performed.
|
||||
if err := option.Performs(boundAct(act)); err != nil {
|
||||
a.logf("REFUSED a warrant for the ask %s: %v", r.ID, err)
|
||||
return nil
|
||||
}
|
||||
open, err := a.open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stillOpen := r.Rehearsal || r.Proposal != nil // a rehearsal and a proposal are about no condition
|
||||
for _, c := range open {
|
||||
stillOpen = stillOpen || c.Key == r.Condition
|
||||
}
|
||||
if !stillOpen {
|
||||
// The asker checks the state is still what it asked about before it acts (to-be 46 §10, step 7).
|
||||
if _, err := a.store.Change(ctx, r.ID, func(x *asked) bool {
|
||||
if x.State != askOpen || x.Acted != "" {
|
||||
return false
|
||||
}
|
||||
x.State, x.Warrant, x.Ended, x.Acted = string(asks.OutcomeChosen), &w, now,
|
||||
"nothing: the condition ended before the answer"
|
||||
return true
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
a.logf("%s, for %s, which ended meanwhile: nothing is done", w.Says(), r.Condition)
|
||||
return nil
|
||||
}
|
||||
// Claimed before acting, by compare-and-set: only the delivery whose write stands acts (security review
|
||||
// of 2026-10-08, finding 9). Not by the warrant's message id, which another publisher could take first:
|
||||
// the controller's own record decides.
|
||||
claimed, err := a.store.Change(ctx, r.ID, func(x *asked) bool {
|
||||
if x.State != askOpen || x.Acted != "" {
|
||||
return false
|
||||
}
|
||||
x.State, x.Warrant, x.Acted = string(asks.OutcomeChosen), &w, "acting"
|
||||
return true
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !claimed {
|
||||
a.logf("the warrant for the ask %s was already taken by another delivery; nothing more is done", r.ID)
|
||||
return nil
|
||||
}
|
||||
r.Acted = "acting"
|
||||
|
||||
why := fmt.Sprintf("%s (ask %s)", w.Says(), r.ID)
|
||||
args := map[string]string{}
|
||||
for k, v := range act.Arguments {
|
||||
args[k] = v
|
||||
}
|
||||
if v, takes := args["why"]; takes && v == "" {
|
||||
args["why"] = why
|
||||
}
|
||||
var acted error
|
||||
outcome := "done"
|
||||
switch {
|
||||
case r.Rehearsal && act.Verb == rehearsalVerb:
|
||||
// A rehearsal's answer performs nothing: it is recorded below as the operator's decision.
|
||||
case r.Proposal != nil && act.Verb == proposalVerb:
|
||||
// A proposed settings layer, set by this controller itself on Approve (novox/hq ADR 0277): the act's
|
||||
// digest of the values is held to the record's own values before anything is set.
|
||||
outcome, acted = a.decideProposal(ctx, *r, act, w)
|
||||
if acted == nil && outcome != "" && !strings.HasPrefix(outcome, "nothing") {
|
||||
outcome = "done: " + outcome
|
||||
}
|
||||
case act.Arguments["silence"] != "":
|
||||
acted = a.silence(ctx, act.Arguments["silence"], conditions.MaxSilence, byWords(w), why)
|
||||
default:
|
||||
acted = a.call(ctx, act, args)
|
||||
}
|
||||
ended := a.now()
|
||||
if acted != nil {
|
||||
outcome = "failed: " + acted.Error()
|
||||
}
|
||||
r.Ended, r.Acted = ended, outcome
|
||||
if _, err := a.store.Change(ctx, r.ID, func(x *asked) bool {
|
||||
if x.Acted != "acting" {
|
||||
return false
|
||||
}
|
||||
x.Ended, x.Acted = ended, outcome
|
||||
return true
|
||||
}); err != nil {
|
||||
a.logf("%s was acted on (%s), and how it ended could NOT be kept: %v", r.ID, outcome, err)
|
||||
}
|
||||
verbArgs := []string{act.Verb}
|
||||
if act.Machine != "" {
|
||||
verbArgs = append(verbArgs, "on "+act.Machine)
|
||||
}
|
||||
keys := make([]string, 0, len(args))
|
||||
for k := range args {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
if k != "why" {
|
||||
verbArgs = append(verbArgs, k+"="+args[k])
|
||||
}
|
||||
}
|
||||
if err := a.record(ctx, link.HandAct{Verb: handActWarrant, Args: verbArgs, Why: why, By: byWords(w),
|
||||
Cause: conditions.CauseOperatorAnswer, Condition: r.Condition, Via: viaWords(w), Ask: r.ID,
|
||||
Proofs: w.Proofs, RequestedBy: r.Condition, Outcome: r.Acted}); err != nil {
|
||||
a.logf("%s was done, and could NOT be recorded in the hand-act log: %v", why, err)
|
||||
}
|
||||
a.logf("%s: %s", why, r.Acted)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handActWarrant is the verb an act the operator chose on a warrant is recorded under: a person's decision,
|
||||
// never a repair (handActVerbs).
|
||||
const handActWarrant = "warrant"
|
||||
|
||||
// byWords is who chose, as the hand-act log says it: "the operator, as telegram identity 42".
|
||||
func byWords(w asks.Warrant) string {
|
||||
if w.By == nil {
|
||||
return "the operator"
|
||||
}
|
||||
return fmt.Sprintf("the %s, as %s identity %s", w.By.Who, w.By.Kind, w.By.Identity)
|
||||
}
|
||||
|
||||
// viaWords is the channel an answer came through: its module and kind, and how the sender was known.
|
||||
func viaWords(w asks.Warrant) string {
|
||||
if w.By == nil {
|
||||
return w.Channel
|
||||
}
|
||||
via := w.Channel + " (" + w.By.Kind + ")"
|
||||
if w.By.Verified != "" {
|
||||
via += ", " + w.By.Verified
|
||||
}
|
||||
return via
|
||||
}
|
||||
|
||||
// errNotGranted is an action whose verb the controller's grant does not name.
|
||||
var errNotGranted = errors.New("the controller's grant does not name this verb")
|
||||
@@ -0,0 +1,151 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// busAsker is an asker on a real bus's `asked` bucket, counting what it performs: two of them are two
|
||||
// controllers sharing one record.
|
||||
type busAskerRig struct {
|
||||
mu sync.Mutex
|
||||
called int
|
||||
acts int
|
||||
open []conditions.Condition
|
||||
sent [][]byte
|
||||
}
|
||||
|
||||
func (rig *busAskerRig) asker(t *testing.T, conn *nats.Conn, now time.Time) *asker {
|
||||
return &asker{
|
||||
open: func(context.Context) ([]conditions.Condition, error) {
|
||||
rig.mu.Lock()
|
||||
defer rig.mu.Unlock()
|
||||
return rig.open, nil
|
||||
},
|
||||
silence: func(context.Context, string, time.Duration, string, string) error { return nil },
|
||||
store: busAsked{conn: conn},
|
||||
publish: func(_ context.Context, subject string, body []byte, _ string) error {
|
||||
rig.mu.Lock()
|
||||
defer rig.mu.Unlock()
|
||||
if subject == asks.AskSubject(askerName) {
|
||||
rig.sent = append(rig.sent, body)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
call: func(context.Context, conditions.Action, map[string]string) error {
|
||||
time.Sleep(20 * time.Millisecond) // long enough for the other delivery to arrive meanwhile
|
||||
rig.mu.Lock()
|
||||
defer rig.mu.Unlock()
|
||||
rig.called++
|
||||
return nil
|
||||
},
|
||||
record: func(context.Context, link.HandAct) error {
|
||||
rig.mu.Lock()
|
||||
defer rig.mu.Unlock()
|
||||
rig.acts++
|
||||
return nil
|
||||
},
|
||||
now: func() time.Time { return now },
|
||||
logf: t.Logf,
|
||||
}
|
||||
}
|
||||
|
||||
func askedBus(t *testing.T) *nats.Conn {
|
||||
t.Helper()
|
||||
conn, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(conn.Close)
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.CreateKeyValue(context.Background(), jetstream.KeyValueConfig{Bucket: broker.AskedBucket}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return conn
|
||||
}
|
||||
|
||||
// The review of 2026-10-09 (L7): two deliveries of one warrant, to two controllers at once, perform its act
|
||||
// exactly once and record it once — the record's compare-and-set decides, never the warrant's message id.
|
||||
func TestTwoAnswersAtOnceActOnce(t *testing.T) {
|
||||
conn := askedBus(t)
|
||||
now := time.Date(2026, 10, 9, 14, 0, 0, 0, time.UTC)
|
||||
rig := &busAskerRig{open: []conditions.Condition{heldCondition()}}
|
||||
first, second := rig.asker(t, conn, now), rig.asker(t, conn, now)
|
||||
if err := first.reconcile(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rig.sent) != 1 {
|
||||
t.Fatalf("asked %d times", len(rig.sent))
|
||||
}
|
||||
var q asks.Ask
|
||||
_ = json.Unmarshal(rig.sent[0], &q)
|
||||
release, _ := q.Option("release")
|
||||
w := asks.Warrant{Ask: q.ID, Asker: askerName, About: q.About, Outcome: asks.OutcomeChosen, Option: release.ID,
|
||||
Label: release.Label, Level: release.Level, Channel: "telegram", Proofs: []string{"P1"}, At: now,
|
||||
AskDigest: q.Digest(), By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}}
|
||||
body, _ := json.Marshal(w)
|
||||
var wg sync.WaitGroup
|
||||
for _, a := range []*asker{first, second, first, second} {
|
||||
wg.Add(1)
|
||||
go func(a *asker) {
|
||||
defer wg.Done()
|
||||
if err := a.Decided(context.Background(), body); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}(a)
|
||||
}
|
||||
wg.Wait()
|
||||
if rig.called != 1 || rig.acts != 1 {
|
||||
t.Fatalf("performed %d time(s), recorded %d time(s)", rig.called, rig.acts)
|
||||
}
|
||||
got, err := busAsked{conn: conn}.Get(context.Background(), q.ID)
|
||||
if err != nil || got == nil || got.Acted != "done" {
|
||||
t.Fatalf("kept as %+v (%v)", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The review of 2026-10-09 (L2): a write decided on a record read earlier never lands over one made since. A
|
||||
// cancel read before the answer was acted on leaves the act's record as it is.
|
||||
func TestAStaleCancelDoesNotWriteOverAnAct(t *testing.T) {
|
||||
conn := askedBus(t)
|
||||
now := time.Date(2026, 10, 9, 14, 0, 0, 0, time.UTC)
|
||||
rig := &busAskerRig{open: []conditions.Condition{heldCondition()}}
|
||||
a := rig.asker(t, conn, now)
|
||||
if err := a.reconcile(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var q asks.Ask
|
||||
_ = json.Unmarshal(rig.sent[0], &q)
|
||||
stale, _ := busAsked{conn: conn}.Get(context.Background(), q.ID)
|
||||
release, _ := q.Option("release")
|
||||
w := asks.Warrant{Ask: q.ID, Asker: askerName, About: q.About, Outcome: asks.OutcomeChosen, Option: release.ID,
|
||||
Label: release.Label, Level: release.Level, Channel: "telegram", At: now, AskDigest: q.Digest(),
|
||||
By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}}
|
||||
body, _ := json.Marshal(w)
|
||||
if err := a.Decided(context.Background(), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := a.cancel(context.Background(), *stale, "the condition ended"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ := busAsked{conn: conn}.Get(context.Background(), q.ID)
|
||||
if got.State != string(asks.OutcomeChosen) || got.Acted != "done" {
|
||||
t.Errorf("a stale cancel wrote over the act: %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,810 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0259 §6: the controller asks the operator for the answers its conditions name, and performs
|
||||
// the one chosen on the router's warrant — once, for its own ask, the option offered, at its level.
|
||||
|
||||
type memAskedStore map[string]asked
|
||||
|
||||
// memAskedMu guards every memAskedStore: Change is a compare-and-set as the bus's is.
|
||||
var memAskedMu sync.Mutex
|
||||
|
||||
func (m memAskedStore) Get(_ context.Context, id string) (*asked, error) {
|
||||
memAskedMu.Lock()
|
||||
defer memAskedMu.Unlock()
|
||||
r, ok := m[id]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
return &r, nil
|
||||
}
|
||||
func (m memAskedStore) Create(_ context.Context, r asked) error {
|
||||
memAskedMu.Lock()
|
||||
defer memAskedMu.Unlock()
|
||||
if _, kept := m[r.ID]; kept {
|
||||
return errors.New("an ask is kept under that id")
|
||||
}
|
||||
m[r.ID] = r
|
||||
return nil
|
||||
}
|
||||
func (m memAskedStore) Change(_ context.Context, id string, change func(*asked) bool) (bool, error) {
|
||||
memAskedMu.Lock()
|
||||
defer memAskedMu.Unlock()
|
||||
r, ok := m[id]
|
||||
if !ok || !change(&r) {
|
||||
return false, nil
|
||||
}
|
||||
m[id] = r
|
||||
return true, nil
|
||||
}
|
||||
func (m memAskedStore) All(context.Context) ([]asked, error) {
|
||||
memAskedMu.Lock()
|
||||
defer memAskedMu.Unlock()
|
||||
var out []asked
|
||||
for _, r := range m {
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
type published struct {
|
||||
subject, id string
|
||||
body []byte
|
||||
}
|
||||
|
||||
type askerRig struct {
|
||||
a *asker
|
||||
open []conditions.Condition
|
||||
store memAskedStore
|
||||
sent []published
|
||||
called []string
|
||||
silenced []string
|
||||
acts []link.HandAct
|
||||
now time.Time
|
||||
}
|
||||
|
||||
func newAskerRig(t *testing.T) *askerRig {
|
||||
r := &askerRig{store: memAskedStore{}, now: time.Date(2026, 10, 8, 14, 0, 0, 0, time.UTC)}
|
||||
r.a = &asker{
|
||||
open: func(context.Context) ([]conditions.Condition, error) { return r.open, nil },
|
||||
silence: func(_ context.Context, key string, d time.Duration, by, why string) error {
|
||||
r.silenced = append(r.silenced, key+" for "+d.String()+" by "+by+" because "+why)
|
||||
// As the controller's conditions do (the confirmation review of 2026-10-09, M1): the condition is
|
||||
// silenced from now on, so what is asked next sees it silenced.
|
||||
for i := range r.open {
|
||||
if r.open[i].Key == key {
|
||||
r.open[i].Silenced = &conditions.Silence{Until: r.now.Add(d), By: by, Why: why, Since: r.now}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
store: r.store,
|
||||
publish: func(_ context.Context, subject string, body []byte, id string) error {
|
||||
r.sent = append(r.sent, published{subject, id, body})
|
||||
return nil
|
||||
},
|
||||
call: func(_ context.Context, a conditions.Action, args map[string]string) error {
|
||||
raw, _ := json.Marshal(args)
|
||||
r.called = append(r.called, a.Verb+"@"+a.Machine+" "+string(raw))
|
||||
return nil
|
||||
},
|
||||
record: func(_ context.Context, act link.HandAct) error { r.acts = append(r.acts, act); return nil },
|
||||
now: func() time.Time { return r.now },
|
||||
logf: t.Logf,
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func heldCondition() conditions.Condition {
|
||||
o := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s",
|
||||
Bound: "24h0m0s", H2: "none: the state is the operator's"}})[0]
|
||||
return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: conditions.Warning, Headline: o.Headline,
|
||||
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}
|
||||
}
|
||||
|
||||
func unitsCondition() conditions.Condition {
|
||||
key := "machine.shanks.units"
|
||||
return conditions.Condition{Key: key, Kind: "machine-units", Severity: conditions.Warning,
|
||||
Headline: "3 failed services on shanks", Explanation: "Needs you: mend or remove them on shanks, or silence this.",
|
||||
Needs: "mend or remove them on shanks, or silence this.", Actions: []conditions.Action{conditions.SilenceAction(key)}}
|
||||
}
|
||||
|
||||
func (r *askerRig) asksSent(t *testing.T) []asks.Ask {
|
||||
t.Helper()
|
||||
var out []asks.Ask
|
||||
for _, p := range r.sent {
|
||||
if p.subject != asks.AskSubject("mesh-controller") {
|
||||
continue
|
||||
}
|
||||
var q asks.Ask
|
||||
if err := json.Unmarshal(p.body, &q); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out = append(out, q)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestAnAskIsMadeForEachConditionThatNamesItsAnswers(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
quiet := conditions.Condition{Key: "machine.ace.silent", Headline: "ace silent", Explanation: "Nothing for you to do. x"}
|
||||
r.open = []conditions.Condition{heldCondition(), unitsCondition(), quiet}
|
||||
if err := r.a.reconcile(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sent := r.asksSent(t)
|
||||
if len(sent) != 2 {
|
||||
t.Fatalf("asked %d times: %+v", len(sent), sent)
|
||||
}
|
||||
byAbout := map[string]asks.Ask{}
|
||||
for _, q := range sent {
|
||||
byAbout[q.About] = q
|
||||
if err := q.Check(r.now); err != nil {
|
||||
t.Errorf("%s: %v", q.About, err)
|
||||
}
|
||||
}
|
||||
held := byAbout[heldCondition().Key]
|
||||
if len(held.Options) != 2 || held.Options[0].Label != "Release" || held.Options[0].Level != asks.Approve ||
|
||||
held.Options[1].ID != "stop" || held.Expires != r.now.Add(askApproveFor) || held.Who != asks.Operator ||
|
||||
held.OnExpiry == "" {
|
||||
t.Errorf("the held delivery is asked %+v", held)
|
||||
}
|
||||
units := byAbout["machine.shanks.units"]
|
||||
if len(units.Options) != 1 || units.Options[0].Level != asks.Acknowledge || units.Expires != r.now.Add(askAcknowledgeFor) {
|
||||
t.Errorf("the failed units are asked %+v", units)
|
||||
}
|
||||
// No second ask while one is open.
|
||||
r.now = r.now.Add(time.Minute)
|
||||
_ = r.a.reconcile(context.Background())
|
||||
if n := len(r.asksSent(t)); n != 2 {
|
||||
t.Errorf("asked again while open: %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAskIsTakenBackWhenItsConditionEndsAndAskedAgainAfterItExpires(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
r.open = []conditions.Condition{heldCondition(), unitsCondition()}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
// The units are silenced, the held delivery lasts past its ask's day.
|
||||
units := unitsCondition()
|
||||
units.Silenced = &conditions.Silence{Until: r.now.Add(48 * time.Hour)}
|
||||
r.open = []conditions.Condition{heldCondition(), units}
|
||||
r.now = r.now.Add(askApproveFor)
|
||||
if err := r.a.reconcile(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var cancels int
|
||||
for _, p := range r.sent {
|
||||
if p.subject == asks.CancelSubject("mesh-controller") {
|
||||
cancels++
|
||||
}
|
||||
}
|
||||
if cancels != 1 {
|
||||
t.Errorf("cancels %d, want the silenced one's", cancels)
|
||||
}
|
||||
if sent := r.asksSent(t); len(sent) != 3 || sent[2].About != heldCondition().Key {
|
||||
t.Errorf("the expired ask was not asked again: %+v", sent)
|
||||
}
|
||||
}
|
||||
|
||||
// warrantFor is the router's warrant for the open ask about a condition, choosing an option by label.
|
||||
func (r *askerRig) warrantFor(t *testing.T, condition, label string) asks.Warrant {
|
||||
t.Helper()
|
||||
for _, a := range r.store {
|
||||
if a.Condition != condition || a.State != askOpen {
|
||||
continue
|
||||
}
|
||||
for _, o := range a.Ask.Options {
|
||||
if o.Label == label {
|
||||
return asks.Warrant{Ask: a.ID, Asker: "mesh-controller", About: condition, Outcome: asks.OutcomeChosen,
|
||||
Option: o.ID, Label: o.Label, Level: o.Level, Channel: "telegram", Proofs: []string{"P1"}, At: r.now,
|
||||
AskDigest: a.Ask.Digest(),
|
||||
By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}}
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Fatalf("no open ask about %s offers %s", condition, label)
|
||||
return asks.Warrant{}
|
||||
}
|
||||
|
||||
func answerWith(t *testing.T, r *askerRig, w asks.Warrant) {
|
||||
t.Helper()
|
||||
body, _ := json.Marshal(w)
|
||||
if err := r.a.Decided(context.Background(), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWarrantIsActedOnOnce(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
r.open = []conditions.Condition{heldCondition()}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
w := r.warrantFor(t, heldCondition().Key, "Release")
|
||||
answerWith(t, r, w)
|
||||
answerWith(t, r, w) // heard again
|
||||
if len(r.called) != 1 {
|
||||
t.Fatalf("called %v", r.called)
|
||||
}
|
||||
want := `mesh-delivery.release@ {"id":"novox/hq@055550802096","why":"the operator, via telegram (user id verified), chose Release (ask ` + w.Ask + `)"}`
|
||||
if r.called[0] != want {
|
||||
t.Errorf("called\n %s\nwant\n %s", r.called[0], want)
|
||||
}
|
||||
if len(r.acts) != 1 {
|
||||
t.Fatalf("hand-acts %+v", r.acts)
|
||||
}
|
||||
act := r.acts[0]
|
||||
if act.Verb != handActWarrant || act.By != "the operator, as telegram identity 42" ||
|
||||
act.Via != "telegram (telegram), user id verified" || act.Ask != w.Ask || strings.Join(act.Proofs, ",") != "P1" ||
|
||||
act.Cause != conditions.CauseOperatorAnswer || act.Condition != heldCondition().Key || act.Outcome != "done" {
|
||||
t.Errorf("the hand-act %+v", act)
|
||||
}
|
||||
if !personsDecision(act) {
|
||||
t.Error("an act on a warrant counts as a repair")
|
||||
}
|
||||
if got := r.store[w.Ask]; got.State != string(asks.OutcomeChosen) || got.Acted != "done" {
|
||||
t.Errorf("kept %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWarrantThatIsNotForItsOwnAskIsRefused(t *testing.T) {
|
||||
for name, change := range map[string]func(*asks.Warrant){
|
||||
"another asker": func(w *asks.Warrant) { w.Asker = "mesh-delivery" },
|
||||
"an ask not held": func(w *asks.Warrant) { w.Ask = "c0000000000000000" },
|
||||
"an option not offered": func(w *asks.Warrant) { w.Option = "delete" },
|
||||
"another level": func(w *asks.Warrant) { w.Level = asks.Acknowledge },
|
||||
"no person": func(w *asks.Warrant) { w.By = nil },
|
||||
"another ask's digest": func(w *asks.Warrant) { w.AskDigest = "sha256:0000" },
|
||||
"no ask's digest": func(w *asks.Warrant) { w.AskDigest = "" },
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
r.open = []conditions.Condition{heldCondition()}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
w := r.warrantFor(t, heldCondition().Key, "Stop")
|
||||
change(&w)
|
||||
answerWith(t, r, w)
|
||||
if len(r.called)+len(r.acts)+len(r.silenced) != 0 {
|
||||
t.Errorf("acted on it: %v %v %v", r.called, r.acts, r.silenced)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEachAnswerCallsExactlyItsVerb(t *testing.T) {
|
||||
plan := "plan-1791454185265004861"
|
||||
waiting := conditions.Condition{Key: "plan." + plan + ".waiting", Severity: conditions.Urgent,
|
||||
Headline: "openrazer delivery waiting to start", Needs: "start it, or stop it.",
|
||||
Explanation: "Needs you: start it, or stop it.", Actions: waitingActions(plan, conditions.Urgent)}
|
||||
module := conditions.Condition{Key: "module.openrazer.g14.unhealthy", Severity: conditions.Warning,
|
||||
Headline: "openrazer not working on g14", Needs: "restart its service openrazer-daemon on g14.",
|
||||
Explanation: "Needs you: restart it.", Actions: []conditions.Action{{Label: "Restart",
|
||||
Verb: "node-service-manager.restart", Machine: "g14", Level: conditions.LevelApprove,
|
||||
Arguments: map[string]string{"unit": "openrazer-daemon.service", "scope": "user"}}}}
|
||||
for _, tc := range []struct {
|
||||
c conditions.Condition
|
||||
label string
|
||||
want string
|
||||
}{
|
||||
{waiting, "Start", `mesh-controller.plans@ {"cause":"operator-answer","go":"` + plan + `","why":"`},
|
||||
{waiting, "Stop", `mesh-controller.plans@ {"cause":"operator-answer","stop":"` + plan + `","why":"`},
|
||||
{module, "Restart", `node-service-manager.restart@g14 {"scope":"user","unit":"openrazer-daemon.service"}`},
|
||||
} {
|
||||
r := newAskerRig(t)
|
||||
r.open = []conditions.Condition{tc.c}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
answerWith(t, r, r.warrantFor(t, tc.c.Key, tc.label))
|
||||
if len(r.called) != 1 || !strings.HasPrefix(r.called[0], tc.want) {
|
||||
t.Errorf("%s: called %v, want %s…", tc.label, r.called, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestASilenceChosenIsTheControllersOwnAndAnAnswerToAnAsk(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
r.open = []conditions.Condition{unitsCondition()}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
w := r.warrantFor(t, "machine.shanks.units", "Silence for a week")
|
||||
w.Level, w.Proofs = asks.Acknowledge, nil
|
||||
w.By = &asks.Person{Who: asks.Operator, Kind: "desktop", Identity: "g14",
|
||||
Verified: "a desk click: whoever was at the operator's session on g14"}
|
||||
w.Channel = "desk-channel"
|
||||
answerWith(t, r, w)
|
||||
if len(r.called) != 0 || len(r.silenced) != 1 || !strings.HasPrefix(r.silenced[0], "machine.shanks.units for 168h0m0s by the operator, as desktop identity g14") {
|
||||
t.Fatalf("silenced %v, called %v", r.silenced, r.called)
|
||||
}
|
||||
if len(r.acts) != 1 || r.acts[0].Cause != conditions.CauseOperatorAnswer || len(r.acts[0].Proofs) != 0 {
|
||||
t.Errorf("%+v", r.acts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAskThatEndedWithoutAChoiceDoesNothing(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
r.open = []conditions.Condition{heldCondition()}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
w := r.warrantFor(t, heldCondition().Key, "Release")
|
||||
w.Outcome, w.Option, w.Label, w.Level, w.By, w.Words = asks.OutcomeExpired, "", "", "", nil, "nobody answered in time"
|
||||
answerWith(t, r, w)
|
||||
if len(r.called)+len(r.acts) != 0 || r.store[w.Ask].State != string(asks.OutcomeExpired) ||
|
||||
!strings.HasPrefix(r.store[w.Ask].Acted, "nothing") {
|
||||
t.Errorf("called %v acts %v kept %+v", r.called, r.acts, r.store[w.Ask])
|
||||
}
|
||||
// And a choice for a condition that ended meanwhile does nothing either.
|
||||
r2 := newAskerRig(t)
|
||||
r2.open = []conditions.Condition{heldCondition()}
|
||||
_ = r2.a.reconcile(context.Background())
|
||||
w2 := r2.warrantFor(t, heldCondition().Key, "Release")
|
||||
r2.open = nil
|
||||
answerWith(t, r2, w2)
|
||||
if len(r2.called) != 0 || r2.store[w2.Ask].Acted != "nothing: the condition ended before the answer" {
|
||||
t.Errorf("%v %+v", r2.called, r2.store[w2.Ask])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWarrantMissedWhileAwayIsReadFromTheRoutersRecord(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
r.open = []conditions.Condition{heldCondition()}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
w := r.warrantFor(t, heldCondition().Key, "Stop")
|
||||
r.a.routerRecord = func(_ context.Context, id string) (*asks.Warrant, error) {
|
||||
if id != w.Ask {
|
||||
return nil, errors.New("another ask")
|
||||
}
|
||||
return &w, nil
|
||||
}
|
||||
r.now = r.now.Add(askCatchUpAfter)
|
||||
if err := r.a.reconcile(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "mesh-delivery.stop@") {
|
||||
t.Errorf("called %v", r.called)
|
||||
}
|
||||
if n := len(r.asksSent(t)); n != 1 {
|
||||
t.Errorf("asked again after the answer: %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
// After review (2026-10-08): a refused ask is not asked again at once; issue 369: nor is the refusal believed
|
||||
// for ever — it is asked again after a wait that doubles with each refusal in a row, and at once when the
|
||||
// channels change.
|
||||
func TestAnAskTheRouterRefusedIsAskedAgainAfterAGrowingWait(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
channels := "channel/telegram=telegram@anchor[choice]own:true"
|
||||
r.a.channels = func(context.Context) string { return channels }
|
||||
r.open = []conditions.Condition{heldCondition()}
|
||||
refuse := func() {
|
||||
t.Helper()
|
||||
sent := r.asksSent(t)
|
||||
refusal, _ := json.Marshal(asks.Warrant{Ask: sent[len(sent)-1].ID, Asker: "mesh-controller",
|
||||
Outcome: asks.OutcomeRefused, Words: "no channel can carry any of its answers now", At: r.now})
|
||||
if err := r.a.Decided(context.Background(), refusal); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
first := r.asksSent(t)[0]
|
||||
refuse()
|
||||
if got := r.store[first.ID]; got.State != string(asks.OutcomeRefused) || !strings.Contains(got.Acted, "nothing") {
|
||||
t.Fatalf("the refusal was kept as %+v", got)
|
||||
}
|
||||
// Each wait: nothing asked before it ends, asked once when it has.
|
||||
for i, wait := range []time.Duration{time.Minute, 2 * time.Minute, 4 * time.Minute, 8 * time.Minute,
|
||||
16 * time.Minute, 30 * time.Minute, 30 * time.Minute} {
|
||||
before := len(r.asksSent(t))
|
||||
r.now = r.now.Add(wait - 10*time.Second)
|
||||
_ = r.a.reconcile(context.Background())
|
||||
if n := len(r.asksSent(t)); n != before {
|
||||
t.Fatalf("refusal %d: asked again before %s", i+1, wait)
|
||||
}
|
||||
r.now = r.now.Add(10 * time.Second)
|
||||
_ = r.a.reconcile(context.Background())
|
||||
if n := len(r.asksSent(t)); n != before+1 {
|
||||
t.Fatalf("refusal %d: not asked again after %s (%d asks)", i+1, wait, n)
|
||||
}
|
||||
refuse()
|
||||
}
|
||||
before := len(r.asksSent(t))
|
||||
channels = "channel/telegram=telegram@anchor[choice,verified-sender]own:true"
|
||||
_ = r.a.reconcile(context.Background())
|
||||
if n := len(r.asksSent(t)); n != before+1 {
|
||||
t.Errorf("not asked again once the channels changed: %d", n-before)
|
||||
}
|
||||
}
|
||||
|
||||
// After review: at most three asks open at once, the most urgent first, then the oldest.
|
||||
func TestAtMostThreeAsksAreOpenTheMostUrgentFirst(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
var open []conditions.Condition
|
||||
for i := 0; i < 4; i++ {
|
||||
c := unitsCondition()
|
||||
c.Key = "machine.m" + string(rune('a'+i)) + ".units"
|
||||
c.Actions = []conditions.Action{conditions.SilenceAction(c.Key)}
|
||||
c.Raised = r.now.Add(-time.Duration(10-i) * time.Hour)
|
||||
open = append(open, c)
|
||||
}
|
||||
urgent := heldCondition()
|
||||
urgent.Severity, urgent.Raised = conditions.Urgent, r.now.Add(-time.Minute)
|
||||
r.open = append(open, urgent)
|
||||
_ = r.a.reconcile(context.Background())
|
||||
sent := r.asksSent(t)
|
||||
if len(sent) != askMostOpen || sent[0].About != urgent.Key || sent[1].About != "machine.ma.units" || sent[2].About != "machine.mb.units" {
|
||||
var about []string
|
||||
for _, q := range sent {
|
||||
about = append(about, q.About)
|
||||
}
|
||||
t.Fatalf("asked %v", about)
|
||||
}
|
||||
}
|
||||
|
||||
// After review: nothing is asked while no router takes asks under the controller's name, and that is said once.
|
||||
func TestNothingIsAskedWithoutARouter(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
var said []string
|
||||
r.a.logf = func(f string, a ...any) { said = append(said, f) }
|
||||
r.a.routerHere = func(context.Context) (bool, error) { return false, nil }
|
||||
r.open = []conditions.Condition{heldCondition()}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
_ = r.a.reconcile(context.Background())
|
||||
if len(r.asksSent(t)) != 0 {
|
||||
t.Error("asked with no router")
|
||||
}
|
||||
n := 0
|
||||
for _, s := range said {
|
||||
if strings.Contains(s, "no router takes asks") {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 1 {
|
||||
t.Errorf("said %d times", n)
|
||||
}
|
||||
}
|
||||
|
||||
// After review: the condition's words keep where an answer is given without a channel; the ask's text does not.
|
||||
func TestTheAskDropsWhereItIsAnsweredAndTheConditionKeepsIt(t *testing.T) {
|
||||
c := heldCondition()
|
||||
if !strings.Contains(c.Explanation, FromMeshMCPServer) {
|
||||
t.Fatalf("the condition lost where it is answered: %q", c.Explanation)
|
||||
}
|
||||
q, _ := askOf("x", c, partsOf(c)[0], time.Now())
|
||||
if strings.Contains(q.Explanation, "mesh MCP server") || !strings.HasPrefix(q.Explanation, "Needs you: release it, or stop it.") {
|
||||
t.Errorf("the ask says %q", q.Explanation)
|
||||
}
|
||||
if askApproveFor >= 24*time.Hour {
|
||||
t.Errorf("an approving ask lasts %s, which the SDK may refuse at its bound", askApproveFor)
|
||||
}
|
||||
}
|
||||
|
||||
// After review (security finding 9): a warrant is acted on only for an ask open in the controller's own record,
|
||||
// once the claim stands, and never when given after the ask expired.
|
||||
func TestAWarrantIsActedOnlyForAnOpenAskItClaimsBeforeItExpired(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
r.open = []conditions.Condition{heldCondition()}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
w := r.warrantFor(t, heldCondition().Key, "Release")
|
||||
late := w
|
||||
late.At = r.store[w.Ask].Ask.Expires.Add(time.Minute)
|
||||
body, _ := json.Marshal(late)
|
||||
_ = r.a.Decided(context.Background(), body)
|
||||
if len(r.called) != 0 {
|
||||
t.Fatalf("acted on a warrant given after the ask expired: %v", r.called)
|
||||
}
|
||||
// Claimed already by another delivery: nothing done here.
|
||||
kept := r.store[w.Ask]
|
||||
kept.Acted = "acting"
|
||||
r.store[w.Ask] = kept
|
||||
body, _ = json.Marshal(w)
|
||||
_ = r.a.Decided(context.Background(), body)
|
||||
if len(r.called) != 0 {
|
||||
t.Fatalf("acted though the claim was another's: %v", r.called)
|
||||
}
|
||||
// Cancelled in its own record: refused.
|
||||
kept.Acted, kept.State = "", askCancelled
|
||||
r.store[w.Ask] = kept
|
||||
_ = r.a.Decided(context.Background(), body)
|
||||
if len(r.called) != 0 {
|
||||
t.Errorf("acted on a cancelled ask: %v", r.called)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0259 §6: a warrant authorises the act its option bound when the controller asked, and no
|
||||
// other. A record of the act changed after the ask — another delivery, another machine, another argument —
|
||||
// is refused and nothing is performed.
|
||||
func TestAWarrantPerformsOnlyTheActItsOptionBound(t *testing.T) {
|
||||
for name, change := range map[string]func(*conditions.Action){
|
||||
"another argument": func(a *conditions.Action) {
|
||||
a.Arguments = map[string]string{"id": "novox/mesh-controller@000000000000"}
|
||||
},
|
||||
"another verb": func(a *conditions.Action) { a.Verb = "mesh-delivery.stop" },
|
||||
"another machine": func(a *conditions.Action) { a.Machine = "anchor" },
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
r.open = []conditions.Condition{heldCondition()}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
w := r.warrantFor(t, heldCondition().Key, "Release")
|
||||
kept := r.store[w.Ask]
|
||||
acts := append([]conditions.Action(nil), kept.Actions...)
|
||||
i := kept.Options[w.Option]
|
||||
change(&acts[i])
|
||||
kept.Actions = acts
|
||||
r.store[w.Ask] = kept
|
||||
answerWith(t, r, w)
|
||||
if len(r.called)+len(r.acts) != 0 {
|
||||
t.Errorf("performed an act the option did not bind: %v %v", r.called, r.acts)
|
||||
}
|
||||
})
|
||||
}
|
||||
// Every option of an ask binds its act.
|
||||
q, _ := askOf("x", heldCondition(), partsOf(heldCondition())[0], time.Now())
|
||||
for _, o := range q.Options {
|
||||
if o.Binds == "" {
|
||||
t.Errorf("the option %s binds nothing", o.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Failure is loud (novox/hq ADR 0259, the self-review of 2026-10-09): a condition that needs the operator and
|
||||
// could not be asked on any channel — no router, or the router refused the ask — is a condition of its own,
|
||||
// cleared once it can be asked again.
|
||||
func TestAnAskThatCannotBeDeliveredIsSaid(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
var raised [][]conditions.Observation
|
||||
r.a.raise = func(_ context.Context, obs []conditions.Observation) error {
|
||||
raised = append(raised, obs)
|
||||
return nil
|
||||
}
|
||||
last := func() []conditions.Observation { return raised[len(raised)-1] }
|
||||
routerHere := false
|
||||
r.a.routerHere = func(context.Context) (bool, error) { return routerHere, nil }
|
||||
channels := "channel/telegram=telegram@anchor[choice]own:true"
|
||||
r.a.channels = func(context.Context) string { return channels }
|
||||
r.open = []conditions.Condition{heldCondition()}
|
||||
|
||||
_ = r.a.reconcile(context.Background())
|
||||
if got := last(); len(got) != 1 || got[0].Kind != "asks-undelivered" ||
|
||||
!strings.Contains(got[0].Summary, heldCondition().Key) || !strings.Contains(got[0].Summary, "no router") {
|
||||
t.Fatalf("no router, said as %+v", got)
|
||||
}
|
||||
|
||||
routerHere = true
|
||||
_ = r.a.reconcile(context.Background())
|
||||
if got := last(); len(got) != 0 {
|
||||
t.Fatalf("asked, and still said undelivered: %+v", got)
|
||||
}
|
||||
first := r.asksSent(t)[0]
|
||||
refusal, _ := json.Marshal(asks.Warrant{Ask: first.ID, Asker: "mesh-controller", Outcome: asks.OutcomeRefused,
|
||||
Words: "no channel can carry any of its answers now", At: r.now})
|
||||
if err := r.a.Decided(context.Background(), refusal); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
if got := last(); len(got) != 1 || !strings.Contains(got[0].Summary, "no channel can carry") {
|
||||
t.Fatalf("the router's refusal, said as %+v", got)
|
||||
}
|
||||
if why, ok := conditions.PlainWords(conditions.Words{Headline: last()[0].Headline, Explanation: last()[0].Explanation,
|
||||
Needs: last()[0].Needs, Resolved: last()[0].Resolved}, ""); !ok {
|
||||
t.Errorf("not plain: %s", why)
|
||||
}
|
||||
r.open = nil
|
||||
_ = r.a.reconcile(context.Background())
|
||||
if got := last(); len(got) != 0 {
|
||||
t.Errorf("nothing needs asking, and still said: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The review of 2026-10-09 (M1): an acknowledging answer never shares an ask with an authorising one. A
|
||||
// condition offering Restart and Silence is asked twice — Restart alone, about the condition, and Silence
|
||||
// alone, apart — so Silence chosen on a channel that only acknowledges leaves the Restart ask open.
|
||||
func TestAnAcknowledgementNeverSharesAnAskWithAnApproval(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
key := "module.shanks.plex.down"
|
||||
c := conditions.Condition{Key: key, Kind: "module-down", Severity: conditions.Urgent, Headline: "Plex down on shanks",
|
||||
Explanation: "Needs you: restart it, or silence this.", Needs: "restart it, or silence this.",
|
||||
Actions: []conditions.Action{
|
||||
{Label: "Restart", Verb: "node-service-manager.restart", Machine: "shanks", Level: conditions.LevelApprove,
|
||||
Arguments: map[string]string{"unit": "plex"}},
|
||||
conditions.SilenceAction(key)}}
|
||||
r.open = []conditions.Condition{c}
|
||||
if err := r.a.reconcile(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sent := r.asksSent(t)
|
||||
if len(sent) != 2 {
|
||||
t.Fatalf("asked %d time(s): %+v", len(sent), sent)
|
||||
}
|
||||
for _, q := range sent {
|
||||
if err := q.Check(r.now); err != nil {
|
||||
t.Errorf("%s: %v", q.About, err)
|
||||
}
|
||||
levels := map[asks.Level]bool{}
|
||||
for _, o := range q.Options {
|
||||
levels[o.Level] = true
|
||||
}
|
||||
if len(levels) != 1 {
|
||||
t.Errorf("the ask about %s mixes levels: %+v", q.About, q.Options)
|
||||
}
|
||||
}
|
||||
byAbout := map[string]asks.Ask{}
|
||||
for _, q := range sent {
|
||||
byAbout[q.About] = q
|
||||
}
|
||||
if q := byAbout[key]; len(q.Options) != 1 || q.Options[0].Label != "Restart" {
|
||||
t.Errorf("the condition's own ask: %+v", q)
|
||||
}
|
||||
if q := byAbout[key+".acknowledge"]; len(q.Options) != 1 || q.Options[0].Level != asks.Acknowledge {
|
||||
t.Errorf("the acknowledging ask: %+v", q)
|
||||
}
|
||||
// Silence chosen: performed, and the Restart ask stays open, never asked twice.
|
||||
answerWith(t, r, r.warrantFor(t, key, "Silence for a week"))
|
||||
if len(r.silenced) != 1 || len(r.called) != 0 {
|
||||
t.Fatalf("silenced %v called %v", r.silenced, r.called)
|
||||
}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
open := 0
|
||||
for _, a := range r.store {
|
||||
if a.State == askOpen && a.Condition == key {
|
||||
open++
|
||||
if a.Part != "" || a.Ask.Options[0].Label != "Restart" {
|
||||
t.Errorf("the open ask is %+v", a)
|
||||
}
|
||||
}
|
||||
}
|
||||
if open != 1 || len(r.asksSent(t)) != 2 {
|
||||
t.Errorf("after the silence: %d open, %d asked", open, len(r.asksSent(t)))
|
||||
}
|
||||
// And the approval still answers: Restart chosen on a channel that proves who answered is performed.
|
||||
answerWith(t, r, r.warrantFor(t, key, "Restart"))
|
||||
if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "node-service-manager.restart@shanks") {
|
||||
t.Errorf("the approval kept through a silence was not performed: %v", r.called)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq issue 353: the controller's grant to ask is composed from the router's assignment and reaches the
|
||||
// bus only when its machine is pushed. Between the two, the bus refuses every ask (measured 2026-10-09, 17:54 to
|
||||
// 17:56 local: seven refusals of mesh.seat.operator-channel.accept.ask.mesh-controller). So nothing is asked
|
||||
// while the bus's user list is behind, it is said once, the conditions that need the operator are raised as
|
||||
// undelivered with what to do, and the asks go out once the bus holds the grant.
|
||||
func TestNothingIsAskedWhileTheBusLacksTheControllersGrant(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
var said []string
|
||||
r.a.logf = func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }
|
||||
var raised [][]conditions.Observation
|
||||
r.a.raise = func(_ context.Context, obs []conditions.Observation) error {
|
||||
raised = append(raised, obs)
|
||||
return nil
|
||||
}
|
||||
held := false
|
||||
r.a.grantHeld = func(context.Context) (bool, string, error) {
|
||||
return held, "the bus's user list on anchor is behind what the mesh composes; `push anchor` carries it", nil
|
||||
}
|
||||
r.open = []conditions.Condition{heldCondition()}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
_ = r.a.reconcile(context.Background())
|
||||
if len(r.asksSent(t)) != 0 {
|
||||
t.Error("asked while the bus lacks the grant")
|
||||
}
|
||||
n := 0
|
||||
for _, s := range said {
|
||||
if strings.Contains(s, "does not hold the controller's grant") {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 1 {
|
||||
t.Errorf("said %d times: %q", n, said)
|
||||
}
|
||||
if len(raised) == 0 || len(raised[len(raised)-1]) != 1 ||
|
||||
!strings.Contains(raised[len(raised)-1][0].Summary, "`push anchor` carries it") ||
|
||||
raised[len(raised)-1][0].Kind != "asks-undelivered" {
|
||||
t.Fatalf("not said as a condition with what to do: %+v", raised)
|
||||
}
|
||||
held = true
|
||||
_ = r.a.reconcile(context.Background())
|
||||
if sent := r.asksSent(t); len(sent) != 1 || sent[0].About != heldCondition().Key {
|
||||
t.Errorf("not asked once the bus holds the grant: %+v", sent)
|
||||
}
|
||||
if last := raised[len(raised)-1]; len(last) != 0 {
|
||||
t.Errorf("the undelivered condition was not cleared: %+v", last)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq issue 369: the router refused while its channels had not yet said they could send; they could twenty
|
||||
// minutes later, and "Questions for you not delivered" repeated that refusal for eleven hours, escalating on it.
|
||||
// The ask is made again; while the router's word on it is awaited the condition stands unchanged (neither
|
||||
// cleared nor raised again); once the router took it, the condition clears; a new refusal is said in its own
|
||||
// words.
|
||||
func TestARefusalIsAskedAgainAndTheUndeliveredConditionClearsOnceTaken(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
var raised [][]conditions.Observation
|
||||
r.a.raise = func(_ context.Context, obs []conditions.Observation) error {
|
||||
raised = append(raised, obs)
|
||||
return nil
|
||||
}
|
||||
last := func() []conditions.Observation { return raised[len(raised)-1] }
|
||||
r.a.channels = func(context.Context) string { return "channel/telegram=telegram@anchor[choice]own:true" }
|
||||
r.open = []conditions.Condition{unitsCondition()}
|
||||
refuse := func(words string) {
|
||||
t.Helper()
|
||||
sent := r.asksSent(t)
|
||||
refusal, _ := json.Marshal(asks.Warrant{Ask: sent[len(sent)-1].ID, Asker: "mesh-controller",
|
||||
Outcome: asks.OutcomeRefused, Words: words, At: r.now})
|
||||
if err := r.a.Decided(context.Background(), refusal); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
refuse("no channel can carry any of its answers now: telegram: telegram has not said whether it can send")
|
||||
_ = r.a.reconcile(context.Background())
|
||||
if got := last(); len(got) != 1 || !strings.Contains(got[0].Summary, "telegram has not said") {
|
||||
t.Fatalf("the refusal, said as %+v", got)
|
||||
}
|
||||
// The wait ends: asked again; until the router's word on it is in, the condition stands as it was.
|
||||
r.now = r.now.Add(askEvery)
|
||||
_ = r.a.reconcile(context.Background())
|
||||
if n := len(r.asksSent(t)); n != 2 {
|
||||
t.Fatalf("not asked again: %d asks", n)
|
||||
}
|
||||
if got := last(); len(got) != 1 {
|
||||
t.Fatalf("cleared while the router's word on the new ask was awaited: the condition would flap")
|
||||
}
|
||||
r.now = r.now.Add(askVerdictWait / 2)
|
||||
_ = r.a.reconcile(context.Background())
|
||||
if got := last(); len(got) != 1 {
|
||||
t.Fatalf("cleared inside the verdict wait: the condition would flap")
|
||||
}
|
||||
// Refused again, now for a reason of today: said in those words.
|
||||
refuse("no channel can carry any of its answers now: telegram: no account is linked")
|
||||
_ = r.a.reconcile(context.Background())
|
||||
if got := last(); len(got) != 1 || !strings.Contains(got[0].Summary, "no account is linked") ||
|
||||
strings.Contains(got[0].Summary, "has not said") {
|
||||
t.Fatalf("an old refusal's words repeated: %+v", got)
|
||||
}
|
||||
// Asked again after the doubled wait, and taken: no refusal comes, and the condition clears.
|
||||
r.now = r.now.Add(2 * askEvery)
|
||||
_ = r.a.reconcile(context.Background())
|
||||
if n := len(r.asksSent(t)); n != 3 {
|
||||
t.Fatalf("not asked again after the doubled wait: %d asks", n)
|
||||
}
|
||||
r.now = r.now.Add(askVerdictWait)
|
||||
_ = r.a.reconcile(context.Background())
|
||||
if got := last(); len(got) != 0 {
|
||||
t.Fatalf("taken, and still said undelivered: %+v", got)
|
||||
}
|
||||
if n := len(r.asksSent(t)); n != 3 {
|
||||
t.Fatalf("an ask taken was asked again: %d asks", n)
|
||||
}
|
||||
// The operator answers it; the condition stays open (its fix takes a while): the old refusal is not said again.
|
||||
taken := r.asksSent(t)[2]
|
||||
r.store.Change(context.Background(), taken.ID, func(x *asked) bool {
|
||||
x.State, x.Ended = string(asks.OutcomeChosen), r.now
|
||||
return true
|
||||
})
|
||||
for i := 0; i < 5; i++ {
|
||||
r.now = r.now.Add(askEvery)
|
||||
_ = r.a.reconcile(context.Background())
|
||||
if got := last(); len(got) != 0 {
|
||||
t.Fatalf("an answered ask brought its old refusal back: %+v", got)
|
||||
}
|
||||
}
|
||||
// Its explanation opens with one verdict, not two.
|
||||
r.open = append(r.open, heldCondition())
|
||||
_ = r.a.reconcile(context.Background())
|
||||
refuse("no channel can carry any of its answers now")
|
||||
_ = r.a.reconcile(context.Background())
|
||||
if e := conditions.Verdict(last()[0].Needs, last()[0].Explanation); strings.Count(e, "Needs you") != 1 {
|
||||
t.Errorf("the explanation says its verdict twice: %q", e)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,339 @@
|
||||
package main
|
||||
|
||||
// The asker on the bus: its asks in the controller's bucket `asked`, its asks and cancels published on the
|
||||
// seat under the controller's name, the verbs a warrant chooses called with the controller's grant, and the
|
||||
// router's record of its asks read under its name (novox/hq ADR 0259).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// askerFrom is the serving controller's asker; nil in any other process.
|
||||
var askerFrom *asker
|
||||
|
||||
// askWithin is how long a verb a warrant chose is given to answer.
|
||||
const askWithin = time.Minute
|
||||
|
||||
type busAsked struct{ conn *nats.Conn }
|
||||
|
||||
func (b busAsked) kv(ctx context.Context) (jetstream.KeyValue, error) {
|
||||
js, err := jetstream.New(b.conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return js.KeyValue(ctx, broker.AskedBucket)
|
||||
}
|
||||
|
||||
func (b busAsked) Get(ctx context.Context, id string) (*asked, error) {
|
||||
kv, err := b.kv(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e, err := kv.Get(ctx, id)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var r asked
|
||||
return &r, json.Unmarshal(e.Value(), &r)
|
||||
}
|
||||
|
||||
// Create keeps a new ask under its id, and only where none is kept: never over another.
|
||||
func (b busAsked) Create(ctx context.Context, r asked) error {
|
||||
kv, err := b.kv(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = kv.Create(ctx, r.ID, body)
|
||||
return err
|
||||
}
|
||||
|
||||
// Change applies change to the ask kept under id by compare-and-set on its key's revision (the review of
|
||||
// 2026-10-09, L2): read, changed, and written only over the revision read; when another write came between,
|
||||
// read again and asked again, at most askChangeTries times. change says whether to write at all.
|
||||
func (b busAsked) Change(ctx context.Context, id string, change func(*asked) bool) (bool, error) {
|
||||
kv, err := b.kv(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for try := 0; try < askChangeTries; try++ {
|
||||
e, err := kv.Get(ctx, id)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
var r asked
|
||||
if err := json.Unmarshal(e.Value(), &r); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !change(&r) {
|
||||
return false, nil
|
||||
}
|
||||
body, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if _, err := kv.Update(ctx, id, body, e.Revision()); err != nil {
|
||||
var api *jetstream.APIError
|
||||
if errors.Is(err, jetstream.ErrKeyExists) || (errors.As(err, &api) && api.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence) {
|
||||
continue
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
return false, fmt.Errorf("the ask %s changed under every one of %d tries", id, askChangeTries)
|
||||
}
|
||||
|
||||
func (b busAsked) All(ctx context.Context) ([]asked, error) {
|
||||
kv, err := b.kv(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lister, err := kv.ListKeys(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = lister.Stop() }()
|
||||
var out []asked
|
||||
for k := range lister.Keys() {
|
||||
e, err := kv.Get(ctx, k)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var r asked
|
||||
if json.Unmarshal(e.Value(), &r) == nil {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// callAction performs an action's verb as the controller, through the grant that names it.
|
||||
func callAction(conn *nats.Conn) func(ctx context.Context, a conditions.Action, args map[string]string) error {
|
||||
return func(ctx context.Context, a conditions.Action, args map[string]string) error {
|
||||
seat, verb, ok := strings.Cut(a.Verb, ".")
|
||||
if !ok {
|
||||
return fmt.Errorf("%q names no seat and verb", a.Verb)
|
||||
}
|
||||
body := map[string]any{}
|
||||
for k, v := range args {
|
||||
body[k] = v
|
||||
}
|
||||
if seat == catalogue.DeliverySeat {
|
||||
_, err := askDeliveryOwner(ctx, conn, verb, body)
|
||||
return err
|
||||
}
|
||||
granted := false
|
||||
for _, v := range broker.VerbsTheControllerActsOnAWarrant {
|
||||
granted = granted || (v.Seat == seat && v.Verb == verb)
|
||||
}
|
||||
if !granted {
|
||||
return fmt.Errorf("%s: %w", a.Verb, errNotGranted)
|
||||
}
|
||||
var answer link.Answer
|
||||
var err error
|
||||
if a.Machine != "" {
|
||||
answer, err = link.AskSeatTool(ctx, conn, seat, verb, a.Machine, body, askWithin)
|
||||
} else {
|
||||
answer, err = link.AskMeshSeatTool(ctx, conn, seat, verb, body, askWithin)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return fmt.Errorf("%s refused: %s", a.Verb, answer.Error)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// routerRecordOf reads the router's record of one of the controller's asks, under its name, and answers
|
||||
// how it ended when it did: the bucket is the one the asks seat's declarer names as its records.
|
||||
func routerRecordOf(conn *nats.Conn, inv *inventory.Inventory) func(ctx context.Context, id string) (*asks.Warrant, error) {
|
||||
return func(ctx context.Context, id string) (*asks.Warrant, error) {
|
||||
bucket, err := asksRecords(ctx, inv)
|
||||
if err != nil || bucket == "" {
|
||||
return nil, err
|
||||
}
|
||||
reply, err := conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET.KV_"+bucket+".$KV."+bucket+"."+askerName+"."+id, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if reply.Header.Get("Status") != "" {
|
||||
return nil, nil // none, or not readable: the event says it
|
||||
}
|
||||
var rec struct {
|
||||
State string `json:"state"`
|
||||
Warrant *asks.Warrant `json:"warrant"`
|
||||
}
|
||||
if json.Unmarshal(reply.Data, &rec) != nil || rec.State == "open" || rec.Warrant == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return rec.Warrant, nil
|
||||
}
|
||||
}
|
||||
|
||||
// asksRecords is the bucket the asks seat's declarer keeps its record of asks in.
|
||||
func asksRecords(ctx context.Context, inv *inventory.Inventory) (string, error) {
|
||||
declared, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, m := range declared {
|
||||
for _, s := range m.DefinesSeats {
|
||||
if s.Name == broker.AsksSeat && len(s.Records) > 0 {
|
||||
return broker.BucketName(m.Module, s.Records[0]), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// routerHereIn says whether a module declaring the asks seat, with its ask named by its caller, is assigned:
|
||||
// without it nothing takes an ask, and asking would only fill a queue nobody reads.
|
||||
func routerHereIn(inv *inventory.Inventory) func(ctx context.Context) (bool, error) {
|
||||
return func(ctx context.Context) (bool, error) {
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
for _, s := range e.Manifest.DefinesSeats {
|
||||
if s.Name == broker.AsksSeat && s.NamedByCaller("ask") && len(e.On) > 0 {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
// grantHeldIn says whether the bus holds the controller's grant to ask (novox/hq issue 353): the user list the
|
||||
// machine holding the bus was last sent is the one the mesh composes now (brokerBehind, the same judgement a
|
||||
// push makes to send that machine first). While it is behind, the controller's ask is refused by the bus,
|
||||
// whatever the record says of the router, so nothing is asked and the operator is told to push that machine.
|
||||
// Judged at most every grantLookEvery: composing the list resolves the bus's machine whole.
|
||||
func grantHeldIn(open *stores) func(ctx context.Context) (bool, string, error) {
|
||||
var mu sync.Mutex
|
||||
var at time.Time
|
||||
var held bool
|
||||
var why string
|
||||
return func(ctx context.Context) (bool, string, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if !at.IsZero() && time.Since(at) < grantLookEvery {
|
||||
return held, why, nil
|
||||
}
|
||||
machine, behind, err := brokerBehind(ctx, open, nil)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
at, held, why = time.Now(), !behind, ""
|
||||
if behind {
|
||||
why = fmt.Sprintf("the bus's user list on %s is behind what the mesh composes, so the bus has not been "+
|
||||
"given the controller's grant to ask; `push %s` carries it", machine, machine)
|
||||
}
|
||||
return held, why, nil
|
||||
}
|
||||
}
|
||||
|
||||
// grantLookEvery is how often the bus's user list is judged against the one its machine was last sent.
|
||||
const grantLookEvery = 30 * time.Second
|
||||
|
||||
// channelsIn is what the channels are now, as a fingerprint: each module claiming a kind of the channel
|
||||
// bench, where, promising what, and whether of its own account. An ask the router refused is asked again
|
||||
// once this changes.
|
||||
func channelsIn(inv *inventory.Inventory) func(ctx context.Context) string {
|
||||
return func(ctx context.Context) string {
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
var parts []string
|
||||
for _, e := range entries {
|
||||
for _, c := range e.Manifest.Claims {
|
||||
if c.Kind == "" || !catalogue.KindedBenches[c.Name] {
|
||||
continue
|
||||
}
|
||||
on := append([]string(nil), e.On...)
|
||||
sort.Strings(on)
|
||||
caps := append([]string(nil), c.Capabilities...)
|
||||
sort.Strings(caps)
|
||||
parts = append(parts, fmt.Sprintf("%s/%s=%s@%s[%s]own:%t", c.Name, c.Kind, e.Manifest.Module,
|
||||
strings.Join(on, ","), strings.Join(caps, ","), e.Manifest.RunsAs != ""))
|
||||
}
|
||||
}
|
||||
sort.Strings(parts)
|
||||
return strings.Join(parts, ";")
|
||||
}
|
||||
}
|
||||
|
||||
// startAsking makes the serving controller's asker and hands it the router's words.
|
||||
func startAsking(ctx context.Context, open *stores, server *link.Server, conn *nats.Conn, keeper *conditions.Keeper) {
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
fmt.Printf("the operator cannot be asked: %v\n", err)
|
||||
return
|
||||
}
|
||||
a := &asker{
|
||||
open: keeper.Open,
|
||||
silence: func(ctx context.Context, key string, d time.Duration, by, why string) error {
|
||||
_, err := keeper.Silence(ctx, key, d, by, why)
|
||||
return err
|
||||
},
|
||||
store: busAsked{conn: conn},
|
||||
publish: func(ctx context.Context, subject string, body []byte, id string) error {
|
||||
_, err := js.Publish(ctx, subject, body, jetstream.WithMsgID(id))
|
||||
return err
|
||||
},
|
||||
call: callAction(conn),
|
||||
// A proposed settings layer is set by this controller itself on the warrant (novox/hq ADR 0277).
|
||||
setLayer: setLayerIn(open),
|
||||
record: func(ctx context.Context, act link.HandAct) error {
|
||||
_, err := link.RecordHandAct(ctx, conn, act)
|
||||
return err
|
||||
},
|
||||
routerRecord: routerRecordOf(conn, open.inventory),
|
||||
routerHere: routerHereIn(open.inventory),
|
||||
grantHeld: grantHeldIn(open),
|
||||
channels: channelsIn(open.inventory),
|
||||
raise: func(ctx context.Context, obs []conditions.Observation) error {
|
||||
return keeper.Reconcile(ctx, sourceAsker, obs)
|
||||
},
|
||||
now: time.Now,
|
||||
logf: func(format string, args ...any) { fmt.Printf(format+"\n", args...) },
|
||||
}
|
||||
if err := server.Decides(a); err != nil {
|
||||
fmt.Printf("the operator's answers cannot be heard, so nothing is asked: %v\n", err)
|
||||
return
|
||||
}
|
||||
askerFrom = a
|
||||
go a.keep(ctx)
|
||||
}
|
||||
@@ -0,0 +1,281 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// Between `assign` and `push` a module's own secrets are not made yet: the push makes them. D1 composed
|
||||
// the machine's declaration without making anything, failed on the missing secret, and raised an urgent
|
||||
// "nothing can be sent to <machine>" — seen live on 2026-10-06, a desktop notification for a machine
|
||||
// the next push sent to without a word (novox/hq issue 275). D1 now composes as the push would, with a
|
||||
// stand-in for what the push makes: pending, not broken, and said only past a bound, as a warning.
|
||||
|
||||
// aKeeper is a module with an own secret the mesh makes — a backup repository's password.
|
||||
func aKeeper() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "keeper", Version: "1",
|
||||
OwnSecrets: catalogue.OwnSecrets{"repository": {Path: "/var/lib/mesh/keeper/repository"}},
|
||||
Resources: []map[string]any{
|
||||
{"id": "state", "type": "directory", "path": "/var/lib/mesh/keeper", "mode": "0700"},
|
||||
}}
|
||||
}
|
||||
|
||||
// pushedBy records a send to a machine as a push does — composed on the send path, so its own secrets
|
||||
// are made — without a bus to carry it.
|
||||
func pushedBy(t *testing.T, open *stores, node string) string {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
plan, settings, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationFor(ctx, open, node, plan, settings)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, err := open.inventory.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := digestOf(body)
|
||||
if err := open.inventory.RecordSent(ctx, record.ID, digest, declared.Builds); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return digest
|
||||
}
|
||||
|
||||
// pushedAndApplied is pushedBy, and the machine reporting it applied that declaration.
|
||||
func pushedAndApplied(t *testing.T, open *stores, node string) {
|
||||
t.Helper()
|
||||
digest := pushedBy(t, open, node)
|
||||
record, err := open.inventory.NodeByName(t.Context(), node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.RecordDoing(t.Context(), record.ID, inventory.Doing{
|
||||
Outcome: inventory.OutcomeApplied, Declared: digest}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// d1 runs D1 once.
|
||||
func d1(t *testing.T, open *stores) []conditions.Observation {
|
||||
t.Helper()
|
||||
got, err := probeDeclarations(t.Context(), &doctor{open: open})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
// **THE WINDOW, REPRODUCED**: assigned and not pushed, a module whose own secret the push makes is
|
||||
// waiting, not uncomposable; past the bound it is a warning naming what the push makes; pushed, nothing.
|
||||
func TestAModuleAssignedAndNotPushedIsAwaitingAPushNotUncomposable(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop") // the machine was pushed before; then the module is assigned
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The read the rest of the mesh asks still refuses it, with the composer's typed error: nothing
|
||||
// can be compared about a secret that does not exist (status), and nothing here string-matches.
|
||||
plan, settings, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = declarationWith(ctx, open, "laptop", plan, settings, gens, Reading)
|
||||
var notMade *catalogue.NotMadeError
|
||||
if !errors.As(err, ¬Made) || notMade.Module != "keeper" || notMade.Name != "repository" {
|
||||
t.Fatalf("a read composition did not say which secret is not made, typed: %v", err)
|
||||
}
|
||||
// Foreseen, it composes, names what the push makes, and made nothing.
|
||||
foreseen, err := declarationWith(ctx, open, "laptop", plan, settings, gens, Foreseeing)
|
||||
if err != nil || len(foreseen.foreseen) != 1 || foreseen.foreseen[0] != "keeper/repository" {
|
||||
t.Fatalf("foreseen: %v %v", foreseen.foreseen, err)
|
||||
}
|
||||
if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "keeper", "repository"); err != nil || held {
|
||||
t.Fatalf("asking ahead of the push made the secret (held %v, %v)", held, err)
|
||||
}
|
||||
|
||||
// Within the bound: nothing at all — no urgent, no warning, nobody notified.
|
||||
if got := d1(t, open); len(got) != 0 {
|
||||
t.Fatalf("a machine waiting for a push raised %+v", got)
|
||||
}
|
||||
|
||||
// Past the bound: a warning, not urgent, saying what the push will make.
|
||||
before := awaitingPushBound
|
||||
awaitingPushBound = -time.Minute
|
||||
t.Cleanup(func() { awaitingPushBound = before })
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.awaiting-push" || got[0].Severity != conditions.Warning ||
|
||||
!strings.Contains(got[0].Summary, "keeper/repository") || !strings.Contains(got[0].Summary, "push laptop") {
|
||||
t.Fatalf("a machine left un-pushed past the bound: %+v", got)
|
||||
}
|
||||
|
||||
// Pushed: the secret is made and D1 says nothing.
|
||||
pushedBy(t, open, "laptop")
|
||||
if got := d1(t, open); len(got) != 0 {
|
||||
t.Fatalf("a pushed machine still raised %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **A REAL FAILURE IS STILL URGENT**: a secret the push would be refused on is not one it will make.
|
||||
// A bus credential nobody issued (issue 203) fails the push, so D1 says it — urgent, in the push's words.
|
||||
func TestASecretThePushCannotMakeIsStillUncomposable(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aTalker())
|
||||
if _, err := assign(ctx, open, "laptop", "talker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || got[0].Severity != conditions.Urgent ||
|
||||
!strings.Contains(got[0].Summary, "module issue talker --node laptop") {
|
||||
t.Fatalf("a push that will be refused was not said urgently: %+v", got)
|
||||
}
|
||||
|
||||
// And a machine whose composition fails for anything else, with a secret waiting beside it, is
|
||||
// uncomposable for that — the stand-in hides nothing.
|
||||
register(t, open, aKeeper())
|
||||
if _, err := assign(ctx, open, "anchor", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
one, two := rivals()
|
||||
register(t, open, one)
|
||||
register(t, open, two)
|
||||
_, _ = assign(ctx, open, "anchor", "rival-one")
|
||||
_, _ = assign(ctx, open, "anchor", "rival-two")
|
||||
keys := map[string]conditions.Severity{}
|
||||
for _, o := range d1(t, open) {
|
||||
keys[o.Key()] = o.Severity
|
||||
}
|
||||
if keys["machine.anchor.uncomposable"] != conditions.Urgent {
|
||||
t.Fatalf("a real failure beside a waiting secret: %v", keys)
|
||||
}
|
||||
}
|
||||
|
||||
// A given secret sealed to a key the machine no longer has is not the mesh's to make again: the push is
|
||||
// refused on it, so D1 is too.
|
||||
func TestAGivenSecretUnderAnOldKeyIsNotForeseen(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "keeper", "repository", "given"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, err := open.inventory.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "issue it again") {
|
||||
t.Fatalf("a given secret under an old key: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The bound is read from when the machine began waiting: the oldest assignment since its last send.
|
||||
func TestAMachineAwaitsAPushSinceItsOldestAssignmentSinceTheLastSend(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop")
|
||||
sent := time.Now()
|
||||
since, err := open.inventory.AwaitingSince(ctx, "laptop")
|
||||
if err != nil || since.After(sent) {
|
||||
t.Fatalf("nothing assigned since the send: waiting since %v (%v), the send was before %v", since, err, sent)
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
since, err = open.inventory.AwaitingSince(ctx, "laptop")
|
||||
if err != nil || since.Before(sent.Add(-time.Second)) {
|
||||
t.Fatalf("assigned after the send: waiting since %v (%v), not from the assignment", since, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **D3 AND D13 WAIT FOR THE SEND**: a holder is expected to answer on a machine once the machine was sent
|
||||
// it and had time to report — never between assign and push.
|
||||
func TestAHolderIsExpectedOnlyOnceSentAndReported(t *testing.T) {
|
||||
now := time.Now()
|
||||
sent := now.Add(-time.Minute)
|
||||
long := now.Add(-time.Hour)
|
||||
cases := []struct {
|
||||
name string
|
||||
send lastSend
|
||||
want bool
|
||||
}{
|
||||
{"never sent", lastSend{}, false},
|
||||
{"sent without it", lastSend{sent: &long, current: true, carried: map[string]string{"other": "c"}}, false},
|
||||
{"sent with it, not reported yet", lastSend{sent: &sent, carried: map[string]string{"keeper": "c"}}, false},
|
||||
{"sent with it and reported", lastSend{sent: &sent, current: true, carried: map[string]string{"keeper": "c"}}, true},
|
||||
{"sent with it long ago, never reported", lastSend{sent: &long, carried: map[string]string{"keeper": "c"}}, true},
|
||||
{"sent before builds were kept", lastSend{sent: &long, current: true}, true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := c.send.settled("keeper", now); got != c.want {
|
||||
t.Errorf("%s: settled %v, want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// And through the stores: assigned, not in the last send; pushed and reported, carried and settled.
|
||||
func TestALastSendIsReadFromTheSendAndTheReport(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop")
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sends, err := readDeliveries(ctx, open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatal("a holder assigned and not pushed is expected to answer")
|
||||
}
|
||||
if !sends["laptop"].settled(overlay.Name, time.Now().Add(time.Hour)) {
|
||||
t.Fatal("a module the machine was sent long ago is not expected to answer")
|
||||
}
|
||||
pushedBy(t, open, "laptop")
|
||||
sends, err = readDeliveries(ctx, open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatal("pushed a moment ago and not reported, a holder is already expected")
|
||||
}
|
||||
if !sends["laptop"].settled("keeper", time.Now().Add(reportGrace+time.Minute)) {
|
||||
t.Fatal("pushed past the grace, a holder is not expected")
|
||||
}
|
||||
if _, ok := sends["laptop"].carried["keeper"]; !ok {
|
||||
t.Fatalf("the send's builds do not carry keeper: %v", sends["laptop"].carried)
|
||||
}
|
||||
pushedAndApplied(t, open, "laptop")
|
||||
if sends, err = readDeliveries(ctx, open.inventory); err != nil || !sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatalf("pushed and reported applied, a holder is not expected to answer (%v)", err)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,902 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The suite's tests of a merge, but for the merge window's own (this file's), plan the merge as it is heard: a
|
||||
// window of nothing closes at once, so a merge with no walk open is cut into its walk at once, as every merge
|
||||
// was planned before novox/hq ADR 0276. The window's tests set it through the controller's settings.
|
||||
func init() { mergeWindowDefault = 0 }
|
||||
|
||||
// windowed is a mesh whose controller's settings give a merge window of 90 seconds and at most 10 minutes, as its
|
||||
// settings file says them, and
|
||||
// modules built from the catalogue (app, notes) and from three repositories of their own.
|
||||
func windowed(t *testing.T) *stores {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
settings := t.TempDir() + "/merge-window.json"
|
||||
if err := os.WriteFile(settings, []byte(`{"merge-window": "90s", "merge-window-at-most": "10m"}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv(mergeWindowFileVar, settings)
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"},
|
||||
inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Ref: "main", BuiltFrom: "c0",
|
||||
Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, m := range []struct{ module, repository, path string }{
|
||||
{"app", "novox/mesh-catalog", "modules/app"},
|
||||
{"notes", "novox/mesh-catalog", "modules/notes"},
|
||||
{"one", "novox/one", ""},
|
||||
{"two", "novox/two", ""},
|
||||
{"three", "novox/three", ""},
|
||||
} {
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1"},
|
||||
inventory.Source{Repository: m.repository, Seat: "git", Path: m.path, Ref: "main", BuiltFrom: "c0",
|
||||
Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return open
|
||||
}
|
||||
|
||||
// t0 is the moment a test's clock starts: merges are dated by it, and heard after it.
|
||||
var t0 = time.Now().UTC().Truncate(time.Second)
|
||||
|
||||
// catalogueMerge is a merge of the catalogue changing one module's directory, made at a moment.
|
||||
func catalogueMerge(commit, module string, made time.Time) link.SourceMoved {
|
||||
return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit,
|
||||
MergedAt: made.Format(time.RFC3339Nano), Paths: []string{"modules/" + module + "/module.json"},
|
||||
ModuleDirs: []string{"modules/" + module}, ModuleDirsSaid: true}
|
||||
}
|
||||
|
||||
// repoMerge is a merge of a repository of one module's own.
|
||||
func repoMerge(repo, commit string, made time.Time) link.SourceMoved {
|
||||
return link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Commit: commit,
|
||||
MergedAt: made.Format(time.RFC3339Nano), Paths: []string{"main.go"}}
|
||||
}
|
||||
|
||||
// hear is a merge heard at a moment.
|
||||
func hear(t *testing.T, open *stores, m link.SourceMoved, now time.Time) {
|
||||
t.Helper()
|
||||
if err := (following{open: open}).hearMerge(t.Context(), m, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// cutAt is the cutter looking at a moment.
|
||||
func cutAt(t *testing.T, open *stores, now time.Time) {
|
||||
t.Helper()
|
||||
if err := cutBatchesHeld(t.Context(), open, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// walks is every plan record that is a walk, newest first, and the batches not yet cut.
|
||||
func walks(t *testing.T, open *stores) (walks, batches []inventory.Plan) {
|
||||
t.Helper()
|
||||
recent, err := open.inventory.RecentPlans(t.Context(), 50)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, p := range recent {
|
||||
if p.Batch() {
|
||||
batches = append(batches, p)
|
||||
} else {
|
||||
walks = append(walks, p)
|
||||
}
|
||||
}
|
||||
return walks, batches
|
||||
}
|
||||
|
||||
// The replay of issue 362: two catalogue merges 18 seconds apart are one batch, cut once into one walk at the
|
||||
// later commit, which names both merges — the earlier carried by the later.
|
||||
func TestTwoMergesSecondsApartAreOneWalkAtTheLaterCommit(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asked := asksWithPaths(t)
|
||||
claude := catalogueMerge("553b7191claude", "app", t0)
|
||||
dunst := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second))
|
||||
hear(t, open, claude, t0.Add(time.Second))
|
||||
hear(t, open, dunst, t0.Add(18*time.Second))
|
||||
if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 1 {
|
||||
t.Fatalf("within the window: %d walk(s), %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
cutAt(t, open, t0.Add(18*time.Second+89*time.Second))
|
||||
if ws, _ := walks(t, open); len(ws) != 0 {
|
||||
t.Fatalf("cut before the window closed: %+v", ws)
|
||||
}
|
||||
cutAt(t, open, t0.Add(18*time.Second+90*time.Second))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 0 {
|
||||
t.Fatalf("after the window: %d walk(s), %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
w := ws[0]
|
||||
if w.Commit != dunst.Commit || len(w.Commits) != 1 || w.Commits[0].Commit != dunst.Commit {
|
||||
t.Fatalf("the walk is at %s %+v, not the later commit", w.Commit, w.Commits)
|
||||
}
|
||||
for _, m := range []string{"app", "notes"} {
|
||||
if _, in := w.Modules[m]; !in {
|
||||
t.Fatalf("the walk does not build %s, which one of its merges moved: %v", m, w.Modules)
|
||||
}
|
||||
}
|
||||
want := []inventory.PlanMerge{{Repository: "novox/mesh-catalog", Commit: claude.Commit, Carried: dunst.Commit},
|
||||
{Repository: "novox/mesh-catalog", Commit: dunst.Commit}}
|
||||
if w.Delivery == nil || !slices.EqualFunc(w.Delivery.Merges, want, sameMerge) {
|
||||
t.Fatalf("the walk answers %+v, want %+v", w.Delivery, want)
|
||||
}
|
||||
if len(*asked) != 2 || (*asked)[0][2] != "main" || (*asked)[1][2] != "main" {
|
||||
t.Fatalf("the walk did not ask its modules at the branch, which holds the commit it carries: %v", *asked)
|
||||
}
|
||||
// Heard again, as the bus may hand it over twice: never a second merge, nor a second walk.
|
||||
hear(t, open, claude, t0.Add(5*time.Minute))
|
||||
if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 {
|
||||
t.Fatalf("a merge heard twice made %d walk(s) and %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
}
|
||||
|
||||
// Merges of three repositories in one window are one walk, one commit for each.
|
||||
func TestThreeRepositoriesInOneWindowAreOneWalk(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksRecorded(t)
|
||||
for i, r := range []string{"one", "two", "three"} {
|
||||
hear(t, open, repoMerge(r, "c-"+r, t0.Add(time.Duration(i)*20*time.Second)),
|
||||
t0.Add(time.Duration(i)*20*time.Second+time.Second))
|
||||
}
|
||||
cutAt(t, open, t0.Add(41*time.Second+90*time.Second))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 0 {
|
||||
t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
w := ws[0]
|
||||
if len(w.Commits) != 3 || len(w.Delivery.Merges) != 3 {
|
||||
t.Fatalf("the walk carries %+v and answers %+v", w.Commits, w.Delivery.Merges)
|
||||
}
|
||||
for _, r := range []string{"one", "two", "three"} {
|
||||
if w.CommitOf("novox/"+r) != "c-"+r {
|
||||
t.Fatalf("the walk carries %s at %q", r, w.CommitOf("novox/"+r))
|
||||
}
|
||||
if _, in := w.Modules[r]; !in {
|
||||
t.Fatalf("the walk does not build %s: %v", r, w.Modules)
|
||||
}
|
||||
}
|
||||
// Each module's ask is recorded at its own repository's commit.
|
||||
for _, r := range []string{"one", "two", "three"} {
|
||||
q, found, err := open.inventory.BuildRequestByID(t.Context(), w.Modules[r].Build)
|
||||
if err != nil || !found || q.Commit != "c-"+r {
|
||||
t.Fatalf("%s's build was not recorded at its own commit: %+v %v %v", r, q, found, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Each merge restarts the window: a merge at second 80 keeps the batch open until second 170; the window
|
||||
// closes at most ten minutes after its first merge however busy.
|
||||
func TestAMergeRestartsTheWindowAndTheMaximumClosesIt(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
hear(t, open, repoMerge("two", "c2", t0.Add(80*time.Second)), t0.Add(80*time.Second))
|
||||
cutAt(t, open, t0.Add(100*time.Second))
|
||||
if ws, _ := walks(t, open); len(ws) != 0 {
|
||||
t.Fatal("the window closed 90 seconds after its first merge, not after its last")
|
||||
}
|
||||
cutAt(t, open, t0.Add(170*time.Second))
|
||||
if ws, _ := walks(t, open); len(ws) != 1 || len(ws[0].Commits) != 2 {
|
||||
t.Fatalf("the window did not close 90 seconds after its last merge: %+v", ws)
|
||||
}
|
||||
|
||||
busy := windowed(t)
|
||||
asksWithPaths(t)
|
||||
var last time.Time
|
||||
for i := 0; i < 12; i++ {
|
||||
last = t0.Add(time.Duration(i) * time.Minute)
|
||||
repo := []string{"one", "two", "three"}[i%3]
|
||||
hear(t, busy, repoMerge(repo, "c"+string(rune('a'+i)), last), last)
|
||||
if ws, _ := walks(t, busy); len(ws) > 0 {
|
||||
if i != 10 {
|
||||
t.Fatalf("a busy window closed at merge %d (minute %d), not at its maximum", i, i)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
ws, _ := walks(t, busy)
|
||||
if len(ws) != 1 {
|
||||
t.Fatalf("ten minutes of merges a minute apart were never cut: %d walk(s)", len(ws))
|
||||
}
|
||||
}
|
||||
|
||||
// `plans` lists the batch being assembled first, in the operator's words: how long is left, when at the latest,
|
||||
// what is grouped, and that the plan is not yet calculated. It says so on the bus as plan-moved, too.
|
||||
func TestTheAssemblingBatchIsShown(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
var said []inventory.Plan
|
||||
was := inventory.PlanSaved
|
||||
inventory.PlanSaved = func(p inventory.Plan) { said = append(said, p) }
|
||||
t.Cleanup(func() { inventory.PlanSaved = was })
|
||||
now := time.Now().UTC()
|
||||
claude := catalogueMerge("553b7191claude", "app", now.Add(-20*time.Second))
|
||||
claude.Number, claude.Title = 174, "claude-code: an agent proposes a section"
|
||||
dunst := catalogueMerge("48bda475dunst", "notes", now.Add(-2*time.Second))
|
||||
dunst.Number, dunst.Title = 175, "dunst: the font the operator chose"
|
||||
hear(t, open, claude, now.Add(-19*time.Second))
|
||||
hear(t, open, dunst, now.Add(-time.Second))
|
||||
hear(t, open, repoMerge("one", "a6bc0931one", now), now)
|
||||
out := captured(t, func() error { return plansCommand(t.Context(), nil) })
|
||||
lines := strings.Split(out, "\n")
|
||||
first := lines[0]
|
||||
for _, want := range []string{"assembling: ", " s left (at the latest ", "grouped: novox/mesh-catalog@48bda475 " +
|
||||
"(answers 553b7191), novox/one@a6bc0931; plan not yet calculated"} {
|
||||
if !strings.Contains(first, want) {
|
||||
t.Fatalf("plans' first line %q does not say %q", first, want)
|
||||
}
|
||||
}
|
||||
// Under it, one line per repository: the pull request, what it answers, what it moves.
|
||||
if len(lines) < 3 || strings.TrimSpace(lines[1]) != "mesh-catalog #175 dunst: the font the operator chose (answers "+
|
||||
"#174 claude-code: an agent proposes a section) · app, notes" ||
|
||||
strings.TrimSpace(lines[2]) != "one a6bc0931 · one" {
|
||||
t.Fatalf("the grouped list reads %q", lines[1:4])
|
||||
}
|
||||
if len(said) == 0 || said[len(said)-1].State != inventory.PlanAssembling || said[len(said)-1].Delivery.Batch == nil ||
|
||||
len(said[len(said)-1].Delivery.Merges) != 3 {
|
||||
t.Fatalf("the batch was not said as assembling with its merges: %+v", said)
|
||||
}
|
||||
}
|
||||
|
||||
// captured is what a command printed.
|
||||
func captured(t *testing.T, run func() error) string {
|
||||
t.Helper()
|
||||
r, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
was := os.Stdout
|
||||
os.Stdout = w
|
||||
runErr := run()
|
||||
os.Stdout = was
|
||||
_ = w.Close()
|
||||
var b bytes.Buffer
|
||||
_, _ = io.Copy(&b, r)
|
||||
if runErr != nil {
|
||||
t.Fatal(runErr)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// One walk at a time: a merge heard while a walk runs joins the next batch, which is cut when the walk ends;
|
||||
// the walk that started is never superseded.
|
||||
func TestAMergeDuringAWalkJoinsTheNextBatch(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(90*time.Second))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 || !ws[0].Open() {
|
||||
t.Fatalf("the first batch was not cut: %+v", ws)
|
||||
}
|
||||
first := ws[0]
|
||||
hear(t, open, repoMerge("two", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
|
||||
cutAt(t, open, t0.Add(5*time.Minute))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != first.ID {
|
||||
t.Fatalf("the merge during the walk: %d walk(s), batches %+v", len(ws), bs)
|
||||
}
|
||||
if got, _ := open.inventory.PlanByID(t.Context(), first.ID); !got.Open() {
|
||||
t.Fatalf("the started walk was %s by the next merge", got.State)
|
||||
}
|
||||
if !strings.HasPrefix(batchWords(bs[0], t0.Add(5*time.Minute)), "queued behind "+first.ID) {
|
||||
t.Fatalf("a queued batch reads %q", batchWords(bs[0], t0.Add(5*time.Minute)))
|
||||
}
|
||||
first.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(t.Context(), &first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(6*time.Minute))
|
||||
ws, bs = walks(t, open)
|
||||
if len(ws) != 2 || len(bs) != 0 || ws[0].Commit != "c2" {
|
||||
t.Fatalf("the queued batch was not cut when the walk ended: %+v %+v", ws, bs)
|
||||
}
|
||||
}
|
||||
|
||||
// A walk waiting for its delivery's word is folded into the next batch's walk, which answers its merges; it
|
||||
// names the walk that took it over.
|
||||
func TestAWaitingWalkIsFoldedIntoTheNextBatch(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(90*time.Second))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 || !ws[0].Waiting() {
|
||||
t.Fatalf("the walk does not wait for its word: %+v", ws)
|
||||
}
|
||||
waiting := ws[0]
|
||||
hear(t, open, repoMerge("two", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
|
||||
cutAt(t, open, t0.Add(2*time.Minute+90*time.Second))
|
||||
ws, bs := walks(t, open)
|
||||
if len(bs) != 0 || len(ws) != 2 {
|
||||
t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
folded, err := open.inventory.PlanByID(ctx, waiting.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
newer := ws[0]
|
||||
if folded.State != inventory.PlanSuperseded || folded.Delivery.TakenOverBy != newer.ID {
|
||||
t.Fatalf("the waiting walk is %s, taken over by %q", folded.State, folded.Delivery.TakenOverBy)
|
||||
}
|
||||
if len(newer.Delivery.Merges) != 2 || newer.CommitOf("novox/one") != "c1" || newer.CommitOf("novox/two") != "c2" {
|
||||
t.Fatalf("the newer walk answers %+v", newer.Delivery.Merges)
|
||||
}
|
||||
if _, in := newer.Modules["one"]; !in {
|
||||
t.Fatalf("what the folded walk was to build is not built: %v", newer.Modules)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge heard after a later merge of its branch was walked is answered by that walk when it builds all it
|
||||
// moves — named at once on a walk done — and joins the next batch when it does not.
|
||||
func TestALateMergeIsAnsweredByTheWalkOfTheLaterOne(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
later := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second))
|
||||
hear(t, open, later, t0.Add(18*time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
w := ws[0]
|
||||
w.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Moves only notes, which the done walk built from the branch after it: answered there, at once.
|
||||
hear(t, open, catalogueMerge("553b7191early", "notes", t0), t0.Add(15*time.Minute))
|
||||
got, _ := open.inventory.PlanByID(ctx, w.ID)
|
||||
if got.State != inventory.PlanDone || len(got.Delivery.Merges) != 2 ||
|
||||
!sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: "553b7191early",
|
||||
Carried: later.Commit}) {
|
||||
t.Fatalf("the late merge is not named by the walk that carried it: %+v", got.Delivery.Merges)
|
||||
}
|
||||
// Moves app, which that walk never built: the next batch walks it.
|
||||
hear(t, open, catalogueMerge("1111aaaaapp", "app", t0.Add(time.Second)), t0.Add(16*time.Minute))
|
||||
_, bs := walks(t, open)
|
||||
if len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "1111aaaaapp" {
|
||||
t.Fatalf("a late merge moving what the walk never built did not join the next batch: %+v", bs)
|
||||
}
|
||||
}
|
||||
|
||||
// A failed walk marks nothing delivered: its earlier merges are walked alone, on their own commit, the newest
|
||||
// first; the first delivered answers the older ones, and the search stops.
|
||||
func TestAFailedWalkWalksItsEarlierMergesAlone(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asked := asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
// Made before the cut marks their modules seen, as merges are: walked again, they are news all the same.
|
||||
oldest := catalogueMerge("aaaa0001", "app", t0.Add(-3*time.Minute))
|
||||
middle := catalogueMerge("bbbb0002", "app", t0.Add(-3*time.Minute+10*time.Second))
|
||||
newest := catalogueMerge("cccc0003", "notes", t0.Add(-3*time.Minute+20*time.Second))
|
||||
for i, m := range []link.SourceMoved{oldest, middle, newest} {
|
||||
hear(t, open, m, t0.Add(time.Duration(i)*10*time.Second+time.Second))
|
||||
}
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
failed := ws[0]
|
||||
failed.State, failed.Note = inventory.PlanFailed, "notes failed to build in tier 0"
|
||||
if err := open.inventory.SavePlan(ctx, &failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(3*time.Minute))
|
||||
got, _ := open.inventory.PlanByID(ctx, failed.ID)
|
||||
if got.State != inventory.PlanFailed || len(got.Delivery.Merges) != 3 {
|
||||
t.Fatalf("the failed walk's record changed: %s %+v", got.State, got.Delivery.Merges)
|
||||
}
|
||||
ws, _ = walks(t, open)
|
||||
alone := ws[0]
|
||||
if alone.ID == failed.ID || alone.Commit != middle.Commit || len(alone.Delivery.Merges) != 1 ||
|
||||
!sameMerge(alone.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: middle.Commit}) {
|
||||
t.Fatalf("the newest earlier merge was not walked alone on its own commit: %+v", alone)
|
||||
}
|
||||
if _, in := alone.Modules["app"]; !in || (*asked)[len(*asked)-1] != [3]string{"novox/mesh-catalog", "modules/app",
|
||||
middle.Commit} {
|
||||
t.Fatalf("the merge walked alone does not build app at its own commit: %v, asked %v", alone.Modules, *asked)
|
||||
}
|
||||
// Retried, the failed walk would name merges the search answers now.
|
||||
if _, err := retryPlan(ctx, open, failed.ID); err == nil || !strings.Contains(err.Error(), "walked alone") {
|
||||
t.Fatalf("a failed walk whose merges are searched was retried: %v", err)
|
||||
}
|
||||
alone.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &alone); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(4*time.Minute))
|
||||
ws, bs := walks(t, open)
|
||||
if ws[0].ID != alone.ID || len(bs) != 0 {
|
||||
t.Fatalf("the search went on after a merge was delivered: %+v", ws[0])
|
||||
}
|
||||
done, _ := open.inventory.PlanByID(ctx, alone.ID)
|
||||
if len(done.Delivery.Merges) != 2 || !sameMerge(done.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog",
|
||||
Commit: oldest.Commit, Carried: middle.Commit}) {
|
||||
t.Fatalf("the oldest merge is not answered by the walk that delivered the one after it: %+v", done.Delivery.Merges)
|
||||
}
|
||||
// A stopped walk starts no search: a person ended it.
|
||||
hear(t, open, catalogueMerge("dddd0004", "app", t0.Add(5*time.Minute)), t0.Add(5*time.Minute))
|
||||
hear(t, open, catalogueMerge("eeee0005", "notes", t0.Add(5*time.Minute+time.Second)), t0.Add(5*time.Minute+time.Second))
|
||||
cutAt(t, open, t0.Add(8*time.Minute))
|
||||
ws, _ = walks(t, open)
|
||||
if _, err := stopWalk(ctx, open.inventory, ws[0].ID, "mesh-delivery for jochen", "not now"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(9*time.Minute))
|
||||
if again, _ := walks(t, open); again[0].ID != ws[0].ID {
|
||||
t.Fatalf("a stopped walk's earlier merge was walked alone: %+v", again[0])
|
||||
}
|
||||
}
|
||||
|
||||
// A delivery group's order holds inside the walk (ADR 0249 unchanged): merges of two repositories from one head
|
||||
// branch name, the node-engine's before the controller's, are tiered so; without the group, one tier.
|
||||
func TestAGroupsOrderBecomesTiersInsideTheWalk(t *testing.T) {
|
||||
for _, grouped := range []bool{true, false} {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
for _, m := range []string{"mesh-host"} {
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"},
|
||||
inventory.Source{Repository: "novox/" + m, Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
host := repoMerge("mesh-host", "h1", t0)
|
||||
ctl := repoMerge("mesh-controller", "k1", t0.Add(time.Second))
|
||||
host.Head, ctl.Head = "feat/together", "feat/together"
|
||||
if !grouped {
|
||||
ctl.Head = "feat/alone"
|
||||
}
|
||||
hear(t, open, ctl, t0.Add(2*time.Second))
|
||||
hear(t, open, host, t0.Add(3*time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 {
|
||||
t.Fatalf("%d walks", len(ws))
|
||||
}
|
||||
tiers := ws[0].Tiers
|
||||
if grouped {
|
||||
if len(tiers) != 2 || !slices.Equal(tiers[0], []string{"mesh-host"}) ||
|
||||
!slices.Equal(tiers[1], []string{"mesh-controller"}) {
|
||||
t.Fatalf("the group's order is not the walk's tiers: %v", tiers)
|
||||
}
|
||||
} else if len(tiers) != 1 {
|
||||
t.Fatalf("two merges of no group were ordered: %v", tiers)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A restarted controller resumes the window where it stood: the batch, its merges and their times are in the
|
||||
// store, a merge handed over again is not doubled, and the batch is cut when its window closes.
|
||||
func TestABatchSurvivesARestart(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
hear(t, open, repoMerge("two", "c2", t0.Add(30*time.Second)), t0.Add(30*time.Second))
|
||||
|
||||
// A new controller: nothing of the first one's but the store.
|
||||
again, err := openStores(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(again.Close)
|
||||
hear(t, again, repoMerge("one", "c1", t0), t0.Add(60*time.Second)) // the bus hands it over again
|
||||
cutAt(t, again, t0.Add(119*time.Second))
|
||||
if ws, bs := walks(t, again); len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 {
|
||||
t.Fatalf("the restarted controller's batch: %d walk(s), %+v", len(ws), bs)
|
||||
}
|
||||
cutAt(t, again, t0.Add(120*time.Second))
|
||||
ws, bs := walks(t, again)
|
||||
if len(ws) != 1 || len(bs) != 0 || len(ws[0].Delivery.Merges) != 2 {
|
||||
t.Fatalf("the restarted controller did not cut the batch at its window: %+v %+v", ws, bs)
|
||||
}
|
||||
}
|
||||
|
||||
// The window's settings reach the controller in its settings file, read at every look; one that says no
|
||||
// duration, or no file, is the default.
|
||||
func TestTheWindowIsTheControllersSetting(t *testing.T) {
|
||||
file := t.TempDir() + "/merge-window.json"
|
||||
t.Setenv(mergeWindowFileVar, file)
|
||||
for _, c := range []struct {
|
||||
content string
|
||||
window, atMost time.Duration
|
||||
}{
|
||||
{`{"merge-window": "60s", "merge-window-at-most": "5m"}`, time.Minute, 5 * time.Minute},
|
||||
{`{"merge-window": 30, "merge-window-at-most": ""}`, 30 * time.Second, mergeWindowAtMostDefault},
|
||||
{`{"merge-window": "soon"}`, mergeWindowDefault, mergeWindowAtMostDefault},
|
||||
} {
|
||||
if err := os.WriteFile(file, []byte(c.content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if w, m := mergeWindowOf(); w != c.window || m != c.atMost {
|
||||
t.Errorf("%s reads as %s and %s, want %s and %s", c.content, w, m, c.window, c.atMost)
|
||||
}
|
||||
}
|
||||
t.Setenv(mergeWindowFileVar, file+".gone")
|
||||
if w, m := mergeWindowOf(); w != mergeWindowDefault || m != mergeWindowAtMostDefault {
|
||||
t.Errorf("no file reads as %s and %s", w, m)
|
||||
}
|
||||
}
|
||||
|
||||
// S16 names every merge a waiting walk answers, never one commit a supersession may have ended (issue 362).
|
||||
func TestAWaitingWalkNamesTheMergesItAnswers(t *testing.T) {
|
||||
f := calm(t0)
|
||||
f.waits = []waitFacts{{id: "plan-2", repository: "novox/mesh-catalog", commit: "48bda475dunst", awaits: "mesh-delivery",
|
||||
since: t0.Add(-31 * time.Minute), merges: []inventory.PlanMerge{
|
||||
{Repository: "novox/mesh-catalog", Commit: "553b7191claude", Carried: "48bda475dunst"},
|
||||
{Repository: "novox/mesh-catalog", Commit: "48bda475dunst"}}}}
|
||||
got := watchWaits(f)
|
||||
if len(got) != 1 || !strings.Contains(got[0].Summary, "novox/mesh-catalog@553b7191") ||
|
||||
!strings.Contains(got[0].Summary, "novox/mesh-catalog@48bda475") {
|
||||
t.Fatalf("the waiting walk does not name both merges: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge heard after a later merge of its branch was cut is named by that walk however its modules read since
|
||||
// the cut, in a repository of one module (review of this change); a walk that never built what it moves names
|
||||
// nothing, and the merge joins the next batch.
|
||||
func TestALateMergeOfAOneModuleRepositoryIsNamed(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
c2 := repoMerge("one", "c2", t0.Add(-50*time.Second))
|
||||
hear(t, open, c2, t0)
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
w := ws[0]
|
||||
w.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, repoMerge("one", "c1", t0.Add(-60*time.Second)), t0.Add(15*time.Minute))
|
||||
got, _ := open.inventory.PlanByID(ctx, w.ID)
|
||||
if len(got.Delivery.Merges) != 2 || !sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/one",
|
||||
Commit: "c1", Carried: "c2"}) {
|
||||
t.Fatalf("the late merge was not named by the walk that carried it: %+v", got.Delivery.Merges)
|
||||
}
|
||||
|
||||
// A walk of two that built only two: a late merge of one is not its to name.
|
||||
hear(t, open, repoMerge("two", "d2", t0.Add(16*time.Minute)), t0.Add(16*time.Minute))
|
||||
cutAt(t, open, t0.Add(18*time.Minute))
|
||||
ws, _ = walks(t, open)
|
||||
w = ws[0]
|
||||
delete(w.Modules, "two")
|
||||
w.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, repoMerge("two", "d1", t0.Add(15*time.Minute)), t0.Add(20*time.Minute))
|
||||
if got, _ := open.inventory.PlanByID(ctx, w.ID); len(got.Delivery.Merges) != 1 {
|
||||
t.Fatalf("a walk that never built what the late merge moves named it: %+v", got.Delivery.Merges)
|
||||
}
|
||||
if _, bs := walks(t, open); len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "d1" {
|
||||
t.Fatalf("the late merge did not join the next batch: %+v", bs)
|
||||
}
|
||||
}
|
||||
|
||||
// One walk at a time holds against the delivery's word too: a waiting walk let go beside a started one takes the
|
||||
// word and starts once that one ended, asking nothing before.
|
||||
func TestAWalkLetGoBesideAStartedOneStartsOnceItEnded(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asked := asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
started, _ := walks(t, open)
|
||||
waiting := inventory.Plan{ID: "plan-waiting", Repository: "novox/two", Branch: "main", Commit: "d1", Created: t0,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"two"}}, Modules: map[string]*inventory.PlanModule{"two": {}},
|
||||
Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}}
|
||||
if err := open.inventory.SavePlan(ctx, &waiting); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := letGo(ctx, open.inventory, waiting.ID, catalogue.DeliverySeat, "its turn"); err != nil {
|
||||
t.Fatalf("the word was refused beside %s: %v", started[0].ID, err)
|
||||
}
|
||||
before := len(*asked)
|
||||
advanceHeld(ctx, open)
|
||||
got, _ := open.inventory.PlanByID(ctx, waiting.ID)
|
||||
if len(*asked) != before || !strings.Contains(got.Note, "starts once "+started[0].ID) {
|
||||
t.Fatalf("a walk let go beside a started one asked (%d → %d) or does not say it waits: %q", before, len(*asked), got.Note)
|
||||
}
|
||||
// Read as a wait, an hour on: its note on the line, never LATE, and no tier of it late for S3.
|
||||
later := time.Now().Add(time.Hour)
|
||||
if line := planLine(got, later); !strings.Contains(line, "starts once "+started[0].ID) || strings.Contains(line, "LATE") {
|
||||
t.Fatalf("a deferred walk reads %q", line)
|
||||
}
|
||||
facts, _, err := gatherPlans(ctx, open.inventory, later, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range facts {
|
||||
if f.id == got.ID {
|
||||
t.Fatalf("a deferred walk is watched as a tier running late: %+v", f)
|
||||
}
|
||||
}
|
||||
done := started[0]
|
||||
done.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) != before+1 {
|
||||
t.Fatalf("the walk did not start once the started one ended: asked %v", *asked)
|
||||
}
|
||||
}
|
||||
|
||||
// A file a merge of the batch removed and a later one brought back is not removed; one removed last is.
|
||||
func TestARemovedFileIsTheLastMergesWord(t *testing.T) {
|
||||
ev := func(commit string, paths, removed []string) inventory.BatchedMerge {
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "one", Base: "main", Commit: commit, Paths: paths, Removed: removed}
|
||||
b, _ := json.Marshal(m)
|
||||
return inventory.BatchedMerge{Repository: "novox/one", Branch: "main", Commit: commit, Event: b,
|
||||
Merged: t0.Add(time.Duration(len(commit)) * time.Second)}
|
||||
}
|
||||
got := combinedMerges([]inventory.BatchedMerge{
|
||||
ev("a", []string{"x/module.json", "y/module.json"}, []string{"x/module.json", "y/module.json"}),
|
||||
ev("bb", []string{"x/module.json"}, nil),
|
||||
ev("ccc", []string{"z.go"}, nil)})
|
||||
if len(got) != 1 || got[0].Commit != "ccc" || !slices.Equal(got[0].Removed, []string{"y/module.json"}) ||
|
||||
len(got[0].Paths) != 3 {
|
||||
t.Fatalf("combined as %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The catch-up hands over what the bus lost after its branch's later merges were cut, and the record keeps it:
|
||||
// an earlier merge is named by the walk that carried it; a merge made before a cut and heard after it, which
|
||||
// no later merge carries, joins the next batch — neither reads as history and is dropped (review of this change).
|
||||
func TestTheCatchUpKeepsWhatACutMadeHistory(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
base := time.Now().UTC().Add(-time.Hour).Truncate(time.Second)
|
||||
hear(t, open, repoMerge("one", "c2", base.Add(10*time.Second)), base.Add(11*time.Second))
|
||||
cutAt(t, open, base.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
w := ws[0]
|
||||
w.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lost := []link.AnnouncedMerge{
|
||||
{SourceMoved: repoMerge("one", "c1", base), At: base.Add(time.Second)},
|
||||
{SourceMoved: repoMerge("one", "c3lost", base.Add(20*time.Second)), At: base.Add(21 * time.Second)},
|
||||
}
|
||||
catalogued := func(ctx context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
|
||||
entries, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
read, err := readForPlanning(ctx, open.inventory)
|
||||
return entries, read, err
|
||||
}
|
||||
if err := catchUpOnMerges(ctx, time.Now(), unheardMerges{announcedList(lost), open.inventory}, catalogued,
|
||||
(following{open}).SourceMoved, t.Logf); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ := open.inventory.PlanByID(ctx, w.ID)
|
||||
if len(got.Delivery.Merges) != 2 || !sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/one",
|
||||
Commit: "c1", Carried: "c2"}) {
|
||||
t.Fatalf("the earlier merge the catch-up handed over is not named by the walk that carried it: %+v",
|
||||
got.Delivery.Merges)
|
||||
}
|
||||
if _, kept, err := open.inventory.MergeOf(ctx, "novox/one", "c3lost"); err != nil || !kept {
|
||||
t.Fatalf("the merge made before the cut and heard after it was dropped: %v %v", kept, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge on the controller's own path never shares a batch with one that waits for mesh-delivery's word
|
||||
// (decided during the build, 2026-10-10): the two are batches of their own, cut one after the other; a walk on
|
||||
// the own path folds no waiting catalogue walk but batches its merges again behind it; the catalogue's walk
|
||||
// still waits for the word, so no catalogue delivery skips its turn.
|
||||
func TestAMergeOnTheControllersPathNeverSharesABatch(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asked := asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, catalogueMerge("aaaa0001", "app", t0), t0)
|
||||
hear(t, open, repoMerge("mesh-controller", "k1", t0.Add(10*time.Second)), t0.Add(10*time.Second))
|
||||
hear(t, open, catalogueMerge("bbbb0002", "notes", t0.Add(20*time.Second)), t0.Add(20*time.Second))
|
||||
_, bs := walks(t, open)
|
||||
var ownBatch, catalogueBatch inventory.Plan
|
||||
for _, b := range bs {
|
||||
if b.OwnPath() {
|
||||
ownBatch = b
|
||||
} else {
|
||||
catalogueBatch = b
|
||||
}
|
||||
}
|
||||
if len(bs) != 2 || ownBatch.ID == "" || catalogueBatch.ID == "" || len(catalogueBatch.Delivery.Merges) != 2 ||
|
||||
len(ownBatch.Delivery.Merges) != 1 {
|
||||
t.Fatalf("a controller merge and two catalogue merges in one window: %+v", bs)
|
||||
}
|
||||
if !strings.Contains(batchWords(ownBatch, t0.Add(30*time.Second)), "own path") {
|
||||
t.Fatalf("the own-path batch does not say so: %q", batchWords(ownBatch, t0.Add(30*time.Second)))
|
||||
}
|
||||
// Both windows closed, the controller's batch is cut first and starts; the catalogue batch queues behind it,
|
||||
// is cut when the controller's walk ended, and waits for its word with both merges.
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || ws[0].Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k1" {
|
||||
t.Fatalf("the controller's batch was not cut first into a started walk: %+v", ws)
|
||||
}
|
||||
ownWalk := ws[0]
|
||||
for _, m := range []string{"app", "notes"} {
|
||||
if _, in := ownWalk.Modules[m]; in {
|
||||
t.Fatalf("the controller's walk builds %s, a catalogue module: it skipped its turn", m)
|
||||
}
|
||||
}
|
||||
if len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != ownWalk.ID || bs[0].OwnPath() {
|
||||
t.Fatalf("the catalogue batch does not queue behind the controller's walk: %+v", bs)
|
||||
}
|
||||
ownWalk.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &ownWalk); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(3*time.Minute))
|
||||
ws, bs = walks(t, open)
|
||||
if len(bs) != 0 || ws[0].ID != catalogueBatch.ID || !ws[0].Waiting() || len(ws[0].Delivery.Merges) != 2 {
|
||||
t.Fatalf("the catalogue batch was not cut into a waiting walk once the controller's ended: %+v %+v", ws, bs)
|
||||
}
|
||||
for _, m := range []string{"app", "notes"} {
|
||||
if _, in := ws[0].Modules[m]; !in {
|
||||
t.Fatalf("the catalogue walk does not build %s: %v", m, ws[0].Modules)
|
||||
}
|
||||
}
|
||||
for _, a := range *asked {
|
||||
if a[1] == "modules/app" || a[1] == "modules/notes" {
|
||||
t.Fatalf("a catalogue module was asked without the word: %v", *asked)
|
||||
}
|
||||
}
|
||||
|
||||
// A catalogue walk waiting for its word when an own-path batch is cut keeps waiting beside the own-path
|
||||
// walk, is not folded into it, and its word is taken meanwhile: it starts once the own-path walk ended.
|
||||
catalogueWalk := ws[0]
|
||||
hear(t, open, repoMerge("mesh-controller", "k2", t0.Add(4*time.Minute)), t0.Add(4*time.Minute))
|
||||
cutAt(t, open, t0.Add(6*time.Minute))
|
||||
ws, _ = walks(t, open)
|
||||
kept, _ := open.inventory.PlanByID(ctx, catalogueWalk.ID)
|
||||
if !kept.Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k2" || ws[0].Waiting() {
|
||||
t.Fatalf("the waiting catalogue walk was not kept waiting beside the controller's walk: %s %+v", kept.State, ws)
|
||||
}
|
||||
if _, in := ws[0].Modules["app"]; in {
|
||||
t.Fatalf("the controller's walk folded the waiting catalogue walk in: %v", ws[0].Modules)
|
||||
}
|
||||
if _, err := letGo(ctx, open.inventory, catalogueWalk.ID, catalogue.DeliverySeat, "its turn"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := len(*asked)
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) != before {
|
||||
t.Fatalf("the catalogue walk started beside the controller's: asked %v", (*asked)[before:])
|
||||
}
|
||||
own2 := ws[0]
|
||||
own2.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &own2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) < before+1 || (*asked)[before][1] != "modules/app" {
|
||||
t.Fatalf("the catalogue walk did not start once the controller's ended: %v", (*asked)[before:])
|
||||
}
|
||||
}
|
||||
|
||||
// `plans` names a walk by what it moves (asked by the operator, 2026-10-10): the repository's pull request and
|
||||
// title, the modules it moves and the machines running them, then the state words; a record naming no pull
|
||||
// request is named by its commit, and a title is cut at fifty runes.
|
||||
func TestAPlanLineNamesWhatItMoves(t *testing.T) {
|
||||
now := time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)
|
||||
p := inventory.Plan{ID: "plan-1", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c1c1c1c1c1",
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"messenger", "telegram"}}, TierEntered: now.Add(-2 * time.Minute),
|
||||
Modules: map[string]*inventory.PlanModule{"messenger": {State: "asked"}, "telegram": {State: "asked"}},
|
||||
Commits: []inventory.PlanCommit{{Repository: "novox/mesh-catalog", Branch: "main", Commit: "c1c1c1c1c1"}},
|
||||
Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{{Repository: "novox/mesh-catalog",
|
||||
Commit: "c1c1c1c1c1", Number: 175, Title: "a tap shows its outcome", Moves: []string{"telegram", "messenger"}}}}}
|
||||
running := func(module string) []string {
|
||||
if module == "messenger" || module == "telegram" {
|
||||
return []string{"novox"}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if got, want := planLineOn(p, now, pauseView{}, tierAtLeast, running),
|
||||
"mesh-catalog #175 a tap shows its outcome · messenger, telegram → novox · tier 1 of 1, building for 2m0s"; got != want {
|
||||
t.Fatalf("the line reads %q, want %q", got, want)
|
||||
}
|
||||
if got := planLine(p, now); !strings.HasPrefix(got, "mesh-catalog #175 a tap shows its outcome · messenger, telegram · tier") {
|
||||
t.Fatalf("without the machines the line reads %q", got)
|
||||
}
|
||||
old := p
|
||||
old.Commits, old.Delivery = nil, nil
|
||||
if got := planLine(old, now); !strings.HasPrefix(got, "mesh-catalog c1c1c1c1 · tier 1 of 1") {
|
||||
t.Fatalf("a record naming no pull request reads %q", got)
|
||||
}
|
||||
long := p
|
||||
long.Delivery.Merges[0].Title = strings.Repeat("abcdefghij", 6)
|
||||
if got := planHeadline(long, nil); !strings.Contains(got, "#175 "+strings.Repeat("abcdefghij", 4)+"abcdefghi…") {
|
||||
t.Fatalf("a long title is not cut at fifty runes: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// sameMerge compares what a walk names of a merge: its repository, commit and the commit carrying it.
|
||||
func sameMerge(a, b inventory.PlanMerge) bool {
|
||||
return a.Repository == b.Repository && a.Commit == b.Commit && a.Carried == b.Carried
|
||||
}
|
||||
|
||||
// A catalogue merge heard after a later merge of its branch was walked without the word (a merge that touched
|
||||
// the bus too, on the controller's own path) is not answered by that walk while the delivery seat has a holder:
|
||||
// its delivery would skip its turn. It joins the next catalogue batch.
|
||||
func TestALateCatalogueMergeIsNotAnsweredByAnOwnPathWalk(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
for _, m := range []struct {
|
||||
module string
|
||||
claims []catalogue.Claim
|
||||
}{
|
||||
{"nats", nil},
|
||||
{"mesh-delivery", []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
|
||||
} {
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1", Claims: m.claims},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + m.module, Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mixed := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "m1xed", MergedAt: t0.Format(time.RFC3339Nano),
|
||||
Paths: []string{"modules/nats/module.json", "modules/app/module.json"}, ModuleDirs: []string{"modules/nats", "modules/app"},
|
||||
ModuleDirsSaid: true}
|
||||
hear(t, open, mixed, t0.Add(time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 || ws[0].Waiting() {
|
||||
t.Fatalf("the mixed merge's walk waited, or was not cut: %+v", ws)
|
||||
}
|
||||
done := ws[0]
|
||||
done.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, catalogueMerge("ear1ier", "app", t0.Add(-30*time.Second)), t0.Add(3*time.Minute))
|
||||
got, _ := open.inventory.PlanByID(ctx, done.ID)
|
||||
_, bs := walks(t, open)
|
||||
if len(got.Delivery.Merges) != 1 || len(bs) != 1 || bs[0].OwnPath() || bs[0].Delivery.Merges[0].Commit != "ear1ier" {
|
||||
t.Fatalf("the late catalogue merge was answered by the own-path walk (%+v) rather than the next catalogue batch (%+v)",
|
||||
got.Delivery.Merges, bs)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// **Every wait of a batch has a bound and a condition** (novox/hq ADR 0276 decision 9): a batch still
|
||||
// assembling a minute past its maximum while no walk is open is a cut the controller failed to make (S18),
|
||||
// and a batch waiting behind an open walk longer than that walk's bound waits on a walk gone wrong (S19).
|
||||
|
||||
// The kinds S18 and S19 raise.
|
||||
const (
|
||||
kindBatchNotCut = "batch-not-cut"
|
||||
kindBatchBehindWalk = "batch-behind-walk"
|
||||
)
|
||||
|
||||
// batchFacts is one batch not yet cut, as the watchdogs read it.
|
||||
type batchFacts struct {
|
||||
id, state, grouped string
|
||||
// atMost is when its window closes at the latest; closed when it closed.
|
||||
atMost, closed time.Time
|
||||
// behind is the open walk it waits behind, and walkBound that walk's bound: a tier's bound for each of its
|
||||
// tiers.
|
||||
behind string
|
||||
walkBound time.Duration
|
||||
}
|
||||
|
||||
// gatherBatches is every batch not yet cut, with the bound of the walk it waits behind.
|
||||
func gatherBatches(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]batchFacts, error) {
|
||||
batches, err := inv.Batches(ctx)
|
||||
if err != nil || len(batches) == 0 {
|
||||
return nil, err
|
||||
}
|
||||
bounds, err := measuredTierBounds(ctx, inv, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []batchFacts
|
||||
for _, b := range batches {
|
||||
f := batchFacts{id: b.ID, state: b.State, grouped: groupedWords(b)}
|
||||
if w := b.Delivery; w != nil && w.Batch != nil {
|
||||
f.atMost, f.closed, f.behind = w.Batch.AtMost, w.Batch.ClosesAt, w.Batch.Behind
|
||||
if f.atMost.Before(f.closed) {
|
||||
f.closed = f.atMost
|
||||
}
|
||||
}
|
||||
if f.behind != "" {
|
||||
if walk, err := inv.PlanByID(ctx, f.behind); err == nil {
|
||||
f.walkBound = bounds.of(walk.Repository) * time.Duration(max(1, len(walk.Tiers)))
|
||||
}
|
||||
}
|
||||
out = append(out, f)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// watchBatchesNotCut is S18: a batch still assembling a minute past its maximum, while no walk is open.
|
||||
func watchBatchesNotCut(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, b := range f.batches {
|
||||
if b.state != inventory.PlanAssembling || b.atMost.IsZero() || f.now.Sub(b.atMost) <= batchLateAfter {
|
||||
continue
|
||||
}
|
||||
late := f.now.Sub(b.atMost)
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id, Kind: kindBatchNotCut,
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the batch %s is still assembling %s past its latest close (%s), with no walk open: "+
|
||||
"the controller did not cut it; grouped: %s", b.id, ago(late), b.atMost.UTC().Format(time.RFC3339), b.grouped),
|
||||
Said: fmt.Sprintf("assembling since %s past its maximum", ago(late)),
|
||||
Headline: "Merged changes are not being delivered",
|
||||
Explanation: fmt.Sprintf("Merges collected for one delivery should have been planned %s ago and were not. "+
|
||||
"Nothing is lost; the mesh keeps them until it plans them.", humanDuration(late)),
|
||||
Resolved: "Merged changes are being delivered again"})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// watchBatchesBehind is S19: a batch waiting behind an open walk longer than that walk's bound, naming it.
|
||||
func watchBatchesBehind(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, b := range f.batches {
|
||||
if b.state != inventory.PlanQueued || b.behind == "" || b.walkBound <= 0 || f.now.Sub(b.closed) <= b.walkBound {
|
||||
continue
|
||||
}
|
||||
in := f.now.Sub(b.closed)
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id, Kind: kindBatchBehindWalk,
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the batch %s has waited %s behind the walk %s, longer than that walk's bound (%s); "+
|
||||
"grouped: %s; `plans %s` says where that walk stands", b.id, ago(in), b.behind, ago(b.walkBound),
|
||||
b.grouped, b.behind),
|
||||
Said: fmt.Sprintf("queued behind %s for %s", b.behind, ago(in)),
|
||||
Headline: "Merged changes wait behind a slow delivery",
|
||||
Explanation: fmt.Sprintf("Merges collected for the next delivery have waited %s for the delivery before "+
|
||||
"them, which is taking longer than it should. Nothing is lost.", humanDuration(in)),
|
||||
Resolved: "Merged changes no longer wait behind a slow delivery"})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// A binding to data moves only by a person (novox/hq ADR 0232, issue 273).
|
||||
//
|
||||
// The resolver keeps each consumer of a provision that keeps its data at the provider it was last
|
||||
// sent (catalogue/bound.go) and says what it would have moved. This is where that is said: on the
|
||||
// push that composed it, and by the self-check's D12 every run, as an urgent condition naming the
|
||||
// consumer, both providers and the pin that confirms the move.
|
||||
|
||||
// The condition kinds of D12.
|
||||
const (
|
||||
// kindBindingKept is a move the resolver refused: the consumer is still where its data is.
|
||||
kindBindingKept = "binding-kept"
|
||||
// kindBindingMoved is a consumer about to be sent another provider than the one on record with
|
||||
// no pin naming it — what the resolver exists to make impossible, said if it ever is not.
|
||||
kindBindingMoved = "binding-moved"
|
||||
// kindBindingMoving is a move a pin asked for, not yet sent: a person's act, said so that the
|
||||
// data is moved before the push that carries it.
|
||||
kindBindingMoving = "binding-moving"
|
||||
)
|
||||
|
||||
// probeBindingsID is the self-check's id for this probe, and the source of what it raises.
|
||||
const probeBindingsID = "D12"
|
||||
|
||||
// keptObservation is the urgent condition for one refused move.
|
||||
func keptObservation(k catalogue.KeptBinding) conditions.Observation {
|
||||
return conditions.Observation{Scope: conditions.ScopeMachine, ID: bindingID(k.Machine, k.Consumer, k.Provision),
|
||||
Token: kindBindingKept, Kind: kindBindingKept, Machine: k.Machine, Also: otherMachines(k.Machine, k.Bound.Node, k.Would.Node),
|
||||
Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("on %s, the mesh %s", k.Machine, k.String())}
|
||||
}
|
||||
|
||||
func bindingID(machine, consumer, provision string) string {
|
||||
return machine + "." + consumer + "." + provision
|
||||
}
|
||||
|
||||
func otherMachines(machine string, nodes ...string) []string {
|
||||
var out []string
|
||||
seen := map[string]bool{machine: true}
|
||||
for _, n := range nodes {
|
||||
if n != "" && !seen[n] {
|
||||
seen[n] = true
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// reportKept says every move a machine's resolution refused, on the push composing it, and raises its
|
||||
// condition at once where this process keeps the conditions: a push is when a person is looking.
|
||||
func reportKept(ctx context.Context, plan catalogue.Resolution) {
|
||||
for _, k := range plan.Kept {
|
||||
fmt.Printf("%s: the mesh %s\n", plan.Node, k)
|
||||
if conditionsFrom != nil {
|
||||
o := keptObservation(k)
|
||||
o.Source = probeBindingsID
|
||||
if _, err := conditionsFrom.Observe(ctx, o); err != nil {
|
||||
fmt.Printf("%s: and the condition for it could not be raised: %v\n", plan.Node, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// probeBindings is D12: every consumer of a provision that keeps its data is bound where it was last
|
||||
// sent, on every machine — the resolver kept it there (said, urgent, until a person pins), or a pin
|
||||
// moves it (said, so the data goes first), and never anything else.
|
||||
func probeBindings(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
inv := d.open.inventory
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []conditions.Observation
|
||||
for _, n := range nodes {
|
||||
plan, _, err := planFor(ctx, d.open, n.Name)
|
||||
if err != nil {
|
||||
if unresolvable(err) {
|
||||
// D1 says it, with the binding that refused it when that is why.
|
||||
continue
|
||||
}
|
||||
return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err)
|
||||
}
|
||||
bound, err := inv.BindingsFor(ctx, n.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pins, err := inv.PinsFor(ctx, n.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, bindingFindings(plan, bound, pins)...)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// bindingFindings is what one machine's resolution says against its record.
|
||||
func bindingFindings(plan catalogue.Resolution, bound map[string]map[string]catalogue.Chosen,
|
||||
pins map[string]catalogue.Chosen) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, k := range plan.Kept {
|
||||
out = append(out, keptObservation(k))
|
||||
}
|
||||
said := map[string]bool{}
|
||||
for _, need := range plan.Needs {
|
||||
if !need.KeepsData || need.ByRecord {
|
||||
continue
|
||||
}
|
||||
was, recorded := bound[need.For][need.Name]
|
||||
now := catalogue.Chosen{Node: need.From, Module: need.Module}
|
||||
if !recorded || was == now || (was.Module == "" && was.Node == now.Node) {
|
||||
continue
|
||||
}
|
||||
id := bindingID(plan.Node, need.For, need.Name)
|
||||
if said[id] {
|
||||
continue
|
||||
}
|
||||
said[id] = true
|
||||
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Machine: plan.Node,
|
||||
Also: otherMachines(plan.Node, was.Node, now.Node), Resolver: conditions.ResolverOperator}
|
||||
if pin, pinned := pins[need.Name]; pinned && pin.Node == now.Node && (pin.Module == "" || pin.Module == now.Module) {
|
||||
o.Token, o.Kind, o.Severity = kindBindingMoving, kindBindingMoving, conditions.Warning
|
||||
o.Summary = fmt.Sprintf("on %s, %s's %s moves from %s to %s at the next push, by the pin — its data "+
|
||||
"is on %s: move it first", plan.Node, need.For, need.Name, was, now, was)
|
||||
} else {
|
||||
o.Token, o.Kind, o.Severity = kindBindingMoved, kindBindingMoved, conditions.Urgent
|
||||
o.Summary = fmt.Sprintf("on %s, %s's %s would be sent %s, and it is bound to %s, where its data is, "+
|
||||
"with no pin naming %s — a move nothing asked for", plan.Node, need.For, need.Name, now, was, now)
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// novox/hq issue 273, ADR 0232: a consumer of a provision that keeps its data moves only by a pin.
|
||||
|
||||
func storeManifests() []catalogue.Manifest {
|
||||
return []catalogue.Manifest{
|
||||
{Module: "store", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}},
|
||||
Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}},
|
||||
Serves: map[string]map[string]any{"postgres-database": {"port": 5432}},
|
||||
Grants: map[string]string{"postgres-database": "/var/lib/mesh/store/grants"}},
|
||||
{Module: "resolver", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}},
|
||||
Claims: []catalogue.Claim{{Name: "mesh-dns-resolver", Scope: catalogue.ScopeMesh}}},
|
||||
{Module: "network", Version: "1", Requires: []string{"wildcard-resolution"}},
|
||||
{Module: "board", Version: "1", Requires: []string{"postgres-database"}},
|
||||
}
|
||||
}
|
||||
|
||||
func need(t *testing.T, plan catalogue.Resolution, consumer, provision string) catalogue.Needed {
|
||||
t.Helper()
|
||||
for _, n := range plan.Needs {
|
||||
if n.For == consumer && n.Name == provision {
|
||||
return n
|
||||
}
|
||||
}
|
||||
t.Fatalf("no %s for %s: %+v", provision, consumer, plan.Needs)
|
||||
return catalogue.Needed{}
|
||||
}
|
||||
|
||||
// The incident through the stores: the laptop runs its own store and a consumer of it, the anchor's
|
||||
// store holds the mesh's seat. The consumer stays beside its data, the resolver follows its seat, the
|
||||
// binding is recorded as sent, and a pin — only a pin — moves it, said before the push that carries it.
|
||||
func TestTheIncidentAConsumerStaysBesideItsDataUntilAPersonPinsIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, m := range storeManifests() {
|
||||
register(t, open, m)
|
||||
}
|
||||
assignAll := func(pairs ...[2]string) {
|
||||
for _, a := range pairs {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
}
|
||||
// The anchor's store and resolver hold the mesh's seats, on record; the laptop runs its own of each.
|
||||
assignAll([2]string{"anchor", "store"}, [2]string{"anchor", "resolver"})
|
||||
for _, seat := range [][2]string{{"mesh-store", "store"}, {"mesh-dns-resolver", "resolver"}} {
|
||||
if err := inv.HoldSeat(ctx, seat[0], catalogue.ScopeMesh, "anchor", seat[1]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
assignAll([2]string{"laptop", "store"}, [2]string{"laptop", "resolver"}, [2]string{"laptop", "network"},
|
||||
[2]string{"laptop", "board"})
|
||||
|
||||
plan, _, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := need(t, plan, "board", "postgres-database"); n.From != "laptop" || n.Module != "store" || !n.KeepsData {
|
||||
t.Fatalf("the consumer was bound to %s/%s (keeps data: %v); its data is beside it", n.From, n.Module, n.KeepsData)
|
||||
}
|
||||
if n := need(t, plan, "network", "wildcard-resolution"); n.From != "anchor" || n.KeepsData {
|
||||
t.Fatalf("the resolver was bound to %s (keeps data: %v); its seat is held on anchor (issue 258)", n.From, n.KeepsData)
|
||||
}
|
||||
|
||||
// Sent, and recorded: only the binding to data.
|
||||
bindings := boundToData(plan, nil)
|
||||
if len(bindings) != 1 || bindings[0].Provider != (catalogue.Chosen{Node: "laptop", Module: "store"}) {
|
||||
t.Fatalf("recorded %+v", bindings)
|
||||
}
|
||||
if err := inv.RecordBindings(ctx, "laptop", bindings); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 {
|
||||
t.Fatalf("a mesh bound where it was sent: %+v, %v", found, err)
|
||||
}
|
||||
|
||||
// The laptop's store taken away: refused, not moved to the anchor's empty one.
|
||||
if err := inv.Unassign(ctx, "laptop", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := planFor(ctx, open, "laptop"); err == nil || !unresolvable(err) ||
|
||||
!strings.Contains(err.Error(), "board on laptop is bound to laptop/store") ||
|
||||
!strings.Contains(err.Error(), "pin laptop postgres-database anchor store") {
|
||||
t.Fatalf("the consumer's store went and it was answered elsewhere: %v", err)
|
||||
}
|
||||
|
||||
// A person pins the anchor's: it moves, said before it is sent, and the record keeps where it was.
|
||||
if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, _, err = planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := need(t, plan, "board", "postgres-database"); n.From != "anchor" {
|
||||
t.Fatalf("pinned to the anchor and bound to %s", n.From)
|
||||
}
|
||||
found, err := probeBindings(ctx, &doctor{open: open})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(found) != 1 || found[0].Kind != kindBindingMoving || found[0].Severity != conditions.Warning ||
|
||||
!strings.Contains(found[0].Summary, "board's postgres-database moves from laptop/store to anchor/store") {
|
||||
t.Fatalf("a pinned move is not said before it is sent: %+v", found)
|
||||
}
|
||||
if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
all, err := inv.Bindings(ctx)
|
||||
if err != nil || len(all) != 1 || all[0].MovedFrom != "laptop/store" {
|
||||
t.Fatalf("%+v, %v", all, err)
|
||||
}
|
||||
if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 {
|
||||
t.Fatalf("a move sent is still said: %+v, %v", found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// What one machine's resolution says against its record.
|
||||
func TestBindingFindingsSayAKeptMoveAndAMoveNothingAskedFor(t *testing.T) {
|
||||
home, anchor := catalogue.Chosen{Node: "home", Module: "store"}, catalogue.Chosen{Node: "anchor", Module: "store"}
|
||||
plan := catalogue.Resolution{Node: "laptop",
|
||||
Kept: []catalogue.KeptBinding{{Machine: "laptop", Consumer: "board", Provision: "postgres-database",
|
||||
Bound: home, Would: anchor}},
|
||||
Needs: []catalogue.Needed{
|
||||
{Name: "postgres-database", For: "board", From: "home", Module: "store", KeepsData: true},
|
||||
{Name: "postgres-database", For: "game", From: "anchor", Module: "store", KeepsData: true},
|
||||
{Name: "wildcard-resolution", For: "network", From: "anchor", Module: "resolver"},
|
||||
}}
|
||||
bound := map[string]map[string]catalogue.Chosen{
|
||||
"board": {"postgres-database": home},
|
||||
"game": {"postgres-database": home},
|
||||
"network": {"wildcard-resolution": {Node: "home", Module: "resolver"}},
|
||||
}
|
||||
found := linted(bindingFindings(plan, bound, nil))
|
||||
if len(found) != 2 {
|
||||
t.Fatalf("found %+v", found)
|
||||
}
|
||||
kept, moved := found[0], found[1]
|
||||
if kept.Kind != kindBindingKept || kept.Severity != conditions.Urgent || kept.Machine != "laptop" ||
|
||||
!strings.Contains(kept.Summary, "would move board's postgres-database from home/store to anchor/store") ||
|
||||
!strings.Contains(kept.Summary, "its data is on home/store") ||
|
||||
!strings.Contains(kept.Summary, "`pin laptop postgres-database anchor store` to confirm a move (and move the data first)") {
|
||||
t.Errorf("kept: %+v", kept)
|
||||
}
|
||||
if moved.Kind != kindBindingMoved || moved.Severity != conditions.Urgent ||
|
||||
!strings.Contains(moved.Summary, "game's postgres-database would be sent anchor/store") {
|
||||
t.Errorf("moved: %+v", moved)
|
||||
}
|
||||
if kept.Key() == moved.Key() {
|
||||
t.Error("two consumers, one condition")
|
||||
}
|
||||
}
|
||||
|
||||
// A push says the move it refused, and raises its condition at once.
|
||||
func TestAPushSaysAKeptMoveAndRaisesItsCondition(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
reportKept(t.Context(), catalogue.Resolution{Node: "laptop", Kept: []catalogue.KeptBinding{{Machine: "laptop",
|
||||
Consumer: "board", Provision: "postgres-database", Bound: catalogue.Chosen{Node: "home", Module: "store"},
|
||||
Would: catalogue.Chosen{Node: "anchor", Module: "store"}}}})
|
||||
open, err := k.Open(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(open) != 1 || open[0].Kind != kindBindingKept || open[0].Severity != conditions.Urgent ||
|
||||
open[0].Source != probeBindingsID {
|
||||
t.Fatalf("raised %+v", open)
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -148,6 +149,8 @@ func buildFrom(result link.BuildResult) inventory.Build {
|
||||
// be, for exactly the modules it needs most. Keeping them is what makes a replay able to
|
||||
// rebuild the graph rather than a list of names.
|
||||
Path: result.Path,
|
||||
// What it was made from (novox/hq issue 280): two builds with one are one build.
|
||||
SourceFingerprint: result.SourceFingerprint,
|
||||
}
|
||||
// When it was asked, which is what orders it against another build of the same module
|
||||
// (novox/hq 04-ISSUES/219) — not when it was heard.
|
||||
@@ -157,9 +160,16 @@ func buildFrom(result link.BuildResult) inventory.Build {
|
||||
for _, ref := range result.Against {
|
||||
kept.Against = append(kept.Against, catalogue.Recorded(ref))
|
||||
}
|
||||
for _, ref := range result.Mirrored {
|
||||
kept.Mirrored = append(kept.Mirrored, catalogue.Recorded(ref))
|
||||
}
|
||||
for _, r := range result.Read {
|
||||
kept.Read = append(kept.Read, inventory.ReadRepository{Repository: r.Repository, Ref: r.Ref})
|
||||
}
|
||||
// What it was made from, as files (novox/hq ADR 0267): the planner maps the next merge onto it.
|
||||
for _, s := range result.Sources {
|
||||
kept.Sources = append(kept.Sources, inventory.BuildSource{Repository: s.Repository, Ref: s.Ref, Paths: s.Paths})
|
||||
}
|
||||
var announced []inventory.Artifact
|
||||
for _, made := range result.Made {
|
||||
announced = append(announced, inventory.Artifact{
|
||||
@@ -393,15 +403,49 @@ func buildBehind(ctx context.Context, wait time.Duration) error {
|
||||
//
|
||||
// Separated from the command so `--behind` can walk a list without a second path to the same act.
|
||||
func buildOne(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error {
|
||||
_, err := buildOneAsked(ctx, source, path, ref, wait, false)
|
||||
_, err := buildOneAsked(ctx, source, path, ref, wait, false, "build")
|
||||
return err
|
||||
}
|
||||
|
||||
// recordAsked keeps a build request once it is asked (novox/hq issue 325), with who asked it, in a store
|
||||
// opened for the purpose: the asks reach here from processes that hold none.
|
||||
func recordAsked(ctx context.Context, r inventory.BuildRequest) {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "build %s is asked and not kept as a build request: %v\n", r.ID, err)
|
||||
return
|
||||
}
|
||||
defer open.Close()
|
||||
recordBuildRequest(ctx, open.inventory, r)
|
||||
}
|
||||
|
||||
// markWaitFailed says what became of a kept build request whose waited ask failed: never handed over, so
|
||||
// nothing runs; or handed over and no longer waited for, so asked with its outcome unknown — still read as in
|
||||
// flight until its outcome or its bound (novox/hq issue 325).
|
||||
func markWaitFailed(ctx context.Context, id string, why error) {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "build %s could not be marked: %v\n", id, err)
|
||||
return
|
||||
}
|
||||
defer open.Close()
|
||||
mark := open.inventory.MarkOutcomeUnknown
|
||||
if errors.Is(why, link.ErrNotHandedOver) {
|
||||
mark = open.inventory.MarkNotAsked
|
||||
}
|
||||
if err := mark(ctx, id, why.Error()); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "build %s could not be marked: %v\n", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
// buildOneAsked is buildOne answering the id it asked with — what a plan keeps to match the outcome
|
||||
// by (novox/hq ADR 0219) — and, for an ask not waited for, optionally a dry run: built and looked
|
||||
// at, never taken in (issue 240), which is what `replay` asks unless told to register.
|
||||
//
|
||||
// asker is who asked, for the build request kept once the ask is made (novox/hq issue 325); empty for an
|
||||
// asker that keeps the request itself, with its own name, once it knows the ask was made.
|
||||
func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wait time.Duration,
|
||||
dryRun bool) (string, error) {
|
||||
dryRun bool, asker string) (string, error) {
|
||||
if dryRun && wait != 0 {
|
||||
return "", errors.New("a dry run waited for is `build --dry-run`")
|
||||
}
|
||||
@@ -411,6 +455,10 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Through a verb, only a repository the catalogue builds from (novox/hq ADR 0266).
|
||||
if err := verbMayAsk(ctx, source, repository); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
ident, err := openIdentity(ctx)
|
||||
if err != nil {
|
||||
@@ -456,6 +504,14 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
|
||||
}
|
||||
defer ask.Close()
|
||||
fmt.Printf(" of %s\n", seat)
|
||||
// Kept once it is asked, so `assign` knows a module is coming while its build runs (novox/hq issue 325);
|
||||
// never before, so an ask that failed never reads as a build in flight. A dry run registers nothing, and
|
||||
// is not kept.
|
||||
keep := !dryRun && asker != ""
|
||||
// Asked at the terminal is what lets its outcome register a module from a repository the catalogue does
|
||||
// not build it from (novox/hq ADR 0266); never through a verb.
|
||||
asked := inventory.BuildRequest{ID: request.ID, Repository: source.Repository, Seat: source.Seat, Path: path,
|
||||
Ref: ref, For: asker, AtTerminal: startedAtTheTerminal()}
|
||||
|
||||
if wait == 0 {
|
||||
// Asked and not waited for (novox/hq issue 176): the outcome is the role's event, and the
|
||||
@@ -465,6 +521,9 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
|
||||
if err := ask.Ask(ctx, request); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if keep {
|
||||
recordAsked(ctx, asked)
|
||||
}
|
||||
if dryRun {
|
||||
fmt.Printf("asked as a dry run, not waited for: `builds --log %s` follows it as it runs; "+
|
||||
"its outcome is not taken in\n", request.ID)
|
||||
@@ -475,8 +534,16 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
|
||||
return request.ID, nil
|
||||
}
|
||||
|
||||
// Waited for: kept while it is waited for, since that can take minutes, and marked when the hand-over
|
||||
// failed (not asked) or the wait did (asked, outcome unknown) — an outcome heard later is the last word.
|
||||
if keep {
|
||||
recordAsked(ctx, asked)
|
||||
}
|
||||
result, err := ask.Submit(ctx, request, wait)
|
||||
if err != nil {
|
||||
if keep {
|
||||
markWaitFailed(ctx, request.ID, err)
|
||||
}
|
||||
return request.ID, err
|
||||
}
|
||||
|
||||
@@ -486,6 +553,10 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
|
||||
}
|
||||
defer open.Close()
|
||||
manifest, kept, err := takeIn(ctx, open.inventory, result)
|
||||
// The assignments pending on this build, made or ended (novox/hq issue 325).
|
||||
for _, line := range settlePendingOnBuild(ctx, open, result, manifest.Module, err) {
|
||||
fmt.Printf(" %s\n", line)
|
||||
}
|
||||
if err != nil {
|
||||
return request.ID, err
|
||||
}
|
||||
@@ -570,6 +641,61 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), and the mesh does not register it: %w",
|
||||
result.On, result.Repository, short(result.Commit), err)
|
||||
}
|
||||
// **Only from the repository the catalogue builds the module from** (novox/hq ADR 0266): else the trunk
|
||||
// below is the trunk of whatever repository was built, which may be one an agent made — and a module named
|
||||
// `sudo` from it would be what the next push sends. Another repository is the operator's, at the terminal.
|
||||
trunk := result.Trunk
|
||||
if was, err := inv.SourceOf(ctx, manifest.Module); err == nil && followedBranch(was.Ref) != "" {
|
||||
trunk = followedBranch(was.Ref)
|
||||
}
|
||||
if trunk == "" {
|
||||
trunk = "main"
|
||||
}
|
||||
repoID, err := mayRegisterFrom(ctx, inv, manifest.Module, recorded, result.ID, result.Path, trunk)
|
||||
if err != nil {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On,
|
||||
manifest.Module, short(result.Commit), err)
|
||||
}
|
||||
// **Only a commit on the trunk is published** (novox/hq ADR 0238): a commit off its repository's
|
||||
// default branch — a pull request's head, a feature branch built by hand, a `rebuild` or `replay
|
||||
// --register` of one — is for checking, and is never a module's version; nothing could then send it.
|
||||
// The branch the module already follows is its trunk — never the branch a build was asked at, or a
|
||||
// build of a feature branch by name would make that branch its trunk.
|
||||
follows := ""
|
||||
if was, err := inv.SourceOf(ctx, manifest.Module); err == nil {
|
||||
follows = followedBranch(was.Ref)
|
||||
}
|
||||
if err := publishable(result, follows); err != nil {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On,
|
||||
manifest.Module, short(result.Commit), err)
|
||||
}
|
||||
// **A build that failed its gate is never registered again** (novox/hq ADR 0236): an outcome heard
|
||||
// twice, or replayed, would otherwise make the build a rollback put back what the module is again,
|
||||
// and the next push would send it.
|
||||
if failed, err := inv.GateFailed(ctx, kept.ID); err != nil {
|
||||
return manifest, kept, err
|
||||
} else if failed {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), which failed its gate on its first machine and was put "+
|
||||
"back: it is recorded and not registered again — a newer build is", result.On, manifest.Module,
|
||||
short(result.Commit))
|
||||
}
|
||||
// **A build whose source is unchanged is never a move** (novox/hq issue 280): a rebuild made from
|
||||
// what the build the mesh stands on was made from registers that build's artifacts at the new
|
||||
// commit, so no machine is sent a new digest for a source nobody changed — an image is not
|
||||
// byte-reproducible, and the bus rebuilt for another module's merge demanded a planned upgrade.
|
||||
if kept.SourceFingerprint != "" {
|
||||
stands, raw, err := inv.StandingBuild(ctx, manifest.Module, kept.ID)
|
||||
if err != nil {
|
||||
return manifest, kept, err
|
||||
}
|
||||
if stands != "" && stands != kept.ID && len(raw) > 0 {
|
||||
if same, err := catalogue.ParseManifest(raw); err == nil && same.Module == manifest.Module {
|
||||
fmt.Printf("%s at %s was made from the source %s was: registered with its artifacts, no move\n",
|
||||
manifest.Module, short(result.Commit), stands)
|
||||
manifest = same
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, manifest, recorded); err != nil {
|
||||
if errors.Is(err, inventory.ErrSuperseded) {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w",
|
||||
@@ -577,6 +703,10 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
||||
}
|
||||
return manifest, kept, err
|
||||
}
|
||||
// Which repository it is registered from, by the forge's own id (novox/hq ADR 0266).
|
||||
if err := inv.SetSourceIdentity(ctx, manifest.Module, repoID); err != nil {
|
||||
return manifest, kept, err
|
||||
}
|
||||
// The keep set just moved, and new bytes just landed (novox/hq ADR 0189). Asked here rather
|
||||
// than on a timer of its own: this is the only moment either is true. Never fatal — the build
|
||||
// worked and the module is registered.
|
||||
@@ -584,12 +714,43 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
||||
return manifest, kept, nil
|
||||
}
|
||||
|
||||
// errOffTheTrunk is a build of a commit off its repository's trunk, which is never published.
|
||||
var errOffTheTrunk = errors.New("the commit is not on its repository's trunk: a commit off the trunk is checked, " +
|
||||
"never published (ADR 0238) — merge it, and the merge builds it")
|
||||
|
||||
// publishable says whether a build's outcome may become a module's version: its commit on the module's
|
||||
// trunk, as the build seat read the forge at the build — the branch the module follows when its source
|
||||
// names one, else its repository's default branch. A build seat that could not say — one older than the
|
||||
// rule, building its own successor — is let through and said, so the rule can reach the mesh.
|
||||
func publishable(result link.BuildResult, follows string) error {
|
||||
if result.Check != nil || result.Checked != nil || result.DryRun {
|
||||
return errors.New("a check or a dry run is never published")
|
||||
}
|
||||
if result.Trunk == "" {
|
||||
fmt.Printf("%s: the build seat did not say whether %s is on its repository's trunk (it predates ADR 0238); "+
|
||||
"registered as before\n", result.ID, short(result.Commit))
|
||||
return nil
|
||||
}
|
||||
trunk := result.Trunk
|
||||
if follows != "" {
|
||||
trunk = follows
|
||||
}
|
||||
on := slices.Contains(result.Branches, trunk) || (trunk == result.Trunk && result.OnTrunk)
|
||||
if !on {
|
||||
return fmt.Errorf("%w (%s is not on %s)", errOffTheTrunk, short(result.Commit), trunk)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildAndShow builds and prints the manifest without recording anything.
|
||||
func buildAndShow(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error {
|
||||
repository, err := cloneFrom(ctx, source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := verbMayAsk(ctx, source, repository); err != nil {
|
||||
return err
|
||||
}
|
||||
ident, err := openIdentity(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -649,6 +810,8 @@ type answers struct {
|
||||
plans []inventory.Plan
|
||||
// paused is whether the build seat takes work, which a plan waiting on it says (ADR 0219).
|
||||
paused pauseView
|
||||
// tierBounds is how long each repository's plan tier may take before it is late (novox/hq issue 296).
|
||||
tierBounds tierBounds
|
||||
// refused is why a machine cannot be worked out at all, by name. A different thing from every
|
||||
// other answer here: those are about a machine that was told something, and this is about one
|
||||
// that cannot be told anything — it never reaches waiting, because nothing was computed for it
|
||||
@@ -695,6 +858,11 @@ type answers struct {
|
||||
// handed to the operator; healsUnread why it could not be read.
|
||||
heals *healsCount
|
||||
healsUnread string
|
||||
// pending is every assignment waiting for its module's build, and every one ended in the last day
|
||||
// (novox/hq issue 325); pendingUnread why they could not be read. One waiting, or one ended other than
|
||||
// applied, is not well: an assignment somebody made is not made yet, or will not be.
|
||||
pending []inventory.PendingAssignment
|
||||
pendingUnread string
|
||||
}
|
||||
|
||||
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
|
||||
@@ -743,6 +911,10 @@ func heldBy(ctx context.Context) map[string]string {
|
||||
// the mesh runs on today this needs the controller's own connection, so it is handed one; on the bus
|
||||
// being built it dials, because a build request is a one-shot and holds nothing else.
|
||||
func askOverOn(seat string) (link.Builders, error) {
|
||||
// The serving controller asks on its own connection (novox/hq issue 327).
|
||||
if serving := servingBus.Load(); serving != nil {
|
||||
return link.BuildsOn(serving, seat), nil
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -801,11 +973,7 @@ func buildSeatAmong(entries []inventory.Entry) string {
|
||||
// with a consumer of its own that is gone when this returns, so nothing accumulates in the server
|
||||
// for the reading, and filtered by subject, so one build's lines are all that travel.
|
||||
func buildLog(ctx context.Context, id string) error {
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot reach the bus to read a build's log: %w", err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,458 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// Where a build may register a module from (novox/hq ADR 0266).
|
||||
//
|
||||
// A build's outcome registers its module, and a registered module is what the next push sends. Before this,
|
||||
// an outcome registered whatever its manifest named, from whichever repository it was built from: an agent
|
||||
// that made a repository of its own, committed `modules/sudo/module.json` granting itself a rule without a
|
||||
// password, and asked the `build` verb for it had its repository registered as the module `sudo` — whose next
|
||||
// push made the agent root on every node. A fork of the node-engine did the same everywhere. The trunk rule
|
||||
// (ADR 0238) did not stop it: the trunk it checked was the trunk of the repository built, which was the
|
||||
// agent's own.
|
||||
//
|
||||
// So **an outcome registers a module only from the repository the catalogue already builds that module
|
||||
// from**; and a module new to the catalogue only from a repository the catalogue already builds another
|
||||
// module from — whose trunk takes a reviewed, approved merge, which is how a merge adds a module (novox/hq
|
||||
// issue 300). Anything else — a module moved to another repository, a module from a repository the
|
||||
// catalogue has never built — is the operator's, at the controller's terminal: allowed only when the build
|
||||
// request was kept as asked there. Judged at the take-in, which every outcome reaches whoever hears it and
|
||||
// whichever verb asked it (`build`, `rebuild`, `replay --register`, `assign` with build), and before the ask
|
||||
// for a call through a verb, so an agent cannot have a build node run a repository the catalogue does not
|
||||
// build from at all.
|
||||
|
||||
// errNotItsSource is an outcome refused for where it was built from.
|
||||
var errNotItsSource = errors.New("not built from the repository the catalogue builds it from")
|
||||
|
||||
// servedVar marks every process the serving controller starts — each verb's command, each child — and the serving
|
||||
// process itself, so none of them can read as the operator at the terminal (novox/hq ADR 0266). Set by serve
|
||||
// before it answers anything, inherited by every child through os.Environ.
|
||||
const servedVar = "MESH_SERVED_BY_THE_CONTROLLER"
|
||||
|
||||
// startedAtTheTerminal says this process was started at the controller's terminal: not the serving controller,
|
||||
// not anything it started, not a verb's command, not a seat call's. The serving controller marks its own
|
||||
// environment (servedVar), so a build asked in it, or by any process it starts, never reads as the terminal's;
|
||||
// runVerb also names the verb and the caller.
|
||||
//
|
||||
// **And a line mesh-cli asked as the controller's terminal** (novox/hq ADR 0272 §4): the serving controller runs it
|
||||
// without the served mark and without a verb, names its caller, and marks it with cliTerminalVar — a mark only the
|
||||
// mesh-cli path sets and every other command line the controller runs is stripped of (commandEnvironment).
|
||||
func startedAtTheTerminal() bool {
|
||||
if os.Getenv(servedVar) != "" || os.Getenv(verbVar) != "" {
|
||||
return false
|
||||
}
|
||||
return os.Getenv(link.CallerVar) == "" || os.Getenv(cliTerminalVar) != ""
|
||||
}
|
||||
|
||||
// cliTerminalVar marks a command line the serving controller runs as the controller's terminal for mesh-cli.
|
||||
const cliTerminalVar = "MESH_CLI_TERMINAL"
|
||||
|
||||
// markServed marks this process, and so everything it starts, as the serving controller's.
|
||||
func markServed() {
|
||||
if err := os.Setenv(servedVar, "1"); err != nil {
|
||||
panic("the serving controller could not mark its environment: " + err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// sourceForms compares sources however each is spelled: a path on a seat's holder (ADR 0111) or a URL — a
|
||||
// build asked of a seat's path is registered with the URL composed from it when its outcome does not echo
|
||||
// the seat. A seat's path is composed into its URL where the seat's holder is known, so both spellings of
|
||||
// one repository are one; where it is not, a seat's path matches only the same seat's same path.
|
||||
type sourceForms struct {
|
||||
bases map[string]string // the seat's clone base, `scheme://host:port`, by seat; "" where it is not known
|
||||
base func(seat string) string
|
||||
}
|
||||
|
||||
// newSourceForms reads the seats' bases from the mesh once, when first needed.
|
||||
func newSourceForms(ctx context.Context, inv *inventory.Inventory) *sourceForms {
|
||||
f := &sourceForms{bases: map[string]string{}}
|
||||
var world *catalogue.World
|
||||
f.base = func(seat string) string {
|
||||
if b, known := f.bases[seat]; known {
|
||||
return b
|
||||
}
|
||||
if world == nil {
|
||||
w := catalogue.World{}
|
||||
if shelf, err := inv.Catalogue(ctx); err == nil {
|
||||
if read, err := theRestOfTheMesh(ctx, inv, shelf, ""); err == nil {
|
||||
w = read
|
||||
}
|
||||
}
|
||||
world = &w
|
||||
}
|
||||
b, err := seatBase(*world, seat)
|
||||
if err != nil {
|
||||
b = ""
|
||||
}
|
||||
f.bases[seat] = b
|
||||
return b
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// canonical is one spelling of a repository: lower case, no `.git`, no trailing slash, and a seat's path as
|
||||
// the URL its holder serves it at where that is known.
|
||||
func (f *sourceForms) canonical(repository, seat string) string {
|
||||
trim := func(s string) string {
|
||||
s = strings.TrimSpace(strings.ToLower(s))
|
||||
s = strings.TrimRight(s, "/")
|
||||
return strings.TrimRight(strings.TrimSuffix(s, ".git"), "/")
|
||||
}
|
||||
if seat == "" {
|
||||
return trim(repository)
|
||||
}
|
||||
if b := f.base(seat); b != "" {
|
||||
return trim(b + "/" + strings.Trim(repository, "/"))
|
||||
}
|
||||
return "seat:" + seat + ":" + trim(strings.Trim(repository, "/"))
|
||||
}
|
||||
|
||||
// same says two sources are one repository.
|
||||
func (f *sourceForms) same(aRepository, aSeat, bRepository, bSeat string) bool {
|
||||
if aRepository == "" || bRepository == "" {
|
||||
return false
|
||||
}
|
||||
return f.canonical(aRepository, aSeat) == f.canonical(bRepository, bSeat)
|
||||
}
|
||||
|
||||
// buildsFrom says the catalogue builds some module from this repository.
|
||||
func (f *sourceForms) buildsFrom(entries []inventory.Entry, repository, seat string) bool {
|
||||
for _, e := range entries {
|
||||
if e.Provided || e.Source.Repository == "" {
|
||||
continue
|
||||
}
|
||||
if f.same(e.Source.Repository, e.Source.Seat, repository, seat) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// forgeFacts is what the mesh's forge says of a repository a module is registered from (novox/hq ADR 0266).
|
||||
type forgeFacts struct {
|
||||
// ID is the forge's own id of the repository: what tells it from one deleted and made again by its name.
|
||||
ID int64
|
||||
// Guarded is whether the branch is protected as a module's trunk must be: no direct push, at least one
|
||||
// required status, and no administrator merging past one. Why says what is missing when it is not.
|
||||
Guarded bool
|
||||
Why string
|
||||
}
|
||||
|
||||
// askTheForge asks the forge, through its module's tools on the bus, for a repository's id and its branch's
|
||||
// protection. A variable so a test needs no forge.
|
||||
var askTheForge = func(ctx context.Context, owner, repo, branch string) (forgeFacts, error) {
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
return forgeFacts{}, err
|
||||
}
|
||||
defer js.Close()
|
||||
bus := link.OverNATS{Conn: js.Conn()}
|
||||
ask := func(tool string, args map[string]any, into any) error {
|
||||
raw, _ := json.Marshal(args)
|
||||
answer, err := link.Ask(ctx, bus, "gitea", tool, raw, 30*time.Second)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return fmt.Errorf("gitea.%s: %s", tool, answer.Error)
|
||||
}
|
||||
return json.Unmarshal(answer.Result, into)
|
||||
}
|
||||
var found struct {
|
||||
Result struct {
|
||||
ID int64 `json:"id"`
|
||||
FullName string `json:"full_name"`
|
||||
} `json:"result"`
|
||||
}
|
||||
if err := ask("gitea_api", map[string]any{"path": "/repos/" + owner + "/" + repo}, &found); err != nil {
|
||||
return forgeFacts{}, err
|
||||
}
|
||||
if found.Result.ID == 0 {
|
||||
return forgeFacts{}, fmt.Errorf("the forge named no id for %s/%s", owner, repo)
|
||||
}
|
||||
var rules struct {
|
||||
Rules []protectionRule `json:"rules"`
|
||||
}
|
||||
if err := ask("gitea_branch_protection_get", map[string]any{"owner": owner, "repo": repo}, &rules); err != nil {
|
||||
return forgeFacts{}, err
|
||||
}
|
||||
facts := judgedRule(rules.Rules, branch)
|
||||
facts.ID = found.Result.ID
|
||||
return facts, nil
|
||||
}
|
||||
|
||||
// protectionRule is a branch protection rule as the forge's tool summarises it.
|
||||
type protectionRule struct {
|
||||
Rule string `json:"rule"`
|
||||
Push bool `json:"push"`
|
||||
RequiredStatuses []string `json:"required_statuses"`
|
||||
AdminMayOverride bool `json:"admin_may_override"`
|
||||
}
|
||||
|
||||
// judgedRule judges the one rule the forge applies to a branch, as the forge picks it: the rule named for the
|
||||
// branch, else the first glob rule, in the forge's order, that covers it (gitea's first matching rule). Never a
|
||||
// later rule that happens to be stronger: the forge does not read it.
|
||||
func judgedRule(rules []protectionRule, branch string) forgeFacts {
|
||||
var applied *protectionRule
|
||||
for i := range rules {
|
||||
if rules[i].Rule == branch {
|
||||
applied = &rules[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if applied == nil {
|
||||
for i := range rules {
|
||||
if ruleCovers(rules[i].Rule, branch) {
|
||||
applied = &rules[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
switch r := applied; {
|
||||
case r == nil:
|
||||
return forgeFacts{Why: fmt.Sprintf("no protection rule covers %s", branch)}
|
||||
case r.Push:
|
||||
return forgeFacts{Why: fmt.Sprintf("the rule %s lets a person push to %s directly", r.Rule, branch)}
|
||||
case len(r.RequiredStatuses) == 0:
|
||||
return forgeFacts{Why: fmt.Sprintf("the rule %s requires no status before a merge into %s", r.Rule, branch)}
|
||||
case r.AdminMayOverride:
|
||||
return forgeFacts{Why: fmt.Sprintf("the rule %s lets an administrator merge into %s past a status", r.Rule, branch)}
|
||||
}
|
||||
return forgeFacts{Guarded: true}
|
||||
}
|
||||
|
||||
// ruleCovers says a protection rule's name — a branch, or a glob of them — covers a branch, as the forge reads it.
|
||||
func ruleCovers(rule, branch string) bool {
|
||||
if rule == branch {
|
||||
return true
|
||||
}
|
||||
if !strings.ContainsAny(rule, "*?[") {
|
||||
return false
|
||||
}
|
||||
var re strings.Builder
|
||||
re.WriteString("^")
|
||||
for i := 0; i < len(rule); i++ {
|
||||
switch c := rule[i]; {
|
||||
case c == '*' && i+1 < len(rule) && rule[i+1] == '*':
|
||||
re.WriteString(".*")
|
||||
i++
|
||||
case c == '*':
|
||||
re.WriteString("[^/]*")
|
||||
case c == '?':
|
||||
re.WriteString("[^/]")
|
||||
default:
|
||||
re.WriteString(regexp.QuoteMeta(string(c)))
|
||||
}
|
||||
}
|
||||
re.WriteString("$")
|
||||
ok, _ := regexp.MatchString(re.String(), branch)
|
||||
return ok
|
||||
}
|
||||
|
||||
// onTheForge is a source's owner and name on the mesh's own forge (the git seat's holder), and whether it is
|
||||
// there at all.
|
||||
func (f *sourceForms) onTheForge(repository, seat string) (owner, name string, ok bool) {
|
||||
var rest string
|
||||
switch {
|
||||
case seat == gitSeat:
|
||||
rest = strings.Trim(repository, "/")
|
||||
case seat == "":
|
||||
base := f.base(gitSeat)
|
||||
if base == "" {
|
||||
return "", "", false
|
||||
}
|
||||
url, prefix := f.canonical(repository, ""), f.canonical(base, "")+"/"
|
||||
if !strings.HasPrefix(url, prefix) {
|
||||
return "", "", false
|
||||
}
|
||||
// The case the forge spells it with: the URL as given, past the base.
|
||||
rest = strings.TrimSuffix(strings.Trim(repository[len(prefix):], "/"), ".git")
|
||||
default:
|
||||
return "", "", false
|
||||
}
|
||||
parts := strings.Split(rest, "/")
|
||||
if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
|
||||
return "", "", false
|
||||
}
|
||||
return parts[0], parts[1], true
|
||||
}
|
||||
|
||||
// mayRegisterFrom says whether a build's outcome may register module from the source it was built from, and the
|
||||
// forge's id of that repository to record (novox/hq ADR 0266). Through any verb, only:
|
||||
//
|
||||
// - from the module's registered repository — the same repository by the forge's own id, not only its name; or,
|
||||
// for a module new to the catalogue, from a repository the catalogue builds another module from;
|
||||
// - and from a repository on the mesh's forge whose trunk is protected as a trunk must be: no direct push, at
|
||||
// least one required status, no administrator merging past one.
|
||||
//
|
||||
// Anything else only when the build request was kept as asked at the controller's terminal, for this very
|
||||
// repository and path. path is the module's directory as built; branch the trunk it is registered from.
|
||||
func mayRegisterFrom(ctx context.Context, inv *inventory.Inventory, module string, built inventory.Source,
|
||||
buildID, path, branch string) (int64, error) {
|
||||
forms := newSourceForms(ctx, inv)
|
||||
was, err := inv.SourceOf(ctx, module)
|
||||
isNew := errors.Is(err, inventory.ErrNoSuchModule)
|
||||
if err != nil && !isNew {
|
||||
return 0, err
|
||||
}
|
||||
terminal, err := askedHereFor(ctx, inv, forms, buildID, built, path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
refuse := func(why string) (int64, error) {
|
||||
return 0, fmt.Errorf("%w: %s. A module is registered from such a source only by a build asked at the "+
|
||||
"controller's terminal, never through a verb: a registered module is what the next push sends, and whoever "+
|
||||
"may call a verb includes agents (novox/hq ADR 0266)", errNotItsSource, why)
|
||||
}
|
||||
|
||||
var why string
|
||||
var alongside []inventory.Entry // the modules a new one's repository already builds
|
||||
switch {
|
||||
case !isNew && forms.same(was.Repository, was.Seat, built.Repository, built.Seat):
|
||||
case !isNew:
|
||||
registered := was.Repository
|
||||
if registered == "" {
|
||||
registered = "no repository (it was handed over by hand)"
|
||||
}
|
||||
why = fmt.Sprintf("%s is built from %s, and this build is of %s", module, sourceWords(registered, was.Seat),
|
||||
sourceWords(built.Repository, built.Seat))
|
||||
default:
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if !e.Provided && e.Source.Repository != "" &&
|
||||
forms.same(e.Source.Repository, e.Source.Seat, built.Repository, built.Seat) {
|
||||
alongside = append(alongside, e)
|
||||
}
|
||||
}
|
||||
if len(alongside) == 0 {
|
||||
why = fmt.Sprintf("%s is new to the catalogue, and %s is no repository the catalogue builds a module from",
|
||||
module, sourceWords(built.Repository, built.Seat))
|
||||
}
|
||||
}
|
||||
if why != "" && !terminal {
|
||||
return refuse(why)
|
||||
}
|
||||
|
||||
owner, name, onForge := forms.onTheForge(built.Repository, built.Seat)
|
||||
if !onForge {
|
||||
if terminal {
|
||||
fmt.Printf("%s: %s is not on the mesh's forge — registered, as asked at the controller's terminal\n",
|
||||
buildID, sourceWords(built.Repository, built.Seat))
|
||||
return 0, nil
|
||||
}
|
||||
return refuse(fmt.Sprintf("%s is not on the mesh's forge, so whether its trunk is protected cannot be read",
|
||||
sourceWords(built.Repository, built.Seat)))
|
||||
}
|
||||
facts, err := askTheForge(ctx, owner, name, branch)
|
||||
if err != nil {
|
||||
if terminal {
|
||||
fmt.Printf("%s: the forge could not be asked about %s/%s (%v) — registered, as asked at the controller's "+
|
||||
"terminal\n", buildID, owner, name, err)
|
||||
return 0, nil
|
||||
}
|
||||
return refuse(fmt.Sprintf("the forge could not say whether %s/%s's %s is protected: %v", owner, name, branch, err))
|
||||
}
|
||||
if terminal {
|
||||
if why != "" || !facts.Guarded {
|
||||
fmt.Printf("%s: %s — registered, as asked at the controller's terminal\n", buildID,
|
||||
strings.Trim(why+"; "+facts.Why, "; "))
|
||||
}
|
||||
return facts.ID, nil
|
||||
}
|
||||
if !facts.Guarded {
|
||||
return refuse(fmt.Sprintf("%s/%s's %s is not protected as a module's trunk must be: %s", owner, name, branch,
|
||||
facts.Why))
|
||||
}
|
||||
// The same repository by the forge's id, not only its name: one deleted and made again is another.
|
||||
recorded := map[string]int64{}
|
||||
if !isNew {
|
||||
id, err := inv.SourceIdentity(ctx, module)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
recorded[module] = id
|
||||
}
|
||||
for _, e := range alongside {
|
||||
id, err := inv.SourceIdentity(ctx, e.Manifest.Module)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
recorded[e.Manifest.Module] = id
|
||||
}
|
||||
for m, id := range recorded {
|
||||
if id != 0 && id != facts.ID {
|
||||
return refuse(fmt.Sprintf("%s/%s is not the repository %s was registered from: the forge knows it as "+
|
||||
"repository %d, and %s was registered from repository %d — one of that name deleted and made again",
|
||||
owner, name, m, facts.ID, m, id))
|
||||
}
|
||||
}
|
||||
return facts.ID, nil
|
||||
}
|
||||
|
||||
// askedHereFor says the build was asked at the controller's terminal, for this repository and this path: a kept
|
||||
// request marked so, whose source is the outcome's (novox/hq ADR 0266).
|
||||
func askedHereFor(ctx context.Context, inv *inventory.Inventory, forms *sourceForms, buildID string,
|
||||
built inventory.Source, path string) (bool, error) {
|
||||
r, found, err := inv.BuildRequestByID(ctx, buildID)
|
||||
if err != nil || !found || !r.AtTerminal {
|
||||
return false, err
|
||||
}
|
||||
if !forms.same(r.Repository, r.Seat, built.Repository, built.Seat) ||
|
||||
strings.Trim(r.Path, "/") != strings.Trim(path, "/") {
|
||||
fmt.Printf("%s was asked at the terminal of %s at %q, and its outcome is of %s at %q: not the terminal's\n",
|
||||
buildID, sourceWords(r.Repository, r.Seat), r.Path, sourceWords(built.Repository, built.Seat), path)
|
||||
return false, nil
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// sourceWords is a source as a person reads it.
|
||||
func sourceWords(repository, seat string) string {
|
||||
if seat == "" {
|
||||
return repository
|
||||
}
|
||||
return buildSource{Repository: repository, Seat: seat}.String()
|
||||
}
|
||||
|
||||
// verbMayAsk refuses, for a call through a verb, a build of a repository the catalogue builds no module from
|
||||
// (novox/hq ADR 0266): the build node would run what an agent wrote, and its outcome could never be
|
||||
// registered anyway. url is the repository as it is cloned. At the terminal anything may be asked.
|
||||
func verbMayAsk(ctx context.Context, source buildSource, url string) error {
|
||||
if startedAtTheTerminal() {
|
||||
return nil
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
entries, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
forms := newSourceForms(ctx, open.inventory)
|
||||
if forms.buildsFrom(entries, source.Repository, source.Seat) || forms.buildsFrom(entries, url, "") {
|
||||
return nil
|
||||
}
|
||||
return terminalRefusal("%s is no repository the catalogue builds a module from, and a build of any other is "+
|
||||
"asked at the controller's terminal only, never through a verb: a build node runs what the repository "+
|
||||
"says, and its outcome would register a module the next push sends — whoever may call a verb includes "+
|
||||
"agents (novox/hq ADR 0266). Nothing was asked", source)
|
||||
}
|
||||
@@ -0,0 +1,384 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"os"
|
||||
"os/exec"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The routes to root a review of ADR 0266 found (novox/hq ADR 0266 §7): an agent makes a repository of its
|
||||
// own — or forks one the mesh builds from — commits a module.json naming a module the mesh runs everywhere
|
||||
// (`sudo`, granting itself a rule without a password; `mesh-host`, the node-engine), and asks the `build` verb
|
||||
// for it. Its outcome was registered under that name from the agent's repository, and the next push sent it.
|
||||
|
||||
// onTrunk is an outcome of a commit on its repository's trunk, as the build seat says it.
|
||||
func onTrunk(id, repository, seat, path string, manifest map[string]any) link.BuildResult {
|
||||
raw, _ := json.Marshal(manifest)
|
||||
r := link.BuildResult{ID: id, Repository: "http://forge.internal:20000/" + repository + ".git", Path: path,
|
||||
Ref: "main", On: "anchor", Commit: "c0ffee0123456789", Manifest: raw,
|
||||
Trunk: "main", OnTrunk: true, Branches: []string{"main"}}
|
||||
if seat != "" {
|
||||
r.Source = &link.SourceOnSeat{Seat: seat, Repository: repository}
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// keptAsked keeps a build request as the asker would: through a verb, or at the terminal.
|
||||
func keptAsked(t *testing.T, inv *inventory.Inventory, id, repository, path string, atTerminal bool) {
|
||||
t.Helper()
|
||||
if err := inv.RecordBuildRequest(t.Context(), inventory.BuildRequest{ID: id, Repository: repository, Seat: "git",
|
||||
Path: path, For: "build", AtTerminal: atTerminal}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// theCatalogue is a mesh whose sudo is built from the catalogue repository and whose node-engine from its own.
|
||||
func theCatalogue(t *testing.T) *stores {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
for _, b := range []link.BuildResult{
|
||||
onTrunk("build-sudo", "novox/mesh-catalog", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "1"}),
|
||||
onTrunk("build-host", "novox/mesh-host", "git", "", map[string]any{"module": "mesh-host", "version": "1"}),
|
||||
} {
|
||||
keptAsked(t, open.inventory, b.ID, b.Source.Repository, b.Path, true)
|
||||
if _, _, err := takeIn(ctx, open.inventory, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return open
|
||||
}
|
||||
|
||||
func TestABuildFromAnAgentsRepositoryIsNotRegisteredAsAModuleTheMeshHolds(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
for _, c := range []struct {
|
||||
name, repository, path, module string
|
||||
}{
|
||||
{"its own repository naming sudo", "agent/sudo", "modules/sudo", "sudo"},
|
||||
{"a fork of the catalogue", "agent/mesh-catalog", "modules/sudo", "sudo"},
|
||||
{"a fork of the node-engine", "agent/mesh-host", "", "mesh-host"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
id := "build-" + strings.ReplaceAll(c.repository, "/", "-")
|
||||
keptAsked(t, open.inventory, id, c.repository, c.path, false) // through the build verb
|
||||
evil := onTrunk(id, c.repository, "git", c.path, map[string]any{"module": c.module, "version": "evil"})
|
||||
_, _, err := takeIn(ctx, open.inventory, evil)
|
||||
if !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("a build of %s was taken in as %s: %v", c.repository, c.module, err)
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := shelf[c.module].Version; got != "1" {
|
||||
t.Fatalf("%s is now %q, from %s", c.module, got, c.repository)
|
||||
}
|
||||
if _, found, _ := open.inventory.BuildByID(ctx, id); !found {
|
||||
t.Errorf("the refused build %s is not recorded", id)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestANewModuleFromARepositoryTheCatalogueDoesNotBuildFromIsNotRegistered(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
keptAsked(t, open.inventory, "build-new", "agent/tools", "", false)
|
||||
_, _, err := takeIn(ctx, open.inventory, onTrunk("build-new", "agent/tools", "git", "",
|
||||
map[string]any{"module": "agent-tools", "version": "1"}))
|
||||
if !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("a new module from an agent's repository was taken in: %v", err)
|
||||
}
|
||||
// And one asked of nobody here — an outcome on the bus no request was kept for — the same.
|
||||
_, _, err = takeIn(ctx, open.inventory, onTrunk("build-unasked", "agent/tools", "git", "",
|
||||
map[string]any{"module": "agent-tools", "version": "1"}))
|
||||
if !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("an outcome nobody asked for was taken in: %v", err)
|
||||
}
|
||||
if shelf, _ := open.inventory.Catalogue(ctx); shelf["agent-tools"].Module != "" {
|
||||
t.Fatal("the refused module is in the catalogue")
|
||||
}
|
||||
}
|
||||
|
||||
// The operator at the terminal may still move a module, or add one from a new repository.
|
||||
func TestAtTheTerminalAnotherRepositoryIsRegistered(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
keptAsked(t, open.inventory, "build-moved", "novox/sudo", "", true)
|
||||
if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-moved", "novox/sudo", "git", "",
|
||||
map[string]any{"module": "sudo", "version": "2"})); err != nil {
|
||||
t.Fatalf("a move the operator asked for at the terminal was refused: %v", err)
|
||||
}
|
||||
if src, _ := open.inventory.SourceOf(ctx, "sudo"); src.Repository != "novox/sudo" {
|
||||
t.Fatalf("sudo is built from %q", src.Repository)
|
||||
}
|
||||
if err := open.inventory.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-external",
|
||||
Repository: "http://forge.internal:20000/someone/app.git", For: "build", AtTerminal: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-external", "someone/app", "", "",
|
||||
map[string]any{"module": "app", "version": "1"})); err != nil {
|
||||
t.Fatalf("a new module the operator asked for at the terminal was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The delivery's flow is untouched: a merge's rebuild of a module from its own repository, and a merge adding
|
||||
// a module to a repository the catalogue builds from (novox/hq issue 300), are registered with no terminal.
|
||||
func TestADeliveryFromTheRegisteredRepositoryIsRegistered(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
rebuilt := onTrunk("build-plan", "novox/mesh-catalog", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "2"})
|
||||
if err := open.inventory.RecordBuildRequest(ctx, inventory.BuildRequest{ID: rebuilt.ID,
|
||||
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/sudo", For: "plan"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, rebuilt); err != nil {
|
||||
t.Fatalf("a plan's build of the module's own repository was refused: %v", err)
|
||||
}
|
||||
added := onTrunk("build-merge", "novox/mesh-catalog", "git", "modules/zram", map[string]any{"module": "zram", "version": "1"})
|
||||
if _, _, err := takeIn(ctx, open.inventory, added); err != nil {
|
||||
t.Fatalf("a module a merge added to the catalogue repository was refused: %v", err)
|
||||
}
|
||||
shelf, _ := open.inventory.Catalogue(ctx)
|
||||
if shelf["sudo"].Version != "2" || shelf["zram"].Module == "" {
|
||||
t.Fatalf("not registered: sudo %q, zram %q", shelf["sudo"].Version, shelf["zram"].Module)
|
||||
}
|
||||
}
|
||||
|
||||
// Through a verb, a build of a repository the catalogue builds nothing from is not even asked: the build node
|
||||
// would run what the agent wrote.
|
||||
func TestAVerbAsksNoBuildOfARepositoryTheCatalogueDoesNotBuildFrom(t *testing.T) {
|
||||
theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
t.Setenv(verbVar, "build")
|
||||
t.Setenv(link.CallerVar, "node-tools.anchor, through the mesh-controller seat")
|
||||
err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, "http://forge.internal:20000/agent/sudo.git")
|
||||
var policy *heldAtTheTerminal
|
||||
if !errors.As(err, &policy) {
|
||||
t.Fatalf("a verb's build of an agent's repository was asked: %v", err)
|
||||
}
|
||||
if err := verbMayAsk(ctx, buildSource{Repository: "novox/mesh-catalog", Seat: "git"},
|
||||
"http://forge.internal:20000/novox/mesh-catalog.git"); err != nil {
|
||||
t.Fatalf("a verb's build of the catalogue repository was refused: %v", err)
|
||||
}
|
||||
t.Setenv(verbVar, "")
|
||||
t.Setenv(link.CallerVar, "")
|
||||
if err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, ""); err != nil {
|
||||
t.Fatalf("the terminal was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// asTheOperator keeps a build as asked at the controller's terminal, as the operator's first build of a module
|
||||
// from a repository the catalogue does not yet build from is (novox/hq ADR 0266), and hands it back.
|
||||
func asTheOperator(t *testing.T, inv *inventory.Inventory, b link.BuildResult) link.BuildResult {
|
||||
t.Helper()
|
||||
repository, seat := b.Repository, ""
|
||||
if b.Source != nil {
|
||||
repository, seat = b.Source.Repository, b.Source.Seat
|
||||
}
|
||||
if err := inv.RecordBuildRequest(t.Context(), inventory.BuildRequest{ID: b.ID, Repository: repository, Seat: seat,
|
||||
Path: b.Path, For: "build", AtTerminal: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// A rollback puts back only a build of the module's own repository: an agent's build of the module's name,
|
||||
// recorded and refused, at the very commit the machine ran before (a fork carries it), is never registered by
|
||||
// the back door of a failed gate.
|
||||
func TestARollbackNeverPutsBackABuildFromAnotherRepository(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
fork := onTrunk("build-1791500000000000000", "agent/mesh-catalog", "git", "modules/sudo",
|
||||
map[string]any{"module": "sudo", "version": "evil"})
|
||||
fork.Commit = "c0ffee0123456789" // the commit sudo was registered at
|
||||
keptAsked(t, inv, fork.ID, "agent/mesh-catalog", "modules/sudo", false)
|
||||
if _, _, err := takeIn(ctx, inv, fork); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("the fork's build was taken in: %v", err)
|
||||
}
|
||||
// Asked after the registered build, whenever the test runs: an id naming a fixed moment read as older
|
||||
// than the registered build once the clock passed it (2026-10-10 02:40 UTC), and the test failed on main.
|
||||
failed := onTrunk(fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano()), "novox/mesh-catalog", "git", "modules/sudo",
|
||||
map[string]any{"module": "sudo", "version": "2"})
|
||||
failed.Commit = "badbadbad0123456"
|
||||
if _, _, err := takeIn(ctx, inv, failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, _, err := inv.BuildByID(ctx, failed.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
previous, found, err := inv.PreviousBuild(ctx, "sudo", "c0ffee0123456789", record)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if found && previous.ID == fork.ID {
|
||||
t.Fatalf("a rollback would put back the fork's build %s", previous.ID)
|
||||
}
|
||||
if !found || previous.ID != "build-sudo" {
|
||||
t.Fatalf("a rollback puts back %q (found %v), want the registered build-sudo", previous.ID, found)
|
||||
}
|
||||
}
|
||||
|
||||
// theForge is what the forge says in a test, by owner/name: a repository not named here is protected as a
|
||||
// trunk must be, with an id of its own.
|
||||
var theForge sync.Map
|
||||
|
||||
func init() {
|
||||
askTheForge = func(_ context.Context, owner, repo, _ string) (forgeFacts, error) {
|
||||
if said, ok := theForge.Load(owner + "/" + repo); ok {
|
||||
switch f := said.(type) {
|
||||
case error:
|
||||
return forgeFacts{}, f
|
||||
case forgeFacts:
|
||||
return f, nil
|
||||
}
|
||||
}
|
||||
h := fnv.New32a()
|
||||
_, _ = h.Write([]byte(owner + "/" + repo))
|
||||
return forgeFacts{ID: int64(h.Sum32()), Guarded: true}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// A module's trunk the forge does not protect — direct pushes, no required status — or a forge that cannot say,
|
||||
// registers nothing through a verb: the trunk rule means nothing on a branch anyone pushes to.
|
||||
func TestATrunkTheForgeDoesNotProtectRegistersNothing(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
theForge.Store("novox/unguarded", forgeFacts{ID: 7, Why: "the rule main lets a person push to main directly"})
|
||||
t.Cleanup(func() { theForge.Delete("novox/unguarded") })
|
||||
first := onTrunk("build-unguarded-1", "novox/unguarded", "git", "", map[string]any{"module": "unguarded", "version": "1"})
|
||||
keptAsked(t, open.inventory, first.ID, "novox/unguarded", "", true)
|
||||
if _, _, err := takeIn(ctx, open.inventory, first); err != nil {
|
||||
t.Fatalf("at the terminal: %v", err)
|
||||
}
|
||||
again := onTrunk("build-unguarded-2", "novox/unguarded", "git", "", map[string]any{"module": "unguarded", "version": "2"})
|
||||
if _, _, err := takeIn(ctx, open.inventory, again); !errors.Is(err, errNotItsSource) ||
|
||||
!strings.Contains(err.Error(), "push to main directly") {
|
||||
t.Fatalf("a rebuild from an unprotected trunk was taken in: %v", err)
|
||||
}
|
||||
theForge.Store("novox/unguarded", errors.New("nothing serves gitea.gitea_api"))
|
||||
if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-unguarded-3", "novox/unguarded", "git", "",
|
||||
map[string]any{"module": "unguarded", "version": "3"})); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("a forge that could not say was read as a protected trunk: %v", err)
|
||||
}
|
||||
if shelf, _ := open.inventory.Catalogue(ctx); shelf["unguarded"].Version != "1" {
|
||||
t.Fatalf("unguarded is %q", shelf["unguarded"].Version)
|
||||
}
|
||||
}
|
||||
|
||||
// A repository deleted and made again under the module's repository's name is another repository: the forge's
|
||||
// id, recorded at registration, tells them apart.
|
||||
func TestARepositoryMadeAgainUnderItsNameIsNotTheModulesSource(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
theForge.Store("novox/remade", forgeFacts{ID: 100, Guarded: true})
|
||||
t.Cleanup(func() { theForge.Delete("novox/remade") })
|
||||
first := onTrunk("build-remade-1", "novox/remade", "git", "", map[string]any{"module": "remade", "version": "1"})
|
||||
keptAsked(t, open.inventory, first.ID, "novox/remade", "", true)
|
||||
if _, _, err := takeIn(ctx, open.inventory, first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if id, _ := open.inventory.SourceIdentity(ctx, "remade"); id != 100 {
|
||||
t.Fatalf("the forge's id was not recorded: %d", id)
|
||||
}
|
||||
theForge.Store("novox/remade", forgeFacts{ID: 101, Guarded: true}) // deleted, and made again by an agent
|
||||
if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-remade-2", "novox/remade", "git", "",
|
||||
map[string]any{"module": "remade", "version": "evil"})); !errors.Is(err, errNotItsSource) ||
|
||||
!strings.Contains(err.Error(), "made again") {
|
||||
t.Fatalf("a repository made again under the name was taken in: %v", err)
|
||||
}
|
||||
// And a new module from it, beside the one registered from the first, the same.
|
||||
if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-remade-3", "novox/remade", "git", "modules/other",
|
||||
map[string]any{"module": "other", "version": "1"})); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("a new module from a repository made again was taken in: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The terminal's mark is the operator's for the repository and path they asked: an outcome of another, under that
|
||||
// build's id, is not theirs.
|
||||
func TestATerminalRequestCoversOnlyWhatItAsked(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-asked", Repository: "novox/app",
|
||||
Seat: "git", Path: "modules/app", For: "build", AtTerminal: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
other := onTrunk("build-asked", "agent/sudo", "git", "modules/app", map[string]any{"module": "sudo", "version": "evil"})
|
||||
if _, _, err := takeIn(ctx, open.inventory, other); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("an outcome of another repository under a terminal request's id was taken in: %v", err)
|
||||
}
|
||||
elsewhere := onTrunk("build-asked", "novox/app", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "evil"})
|
||||
if _, _, err := takeIn(ctx, open.inventory, elsewhere); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("an outcome of another path under a terminal request's id was taken in: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The serving controller, and everything it starts, is never the terminal: a build asked in it reads as asked
|
||||
// through the mesh even when no verb and no caller is named.
|
||||
func TestTheServingControllerIsNeverTheTerminal(t *testing.T) {
|
||||
t.Setenv(verbVar, "")
|
||||
t.Setenv(link.CallerVar, "")
|
||||
t.Setenv(servedVar, "")
|
||||
if !startedAtTheTerminal() {
|
||||
t.Fatal("a process started by hand is not the terminal")
|
||||
}
|
||||
markServed()
|
||||
if startedAtTheTerminal() {
|
||||
t.Fatal("the serving controller reads as the terminal")
|
||||
}
|
||||
child := exec.Command(os.Args[0], "-test.run=^$")
|
||||
child.Env = os.Environ()
|
||||
if !slices.Contains(child.Env, servedVar+"=1") {
|
||||
t.Fatal("what the serving controller starts does not carry its mark")
|
||||
}
|
||||
}
|
||||
|
||||
// The forge applies one rule to a branch: the rule named for it, else the first glob that covers it. A stronger
|
||||
// rule later in the list is not what guards the branch, and is not read as if it were (the confirmation review).
|
||||
func TestTheRuleJudgedIsTheOneTheForgeApplies(t *testing.T) {
|
||||
guarded := protectionRule{Rule: "*", RequiredStatuses: []string{"mesh/merge-gate"}}
|
||||
open := protectionRule{Rule: "main", Push: true, RequiredStatuses: []string{"mesh/merge-gate"}}
|
||||
if f := judgedRule([]protectionRule{guarded, open}, "main"); f.Guarded {
|
||||
t.Error("an exact rule letting pushes was passed over for a glob that guards")
|
||||
}
|
||||
if f := judgedRule([]protectionRule{{Rule: "ma*", RequiredStatuses: nil}, {Rule: "*", RequiredStatuses: []string{"x"}}},
|
||||
"main"); f.Guarded || !strings.Contains(f.Why, "ma*") {
|
||||
t.Errorf("the first glob covering the branch was not the one judged: %+v", f)
|
||||
}
|
||||
if f := judgedRule([]protectionRule{{Rule: "release/*"}, {Rule: "main", RequiredStatuses: []string{"x"}}}, "main"); !f.Guarded {
|
||||
t.Errorf("the rule named for the branch was not judged: %+v", f)
|
||||
}
|
||||
if f := judgedRule(nil, "main"); f.Guarded {
|
||||
t.Error("no rule is no protection")
|
||||
}
|
||||
}
|
||||
|
||||
// An id the forge could not give keeps the id already recorded.
|
||||
func TestAnUnknownIdentityKeepsTheRecordedOne(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.SetSourceIdentity(ctx, "sudo", 100); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetSourceIdentity(ctx, "sudo", 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if id, _ := open.inventory.SourceIdentity(ctx, "sudo"); id != 100 {
|
||||
t.Fatalf("the recorded id became %d", id)
|
||||
}
|
||||
}
|
||||
@@ -19,11 +19,11 @@ func TestABuildHeardIsRecordedAndRegistered(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
manifest, _ := json.Marshal(map[string]any{"module": "shop", "version": "3"})
|
||||
m, _, err := takeIn(ctx, open.inventory, link.BuildResult{
|
||||
m, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, link.BuildResult{
|
||||
ID: "b-1", Repository: "http://forge.internal:20000/novox/shop.git", Path: "modules/shop",
|
||||
Ref: "main", On: "anchor", Commit: "abcdef0123", Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/shop"},
|
||||
})
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -78,7 +78,7 @@ func TestABuildAtACommitKeepsTheBranchTheModuleFollows(t *testing.T) {
|
||||
Path: "modules/unifi", Ref: ref, On: "anchor", Commit: commit, Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, result("b-1", "main", "1111111aaaa")); err != nil {
|
||||
if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, result("b-1", "main", "1111111aaaa"))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, result("b-2", "9c97a8a", "9c97a8a1d2c3")); err != nil {
|
||||
@@ -117,7 +117,7 @@ func TestAnOlderBuildHeardLaterDoesNotReplaceTheNewer(t *testing.T) {
|
||||
Path: "modules/postgres", Ref: "main", On: "anchor", Commit: "efff5415", Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, result(newer, "4bcd5f73")); err != nil {
|
||||
if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, result(newer, "4bcd5f73"))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, err := takeIn(ctx, open.inventory, result(older, "0ab07fa9"))
|
||||
|
||||
@@ -0,0 +1,417 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The bus as a planned step (novox/hq to-be 45 §8, ADR 0227 rule 8, ADR 0236).
|
||||
//
|
||||
// **A bus upgrade is never rolled out.** The bus carries every declaration, every report and the
|
||||
// controller's own lease; a new bus build that does not come up is a mesh nobody can tell anything,
|
||||
// and a version whose data format moved (2.10 → 2.11) cannot be undone by putting the old one back.
|
||||
// So nothing sends a new bus build on its own: its policy is `record` whatever anyone says (the
|
||||
// catalogue's DerivedUpgrade, and `upgrade` refuses a roll-out), a plan builds it and sends nothing, a
|
||||
// cascade holds the machine (ADR 0221), and a push naming that machine is refused while a new bus build
|
||||
// waits for it (busHeld). The one way is this verb: a person, with why, the streams snapshotted first,
|
||||
// whether it can be reverted said before it starts, the step said as `bus-maintenance` while it runs,
|
||||
// and every stream, durable consumer and a round trip checked after (H-bus) — or the step said failed,
|
||||
// with its snapshot as the way back.
|
||||
|
||||
// busStepProbe is the registry's row for the step; its kinds.
|
||||
const (
|
||||
busStepProbe = "DB"
|
||||
kindBusMaintenance = "bus-maintenance"
|
||||
kindBusUpgradeFailed = "bus-upgrade-failed"
|
||||
)
|
||||
|
||||
// busStepBound is how long after its start a bus upgrade must be followed by a healthy bus.
|
||||
var busStepBound = 15 * time.Minute
|
||||
|
||||
// takeBusSnapshot snapshots every stream before a bus upgrade and answers where the snapshot is: the bus
|
||||
// machine's backup holder backs the bus module up now, whose dump is the streams' snapshot (novox/hq ADR
|
||||
// 0235). A variable so a test takes none. A person who took one by hand says where with --snapshot-taken,
|
||||
// and then none is taken — for a bus whose module does not yet carry the snapshot program.
|
||||
var takeBusSnapshot = snapshotTheBusNow
|
||||
|
||||
// busPending is what a bus upgrade would do: the bus's module, the machines running it, and, per
|
||||
// machine, the build it was last sent against the build the mesh holds. Empty machines: the mesh holds
|
||||
// no bus module.
|
||||
type busPending struct {
|
||||
module string
|
||||
machines []string
|
||||
from map[string]string
|
||||
to string
|
||||
// same are the commits whose build was made from the same source as the build the mesh holds, or
|
||||
// made the same artifacts and manifest (novox/hq issue 280).
|
||||
same map[string]bool
|
||||
}
|
||||
|
||||
// moves is whether sending the machine would replace its bus: a build it was not last sent, unless the
|
||||
// two builds were made from the same source, or made the same artifacts from the same manifest — a
|
||||
// rebuild of the same source for another module's merge changes nothing the machine runs, whatever
|
||||
// image digest it made (novox/hq issue 280).
|
||||
func (b busPending) moves(machine string) bool {
|
||||
from, known := b.from[machine]
|
||||
if b.module == "" || b.to == "" || (known && sameCommit(from, b.to)) {
|
||||
return false
|
||||
}
|
||||
return !known || !b.same[from]
|
||||
}
|
||||
|
||||
// pendingBus reads what a bus upgrade would do.
|
||||
func pendingBus(ctx context.Context, inv *inventory.Inventory) (busPending, error) {
|
||||
var b busPending
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return b, err
|
||||
}
|
||||
for name, m := range shelf {
|
||||
if catalogue.ProvidesBus(m) {
|
||||
b.module = name
|
||||
}
|
||||
}
|
||||
if b.module == "" {
|
||||
return b, nil
|
||||
}
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return b, err
|
||||
}
|
||||
b.to = current[b.module].Commit
|
||||
if b.machines, err = inv.Running(ctx, b.module); err != nil {
|
||||
return b, err
|
||||
}
|
||||
b.from, b.same = map[string]string{}, map[string]bool{}
|
||||
made, err := inv.BuildFingerprints(ctx, b.module)
|
||||
if err != nil {
|
||||
return b, err
|
||||
}
|
||||
for commit, refs := range made {
|
||||
b.same[commit] = refs != "" && refs == made[b.to]
|
||||
}
|
||||
sources, err := inv.BuildSourceFingerprints(ctx, b.module)
|
||||
if err != nil {
|
||||
return b, err
|
||||
}
|
||||
for commit, src := range sources {
|
||||
if src != "" && src == sources[b.to] {
|
||||
b.same[commit] = true
|
||||
}
|
||||
}
|
||||
for _, n := range b.machines {
|
||||
sent, known, err := inv.SentBuilds(ctx, n)
|
||||
if err != nil {
|
||||
return b, err
|
||||
}
|
||||
if known {
|
||||
if c, carried := sent[b.module]; carried {
|
||||
b.from[n] = c
|
||||
}
|
||||
}
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// busHeld names the machines a push may not send because sending them would replace the bus: the
|
||||
// planned step's, not a push's (ADR 0236). Said with the remedy.
|
||||
func busHeld(ctx context.Context, inv *inventory.Inventory, machines []string) (map[string]string, error) {
|
||||
b, err := pendingBus(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]string{}
|
||||
for _, n := range machines {
|
||||
for _, holder := range b.machines {
|
||||
if n == holder && b.moves(n) {
|
||||
out[n] = fmt.Sprintf("sending %s would replace the bus (%s %s → %s), which is a planned step: "+
|
||||
"`bus upgrade --why …` snapshots its streams first and checks them after (novox/hq ADR 0236)",
|
||||
n, b.module, short(orNotKnown(b.from[n])), short(b.to))
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// busCommand is `bus` — what a bus upgrade would do and how the last went — and `bus upgrade`.
|
||||
func busCommand(ctx context.Context, args []string) error {
|
||||
sub := ""
|
||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
sub, args = args[0], args[1:]
|
||||
}
|
||||
// The view's credential, a terminal line like a person's (bus_view.go).
|
||||
switch sub {
|
||||
case "view-credential":
|
||||
return busViewCredential(ctx, args)
|
||||
case "view-revoke":
|
||||
return busViewRevoke(ctx, args)
|
||||
}
|
||||
set := flag.NewFlagSet("bus", flag.ContinueOnError)
|
||||
snapshot := set.String("snapshot-taken", "", "where the streams' snapshot a person took is, while the mesh takes none itself")
|
||||
reversible := set.Bool("reversible", false, "the new version can be undone by putting the old one back")
|
||||
irreversible := set.Bool("irreversible", false, "the new version cannot be undone by putting the old one back, "+
|
||||
"and this is the person's explicit word that it runs anyway")
|
||||
why := addHandActFlags(set)
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New(busUsage)
|
||||
}
|
||||
switch sub {
|
||||
case "":
|
||||
return busStatus(ctx)
|
||||
case "upgrade":
|
||||
default:
|
||||
return fmt.Errorf("bus says what a bus upgrade would do, or `bus upgrade`, `bus view-credential`, `bus view-revoke` — not %q", sub)
|
||||
}
|
||||
// Everything refused before anything is done.
|
||||
if err := why.require("bus upgrade"); err != nil {
|
||||
return err
|
||||
}
|
||||
if *reversible == *irreversible {
|
||||
return errors.New("bus upgrade says, before it starts, whether the new version can be undone by putting the " +
|
||||
"old one back: --reversible, or --irreversible as your explicit word that it runs anyway (to-be 45 §8). " +
|
||||
"Nothing was done")
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
b, err := pendingBus(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if b.module == "" {
|
||||
return errors.New("the mesh holds no module that provides its bus: there is nothing to upgrade")
|
||||
}
|
||||
var moving []string
|
||||
for _, n := range b.machines {
|
||||
if b.moves(n) {
|
||||
moving = append(moving, n)
|
||||
}
|
||||
}
|
||||
if len(moving) == 0 {
|
||||
fmt.Printf("every machine running %s runs the build the mesh holds (%s): nothing to upgrade\n", b.module, short(b.to))
|
||||
return nil
|
||||
}
|
||||
where := strings.TrimSpace(*snapshot)
|
||||
if where == "" {
|
||||
for _, n := range moving {
|
||||
fmt.Printf("snapshotting the bus's streams on %s first (its backup holder, ADR 0235)…\n", n)
|
||||
if where, err = takeBusSnapshot(ctx, b.module, n); err != nil {
|
||||
return fmt.Errorf("the streams could not be snapshotted, so the bus is not replaced: %w — a snapshot "+
|
||||
"taken by hand is said with --snapshot-taken <where>", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
from := map[string]bool{}
|
||||
for _, n := range moving {
|
||||
from[orNotKnown(b.from[n])] = true
|
||||
}
|
||||
step, err := inv.StartBusStep(ctx, inventory.BusStep{Module: b.module, Machines: moving,
|
||||
From: strings.Join(sortedKeys(from), ", "), To: b.to, Snapshot: where, Reversible: *reversible,
|
||||
By: link.Caller(), Why: strings.TrimSpace(*why.why)})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cause := "bus-upgrade"
|
||||
if strings.TrimSpace(*why.cause) == "" {
|
||||
why.cause = &cause
|
||||
}
|
||||
why.record(ctx, "bus upgrade", append([]string{b.module}, moving...))
|
||||
fmt.Printf("bus upgrade %d: %s %s → %s on %s; streams snapshotted at %s; %s\n", step.ID, b.module, step.From,
|
||||
short(b.to), strings.Join(moving, ", "), where, map[bool]string{true: "reversible: putting the old build back undoes it",
|
||||
false: "NOT reversible: the snapshot is the only way back"}[*reversible])
|
||||
sent, err := sendRollout(withBusStep(withScope(ctx, sendScope{person: true})), open, moving)
|
||||
if err != nil {
|
||||
_ = inv.EndBusStep(ctx, step.ID, "failed", "the send was refused: "+err.Error())
|
||||
return fmt.Errorf("the bus's machine could not be sent its new build: %w — nothing was replaced", err)
|
||||
}
|
||||
fmt.Printf("sent %s; `bus-maintenance` is open until the bus answers healthy again — every stream, every durable "+
|
||||
"consumer, a round trip to the machines (H-bus) — within %s, or the step is said failed with its snapshot "+
|
||||
"as the way back. `bus` says how it went\n", strings.Join(sent, ", "), busStepBound)
|
||||
return nil
|
||||
}
|
||||
|
||||
// busStatus is `bus`: what an upgrade would do, and the last step.
|
||||
func busStatus(ctx context.Context) error {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
b, err := pendingBus(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if b.module == "" {
|
||||
fmt.Println("the mesh holds no module that provides its bus")
|
||||
} else {
|
||||
fmt.Printf("the bus is %s, built %s, on %s; never rolled out — a planned step (`bus upgrade`)\n", b.module,
|
||||
short(b.to), orNone(strings.Join(b.machines, ", ")))
|
||||
for _, n := range b.machines {
|
||||
state := "runs it"
|
||||
if b.moves(n) {
|
||||
state = "runs " + short(orNotKnown(b.from[n])) + ": `bus upgrade` replaces it"
|
||||
}
|
||||
fmt.Printf(" %-10s %s\n", n, state)
|
||||
}
|
||||
}
|
||||
fmt.Println(" `bus upgrade` has the bus machine's backup holder snapshot the streams first (ADR 0235)")
|
||||
s, found, err := open.inventory.LatestBusStep(ctx)
|
||||
if err != nil || !found {
|
||||
return err
|
||||
}
|
||||
state := "running since " + s.Started.Local().Format("2006-01-02 15:04")
|
||||
if s.Ended != nil {
|
||||
state = s.Outcome + " at " + s.Ended.Local().Format("2006-01-02 15:04")
|
||||
}
|
||||
fmt.Printf("last step %d: %s → %s on %s by %s (%s): %s; snapshot %s\n", s.ID, s.From, short(s.To),
|
||||
strings.Join(s.Machines, ", "), orNone(s.By), s.Why, state, s.Snapshot)
|
||||
if s.Found != "" {
|
||||
fmt.Printf(" %s\n", s.Found)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// probeBusStep is DB: a bus upgrade running is said as `bus-maintenance`; the bus healthy again after
|
||||
// the machines reported the new build ends it done; past its bound, unhealthy, it ends failed and is
|
||||
// said — urgent, with its snapshot — while the bus is still not healthy.
|
||||
func probeBusStep(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
inv := d.open.inventory
|
||||
s, found, err := inv.LatestBusStep(ctx)
|
||||
if err != nil || !found {
|
||||
return nil, err
|
||||
}
|
||||
if s.Ended != nil && s.Outcome != "failed" {
|
||||
return nil, nil
|
||||
}
|
||||
problems, err := busHealth(ctx, d)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
applied := true
|
||||
for _, r := range reports {
|
||||
for _, n := range s.Machines {
|
||||
if r.Node == n && (!r.Current || r.Outcome != inventory.OutcomeApplied || r.At == nil || r.At.Before(s.Started)) {
|
||||
applied = false
|
||||
problems = append(problems, n+" has not reported the new bus applied")
|
||||
}
|
||||
}
|
||||
}
|
||||
id := s.Module
|
||||
if s.Ended == nil {
|
||||
switch {
|
||||
case applied && len(problems) == 0:
|
||||
return nil, inv.EndBusStep(ctx, s.ID, "done", "the bus answered healthy after the upgrade")
|
||||
case time.Since(s.Started) > busStepBound:
|
||||
found := strings.Join(problems, "; ")
|
||||
if err := inv.EndBusStep(ctx, s.ID, "failed", found); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.Found = found
|
||||
default:
|
||||
return []conditions.Observation{{Scope: conditions.ScopeBus, ID: id, Token: "maintenance",
|
||||
Kind: kindBusMaintenance, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the bus is being upgraded (step %d, %s → %s on %s, by %s: %s); its snapshot is %s",
|
||||
s.ID, s.From, short(s.To), strings.Join(s.Machines, ", "), orNone(s.By), s.Why, s.Snapshot),
|
||||
Said: orNone(strings.Join(problems, "; "))}}, nil
|
||||
}
|
||||
}
|
||||
if len(problems) == 0 {
|
||||
return nil, nil // failed, and healthy since: nothing wrong now
|
||||
}
|
||||
way := "put the old build back"
|
||||
if !s.Reversible {
|
||||
way = "restore the snapshot"
|
||||
}
|
||||
return []conditions.Observation{{Scope: conditions.ScopeBus, ID: id, Token: "upgrade-failed",
|
||||
Kind: kindBusUpgradeFailed, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("the bus upgrade (step %d, %s → %s) did not end healthy within %s: %s — the way back is to %s (%s)",
|
||||
s.ID, s.From, short(s.To), busStepBound, strings.Join(problems, "; "), way, s.Snapshot)}}, nil
|
||||
}
|
||||
|
||||
func sortedKeys(set map[string]bool) []string {
|
||||
out := make([]string, 0, len(set))
|
||||
for k := range set {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// busSnapshotWithin is how long the bus machine's backup holder is given to take the bus's snapshot.
|
||||
var busSnapshotWithin = 15 * time.Minute
|
||||
|
||||
// snapshotTheBusNow asks the bus machine's backup holder to back the bus module up now — its dump is
|
||||
// the streams' snapshot (novox/hq ADR 0235) — and waits until it says a backup newer than the ask:
|
||||
// where the snapshot is, as a person reads it. The serving controller's connection, or one of its own.
|
||||
func snapshotTheBusNow(ctx context.Context, module, node string) (string, error) {
|
||||
var where string
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
asked := time.Now()
|
||||
answer, err := link.AskSeatTool(ctx, conn, catalogue.BackupSeat, "now", node,
|
||||
map[string]any{"module": module}, 30*time.Second)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s's backup holder was not asked to take the bus's snapshot: %w", node, err)
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return fmt.Errorf("%s's backup holder would not take the bus's snapshot: %s", node, answer.Error)
|
||||
}
|
||||
deadline := time.Now().Add(busSnapshotWithin)
|
||||
for {
|
||||
answer, err := link.AskSeatTool(ctx, conn, catalogue.BackupSeat, "backed-up", node, map[string]any{}, 10*time.Second)
|
||||
if err == nil && answer.Error == "" {
|
||||
if measured, err := readHolder(answer.Result); err == nil {
|
||||
for item, m := range measured[module] {
|
||||
if m.LastBackup != nil && m.LastBackup.After(asked) && m.Error == "" {
|
||||
where = fmt.Sprintf("%s's restore point of %s (%s) taken %s", node, module, item,
|
||||
m.LastBackup.UTC().Format(time.RFC3339))
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("%s's backup holder did not say the bus's snapshot was taken within %s; the bus is "+
|
||||
"not replaced", node, busSnapshotWithin)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(10 * time.Second):
|
||||
}
|
||||
}
|
||||
})
|
||||
return where, err
|
||||
}
|
||||
|
||||
// errBusWaits is a send refused because it would replace the bus outside its planned step.
|
||||
var errBusWaits = errors.New("a new bus build waits for its planned step")
|
||||
|
||||
type busStepKey struct{}
|
||||
|
||||
// withBusStep marks a send as the bus's planned step: the one send that may replace the bus.
|
||||
func withBusStep(ctx context.Context) context.Context {
|
||||
return context.WithValue(ctx, busStepKey{}, true)
|
||||
}
|
||||
|
||||
func busStepSending(ctx context.Context) bool { on, _ := ctx.Value(busStepKey{}).(bool); return on }
|
||||
@@ -0,0 +1,117 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The view's credential: the one read-only user a page in a browser connects to the bus as, over the
|
||||
// bus module's WebSocket listener (novox/hq research 036, gap G1; broker.KindView).
|
||||
//
|
||||
// **A terminal line, like a person's credential** (operator.go): printed once, never stored — the mesh
|
||||
// keeps a hash — and revoked by forgetting the row, which the next composition of the user list makes
|
||||
// real. There is one view; issuing it again rotates its password.
|
||||
const busUsage = "bus [upgrade --why … --reversible|--irreversible [--snapshot-taken <where>] | view-credential | view-revoke]"
|
||||
|
||||
// busWebSocketPort is the port the bus module's WebSocket listener is published on, mirrored from the
|
||||
// nats module's manifest (its `bus-websocket` opening), because the credential names where to connect
|
||||
// and the controller does not read the module's configuration. Reached across the overlay only: the
|
||||
// opening is from the mesh, and the mesh's filter admits nothing else.
|
||||
const busWebSocketPort = 4223
|
||||
|
||||
func busViewCredential(ctx context.Context, args []string) error {
|
||||
if len(args) != 0 {
|
||||
return errors.New("bus view-credential takes nothing: there is one view, and this prints its credential once")
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
// Refused here rather than at the next composition, where it would stop the whole file.
|
||||
if _, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindView, PasswordHash: "x"}); err != nil {
|
||||
return err
|
||||
}
|
||||
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: broker.ViewUser, Kind: inventory.BusView})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
where, err := broker.FromEnvironment()
|
||||
if err != nil && !errors.Is(err, broker.ErrNotConfigured) {
|
||||
return err
|
||||
}
|
||||
host := where.Address
|
||||
if h, _, err := net.SplitHostPort(where.Address); err == nil {
|
||||
host = h
|
||||
}
|
||||
websocket := ""
|
||||
if host != "" {
|
||||
websocket = "ws://" + net.JoinHostPort(host, strconv.Itoa(busWebSocketPort))
|
||||
}
|
||||
held, err := json.Marshal(struct {
|
||||
WebSocket string `json:"websocket,omitempty"`
|
||||
URL string `json:"url,omitempty"`
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
User string `json:"user"`
|
||||
Password string `json:"password"`
|
||||
InboxPrefix string `json:"inbox_prefix"`
|
||||
Hears []string `json:"hears"`
|
||||
Reads string `json:"reads"`
|
||||
HowToRead string `json:"how_to_read"`
|
||||
}{
|
||||
WebSocket: websocket, URL: "nats://" + where.Address, Fingerprint: where.Fingerprint,
|
||||
User: broker.ViewUser, Password: password,
|
||||
// The client must make its inboxes under the view's own prefix: its subscribe grant is
|
||||
// `_INBOX.view.>` and no wider (design 25 §4), and a client's default inbox is not under it.
|
||||
InboxPrefix: "_INBOX." + broker.ViewUser,
|
||||
Hears: broker.ViewHears, Reads: broker.ViewBucket,
|
||||
HowToRead: "direct reads only, no watch (a consumer is refused): list with a request to $JS.API.DIRECT.GET.KV_" +
|
||||
broker.ViewBucket + ` carrying {"multi_last":["$KV.` + broker.ViewBucket + `.>"]}, answered until a 204 status; ` +
|
||||
"read one key with $JS.API.DIRECT.GET.KV_" + broker.ViewBucket + ".$KV." + broker.ViewBucket + ".<number> " +
|
||||
"(nats.js: kvm.open(bucket, {allow_direct: true}), never create); re-read the key an event's number names",
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Printf("issued the view, which hears %s and reads the bucket %s, and nothing else\n",
|
||||
strings.Join(broker.ViewHears, ", "), broker.ViewBucket)
|
||||
fmt.Println(" this is the only time the credential is printed; the mesh keeps a hash")
|
||||
fmt.Println(" it works once the bus has been told, which is the next push to the machine holding mesh-broker —")
|
||||
fmt.Println(" and while a new build of the bus module waits for that machine, the next `bus upgrade` a person starts,")
|
||||
fmt.Println(" which is also what brings the WebSocket listener it connects through")
|
||||
fmt.Println()
|
||||
fmt.Println(string(held))
|
||||
return nil
|
||||
}
|
||||
|
||||
func busViewRevoke(ctx context.Context, args []string) error {
|
||||
if len(args) != 0 {
|
||||
return errors.New("bus view-revoke takes nothing: there is one view")
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
|
||||
if err := open.inventory.ForgetBusUser(ctx, broker.ViewUser); err != nil {
|
||||
return err
|
||||
}
|
||||
// **Revoked at the next composition, not now** — as a person is (operator revoke): the bus's users
|
||||
// are a file, and the credential stops working when the file no longer names it.
|
||||
fmt.Println("the view is forgotten, and its credential stops working at the next composition — " +
|
||||
"push the machine holding mesh-broker to make it so")
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A send held for the bus's planned step (novox/hq issue 336).
|
||||
//
|
||||
// **A wait only a person can end is said to that person at once.** No send replaces the bus but its planned
|
||||
// step (sendToEach, ADR 0236), and the step is a person's `bus upgrade`. So while a new bus build waits, every
|
||||
// walk whose tier sends to the bus's machine is refused, and tries again each tick. The refusal was kept only
|
||||
// as the walk's note; nothing was raised, and the operator found the hold by asking why a walk did not move.
|
||||
// Row S17 raises it on the first tick after the first refusal, in the bus's scope, for the operator: what
|
||||
// waits, behind which bus build, since when, and the verb. A walk held only by it is not late, so S3 leaves it
|
||||
// out, as it leaves out a walk under a paused build seat. It clears once the bus's machine has been sent the
|
||||
// build the mesh holds — the step was taken, and no send is refused for the bus any more — whatever the walks'
|
||||
// notes still say. Whether the new bus came up healthy is the step's own condition (probe DB).
|
||||
|
||||
// kindBusStepWaiting is S17's kind: bus.<module>.step-waiting.
|
||||
const kindBusStepWaiting = "bus-step-waiting"
|
||||
|
||||
// busFacts is a new bus build waiting for its planned step, and the sends held for it.
|
||||
type busFacts struct {
|
||||
module, to string
|
||||
// from is the build each machine of the bus runs; machines those whose bus the step would replace.
|
||||
from map[string]string
|
||||
machines []string
|
||||
waits []busWaitFacts
|
||||
}
|
||||
|
||||
// busWaitFacts is one walk whose send was refused because it would replace the bus.
|
||||
type busWaitFacts struct {
|
||||
plan, repository, commit string
|
||||
// modules are what its tier sends: what waits.
|
||||
modules []string
|
||||
// since is the first refusal: as this controller saw it, or, read back, the save that kept the refusal.
|
||||
since time.Time
|
||||
}
|
||||
|
||||
// busRefusedFirst is when this controller first saw each walk refused for the bus's step. The walk's note
|
||||
// keeps the refusal across a restart; this keeps its moment more exactly than the walk's last save.
|
||||
var busRefusedFirst = &firstSeen{at: map[string]time.Time{}}
|
||||
|
||||
type firstSeen struct {
|
||||
mu sync.Mutex
|
||||
at map[string]time.Time
|
||||
}
|
||||
|
||||
// mark keeps the first moment an id was seen.
|
||||
func (s *firstSeen) mark(id string, at time.Time) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if _, seen := s.at[id]; !seen {
|
||||
s.at[id] = at
|
||||
}
|
||||
}
|
||||
|
||||
// of is when an id was first seen; zero when it was not.
|
||||
func (s *firstSeen) of(id string) time.Time {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.at[id]
|
||||
}
|
||||
|
||||
// keepOnly forgets every id not given: a walk no longer held is not held since then.
|
||||
func (s *firstSeen) keepOnly(ids map[string]bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for id := range s.at {
|
||||
if !ids[id] {
|
||||
delete(s.at, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// refusedForTheBus is whether an error is the refusal of a send for the bus's planned step.
|
||||
func refusedForTheBus(err error) bool {
|
||||
return err != nil && strings.Contains(err.Error(), errBusWaits.Error())
|
||||
}
|
||||
|
||||
// busWaitsOf is the sends held for the bus's step, from the open walks: each walk not waiting for its
|
||||
// delivery's word whose note keeps a refusal for this very bus build, while the build would still replace
|
||||
// the bus on a machine. first is when this controller first saw a walk refused, zero when it did not.
|
||||
func busWaitsOf(plans []inventory.Plan, b busPending, first func(string) time.Time) busFacts {
|
||||
f := busFacts{module: b.module, to: b.to, from: b.from}
|
||||
for _, n := range b.machines {
|
||||
if b.moves(n) {
|
||||
f.machines = append(f.machines, n)
|
||||
}
|
||||
}
|
||||
if len(f.machines) == 0 {
|
||||
return f
|
||||
}
|
||||
for _, p := range plans {
|
||||
if !p.Open() || p.Waiting() || !strings.Contains(p.Note, errBusWaits.Error()) || !strings.Contains(p.Note, short(b.to)) {
|
||||
continue
|
||||
}
|
||||
since := p.Updated
|
||||
if seen := first(p.ID); !seen.IsZero() && (since.IsZero() || seen.Before(since)) {
|
||||
since = seen
|
||||
}
|
||||
var modules []string
|
||||
if p.Tier >= 0 && p.Tier < len(p.Tiers) {
|
||||
modules = append(modules, p.Tiers[p.Tier]...)
|
||||
} else {
|
||||
modules = planModules(p)
|
||||
}
|
||||
sort.Strings(modules)
|
||||
f.waits = append(f.waits, busWaitFacts{plan: p.ID, repository: p.Repository, commit: p.Commit, modules: modules,
|
||||
since: since})
|
||||
}
|
||||
sort.Slice(f.waits, func(i, j int) bool { return f.waits[i].since.Before(f.waits[j].since) })
|
||||
return f
|
||||
}
|
||||
|
||||
// heldByTheBus is the walks of a bus's facts, by id: what S3 leaves out.
|
||||
func (b busFacts) heldByTheBus() map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, w := range b.waits {
|
||||
out[w.plan] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// busUpgradeVerb is the call that ends the wait, as the summary names it: through the mesh MCP server, with
|
||||
// why, saying whether the new version can be undone ("reversible": "true") or not ("irreversible": "true").
|
||||
const busUpgradeVerb = "`mesh_call mesh-controller.bus {\"upgrade\": \"true\", \"why\": \"…\", \"reversible\" or \"irreversible\": \"true\"}`"
|
||||
|
||||
// watchBusWaits is S17: a send held for the bus's planned step, said at once to the operator.
|
||||
func watchBusWaits(f *signalFacts) []conditions.Observation {
|
||||
b := f.bus
|
||||
if len(b.waits) == 0 || len(b.machines) == 0 {
|
||||
return nil
|
||||
}
|
||||
since := b.waits[0].since
|
||||
var walks, what []string
|
||||
for _, w := range b.waits {
|
||||
walks = append(walks, fmt.Sprintf("%s (%s %s, tier: %s)", w.plan, w.repository, short(w.commit),
|
||||
strings.Join(w.modules, ", ")))
|
||||
what = append(what, deliveryWhat(w.modules, w.repository))
|
||||
}
|
||||
var from []string
|
||||
for _, n := range b.machines {
|
||||
from = append(from, short(orNotKnown(b.from[n])))
|
||||
}
|
||||
machines := namesWords(b.machines, 3)
|
||||
in := f.now.Sub(since)
|
||||
return []conditions.Observation{{Scope: conditions.ScopeBus, ID: b.module, Token: "step-waiting",
|
||||
Kind: kindBusStepWaiting, Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
|
||||
Machine: b.machines[0], Also: b.machines[1:],
|
||||
Summary: fmt.Sprintf("sends to %s wait for the bus's planned step: a new bus build (%s %s → %s) would replace "+
|
||||
"the bus there, which only a person's %s does; waiting since %s: %s", strings.Join(b.machines, ", "),
|
||||
b.module, strings.Join(sortedUnique(from), ", "), short(b.to), busUpgradeVerb,
|
||||
since.UTC().Format(time.RFC3339), strings.Join(walks, "; ")),
|
||||
Said: fmt.Sprintf("%d walk(s) refused since %s", len(b.waits), since.UTC().Format(time.RFC3339)),
|
||||
Headline: clipWords("Sends to "+machines+" wait for a bus upgrade", conditions.HeadlineMax),
|
||||
Explanation: clipWords(fmt.Sprintf("A new version of the mesh's message system is built, and only a person "+
|
||||
"installs it. Until then nothing else is sent to %s: %s waits, for %s so far.", machines,
|
||||
namesWords(sortedUnique(what), 3), humanDuration(in)), conditions.ExplanationMax),
|
||||
Needs: "start the bus upgrade " + FromMeshMCPServer,
|
||||
Resolved: "Resolved: the bus upgrade started, and sends go on"}}
|
||||
}
|
||||
|
||||
// sortedUnique is a list sorted, each once.
|
||||
func sortedUnique(xs []string) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, x := range xs {
|
||||
if !seen[x] {
|
||||
seen[x] = true
|
||||
out = append(out, x)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// clipWords keeps a plain sentence within a bound, at a word.
|
||||
func clipWords(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
cut := strings.LastIndex(s[:n-1], " ")
|
||||
if cut <= 0 {
|
||||
cut = n - 1
|
||||
}
|
||||
return s[:cut] + "…"
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq issue 336: a send held because it would replace the bus outside its planned step waits for a
|
||||
// person, so a person is told at once — what waits, behind which bus build, since when, and the verb that
|
||||
// ends it — and the wait is not read as a walk running late.
|
||||
|
||||
// aBusOnAnchor is a new bus build waiting for its step on anchor: nats 88135ad0 there, 32307bd1 held.
|
||||
func aBusOnAnchor() busPending {
|
||||
return busPending{module: "nats", machines: []string{"anchor"}, from: map[string]string{"anchor": "88135ad0aaaa"},
|
||||
to: "32307bd1bbbb", same: map[string]bool{}}
|
||||
}
|
||||
|
||||
// refusedNote is the note a walk keeps when its send was refused for the bus, as advanceHeld writes it.
|
||||
func refusedNote(b busPending, tier int) string {
|
||||
held := "sending anchor would replace the bus (nats " + short(b.from["anchor"]) + " → " + short(b.to) +
|
||||
"), which is a planned step: `bus upgrade --why …` snapshots its streams first and checks them after (novox/hq ADR 0236)"
|
||||
return "tier " + string(rune('0'+tier)) + ": " + errBusWaits.Error() + ": " + held + " — tried again"
|
||||
}
|
||||
|
||||
func TestASendHeldForTheBusStepIsFoundFromTheWalksItHolds(t *testing.T) {
|
||||
now := time.Date(2026, 10, 8, 18, 30, 0, 0, time.UTC)
|
||||
b := aBusOnAnchor()
|
||||
walk := func(id, repo, note string, updated time.Time) inventory.Plan {
|
||||
return inventory.Plan{ID: id, Repository: repo, Commit: "c0ffee001122", State: inventory.PlanRolling, Tier: 0,
|
||||
Tiers: [][]string{{"mesh-host"}}, Modules: map[string]*inventory.PlanModule{"mesh-host": {}}, Note: note,
|
||||
Updated: updated}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
bus busPending
|
||||
plans []inventory.Plan
|
||||
first map[string]time.Time
|
||||
want []string
|
||||
since time.Time
|
||||
}{
|
||||
{"a walk refused for the bus", b,
|
||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now.Add(-27*time.Minute))}, nil,
|
||||
[]string{"plan-1"}, now.Add(-27 * time.Minute)},
|
||||
{"refused earlier than its last save, as this controller saw it", b,
|
||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now.Add(-5*time.Minute))},
|
||||
map[string]time.Time{"plan-1": now.Add(-28 * time.Minute)}, []string{"plan-1"}, now.Add(-28 * time.Minute)},
|
||||
{"a walk refused for another reason", b,
|
||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", "tier 0: the build seat is paused — tried again", now)}, nil,
|
||||
nil, time.Time{}},
|
||||
{"refused for an older bus build than the one held now", func() busPending { o := b; o.to = "99999999cccc"; return o }(),
|
||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)}, nil, nil, time.Time{}},
|
||||
{"the bus already runs the build held: its step started", func() busPending {
|
||||
o := aBusOnAnchor()
|
||||
o.from = map[string]string{"anchor": o.to}
|
||||
return o
|
||||
}(), []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)}, nil, nil, time.Time{}},
|
||||
{"a walk waiting for its delivery's word asks no send", b, func() []inventory.Plan {
|
||||
p := walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)
|
||||
p.Delivery = &inventory.PlanDelivery{Awaits: "mesh-delivery"}
|
||||
return []inventory.Plan{p}
|
||||
}(), nil, nil, time.Time{}},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
got := busWaitsOf(c.plans, c.bus, func(id string) time.Time { return c.first[id] })
|
||||
var ids []string
|
||||
for _, w := range got.waits {
|
||||
ids = append(ids, w.plan)
|
||||
}
|
||||
if strings.Join(ids, ",") != strings.Join(c.want, ",") {
|
||||
t.Fatalf("held for the bus: %v, want %v", ids, c.want)
|
||||
}
|
||||
if len(c.want) > 0 {
|
||||
if !got.waits[0].since.Equal(c.since) {
|
||||
t.Errorf("waiting since %s, want %s", got.waits[0].since, c.since)
|
||||
}
|
||||
if strings.Join(got.machines, ",") != "anchor" || got.to != b.to || got.module != "nats" {
|
||||
t.Errorf("behind %+v", got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// **Said at once, not as lateness, and cleared when the step starts**: the walk held only for the bus raises
|
||||
// the bus's condition on the first tick, before any tier bound, naming what waits, the bus build from and to,
|
||||
// since when and `bus upgrade`, for the operator; S3 says nothing of it even past its bound; and the
|
||||
// condition clears once the bus's machine runs the new build.
|
||||
func TestASendHeldForTheBusStepIsSaidAtOnceAndNotAsLateness(t *testing.T) {
|
||||
now := time.Date(2026, 10, 8, 18, 30, 0, 0, time.UTC)
|
||||
b := aBusOnAnchor()
|
||||
held := func(entered time.Duration) *signalFacts {
|
||||
f := calm(now)
|
||||
f.plans = []planFacts{{id: "plan-1", repository: "novox/mesh-host", commit: "c0ffee00", tier: 0, tiers: 1,
|
||||
entered: now.Add(-entered), bound: 30 * time.Minute, waiting: refusedNote(b, 0), bus: true}}
|
||||
f.bus = busFacts{module: "nats", to: b.to, from: b.from, machines: []string{"anchor"},
|
||||
waits: []busWaitFacts{{plan: "plan-1", repository: "novox/mesh-host", commit: "c0ffee001122",
|
||||
modules: []string{"mesh-host"}, since: now.Add(-time.Minute)}}}
|
||||
return f
|
||||
}
|
||||
|
||||
f := held(time.Minute)
|
||||
got := watchBusWaits(f)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("a send held for the bus a minute ago raised %+v", got)
|
||||
}
|
||||
o := got[0]
|
||||
if o.Key() != "bus.nats.step-waiting" || o.Kind != kindBusStepWaiting || o.Resolver != conditions.ResolverOperator {
|
||||
t.Fatalf("raised %s (%s), resolver %q", o.Key(), o.Kind, o.Resolver)
|
||||
}
|
||||
for _, want := range []string{"anchor", "mesh-host", "88135ad0", "32307bd1", "mesh_call mesh-controller.bus",
|
||||
`"upgrade": "true"`, `"reversible"`, `"irreversible"`, "2026-10-08T18:29:00Z", "plan-1"} {
|
||||
if !strings.Contains(o.Summary, want) {
|
||||
t.Errorf("its summary does not say %q: %s", want, o.Summary)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(o.Explanation, "mesh-host") || !strings.Contains(o.Headline, "bus upgrade") || o.Needs == "" {
|
||||
t.Errorf("its words do not say what waits and what the operator does: %+v", o)
|
||||
}
|
||||
if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Explanation: o.Explanation,
|
||||
Resolved: o.Resolved, Needs: o.Needs}, "anchor"); !ok {
|
||||
t.Errorf("its words are not plain: %s", why)
|
||||
}
|
||||
|
||||
// Past S3's bound: still the bus's wait, never a stalled walk.
|
||||
late := held(45 * time.Minute)
|
||||
if s3 := watchPlans(late); len(s3) != 0 {
|
||||
t.Fatalf("a walk held only by the bus step was said stalled: %+v", s3)
|
||||
}
|
||||
// A walk held for something else past its bound is still stalled.
|
||||
other := held(45 * time.Minute)
|
||||
other.plans[0].bus = false
|
||||
if s3 := watchPlans(other); len(s3) != 1 {
|
||||
t.Fatalf("a walk held for something else past its bound raised %+v", s3)
|
||||
}
|
||||
|
||||
// Through the keeper: open at the first tick, cleared when the step started.
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: &conditions.Told{},
|
||||
Now: func() time.Time { return now }})
|
||||
defer k.Close(context.Background())
|
||||
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
||||
w.see(t.Context(), held(time.Minute))
|
||||
open, err := k.Open(t.Context())
|
||||
if err != nil || len(open) != 1 || open[0].Key != "bus.nats.step-waiting" {
|
||||
t.Fatalf("after the first tick, open: %+v (%v)", open, err)
|
||||
}
|
||||
started := held(time.Minute)
|
||||
started.bus = busFacts{module: "nats", to: b.to}
|
||||
started.plans[0].bus = false
|
||||
w.see(t.Context(), started)
|
||||
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||
t.Fatalf("the step started, and open: %+v", open)
|
||||
}
|
||||
}
|
||||
@@ -53,9 +53,26 @@ func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Ra
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// And the work queues of seats that name their caller or their kind, with each holder's worker
|
||||
// (novox/hq ADR 0259 §3): an ask queues until the router takes it, a channel's work until that kind
|
||||
// takes it.
|
||||
trafficStreams, trafficWorkers, err := seatTrafficObjects(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Every one tried, and every failure named: one module's consumer the bus refuses is no reason
|
||||
// the modules after it in the list hear nothing (novox/hq issue 208, where this runs on each send).
|
||||
var failed []error
|
||||
for _, s := range trafficStreams {
|
||||
if err := r.EnsureStream(s); err != nil {
|
||||
failed = append(failed, fmt.Errorf("the work queue %s: %w", s.Name, err))
|
||||
}
|
||||
}
|
||||
for _, c := range trafficWorkers {
|
||||
if err := r.EnsureConsumer(c); err != nil {
|
||||
failed = append(failed, fmt.Errorf("the worker %s on %s: %w", c.Name, c.Stream, err))
|
||||
}
|
||||
}
|
||||
for _, c := range consumers {
|
||||
if err := r.EnsureConsumer(c.Consumer); err != nil {
|
||||
failed = append(failed, fmt.Errorf("how %s on %s hears what it consumes: %w", c.Module, c.Node, err))
|
||||
@@ -130,6 +147,37 @@ func moduleConsumers(ctx context.Context, inv *inventory.Inventory) ([]broker.Mo
|
||||
return broker.ConsumersOf(users), nil
|
||||
}
|
||||
|
||||
// seatTrafficObjects is the work queues and workers of seats that name their caller or their kind, from
|
||||
// the records the user list is composed from.
|
||||
func seatTrafficObjects(ctx context.Context, inv *inventory.Inventory) ([]broker.Stream, []broker.Consumer, error) {
|
||||
records, err := inv.BusRecords(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
users, err := broker.Users(records)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
streams, workers := broker.SeatTrafficObjects(users)
|
||||
// And the queue of every such seat the catalogue declares, held or not: work queues from registration,
|
||||
// so what is submitted before a holder is assigned waits for it (the correctness review of 2026-10-08).
|
||||
declared, err := inv.DeclaredTrafficSeats(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
have := map[string]bool{}
|
||||
for _, s := range streams {
|
||||
have[s.Name] = true
|
||||
}
|
||||
for _, s := range broker.TrafficQueues(declared) {
|
||||
if !have[s.Name] {
|
||||
streams = append(streams, s)
|
||||
have[s.Name] = true
|
||||
}
|
||||
}
|
||||
return streams, workers, nil
|
||||
}
|
||||
|
||||
// moduleConsumerCount is how many modules hear what they consume, for the raise's one line.
|
||||
func moduleConsumerCount(ctx context.Context, inv *inventory.Inventory) (int, error) {
|
||||
consumers, err := moduleConsumers(ctx, inv)
|
||||
|
||||
@@ -2,7 +2,6 @@ package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -12,6 +11,8 @@ import (
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// novox/hq issue 208: the bus's objects a declaration implies are asserted whenever one is sent, not
|
||||
@@ -158,10 +159,7 @@ func TestASendThatCannotAssertTheBusSaysSoAndRaisesACondition(t *testing.T) {
|
||||
// Against a real bus, through the send's own grant: a module assigned after start has its consumer
|
||||
// once a declaration is sent, and a holder assigned after start its seat's worker.
|
||||
func TestNatsAModuleAssignedAfterStartGetsItsConsumerAtItsFirstSend(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
open := aMesh(t)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
|
||||
@@ -37,8 +37,6 @@ func TestAPushAnswersBeforeItSends(t *testing.T) {
|
||||
}{
|
||||
{"push", map[string]any{"node": "anchor", "why": "w"}, true},
|
||||
{"push", map[string]any{"why": "w"}, true},
|
||||
{"command", map[string]any{"command": "push anchor --why w"}, true},
|
||||
{"command", map[string]any{"command": "push --behind --why=w"}, true},
|
||||
{"command", map[string]any{"command": "builds"}, false},
|
||||
{"status", map[string]any{}, false},
|
||||
{"assign", map[string]any{"node": "anchor", "module": "m"}, false},
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The change plan (novox/hq ADR 0238): **one commit, one plan** — what a change does to the mesh, computed
|
||||
// from its diffset (a repository, the branch it merges into, the commit at hand) by the planner, never by
|
||||
// a mapping of its own. reachOfMerge answers what moves and what follows it; this adds where each module
|
||||
// goes — the deploy plan, machine by machine in the build plan's order — and what is not an ordinary
|
||||
// send. A pull request's check posts it with its verdict; the release a merge makes follows the same
|
||||
// planner, so the two can be compared.
|
||||
|
||||
// policyOf is a module's upgrade policy, as the controller holds it; false when it cannot be read.
|
||||
type policyOf func(module string) (inventory.Upgrade, bool)
|
||||
|
||||
// changePlanOf is the change plan of a reach: pure, so it is tested without a store.
|
||||
func changePlanOf(repository, base, head string, r mergeReach, entries []inventory.Entry, policy policyOf) link.ChangePlan {
|
||||
p := link.ChangePlan{Repository: repository, Base: base, Head: head, Moved: r.Moved(), Dependents: r.Dependents(),
|
||||
New: r.Added, Unread: r.Unread, Tiers: r.Plan.Tiers}
|
||||
byName := map[string]inventory.Entry{}
|
||||
for _, e := range entries {
|
||||
byName[e.Manifest.Module] = e
|
||||
}
|
||||
machines := map[string]*link.MachinePlan{}
|
||||
machine := func(name string) *link.MachinePlan {
|
||||
if machines[name] == nil {
|
||||
machines[name] = &link.MachinePlan{Machine: name}
|
||||
}
|
||||
return machines[name]
|
||||
}
|
||||
for _, tier := range r.Plan.Tiers {
|
||||
for _, name := range tier {
|
||||
e, held := byName[name]
|
||||
if !held {
|
||||
continue
|
||||
}
|
||||
u, known := policy(name)
|
||||
waits := known && !u.RollOut
|
||||
for _, on := range e.On {
|
||||
if waits {
|
||||
machine(on).Waits = append(machine(on).Waits, name)
|
||||
} else {
|
||||
machine(on).Receives = append(machine(on).Receives, name)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case (name == "nats" || catalogue.ProvidesBus(e.Manifest)) && len(e.On) > 0:
|
||||
// Said before the merge (novox/hq issue 336): a new bus build holds every send to the bus's machine
|
||||
// until a person takes the step, so merging it is a promise to take it.
|
||||
p.Steps = append(p.Steps, fmt.Sprintf("%s: %s is never sent by an ordinary send, and until %s runs, "+
|
||||
"nothing else is sent to %s either — unless the new build turns out the same as the one running there "+
|
||||
"(issue 280)", busUpgradeNeeded, name, busUpgradeVerb, strings.Join(e.On, ", ")))
|
||||
case name == "nats" || catalogue.ProvidesBus(e.Manifest):
|
||||
p.Steps = append(p.Steps, "a planned bus step: the bus is upgraded by `bus upgrade`, never by an ordinary send")
|
||||
case waits && len(e.On) > 0:
|
||||
p.Steps = append(p.Steps, fmt.Sprintf("%s waits for a person: its policy records (%s)", name,
|
||||
orNone(u.From)))
|
||||
}
|
||||
if len(e.Manifest.Provides) > 0 && len(e.On) > 0 {
|
||||
var offers []string
|
||||
for _, o := range e.Manifest.Provides {
|
||||
offers = append(offers, o.Name)
|
||||
}
|
||||
p.Steps = append(p.Steps, fmt.Sprintf("%s provides %s: its consumers are sent again after it",
|
||||
name, strings.Join(offers, ", ")))
|
||||
}
|
||||
if e.Manifest.Data != nil && len(e.On) > 0 {
|
||||
p.Steps = append(p.Steps, fmt.Sprintf("%s keeps data (ADR 0233): what it holds is backed up before it moves", name))
|
||||
}
|
||||
}
|
||||
}
|
||||
var names []string
|
||||
for name := range machines {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
p.Machines = append(p.Machines, *machines[name])
|
||||
}
|
||||
p.Summary = summaryOf(p)
|
||||
return p
|
||||
}
|
||||
|
||||
// busUpgradeNeeded is how a change plan says that merging it holds the bus's machine for a person's step.
|
||||
const busUpgradeNeeded = "merging this needs a person's bus upgrade"
|
||||
|
||||
// summaryOf is a change plan in one line: what it builds, where it goes, and whether the bus moves.
|
||||
func summaryOf(p link.ChangePlan) string {
|
||||
if len(p.Moved) == 0 && len(p.New) == 0 {
|
||||
return "builds nothing: the change touches no module of the mesh's graph"
|
||||
}
|
||||
var parts []string
|
||||
what := strings.Join(p.Moved, ", ")
|
||||
if len(p.Dependents) > 0 {
|
||||
what += fmt.Sprintf(" (+%d dependent(s))", len(p.Dependents))
|
||||
}
|
||||
if len(p.New) > 0 {
|
||||
if what != "" {
|
||||
what += ", "
|
||||
}
|
||||
what += "new: " + strings.Join(p.New, ", ")
|
||||
}
|
||||
var to []string
|
||||
for _, m := range p.Machines {
|
||||
if len(m.Receives) > 0 {
|
||||
to = append(to, m.Machine)
|
||||
}
|
||||
}
|
||||
if len(to) > 0 {
|
||||
parts = append(parts, "builds "+what+" → "+strings.Join(to, ", "))
|
||||
} else {
|
||||
parts = append(parts, "builds "+what+", sent nowhere")
|
||||
}
|
||||
bus := "no bus step"
|
||||
for _, s := range p.Steps {
|
||||
switch {
|
||||
case strings.HasPrefix(s, busUpgradeNeeded):
|
||||
bus = busUpgradeNeeded
|
||||
case strings.HasPrefix(s, "a planned bus step") && bus == "no bus step":
|
||||
bus = "a bus step"
|
||||
}
|
||||
}
|
||||
parts = append(parts, bus)
|
||||
waiting := 0
|
||||
for _, m := range p.Machines {
|
||||
waiting += len(m.Waits)
|
||||
}
|
||||
if waiting > 0 {
|
||||
parts = append(parts, fmt.Sprintf("%d wait(s) for a person", waiting))
|
||||
}
|
||||
return strings.Join(parts, "; ")
|
||||
}
|
||||
|
||||
// planText is a change plan as a person reads it, for the pull request's comment.
|
||||
func planText(p link.ChangePlan) string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "change plan of %s at %.8s into %s: %s\n", p.Repository, p.Head, p.Base, p.Summary)
|
||||
for i, tier := range p.Tiers {
|
||||
fmt.Fprintf(&b, " tier %d: %s\n", i, strings.Join(tier, ", "))
|
||||
}
|
||||
for _, m := range p.Machines {
|
||||
line := " " + m.Machine + ": "
|
||||
if len(m.Receives) > 0 {
|
||||
line += "receives " + strings.Join(m.Receives, ", ")
|
||||
}
|
||||
if len(m.Waits) > 0 {
|
||||
if len(m.Receives) > 0 {
|
||||
line += "; "
|
||||
}
|
||||
line += "waits for a person: " + strings.Join(m.Waits, ", ")
|
||||
}
|
||||
b.WriteString(line + "\n")
|
||||
}
|
||||
for _, s := range p.Steps {
|
||||
b.WriteString(" - " + s + "\n")
|
||||
}
|
||||
if len(p.Unread) > 0 {
|
||||
fmt.Fprintf(&b, " read by no module's build: %s\n", strings.Join(p.Unread, ", "))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **One commit, one change plan** (novox/hq ADR 0238): the planner's reach, laid out machine by machine in
|
||||
// the build plan's order, with what is not an ordinary send said — the bus step, a module that waits for a
|
||||
// person, a provider whose consumers follow it.
|
||||
func TestAChangePlanSaysWhatEachMachineReceives(t *testing.T) {
|
||||
const catalogue_ = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||
entry := func(name, path string, on ...string) inventory.Entry {
|
||||
e := fromRepo(name, catalogue_, path)
|
||||
e.Source.BuiltFrom = "old"
|
||||
e.On = on
|
||||
return e
|
||||
}
|
||||
nats := entry("nats", "modules/nats", "anchor")
|
||||
nats.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
|
||||
gitea := entry("gitea", "modules/gitea", "anchor")
|
||||
held := entry("photos", "modules/photos", "anchor", "laptop")
|
||||
tools := fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools")
|
||||
tools.On = []string{"anchor", "laptop"}
|
||||
entries := []inventory.Entry{nats, gitea, held, tools}
|
||||
edges := []inventory.Edge{{From: "node-tools", To: "nats", Kind: inventory.EdgeStandsOn}}
|
||||
policy := func(module string) (inventory.Upgrade, bool) {
|
||||
if module == "photos" {
|
||||
return inventory.Upgrade{RollOut: false, From: "a person"}, true
|
||||
}
|
||||
return inventory.Upgrade{RollOut: true}, true
|
||||
}
|
||||
plan := func(paths ...string) link.ChangePlan {
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: paths}
|
||||
return changePlanOf("novox/mesh-catalog", "main", "head", reachOfMerge(m, entries, nil, edges), entries, policy)
|
||||
}
|
||||
|
||||
p := plan("modules/gitea/index.ts")
|
||||
if p.Summary != "builds gitea → anchor; no bus step" {
|
||||
t.Errorf("one module's change reads %q", p.Summary)
|
||||
}
|
||||
|
||||
p = plan("modules/nats/Dockerfile", "modules/photos/x.js")
|
||||
if !strings.Contains(p.Summary, "needs a person's bus upgrade") || !strings.Contains(p.Summary, "2 wait(s) for a person") ||
|
||||
!strings.Contains(p.Summary, "(+1 dependent(s))") {
|
||||
t.Errorf("the bus and a held module read %q", p.Summary)
|
||||
}
|
||||
got := map[string]string{}
|
||||
for _, m := range p.Machines {
|
||||
got[m.Machine] = strings.Join(m.Receives, ",") + "|" + strings.Join(m.Waits, ",")
|
||||
}
|
||||
// The bus first, what stands on it after: the build plan's order, per machine.
|
||||
if got["anchor"] != "nats,node-tools|photos" || got["laptop"] != "node-tools|photos" {
|
||||
t.Errorf("the deploy plan reads %v", got)
|
||||
}
|
||||
text := strings.Join(p.Steps, "\n")
|
||||
// Said before the merge (novox/hq issue 336): merging it holds every send to the bus's machine until a
|
||||
// person runs the bus's step, and the verb that does.
|
||||
for _, want := range []string{"merging this needs a person's bus upgrade", "nothing else is sent to anchor",
|
||||
"mesh_call mesh-controller.bus", "the same as the one running there", "photos waits for a person",
|
||||
"nats provides mesh-bus"} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Errorf("the steps do not say %q:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
|
||||
p = plan("merge-check.sh")
|
||||
if !strings.HasPrefix(p.Summary, "builds nothing") || len(p.Machines) != 0 || strings.Join(p.Unread, ",") != "merge-check.sh" {
|
||||
t.Errorf("a root file's plan reads %+v", p)
|
||||
}
|
||||
if !strings.Contains(planText(p), "read by no module's build: merge-check.sh") {
|
||||
t.Errorf("the plan's text does not say why nothing is built:\n%s", planText(p))
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
@@ -31,6 +32,9 @@ import (
|
||||
// provision it wants that a manifest given here offers. An overflow is refused in the pull request
|
||||
// that introduces it — a new requirement, a lowered bound, a longer slug — instead of on the
|
||||
// provider's machine when a real machine's name first meets the module's.
|
||||
// SomeManifestsVar, set by the merge gate, says the manifests given are only some of their repository's.
|
||||
const SomeManifestsVar = "MESH_MODULE_CHECK_SOME"
|
||||
|
||||
func moduleCheck(paths []string, out io.Writer) error {
|
||||
return moduleCheckFor(paths, catalogue.DefaultLongestMachine, out)
|
||||
}
|
||||
@@ -64,6 +68,13 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
}
|
||||
// A definition names no installation (novox/hq ADR 0112, ADR 0155): judged here, in the
|
||||
// catalogue-wide test, and at registration, which refuses in the same words.
|
||||
if wrong := catalogue.TrustProblems(m); len(wrong) > 0 {
|
||||
for _, p := range wrong {
|
||||
fmt.Fprintf(out, "%s: %s\n", path, p)
|
||||
}
|
||||
failed += len(wrong)
|
||||
faulted[m.Module] = true
|
||||
}
|
||||
if named := catalogue.InstallationProblems(m); len(named) > 0 {
|
||||
for _, p := range named {
|
||||
fmt.Fprintf(out, "%s: %s\n", path, p)
|
||||
@@ -77,12 +88,24 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
// Between the manifests: a seat declared twice, a use of a seat nothing declares, a claim on
|
||||
// a seat that does not exist. Run only over what parsed, because a problem inside one manifest
|
||||
// has already been said and would be said again here in a worse form.
|
||||
//
|
||||
// **Over some of a repository's manifests, a seat none of them declares is a note** (novox/hq issue 364), as
|
||||
// this command's own word says above: the merge gate passes only the manifests a change touches, and says so
|
||||
// with SomeManifestsVar, so a module that uses or claims a seat another module declares (the operator
|
||||
// channel's, a channel bench) was refused there for a manifest it was not given. Given every manifest — the
|
||||
// catalogue's own check, and registration — it stays a refusal.
|
||||
some := os.Getenv(SomeManifestsVar) != ""
|
||||
problems := catalogue.CatalogueProblems(shelf)
|
||||
sort.Strings(problems)
|
||||
for _, p := range problems {
|
||||
if some && catalogue.IsUndeclaredSeat(p) {
|
||||
fmt.Fprintf(out, "note: %s among the manifests given; registration judges it against the whole catalogue, "+
|
||||
"and passing the declaring module's manifest too judges it here\n", p)
|
||||
continue
|
||||
}
|
||||
fmt.Fprintln(out, p)
|
||||
failed++
|
||||
}
|
||||
failed += len(problems)
|
||||
|
||||
// Between the manifests too: an identity against the bounds of the provisions it wants, which
|
||||
// only the provider's manifest states.
|
||||
@@ -93,11 +116,49 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
}
|
||||
failed += len(identities)
|
||||
|
||||
// And the data each module keeps (novox/hq ADR 0233): a provider that grants says what it keeps for
|
||||
// its consumers, a directory a container writes is declared, and no backup line is written by hand.
|
||||
data := catalogue.DataProblems(shelf)
|
||||
sort.Strings(data)
|
||||
for _, p := range data {
|
||||
fmt.Fprintln(out, p)
|
||||
}
|
||||
failed += len(data)
|
||||
|
||||
var names []string
|
||||
for name := range shelf {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
|
||||
// **Every long-running resource says how it is ready** (novox/hq ADR 0240 rule 8): warned until the
|
||||
// date, refused from it. The count is the catalogue's: its merge check keeps the number and lets a
|
||||
// change lower it, never raise it.
|
||||
undeclared := 0
|
||||
required := !checkNow().Before(catalogue.HealthRequiredFrom)
|
||||
for _, name := range names {
|
||||
missing := catalogue.Undeclared(shelf[name])
|
||||
undeclared += len(missing)
|
||||
if len(missing) == 0 {
|
||||
continue
|
||||
}
|
||||
if required {
|
||||
for _, id := range missing {
|
||||
fmt.Fprintf(out, "%s: %s stays up and does not say how it is ready: a long-running resource declares "+
|
||||
"health since %s (novox/hq ADR 0240 rule 8)\n", name, id, catalogue.HealthRequiredFrom.Format("2006-01-02"))
|
||||
}
|
||||
failed += len(missing)
|
||||
faulted[name] = true
|
||||
}
|
||||
}
|
||||
|
||||
// **A file that asks for a setting and does not say whether it is trusted counts as trusted** (novox/hq issue
|
||||
// 339): listed and counted, never refused, so an author can opt out a file nothing trusts.
|
||||
unsaid := 0
|
||||
for _, name := range names {
|
||||
unsaid += len(catalogue.UnsaidTrust(shelf[name]))
|
||||
}
|
||||
|
||||
for _, name := range names {
|
||||
m := shelf[name]
|
||||
if faulted[name] {
|
||||
@@ -121,8 +182,38 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
if len(m.Reads) > 0 {
|
||||
fmt.Fprintf(out, ", reads %s", strings.Join(m.Reads, ", "))
|
||||
}
|
||||
// The data it keeps, by class, so a reviewer sees what the mesh will protect and how.
|
||||
if items := m.DataItems(); len(items) > 0 {
|
||||
kept := make([]string, 0, len(items))
|
||||
for _, it := range items {
|
||||
kept = append(kept, it.ID+" ("+it.Class+")")
|
||||
}
|
||||
fmt.Fprintf(out, ", keeps %s", strings.Join(kept, ", "))
|
||||
}
|
||||
// How what it runs is ready (ADR 0240): each declared check, and what is judged by liveness alone.
|
||||
var checks []string
|
||||
for _, r := range m.Resources {
|
||||
if h, has, _ := catalogue.ReadHealth(r); has {
|
||||
checks = append(checks, fmt.Sprintf("%v by %s", r["id"], catalogue.HealthWords(h)))
|
||||
}
|
||||
}
|
||||
if len(checks) > 0 {
|
||||
fmt.Fprintf(out, ", ready: %s", strings.Join(checks, "; "))
|
||||
}
|
||||
if missing := catalogue.UnsaidTrust(m); len(missing) > 0 {
|
||||
fmt.Fprintf(out, "; WARNING: %s ask(s) for a setting and do(es) not say whether it is trusted, so it counts as "+
|
||||
"trusted: set at the terminal alone; say %q false where nothing trusts it (novox/hq issue 339)",
|
||||
strings.Join(missing, ", "), catalogue.TrustedField)
|
||||
}
|
||||
if missing := catalogue.Undeclared(m); len(missing) > 0 {
|
||||
fmt.Fprintf(out, "; WARNING: %s stay(s) up and say(s) not how it is ready — judged by liveness alone, "+
|
||||
"refused from %s (ADR 0240 rule 8)", strings.Join(missing, ", "), catalogue.HealthRequiredFrom.Format("2006-01-02"))
|
||||
}
|
||||
fmt.Fprintln(out)
|
||||
}
|
||||
// The count the catalogue keeps (ADR 0240 rule 8), in a line its merge check reads.
|
||||
fmt.Fprintf(out, "%s %d\n", UndeclaredHealthLine, undeclared)
|
||||
fmt.Fprintf(out, "%s %d\n", UnsaidTrustLine, unsaid)
|
||||
if failed > 0 {
|
||||
return fmt.Errorf("%d problem(s) in %d manifest(s)", failed, len(paths))
|
||||
}
|
||||
@@ -133,6 +224,17 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// UndeclaredHealthLine starts the line `module check` says the count of long-running resources without
|
||||
// `health` in, over the manifests given: the catalogue's merge check compares it with the number it keeps.
|
||||
const UndeclaredHealthLine = "long-running resources without health:"
|
||||
|
||||
// UnsaidTrustLine starts the line `module check` says the count of files that ask for a setting and do not say
|
||||
// whether it is trusted, and so count as trusted (novox/hq issue 339).
|
||||
const UnsaidTrustLine = "files asking for a setting without saying whether it is trusted:"
|
||||
|
||||
// checkNow is the clock `module check` judges the date by; a test sets it.
|
||||
var checkNow = time.Now
|
||||
|
||||
func joinInvokes(invokes []string) string {
|
||||
if len(invokes) == 1 && invokes[0] == "*" {
|
||||
return "every tool"
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// `check-here` is a pull request's merge check run on the machine at hand **exactly as the build seat
|
||||
// runs it** (novox/hq issue 286): the same code (builder.Check), the same ask the controller would make of
|
||||
// the seat — the gate's modules and judge by the planner's own answer over the facts snapshot, the
|
||||
// repositories beside it at the refs the snapshot says the seat clones them at — in the toolchain image
|
||||
// the mesh holds, as the user the build seat runs as, against a throwaway store and bus of the versions
|
||||
// the mesh runs.
|
||||
//
|
||||
// **The build seat is the reference.** A merge-check.sh that passed on an agent's machine failed on the
|
||||
// seat for three reasons that were each the agent's environment, never the change: a newer Go whose gofmt
|
||||
// lays a file out differently, a sibling checkout at the agent's feature branch where the seat had the
|
||||
// commit the mesh runs, and a different user. Run here, a check sees what the seat will.
|
||||
//
|
||||
// check-here [--tree <checkout>] [--base main] [--registry <artifact store>] [--forge <url of the owner>]
|
||||
// [--number <n>] [--user uid:gid] [--facts store|<file>] [--keep]
|
||||
//
|
||||
// The checkout's HEAD is what is checked, and it must be committed: the seat checks a commit, never a
|
||||
// working tree.
|
||||
func checkHereCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("check-here", flag.ContinueOnError)
|
||||
tree := set.String("tree", ".", "the change's checkout; its HEAD is checked")
|
||||
base := set.String("base", "main", "the branch the change would merge into")
|
||||
registry := set.String("registry", os.Getenv("MESH_REGISTRY"), "the artifact store holding the facts and the toolchains")
|
||||
forge := set.String("forge", "", "where the repositories beside it are cloned from, as <forge>/<repository>.git; "+
|
||||
"the checkout's origin without its own name when not given")
|
||||
number := set.Int("number", 0, "the pull request's number, when there is one")
|
||||
// The build seat's service runs as root, and its check containers run as the builder does.
|
||||
user := set.String("user", "0:0", "the user the check's containers run as: the build seat's")
|
||||
keep := set.Bool("keep", false, "keep the workspace afterwards")
|
||||
factsFrom := set.String("facts", "store", "the facts snapshot: `store`, the one the artifact store holds — "+
|
||||
"what the seat reads — or a file")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
}
|
||||
if *registry == "" {
|
||||
return errors.New("check-here reads the facts and the toolchains from the artifact store: --registry <host:port> or MESH_REGISTRY")
|
||||
}
|
||||
dir, err := filepath.Abs(*tree)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
git := func(args ...string) (string, error) {
|
||||
cmd := exec.CommandContext(ctx, "git", args...)
|
||||
cmd.Dir = dir
|
||||
out, err := cmd.Output()
|
||||
return strings.TrimSpace(string(out)), err
|
||||
}
|
||||
if dirty, err := git("status", "--porcelain", "--untracked-files=no"); err != nil {
|
||||
return fmt.Errorf("%s is not a checkout: %w", dir, err)
|
||||
} else if dirty != "" {
|
||||
return errors.New("the checkout has changes not committed: the build seat checks a commit, so commit first")
|
||||
}
|
||||
head, err := git("rev-parse", "HEAD")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
origin, err := git("remote", "get-url", "origin")
|
||||
if err != nil {
|
||||
return fmt.Errorf("the checkout has no origin to say which repository it is: %w", err)
|
||||
}
|
||||
owner, repo, prefix := ownerRepoOf(origin)
|
||||
if *forge == "" {
|
||||
*forge = prefix
|
||||
}
|
||||
if _, err := git("fetch", "--quiet", "origin", *base); err != nil {
|
||||
return fmt.Errorf("cannot fetch %s to say what the change touches: %w", *base, err)
|
||||
}
|
||||
// Without rename detection, so a file moved out of a build source is said under its old name as well:
|
||||
// its going is a change to the build that held it (novox/hq ADR 0267).
|
||||
changedText, err := git("diff", "--name-only", "--no-renames", "origin/"+*base+"...HEAD")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var paths, removed []string
|
||||
for _, p := range strings.Split(changedText, "\n") {
|
||||
if p = strings.TrimSpace(p); p == "" {
|
||||
continue
|
||||
}
|
||||
paths = append(paths, p)
|
||||
if _, err := os.Stat(filepath.Join(dir, p)); os.IsNotExist(err) {
|
||||
removed = append(removed, p)
|
||||
}
|
||||
}
|
||||
|
||||
_ = os.Setenv("MESH_REGISTRY", *registry)
|
||||
f, err := readFacts(ctx, *factsFrom)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the facts snapshot cannot be read: %w", err)
|
||||
}
|
||||
entries, read, edges, err := graphOfFacts(f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p := link.PullUpdated{Owner: owner, Repo: repo, Number: *number, Base: *base, Commit: head, Paths: paths,
|
||||
Removed: removed, ModuleDirs: moduleDirsIn(dir, paths), ModuleDirsSaid: true}
|
||||
scope := pullScope(p, entries, read, edges)
|
||||
_, scriptErr := os.Stat(filepath.Join(dir, builder.CheckScript))
|
||||
if !scope.gated() && !scope.Mesh {
|
||||
fmt.Printf("%s: %s, and the repository is not the mesh's: the build seat runs nothing for it\n",
|
||||
owner+"/"+repo, noModuleTouched)
|
||||
return nil
|
||||
}
|
||||
if !scope.gated() && scriptErr != nil {
|
||||
fmt.Printf("%s: %s; %s\n", owner+"/"+repo, noModuleTouched, noMergeCheck)
|
||||
return nil
|
||||
}
|
||||
|
||||
toolchains := map[string]string{}
|
||||
for language, reference := range f.Versions.Toolchains {
|
||||
toolchains[language] = catalogue.Rerouted(reference, *registry)
|
||||
}
|
||||
if len(toolchains) == 0 {
|
||||
return errors.New("the facts snapshot names no toolchain: it was taken by a controller from before " +
|
||||
"issue 286, and the seat's toolchain cannot be known here")
|
||||
}
|
||||
beside := map[string]builder.Beside{}
|
||||
for d, ref := range f.Beside {
|
||||
from := d
|
||||
if d == "mesh-controller-main" {
|
||||
from = "mesh-controller"
|
||||
}
|
||||
beside[d] = builder.Beside{Repository: strings.TrimSuffix(*forge, "/") + "/" + from + ".git", Ref: ref}
|
||||
}
|
||||
id := fmt.Sprintf("check-here-%d", time.Now().UnixNano())
|
||||
spec := builder.CheckSpec{ID: id, Repository: dir, Ref: head, Owner: owner, Repo: repo, Number: *number,
|
||||
Paths: paths, Beside: beside, Modules: scope.Modules, New: scope.New, Manifests: scope.Manifests,
|
||||
Base: *base, Judge: scope.Judge, Toolchain: toolchains["go"], Toolchains: toolchains, User: *user}
|
||||
|
||||
workspace, err := os.MkdirTemp("", "mesh-check-here-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !*keep {
|
||||
defer removeWorkspace(spec.Toolchain, workspace, *user)
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "checking %s/%s at %.8s as the build seat would, against the facts of %s, in %s\n",
|
||||
owner, repo, head, f.Taken.Format(time.RFC3339), workspace)
|
||||
v, err := builder.Check(ctx, builder.Command, spec, workspace, *registry, builder.GitCredential{},
|
||||
func(step, message string) {
|
||||
if step != "output" {
|
||||
fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("the check could not run — on the seat an error, never a pass: %w", err)
|
||||
}
|
||||
fmt.Println(v.Report)
|
||||
fmt.Println()
|
||||
fmt.Printf("mesh/merge-gate: %s — %s\n", strings.ToUpper(v.Gate.Verdict), v.Gate.Summary)
|
||||
if v.Repo != nil {
|
||||
fmt.Printf("mesh/repo-check: %s — %s\n", strings.ToUpper(v.Repo.Verdict), v.Repo.Summary)
|
||||
}
|
||||
for _, l := range []*builder.Layer{v.Gate, v.Repo} {
|
||||
if l != nil && l.Verdict != "pass" && l.Verdict != "warning" {
|
||||
return errors.New("the build seat would not pass this change")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ownerRepoOf reads owner, repository and the owner's URL from a remote: ssh://git@host:222/novox/mesh-host.git
|
||||
// → novox, mesh-host, ssh://git@host:222/novox.
|
||||
func ownerRepoOf(remote string) (string, string, string) {
|
||||
trimmed := strings.TrimSuffix(strings.TrimSuffix(remote, "/"), ".git")
|
||||
cut := strings.LastIndexAny(trimmed, "/:")
|
||||
if cut < 0 {
|
||||
return "", trimmed, ""
|
||||
}
|
||||
repo, prefix := trimmed[cut+1:], trimmed[:cut]
|
||||
owner := prefix
|
||||
if at := strings.LastIndexAny(prefix, "/:"); at >= 0 {
|
||||
owner = prefix[at+1:]
|
||||
}
|
||||
return owner, repo, prefix
|
||||
}
|
||||
|
||||
// removeWorkspace removes what the check left, written as the seat's user: by a container of that user
|
||||
// when it is not this one.
|
||||
func removeWorkspace(image, workspace, user string) {
|
||||
if image != "" && user != fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid()) {
|
||||
// Everything in it — the check's HOME is the workspace, so the toolchain's own files are there too.
|
||||
_ = exec.Command("docker", "run", "--rm", "--user", user, "--volume", workspace+":/workspace", image,
|
||||
"sh", "-c", "rm -rf /workspace/* /workspace/.[!.]*").Run()
|
||||
}
|
||||
_ = os.RemoveAll(workspace)
|
||||
}
|
||||
@@ -92,3 +92,34 @@ func TestRegistrationRefusesADefinitionNamingAnInstallation(t *testing.T) {
|
||||
t.Fatalf("a name declared on purpose passes; got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// **The controller's own manifest names every verb of its seat** (novox/hq ADR 0132): its tools lagged
|
||||
// the seat's verbs for weeks, and `module check` — the gate's first step for a change touching it —
|
||||
// refused it. Held here, so a verb added to the table without the manifest fails this repository's
|
||||
// own suite rather than its next pull request's gate.
|
||||
func TestTheControllersManifestServesEveryVerbOfItsSeat(t *testing.T) {
|
||||
var out strings.Builder
|
||||
if err := moduleCheck([]string{"../../module.json"}, &out); err != nil {
|
||||
t.Fatalf("the controller's own module.json fails module check: %v\n%s", err, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A module that uses a seat another module declares (novox/hq issue 364): refused over the whole catalogue when the
|
||||
// declarer is missing, a note when the gate says it gives only the manifests a change touches.
|
||||
func TestASeatAnotherModuleDeclaresIsANoteOverSomeManifests(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
user := filepath.Join(dir, "user.json")
|
||||
os.WriteFile(user, []byte(`{"module":"asker","version":"1","uses":["operator-channel"]}`), 0o600)
|
||||
var out bytes.Buffer
|
||||
if err := moduleCheck([]string{user}, &out); err == nil {
|
||||
t.Fatalf("a use of a seat nothing given declares passed the whole-catalogue check:\n%s", out.String())
|
||||
}
|
||||
t.Setenv(SomeManifestsVar, "1")
|
||||
out.Reset()
|
||||
if err := moduleCheck([]string{user}, &out); err != nil {
|
||||
t.Fatalf("over some manifests the use was refused:\n%s", out.String())
|
||||
}
|
||||
if !strings.Contains(out.String(), "note: asker uses the seat \"operator-channel\"") {
|
||||
t.Fatalf("the note was not said:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,400 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A pull request's merge check (novox/hq to-be 45 §9, ADR 0237 as amended 2026-10-06): the forge
|
||||
// announces every pull request's new head, the controller decides what is checked, asks the build seat to
|
||||
// check it, and says the verdict as `checked`, which the forge's holder sets as the pull request's
|
||||
// statuses. **Before merge, never after**: every check the mesh had ran after a merge, on a machine.
|
||||
//
|
||||
// **The mesh's module graph decides, not the repository.** The controller holds the graph — every module,
|
||||
// the repository and directory it is built from — and maps the pull request's changed paths onto it by
|
||||
// the planner's own answer (reachOfMerge, touchedBy — the one place a changed file is mapped onto modules,
|
||||
// for the merge handler, the release planner, the merge gate and this check; issue 280): **a changed file
|
||||
// touches exactly the modules whose build reads it** — a module's own directory (the whole repository for
|
||||
// one built from its root), or a repository its recipe packages. A file no build reads — a script at the
|
||||
// root, a README — touches no module. A directory the change adds a module.json in, which the graph does
|
||||
// not hold yet (said by the head, issue 278), is a new module and is checked too.
|
||||
//
|
||||
// - touches a module: the build seat runs **the gate** — `mesh/merge-gate`, the touched manifests, every
|
||||
// machine composed with the change, the replays — and the repository's own merge-check.sh beside it;
|
||||
// - touches none, in a repository that is the mesh's (it sources a module on some branch, or shares the
|
||||
// core's owner): the gate is a pass that says so — a fact, not a missing check — and the repository's
|
||||
// own merge-check.sh runs as `mesh/repo-check`, a warning when it has none;
|
||||
// - touches none, anywhere else: the gate is a pass that says so, and nothing more is said.
|
||||
//
|
||||
// What is checked is decided here and run there (internal/builder/check.go).
|
||||
|
||||
// checkTimeout is how long one check may run on the build seat. Said here so the ask's watchdog (S6)
|
||||
// and the builder agree on what late means.
|
||||
const checkTimeout = 45 * time.Minute
|
||||
|
||||
// noModuleTouched is the gate's word for a change that touches nothing of the graph.
|
||||
const noModuleTouched = "the change touches no module of the mesh's graph"
|
||||
|
||||
// noMergeCheck is the repository layer's word for a repository of the mesh with no merge-check.sh.
|
||||
const noMergeCheck = "the repository declares no merge-check.sh: none of its own tests run before it merges"
|
||||
|
||||
// coreModules are the modules whose repositories are the mesh's core, by the directory a check finds
|
||||
// each beside it — and whose owner is the mesh's own.
|
||||
var coreModules = map[string]string{"mesh-controller": "mesh-controller", "mesh-host": "mesh-host",
|
||||
"node-tools": "mesh-tools", "nats": "mesh-catalog"}
|
||||
|
||||
// checkScope is what a pull request reaches of the mesh's graph, as the planner reckons it.
|
||||
type checkScope struct {
|
||||
// Modules are the modules a merge of the change would move itself — built from the repository into
|
||||
// the pull request's base and reading a changed file, or packaging the repository's source — and
|
||||
// Dependents those the plan would build after them; New the directories it adds a module in.
|
||||
Modules []string
|
||||
Dependents []string
|
||||
New []string
|
||||
// Manifests are the moved modules' and the new ones' manifests in the change's tree.
|
||||
Manifests []string
|
||||
// Width is how many modules a merge would build, in how many tiers; Unread the changed files no
|
||||
// module's build reads.
|
||||
Width, Tiers int
|
||||
Unread []string
|
||||
// Mesh says the repository is the mesh's: modules are built from it on some branch, its owner is the
|
||||
// core's, or the change adds a module to it.
|
||||
Mesh bool
|
||||
// Judge is who judges the gate (link.JudgeSelf, link.JudgeValidator, or the running controller).
|
||||
Judge string
|
||||
// From is a module built from the repository, for how the mesh clones it; nil when none is.
|
||||
From *inventory.Entry
|
||||
// Reach is the planner's whole answer, which the change plan is made from.
|
||||
Reach mergeReach
|
||||
}
|
||||
|
||||
func (s checkScope) gated() bool { return len(s.Modules)+len(s.New) > 0 }
|
||||
|
||||
// pullScope is what a pull request reaches: **the planner's own answer** (reachOfMerge), asked as if the
|
||||
// head were merged into the base — never a mapping of its own, so a change to what a merge touches
|
||||
// changes what is checked with it (novox/hq ADR 0238).
|
||||
func pullScope(p link.PullUpdated, entries []inventory.Entry, read map[string][]inventory.ReadRepository,
|
||||
edges []inventory.Edge) checkScope {
|
||||
m := link.SourceMoved{Owner: p.Owner, Repo: p.Repo, Base: p.Base, CloneURL: p.CloneURL, Commit: p.Commit,
|
||||
Paths: p.Paths, PathsTruncated: p.PathsTruncated, Removed: p.Removed, ModuleDirs: p.ModuleDirs,
|
||||
ModuleDirsSaid: p.ModuleDirsSaid}
|
||||
r := reachOfMerge(m, entries, read, edges)
|
||||
s := checkScope{Modules: r.Moved(), Dependents: r.Dependents(), New: r.Added, Unread: r.Unread,
|
||||
Width: len(r.Plan.Modules), Tiers: len(r.Plan.Tiers), Reach: r}
|
||||
for _, e := range append(append([]inventory.Entry{}, r.Touched...), r.Deleted...) {
|
||||
s.Manifests = append(s.Manifests, path.Join(strings.Trim(e.Source.Path, "/"), moduleManifestFile))
|
||||
switch e.Manifest.Module {
|
||||
case "mesh-controller":
|
||||
s.Judge = link.JudgeSelf
|
||||
case "mesh-host":
|
||||
if s.Judge == "" {
|
||||
s.Judge = link.JudgeValidator
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, d := range r.Added {
|
||||
s.Manifests = append(s.Manifests, path.Join(d, moduleManifestFile))
|
||||
}
|
||||
sort.Strings(s.Manifests)
|
||||
s.Manifests = slices.Compact(s.Manifests)
|
||||
|
||||
// Whose repository it is, for how it is cloned and whether its own check is the mesh's to run.
|
||||
owners := map[string]bool{}
|
||||
for i, e := range entries {
|
||||
if e.Provided {
|
||||
continue
|
||||
}
|
||||
if _, core := coreModules[e.Manifest.Module]; core {
|
||||
if owner := sourceOwner(e.Source.Repository); owner != "" {
|
||||
owners[owner] = true
|
||||
}
|
||||
}
|
||||
if sameRepository(e.Source.Repository, m) && s.From == nil {
|
||||
s.From = &entries[i]
|
||||
}
|
||||
}
|
||||
s.Mesh = s.From != nil || owners[strings.ToLower(p.Owner)] || s.gated()
|
||||
return s
|
||||
}
|
||||
|
||||
// sourceOwner is the owner of a recorded repository, a path on the git seat or a URL: novox/mesh-host → novox.
|
||||
func sourceOwner(repository string) string {
|
||||
parts := strings.Split(strings.Trim(strings.TrimSuffix(repository, ".git"), "/"), "/")
|
||||
if len(parts) < 2 {
|
||||
return ""
|
||||
}
|
||||
return strings.ToLower(parts[len(parts)-2])
|
||||
}
|
||||
|
||||
// PullUpdated decides a pull request's merge check, and asks for it when there is something to run.
|
||||
func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
||||
inv := f.open.inventory
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
read, err := readForPlanning(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
edges, err := inv.Dependencies(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
scope := pullScope(p, entries, read, edges)
|
||||
// The change plan of the commit at hand (ADR 0238): what a merge of it would build and send, posted
|
||||
// with the verdict whatever the verdict is.
|
||||
plan := changePlanOf(p.Owner+"/"+p.Repo, p.Base, p.Commit, scope.Reach, entries, func(module string) (inventory.Upgrade, bool) {
|
||||
u, err := inv.UpgradeOf(ctx, module)
|
||||
return u, err == nil
|
||||
})
|
||||
fmt.Print(planText(plan))
|
||||
direct := link.Checked{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Commit: p.Commit,
|
||||
ID: link.NewBuildID(time.Now()), Verdict: "pass", Summary: noModuleTouched,
|
||||
Gate: &link.CheckLayer{Verdict: "pass", Summary: noModuleTouched}, Plan: &plan}
|
||||
switch {
|
||||
case !scope.gated() && !scope.Mesh:
|
||||
fmt.Printf("%s/%s#%d (%.8s): %s, and the repository is not the mesh's: said, nothing run\n",
|
||||
p.Owner, p.Repo, p.Number, p.Commit, noModuleTouched)
|
||||
sayChecked(ctx, direct)
|
||||
return nil
|
||||
case !scope.gated() && p.MergeCheckSaid && !p.MergeCheck:
|
||||
direct.RepoCheck = &link.CheckLayer{Verdict: "warning", Summary: noMergeCheck}
|
||||
fmt.Printf("%s/%s#%d (%.8s): %s; %s\n", p.Owner, p.Repo, p.Number, p.Commit, noModuleTouched, noMergeCheck)
|
||||
sayChecked(ctx, direct)
|
||||
return nil
|
||||
}
|
||||
request, err := checkRequestFor(ctx, f.open, p, scope, entries)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
request.Check.Plan = &plan
|
||||
seat := buildSeatHeld(ctx)
|
||||
ask, err := askOverOn(seat)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer ask.Close()
|
||||
if err := ask.Ask(ctx, request); err != nil {
|
||||
return err
|
||||
}
|
||||
what := "its own merge-check.sh alone: " + noModuleTouched
|
||||
if scope.gated() {
|
||||
what = fmt.Sprintf("the gate over %s (a merge would build %d module(s) in %d tier(s))",
|
||||
strings.Join(append(append([]string{}, scope.Modules...), prefixedAll("new:", scope.New)...), ", "),
|
||||
scope.Width, scope.Tiers)
|
||||
}
|
||||
fmt.Printf("%s/%s#%d (%.8s): asked %s to check it before it merges — %s — as %s\n", p.Owner, p.Repo, p.Number,
|
||||
p.Commit, seat, what, request.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkRequestFor is the ask for one pull request's head: the repository as the mesh clones it, the head,
|
||||
// and what is read beside it.
|
||||
func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scope checkScope,
|
||||
entries []inventory.Entry) (link.BuildRequest, error) {
|
||||
shelf := map[string]catalogue.Manifest{}
|
||||
for _, e := range entries {
|
||||
shelf[e.Manifest.Module] = e.Manifest
|
||||
}
|
||||
world, err := theRestOfTheMesh(ctx, open.inventory, shelf, "")
|
||||
if err != nil {
|
||||
return link.BuildRequest{}, err
|
||||
}
|
||||
clone := func(s inventory.Source) (string, error) {
|
||||
if s.Seat == "" {
|
||||
return s.Repository, nil
|
||||
}
|
||||
return clonedFromSeat(world, s.Seat, s.Repository)
|
||||
}
|
||||
// As the mesh clones a module built from it; a repository no module is built from, from the forge.
|
||||
source := inventory.Source{Seat: gitSeat, Repository: p.Owner + "/" + p.Repo}
|
||||
if scope.From != nil {
|
||||
source = scope.From.Source
|
||||
}
|
||||
repository, err := clone(source)
|
||||
if err != nil {
|
||||
return link.BuildRequest{}, err
|
||||
}
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return link.BuildRequest{}, err
|
||||
}
|
||||
// Beside it, at what the mesh runs: each core repository by the module the mesh builds from it.
|
||||
beside := map[string]link.CheckedOut{}
|
||||
for _, e := range entries {
|
||||
dir, core := coreModules[e.Manifest.Module]
|
||||
if !core || e.Provided || e.Source.Repository == "" {
|
||||
continue
|
||||
}
|
||||
url, err := clone(e.Source)
|
||||
if err != nil {
|
||||
return link.BuildRequest{}, err
|
||||
}
|
||||
refs := besideRefs(dir, current[e.Manifest.Module].Commit)
|
||||
beside[dir] = link.CheckedOut{Repository: url, Ref: refs[dir]}
|
||||
if dir == "mesh-controller" {
|
||||
beside["mesh-controller-main"] = link.CheckedOut{Repository: url, Ref: refs["mesh-controller-main"]}
|
||||
if e.Source.Seat != "" {
|
||||
if lab, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository,
|
||||
"mesh-lab")}); err == nil {
|
||||
beside["mesh-lab"] = link.CheckedOut{Repository: lab, Ref: refs["mesh-lab"]}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return link.BuildRequest{
|
||||
ID: link.NewBuildID(time.Now()),
|
||||
Repository: repository,
|
||||
Ref: p.Commit,
|
||||
Held: heldBy(ctx),
|
||||
Seats: seatBases(ctx),
|
||||
Source: sourceOnSeat(source),
|
||||
Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base,
|
||||
Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents, New: scope.New,
|
||||
Manifests: scope.Manifests, Judge: scope.Judge},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// besideRefs is the ref each repository is cloned at beside a check, by the directory it is found under:
|
||||
// a core repository at the commit the mesh runs of the module built from it (`running`) — but the
|
||||
// catalogue, whose checkout beside is what tests read its files from, at its main, what the next merge
|
||||
// builds from; and beside the controller its main, for a judge the running controller predates, and the
|
||||
// lab's main, whose replays every check runs. **One rule, read by the check the controller asks for and by
|
||||
// the facts snapshot** (Facts.Beside), so a check run by hand clones what the build seat clones.
|
||||
func besideRefs(dir, running string) map[string]string {
|
||||
switch dir {
|
||||
case "mesh-catalog":
|
||||
return map[string]string{dir: "main"}
|
||||
case "mesh-controller":
|
||||
return map[string]string{dir: running, "mesh-controller-main": "main", "mesh-lab": "main"}
|
||||
}
|
||||
return map[string]string{dir: running}
|
||||
}
|
||||
|
||||
// siblingOf is another repository of the same owner: novox/mesh-controller → novox/mesh-lab.
|
||||
func siblingOf(repository, name string) string {
|
||||
if cut := strings.LastIndex(repository, "/"); cut >= 0 {
|
||||
return repository[:cut+1] + name
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// sourceOnSeat is a source's seat form, nil for one on no seat.
|
||||
func sourceOnSeat(s inventory.Source) *link.SourceOnSeat {
|
||||
if s.Seat == "" {
|
||||
return nil
|
||||
}
|
||||
return &link.SourceOnSeat{Seat: s.Seat, Repository: s.Repository}
|
||||
}
|
||||
|
||||
// checkEvents is where the serving controller says a check's verdict; nil in a command.
|
||||
var checkEvents link.Bus
|
||||
|
||||
// maxCheckReport is how much of a check's report travels in its verdict: enough for the failures and
|
||||
// the machines, never a log.
|
||||
const maxCheckReport = 60 << 10
|
||||
|
||||
// checked says a merge check's verdict as the controller's `checked`. Nothing is recorded or
|
||||
// registered: a check builds nothing (issue 240's rule for a dry run, kept for a check).
|
||||
func checked(ctx context.Context, result link.BuildResult) {
|
||||
sayChecked(ctx, checkedOf(result))
|
||||
}
|
||||
|
||||
// checkedOf is what a check's outcome says: each layer, and an error — never a pass — for a check that
|
||||
// could not run.
|
||||
func checkedOf(result link.BuildResult) link.Checked {
|
||||
c := link.Checked{ID: result.ID, On: result.On, Commit: result.Ref}
|
||||
if result.Checked != nil {
|
||||
c.Owner, c.Repo, c.Number = result.Checked.Owner, result.Checked.Repo, result.Checked.Number
|
||||
}
|
||||
switch {
|
||||
case result.Check != nil:
|
||||
c.Verdict, c.Summary, c.Report = result.Check.Verdict, result.Check.Summary, result.Check.Report
|
||||
c.Gate, c.RepoCheck = result.Check.Gate, result.Check.RepoCheck
|
||||
if c.Gate == nil {
|
||||
// A build seat from before the layers: its verdict is the gate's.
|
||||
c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
|
||||
}
|
||||
case result.Failed != "":
|
||||
// The check could not run: an error, never read as a pass — on both layers it was asked for.
|
||||
c.Verdict, c.Summary = "error", "the check could not run: "+firstLine(result.Failed)
|
||||
default:
|
||||
c.Verdict, c.Summary = "error", "the build seat answered the check with no verdict"
|
||||
}
|
||||
if c.Verdict == "" {
|
||||
c.Verdict = "error"
|
||||
}
|
||||
if result.Check == nil {
|
||||
c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
|
||||
c.RepoCheck = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
|
||||
}
|
||||
if result.Checked != nil && c.Gate != nil && len(c.Gate.Modules) == 0 {
|
||||
c.Gate.Modules = append(append([]string{}, result.Checked.Modules...), prefixedAll("new:", result.Checked.New)...)
|
||||
}
|
||||
if result.Checked != nil && c.Gate != nil && len(c.Gate.Dependents) == 0 {
|
||||
c.Gate.Dependents = result.Checked.Dependents
|
||||
}
|
||||
if result.Checked != nil {
|
||||
c.Plan = result.Checked.Plan
|
||||
// A delivery group's composed check (novox/hq ADR 0239): every head it judged, this one first.
|
||||
if g := result.Checked; g.Group != "" {
|
||||
c.Group = g.Group
|
||||
c.Members = append(c.Members, link.CheckedMember{Owner: g.Owner, Repo: g.Repo, Number: g.Number,
|
||||
Commit: result.Ref})
|
||||
for _, m := range g.Members {
|
||||
c.Members = append(c.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo, Number: m.Number,
|
||||
Commit: m.Ref})
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, l := range []*link.CheckLayer{c.Gate, c.RepoCheck} {
|
||||
if l != nil && l.Verdict == "" {
|
||||
l.Verdict = "error"
|
||||
}
|
||||
}
|
||||
if len(c.Report) > maxCheckReport {
|
||||
c.Report = "…" + c.Report[len(c.Report)-maxCheckReport:]
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func prefixedAll(prefix string, items []string) []string {
|
||||
out := make([]string, 0, len(items))
|
||||
for _, i := range items {
|
||||
out = append(out, prefix+i)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sayChecked says a merge check's verdict on the bus, where the forge's holder hears it.
|
||||
func sayChecked(ctx context.Context, c link.Checked) {
|
||||
repo := "none"
|
||||
if c.RepoCheck != nil {
|
||||
repo = strings.ToUpper(c.RepoCheck.Verdict) + " — " + c.RepoCheck.Summary
|
||||
}
|
||||
fmt.Printf("%s: %s/%s#%d at %.8s checked on %s: gate %s — %s; repository %s\n", c.ID, c.Owner, c.Repo, c.Number,
|
||||
c.Commit, orSomewhere(c.On), strings.ToUpper(c.Verdict), c.Summary, repo)
|
||||
if checkEvents == nil {
|
||||
return
|
||||
}
|
||||
body, err := json.Marshal(c)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
stating, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer stop()
|
||||
if err := checkEvents.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeyChecked, body); err != nil {
|
||||
fmt.Printf("%s: the verdict could not be said, so the pull request is not told it: %v\n", c.ID, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **The mesh's module graph decides what a pull request's check runs**, not the repository (novox/hq
|
||||
// ADR 0237 as amended): a change is mapped onto the graph by the rule a merge is (issue 278), the gate
|
||||
// runs when it touches a module — a new one included — and a repository that is the mesh's and touches
|
||||
// none still has its own merge-check.sh run.
|
||||
func TestThePullRequestIsMappedOntoTheModuleGraph(t *testing.T) {
|
||||
const catalogue = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||
const controller = "http://forge.internal:20000/novox/mesh-controller.git"
|
||||
const host = "http://forge.internal:20000/novox/mesh-host.git"
|
||||
photos := fromRepo("photos", "http://forge.internal:20000/novox/photos.git", "")
|
||||
photos.Source.Ref = "nox-mesh"
|
||||
snake := fromRepo("snake", "jschoubben/snake", "")
|
||||
snake.Source.Seat = "git"
|
||||
entries := []inventory.Entry{
|
||||
fromRepo("gitea", catalogue, "modules/gitea"),
|
||||
fromRepo("keycloak", catalogue, "modules/keycloak"),
|
||||
fromRepo("nats", catalogue, "modules/nats"),
|
||||
fromRepo("mesh-controller", controller, ""),
|
||||
fromRepo("mesh-host", host, ""),
|
||||
photos, snake,
|
||||
}
|
||||
pull := func(owner, repo, base string, paths []string, dirs ...string) link.PullUpdated {
|
||||
return link.PullUpdated{Owner: owner, Repo: repo, Base: base, Commit: "abc", Paths: paths,
|
||||
ModuleDirs: dirs, ModuleDirsSaid: true}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
p link.PullUpdated
|
||||
modules, new, manifest string
|
||||
mesh bool
|
||||
judge string
|
||||
}{
|
||||
{"one module's own files", pull("novox", "mesh-catalog", "main", []string{"modules/gitea/index.ts"}, "modules/gitea"),
|
||||
"gitea", "", "modules/gitea/module.json", true, ""},
|
||||
{"a file no module's build reads touches no module (issue 280)",
|
||||
pull("novox", "mesh-catalog", "main", []string{"merge-check.sh", "README.md"}), "", "", "", true, ""},
|
||||
{"files the announcer could not list: everything built from it",
|
||||
link.PullUpdated{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "abc", PathsTruncated: true,
|
||||
Paths: []string{"README.md"}}, "gitea,keycloak,nats", "",
|
||||
"modules/gitea/module.json,modules/keycloak/module.json,modules/nats/module.json", true, ""},
|
||||
{"a new module, said by the head", pull("novox", "mesh-catalog", "main",
|
||||
[]string{"modules/newmod/index.ts", "modules/newmod/module.json"}, "modules/newmod"),
|
||||
"", "modules/newmod", "modules/newmod/module.json", true, ""},
|
||||
{"the controller judges itself", pull("novox", "mesh-controller", "main", []string{"cmd/x.go"}),
|
||||
"mesh-controller", "", "module.json", true, "self"},
|
||||
{"the node-engine is judged with its validator", pull("novox", "mesh-host", "main", []string{"validate/v.go"}),
|
||||
"mesh-host", "", "module.json", true, "validator"},
|
||||
{"the core's owner, no module: the mesh's, its own check alone", pull("novox", "hq", "main", []string{"README.md"}),
|
||||
"", "", "", true, ""},
|
||||
{"a branch nothing is built from: the mesh's repository, no module", pull("novox", "photos", "master",
|
||||
[]string{"server/x.js"}), "", "", "", true, ""},
|
||||
{"the branch a module is built from", pull("novox", "photos", "nox-mesh", []string{"server/x.js"}),
|
||||
"photos", "", "module.json", true, ""},
|
||||
{"a repository on the forge's seat", pull("jschoubben", "snake", "main", []string{"index.html"}),
|
||||
"snake", "", "module.json", true, ""},
|
||||
{"a repository of nobody's, touching nothing", pull("someone", "dotfiles", "main", []string{"x"}),
|
||||
"", "", "", false, ""},
|
||||
{"a repository adding a module at its root", pull("someone", "newapp", "main", []string{"module.json", "x.js"}),
|
||||
"", ".", "module.json", true, ""},
|
||||
} {
|
||||
s := pullScope(c.p, entries, nil, nil)
|
||||
got := []string{strings.Join(s.Modules, ","), strings.Join(s.New, ","), strings.Join(s.Manifests, ",")}
|
||||
want := []string{c.modules, c.new, c.manifest}
|
||||
for i, what := range []string{"modules", "new", "manifests"} {
|
||||
if got[i] != want[i] {
|
||||
t.Errorf("%s: %s %q, wanted %q", c.what, what, got[i], want[i])
|
||||
}
|
||||
}
|
||||
if s.Mesh != c.mesh || s.Judge != c.judge {
|
||||
t.Errorf("%s: the mesh's %v judged by %q, wanted %v by %q", c.what, s.Mesh, s.Judge, c.mesh, c.judge)
|
||||
}
|
||||
if s.gated() != (c.modules != "" || c.new != "") {
|
||||
t.Errorf("%s: gated %v", c.what, s.gated())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A module whose build packages another repository's source is touched by a change to it.
|
||||
func TestAPullRequestTouchesWhatPackagesItsRepository(t *testing.T) {
|
||||
entries := []inventory.Entry{fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools")}
|
||||
read := map[string][]inventory.ReadRepository{"node-tools": {{Repository: "http://forge.internal:20000/novox/mesh-sdk.git"}}}
|
||||
s := pullScope(link.PullUpdated{Owner: "novox", Repo: "mesh-sdk", Base: "main", Commit: "abc", Paths: []string{"go/x.go"}}, entries, read, nil)
|
||||
if strings.Join(s.Modules, ",") != "node-tools" || len(s.Manifests) != 0 {
|
||||
t.Fatalf("a change to what node-tools packages touched %v (manifests %v)", s.Modules, s.Manifests)
|
||||
}
|
||||
}
|
||||
|
||||
// Each layer is said; a check that could not run is an error on both, never a pass; a build seat from
|
||||
// before the layers is read as the gate.
|
||||
func TestAChecksLayersAreEachSaidAndAnErrorIsNeverAPass(t *testing.T) {
|
||||
asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog", Number: 3, Modules: []string{"gitea"}}
|
||||
c := checkedOf(link.BuildResult{ID: "b", Ref: "abc", Checked: asked, Failed: "the facts snapshot cannot be read"})
|
||||
if c.Verdict != "error" || c.Gate == nil || c.Gate.Verdict != "error" || c.RepoCheck == nil || c.RepoCheck.Verdict != "error" {
|
||||
t.Fatalf("a check that could not run said %+v", c)
|
||||
}
|
||||
if strings.Join(c.Gate.Modules, ",") != "gitea" {
|
||||
t.Errorf("the gate names %v", c.Gate.Modules)
|
||||
}
|
||||
c = checkedOf(link.BuildResult{ID: "b", Ref: "abc", Checked: asked, Check: &link.CheckOutcome{Verdict: "warning", Summary: "wide"}})
|
||||
if c.Gate == nil || c.Gate.Verdict != "warning" || c.RepoCheck != nil {
|
||||
t.Fatalf("an outcome without layers said %+v", c)
|
||||
}
|
||||
c = checkedOf(link.BuildResult{ID: "b", Ref: "abc", Checked: asked, Check: &link.CheckOutcome{Verdict: "pass",
|
||||
Gate: &link.CheckLayer{Verdict: "pass"}, RepoCheck: &link.CheckLayer{Verdict: "fail", Summary: "its merge-check.sh failed"}}})
|
||||
if c.RepoCheck.Verdict != "fail" || c.Gate.Verdict != "pass" {
|
||||
t.Fatalf("the layers said %+v / %+v", c.Gate, c.RepoCheck)
|
||||
}
|
||||
}
|
||||
|
||||
// **The check asks the planner, never a mapping of its own** (novox/hq ADR 0238): what a pull request
|
||||
// reaches is reachOfMerge's answer — the same that plans a merge — so a file at the root touches no
|
||||
// module in both, and what stands on a touched module is named as its dependents in both.
|
||||
func TestAPullRequestReachesWhatAMergeOfItWouldPlan(t *testing.T) {
|
||||
const catalogue = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||
nats := fromRepo("nats", catalogue, "modules/nats")
|
||||
nats.Source.BuiltFrom = "old"
|
||||
gitea := fromRepo("gitea", catalogue, "modules/gitea")
|
||||
gitea.Source.BuiltFrom = "old"
|
||||
tools := fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools")
|
||||
entries := []inventory.Entry{nats, gitea, tools}
|
||||
// node-tools stands on the bus's image; a code edge, so a plan takes it along.
|
||||
edges := []inventory.Edge{{From: "node-tools", To: "nats", Kind: inventory.EdgeStandsOn}}
|
||||
read := map[string][]inventory.ReadRepository{}
|
||||
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
paths []string
|
||||
moved, dependents string
|
||||
width int
|
||||
unread string
|
||||
}{
|
||||
{"a file at the root, read by no build", []string{"merge-check.sh"}, "", "", 0, "merge-check.sh"},
|
||||
{"the bus's own directory", []string{"modules/nats/Dockerfile"}, "nats", "node-tools", 2, ""},
|
||||
{"both, and a README", []string{"modules/gitea/index.ts", "modules/nats/x", "README.md"}, "gitea,nats", "node-tools", 3, "README.md"},
|
||||
} {
|
||||
p := link.PullUpdated{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: c.paths}
|
||||
s := pullScope(p, entries, read, edges)
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: c.paths}
|
||||
r := reachOfMerge(m, entries, read, edges)
|
||||
if got := strings.Join(s.Modules, ","); got != c.moved || got != strings.Join(r.Moved(), ",") {
|
||||
t.Errorf("%s: the check reaches %q, the planner %v, wanted %q", c.what, got, r.Moved(), c.moved)
|
||||
}
|
||||
if got := strings.Join(s.Dependents, ","); got != c.dependents {
|
||||
t.Errorf("%s: dependents %q, wanted %q", c.what, got, c.dependents)
|
||||
}
|
||||
if s.Width != c.width || s.Width != len(r.Plan.Modules) {
|
||||
t.Errorf("%s: a merge would build %d, the planner says %d, wanted %d", c.what, s.Width, len(r.Plan.Modules), c.width)
|
||||
}
|
||||
if got := strings.Join(s.Unread, ","); got != c.unread {
|
||||
t.Errorf("%s: unread %q, wanted %q", c.what, got, c.unread)
|
||||
}
|
||||
}
|
||||
|
||||
// A repository whose build another module's recipe packages: that module is reached through the
|
||||
// planner's `read`, and what stands on it after it.
|
||||
read["gitea"] = []inventory.ReadRepository{{Repository: "http://forge.internal:20000/novox/mesh-sdk.git"}}
|
||||
s := pullScope(link.PullUpdated{Owner: "novox", Repo: "mesh-sdk", Base: "main", Commit: "head",
|
||||
Paths: []string{"src/index.ts"}}, entries, read, edges)
|
||||
if strings.Join(s.Modules, ",") != "gitea" || len(s.Manifests) != 0 {
|
||||
t.Fatalf("a change to the source gitea packages reaches %v (manifests %v)", s.Modules, s.Manifests)
|
||||
}
|
||||
}
|
||||
|
||||
// **Only a commit on the trunk is published** (novox/hq ADR 0238): a build of a commit off its repository's
|
||||
// default branch — a pull request's head, a branch built by hand, a rebuild or replay of one — is recorded
|
||||
// and never registered, so nothing can send it; a check or a dry run never is either.
|
||||
func TestABuildOffTheTrunkIsNeverPublished(t *testing.T) {
|
||||
on := link.BuildResult{ID: "b", Commit: "abc", Trunk: "main", OnTrunk: true}
|
||||
if err := publishable(on, ""); err != nil {
|
||||
t.Errorf("a build on the trunk was refused: %v", err)
|
||||
}
|
||||
off := link.BuildResult{ID: "b", Commit: "abc", Trunk: "main"}
|
||||
if err := publishable(off, ""); !errors.Is(err, errOffTheTrunk) || !strings.Contains(err.Error(), "main") {
|
||||
t.Errorf("a build off the trunk was let through: %v", err)
|
||||
}
|
||||
for _, r := range []link.BuildResult{
|
||||
{ID: "c", Trunk: "main", OnTrunk: true, Check: &link.CheckOutcome{Verdict: "pass"}},
|
||||
{ID: "d", Trunk: "main", OnTrunk: true, Checked: &link.CheckRequest{}},
|
||||
{ID: "e", Trunk: "main", OnTrunk: true, DryRun: true},
|
||||
} {
|
||||
if publishable(r, "") == nil {
|
||||
t.Errorf("%s, a check or a dry run, was publishable", r.ID)
|
||||
}
|
||||
}
|
||||
// A module that follows a branch other than the default: that branch is its trunk, and the default
|
||||
// is not.
|
||||
follows := link.BuildResult{ID: "g", Commit: "abc", Trunk: "master", Branches: []string{"nox-mesh"}}
|
||||
if err := publishable(follows, "nox-mesh"); err != nil {
|
||||
t.Errorf("a commit on the branch a module follows was refused: %v", err)
|
||||
}
|
||||
if err := publishable(link.BuildResult{ID: "h", Commit: "abc", Trunk: "master", OnTrunk: true,
|
||||
Branches: []string{"master"}}, "nox-mesh"); err == nil {
|
||||
t.Error("a commit on the default but not on the branch the module follows was published")
|
||||
}
|
||||
if err := publishable(link.BuildResult{ID: "i", Commit: "abc", Trunk: "main", Branches: []string{"feat/x"}}, ""); err == nil {
|
||||
t.Error("a feature branch's commit was published")
|
||||
}
|
||||
// A build seat older than the rule says nothing: let through and said, so the rule can reach the mesh.
|
||||
if err := publishable(link.BuildResult{ID: "f", Commit: "abc"}, ""); err != nil {
|
||||
t.Errorf("a build seat that said nothing was refused: %v", err)
|
||||
}
|
||||
}
|
||||
+176
-75
@@ -17,14 +17,173 @@ import (
|
||||
// moment the keep set moved, so it is the moment to say what may go — and it needs no timer of
|
||||
// its own. Reclaiming the bytes is the store's own nightly step; this only decides.
|
||||
//
|
||||
// **And run when a person asks** (novox/hq ADR 0251): the `collect` verb is the same sweep, with
|
||||
// bounds of its own. One implementation, so what a person runs is exactly what a build runs.
|
||||
//
|
||||
// Never fatal to a build. The build succeeded, the module is registered, and a store that could
|
||||
// not be reached is a thing to say rather than a reason to undo any of that. The next build asks
|
||||
// again, and the references it could not collect are still uncollected, so nothing is lost by
|
||||
// having failed.
|
||||
|
||||
// sweepBounds is how much one sweep may ask of the store.
|
||||
type sweepBounds struct {
|
||||
// most is how many artifacts it lets go of at most.
|
||||
most int
|
||||
// budget is the longest it keeps its caller waiting.
|
||||
budget time.Duration
|
||||
// platforms is whether an index is let go of with its own platform manifests (novox/hq ADR 0257).
|
||||
// Only a sweep a person confirmed, after its dry run listed what stays and names a platform: the
|
||||
// records cannot see an unrecorded index, or a copy in progress, naming the same platform, and the
|
||||
// store's tools can.
|
||||
platforms bool
|
||||
}
|
||||
|
||||
// afterBuild are the bounds of the sweep inside somebody's build.
|
||||
//
|
||||
// **Bounded, because this runs inside somebody's build.** The first sweep of a mesh that has never
|
||||
// collected has the whole history to get through, and a person waiting on `build` should not pay for
|
||||
// it. What is left over is simply offered again next time — builds are frequent, and the point is
|
||||
// that the store stops growing, not that it empties tonight.
|
||||
var afterBuild = sweepBounds{most: mostPerSweep, budget: sweepBudget}
|
||||
|
||||
// sweepResult is what one sweep did.
|
||||
type sweepResult struct {
|
||||
// HoldersWritten is how many kept archives the store now holds by a manifest it did not before.
|
||||
HoldersWritten int
|
||||
// Missing is how many kept archives the store does not have at all.
|
||||
Missing int
|
||||
// Held is whether every kept archive could be asked to be held; false stops the sweep before it
|
||||
// lets anything go.
|
||||
Held bool
|
||||
// LetGo is what the store let go of, and the record now says collected.
|
||||
LetGo []string
|
||||
// Skipped is how many references the sweep will not address (novox/hq issue 226).
|
||||
Skipped int
|
||||
// Indexes is how many eligible indexes a sweep a person did not confirm left for one that is: an
|
||||
// index goes only with its platform manifests (novox/hq ADR 0257 §4).
|
||||
Indexes int
|
||||
// Left is how many eligible references were not asked about this time.
|
||||
Left int
|
||||
// Bounded is whether it stopped at its bounds rather than at a refusal.
|
||||
Bounded bool
|
||||
// Stopped says why the sweep stopped before the end; empty when it did not.
|
||||
Stopped string
|
||||
}
|
||||
|
||||
// sweep holds every kept archive, then asks the store to let go of each reference in order, records
|
||||
// what it let go of, and says what it did.
|
||||
func sweep(ctx context.Context, inv *inventory.Inventory, store artifacts.Store, references, kept []string,
|
||||
bounds sweepBounds) sweepResult {
|
||||
within, stop := context.WithTimeout(ctx, bounds.budget)
|
||||
defer stop()
|
||||
var r sweepResult
|
||||
|
||||
// **Hold before letting go** (novox/hq issue 253). The store's collector keeps only what a
|
||||
// manifest names, and archives were published as bare blobs, so every kept archive is first
|
||||
// held by its manifest — which backfills the ones published before holders, a few at a time
|
||||
// as builds come, and is two HEADs each once done. A kept archive that could not be held stops
|
||||
// the sweep before it deletes anything: "everything kept is held" is the precondition the
|
||||
// collector's safety rests on, and a store refusing a hold would refuse the deletes too.
|
||||
// **An index goes with its platform manifests only when a person confirmed it** (novox/hq ADR
|
||||
// 0257), and a kept index keeps its own: which platform manifests the kept indexes name is read
|
||||
// from the store for every repository the sweep will touch, before the first delete. A single
|
||||
// read that fails stops the sweep before it deletes anything, as a kept archive that cannot be
|
||||
// held does. The sweep after a build leaves an eligible index for such a collect, below.
|
||||
store.Spare = nil
|
||||
if bounds.platforms {
|
||||
if inv == nil {
|
||||
r.Stopped = "no records to read which platform manifests a kept index names, so nothing was let go"
|
||||
r.Left = len(references)
|
||||
return r
|
||||
}
|
||||
images, err := inv.KeptImages(ctx)
|
||||
if err == nil {
|
||||
store.Spare, err = store.SpareKeptIndexes(within, images, references)
|
||||
}
|
||||
if err != nil {
|
||||
r.Stopped = fmt.Sprintf("which platform manifests a kept index names could not be read, so nothing was let go: %v", err)
|
||||
r.Left = len(references)
|
||||
return r
|
||||
}
|
||||
}
|
||||
|
||||
wrote, missing, err := holdKept(within, store, kept)
|
||||
r.HoldersWritten, r.Missing = wrote, missing
|
||||
if err != nil {
|
||||
r.Stopped = fmt.Sprintf("not every kept archive could be held, so nothing was let go: %v", err)
|
||||
r.Left = len(references)
|
||||
return r
|
||||
}
|
||||
r.Held = true
|
||||
|
||||
for i, reference := range references {
|
||||
if i >= bounds.most || within.Err() != nil {
|
||||
r.Left = len(references) - i
|
||||
r.Bounded = true
|
||||
if i >= bounds.most {
|
||||
r.Stopped = fmt.Sprintf("asked about %d, the most one sweep asks", bounds.most)
|
||||
} else {
|
||||
r.Stopped = fmt.Sprintf("ran out of its %s", bounds.budget)
|
||||
}
|
||||
break
|
||||
}
|
||||
if !bounds.platforms {
|
||||
// **An index waits for a confirmed collect** (novox/hq ADR 0257 §4). Let go of alone,
|
||||
// its platform manifests would stay in the store and the record would say collected, so
|
||||
// no later sweep would offer it again and they would stay for ever. Left eligible, the
|
||||
// next collect a person confirms takes it with them.
|
||||
index, err := store.IsIndex(within, reference)
|
||||
if err != nil && !errors.Is(err, artifacts.Gone) && !errors.Is(err, artifacts.ErrNotOurs) {
|
||||
r.Stopped = fmt.Sprintf("the artifact store could not say whether %s is an index, so nothing more was asked of it: %v", reference, err)
|
||||
r.Left = len(references) - i
|
||||
break
|
||||
}
|
||||
if index {
|
||||
r.Indexes++
|
||||
continue
|
||||
}
|
||||
}
|
||||
err := store.LetGo(within, reference)
|
||||
if err == nil || errors.Is(err, artifacts.Gone) {
|
||||
// Gone is the outcome wanted, already true. Recorded so the next sweep does not ask
|
||||
// again for ever.
|
||||
r.LetGo = append(r.LetGo, reference)
|
||||
continue
|
||||
}
|
||||
if errors.Is(err, artifacts.ErrNotOurs) {
|
||||
// **A fact about this record, so this record is skipped** (novox/hq issue 226). Not
|
||||
// marked collected — the mesh did not remove it and should not claim to — and not a
|
||||
// reason to stop, because the store was never asked. One of these at the front of
|
||||
// the oldest-first order ended every sweep until this.
|
||||
r.Skipped++
|
||||
if r.Skipped == 1 {
|
||||
fmt.Fprintf(os.Stderr, "the sweep will not address %s and went on: %v\n", reference, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
// **Stopped at the first refusal by the STORE, not pushed through.** A store that refuses
|
||||
// one refuses all of them — deletion disabled, the store down, the network gone — so
|
||||
// going on would be a hundred identical failures and a hundred identical log lines in
|
||||
// front of whoever was building something.
|
||||
r.Stopped = fmt.Sprintf("the artifact store kept %s, so nothing more was asked of it: %v", reference, err)
|
||||
r.Left = len(references) - i
|
||||
break
|
||||
}
|
||||
|
||||
if len(r.LetGo) > 0 {
|
||||
// Recorded outside `within`: the deletions happened, and losing the record of them because
|
||||
// the sweep ran out of budget would mean asking about them again for ever.
|
||||
if err := inv.MarkCollected(ctx, r.LetGo); err != nil {
|
||||
r.Stopped = fmt.Sprintf("the store let go of %d artifact(s) and the record of it did not keep: %v",
|
||||
len(r.LetGo), err)
|
||||
}
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// collect asks the store to let go of everything the mesh made and no longer keeps, and records
|
||||
// what it let go of. Says what it did and what it could not; returns nothing, because nothing
|
||||
// upstream should branch on it.
|
||||
// what it let go of — after a build. Says what it did and what it could not; returns nothing,
|
||||
// because nothing upstream should branch on it.
|
||||
func collect(ctx context.Context, inv *inventory.Inventory) {
|
||||
references, err := inv.ToCollect(ctx)
|
||||
if err != nil {
|
||||
@@ -55,86 +214,28 @@ func collect(ctx context.Context, inv *inventory.Inventory) {
|
||||
return
|
||||
}
|
||||
|
||||
// **Bounded, because this runs inside somebody's build.** The first sweep of a mesh that has
|
||||
// never collected has the whole history to get through, and a person waiting on `build` should
|
||||
// not pay for it. Two bounds, and what is left over is simply offered again next time —
|
||||
// builds are frequent, and the point is that the store stops growing, not that it empties
|
||||
// tonight.
|
||||
within, stop := context.WithTimeout(ctx, sweepBudget)
|
||||
defer stop()
|
||||
store := artifacts.Store{Address: address}
|
||||
|
||||
// **Hold before letting go** (novox/hq issue 253). The store's collector keeps only what a
|
||||
// manifest names, and archives were published as bare blobs, so every kept archive is first
|
||||
// held by its manifest — which backfills the ones published before holders, a few at a time
|
||||
// as builds come, and is two HEADs each once done. A kept archive that could not be held stops
|
||||
// the sweep before it deletes anything: "everything kept is held" is the precondition the
|
||||
// collector's safety rests on, and a store refusing a hold would refuse the deletes too.
|
||||
wrote, missing, err := holdKept(within, store, kept)
|
||||
if wrote > 0 {
|
||||
fmt.Fprintf(os.Stderr, "the artifact store now holds %d more kept archive(s) by a manifest\n", wrote)
|
||||
r := sweep(ctx, inv, artifacts.Store{Address: address}, references, kept, afterBuild)
|
||||
if r.HoldersWritten > 0 {
|
||||
fmt.Fprintf(os.Stderr, "the artifact store now holds %d more kept archive(s) by a manifest\n", r.HoldersWritten)
|
||||
}
|
||||
if missing > 0 {
|
||||
if r.Missing > 0 {
|
||||
fmt.Fprintf(os.Stderr, "%d archive(s) the mesh keeps are not in the artifact store at all; "+
|
||||
"`collection` lists them\n", missing)
|
||||
"`collection` lists them\n", r.Missing)
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "not every kept archive could be held, so nothing was let go: %v\n", err)
|
||||
return
|
||||
if len(r.LetGo) > 0 {
|
||||
fmt.Fprintf(os.Stderr, "the artifact store let go of %d artifact(s) the mesh no longer keeps\n", len(r.LetGo))
|
||||
}
|
||||
|
||||
var done []string
|
||||
var left, skipped int
|
||||
for i, reference := range references {
|
||||
if i >= mostPerSweep || within.Err() != nil {
|
||||
left = len(references) - i
|
||||
break
|
||||
}
|
||||
err := store.LetGo(within, reference)
|
||||
if err == nil || errors.Is(err, artifacts.Gone) {
|
||||
// Gone is the outcome wanted, already true. Recorded so the next sweep does not ask
|
||||
// again for ever.
|
||||
done = append(done, reference)
|
||||
continue
|
||||
}
|
||||
if errors.Is(err, artifacts.ErrNotOurs) {
|
||||
// **A fact about this record, so this record is skipped** (novox/hq issue 226). Not
|
||||
// marked collected — the mesh did not remove it and should not claim to — and not a
|
||||
// reason to stop, because the store was never asked. One of these at the front of
|
||||
// the oldest-first order ended every sweep until this.
|
||||
skipped++
|
||||
if skipped == 1 {
|
||||
fmt.Fprintf(os.Stderr,
|
||||
"the sweep will not address %s and went on: %v\n", reference, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
// **Stopped at the first refusal by the STORE, not pushed through.** A store that refuses
|
||||
// one refuses all of them — deletion disabled, the store down, the network gone — so
|
||||
// going on would be a hundred identical failures and a hundred identical log lines in
|
||||
// front of whoever was building something.
|
||||
fmt.Fprintf(os.Stderr, "the artifact store kept %s, so nothing more was asked of it: %v\n",
|
||||
reference, err)
|
||||
left = len(references) - i
|
||||
break
|
||||
if r.Stopped != "" && !r.Bounded {
|
||||
fmt.Fprintln(os.Stderr, r.Stopped)
|
||||
}
|
||||
|
||||
if len(done) > 0 {
|
||||
// Recorded outside `within`: the deletions happened, and losing the record of them because
|
||||
// the sweep ran out of budget would mean asking about them again for ever.
|
||||
if err := inv.MarkCollected(ctx, done); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "the store let go of %d artifact(s) and the record of it did not keep: %v\n",
|
||||
len(done), err)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "the artifact store let go of %d artifact(s) the mesh no longer keeps\n",
|
||||
len(done))
|
||||
if r.Held && r.Left > 0 {
|
||||
fmt.Fprintf(os.Stderr, "%d more to collect; the next build asks again\n", r.Left)
|
||||
}
|
||||
if left > 0 {
|
||||
fmt.Fprintf(os.Stderr, "%d more to collect; the next build asks again\n", left)
|
||||
if r.Skipped > 0 {
|
||||
fmt.Fprintf(os.Stderr, "%d artifact(s) the sweep will not address were skipped\n", r.Skipped)
|
||||
}
|
||||
if skipped > 0 {
|
||||
fmt.Fprintf(os.Stderr, "%d artifact(s) the sweep will not address were skipped\n", skipped)
|
||||
if r.Indexes > 0 {
|
||||
fmt.Fprintf(os.Stderr, "%d eligible index(es) wait for a collect a person confirms, which takes their platforms too\n", r.Indexes)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
@@ -46,7 +47,7 @@ func keeperOn(ctx context.Context, conn *nats.Conn) (*conditions.Keeper, error)
|
||||
Say: func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) },
|
||||
// What status leads with changed: composed again soon (a nudge outside the serving controller
|
||||
// does nothing).
|
||||
Changed: statusFrom.nudge,
|
||||
Changed: func() { statusFrom.nudge(); askerFrom.nudge() },
|
||||
// Written under the lease, carrying its epoch (novox/hq to-be 45 §6).
|
||||
Epoch: func() (uint64, error) { return theLease.epoch(context.WithoutCancel(ctx)) }}), nil
|
||||
}
|
||||
@@ -107,19 +108,20 @@ func conditionsCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
switch sub {
|
||||
case "list":
|
||||
return listConditions(ctx, args)
|
||||
return listConditions(ctx, args, os.Stdout)
|
||||
case "show":
|
||||
return showCondition(ctx, args)
|
||||
return showCondition(ctx, args, os.Stdout)
|
||||
case "silence":
|
||||
return silenceCondition(ctx, args)
|
||||
case "history":
|
||||
return conditionHistory(ctx, args)
|
||||
return conditionHistory(ctx, args, os.Stdout)
|
||||
}
|
||||
return errors.New(conditionsUsage)
|
||||
}
|
||||
|
||||
func listConditions(ctx context.Context, args []string) error {
|
||||
func listConditions(ctx context.Context, args []string, w io.Writer) error {
|
||||
set := flag.NewFlagSet("conditions", flag.ContinueOnError)
|
||||
usageTo(set, w)
|
||||
scope := set.String("scope", "", "only this scope: "+strings.Join(conditions.Scopes, ", "))
|
||||
severity := set.String("severity", "", "only urgent, or only warning")
|
||||
machine := set.String("machine", "", "only those about this machine")
|
||||
@@ -144,26 +146,64 @@ func listConditions(ctx context.Context, args []string) error {
|
||||
}
|
||||
}
|
||||
if *asJSON {
|
||||
if out == nil {
|
||||
out = []conditions.Condition{}
|
||||
}
|
||||
return printJSON(map[string]any{"conditions": out, "open": len(open),
|
||||
"note": "urgent first, then oldest first; a condition clears when observation says so, never by hand"})
|
||||
return printJSONTo(w, map[string]any{"conditions": inBrief(out), "open": len(open), "counted": counted(out),
|
||||
"note": "urgent first, then oldest first; a condition clears when observation says so, never by hand; " +
|
||||
"each with its newest evidence — `conditions key=<key>` gives one whole"})
|
||||
}
|
||||
if len(out) == 0 {
|
||||
if len(open) == 0 {
|
||||
fmt.Println("no open conditions")
|
||||
fmt.Fprintln(w, "no open conditions")
|
||||
} else {
|
||||
fmt.Printf("none of the %d open condition(s) is about that\n", len(open))
|
||||
fmt.Fprintf(w, "none of the %d open condition(s) is about that\n", len(open))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for _, line := range conditionLines(out, time.Now()) {
|
||||
fmt.Println(line)
|
||||
fmt.Fprintln(w, line)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// briefEvidence is how many observations a condition carries in a list of them: its newest. The store
|
||||
// keeps ten, and a list of hundreds of conditions with ten each was the larger half of an answer more
|
||||
// than the bus carries in one message (novox/hq issue 314). One condition whole is `conditions key=`.
|
||||
const briefEvidence = 1
|
||||
|
||||
// conditionInBrief is a condition as a list carries it: its newest evidence, with how much more there is
|
||||
// and where to read it. Everything else of it, so a reader of the list — the operator's channel among
|
||||
// them — reads the same condition it always did.
|
||||
type conditionInBrief struct {
|
||||
conditions.Condition
|
||||
// EvidenceKept is how many observations the condition keeps, when the list shows fewer.
|
||||
EvidenceKept int `json:"evidence-kept,omitempty"`
|
||||
More string `json:"more,omitempty"`
|
||||
}
|
||||
|
||||
// inBrief is a list of conditions in brief; never null.
|
||||
func inBrief(list []conditions.Condition) []conditionInBrief {
|
||||
out := make([]conditionInBrief, 0, len(list))
|
||||
for _, c := range list {
|
||||
b := conditionInBrief{Condition: c}
|
||||
if len(c.Evidence) > briefEvidence {
|
||||
b.EvidenceKept = len(c.Evidence)
|
||||
b.Evidence = c.Evidence[:briefEvidence:briefEvidence]
|
||||
b.More = c.Show()
|
||||
}
|
||||
out = append(out, b)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// counted is how many of a list each source raised of each kind — `D14 stalled: 689` — so a list of
|
||||
// hundreds says its shape before its lines.
|
||||
func counted(list []conditions.Condition) map[string]int {
|
||||
out := map[string]int{}
|
||||
for _, c := range list {
|
||||
out[c.Source+" "+c.Kind]++
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// concerns says whether a condition is about a machine: it names it, or its key does.
|
||||
func concerns(c conditions.Condition, machine string) bool {
|
||||
if c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) {
|
||||
@@ -195,8 +235,9 @@ func conditionLines(list []conditions.Condition, now time.Time) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
func showCondition(ctx context.Context, args []string) error {
|
||||
func showCondition(ctx context.Context, args []string, w io.Writer) error {
|
||||
set := flag.NewFlagSet("conditions show", flag.ContinueOnError)
|
||||
usageTo(set, w)
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
rest, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
@@ -228,34 +269,34 @@ func showCondition(ctx context.Context, args []string) error {
|
||||
"history --key %s` what became of it", key, key)
|
||||
}
|
||||
if *asJSON {
|
||||
return printJSON(c)
|
||||
return printJSONTo(w, c)
|
||||
}
|
||||
now := time.Now()
|
||||
fmt.Printf("%s %s\n %s\n\n", strings.ToUpper(string(c.Severity)), c.Key, c.Summary)
|
||||
fmt.Printf(" kind %s\n about %s %s", c.Kind, c.Subject.Scope, c.Subject.ID)
|
||||
fmt.Fprintf(w, "%s %s\n %s\n\n", strings.ToUpper(string(c.Severity)), c.Key, c.Summary)
|
||||
fmt.Fprintf(w, " kind %s\n about %s %s", c.Kind, c.Subject.Scope, c.Subject.ID)
|
||||
if c.Subject.Machine != "" {
|
||||
fmt.Printf(", on %s", c.Subject.Machine)
|
||||
fmt.Fprintf(w, ", on %s", c.Subject.Machine)
|
||||
}
|
||||
fmt.Printf("\n raised by %s\n since %s (%s ago), observed %d time(s), last %s ago\n",
|
||||
fmt.Fprintf(w, "\n raised by %s\n since %s (%s ago), observed %d time(s), last %s ago\n",
|
||||
c.Source, c.Raised.Local().Format("2006-01-02 15:04:05"), roughly(now.Sub(c.Raised)), c.Observations,
|
||||
now.Sub(c.LastObserved).Round(time.Second))
|
||||
if c.Count > 1 {
|
||||
fmt.Printf(" raised %d times, each within ten minutes of clearing\n", c.Count)
|
||||
fmt.Fprintf(w, " raised %d times, each within ten minutes of clearing\n", c.Count)
|
||||
}
|
||||
fmt.Printf(" resolved by %s\n", resolverWords(c.Resolver))
|
||||
fmt.Fprintf(w, " resolved by %s\n", resolverWords(c.Resolver))
|
||||
if c.Silenced != nil {
|
||||
fmt.Printf(" silenced until %s by %s: %s\n", c.Silenced.Until.Local().Format("2006-01-02 15:04"),
|
||||
fmt.Fprintf(w, " silenced until %s by %s: %s\n", c.Silenced.Until.Local().Format("2006-01-02 15:04"),
|
||||
c.Silenced.By, c.Silenced.Why)
|
||||
}
|
||||
if len(c.Tried) > 0 {
|
||||
fmt.Println("\n tried:")
|
||||
fmt.Fprintln(w, "\n tried:")
|
||||
for _, t := range c.Tried {
|
||||
fmt.Printf(" %s %s — %s: %s\n", t.At.Local().Format("2006-01-02 15:04"), orHealer(t.By), t.What, t.Outcome)
|
||||
fmt.Fprintf(w, " %s %s — %s: %s\n", t.At.Local().Format("2006-01-02 15:04"), orHealer(t.By), t.What, t.Outcome)
|
||||
}
|
||||
}
|
||||
fmt.Println("\n evidence, newest first:")
|
||||
fmt.Fprintln(w, "\n evidence, newest first:")
|
||||
for _, e := range c.Evidence {
|
||||
fmt.Printf(" %s %s\n", e.At.Local().Format("2006-01-02 15:04:05"), e.Said)
|
||||
fmt.Fprintf(w, " %s %s\n", e.At.Local().Format("2006-01-02 15:04:05"), e.Said)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -340,8 +381,9 @@ func parseFor(s string) (time.Duration, error) {
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func conditionHistory(ctx context.Context, args []string) error {
|
||||
func conditionHistory(ctx context.Context, args []string, w io.Writer) error {
|
||||
set := flag.NewFlagSet("conditions history", flag.ContinueOnError)
|
||||
usageTo(set, w)
|
||||
days := set.Int("days", 7, "how many days back, at most 90")
|
||||
key := set.String("key", "", "only this condition")
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
@@ -382,10 +424,10 @@ func conditionHistory(ctx context.Context, args []string) error {
|
||||
if out == nil {
|
||||
out = []conditions.Event{}
|
||||
}
|
||||
return printJSON(map[string]any{"history": out, "days": *days})
|
||||
return printJSONTo(w, map[string]any{"history": out, "days": *days})
|
||||
}
|
||||
if len(out) == 0 {
|
||||
fmt.Printf("nothing was raised, changed or cleared in the last %d day(s)\n", *days)
|
||||
fmt.Fprintf(w, "nothing was raised, changed or cleared in the last %d day(s)\n", *days)
|
||||
return nil
|
||||
}
|
||||
for _, e := range out {
|
||||
@@ -402,7 +444,7 @@ func conditionHistory(ctx context.Context, args []string) error {
|
||||
line += " — " + e.Why
|
||||
}
|
||||
}
|
||||
fmt.Println(line)
|
||||
fmt.Fprintln(w, line)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// stalledConditions are n open conditions like those of 2026-10-08: one per delivery held past its bound,
|
||||
// each keeping all its evidence.
|
||||
func stalledConditions(n int) []conditions.Condition {
|
||||
at := time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC)
|
||||
var out []conditions.Condition
|
||||
for i := 0; i < n; i++ {
|
||||
id := fmt.Sprintf("d-%012x", 0x51a11ed+i)
|
||||
lines := stalledObservations([]stalledLine{{ID: id, State: "checking", For: "13h2m", Bound: "45m",
|
||||
H2: "close: recheck", Says: "the check was asked and nothing has answered"}})
|
||||
o := lines[0]
|
||||
c := conditions.Condition{Key: o.Key(), Kind: o.Kind, Subject: conditions.Subject{Scope: o.Scope, ID: o.ID},
|
||||
Severity: o.Severity, Summary: o.Summary, Source: probeDeliveriesID, Raised: at, LastObserved: at,
|
||||
Observations: 160, Count: 1, Resolver: conditions.ResolverSelf}
|
||||
for e := 0; e < conditions.KeptEvidence; e++ {
|
||||
c.Evidence = append(c.Evidence, conditions.Evidence{At: at.Add(time.Duration(e) * 5 * time.Minute),
|
||||
Said: "checking for " + (13*time.Hour + time.Duration(e)*time.Minute).String()})
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// novox/hq issue 314: a list of hundreds of open conditions carries each one's newest evidence, with
|
||||
// how much more it keeps and where to read it, and the verb's answer says it once — as data, not again
|
||||
// as the text the command printed. On 2026-10-08, 689 stalled deliveries made that answer more than the
|
||||
// bus carries in one message; now it is less than half of one.
|
||||
func TestAListOfConditionsCarriesEachInBrief(t *testing.T) {
|
||||
list := stalledConditions(689)
|
||||
printed, _ := json.MarshalIndent(map[string]any{"conditions": inBrief(list), "open": len(list),
|
||||
"counted": counted(list)}, "", " ")
|
||||
var parsed any
|
||||
if err := json.Unmarshal(printed, &parsed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wire, _ := json.Marshal(map[string]any{"result": verbAnswer{OK: true, Answer: parsed,
|
||||
Output: "its answer, as data, is `answer`\n"}})
|
||||
before, _ := json.MarshalIndent(map[string]any{"conditions": list}, "", " ")
|
||||
t.Logf("689 stalled deliveries: %d bytes on the wire, where the whole list printed was %d and was sent twice",
|
||||
len(wire), len(before))
|
||||
if len(wire) >= 512<<10 {
|
||||
t.Fatalf("the answer of 689 conditions is %d bytes, not under half of one message of the bus", len(wire))
|
||||
}
|
||||
var doc struct {
|
||||
Conditions []conditionInBrief `json:"conditions"`
|
||||
Counted map[string]int `json:"counted"`
|
||||
}
|
||||
if err := json.Unmarshal(printed, &doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c := doc.Conditions[0]
|
||||
if len(doc.Conditions) != 689 || len(c.Evidence) != 1 || c.EvidenceKept != conditions.KeptEvidence ||
|
||||
c.Key != list[0].Key || c.Summary != list[0].Summary || !strings.Contains(c.More, "key="+c.Key) {
|
||||
t.Fatalf("a condition in brief reads %+v", c)
|
||||
}
|
||||
if doc.Counted["D14 stalled"] != 689 {
|
||||
t.Fatalf("counted %v", doc.Counted)
|
||||
}
|
||||
if one := inBrief(list[:1])[0]; len(list[0].Evidence) != conditions.KeptEvidence || one.Evidence[0] != list[0].Evidence[0] {
|
||||
t.Fatal("the brief changed the condition it was made from, or kept other than the newest")
|
||||
}
|
||||
}
|
||||
|
||||
// The self-check's verdict carries at most overviewFindings of one probe's findings, with their count and
|
||||
// how to read them all; `doctor probe=` gives that probe whole (novox/hq issue 314).
|
||||
func TestAVerdictCarriesAProbesFindingsInBrief(t *testing.T) {
|
||||
var found []string
|
||||
for i := 0; i < 689; i++ {
|
||||
found = append(found, fmt.Sprintf("delivery d-%012x has been checking for 13h, past its bound of 45m", i))
|
||||
}
|
||||
run := doctorRun{Run: "run-1", Probes: []probeVerdict{{ID: "D14", Verdict: "failed", Found: found},
|
||||
{ID: "D1", Verdict: "passed"}}}
|
||||
answer, err := verdictAnswer(run, time.Now(), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
brief := answer["run"].(doctorRun).Probes[0]
|
||||
if len(brief.Found) != overviewFindings || brief.FoundCount != 689 || !strings.Contains(brief.More, "probe=D14") {
|
||||
t.Fatalf("the verdict carries %d findings of D14, count %d, more %q", len(brief.Found), brief.FoundCount, brief.More)
|
||||
}
|
||||
if len(run.Probes[0].Found) != 689 {
|
||||
t.Fatal("the overview cut the run it was made from")
|
||||
}
|
||||
whole, err := verdictAnswer(run, time.Now(), "d14")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p := whole["run"].(doctorRun).Probes; len(p) != 1 || len(p[0].Found) != 689 || p[0].More != "" {
|
||||
t.Fatalf("doctor probe=D14 answered %+v", p)
|
||||
}
|
||||
if _, err := verdictAnswer(run, time.Now(), "D99"); err == nil {
|
||||
t.Fatal("a probe the run does not have was answered")
|
||||
}
|
||||
if !strings.Contains(doctorText(answer), "probe=D14` gives them all") {
|
||||
t.Fatalf("the text does not say how to read them all:\n%s", doctorText(answer))
|
||||
}
|
||||
}
|
||||
@@ -86,7 +86,7 @@ func TestASilenceThroughTheVerbHoldsAndTheConditionStaysOpen(t *testing.T) {
|
||||
func TestUnreadableConditionsAreNotAWellMesh(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
_, store := withConditionsInMemory(t)
|
||||
store.Fail = errors.New("the bus is away")
|
||||
store.SetFail(errors.New("the bus is away"))
|
||||
asked, err := theThreeQuestions(t.Context(), open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -98,7 +98,7 @@ func TestUnreadableConditionsAreNotAWellMesh(t *testing.T) {
|
||||
if !strings.HasPrefix(said, "the open conditions could NOT be read") || strings.Contains(said, "no open conditions") {
|
||||
t.Fatalf("%s", said)
|
||||
}
|
||||
store.Fail = nil
|
||||
store.SetFail(nil)
|
||||
asked, _ = theThreeQuestions(t.Context(), open)
|
||||
said = printed(t, func() error { return printStatus(asked) })
|
||||
if asked.well() && !strings.Contains(said, "no open conditions;") {
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// **A finding one look can be wrong about is raised on the second look in a row** (novox/hq issue 277).
|
||||
//
|
||||
// D2 raised its resolver urgent on one unanswered question, asked once, while the resolver's machine
|
||||
// was loaded by a push and a build starting; thirty questions right after were answered at once. A
|
||||
// single sample of something that is answered over the network, or timed on a machine under load,
|
||||
// is not the invariant failing. So a source marks such a finding `Confirm`, and this holds it back:
|
||||
//
|
||||
// - raised when the source's previous look saw it too — two runs of the self-check in a row (five
|
||||
// minutes apart), or two ticks of the watchdogs (half a minute) — at the severity the source says;
|
||||
// - kept while it is already open, however it is seen, so a condition the next look still sees is
|
||||
// never cleared and raised again (flapping is not news; a reopening within ReopenWithin still is);
|
||||
// - cleared, as everything is, by the look that no longer sees it.
|
||||
//
|
||||
// A finding held back is not a pass: the self-check's verdict names it as unconfirmed. A finding that is
|
||||
// a definite answer — a resolver that answered wrongly, a stream that is not there — is not marked, and
|
||||
// is raised at once.
|
||||
type confirming struct {
|
||||
mu sync.Mutex
|
||||
// last is, by source, the keys of the Confirm findings its previous look saw.
|
||||
last map[string]map[string]bool
|
||||
}
|
||||
|
||||
// pass splits one source's findings into what is raised now and what is held for the next look, and
|
||||
// remembers what it saw. An open condition that cannot be read is kept: unknown is not a reason to
|
||||
// hold a finding back.
|
||||
func (c *confirming) pass(ctx context.Context, keeper *conditions.Keeper, source string,
|
||||
found []conditions.Observation) (raise, held []conditions.Observation) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.last == nil {
|
||||
c.last = map[string]map[string]bool{}
|
||||
}
|
||||
before, seen := c.last[source], map[string]bool{}
|
||||
for _, o := range found {
|
||||
if !o.Confirm {
|
||||
raise = append(raise, o)
|
||||
continue
|
||||
}
|
||||
key := o.Key()
|
||||
seen[key] = true
|
||||
if before[key] || isOpen(ctx, keeper, key) {
|
||||
raise = append(raise, o)
|
||||
continue
|
||||
}
|
||||
held = append(held, o)
|
||||
}
|
||||
c.last[source] = seen
|
||||
return raise, held
|
||||
}
|
||||
|
||||
// isOpen says whether a condition is open; one that cannot be read is taken as open.
|
||||
func isOpen(ctx context.Context, keeper *conditions.Keeper, key string) bool {
|
||||
if keeper == nil {
|
||||
return false
|
||||
}
|
||||
_, open, err := keeper.Get(ctx, key)
|
||||
return open || err != nil
|
||||
}
|
||||
@@ -0,0 +1,931 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A module declares the data it holds, and the mesh protects and watches it from that declaration
|
||||
// (novox/hq ADR 0233).
|
||||
//
|
||||
// The self-check's D13 composes what every machine declares, asks each machine's backup holder what
|
||||
// it measured of every item — size, newest write, newest good backup, the redundant storage it is on —
|
||||
// and keeps both. From that, and from what every provider says it holds for its consumers, it raises,
|
||||
// each URGENT for what is irreplaceable and a WARNING for what is valuable (the operator's ranking):
|
||||
//
|
||||
// - `data-shrank`: an item holds less than half of its largest size in seven days, and at least
|
||||
// shrinkFloor less; `data-missing`: its path is gone;
|
||||
// - `empty-replacement`: an item, or a consumer's data at a provider, is less than half the size of a
|
||||
// copy of the same thing kept elsewhere — on 2026-10-05 five applications ran for twenty hours on
|
||||
// empty databases while their real ones sat on another machine (issue 273);
|
||||
// - `data-held-twice` (warning): a consumer has active data at two providers and their sizes cannot
|
||||
// be compared;
|
||||
// - `data-quiet`: an item said to be written all the time has not been, within its bound;
|
||||
// - `backup-stale`: an item's newest good backup is older than its bound, or there is none;
|
||||
// - `array-degraded`: the redundant storage an item is on is not healthy, or cannot be read;
|
||||
// `protection-missing`: an item said to be protected by redundancy is on storage that is not;
|
||||
// - `cleanup-waiting` (warning): an item retired more than thirty days, waiting for a person.
|
||||
//
|
||||
// And it retires: an irreplaceable or valuable item in a module's own directory that its machine no
|
||||
// longer declares — its module unassigned — is kept, marked retired with when and why, and listed by
|
||||
// `cleanup list` until `cleanup delete` removes it. The node-engine never deletes a directory with
|
||||
// anything in it; this is the record of what it kept. An operator's path is never retired or deleted.
|
||||
|
||||
// The condition kinds of D13.
|
||||
const (
|
||||
kindDataShrank = "data-shrank"
|
||||
kindEmptyReplacement = "empty-replacement"
|
||||
kindDataHeldTwice = "data-held-twice"
|
||||
kindDataQuiet = "data-quiet"
|
||||
kindBackupStale = "backup-stale"
|
||||
kindDataUnmeasured = "data-unmeasured"
|
||||
// kindDataMissing is a watched item whose path is gone.
|
||||
kindDataMissing = "data-missing"
|
||||
// kindArrayDegraded is redundant storage watched data is on that is not healthy, or cannot be read.
|
||||
kindArrayDegraded = "array-degraded"
|
||||
// kindProtectionMissing is an item said to be protected by redundancy, on storage that is not.
|
||||
kindProtectionMissing = "protection-missing"
|
||||
)
|
||||
|
||||
// probeDataID is the self-check's id for this probe.
|
||||
const probeDataID = "D13"
|
||||
|
||||
// The bounds the findings are read against.
|
||||
var (
|
||||
// shrinkWindow is how far back the largest size is looked for.
|
||||
shrinkWindow = 7 * 24 * time.Hour
|
||||
// shrinkFloor is the least loss that is worth saying: two empty databases differ by a few
|
||||
// megabytes, and half of almost nothing is noise.
|
||||
shrinkFloor int64 = 16 << 20
|
||||
// dataAsk is how long one machine's holder, or one provider, is given to answer.
|
||||
dataAsk = 8 * time.Second
|
||||
)
|
||||
|
||||
// keyOfItem is one item's condition id: its machine, module and item.
|
||||
func keyOfItem(machine, module, item string) string { return machine + "." + module + "." + item }
|
||||
|
||||
// holderAnswer is what a node-backup holder's `backed-up` says of one module (ADR 0233 adds Data).
|
||||
type holderAnswer struct {
|
||||
Module string `json:"module"`
|
||||
Data []holderItem `json:"data"`
|
||||
}
|
||||
|
||||
// holderItem is one item as the holder measured it.
|
||||
type holderItem struct {
|
||||
Item string `json:"item"`
|
||||
Class string `json:"class"`
|
||||
Path string `json:"path"`
|
||||
SizeBytes *int64 `json:"size_bytes"`
|
||||
LastWrite *time.Time `json:"last_write"`
|
||||
MeasuredAt *time.Time `json:"measured_at"`
|
||||
LastBackup *time.Time `json:"last_backup"`
|
||||
Error string `json:"error,omitempty"`
|
||||
// Precision is what the size is: exact, a dataset's, partial, or none (ADR 0233).
|
||||
Precision string `json:"precision,omitempty"`
|
||||
// Redundancy is the redundant storage the item is on, where the holder could tell (ADR 0233).
|
||||
Redundancy *inventory.Redundancy `json:"redundancy,omitempty"`
|
||||
}
|
||||
|
||||
// readHolder reads a holder's answer into measurements by module and item.
|
||||
func readHolder(raw json.RawMessage) (map[string]map[string]inventory.Measurement, error) {
|
||||
var modules []holderAnswer
|
||||
if err := json.Unmarshal(raw, &modules); err != nil {
|
||||
return nil, fmt.Errorf("its answer is not readable: %w", err)
|
||||
}
|
||||
out := map[string]map[string]inventory.Measurement{}
|
||||
for _, m := range modules {
|
||||
for _, it := range m.Data {
|
||||
if out[m.Module] == nil {
|
||||
out[m.Module] = map[string]inventory.Measurement{}
|
||||
}
|
||||
out[m.Module][it.Item] = inventory.Measurement{Path: it.Path, Size: it.SizeBytes, LastWrite: it.LastWrite,
|
||||
MeasuredAt: it.MeasuredAt, LastBackup: it.LastBackup, Error: it.Error, Redundancy: it.Redundancy,
|
||||
Precision: it.Precision}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// declaredOn is every data item a machine's composition declares, and the module there that holds
|
||||
// node-backup to measure them — empty for none.
|
||||
func declaredOn(plan catalogue.Resolution) ([]inventory.DeclaredData, string) {
|
||||
var out []inventory.DeclaredData
|
||||
held := ""
|
||||
for _, m := range plan.Modules {
|
||||
for _, c := range m.Claims {
|
||||
if s, known := catalogue.SeatNamed(c.Name); known && s.Name == catalogue.BackupSeat {
|
||||
held = m.Module
|
||||
}
|
||||
}
|
||||
for _, it := range m.DataItems() {
|
||||
out = append(out, inventory.DeclaredData{Module: m.Module, Item: it.ID, Class: it.Class,
|
||||
Owned: it.OwnedByModule(), Protection: it.Protection()})
|
||||
}
|
||||
}
|
||||
return out, held
|
||||
}
|
||||
|
||||
// consumerCopy is one provider's account of one consumer: where, how big, and whether still active.
|
||||
type consumerCopy struct {
|
||||
Node, Module, Consumer string
|
||||
Size *int64
|
||||
Retired bool
|
||||
// Class is how precious the consumer's data is: the stricter of what the provider keeps for its
|
||||
// consumers and what the consumer says it keeps there (`kept-by`).
|
||||
Class string
|
||||
}
|
||||
|
||||
// probeData is D13.
|
||||
func probeData(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
if d.js == nil {
|
||||
return nil, errors.New("no bus to ask the machines over")
|
||||
}
|
||||
open := d.open
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nodes, err := open.inventory.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
heard := heardMachines(d)
|
||||
now := time.Now()
|
||||
delivered, err := readDeliveries(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
type machine struct {
|
||||
name string
|
||||
declared []inventory.DeclaredData
|
||||
held bool
|
||||
measured map[string]map[string]inventory.Measurement
|
||||
askErr error
|
||||
}
|
||||
var machines []*machine
|
||||
for _, n := range nodes {
|
||||
plan, _, err := planFor(ctx, open, n.Name)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
// A machine that cannot be worked out declares nothing this run — which is not the same as
|
||||
// declaring nothing: retiring its data on that would be acting on an unreadable result.
|
||||
continue
|
||||
}
|
||||
declared, holder := declaredOn(plan)
|
||||
// **A holder is asked only once its machine has been sent it and had time to report** (novox/hq
|
||||
// issue 275): assigned and not pushed yet, it is not there to answer, and "did not say what it
|
||||
// measured" about it was a warning for a push nobody had made yet.
|
||||
held := holder != "" && delivered[n.Name].settled(holder, now)
|
||||
machines = append(machines, &machine{name: n.Name, declared: declared, held: held})
|
||||
}
|
||||
// Every holder asked at once, as D8 asks every ban list.
|
||||
var wg sync.WaitGroup
|
||||
for _, m := range machines {
|
||||
if !m.held || !heard[m.name] {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(m *machine) {
|
||||
defer wg.Done()
|
||||
asking, cancel := context.WithTimeout(ctx, dataAsk)
|
||||
defer cancel()
|
||||
raw, err := askSeatTool(asking, d.js.Conn(), catalogue.BackupSeat, "backed-up", m.name)
|
||||
if err == nil {
|
||||
m.measured, err = readHolder(raw)
|
||||
}
|
||||
m.askErr = err
|
||||
}(m)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
var out []conditions.Observation
|
||||
for _, m := range machines {
|
||||
if m.askErr != nil {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name, Token: kindDataUnmeasured,
|
||||
Kind: kindDataUnmeasured, Machine: m.name, Severity: conditions.Warning, Confirm: true,
|
||||
Summary: fmt.Sprintf("%s's backup holder did not say what it measured of the data declared there, so "+
|
||||
"nothing about that data is known", m.name),
|
||||
Said: firstLine(m.askErr.Error())})
|
||||
}
|
||||
why := fmt.Sprintf("no longer declared on %s: its module was unassigned there, or is no longer pulled in", m.name)
|
||||
change, err := open.inventory.RecordData(ctx, m.name, m.declared, m.measured, why, now)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("what %s holds could not be kept: %w", m.name, err)
|
||||
}
|
||||
for _, r := range change.Retired {
|
||||
log.Printf("data: %s of %s on %s RETIRED, kept at %s: %s — `cleanup list` shows it, and only `cleanup "+
|
||||
"delete` removes it (novox/hq ADR 0233)", r.Item, r.Module, r.Machine, orUnknownPath(r.Path), why)
|
||||
}
|
||||
for _, r := range change.Reenabled {
|
||||
log.Printf("data: %s of %s on %s is declared again, no longer retired", r.Item, r.Module, r.Machine)
|
||||
}
|
||||
}
|
||||
|
||||
records, err := open.inventory.Data(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
peaks, err := open.inventory.DataPeaks(ctx, now.Add(-shrinkWindow))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bindings, err := open.inventory.Bindings(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
upgraded, keptBy := keptByClasses(bindings, shelf)
|
||||
copies, err := consumerCopies(ctx, d.js.Conn(), open.inventory, shelf, keptBy)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, dataFindings(records, peaks, shelf, copies, upgraded, now)...)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func orUnknownPath(p string) string {
|
||||
if p == "" {
|
||||
return "a path its backup holder never named"
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// consumerCopies asks every provider of a provision whose consumers' data is kept what it holds, at
|
||||
// once. One that cannot answer is passed over: it says nothing about any copy, which is not a finding.
|
||||
func consumerCopies(ctx context.Context, conn *nats.Conn, inv *inventory.Inventory,
|
||||
shelf map[string]catalogue.Manifest, keptBy map[string]string) ([]consumerCopy, error) {
|
||||
instances, err := providerInstances(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var asked []providerInstance
|
||||
for _, p := range instances {
|
||||
m := shelf[p.Module]
|
||||
keeps := false
|
||||
for provision := range m.Grants {
|
||||
keeps = keeps || m.KeepsConsumerData(provision)
|
||||
}
|
||||
if keeps {
|
||||
asked = append(asked, p)
|
||||
}
|
||||
}
|
||||
states := make([]*link.RetirementState, len(asked))
|
||||
var wg sync.WaitGroup
|
||||
for i, p := range asked {
|
||||
wg.Add(1)
|
||||
go func(i int, p providerInstance) {
|
||||
defer wg.Done()
|
||||
asking, cancel := context.WithTimeout(ctx, dataAsk)
|
||||
defer cancel()
|
||||
if s, err := askRetirement(asking, conn, p); err == nil {
|
||||
states[i] = &s
|
||||
}
|
||||
}(i, p)
|
||||
}
|
||||
wg.Wait()
|
||||
var out []consumerCopy
|
||||
for i, p := range asked {
|
||||
s := states[i]
|
||||
if s == nil {
|
||||
continue
|
||||
}
|
||||
class := consumersClass(shelf[p.Module])
|
||||
for _, c := range s.Held {
|
||||
cp := consumerCopy{Node: p.Node, Module: p.Module, Consumer: c,
|
||||
Class: catalogue.StricterClass(class, keptBy[p.Module+"/"+c])}
|
||||
if size, ok := s.HeldSizes[c]; ok && size >= 0 {
|
||||
size := size
|
||||
cp.Size = &size
|
||||
}
|
||||
out = append(out, cp)
|
||||
}
|
||||
for _, r := range s.Retired {
|
||||
if r.Kind != "" && r.Kind != "consumer" {
|
||||
continue
|
||||
}
|
||||
cp := consumerCopy{Node: p.Node, Module: p.Module, Consumer: r.Consumer, Retired: true,
|
||||
Class: catalogue.StricterClass(class, keptBy[p.Module+"/"+r.Consumer])}
|
||||
if r.SizeBytes != nil && *r.SizeBytes >= 0 {
|
||||
cp.Size = r.SizeBytes
|
||||
}
|
||||
out = append(out, cp)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// severityOf is how loud a finding about data of a class is: urgent for what is irreplaceable, a warning
|
||||
// for anything else watched (the operator's ranking, ADR 0233).
|
||||
func severityOf(class string) conditions.Severity {
|
||||
if class == catalogue.ClassIrreplaceable {
|
||||
return conditions.Urgent
|
||||
}
|
||||
return conditions.Warning
|
||||
}
|
||||
|
||||
// consumersClass is the most precious class a provider keeps any of its consumers' data as.
|
||||
func consumersClass(m catalogue.Manifest) string {
|
||||
class := catalogue.ClassNone
|
||||
for provision := range m.Grants {
|
||||
if c, ok := m.ConsumerDataOf(provision); ok {
|
||||
class = catalogue.StricterClass(class, c.Class)
|
||||
} else if m.KeepsConsumerData(provision) {
|
||||
class = catalogue.StricterClass(class, catalogue.ClassValuable)
|
||||
}
|
||||
}
|
||||
return class
|
||||
}
|
||||
|
||||
// keptByClasses is what consumers say of the data they keep with their providers (`kept-by`), read
|
||||
// through where each is bound: by provider module and consumer identity, the class of that consumer's
|
||||
// data there; and by provider item key (machine/module/item), the class the item holding it is held to.
|
||||
func keptByClasses(bindings []inventory.Binding, shelf map[string]catalogue.Manifest) (map[string]string, map[string]string) {
|
||||
upgraded, keptBy := map[string]string{}, map[string]string{}
|
||||
for _, b := range bindings {
|
||||
m, ok := shelf[b.Consumer]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
k, said := m.KeptByOf(b.Provision)
|
||||
if !said {
|
||||
continue
|
||||
}
|
||||
identity := catalogue.ConsumerIdentity(b.Machine, catalogue.IdentitySource(m.Slug, m.Module))
|
||||
key := b.Provider.Module + "/" + identity
|
||||
keptBy[key] = catalogue.StricterClass(keptBy[key], k.Class)
|
||||
if pc, ok := shelf[b.Provider.Module].ConsumerDataOf(b.Provision); ok && pc.In != "" {
|
||||
if _, own := shelf[b.Provider.Module].DataItem(pc.In); own {
|
||||
item := b.Provider.Node + "/" + b.Provider.Module + "/" + pc.In
|
||||
upgraded[item] = catalogue.StricterClass(upgraded[item], k.Class)
|
||||
}
|
||||
}
|
||||
}
|
||||
return upgraded, keptBy
|
||||
}
|
||||
|
||||
// dataFindings is every condition the data on record raises now. A function of what is known, so the
|
||||
// incident's shape is tested without a mesh. upgraded is the class an item is held to where a consumer
|
||||
// of its module keeps data in it more precious than its own class says (`kept-by`), by its key.
|
||||
func dataFindings(records []inventory.DataRecord, peaks map[string]int64, shelf map[string]catalogue.Manifest,
|
||||
copies []consumerCopy, upgraded map[string]string, now time.Time) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
byItem := map[string][]inventory.DataRecord{}
|
||||
arrays := map[string][]inventory.DataRecord{}
|
||||
type shrunk struct {
|
||||
machine, dataset, class string
|
||||
size, peak int64
|
||||
items []string
|
||||
}
|
||||
shrunkDatasets := map[string]shrunk{}
|
||||
for _, r := range records {
|
||||
if r.DeletedAt != nil {
|
||||
continue
|
||||
}
|
||||
class := catalogue.StricterClass(r.Class, upgraded[r.Key()])
|
||||
r.Class = class
|
||||
byItem[r.Module+"/"+r.Item] = append(byItem[r.Module+"/"+r.Item], r)
|
||||
item, declared := shelf[r.Module].DataItem(r.Item)
|
||||
id := keyOfItem(r.Machine, r.Module, r.Item)
|
||||
if r.Retired() {
|
||||
if now.Sub(*r.RetiredAt) > cleanupAfter {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: "cleanup",
|
||||
Kind: kindCleanupWaiting, Machine: r.Machine, Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s of %s on %s (%s, %s) has been retired %d days — kept where it was since %s; "+
|
||||
"`cleanup delete %s %s %s --why …` once a person has decided, or assign %s there again",
|
||||
r.Item, r.Module, r.Machine, r.Class, sizeWords(r.Size), int(now.Sub(*r.RetiredAt).Hours()/24),
|
||||
r.RetiredWhy, r.Machine, r.Module, r.Item, r.Module),
|
||||
Said: "kept at " + orUnknownPath(r.Path)})
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !catalogue.Watched(class) {
|
||||
continue
|
||||
}
|
||||
severity := severityOf(class)
|
||||
if r.Redundancy != nil {
|
||||
where := r.Machine + "/" + r.Redundancy.Kind + ":" + r.Redundancy.Where
|
||||
arrays[where] = append(arrays[where], r)
|
||||
} else if declared && item.Redundancy != "" && r.MeasuredAt != nil && r.MeasureError == "" {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindProtectionMissing,
|
||||
Kind: kindProtectionMissing, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s of %s on %s (%s) is said to be protected by the redundancy of the storage it is on, "+
|
||||
"and it is on nothing the backup holder can read as redundant: it has no protection the mesh can see",
|
||||
r.Item, r.Module, r.Machine, class),
|
||||
Said: orUnknownPath(r.Path) + " is on no redundant storage the backup holder can read"})
|
||||
}
|
||||
if r.MeasureError != "" && strings.Contains(r.MeasureError, "does not exist") {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataMissing,
|
||||
Kind: kindDataMissing, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s of %s on %s (%s) is gone: where it was declared, nothing exists any more", r.Item,
|
||||
r.Module, r.Machine, class),
|
||||
Said: orUnknownPath(r.Path) + " does not exist: " + firstLine(r.MeasureError)})
|
||||
} else if peak, ok := peaks[r.Key()]; ok && r.Size != nil && inventory.Comparable(r.Precision) &&
|
||||
*r.Size*2 < peak && peak-*r.Size >= shrinkFloor && inventory.Dataset(r.Precision) != "" {
|
||||
// Several items on one dataset share its size: one condition for the dataset, as loud as the
|
||||
// most precious item on it.
|
||||
k := r.Machine + "/" + inventory.Dataset(r.Precision)
|
||||
ds := shrunkDatasets[k]
|
||||
ds.machine, ds.dataset, ds.size, ds.peak = r.Machine, inventory.Dataset(r.Precision), *r.Size, peak
|
||||
ds.class = catalogue.StricterClass(ds.class, class)
|
||||
ds.items = append(ds.items, r.Module+"/"+r.Item)
|
||||
shrunkDatasets[k] = ds
|
||||
} else if peak, ok := peaks[r.Key()]; ok && r.Size != nil && inventory.Comparable(r.Precision) &&
|
||||
*r.Size*2 < peak && peak-*r.Size >= shrinkFloor {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataShrank,
|
||||
Kind: kindDataShrank, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s of %s on %s (%s) shrank to %s from %s within %d days — more than half of what it "+
|
||||
"held is gone. If that was meant, silence this with why; if not, `node-backup.restore` puts the last "+
|
||||
"good copy beside it", r.Item, r.Module, r.Machine, class, sizeWords(r.Size), sizeWords(&peak),
|
||||
int(shrinkWindow.Hours()/24))})
|
||||
}
|
||||
if !declared {
|
||||
continue
|
||||
}
|
||||
if within := item.ActiveWithin(); within > 0 && r.LastWrite != nil && now.Sub(*r.LastWrite) > within {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataQuiet,
|
||||
Kind: kindDataQuiet, Machine: r.Machine, Severity: severity,
|
||||
Summary: fmt.Sprintf("%s of %s on %s is written all the time, and has not been since %s (its bound is %s): "+
|
||||
"whatever writes it has stopped", r.Item, r.Module, r.Machine, r.LastWrite.UTC().Format(time.RFC3339),
|
||||
within)})
|
||||
}
|
||||
// A backup is required of what is irreplaceable and copied; of what is valuable it is the standard
|
||||
// plan, said only where the machine was measured — where a holder is there to take it.
|
||||
if item.BackedUp() && (class == catalogue.ClassIrreplaceable || r.MeasuredAt != nil) {
|
||||
within := item.BackupWithin()
|
||||
switch {
|
||||
case r.LastBackup == nil && now.Sub(r.FirstSeen) > within:
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindBackupStale,
|
||||
Kind: kindBackupStale, Machine: r.Machine, Severity: severity,
|
||||
Summary: fmt.Sprintf("%s of %s on %s is %s and has no good backup on record, %s after it was first "+
|
||||
"declared — is node-backup held there, and do its nights succeed? (`node-backup.backed-up`)",
|
||||
r.Item, r.Module, r.Machine, class, now.Sub(r.FirstSeen).Round(time.Hour))})
|
||||
case r.LastBackup != nil && now.Sub(*r.LastBackup) > within:
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindBackupStale,
|
||||
Kind: kindBackupStale, Machine: r.Machine, Severity: severity,
|
||||
Summary: fmt.Sprintf("%s of %s on %s is %s and its newest good backup is from %s, older than its bound "+
|
||||
"of %s", r.Item, r.Module, r.Machine, class, r.LastBackup.UTC().Format(time.RFC3339), within)})
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, k := range keysSorted(shrunkDatasets) {
|
||||
ds := shrunkDatasets[k]
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine,
|
||||
ID: ds.machine + ".dataset." + strings.ReplaceAll(ds.dataset, "/", "-"), Token: kindDataShrank,
|
||||
Kind: kindDataShrank, Machine: ds.machine, Severity: severityOf(ds.class), Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("the dataset on %s that holds %s shrank to %s from %s within %d days — more than half of "+
|
||||
"what it held is gone", ds.machine, strings.Join(ds.items, ", "), sizeWords(&ds.size), sizeWords(&ds.peak),
|
||||
int(shrinkWindow.Hours()/24)),
|
||||
Said: "the dataset " + ds.dataset})
|
||||
}
|
||||
// The redundant storage watched data is on: one condition per array, as loud as the most precious
|
||||
// item on it — the array, not each item, is what degrades.
|
||||
for _, where := range keysSorted(arrays) {
|
||||
rs := arrays[where]
|
||||
red := rs[0].Redundancy
|
||||
if red.Healthy != nil && *red.Healthy {
|
||||
continue
|
||||
}
|
||||
class, machine := catalogue.ClassValuable, rs[0].Machine
|
||||
var names []string
|
||||
for _, r := range rs {
|
||||
class = catalogue.StricterClass(class, r.Class)
|
||||
names = append(names, r.Module+"/"+r.Item)
|
||||
}
|
||||
state := "could not be read"
|
||||
if red.Healthy != nil {
|
||||
state = "is NOT healthy"
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine,
|
||||
ID: machine + ".array." + strings.NewReplacer("/", "-", ":", "-").Replace(red.Kind+"-"+red.Where),
|
||||
Token: kindArrayDegraded, Kind: kindArrayDegraded, Machine: machine, Severity: severityOf(class),
|
||||
Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("the %s storage on %s that protects %s %s", red.Kind, machine, strings.Join(names, ", "),
|
||||
state),
|
||||
Said: fmt.Sprintf("the %s storage %s %s: %s", red.Kind, red.Where, state, firstLine(red.Said))})
|
||||
}
|
||||
// The same item on several machines: a copy that is in use and far smaller than one kept elsewhere is
|
||||
// an empty replacement. Only against a retired copy — a module running on two machines on purpose
|
||||
// keeps two different sets of data.
|
||||
for _, key := range keysSorted(byItem) {
|
||||
rs := byItem[key]
|
||||
for _, a := range rs {
|
||||
if a.Retired() || a.Size == nil || !catalogue.Watched(a.Class) || !inventory.Comparable(a.Precision) {
|
||||
continue
|
||||
}
|
||||
for _, o := range rs {
|
||||
if o.Machine == a.Machine || !o.Retired() || o.Size == nil || !inventory.Comparable(o.Precision) ||
|
||||
!replacedByLess(*a.Size, *o.Size) {
|
||||
continue
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine,
|
||||
ID: keyOfItem(a.Machine, a.Module, a.Item), Token: kindEmptyReplacement, Kind: kindEmptyReplacement,
|
||||
Machine: a.Machine, Also: []string{o.Machine}, Severity: severityOf(a.Class), Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s of %s on %s holds %s, and the copy %s kept on %s holds %s: %s is running on "+
|
||||
"an empty replacement of its data. Move the data, or assign it back where its data is",
|
||||
a.Item, a.Module, a.Machine, sizeWords(a.Size), o.Module, o.Machine, sizeWords(o.Size), a.Module)})
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
out = append(out, consumerFindings(copies)...)
|
||||
return out
|
||||
}
|
||||
|
||||
// replacedByLess is whether a copy in use is an empty replacement of a copy kept elsewhere: less than
|
||||
// half of it, and at least shrinkFloor less.
|
||||
func replacedByLess(inUse, kept int64) bool {
|
||||
return inUse*2 < kept && kept-inUse >= shrinkFloor
|
||||
}
|
||||
|
||||
// consumerFindings is the same question of consumers' data at providers: one consumer, the same
|
||||
// provider module on two machines.
|
||||
func consumerFindings(copies []consumerCopy) []conditions.Observation {
|
||||
by := map[string][]consumerCopy{}
|
||||
for _, c := range copies {
|
||||
k := c.Module + "/" + c.Consumer
|
||||
by[k] = append(by[k], c)
|
||||
}
|
||||
var out []conditions.Observation
|
||||
for _, k := range keysSorted(by) {
|
||||
cs := by[k]
|
||||
if len(cs) < 2 {
|
||||
continue
|
||||
}
|
||||
found := false
|
||||
for _, a := range cs {
|
||||
if a.Retired || a.Size == nil {
|
||||
continue
|
||||
}
|
||||
for _, o := range cs {
|
||||
if o.Node == a.Node || o.Size == nil || !replacedByLess(*a.Size, *o.Size) {
|
||||
continue
|
||||
}
|
||||
state := "active"
|
||||
if o.Retired {
|
||||
state = "retired"
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeProvider,
|
||||
ID: a.Module + "." + a.Node + "." + a.Consumer, Token: kindEmptyReplacement, Kind: kindEmptyReplacement,
|
||||
Machine: a.Node, Also: []string{o.Node}, Severity: severityOf(catalogue.StricterClass(a.Class, o.Class)),
|
||||
Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s's data at %s on %s holds %s, and its %s copy at %s on %s holds %s: the "+
|
||||
"consumer is using an empty replacement of its data (issue 273's shape). Pin it back to %s, or move "+
|
||||
"the data first", a.Consumer, a.Module, a.Node, sizeWords(a.Size), state, o.Module, o.Node,
|
||||
sizeWords(o.Size), o.Node)})
|
||||
found = true
|
||||
break
|
||||
}
|
||||
if found {
|
||||
break
|
||||
}
|
||||
}
|
||||
if found {
|
||||
continue
|
||||
}
|
||||
var active []consumerCopy
|
||||
for _, c := range cs {
|
||||
if !c.Retired {
|
||||
active = append(active, c)
|
||||
}
|
||||
}
|
||||
if len(active) >= 2 {
|
||||
var where []string
|
||||
var also []string
|
||||
for _, c := range active {
|
||||
where = append(where, c.Node+" ("+sizeWords(c.Size)+")")
|
||||
also = append(also, c.Node)
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeProvider,
|
||||
ID: active[0].Module + "." + active[0].Consumer, Token: kindDataHeldTwice, Kind: kindDataHeldTwice,
|
||||
Machine: active[0].Node, Also: also[1:], Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s has active data at %s on %d machines — %s — and only one is the one it uses",
|
||||
active[0].Consumer, active[0].Module, len(active), strings.Join(where, ", "))})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func keysSorted[V any](m map[string]V) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// ---- the `data` verb ---------------------------------------------------------------------------
|
||||
|
||||
const dataUsage = "data [--json] [--machine <name>] [--retired]"
|
||||
|
||||
// dataRow is one item as `data` lists it.
|
||||
type dataRow struct {
|
||||
Machine string `json:"machine"`
|
||||
Module string `json:"module"`
|
||||
Item string `json:"item"`
|
||||
Class string `json:"class"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Protection string `json:"protection,omitempty"`
|
||||
// Array is the redundant storage it is on and its state, where its holder could tell.
|
||||
Array string `json:"array,omitempty"`
|
||||
Unmeasured string `json:"unmeasured,omitempty"`
|
||||
// Precision says what the size is: exact, a dataset's whole size, partial, or none.
|
||||
Precision string `json:"precision,omitempty"`
|
||||
SizeBytes *int64 `json:"size-bytes,omitempty"`
|
||||
LastWrite string `json:"last-write,omitempty"`
|
||||
MeasuredAt string `json:"measured-at,omitempty"`
|
||||
LastBackup string `json:"last-backup,omitempty"`
|
||||
BackupDue string `json:"backup-within,omitempty"`
|
||||
Retired string `json:"retired,omitempty"`
|
||||
RetiredWhy string `json:"retired-why,omitempty"`
|
||||
Deleted string `json:"deleted,omitempty"`
|
||||
}
|
||||
|
||||
// dataCommand is `data`: every item every machine declares, or held retired, as the self-check last
|
||||
// found it.
|
||||
func dataCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("data", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
only := set.String("machine", "", "one machine")
|
||||
retiredOnly := set.Bool("retired", false, "only what is retired")
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New(dataUsage)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
records, err := open.inventory.Data(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rows := dataRows(records, shelf, *only, *retiredOnly)
|
||||
if *asJSON {
|
||||
return printJSON(map[string]any{"data": rows})
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
fmt.Println("no data on record: the self-check (D13) records what each machine declares on its next run")
|
||||
return nil
|
||||
}
|
||||
for _, r := range rows {
|
||||
state := ""
|
||||
switch {
|
||||
case r.Deleted != "":
|
||||
state = " DELETED " + r.Deleted
|
||||
case r.Retired != "":
|
||||
state = " RETIRED " + r.Retired + " — " + r.RetiredWhy
|
||||
}
|
||||
fmt.Printf("%s %s/%s %s %s %s protected by %s%s\n", r.Machine, r.Module, r.Item, r.Class,
|
||||
sizeWords(r.SizeBytes), orUnknownPath(r.Path), orNothingWord(r.Protection), state)
|
||||
if r.Array != "" {
|
||||
fmt.Printf(" on %s\n", r.Array)
|
||||
}
|
||||
if r.Precision != "" && r.Precision != "exact" {
|
||||
fmt.Printf(" size: %s\n", r.Precision)
|
||||
}
|
||||
if r.Unmeasured != "" {
|
||||
fmt.Printf(" not measured: %s\n", r.Unmeasured)
|
||||
}
|
||||
if r.Class == catalogue.ClassCache {
|
||||
continue
|
||||
}
|
||||
fmt.Printf(" last write %s, measured %s, last backup %s%s\n", orNever(r.LastWrite), orNever(r.MeasuredAt),
|
||||
orNever(r.LastBackup), within(r.BackupDue))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func dataRows(records []inventory.DataRecord, shelf map[string]catalogue.Manifest, only string, retiredOnly bool) []dataRow {
|
||||
rows := []dataRow{}
|
||||
stamp := func(t *time.Time) string {
|
||||
if t == nil {
|
||||
return ""
|
||||
}
|
||||
return t.UTC().Format(time.RFC3339)
|
||||
}
|
||||
for _, r := range records {
|
||||
if only != "" && r.Machine != only {
|
||||
continue
|
||||
}
|
||||
if retiredOnly && !r.Retired() {
|
||||
continue
|
||||
}
|
||||
row := dataRow{Machine: r.Machine, Module: r.Module, Item: r.Item, Class: r.Class, Path: r.Path,
|
||||
Protection: r.Protection, Unmeasured: r.MeasureError, Precision: r.Precision, SizeBytes: r.Size, LastWrite: stamp(r.LastWrite), MeasuredAt: stamp(r.MeasuredAt),
|
||||
LastBackup: stamp(r.LastBackup), Retired: stamp(r.RetiredAt), RetiredWhy: r.RetiredWhy,
|
||||
Deleted: stamp(r.DeletedAt)}
|
||||
if it, ok := shelf[r.Module].DataItem(r.Item); ok && it.BackedUp() {
|
||||
row.BackupDue = it.BackupWithin().String()
|
||||
}
|
||||
if red := r.Redundancy; red != nil {
|
||||
state := "state unread"
|
||||
if red.Healthy != nil && *red.Healthy {
|
||||
state = "healthy"
|
||||
} else if red.Healthy != nil {
|
||||
state = "NOT HEALTHY"
|
||||
}
|
||||
row.Array = red.Kind + " " + red.Where + ", " + state
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
func orNothingWord(s string) string {
|
||||
if s == "" || s == "none" {
|
||||
return "nothing"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func orNever(s string) string {
|
||||
if s == "" {
|
||||
return "never"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func within(s string) string {
|
||||
if s == "" {
|
||||
return " (not backed up)"
|
||||
}
|
||||
return " (bound " + s + ")"
|
||||
}
|
||||
|
||||
// ---- cleanup of retired own data ---------------------------------------------------------------
|
||||
|
||||
// The tools a node-backup holder serves to delete one retired item (novox/hq ADR 0233): the first
|
||||
// takes a last restore point of it, tagged as retired, and only then removes it — in the background,
|
||||
// because a large item outlasts any call — and the second says how that went. Module tools, not seat
|
||||
// verbs: only the controller's `cleanup delete` calls them, as it calls a provider's provisioner_delete.
|
||||
const (
|
||||
ToolDeleteRetired = "backup_delete_retired"
|
||||
ToolDeletedOutcome = "backup_deleted"
|
||||
)
|
||||
|
||||
// deletionWait is how long `cleanup delete` follows a deletion before handing it back to the person.
|
||||
var deletionWait = 8 * time.Minute
|
||||
|
||||
// deletionPoll is how often it asks.
|
||||
var deletionPoll = 5 * time.Second
|
||||
|
||||
// deletion is a holder's account of one deletion.
|
||||
type deletion struct {
|
||||
Started bool `json:"started"`
|
||||
Running bool `json:"running"`
|
||||
Done bool `json:"done"`
|
||||
OK bool `json:"ok"`
|
||||
Snapshot string `json:"snapshot"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
|
||||
// retiredData is every retired item on record, as `cleanup list` shows them.
|
||||
func retiredData(records []inventory.DataRecord, now time.Time) []retiredRow {
|
||||
var out []retiredRow
|
||||
for _, r := range records {
|
||||
if !r.Retired() {
|
||||
continue
|
||||
}
|
||||
out = append(out, retiredRow{Node: r.Machine, Module: r.Module, Consumer: r.Item, Kind: retiredDataKind,
|
||||
RetiredAt: r.RetiredAt.UTC().Format(time.RFC3339), AgeDays: int(now.Sub(*r.RetiredAt).Hours() / 24),
|
||||
SizeBytes: r.Size, Why: r.RetiredWhy, Path: r.Path, Class: r.Class})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// retiredDataKind is what `cleanup list` calls a module's own retired data, beside a provider's consumer.
|
||||
const retiredDataKind = "own-data"
|
||||
|
||||
// holderOn is the module holding node-backup on a machine.
|
||||
func holderOn(ctx context.Context, inv *inventory.Inventory, machine string) (string, error) {
|
||||
held, err := inv.Holdings(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, h := range held {
|
||||
if s, known := catalogue.SeatNamed(h.Claim); known && s.Name == catalogue.BackupSeat && h.Node == machine {
|
||||
return h.Module, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("nothing holds %s on %s, and it is the backup holder that deletes retired data there "+
|
||||
"(after a last restore point)", catalogue.BackupSeat, machine)
|
||||
}
|
||||
|
||||
// deleteRetiredData has a machine's backup holder delete one retired item: never one declared now, and
|
||||
// never one not retired. The holder takes a last restore point of it first, so the deletion can be
|
||||
// undone until a person forgets that restore point; the record says deleted only once the holder says
|
||||
// it is.
|
||||
func deleteRetiredData(ctx context.Context, conn *nats.Conn, open *stores, r inventory.DataRecord, f handActFlags) error {
|
||||
inv := open.inventory
|
||||
if !r.Retired() {
|
||||
return fmt.Errorf("%s of %s on %s is not retired — only retired data is deleted. Nothing was done",
|
||||
r.Item, r.Module, r.Machine)
|
||||
}
|
||||
if r.Path == "" {
|
||||
return fmt.Errorf("%s of %s on %s was never measured, so where it is was never said; nothing was deleted",
|
||||
r.Item, r.Module, r.Machine)
|
||||
}
|
||||
if plan, _, err := planFor(ctx, open, r.Machine); err == nil {
|
||||
for _, m := range plan.Modules {
|
||||
if _, still := m.DataItem(r.Item); still && m.Module == r.Module {
|
||||
return fmt.Errorf("%s runs on %s again and declares %s: it is not retired any more. Nothing was done",
|
||||
r.Module, r.Machine, r.Item)
|
||||
}
|
||||
}
|
||||
}
|
||||
holder, err := holderOn(ctx, inv, r.Machine)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.record(ctx, "cleanup delete", []string{r.Machine, r.Module, r.Item})
|
||||
args := map[string]any{"module": r.Module, "item": r.Item, "path": r.Path, "confirm": r.Item,
|
||||
"why": strings.TrimSpace(*f.why), "by": link.Caller(), "via": link.ViaController}
|
||||
ask := func(tool string) (deletion, error) {
|
||||
var d deletion
|
||||
answer, err := link.AskModuleToolOn(ctx, conn, holder, tool, r.Machine, args, 25*time.Second)
|
||||
if err != nil {
|
||||
return d, err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return d, fmt.Errorf("%s on %s refused: %s", holder, r.Machine, answer.Error)
|
||||
}
|
||||
return d, unmarshalAnswer(answer, &d)
|
||||
}
|
||||
d, err := ask(ToolDeleteRetired)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for waited := time.Duration(0); !d.Done && waited < deletionWait; waited += deletionPoll {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(deletionPoll):
|
||||
}
|
||||
if d, err = ask(ToolDeletedOutcome); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case !d.Done:
|
||||
fmt.Printf("%s on %s is still taking the last restore point of %s and deleting it; `cleanup list` keeps "+
|
||||
"showing it until the holder says it is done — the same `cleanup delete` again reads how it went\n",
|
||||
holder, r.Machine, r.Path)
|
||||
return nil
|
||||
case !d.OK:
|
||||
return fmt.Errorf("%s on %s did NOT delete %s: %s", holder, r.Machine, r.Path, d.Error)
|
||||
}
|
||||
if err := inv.MarkDataDeleted(ctx, r.Machine, r.Module, r.Item, link.Caller(), strings.TrimSpace(*f.why), time.Now()); err != nil {
|
||||
return fmt.Errorf("%s deleted %s on %s, and it could not be recorded: %w", holder, r.Path, r.Machine, err)
|
||||
}
|
||||
fmt.Printf("%s on %s deleted %s of %s (%s, %s); its last restore point is %s, kept until a person forgets it\n",
|
||||
holder, r.Machine, r.Item, r.Module, r.Path, sizeWords(r.Size), orNever(d.Snapshot))
|
||||
return nil
|
||||
}
|
||||
|
||||
// keptOnUnassign says, for an unassignment, the irreplaceable and valuable data each module leaves in its
|
||||
// own directories on the machine:
|
||||
// kept, and retired at the self-check's next run.
|
||||
func keptOnUnassign(ctx context.Context, inv *inventory.Inventory, machine string, modules []string) []string {
|
||||
records, err := inv.Data(ctx)
|
||||
if err != nil {
|
||||
return []string{"what it leaves behind could not be read from the mesh's record: " + err.Error()}
|
||||
}
|
||||
var out []string
|
||||
for _, r := range records {
|
||||
if r.Machine != machine || r.DeletedAt != nil || !catalogue.Retires(r.Class) || !r.Owned {
|
||||
continue
|
||||
}
|
||||
for _, m := range modules {
|
||||
if r.Module == m {
|
||||
out = append(out, fmt.Sprintf("%s's %s (%s, %s) stays where it is: it is %s, so it is retired, "+
|
||||
"never removed — `cleanup list` shows it, `cleanup delete` alone removes it (novox/hq ADR 0233)",
|
||||
r.Module, r.Item, orUnknownPath(r.Path), sizeWords(r.Size), r.Class))
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,465 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
func bytesOf(n int64) *int64 { return &n }
|
||||
|
||||
func when(t time.Time) *time.Time { return &t }
|
||||
|
||||
func shelfFor(t *testing.T, manifests ...string) map[string]catalogue.Manifest {
|
||||
t.Helper()
|
||||
out := map[string]catalogue.Manifest{}
|
||||
for _, raw := range manifests {
|
||||
m, err := catalogue.ParseManifest([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out[m.Module] = m
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
const houseManifest = `{"module":"house","version":"1",
|
||||
"data":{"own":[{"id":"config","path":"${dir:config}","class":"irreplaceable","active":"1d"}]},
|
||||
"resources":[{"id":"config","type":"directory","mode":"0700"}]}`
|
||||
|
||||
func findingsByKind(obs []conditions.Observation) map[string]conditions.Observation {
|
||||
out := map[string]conditions.Observation{}
|
||||
for _, o := range linted(obs) {
|
||||
out[o.Kind] = o
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// THE INCIDENT (issue 273), replayed against what D13 reads: five applications on the home server bound,
|
||||
// by one changed rule, to the store on the control node, which made each an empty database — while
|
||||
// their real databases, hundreds of megabytes each, sat on the home server's own store, by then retired
|
||||
// because the mesh no longer asked for them there. Each is an empty replacement, naming both machines
|
||||
// and the pin back: a warning for the store's consumers, whose data is valuable; urgent for one that says
|
||||
// its data there is irreplaceable (`kept-by`).
|
||||
func TestAnEmptyReplacementOfAConsumersDataIsSaid(t *testing.T) {
|
||||
var copies []consumerCopy
|
||||
for _, app := range []string{"mesh_home_board", "mesh_home_flows", "mesh_home_agents", "mesh_home_game", "mesh_home_cars"} {
|
||||
copies = append(copies,
|
||||
consumerCopy{Node: "home", Module: "postgres", Consumer: app, Size: bytesOf(400 << 20), Retired: true, Class: "valuable"},
|
||||
consumerCopy{Node: "anchor", Module: "postgres", Consumer: app, Size: bytesOf(9 << 20), Class: "valuable"})
|
||||
}
|
||||
copies[1].Class = "irreplaceable" // the photo site's own database says so
|
||||
got := linted(dataFindings(nil, nil, nil, copies, nil, time.Now()))
|
||||
if len(got) != 5 {
|
||||
t.Fatalf("%d findings for five empty replacements: %+v", len(got), got)
|
||||
}
|
||||
for i, o := range got {
|
||||
want := conditions.Warning
|
||||
if strings.Contains(o.ID, "mesh_home_board") {
|
||||
want = conditions.Urgent
|
||||
}
|
||||
_ = i
|
||||
if o.Kind != kindEmptyReplacement || o.Severity != want || o.Machine != "anchor" ||
|
||||
len(o.Also) != 1 || o.Also[0] != "home" || !strings.Contains(o.Summary, "Pin it back to home") {
|
||||
t.Errorf("%+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
// While the old copy is still active (the first ten minutes), it is the same finding.
|
||||
copies[0].Retired = false
|
||||
copies[1].Class = "valuable"
|
||||
if got := dataFindings(nil, nil, nil, copies[:2], nil, time.Now()); len(got) != 1 || got[0].Kind != kindEmptyReplacement {
|
||||
t.Fatalf("with the old copy still active: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A move a person made — the data moved first, then pinned — leaves a full copy at the new provider and
|
||||
// a retired one at the old: nothing to say here; `cleanup` covers the old one.
|
||||
func TestADeliberateMoveIsNoEmptyReplacement(t *testing.T) {
|
||||
copies := []consumerCopy{
|
||||
{Node: "home", Module: "postgres", Consumer: "mesh_home_board", Size: bytesOf(400 << 20), Retired: true},
|
||||
{Node: "anchor", Module: "postgres", Consumer: "mesh_home_board", Size: bytesOf(402 << 20)},
|
||||
}
|
||||
if got := dataFindings(nil, nil, nil, copies, nil, time.Now()); len(got) != 0 {
|
||||
t.Fatalf("a deliberate move raised %+v", got)
|
||||
}
|
||||
// Two small databases differing by less than the floor are not a finding either.
|
||||
copies[0].Size, copies[1].Size = bytesOf(12<<20), bytesOf(8<<20)
|
||||
if got := dataFindings(nil, nil, nil, copies, nil, time.Now()); len(got) != 0 {
|
||||
t.Fatalf("noise between two empty databases raised %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Where a provider cannot say sizes, a consumer active at two providers is still said — as a warning,
|
||||
// since which one is empty cannot be told.
|
||||
func TestConsumerDataActiveTwiceWithoutSizesIsAWarning(t *testing.T) {
|
||||
copies := []consumerCopy{
|
||||
{Node: "home", Module: "minio", Consumer: "mesh_home_photos"},
|
||||
{Node: "anchor", Module: "minio", Consumer: "mesh_home_photos"},
|
||||
}
|
||||
got := linted(dataFindings(nil, nil, nil, copies, nil, time.Now()))
|
||||
if len(got) != 1 || got[0].Kind != kindDataHeldTwice || got[0].Severity != conditions.Warning {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The same incident for a module's own data: a module unassigned from one machine and assigned on
|
||||
// another starts over in an empty directory while its full one is kept, retired, where it was.
|
||||
func TestAnEmptyReplacementOfAModulesOwnDataIsUrgent(t *testing.T) {
|
||||
now := time.Now()
|
||||
retired := now.Add(-time.Hour)
|
||||
records := []inventory.DataRecord{
|
||||
{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable", Path: "/var/lib/house/config",
|
||||
Size: bytesOf(2 << 30), RetiredAt: &retired, FirstSeen: now.Add(-90 * 24 * time.Hour)},
|
||||
{Machine: "anchor", Module: "house", Item: "config", Class: "irreplaceable", Path: "/var/lib/house/config",
|
||||
Size: bytesOf(1 << 20), FirstSeen: now.Add(-time.Hour), LastWrite: when(now)},
|
||||
}
|
||||
got := findingsByKind(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now))
|
||||
o, ok := got[kindEmptyReplacement]
|
||||
if !ok || o.Severity != conditions.Urgent || o.Machine != "anchor" || o.Also[0] != "home" {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
// The same of a valuable item is a warning.
|
||||
records[0].Class, records[1].Class = "valuable", "valuable"
|
||||
if o := findingsByKind(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now))[kindEmptyReplacement]; o.Severity != conditions.Warning {
|
||||
t.Fatalf("a valuable empty replacement: %+v", o)
|
||||
}
|
||||
records[0].Class, records[1].Class = "irreplaceable", "irreplaceable"
|
||||
// Two machines running a module on purpose, both active, keep two sets of data: nothing to say.
|
||||
records[0].RetiredAt = nil
|
||||
if got := findingsByKind(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now)); got[kindEmptyReplacement].Kind != "" {
|
||||
t.Fatalf("two active copies were read as a replacement: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// An irreplaceable item that lost more than half of its largest size in a week is urgent; a smaller loss,
|
||||
// or a loss under the floor, is not a finding.
|
||||
func TestAShrinkOfMoreThanHalfIsUrgent(t *testing.T) {
|
||||
now := time.Now()
|
||||
r := inventory.DataRecord{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable",
|
||||
Size: bytesOf(300 << 20), FirstSeen: now.Add(-30 * 24 * time.Hour), LastWrite: when(now), LastBackup: when(now)}
|
||||
shelf := shelfFor(t, houseManifest)
|
||||
o := findingsByKind(dataFindings([]inventory.DataRecord{r}, map[string]int64{r.Key(): 1 << 30}, shelf, nil, nil, now))[kindDataShrank]
|
||||
if o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "shrank") {
|
||||
t.Fatalf("%+v", o)
|
||||
}
|
||||
for _, peak := range []int64{500 << 20, 20 << 20} {
|
||||
if got := findingsByKind(dataFindings([]inventory.DataRecord{r}, map[string]int64{r.Key(): peak}, shelf, nil, nil, now)); got[kindDataShrank].Kind != "" {
|
||||
t.Errorf("a peak of %d raised a shrink", peak)
|
||||
}
|
||||
}
|
||||
small := r
|
||||
small.Size = bytesOf(1 << 20)
|
||||
if got := findingsByKind(dataFindings([]inventory.DataRecord{small}, map[string]int64{r.Key(): 10 << 20}, shelf, nil, nil, now)); got[kindDataShrank].Kind != "" {
|
||||
t.Error("a loss under the floor raised a shrink")
|
||||
}
|
||||
}
|
||||
|
||||
// Data said to be written all the time and not written; data with no backup or an old one — urgent when
|
||||
// irreplaceable, a warning when valuable; and a new item given its bound before it is said.
|
||||
func TestQuietDataAndMissingBackupsAreSaidByClass(t *testing.T) {
|
||||
now := time.Now()
|
||||
shelf := shelfFor(t, houseManifest)
|
||||
r := inventory.DataRecord{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable",
|
||||
Size: bytesOf(1 << 30), FirstSeen: now.Add(-10 * 24 * time.Hour), LastWrite: when(now.Add(-3 * 24 * time.Hour)),
|
||||
LastBackup: when(now.Add(-72 * time.Hour))}
|
||||
got := findingsByKind(dataFindings([]inventory.DataRecord{r}, nil, shelf, nil, nil, now))
|
||||
if got[kindDataQuiet].Severity != conditions.Urgent || got[kindBackupStale].Severity != conditions.Urgent {
|
||||
t.Fatalf("irreplaceable: %+v", got)
|
||||
}
|
||||
valuable := r
|
||||
valuable.Class, valuable.MeasuredAt = "valuable", when(now) // measured: a holder is there to take its backup
|
||||
if got := findingsByKind(dataFindings([]inventory.DataRecord{valuable}, nil, shelf, nil, nil, now)); got[kindDataQuiet].Severity != conditions.Warning ||
|
||||
got[kindBackupStale].Severity != conditions.Warning {
|
||||
t.Fatalf("valuable: %+v", got)
|
||||
}
|
||||
never := r
|
||||
never.LastBackup = nil
|
||||
if o := findingsByKind(dataFindings([]inventory.DataRecord{never}, nil, shelf, nil, nil, now))[kindBackupStale]; !strings.Contains(o.Summary, "no good backup") {
|
||||
t.Fatalf("never backed up: %+v", o)
|
||||
}
|
||||
fresh := never
|
||||
fresh.FirstSeen, fresh.LastWrite = now.Add(-time.Hour), when(now)
|
||||
if got := dataFindings([]inventory.DataRecord{fresh}, nil, shelf, nil, nil, now); len(got) != 0 {
|
||||
t.Fatalf("an item declared an hour ago, before its first night, raised %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// An item retired more than thirty days waits for a person; less, it is only listed.
|
||||
func TestRetiredDataWaitingThirtyDaysIsSaid(t *testing.T) {
|
||||
now := time.Now()
|
||||
old, recent := now.Add(-31*24*time.Hour), now.Add(-2*24*time.Hour)
|
||||
records := []inventory.DataRecord{
|
||||
{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable", Size: bytesOf(1 << 30), RetiredAt: &old},
|
||||
{Machine: "home", Module: "attic", Item: "boxes", Class: "irreplaceable", Size: bytesOf(1 << 30), RetiredAt: &recent},
|
||||
}
|
||||
got := linted(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now))
|
||||
if len(got) != 1 || got[0].Kind != kindCleanupWaiting || !strings.Contains(got[0].Summary, "cleanup delete home house config") {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
if rows := retiredData(records, now); len(rows) != 2 || rows[0].Kind != retiredDataKind {
|
||||
t.Fatalf("cleanup list: %+v", rows)
|
||||
}
|
||||
}
|
||||
|
||||
// The holder's answer reads into measurements, by module and item.
|
||||
func TestTheHoldersAnswerIsRead(t *testing.T) {
|
||||
raw := []byte(`[{"module":"postgres","runs":1,"paths":["/var/lib/mesh-store/dumps"],"lastNight":null,"restorePoints":3,
|
||||
"data":[{"item":"store","class":"irreplaceable","path":"/var/lib/mesh-store","covered_by":"/var/lib/mesh-store/dumps",
|
||||
"size_bytes":1073741824,"last_write":"2026-10-06T10:00:00Z","measured_at":"2026-10-06T10:05:00Z","last_backup":"2026-10-06T03:10:00Z"}]}]`)
|
||||
got, err := readHolder(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := got["postgres"]["store"]
|
||||
if m.Path != "/var/lib/mesh-store" || m.Size == nil || *m.Size != 1<<30 || m.LastBackup == nil || m.MeasuredAt == nil {
|
||||
t.Fatalf("%+v", m)
|
||||
}
|
||||
// An older holder, which says no data, reads as nothing measured rather than a failure.
|
||||
if got, err := readHolder([]byte(`[{"module":"postgres","runs":1,"paths":[]}]`)); err != nil || len(got) != 0 {
|
||||
t.Fatalf("%v, %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeHolder answers node-backup's `backed-up` on one machine over a real bus, with what it is told it
|
||||
// measured.
|
||||
type fakeHolder struct {
|
||||
mu sync.Mutex
|
||||
modules []map[string]any
|
||||
}
|
||||
|
||||
func (f *fakeHolder) set(modules ...map[string]any) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.modules = modules
|
||||
}
|
||||
|
||||
func (f *fakeHolder) serve(t *testing.T, conn *nats.Conn, node string) {
|
||||
t.Helper()
|
||||
sub, err := conn.Subscribe(link.NodeSeatToolSubject(catalogue.BackupSeat, "backed-up", node), func(m *nats.Msg) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
body, _ := json.Marshal(map[string]any{"result": f.modules, "error": "", "node": node})
|
||||
_ = m.Respond(body)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
if err := conn.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func measuredHouse(path string, size int64, at time.Time) map[string]any {
|
||||
return map[string]any{"module": "house", "runs": 0, "paths": []string{path}, "data": []map[string]any{{
|
||||
"item": "config", "class": "irreplaceable", "path": path, "covered_by": path, "size_bytes": size,
|
||||
"last_write": at, "measured_at": at, "last_backup": at}}}
|
||||
}
|
||||
|
||||
// UNASSIGNING A MODULE WITH IRREPLACEABLE DATA KEEPS THE DATA, and assigning it elsewhere onto an empty
|
||||
// directory is an empty replacement — through the real stores and a real bus. The unassignment says the
|
||||
// data stays; the self-check's next run retires it (kept, listed by cleanup), and when the module comes
|
||||
// up on another machine with an empty directory while the full one waits retired, that is urgent.
|
||||
func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "keeper", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}})
|
||||
house, err := catalogue.ParseManifest([]byte(houseManifest))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, house)
|
||||
if _, err := assign(ctx, open, "laptop", "house"); err == nil {
|
||||
t.Fatal("irreplaceable data was assigned to a machine with nothing to back it up")
|
||||
}
|
||||
for _, node := range []string{"laptop", "anchor"} {
|
||||
if _, err := assign(ctx, open, node, "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "house"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Sent, and applied: a holder is asked only once it has been (novox/hq issue 275).
|
||||
for _, node := range []string{"laptop", "anchor"} {
|
||||
pushedAndApplied(t, open, node)
|
||||
}
|
||||
|
||||
conn := onATestBus(t)
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
laptop, anchor := &fakeHolder{}, &fakeHolder{}
|
||||
laptop.serve(t, conn, "laptop")
|
||||
anchor.serve(t, conn, "anchor")
|
||||
now := time.Now()
|
||||
heard := &watchdogs{last: &signalFacts{now: now, machines: []machineFacts{
|
||||
{name: "laptop", lastHeard: now}, {name: "anchor", lastHeard: now}}}}
|
||||
d := &doctor{open: open, js: js, watchdogs: heard}
|
||||
var d13 probe
|
||||
for _, p := range probeRegistry {
|
||||
if p.ID == probeDataID {
|
||||
d13 = p
|
||||
}
|
||||
}
|
||||
run := func() []conditions.Observation {
|
||||
t.Helper()
|
||||
probing := context.WithValue(ctx, probeAsksKey{}, d13)
|
||||
obs, err := probeData(probing, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return obs
|
||||
}
|
||||
|
||||
laptop.set(measuredHouse("/var/lib/house/config", 2<<30, now))
|
||||
if obs := run(); len(obs) != 0 {
|
||||
t.Fatalf("a measured, backed-up item raised %+v", obs)
|
||||
}
|
||||
r, err := inv.DataOf(ctx, "laptop", "house", "config")
|
||||
if err != nil || r.Path != "/var/lib/house/config" || r.Size == nil || *r.Size != 2<<30 || r.LastBackup == nil {
|
||||
t.Fatalf("what the holder measured was not kept: %+v, %v", r, err)
|
||||
}
|
||||
|
||||
said, err := unassign(ctx, open, "laptop", "house")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(said, "house's config (/var/lib/house/config, 2.0 GB) stays where it is") {
|
||||
t.Fatalf("the unassignment does not say the data stays:\n%s", said)
|
||||
}
|
||||
laptop.set()
|
||||
run()
|
||||
r, err = inv.DataOf(ctx, "laptop", "house", "config")
|
||||
if err != nil || !r.Retired() || r.Path != "/var/lib/house/config" {
|
||||
t.Fatalf("unassigned, the irreplaceable item is not kept retired: %+v, %v", r, err)
|
||||
}
|
||||
records, _ := inv.Data(ctx)
|
||||
if rows := retiredData(records, time.Now()); len(rows) != 1 || rows[0].Path != "/var/lib/house/config" {
|
||||
t.Fatalf("cleanup list: %+v", rows)
|
||||
}
|
||||
|
||||
// Assigned on the anchor, onto an empty directory.
|
||||
if _, err := assign(ctx, open, "anchor", "house"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
anchor.set(measuredHouse("/var/lib/house/config", 300<<10, time.Now()))
|
||||
obs := findingsByKind(run())
|
||||
o, ok := obs[kindEmptyReplacement]
|
||||
if !ok || o.Severity != conditions.Urgent || o.Machine != "anchor" || o.Also[0] != "laptop" {
|
||||
t.Fatalf("an empty replacement of a module's data was not urgent: %+v", obs)
|
||||
}
|
||||
}
|
||||
|
||||
// Data on redundant storage: the array it is on is watched, one condition per array as loud as the most
|
||||
// precious item on it; an item said to be on redundancy and found on plain storage is said; an item
|
||||
// whose path is gone is said.
|
||||
func TestTheArrayUnderDataIsWatched(t *testing.T) {
|
||||
now := time.Now()
|
||||
media := `{"module":"media","version":"1","accesses":[{"id":"films","mode":"read"},{"id":"shows","mode":"read"}],
|
||||
"data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"an array, no room to copy"},
|
||||
{"id":"shows","path":"${access:shows}","class":"irreplaceable","redundancy":"an array, no room to copy"}]}}`
|
||||
shelf := shelfFor(t, media)
|
||||
sick := false
|
||||
on := func(item string, healthy *bool) inventory.DataRecord {
|
||||
return inventory.DataRecord{Machine: "home", Module: "media", Item: item, Class: "irreplaceable", Owned: false,
|
||||
Path: "/tank/" + item, Size: bytesOf(40 << 40), FirstSeen: now.Add(-24 * time.Hour), MeasuredAt: when(now),
|
||||
Redundancy: &inventory.Redundancy{Kind: "zfs", Where: "tank", Healthy: healthy, Said: "pool 'tank' is DEGRADED"}}
|
||||
}
|
||||
got := linted(dataFindings([]inventory.DataRecord{on("films", &sick), on("shows", &sick)}, nil, shelf, nil, nil, now))
|
||||
if len(got) != 1 || got[0].Kind != kindArrayDegraded || got[0].Severity != conditions.Urgent ||
|
||||
!strings.Contains(got[0].Summary, "media/films, media/shows") {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
well := true
|
||||
if got := dataFindings([]inventory.DataRecord{on("films", &well)}, nil, shelf, nil, nil, now); len(got) != 0 {
|
||||
t.Fatalf("a healthy array raised %+v", got)
|
||||
}
|
||||
plain := on("films", nil)
|
||||
plain.Redundancy = nil
|
||||
if o := findingsByKind(dataFindings([]inventory.DataRecord{plain}, nil, shelf, nil, nil, now))[kindProtectionMissing]; o.Severity != conditions.Urgent {
|
||||
t.Fatalf("redundancy said and not found: %+v", o)
|
||||
}
|
||||
gone := on("films", &well)
|
||||
gone.MeasureError, gone.Size = "/tank/films does not exist", bytesOf(0)
|
||||
if o := findingsByKind(dataFindings([]inventory.DataRecord{gone}, map[string]int64{gone.Key(): 40 << 40}, shelf, nil, nil, now))[kindDataMissing]; o.Severity != conditions.Urgent {
|
||||
t.Fatalf("a vanished library: %+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
// What a consumer keeps with its provider as irreplaceable holds the provider's item to that class:
|
||||
// the photo site's objects make the object store's data an urgent matter.
|
||||
func TestKeptByHoldsTheProvidersItemToTheConsumersClass(t *testing.T) {
|
||||
objects := `{"module":"objects","version":"1","provides":[{"name":"s3-bucket","scope":"mesh"}],"grants":{"s3-bucket":"${dir:g}"},
|
||||
"data":{"own":[{"id":"data","path":"${dir:data}","class":"valuable"}],"consumers":{"s3-bucket":{"class":"valuable","in":"data"}}},
|
||||
"resources":[{"id":"g","type":"directory","mode":"0700"},{"id":"data","type":"directory","mode":"0700"}]}`
|
||||
photos := `{"module":"photos","version":"1","requires":["s3-bucket"],"data":{"kept-by":{"s3-bucket":{"class":"irreplaceable"}}}}`
|
||||
shelf := shelfFor(t, objects, photos)
|
||||
bindings := []inventory.Binding{{Machine: "anchor", Consumer: "photos", Provision: "s3-bucket",
|
||||
Provider: catalogue.Chosen{Node: "anchor", Module: "objects"}}}
|
||||
upgraded, keptBy := keptByClasses(bindings, shelf)
|
||||
if upgraded["anchor/objects/data"] != "irreplaceable" || keptBy["objects/mesh_anchor_photos"] != "irreplaceable" {
|
||||
t.Fatalf("upgraded %v, kept by %v", upgraded, keptBy)
|
||||
}
|
||||
now := time.Now()
|
||||
r := inventory.DataRecord{Machine: "anchor", Module: "objects", Item: "data", Class: "valuable", Owned: true,
|
||||
Size: bytesOf(10 << 30), FirstSeen: now.Add(-10 * 24 * time.Hour), MeasuredAt: when(now), LastBackup: when(now.Add(-72 * time.Hour))}
|
||||
if o := findingsByKind(dataFindings([]inventory.DataRecord{r}, nil, shelf, nil, upgraded, now))[kindBackupStale]; o.Severity != conditions.Urgent {
|
||||
t.Fatalf("the photos' store without a backup: %+v", o)
|
||||
}
|
||||
if o := findingsByKind(dataFindings([]inventory.DataRecord{r}, nil, shelf, nil, nil, now))[kindBackupStale]; o.Severity != conditions.Warning {
|
||||
t.Fatalf("a valuable store without a backup: %+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
// Items measured from one dataset's counters share its size: a shrink of the dataset is one condition
|
||||
// naming every item on it, not one per item; and a partial walk's lower bound is never compared.
|
||||
func TestADatasetShrinksOnceAndAPartialSizeIsNeverCompared(t *testing.T) {
|
||||
now := time.Now()
|
||||
media := `{"module":"media","version":"1","accesses":[{"id":"films","mode":"read"},{"id":"shows","mode":"read"}],
|
||||
"data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"an array","measure":"dataset"},
|
||||
{"id":"shows","path":"${access:shows}","class":"irreplaceable","redundancy":"an array","measure":"dataset"}]}}`
|
||||
shelf := shelfFor(t, media, houseManifest)
|
||||
well := true
|
||||
on := func(item string, size int64) inventory.DataRecord {
|
||||
return inventory.DataRecord{Machine: "home", Module: "media", Item: item, Class: "irreplaceable",
|
||||
Size: bytesOf(size), FirstSeen: now.Add(-24 * time.Hour), MeasuredAt: when(now),
|
||||
Precision: "dataset tank/media: its whole size",
|
||||
Redundancy: &inventory.Redundancy{Kind: "zfs", Where: "tank", Healthy: &well}}
|
||||
}
|
||||
films, shows := on("films", 30<<40), on("shows", 30<<40)
|
||||
peaks := map[string]int64{films.Key(): 90 << 40, shows.Key(): 90 << 40}
|
||||
got := linted(dataFindings([]inventory.DataRecord{films, shows}, peaks, shelf, nil, nil, now))
|
||||
if len(got) != 1 || got[0].Kind != kindDataShrank || got[0].Severity != conditions.Urgent ||
|
||||
!strings.Contains(got[0].Summary, "media/films, media/shows") {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
partial := inventory.DataRecord{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable",
|
||||
Size: bytesOf(1 << 20), FirstSeen: now.Add(-24 * time.Hour), MeasuredAt: when(now), LastWrite: when(now),
|
||||
LastBackup: when(now), Precision: "partial: measured partially"}
|
||||
if got := dataFindings([]inventory.DataRecord{partial}, map[string]int64{partial.Key(): 1 << 30}, shelf, nil, nil, now); len(got) != 0 {
|
||||
t.Fatalf("a partial size was compared: %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// What a consumer gave up on, answered by the serving controller (novox/hq issue 330).
|
||||
//
|
||||
// **In this process, on its own connection**: DEAD_LETTERS is read and changed on the bus, and the
|
||||
// serving controller is already on it. A verb run as a fresh process would open a connection of its own
|
||||
// for each call (novox/hq issue 327).
|
||||
|
||||
// busHandles are the serving controller's connection and JetStream handle.
|
||||
type busHandles struct {
|
||||
conn *nats.Conn
|
||||
js nats.JetStreamContext
|
||||
}
|
||||
|
||||
// defaultDeadLetters is how many the list says when not asked for more.
|
||||
const defaultDeadLetters = 50
|
||||
|
||||
// causeDeadLetter is the cause a delivery or drop gives when the caller gives none.
|
||||
const causeDeadLetter = "dead-letter"
|
||||
|
||||
// deadLettersAnswer is what `dead-letters` answers: the list, one whole, or what came of delivering one
|
||||
// again or dropping it.
|
||||
func deadLettersAnswer(ctx context.Context, a *verbArguments) (any, error) {
|
||||
serving := servingBus.Load()
|
||||
if serving == nil {
|
||||
return nil, errors.New("this controller is not serving, so it does not read DEAD_LETTERS: ask again, " +
|
||||
"and the serving controller answers")
|
||||
}
|
||||
on := &busHandles{conn: serving.Conn(), js: serving.Context()}
|
||||
// The shape first, and every argument it reads; one given beside it is refused before anything is
|
||||
// done, as every verb refuses what it would pass over (novox/hq issue 244).
|
||||
var deliver, drop, why, cause, idText, consumer, limit string
|
||||
switch {
|
||||
case a.given["deliver"] != "" || a.given["drop"] != "":
|
||||
deliver, drop, why, cause = a.str("deliver"), a.str("drop"), a.str("why"), a.str("cause")
|
||||
case a.given["id"] != "":
|
||||
idText = a.str("id")
|
||||
default:
|
||||
consumer, limit = a.str("consumer"), a.str("limit")
|
||||
}
|
||||
if unused := a.unused(); len(unused) > 0 {
|
||||
return nil, fmt.Errorf("dead-letters did not use %s together with %s, and an argument a verb would pass "+
|
||||
"over is refused: nothing was done", quoteAll(unused), quoteAll(a.usedGiven()))
|
||||
}
|
||||
switch {
|
||||
case deliver != "" && drop != "":
|
||||
return nil, errors.New("dead-letters delivers one again or drops one, not both. Nothing was done")
|
||||
case deliver != "" || drop != "":
|
||||
act, text := "deliver", deliver
|
||||
if drop != "" {
|
||||
act, text = "drop", drop
|
||||
}
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return nil, fmt.Errorf("dead-letters %s is a hand act, and says why: why is required and recorded in "+
|
||||
"the hand-act log (novox/hq to-be 45 §7). Nothing was done", act)
|
||||
}
|
||||
id, err := deadLetterID(text)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return actOnDeadLetter(ctx, on, act, id, why, cause)
|
||||
case idText != "":
|
||||
id, err := deadLetterID(idText)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return link.DeadLetterNamed(on.js, id)
|
||||
}
|
||||
most := defaultDeadLetters
|
||||
if limit != "" {
|
||||
n, err := strconv.Atoi(limit)
|
||||
if err != nil || n <= 0 {
|
||||
return nil, fmt.Errorf("limit is a number of dead letters, not %q", limit)
|
||||
}
|
||||
most = n
|
||||
}
|
||||
held, total, err := link.DeadLetters(on.js, consumer, most)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
answer := map[string]any{"dead_letters": held, "held": total,
|
||||
"note": "newest first; with id, one whole; deliver or drop one with why"}
|
||||
if total == 0 {
|
||||
answer["note"] = "no consumer gave up on a message that is still kept"
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// deadLetterID is a dead letter's id as a caller wrote it.
|
||||
func deadLetterID(text string) (uint64, error) {
|
||||
id, err := strconv.ParseUint(strings.TrimSpace(text), 10, 64)
|
||||
if err != nil || id == 0 {
|
||||
return 0, fmt.Errorf("a dead letter's id is its number in %s, as dead-letters lists it, not %q",
|
||||
broker.DeadLettersStream, text)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// actOnDeadLetter delivers one again or drops it, recorded in the hand-act log before it is done. A log
|
||||
// that cannot be written is said, and the act still happens: the log is never the reason a person's act
|
||||
// is refused (handacts.go).
|
||||
func actOnDeadLetter(ctx context.Context, on *busHandles, act string, id uint64, why, cause string) (any, error) {
|
||||
d, err := link.DeadLetterNamed(on.js, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if act == "deliver" {
|
||||
// Refused before it is recorded: an act that cannot be done is not an act.
|
||||
if _, err := link.AgainTo(on.js, d); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if cause == "" {
|
||||
cause = causeDeadLetter
|
||||
}
|
||||
by := link.CallerIn(ctx)
|
||||
if by == "" {
|
||||
by = "a seat call whose caller the bus did not name"
|
||||
}
|
||||
answer := map[string]any{"dead_letter": d.ID, "consumer": d.Who, "subject": d.Subject}
|
||||
recorded, logErr := link.RecordHandAct(ctx, on.conn, link.HandAct{Verb: "dead-letters " + act,
|
||||
Args: []string{strconv.FormatUint(id, 10), d.Stream + "." + d.Consumer}, Why: why, Cause: cause,
|
||||
Condition: conditions.Key(conditions.ScopeBus, d.Stream+"."+d.Consumer, link.AdvisoryMaxDeliveries),
|
||||
By: by + ", through the " + catalogue.ControllerSeatName + " seat"})
|
||||
if logErr != nil {
|
||||
answer["unrecorded"] = "the hand-act log could not be written, and the act was done all the same: " + logErr.Error()
|
||||
} else {
|
||||
answer["recorded"] = recorded.ID
|
||||
}
|
||||
switch act {
|
||||
case "deliver":
|
||||
_, to, err := link.DeliverAgain(on.js, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
answer["delivered_on"] = to
|
||||
answer["done"] = fmt.Sprintf("dead letter %d was delivered again to %s, and nobody else; it is no longer kept",
|
||||
id, consumerWho(d.Stream, d.Consumer))
|
||||
case "drop":
|
||||
if _, err := link.DropDeadLetter(on.js, id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
answer["done"] = fmt.Sprintf("dead letter %d, which %s gave up on, was dropped for good", id,
|
||||
consumerWho(d.Stream, d.Consumer))
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// The serving controller's bus, with the mesh's streams, for the verb to read and act on.
|
||||
func servingDeadLetters(t *testing.T) *broker.JetStream {
|
||||
t.Helper()
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := servingBus.Load()
|
||||
servingBus.Store(js)
|
||||
t.Cleanup(func() { servingBus.Store(before) })
|
||||
return js
|
||||
}
|
||||
|
||||
func askDeadLetters(t *testing.T, args map[string]any) (any, error) {
|
||||
t.Helper()
|
||||
a, err := readArguments("dead-letters", args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return deadLettersAnswer(context.Background(), a)
|
||||
}
|
||||
|
||||
func TestDeadLettersListsDropsAndRecordsWhy(t *testing.T) {
|
||||
js := servingDeadLetters(t)
|
||||
if _, err := js.Context().Publish("mesh.mod.gitea.event.pull.merged", []byte(`{"n":1}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, err := link.KeepDeadLetter(js.Context(), []byte(`{"stream":"EVENTS","consumer":"media_sonarr","stream_seq":1,"deliveries":5}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
answer, err := askDeadLetters(t, map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
listed := answer.(map[string]any)
|
||||
if listed["held"] != 1 || len(listed["dead_letters"].([]link.DeadLetter)) != 1 {
|
||||
t.Fatalf("listed %v", listed)
|
||||
}
|
||||
|
||||
// Refused before anything is done: an act without why, an argument the shape passes over, both acts.
|
||||
for _, args := range []map[string]any{
|
||||
{"drop": "1"},
|
||||
{"drop": "1", "why": "x", "limit": "3"},
|
||||
{"drop": "1", "deliver": "1", "why": "x"},
|
||||
{"why": "x"},
|
||||
{"id": "nought"},
|
||||
} {
|
||||
if _, err := askDeadLetters(t, args); err == nil {
|
||||
t.Errorf("%v was done", args)
|
||||
}
|
||||
}
|
||||
if _, total, _ := link.DeadLetters(js.Context(), "", 0); total != 1 {
|
||||
t.Fatalf("a refused call changed what is kept: %d left", total)
|
||||
}
|
||||
|
||||
done, err := askDeadLetters(t, map[string]any{"drop": "1", "why": "the media server took the download in by hand"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if said := done.(map[string]any); said["recorded"] == nil || !strings.Contains(said["done"].(string), "dropped") {
|
||||
t.Fatalf("answered %v", said)
|
||||
}
|
||||
acts, err := link.HandActs(context.Background(), js.Conn(), time.Now().Add(-time.Minute))
|
||||
if err != nil || len(acts) != 1 || acts[0].Verb != "dead-letters drop" || acts[0].Cause != causeDeadLetter ||
|
||||
!strings.Contains(acts[0].Condition, "media_sonarr") {
|
||||
t.Fatalf("recorded %+v (%v)", acts, err)
|
||||
}
|
||||
if !personsDecision(acts[0]) {
|
||||
t.Error("dropping a dead letter is counted as a repair, so S15 would want a healer for it")
|
||||
}
|
||||
if _, err := askDeadLetters(t, map[string]any{"id": "1"}); err == nil {
|
||||
t.Errorf("dead letter %d is still answered after it was dropped", d.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// Open while DEAD_LETTERS holds a message for the consumer, in words the operator reads in one pass:
|
||||
// what is held, and where to act.
|
||||
func TestAConsumersDeadLettersAreSaidUntilActedOn(t *testing.T) {
|
||||
f := &signalFacts{now: time.Now(), deadLetters: map[string]int{"EVENTS.media_sonarr": 4, "EVENTS.controller": 1}}
|
||||
found := watchDeadLetters(f)
|
||||
if len(found) != 2 {
|
||||
t.Fatalf("said %d conditions", len(found))
|
||||
}
|
||||
for _, o := range found {
|
||||
if o.Kind != "max-deliveries" || o.Severity != conditions.Warning || o.Needs == "" {
|
||||
t.Errorf("%+v", o)
|
||||
}
|
||||
if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Needs: o.Needs,
|
||||
Explanation: o.Explanation, Resolved: o.Resolved}, "media"); !ok {
|
||||
t.Errorf("%q is not plain: %s", o.Headline, why)
|
||||
}
|
||||
if !strings.Contains(o.Needs, "mesh MCP server") {
|
||||
t.Errorf("does not say where to act: %q", o.Needs)
|
||||
}
|
||||
}
|
||||
sonarr := found[1]
|
||||
if sonarr.ID != "EVENTS.media_sonarr" || sonarr.Machine != "media" ||
|
||||
sonarr.Headline != "Sonarr on media could not handle 4 messages" ||
|
||||
!strings.Contains(sonarr.Summary, "DEAD_LETTERS") {
|
||||
t.Errorf("%+v", sonarr)
|
||||
}
|
||||
if found[0].Headline != "The controller could not handle a message" {
|
||||
t.Errorf("%q", found[0].Headline)
|
||||
}
|
||||
// None held, none said: it clears when they are delivered again or dropped.
|
||||
if left := watchDeadLetters(&signalFacts{now: time.Now()}); len(left) != 0 {
|
||||
t.Fatalf("%v", left)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,768 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The controller's part in a delivery (novox/hq ADR 0239, to-be 47).
|
||||
//
|
||||
// **A delivery is mesh-delivery's; the walk is the controller's.** A delivery is one commit in one
|
||||
// repository from its pull request's head to every machine, and a delivery group a few of them sharing a
|
||||
// branch name; their states, order, holds and record belong to the module holding the `mesh-delivery`
|
||||
// seat. The controller keeps what it already owned — the planner, the gate, registration, sending, the
|
||||
// rollback — and the **walk**: one trunk commit's plan, tier by tier across the machines, one machine
|
||||
// first and judged at the gate (ADR 0236). What changes here is when a walk starts.
|
||||
//
|
||||
// - A walk that moves the controller, the node-engine, the node tools, the bus or mesh-delivery itself is
|
||||
// **on the controller's own path**: started by the merge, as every plan was. mesh-delivery cannot gate
|
||||
// or deliver itself, and nothing the core needs to be repaired may wait for it.
|
||||
// - Any other walk, **while the seat has a holder on record**, is opened by the merge and waits — nothing
|
||||
// asked, nothing registered, nothing sent — until mesh-delivery says `deliver`, or a person says
|
||||
// `plans go`. Nothing of it is registered before then, so no other send can carry it to a machine
|
||||
// before its turn (ADR 0236 §4a carries everything registered).
|
||||
// - With no holder on record, every walk starts at the merge, exactly as before this existed.
|
||||
//
|
||||
// The verbs mesh-delivery asks with are here: `delivery plan` (the planner's one answer for a diffset),
|
||||
// `delivery order` (a group's order from the graph), `delivery check` (a group's heads composed), `delivery
|
||||
// go`, `delivery stop` and `delivery walks`.
|
||||
|
||||
// onTheControllersPath are the modules whose walk never waits for the delivery's owner, and what each is.
|
||||
var onTheControllersPath = map[string]string{
|
||||
"mesh-controller": "the controller",
|
||||
"mesh-host": "the node-engine",
|
||||
"node-tools": "the node tools",
|
||||
"nats": "the bus",
|
||||
catalogue.DeliverySeat: "the delivery's owner",
|
||||
}
|
||||
|
||||
// deliverySeatHeld is whether a module claiming the delivery seat is assigned somewhere: the seat has a
|
||||
// holder on record. Read from the catalogue the merge handler already holds; never from whether the
|
||||
// holder answers, so a walk does not start by itself because mesh-delivery is down — that wait is said.
|
||||
func deliverySeatHeld(entries []inventory.Entry) bool {
|
||||
for _, e := range entries {
|
||||
if len(e.On) > 0 && e.Manifest.ClaimsSeat(catalogue.DeliverySeat) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// awaitsFor is what a new walk waits for: nil when it starts at once — no holder on record, or a module
|
||||
// on the controller's own path among those it moves.
|
||||
func awaitsFor(entries []inventory.Entry, modules []string) *inventory.PlanDelivery {
|
||||
if !deliverySeatHeld(entries) {
|
||||
return nil
|
||||
}
|
||||
for _, m := range modules {
|
||||
if _, own := onTheControllersPath[m]; own {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}
|
||||
}
|
||||
|
||||
// waitingNote is what a walk waiting for its word says, wherever it is read.
|
||||
func waitingNote(p inventory.Plan) string {
|
||||
return fmt.Sprintf("published; its walk waits for %s's word — `plans go %s --why …` starts it by hand",
|
||||
p.Delivery.Awaits, p.ID)
|
||||
}
|
||||
|
||||
// letGo gives a waiting walk its word: by the delivery's owner, or a person. The next advance asks its
|
||||
// first tier. Refused for a walk that does not wait.
|
||||
func letGo(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, error) {
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, err
|
||||
}
|
||||
defer release()
|
||||
p, err := inv.PlanByID(ctx, id)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, err
|
||||
}
|
||||
switch {
|
||||
case !p.Open():
|
||||
return p, fmt.Errorf("%s is %s: there is no walk to start", p.ID, p.State)
|
||||
case p.Delivery == nil || p.Delivery.Awaits == "":
|
||||
return p, fmt.Errorf("%s waits for nobody: it started at its merge", p.ID)
|
||||
case p.Delivery.Go != nil:
|
||||
return p, fmt.Errorf("%s was let go by %s at %s already", p.ID, p.Delivery.By,
|
||||
p.Delivery.Go.Local().Format("15:04:05"))
|
||||
}
|
||||
// **One walk at a time** (novox/hq ADR 0276): the word is taken, and the walk starts once no other walk is
|
||||
// started (advanceOnce), so the delivery's owner says it once and is not refused.
|
||||
now := time.Now().UTC()
|
||||
p.Delivery.Go, p.Delivery.By, p.Delivery.Why = &now, by, why
|
||||
p.Note = "let go by " + by + "; its first tier is asked next"
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
return p, err
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// stopWalk ends a walk on its delivery's word: failed, said as stopped by whom, why. What it asked still
|
||||
// builds and registers; nothing further is asked or sent.
|
||||
func stopWalk(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, error) {
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, err
|
||||
}
|
||||
defer release()
|
||||
p, err := inv.PlanByID(ctx, id)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, err
|
||||
}
|
||||
if !p.Open() {
|
||||
return p, fmt.Errorf("%s is already %s", p.ID, p.State)
|
||||
}
|
||||
if p.Delivery == nil {
|
||||
p.Delivery = &inventory.PlanDelivery{}
|
||||
}
|
||||
p.Delivery.Stopped, p.Delivery.StoppedWhy = by, why
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = fmt.Sprintf("stopped by %s at tier %d: %s", by, p.Tier, why)
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
return p, err
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// orderMember is one member of a group, as mesh-delivery says it.
|
||||
type orderMember struct {
|
||||
ID string `json:"id"`
|
||||
Repository string `json:"repository"`
|
||||
Base string `json:"base,omitempty"`
|
||||
Head string `json:"head,omitempty"`
|
||||
Number int `json:"number,omitempty"`
|
||||
Paths []string `json:"paths,omitempty"`
|
||||
PathsTruncated bool `json:"paths_truncated,omitempty"`
|
||||
Removed []string `json:"removed,omitempty"`
|
||||
ModuleDirs []string `json:"module_dirs,omitempty"`
|
||||
ModuleDirsSaid bool `json:"module_dirs_said,omitempty"`
|
||||
CloneURL string `json:"clone_url,omitempty"`
|
||||
// After are the repositories its pull request says it goes after (`after: <repository>`).
|
||||
After []string `json:"after,omitempty"`
|
||||
}
|
||||
|
||||
// pull is the member as the forge announced it.
|
||||
func (m orderMember) pull() link.PullUpdated {
|
||||
owner, repo, _ := strings.Cut(m.Repository, "/")
|
||||
base := m.Base
|
||||
if base == "" {
|
||||
base = "main"
|
||||
}
|
||||
return link.PullUpdated{Owner: owner, Repo: repo, Number: m.Number, Base: base, Commit: m.Head,
|
||||
CloneURL: m.CloneURL, Paths: m.Paths, PathsTruncated: m.PathsTruncated, Removed: m.Removed,
|
||||
ModuleDirs: m.ModuleDirs, ModuleDirsSaid: m.ModuleDirsSaid}
|
||||
}
|
||||
|
||||
// orderPair is one "before" among a group's members, why, and the rules it won over.
|
||||
type orderPair struct {
|
||||
Before string `json:"before"`
|
||||
After string `json:"after"`
|
||||
Why string `json:"why"`
|
||||
// Over is the rules that ordered the pair the other way and lost to Why, by precedence (novox/hq ADR
|
||||
// 0249): said, so a person reading the order sees which rule won.
|
||||
Over []string `json:"over,omitempty"`
|
||||
}
|
||||
|
||||
// orderContradiction is two members that rules of one rank order both ways: no precedence resolves it, so
|
||||
// the group is refused, the rules named, and Said says how a person declares the order.
|
||||
type orderContradiction struct {
|
||||
Members []string `json:"members"`
|
||||
Rules []orderPair `json:"rules"`
|
||||
Said string `json:"said"`
|
||||
}
|
||||
|
||||
// orderReach is what a member moves, as the planner says.
|
||||
type orderReach struct {
|
||||
Moved []string `json:"moved,omitempty"`
|
||||
Dependents []string `json:"dependents,omitempty"`
|
||||
New []string `json:"new,omitempty"`
|
||||
Manifests []string `json:"manifests,omitempty"`
|
||||
}
|
||||
|
||||
// groupOrder is a group's order: the members in it, every pair and why, and the cycle when there is one —
|
||||
// with each contradiction no precedence resolves.
|
||||
type groupOrder struct {
|
||||
Order []string `json:"order"`
|
||||
Pairs []orderPair `json:"pairs,omitempty"`
|
||||
Cycle []string `json:"cycle,omitempty"`
|
||||
Contradictions []orderContradiction `json:"contradictions,omitempty"`
|
||||
Reach map[string]orderReach `json:"reach,omitempty"`
|
||||
}
|
||||
|
||||
// The reasons a pair is ordered, in the words the delivery plan shows.
|
||||
const (
|
||||
orderDeclared = "declared"
|
||||
orderBuiltBy = "built by"
|
||||
orderVersionSkew = "version skew"
|
||||
orderEngineFirst = "engine before controller"
|
||||
)
|
||||
|
||||
// orderRank is a rule's precedence when two rules order one pair both ways (novox/hq ADR 0249), the lower
|
||||
// winning: what a person declared; then what the graph and the change say (built by, version skew); then
|
||||
// the rollout default, which infers a direction from the modules' names alone.
|
||||
func orderRank(why string) int {
|
||||
switch why {
|
||||
case orderDeclared:
|
||||
return 0
|
||||
case orderBuiltBy, orderVersionSkew:
|
||||
return 1
|
||||
case orderEngineFirst:
|
||||
return 2
|
||||
}
|
||||
return 3
|
||||
}
|
||||
|
||||
// orderOf is a group's order (novox/hq ADR 0239 decision 4, ADR 0249), pure. A member goes before another
|
||||
// when:
|
||||
//
|
||||
// - its pull request is named in the other's `after:` lines (declared);
|
||||
// - it moves a module the other's moved modules are built by, stand on, package or declare (built by);
|
||||
// - it moves the controller and the other changes any module's manifest (version skew: the newer
|
||||
// controller parses what the newer manifest says) — the node-engine's excepted, which goes first;
|
||||
// - it moves the node-engine and the other moves the controller (the witness reads nothing the controller
|
||||
// does not yet grant, and a controller sends nothing an older engine would refuse — ADR 0236's rollout
|
||||
// order).
|
||||
//
|
||||
// When rules order one pair both ways, the one of higher precedence wins (orderRank) and the pair says
|
||||
// what it won over. Rules of one rank both ways are a contradiction: both are kept as pairs, the members
|
||||
// are the cycle, and the contradiction says how to declare the order. Otherwise, by repository then id, so
|
||||
// every reading gives one order. A cycle through three members or more is named by its members, and none
|
||||
// of them is ordered.
|
||||
func orderOf(members []orderMember, reach map[string]orderReach, edges []inventory.Edge) groupOrder {
|
||||
out := groupOrder{Reach: reach}
|
||||
byID := map[string]orderMember{}
|
||||
for _, m := range members {
|
||||
byID[m.ID] = m
|
||||
}
|
||||
var claims []orderPair
|
||||
add := func(before, after, why string) {
|
||||
if before == after {
|
||||
return
|
||||
}
|
||||
for _, p := range claims {
|
||||
if p.Before == before && p.After == after && p.Why == why {
|
||||
return
|
||||
}
|
||||
}
|
||||
claims = append(claims, orderPair{Before: before, After: after, Why: why})
|
||||
}
|
||||
named := func(said, repository string) bool {
|
||||
said = strings.TrimSuffix(strings.TrimSpace(said), ".git")
|
||||
if strings.EqualFold(said, repository) {
|
||||
return true
|
||||
}
|
||||
_, repo, _ := strings.Cut(repository, "/")
|
||||
return strings.EqualFold(said, repo)
|
||||
}
|
||||
for _, a := range members {
|
||||
for _, b := range members {
|
||||
if a.ID == b.ID {
|
||||
continue
|
||||
}
|
||||
ra, rb := reach[a.ID], reach[b.ID]
|
||||
for _, said := range b.After {
|
||||
if named(said, a.Repository) {
|
||||
add(a.ID, b.ID, orderDeclared)
|
||||
}
|
||||
}
|
||||
for _, e := range edges {
|
||||
if slices.Contains(ra.Moved, e.To) && slices.Contains(rb.Moved, e.From) && e.From != e.To {
|
||||
add(a.ID, b.ID, orderBuiltBy)
|
||||
break
|
||||
}
|
||||
}
|
||||
if slices.Contains(ra.Moved, "mesh-controller") && len(rb.Manifests) > 0 &&
|
||||
!slices.Contains(rb.Moved, "mesh-controller") && !slices.Contains(rb.Moved, "mesh-host") {
|
||||
add(a.ID, b.ID, orderVersionSkew)
|
||||
}
|
||||
if slices.Contains(ra.Moved, "mesh-host") && slices.Contains(rb.Moved, "mesh-controller") &&
|
||||
!slices.Contains(ra.Moved, "mesh-controller") {
|
||||
add(a.ID, b.ID, orderEngineFirst)
|
||||
}
|
||||
}
|
||||
}
|
||||
out.Pairs, out.Contradictions = resolveClaims(claims, byID)
|
||||
sort.Slice(out.Pairs, func(i, j int) bool {
|
||||
if out.Pairs[i].Before != out.Pairs[j].Before {
|
||||
return out.Pairs[i].Before < out.Pairs[j].Before
|
||||
}
|
||||
return out.Pairs[i].After < out.Pairs[j].After
|
||||
})
|
||||
// Kahn's walk, the next always the first by repository and id among those with nothing before them.
|
||||
waiting := map[string]int{}
|
||||
for _, m := range members {
|
||||
waiting[m.ID] = 0
|
||||
}
|
||||
for _, p := range out.Pairs {
|
||||
waiting[p.After]++
|
||||
}
|
||||
done := map[string]bool{}
|
||||
for len(done) < len(members) {
|
||||
var ready []orderMember
|
||||
for _, m := range members {
|
||||
if !done[m.ID] && waiting[m.ID] == 0 {
|
||||
ready = append(ready, m)
|
||||
}
|
||||
}
|
||||
if len(ready) == 0 {
|
||||
for _, m := range members {
|
||||
if !done[m.ID] {
|
||||
out.Cycle = append(out.Cycle, m.ID)
|
||||
}
|
||||
}
|
||||
sort.Strings(out.Cycle)
|
||||
return out
|
||||
}
|
||||
sort.Slice(ready, func(i, j int) bool {
|
||||
if ready[i].Repository != ready[j].Repository {
|
||||
return ready[i].Repository < ready[j].Repository
|
||||
}
|
||||
return ready[i].ID < ready[j].ID
|
||||
})
|
||||
next := ready[0]
|
||||
done[next.ID] = true
|
||||
out.Order = append(out.Order, next.ID)
|
||||
for _, p := range out.Pairs {
|
||||
if p.Before == next.ID {
|
||||
waiting[p.After]--
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// orderRules is the rules in the order a pair names them when more than one says the same.
|
||||
var orderRules = []string{orderDeclared, orderBuiltBy, orderVersionSkew, orderEngineFirst}
|
||||
|
||||
// resolveClaims is the pairs the rules' claims come to (novox/hq ADR 0249): for each two members, the claims
|
||||
// of the highest precedence decide, and the claims they overrule are said on the pair. Claims of one rank
|
||||
// both ways are a contradiction: both pairs are kept, so the walk finds the cycle, and the contradiction
|
||||
// says how a person declares the order.
|
||||
func resolveClaims(claims []orderPair, byID map[string]orderMember) ([]orderPair, []orderContradiction) {
|
||||
byTwo := map[[2]string][]orderPair{}
|
||||
var keys [][2]string
|
||||
for _, c := range claims {
|
||||
k := [2]string{min(c.Before, c.After), max(c.Before, c.After)}
|
||||
if _, ok := byTwo[k]; !ok {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
byTwo[k] = append(byTwo[k], c)
|
||||
}
|
||||
slices.SortFunc(keys, func(a, b [2]string) int {
|
||||
if c := strings.Compare(a[0], b[0]); c != 0 {
|
||||
return c
|
||||
}
|
||||
return strings.Compare(a[1], b[1])
|
||||
})
|
||||
ruleIndex := func(why string) int { return slices.Index(orderRules, why) }
|
||||
var pairs []orderPair
|
||||
var contradictions []orderContradiction
|
||||
for _, k := range keys {
|
||||
cs := byTwo[k]
|
||||
slices.SortStableFunc(cs, func(a, b orderPair) int { return ruleIndex(a.Why) - ruleIndex(b.Why) })
|
||||
top := orderRank(cs[0].Why)
|
||||
var won []orderPair
|
||||
ways := map[string]bool{}
|
||||
for _, c := range cs {
|
||||
if orderRank(c.Why) == top {
|
||||
won = append(won, c)
|
||||
ways[c.Before] = true
|
||||
}
|
||||
}
|
||||
if len(ways) == 1 {
|
||||
p := orderPair{Before: won[0].Before, After: won[0].After, Why: won[0].Why}
|
||||
for _, c := range cs {
|
||||
if c.Before != p.Before && !slices.Contains(p.Over, c.Why) {
|
||||
p.Over = append(p.Over, c.Why)
|
||||
}
|
||||
}
|
||||
pairs = append(pairs, p)
|
||||
continue
|
||||
}
|
||||
// One rank both ways: no precedence resolves it.
|
||||
var first, second *orderPair
|
||||
var said []string
|
||||
for i := range won {
|
||||
c := won[i]
|
||||
if c.Before == k[0] && first == nil {
|
||||
first = &won[i]
|
||||
}
|
||||
if c.Before == k[1] && second == nil {
|
||||
second = &won[i]
|
||||
}
|
||||
said = append(said, fmt.Sprintf("%s puts %s first", c.Why, c.Before))
|
||||
}
|
||||
pairs = append(pairs, orderPair{Before: first.Before, After: first.After, Why: first.Why},
|
||||
orderPair{Before: second.Before, After: second.After, Why: second.Why})
|
||||
a, b := byID[k[0]], byID[k[1]]
|
||||
how := fmt.Sprintf("neither outranks the other: declare the order with a line `after: %s` in %s's "+
|
||||
"description, or `after: %s` in %s's", a.Repository, pullName(b), b.Repository, pullName(a))
|
||||
if top == orderRank(orderDeclared) {
|
||||
how = "each pull request declares it goes after the other: remove one of the `after:` lines"
|
||||
}
|
||||
contradictions = append(contradictions, orderContradiction{Members: []string{k[0], k[1]}, Rules: won,
|
||||
Said: fmt.Sprintf("%s and %s: %s; %s", k[0], k[1], strings.Join(said, ", "), how)})
|
||||
}
|
||||
return pairs, contradictions
|
||||
}
|
||||
|
||||
// pullName is a member as a person finds it: its pull request, or its id when no number was said.
|
||||
func pullName(m orderMember) string {
|
||||
if m.Number > 0 {
|
||||
return fmt.Sprintf("%s#%d", m.Repository, m.Number)
|
||||
}
|
||||
return m.ID
|
||||
}
|
||||
|
||||
// reachOfMembers is each member's reach, as the planner says it.
|
||||
func reachOfMembers(members []orderMember, entries []inventory.Entry, read map[string][]inventory.ReadRepository,
|
||||
edges []inventory.Edge) map[string]orderReach {
|
||||
out := map[string]orderReach{}
|
||||
for _, m := range members {
|
||||
s := pullScope(m.pull(), entries, read, edges)
|
||||
out[m.ID] = orderReach{Moved: s.Modules, Dependents: s.Dependents, New: s.New, Manifests: s.Manifests}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// deliveryCommand is `delivery`, the controller's verbs for the delivery's owner:
|
||||
//
|
||||
// delivery plan --repository owner/repo --head <commit> [--base main] --paths a,b [--module-dirs …] [--removed …]
|
||||
// delivery order --members <json>
|
||||
// delivery check --group <id> --members <json>
|
||||
// delivery go <plan> [--by <who>] [--why <text>]
|
||||
// delivery stop <plan> --why <text> [--by <who>]
|
||||
// delivery walks [-n 50] [--plan <id>]
|
||||
func deliveryCommand(ctx context.Context, args []string) error {
|
||||
if len(args) == 0 {
|
||||
return errors.New("delivery plan|order|check|go|stop|walks")
|
||||
}
|
||||
sub, rest := args[0], args[1:]
|
||||
set := flag.NewFlagSet("delivery "+sub, flag.ContinueOnError)
|
||||
repository := set.String("repository", "", "owner/repository")
|
||||
base := set.String("base", "main", "the branch it merges into")
|
||||
head := set.String("head", "", "the commit at hand")
|
||||
paths := set.String("paths", "", "the files it changes, comma-separated")
|
||||
moduleDirs := set.String("module-dirs", "", "the directories holding a module.json at the head, comma-separated")
|
||||
removed := set.String("removed", "", "the files it deletes, comma-separated")
|
||||
membersJSON := set.String("members", "", "a group's members, as JSON")
|
||||
group := set.String("group", "", "a delivery group's id")
|
||||
by := set.String("by", catalogue.DeliverySeat, "who says it")
|
||||
why := set.String("why", "", "why")
|
||||
limit := set.Int("n", 50, "how many ended walks to answer beside the open ones")
|
||||
planID := set.String("plan", "", "one walk")
|
||||
positionals, err := parseAround(set, rest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var members []orderMember
|
||||
if *membersJSON != "" {
|
||||
if err := json.Unmarshal([]byte(*membersJSON), &members); err != nil {
|
||||
return fmt.Errorf("the members are not readable: %w", err)
|
||||
}
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
answer := func(v any) error {
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
enc.SetIndent("", " ")
|
||||
return enc.Encode(v)
|
||||
}
|
||||
switch sub {
|
||||
case "plan":
|
||||
if *repository == "" || *head == "" && *paths == "" {
|
||||
return errors.New("delivery plan --repository owner/repo --head <commit> --paths a,b")
|
||||
}
|
||||
m := orderMember{ID: *repository + "@" + *head, Repository: *repository, Base: *base, Head: *head,
|
||||
Paths: splitList(*paths), Removed: splitList(*removed)}
|
||||
if d := moduleDirsOf(*moduleDirs); d != nil {
|
||||
m.ModuleDirs, m.ModuleDirsSaid = *d, true
|
||||
}
|
||||
entries, read, edges, err := theGraph(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s := pullScope(m.pull(), entries, read, edges)
|
||||
plan := changePlanOf(*repository, *base, *head, s.Reach, entries, func(module string) (inventory.Upgrade, bool) {
|
||||
u, err := inv.UpgradeOf(ctx, module)
|
||||
return u, err == nil
|
||||
})
|
||||
return answer(map[string]any{"plan": plan, "reach": orderReach{Moved: s.Modules, Dependents: s.Dependents,
|
||||
New: s.New, Manifests: s.Manifests}, "mesh": s.Mesh, "gated": s.gated()})
|
||||
case "order":
|
||||
if len(members) == 0 {
|
||||
return errors.New("delivery order --members <json>: a group has members")
|
||||
}
|
||||
entries, read, edges, err := theGraph(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return answer(orderOf(members, reachOfMembers(members, entries, read, edges), edges))
|
||||
case "check":
|
||||
if *group == "" && len(members) == 1 {
|
||||
// One head, checked again as the forge's announcement would have it (a recheck): the controller's
|
||||
// own path for a pull request, run because the delivery's owner asked.
|
||||
if err := sayingOnTheBus(ctx, func() error { return (following{open}).PullUpdated(ctx, members[0].pull()) }); err != nil {
|
||||
return err
|
||||
}
|
||||
return answer(map[string]any{"rechecked": members[0].ID})
|
||||
}
|
||||
if *group == "" || len(members) < 2 {
|
||||
return errors.New("delivery check --group <id> --members <json> (two heads or more), or --members <one head>")
|
||||
}
|
||||
id, err := askGroupCheck(ctx, open, *group, members)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return answer(map[string]any{"asked": id, "group": *group})
|
||||
case "go":
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("delivery go <plan> [--by <who>] [--why <text>]")
|
||||
}
|
||||
var p inventory.Plan
|
||||
if err := sayingOnTheBus(ctx, func() (err error) {
|
||||
p, err = letGo(ctx, inv, positionals[0], *by, strings.TrimSpace(*why))
|
||||
return err
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s (%s at %s) is let go by %s; its first tier is asked at the next pass\n", p.ID, p.Repository,
|
||||
short(p.Commit), *by)
|
||||
return nil
|
||||
case "stop":
|
||||
if len(positionals) != 1 || strings.TrimSpace(*why) == "" {
|
||||
return errors.New("delivery stop <plan> --why <text> [--by <who>]")
|
||||
}
|
||||
var p inventory.Plan
|
||||
if err := sayingOnTheBus(ctx, func() (err error) {
|
||||
p, err = stopWalk(ctx, inv, positionals[0], *by, strings.TrimSpace(*why))
|
||||
return err
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s stopped by %s at tier %d of %d; what was asked still builds and registers, nothing further "+
|
||||
"is asked or sent\n", p.ID, *by, p.Tier, len(p.Tiers))
|
||||
return nil
|
||||
case "walks":
|
||||
var walks []inventory.Plan
|
||||
if *planID != "" {
|
||||
p, err := inv.PlanByID(ctx, *planID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
walks = []inventory.Plan{p}
|
||||
} else {
|
||||
if walks, err = inv.OpenPlans(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
// And the batch being assembled (novox/hq ADR 0276): never a walk to link or let go, shown.
|
||||
batches, err := inv.Batches(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
walks = append(walks, batches...)
|
||||
recent, err := inv.RecentPlans(ctx, *limit)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, p := range recent {
|
||||
if !slices.ContainsFunc(walks, func(w inventory.Plan) bool { return w.ID == p.ID }) {
|
||||
walks = append(walks, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return answer(map[string]any{"held": deliverySeatHeld(entries), "walks": walks,
|
||||
"own-path": sortedKeysOf(ownPathWords())})
|
||||
}
|
||||
return fmt.Errorf("delivery %s: plan, order, check, go, stop or walks", sub)
|
||||
}
|
||||
|
||||
// ownPathWords is the controller's own path as words, for an answer.
|
||||
func ownPathWords() map[string]string { return onTheControllersPath }
|
||||
|
||||
// theGraph is what the planner reads.
|
||||
func theGraph(ctx context.Context, inv *inventory.Inventory) ([]inventory.Entry, map[string][]inventory.ReadRepository,
|
||||
[]inventory.Edge, error) {
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
read, err := readForPlanning(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
edges, err := inv.Dependencies(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
return entries, read, edges, nil
|
||||
}
|
||||
|
||||
// groupPrimary is the head a group's composed check is run from: the one moving the controller, which then
|
||||
// judges the group by itself; else the one moving the node-engine, whose validator judges; else the first.
|
||||
func groupPrimary(members []orderMember, reach map[string]orderReach) int {
|
||||
for _, want := range []string{"mesh-controller", "mesh-host"} {
|
||||
for i, m := range members {
|
||||
if slices.Contains(reach[m.ID].Moved, want) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// askGroupCheck asks the build seat for a group's composed check: every head laid over the mesh in turn,
|
||||
// judged as one future state (novox/hq ADR 0239). The verdict comes back as `checked` with the group's id,
|
||||
// for mesh-delivery; the forge's holder sets no status from it.
|
||||
func askGroupCheck(ctx context.Context, open *stores, group string, members []orderMember) (string, error) {
|
||||
entries, read, edges, err := theGraph(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
reach := reachOfMembers(members, entries, read, edges)
|
||||
primary := groupPrimary(members, reach)
|
||||
p := members[primary].pull()
|
||||
scope := pullScope(p, entries, read, edges)
|
||||
// The gate runs over what any member moves; a judge from the primary alone.
|
||||
for _, m := range members {
|
||||
r := reach[m.ID]
|
||||
scope.Modules = appendNew(scope.Modules, r.Moved...)
|
||||
scope.Dependents = appendNew(scope.Dependents, r.Dependents...)
|
||||
}
|
||||
request, err := checkRequestFor(ctx, open, p, scope, entries)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
request.Check.Group = group
|
||||
world, err := theRestOfTheMesh(ctx, open.inventory, shelfOf(entries), "")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for i, m := range members {
|
||||
if i == primary {
|
||||
continue
|
||||
}
|
||||
mp := m.pull()
|
||||
// As the mesh clones a module built from it; a repository no module is built from, from the forge.
|
||||
source := inventory.Source{Seat: gitSeat, Repository: mp.Owner + "/" + mp.Repo}
|
||||
for _, e := range entries {
|
||||
if !e.Provided && sameRepository(e.Source.Repository, link.SourceMoved{Owner: mp.Owner, Repo: mp.Repo}) {
|
||||
source = e.Source
|
||||
break
|
||||
}
|
||||
}
|
||||
url := source.Repository
|
||||
if source.Seat != "" {
|
||||
url, err = clonedFromSeat(world, source.Seat, source.Repository)
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s cannot be cloned for the group's check: %w", m.ID, err)
|
||||
}
|
||||
request.Check.Members = append(request.Check.Members, link.GroupMember{Owner: mp.Owner, Repo: mp.Repo,
|
||||
Number: mp.Number, Repository: url, Ref: mp.Commit, Paths: mp.Paths})
|
||||
}
|
||||
seat := buildSeatHeld(ctx)
|
||||
ask, err := askOverOn(seat)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer ask.Close()
|
||||
if err := ask.Ask(ctx, request); err != nil {
|
||||
return "", err
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "group %s: asked %s to check %d head(s) composed together, as %s\n", group, seat,
|
||||
len(members), request.ID)
|
||||
return request.ID, nil
|
||||
}
|
||||
|
||||
// appendNew appends what is not there yet.
|
||||
func appendNew(to []string, items ...string) []string {
|
||||
for _, i := range items {
|
||||
if !slices.Contains(to, i) {
|
||||
to = append(to, i)
|
||||
}
|
||||
}
|
||||
return to
|
||||
}
|
||||
|
||||
// shelfOf is the catalogue's manifests by name.
|
||||
func shelfOf(entries []inventory.Entry) map[string]catalogue.Manifest {
|
||||
shelf := map[string]catalogue.Manifest{}
|
||||
for _, e := range entries {
|
||||
shelf[e.Manifest.Module] = e.Manifest
|
||||
}
|
||||
return shelf
|
||||
}
|
||||
|
||||
// sayPlanMoved says a walk kept in a new state as the controller's `plan-moved`, the plan whole: what the
|
||||
// delivery it walks reads its steps from. Never in the way of the walk: said beside it, and a save that could
|
||||
// not be said is logged — the delivery's owner, which also asks for the walks it follows, finds it by
|
||||
// comparison. Records arriving out of order are told apart by the plan's revision.
|
||||
func sayPlanMoved(ctx context.Context, bus link.Bus, p inventory.Plan) {
|
||||
go publishPlanMoved(ctx, bus, p)
|
||||
}
|
||||
|
||||
func publishPlanMoved(ctx context.Context, bus link.Bus, p inventory.Plan) {
|
||||
body, err := json.Marshal(p)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
stating, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer stop()
|
||||
if err := bus.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeyPlanMoved, body); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: kept, and could not be said as plan-moved: %v\n", p.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// sayingOnTheBus runs a command that keeps walks or says verdicts with the bus to say them on: the serving
|
||||
// controller's, or a connection of the command's own — a verb runs as a command of its own, and what it kept
|
||||
// would otherwise be said by nobody until the delivery's owner read the walks back. Without a bus the command
|
||||
// still runs; what it did is found by comparison.
|
||||
func sayingOnTheBus(ctx context.Context, f func() error) error {
|
||||
if checkEvents != nil {
|
||||
return f()
|
||||
}
|
||||
ran := false
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
js, err := conn.JetStream()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bus := link.OverNATS{Conn: conn, JS: js}
|
||||
checkEvents = bus
|
||||
inventory.PlanSaved = func(p inventory.Plan) { publishPlanMoved(ctx, bus, p) }
|
||||
defer func() { checkEvents, inventory.PlanSaved = nil, nil }()
|
||||
ran = true
|
||||
return f()
|
||||
})
|
||||
if !ran {
|
||||
fmt.Fprintf(os.Stderr, "the bus cannot be reached (%v): what this does is not said, and is found by comparison\n", err)
|
||||
return f()
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A delivery held past its bound, said by the controller and healed by H2 from mesh-delivery's own table
|
||||
// (novox/hq ADR 0239 decision 9, to-be 47 Phase B). The conditions are the controller's, one owner: the
|
||||
// self-check reads the delivery owner's `stalled` (probe D14) and raises `delivery.<id>.stalled`; healer
|
||||
// H2 calls the owner's `close`, which takes only the transition the table names for that state, and only
|
||||
// the next observation says whether it worked (ADR 0231).
|
||||
|
||||
// probeDeliveriesID is the probe that reads the delivery's owner.
|
||||
const probeDeliveriesID = "D14"
|
||||
|
||||
// kindDeliveryStalled is what a delivery held past its bound raises: H2's kind, in the delivery scope.
|
||||
const kindDeliveryStalled = "stalled"
|
||||
|
||||
// deliveryOwnerWithin is how long the owner is given to answer.
|
||||
var deliveryOwnerWithin = 10 * time.Second
|
||||
|
||||
// stalledLine is one delivery past its bound, as mesh-delivery's `stalled` says it.
|
||||
type stalledLine struct {
|
||||
ID string `json:"id"`
|
||||
// Number is the pull request's, for a line of a head the forge never announced (novox/hq issue 347).
|
||||
Number int `json:"number,omitempty"`
|
||||
State string `json:"state"`
|
||||
For string `json:"for"`
|
||||
Bound string `json:"bound"`
|
||||
H2 string `json:"h2"`
|
||||
Says string `json:"says"`
|
||||
}
|
||||
|
||||
// operatorsOnly is whether the table leaves H2 nothing to do for the line: the state is the operator's.
|
||||
func (l stalledLine) operatorsOnly() bool { return l.H2 == "" || strings.HasPrefix(l.H2, "none") }
|
||||
|
||||
// askDeliveryOwner asks the holder of the mesh-delivery seat one of the verbs the controller is granted;
|
||||
// a seam a test replaces. The holder's own refusal is an error naming it.
|
||||
var askDeliveryOwner = func(ctx context.Context, conn *nats.Conn, verb string, args map[string]any) (json.RawMessage, error) {
|
||||
granted := false
|
||||
for _, v := range broker.VerbsTheControllerAsksTheDeliveryOwner {
|
||||
granted = granted || v.Verb == verb
|
||||
}
|
||||
if !granted {
|
||||
return nil, fmt.Errorf("the controller asks %s.%s, which its grant does not name", catalogue.DeliverySeat, verb)
|
||||
}
|
||||
if conn == nil {
|
||||
return nil, errors.New("this controller is not on the bus")
|
||||
}
|
||||
answer, err := link.AskMeshSeatTool(ctx, conn, catalogue.DeliverySeat, verb, args, deliveryOwnerWithin)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return nil, fmt.Errorf("%s.%s refused: %s", catalogue.DeliverySeat, verb, answer.Error)
|
||||
}
|
||||
return unwrapToolResult(answer.Result), nil
|
||||
}
|
||||
|
||||
// unwrapToolResult is a tool's answer whatever the runtime wrapped it in: the JSON itself, or the
|
||||
// protocol's content list holding it as text.
|
||||
func unwrapToolResult(raw json.RawMessage) json.RawMessage {
|
||||
var wrapped struct {
|
||||
Content []struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"content"`
|
||||
}
|
||||
if json.Unmarshal(raw, &wrapped) == nil && len(wrapped.Content) > 0 && json.Valid([]byte(wrapped.Content[0].Text)) {
|
||||
return json.RawMessage(wrapped.Content[0].Text)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
// deliveriesStalled is what the owner says is held past its bound; nothing when no holder is on record,
|
||||
// or none answers (D3 says that one).
|
||||
func deliveriesStalled(ctx context.Context, conn *nats.Conn, held bool) ([]stalledLine, error) {
|
||||
if !held {
|
||||
return nil, nil
|
||||
}
|
||||
raw, err := askDeliveryOwner(ctx, conn, "stalled", map[string]any{})
|
||||
if errors.Is(err, link.ErrNothingServes) {
|
||||
return nil, nil // the holder not running is D3's holder-silent, said once there
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var lines []stalledLine
|
||||
if err := json.Unmarshal(raw, &lines); err != nil {
|
||||
return nil, fmt.Errorf("%s.stalled answered something unreadable: %w", catalogue.DeliverySeat, err)
|
||||
}
|
||||
return lines, nil
|
||||
}
|
||||
|
||||
// stalledObservations are the conditions of what is stalled.
|
||||
func stalledObservations(lines []stalledLine) []conditions.Observation {
|
||||
out := make([]conditions.Observation, 0, len(lines))
|
||||
for _, l := range lines {
|
||||
o := conditions.Observation{Scope: conditions.ScopeDelivery, ID: l.ID, Kind: kindDeliveryStalled,
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the delivery %s has been %s for %s, past its bound of %s (%s): healer H2 may %s — "+
|
||||
"`mesh-delivery.show %s`", l.ID, l.State, l.For, l.Bound, l.Says, l.H2, l.ID),
|
||||
Said: fmt.Sprintf("%s for %s", l.State, l.For)}
|
||||
if l.operatorsOnly() {
|
||||
o.Resolver = conditions.ResolverOperator
|
||||
}
|
||||
o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions = stalledWords(l, o)
|
||||
out = append(out, o)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// probeDeliveries is D14: every delivery held past its state's bound is said.
|
||||
func probeDeliveries(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
entries, err := d.open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var conn *nats.Conn
|
||||
if d.js != nil {
|
||||
conn = d.js.Conn()
|
||||
}
|
||||
lines, err := deliveriesStalled(ctx, conn, deliverySeatHeld(entries))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return stalledObservations(lines), nil
|
||||
}
|
||||
|
||||
// --- H2, for a delivery ---------------------------------------------------------------------------------
|
||||
|
||||
// connOf is the bus a healer asks over.
|
||||
func (h *healing) connOf() *nats.Conn {
|
||||
if h.js == nil {
|
||||
return nil
|
||||
}
|
||||
return h.js.Conn()
|
||||
}
|
||||
|
||||
// appliesToAStalledDelivery is H2's for a delivery: the owner still lists it, with a transition the table
|
||||
// lets H2 take; never one whose state is the operator's.
|
||||
func appliesToAStalledDelivery(ctx context.Context, h *healing, c conditions.Condition) (string, bool, string, error) {
|
||||
lines, err := deliveriesStalled(ctx, h.connOf(), true)
|
||||
if err != nil {
|
||||
return "", false, "", err
|
||||
}
|
||||
for _, l := range lines {
|
||||
if l.ID != c.Subject.ID {
|
||||
continue
|
||||
}
|
||||
if l.operatorsOnly() {
|
||||
return "", false, "the delivery is " + l.State + ", a state the table leaves to the operator", nil
|
||||
}
|
||||
return c.Key, true, "", nil
|
||||
}
|
||||
return "", false, "the delivery's owner no longer lists it as stalled: its condition clears on the next look", nil
|
||||
}
|
||||
|
||||
// repairDelivery is H2 for a delivery: the owner's `close`, which reads the walk again and takes only the
|
||||
// transition its table names. A refusal is no repair, said; the next observation says whether it worked.
|
||||
func repairDelivery(ctx context.Context, h *healing, c conditions.Condition) (string, string, error) {
|
||||
raw, err := askDeliveryOwner(ctx, h.connOf(), "close", map[string]any{"id": c.Subject.ID, "why": c.Key})
|
||||
if err != nil {
|
||||
if errors.Is(err, link.ErrNothingServes) {
|
||||
return "", "", err
|
||||
}
|
||||
return "closed nothing", oneLine(err.Error()), nil
|
||||
}
|
||||
var said string
|
||||
if json.Unmarshal(raw, &said) != nil {
|
||||
said = string(raw)
|
||||
}
|
||||
return "asked " + catalogue.DeliverySeat + " to close " + c.Subject.ID, said, nil
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0239 decision 9, to-be 47 Phase B: a delivery held past its bound is the controller's
|
||||
// condition, read from mesh-delivery's `stalled`; H2 takes the table's transition through its `close`.
|
||||
|
||||
// ownerAnswers replaces the delivery owner with one that answers stalled with these lines and records
|
||||
// what close was asked.
|
||||
func ownerAnswers(t *testing.T, lines []stalledLine, closeErr error) *[]string {
|
||||
t.Helper()
|
||||
var closed []string
|
||||
was := askDeliveryOwner
|
||||
askDeliveryOwner = func(_ context.Context, _ *nats.Conn, verb string, args map[string]any) (json.RawMessage, error) {
|
||||
switch verb {
|
||||
case "stalled":
|
||||
raw, _ := json.Marshal(lines)
|
||||
return raw, nil
|
||||
case "close":
|
||||
closed = append(closed, args["id"].(string))
|
||||
if closeErr != nil {
|
||||
return nil, closeErr
|
||||
}
|
||||
return json.RawMessage(`"` + args["id"].(string) + `: delivering → delivered, as its walk's record says"`), nil
|
||||
}
|
||||
t.Fatalf("asked the owner %s", verb)
|
||||
return nil, nil
|
||||
}
|
||||
t.Cleanup(func() { askDeliveryOwner = was })
|
||||
return &closed
|
||||
}
|
||||
|
||||
func holdTheDeliverySeat(t *testing.T, open *stores) {
|
||||
t.Helper()
|
||||
m := catalogue.Manifest{Module: "mesh-delivery", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}
|
||||
if err := open.inventory.RegisterModule(t.Context(), m, inventory.Source{Repository: "novox/mesh-catalog",
|
||||
Path: "modules/mesh-delivery", BuiltFrom: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.Assign(t.Context(), "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
var twoStalled = []stalledLine{
|
||||
{ID: "novox/app@aaaaaaaaaaaa", State: "delivering", For: "3h0m0s", Bound: "2h0m0s",
|
||||
H2: "close, by done or superseded or failed, when the walk's record says so", Says: "its walk runs"},
|
||||
{ID: "novox/lab@bbbbbbbbbbbb", State: "held", For: "49h0m0s", Bound: "24h0m0s",
|
||||
H2: "none: the state is the operator's", Says: "it waits for the operator"},
|
||||
}
|
||||
|
||||
func TestD14SaysEveryDeliveryHeldPastItsBound(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
ownerAnswers(t, twoStalled, nil)
|
||||
d := &doctor{open: open}
|
||||
got, err := probeDeliveries(ctx, d)
|
||||
if err != nil || len(got) != 0 {
|
||||
t.Fatalf("with no holder on record D14 said %+v %v", got, err)
|
||||
}
|
||||
holdTheDeliverySeat(t, open)
|
||||
got, err = probeDeliveries(ctx, d)
|
||||
if err != nil || len(got) != 2 {
|
||||
t.Fatalf("D14 said %+v %v", got, err)
|
||||
}
|
||||
if got[0].Key() != "delivery.novox/app_aaaaaaaaaaaa.stalled" || got[0].Severity != conditions.Warning ||
|
||||
got[0].Resolver != "" || !strings.Contains(got[0].Summary, "mesh-delivery.show novox/app@aaaaaaaaaaaa") {
|
||||
t.Fatalf("the first is %+v", got[0])
|
||||
}
|
||||
if got[1].Resolver != conditions.ResolverOperator {
|
||||
t.Fatalf("a held delivery is not the operator's: %+v", got[1])
|
||||
}
|
||||
// The holder not running is D3's to say, not D14's.
|
||||
was := askDeliveryOwner
|
||||
askDeliveryOwner = func(context.Context, *nats.Conn, string, map[string]any) (json.RawMessage, error) {
|
||||
return nil, link.ErrNothingServes
|
||||
}
|
||||
defer func() { askDeliveryOwner = was }()
|
||||
if got, err := probeDeliveries(ctx, d); err != nil || len(got) != 0 {
|
||||
t.Fatalf("an owner that is down was said by D14: %+v %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// H2 for a delivery: close asked for the one whose state the table gives H2, never for the operator's; a
|
||||
// refusal is no repair.
|
||||
func TestH2ClosesAStalledDeliveryThroughItsOwnerOnly(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
h, told, _ := healingOn(t, open)
|
||||
closed := ownerAnswers(t, twoStalled, nil)
|
||||
for _, o := range stalledObservations(twoStalled) {
|
||||
o.Source = probeDeliveriesID
|
||||
if _, err := conditionsFrom.Observe(ctx, o); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
h.tick(ctx)
|
||||
if !slices.Equal(*closed, []string{"novox/app@aaaaaaaaaaaa"}) {
|
||||
t.Fatalf("close was asked for %v", *closed)
|
||||
}
|
||||
acts := told.said()
|
||||
if len(acts) != 1 || acts[0].Healer != "H2" || acts[0].Condition != "delivery.novox/app_aaaaaaaaaaaa.stalled" {
|
||||
t.Fatalf("said %+v", acts)
|
||||
}
|
||||
// A refusal: closed nothing, said so.
|
||||
c, _, _ := conditionsFrom.Get(ctx, "delivery.novox/app_aaaaaaaaaaaa.stalled")
|
||||
ownerAnswers(t, twoStalled, errors.New("mesh-delivery.close refused: still delivering"))
|
||||
act, said, err := repairDelivery(ctx, h, c)
|
||||
if err != nil || act != "closed nothing" || !strings.Contains(said, "still delivering") {
|
||||
t.Fatalf("a refused close reads %q %q %v", act, said, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The controller may ask the delivery's owner what D14 and H2 ask, on its flat subjects, and nothing else
|
||||
// of it; and over a real bus the answer — wrapped as the runtime wraps it — is read.
|
||||
func TestTheDeliveryOwnerIsAskedOverTheBus(t *testing.T) {
|
||||
granted, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// And release and stop, which the operator's warrant chooses (novox/hq ADR 0259).
|
||||
for _, verb := range []string{"stalled", "close", "release", "stop"} {
|
||||
if !slices.Contains(granted.Publish, link.SeatToolSubject(catalogue.DeliverySeat, verb)) {
|
||||
t.Errorf("the controller may not ask %s.%s", catalogue.DeliverySeat, verb)
|
||||
}
|
||||
}
|
||||
if _, err := askDeliveryOwner(t.Context(), nil, "retire-history", nil); err == nil || !strings.Contains(err.Error(), "grant") {
|
||||
t.Fatalf("a verb the grant does not name was asked: %v", err)
|
||||
}
|
||||
conn, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
if _, err := deliveriesStalled(t.Context(), conn, true); err != nil {
|
||||
t.Fatalf("an owner not running is D3's, not an error: %v", err)
|
||||
}
|
||||
lines, _ := json.Marshal(twoStalled)
|
||||
wrapped, _ := json.Marshal(map[string]any{"content": []map[string]any{{"type": "text", "text": string(lines)}}})
|
||||
sub, err := conn.Subscribe(link.SeatToolSubject(catalogue.DeliverySeat, "stalled"), func(m *nats.Msg) {
|
||||
body, _ := json.Marshal(link.Answer{Result: wrapped})
|
||||
_ = m.Respond(body)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
got, err := deliveriesStalled(ctx, conn, true)
|
||||
if err != nil || len(got) != 2 || got[0].ID != "novox/app@aaaaaaaaaaaa" {
|
||||
t.Fatalf("over the bus: %+v %v", got, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,393 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0239 decision 4: a group's order, from the graph and the pull requests, the same on every
|
||||
// reading — the build agent before what it builds, the controller before a manifest that needs it, the
|
||||
// node-engine before the controller, a declared `after:` — and a contradiction named, never ordered.
|
||||
func TestAGroupIsOrderedByTheGraphAndWhatItsPullRequestsSay(t *testing.T) {
|
||||
members := []orderMember{
|
||||
{ID: "cat", Repository: "novox/mesh-catalog"},
|
||||
{ID: "ctl", Repository: "novox/mesh-controller"},
|
||||
{ID: "host", Repository: "novox/mesh-host"},
|
||||
{ID: "agent", Repository: "novox/build-agent"},
|
||||
{ID: "app", Repository: "novox/app", After: []string{"lab"}},
|
||||
{ID: "lab", Repository: "novox/lab"},
|
||||
}
|
||||
reach := map[string]orderReach{
|
||||
"cat": {Moved: []string{"gitea"}, Manifests: []string{"modules/gitea/module.json"}},
|
||||
"ctl": {Moved: []string{"mesh-controller"}, Manifests: []string{"module.json"}},
|
||||
"host": {Moved: []string{"mesh-host"}, Manifests: []string{"module.json"}},
|
||||
"agent": {Moved: []string{"build-agent"}},
|
||||
"app": {Moved: []string{"app"}},
|
||||
"lab": {Moved: []string{"lab"}},
|
||||
}
|
||||
edges := []inventory.Edge{{From: "app", To: "build-agent", Kind: inventory.EdgeBuiltBy},
|
||||
{From: "gitea", To: "postgres", Kind: inventory.EdgeDeclared}}
|
||||
got := orderOf(members, reach, edges)
|
||||
if len(got.Cycle) > 0 {
|
||||
t.Fatalf("a cycle where there is none: %v", got.Cycle)
|
||||
}
|
||||
// By repository among those with nothing before them: the build agent, the lab, then what waited on both.
|
||||
want := []string{"agent", "lab", "app", "host", "ctl", "cat"}
|
||||
if !reflect.DeepEqual(got.Order, want) {
|
||||
t.Fatalf("ordered %v, wanted %v; pairs %+v", got.Order, want, got.Pairs)
|
||||
}
|
||||
why := map[string]string{}
|
||||
for _, p := range got.Pairs {
|
||||
why[p.Before+">"+p.After] = p.Why
|
||||
}
|
||||
for pair, reason := range map[string]string{"host>ctl": orderEngineFirst, "ctl>cat": orderVersionSkew,
|
||||
"ctl>host": "", "agent>app": orderBuiltBy, "lab>app": orderDeclared} {
|
||||
if why[pair] != reason {
|
||||
t.Errorf("%s is ordered %q, wanted %q", pair, why[pair], reason)
|
||||
}
|
||||
}
|
||||
// The same members in another order read the same.
|
||||
shuffled := []orderMember{members[5], members[3], members[0], members[4], members[2], members[1]}
|
||||
if again := orderOf(shuffled, reach, edges); !reflect.DeepEqual(again.Order, want) {
|
||||
t.Fatalf("another reading ordered %v", again.Order)
|
||||
}
|
||||
|
||||
// A declared order against an inferred one wins (novox/hq ADR 0249), and the pair says what it won over.
|
||||
members[1].After = []string{"mesh-catalog"}
|
||||
got = orderOf(members, reach, edges)
|
||||
if len(got.Cycle) > 0 || len(got.Contradictions) > 0 {
|
||||
t.Fatalf("a declared order did not win: cycle %v, %+v", got.Cycle, got.Contradictions)
|
||||
}
|
||||
if want := []string{"agent", "lab", "app", "cat", "host", "ctl"}; !reflect.DeepEqual(got.Order, want) {
|
||||
t.Fatalf("ordered %v, wanted %v", got.Order, want)
|
||||
}
|
||||
if !slices.ContainsFunc(got.Pairs, func(p orderPair) bool {
|
||||
return p.Before == "cat" && p.After == "ctl" && p.Why == orderDeclared && reflect.DeepEqual(p.Over, []string{orderVersionSkew})
|
||||
}) || slices.ContainsFunc(got.Pairs, func(p orderPair) bool { return p.Before == "ctl" && p.After == "cat" }) {
|
||||
t.Fatalf("the declared pair does not say it won over version skew: %+v", got.Pairs)
|
||||
}
|
||||
|
||||
// Declared both ways is a contradiction no precedence resolves: a cycle, and how to end it.
|
||||
members[0].After = []string{"novox/mesh-controller"}
|
||||
got = orderOf(members, reach, edges)
|
||||
if !reflect.DeepEqual(got.Cycle, []string{"cat", "ctl"}) || len(got.Contradictions) != 1 {
|
||||
t.Fatalf("the cycle is %v, ordered %v, contradictions %+v", got.Cycle, got.Order, got.Contradictions)
|
||||
}
|
||||
if c := got.Contradictions[0]; !reflect.DeepEqual(c.Members, []string{"cat", "ctl"}) ||
|
||||
!strings.Contains(c.Said, "remove one of the `after:` lines") {
|
||||
t.Fatalf("the contradiction is said %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0249: when two rules order one pair both ways, the rule of higher precedence wins and the
|
||||
// pair says what it overruled; rules of one rank both ways are refused, naming both and how to declare.
|
||||
func TestAnOrderRuleYieldsToAStrongerOne(t *testing.T) {
|
||||
members := []orderMember{
|
||||
{ID: "ctl", Repository: "novox/mesh-controller", Number: 7},
|
||||
{ID: "host", Repository: "novox/mesh-host", Number: 9},
|
||||
}
|
||||
reach := map[string]orderReach{
|
||||
"ctl": {Moved: []string{"mesh-controller", "build-agent"}},
|
||||
"host": {Moved: []string{"mesh-host"}},
|
||||
}
|
||||
builtBy := []inventory.Edge{{From: "mesh-host", To: "build-agent", Kind: inventory.EdgeBuiltBy}}
|
||||
|
||||
// Built by outranks engine before controller.
|
||||
got := orderOf(members, reach, builtBy)
|
||||
if !reflect.DeepEqual(got.Order, []string{"ctl", "host"}) || len(got.Cycle) > 0 {
|
||||
t.Fatalf("ordered %v, cycle %v", got.Order, got.Cycle)
|
||||
}
|
||||
if want := []orderPair{{Before: "ctl", After: "host", Why: orderBuiltBy, Over: []string{orderEngineFirst}}}; !reflect.DeepEqual(got.Pairs, want) {
|
||||
t.Fatalf("pairs %+v, wanted %+v", got.Pairs, want)
|
||||
}
|
||||
|
||||
// A declared line outranks both.
|
||||
members[0].After = []string{"mesh-host"}
|
||||
got = orderOf(members, reach, builtBy)
|
||||
if !reflect.DeepEqual(got.Order, []string{"host", "ctl"}) || len(got.Cycle) > 0 ||
|
||||
!reflect.DeepEqual(got.Pairs[0].Over, []string{orderBuiltBy}) || got.Pairs[0].Why != orderDeclared {
|
||||
t.Fatalf("a declared order did not win: %v %+v", got.Order, got.Pairs)
|
||||
}
|
||||
|
||||
// Without the build agent, engine before controller stands alone.
|
||||
members[0].After = nil
|
||||
got = orderOf(members, map[string]orderReach{"ctl": {Moved: []string{"mesh-controller"}}, "host": reach["host"]}, builtBy)
|
||||
if !reflect.DeepEqual(got.Order, []string{"host", "ctl"}) || got.Pairs[0].Why != orderEngineFirst || len(got.Pairs[0].Over) > 0 {
|
||||
t.Fatalf("engine before controller alone: %v %+v", got.Order, got.Pairs)
|
||||
}
|
||||
|
||||
// Built by both ways is one rank against itself: refused, both rules named, and how to declare.
|
||||
both := append(builtBy, inventory.Edge{From: "build-agent", To: "mesh-host", Kind: inventory.EdgeStandsOn})
|
||||
got = orderOf(members, reach, both)
|
||||
if !reflect.DeepEqual(got.Cycle, []string{"ctl", "host"}) || len(got.Order) > 0 || len(got.Contradictions) != 1 {
|
||||
t.Fatalf("a contradiction was ordered: %v, cycle %v, %+v", got.Order, got.Cycle, got.Contradictions)
|
||||
}
|
||||
said := got.Contradictions[0].Said
|
||||
for _, w := range []string{"built by puts ctl first", "built by puts host first",
|
||||
"`after: novox/mesh-controller` in novox/mesh-host#9's description", "`after: novox/mesh-host` in novox/mesh-controller#7's"} {
|
||||
if !strings.Contains(said, w) {
|
||||
t.Errorf("the contradiction does not say %q: %s", w, said)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0239 decision 8: a walk waits for the delivery's owner only while the seat has a holder on
|
||||
// record, and never one that moves a module on the controller's own path.
|
||||
func TestAWalkWaitsOnlyWhileTheDeliverySeatIsHeldAndNeverForTheCore(t *testing.T) {
|
||||
holder := inventory.Entry{Manifest: catalogue.Manifest{Module: "mesh-delivery",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, On: []string{"anchor"}}
|
||||
unassigned := holder
|
||||
unassigned.On = nil
|
||||
for _, c := range []struct {
|
||||
entries []inventory.Entry
|
||||
moved []string
|
||||
waits bool
|
||||
}{
|
||||
{nil, []string{"gitea"}, false},
|
||||
{[]inventory.Entry{unassigned}, []string{"gitea"}, false},
|
||||
{[]inventory.Entry{holder}, []string{"gitea", "plex"}, true},
|
||||
{[]inventory.Entry{holder}, []string{"gitea", "mesh-controller"}, false},
|
||||
{[]inventory.Entry{holder}, []string{"mesh-host"}, false},
|
||||
{[]inventory.Entry{holder}, []string{"node-tools"}, false},
|
||||
{[]inventory.Entry{holder}, []string{"nats"}, false},
|
||||
{[]inventory.Entry{holder}, []string{"mesh-delivery", "gitea"}, false},
|
||||
} {
|
||||
d := awaitsFor(c.entries, c.moved)
|
||||
if (d != nil) != c.waits {
|
||||
t.Errorf("held %v, moving %v: waits %v, wanted %v", len(c.entries) > 0 && len(c.entries[0].On) > 0,
|
||||
c.moved, d != nil, c.waits)
|
||||
}
|
||||
if d != nil && d.Awaits != catalogue.DeliverySeat {
|
||||
t.Errorf("waits for %q", d.Awaits)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0239: with mesh-delivery on record, a merge opens its walk and asks nothing until the
|
||||
// delivery's word; the word starts it; the core's own merge never waits; a person starts a waiting walk by
|
||||
// hand when the owner is down, and the owner's stop ends one as stopped.
|
||||
func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asked := asksRecorded(t)
|
||||
withConditionsInMemory(t)
|
||||
for _, name := range []string{"app", "mesh-delivery"} {
|
||||
m := catalogue.Manifest{Module: name, Version: "1"}
|
||||
if name == "mesh-delivery" {
|
||||
m.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
|
||||
Path: "modules/" + name, Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
merge := func(commit string, paths ...string) inventory.Plan {
|
||||
t.Helper()
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit, Paths: paths,
|
||||
ModuleDirs: []string{"modules/app", "modules/mesh-delivery"}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recent, err := inv.RecentPlans(ctx, 1)
|
||||
if err != nil || len(recent) != 1 || recent[0].Commit != commit {
|
||||
t.Fatalf("no plan for %s: %v %v", commit, recent, err)
|
||||
}
|
||||
return recent[0]
|
||||
}
|
||||
|
||||
// finish ends a walk as done, so the next merge has nothing of it to take over.
|
||||
finish := func(id string) {
|
||||
t.Helper()
|
||||
w, err := inv.PlanByID(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.State = inventory.PlanDone
|
||||
if err := inv.SavePlan(ctx, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// No holder on record: the merge starts its walk, as before.
|
||||
p := merge("c1aaaaaaaa", "modules/app/index.ts")
|
||||
if p.Waiting() || len(*asked) != 1 {
|
||||
t.Fatalf("with no holder on record the walk waited (%v) or asked %v", p.Waiting(), *asked)
|
||||
}
|
||||
|
||||
// The holder on record: the next merge waits, asking nothing, and an advance asks nothing either. One walk
|
||||
// is open at a time (novox/hq ADR 0276): the first ends before the next merge's batch is cut.
|
||||
finish(p.ID)
|
||||
if _, err := inv.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p = merge("c2bbbbbbbb", "modules/app/index.ts")
|
||||
if !p.Waiting() || len(*asked) != 1 || !strings.Contains(p.Note, "plans go "+p.ID) {
|
||||
t.Fatalf("the walk did not wait for its word: waiting %v, asked %v, note %q", p.Waiting(), *asked, p.Note)
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) != 1 {
|
||||
t.Fatalf("a waiting walk was advanced into asking: %v", *asked)
|
||||
}
|
||||
if line := planLine(p, p.Created); strings.Contains(line, "LATE") || !strings.Contains(line, "waits for") {
|
||||
t.Errorf("a waiting walk reads %q", line)
|
||||
}
|
||||
|
||||
// The delivery's word starts it.
|
||||
if err := deliveryCommand(ctx, []string{"go", p.ID, "--by", catalogue.DeliverySeat, "--why", "its turn"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := deliveryCommand(ctx, []string{"go", p.ID}); err == nil {
|
||||
t.Fatal("a walk was let go twice")
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) != 2 {
|
||||
t.Fatalf("the word did not start the walk: %v", *asked)
|
||||
}
|
||||
got, err := inv.PlanByID(ctx, p.ID)
|
||||
if err != nil || got.Delivery == nil || got.Delivery.By != catalogue.DeliverySeat || got.Delivery.Why != "its turn" {
|
||||
t.Fatalf("the word was not kept: %+v %v", got.Delivery, err)
|
||||
}
|
||||
|
||||
// The delivery's owner's own merge never waits for it. A walk that started is never taken over (novox/hq
|
||||
// ADR 0276): the merge's batch is cut once it ended.
|
||||
finish(p.ID)
|
||||
p = merge("c3cccccccc", "modules/mesh-delivery/main.go")
|
||||
if p.Waiting() || len(*asked) != 3 {
|
||||
t.Fatalf("mesh-delivery's own walk waited for mesh-delivery: %v %v", p.Waiting(), *asked)
|
||||
}
|
||||
|
||||
// The owner down: a person starts a waiting walk, with why.
|
||||
finish(p.ID)
|
||||
p = merge("c4dddddddd", "modules/app/index.ts")
|
||||
if !p.Waiting() {
|
||||
t.Fatal("the walk did not wait")
|
||||
}
|
||||
if err := plansCommand(ctx, []string{"go", p.ID}); err == nil || !strings.Contains(err.Error(), "--why") {
|
||||
t.Fatalf("a walk was started by hand without why: %v", err)
|
||||
}
|
||||
if err := plansCommand(ctx, []string{"go", p.ID, "--why", "mesh-delivery is down"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
got, _ = inv.PlanByID(ctx, p.ID)
|
||||
if got.Waiting() || !strings.HasPrefix(got.Delivery.By, "a person") || len(*asked) < 4 {
|
||||
t.Fatalf("a person's word did not start the walk: %+v, asked %v", got.Delivery, *asked)
|
||||
}
|
||||
|
||||
// The owner's stop: failed, said as stopped by it.
|
||||
finish(p.ID)
|
||||
p = merge("c5eeeeeeee", "modules/app/index.ts")
|
||||
if err := deliveryCommand(ctx, []string{"stop", p.ID, "--why", "the operator stopped it", "--by",
|
||||
"mesh-delivery for jochen"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ = inv.PlanByID(ctx, p.ID)
|
||||
if got.State != inventory.PlanFailed || got.Delivery.Stopped != "mesh-delivery for jochen" ||
|
||||
!strings.Contains(got.Note, "the operator stopped it") {
|
||||
t.Fatalf("the stop was kept as %s %+v %q", got.State, got.Delivery, got.Note)
|
||||
}
|
||||
|
||||
// Unassigned: the mesh is back on the controller's own path.
|
||||
if err := inv.Unassign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asks := len(*asked)
|
||||
if p = merge("c6ffffffff", "modules/app/index.ts"); p.Waiting() || len(*asked) != asks+1 {
|
||||
t.Fatalf("with the holder gone the walk waited: %v", p.Waiting())
|
||||
}
|
||||
}
|
||||
|
||||
// The verbs mesh-delivery asks with become the commands they name, and nothing a caller sends is passed over.
|
||||
func TestTheDeliveryVerbsComposeTheirCommands(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
want []string
|
||||
}{
|
||||
{"deliver", map[string]any{"plan": "plan-1", "why": "its turn"},
|
||||
[]string{"delivery", "go", "plan-1", "--by", "mesh-delivery", "--why", "its turn"}},
|
||||
{"delivery-stop", map[string]any{"plan": "plan-1", "why": "w", "by": "jochen"},
|
||||
[]string{"delivery", "stop", "plan-1", "--why", "w", "--by", "mesh-delivery for jochen"}},
|
||||
{"delivery-walks", map[string]any{}, []string{"delivery", "walks"}},
|
||||
{"delivery-walks", map[string]any{"plan": "plan-1"}, []string{"delivery", "walks", "--plan", "plan-1"}},
|
||||
{"delivery-order", map[string]any{"members": "[]"}, []string{"delivery", "order", "--members", "[]"}},
|
||||
{"delivery-check", map[string]any{"group": "feat/x", "members": "[]"},
|
||||
[]string{"delivery", "check", "--group", "feat/x", "--members", "[]"}},
|
||||
{"delivery-plan", map[string]any{"repository": "novox/a", "paths": "x", "head": "c0"},
|
||||
[]string{"delivery", "plan", "--repository", "novox/a", "--paths", "x", "--head", "c0"}},
|
||||
{"plans", map[string]any{"go": "plan-1", "why": "down"}, []string{"plans", "go", "plan-1", "--why", "down"}},
|
||||
} {
|
||||
got, err := argvFor(c.verb, c.args)
|
||||
if err != nil || !reflect.DeepEqual(got, c.want) {
|
||||
t.Errorf("%s %v → %v %v, wanted %v", c.verb, c.args, got, err, c.want)
|
||||
}
|
||||
}
|
||||
if _, err := argvFor("deliver", map[string]any{}); err == nil {
|
||||
t.Error("deliver without a walk was composed")
|
||||
}
|
||||
if _, err := argvFor("delivery-stop", map[string]any{"plan": "p"}); err == nil {
|
||||
t.Error("a stop without why was composed")
|
||||
}
|
||||
}
|
||||
|
||||
// A verb runs as a command of its own: what it keeps of a walk is still said as plan-moved, on a bus of the
|
||||
// command's own, so the delivery's owner hears it at once and not only when it reads the walks back.
|
||||
func TestAWalkLetGoByAVerbIsSaidAsPlanMoved(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
url := testbus.URL(t)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := handActConn
|
||||
handActConn = js.Conn()
|
||||
t.Cleanup(func() { handActConn = before })
|
||||
heard := make(chan *nats.Msg, 4)
|
||||
sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), heard)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
|
||||
waiting := inventory.Plan{ID: "plan-waits", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c7c7c7c7",
|
||||
Created: time.Now().UTC(), State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
|
||||
Modules: map[string]*inventory.PlanModule{"app": {}},
|
||||
Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}}
|
||||
if err := open.inventory.SavePlan(ctx, &waiting); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case m := <-heard:
|
||||
var said inventory.Plan
|
||||
if err := json.Unmarshal(m.Data, &said); err != nil || said.ID != waiting.ID || said.Delivery == nil ||
|
||||
said.Delivery.Go == nil {
|
||||
t.Fatalf("plan-moved said %s (%v)", m.Data, err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("a walk let go by a verb was not said")
|
||||
}
|
||||
if checkEvents != nil || inventory.PlanSaved != nil {
|
||||
t.Fatal("the command's own bus was left in place")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,333 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/secrets"
|
||||
)
|
||||
|
||||
// A module's own secret given at the operator's desk (novox/hq ADR 0259 §10, ADR 0277).
|
||||
//
|
||||
// mesh-controller secret ask <node> <module> <name> [--at <desk>]
|
||||
//
|
||||
// **The value never passes through whoever asked for it.** An agent, or the operator at the mesh MCP
|
||||
// server, calls `secret-ask` (or `give`) with the machine, the module, the secret's name and the desk — never
|
||||
// a value. The controller makes a sealing keypair for this one call, asks the desk's `node-launcher.secret` to
|
||||
// prompt the operator without showing what is typed, and is answered with what was typed **sealed to that
|
||||
// key**: no plaintext on the bus, in a runtime's log or in any call's record. It opens it here, seals it to the
|
||||
// module's machine exactly as `secret accept` does, and forgets it. What it answers says only that the
|
||||
// value was taken, or why not.
|
||||
//
|
||||
// **Bounded, and the prompt says who asked** (ADR 0277): one open prompt per secret and few an hour, read from
|
||||
// the store before the prompt opens (inventory.OpenSecretAsk), so an agent cannot keep a prompt in front of the
|
||||
// operator until they type. The prompt names the module, the secret, the machine and who asked — the caller as
|
||||
// the bus named it, never a word the caller chose — written by the desk's launcher from those names alone.
|
||||
//
|
||||
// **What remains** (ADR 0234's accepted residual risk): on an X11 desk any program of the operator's
|
||||
// account can read the keys as they are typed. And a program that calls the desk's prompt itself, with a
|
||||
// key of its own, is answered with what the operator typed into a prompt they did not ask for — as it could
|
||||
// draw a window of its own. The prompt says who asks and for what, so the operator types only into a
|
||||
// prompt they started.
|
||||
|
||||
// deskPromptWithin is how long the prompt waits for the operator: below the runtime's thirty seconds for
|
||||
// one call, as the launcher's menu is.
|
||||
const deskPromptWithin = 25
|
||||
|
||||
// deskGive is the desk path, its four reaches given so a test needs no store and no bus.
|
||||
type deskGive struct {
|
||||
// declares refuses a module or a secret the mesh would refuse, before anybody is asked to type.
|
||||
declares func(module, name string) error
|
||||
// known refuses a machine the mesh does not know, before anybody is asked to type; nil knows every one
|
||||
// (a test that does not look).
|
||||
known func(machine string) error
|
||||
// trusted says a module runs as an account of its own: its secret is never taken at a desk (below). Nil is
|
||||
// never (a test that does not look).
|
||||
trusted func(module string) (bool, error)
|
||||
// ask asks one machine's node-launcher.secret and answers its result, or the holder's refusal.
|
||||
ask func(machine string, args map[string]any) (json.RawMessage, error)
|
||||
// accept seals the value as `secret accept` does, and says whether it lives until the module's start.
|
||||
accept func(value string) (untilStart bool, err error)
|
||||
// record writes the act in the hand-act log.
|
||||
record func(link.HandAct) error
|
||||
// announce raises the condition that says a module's own secret was given (secretGivenObservation), on
|
||||
// every channel; nil announces nothing (a test that does not look).
|
||||
announce func(node, module, name, how string) error
|
||||
// askedBy is who asked, as the bus named the caller: said in the prompt and recorded.
|
||||
askedBy string
|
||||
// open records the ask and holds the bounds (one open per secret, few an hour), answering the record's id;
|
||||
// nil keeps no record (a test that does not look). end closes it with how it ended.
|
||||
open func(node, module, name, desk string) (int64, error)
|
||||
end func(id int64, outcome string) error
|
||||
}
|
||||
|
||||
// askedByName is the caller as the prompt names it: the first clause of what the bus said, in the characters
|
||||
// a name has, at most 80 of them. The desk's launcher refuses anything else, so no words of the caller's own
|
||||
// reach the prompt.
|
||||
func askedByName(caller string) string {
|
||||
first, _, _ := strings.Cut(caller, ",")
|
||||
var b strings.Builder
|
||||
for _, r := range strings.TrimSpace(first) {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '.', r == '_', r == '/', r == '@',
|
||||
r == '-', r == ' ':
|
||||
b.WriteRune(r)
|
||||
default:
|
||||
b.WriteRune('-')
|
||||
}
|
||||
if b.Len() >= 80 {
|
||||
break
|
||||
}
|
||||
}
|
||||
name := strings.TrimSpace(b.String())
|
||||
if name == "" || strings.HasPrefix(name, "-") {
|
||||
return "an unnamed caller"
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// errNothingGiven is a prompt dismissed, or not answered in time: nothing changes.
|
||||
var errNothingGiven = errors.New("nothing was given")
|
||||
|
||||
// give asks the desk for the value and seals it; it answers the words said to the caller.
|
||||
func (d deskGive) give(node, module, name, desk string) (string, error) {
|
||||
for what, v := range map[string]string{"the machine": node, "the module": module, "the secret": name, "the desk": desk} {
|
||||
if strings.TrimSpace(v) == "" {
|
||||
return "", fmt.Errorf("%s is not named", what)
|
||||
}
|
||||
}
|
||||
if d.known != nil {
|
||||
for what, machine := range map[string]string{"the machine the secret is for": node, "the desk": desk} {
|
||||
if err := d.known(machine); err != nil {
|
||||
return "", fmt.Errorf("nobody was asked to type anything: %s, %s, is not a machine this mesh knows: %w",
|
||||
what, machine, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := d.declares(module, name); err != nil {
|
||||
return "", fmt.Errorf("nobody was asked to type anything: %w", err)
|
||||
}
|
||||
// **A trusted party's secret is not taken at a desk** (the confirmation review of 2026-10-09, N1-give). The
|
||||
// prompt's answer comes back over the desk machine's bus, on a subject that machine's runtime answers — and
|
||||
// on a desk machine agents run as the operator, who holds that runtime's credential. An agent could answer
|
||||
// first, with a bot token of its own sealed to this call's key, and the channel the operator's answers are
|
||||
// proven on would be the agent's. So the value of a module running as its own account is typed at the
|
||||
// controller's terminal, where no bus carries it.
|
||||
if d.trusted != nil {
|
||||
trusted, err := d.trusted(module)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("nobody was asked to type anything: whether %s is a trusted party could not be read: %w", module, err)
|
||||
}
|
||||
if trusted {
|
||||
return "", fmt.Errorf("nobody was asked to type anything: %s runs as an account of its own and proves the "+
|
||||
"operator's answers, so its %s is given at the controller's terminal alone — there, run `mesh-controller "+
|
||||
"secret accept %s %s %s` and type it at its prompt. A desk's prompt is answered over the desk machine's "+
|
||||
"bus, where an agent may answer first (novox/hq ADR 0259 §10)", module, name, node, module, name)
|
||||
}
|
||||
}
|
||||
// The bounds, read and kept before anybody is asked to type (novox/hq ADR 0277): one open prompt per secret,
|
||||
// few an hour. How the ask ends is recorded whatever happens below.
|
||||
outcome := "failed"
|
||||
if d.open != nil {
|
||||
id, err := d.open(node, module, name, desk)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("nobody was asked to type anything: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if d.end != nil {
|
||||
_ = d.end(id, outcome)
|
||||
}
|
||||
}()
|
||||
}
|
||||
public, private, err := secrets.Keypair()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("no key could be made to take the value: %w", err)
|
||||
}
|
||||
// By name, never by words: the holder writes the prompt from these, and says the controller asks, which
|
||||
// the bus alone makes true (broker.ControllerOnly). Who asked is the bus's word on the caller, cut to a
|
||||
// name's characters — never an argument of the call.
|
||||
raw, err := d.ask(desk, map[string]any{
|
||||
"module": module,
|
||||
"secret": name,
|
||||
"node": node,
|
||||
"asked_by": askedByName(d.askedBy),
|
||||
"seal_to": public,
|
||||
"timeout_seconds": deskPromptWithin,
|
||||
})
|
||||
if err != nil {
|
||||
outcome = "refused"
|
||||
return "", fmt.Errorf("the desk on %s could not be asked: %w", desk, err)
|
||||
}
|
||||
var answer struct {
|
||||
Sealed string `json:"sealed"`
|
||||
Cancelled bool `json:"cancelled"`
|
||||
TimedOut bool `json:"timed_out"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &answer); err != nil {
|
||||
return "", fmt.Errorf("the desk on %s answered something that is not the prompt's answer", desk)
|
||||
}
|
||||
switch {
|
||||
case answer.TimedOut:
|
||||
outcome = "timed-out"
|
||||
return "", fmt.Errorf("%w: the prompt on %s was not answered within %d seconds", errNothingGiven, desk, deskPromptWithin)
|
||||
case answer.Cancelled:
|
||||
outcome = "dismissed"
|
||||
return "", fmt.Errorf("%w: the prompt on %s was dismissed", errNothingGiven, desk)
|
||||
case answer.Sealed == "":
|
||||
return "", fmt.Errorf("the desk on %s answered no sealed value", desk)
|
||||
}
|
||||
opened, err := secrets.Open(private, answer.Sealed)
|
||||
if err != nil {
|
||||
// Never the value, never what failed to open: only that it was not sealed to this call.
|
||||
return "", fmt.Errorf("the desk on %s answered a value not sealed to this call; nothing was taken", desk)
|
||||
}
|
||||
value := asSupplied(string(opened))
|
||||
for i := range opened {
|
||||
opened[i] = 0
|
||||
}
|
||||
if strings.TrimSpace(value) == "" {
|
||||
outcome = "empty"
|
||||
return "", fmt.Errorf("%w: the prompt on %s was answered empty", errNothingGiven, desk)
|
||||
}
|
||||
untilStart, err := d.accept(value)
|
||||
value = ""
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
outcome = "given"
|
||||
act := link.HandAct{Verb: "secret accept", Args: []string{node, module, name, "--at-desk", desk},
|
||||
Why: fmt.Sprintf("the operator gave %s for %s on %s at the desk on %s, asked by %s", name, module, node, desk,
|
||||
askedByName(d.askedBy)),
|
||||
Cause: "given-at-the-desk"}
|
||||
recorded := ""
|
||||
if err := d.record(act); err != nil {
|
||||
recorded = fmt.Sprintf("\n this act could NOT be recorded in the hand-act log, and is done anyway: %v", err)
|
||||
}
|
||||
if d.announce != nil {
|
||||
if err := d.announce(node, module, name, "at the desk on "+desk); err != nil {
|
||||
recorded += fmt.Sprintf("\n this change could NOT be announced on the operator's channels: %v", err)
|
||||
}
|
||||
}
|
||||
words := fmt.Sprintf("%s on %s now holds %q, given at the desk on %s and sealed to %s; the mesh cannot read it "+
|
||||
"back.\n run `push %s` to send it", module, node, name, desk, node, node)
|
||||
if untilStart {
|
||||
words += fmt.Sprintf("\n it lives until %s next starts well under the mesh, and is then replaced with a value "+
|
||||
"the mesh makes (ADR 0228)", module)
|
||||
}
|
||||
return words + recorded, nil
|
||||
}
|
||||
|
||||
// askAtDesk is `secret ask <node> <module> <name> [--at <machine>]`, and the terminal's `secret accept … --at-desk
|
||||
// <machine>`: the desk path, on this controller's stores and bus. The desk is the module's machine unless named.
|
||||
func askAtDesk(ctx context.Context, node, module, name, desk string) error {
|
||||
if desk == "" {
|
||||
desk = node
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
d := deskGive{
|
||||
askedBy: link.Caller(),
|
||||
open: func(node, module, name, desk string) (int64, error) {
|
||||
return open.inventory.OpenSecretAsk(ctx, node, module, name, askedByName(link.Caller()), desk)
|
||||
},
|
||||
end: func(id int64, outcome string) error { return open.inventory.EndSecretAsk(ctx, id, outcome) },
|
||||
declares: func(module, name string) error { return open.inventory.DeclaresOwnSecret(ctx, module, name) },
|
||||
known: func(machine string) error {
|
||||
_, err := open.inventory.NodeByName(ctx, machine)
|
||||
return err
|
||||
},
|
||||
trusted: func(module string) (bool, error) { return open.inventory.RunsAsItsOwnAccount(ctx, module) },
|
||||
ask: func(machine string, args map[string]any) (json.RawMessage, error) {
|
||||
var result json.RawMessage
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
answer, err := link.AskSeatTool(ctx, conn, "node-launcher", "secret", machine, args,
|
||||
time.Duration(deskPromptWithin+5)*time.Second)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return errors.New(answer.Error)
|
||||
}
|
||||
result = answer.Result
|
||||
return nil
|
||||
})
|
||||
return result, err
|
||||
},
|
||||
accept: func(value string) (bool, error) {
|
||||
return open.inventory.AcceptGivenSecret(ctx, node, module, name, value)
|
||||
},
|
||||
record: func(act link.HandAct) error {
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
_, err := link.RecordHandAct(ctx, conn, act)
|
||||
return err
|
||||
})
|
||||
},
|
||||
announce: func(node, module, name, how string) error { return announceSecretGiven(ctx, node, module, name, how) },
|
||||
}
|
||||
words, err := d.give(node, module, name, desk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(words)
|
||||
return nil
|
||||
}
|
||||
|
||||
// kindSecretGiven is the condition every value given for a module's own secret raises (the review of 2026-10-09,
|
||||
// M4): on every channel, so a bot token changed by somebody else — a channel that now answers for them — is
|
||||
// heard of. It stays until the operator silences or clears it.
|
||||
const kindSecretGiven = "secret-given"
|
||||
|
||||
// secretGivenObservation is that condition: which secret, of which module on which machine, how and when.
|
||||
// The summary, for whoever looks closer, names the secret and the time. The words the operator reads are
|
||||
// held to the plain rule (conditions.PlainWords): no clock time — the channel says when, in the operator's
|
||||
// time — and the secret's name said as words. Words that broke the rule were replaced by the keeper with
|
||||
// "needs a look … a problem it calls secret given" (hq issue 359), which told the operator nothing.
|
||||
func secretGivenObservation(node, module, name, how string, at time.Time) conditions.Observation {
|
||||
key := node + "." + module + "." + name
|
||||
where := module + " on " + node
|
||||
// Within the bounds whatever the names' length: the module and machine, else the module, else the machine.
|
||||
headline := "New secret given for " + where
|
||||
for _, h := range []string{"New secret given for " + module, "New secret given on " + node} {
|
||||
if len(headline) > conditions.HeadlineMax {
|
||||
headline = h
|
||||
}
|
||||
}
|
||||
resolved := "You saw that " + module + " was given a new secret"
|
||||
if len(resolved) > conditions.HeadlineMax+20 {
|
||||
resolved = "You saw that a new secret was given on " + node
|
||||
}
|
||||
return conditions.Observation{Scope: conditions.ScopeMachine, ID: key, Token: kindSecretGiven, Kind: kindSecretGiven,
|
||||
Machine: node, Severity: conditions.Urgent, Source: kindSecretGiven,
|
||||
Summary: fmt.Sprintf("%s of %s on %s was given %s at %s", name, module, node, how,
|
||||
at.Local().Format("2006-01-02 15:04")),
|
||||
Headline: headline,
|
||||
Explanation: fmt.Sprintf("The secret %s of %s was given %s. If you gave it, nothing else is needed. If you "+
|
||||
"did not, somebody else now holds what %s acts with.", secretNameWords(name), where, how, module),
|
||||
Needs: "silence this if you just gave it; if you did not, give it again yourself so that only you hold it.",
|
||||
Resolved: resolved,
|
||||
Actions: []conditions.Action{conditions.SilenceAction(conditions.Key(conditions.ScopeMachine, key, kindSecretGiven))}}
|
||||
}
|
||||
|
||||
// secretNameWords is a secret's name as the operator reads it: "telegram-token" is "telegram token".
|
||||
func secretNameWords(name string) string {
|
||||
return strings.Join(strings.FieldsFunc(name, func(r rune) bool { return r == '-' || r == '_' || r == '.' }), " ")
|
||||
}
|
||||
|
||||
// announceSecretGiven raises it on this controller's keeper.
|
||||
func announceSecretGiven(ctx context.Context, node, module, name, how string) error {
|
||||
return withKeeper(ctx, func(k *conditions.Keeper) error {
|
||||
_, err := k.Observe(ctx, secretGivenObservation(node, module, name, how, time.Now()))
|
||||
return err
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,512 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/secrets"
|
||||
)
|
||||
|
||||
const typed = "123456789:AAEhBP0av28P4XFQnIuR-o-7Xnz1kkUzW3g"
|
||||
|
||||
// aDesk is the desk path with a prompt the test answers as the operator would, and what it was asked kept.
|
||||
func aDesk(t *testing.T, answer func(args map[string]any) (json.RawMessage, error)) (deskGive, *[]string, *[]link.HandAct, *[]map[string]any) {
|
||||
t.Helper()
|
||||
var accepted []string
|
||||
var acts []link.HandAct
|
||||
var asked []map[string]any
|
||||
return deskGive{
|
||||
declares: func(module, name string) error {
|
||||
if module != "telegram" || name != "telegram-token" {
|
||||
return errors.New(module + " does not declare " + name + " as an own secret")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
ask: func(machine string, args map[string]any) (json.RawMessage, error) {
|
||||
asked = append(asked, args)
|
||||
return answer(args)
|
||||
},
|
||||
accept: func(value string) (bool, error) { accepted = append(accepted, value); return false, nil },
|
||||
record: func(a link.HandAct) error { acts = append(acts, a); return nil },
|
||||
}, &accepted, &acts, &asked
|
||||
}
|
||||
|
||||
func sealedTo(t *testing.T, value string) func(args map[string]any) (json.RawMessage, error) {
|
||||
return func(args map[string]any) (json.RawMessage, error) {
|
||||
sealed, err := secrets.Seal(args["seal_to"].(string), []byte(value+"\n"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(map[string]any{"sealed": sealed})
|
||||
return raw, nil
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0259 §10: the value typed at the desk is sealed as `secret accept` seals it, and is in no
|
||||
// answer, no prompt argument and no act recorded.
|
||||
func TestASecretGivenAtTheDeskIsSealedAndSaidNowhere(t *testing.T) {
|
||||
d, accepted, acts, asked := aDesk(t, sealedTo(t, typed))
|
||||
words, err := d.give("anchor", "telegram", "telegram-token", "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(*accepted) != 1 || (*accepted)[0] != typed {
|
||||
t.Fatalf("the value sealed is not what was typed, its line ending taken off")
|
||||
}
|
||||
if len(*acts) != 1 || (*acts)[0].Verb != "secret accept" || (*acts)[0].Cause != "given-at-the-desk" ||
|
||||
!strings.Contains((*acts)[0].Why, "at the desk on laptop") {
|
||||
t.Errorf("the act: %+v", *acts)
|
||||
}
|
||||
raw, _ := json.Marshal(struct {
|
||||
Words string
|
||||
Acts []link.HandAct
|
||||
Asked []map[string]any
|
||||
}{words, *acts, *asked})
|
||||
if strings.Contains(string(raw), typed) || strings.Contains(string(raw), "AAEhBP0") {
|
||||
t.Fatal("the value appears in what was said, asked or recorded")
|
||||
}
|
||||
if !strings.Contains(words, "push anchor") || !strings.Contains(words, "given at the desk on laptop") {
|
||||
t.Errorf("%q", words)
|
||||
}
|
||||
if p := (*asked)[0]; p["seal_to"] == "" || p["timeout_seconds"] != deskPromptWithin {
|
||||
t.Errorf("the prompt was asked %v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNothingIsAskedForASecretTheMeshWouldRefuse(t *testing.T) {
|
||||
for _, c := range [][2]string{{"telegram", "chat-id"}, {"nobody", "telegram-token"}} {
|
||||
d, accepted, _, asked := aDesk(t, sealedTo(t, typed))
|
||||
if _, err := d.give("anchor", c[0], c[1], "laptop"); err == nil || !strings.Contains(err.Error(), "nobody was asked") {
|
||||
t.Errorf("%v: %v", c, err)
|
||||
}
|
||||
if len(*asked) != 0 || len(*accepted) != 0 {
|
||||
t.Errorf("%v: the operator was asked anyway", c)
|
||||
}
|
||||
}
|
||||
d, _, _, _ := aDesk(t, sealedTo(t, typed))
|
||||
if _, err := d.give("anchor", "telegram", "telegram-token", ""); err == nil {
|
||||
t.Error("no desk was refused nowhere")
|
||||
}
|
||||
}
|
||||
|
||||
func TestADismissedEmptyLateOrForeignAnswerTakesNothing(t *testing.T) {
|
||||
for want, answer := range map[string]func(map[string]any) (json.RawMessage, error){
|
||||
"not answered within 25 seconds": func(map[string]any) (json.RawMessage, error) {
|
||||
return json.RawMessage(`{"cancelled":true,"timed_out":true}`), nil
|
||||
},
|
||||
"was dismissed": func(map[string]any) (json.RawMessage, error) { return json.RawMessage(`{"cancelled":true}`), nil },
|
||||
"answered empty": sealedTo(t, " "),
|
||||
"not sealed to this call": func(map[string]any) (json.RawMessage, error) {
|
||||
other, _, _ := secrets.Keypair()
|
||||
sealed, _ := secrets.Seal(other, []byte(typed))
|
||||
raw, _ := json.Marshal(map[string]any{"sealed": sealed})
|
||||
return raw, nil
|
||||
},
|
||||
"could not be asked": func(map[string]any) (json.RawMessage, error) { return nil, errors.New("no session answers") },
|
||||
} {
|
||||
d, accepted, acts, _ := aDesk(t, answer)
|
||||
_, err := d.give("anchor", "telegram", "telegram-token", "laptop")
|
||||
if err == nil || !strings.Contains(err.Error(), want) || strings.Contains(err.Error(), typed) {
|
||||
t.Errorf("want %q, got %v", want, err)
|
||||
}
|
||||
if len(*accepted) != 0 || len(*acts) != 0 {
|
||||
t.Errorf("%s: something was taken or recorded", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheGiveVerbRunsTheDeskPathAndTheControllerMayAskTheDesk(t *testing.T) {
|
||||
argv, err := argvFor("give", map[string]any{"node": "anchor", "module": "telegram", "secret": "telegram-token", "at": "laptop"})
|
||||
if err != nil || strings.Join(argv, " ") != "secret ask anchor telegram telegram-token --at laptop" {
|
||||
t.Fatalf("%v %v", argv, err)
|
||||
}
|
||||
if _, err := argvFor("give", map[string]any{"node": "anchor", "module": "telegram", "secret": "telegram-token"}); err == nil {
|
||||
t.Error("give without a desk was taken")
|
||||
}
|
||||
// secret-ask is the same line, with the desk the module's machine unless named (novox/hq ADR 0277).
|
||||
argv, err = argvFor("secret-ask", map[string]any{"node": "anchor", "module": "telegram", "secret": "telegram-token"})
|
||||
if err != nil || strings.Join(argv, " ") != "secret ask anchor telegram telegram-token" {
|
||||
t.Fatalf("%v %v", argv, err)
|
||||
}
|
||||
argv, err = argvFor("secret-ask", map[string]any{"node": "anchor", "module": "telegram", "secret": "telegram-token", "at": "laptop"})
|
||||
if err != nil || strings.Join(argv, " ") != "secret ask anchor telegram telegram-token --at laptop" {
|
||||
t.Fatalf("%v %v", argv, err)
|
||||
}
|
||||
perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found := false
|
||||
for _, p := range perms.Publish {
|
||||
found = found || p == "mesh.seat.node-launcher.tool.secret.*"
|
||||
}
|
||||
if !found {
|
||||
t.Error("the controller may not ask the desk's prompt")
|
||||
}
|
||||
}
|
||||
|
||||
// The review of 2026-10-09 (M4): the desk's prompt says who asks in words the caller does not choose — the
|
||||
// controller, which the bus alone lets ask it — and what for, from names the controller checked; the prompt
|
||||
// carries no free text of the caller's.
|
||||
func TestThePromptIsAskedByNameNeverByWordsTheCallerChose(t *testing.T) {
|
||||
d, _, _, asked := aDesk(t, sealedTo(t, typed))
|
||||
if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := (*asked)[0]
|
||||
if p["module"] != "telegram" || p["secret"] != "telegram-token" || p["node"] != "anchor" {
|
||||
t.Errorf("the prompt was not asked by name: %v", p)
|
||||
}
|
||||
for _, free := range []string{"prompt", "message"} {
|
||||
if _, there := p[free]; there {
|
||||
t.Errorf("the prompt carries the caller's %s: %v", free, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Every value given for a module's own secret is announced as a condition, on every channel (the review of
|
||||
// 2026-10-09, M4): a bot token changed by somebody else is a channel that now answers for them.
|
||||
func TestAValueGivenAtTheDeskIsAnnounced(t *testing.T) {
|
||||
d, _, _, _ := aDesk(t, sealedTo(t, typed))
|
||||
var said []string
|
||||
d.announce = func(node, module, name, how string) error {
|
||||
said = append(said, node+" "+module+" "+name+" "+how)
|
||||
return nil
|
||||
}
|
||||
if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(said) != 1 || !strings.Contains(said[0], "anchor telegram telegram-token") || !strings.Contains(said[0], "laptop") {
|
||||
t.Fatalf("announced %v", said)
|
||||
}
|
||||
o := secretGivenObservation("anchor", "telegram", "telegram-token", "at the desk on laptop", time.Date(2026, 10, 9, 12, 3, 0, 0, time.UTC))
|
||||
if o.Severity != conditions.Urgent || !strings.Contains(o.Explanation, "telegram token") ||
|
||||
len(o.Actions) == 0 || o.Key() == "" {
|
||||
t.Errorf("the announcement %+v", o)
|
||||
}
|
||||
if strings.Contains(o.Summary+o.Explanation+o.Said, typed) {
|
||||
t.Error("the announcement carries the value")
|
||||
}
|
||||
}
|
||||
|
||||
// **The secret-given condition says itself in its own plain words** (hq issue 359): the words it carries pass
|
||||
// the plain rule, from the controller's terminal and from a desk, so the keeper keeps them — they once held a
|
||||
// clock time, and the operator read "Novox needs a look … a problem it calls secret given" instead. Its
|
||||
// severity and its answer stay: it is heard on every channel, and silenced by the operator.
|
||||
func TestTheSecretGivenConditionSaysItselfInPlainWords(t *testing.T) {
|
||||
at := time.Date(2026, 10, 9, 23, 32, 0, 0, time.Local)
|
||||
for _, how := range []string{"at the controller's terminal", "at the desk on laptop"} {
|
||||
o := secretGivenObservation("anchor", "telegram", "telegram-token", how, at)
|
||||
w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs,
|
||||
Actions: o.Actions}
|
||||
if why, ok := conditions.PlainWords(w, o.Machine); !ok {
|
||||
t.Fatalf("given %s, the words are not plain: %s", how, why)
|
||||
}
|
||||
k, _ := withConditionsInMemory(t)
|
||||
c, err := k.Observe(t.Context(), o)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.Headline != "New secret given for telegram on anchor" || c.Severity != conditions.Urgent ||
|
||||
!strings.HasPrefix(c.Explanation, conditions.NeedsYou+" silence this") ||
|
||||
!strings.Contains(c.Explanation, "telegram token of telegram on anchor was given "+how) ||
|
||||
len(c.Actions) != 1 || c.Actions[0].Label != "Silence for a week" {
|
||||
t.Errorf("given %s, the keeper said %q / %q (%s, %v)", how, c.Headline, c.Explanation, c.Severity, c.Actions)
|
||||
}
|
||||
if strings.Contains(c.Headline+c.Explanation+c.Resolved+c.Needs, typed) {
|
||||
t.Error("the words carry the value")
|
||||
}
|
||||
}
|
||||
// A long module name keeps the headline and the resolved line within their bounds.
|
||||
for _, module := range []string{"a-module-with-a-rather-long-name-indeed",
|
||||
"a-module-with-a-name-so-long-that-no-headline-could-ever-hold-it"} {
|
||||
o := secretGivenObservation("anchor", module, "api-key", "at the controller's terminal", at)
|
||||
if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Explanation: o.Explanation,
|
||||
Resolved: o.Resolved, Needs: o.Needs, Actions: o.Actions}, o.Machine); !ok {
|
||||
t.Errorf("the module %s: %s", module, why)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The bus lets the controller alone ask the desk's prompt (the review of 2026-10-09, M4): the runtime, which
|
||||
// carries every agent's calls, and a person granted every tool are denied it, however wide their grant.
|
||||
func TestOnlyTheControllerMayAskTheDesksPrompt(t *testing.T) {
|
||||
for _, p := range []broker.Principal{
|
||||
{Kind: broker.KindNodeTools, Node: "laptop"},
|
||||
{Kind: broker.KindPerson, Module: "operator", Invokes: []string{"*"}},
|
||||
{Kind: broker.KindModule, Node: "laptop", Module: "lab", Invokes: []string{"seat:node-launcher.secret"}},
|
||||
} {
|
||||
perms, err := broker.PermissionsFor(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, subject := range []string{"mesh.seat.node-launcher.tool.secret.laptop", "mesh.seat.node-launcher.tool.secret",
|
||||
"mesh.mod.rofi.tool.node-launcher.secret", "mesh.mod.rofi.tool.node-launcher.secret.laptop"} {
|
||||
if broker.MayPublish(perms, subject) {
|
||||
t.Errorf("%s may publish %s", p.Username(), subject)
|
||||
}
|
||||
}
|
||||
}
|
||||
perms, _ := broker.PermissionsFor(broker.Principal{Kind: broker.KindController})
|
||||
if !broker.MayPublish(perms, "mesh.seat.node-launcher.tool.secret.laptop") {
|
||||
t.Error("the controller may not ask the desk's prompt")
|
||||
}
|
||||
}
|
||||
|
||||
// A value for a secret comes from the terminal or the desk, never through a verb (the review of 2026-10-09,
|
||||
// M4): `secret accept` with a value, run for a verb, is refused before anything is read.
|
||||
func TestASecretValueIsNeverAcceptedThroughAVerb(t *testing.T) {
|
||||
t.Setenv(verbVar, "mesh-controller.command")
|
||||
for _, args := range [][]string{
|
||||
{"accept", "anchor", "telegram", "telegram-token", "--from", "/dev/null"},
|
||||
{"accept", "anchor", "app", "db", "--from", "/dev/null", "--provider", "store"},
|
||||
} {
|
||||
err := secretCommand(context.Background(), args)
|
||||
if err == nil || !strings.Contains(err.Error(), "never through a verb") {
|
||||
t.Errorf("%v: %v", args, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The `give` and `secret-ask` verbs' own line passes the terminal-only rule of ADR 0266, and no `secret accept`
|
||||
// does: a value, a file, a provider or an extra word is still the terminal's alone (novox/hq ADR 0277).
|
||||
func TestOnlyTheGiveLinePassesTheTerminalRuleForSecrets(t *testing.T) {
|
||||
for _, argv := range [][]string{
|
||||
{"secret", "ask", "anchor", "telegram", "telegram-token", "--at", "laptop"},
|
||||
{"secret", "ask", "anchor", "telegram", "telegram-token"},
|
||||
} {
|
||||
if err := terminalOnly(argv); err != nil {
|
||||
t.Errorf("%v refused: %v", argv, err)
|
||||
}
|
||||
}
|
||||
for _, argv := range [][]string{
|
||||
{"secret", "accept", "anchor", "telegram", "telegram-token", "--at-desk", "laptop"},
|
||||
{"secret", "ask", "anchor", "telegram", "telegram-token", "--from", "/tmp/x"},
|
||||
{"secret", "ask", "anchor", "telegram", "telegram-token", "--at", "laptop", "--local"},
|
||||
{"secret", "ask", "anchor", "telegram", "--at", "laptop"},
|
||||
{"secret", "accept", "anchor", "telegram", "telegram-token"},
|
||||
{"secret", "accept", "anchor", "telegram", "telegram-token", "--from", "/tmp/x"},
|
||||
{"secret", "accept", "anchor", "telegram", "telegram-token", "--at-desk", "laptop", "--local"},
|
||||
{"secret", "accept", "anchor", "telegram", "--provider", "--at-desk", "laptop"},
|
||||
{"secret", "export", "anchor", "telegram", "telegram-token", "--at-desk", "laptop"},
|
||||
} {
|
||||
if err := terminalOnly(argv); err == nil {
|
||||
t.Errorf("%v passed the terminal rule", argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The confirmation review of 2026-10-09, N1-give: a desk's prompt is answered over the desk machine's bus, and
|
||||
// on a desk machine agents run as the operator, who holds its runtime's credential — so a trusted party's
|
||||
// secret (a module running as an account of its own: the Telegram bot's token) is never taken at a desk.
|
||||
// Refused before anybody is asked to type, whoever called, naming the terminal's line.
|
||||
func TestATrustedPartysSecretIsNeverTakenAtADesk(t *testing.T) {
|
||||
d, accepted, acts, asked := aDesk(t, sealedTo(t, typed))
|
||||
d.trusted = func(module string) (bool, error) { return module == "telegram", nil }
|
||||
_, err := d.give("anchor", "telegram", "telegram-token", "laptop")
|
||||
if err == nil || !strings.Contains(err.Error(), "controller's terminal alone") ||
|
||||
!strings.Contains(err.Error(), "secret accept anchor telegram telegram-token") {
|
||||
t.Fatalf("a trusted party's secret was taken at the desk, or refused without the line: %v", err)
|
||||
}
|
||||
if len(*asked)+len(*accepted)+len(*acts) != 0 {
|
||||
t.Errorf("asked %v, accepted %d, recorded %v", *asked, len(*accepted), *acts)
|
||||
}
|
||||
d.trusted = func(string) (bool, error) { return false, errors.New("the store did not answer") }
|
||||
if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); err == nil || len(*asked) != 0 {
|
||||
t.Errorf("a module not known to be untrusted was asked at the desk: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// And who may answer the desk's prompt at all: only the runtime of the machine it is asked on, carrying the
|
||||
// launcher that holds the seat there — never the controller, another machine's runtime, or a module's own
|
||||
// account (the confirmation review of 2026-10-09, N1-give).
|
||||
func TestOnlyTheDeskMachinesLauncherMayAnswerItsPrompt(t *testing.T) {
|
||||
launcher := broker.Declared{Module: "rofi", Holds: []broker.Seat{{Name: "node-launcher", Scope: "node",
|
||||
Serves: []string{"run", "secret"}}}}
|
||||
subject := "mesh.seat.node-launcher.tool.secret.laptop"
|
||||
for _, c := range []struct {
|
||||
p broker.Principal
|
||||
answers bool
|
||||
}{
|
||||
{broker.Principal{Kind: broker.KindNodeTools, Node: "laptop", Module: broker.RuntimeModule, Carries: []broker.Declared{launcher}}, true},
|
||||
{broker.Principal{Kind: broker.KindNodeTools, Node: "anchor", Module: broker.RuntimeModule, Carries: []broker.Declared{launcher}}, false},
|
||||
{broker.Principal{Kind: broker.KindController}, false},
|
||||
{broker.Principal{Kind: broker.KindModule, Node: "laptop", Module: "lab"}, false},
|
||||
{broker.Principal{Kind: broker.KindNode, Node: "laptop"}, false},
|
||||
} {
|
||||
perms, err := broker.PermissionsFor(c.p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := broker.MaySubscribe(perms, subject); got != c.answers {
|
||||
t.Errorf("%s may answer %s: %v, want %v", c.p.Username(), subject, got, c.answers)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// N1-give at the controller's terminal (the confirmation review of 2026-10-09): a trusted party's secret is
|
||||
// announced before it is kept, and not kept when the announcement fails; another module's is kept first and
|
||||
// a failed announcement is said, not undone.
|
||||
func TestATrustedPartysSecretGivenAtTheTerminalIsAnnouncedBeforeItIsKept(t *testing.T) {
|
||||
var order []string
|
||||
announce := func(fail bool) func() error {
|
||||
return func() error {
|
||||
order = append(order, "announce")
|
||||
if fail {
|
||||
return errors.New("no channel")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
keep := func() (bool, error) { order = append(order, "keep"); return false, nil }
|
||||
|
||||
order = nil
|
||||
if _, unannounced, err := keepGiven(true, announce(false), keep); err != nil || unannounced != nil ||
|
||||
strings.Join(order, ",") != "announce,keep" {
|
||||
t.Errorf("trusted: %v %v, order %v; want announced, then kept", unannounced, err, order)
|
||||
}
|
||||
order = nil
|
||||
if _, _, err := keepGiven(true, announce(true), keep); err == nil || strings.Join(order, ",") != "announce" {
|
||||
t.Errorf("trusted, announcement failed: %v, order %v; want refused and nothing kept", err, order)
|
||||
}
|
||||
order = nil
|
||||
if _, unannounced, err := keepGiven(false, announce(true), keep); err != nil || unannounced == nil ||
|
||||
strings.Join(order, ",") != "keep,announce" {
|
||||
t.Errorf("not trusted: %v %v, order %v; want kept, then the failed announcement said", unannounced, err, order)
|
||||
}
|
||||
order = nil
|
||||
failing := func() (bool, error) { order = append(order, "keep"); return false, errors.New("store away") }
|
||||
if _, _, err := keepGiven(false, announce(false), failing); err == nil || strings.Join(order, ",") != "keep" {
|
||||
t.Errorf("not trusted, keep failed: %v, order %v; want refused and nothing announced", err, order)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine the mesh does not know, as the secret's or as the desk, is refused before anybody is asked to type.
|
||||
func TestAGiveNamingAMachineTheMeshDoesNotKnowAsksNobody(t *testing.T) {
|
||||
for _, unknown := range []string{"elsewhere", "nodesk"} {
|
||||
d, accepted, acts, asked := aDesk(t, func(map[string]any) (json.RawMessage, error) {
|
||||
t.Fatal("the desk was asked")
|
||||
return nil, nil
|
||||
})
|
||||
d.known = func(machine string) error {
|
||||
if machine == unknown {
|
||||
return errors.New("no node " + machine)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
node, desk := "anchor", "laptop"
|
||||
if unknown == "elsewhere" {
|
||||
node = unknown
|
||||
} else {
|
||||
desk = unknown
|
||||
}
|
||||
_, err := d.give(node, "telegram", "telegram-token", desk)
|
||||
if err == nil || !strings.Contains(err.Error(), "nobody was asked") || !strings.Contains(err.Error(), unknown) {
|
||||
t.Errorf("%s: %v", unknown, err)
|
||||
}
|
||||
if len(*accepted)+len(*acts)+len(*asked) != 0 {
|
||||
t.Errorf("%s: something happened: %v %v %v", unknown, *accepted, *acts, *asked)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0277: the prompt names who asked — the controller's word on the bus's caller, cut to a name's
|
||||
// characters — and never a word the caller chose: there is no argument for it.
|
||||
func TestThePromptNamesWhoAskedFromTheBussWordAlone(t *testing.T) {
|
||||
d, _, acts, asked := aDesk(t, sealedTo(t, typed))
|
||||
d.askedBy = "g14/claude-code, through the mesh-controller seat"
|
||||
if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := (*asked)[0]["asked_by"]; got != "g14/claude-code" {
|
||||
t.Errorf("asked_by %q", got)
|
||||
}
|
||||
if len(*acts) != 1 || !strings.Contains((*acts)[0].Why, "asked by g14/claude-code") {
|
||||
t.Errorf("the record: %+v", *acts)
|
||||
}
|
||||
for in, want := range map[string]string{
|
||||
"jochen at a shell on novox": "jochen at a shell on novox",
|
||||
"laptop/agent": "laptop/agent",
|
||||
"Your bank asks\nType your PIN, now": "Your bank asks-Type your PIN",
|
||||
"": "an unnamed caller",
|
||||
"<b>x</b>": "an unnamed caller",
|
||||
strings.Repeat("a", 100): strings.Repeat("a", 80),
|
||||
"--prompt, something else": "an unnamed caller",
|
||||
} {
|
||||
if got := askedByName(in); got != want {
|
||||
t.Errorf("askedByName(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
// The verb's schema has no argument that reaches the prompt's words.
|
||||
for _, verb := range []string{"give", "secret-ask"} {
|
||||
for _, free := range []string{"asked_by", "prompt", "message", "value", "from"} {
|
||||
if _, err := argvFor(verb, map[string]any{"node": "anchor", "module": "telegram", "secret": "telegram-token",
|
||||
"at": "laptop", free: "x"}); err == nil {
|
||||
t.Errorf("%s takes %s", verb, free)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An ask for a secret is bounded before anybody is asked to type (ADR 0277): the record refuses it, nothing is
|
||||
// asked; and how every ask ends is recorded.
|
||||
func TestASecretAskIsBoundedAndItsEndRecorded(t *testing.T) {
|
||||
d, accepted, _, asked := aDesk(t, sealedTo(t, typed))
|
||||
var ended []string
|
||||
d.open = func(node, module, name, desk string) (int64, error) { return 7, nil }
|
||||
d.end = func(id int64, outcome string) error {
|
||||
ended = append(ended, fmt.Sprintf("%d %s", id, outcome))
|
||||
return nil
|
||||
}
|
||||
if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.open = func(node, module, name, desk string) (int64, error) {
|
||||
return 0, errors.New("an ask for telegram-token of telegram on anchor is still open")
|
||||
}
|
||||
_, err := d.give("anchor", "telegram", "telegram-token", "laptop")
|
||||
if err == nil || !strings.Contains(err.Error(), "nobody was asked to type anything") || !strings.Contains(err.Error(), "still open") {
|
||||
t.Errorf("a second ask: %v", err)
|
||||
}
|
||||
if len(*asked) != 1 || len(*accepted) != 1 {
|
||||
t.Errorf("asked %d, accepted %d", len(*asked), len(*accepted))
|
||||
}
|
||||
d.open = func(node, module, name, desk string) (int64, error) { return 8, nil }
|
||||
d.ask = func(string, map[string]any) (json.RawMessage, error) {
|
||||
return json.RawMessage(`{"cancelled":true}`), nil
|
||||
}
|
||||
if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); !errors.Is(err, errNothingGiven) {
|
||||
t.Errorf("a dismissed prompt: %v", err)
|
||||
}
|
||||
if strings.Join(ended, "; ") != "7 given; 8 dismissed" {
|
||||
t.Errorf("ended: %v", ended)
|
||||
}
|
||||
}
|
||||
|
||||
// A secret ask cannot be turned into a secret read: the line carries no value, the answer carries none, and every
|
||||
// other `secret` line is the terminal's.
|
||||
func TestASecretAskIsNeverASecretRead(t *testing.T) {
|
||||
t.Setenv(verbVar, "mesh-controller.secret-ask")
|
||||
for _, args := range [][]string{
|
||||
{"ask", "anchor", "telegram", "telegram-token", "the-value"},
|
||||
{"ask", "anchor", "telegram"},
|
||||
{"ask", "anchor", "telegram", "telegram-token", "--from", "/dev/null"},
|
||||
} {
|
||||
if err := secretCommand(context.Background(), args); err == nil {
|
||||
t.Errorf("secret %v was taken", args)
|
||||
}
|
||||
}
|
||||
for _, args := range [][]string{{"recover", "anchor", "telegram", "telegram-token"}, {"export"}} {
|
||||
if err := terminalOnly(append([]string{"secret"}, args...)); err == nil {
|
||||
t.Errorf("secret %v passed the terminal rule", args)
|
||||
}
|
||||
}
|
||||
}
|
||||
+150
-15
@@ -75,7 +75,8 @@ type probe struct {
|
||||
// probe added to a design is a row added here.
|
||||
var probeRegistry = []probe{
|
||||
{ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation",
|
||||
From: "issues 236, 263", Kind: "declaration-refused", Phase: 1, run: probeDeclarations},
|
||||
From: "issues 236, 263, 275", Kind: "declaration-refused", Raises: []string{kindAwaitingPush}, Phase: 1,
|
||||
run: probeDeclarations},
|
||||
{ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6",
|
||||
From: "issue 262", Kind: "resolver-wrong", Phase: 1, run: probeResolvers},
|
||||
{ID: "D3", Asserts: "every seat on record that serves verbs has a live holder that answers, on every " +
|
||||
@@ -97,8 +98,62 @@ var probeRegistry = []probe{
|
||||
From: "issue 265", Kind: "status-slow", Phase: 1, run: probeStatus},
|
||||
{ID: "D10", Asserts: "every machine runs the node-engine and node tools builds the mesh holds, or is inside " +
|
||||
"a plan's window", From: "the version split", Kind: "core-behind", Phase: 1, run: probeCoreBuilds},
|
||||
{ID: "D11", Asserts: "no provider holds a consumer retired more than thirty days without a person deciding " +
|
||||
"its cleanup", From: "ADR 0230", Kind: kindCleanupWaiting, Phase: 2, run: probeRetired},
|
||||
{ID: probeBindingsID, Asserts: "every consumer of a provision that keeps its data is bound where it was last " +
|
||||
"sent, or moves by a pin", From: "issue 273, ADR 0232", Kind: kindBindingMoved,
|
||||
Raises: []string{kindBindingKept, kindBindingMoving}, Phase: 2, run: probeBindings},
|
||||
{ID: probeDataID, Asserts: "every item of data a machine declares is measured, is there, holds what it held, is " +
|
||||
"written where it should be, is backed up within its bound or sits on healthy redundant storage, and is no " +
|
||||
"empty replacement of a copy kept elsewhere; what a machine no longer declares that is irreplaceable or " +
|
||||
"valuable is retired, not forgotten", From: "issue 273, ADR 0233",
|
||||
Kind: kindDataShrank, Raises: []string{kindEmptyReplacement, kindDataHeldTwice, kindDataQuiet, kindBackupStale,
|
||||
kindDataUnmeasured, kindDataMissing, kindArrayDegraded, kindProtectionMissing, kindCleanupWaiting},
|
||||
Phase: 2, run: probeData,
|
||||
Asks: []broker.SeatVerb{{Seat: "node-backup", Verb: "backed-up"}}},
|
||||
// The delivery's owner (novox/hq ADR 0239): what it holds past a bound of its own table is said here, by
|
||||
// the controller, and H2 works it through the owner's `close`.
|
||||
{ID: probeDeliveriesID, Asserts: "no delivery is held past its state's bound unsaid: mesh-delivery's " +
|
||||
"`stalled`, each with the transition its table lets healer H2 take", From: "ADR 0239",
|
||||
Kind: kindDeliveryStalled, Phase: 3, run: probeDeliveries},
|
||||
// A client of the bus reconnecting in a loop (novox/hq issue 327), from the server's record of closed
|
||||
// connections, which the bus's own module reads.
|
||||
{ID: probeReconnectsID, Asserts: "no user of the bus had its connection dropped more than twelve times in the " +
|
||||
"last hour: the bus module's nats_closed_connections", From: "issue 327", Kind: kindBusReconnects,
|
||||
Phase: 1, run: probeReconnects},
|
||||
// The account agents run as (novox/hq ADR 0266): where a machine names one, its node-engine has judged it
|
||||
// unable to become root without a person — what ADR 0259 §8 rests an authorised answer on.
|
||||
{ID: agentAccountProbe, Asserts: "every machine that names an agent account has it judged, on its node-engine's " +
|
||||
"newest statement, unable to become root without a person", From: "ADR 0266, ADR 0259 §8",
|
||||
Kind: kindAgentCanBecomeRoot, Phase: 1, run: probeAgentAccounts},
|
||||
// Root where the trusted parties run (novox/hq ADR 0259 §8): while an agent can become root there without a
|
||||
// person, an answer proven there proves nothing.
|
||||
{ID: "D-root", Asserts: "no agent can become root without a person on a machine where the router or a channel " +
|
||||
"proving its sender runs: not by its own account, and not through a tool that runs its command as an account " +
|
||||
"that can", From: "ADR 0259 §8", Kind: kindRootNotFree, Phase: 2, run: probeAgentRoot},
|
||||
{ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals",
|
||||
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs},
|
||||
// The core's health definitions (novox/hq to-be 45 §8, ADR 0236): what a core component's new build is
|
||||
// judged by on its first machine, run against every machine between upgrades too.
|
||||
{ID: "H-controller", Asserts: "the controller lease is held, renewed in time, by a controller that says it is " +
|
||||
"ready: its self-check ran and status answered in full within ten seconds", From: "ADR 0236, to-be 45 §8",
|
||||
Kind: kindCoreUnhealthy, Phase: 4, run: probeControllerHealth},
|
||||
{ID: "H-engine", Asserts: "every machine heard from has reported its current declaration, under a node-engine " +
|
||||
"build it names", From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, run: probeEngineHealth},
|
||||
{ID: "H-tools", Asserts: "every machine heard from that runs the node tools has them answering the bus",
|
||||
From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, run: probeToolsHealth},
|
||||
{ID: "H-bus", Asserts: "every stream and durable consumer the mesh defines is on the bus, and a request crosses " +
|
||||
"it to the machines' node tools and back", From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4,
|
||||
run: probeBusHealth},
|
||||
// The gate's verdicts and the witnesses' rollbacks (ADR 0236): each build that failed its gate keeps its
|
||||
// condition until a newer build passes; each rollback a witness stands by is said.
|
||||
{ID: gateProbe, Asserts: "no build that failed its gate, and no core component a witness put back, goes unsaid; " +
|
||||
"a newer build that passes its gate clears it", From: "ADR 0236, to-be 45 §8", Kind: kindRolledBack,
|
||||
Raises: []string{kindRollbackFailed}, Phase: 4, run: probeGates},
|
||||
// The bus's planned step (ADR 0236): open while a person's bus upgrade runs, then checked by H-bus.
|
||||
{ID: busStepProbe, Asserts: "a bus upgrade a person started is said while it runs, and is followed by the bus's " +
|
||||
"health within its bound — or is said failed, with its snapshot as the way back", From: "ADR 0236, to-be 45 §8",
|
||||
Kind: kindBusMaintenance, Raises: []string{kindBusUpgradeFailed}, Phase: 4, run: probeBusStep},
|
||||
}
|
||||
|
||||
// probeVerdict is one probe's outcome in a run.
|
||||
@@ -106,8 +161,55 @@ type probeVerdict struct {
|
||||
ID string `json:"id"`
|
||||
Verdict string `json:"verdict"`
|
||||
Found []string `json:"found,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Took string `json:"took,omitempty"`
|
||||
// Unconfirmed are findings one look can be wrong about, seen by this run and not the one before:
|
||||
// raised if the next run sees them too (confirm.go). Not a pass, and not yet a condition.
|
||||
Unconfirmed []string `json:"unconfirmed,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Took string `json:"took,omitempty"`
|
||||
// FoundCount and More are an overview's: how many findings the probe had when the overview shows
|
||||
// fewer, and how to read them all (novox/hq issue 314). Never in a heartbeat.
|
||||
FoundCount int `json:"found-count,omitempty"`
|
||||
More string `json:"more,omitempty"`
|
||||
}
|
||||
|
||||
// overviewFindings is how many findings of one probe an overview carries. On 2026-10-08 D14 found 689
|
||||
// stalled deliveries, and an overview carrying every one of them, beside the conditions they raised,
|
||||
// was more than the bus carries in one message (novox/hq issue 314); one probe's verdict whole is
|
||||
// `doctor probe=<id>`.
|
||||
const overviewFindings = 20
|
||||
|
||||
// inOverview is a run as an overview carries it: each probe's findings at most overviewFindings, with
|
||||
// their count and how to read them all. The run itself is not changed.
|
||||
func inOverview(run doctorRun) doctorRun {
|
||||
probes := make([]probeVerdict, len(run.Probes))
|
||||
for i, p := range run.Probes {
|
||||
total := len(p.Found) + len(p.Unconfirmed)
|
||||
if total > overviewFindings {
|
||||
p.FoundCount = len(p.Found)
|
||||
if len(p.Found) > overviewFindings {
|
||||
p.Found = p.Found[:overviewFindings:overviewFindings]
|
||||
}
|
||||
if room := overviewFindings - len(p.Found); len(p.Unconfirmed) > room {
|
||||
p.Unconfirmed = p.Unconfirmed[:room:room]
|
||||
}
|
||||
p.More = fmt.Sprintf("%d found and %d unconfirmed, %d of them shown — `doctor probe=%s` gives them all",
|
||||
p.FoundCount, total-p.FoundCount, len(p.Found)+len(p.Unconfirmed), p.ID)
|
||||
}
|
||||
probes[i] = p
|
||||
}
|
||||
run.Probes = probes
|
||||
return run
|
||||
}
|
||||
|
||||
// oneProbe is a run with only one probe's verdict, whole.
|
||||
func oneProbe(run doctorRun, id string) (doctorRun, error) {
|
||||
for _, p := range run.Probes {
|
||||
if strings.EqualFold(p.ID, id) {
|
||||
run.Probes = []probeVerdict{p}
|
||||
return run, nil
|
||||
}
|
||||
}
|
||||
return doctorRun{}, fmt.Errorf("the run %s has no probe %q — `doctor probes=true` lists them", run.Run, id)
|
||||
}
|
||||
|
||||
// doctorCounts are a run's verdicts, counted.
|
||||
@@ -142,6 +244,8 @@ type doctor struct {
|
||||
teller conditions.Teller
|
||||
watchdogs *watchdogs
|
||||
host string
|
||||
// confirm holds back what one run alone saw of a finding a single look can be wrong about.
|
||||
confirm confirming
|
||||
|
||||
running sync.Mutex
|
||||
mu sync.Mutex
|
||||
@@ -253,21 +357,27 @@ func (d *doctor) runOnce(ctx context.Context, why string) doctorRun {
|
||||
case r.err != nil:
|
||||
v.Verdict, v.Error = verdictFailedToRun, r.err.Error()
|
||||
run.Counts.FailedToRun++
|
||||
// A probe that could not run once — a question timed out on a loaded machine — is said in
|
||||
// the verdict at once, and raised as a condition when the next run cannot run it either.
|
||||
blind = append(blind, conditions.Observation{Scope: conditions.ScopeProbe, ID: p.ID, Kind: "probe-failed",
|
||||
Token: "failed", Severity: conditions.Warning,
|
||||
Token: "failed", Severity: conditions.Warning, Confirm: true,
|
||||
Summary: fmt.Sprintf("the probe %s (%s) could not run: what it checks is not known — never a pass", p.ID, p.Asserts),
|
||||
Said: firstLine(r.err.Error())})
|
||||
default:
|
||||
if err := d.keeper.Reconcile(ctx, p.ID, kindedAs(r.obs, p.Kind)); err != nil {
|
||||
raise, held := d.confirm.pass(ctx, d.keeper, p.ID, kindedAs(r.obs, p.Kind))
|
||||
if err := d.keeper.Reconcile(ctx, p.ID, raise); err != nil {
|
||||
v.Error = "what it found could not be kept: " + err.Error()
|
||||
}
|
||||
if len(r.obs) == 0 {
|
||||
for _, o := range held {
|
||||
v.Unconfirmed = append(v.Unconfirmed, o.Summary)
|
||||
}
|
||||
if len(raise) == 0 {
|
||||
v.Verdict = verdictPass
|
||||
run.Counts.Passed++
|
||||
} else {
|
||||
v.Verdict = verdictFail
|
||||
run.Counts.Failed++
|
||||
for _, o := range r.obs {
|
||||
for _, o := range raise {
|
||||
v.Found = append(v.Found, o.Summary)
|
||||
}
|
||||
}
|
||||
@@ -277,6 +387,7 @@ func (d *doctor) runOnce(ctx context.Context, why string) doctorRun {
|
||||
}
|
||||
run.Probes = append(run.Probes, v)
|
||||
}
|
||||
blind, _ = d.confirm.pass(ctx, d.keeper, sourceDoctor, blind)
|
||||
if err := d.keeper.Reconcile(ctx, sourceDoctor, blind); err != nil {
|
||||
fmt.Printf("the self-check's own failures could not be kept: %v\n", err)
|
||||
}
|
||||
@@ -334,12 +445,13 @@ func doctorCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
set := flag.NewFlagSet("doctor", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
probe := set.String("probe", "", "one probe's verdict, with every finding")
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New("doctor [run|probes|signals] [--json]")
|
||||
return errors.New("doctor [run|probes|signals] [--probe <id>] [--json]")
|
||||
}
|
||||
answer, err := doctorAnswer(ctx, sub)
|
||||
answer, err := doctorAnswer(ctx, sub, *probe)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -351,12 +463,18 @@ func doctorCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
|
||||
// doctorAnswer is what the verb answers, as data.
|
||||
func doctorAnswer(ctx context.Context, sub string) (any, error) {
|
||||
//
|
||||
// A verdict is an overview — each probe's findings at most overviewFindings — unless probe names one,
|
||||
// which is then answered alone and whole (novox/hq issue 314).
|
||||
func doctorAnswer(ctx context.Context, sub, probe string) (any, error) {
|
||||
if probe != "" && sub != "" && sub != "run" {
|
||||
return nil, fmt.Errorf("probe names one probe of a verdict; %s has none", sub)
|
||||
}
|
||||
switch sub {
|
||||
case "":
|
||||
if doctorFrom != nil {
|
||||
if run := doctorFrom.lastRun(); run != nil {
|
||||
return verdictAnswer(*run, time.Now()), nil
|
||||
return verdictAnswer(*run, time.Now(), probe)
|
||||
}
|
||||
return nil, fmt.Errorf("the self-check has not finished its first run yet: it runs %s after the "+
|
||||
"controller starts, then every %s — `doctor run` runs it now", doctorFirstAfter, doctorEvery)
|
||||
@@ -365,7 +483,7 @@ func doctorAnswer(ctx context.Context, sub string) (any, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return verdictAnswer(run, time.Now()), nil
|
||||
return verdictAnswer(run, time.Now(), probe)
|
||||
case "run":
|
||||
d := doctorFrom
|
||||
if d == nil {
|
||||
@@ -376,7 +494,7 @@ func doctorAnswer(ctx context.Context, sub string) (any, error) {
|
||||
defer closeIt()
|
||||
d = local
|
||||
}
|
||||
return verdictAnswer(d.runOnce(ctx, "asked by "+link.Caller()), time.Now()), nil
|
||||
return verdictAnswer(d.runOnce(ctx, "asked by "+link.Caller()), time.Now(), probe)
|
||||
case "probes":
|
||||
return probesAnswer(), nil
|
||||
case "signals":
|
||||
@@ -390,9 +508,20 @@ func doctorAnswer(ctx context.Context, sub string) (any, error) {
|
||||
}
|
||||
|
||||
// verdictAnswer is a run as the verb answers it, with its age.
|
||||
func verdictAnswer(run doctorRun, now time.Time) map[string]any {
|
||||
func verdictAnswer(run doctorRun, now time.Time, probe string) (map[string]any, error) {
|
||||
if probe != "" {
|
||||
one, err := oneProbe(run, probe)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
run = one
|
||||
} else {
|
||||
run = inOverview(run)
|
||||
}
|
||||
return map[string]any{"run": run, "age": now.Sub(run.At).Round(time.Second).String(),
|
||||
"note": "a probe that could not run is never a pass; each failure is an open condition until a run passes it"}
|
||||
"note": "a probe that could not run is never a pass; each failure is an open condition until a run passes it, " +
|
||||
"and one a single look can be wrong about — an unanswered question, a slow answer — is raised when two " +
|
||||
"runs in a row see it"}, nil
|
||||
}
|
||||
|
||||
// probesAnswer is the registry.
|
||||
@@ -458,6 +587,12 @@ func doctorText(answer any) string {
|
||||
for _, f := range p.Found {
|
||||
fmt.Fprintf(&b, " %s\n", f)
|
||||
}
|
||||
for _, f := range p.Unconfirmed {
|
||||
fmt.Fprintf(&b, " unconfirmed, raised if the next run sees it too: %s\n", f)
|
||||
}
|
||||
if p.More != "" {
|
||||
fmt.Fprintf(&b, " … %s\n", p.More)
|
||||
}
|
||||
if p.Error != "" {
|
||||
fmt.Fprintf(&b, " %s\n", p.Error)
|
||||
}
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
@@ -20,6 +19,8 @@ import (
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// The self-check (novox/hq to-be 45 §4): a probe that fails raises its condition, one that cannot run
|
||||
@@ -74,6 +75,15 @@ func TestARunKeepsWhatEachProbeFoundAndSaysItsHeartbeat(t *testing.T) {
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||
defer k.Close(context.Background())
|
||||
d := &doctor{keeper: k, teller: told, host: "anchor"}
|
||||
// A probe that cannot run is said in the verdict at once, and as a condition when the next run cannot
|
||||
// run it either (novox/hq issue 277): one timed-out question is not a probe gone blind.
|
||||
first := d.runOnce(t.Context(), "a test")
|
||||
if first.Counts != (doctorCounts{Passed: 1, Failed: 1, FailedToRun: 2, Deferred: 1}) {
|
||||
t.Fatalf("counted %+v", first.Counts)
|
||||
}
|
||||
if open, _ := k.Open(t.Context()); len(open) != 1 || open[0].Key != "machine.anchor.refused" {
|
||||
t.Fatalf("after one run, open: %+v", open)
|
||||
}
|
||||
run := d.runOnce(t.Context(), "a test")
|
||||
if run.Counts != (doctorCounts{Passed: 1, Failed: 1, FailedToRun: 2, Deferred: 1}) {
|
||||
t.Fatalf("counted %+v", run.Counts)
|
||||
@@ -93,7 +103,7 @@ func TestARunKeepsWhatEachProbeFoundAndSaysItsHeartbeat(t *testing.T) {
|
||||
}
|
||||
}
|
||||
// The heartbeat, in the shape mesh-watcher reads (the contract with the operator's channel).
|
||||
if len(told.Names) != 1 || told.Names[0] != conditions.HeartbeatEvent {
|
||||
if len(told.Names) != 2 || told.Names[1] != conditions.HeartbeatEvent {
|
||||
t.Fatalf("said %v", told.Names)
|
||||
}
|
||||
if d.lastRunEnded().IsZero() || d.lastRun().Run != run.Run {
|
||||
@@ -178,6 +188,7 @@ func TestAMachineWhoseDeclarationDoesNotComposeIsSaid(t *testing.T) {
|
||||
}
|
||||
}
|
||||
got, err := probeDeclarations(ctx, &doctor{open: open})
|
||||
linted(got)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -240,10 +251,7 @@ func TestAResolverAnsweringNoSuchNameForIPv6IsWrong(t *testing.T) {
|
||||
// **D6, D7: what the controller defines is what it finds**, and a consumer deleted or a stream
|
||||
// redefined is said — against a real bus, raised by the same derivation the controller starts with.
|
||||
func TestNatsTheBusIsWhatTheControllerDefines(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
open := aMesh(t)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
@@ -290,10 +298,7 @@ func TestNatsTheBusIsWhatTheControllerDefines(t *testing.T) {
|
||||
|
||||
// **S9 hears the bus**: a consumer that gives up on a message, and one deleted, as the server says.
|
||||
func TestNatsTheBusSaysAConsumerGaveUpAndOneWasDeleted(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -45,7 +45,7 @@ func TestAMachineIsSentTheEpochOnlyOnceItSaysItReadsOne(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &bodiesDelivery{recordedDelivery: recordedDelivery{inv: inv}, bodies: map[string][]byte{}}
|
||||
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, composeForPush(open, gens), d, ""); err != nil {
|
||||
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, composeForPush(open, gens), d, "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
carried := func(node string) (epoch float64, has bool) {
|
||||
@@ -92,7 +92,7 @@ func TestNothingIsComposedOrSentWithoutTheLease(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &bodiesDelivery{recordedDelivery: recordedDelivery{inv: open.inventory}, bodies: map[string][]byte{}}
|
||||
refused, err := sendRound(ctx, open, []string{"anchor"}, composeForPush(open, gens), d, "")
|
||||
refused, err := sendRound(ctx, open, []string{"anchor"}, composeForPush(open, gens), d, "", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,658 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/validate"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
snapshot "github.com/novox/mesh-controller/internal/facts"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The facts snapshot (novox/hq to-be 45 §9, ADR 0227 rule 9): what a merge check needs to judge a change
|
||||
// against the mesh that runs, written by the controller to the artifact store, where the build seat reads
|
||||
// it. internal/facts says what it holds and what it never holds; this composes it from the store and
|
||||
// keeps it current.
|
||||
|
||||
// factsEvery is how often the snapshot is composed. It is kept when it moved — a machine, an
|
||||
// assignment, a seat, a setting, a build — or once a day when nothing did, so its age says the
|
||||
// controller is still writing it (S14).
|
||||
var factsEvery = 10 * time.Minute
|
||||
|
||||
// factsDaily is how old a snapshot of an unchanged mesh may grow before it is written again.
|
||||
const factsDaily = 24 * time.Hour
|
||||
|
||||
// factsStaleAfter is S14's bound: a snapshot older than this is one no check should be fed.
|
||||
const factsStaleAfter = 48 * time.Hour
|
||||
|
||||
// factsExport is what this controller knows of the snapshot it keeps: when the newest was taken, its
|
||||
// content, and the last attempt's error.
|
||||
type factsExport struct {
|
||||
mu sync.Mutex
|
||||
taken time.Time
|
||||
content string
|
||||
digest string
|
||||
err error
|
||||
began time.Time
|
||||
}
|
||||
|
||||
// exportedFacts is this process's export, read by S14.
|
||||
var exportedFacts = &factsExport{}
|
||||
|
||||
func (e *factsExport) last() (taken time.Time, digest string, began time.Time, err error) {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
return e.taken, e.digest, e.began, e.err
|
||||
}
|
||||
|
||||
func (e *factsExport) kept(f snapshot.Facts, content, digest string) {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
e.taken, e.content, e.digest, e.err = f.Taken, content, digest, nil
|
||||
}
|
||||
|
||||
func (e *factsExport) failed(err error) {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
e.err = err
|
||||
}
|
||||
|
||||
// exportingFacts keeps the snapshot current for as long as this controller holds the lease: ctx ends
|
||||
// when it stops acting.
|
||||
func exportingFacts(ctx context.Context, open *stores, busVersion func() string) {
|
||||
exportedFacts.mu.Lock()
|
||||
exportedFacts.began = time.Now()
|
||||
exportedFacts.mu.Unlock()
|
||||
// What the store holds already, so a restarted controller neither writes an unchanged snapshot again
|
||||
// nor reads its age as zero.
|
||||
if address, err := factsStore(ctx, open); err == nil {
|
||||
if body, digest, err := (artifacts.Store{Address: address}).GetTagged(ctx, snapshot.Repository, snapshot.Tag); err == nil {
|
||||
if f, err := snapshot.Decode(body); err == nil {
|
||||
content, _ := f.Content()
|
||||
exportedFacts.kept(f, content, digest)
|
||||
}
|
||||
}
|
||||
}
|
||||
failing := ""
|
||||
first := time.NewTimer(time.Minute)
|
||||
defer first.Stop()
|
||||
tick := time.NewTicker(factsEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-first.C:
|
||||
case <-tick.C:
|
||||
}
|
||||
wrote, err := exportFacts(ctx, open, busVersion(), false)
|
||||
why := ""
|
||||
if err != nil {
|
||||
why = err.Error()
|
||||
exportedFacts.failed(err)
|
||||
}
|
||||
if why != failing {
|
||||
if why != "" {
|
||||
fmt.Printf("the facts snapshot cannot be kept: %s\n", why)
|
||||
} else {
|
||||
fmt.Println("the facts snapshot is kept again")
|
||||
}
|
||||
failing = why
|
||||
}
|
||||
if wrote != "" {
|
||||
fmt.Printf("the facts snapshot moved and is kept as %s\n", short(strings.TrimPrefix(wrote, "sha256:")))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// exportFacts composes the snapshot and keeps it when it moved, or when the one kept is a day old, or
|
||||
// when told to. Answers the digest it kept, empty when it kept nothing.
|
||||
func exportFacts(ctx context.Context, open *stores, busVersion string, force bool) (string, error) {
|
||||
f, err := gatherFacts(ctx, open, busVersion)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
content, err := f.Content()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
exportedFacts.mu.Lock()
|
||||
unchanged := content == exportedFacts.content && time.Since(exportedFacts.taken) < factsDaily
|
||||
exportedFacts.mu.Unlock()
|
||||
if unchanged && !force {
|
||||
return "", nil
|
||||
}
|
||||
body, err := f.Encode()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
address, err := factsStore(ctx, open)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
digest, err := (artifacts.Store{Address: address}).PutTagged(ctx, snapshot.Repository, snapshot.Tag, snapshot.MediaType, body)
|
||||
if err != nil && digest == "" {
|
||||
return "", err
|
||||
}
|
||||
exportedFacts.kept(f, content, digest)
|
||||
return digest, err
|
||||
}
|
||||
|
||||
// factsStore is the artifact store as this controller reaches it.
|
||||
func factsStore(ctx context.Context, open *stores) (string, error) {
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
address, err := artifactStoreAddress(ctx, open.inventory, shelf, "")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if address == "" {
|
||||
return "", errors.New("the artifact store is not on the private network, so there is nowhere to keep the facts")
|
||||
}
|
||||
return address, nil
|
||||
}
|
||||
|
||||
// gatherFacts composes one snapshot from the store: read only, nothing made, nothing sent.
|
||||
func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot.Facts, error) {
|
||||
inv := open.inventory
|
||||
f := snapshot.Facts{Format: snapshot.Format, Taken: time.Now().UTC(), Controller: snapshot.Build{Version: version}}
|
||||
f.Versions.Bus = busVersion
|
||||
storeVersion, err := inv.ServerVersion(ctx)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, fmt.Errorf("the store will not say its version: %w", err)
|
||||
}
|
||||
f.Versions.Store = storeVersion
|
||||
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
overlays, err := inv.Overlays(ctx)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
place := map[string]inventory.Overlay{}
|
||||
for _, o := range overlays {
|
||||
place[o.Name] = o
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
shelf := map[string]catalogue.Manifest{}
|
||||
for _, e := range entries {
|
||||
shelf[e.Manifest.Module] = e.Manifest
|
||||
}
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
read, err := readForPlanning(ctx, inv)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
edges, err := inv.Dependencies(ctx)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
holdings, err := inv.Holdings(ctx)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
|
||||
// **Every name first**, so text read afterwards — a setting naming a machine, a problem naming a
|
||||
// site — has it replaced wherever it appears.
|
||||
scrub := snapshot.NewScrubber()
|
||||
domains := map[string]string{}
|
||||
for _, n := range nodes {
|
||||
scrub.Machine(n.Name)
|
||||
if n.Account != "" && n.Account != "root" {
|
||||
scrub.Account(n.Account)
|
||||
}
|
||||
d, err := inv.PublicDomainOf(ctx, n.Name)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
domains[n.Name] = scrub.Domain(d)
|
||||
}
|
||||
for _, o := range overlays {
|
||||
scrub.Site(o.Site)
|
||||
}
|
||||
|
||||
// What a merge check runs in and reads beside it, as the build seat would be asked for it.
|
||||
if held, err := inv.Held(ctx); err == nil {
|
||||
for language, reference := range builder.ToolchainsOf(held) {
|
||||
if f.Versions.Toolchains == nil {
|
||||
f.Versions.Toolchains = map[string]string{}
|
||||
}
|
||||
f.Versions.Toolchains[language] = catalogue.Recorded(reference)
|
||||
}
|
||||
} else {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if dir, core := coreModules[e.Manifest.Module]; core && !e.Provided && e.Source.Repository != "" {
|
||||
for d, ref := range besideRefs(dir, current[e.Manifest.Module].Commit) {
|
||||
if f.Beside == nil {
|
||||
f.Beside = map[string]string{}
|
||||
}
|
||||
f.Beside[d] = ref
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if c, ok := current["mesh-controller"]; ok {
|
||||
f.Controller.Commit = c.Commit
|
||||
}
|
||||
|
||||
gens, gensErr := generators(ctx, open)
|
||||
hostShelf := shelf[hostModule]
|
||||
meshWide := map[string]bool{}
|
||||
engines := map[string]bool{}
|
||||
for _, n := range nodes {
|
||||
m := snapshot.Machine{Name: scrub.Machine(n.Name), Length: len(n.Name), Adopted: n.Adopted,
|
||||
AccountHome: scrub.Text(n.AccountHome), NodeEngine: n.HostVersion, PublicDomain: domains[n.Name],
|
||||
AgentAccount: scrub.Account(n.AgentAccount), AgentAccountHome: scrub.Text(n.AgentAccountHome)}
|
||||
switch n.Account {
|
||||
case "", "root":
|
||||
m.Account = n.Account
|
||||
default:
|
||||
m.Account = scrub.Account(n.Account)
|
||||
}
|
||||
if n.HostVersion != "" {
|
||||
engines[n.HostVersion] = true
|
||||
}
|
||||
m.System = systemOf(hostShelf, n.HostVersion)
|
||||
m.Libc = libcOf(m.System)
|
||||
reported, err := inv.DescribedOf(ctx, n.Name)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
m.Architecture, m.Kernel = reported.Architecture, reported.Kernel
|
||||
outward, err := inv.OutwardLinksOf(ctx, n.Name)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
for _, l := range outward {
|
||||
m.OutwardLinks = append(m.OutwardLinks, scrub.Text(l))
|
||||
}
|
||||
capabilities, err := inv.Profile(ctx, n.Name)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
for _, c := range capabilities {
|
||||
kept := snapshot.Capability{Name: c.Name, Present: c.Present}
|
||||
// The detail only where it is a version: everything else a detector says — a ruleset, a
|
||||
// device, a path — is the machine's own business and no check reads it.
|
||||
if c.Present && (c.Name == "container-runtime" || c.Name == "package-manager") {
|
||||
kept.Detail = scrub.Text(c.Detail)
|
||||
}
|
||||
m.Capabilities = append(m.Capabilities, kept)
|
||||
}
|
||||
if o, ok := place[n.Name]; ok {
|
||||
m.Site, m.Hub, m.Public, m.OnNetwork = scrub.Site(o.Site), o.Hub, o.Endpoint != "", o.Address != ""
|
||||
}
|
||||
assigned, err := inv.Assigned(ctx, n.Name)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
m.Assigned = assigned
|
||||
sent, known, err := inv.SentBuilds(ctx, n.Name)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
if known && slices.Contains(assigned, broker.RuntimeModule) {
|
||||
m.NodeTools = sent[broker.RuntimeModule]
|
||||
}
|
||||
pins, err := inv.PinsFor(ctx, n.Name)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
for provision, c := range pins {
|
||||
m.Pins = append(m.Pins, snapshot.Pin{Provision: provision, Machine: scrub.Machine(c.Node), Module: c.Module})
|
||||
}
|
||||
for _, module := range assigned {
|
||||
held, err := inv.SecretsOf(ctx, n.Name, module)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
for _, h := range held {
|
||||
if h.Origin == inventory.OriginAccepted {
|
||||
m.Accepted = append(m.Accepted, snapshot.Accepted{Module: module, Name: h.Name,
|
||||
Provider: scrub.Machine(h.Provider), Local: h.Local})
|
||||
}
|
||||
}
|
||||
layers, err := inv.SettingsFor(ctx, n.Name, module)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
for _, layer := range layers {
|
||||
if layer.From == catalogue.MeshWideLayer {
|
||||
if !meshWide[module] {
|
||||
meshWide[module] = true
|
||||
f.Settings = append(f.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Values)})
|
||||
}
|
||||
continue
|
||||
}
|
||||
m.Settings = append(m.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Values)})
|
||||
}
|
||||
}
|
||||
m.Declaration = declarationFacts(ctx, open, n.Name, gens, gensErr, scrub)
|
||||
if ctx.Err() != nil {
|
||||
return snapshot.Facts{}, ctx.Err()
|
||||
}
|
||||
f.Machines = append(f.Machines, m)
|
||||
}
|
||||
for e := range engines {
|
||||
f.Versions.NodeEngines = append(f.Versions.NodeEngines, e)
|
||||
}
|
||||
|
||||
// Seats and their holders, and from them the roles a machine is named by.
|
||||
roles := map[string][]string{}
|
||||
seats := map[string]*snapshot.Seat{}
|
||||
for _, h := range holdings {
|
||||
key := h.Claim + "\x00" + h.Scope
|
||||
s, ok := seats[key]
|
||||
if !ok {
|
||||
s = &snapshot.Seat{Name: h.Claim, Scope: h.Scope}
|
||||
seats[key] = s
|
||||
}
|
||||
s.Holders = append(s.Holders, snapshot.Holder{Machine: scrub.Machine(h.Node), Module: h.Module})
|
||||
if h.Scope == catalogue.ScopeMesh {
|
||||
role := "holds " + h.Claim
|
||||
if h.Claim == catalogue.ControllerSeatName {
|
||||
role = "the control node"
|
||||
}
|
||||
roles[h.Node] = append(roles[h.Node], role)
|
||||
}
|
||||
}
|
||||
for _, s := range seats {
|
||||
f.Seats = append(f.Seats, *s)
|
||||
}
|
||||
for i := range f.Machines {
|
||||
for _, n := range nodes {
|
||||
if scrub.Machine(n.Name) != f.Machines[i].Name {
|
||||
continue
|
||||
}
|
||||
f.Machines[i].Roles = roles[n.Name]
|
||||
if f.Machines[i].Hub {
|
||||
f.Machines[i].Roles = append(f.Machines[i].Roles, "the hub")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Every module, as the mesh holds it, and where it is built from.
|
||||
newest := map[string]inventory.Source{}
|
||||
count := map[string]int{}
|
||||
for _, e := range entries {
|
||||
raw, err := json.Marshal(e.Manifest)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
mod := snapshot.Module{Name: e.Manifest.Module, Repository: snapshot.RepositoryName(e.Source.Repository), Path: e.Source.Path,
|
||||
Commit: e.Source.BuiltFrom, Provided: e.Provided, RollOut: current[e.Manifest.Module].RollOut,
|
||||
Manifest: raw}
|
||||
for _, r := range read[e.Manifest.Module] {
|
||||
// The module's own build source is said apart: a gate that predates it would read an own
|
||||
// entry among Reads as a context of its own repository.
|
||||
if r.Own {
|
||||
if len(r.Paths) > 0 {
|
||||
mod.Sources = append(mod.Sources, snapshot.BuildSource{Own: true, Paths: r.Paths})
|
||||
}
|
||||
continue
|
||||
}
|
||||
mod.Reads = append(mod.Reads, snapshot.RepositoryName(r.Repository))
|
||||
if len(r.Paths) > 0 {
|
||||
mod.Sources = append(mod.Sources, snapshot.BuildSource{Repository: snapshot.RepositoryName(r.Repository), Paths: r.Paths})
|
||||
}
|
||||
}
|
||||
f.Modules = append(f.Modules, mod)
|
||||
if e.Provided || e.Source.Repository == "" {
|
||||
continue
|
||||
}
|
||||
count[e.Source.Repository]++
|
||||
if was, ok := newest[e.Source.Repository]; !ok || e.Source.Seen.After(was.Seen) {
|
||||
newest[e.Source.Repository] = e.Source
|
||||
}
|
||||
}
|
||||
for repository, s := range newest {
|
||||
commit := s.Head
|
||||
if commit == "" {
|
||||
commit = s.BuiltFrom
|
||||
}
|
||||
f.Sources = append(f.Sources, snapshot.Source{Repository: snapshot.RepositoryName(repository), Commit: commit,
|
||||
Modules: count[repository]})
|
||||
}
|
||||
for _, e := range edges {
|
||||
f.Edges = append(f.Edges, snapshot.Edge{From: e.From, To: e.To, Kind: e.Kind})
|
||||
}
|
||||
f.Sorted()
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// declarationFacts is how one machine's declaration composes now, as the next push would compose it and
|
||||
// without making anything (D1's composition), and whether the node-engine's validator takes it.
|
||||
func declarationFacts(ctx context.Context, open *stores, node string, gens map[string]catalogue.Generator,
|
||||
gensErr error, scrub *snapshot.Scrubber) snapshot.Declaration {
|
||||
var d snapshot.Declaration
|
||||
if gensErr != nil {
|
||||
d.Problems = []string{scrub.Text("the private network cannot be computed: " + oneLine(gensErr.Error()))}
|
||||
return d
|
||||
}
|
||||
declared, problems, err := composedAndValidated(ctx, open, node, gens, Foreseeing)
|
||||
if err != nil {
|
||||
d.Problems = []string{scrub.Text(oneLine(err.Error()))}
|
||||
return d
|
||||
}
|
||||
for _, p := range problems {
|
||||
d.Problems = append(d.Problems, scrub.Text(p))
|
||||
}
|
||||
d.Composes = len(problems) == 0
|
||||
if body, err := declared.Body(); err == nil {
|
||||
d.Digest = fmt.Sprintf("sha256:%x", sha256.Sum256(body))
|
||||
}
|
||||
// Scrubbed like every other word: a resource is named after the machine a grant is for.
|
||||
for _, r := range resourceNames(declared.Resources) {
|
||||
d.Resources = append(d.Resources, scrub.Text(r))
|
||||
}
|
||||
for module, why := range declared.leftOutWhy {
|
||||
if d.LeftOut == nil {
|
||||
d.LeftOut = map[string]string{}
|
||||
}
|
||||
d.LeftOut[module] = scrub.Text(why)
|
||||
}
|
||||
for _, o := range declared.withheld {
|
||||
d.Withheld = append(d.Withheld, scrub.Text(o.String()))
|
||||
}
|
||||
for _, u := range declared.unbound {
|
||||
d.Unbound = append(d.Unbound, scrub.Text(u.String()))
|
||||
}
|
||||
sort.Strings(d.Withheld)
|
||||
sort.Strings(d.Unbound)
|
||||
return d
|
||||
}
|
||||
|
||||
// composedAndValidated composes one machine's declaration — as a push would (Allocating) or as the next
|
||||
// push will without making anything (Foreseeing) — with the order it was last sent, and runs the
|
||||
// node-engine's own validator over the body. An error is that it did not compose; problems are what the
|
||||
// validator refuses.
|
||||
func composedAndValidated(ctx context.Context, open *stores, node string, gens map[string]catalogue.Generator,
|
||||
choosing Choosing) (sendable, []string, error) {
|
||||
plan, settings, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
return sendable{}, nil, err
|
||||
}
|
||||
declared, err := declarationWith(ctx, open, node, plan, settings, gens, choosing)
|
||||
if err != nil {
|
||||
return sendable{}, nil, err
|
||||
}
|
||||
record, err := open.inventory.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return sendable{}, nil, err
|
||||
}
|
||||
if declared.Sequence, err = open.inventory.Sequence(ctx, record.ID); err != nil {
|
||||
return sendable{}, nil, err
|
||||
}
|
||||
if declared.Epoch, err = open.inventory.SentEpoch(ctx, record.ID); err != nil {
|
||||
return sendable{}, nil, err
|
||||
}
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
return sendable{}, nil, err
|
||||
}
|
||||
return declared, validate.Declaration(body), nil
|
||||
}
|
||||
|
||||
// resourceNames are a declaration's resources as `type:id`, sorted.
|
||||
func resourceNames(resources []map[string]any) []string {
|
||||
out := make([]string, 0, len(resources))
|
||||
for _, r := range resources {
|
||||
kind, _ := r["type"].(string)
|
||||
id, _ := r["id"].(string)
|
||||
out = append(out, kind+":"+id)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// systemOf is the system a node-engine of that version was built for, read from the build the mesh
|
||||
// holds: the artifact a resource delivered into `versions/<version>` came from is named for its system
|
||||
// (`host-arch`). Empty when the version is not a delivered one — an engine placed by hand.
|
||||
func systemOf(host catalogue.Manifest, version string) string {
|
||||
if version == "" {
|
||||
return ""
|
||||
}
|
||||
for _, r := range host.Resources {
|
||||
path, _ := r["path"].(string)
|
||||
if !strings.HasSuffix(path, "/versions/"+version) {
|
||||
continue
|
||||
}
|
||||
source, _ := r["source"].(string)
|
||||
for _, part := range strings.Split(source, "/") {
|
||||
if system, ok := strings.CutPrefix(part, "host-"); ok && system != "" {
|
||||
return system
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// libcOf is the C library of a system the node-engine is built for.
|
||||
func libcOf(system string) string {
|
||||
switch system {
|
||||
case "arch":
|
||||
return "glibc"
|
||||
case "alpine":
|
||||
return "musl"
|
||||
case "android":
|
||||
return "bionic"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// factsCommand is `facts`: what the controller keeps, and keeping it now.
|
||||
//
|
||||
// facts the snapshot the artifact store holds: when, which, how many machines
|
||||
// facts show the same, whole, as JSON
|
||||
// facts export compose and keep one now
|
||||
// facts compose compose one and print it, keeping nothing
|
||||
func factsCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("facts", flag.ContinueOnError)
|
||||
rest, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
what := ""
|
||||
if len(rest) > 0 {
|
||||
what = rest[0]
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
switch what {
|
||||
case "", "show":
|
||||
address, err := factsStore(ctx, open)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body, digest, err := (artifacts.Store{Address: address}).GetTagged(ctx, snapshot.Repository, snapshot.Tag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if what == "show" {
|
||||
_, err := os.Stdout.Write(body)
|
||||
return err
|
||||
}
|
||||
f, err := snapshot.Decode(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("the facts snapshot kept as %s:%s is %s, taken %s (%s ago) by controller %s\n",
|
||||
snapshot.Repository, snapshot.Tag, short(strings.TrimPrefix(digest, "sha256:")),
|
||||
f.Taken.Format(time.RFC3339), ago(time.Since(f.Taken)), orNone(f.Controller.Commit))
|
||||
fmt.Printf(" %d machine(s), %d module(s), %d seat(s); the longest machine name is %d characters\n",
|
||||
len(f.Machines), len(f.Modules), len(f.Seats), f.Longest())
|
||||
fmt.Printf(" the bus runs %s, the store %s\n", orNone(f.Versions.Bus), orNone(f.Versions.Store))
|
||||
for _, m := range f.Machines {
|
||||
state := "composes"
|
||||
if !m.Declaration.Composes {
|
||||
state = "does NOT compose: " + strings.Join(m.Declaration.Problems, "; ")
|
||||
}
|
||||
fmt.Printf(" %-12s %s; %d module(s); %s\n", m.Name, m.Described(), len(m.Assigned), state)
|
||||
}
|
||||
return nil
|
||||
case "export", "compose":
|
||||
busVersion := ""
|
||||
if server, err := connectLink(ctx, nil, nil, nil); err == nil {
|
||||
busVersion = busVersionOf(server)
|
||||
server.Close()
|
||||
}
|
||||
if what == "compose" {
|
||||
f, err := gatherFacts(ctx, open, busVersion)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := f.Encode()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = os.Stdout.Write(append(body, '\n'))
|
||||
return err
|
||||
}
|
||||
digest, err := exportFacts(ctx, open, busVersion, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("the facts snapshot is kept as %s:%s, %s\n", snapshot.Repository, snapshot.Tag, digest)
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("facts [show|export|compose], not %q", what)
|
||||
}
|
||||
|
||||
// busVersionOf is the bus server's release, as it told this connection.
|
||||
func busVersionOf(server *link.Server) string {
|
||||
if bus, ok := server.Bus().(link.OverNATS); ok && bus.Conn != nil {
|
||||
return bus.Conn.ConnectedServerVersion()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
snapshot "github.com/novox/mesh-controller/internal/facts"
|
||||
)
|
||||
|
||||
// The facts snapshot (novox/hq to-be 45 §9): composed from the store, every machine under a pseudonym
|
||||
// of its name's length, and nothing of the installation in it — no secret, no address, no name.
|
||||
|
||||
// aMeshWithSecrets is aMesh with a provider and its consumers, a value given by hand, settings carrying
|
||||
// a password, an address and a machine's name, and a push's worth of credentials made.
|
||||
func aMeshWithSecrets(t *testing.T) (*stores, []string) {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "objects", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "s3-bucket", Scope: catalogue.ScopeMesh,
|
||||
Identity: &catalogue.OfferIdentity{Max: 20, In: "an S3 access key"}}},
|
||||
Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}})
|
||||
register(t, open, catalogue.Manifest{Module: "files", Version: "1", Requires: []string{"s3-bucket"},
|
||||
Resources: []map[string]any{{"id": "config", "type": "file", "path": "/etc/files/config.json",
|
||||
"mode": "0600", "content": "{}", "merge": "json"}}})
|
||||
for _, a := range [][2]string{{"anchor", "objects"}, {"laptop", "files"}} {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
secrets := []string{"Hunter2-Is-Not-A-Password-9f8e7d", "0123456789abcdefABCDEF0123456789zz"}
|
||||
if err := open.inventory.SetSettings(ctx, "", "files", map[string]any{
|
||||
"admin_password": secrets[0], "upstream": "10.77.0.9", "hub": "anchor"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "laptop", "files", map[string]any{
|
||||
"note": "reach me at 192.168.1.135, token " + secrets[1]}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A push's worth of composition, which makes the pair credential the consumer is sent.
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, node := range []string{"laptop", "anchor"} {
|
||||
if _, _, err := composedAndValidated(ctx, open, node, gens, Allocating); err != nil {
|
||||
t.Fatalf("%s does not compose: %v", node, err)
|
||||
}
|
||||
}
|
||||
issued, err := open.inventory.SecretsFrom(ctx, "anchor")
|
||||
if err != nil || len(issued) == 0 {
|
||||
t.Fatalf("no credential was made for the consumer: %v", err)
|
||||
}
|
||||
for _, s := range issued {
|
||||
// Sealed, never kept plain (ADR 0004): the sealed blobs are what the store holds, and none may leave.
|
||||
secrets = append(secrets, s.ForConsumer, s.ForProvider)
|
||||
}
|
||||
return open, secrets
|
||||
}
|
||||
|
||||
func TestTheFactsCarryNoSecretNoAddressAndNoName(t *testing.T) {
|
||||
open, secrets := aMeshWithSecrets(t)
|
||||
f, err := gatherFacts(t.Context(), open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := f.Encode()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := string(body)
|
||||
for _, s := range secrets {
|
||||
if s != "" && strings.Contains(text, s) {
|
||||
t.Errorf("a secret is in the snapshot: %q", s)
|
||||
}
|
||||
}
|
||||
for _, leaked := range []string{"anchor", "laptop", "10.77.0.", "192.168.1.135", ".example:51820"} {
|
||||
if strings.Contains(text, leaked) {
|
||||
t.Errorf("%q is in the snapshot", leaked)
|
||||
}
|
||||
}
|
||||
// Every address it carries is a documentation address.
|
||||
for _, a := range regexp.MustCompile(`\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b`).FindAllString(text, -1) {
|
||||
if !strings.HasPrefix(a, "192.0.2.") && !strings.HasPrefix(a, "198.51.100.") && !strings.HasPrefix(a, "203.0.113.") {
|
||||
t.Errorf("%s is an address outside the documentation ranges", a)
|
||||
}
|
||||
}
|
||||
|
||||
// What a check needs is there: every machine, its length, its modules, its declaration composing.
|
||||
if len(f.Machines) != 2 || f.Longest() != len("laptop") {
|
||||
t.Fatalf("machines %+v, longest %d", f.Machines, f.Longest())
|
||||
}
|
||||
anchor := snapshot.Pseudonym("machine", "anchor")
|
||||
m, ok := f.Machine(anchor)
|
||||
if !ok || !m.Hub || !strings.Contains(m.Described(), "the hub") || len(m.Name) != len("anchor") {
|
||||
t.Fatalf("the anchor reads as %+v", m)
|
||||
}
|
||||
if !m.Declaration.Composes || m.Declaration.Digest == "" || len(m.Declaration.Resources) == 0 {
|
||||
t.Errorf("the anchor's declaration reads as %+v", m.Declaration)
|
||||
}
|
||||
laptop, _ := f.Machine(snapshot.Pseudonym("machine", "laptop"))
|
||||
if strings.Join(laptop.Assigned, ",") != "files,mesh-wireguard" && !strings.Contains(strings.Join(laptop.Assigned, ","), "files") {
|
||||
t.Errorf("the laptop's assignments read as %v", laptop.Assigned)
|
||||
}
|
||||
var meshWide map[string]any
|
||||
for _, s := range f.Settings {
|
||||
if s.Module == "files" {
|
||||
meshWide = s.Values
|
||||
}
|
||||
}
|
||||
if meshWide["admin_password"] != snapshot.Withheld || meshWide["hub"] != anchor {
|
||||
t.Errorf("the mesh-wide settings read as %v", meshWide)
|
||||
}
|
||||
if f.Versions.Bus != "2.11.17" || f.Versions.Store == "" {
|
||||
t.Errorf("versions read as %+v", f.Versions)
|
||||
}
|
||||
var objects bool
|
||||
for _, mod := range f.Modules {
|
||||
objects = objects || mod.Name == "objects" && len(mod.Manifest) > 0
|
||||
}
|
||||
if !objects {
|
||||
t.Error("the modules the mesh holds are not in the snapshot")
|
||||
}
|
||||
|
||||
// And an unchanged mesh is the same content a moment later.
|
||||
again, err := gatherFacts(t.Context(), open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a, _ := f.Content()
|
||||
b, _ := again.Content()
|
||||
if a != b {
|
||||
t.Error("two snapshots of an unchanged mesh differ, so it would be written again every ten minutes")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A directory the node-engine uses as found (novox/hq issue 339) waits for a person to hand it over at the
|
||||
// machine. Found before this send, it is no fault of the build: the gate passes with the wait carried, so an
|
||||
// urgent fix of that module still goes through. Found by this send, the send brought it, and the gate holds.
|
||||
func foundDirectory(module string, since time.Time) inventory.ResourceHealth {
|
||||
return inventory.ResourceHealth{Module: module, Resource: module + ".data", Kind: link.KindDirectory,
|
||||
Target: "/srv/" + module, State: link.StateUnhealthy, Since: since,
|
||||
Reason: link.ReasonUsedAsFound + " owned by 1000:1000, mode 700, as found; root, mode 755 was declared and " +
|
||||
"not given it — `nox node hand-over laptop <directory>` on the control-node, with its path, hands it to the mesh"}
|
||||
}
|
||||
|
||||
func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) {
|
||||
now := time.Now()
|
||||
sent := now.Add(-time.Minute)
|
||||
f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now,
|
||||
Resources: []inventory.ResourceHealth{foundDirectory("notes", sent.Add(-24*time.Hour))}}}}
|
||||
h, why := moduleHealthWord("notes", "laptop", sent, f)
|
||||
if h != healthPerson || !strings.Contains(why, "notes.data") || !strings.Contains(why, "hand-over") {
|
||||
t.Fatalf("a directory found before the send reads %v %q; want a wait for a person", h, why)
|
||||
}
|
||||
// Found by this very send: the send brought it, and it is not passed.
|
||||
f.health["laptop"] = inventory.NodeHealth{Node: "laptop", HeardAt: now,
|
||||
Resources: []inventory.ResourceHealth{foundDirectory("notes", sent.Add(time.Second))}}
|
||||
if h, why := moduleHealthWord("notes", "laptop", sent, f); h != healthNotYet {
|
||||
t.Fatalf("a directory this send found reads %v %q; want not yet", h, why)
|
||||
}
|
||||
// A container down beside the old wait is a fault, as before.
|
||||
f.health["laptop"] = inventory.NodeHealth{Node: "laptop", HeardAt: now, Resources: []inventory.ResourceHealth{
|
||||
foundDirectory("notes", sent.Add(-time.Hour)),
|
||||
{Module: "notes", Resource: "notes.web", Kind: "container", Target: "notes", State: link.StateUnhealthy, Reason: "down"}}}
|
||||
if h, why := moduleHealthWord("notes", "laptop", sent, f); h != healthNotYet {
|
||||
t.Fatalf("a container down beside the wait reads %v %q; want not yet", h, why)
|
||||
}
|
||||
}
|
||||
|
||||
// The whole walk: a module whose directory was used as found long before still gets its fix to every machine,
|
||||
// its pass kept and the wait said; a directory this very send found holds it and puts it back.
|
||||
func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
found time.Duration // when the directory was found, against now
|
||||
passes bool
|
||||
}{
|
||||
{"found a day before the send", -24 * time.Hour, true},
|
||||
{"found by this send", time.Hour, false},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
|
||||
return map[string]bool{"anchor": true, "laptop": true}, nil
|
||||
}
|
||||
backlogFacts := gatherGateFacts
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
f, err := backlogFacts(ctx, open, component)
|
||||
// The used-as-found condition, raised after the send (its second statement): its own kind, never a
|
||||
// fault the gate reads as the build's.
|
||||
f.judged, f.openErr = true, nil
|
||||
f.open = append(f.open, conditions.Condition{Key: usedAsFoundKey("app", "anchor"), Kind: kindUsedAsFound,
|
||||
Subject: conditions.Subject{Scope: conditions.ScopeModule, ID: "app.anchor", Machine: "anchor"},
|
||||
Raised: time.Now()})
|
||||
f.health = map[string]inventory.NodeHealth{}
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
f.health[n] = inventory.NodeHealth{Node: n, HeardAt: time.Now(), Resources: []inventory.ResourceHealth{
|
||||
{Module: "app", Resource: "app.web", Kind: "container", Target: "app", State: link.StateHealthy},
|
||||
foundDirectory("app", time.Now().Add(c.found)),
|
||||
{Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateHealthy}}}
|
||||
}
|
||||
return f, err
|
||||
}
|
||||
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
||||
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
|
||||
gateSettle, gateEvery, gateBound = 0, 0, 300*time.Millisecond
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
advancePlans(ctx, b.open)
|
||||
if p := b.release(t); p.State != inventory.PlanRolling {
|
||||
break
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
p := b.release(t)
|
||||
v, found, err := inv.GateOf(ctx, "build-app-c2")
|
||||
if c.passes {
|
||||
if p.State != inventory.PlanDone || err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("the walk is %s (%s); app's verdict %+v: want the fix through", p.State, p.Note, v)
|
||||
}
|
||||
if !strings.Contains(v.Why+p.Note, "hand it over") {
|
||||
t.Errorf("the wait is not carried: verdict %q, walk %q", v.Why, p.Note)
|
||||
}
|
||||
return
|
||||
}
|
||||
if p.State == inventory.PlanDone {
|
||||
t.Fatalf("a directory this send found let the walk through: %s", p.Note)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The condition says the wait in its own kind: the operator's, never urgent, and cleared once handed over.
|
||||
func TestADirectoryUsedAsFoundIsItsOwnCondition(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
ctx := t.Context()
|
||||
rs := map[string][]inventory.ResourceHealth{"notes": {foundDirectory("notes", time.Now().Add(-time.Hour))}}
|
||||
for i := 0; i < 2; i++ {
|
||||
if err := judgeModuleHealth(ctx, nil, k, "laptop", rs, map[string]int{"notes": i + 1}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
open, _ := k.Open(ctx)
|
||||
var got *conditions.Condition
|
||||
for i, c := range open {
|
||||
if c.Key == usedAsFoundKey("notes", "laptop") {
|
||||
got = &open[i]
|
||||
}
|
||||
if c.Kind == kindModuleUnhealthy {
|
||||
t.Fatalf("raised as a fault: %+v", c)
|
||||
}
|
||||
}
|
||||
if got == nil || got.Resolver != conditions.ResolverOperator || got.Severity == conditions.Urgent ||
|
||||
!strings.Contains(got.Summary, "notes.data") {
|
||||
t.Fatalf("the condition: %+v", got)
|
||||
}
|
||||
// Long open is still not urgent: only a person can hand it over, and nothing is broken by the wait.
|
||||
if err := judgeModuleHealth(ctx, nil, k, "laptop", rs, map[string]int{"notes": 3}, time.Now().Add(48*time.Hour)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open, _ = k.Open(ctx)
|
||||
for _, c := range open {
|
||||
if c.Key == usedAsFoundKey("notes", "laptop") && c.Severity == conditions.Urgent {
|
||||
t.Fatal("a directory used as found became urgent")
|
||||
}
|
||||
}
|
||||
if err := judgeModuleHealth(ctx, nil, k, "laptop", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open, _ = k.Open(ctx)
|
||||
for _, c := range open {
|
||||
if c.Key == usedAsFoundKey("notes", "laptop") {
|
||||
t.Fatal("not cleared once handed over")
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,453 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// What the review of issue 318's fix found (novox/hq ADR 0254): every module of a send leaves it with a
|
||||
// verdict, a pass is counted no faster than the gate's spacing, a carried build's verdict carries only its own
|
||||
// wait, and a provider waiting for a login says who waits on it.
|
||||
|
||||
// withGateBounds sets the gate's bounds for one test.
|
||||
func withGateBounds(t *testing.T, settle, every, bound time.Duration) {
|
||||
t.Helper()
|
||||
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
||||
gateSettle, gateEvery, gateBound = settle, every, bound
|
||||
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
|
||||
}
|
||||
|
||||
// factsOn is a judging's facts for one machine that applied its send: the node tools answer, and what it says
|
||||
// of its modules' resources.
|
||||
func factsOn(t *testing.T, machine string, since time.Time, rs ...inventory.ResourceHealth) func() gateFacts {
|
||||
return func() gateFacts {
|
||||
now := time.Now()
|
||||
at := now
|
||||
return gateFacts{now: now,
|
||||
reports: map[string]inventory.Reported{machine: {Node: machine, Outcome: inventory.OutcomeApplied, At: &at, Current: true}},
|
||||
engines: map[string]string{},
|
||||
rolledBack: map[string][]lease.Rollback{},
|
||||
served: map[string]served{machine: {runtime: true, tools: map[string]bool{}}},
|
||||
health: map[string]inventory.NodeHealth{machine: {Node: machine, HeardAt: now, Resources: rs}},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func healthyContainer(module string) inventory.ResourceHealth {
|
||||
return inventory.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module,
|
||||
State: link.StateHealthy}
|
||||
}
|
||||
|
||||
// **A fault decided before the settle time does not strand the module beside it** (review (a)): the node
|
||||
// tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass,
|
||||
// counted over the gate's spacing — before it says its own. The broken one alone is put back.
|
||||
func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
withGateBounds(t, 150*time.Millisecond, 20*time.Millisecond, 10*time.Second)
|
||||
since := time.Now().Add(-time.Second)
|
||||
base := factsOn(t, "laptop", since, healthyContainer("app"))
|
||||
wasGather := gatherGateFacts
|
||||
t.Cleanup(func() { gatherGateFacts = wasGather })
|
||||
gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) {
|
||||
f := base()
|
||||
f.rolledBack["laptop"] = []lease.Rollback{{Component: lease.ComponentNodeTools, Outcome: lease.OutcomeRolledBack,
|
||||
At: since.Add(100 * time.Millisecond), Why: "the node tools did not answer"}}
|
||||
return f, nil
|
||||
}
|
||||
g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since}
|
||||
pairs := []judged{{module: broker_runtime, node: "laptop"}, {module: "app", node: "laptop"}}
|
||||
judgings := 0
|
||||
for deadline := time.Now().Add(5 * time.Second); g.Verdict == "" && time.Now().Before(deadline); {
|
||||
if _, err := judgeMoves(ctx, open, g, pairs, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
judgings++
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
}
|
||||
if g.Verdict != inventory.GateFailed || judgings < gatePasses {
|
||||
t.Fatalf("verdict %q after %d judging(s): %+v; want failed, after the module beside it was judged", g.Verdict,
|
||||
judgings, g)
|
||||
}
|
||||
if !reflect.DeepEqual(g.Passing, []string{"app"}) || !reflect.DeepEqual(g.Failing, []string{broker_runtime}) {
|
||||
t.Fatalf("passing %v, failing %v; want app kept and the node tools put back", g.Passing, g.Failing)
|
||||
}
|
||||
if !strings.Contains(g.Why, "witness") {
|
||||
t.Errorf("the verdict does not say what broke: %q", g.Why)
|
||||
}
|
||||
}
|
||||
|
||||
// broker_runtime is the node tools' module name, a core component a witness judges.
|
||||
const broker_runtime = "node-tools"
|
||||
|
||||
// **A pass is counted only the gate's spacing after the one before** (review (c)): a send judged every tick
|
||||
// while a module beside it is not yet healthy counts the healthy one once.
|
||||
func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
withGateBounds(t, 0, time.Hour, time.Hour)
|
||||
since := time.Now().Add(-time.Minute)
|
||||
base := factsOn(t, "laptop", since, healthyContainer("app"), inventory.ResourceHealth{Module: "late",
|
||||
Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy, Reason: "down"})
|
||||
wasGather := gatherGateFacts
|
||||
t.Cleanup(func() { gatherGateFacts = wasGather })
|
||||
gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) { return base(), nil }
|
||||
g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since}
|
||||
pairs := []judged{{module: "app", node: "laptop"}, {module: "late", node: "laptop"}}
|
||||
now := time.Now()
|
||||
for i := 0; i < 3; i++ {
|
||||
if _, err := judgeMoves(ctx, open, g, pairs, now.Add(time.Duration(i)*time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if g.Healthy["app"] != 1 || g.Healthy["late"] != 0 {
|
||||
t.Fatalf("counted %v in three judgings within a second; want app once and late never", g.Healthy)
|
||||
}
|
||||
}
|
||||
|
||||
// **A carried build's pass carries its own wait, never another's** (review (e)).
|
||||
func TestACarriedPassCarriesOnlyItsOwnWait(t *testing.T) {
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
since := time.Now().Add(-time.Minute)
|
||||
g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since, Verdict: inventory.GatePassed,
|
||||
Why: "healthy 3 times over 2m0s" + waitsSaid(map[string]string{"late": "relogin needed on laptop: late waits"}),
|
||||
Waits: map[string]string{"late": "relogin needed on laptop: late waits"},
|
||||
Carried: []inventory.CarriedMove{{Module: "app", Node: "laptop", From: "c1", To: "c2", Build: "build-app-c2"},
|
||||
{Module: "late", Node: "laptop", From: "c1", To: "c2", Build: "build-late-c2"}}}
|
||||
passCarried(ctx, b.open, &inventory.Plan{ID: "release-test"}, g, "")
|
||||
app, _, _ := inv.GateOf(ctx, "build-app-c2")
|
||||
late, _, _ := inv.GateOf(ctx, "build-late-c2")
|
||||
if strings.Contains(app.Why, "waits for a person") || app.Verdict != inventory.GatePassed {
|
||||
t.Errorf("app's pass: %+v; want it without late's wait", app)
|
||||
}
|
||||
if !strings.Contains(late.Why, "relogin needed on laptop") {
|
||||
t.Errorf("late's pass: %+v; want its own wait", late)
|
||||
}
|
||||
}
|
||||
|
||||
// **The tier path keeps the pass of the module the gate is kept on** (review (b)): a plan's first send
|
||||
// carried its own module, healthy, and a build waiting on that machine that never became healthy. At the
|
||||
// bound the waiting one is put back and marked; the plan's own module keeps its pass, so no walk waits on it.
|
||||
func TestThePlansOwnModuleKeepsItsPassWhenItsSendFails(t *testing.T) {
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := g.open.inventory
|
||||
// `late` waits on anchor for a gate: c1 sent, c2 registered and built.
|
||||
for _, b := range []inventory.Build{
|
||||
{ID: "build-late-1", Module: "late", Commit: "l1", Repository: "novox/mesh-catalog", Path: "modules/late",
|
||||
Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)},
|
||||
{ID: "build-late-2", Module: "late", Commit: "l2", Repository: "novox/mesh-catalog", Path: "modules/late",
|
||||
Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)},
|
||||
} {
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: "late", Version: b.Commit})
|
||||
b.Manifest = manifest
|
||||
b.Made = []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + b.Commit}}
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "late", Version: b.Commit}, inventory.Source{
|
||||
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/late", BuiltFrom: b.Commit, Head: b.Commit,
|
||||
Asked: b.Asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b.Commit == "l1" {
|
||||
if _, err := inv.Assign(ctx, "anchor", "late"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, g.open, "anchor"), "d-anchor-late", map[string]string{"app": "c1",
|
||||
"late": "l1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
wasMoves := machineMoves
|
||||
t.Cleanup(func() { machineMoves = wasMoves })
|
||||
machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
|
||||
modules, err := open.inventory.Assigned(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sent, known, err := open.inventory.SentBuilds(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f.moves(node, modules, sent, known, all), nil
|
||||
}
|
||||
gather := gatherGateFacts
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
f, err := gather(ctx, open, component)
|
||||
f.health = map[string]inventory.NodeHealth{"anchor": {Node: "anchor", HeardAt: time.Now(), Resources: []inventory.ResourceHealth{
|
||||
healthyContainer("app"), {Module: "late", Resource: "late.web", Kind: "container", Target: "late",
|
||||
State: link.StateUnhealthy, Reason: "down"}}}}
|
||||
return f, err
|
||||
}
|
||||
gateEvery, gateBound = 0, 400*time.Millisecond
|
||||
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); {
|
||||
advancePlans(ctx, g.open)
|
||||
if p := g.plan(t); p.State == inventory.PlanFailed || p.State == inventory.PlanDone {
|
||||
break
|
||||
}
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
}
|
||||
p := g.plan(t)
|
||||
if p.State != inventory.PlanFailed {
|
||||
t.Fatalf("the plan is %s: %s; want it failed on late", p.State, p.Note)
|
||||
}
|
||||
if v, found, err := inv.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed ||
|
||||
!strings.Contains(v.Why, "on its own") {
|
||||
t.Fatalf("app's verdict: %+v (found %v, %v); want its own pass kept", v, found, err)
|
||||
}
|
||||
if failed, _ := inv.GateFailed(ctx, "build-late-2"); !failed {
|
||||
t.Fatal("late's build is not marked failed at its gate")
|
||||
}
|
||||
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" || current["late"].Commit != "l1" {
|
||||
t.Fatalf("registered app %s, late %s; want app kept at c2 and late put back to l1", current["app"].Commit,
|
||||
current["late"].Commit)
|
||||
}
|
||||
}
|
||||
|
||||
// **A provider waiting for a login says who waits on it** (review (g)): its consumer, failing a check that
|
||||
// needs it, is held under it, and the provider's relogin-needed condition lists it and is urgent.
|
||||
func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
k := conditionsFrom
|
||||
register(t, open, catalogue.Manifest{Module: "db", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}})
|
||||
register(t, open, catalogue.Manifest{Module: "shop", Version: "1", Requires: []string{"postgres-database"}})
|
||||
for _, a := range [][2]string{{"anchor", "db"}, {"laptop", "shop"}} {
|
||||
if _, err := inv.Assign(ctx, a[0], a[1]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.RecordBindings(ctx, "laptop", []inventory.Binding{{Machine: "laptop", Consumer: "shop",
|
||||
Provision: "postgres-database", Provider: catalogue.Chosen{Node: "anchor", Module: "db"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
at := h0
|
||||
say := func(machine string, rs ...link.ResourceHealth) {
|
||||
t.Helper()
|
||||
at = at.Add(time.Second)
|
||||
for i := range rs {
|
||||
rs[i].Since = at
|
||||
}
|
||||
if err := stateHealth(ctx, inv, k, machine, link.Health{Contract: link.ReadinessContract, At: at, Resources: rs}, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
account := link.ResourceHealth{Module: "db", Resource: "db.operator", Kind: link.KindAccount, Target: "operator",
|
||||
Account: "operator", State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": operator is in the group db"}
|
||||
unit := link.ResourceHealth{Module: "db", Resource: "db.server", Kind: link.KindUnit, Target: "db.service",
|
||||
Account: "operator", State: link.StateUnhealthy, Reason: "failed in the account's own service manager (exit-code)"}
|
||||
shop := link.ResourceHealth{Module: "shop", Resource: "shop.web", Kind: "container", Target: "shop",
|
||||
State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"}
|
||||
for look := 0; look < 2; look++ {
|
||||
say("anchor", account, unit)
|
||||
say("laptop", shop)
|
||||
}
|
||||
list, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keys []string
|
||||
var c conditions.Condition
|
||||
for _, x := range list {
|
||||
keys = append(keys, x.Key)
|
||||
if x.Key == "module.db.anchor.relogin-needed" {
|
||||
c = x
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(keys, []string{"module.db.anchor.relogin-needed"}) {
|
||||
t.Fatalf("open %v; want the provider's wait alone, its consumer held under it", keys)
|
||||
}
|
||||
if c.Severity != conditions.Urgent || !strings.Contains(c.Evidence[0].Said, "shop on laptop") {
|
||||
t.Fatalf("the provider's wait: %s, %q; want it urgent and naming its consumer", c.Severity, c.Evidence[0].Said)
|
||||
}
|
||||
}
|
||||
|
||||
// **A broken module is put back at once** (issue 318 review): the laptop's witness put the node tools back
|
||||
// within a minute of a send that also moved `app`, and `app` reads not yet healthy because of it. The node
|
||||
// tools are marked and put back in the very judging that found them broken — their registered build is the
|
||||
// one before, and the laptop is sent it — while `app` goes on being judged, recovers, and keeps its own pass.
|
||||
func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
withConditionsInMemory(t)
|
||||
was := doctorFrom
|
||||
doctorFrom = nil
|
||||
t.Cleanup(func() { doctorFrom = was })
|
||||
old, recent := time.Now().Add(-3*time.Hour), time.Now().Add(-time.Minute)
|
||||
build := func(module, commit string, asked time.Time) {
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: module, Version: "1"})
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + module + "-" + commit, Module: module, Commit: commit,
|
||||
Repository: "novox/mesh-catalog", Path: "modules/" + module, Manifest: manifest, Asked: asked, At: asked,
|
||||
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + module + commit}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: module, Version: "1"}, inventory.Source{
|
||||
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + module, BuiltFrom: commit, Head: commit,
|
||||
Asked: asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, m := range []string{"app", broker_runtime} {
|
||||
build(m, "c1", old)
|
||||
if _, err := inv.Assign(ctx, "laptop", m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop", map[string]string{"app": "c1", broker_runtime: "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
build("app", "c2", recent)
|
||||
build(broker_runtime, "c2", recent)
|
||||
|
||||
wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts
|
||||
t.Cleanup(func() {
|
||||
machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
|
||||
})
|
||||
machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
|
||||
modules, _ := open.inventory.Assigned(ctx, node)
|
||||
sent, known, err := open.inventory.SentBuilds(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f.moves(node, modules, sent, known, all), nil
|
||||
}
|
||||
releaseHeard = func(context.Context, *stores) (map[string]bool, error) { return map[string]bool{"laptop": true}, nil }
|
||||
var sends []string
|
||||
n := 0
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, node := range names {
|
||||
n++
|
||||
carried := map[string]string{"app": current["app"].Commit, broker_runtime: current[broker_runtime].Commit}
|
||||
sends = append(sends, node+":"+carried[broker_runtime])
|
||||
digest := "d-" + node + "-" + time.Now().Format("150405.000000") + string(rune('a'+n))
|
||||
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
|
||||
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
var seen []string // the node tools' registered build at each judging
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
now := time.Now()
|
||||
f := gateFacts{now: now, reports: map[string]inventory.Reported{}, engines: map[string]string{},
|
||||
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
|
||||
reports, _ := open.inventory.LastReports(ctx)
|
||||
for _, r := range reports {
|
||||
f.reports[r.Node] = r
|
||||
}
|
||||
current, _ := open.inventory.CurrentBuilds(ctx)
|
||||
seen = append(seen, current[broker_runtime].Commit)
|
||||
app := healthyContainer("app")
|
||||
if current[broker_runtime].Commit == "c2" {
|
||||
// The witness reverted the node tools; app cannot reach them yet.
|
||||
f.rolledBack["laptop"] = []lease.Rollback{{Component: lease.ComponentNodeTools, Outcome: lease.OutcomeRolledBack,
|
||||
From: "c2", To: "c1", At: now, Why: "the node tools did not answer"}}
|
||||
app.State, app.Reason = link.StateUnhealthy, "down"
|
||||
}
|
||||
f.served["laptop"] = served{runtime: current[broker_runtime].Commit == "c1", tools: map[string]bool{}}
|
||||
f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now, Resources: []inventory.ResourceHealth{app}}}
|
||||
return f, nil
|
||||
}
|
||||
withGateBounds(t, 100*time.Millisecond, 0, 10*time.Second)
|
||||
|
||||
release := func() inventory.Plan {
|
||||
t.Helper()
|
||||
plans, _ := inv.RecentPlans(ctx, 10)
|
||||
for _, p := range plans {
|
||||
if p.Release != nil {
|
||||
return p
|
||||
}
|
||||
}
|
||||
t.Fatal("no walk")
|
||||
return inventory.Plan{}
|
||||
}
|
||||
// Judged until the first judging has found the node tools broken, and one more.
|
||||
for i := 0; i < 20 && len(seen) < 2; i++ {
|
||||
advancePlans(ctx, open)
|
||||
}
|
||||
if len(seen) < 2 || seen[0] != "c2" || seen[1] != "c1" {
|
||||
t.Fatalf("the node tools' registered build at each judging: %v; want c2, then c1 at the very next judging", seen)
|
||||
}
|
||||
if failed, _ := inv.GateFailed(ctx, "build-"+broker_runtime+"-c2"); !failed {
|
||||
t.Fatal("the node tools' build is not marked failed at once")
|
||||
}
|
||||
if p := release(); p.State != inventory.PlanRolling || p.Release.Gate == nil || p.Release.Gate.Verdict != "" {
|
||||
t.Fatalf("the walk is %s with gate %+v; want it still judging app", p.State, p.Release.Gate)
|
||||
}
|
||||
if !slices.Contains(sends, "laptop:c1") {
|
||||
t.Fatalf("sends %v; want the laptop sent the node tools' earlier build at once", sends)
|
||||
}
|
||||
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); {
|
||||
advancePlans(ctx, open)
|
||||
if release().State != inventory.PlanRolling {
|
||||
break
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
p := release()
|
||||
if p.State != inventory.PlanFailed {
|
||||
t.Fatalf("the walk is %s: %s; want failed, for the node tools", p.State, p.Note)
|
||||
}
|
||||
if v, found, _ := inv.GateOf(ctx, "build-app-c2"); !found || v.Verdict != inventory.GatePassed || !strings.Contains(v.Why, "on its own") {
|
||||
t.Fatalf("app's verdict: %+v; want its own pass", v)
|
||||
}
|
||||
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" {
|
||||
t.Fatalf("app is registered at %s; want it kept at c2", current["app"].Commit)
|
||||
}
|
||||
}
|
||||
|
||||
// **A move from a build not known excuses no wait** (issue 318 review): a plan's own module whose previous
|
||||
// build was not recorded, or whose previous manifest is not kept, cannot be shown to have added the group.
|
||||
func TestAMoveFromAnUnknownBuildExcusesNoWait(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
record := func(commit string, m catalogue.Manifest) {
|
||||
raw, _ := json.Marshal(m)
|
||||
at := time.Now().Add(-time.Hour)
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-lights-" + commit, Module: "lights", Commit: commit,
|
||||
Repository: "novox/mesh-catalog", Path: "modules/lights", Manifest: raw, Asked: at, At: at}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
record("c1", catalogue.Manifest{Module: "lights"})
|
||||
record("c2", catalogue.Manifest{Module: "lights", Resources: []map[string]any{{"id": "operator", "type": "user",
|
||||
"name": "operator", "groups": []any{"lights"}}}})
|
||||
pairs := []judged{{module: "lights", node: "laptop"}}
|
||||
shelf := map[string]catalogue.Manifest{}
|
||||
for _, c := range []struct {
|
||||
from string
|
||||
want bool
|
||||
}{{"c1", true}, {"", false}, {"c0-never-built", false}} {
|
||||
g := &inventory.PlanGate{From: c.from, To: "c2"}
|
||||
if got := movesAddingGroups(ctx, inv, g, pairs, shelf)["lights"]; got != c.want {
|
||||
t.Errorf("a move from %q adds a group: %v; want %v", c.from, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,595 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The gate on a plan's first machine and the rollback after it (novox/hq ADR 0236, to-be 45 §8).
|
||||
|
||||
// gateMesh is a mesh with `app` running on anchor and laptop at build c1, a newer build c2 registered,
|
||||
// a plan whose tier built c2, and every send recorded and answered — the machine applies what it is
|
||||
// sent and reports it — with the gate's bounds shortened to judge in three steps.
|
||||
type gateMesh struct {
|
||||
open *stores
|
||||
sent [][]string
|
||||
health map[string]health // per machine, what a judging finds; healthy when unsaid
|
||||
keeper *conditions.Keeper
|
||||
told *conditions.Told
|
||||
}
|
||||
|
||||
func aGateMesh(t *testing.T) *gateMesh {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
g := &gateMesh{open: open, health: map[string]health{}}
|
||||
g.keeper, _ = withConditionsInMemory(t)
|
||||
g.told = &conditions.Told{}
|
||||
was := doctorFrom
|
||||
doctorFrom = &doctor{open: open, keeper: g.keeper, teller: g.told}
|
||||
t.Cleanup(func() { doctorFrom = was })
|
||||
|
||||
for _, b := range []inventory.Build{
|
||||
{ID: "build-1", Module: "app", Commit: "c1", Repository: "novox/mesh-catalog", Path: "modules/app",
|
||||
Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)},
|
||||
{ID: "build-2", Module: "app", Commit: "c2", Repository: "novox/mesh-catalog", Path: "modules/app",
|
||||
Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)},
|
||||
} {
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: b.Commit})
|
||||
b.Manifest = manifest
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
register := func(commit string, asked time.Time) {
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: commit},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: commit,
|
||||
Head: commit, Asked: asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
register("c1", time.Now().Add(-2*time.Hour))
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.Assign(ctx, n, "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"app": "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
register("c2", time.Now().Add(-time.Minute))
|
||||
|
||||
// Every send: recorded with the build the module is at, applied and reported by the machine.
|
||||
n := 0
|
||||
wasSend := sendRollout
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
g.sent = append(g.sent, append([]string(nil), names...))
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, node := range names {
|
||||
n++
|
||||
digest := fmt.Sprintf("d-%s-%d", node, n)
|
||||
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, map[string]string{"app": current["app"].Commit}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
|
||||
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = wasSend })
|
||||
|
||||
// What a judging reads: the store's reports, and a health the test says per machine.
|
||||
wasGather := gatherGateFacts
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
|
||||
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
|
||||
reports, err := open.inventory.LastReports(ctx)
|
||||
if err != nil {
|
||||
return f, err
|
||||
}
|
||||
for _, r := range reports {
|
||||
if g.health[r.Node] == healthBroken {
|
||||
r.Outcome = inventory.OutcomeFailed
|
||||
}
|
||||
f.reports[r.Node] = r
|
||||
}
|
||||
for node, h := range g.health {
|
||||
if h == healthNotYet {
|
||||
// Not reported on the send: neither the send made last, nor the gate's own (issue 352).
|
||||
f.rolledBack[node] = nil
|
||||
r := f.reports[node]
|
||||
r.Current, r.Declared, r.ReportedSequence = false, "", 0
|
||||
f.reports[node] = r
|
||||
}
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
t.Cleanup(func() { gatherGateFacts = wasGather })
|
||||
|
||||
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
||||
// One judging per advance until a test says otherwise: a pass waits gateEvery for the next.
|
||||
gateSettle, gateEvery = 0, time.Hour
|
||||
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
|
||||
|
||||
built := time.Now().UTC()
|
||||
plan := inventory.Plan{ID: "plan-gate", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
|
||||
Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
|
||||
Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2",
|
||||
Build: "build-2"}}}
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
func (g *gateMesh) plan(t *testing.T) inventory.Plan {
|
||||
t.Helper()
|
||||
p, err := g.open.inventory.PlanByID(t.Context(), "plan-gate")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// A module's build that fails its gate on the first machine is put back there — the previous build
|
||||
// registered and sent to it again — and never reaches the second machine; it is marked, said as a
|
||||
// condition and an event, never registered or rolled back again.
|
||||
func TestABuildThatFailsItsGateIsRolledBackOnItsFirstMachineAndGoesNoFurther(t *testing.T) {
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := g.open.inventory
|
||||
|
||||
advancePlans(ctx, g.open) // the first machine is sent the new build
|
||||
if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}}) {
|
||||
t.Fatalf("sent %v, not the first machine alone", g.sent)
|
||||
}
|
||||
if p := g.plan(t); p.Modules["app"].Previous != "c1" {
|
||||
t.Fatalf("the build the first machine ran before was not kept: %+v", p.Modules["app"])
|
||||
}
|
||||
|
||||
g.health["anchor"] = healthBroken // the new build breaks its first machine, after one good judging
|
||||
gateEvery = 0
|
||||
advancePlans(ctx, g.open)
|
||||
|
||||
p := g.plan(t)
|
||||
gate := p.Modules["app"].Gate
|
||||
if p.State != inventory.PlanFailed || gate == nil || gate.Verdict != inventory.GateFailed ||
|
||||
gate.Rollback != inventory.RolledBack {
|
||||
t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate)
|
||||
}
|
||||
if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"anchor"}}) {
|
||||
t.Fatalf("sent %v: the rollback goes to the first machine, and nothing reaches laptop", g.sent)
|
||||
}
|
||||
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" {
|
||||
t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit)
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c1" {
|
||||
t.Fatalf("anchor was last sent %v, not the previous build", sent)
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["app"] != "c1" {
|
||||
t.Fatalf("laptop was sent the failed build: %v", sent)
|
||||
}
|
||||
if failed, err := inv.GateFailed(ctx, "build-2"); err != nil || !failed {
|
||||
t.Fatalf("the build is not marked failed at its gate: %v %v", failed, err)
|
||||
}
|
||||
|
||||
// Said: a condition for the operator, and the event.
|
||||
open, err := g.keeper.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var key string
|
||||
for _, c := range open {
|
||||
if c.Kind == kindRolledBack {
|
||||
key = c.Key
|
||||
}
|
||||
}
|
||||
if key != "build.app.anchor.rolled-back" {
|
||||
t.Fatalf("no rolled-back condition for app on anchor: %+v", open)
|
||||
}
|
||||
saidIt := false
|
||||
for _, e := range g.told.Said() {
|
||||
saidIt = saidIt || e.Event == link.KeyRolledBack
|
||||
}
|
||||
if !saidIt {
|
||||
t.Fatalf("the rollback was not said as an event: %+v", g.told.Said())
|
||||
}
|
||||
|
||||
// Never again: more passes send nothing, a second judging rolls nothing back, and the failed build
|
||||
// heard again is not registered.
|
||||
advancePlans(ctx, g.open)
|
||||
state := p.Modules["app"]
|
||||
gateFailed(ctx, g.open, &p, "app", state, []string{"anchor"}, "again")
|
||||
if len(g.sent) != 2 {
|
||||
t.Fatalf("sent again after the rollback: %v", g.sent)
|
||||
}
|
||||
_, _, err = takeIn(ctx, inv, link.BuildResult{ID: "build-2", Repository: "novox/mesh-catalog", Path: "modules/app",
|
||||
Commit: "c2", Manifest: mustJSON(t, catalogue.Manifest{Module: "app", Version: "c2"}),
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}})
|
||||
if err == nil || !strings.Contains(err.Error(), "failed its gate") {
|
||||
t.Fatalf("the failed build was registered again: %v", err)
|
||||
}
|
||||
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" {
|
||||
t.Fatalf("the module moved to %s", current["app"].Commit)
|
||||
}
|
||||
if _, err := retryPlan(ctx, g.open, "plan-gate"); err == nil || !strings.Contains(err.Error(), "failed its gate") {
|
||||
t.Fatalf("a plan stopped at a failed gate was retried: %v", err)
|
||||
}
|
||||
|
||||
// The probe keeps the condition while the newest verdict is the failure.
|
||||
obs, err := probeGates(ctx, doctorFrom)
|
||||
if err != nil || len(obs) != 1 || obs[0].Key() != key {
|
||||
t.Fatalf("the gate's probe found %+v %v", obs, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A passing build rolls everywhere with no hand: judged healthy on its first machine three times, then
|
||||
// the rest are sent, the verdict kept, and the plan done.
|
||||
func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) {
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
gateEvery = 0
|
||||
for i := 0; i < 6; i++ {
|
||||
advancePlans(ctx, g.open)
|
||||
}
|
||||
p := g.plan(t)
|
||||
if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"laptop"}}) {
|
||||
t.Fatalf("sent %v", g.sent)
|
||||
}
|
||||
gate := p.Modules["app"].Gate
|
||||
if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gatePasses {
|
||||
t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate)
|
||||
}
|
||||
if v, found, err := g.open.inventory.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("the verdict was not kept: %+v %v %v", v, found, err)
|
||||
}
|
||||
if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 {
|
||||
t.Fatalf("a passing build left a condition: %+v %v", obs, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A first machine that never becomes healthy fails its gate at the bound, and is put back.
|
||||
func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) {
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
advancePlans(ctx, g.open)
|
||||
g.health["anchor"] = healthNotYet
|
||||
gateEvery = 0
|
||||
advancePlans(ctx, g.open)
|
||||
if p := g.plan(t); !p.Open() || len(g.sent) != 1 {
|
||||
t.Fatalf("judged before the bound: %s %v", p.State, g.sent)
|
||||
}
|
||||
gateBound = -time.Second
|
||||
advancePlans(ctx, g.open)
|
||||
p := g.plan(t)
|
||||
if gate := p.Modules["app"].Gate; p.State != inventory.PlanFailed || gate.Verdict != inventory.GateFailed ||
|
||||
!strings.Contains(gate.Why, "not healthy within") || gate.Rollback != inventory.RolledBack {
|
||||
t.Fatalf("the plan is %s, its gate %+v", p.State, gate)
|
||||
}
|
||||
}
|
||||
|
||||
// The health a judging finds, per core component and for a module.
|
||||
func TestTheHealthDefinitions(t *testing.T) {
|
||||
now := time.Now()
|
||||
since := now.Add(-time.Minute)
|
||||
applied := func(node string) gateFacts {
|
||||
at := now
|
||||
return gateFacts{now: now, reports: map[string]inventory.Reported{node: {Node: node,
|
||||
Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, engines: map[string]string{},
|
||||
served: map[string]served{}, rolledBack: map[string][]lease.Rollback{}}
|
||||
}
|
||||
m := catalogue.Manifest{Module: "app", Tools: []string{"app_list"}}
|
||||
|
||||
f := applied("anchor")
|
||||
f.served["anchor"] = served{runtime: true, tools: map[string]bool{}}
|
||||
if h, why := judgeHealth("app", "", m, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "tools") {
|
||||
t.Errorf("a module whose tools are not served is %v (%s)", h, why)
|
||||
}
|
||||
f.served["anchor"].tools["app"] = true
|
||||
if h, why := judgeHealth("app", "", m, "anchor", since, f); h != healthGood {
|
||||
t.Errorf("a module applied with its tools served is %v (%s)", h, why)
|
||||
}
|
||||
// A condition raised about it on that machine since it was sent: not yet healthy.
|
||||
f.judged = true
|
||||
f.open = []conditions.Condition{{Key: "provider.app.anchor.x.failing", Subject: conditions.Subject{
|
||||
Scope: conditions.ScopeProvider, ID: "app.anchor.x", Machine: "anchor"}, Raised: now, Summary: "failing"}}
|
||||
if h, _ := judgeHealth("app", "", m, "anchor", since, f); h != healthNotYet {
|
||||
t.Errorf("a module with a new condition about it is %v", h)
|
||||
}
|
||||
f.open[0].Raised = since.Add(-time.Hour)
|
||||
if h, _ := judgeHealth("app", "", m, "anchor", since, f); h != healthGood {
|
||||
t.Errorf("a condition older than the send counted against it: %v", h)
|
||||
}
|
||||
// A witness that put it back: broken.
|
||||
f.rolledBack["anchor"] = []lease.Rollback{{Component: lease.ComponentNodeTools, From: "sha256:aa", To: "sha256:bb",
|
||||
Outcome: lease.OutcomeRolledBack, Why: "x", At: now}}
|
||||
if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthBroken {
|
||||
t.Errorf("a component a witness put back is %v", h)
|
||||
}
|
||||
// The node tools answer, or not.
|
||||
f = applied("anchor")
|
||||
if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthNotYet {
|
||||
t.Errorf("node tools not answering are %v", h)
|
||||
}
|
||||
f.served["anchor"] = served{runtime: true}
|
||||
if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthGood {
|
||||
t.Errorf("node tools answering are %v", h)
|
||||
}
|
||||
// The controller: the lease held since the send, by a controller that says it is ready.
|
||||
f = applied("control")
|
||||
f.holder = &lease.Holder{Taken: since.Add(-time.Hour), Health: &lease.Health{Ready: true}}
|
||||
if h, _ := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthNotYet {
|
||||
t.Errorf("a lease held by a controller older than the build is %v", h)
|
||||
}
|
||||
f.holder.Taken = now
|
||||
if h, _ := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthGood {
|
||||
t.Errorf("a new controller holding the lease and ready is %v", h)
|
||||
}
|
||||
f.holder.Health = &lease.Health{Why: "the self-check has not finished its first run"}
|
||||
if h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthNotYet ||
|
||||
!strings.Contains(why, "first run") {
|
||||
t.Errorf("a controller not ready is %v (%s)", h, why)
|
||||
}
|
||||
// What the module runs (novox/hq ADR 0240 §4): a judging passes only when every long-running
|
||||
// resource of it on that machine is stated healthy since the send; starting and unhealthy wait.
|
||||
f = applied("anchor")
|
||||
stated := func(state, reason string, heard time.Time) {
|
||||
f.health = map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Contract: 1, SaidAt: heard, HeardAt: heard,
|
||||
Resources: []inventory.ResourceHealth{
|
||||
{Module: "app", Resource: "app.server", Kind: "container", Target: "app-server", State: state, Reason: reason},
|
||||
{Module: "other", Resource: "other.server", Kind: "container", State: link.StateUnhealthy, Reason: "down"}}}}
|
||||
}
|
||||
plain := catalogue.Manifest{Module: "app"}
|
||||
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthGood {
|
||||
t.Errorf("a machine whose engine states no health is judged as before, not %v (%s)", h, why)
|
||||
}
|
||||
stated(link.StateStarting, "", now)
|
||||
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "starting") {
|
||||
t.Errorf("a resource still starting is not yet a pass: %v (%s)", h, why)
|
||||
}
|
||||
stated(link.StateUnhealthy, "restarting", now)
|
||||
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "restarting") {
|
||||
t.Errorf("a resource unhealthy fails the judging: %v (%s)", h, why)
|
||||
}
|
||||
stated(link.StateHealthy, "", since.Add(-time.Minute))
|
||||
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet {
|
||||
t.Errorf("a statement from before the send says nothing of the new build: %v (%s)", h, why)
|
||||
}
|
||||
stated(link.StateHealthy, "", now)
|
||||
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthGood {
|
||||
t.Errorf("every resource of it healthy since the send — another module's state is not its — is %v (%s)", h, why)
|
||||
}
|
||||
f.healthErr = errors.New("the store is away")
|
||||
if h, _ := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet {
|
||||
t.Errorf("health that cannot be read is never read as healthy: %v", h)
|
||||
}
|
||||
// And the condition it raises after the send holds it, as every condition about it does.
|
||||
f.healthErr = nil
|
||||
f.judged = true
|
||||
f.open = []conditions.Condition{{Key: "module.app.anchor.unhealthy", Kind: kindModuleUnhealthy, Subject: conditions.Subject{
|
||||
Scope: conditions.ScopeModule, ID: "app.anchor", Machine: "anchor"}, Raised: now, Summary: "app on anchor is not healthy"}}
|
||||
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "module.app.anchor.unhealthy") {
|
||||
t.Errorf("a module held on its own unhealthy condition is %v (%s)", h, why)
|
||||
}
|
||||
|
||||
// A machine that refused what it was sent: broken.
|
||||
f = applied("anchor")
|
||||
r := f.reports["anchor"]
|
||||
r.Outcome = inventory.OutcomeRefused
|
||||
f.reports["anchor"] = r
|
||||
if h, _ := judgeHealth("app", "", catalogue.Manifest{}, "anchor", since, f); h != healthBroken {
|
||||
t.Errorf("a refusal is %v", h)
|
||||
}
|
||||
}
|
||||
|
||||
// The bus is never rolled out: its policy records whatever is said, a person's roll-out is refused,
|
||||
// a plan builds it and sends nothing, and a push naming its machine is refused while a new build waits.
|
||||
func TestTheBusIsNeverRolledOutAutomatically(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
bus := catalogue.Manifest{Module: "nats", Version: "2", Provides: []catalogue.Offer{{Name: "mesh-bus"}},
|
||||
Upgrade: &catalogue.UpgradePolicy{Policy: catalogue.PolicyRoll}}
|
||||
if err := inv.RegisterModule(ctx, bus, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
|
||||
Path: "modules/nats", BuiltFrom: "n2", Head: "n2"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "anchor", "nats"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"nats": "n1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
u, err := inv.UpgradeOf(ctx, "nats")
|
||||
if err != nil || u.RollOut || u.From != catalogue.FromBus {
|
||||
t.Fatalf("the bus's policy is %+v %v", u, err)
|
||||
}
|
||||
if err := inv.SetUpgradeOf(ctx, "nats", inventory.Upgrade{RollOut: true}); !errors.Is(err, inventory.ErrBusIsPlanned) {
|
||||
t.Fatalf("a person rolled the bus out: %v", err)
|
||||
}
|
||||
var sent [][]string
|
||||
was := sendRollout
|
||||
sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) {
|
||||
sent = append(sent, names)
|
||||
return names, nil
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = was })
|
||||
now := time.Now().UTC()
|
||||
plan := inventory.Plan{ID: "plan-bus", Repository: "novox/mesh-catalog", Commit: "n2", Created: now,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"nats"}},
|
||||
Modules: map[string]*inventory.PlanModule{"nats": {State: "built", BuiltAt: &now, Commit: "n2", Build: "b"}}}
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, open)
|
||||
if p, _ := inv.PlanByID(ctx, "plan-bus"); p.State != inventory.PlanDone || len(sent) != 0 {
|
||||
t.Fatalf("a plan sent the bus: %s %v", p.State, sent)
|
||||
}
|
||||
held, err := busHeld(ctx, inv, []string{"anchor", "laptop"})
|
||||
if err != nil || !strings.Contains(held["anchor"], "planned step") || held["laptop"] != "" {
|
||||
t.Fatalf("a push may send the bus's machine: %v %v", held, err)
|
||||
}
|
||||
// Nor may any other send — a plan's for another module on that machine carried the bus with it.
|
||||
if _, err := sendToEach(ctx, open, []string{"laptop", "anchor"}); !errors.Is(err, errBusWaits) {
|
||||
t.Fatalf("a send to the bus's machine was not refused: %v", err)
|
||||
}
|
||||
// A rebuild that made the same artifacts is no move.
|
||||
for _, b := range []inventory.Build{{ID: "nb1", Module: "nats", Commit: "n1"}, {ID: "nb2", Module: "nats", Commit: "n2"}} {
|
||||
b.Made = []inventory.Artifact{{Name: "server", Kind: "image", Reference: "registry/nats@sha256:same"}}
|
||||
b.Asked, b.At = time.Now(), time.Now()
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if held, err := busHeld(ctx, inv, []string{"anchor"}); err != nil || len(held) != 0 {
|
||||
t.Fatalf("a rebuild that changes nothing held the bus's machine: %v %v", held, err)
|
||||
}
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "nb3", Module: "nats", Commit: "n2", Asked: time.Now().Add(time.Second),
|
||||
At: time.Now().Add(time.Second), Made: []inventory.Artifact{{Name: "server", Kind: "image",
|
||||
Reference: "registry/nats@sha256:new"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The planned step refuses to start without its word on reversibility, and without a snapshot taken
|
||||
// first by the bus machine's backup holder.
|
||||
if err := busCommand(ctx, []string{"upgrade", "--why", "2.11"}); err == nil || !strings.Contains(err.Error(), "reversible") {
|
||||
t.Fatalf("a bus upgrade started without saying whether it can be reverted: %v", err)
|
||||
}
|
||||
wasSnapshot := takeBusSnapshot
|
||||
t.Cleanup(func() { takeBusSnapshot = wasSnapshot })
|
||||
takeBusSnapshot = func(context.Context, string, string) (string, error) { return "", errors.New("no holder answers") }
|
||||
if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "snapshotted") || len(sent) != 0 {
|
||||
t.Fatalf("a bus upgrade started without its snapshot: %v, sent %v", err, sent)
|
||||
}
|
||||
takeBusSnapshot = func(_ context.Context, module, node string) (string, error) {
|
||||
return node + "'s restore point of " + module, nil
|
||||
}
|
||||
if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(sent, [][]string{{"anchor"}}) {
|
||||
t.Fatalf("the step sent %v, not the bus's machine", sent)
|
||||
}
|
||||
s, found, err := inv.LatestBusStep(ctx)
|
||||
if err != nil || !found || s.Snapshot != "anchor's restore point of nats" || s.Ended != nil ||
|
||||
!reflect.DeepEqual(s.Machines, []string{"anchor"}) {
|
||||
t.Fatalf("the step is %+v %v %v", s, found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge that deletes a module's directory builds nothing for it: the module is forgotten where
|
||||
// nothing holds it, and a plan whose build finds no manifest goes on instead of failing.
|
||||
func TestAMergeThatDeletesAModulePlansNothingToBuildForIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asked := asksRecorded(t)
|
||||
for _, name := range []string{"gone", "kept"} {
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: name, Version: "1"}, inventory.Source{
|
||||
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + name, BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1deadbeef",
|
||||
Paths: []string{"modules/gone/module.json", "modules/gone/index.ts"},
|
||||
Removed: []string{"modules/gone/module.json", "modules/gone/index.ts"}}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(*asked) != 0 {
|
||||
t.Fatalf("a deleted module was asked to build: %v", *asked)
|
||||
}
|
||||
if plans, _ := inv.OpenPlans(ctx); len(plans) != 0 {
|
||||
t.Fatalf("a merge that only deleted a module made a plan: %+v", plans)
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, still := shelf["gone"]; still {
|
||||
t.Fatal("a deleted module nothing holds was not forgotten")
|
||||
}
|
||||
|
||||
// Without the announcer saying what went: the build finds no manifest, and the plan goes on.
|
||||
asked0 := time.Now().UTC().Add(-time.Minute)
|
||||
plan := inventory.Plan{ID: "plan-deleted", Repository: "novox/mesh-catalog", Commit: "c2", Created: asked0,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"kept"}},
|
||||
Modules: map[string]*inventory.PlanModule{"kept": {State: "asked", AskedAt: &asked0, Build: "build-k"}}}
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
planBuilt(ctx, open, "kept", "", "http://forge/novox/mesh-catalog.git has no module.json at modules/kept, so "+
|
||||
"there is nothing saying what it is: open …/module.json: no such file or directory", asked0, "build-k")
|
||||
p, _ := inv.PlanByID(ctx, "plan-deleted")
|
||||
if p.State == inventory.PlanFailed || p.Modules["kept"].State != planDeleted {
|
||||
t.Fatalf("a module deleted at its source failed the plan: %s %+v", p.State, p.Modules["kept"])
|
||||
}
|
||||
}
|
||||
|
||||
func mustJSON(t *testing.T, v any) []byte {
|
||||
t.Helper()
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func nodeID(t *testing.T, open *stores, name string) string {
|
||||
t.Helper()
|
||||
n, err := open.inventory.NodeByName(context.Background(), name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return n.ID
|
||||
}
|
||||
|
||||
// What a machine's witness says in its reports is a condition while it says it, by the host's words:
|
||||
// `core.<component>.<node>.<outcome>`, urgent for a rollback, a warning when it could not judge — and
|
||||
// gone with the first report that no longer carries it.
|
||||
func TestAWitnessesVerdictIsAConditionWhileItsReportsSayIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
d := &doctor{open: open}
|
||||
at := time.Now().UTC()
|
||||
witnessed.heard("control", []lease.Rollback{
|
||||
{Component: lease.ComponentController, From: "sha256:new", To: "sha256:old", Outcome: lease.OutcomeRolledBack,
|
||||
Why: "the new controller did not take the lease within 60s", At: at},
|
||||
{Component: lease.ComponentNodeTools, From: "sha256:t2", Outcome: lease.OutcomeUnwitnessed, Why: "no grant", At: at},
|
||||
}, at)
|
||||
t.Cleanup(func() { witnessed.heard("control", nil, time.Now()) })
|
||||
obs, err := probeGates(t.Context(), d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := map[string]conditions.Severity{}
|
||||
for _, o := range obs {
|
||||
got[o.Key()] = o.Severity
|
||||
}
|
||||
want := map[string]conditions.Severity{"core.controller.control.rolled-back": conditions.Urgent,
|
||||
"core.node-tools.control.unwitnessed": conditions.Warning}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("the witness's verdicts are %v", got)
|
||||
}
|
||||
witnessed.heard("control", nil, time.Now())
|
||||
if obs, err := probeGates(t.Context(), d); err != nil || len(obs) != 0 {
|
||||
t.Fatalf("a verdict the reports no longer carry is still said: %+v %v", obs, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq issue 274: a provider is granted exactly the consumers whose own resolution binds them to
|
||||
// it — not every consumer a pair credential from it was ever made for.
|
||||
|
||||
// grantOf is the grant of one provision to one consuming module, and whether there is one at all.
|
||||
func grantOf(grants []catalogue.Grant, provision, consumer, module string) (catalogue.Grant, bool) {
|
||||
for _, g := range grants {
|
||||
if g.Provision == provision && g.Consumer == consumer && (g.From == module || g.From == "") {
|
||||
return g, true
|
||||
}
|
||||
}
|
||||
return catalogue.Grant{}, false
|
||||
}
|
||||
|
||||
// The morning after issue 273, through the stores: a consumer was bound to the store on another
|
||||
// machine, a credential from there was made, and a person pinned it back to the store beside it. Both
|
||||
// credentials are on record. The store it left is no longer granted it — so it retires it and keeps
|
||||
// its data (ADR 0230) — and says so; the store it is bound to keeps its grant; and the credential from
|
||||
// the store it left stays on record.
|
||||
func TestAConsumerPinnedBackIsNoLongerGrantedByTheProviderItLeft(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
for _, m := range storeManifests() {
|
||||
register(t, open, m)
|
||||
}
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.HoldSeat(ctx, "mesh-store", catalogue.ScopeMesh, "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, a := range [][2]string{{"laptop", "store"}, {"laptop", "board"}} {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
|
||||
// Bound to the anchor's store, and sent so: the credential from the anchor is made and recorded.
|
||||
if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, _, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := need(t, plan, "board", "postgres-database"); n.From != "anchor" {
|
||||
t.Fatalf("pinned to the anchor and bound to %s", n.From)
|
||||
}
|
||||
if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
grants, _, unbound, err := grantsFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); !ok || g.From != "board" || len(unbound) != 0 {
|
||||
t.Fatalf("a consumer bound to the anchor is not granted there: %+v, unbound %+v", grants, unbound)
|
||||
}
|
||||
|
||||
// Pinned back beside its data, as the operator did.
|
||||
if err := inv.PinProvision(ctx, "laptop", "postgres-database", "laptop", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, _, err = planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := need(t, plan, "board", "postgres-database"); n.From != "laptop" {
|
||||
t.Fatalf("pinned back to the laptop and bound to %s", n.From)
|
||||
}
|
||||
if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
holders, err := inv.HoldersOf(ctx, "postgres-database", "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(holders) != 2 {
|
||||
t.Fatalf("today's state is two credentials on record, one from each store: %+v", holders)
|
||||
}
|
||||
|
||||
// The store it left: no longer granted, and said.
|
||||
grants, _, unbound, err = grantsFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); ok && g.From != "" {
|
||||
t.Fatalf("the anchor's store is still granted a consumer bound to the laptop's: %+v", g)
|
||||
}
|
||||
if len(unbound) != 1 || unbound[0].Module != "board" || unbound[0].Provider != "anchor" ||
|
||||
strings.Join(unbound[0].BoundTo, ",") != "laptop" {
|
||||
t.Fatalf("the consumer that moved is not the one said: %+v", unbound)
|
||||
}
|
||||
planned, settings, err := planFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationFor(ctx, open, "anchor", planned, settings)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := declared.Received["store"]["postgres-database"]; len(got) != 0 {
|
||||
t.Fatalf("the anchor's store is still told about %+v", got)
|
||||
}
|
||||
said := printed(t, func() error { reportLeftOut("anchor", declared); return nil })
|
||||
if !strings.Contains(said, "board on laptop is bound to laptop for postgres-database, not to anchor") ||
|
||||
!strings.Contains(said, "cleanup delete") {
|
||||
t.Fatalf("the push does not say whom the anchor no longer grants:\n%s", said)
|
||||
}
|
||||
|
||||
// The store it is bound to: granted.
|
||||
grants, _, unbound, err = grantsFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); !ok || g.From != "board" || len(unbound) != 0 {
|
||||
t.Fatalf("the consumer is not granted by the store it is bound to: %+v, unbound %+v", grants, unbound)
|
||||
}
|
||||
|
||||
// And the credential from the store it left is kept: the key to the login and data held there.
|
||||
if holders, err = inv.HoldersOf(ctx, "postgres-database", "laptop"); err != nil || len(holders) != 2 {
|
||||
t.Fatalf("a credential was forgotten while its provider still holds the login: %+v, %v", holders, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A consumer whose resolution cannot be read is an error, never a consumer bound nowhere — withdrawing
|
||||
// a grant on that reading would take its access away (issue 152).
|
||||
func TestAConsumerWhoseResolutionCannotBeReadIsNotWithdrawn(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
for _, m := range storeManifests() {
|
||||
register(t, open, m)
|
||||
}
|
||||
for _, a := range [][2]string{{"anchor", "store"}, {"laptop", "board"}} {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
if _, _, err := planFor(ctx, open, "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The laptop's key changes to one nothing can seal to: its resolution cannot be completed, and
|
||||
// that is not its set failing to compose.
|
||||
laptop, err := inv.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSealingKey(ctx, laptop.ID, "not a key"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := planFor(ctx, open, "laptop"); err == nil || unresolvable(err) {
|
||||
t.Fatalf("the seam this test relies on moved: %v", err)
|
||||
}
|
||||
grants, _, unbound, err := grantsFor(ctx, open, "anchor")
|
||||
if err == nil {
|
||||
t.Fatalf("an unreadable consumer was answered: grants %+v, unbound %+v", grants, unbound)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "what laptop asked of postgres-database cannot be read") {
|
||||
t.Fatalf("the error does not say whose resolution could not be read: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The rule itself, on a resolution: bound is the provider a need for exactly that credential is
|
||||
// answered by, never one answered by a record, and a different local name is a different credential.
|
||||
func TestBindsFromIsTheProviderOfThatCredential(t *testing.T) {
|
||||
r := catalogue.Resolution{Needs: []catalogue.Needed{
|
||||
{Name: "postgres-database", For: "board", From: "laptop"},
|
||||
{Name: "postgres-database", For: "board", From: "laptop", Local: "reports"},
|
||||
{Name: "postgres-database", For: "wiki", From: "anchor"},
|
||||
{Name: "a-licence", For: "board", From: "the-licence", ByRecord: true},
|
||||
}}
|
||||
s := inventory.Secret{Name: "postgres-database", ConsumerModule: "board"}
|
||||
if got := r.BindsFrom(s.Name, s.ConsumerModule, s.Local); strings.Join(got, ",") != "laptop" {
|
||||
t.Fatalf("board's credential is bound to %v", got)
|
||||
}
|
||||
if got := r.BindsFrom("postgres-database", "board", "archive"); len(got) != 0 {
|
||||
t.Fatalf("a local name nothing asks for is bound to %v", got)
|
||||
}
|
||||
if got := r.BindsFrom("a-licence", "board", ""); len(got) != 0 {
|
||||
t.Fatalf("a need answered by a record is bound to %v", got)
|
||||
}
|
||||
}
|
||||
+199
-14
@@ -6,14 +6,16 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
@@ -27,6 +29,107 @@ import (
|
||||
// step of a procedure is not a repair. `conditions silence` joins them when the condition store does
|
||||
// (Phase 1).
|
||||
|
||||
// handActVerb is one verb that writes the hand-act log, and whether what it records is a repair.
|
||||
type handActVerb struct {
|
||||
Verb string
|
||||
// Decision says why an act of this verb is a person's decision by design rather than a repair a
|
||||
// healer could take over; empty for a repair.
|
||||
Decision string
|
||||
// DecidedFor limits Decision to these causes; empty, it holds for every act of the verb.
|
||||
DecidedFor []string
|
||||
// DecidedWhen limits Decision to the acts it answers true for: what the controller read the act to
|
||||
// be from what it did (a push's kind), never a word the person gave.
|
||||
DecidedWhen func(link.HandAct) bool
|
||||
}
|
||||
|
||||
// causeLeakedInLogs is the cause a rotation after a value was printed into a log gives.
|
||||
const causeLeakedInLogs = "leaked-in-logs"
|
||||
|
||||
// causeDrill is the cause of every act `hand-act drill` records, and the one cause `hand-act record`
|
||||
// refuses: a drill has its own verb, so whether an act was a drill is said by the verb a person chose,
|
||||
// never by a word typed into a repair's cause (novox/hq issue 292).
|
||||
const causeDrill = "drill"
|
||||
|
||||
// handActVerbs is every verb that writes the hand-act log (novox/hq to-be 45 §7). **S15 reads it**:
|
||||
// an act recorded by a verb whose entry names a decision is the mesh working as decided, never a
|
||||
// repair, and does not count toward `healer-wanted` — whatever cause it gives. A verb not listed, or
|
||||
// listed without a decision, counts, so a new verb is a repair until its entry says otherwise.
|
||||
var handActVerbs = []handActVerb{
|
||||
// Repairs: each repeated is a healer the mesh lacks. A push by hand is exactly what roll-out by
|
||||
// default (ADR 0236) exists to end — except a push that only moved builds a `record` policy held for
|
||||
// a person's word (ADR 0242), which the push itself reads from what it carried (recorded_push.go).
|
||||
{Verb: "push", Decision: "a recorded build moves only by a person's push: that push is the word its " +
|
||||
"upgrade policy asks for (ADR 0242)", DecidedWhen: pushedRecorded},
|
||||
// Stopping or starting a walk the operator chose on a warrant (novox/hq ADR 0259) is their decision.
|
||||
{Verb: "plans stop", Decision: "the operator's answer to an ask is their decision, not a repair (ADR 0259)",
|
||||
DecidedFor: []string{conditions.CauseOperatorAnswer}},
|
||||
{Verb: "plans close"},
|
||||
// A walk started by a person instead of its delivery's owner (novox/hq ADR 0239): the owner down, or
|
||||
// not trusted with it — either is a repair the owner should have made. Unless the operator chose it on
|
||||
// a warrant (ADR 0259).
|
||||
{Verb: "plans go", Decision: "the operator's answer to an ask is their decision, not a repair (ADR 0259)",
|
||||
DecidedFor: []string{conditions.CauseOperatorAnswer}},
|
||||
// An act the operator chose on a warrant (novox/hq ADR 0259): asked by the controller, answered on a
|
||||
// channel that proved who answered, performed by the controller as itself.
|
||||
{Verb: handActWarrant, Decision: "the operator chose it, answering what the controller asked (ADR 0259)"},
|
||||
{Verb: "broker consumer-reset"},
|
||||
// Silencing the same condition twice says the condition, or what it watches, wants mending — unless
|
||||
// it is the operator's answer on a notification: a decision to live with it (novox/hq ADR 0258).
|
||||
{Verb: "conditions silence", Decision: "the operator's answer on a notification is their decision, " +
|
||||
"not a repair (ADR 0258)", DecidedFor: []string{conditions.CauseOperatorAnswer}},
|
||||
// An act done outside the mesh: the mesh cannot tell a repair from a decision there, so it counts —
|
||||
// except a drill recorded through it before `hand-act drill` existed (2026-10-07). It refuses the
|
||||
// cause since, so no act recorded through it now carries it.
|
||||
{Verb: "hand-act record", Decision: "a drill recorded before `hand-act drill` existed: a person's " +
|
||||
"deliberate test, never a repair", DecidedFor: []string{causeDrill}},
|
||||
// A drill: something broken on purpose to see the mesh raise and clear it. A person's test, never a
|
||||
// repair, however often it is run.
|
||||
{Verb: "hand-act drill", Decision: "a drill is a person's deliberate test of the mesh, never a repair"},
|
||||
// A person's decisions by design.
|
||||
{Verb: "retire approve", Decision: "nothing is retired past its bound without a person (ADR 0230)"},
|
||||
{Verb: "retire reject", Decision: "keeping a consumer active is a person's word (ADR 0230)"},
|
||||
{Verb: "cleanup delete", Decision: "nothing retired is deleted without a person (ADR 0230)"},
|
||||
// What becomes of a message a consumer gave up on (novox/hq issue 330): kept until a person says.
|
||||
{Verb: "dead-letters deliver", Decision: "a message a consumer gave up on is delivered again only on a " +
|
||||
"person's word (issue 330)"},
|
||||
{Verb: "dead-letters drop", Decision: "a message a consumer gave up on is let go only on a person's word " +
|
||||
"(issue 330)"},
|
||||
// The sweep run on a person's word rather than after a build: the same decision the records make, at
|
||||
// a moment the person chose (ADR 0251) — never a repair.
|
||||
{Verb: "collect", Decision: "letting the store go of what the records keep for no reason, now rather " +
|
||||
"than at the next build, is a person's word (ADR 0251)"},
|
||||
// Recording a copy no record names as the mesh's: a person's reading of the store, never a repair
|
||||
// (ADR 0257).
|
||||
{Verb: "mirrors", Decision: "which copies in the store no record names are the mesh's is a person's " +
|
||||
"word (ADR 0257)"},
|
||||
{Verb: "bus upgrade", Decision: "the bus is never rolled by the mesh: replacing it is a planned step a " +
|
||||
"person starts (ADR 0236)"},
|
||||
{Verb: "upgrade release-backlog", Decision: "after a release plan failed, the next opens only when a " +
|
||||
"person releases it (ADR 0236)"},
|
||||
// A leak is judged by a person — which value was exposed, to whom — and its rotation is the answer
|
||||
// to that judgement. Several values rotate for one leak, and a leak that recurs is a defect of the
|
||||
// module that prints them, an issue against it, not a healer that rotates. A rotation for any other
|
||||
// cause — a credential that stopped working — counts: a schedule or a healer could take it over.
|
||||
// A value given at the desk (novox/hq ADR 0259 §10): an outside party's key, such as a bot token, which
|
||||
// only a person can give. Their word, never a repair.
|
||||
{Verb: "secret accept", Decision: "a value an outside party issued is given by a person, at their desk"},
|
||||
{Verb: "secret rotate", Decision: "a value a person judged disclosed is replaced on their word",
|
||||
DecidedFor: []string{causeLeakedInLogs}},
|
||||
}
|
||||
|
||||
// personsDecision is whether an act in the log is a person's decision by design, by the verb that
|
||||
// recorded it (handActVerbs).
|
||||
func personsDecision(a link.HandAct) bool {
|
||||
for _, v := range handActVerbs {
|
||||
if v.Verb != a.Verb {
|
||||
continue
|
||||
}
|
||||
return v.Decision != "" && (len(v.DecidedFor) == 0 || slices.Contains(v.DecidedFor, a.Cause)) &&
|
||||
(v.DecidedWhen == nil || v.DecidedWhen(a))
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// handActFlags are the flags every repairing verb takes.
|
||||
type handActFlags struct {
|
||||
why, cause, condition *string
|
||||
@@ -61,14 +164,10 @@ func onTheBus(f func(*nats.Conn) error) error {
|
||||
if handActConn != nil {
|
||||
return f(handActConn)
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot reach the bus: %w", err)
|
||||
}
|
||||
defer js.Close()
|
||||
return f(js.Conn())
|
||||
}
|
||||
@@ -82,6 +181,19 @@ func (f handActFlags) record(ctx context.Context, verb string, args []string) {
|
||||
}
|
||||
act := link.HandAct{Verb: verb, Args: args, Why: strings.TrimSpace(*f.why),
|
||||
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
|
||||
// A push naming one machine says whether it only moves recorded builds (novox/hq issue 301):
|
||||
// read from what it carries, before it is sent.
|
||||
if verb == "push" && len(args) == 1 && !strings.HasPrefix(args[0], "-") {
|
||||
switch carried, why, err := recordedPushOf(ctx, args[0]); {
|
||||
case err != nil:
|
||||
fmt.Fprintf(os.Stderr, "whether this push only moves recorded builds could not be read, so it is "+
|
||||
"recorded as a push by hand: %v\n", err)
|
||||
case len(carried) > 0:
|
||||
act.Kind, act.Carried = link.KindRecordedBuilds, carried
|
||||
default:
|
||||
fmt.Printf("a push by hand, not of recorded builds only: %s\n", why)
|
||||
}
|
||||
}
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
written, err := link.RecordHandAct(ctx, conn, act)
|
||||
act = written
|
||||
@@ -91,6 +203,12 @@ func (f handActFlags) record(ctx context.Context, verb string, args []string) {
|
||||
fmt.Fprintf(os.Stderr, "this act by hand could NOT be recorded in the hand-act log, and is done anyway: %v\n", err)
|
||||
return
|
||||
}
|
||||
if act.Kind == link.KindRecordedBuilds {
|
||||
fmt.Printf("recorded as %s in the hand-act log: a push of recorded builds (%s) by %s, because %q "+
|
||||
"— the person's word their upgrade policy asks for, which no healer is wanted for\n", act.ID,
|
||||
strings.Join(act.Carried, "; "), act.By, act.Why)
|
||||
return
|
||||
}
|
||||
fmt.Printf("recorded as %s in the hand-act log: %s, because %q (cause: %s)\n", act.ID, act.By, act.Why, act.Cause)
|
||||
}
|
||||
|
||||
@@ -114,6 +232,10 @@ func handActCommand(ctx context.Context, args []string) error {
|
||||
return errors.New("hand-act record says the cause too: --cause <word>, the word a second " +
|
||||
"act for the same reason will use — it is how a repair done twice is found")
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(*f.cause), causeDrill) {
|
||||
return errors.New("a drill is not recorded as a repair: hand-act drill <what was done> --why <text> " +
|
||||
"records it as the person's deliberate test it is, which no healer is wanted for. Nothing was recorded")
|
||||
}
|
||||
act := link.HandAct{Verb: "hand-act record", Args: []string{what}, Why: strings.TrimSpace(*f.why),
|
||||
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
@@ -126,13 +248,26 @@ func handActCommand(ctx context.Context, args []string) error {
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if len(args) > 0 && args[0] == "drill" {
|
||||
return handActDrill(ctx, args[1:])
|
||||
}
|
||||
if len(args) > 0 && args[0] == "warrant" {
|
||||
return handActWarrantCommand(ctx, args[1:])
|
||||
}
|
||||
if len(args) > 0 && args[0] != "list" && !strings.HasPrefix(args[0], "-") {
|
||||
return errors.New("hand-act record <what> --why <text> --cause <word> | hand-acts [--days N] [--json]")
|
||||
return errors.New("hand-act record <what> --why <text> --cause <word> | hand-act drill <what> --why <text> " +
|
||||
"| hand-acts [--days N] [--json]")
|
||||
}
|
||||
if len(args) > 0 && args[0] == "list" {
|
||||
args = args[1:]
|
||||
}
|
||||
return listHandActs(ctx, args, os.Stdout)
|
||||
}
|
||||
|
||||
// listHandActs is `hand-acts`: what was done by hand lately, and the causes done more than once.
|
||||
func listHandActs(ctx context.Context, args []string, w io.Writer) error {
|
||||
set := flag.NewFlagSet("hand-acts", flag.ContinueOnError)
|
||||
usageTo(set, w)
|
||||
days := set.Int("days", 14, "how many days back")
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
@@ -144,27 +279,34 @@ func handActCommand(ctx context.Context, args []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
repeated := link.RepeatedCauses(acts, now)
|
||||
repeated := link.RepeatedCauses(repairs(acts), now)
|
||||
if *asJSON {
|
||||
body, err := json.MarshalIndent(map[string]any{"acts": acts, "repeated": repeated}, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
fmt.Fprintln(w, string(body))
|
||||
return nil
|
||||
}
|
||||
if len(acts) == 0 {
|
||||
fmt.Printf("nothing was done by hand in the last %d day(s)\n", *days)
|
||||
fmt.Fprintf(w, "nothing was done by hand in the last %d day(s)\n", *days)
|
||||
return nil
|
||||
}
|
||||
for i := len(acts) - 1; i >= 0; i-- {
|
||||
a := acts[i]
|
||||
fmt.Printf("%s %s %s %s\n by %s — %s (cause: %s", a.At.Local().Format("2006-01-02 15:04"), a.ID,
|
||||
fmt.Fprintf(w, "%s %s %s %s\n by %s — %s (cause: %s", a.At.Local().Format("2006-01-02 15:04"), a.ID,
|
||||
a.Verb, strings.Join(a.Args, " "), a.By, a.Why, a.Cause)
|
||||
if a.Condition != "" {
|
||||
fmt.Printf(", condition %s", a.Condition)
|
||||
fmt.Fprintf(w, ", condition %s", a.Condition)
|
||||
}
|
||||
fmt.Fprintln(w, ")")
|
||||
if pushedRecorded(a) {
|
||||
carried := strings.Join(a.Carried, "; ")
|
||||
if carried == "" {
|
||||
carried = recordedBefore[a.ID]
|
||||
}
|
||||
fmt.Fprintf(w, " a push of recorded builds, no repair: %s\n", carried)
|
||||
}
|
||||
fmt.Println(")")
|
||||
}
|
||||
if len(repeated) > 0 {
|
||||
causes := make([]string, 0, len(repeated))
|
||||
@@ -172,13 +314,56 @@ func handActCommand(ctx context.Context, args []string) error {
|
||||
causes = append(causes, fmt.Sprintf("%s ×%d", c, n))
|
||||
}
|
||||
sort.Strings(causes)
|
||||
fmt.Printf("\ndone by hand more than once in a fortnight — a healer is wanted (to-be 45 S15): %s\n",
|
||||
fmt.Fprintf(w, "\ndone by hand more than once in a fortnight — a healer is wanted (to-be 45 S15): %s\n",
|
||||
strings.Join(causes, ", "))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// handActDrill is `hand-act drill`: an act done on purpose to test the mesh — a module stopped, a
|
||||
// process killed — recorded so the conditions it raises are read as the drill they are. Its cause is
|
||||
// always causeDrill and S15 never counts it (handActVerbs).
|
||||
func handActDrill(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("hand-act drill", flag.ContinueOnError)
|
||||
why := set.String("why", "", "what the drill tests — recorded in the hand-act log (novox/hq to-be 45 §7)")
|
||||
condition := set.String("condition", "", "the key of the condition the drill is meant to raise, if any")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
what := strings.TrimSpace(strings.Join(positionals, " "))
|
||||
if what == "" {
|
||||
return errors.New("hand-act drill <what was done on purpose> --why <what it tests> [--condition <key>]")
|
||||
}
|
||||
if strings.TrimSpace(*why) == "" {
|
||||
return errors.New("a drill says what it tests: --why <text> (recorded in the hand-act log, novox/hq " +
|
||||
"to-be 45 §7). Nothing was recorded")
|
||||
}
|
||||
act := link.HandAct{Verb: "hand-act drill", Args: []string{what}, Why: strings.TrimSpace(*why),
|
||||
Cause: causeDrill, Condition: strings.TrimSpace(*condition)}
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
written, err := link.RecordHandAct(ctx, conn, act)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the drill could not be recorded: %w", err)
|
||||
}
|
||||
fmt.Printf("recorded as %s: %s drilled %q, because %q — a drill, which no healer is wanted for\n",
|
||||
written.ID, written.By, what, written.Why)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// repairs are the acts that are not a person's decision by design: what S15 counts.
|
||||
func repairs(acts []link.HandAct) []link.HandAct {
|
||||
out := make([]link.HandAct, 0, len(acts))
|
||||
for _, a := range acts {
|
||||
if !personsDecision(a) {
|
||||
out = append(out, a)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// handActsThisWeek is how many acts were done by hand in the last seven days, for `status`; -1 when
|
||||
// the log could not be read, which status says rather than reading as none.
|
||||
func handActsThisWeek(ctx context.Context) (int, string) {
|
||||
|
||||
@@ -22,10 +22,6 @@ func TestARepairByHandWithoutAReasonIsRefused(t *testing.T) {
|
||||
{"plans", map[string]any{"close": "plan-1"}},
|
||||
{"plans", map[string]any{"stop": "plan-1"}},
|
||||
{"hand-act", map[string]any{"what": "restarted the proxy", "cause": "proxy-stuck"}},
|
||||
{"command", map[string]any{"command": "push anchor"}},
|
||||
{"command", map[string]any{"command": "plans close plan-1"}},
|
||||
{"command", map[string]any{"command": "broker consumer-reset EVENTS controller"}},
|
||||
{"command", map[string]any{"command": "hand-act record restarted --cause x"}},
|
||||
} {
|
||||
argv, err := argvFor(c.verb, c.args)
|
||||
if c.verb == "plans" && err == nil {
|
||||
@@ -65,7 +61,6 @@ func TestARepairByHandCarriesItsReason(t *testing.T) {
|
||||
{"plans", map[string]any{"retry": "plan-1"}, "plans retry plan-1"},
|
||||
{"hand-act", map[string]any{"what": "restarted", "why": "hung", "cause": "proxy", "condition": "machine.a.silent"},
|
||||
"hand-act record restarted --why hung --cause proxy --condition machine.a.silent"},
|
||||
{"command", map[string]any{"command": "push anchor --why stuck"}, "push anchor --why stuck"},
|
||||
{"command", map[string]any{"command": "plans plan-1"}, "plans plan-1"},
|
||||
} {
|
||||
argv, err := argvFor(c.verb, c.args)
|
||||
@@ -92,3 +87,32 @@ func TestDurationsAreSummarisedPerSubject(t *testing.T) {
|
||||
t.Fatalf("a minute between words suggests %q", got[1].Suggests)
|
||||
}
|
||||
}
|
||||
|
||||
// **A drill has its own verb** — `hand-act drill`, the seat's `drill` — and `hand-act record` refuses
|
||||
// the cause, so a repair cannot pass for a drill by the word it gives.
|
||||
func TestADrillIsRecordedThroughItsOwnVerb(t *testing.T) {
|
||||
err := handActCommand(context.Background(), []string{"record", "stopped searxng", "--why", "a test", "--cause", "drill"})
|
||||
if err == nil || !strings.Contains(err.Error(), "hand-act drill") {
|
||||
t.Errorf("hand-act record --cause drill was not sent to the drill verb: %v", err)
|
||||
}
|
||||
if err := handActCommand(context.Background(), []string{"drill", "stopped searxng"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "--why") {
|
||||
t.Errorf("a drill without what it tests: %v", err)
|
||||
}
|
||||
if err := handActCommand(context.Background(), []string{"drill", "--why", "a test"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "hand-act drill <what") {
|
||||
t.Errorf("a drill without what was done: %v", err)
|
||||
}
|
||||
argv, err := argvFor("drill", map[string]any{"what": "stopped searxng", "why": "ADR 0240 phase A",
|
||||
"condition": "machine.ace.module.searxng.unhealthy"})
|
||||
if want := "hand-act drill stopped searxng --why ADR 0240 phase A --condition machine.ace.module.searxng.unhealthy"; err != nil ||
|
||||
strings.Join(argv, " ") != want {
|
||||
t.Errorf("the seat's drill: %v %v, want %q", argv, err, want)
|
||||
}
|
||||
if _, err := argvFor("drill", map[string]any{"what": "stopped searxng"}); err == nil {
|
||||
t.Error("the seat's drill without why was not refused")
|
||||
}
|
||||
if repairingCommand([]string{"hand-act", "drill", "x"}) != "hand-act drill" {
|
||||
t.Error("a drill through `command` is not held to why")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A hand-over's line is judged before anything is asked (novox/hq issue 356): a node's name and the directory's
|
||||
// absolute path exactly as the engine states it — no `..`, no doubled or trailing separator, nothing relative.
|
||||
func TestAHandOverLineIsJudgedBeforeItIsAsked(t *testing.T) {
|
||||
for _, args := range [][]string{
|
||||
{},
|
||||
{"laptop"},
|
||||
{"laptop", "/srv/notes", "extra"},
|
||||
{"", "/srv/notes"},
|
||||
{"--node", "/srv/notes"},
|
||||
{"laptop", "srv/notes"},
|
||||
{"laptop", "/srv/../etc"},
|
||||
{"laptop", "/srv//notes"},
|
||||
{"laptop", "/srv/notes/"},
|
||||
{"laptop", "/srv/notes/."},
|
||||
} {
|
||||
if _, _, err := handOverLine(args); err == nil {
|
||||
t.Errorf("%q was taken", args)
|
||||
}
|
||||
}
|
||||
node, path, err := handOverLine([]string{"laptop", "/srv/notes"})
|
||||
if err != nil || node != "laptop" || path != "/srv/notes" {
|
||||
t.Fatalf("read as %q %q %v", node, path, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Who hands over is the line's caller in the words every verb's caller is recorded in — the operator through
|
||||
// mesh-cli — or the controller's terminal when nobody is named.
|
||||
func TestAHandOverNamesWhoAsked(t *testing.T) {
|
||||
t.Setenv(link.CallerVar, " jo through mesh-cli on anchor ")
|
||||
if by := handOverBy(); by != "jo through mesh-cli on anchor" {
|
||||
t.Fatalf("by %q", by)
|
||||
}
|
||||
t.Setenv(link.CallerVar, "")
|
||||
if by := handOverBy(); by != "the controller's terminal" {
|
||||
t.Fatalf("by %q", by)
|
||||
}
|
||||
}
|
||||
|
||||
// **A hand-over is the controller's terminal's alone** (novox/hq issue 356, ADR 0266): through any verb, and
|
||||
// through mesh-cli outside the terminal, `node hand-over` is refused and nothing runs — at the next apply root
|
||||
// gives the directory to the account the module declares, and whoever may call a verb includes agents.
|
||||
func TestAHandOverIsRefusedThroughEveryVerb(t *testing.T) {
|
||||
line := []string{"node", "hand-over", "laptop", "/srv/notes"}
|
||||
if err := terminalOnly(line); err == nil {
|
||||
t.Fatal("node hand-over passed as a verb's line")
|
||||
}
|
||||
if _, err := argvFor("command", map[string]any{"command": "node hand-over laptop /srv/notes"}); err == nil {
|
||||
t.Fatal("the command verb composed node hand-over")
|
||||
}
|
||||
if _, err := ordinaryLine(line); err == nil {
|
||||
t.Fatal("node hand-over composed as an ordinary mesh-cli line")
|
||||
}
|
||||
if _, err := argvFor("node", map[string]any{"node": "laptop", "hand-over": "/srv/notes"}); err == nil {
|
||||
t.Fatal("the node verb composed a hand-over")
|
||||
}
|
||||
}
|
||||
|
||||
// The condition's words are plain and name no machine's binary; the operator's line, with the node and the path
|
||||
// (novox/hq ADR 0272), is in the summary the module's health gives (moduleHealthWord) and in the evidence.
|
||||
func TestTheUsedAsFoundConditionNamesTheOperatorsLine(t *testing.T) {
|
||||
rs := []inventory.ResourceHealth{foundDirectory("notes", time.Now().Add(-24*time.Hour))}
|
||||
o := usedAsFoundObservation("notes", "laptop", "notes on laptop uses notes.data as found", rs)
|
||||
if strings.Contains(o.Needs, "mesh-host") || strings.Contains(o.Explanation, "mesh-host") ||
|
||||
!strings.Contains(o.Needs, "control-node") {
|
||||
t.Fatalf("the condition's words: %q %q", o.Needs, o.Explanation)
|
||||
}
|
||||
if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Needs: o.Needs,
|
||||
Resolved: o.Resolved}, "laptop"); !ok {
|
||||
t.Fatalf("not plain: %s", why)
|
||||
}
|
||||
f := gateFacts{now: time.Now(), health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: time.Now(),
|
||||
Resources: rs}}}
|
||||
h, why := moduleHealthWord("notes", "laptop", time.Now().Add(-time.Hour), f)
|
||||
if h != healthPerson || !strings.Contains(why, "`nox node hand-over laptop <directory>` on the control-node") ||
|
||||
strings.Contains(why, "mesh-host") || strings.Contains(why, "/srv/") {
|
||||
t.Fatalf("the module's health reads %v %q; want the operator's line", h, why)
|
||||
}
|
||||
}
|
||||
|
||||
// **Nothing is asked of a node the mesh does not know, and the engine's refusal is the command's failure**
|
||||
// (review of issue 356): a refused hand-over never exits as a success.
|
||||
func TestAHandOverAsksOnlyAKnownNodeAndFailsOnARefusal(t *testing.T) {
|
||||
t.Setenv(link.CallerVar, "jo through mesh-cli on anchor")
|
||||
asked := 0
|
||||
ask := func(answer link.HandOverAnswer) func(node, path, by string) (link.HandOverAnswer, error) {
|
||||
return func(node, path, by string) (link.HandOverAnswer, error) {
|
||||
asked++
|
||||
if node != "laptop" || path != "/srv/notes" || by != "jo through mesh-cli on anchor" {
|
||||
t.Fatalf("asked %q %q %q", node, path, by)
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
}
|
||||
unknown := func(string) error { return errors.New("no node called laptop") }
|
||||
known := func(string) error { return nil }
|
||||
var out bytes.Buffer
|
||||
err := handOverAsked([]string{"laptop", "/srv/notes"}, unknown, ask(link.HandOverAnswer{Said: "x"}), &out)
|
||||
if err == nil || asked != 0 || !strings.Contains(err.Error(), "nothing was asked") {
|
||||
t.Fatalf("an unknown node: %v, asked %d", err, asked)
|
||||
}
|
||||
err = handOverAsked([]string{"laptop", "/srv/../etc"}, known, ask(link.HandOverAnswer{Said: "x"}), &out)
|
||||
if err == nil || asked != 0 {
|
||||
t.Fatalf("a refused line was asked: %v, asked %d", err, asked)
|
||||
}
|
||||
err = handOverAsked([]string{"laptop", "/srv/notes"}, known,
|
||||
ask(link.HandOverAnswer{Refused: "/srv/notes is not used as found; nothing was handed over"}), &out)
|
||||
if err == nil || !strings.Contains(err.Error(), "laptop refused: /srv/notes is not used as found") || out.Len() != 0 {
|
||||
t.Fatalf("a refusal: %v, printed %q", err, out.String())
|
||||
}
|
||||
err = handOverAsked([]string{"laptop", "/srv/notes"}, known, ask(link.HandOverAnswer{Said: "handed over"}), &out)
|
||||
if err != nil || !strings.HasPrefix(out.String(), "handed over\n") || !strings.Contains(out.String(), "`nox push laptop`") {
|
||||
t.Fatalf("a record: %v, printed %q", err, out.String())
|
||||
}
|
||||
failing := func(string, string, string) (link.HandOverAnswer, error) {
|
||||
return link.HandOverAnswer{}, errors.New("no engine")
|
||||
}
|
||||
if err := handOverAsked([]string{"laptop", "/srv/notes"}, known, failing, &out); err == nil {
|
||||
t.Fatal("an ask that failed was a success")
|
||||
}
|
||||
}
|
||||
|
||||
// The setuid search's line asks only a known node, one name and nothing else, and fails on a refusal
|
||||
// (novox/hq issue 361).
|
||||
func TestASetuidSearchAsksOnlyAKnownNodeAndFailsOnARefusal(t *testing.T) {
|
||||
t.Setenv(link.CallerVar, "jo through mesh-cli on anchor")
|
||||
asked := 0
|
||||
ask := func(answer link.HandOverAnswer) func(node, by string) (link.HandOverAnswer, error) {
|
||||
return func(node, by string) (link.HandOverAnswer, error) {
|
||||
asked++
|
||||
if node != "novox" || by != "jo through mesh-cli on anchor" {
|
||||
t.Fatalf("asked %q %q", node, by)
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
}
|
||||
known := func(string) error { return nil }
|
||||
var out bytes.Buffer
|
||||
for _, args := range [][]string{nil, {"novox", "extra"}, {"-x"}} {
|
||||
if err := setuidSearchAsked(args, known, ask(link.HandOverAnswer{Said: "x"}), &out); err == nil || asked != 0 {
|
||||
t.Fatalf("%v was asked: %v", args, err)
|
||||
}
|
||||
}
|
||||
if err := setuidSearchAsked([]string{"novox"}, func(string) error { return errors.New("no node called novox") },
|
||||
ask(link.HandOverAnswer{Said: "x"}), &out); err == nil || asked != 0 {
|
||||
t.Fatalf("an unknown node: %v", err)
|
||||
}
|
||||
if err := setuidSearchAsked([]string{"novox"}, known, ask(link.HandOverAnswer{Refused: "no; no search was started"}),
|
||||
&out); err == nil || !strings.Contains(err.Error(), "novox refused") || out.Len() != 0 {
|
||||
t.Fatalf("a refusal: %v, printed %q", err, out.String())
|
||||
}
|
||||
if err := setuidSearchAsked([]string{"novox"}, known, ask(link.HandOverAnswer{Said: "a new search starts"}),
|
||||
&out); err != nil || !strings.HasPrefix(out.String(), "a new search starts\n") {
|
||||
t.Fatalf("a start: %v, printed %q", err, out.String())
|
||||
}
|
||||
}
|
||||
@@ -111,9 +111,11 @@ var healerRegistry = []healerRow{
|
||||
{ID: "H2", Kinds: []string{"stalled"},
|
||||
Condition: "a plan stalled on a wait that is superseded — a newer plan of its repository and branch " +
|
||||
"exists — or already finished — every module of every tier built or failed, and every one that " +
|
||||
"rolls out sent (S3)",
|
||||
"rolls out sent (S3); or a delivery held past its state's bound where mesh-delivery's table lets H2 " +
|
||||
"take a transition (D14, novox/hq ADR 0239)",
|
||||
Repair: "close the plan with its note, as `plans close` does: superseded, naming the newer plan, or " +
|
||||
"done; what it asked still builds and registers",
|
||||
"done; what it asked still builds and registers — or, for a delivery, mesh-delivery's `close`, which " +
|
||||
"reads its walk again and takes only the transition its table names",
|
||||
Then: "resolver operator, urgent", From: "a stuck plan closed by hand (214, 254)",
|
||||
Budget: 1, Window: 24 * time.Hour, Settle: 2 * time.Minute, ActsIn: actsInController, Event: link.KeyHealerActed,
|
||||
applies: appliesToAStalePlan, repair: repairPlan},
|
||||
@@ -614,13 +616,16 @@ func lastReportOf(ctx context.Context, inv *inventory.Inventory, node string) (i
|
||||
// planStale is why a plan's wait is superseded or finished, and the state closing it leaves it in; empty
|
||||
// when it is neither, which is not H2's to repair.
|
||||
func planStale(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) (state, why string, err error) {
|
||||
if p.Release != nil {
|
||||
return "", "", nil // a release plan walks machines, and its own gate says when it is done (ADR 0236)
|
||||
}
|
||||
recent, err := inv.RecentPlans(ctx, 50)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
for _, newer := range recent {
|
||||
if newer.ID == p.ID || !newer.Created.After(p.Created) || !repositoryMatches(newer.Repository, p.Repository) ||
|
||||
newer.Branch != p.Branch || newer.State == inventory.PlanSuperseded {
|
||||
newer.Branch != p.Branch || newer.State == inventory.PlanSuperseded || newer.Batch() {
|
||||
continue
|
||||
}
|
||||
return inventory.PlanSuperseded, fmt.Sprintf("superseded by %s (%s %s), a newer merge of the same repository "+
|
||||
@@ -649,6 +654,9 @@ func planStale(ctx context.Context, inv *inventory.Inventory, p inventory.Plan)
|
||||
|
||||
// appliesToAStalePlan is H2's: the plan is open, and its wait is superseded or finished.
|
||||
func appliesToAStalePlan(ctx context.Context, h *healing, c conditions.Condition) (string, bool, string, error) {
|
||||
if c.Subject.Scope == conditions.ScopeDelivery {
|
||||
return appliesToAStalledDelivery(ctx, h, c)
|
||||
}
|
||||
p, err := h.open.inventory.PlanByID(ctx, c.Subject.ID)
|
||||
if err != nil {
|
||||
return "", false, "", err
|
||||
@@ -669,6 +677,9 @@ func appliesToAStalePlan(ctx context.Context, h *healing, c conditions.Condition
|
||||
|
||||
// repairPlan is H2: the plan closed with its note, under the plans' hold, by compare-and-set.
|
||||
func repairPlan(ctx context.Context, h *healing, c conditions.Condition) (string, string, error) {
|
||||
if c.Subject.Scope == conditions.ScopeDelivery {
|
||||
return repairDelivery(ctx, h, c)
|
||||
}
|
||||
inv := h.open.inventory
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
|
||||
@@ -18,6 +18,8 @@ import (
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// The test generated from the healer registry (novox/hq to-be 45 §7, ADR 0227 rule 7 "how it is
|
||||
@@ -434,10 +436,7 @@ func TestH4ResetsOnlyWhatTheTableMarksResettable(t *testing.T) {
|
||||
// asserts the bus's objects the way a send does, and D6's next run clears it — the healer never does.
|
||||
// Deleted again within the hour, the budget is spent: the operator's, urgent, and H3 stops.
|
||||
func TestNatsH3AssertsAMissingConsumerAgainAndBrakesAfterItsBudget(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
js, err := broker.Dial(url)
|
||||
@@ -530,10 +529,7 @@ func TestNatsH3AssertsAMissingConsumerAgainAndBrakesAfterItsBudget(t *testing.T)
|
||||
// once replayed — is re-made from now by the mesh itself, and the controller's bound subscription still
|
||||
// receives what comes next: a reset that left the controller deaf would be the incident.
|
||||
func TestNatsH4ResetsTheControllersEventsConsumerAndItStillDelivers(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
js, err := broker.Dial(url)
|
||||
@@ -619,10 +615,7 @@ func TestNatsH4ResetsTheControllersEventsConsumerAndItStillDelivers(t *testing.T
|
||||
|
||||
// **H1's question reaches the machine**, on the subject its grant lets it hear and nothing else.
|
||||
func TestNatsAskToReportReachesTheMachine(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// Every catalogue module that runs something long-lived declares how it is ready (novox/hq ADR 0240 rule
|
||||
// 8): `module check` warns and counts the undeclared before the date, and refuses them from it.
|
||||
func TestModuleCheckCountsTheUndeclaredAndRefusesThemFromTheDate(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
digest := "@sha256:" + strings.Repeat("a", 64)
|
||||
path := filepath.Join(dir, "module.json")
|
||||
os.WriteFile(path, []byte(`{"module":"web","listens":[{"name":"web","port":80,"from":"mesh"}],"resources":[
|
||||
{"id":"server","type":"container","name":"web","image":"registry.example/web`+digest+`","ports":["80"],
|
||||
"health":{"kind":"http","endpoint":"web"}},
|
||||
{"id":"worker","type":"container","name":"web-worker","image":"registry.example/web`+digest+`"},
|
||||
{"id":"seed","type":"container","name":"web-seed","image":"registry.example/web`+digest+`","run-once":true}]}`), 0o600)
|
||||
defer func() { checkNow = time.Now }()
|
||||
|
||||
checkNow = func() time.Time { return catalogue.HealthRequiredFrom.Add(-time.Hour) }
|
||||
var out bytes.Buffer
|
||||
if err := moduleCheck([]string{path}, &out); err != nil {
|
||||
t.Fatalf("refused before the date: %v\n%s", err, out.String())
|
||||
}
|
||||
for _, want := range []string{"ready: server by http / on web every 30s", "WARNING: worker stay(s) up",
|
||||
catalogue.HealthRequiredFrom.Format("2006-01-02"), UndeclaredHealthLine + " 1"} {
|
||||
if !strings.Contains(out.String(), want) {
|
||||
t.Errorf("the check does not say %q:\n%s", want, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
checkNow = func() time.Time { return catalogue.HealthRequiredFrom }
|
||||
out.Reset()
|
||||
if err := moduleCheck([]string{path}, &out); err == nil {
|
||||
t.Fatalf("a long-running resource without health passed after the date:\n%s", out.String())
|
||||
}
|
||||
if !strings.Contains(out.String(), "web: worker stays up and does not say how it is ready") {
|
||||
t.Errorf("the refusal does not name the resource:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A file that asks for a setting without saying whether it is trusted counts as trusted (novox/hq issue 339):
|
||||
// `module check` lists and counts it, and never refuses it — there is nothing unsafe to refuse.
|
||||
func TestModuleCheckListsUnmarkedFilesAndNeverRefusesThem(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "module.json")
|
||||
os.WriteFile(path, []byte(`{"module":"power","resources":[
|
||||
{"id":"logind","type":"file","path":"/etc/systemd/logind.conf.d/power.conf","mode":"0644","trusted":true,
|
||||
"content":"HandleLidSwitch=${setting:lid}\n"},
|
||||
{"id":"note","type":"file","path":"/var/lib/power/note","mode":"0644","content":"${setting:greeting}\n"}]}`), 0o600)
|
||||
defer func() { checkNow = time.Now }()
|
||||
for _, at := range []time.Time{time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC), time.Date(2036, 1, 1, 0, 0, 0, 0, time.UTC)} {
|
||||
checkNow = func() time.Time { return at }
|
||||
var out bytes.Buffer
|
||||
if err := moduleCheck([]string{path}, &out); err != nil {
|
||||
t.Fatalf("refused at %v: %v\n%s", at, err, out.String())
|
||||
}
|
||||
for _, want := range []string{"note ask(s) for a setting and do(es) not say whether it is trusted, so it counts as trusted",
|
||||
UnsaidTrustLine + " 1"} {
|
||||
if !strings.Contains(out.String(), want) {
|
||||
t.Errorf("the check does not say %q:\n%s", want, out.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
)
|
||||
|
||||
// The core components' health definitions, as probes in the self-check's registry (novox/hq to-be 45
|
||||
// §8, §4 `H-*`). The same definitions judge a core component's new build on its first machine (the
|
||||
// gate, gate.go); here they are run against every machine on the self-check's schedule, so a core
|
||||
// component that stops being healthy between upgrades is said too.
|
||||
//
|
||||
// controller holds the lease, and says it is ready: its self-check ran, `status` answered in bound
|
||||
// node-engine has reported its current declaration, under a build the mesh delivered
|
||||
// node tools announced, and answer the bus's discovery
|
||||
// bus every stream and durable consumer the mesh defines is there, and a request crosses the
|
||||
// bus to every machine's node tools and back
|
||||
const kindCoreUnhealthy = "core-unhealthy"
|
||||
|
||||
// probeControllerHealth is H-controller: the lease is held, fresh, by a controller that says it is
|
||||
// ready — or one that started less than the witness's bound ago.
|
||||
func probeControllerHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
h, found, err := theLease.holder(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
unhealthy := func(why string) []conditions.Observation {
|
||||
node := d.host
|
||||
if found && h.Host != "" {
|
||||
node = h.Host
|
||||
}
|
||||
return []conditions.Observation{{Scope: conditions.ScopeCore, ID: lease.ComponentController + "." + node,
|
||||
Token: "unhealthy", Kind: kindCoreUnhealthy, Machine: node, Severity: conditions.Urgent,
|
||||
Summary: "the controller is not healthy: " + why}}
|
||||
}
|
||||
switch {
|
||||
case !found:
|
||||
return unhealthy("nobody holds the controller lease"), nil
|
||||
case time.Since(h.Renewed) > lease.FreshWithin:
|
||||
return unhealthy(fmt.Sprintf("the lease was last renewed %s ago", time.Since(h.Renewed).Round(time.Second))), nil
|
||||
case (h.Health == nil || !h.Health.Ready) && time.Since(h.Taken) > lease.ReadyWithin:
|
||||
why := "the controller holding the lease does not say it is ready"
|
||||
if h.Health != nil && h.Health.Why != "" {
|
||||
why += ": " + h.Health.Why
|
||||
}
|
||||
return unhealthy(why), nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// probeEngineHealth is H-engine: every machine heard from has reported its current declaration — the
|
||||
// last one it was sent — under a node-engine build the mesh delivered, or is inside the bound of a send.
|
||||
func probeEngineHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
return machinesHealth(ctx, d, hostModule)
|
||||
}
|
||||
|
||||
// probeToolsHealth is H-tools: every machine heard from that is assigned the node tools has them
|
||||
// announced, answering the bus's discovery.
|
||||
func probeToolsHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
return machinesHealth(ctx, d, broker.RuntimeModule)
|
||||
}
|
||||
|
||||
// machinesHealth judges a component on every machine running it and heard from, by its health
|
||||
// definition, as the gate does — with no condition taken as the build's doing.
|
||||
func machinesHealth(ctx context.Context, d *doctor, component string) ([]conditions.Observation, error) {
|
||||
inv := d.open.inventory
|
||||
running, err := inv.Running(ctx, component)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f, err := gatherGateFacts(ctx, d.open, coreComponent(component))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f.judged = false
|
||||
heard := heardMachines(d)
|
||||
var out []conditions.Observation
|
||||
for _, node := range running {
|
||||
if !heard[node] {
|
||||
continue // a machine not heard from is S1's
|
||||
}
|
||||
if r, said := f.reports[node]; said && !r.Current && r.Sent != nil && time.Since(*r.Sent) < gateBound {
|
||||
continue // inside the bound of a send: S2's, and the gate's
|
||||
}
|
||||
// What the machine did with what it was sent is not the component's health: the node-engine's is
|
||||
// that it reported its current declaration at all, the node tools' that they answer.
|
||||
if r := f.reports[node]; r.Current || component == broker.RuntimeModule {
|
||||
now := time.Now()
|
||||
r.Current, r.Outcome = true, inventory.OutcomeApplied
|
||||
if r.At == nil {
|
||||
r.At = &now
|
||||
}
|
||||
f.reports[node] = r
|
||||
}
|
||||
if component == hostModule {
|
||||
// A node-engine reporting a build the mesh delivered, current or previous, is the version
|
||||
// split's (D10), not ill health; a build the mesh did not deliver is.
|
||||
f.engines[node] = deliveredOr(shelf[component], f.engines[node])
|
||||
}
|
||||
h, why := judgeHealth(component, coreComponent(component), shelf[component], node, time.Time{}, f)
|
||||
if h == healthGood {
|
||||
continue
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: coreComponent(component) + "." + node,
|
||||
Token: "unhealthy", Kind: kindCoreUnhealthy, Machine: node, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the %s on %s is not healthy: %s", componentWords(component), node, why)})
|
||||
}
|
||||
return sortedFound(out), nil
|
||||
}
|
||||
|
||||
// deliveredOr is the reported engine build, or the current delivered one when the report names any
|
||||
// build at all: what H-engine judges is that it reports, not which.
|
||||
func deliveredOr(m catalogue.Manifest, reported string) string {
|
||||
if reported == "" {
|
||||
return reported
|
||||
}
|
||||
if v := deliveredVersions(m); len(v) > 0 {
|
||||
return v[0]
|
||||
}
|
||||
return reported
|
||||
}
|
||||
|
||||
// componentWords is a core component as a sentence names it.
|
||||
func componentWords(component string) string {
|
||||
switch component {
|
||||
case hostModule:
|
||||
return "node-engine"
|
||||
case broker.RuntimeModule:
|
||||
return "node tools"
|
||||
case catalogue.ControllerSeatName:
|
||||
return "controller"
|
||||
}
|
||||
return component
|
||||
}
|
||||
|
||||
// probeBusHealth is H-bus: every stream and durable consumer the mesh defines is on the bus, and a
|
||||
// request crosses it to every machine's node tools and back.
|
||||
func probeBusHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
problems, err := busHealth(ctx, d)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(problems) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return []conditions.Observation{{Scope: conditions.ScopeBus, ID: "mesh", Token: "unhealthy",
|
||||
Kind: kindCoreUnhealthy, Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("the bus is not healthy: %s", strings.Join(problems, "; ")),
|
||||
Said: strings.Join(problems, "; ")}}, nil
|
||||
}
|
||||
|
||||
// busHealth is the bus's health definition, as what is wanting: nothing when healthy. What the bus's
|
||||
// planned step checks after the bus is replaced, too.
|
||||
var busHealth = func(ctx context.Context, d *doctor) ([]string, error) {
|
||||
if d.js == nil {
|
||||
return nil, errors.New("this controller has no bus to ask")
|
||||
}
|
||||
streams, consumers, err := expectedBusObjects(ctx, d.open.inventory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js := d.js.Context()
|
||||
var problems []string
|
||||
for _, s := range streams {
|
||||
if _, err := js.StreamInfo(s.Name, nats.Context(ctx)); errors.Is(err, nats.ErrStreamNotFound) {
|
||||
problems = append(problems, "the stream "+s.Name+" is missing")
|
||||
} else if err != nil {
|
||||
return nil, fmt.Errorf("the stream %s cannot be read: %w", s.Name, err)
|
||||
}
|
||||
}
|
||||
for _, c := range consumers {
|
||||
_, err := js.ConsumerInfo(c.Stream, c.Name, nats.Context(ctx))
|
||||
if errors.Is(err, nats.ErrConsumerNotFound) || errors.Is(err, nats.ErrStreamNotFound) {
|
||||
problems = append(problems, consumerWords(c)+" is missing")
|
||||
} else if err != nil {
|
||||
return nil, fmt.Errorf("%s cannot be read: %w", consumerWords(c), err)
|
||||
}
|
||||
}
|
||||
// The round trip: every machine heard from that runs the node tools answers across the bus.
|
||||
running, err := d.open.inventory.Running(ctx, broker.RuntimeModule)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
heard := heardMachines(d)
|
||||
var expected []string
|
||||
for _, n := range running {
|
||||
if heard[n] {
|
||||
expected = append(expected, n)
|
||||
}
|
||||
}
|
||||
if len(expected) > 0 {
|
||||
answered, err := servedOnTheBus(ctx, d.js.Conn())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var silent []string
|
||||
for _, n := range expected {
|
||||
if !answered[n].runtime {
|
||||
silent = append(silent, n)
|
||||
}
|
||||
}
|
||||
// One machine's tools silent is that machine's (H-tools); none answering is the bus.
|
||||
if len(silent) == len(expected) {
|
||||
slices.Sort(silent)
|
||||
problems = append(problems, "no request crossed the bus and came back: no machine's node tools answered ("+
|
||||
strings.Join(silent, ", ")+")")
|
||||
}
|
||||
}
|
||||
return problems, nil
|
||||
}
|
||||
@@ -120,6 +120,10 @@ func heldMachines(ctx context.Context, open *stores, names []string) (map[string
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f, err := readMoveFacts(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, node := range names {
|
||||
plan, _, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
@@ -133,7 +137,25 @@ func heldMachines(ctx context.Context, open *stores, names []string) (map[string
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if why := heldBack(node, modules, sent, known, current, plans); len(why) > 0 {
|
||||
// A rebuild that put the same thing on the machine is no move (ADR 0236): read as the build it
|
||||
// runs.
|
||||
same := map[string]string{}
|
||||
for m, was := range sent {
|
||||
same[m] = was
|
||||
if f.identical(m, was, current[m].Commit) {
|
||||
same[m] = current[m].Commit
|
||||
}
|
||||
}
|
||||
why := heldBack(node, modules, same, known, current, plans)
|
||||
// And a build no gate has seen is not carried by a send that does not judge it (ADR 0236).
|
||||
for _, mv := range f.moves(node, modules, same, known, false) {
|
||||
if planStillToSend(plans, mv.Module, node) == "" {
|
||||
why = append(why, fmt.Sprintf("%s would move from %s to %s, which has passed no gate yet — a release "+
|
||||
"plan sends it, one machine at a time, judged", mv.Module, buildName(mv.From), buildName(mv.To)))
|
||||
}
|
||||
}
|
||||
if len(why) > 0 {
|
||||
sort.Strings(why)
|
||||
out[node] = why
|
||||
}
|
||||
}
|
||||
@@ -157,7 +179,7 @@ func sayHeld(w io.Writer, node string, why []string) {
|
||||
// machines that could not be composed, as refusals.
|
||||
func flushBehind(ctx context.Context, open *stores, nodes []inventory.Node, handled map[string]bool,
|
||||
compose func(held context.Context, node string) (sendable, error), d delivery, holder string,
|
||||
w io.Writer) ([]string, error) {
|
||||
keep keeping, w io.Writer) ([]string, error) {
|
||||
inv := open.inventory
|
||||
var refusals []string
|
||||
// Bounded by the node count: a node is marked handled the round it is considered and is never
|
||||
@@ -215,7 +237,7 @@ func flushBehind(ctx context.Context, open *stores, nodes []inventory.Node, hand
|
||||
// a refusal and reported at the end, and the others are still sent (novox/hq ADR 0066).
|
||||
// Held for this round only, and after the last round's were given back, so two pushes
|
||||
// cascading into each other's machines never each wait on the other.
|
||||
refused, err := sendRound(ctx, open, sending, compose, d, holder)
|
||||
refused, err := sendRound(ctx, open, sending, compose, d, holder, keep)
|
||||
refusals = append(refusals, refused...)
|
||||
if err != nil {
|
||||
return refusals, err
|
||||
@@ -232,6 +254,7 @@ func composeForPush(open *stores, gens map[string]catalogue.Generator) func(held
|
||||
return sendable{}, err
|
||||
}
|
||||
reportUnhostable(node, plan)
|
||||
reportKept(held, plan)
|
||||
declared, err := declarationWith(held, open, node, plan, settings, gens, Allocating)
|
||||
if err == nil {
|
||||
reportLeftOut(node, declared)
|
||||
|
||||
@@ -193,13 +193,17 @@ func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// Recorded by a person's choice: the default rolls out since novox/hq ADR 0236.
|
||||
if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{Why: "each machine checked by hand"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
compose := composeForPush(open, gens)
|
||||
d := &recordedDelivery{inv: inv}
|
||||
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, ""); err != nil {
|
||||
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if builds, known, err := inv.SentBuilds(ctx, "laptop"); err != nil || !known || builds["resolver"] != "c1c1c1c1c1" {
|
||||
@@ -214,16 +218,16 @@ func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
|
||||
}
|
||||
before := digestOfLaptop()
|
||||
|
||||
// The change merges; the policy is the default, record. `push anchor` sends the anchor...
|
||||
// The change merges; the policy is record. `push anchor` sends the anchor...
|
||||
aResolver(t, open, "c2c2c2c2c2", asked.Add(time.Minute))
|
||||
d.declared = nil
|
||||
if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, ""); err != nil {
|
||||
if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// ...and its cascade leaves the laptop, saying so.
|
||||
var said bytes.Buffer
|
||||
d.declared = nil
|
||||
refused, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true}, compose, d, "", &said)
|
||||
refused, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true}, compose, d, "", nil, &said)
|
||||
if err != nil || len(refused) != 0 {
|
||||
t.Fatalf("the cascade failed: %v %v", refused, err)
|
||||
}
|
||||
@@ -244,13 +248,13 @@ func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
|
||||
// is held, and that what else it is owed waits with it.
|
||||
aThirdMachine(t, open)
|
||||
d.declared = nil
|
||||
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, ""); err != nil {
|
||||
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.declared = nil
|
||||
said.Reset()
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
|
||||
compose, d, "", &said); err != nil {
|
||||
compose, d, "", nil, &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(d.declared) != 0 || digestOfLaptop() != before {
|
||||
@@ -260,13 +264,27 @@ func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
|
||||
t.Fatalf("the push did not say both:\n%s", said.String())
|
||||
}
|
||||
|
||||
// A policy that rolls out: the laptop is a consequence like any other, and sent.
|
||||
// A policy that rolls out, and a build no gate has seen: still held — a cascade does not judge it
|
||||
// (novox/hq ADR 0236).
|
||||
if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{RollOut: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said.Reset()
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
|
||||
compose, d, "", &said); err != nil {
|
||||
compose, d, "", nil, &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(d.declared) != 0 || !strings.Contains(said.String(), "has passed no gate yet") {
|
||||
t.Fatalf("a cascade carried a build no gate has seen: %v\n%s", d.declared, said.String())
|
||||
}
|
||||
// Once it passed a gate on some machine, the laptop is a consequence like any other, and sent.
|
||||
if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: "build-c2", Module: "resolver", Commit: "c2c2c2c2c2",
|
||||
Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said.Reset()
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
|
||||
compose, d, "", nil, &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(d.declared, []string{"laptop"}) || digestOfLaptop() == before {
|
||||
@@ -293,18 +311,18 @@ func TestANamedPushStillSendsAConsequenceNothingHolds(t *testing.T) {
|
||||
}
|
||||
compose := composeForPush(open, gens)
|
||||
d := &recordedDelivery{inv: inv}
|
||||
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, ""); err != nil {
|
||||
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// `push spare`, the machine just placed: the others' peers change with it.
|
||||
aThirdMachine(t, open)
|
||||
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, ""); err != nil {
|
||||
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.declared = nil
|
||||
var said bytes.Buffer
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", &said); err != nil {
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(d.declared, []string{"anchor", "laptop"}) {
|
||||
@@ -324,7 +342,7 @@ func TestANamedPushStillSendsAConsequenceNothingHolds(t *testing.T) {
|
||||
}
|
||||
d.declared = nil
|
||||
said.Reset()
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", &said); err != nil {
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The anchor, the hub, may still be settling from the machine placed above; the laptop is the
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// hideTyping turns a terminal's echo off while a secret is typed at it, and gives back what restores it. On
|
||||
// anything that is not a terminal (a pipe, a file) it does nothing.
|
||||
func hideTyping(f *os.File) func() {
|
||||
fd := int(f.Fd())
|
||||
before, err := unix.IoctlGetTermios(fd, unix.TCGETS)
|
||||
if err != nil {
|
||||
return func() {}
|
||||
}
|
||||
hidden := *before
|
||||
hidden.Lflag &^= unix.ECHO
|
||||
if err := unix.IoctlSetTermios(fd, unix.TCSETS, &hidden); err != nil {
|
||||
return func() {}
|
||||
}
|
||||
return func() {
|
||||
_ = unix.IoctlSetTermios(fd, unix.TCSETS, before)
|
||||
_, _ = os.Stderr.WriteString("\n")
|
||||
}
|
||||
}
|
||||
@@ -23,11 +23,11 @@ func TestASendRoundGivesItsHoldBackOnEveryWayOut(t *testing.T) {
|
||||
|
||||
for name, round := range map[string]func() error{
|
||||
"a body that cannot be marshalled": func() error {
|
||||
_, err := sendRound(ctx, open, []string{"anchor"}, unmarshallable, fine, "")
|
||||
_, err := sendRound(ctx, open, []string{"anchor"}, unmarshallable, fine, "", nil)
|
||||
return err
|
||||
},
|
||||
"a send that fails": func() error {
|
||||
_, err := sendRound(ctx, open, []string{"anchor"}, plain, failing, "")
|
||||
_, err := sendRound(ctx, open, []string{"anchor"}, plain, failing, "", nil)
|
||||
return err
|
||||
},
|
||||
} {
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// The live case of 2026-10-07 (novox/hq issue 299), replayed: the node-uplink seat gained its first
|
||||
// verbs, `resolvers` and `links`, both optional while its holders catch up, and the holders serve
|
||||
// neither. The working set is read from the store, whose rows carry no optional mark, so the seat comes
|
||||
// back through UseSeats as it does live. Its holder on every machine answers nothing for it on the bus,
|
||||
// and must not be judged silent: a seat whose verbs are all optional asks no holder to answer.
|
||||
func TestAHolderOfASeatWhoseVerbsAreAllOptionalIsNotSilent(t *testing.T) {
|
||||
defer catalogue.UseSeats(catalogue.DefaultSeats())
|
||||
var rows []catalogue.Seat
|
||||
for _, s := range catalogue.DefaultSeats() {
|
||||
stored := s
|
||||
stored.Serves = nil
|
||||
for _, v := range s.Serves {
|
||||
v.Optional = false // the store never keeps the mark
|
||||
stored.Serves = append(stored.Serves, v)
|
||||
}
|
||||
rows = append(rows, stored)
|
||||
}
|
||||
catalogue.UseSeats(rows)
|
||||
|
||||
uplink, ok := catalogue.SeatNamed("node-uplink")
|
||||
if !ok || len(uplink.Serves) == 0 {
|
||||
t.Fatalf("node-uplink must serve verbs for this replay: %+v", uplink)
|
||||
}
|
||||
for _, v := range uplink.Serves {
|
||||
if !v.Optional {
|
||||
t.Fatalf("node-uplink's %s is required; this replay is of a seat whose verbs are all optional", v.Name)
|
||||
}
|
||||
}
|
||||
|
||||
nodes := []string{"anchor", "home-server", "workstation", "laptop"}
|
||||
heard := map[string]bool{}
|
||||
var recorded []catalogue.Held
|
||||
for _, n := range nodes {
|
||||
heard[n] = true
|
||||
recorded = append(recorded, catalogue.Held{Claim: "node-uplink", Scope: catalogue.ScopeNode, Node: n,
|
||||
Module: "networkmanager"})
|
||||
}
|
||||
expected := holdersToHear(catalogue.SeatsWithAProtocol(), recorded, nil, heard, nil, time.Now())
|
||||
if _, asked := expected["node-uplink"]; asked {
|
||||
t.Fatalf("node-uplink's holders are expected to answer though every verb of the seat is optional: %v",
|
||||
expected["node-uplink"])
|
||||
}
|
||||
// Nothing answers for the seat, as live: no silence is said about it.
|
||||
for _, o := range silentHolders(expected, map[string]map[string]bool{}) {
|
||||
if o.ID == "node-uplink."+o.Machine {
|
||||
t.Errorf("a holder serving none of a seat's optional verbs was judged silent: %s", o.Summary)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Silence is still judged on a required verb: a seat with one required and one optional verb, held on a
|
||||
// machine that is heard from, whose holder answers nothing — silent.
|
||||
func TestAHolderServingNoRequiredVerbIsStillSilent(t *testing.T) {
|
||||
seat := catalogue.Seat{Name: "example-seat", Scope: catalogue.ScopeNode, Serves: []catalogue.Verb{
|
||||
{Name: "state"}, {Name: "later", Optional: true}}}
|
||||
recorded := []catalogue.Held{{Claim: "example-seat", Scope: catalogue.ScopeNode, Node: "laptop", Module: "m"}}
|
||||
expected := holdersToHear([]catalogue.Seat{seat}, recorded, nil, map[string]bool{"laptop": true}, nil, time.Now())
|
||||
if !expected["example-seat"]["laptop"] {
|
||||
t.Fatalf("a holder of a seat with a required verb is not expected to answer: %v", expected)
|
||||
}
|
||||
out := silentHolders(expected, map[string]map[string]bool{})
|
||||
if len(out) != 1 || out[0].ID != "example-seat.laptop" || out[0].Token != "silent" {
|
||||
t.Fatalf("a holder serving no required verb is not said to be silent: %+v", out)
|
||||
}
|
||||
if got := silentHolders(expected, map[string]map[string]bool{"example-seat": {"laptop": true}}); len(got) != 0 {
|
||||
t.Fatalf("a holder that answers is said to be silent: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHoldingNeedsAnAnswer(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
serves []catalogue.Verb
|
||||
want bool
|
||||
}{
|
||||
{"no verb", nil, false},
|
||||
{"only optional verbs", []catalogue.Verb{{Name: "a", Optional: true}, {Name: "b", Optional: true}}, false},
|
||||
{"a required verb among optional ones", []catalogue.Verb{{Name: "a", Optional: true}, {Name: "b"}}, true},
|
||||
{"only required verbs", []catalogue.Verb{{Name: "a"}}, true},
|
||||
} {
|
||||
if got := holdingNeedsAnAnswer(catalogue.Seat{Name: "s", Serves: c.serves}); got != c.want {
|
||||
t.Errorf("%s: holdingNeedsAnAnswer = %v, want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The control-node withholds the login shell's `execute` (novox/hq ADR 0268): its holder there serves
|
||||
// nothing on the seat, and must not be judged silent for it, as the store's rows read it back.
|
||||
func TestALoginShellWithholdingExecuteIsNotSilent(t *testing.T) {
|
||||
defer catalogue.UseSeats(catalogue.DefaultSeats())
|
||||
var rows []catalogue.Seat
|
||||
for _, s := range catalogue.DefaultSeats() {
|
||||
stored := s
|
||||
stored.Serves = nil
|
||||
for _, v := range s.Serves {
|
||||
v.Optional = false // the store never keeps the mark
|
||||
stored.Serves = append(stored.Serves, v)
|
||||
}
|
||||
rows = append(rows, stored)
|
||||
}
|
||||
catalogue.UseSeats(rows)
|
||||
recorded := []catalogue.Held{{Claim: catalogue.LoginShellSeat, Scope: catalogue.ScopeNode, Node: "anchor", Module: "zsh"}}
|
||||
expected := holdersToHear(catalogue.SeatsWithAProtocol(), recorded, nil, map[string]bool{"anchor": true}, nil, time.Now())
|
||||
if _, asked := expected[catalogue.LoginShellSeat]; asked {
|
||||
t.Fatalf("the login shell's holder is expected to answer, so withholding execute would be said silent: %v",
|
||||
expected[catalogue.LoginShellSeat])
|
||||
}
|
||||
}
|
||||
@@ -96,7 +96,7 @@ func TestAnOverflowingConsumerNeverRefusesItsProvidersMachine(t *testing.T) {
|
||||
if len(declared.withheld) != 1 || declared.withheld[0].Identity != "mesh_laptop_photoalbum" {
|
||||
t.Fatalf("the overflowing consumer is not the one withheld: %+v", declared.withheld)
|
||||
}
|
||||
grants, _, err := grantsFor(ctx, open, "anchor")
|
||||
grants, _, _, err := grantsFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// novox/hq issue 348: on 2026-10-09 the control node's resolver stopped answering on its private address
|
||||
// at 10:57:57 UTC; the machine's network condition was raised at 10:58:45. The node-engine and the
|
||||
// controller were sent at 10:59:34. The new node-engine's first statement judged the names once — unknown,
|
||||
// "one look failed; a second decides" — and that statement cleared the condition, though its last evidence
|
||||
// still said "connection refused". The next look raised it again at 11:00:23, after the send, and both
|
||||
// builds failed their gate at 11:10 with "raised since it was sent" and were put back, for a fault that
|
||||
// began before they were sent.
|
||||
|
||||
// namesRefused is the control node's names part as its node-engine said it in the outage: its own
|
||||
// resolver, at its own address, refusing.
|
||||
func namesRefused(state string, streak int) link.NetworkPart {
|
||||
p := link.NetworkPart{Part: link.PartNames, State: state, Since: h0, Streak: streak}
|
||||
if state == link.StateUnhealthy {
|
||||
p.Reason = "1 of its 2 resolvers do not answer as the mesh's do"
|
||||
p.Said = "10.77.0.1 — anchor.internal (IPv4): read udp 10.77.0.1:35244->10.77.0.1:53: read: connection refused"
|
||||
p.Toward = []string{"10.77.0.1"}
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func networkSaying(parts ...link.NetworkPart) *link.NetworkHealth {
|
||||
state := link.StateHealthy
|
||||
for _, p := range parts {
|
||||
switch {
|
||||
case p.State == link.StateUnhealthy:
|
||||
state = link.StateUnhealthy
|
||||
case p.State == link.StateUnknown && state == link.StateHealthy:
|
||||
state = link.StateUnknown
|
||||
}
|
||||
}
|
||||
return &link.NetworkHealth{State: state, Since: h0, Parts: parts}
|
||||
}
|
||||
|
||||
// TestReplay348 replays the statements of the outage: the condition raised before the send is not
|
||||
// cleared by the restarted engine's first, undecided statement, and the gate does not count it against
|
||||
// the builds sent after it began.
|
||||
func TestReplay348(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv, k := open.inventory, conditionsFrom
|
||||
say := func(at time.Time, n *link.NetworkHealth) {
|
||||
t.Helper()
|
||||
if err := stateHealth(ctx, inv, k, "anchor", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
network := func() (conditions.Condition, bool) {
|
||||
t.Helper()
|
||||
list, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range list {
|
||||
if c.Key == "machine.anchor.network" {
|
||||
return c, true
|
||||
}
|
||||
}
|
||||
return conditions.Condition{}, false
|
||||
}
|
||||
|
||||
// 10:58:45 — the second failing look: raised.
|
||||
say(h0, networkSaying(namesRefused(link.StateUnhealthy, 2)))
|
||||
raised, ok := network()
|
||||
if !ok {
|
||||
t.Fatal("the resolver refusing on the control node raised nothing")
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
sent := time.Now().UTC()
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
|
||||
// 10:59:42 — the restarted engine's first statement: one look failed, a second decides.
|
||||
say(h0.Add(time.Minute), networkSaying(namesRefused(link.StateUnknown, 1)))
|
||||
if _, ok := network(); !ok {
|
||||
t.Fatal("a statement that judged nothing yet cleared the condition: the restarted engine's first look " +
|
||||
"said the fault was gone while it still refused")
|
||||
}
|
||||
// 11:00:23 — its second look: unhealthy again, the same raising.
|
||||
say(h0.Add(2*time.Minute), networkSaying(namesRefused(link.StateUnhealthy, 2)))
|
||||
again, ok := network()
|
||||
if !ok || !again.Raised.Equal(raised.Raised) || again.Count != 1 {
|
||||
t.Fatalf("the same raising was not kept: raised %s (first %s), count %d", again.Raised, raised.Raised, again.Count)
|
||||
}
|
||||
|
||||
// The gate on the control node, for a build sent after the fault began.
|
||||
open2, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := gateFacts{judged: true, open: open2}
|
||||
if w := aboutTheMachine("anchor", []string{"mesh-host"}, sent, f); w.whole != "" || len(w.on) != 0 {
|
||||
t.Fatalf("a fault from before the send held the build: %+v", w)
|
||||
}
|
||||
|
||||
// Decided healthy: cleared.
|
||||
say(h0.Add(3*time.Minute), networkSaying(link.NetworkPart{Part: link.PartNames, State: link.StateHealthy, Since: h0}))
|
||||
if c, ok := network(); ok {
|
||||
t.Fatalf("a statement that decides the names healthy left %s open", c.Key)
|
||||
}
|
||||
}
|
||||
|
||||
// A fault there at the send, cleared and reopened after it, is not raised since the send; one that cleared
|
||||
// before the send and came back after it is — a send that breaks a recovered machine fails its gate (review
|
||||
// of mesh-controller PR 179, A2). Read through OpenAt, by both of the gate's readings.
|
||||
func TestAFaultThatFlappedAfterTheSendIsNotTheSendsAndOneThatRecoveredBeforeItIs(t *testing.T) {
|
||||
since := h0
|
||||
network := func(first time.Time, gaps ...conditions.Gap) conditions.Condition {
|
||||
raised := since.Add(time.Minute)
|
||||
if len(gaps) > 0 {
|
||||
raised = gaps[len(gaps)-1].Reopened
|
||||
}
|
||||
return conditions.Condition{Key: "machine.anchor.network", Kind: kindMachineNetwork,
|
||||
Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor"},
|
||||
Summary: "anchor's network is not healthy", Source: sourceNetwork, First: first, Gaps: gaps, Raised: raised}
|
||||
}
|
||||
held := func(c conditions.Condition) bool {
|
||||
return aboutTheMachine("anchor", []string{"mesh-controller"}, since, gateFacts{judged: true,
|
||||
open: []conditions.Condition{c}}).whole != ""
|
||||
}
|
||||
// The day's case: raised before the send, cleared 8 s after it, reopened 49 s after it.
|
||||
flapped := network(since.Add(-49*time.Second),
|
||||
conditions.Gap{Cleared: since.Add(8 * time.Second), Reopened: since.Add(49 * time.Second)})
|
||||
if held(flapped) {
|
||||
t.Fatal("a fault there at the send, flapping after it, held the machine")
|
||||
}
|
||||
// Recovered before the send, broken again after it: the send's.
|
||||
recovered := network(since.Add(-time.Hour),
|
||||
conditions.Gap{Cleared: since.Add(-30 * time.Second), Reopened: since.Add(20 * time.Second)})
|
||||
if !held(recovered) {
|
||||
t.Fatal("a machine recovered at the send and broken after it passed the gate")
|
||||
}
|
||||
// An older gap, before the send, and the fault there at the send: not the send's.
|
||||
twice := network(since.Add(-time.Hour),
|
||||
conditions.Gap{Cleared: since.Add(-50 * time.Minute), Reopened: since.Add(-45 * time.Minute)},
|
||||
conditions.Gap{Cleared: since.Add(10 * time.Second), Reopened: since.Add(30 * time.Second)})
|
||||
if held(twice) {
|
||||
t.Fatal("a fault there at the send, with an older gap, held the machine")
|
||||
}
|
||||
// Raised after the send, never cleared: the send's.
|
||||
if !held(network(time.Time{})) {
|
||||
t.Fatal("a fault raised after the send held nothing")
|
||||
}
|
||||
// And a module's own, through judgeHealth.
|
||||
at := since.Add(2 * time.Minute)
|
||||
g := gateFacts{judged: true, now: at, reports: map[string]inventory.Reported{"anchor": {Node: "anchor",
|
||||
Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, engines: map[string]string{},
|
||||
served: map[string]served{}, rolledBack: map[string][]lease.Rollback{},
|
||||
open: []conditions.Condition{{Key: "provider.app.anchor.x.failing", Subject: conditions.Subject{
|
||||
Scope: conditions.ScopeProvider, ID: "app.anchor.x", Machine: "anchor"}, Summary: "failing",
|
||||
First: since.Add(-time.Hour), Raised: since.Add(time.Minute),
|
||||
Gaps: []conditions.Gap{{Cleared: since.Add(5 * time.Second), Reopened: since.Add(time.Minute)}}}}}
|
||||
if _, why := judgeHealth("app", "", catalogue.Manifest{Module: "app"}, "anchor", since, g); strings.HasPrefix(why, "raised since it was sent") {
|
||||
t.Fatalf("a module's own fault there at the send: %s", why)
|
||||
}
|
||||
g.open[0].Gaps[0].Cleared = since.Add(-5 * time.Second)
|
||||
if _, why := judgeHealth("app", "", catalogue.Manifest{Module: "app"}, "anchor", since, g); !strings.HasPrefix(why, "raised since it was sent") {
|
||||
t.Fatalf("a module's own fault, recovered at the send and back after it, was not counted: %s", why)
|
||||
}
|
||||
}
|
||||
|
||||
// Only an undecided part holds a condition that names it; a condition about another part clears, and a
|
||||
// statement unknown as a whole holds every part (review of PR 179, A4). Pure.
|
||||
func TestAnUndecidedPartHoldsOnlyWhatNamesIt(t *testing.T) {
|
||||
f := netFacts(map[string]*inventory.NetworkHealth{
|
||||
"anchor": aNetwork(link.StateUnknown, inventory.NetworkPart{Part: link.PartNames, State: link.StateUnknown, Streak: 1},
|
||||
inventory.NetworkPart{Part: link.PartRoute, State: link.StateHealthy}),
|
||||
"laptop": aNetwork(link.StateHealthy, inventory.NetworkPart{Part: link.PartNames, State: link.StateHealthy}),
|
||||
"spare": aNetwork(link.StateStarting, inventory.NetworkPart{Part: link.PartTunnel, State: link.StateHealthy}),
|
||||
"other": aNetwork(link.StateStarting, inventory.NetworkPart{Part: link.PartTunnel, State: link.StateStarting}),
|
||||
})
|
||||
u := undecidedParts(f)
|
||||
if !u["anchor"][link.PartNames] || u["anchor"][link.PartRoute] || u["laptop"] != nil || !u["spare"]["*"] ||
|
||||
!u["other"][link.PartTunnel] || u["other"]["*"] {
|
||||
t.Fatalf("undecided: %v", u)
|
||||
}
|
||||
about := func(machine, said string, also ...string) conditions.Condition {
|
||||
return conditions.Condition{Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: machine,
|
||||
Machine: machine, Also: also}, Evidence: []conditions.Evidence{{Said: said}}}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
c conditions.Condition
|
||||
held bool
|
||||
}{
|
||||
{about("anchor", "names since 2026-10-09 10:58:45 UTC: 10.77.0.1 — refused"), true},
|
||||
{about("anchor", "route since 2026-10-09 10:58:45 UTC: no default route"), false},
|
||||
{about("laptop", "names since 2026-10-09 10:58:45 UTC: refused"), false},
|
||||
{about("spare", "route since …: no default route"), true},
|
||||
{about("hub", "anchor: names: refused", "anchor"), true},
|
||||
{about("hub", "anchor: tunnel: no handshake", "anchor"), false},
|
||||
} {
|
||||
if got := heldUndecided(c.c, u); got != c.held {
|
||||
t.Errorf("%s %q held %v, want %v", c.c.Subject.Machine, c.c.Evidence[0].Said, got, c.held)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A release walks its modules without a record per module: D10 counts what its tier names as rolling,
|
||||
// so the node-engine a release walks is not "behind, and no plan is rolling it out" on its first machine.
|
||||
func TestAReleaseRollsOutWhatItsTierNames(t *testing.T) {
|
||||
plans := []inventory.Plan{
|
||||
{ID: "release-1", State: inventory.PlanRolling, Tiers: [][]string{{"mesh-host"}}, Modules: map[string]*inventory.PlanModule{}},
|
||||
{ID: "plan-2", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{"letta": {}}},
|
||||
}
|
||||
got := rollingModules(plans)
|
||||
if !got["mesh-host"] || !got["letta"] || len(got) != 2 {
|
||||
t.Fatalf("rolling: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,356 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
)
|
||||
|
||||
// novox/hq issue 352: on 2026-10-09 a release's gate on the control node read the machine's report against a
|
||||
// newer send another plan had just made there — not against its own send — and failed three builds the
|
||||
// machine had reported healthy ("has not reported on what it was sent"), put them back on every machine,
|
||||
// to a controller older than the store's schema, and that controller then judged the newer plan's
|
||||
// controller passed from the put-back build's health.
|
||||
|
||||
// TestReplay352 replays the walk on the backlog fixture: the release sends anchor and anchor reports;
|
||||
// another send reaches anchor, unreported; the gate still passes. And a send that moves a judged module
|
||||
// to another build supersedes the judging: no verdict, nothing put back.
|
||||
func TestReplay352(t *testing.T) {
|
||||
t.Run("a newer send to the judged machine does not unreport the gate's", testANewerSendDoesNotUnreportTheGatesOwn)
|
||||
t.Run("a send that moves the module supersedes the judging", testASendThatMovesTheModuleSupersedesTheJudging)
|
||||
}
|
||||
|
||||
func testANewerSendDoesNotUnreportTheGatesOwn(t *testing.T) {
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
advancePlans(ctx, b.open) // anchor is sent, and the fixture reports it applied
|
||||
// 16:31:37 — another plan sends anchor a newer declaration, which it has not reported on.
|
||||
carried, _, _ := inv.SentBuilds(ctx, "anchor")
|
||||
if err := inv.RecordSent(ctx, nodeID(t, b.open, "anchor"), "d-anchor-newer", carried); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reports, _ := inv.LastReports(ctx)
|
||||
for _, r := range reports {
|
||||
if r.Node == "anchor" && r.Current {
|
||||
t.Fatal("the fixture's newer send reads as reported")
|
||||
}
|
||||
}
|
||||
gateEvery, gateBound = 0, 0 // past the bound at once: before the fix, "has not reported" fails it here
|
||||
for i := 0; i < 4; i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
p := b.release(t)
|
||||
if p.State == inventory.PlanFailed || strings.Contains(p.Note, "has not reported") {
|
||||
t.Fatalf("the release failed on the newer send: %s %s", p.State, p.Note)
|
||||
}
|
||||
if g := p.Release.Gate; g != nil && (g.Sent == nil || g.Sent["anchor"].Digest == "") {
|
||||
t.Fatalf("the gate does not keep what it sent: %+v", g)
|
||||
}
|
||||
if v, found, err := inv.GateOf(ctx, "build-app-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("app's pass on anchor was not kept: %+v %v %v", v, found, err)
|
||||
}
|
||||
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" {
|
||||
t.Fatalf("app was put back to %s", current["app"].Commit)
|
||||
}
|
||||
}
|
||||
|
||||
// A plan's own first send waits while a release judges the same module on that machine with another build.
|
||||
func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) {
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
advancePlans(ctx, b.open) // the release judges app c2 on anchor
|
||||
_, _, err := gatedSend(ctx, b.open, "anchor", []inventory.CarriedMove{{Module: "app", Node: "anchor", From: "c2", To: "c3", Build: "build-app-c3"}})
|
||||
if !errors.Is(err, errWalkedElsewhere) || !strings.Contains(err.Error(), "release-") {
|
||||
t.Fatalf("a newer build of a judged module was sent under the release's gate: %v", err)
|
||||
}
|
||||
if len(b.sent) != 1 {
|
||||
t.Fatalf("sent %v", b.sent)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge plan's judging is superseded the same way: another send moved its module on the first machine.
|
||||
func TestAPlansJudgingIsSupersededByASendThatMovesItsModule(t *testing.T) {
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := g.open.inventory
|
||||
advancePlans(ctx, g.open) // anchor is sent app c2 first
|
||||
if err := inv.RecordSent(ctx, nodeID(t, g.open, "anchor"), "d-anchor-c3", map[string]string{"app": "c3"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gateEvery = 0
|
||||
advancePlans(ctx, g.open)
|
||||
p := g.plan(t)
|
||||
if p.State != inventory.PlanSuperseded || !strings.Contains(p.Note, "superseded") || !strings.Contains(p.Note, "c3") {
|
||||
t.Fatalf("the plan is %s: %s", p.State, p.Note)
|
||||
}
|
||||
// Nothing put back: the registered build stands, the build is not marked, and the plan's gate made no
|
||||
// rollback (a release may walk what the other send left waiting on anchor; that is not a put-back).
|
||||
if r := p.Modules["app"].Gate.Rollback; r != "" {
|
||||
t.Fatalf("a superseded judging made a rollback: %q", r)
|
||||
}
|
||||
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" {
|
||||
t.Fatalf("app was put back to %s", current["app"].Commit)
|
||||
}
|
||||
if failed, _ := inv.GateFailed(ctx, "build-2"); failed {
|
||||
t.Fatal("a superseded build was marked failed")
|
||||
}
|
||||
}
|
||||
|
||||
func testASendThatMovesTheModuleSupersedesTheJudging(t *testing.T) {
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
advancePlans(ctx, b.open)
|
||||
// Another send moves app on anchor to a build this gate does not judge.
|
||||
carried, _, _ := inv.SentBuilds(ctx, "anchor")
|
||||
carried["app"] = "c3"
|
||||
if err := inv.RecordSent(ctx, nodeID(t, b.open, "anchor"), "d-anchor-c3", carried); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gateEvery = 0
|
||||
advancePlans(ctx, b.open)
|
||||
p := b.release(t)
|
||||
if p.State != inventory.PlanSuperseded || !strings.Contains(p.Note, "superseded") || !strings.Contains(p.Note, "c3") {
|
||||
t.Fatalf("the release is %s: %s", p.State, p.Note)
|
||||
}
|
||||
if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}}) {
|
||||
t.Fatalf("sent %v: a superseded judging puts nothing back", b.sent)
|
||||
}
|
||||
if _, found, _ := inv.GateOf(ctx, "build-app-c2"); found {
|
||||
t.Fatal("a superseded judging kept a verdict")
|
||||
}
|
||||
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" {
|
||||
t.Fatalf("app was put back to %s", current["app"].Commit)
|
||||
}
|
||||
}
|
||||
|
||||
// A report is on the gate's own send: the declaration itself, or one sequenced after it; a gate kept
|
||||
// without its send reads the report against the send made last, as before. Pure.
|
||||
func TestAReportIsHeldAgainstTheGatesOwnSend(t *testing.T) {
|
||||
sent := inventory.SentDeclaration{Digest: "d-490", Sequence: 490}
|
||||
for _, c := range []struct {
|
||||
r inventory.Reported
|
||||
want bool
|
||||
}{
|
||||
{inventory.Reported{Declared: "d-490", Current: false}, true},
|
||||
{inventory.Reported{Declared: "d-491", ReportedSequence: 491, Current: true}, true},
|
||||
{inventory.Reported{Declared: "d-489", ReportedSequence: 489, Current: false}, false},
|
||||
{inventory.Reported{Declared: "other", ReportedSequence: 490}, true}, // the same sequence, said by another digest
|
||||
{inventory.Reported{Declared: "d-495", ReportedSequence: 495, Current: true}, true}, // the last send: this one or a later one
|
||||
{inventory.Reported{Declared: "", Current: false}, false},
|
||||
} {
|
||||
if got := sent.ReportsOn(c.r); got != c.want {
|
||||
t.Errorf("%+v on %+v: %v", c.r, sent, got)
|
||||
}
|
||||
}
|
||||
byDigest := inventory.SentDeclaration{Digest: "d-1"}
|
||||
if !byDigest.ReportsOn(inventory.Reported{Declared: "d-1"}) || byDigest.ReportsOn(inventory.Reported{ReportedSequence: 5}) ||
|
||||
!byDigest.ReportsOn(inventory.Reported{Current: true}) {
|
||||
t.Fatal("a send kept without a sequence is matched by its digest and by the last send alone")
|
||||
}
|
||||
f := gateFacts{sent: map[string]inventory.SentDeclaration{"anchor": sent}}
|
||||
if !f.reportedOn("anchor", inventory.Reported{Declared: "d-490"}) || f.reportedOn("anchor", inventory.Reported{Declared: "d-1"}) {
|
||||
t.Fatal("a gate that kept its send read the report against something other than it")
|
||||
}
|
||||
if !f.reportedOn("laptop", inventory.Reported{Current: true}) || f.reportedOn("laptop", inventory.Reported{Current: false}) {
|
||||
t.Fatal("a gate that did not keep its send does not read the report against the send made last")
|
||||
}
|
||||
// Through the merge plan's first-machine wait too.
|
||||
at := time.Now()
|
||||
state := inventory.PlanModule{First: []string{"anchor"}, FirstAt: &at,
|
||||
Gate: &inventory.PlanGate{Machines: []string{"anchor"}, Sent: map[string]inventory.SentDeclaration{"anchor": sent}}}
|
||||
reports := []inventory.Reported{{Node: "anchor", At: &at, Outcome: inventory.OutcomeApplied, Current: false, Declared: "d-490"}}
|
||||
if step := nextRollout(state, []string{"anchor", "laptop"}, false, reports, at.Add(time.Minute), time.Hour); step.waiting != "" || step.failed != "" {
|
||||
t.Fatalf("the first machine's report on the plan's own send read as none: %+v", step)
|
||||
}
|
||||
reports[0].Declared = "d-480"
|
||||
if step := nextRollout(state, []string{"anchor", "laptop"}, false, reports, at.Add(time.Minute), time.Hour); step.waiting == "" {
|
||||
t.Fatalf("a report on an older send read as the plan's: %+v", step)
|
||||
}
|
||||
}
|
||||
|
||||
// A gate judges only the build the machine was last sent: last sent another build of the module, the
|
||||
// judging is superseded, whatever the machine reports. Pure.
|
||||
func TestAGateJudgesOnlyTheBuildTheMachineWasLastSent(t *testing.T) {
|
||||
at := time.Now()
|
||||
f := gateFacts{now: at, reports: map[string]inventory.Reported{"anchor": {Node: "anchor", Outcome: inventory.OutcomeApplied,
|
||||
At: &at, Current: true}}, engines: map[string]string{}, served: map[string]served{}, rolledBack: map[string][]lease.Rollback{},
|
||||
commits: map[string]string{"mesh-controller": "e6b00e2e"}, sentBuilds: map[string]map[string]string{"anchor": {"mesh-controller": "ef26d4cb"}}}
|
||||
taken := at.Add(-30 * time.Second)
|
||||
f.holder = &lease.Holder{Taken: taken, Health: &lease.Health{Ready: true}}
|
||||
h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "anchor", at.Add(-time.Minute), f)
|
||||
if h != healthSuperseded || !strings.Contains(why, "ef26d4cb") || !strings.Contains(why, "e6b00e2e") {
|
||||
t.Fatalf("a controller build the machine no longer runs: %v %q", h, why)
|
||||
}
|
||||
f.sentBuilds["anchor"]["mesh-controller"] = "e6b00e2e"
|
||||
if h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "anchor", at.Add(-time.Minute), f); h == healthSuperseded {
|
||||
t.Fatalf("the build sent read as another: %q", why)
|
||||
}
|
||||
delete(f.sentBuilds, "anchor")
|
||||
if h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "anchor", at.Add(-time.Minute), f); h == healthSuperseded {
|
||||
t.Fatalf("a machine whose send is not known read as superseded: %q", why)
|
||||
}
|
||||
g := &inventory.PlanGate{To: "c2", Carried: []inventory.CarriedMove{{Module: "late", Node: "anchor", To: "c5"}}}
|
||||
if got := judgedCommits(g, []judged{{"app", "anchor"}, {"late", "anchor"}}); got["app"] != "c2" || got["late"] != "c5" {
|
||||
t.Fatalf("judged commits %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A move of another build of a module to a machine where a release or a plan is judging that module
|
||||
// waits for that judging; the same build to that machine is already there. Pure.
|
||||
func TestAWalkWaitsForAJudgingOfTheSameModuleOnThatMachine(t *testing.T) {
|
||||
at := time.Now()
|
||||
release := inventory.Plan{ID: "release-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{
|
||||
Gate: &inventory.PlanGate{Machines: []string{"novox"}, Carried: []inventory.CarriedMove{
|
||||
{Module: "mesh-controller", Node: "novox", From: "ef26d4cb", To: "2913c54c"}}}}}
|
||||
merge := inventory.Plan{ID: "plan-1", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
|
||||
"app": {First: []string{"anchor"}, FirstAt: &at, Commit: "c2", Gate: &inventory.PlanGate{Machines: []string{"anchor"}}}}}
|
||||
f := moveFacts{plans: []inventory.Plan{release, merge}}
|
||||
for _, c := range []struct {
|
||||
module, node, to, want string
|
||||
}{
|
||||
{"mesh-controller", "novox", "e6b00e2e", "release-1"}, // the day's case: a newer controller to the judged machine
|
||||
{"mesh-controller", "novox", "2913c54c", ""}, // the same build: already there
|
||||
{"mesh-controller", "ace", "2913c54c", "release-1"}, // another machine while the first is judged
|
||||
{"mesh-host", "novox", "x", ""}, // a module the release does not carry
|
||||
{"app", "anchor", "c2", ""},
|
||||
{"app", "anchor", "c3", "plan-1"},
|
||||
{"app", "laptop", "c2", "plan-1"},
|
||||
} {
|
||||
if got := f.walkedBy(c.module, c.node, c.to); got != c.want {
|
||||
t.Errorf("%s %s to %s: walked by %q, want %q", c.module, c.to, c.node, got, c.want)
|
||||
}
|
||||
}
|
||||
release.Release.Gate.Verdict = inventory.GatePassed
|
||||
merge.Modules["app"].Gate.Verdict = inventory.GatePassed
|
||||
if f.walkedBy("mesh-controller", "novox", "e6b00e2e") != "" || f.walkedBy("app", "laptop", "c3") != "" {
|
||||
t.Fatal("a passed judging still holds a move")
|
||||
}
|
||||
release.Release.Gate.Verdict = ""
|
||||
f.plans[0].State = inventory.PlanSuperseded
|
||||
if f.walkedBy("mesh-controller", "novox", "e6b00e2e") != "" {
|
||||
t.Fatal("a closed release still holds a move")
|
||||
}
|
||||
}
|
||||
|
||||
// The controller is never put back to a build that reaches less of the store's schema than the store
|
||||
// has, or to one that never said what it reaches: the current build is kept, and the condition says so.
|
||||
func TestTheControllerIsNotPutBackToABuildOlderThanTheStoresSchema(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
keeper, _ := withConditionsInMemory(t)
|
||||
told := &conditions.Told{}
|
||||
was := doctorFrom
|
||||
doctorFrom = &doctor{open: open, keeper: keeper, teller: told}
|
||||
t.Cleanup(func() { doctorFrom = was })
|
||||
wasSend := sendRollout
|
||||
var sent [][]string
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
sent = append(sent, names)
|
||||
return names, nil
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = wasSend })
|
||||
|
||||
build := func(id, commit, digest string, asked time.Time) inventory.Build {
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: commit})
|
||||
b := inventory.Build{ID: id, Module: "mesh-controller", Commit: commit, Repository: "novox/mesh-controller", Path: ".",
|
||||
Manifest: manifest, Asked: asked, At: asked, Made: []inventory.Artifact{{Name: "controller", Kind: catalogue.ArtifactBundle,
|
||||
Reference: "mesh-artifact://mesh-controller/controller/blobs/sha256:" + digest}}}
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: commit},
|
||||
inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Path: ".", BuiltFrom: commit, Head: commit, Asked: asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
previous := build("build-old", "ef26d4cb", strings.Repeat("1", 64), time.Now().Add(-2*time.Hour))
|
||||
failed := build("build-new", "e6b00e2e", strings.Repeat("2", 64), time.Now().Add(-time.Minute))
|
||||
if versionOfBuild(previous) != strings.Repeat("1", 12) {
|
||||
t.Fatalf("the build's version is %q", versionOfBuild(previous))
|
||||
}
|
||||
applied, err := inv.SchemaApplied(ctx)
|
||||
if err != nil || applied < 87 {
|
||||
t.Fatalf("the store's schema reaches %d (%v)", applied, err)
|
||||
}
|
||||
// The build before never recorded what it reads: not proved, refused.
|
||||
if why, ok := controllerSchemaAllows(ctx, inv, previous); ok || !strings.Contains(why, "never recorded") {
|
||||
t.Fatalf("an unknown reach: %v %q", ok, why)
|
||||
}
|
||||
// It reads less than the store has: refused, naming both.
|
||||
if err := inv.RecordSchemaReach(ctx, versionOfBuild(previous), applied-1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if why, ok := controllerSchemaAllows(ctx, inv, previous); ok || !strings.Contains(why, "is at") {
|
||||
t.Fatalf("a reach behind the store: %v %q", ok, why)
|
||||
}
|
||||
// Through the gate: the failed build is marked, nothing is put back, nothing is sent, the condition is urgent.
|
||||
at := time.Now().Add(-5 * time.Minute)
|
||||
state := &inventory.PlanModule{Build: failed.ID, Commit: failed.Commit, Previous: previous.Commit, First: []string{"anchor"}, FirstAt: &at}
|
||||
p := inventory.Plan{ID: "plan-352", Repository: "novox/mesh-controller", Branch: "main", Commit: failed.Commit, Created: at,
|
||||
State: inventory.PlanRolling, Tiers: [][]string{{"mesh-controller"}}, Modules: map[string]*inventory.PlanModule{"mesh-controller": state}}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gateFailed(ctx, open, &p, "mesh-controller", state, []string{"anchor"}, "not healthy within 10m0s of its apply")
|
||||
if state.Gate.Rollback != inventory.NotRolledBack || !strings.Contains(p.Note, "NOT put back") || !strings.Contains(p.Note, "the current build is kept") {
|
||||
t.Fatalf("rollback %q: %s", state.Gate.Rollback, p.Note)
|
||||
}
|
||||
if len(sent) != 0 {
|
||||
t.Fatalf("sent %v: nothing is put back", sent)
|
||||
}
|
||||
if current, _ := inv.CurrentBuilds(ctx); current["mesh-controller"].Commit != failed.Commit {
|
||||
t.Fatalf("the module was put back to %s", current["mesh-controller"].Commit)
|
||||
}
|
||||
if marked, _ := inv.GateFailed(ctx, failed.ID); !marked {
|
||||
t.Fatal("the failed build is not marked failed at its gate")
|
||||
}
|
||||
open2, _ := keeper.Open(ctx)
|
||||
var found bool
|
||||
for _, c := range open2 {
|
||||
if c.Kind == kindRollbackFailed && c.Severity == conditions.Urgent && strings.Contains(c.Summary, "current build is kept") {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("no urgent rollback-failed condition saying the current build is kept: %+v", open2)
|
||||
}
|
||||
// Reaching the store: allowed.
|
||||
if err := inv.RecordSchemaReach(ctx, versionOfBuild(previous), applied); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if why, ok := controllerSchemaAllows(ctx, inv, previous); !ok {
|
||||
t.Fatalf("a build that reads the whole schema was refused: %q", why)
|
||||
}
|
||||
// A build with no bundle to know it by: refused.
|
||||
if why, ok := controllerSchemaAllows(ctx, inv, inventory.Build{Commit: "x"}); ok || !strings.Contains(why, "names no bundle") {
|
||||
t.Fatalf("a build without a bundle: %v %q", ok, why)
|
||||
}
|
||||
}
|
||||
|
||||
// The lease's holder names the build the declaration told it it is, and the version stamp only without one.
|
||||
func TestTheHolderNamesTheBuildTheDeclarationToldIt(t *testing.T) {
|
||||
t.Setenv(RunningBuildVar, " ad62528c47c7 ")
|
||||
if h := holderOf("x"); h.Build != "ad62528c47c7" {
|
||||
t.Fatalf("the holder's build is %q", h.Build)
|
||||
}
|
||||
t.Setenv(RunningBuildVar, "")
|
||||
if h := holderOf("x"); h.Build != version {
|
||||
t.Fatalf("without a declared version the holder's build is %q", h.Build)
|
||||
}
|
||||
if reach, err := schemaReach(); err != nil || reach < 87 {
|
||||
t.Fatalf("this build's reach is %d (%v)", reach, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// aMachineJoining is a machine with a live token issued for a tunnel key, given its address — what
|
||||
// `token issue --overlay-key` records before it pushes the hub (novox/hq ADR 0169).
|
||||
func aMachineJoining(t *testing.T, ctx context.Context, inv *inventory.Inventory, name string,
|
||||
validFor time.Duration) string {
|
||||
t.Helper()
|
||||
record, err := inv.AddNode(ctx, name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.IssueToken(ctx, name, validFor); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := aPublicKey(t)
|
||||
if err := inv.RecordOverlayKey(ctx, record.ID, key); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.BindTokenToKey(ctx, record.ID, key); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cidr, err := overlayRange(ctx, inv)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.AssignAddress(ctx, record.ID, cidr); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
// hubPeers are the keys the hub's composed tunnel carries.
|
||||
func hubPeers(t *testing.T, ctx context.Context, inv *inventory.Inventory) map[string]bool {
|
||||
t.Helper()
|
||||
g, err := network(ctx, inv, map[string]bool{"anchor": true, "laptop": true}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := map[string]bool{}
|
||||
for _, p := range g.Graph()["anchor"] {
|
||||
out[p.Key] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// **A machine joining is a peer of the hub while its token can be used, and not after** (novox/hq
|
||||
// ADR 0169): the hub's composed tunnel carries a machine whose token was issued for its key, so the
|
||||
// tunnel answers the first time it knocks; a token that expired unused takes the peer with it at the
|
||||
// hub's next composition.
|
||||
func TestAMachineJoiningIsAPeerOfTheHubUntilItsTokenExpires(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
|
||||
joining := aMachineJoining(t, ctx, inv, "joiner", time.Hour)
|
||||
expired := aMachineJoining(t, ctx, inv, "late", 2*time.Second)
|
||||
time.Sleep(2100 * time.Millisecond)
|
||||
|
||||
peers := hubPeers(t, ctx, inv)
|
||||
if !peers[joining] {
|
||||
t.Fatalf("the hub does not carry the machine its live token was issued for: %v", peers)
|
||||
}
|
||||
if peers[expired] {
|
||||
t.Fatalf("the hub still carries a machine whose token expired unused: %v", peers)
|
||||
}
|
||||
|
||||
// A token issued without a key gives the hub nothing to carry: there is no key to carry.
|
||||
if _, err := inv.AddNode(ctx, "keyless"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.IssueToken(ctx, "keyless", time.Hour); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if after := hubPeers(t, ctx, inv); len(after) != len(peers) {
|
||||
t.Fatalf("a token issued without a key changed the hub's peers: %v, was %v", after, peers)
|
||||
}
|
||||
}
|
||||
|
||||
// **A tunnel key that is not one is refused before anything is recorded** (novox/hq ADR 0169): a
|
||||
// token issued for it would be a tunnel the hub could never answer.
|
||||
func TestATokenIsNotIssuedForSomethingThatIsNotATunnelKey(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
record, err := open.inventory.AddNode(ctx, "joiner")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := throughTheTunnel(ctx, open, record, "not-a-key", "10.77.0.1:4222"); err == nil {
|
||||
t.Fatal("a token was issued for something that is not a tunnel key")
|
||||
}
|
||||
if held := placementOf(t, ctx, open.inventory, "joiner"); held.Key != "" || held.Address != "" {
|
||||
t.Fatalf("a refused key left a record behind: %+v", held)
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,6 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -14,6 +13,8 @@ import (
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// Two controllers at once (novox/hq to-be 45 §6, issue 204; the half of replay R1 that lives here): two
|
||||
@@ -22,15 +23,12 @@ import (
|
||||
// one that lost it acts no more — no declaration composed, no plan and no condition written — the
|
||||
// moment it lost it.
|
||||
//
|
||||
// MESH_TEST_POSTGRES=… MESH_TEST_NATS=nats://127.0.0.1:14222 go test ./cmd/mesh-controller/ -run Controllers
|
||||
// MESH_TEST_POSTGRES=… go test ./cmd/mesh-controller/ -run Controllers (each test on a bus of its own: internal/testbus)
|
||||
|
||||
// aBusForTheLease is the test bus with the controller's lease bucket new.
|
||||
func aBusForTheLease(t *testing.T) (string, jetstream.KeyValue) {
|
||||
t.Helper()
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -232,7 +232,7 @@ func licenceKey(ctx context.Context, args []string) error {
|
||||
// and printing the value here would put the one copy that matters on a terminal.
|
||||
fmt.Printf("sealed to %d holder(s). The mesh has discarded the key and cannot read it back\n",
|
||||
sealed)
|
||||
fmt.Printf(" run `push` to deliver it\n")
|
||||
fmt.Printf(" run `push --behind` to deliver it\n")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -340,7 +340,7 @@ func licenceSetGrant(ctx context.Context, args []string) error {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s now holds a refresh token for %s, sealed to that node's key and readable by it "+
|
||||
"alone.\n the control plane stored the box without opening it; run `push` to deliver it\n",
|
||||
"alone.\n the control plane stored the box without opening it; run `push --behind` to deliver it\n",
|
||||
"the manager", name)
|
||||
return nil
|
||||
}
|
||||
@@ -423,7 +423,7 @@ func licenceSubmitRefresh(ctx context.Context, args []string) error {
|
||||
"manager node alone"
|
||||
}
|
||||
fmt.Printf("submitted a refresh for %s: a new access token sealed to %d holder(s), and %s.\n"+
|
||||
" run `push` to deliver it\n", name, sealed, rotatedNote)
|
||||
" run `push --behind` to deliver it\n", name, sealed, rotatedNote)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -456,7 +456,7 @@ func licenceRefresh(ctx context.Context, args []string) error {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("refreshed %s: a new access token sealed to %d holder(s), and the refresh token left "+
|
||||
"with its manager.\n run `push` to deliver it\n", name, sealed)
|
||||
"with its manager.\n run `push --behind` to deliver it\n", name, sealed)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,413 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A machine says how its network is (novox/hq ADR 0241, which extends ADR 0240 from what a module runs
|
||||
// to the machine it runs on).
|
||||
//
|
||||
// **Every machine's node-engine judges its own networking** — the resolver file the uplink holder
|
||||
// declared, the names through every resolver it lists, the tunnel's handshake with the hub, the bus, the
|
||||
// default route — on the two-look rule, and states it beside its resources. The controller keeps the
|
||||
// newest statement per machine and raises from all of them together:
|
||||
//
|
||||
// - **an outside writer of the resolver file is its own finding**, `machine.<m>.<owner>.rewritten`:
|
||||
// the file the uplink holder declares was rewritten by another program, named where the engine could
|
||||
// name it. The names failing through what that program wrote are that finding's consequence, said in
|
||||
// it — never a second condition;
|
||||
// - **what is the machine's own** — its route, its tunnel, its resolvers answering wrong, a resolver
|
||||
// that is no mesh machine — is `machine.<m>.network`;
|
||||
// - **what points at another machine is said once, there** (the provider hold of ADR 0240 rule 5, for
|
||||
// the network): a failure toward the hub or toward a mesh resolver is held under that machine when it
|
||||
// is down on the record — silent, or its own network unhealthy — or when a second machine finds the
|
||||
// same; then `machine.<x>.unreachable` names every machine that cannot reach it, and none of them
|
||||
// raises anything of its own for it. One machine alone failing toward a healthy one is its own.
|
||||
//
|
||||
// Each is a warning; urgent on the control node, or when the bus cannot be reached, or for the hub.
|
||||
// Cleared on the first statement that no longer says it. An engine older than this judging says nothing
|
||||
// of its network, and nothing is raised for it.
|
||||
|
||||
// The conditions a machine's network raises.
|
||||
const (
|
||||
kindMachineNetwork = "machine-network"
|
||||
kindNetworkRewritten = "network-rewritten"
|
||||
kindNetworkUnreachable = "network-unreachable"
|
||||
sourceNetwork = "network"
|
||||
)
|
||||
|
||||
// networkKinds are the kinds this judging owns: every open one it no longer says, it clears.
|
||||
var networkKinds = []string{kindMachineNetwork, kindNetworkRewritten, kindNetworkUnreachable}
|
||||
|
||||
// networkFacts is what one judging of every machine's network reads.
|
||||
type networkFacts struct {
|
||||
healths map[string]inventory.NodeHealth
|
||||
// byAddress is each machine's address on the private network; hub the hub's name; control the
|
||||
// control node's.
|
||||
byAddress map[string]string
|
||||
hub string
|
||||
control string
|
||||
// silent is every machine whose silence is an open condition.
|
||||
silent map[string]bool
|
||||
}
|
||||
|
||||
// judgeNetworks raises and clears every machine's network conditions from every machine's newest
|
||||
// statement, after one machine's statement was kept.
|
||||
func judgeNetworks(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, now time.Time) error {
|
||||
healths, err := inv.Healths(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
overlays, err := inv.Overlays(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
open, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f := networkFacts{healths: healths, byAddress: map[string]string{}, control: controlHost(ctx, inv),
|
||||
silent: map[string]bool{}}
|
||||
for _, o := range overlays {
|
||||
if o.Address != "" {
|
||||
f.byAddress[o.Address] = o.Name
|
||||
}
|
||||
if o.Hub {
|
||||
f.hub = o.Name
|
||||
}
|
||||
}
|
||||
for _, c := range open {
|
||||
if c.Subject.Scope == conditions.ScopeMachine && c.Kind == "silent" {
|
||||
f.silent[c.Subject.ID] = true
|
||||
}
|
||||
}
|
||||
var problems []string
|
||||
said := map[string]bool{}
|
||||
for _, o := range networkObservations(f) {
|
||||
said[o.Key()] = true
|
||||
if _, err := k.Observe(ctx, o); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
}
|
||||
undecided := undecidedParts(f)
|
||||
for _, c := range open {
|
||||
if !slices.Contains(networkKinds, c.Kind) || said[c.Key] || heldUndecided(c, undecided) {
|
||||
continue
|
||||
}
|
||||
why := "no machine says it any more"
|
||||
if c.Subject.Machine != "" {
|
||||
why = c.Subject.Machine + "'s network no longer says it"
|
||||
}
|
||||
if _, err := k.Clear(ctx, c.Key, why); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
return fmt.Errorf("%s", strings.Join(problems, "; "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// undecidedParts is, per machine, every part of its newest statement not yet judged: starting, or unknown
|
||||
// — one look failed and a second decides (novox/hq issue 348). Such a part does not say its fault is gone.
|
||||
// A statement whose parts are all decided but whose whole is unknown or starting holds every part. Pure.
|
||||
//
|
||||
// On 2026-10-09 the control node's resolver refused every question from 10:58 to 11:18 UTC. A build of
|
||||
// the node-engine sent at 10:59:34 restarted it; its first statement judged the names once (unknown, "one
|
||||
// look failed; a second decides"), and that statement cleared the control node's network condition while
|
||||
// its last evidence still said "connection refused". The second look raised it again forty seconds later —
|
||||
// after the send — and the gate failed the build for a fault from before it.
|
||||
func undecidedParts(f networkFacts) map[string]map[string]bool {
|
||||
out := map[string]map[string]bool{}
|
||||
for m, h := range f.healths {
|
||||
if h.Network == nil {
|
||||
continue
|
||||
}
|
||||
parts := map[string]bool{}
|
||||
for _, p := range h.Network.Parts {
|
||||
if p.State == link.StateUnknown || p.State == link.StateStarting {
|
||||
parts[p.Part] = true
|
||||
}
|
||||
}
|
||||
if len(parts) == 0 && (h.Network.State == link.StateUnknown || h.Network.State == link.StateStarting) {
|
||||
parts["*"] = true
|
||||
}
|
||||
if len(parts) > 0 {
|
||||
out[m] = parts
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// heldUndecided says an open network condition is kept rather than cleared: a part its newest evidence
|
||||
// names is undecided in the newest statement of a machine it is about. A condition about other parts
|
||||
// clears as before. Pure.
|
||||
func heldUndecided(c conditions.Condition, undecided map[string]map[string]bool) bool {
|
||||
said := ""
|
||||
if len(c.Evidence) > 0 {
|
||||
said = c.Evidence[0].Said
|
||||
}
|
||||
for _, m := range append([]string{c.Subject.Machine}, c.Subject.Also...) {
|
||||
parts := undecided[m]
|
||||
if parts["*"] {
|
||||
return true
|
||||
}
|
||||
for part := range parts {
|
||||
if strings.Contains(said, part+" since ") || strings.Contains(said, part+": ") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// pointed is one machine's failing part that points at another machine.
|
||||
type pointed struct {
|
||||
from string
|
||||
part inventory.NetworkPart
|
||||
}
|
||||
|
||||
// networkObservations is every network condition the statements say now. Pure.
|
||||
func networkObservations(f networkFacts) []conditions.Observation {
|
||||
machines := make([]string, 0, len(f.healths))
|
||||
for m := range f.healths {
|
||||
machines = append(machines, m)
|
||||
}
|
||||
sort.Strings(machines)
|
||||
|
||||
unhealthy := func(m string) []inventory.NetworkPart {
|
||||
h := f.healths[m]
|
||||
if h.Network == nil {
|
||||
return nil
|
||||
}
|
||||
var out []inventory.NetworkPart
|
||||
for _, p := range h.Network.Parts {
|
||||
if p.State == link.StateUnhealthy {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
// The machines a part points at, other than its own: the hub, and each mesh resolver by its address.
|
||||
// An address that is no mesh machine's — the resolver a VPN client wrote in — is the machine's own.
|
||||
targets := func(m string, p inventory.NetworkPart) ([]string, bool) {
|
||||
if len(p.Toward) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
var out []string
|
||||
for _, t := range p.Toward {
|
||||
x := f.byAddress[t]
|
||||
if t == link.TowardHub {
|
||||
x = f.hub
|
||||
}
|
||||
if x == "" || x == m {
|
||||
return nil, false
|
||||
}
|
||||
if !slices.Contains(out, x) {
|
||||
out = append(out, x)
|
||||
}
|
||||
}
|
||||
return out, true
|
||||
}
|
||||
|
||||
// First pass: what points at whom.
|
||||
pointing := map[string][]pointed{}
|
||||
for _, m := range machines {
|
||||
for _, p := range unhealthy(m) {
|
||||
if xs, ok := targets(m, p); ok {
|
||||
for _, x := range xs {
|
||||
pointing[x] = append(pointing[x], pointed{from: m, part: p})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
from := func(x string) []string {
|
||||
var out []string
|
||||
for _, pt := range pointing[x] {
|
||||
if !slices.Contains(out, pt.from) {
|
||||
out = append(out, pt.from)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
// A machine is down on the record when its silence is open or its own network is unhealthy, or when
|
||||
// two machines find it unreachable: then what points at it is held there.
|
||||
down := func(x string) bool {
|
||||
return f.silent[x] || len(unhealthy(x)) > 0 || len(from(x)) >= 2
|
||||
}
|
||||
|
||||
var out []conditions.Observation
|
||||
saysOwn := map[string]bool{}
|
||||
for _, m := range machines {
|
||||
parts := unhealthy(m)
|
||||
if len(parts) == 0 {
|
||||
continue
|
||||
}
|
||||
var own []inventory.NetworkPart
|
||||
var rewritten *inventory.NetworkPart
|
||||
for i, p := range parts {
|
||||
if p.Part == link.PartResolvConf {
|
||||
rewritten = &parts[i]
|
||||
continue
|
||||
}
|
||||
if xs, ok := targets(m, p); ok && allDown(xs, down) {
|
||||
continue // held at the machines it points at
|
||||
}
|
||||
own = append(own, p)
|
||||
}
|
||||
if rewritten != nil {
|
||||
out = append(out, rewrittenObservation(m, *rewritten, parts, f))
|
||||
// The names failing through what another program wrote are that finding's, said in it.
|
||||
kept := own[:0]
|
||||
for _, p := range own {
|
||||
if p.Part != link.PartNames {
|
||||
kept = append(kept, p)
|
||||
}
|
||||
}
|
||||
own = kept
|
||||
}
|
||||
if len(own) > 0 {
|
||||
out = append(out, machineNetworkObservation(m, own, f, from(m)))
|
||||
saysOwn[m] = true
|
||||
}
|
||||
}
|
||||
// Said once, at the machine everybody points at — unless its own network condition already says it
|
||||
// (listed there), or its silence does.
|
||||
targetsSorted := make([]string, 0, len(pointing))
|
||||
for x := range pointing {
|
||||
targetsSorted = append(targetsSorted, x)
|
||||
}
|
||||
sort.Strings(targetsSorted)
|
||||
for _, x := range targetsSorted {
|
||||
if !down(x) || saysOwn[x] || f.silent[x] {
|
||||
continue
|
||||
}
|
||||
out = append(out, unreachableObservation(x, pointing[x], from(x), f))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func allDown(xs []string, down func(string) bool) bool {
|
||||
for _, x := range xs {
|
||||
if !down(x) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(xs) > 0
|
||||
}
|
||||
|
||||
// rewrittenObservation is the resolver file rewritten by another program: its own finding, naming the
|
||||
// writer where the engine could, and what it costs the machine.
|
||||
func rewrittenObservation(m string, p inventory.NetworkPart, all []inventory.NetworkPart, f networkFacts) conditions.Observation {
|
||||
writer := ""
|
||||
if p.Writer != "" {
|
||||
writer = " (" + p.Writer + ")"
|
||||
}
|
||||
cost := "the names through it are not yet judged"
|
||||
said := []string{p.Part + ": " + p.Said}
|
||||
for _, q := range all {
|
||||
if q.Part == link.PartNames {
|
||||
cost = strings.TrimSuffix(q.Reason, ".")
|
||||
said = append(said, q.Part+": "+q.Said)
|
||||
}
|
||||
}
|
||||
if cost == "the names through it are not yet judged" {
|
||||
cost = "the names still resolve through what it wrote"
|
||||
}
|
||||
id := m
|
||||
if p.Owner != "" {
|
||||
// Named by the module whose file it is: a send that moved that module is what the gate
|
||||
// holds it on (issue 281's rule — what names a moved module is that module's).
|
||||
id = m + "." + p.Owner
|
||||
}
|
||||
severity := conditions.Warning
|
||||
if m == f.control {
|
||||
severity = conditions.Urgent
|
||||
}
|
||||
summary := fmt.Sprintf("the resolver file on %s was rewritten by another program%s — %s until the "+
|
||||
"node-engine writes it back at its next reconcile, or that program gives it back", m, writer, cost)
|
||||
if p.Owner != "" {
|
||||
summary = fmt.Sprintf("the resolver file %s writes on %s was rewritten by another program%s — %s until "+
|
||||
"the node-engine writes it back at its next reconcile, or that program gives it back", p.Owner, m, writer, cost)
|
||||
}
|
||||
return conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: "rewritten", Kind: kindNetworkRewritten,
|
||||
Machine: m, Severity: severity, Source: sourceNetwork, Summary: summary,
|
||||
Said: fmt.Sprintf("since %s: %s", p.Since.UTC().Format("2006-01-02 15:04:05 MST"), strings.Join(said, " | "))}
|
||||
}
|
||||
|
||||
// machineNetworkObservation is what is wrong with a machine's own networking.
|
||||
func machineNetworkObservation(m string, parts []inventory.NetworkPart, f networkFacts, waiting []string) conditions.Observation {
|
||||
var words, said []string
|
||||
severity := conditions.Warning
|
||||
for _, p := range parts {
|
||||
words = append(words, p.Reason)
|
||||
said = append(said, fmt.Sprintf("%s since %s: %s", p.Part, p.Since.UTC().Format("2006-01-02 15:04:05 MST"), p.Said))
|
||||
if p.Part == link.PartBus {
|
||||
severity = conditions.Urgent
|
||||
}
|
||||
}
|
||||
if m == f.control || m == f.hub {
|
||||
severity = conditions.Urgent
|
||||
}
|
||||
summary := fmt.Sprintf("%s's network is not healthy: %s", m, strings.Join(words, "; "))
|
||||
if len(waiting) > 0 {
|
||||
severity = conditions.Urgent
|
||||
summary += fmt.Sprintf("; %s cannot reach it", strings.Join(waiting, ", "))
|
||||
}
|
||||
return conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Token: "network", Kind: kindMachineNetwork,
|
||||
Machine: m, Severity: severity, Source: sourceNetwork, Summary: summary, Said: strings.Join(said, " | ")}
|
||||
}
|
||||
|
||||
// unreachableObservation is one machine others cannot reach, said once there.
|
||||
func unreachableObservation(x string, pts []pointed, from []string, f networkFacts) conditions.Observation {
|
||||
var what []string
|
||||
var said []string
|
||||
for _, pt := range pts {
|
||||
w := map[string]string{link.PartTunnel: "the tunnel to it", link.PartBus: "the bus on it",
|
||||
link.PartNames: "its resolver"}[pt.part.Part]
|
||||
if w == "" {
|
||||
w = pt.part.Part
|
||||
}
|
||||
if !slices.Contains(what, w) {
|
||||
what = append(what, w)
|
||||
}
|
||||
said = append(said, fmt.Sprintf("%s: %s: %s", pt.from, pt.part.Part, pt.part.Said))
|
||||
}
|
||||
severity := conditions.Warning
|
||||
if x == f.hub || x == f.control || slices.Contains(what, "the bus on it") {
|
||||
severity = conditions.Urgent
|
||||
}
|
||||
return conditions.Observation{Scope: conditions.ScopeMachine, ID: x, Token: "unreachable", Kind: kindNetworkUnreachable,
|
||||
Machine: x, Also: from, Severity: severity, Source: sourceNetwork,
|
||||
Summary: fmt.Sprintf("%s cannot be reached from %s: %s", x, strings.Join(from, ", "), strings.Join(what, ", ")),
|
||||
Said: strings.Join(said, " | ")}
|
||||
}
|
||||
|
||||
// networkLines is what `node show` says of a machine's networking.
|
||||
func networkLines(h inventory.NodeHealth, had bool, now time.Time) []string {
|
||||
if !had || h.Network == nil {
|
||||
return []string{" its node-engine does not say how its network is — it is older than that judging (ADR 0241)"}
|
||||
}
|
||||
out := []string{fmt.Sprintf(" its network: %s since %s", h.Network.State, h.Network.Since.Local().Format("2006-01-02 15:04"))}
|
||||
for _, p := range h.Network.Parts {
|
||||
line := fmt.Sprintf(" %-10s %s", p.State, p.Part)
|
||||
if p.Reason != "" {
|
||||
line += " — " + p.Reason
|
||||
}
|
||||
if p.Writer != "" {
|
||||
line += " (" + p.Writer + ")"
|
||||
}
|
||||
out = append(out, line)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,306 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A machine says how its network is (novox/hq ADR 0241, "how it is checked"): the resolver file rewritten
|
||||
// by another program is one finding, naming the writer, with the names it costs said in it; what is the
|
||||
// machine's own is `machine.<m>.network`; what points at another machine that is down is said once, there;
|
||||
// one machine alone failing toward a healthy one is its own; the control node, the hub and the bus are
|
||||
// urgent; an engine that says nothing of its network raises nothing; and the gate waits on what is shown to
|
||||
// be another's.
|
||||
|
||||
func aNetwork(state string, parts ...inventory.NetworkPart) *inventory.NetworkHealth {
|
||||
for i := range parts {
|
||||
if parts[i].State == "" {
|
||||
parts[i].State = link.StateUnhealthy
|
||||
}
|
||||
parts[i].Since = h0
|
||||
}
|
||||
return &inventory.NetworkHealth{State: state, Since: h0, Parts: parts}
|
||||
}
|
||||
|
||||
func netFacts(healths map[string]*inventory.NetworkHealth) networkFacts {
|
||||
f := networkFacts{healths: map[string]inventory.NodeHealth{}, hub: "anchor", control: "anchor",
|
||||
byAddress: map[string]string{"10.77.0.1": "anchor", "10.77.0.2": "laptop", "10.77.0.3": "spare"},
|
||||
silent: map[string]bool{}}
|
||||
for m, n := range healths {
|
||||
f.healths[m] = inventory.NodeHealth{Node: m, Network: n}
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
var (
|
||||
rewrittenByVPN = inventory.NetworkPart{Part: link.PartResolvConf, Reason: "the resolver file was rewritten by another program",
|
||||
Said: "/etc/resolv.conf lists 172.16.5.5 where 10.77.0.1 is declared", Writer: "FortiClient", Owner: "networkmanager"}
|
||||
namesThroughVPN = inventory.NetworkPart{Part: link.PartNames, Reason: "mesh names do not resolve",
|
||||
Said: "172.16.5.5 — anchor.internal (IPv4): says no such name", Toward: []string{"172.16.5.5"}}
|
||||
tunnelDown = inventory.NetworkPart{Part: link.PartTunnel, Reason: "the tunnel to the hub has not handshaken for over five minutes",
|
||||
Said: "mesh0's newest handshake with the hub was 9m0s ago", Toward: []string{link.TowardHub}}
|
||||
noRoute = inventory.NetworkPart{Part: link.PartRoute, Reason: "the machine has no default route", Said: "no default route"}
|
||||
)
|
||||
|
||||
func keysOf(obs []conditions.Observation) []string {
|
||||
var keys []string
|
||||
for _, o := range obs {
|
||||
keys = append(keys, o.Key())
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
func TestAResolverFileRewrittenIsOneFindingNamingItsWriterAndWhatItCosts(t *testing.T) {
|
||||
obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
|
||||
"anchor": aNetwork(link.StateHealthy),
|
||||
"laptop": aNetwork(link.StateUnhealthy, rewrittenByVPN, namesThroughVPN),
|
||||
}))
|
||||
if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" {
|
||||
t.Fatalf("want one finding, the rewrite, got %v", keys)
|
||||
}
|
||||
o := obs[0]
|
||||
for _, want := range []string{"laptop", "rewritten by another program (FortiClient)", "mesh names do not resolve",
|
||||
"next reconcile"} {
|
||||
if !strings.Contains(o.Summary, want) {
|
||||
t.Errorf("the summary does not say %q: %s", want, o.Summary)
|
||||
}
|
||||
}
|
||||
if strings.Contains(o.Summary, "172.16.") || strings.Contains(o.Summary, "/etc/") {
|
||||
t.Errorf("an address or a path reached the summary: %s", o.Summary)
|
||||
}
|
||||
if !strings.Contains(o.Said, "172.16.5.5") || o.Severity != conditions.Warning || o.Machine != "laptop" {
|
||||
t.Errorf("the evidence or the severity is wrong: %+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOneMachineFailingTowardAHealthyHubIsItsOwn(t *testing.T) {
|
||||
obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
|
||||
"anchor": aNetwork(link.StateHealthy),
|
||||
"laptop": aNetwork(link.StateUnhealthy, tunnelDown),
|
||||
"spare": aNetwork(link.StateHealthy),
|
||||
}))
|
||||
if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" {
|
||||
t.Fatalf("want the laptop's own, got %v", keys)
|
||||
}
|
||||
if obs[0].Severity != conditions.Warning {
|
||||
t.Fatalf("a laptop's own tunnel is a warning, got %s", obs[0].Severity)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTwoMachinesThatCannotReachTheHubAreSaidOnceAtTheHub(t *testing.T) {
|
||||
obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
|
||||
"anchor": aNetwork(link.StateHealthy),
|
||||
"laptop": aNetwork(link.StateUnhealthy, tunnelDown),
|
||||
"spare": aNetwork(link.StateUnhealthy, tunnelDown),
|
||||
}))
|
||||
if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.unreachable" {
|
||||
t.Fatalf("want one condition at the hub, got %v", keys)
|
||||
}
|
||||
o := obs[0]
|
||||
if o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop, spare") || len(o.Also) != 2 {
|
||||
t.Fatalf("the hub's condition is %+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhatPointsAtASilentHubIsHeldUnderItsSilence(t *testing.T) {
|
||||
f := netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, tunnelDown)})
|
||||
f.silent["anchor"] = true
|
||||
if obs := networkObservations(f); len(obs) != 0 {
|
||||
t.Fatalf("the hub's silence says it; got %v", keysOf(obs))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAHubWhoseOwnNetworkIsUnhealthyListsWhoCannotReachIt(t *testing.T) {
|
||||
obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
|
||||
"anchor": aNetwork(link.StateUnhealthy, noRoute),
|
||||
"laptop": aNetwork(link.StateUnhealthy, tunnelDown),
|
||||
}))
|
||||
if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.network" {
|
||||
t.Fatalf("want the hub's own, holding the laptop's, got %v", keys)
|
||||
}
|
||||
if o := obs[0]; o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop cannot reach it") {
|
||||
t.Fatalf("the hub's condition is %+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOneMachineFailingAHealthyMeshResolverIsItsOwnAndTwoAreTheResolvers(t *testing.T) {
|
||||
silentResolver := inventory.NetworkPart{Part: link.PartNames, Reason: "1 of its 2 resolvers do not answer as the mesh's do",
|
||||
Said: "10.77.0.3 — no answer within 1s", Toward: []string{"10.77.0.3"}}
|
||||
obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
|
||||
"anchor": aNetwork(link.StateHealthy), "spare": aNetwork(link.StateHealthy),
|
||||
"laptop": aNetwork(link.StateUnhealthy, silentResolver),
|
||||
}))
|
||||
if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" {
|
||||
t.Fatalf("one machine alone: want its own, got %v", keys)
|
||||
}
|
||||
obs = networkObservations(netFacts(map[string]*inventory.NetworkHealth{
|
||||
"anchor": aNetwork(link.StateUnhealthy, silentResolver), "spare": aNetwork(link.StateHealthy),
|
||||
"laptop": aNetwork(link.StateUnhealthy, silentResolver),
|
||||
}))
|
||||
if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.spare.unreachable" {
|
||||
t.Fatalf("two machines: want it said once at the resolver's machine, got %v", keys)
|
||||
}
|
||||
if !strings.Contains(obs[0].Summary, "its resolver") {
|
||||
t.Fatalf("the resolver's machine is said %s", obs[0].Summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheControlNodeAndTheBusAreUrgent(t *testing.T) {
|
||||
f := netFacts(map[string]*inventory.NetworkHealth{"anchor": aNetwork(link.StateUnhealthy, rewrittenByVPN)})
|
||||
if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent {
|
||||
t.Fatalf("the control node's rewritten file: %+v", obs)
|
||||
}
|
||||
bus := inventory.NetworkPart{Part: link.PartBus, Reason: "the bus cannot be reached", Said: "no link"}
|
||||
f = netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, bus, noRoute)})
|
||||
if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent {
|
||||
t.Fatalf("the bus unreachable from the laptop: %+v", obs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnEngineThatSaysNothingOfItsNetworkRaisesNothing(t *testing.T) {
|
||||
f := netFacts(map[string]*inventory.NetworkHealth{"laptop": nil, "anchor": aNetwork(link.StateHealthy)})
|
||||
if obs := networkObservations(f); len(obs) != 0 {
|
||||
t.Fatalf("got %v", keysOf(obs))
|
||||
}
|
||||
}
|
||||
|
||||
// Through the store and the keeper: the statement kept, the rewrite raised from it, cleared when the file
|
||||
// is written back, and node show saying it.
|
||||
func TestARewrittenResolverFileIsRaisedFromTheStatementAndClearedWhenWrittenBack(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv, k := open.inventory, conditionsFrom
|
||||
say := func(at time.Time, n *link.NetworkHealth) {
|
||||
t.Helper()
|
||||
if err := stateHealth(ctx, inv, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
openKeys := func() []string {
|
||||
t.Helper()
|
||||
list, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keys []string
|
||||
for _, c := range list {
|
||||
keys = append(keys, c.Key)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
healthy := &link.NetworkHealth{State: link.StateHealthy, Since: h0, Parts: []link.NetworkPart{
|
||||
{Part: link.PartResolvConf, State: link.StateHealthy, Since: h0}}}
|
||||
rewritten := &link.NetworkHealth{State: link.StateUnhealthy, Since: h0.Add(time.Minute), Parts: []link.NetworkPart{
|
||||
{Part: link.PartResolvConf, State: link.StateUnhealthy, Reason: rewrittenByVPN.Reason, Said: rewrittenByVPN.Said,
|
||||
Writer: "FortiClient", Owner: "networkmanager", Since: h0.Add(time.Minute)},
|
||||
{Part: link.PartNames, State: link.StateUnhealthy, Reason: namesThroughVPN.Reason, Said: namesThroughVPN.Said,
|
||||
Toward: namesThroughVPN.Toward, Since: h0.Add(time.Minute)}}}
|
||||
|
||||
say(h0, healthy)
|
||||
if keys := openKeys(); len(keys) != 0 {
|
||||
t.Fatalf("a healthy network raised %v", keys)
|
||||
}
|
||||
say(h0.Add(time.Minute), rewritten)
|
||||
if keys := openKeys(); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" {
|
||||
t.Fatalf("the rewrite raised %v", keys)
|
||||
}
|
||||
kept, had, err := inv.HealthOf(ctx, "laptop")
|
||||
if err != nil || !had || kept.Network == nil || kept.Network.Parts[0].Writer != "FortiClient" {
|
||||
t.Fatalf("the statement's network was not kept: %+v %v", kept.Network, err)
|
||||
}
|
||||
if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "FortiClient") ||
|
||||
!strings.Contains(lines, "unhealthy resolv-conf") {
|
||||
t.Fatalf("node show says:\n%s", lines)
|
||||
}
|
||||
say(h0.Add(2*time.Minute), healthy)
|
||||
if keys := openKeys(); len(keys) != 0 {
|
||||
t.Fatalf("written back, still open: %v", keys)
|
||||
}
|
||||
// An engine older than the judging says no network: nothing raised, and node show says it is not known.
|
||||
say(h0.Add(3*time.Minute), nil)
|
||||
kept, had, _ = inv.HealthOf(ctx, "laptop")
|
||||
if keys := openKeys(); len(keys) != 0 || kept.Network != nil {
|
||||
t.Fatalf("an older engine: %v %+v", keys, kept.Network)
|
||||
}
|
||||
if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "older than that judging") {
|
||||
t.Fatalf("node show says:\n%s", lines)
|
||||
}
|
||||
}
|
||||
|
||||
// The gate: a resolver file another program rewrote waits the judging rather than failing it at the
|
||||
// bound; the same, when the send moved the module whose file it is, is that module's; a machine's own
|
||||
// network fault holds the machine as a whole, as before.
|
||||
func TestTheGateWaitsOnARewriteItDidNotMakeAndHoldsTheOwnerOnOneItMoved(t *testing.T) {
|
||||
since := h0
|
||||
rewrite := conditions.Condition{Key: "machine.laptop.networkmanager.rewritten", Kind: kindNetworkRewritten,
|
||||
Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop.networkmanager", Machine: "laptop"},
|
||||
Summary: "the resolver file was rewritten", Raised: since.Add(time.Minute), Source: sourceNetwork}
|
||||
f := gateFacts{judged: true, open: []conditions.Condition{rewrite}}
|
||||
if w := aboutTheMachine("laptop", []string{"letta"}, since, f); w.waiting == "" || w.whole != "" || len(w.on) != 0 {
|
||||
t.Fatalf("a rewrite the send did not make: %+v", w)
|
||||
}
|
||||
if w := aboutTheMachine("laptop", []string{"networkmanager", "letta"}, since, f); w.on["networkmanager"] == "" ||
|
||||
w.on["letta"] != "" || w.waiting != "" {
|
||||
t.Fatalf("a rewrite of the file a moved module owns: %+v", w)
|
||||
}
|
||||
own := conditions.Condition{Key: "machine.laptop.network", Kind: kindMachineNetwork,
|
||||
Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop", Machine: "laptop"},
|
||||
Summary: "laptop's network is not healthy", Raised: since.Add(time.Minute), Source: sourceNetwork}
|
||||
if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true,
|
||||
open: []conditions.Condition{own}}); w.whole == "" || w.waiting != "" {
|
||||
t.Fatalf("the machine's own network: %+v", w)
|
||||
}
|
||||
unreachable := conditions.Condition{Key: "machine.anchor.unreachable", Kind: kindNetworkUnreachable,
|
||||
Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor", Also: []string{"laptop", "spare"}},
|
||||
Summary: "anchor cannot be reached", Raised: since.Add(time.Minute), Source: sourceNetwork}
|
||||
if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true,
|
||||
open: []conditions.Condition{unreachable}}); w.waiting == "" || w.whole != "" {
|
||||
t.Fatalf("a machine that cannot reach the hub: %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTheDrillsStatementsRaiseAndClearTheRewrite replays the drill of ADR 0241 (mesh-host
|
||||
// internal/network TestDrill…): what a node-engine said in a throwaway container while its resolver file
|
||||
// was declared, rewritten as a VPN client rewrites it, and written back — recorded, with the mesh's names
|
||||
// and addresses replaced by this test mesh's. Healthy raises nothing; the rewrite, on its second look, is
|
||||
// raised as one finding naming the writer; written back, it clears.
|
||||
func TestTheDrillsStatementsRaiseAndClearTheRewrite(t *testing.T) {
|
||||
raw, err := os.ReadFile("testdata/network-drill.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var said []link.Health
|
||||
if err := json.Unmarshal(raw, &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
k := conditionsFrom
|
||||
var raisedAt []int
|
||||
for i, h := range said {
|
||||
if err := stateHealth(ctx, open.inventory, k, "laptop", h, h.At); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
list, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range list {
|
||||
if c.Key != "machine.laptop.networkmanager.rewritten" || !strings.Contains(c.Summary, "(FortiClient)") {
|
||||
t.Fatalf("statement %d raised %s: %s", i, c.Key, c.Summary)
|
||||
}
|
||||
raisedAt = append(raisedAt, i)
|
||||
}
|
||||
}
|
||||
// Five statements: healthy, healthy, one failing look (still healthy), unhealthy, written back.
|
||||
if len(raisedAt) != 1 || raisedAt[0] != 3 {
|
||||
t.Fatalf("the rewrite was open after statements %v; want after the fourth alone", raisedAt)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A failed unit is never silent (novox/hq issue 315).
|
||||
//
|
||||
// **A module's failed unit was never raised, and neither was the machine's.** Liveness judges what a
|
||||
// module runs long-lived; a module whose daemon is a package's unit, started by D-Bus activation, states
|
||||
// no service, and its unit failed at every start while the machine read healthy. The profile's
|
||||
// `service-manager` said `degraded`, and nothing raised that.
|
||||
//
|
||||
// The node-engine now reads every failed unit in the managers the mesh places units in, and says whose
|
||||
// each is:
|
||||
//
|
||||
// - **a module's** — the declaration states it, writes its file, or installs the package its file
|
||||
// belongs to — is among the module's resources, unhealthy, of kind `unit`: the module's own
|
||||
// `module.<module>.<machine>.unhealthy`, naming the unit, raised on the second statement and held by
|
||||
// the gate as any unhealthy resource is (ADR 0240 §4);
|
||||
// - **the machine's** — no module places it: a mount of the machine's own table, a unit a removed
|
||||
// package left behind — is one finding for the machine, `machine.<m>.units`, listing them, so a
|
||||
// degraded service manager is never silent. A warning; cleared on the first statement that lists none.
|
||||
//
|
||||
// The engine applies the two-look rule itself (ADR 0241 §2), so the machine's finding is raised on the
|
||||
// statement that first says it. It is nothing a send did — no module places what it lists — so the gate
|
||||
// never holds a send on it. An engine older than this reading says nothing of its units, and nothing is
|
||||
// raised or cleared for it.
|
||||
|
||||
// The condition a machine's own failed units raise.
|
||||
const (
|
||||
kindMachineUnits = "machine-units"
|
||||
sourceUnits = "units"
|
||||
)
|
||||
|
||||
// unitsKept is a statement's units as the inventory keeps them.
|
||||
func unitsKept(u *link.UnitsHealth) *inventory.UnitsHealth {
|
||||
if u == nil {
|
||||
return nil
|
||||
}
|
||||
out := &inventory.UnitsHealth{State: u.State, Failed: []inventory.FailedUnit{}, Unread: u.Unread}
|
||||
for _, f := range u.Failed {
|
||||
out.Failed = append(out.Failed, inventory.FailedUnit{Unit: f.Unit, Scope: f.Scope, Load: f.Load, Result: f.Result,
|
||||
Resource: f.Resource, Since: f.Since})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// judgeUnits raises the machine's own failed units as one finding, or clears it when the statement lists
|
||||
// none. A statement that says nothing of units — an older engine — leaves it as it is.
|
||||
func judgeUnits(ctx context.Context, k *conditions.Keeper, node string, u *link.UnitsHealth) error {
|
||||
if k == nil || u == nil {
|
||||
return nil
|
||||
}
|
||||
o, raise := machineUnitsObservation(node, u)
|
||||
if raise {
|
||||
_, err := k.Observe(ctx, o)
|
||||
return err
|
||||
}
|
||||
open, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, c := range open {
|
||||
if c.Kind == kindMachineUnits && c.Key == o.Key() {
|
||||
why := fmt.Sprintf("%s's service managers list no failed unit the mesh does not place", node)
|
||||
if u.State == link.UnitsRunning {
|
||||
why = fmt.Sprintf("%s's service managers are running, no unit failed", node)
|
||||
}
|
||||
_, err := k.Clear(ctx, c.Key, why)
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// machineUnitsObservation is the machine's own failed units in one finding, and whether there are any.
|
||||
// The summary names each unit and its manager, which is what a person looks for; how each failed and
|
||||
// since when is evidence. Pure.
|
||||
func machineUnitsObservation(node string, u *link.UnitsHealth) (conditions.Observation, bool) {
|
||||
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Token: "units", Kind: kindMachineUnits,
|
||||
Machine: node, Severity: conditions.Warning, Source: sourceUnits}
|
||||
if len(u.Failed) == 0 {
|
||||
return o, false
|
||||
}
|
||||
var names, said, plain []string
|
||||
for _, f := range u.Failed {
|
||||
plain = append(plain, unitPlainWords(f.Unit))
|
||||
where := ""
|
||||
if f.Scope == "user" {
|
||||
where = " (the account's own manager)"
|
||||
}
|
||||
names = append(names, f.Unit+where)
|
||||
line := fmt.Sprintf("%s in the %s manager: %s", f.Unit, orSystem(f.Scope), orNotSaid(f.Result))
|
||||
if f.Load != "" && f.Load != "loaded" {
|
||||
line += ", its unit " + f.Load
|
||||
}
|
||||
if f.Resource != "" {
|
||||
line += ", named by the mesh's own " + f.Resource
|
||||
}
|
||||
line += ", since " + f.Since.UTC().Format("2006-01-02 15:04:05 MST")
|
||||
said = append(said, line)
|
||||
}
|
||||
o.Summary = fmt.Sprintf("%s's service manager is degraded: %d failed unit(s) no module places — %s. "+
|
||||
"Each is the machine's own: mend or remove it there, or have a module place it",
|
||||
node, len(u.Failed), strings.Join(names, ", "))
|
||||
o.Said = strings.Join(said, "; ")
|
||||
failed := "1 failed service"
|
||||
if len(u.Failed) > 1 {
|
||||
failed = fmt.Sprintf("%d failed services", len(u.Failed))
|
||||
}
|
||||
o.Headline = fmt.Sprintf("%s on %s", conditions.Capital(failed), node)
|
||||
o.Needs = fmt.Sprintf("mend or remove them on %s, or silence this if they do not matter.", node)
|
||||
o.Explanation = fmt.Sprintf("On %s, %s failed. No module manages them, so the mesh does not repair them.",
|
||||
node, namesWords(plain, 3))
|
||||
o.Actions = []conditions.Action{conditions.SilenceAction(o.Key())}
|
||||
o.Resolved = "No failed services on " + node + " any more"
|
||||
return o, true
|
||||
}
|
||||
|
||||
func orSystem(scope string) string {
|
||||
if scope == "" {
|
||||
return "system"
|
||||
}
|
||||
return scope
|
||||
}
|
||||
|
||||
// unitsLines is what `node show` says of a machine's service managers.
|
||||
func unitsLines(h inventory.NodeHealth, had bool, _ time.Time) []string {
|
||||
if !had || h.Units == nil {
|
||||
return []string{" its node-engine does not say which units failed — it is older than that reading (issue 315)"}
|
||||
}
|
||||
out := []string{" its service managers: " + h.Units.State}
|
||||
for _, f := range h.Units.Failed {
|
||||
line := fmt.Sprintf(" failed %s (%s) — no module places it", f.Unit, orSystem(f.Scope))
|
||||
if f.Result != "" {
|
||||
line += ", " + f.Result
|
||||
}
|
||||
out = append(out, line)
|
||||
}
|
||||
for _, r := range h.Resources {
|
||||
if r.Kind == link.KindUnit {
|
||||
out = append(out, fmt.Sprintf(" failed %s — %s's, %s", r.Target, r.Module, r.Reason))
|
||||
}
|
||||
}
|
||||
for _, w := range h.Units.Unread {
|
||||
out = append(out, " unread "+w)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A failed unit is never silent (novox/hq issue 315, "how it is checked"): a module's failed unit raises
|
||||
// the module's condition on the second statement, naming the unit; the machine's own failed units raise
|
||||
// one finding for the machine, listing them, cleared when none is listed; an engine that says nothing of
|
||||
// its units raises and clears nothing; and the gate never holds a send on the machine's own.
|
||||
|
||||
// desktop is the workstation as found on 2026-10-08, with the mesh's names: the Razer daemon's packaged
|
||||
// user unit, the module's; two mounts of the machine's own table and a unit a removed package left
|
||||
// behind, the machine's.
|
||||
func desktop(at time.Time) link.Health {
|
||||
return link.Health{Contract: link.ReadinessContract, At: at,
|
||||
Resources: []link.ResourceHealth{{Module: "openrazer", Resource: "openrazer.daemon", Kind: link.KindUnit,
|
||||
Target: "openrazer-daemon.service", State: link.StateUnhealthy,
|
||||
Reason: "failed in the account's own service manager (exit-code)", Since: h0, Streak: 2}},
|
||||
Units: &link.UnitsHealth{State: link.UnitsDegraded, Failed: []link.FailedUnit{
|
||||
{Unit: "mnt-recalbox.mount", Scope: "system", Load: "loaded", Result: "exit-code", Since: h0},
|
||||
{Unit: "storage-media.mount", Scope: "system", Load: "loaded", Result: "timeout", Since: h0},
|
||||
{Unit: "greenclip.service", Scope: "user", Load: "not-found", Result: "start-limit-hit", Since: h0},
|
||||
}}}
|
||||
}
|
||||
|
||||
func TestAFailedUnitIsTheModulesConditionAndTheMachinesOwnAreOneFinding(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv, k := open.inventory, conditionsFrom
|
||||
openConditions := func() map[string]conditions.Condition {
|
||||
t.Helper()
|
||||
list, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := map[string]conditions.Condition{}
|
||||
for _, c := range list {
|
||||
out[c.Key] = c
|
||||
}
|
||||
return out
|
||||
}
|
||||
say := func(at time.Time, h link.Health) {
|
||||
t.Helper()
|
||||
if err := stateHealth(ctx, inv, k, "laptop", h, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
say(h0, desktop(h0))
|
||||
got := openConditions()
|
||||
machine, raised := got["machine.laptop.units"]
|
||||
if !raised || len(got) != 1 {
|
||||
t.Fatalf("the machine's own failed units are one finding at once (the engine looked twice): %v", got)
|
||||
}
|
||||
for _, unit := range []string{"mnt-recalbox.mount", "storage-media.mount", "greenclip.service"} {
|
||||
if !strings.Contains(machine.Summary, unit) {
|
||||
t.Errorf("the finding names %s: %s", unit, machine.Summary)
|
||||
}
|
||||
}
|
||||
if strings.Contains(machine.Summary, "openrazer") || machine.Severity != conditions.Warning {
|
||||
t.Errorf("the module's unit is the module's, and the machine's a warning: %+v", machine)
|
||||
}
|
||||
|
||||
say(h0.Add(time.Minute), desktop(h0.Add(time.Minute)))
|
||||
got = openConditions()
|
||||
module, raised := got["module.openrazer.laptop.unhealthy"]
|
||||
if !raised || !strings.Contains(module.Summary, "openrazer-daemon.service") ||
|
||||
!strings.Contains(module.Summary, "account's own service manager") {
|
||||
t.Fatalf("the module's failed unit raises its condition on the second statement, naming the unit: %+v", got)
|
||||
}
|
||||
|
||||
kept, had, err := inv.HealthOf(ctx, "laptop")
|
||||
if err != nil || !had || kept.Units == nil || len(kept.Units.Failed) != 3 {
|
||||
t.Fatalf("the statement's units were not kept: %+v %v", kept.Units, err)
|
||||
}
|
||||
lines := strings.Join(unitsLines(kept, had, h0), "\n")
|
||||
for _, want := range []string{"degraded", "storage-media.mount (system) — no module places it",
|
||||
"openrazer-daemon.service — openrazer's"} {
|
||||
if !strings.Contains(lines, want) {
|
||||
t.Errorf("node show does not say %q:\n%s", want, lines)
|
||||
}
|
||||
}
|
||||
|
||||
// An older engine says nothing of its units: nothing cleared on its word.
|
||||
older := desktop(h0.Add(2 * time.Minute))
|
||||
older.Units = nil
|
||||
say(h0.Add(2*time.Minute), older)
|
||||
if _, still := openConditions()["machine.laptop.units"]; !still {
|
||||
t.Fatal("a statement saying nothing of units cleared the machine's finding")
|
||||
}
|
||||
|
||||
// The operator mends the mounts and removes the leftover link; the module's unit still fails.
|
||||
mended := desktop(h0.Add(3 * time.Minute))
|
||||
mended.Units.Failed = []link.FailedUnit{}
|
||||
say(h0.Add(3*time.Minute), mended)
|
||||
got = openConditions()
|
||||
if _, still := got["machine.laptop.units"]; still {
|
||||
t.Fatalf("no failed unit of the machine's own, still open: %v", got)
|
||||
}
|
||||
if _, still := got["module.openrazer.laptop.unhealthy"]; !still {
|
||||
t.Fatalf("the module's unit still fails, and its condition stands: %v", got)
|
||||
}
|
||||
|
||||
// And the module's unit mended: running, nothing open.
|
||||
well := link.Health{Contract: link.ReadinessContract, At: h0.Add(4 * time.Minute),
|
||||
Units: &link.UnitsHealth{State: link.UnitsRunning, Failed: []link.FailedUnit{}}}
|
||||
say(h0.Add(4*time.Minute), well)
|
||||
if got = openConditions(); len(got) != 0 {
|
||||
t.Fatalf("a running service manager leaves nothing open: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheGateNeverHoldsASendOnTheMachinesOwnUnits(t *testing.T) {
|
||||
since := h0
|
||||
units := conditions.Condition{Key: "machine.laptop.units", Kind: kindMachineUnits,
|
||||
Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop", Machine: "laptop"},
|
||||
Summary: "laptop's service manager is degraded", Raised: since.Add(time.Minute), Source: sourceUnits}
|
||||
if w := aboutTheMachine("laptop", []string{"openrazer"}, since, gateFacts{judged: true,
|
||||
open: []conditions.Condition{units}}); w.whole != "" || w.waiting != "" || len(w.on) != 0 {
|
||||
t.Fatalf("the machine's own failed units held a send: %+v", w)
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/identity"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/licences"
|
||||
@@ -53,6 +54,9 @@ func run() error {
|
||||
return fmt.Errorf("no command given")
|
||||
}
|
||||
|
||||
// Every connection this process dials says what it is (novox/hq issue 327).
|
||||
broker.ConnectionName = connectionName(args[0], os.Getenv(verbVar))
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
// Whatever this process holds of the controller's lease is given back as it ends (novox/hq to-be
|
||||
@@ -74,6 +78,8 @@ func run() error {
|
||||
return rotateCommand(ctx, args[1:])
|
||||
case "ask":
|
||||
return askCommand(ctx, args[1:])
|
||||
case "rehearse":
|
||||
return rehearseCommand(ctx, args[1:])
|
||||
case "builds":
|
||||
return buildsCommand(ctx, args[1:])
|
||||
// The build queue, controlled by hand (novox/hq ADR 0219).
|
||||
@@ -93,8 +99,20 @@ func run() error {
|
||||
return pauseCommand(ctx, args[0], args[1:])
|
||||
case "collection":
|
||||
return collectionCommand(ctx, args[1:])
|
||||
// What the records say the registries may keep, asked on demand (novox/hq ADR 0251).
|
||||
case "artifacts":
|
||||
return artifactsCommand(ctx, args[1:])
|
||||
case "collect":
|
||||
return collectCommand(ctx, args[1:])
|
||||
case "images":
|
||||
return imagesCommand(ctx, args[1:])
|
||||
case "mirrors":
|
||||
return mirrorsCommand(ctx, args[1:])
|
||||
case "plans":
|
||||
return plansCommand(ctx, args[1:])
|
||||
case "delivery":
|
||||
// The verbs the delivery's owner asks with (novox/hq ADR 0239).
|
||||
return deliveryCommand(ctx, args[1:])
|
||||
case "pin":
|
||||
return pinCommand(ctx, args[1:], true)
|
||||
case "unpin":
|
||||
@@ -114,8 +132,20 @@ func run() error {
|
||||
return brokerCommand(ctx, args[1:])
|
||||
case "serve":
|
||||
return serve(ctx)
|
||||
// The facts snapshot a merge check is fed (novox/hq to-be 45 §9).
|
||||
case "facts":
|
||||
return factsCommand(ctx, args[1:])
|
||||
// The merge gate: every machine of the snapshot composed with a change (novox/hq to-be 45 §9).
|
||||
case "merge-gate":
|
||||
return mergeGateCommand(ctx, args[1:])
|
||||
// A pull request's merge check run here exactly as the build seat runs it (novox/hq issue 286).
|
||||
case "check-here":
|
||||
return checkHereCommand(ctx, args[1:])
|
||||
case "upgrade":
|
||||
return upgradeCommand(ctx, args[1:])
|
||||
// The bus as a planned step (novox/hq to-be 45 §8, ADR 0236).
|
||||
case "bus":
|
||||
return busCommand(ctx, args[1:])
|
||||
case "declare":
|
||||
return declare(ctx, args[1:])
|
||||
case "overlay":
|
||||
@@ -164,6 +194,15 @@ func run() error {
|
||||
// What the healers did, and their brake (novox/hq to-be 45 §7).
|
||||
case "healers":
|
||||
return healersCommand(ctx, args[1:])
|
||||
// A consumer the mesh stopped asking for: retired, waiting for a person, deleted only by one
|
||||
// (novox/hq ADR 0230).
|
||||
case "retire":
|
||||
return retireCommand(ctx, args[1:])
|
||||
case "cleanup":
|
||||
return cleanupCommand(ctx, args[1:])
|
||||
// The data every machine declares, as the self-check last found it (novox/hq ADR 0233).
|
||||
case "data":
|
||||
return dataCommand(ctx, args[1:])
|
||||
case "version":
|
||||
fmt.Println(version)
|
||||
return nil
|
||||
@@ -179,6 +218,10 @@ func run() error {
|
||||
func usage() {
|
||||
fmt.Fprint(os.Stderr, `mesh-controller — the control plane
|
||||
|
||||
On a node the operator types these as 'mesh-cli <command>' (in zsh, 'nox <command>'): asked
|
||||
through the node's engine, and run as the controller's terminal only on the control-node
|
||||
(novox/hq ADR 0272).
|
||||
|
||||
migrate bring each context's schema up to date
|
||||
prepare the same, asked the way the mesh asks any module (ADR 0135)
|
||||
node add <name> [--adopted] create a node record; --adopted: the machine is in use
|
||||
@@ -210,6 +253,16 @@ func usage() {
|
||||
upgrade <name> roll-out [--together] ...send it to the machines running it
|
||||
upgrade <name> record ...record that they are behind, and send nothing
|
||||
status [--json] what is wrong, what is quiet, and what is out of date
|
||||
retire [--json] every provider waiting for a person to approve a retirement (ADR 0230)
|
||||
retire approve <node> <module> --why <text> retire what it waits with: access off, data kept
|
||||
retire reject <node> <module> --why <text> keep them active; a warning stays open
|
||||
cleanup [list] [--json] every retired consumer per provider: age, size, why
|
||||
cleanup delete <node> <module> <consumer> --why <text> the provider deletes that one retired consumer
|
||||
cleanup delete --older-than <days> --why <text> [--confirm] list those older; delete only with --confirm
|
||||
artifacts [--json] [--repository <r>] [--collected] every artifact the mesh made: kept and why, eligible, collected (ADR 0251)
|
||||
collect [--json] [--most <n>] what the store's sweep would let go of, now; nothing is done
|
||||
collect --confirm --why <text> [--most <n>] run the sweep now: hold every kept archive, let go of the eligible
|
||||
images <node> [--json] every container image the machine's declaration names, now and as last sent
|
||||
seats [--json] every seat this mesh defines, what it delivers, and who holds it
|
||||
seat rename <from> <to> rename a seat; its former name still resolves (ADR 0122)
|
||||
seat <name> --to <node>/<module> hand a seat to that assignment as one act; never empty in between (ADR 0131)
|
||||
@@ -314,6 +367,12 @@ func parseAround(set *flag.FlagSet, args []string) ([]string, error) {
|
||||
// registered, the same as the waiting command does. Said either way, so the daemon's log tells what
|
||||
// became of a build nobody was watching.
|
||||
func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
||||
// **A merge check builds nothing** (novox/hq to-be 45 §9): its verdict is said, and nothing of it is
|
||||
// recorded or registered.
|
||||
if result.Check != nil || result.Checked != nil {
|
||||
checked(ctx, result)
|
||||
return nil
|
||||
}
|
||||
// **A dry run is looked at, never taken in** (novox/hq issue 240). On 2026-10-04 a dry run of an
|
||||
// unmerged branch was heard here like any build, registered, and its definition reached a machine
|
||||
// before anyone had reviewed it.
|
||||
@@ -322,6 +381,15 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
||||
return nil
|
||||
}
|
||||
manifest, _, err := takeIn(ctx, b.inv, result)
|
||||
// The assignments pending on this build, made or ended (novox/hq issue 325), said in the daemon's log
|
||||
// after what became of the build.
|
||||
if b.open != nil {
|
||||
defer func() {
|
||||
for _, line := range settlePendingOnBuild(ctx, b.open, result, manifest.Module, err) {
|
||||
fmt.Printf("%s: %s\n", result.ID, line)
|
||||
}
|
||||
}()
|
||||
}
|
||||
// When it was asked, so a plan takes as its outcome only a build asked for it or after it
|
||||
// (novox/hq 04-ISSUES/219). Zero when the id does not say.
|
||||
asked, _ := link.BuildAskedAt(result.ID)
|
||||
@@ -358,3 +426,18 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
||||
statusFrom.nudge()
|
||||
return nil
|
||||
}
|
||||
|
||||
// verbVar carries the seat verb a command runs for, from the serving controller to the process it starts.
|
||||
const verbVar = "MESH_VERB"
|
||||
|
||||
// connectionName is what this process's connections say they are in the bus's list (novox/hq issue 327):
|
||||
// the serving controller, a verb's own process and which verb, or a command run at a shell and which.
|
||||
func connectionName(command, verb string) string {
|
||||
switch {
|
||||
case command == "serve":
|
||||
return "mesh-controller serving"
|
||||
case verb != "":
|
||||
return "mesh-controller verb " + verb
|
||||
}
|
||||
return "mesh-controller command " + command
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,414 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
snapshot "github.com/novox/mesh-controller/internal/facts"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The merge gate (novox/hq to-be 45 §9): a change judged against every machine of the snapshot, by the
|
||||
// incidents it is written from. Each case raises a mesh, takes its snapshot, and judges a pull request's
|
||||
// tree against it in throwaway stores of its own.
|
||||
|
||||
// catalogueMesh is two machines and a catalogue repository: a resolver and an object store on the
|
||||
// anchor, and on the laptop a network manager requiring the resolver, an album requiring the object
|
||||
// store, and a login manager. Every module is registered from novox/mesh-catalog, as the mesh's are.
|
||||
func catalogueMesh(t *testing.T) (snapshot.Facts, map[string]string) {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
manifests := map[string]string{
|
||||
"objects": `{"module":"objects","version":"1",
|
||||
"provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}],
|
||||
"receives":{"s3-bucket":"/var/lib/mesh/objects/mesh.json"}}`,
|
||||
"resolver": `{"module":"resolver","version":"1",
|
||||
"provides":[{"name":"wildcard-resolution","scope":"mesh","identity":false}]}`,
|
||||
"networkmanager": `{"module":"networkmanager","version":"1","requires":["wildcard-resolution"]}`,
|
||||
"album": `{"module":"album","version":"1","requires":["s3-bucket"]}`,
|
||||
"lemurs": `{"module":"lemurs","version":"1","capabilities":["systemd"],
|
||||
"resources":[{"id":"service","type":"service","unit":"lemurs.service","state":"running","boot":"enabled"}]}`,
|
||||
}
|
||||
for name, raw := range manifests {
|
||||
m, err := catalogue.ParseManifest([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog",
|
||||
Path: "modules/" + name, BuiltFrom: "c0ffee"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "resolver"}, {"laptop", "networkmanager"},
|
||||
{"laptop", "album"}, {"laptop", "lemurs"}} {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
f, err := gatherFacts(ctx, open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return f, manifests
|
||||
}
|
||||
|
||||
// aTree is a checkout of the catalogue repository holding these manifests.
|
||||
func aTree(t *testing.T, manifests map[string]string) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
for name, raw := range manifests {
|
||||
at := filepath.Join(dir, "modules", name)
|
||||
if err := os.MkdirAll(at, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(at, "module.json"), []byte(raw), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
func gateJudged(t *testing.T, f snapshot.Facts, tree string, changed ...string) mergeVerdict {
|
||||
t.Helper()
|
||||
admin := os.Getenv("MESH_TEST_POSTGRES")
|
||||
in := mergeCheckInput{facts: f, admin: admin, changed: changed}
|
||||
if tree != "" {
|
||||
in.repository, in.tree = "novox/mesh-catalog", tree
|
||||
}
|
||||
v, err := judgeChange(t.Context(), in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func withEdit(manifests map[string]string, name, raw string) map[string]string {
|
||||
out := map[string]string{}
|
||||
for k, v := range manifests {
|
||||
out[k] = v
|
||||
}
|
||||
if raw == "" {
|
||||
delete(out, name)
|
||||
} else {
|
||||
out[name] = raw
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// The mesh as it is passes: every machine composes in the gate's store as the controller composed it.
|
||||
func TestTheMeshAsItIsPassesTheGate(t *testing.T) {
|
||||
f, manifests := catalogueMesh(t)
|
||||
for _, m := range f.Machines {
|
||||
if !m.Declaration.Composes {
|
||||
t.Fatalf("the mesh itself does not compose: %+v", m.Declaration)
|
||||
}
|
||||
}
|
||||
v := gateJudged(t, f, aTree(t, manifests))
|
||||
if v.Verdict != "pass" {
|
||||
t.Fatalf("an unchanged catalogue does not pass:\n%s", v.Report())
|
||||
}
|
||||
for _, m := range v.Machines {
|
||||
if !m.Base.Composes || !m.Change.Composes {
|
||||
t.Errorf("%s does not compose in the gate's store as it does on the mesh: %+v / %+v", m.Described, m.Base, m.Change)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **Issue 263**: a change makes the network manager require the object store's provision; on a machine
|
||||
// whose name is six characters its identity is 26 against a bound of 20. Refused in the pull request,
|
||||
// naming the module, and not on the anchor after it merged.
|
||||
func TestIssue263AnIdentityARealMachineNameOverflowsFailsThePullRequest(t *testing.T) {
|
||||
f, manifests := catalogueMesh(t)
|
||||
tree := aTree(t, withEdit(manifests, "networkmanager",
|
||||
`{"module":"networkmanager","version":"1","requires":["wildcard-resolution","s3-bucket"]}`))
|
||||
v := gateJudged(t, f, tree)
|
||||
if v.Verdict != "fail" {
|
||||
t.Fatalf("the overflow passed the gate:\n%s", v.Report())
|
||||
}
|
||||
report := v.Report()
|
||||
if !strings.Contains(report, "networkmanager") || !strings.Contains(report, "s3-bucket") {
|
||||
t.Errorf("the refusal does not name the module and the provision:\n%s", report)
|
||||
}
|
||||
}
|
||||
|
||||
// **Issue 236**: a login manager's service that omits its state passed the catalogue check and was
|
||||
// refused whole by the node-engine on the first machine. A change to a module a machine runs, and a
|
||||
// module nobody runs yet, are both refused before merge, naming the machine and the module.
|
||||
func TestIssue236AManifestTheNodeEngineRefusesFailsThePullRequest(t *testing.T) {
|
||||
f, manifests := catalogueMesh(t)
|
||||
broken := `{"module":"lemurs","version":"1","capabilities":["systemd"],
|
||||
"resources":[{"id":"service","type":"service","unit":"lemurs.service","boot":"enabled"}]}`
|
||||
v := gateJudged(t, f, aTree(t, withEdit(manifests, "lemurs", broken)))
|
||||
if v.Verdict != "fail" || !strings.Contains(v.Report(), "lemurs") {
|
||||
t.Fatalf("a service the node-engine refuses passed the gate:\n%s", v.Report())
|
||||
}
|
||||
laptop := snapshot.Pseudonym("machine", "laptop")
|
||||
if !strings.Contains(v.Report(), laptop) {
|
||||
t.Errorf("the refusal does not name the machine it would be refused on:\n%s", v.Report())
|
||||
}
|
||||
|
||||
// And as a new module, which no machine runs: tried on one that could.
|
||||
newcomer := strings.ReplaceAll(broken, `"lemurs"`, `"greeter"`)
|
||||
newcomer = strings.ReplaceAll(newcomer, "lemurs.service", "greeter.service")
|
||||
v = gateJudged(t, f, aTree(t, withEdit(manifests, "greeter", newcomer)))
|
||||
if v.Verdict != "fail" || !strings.Contains(v.Report(), "greeter, assigned to") {
|
||||
t.Fatalf("a new module the node-engine would refuse passed the gate:\n%s", v.Report())
|
||||
}
|
||||
}
|
||||
|
||||
// **Version skew**: a manifest the controller judging it cannot read — the one the mesh runs, for a
|
||||
// catalogue change — fails here, not at registration after the merge.
|
||||
func TestAManifestTheRunningControllerCannotReadFailsThePullRequest(t *testing.T) {
|
||||
f, manifests := catalogueMesh(t)
|
||||
v := gateJudged(t, f, aTree(t, withEdit(manifests, "album",
|
||||
`{"module":"album","version":"1","requires":["s3-bucket"],"a-field-of-a-newer-controller":true}`)))
|
||||
if v.Verdict != "fail" || !strings.Contains(v.Report(), "refuses it") {
|
||||
t.Fatalf("a manifest this controller cannot read passed:\n%s", v.Report())
|
||||
}
|
||||
}
|
||||
|
||||
// A module a machine runs, removed from its source, fails until it is unassigned (ADR 0236).
|
||||
func TestAModuleAMachineRunsRemovedFromItsSourceFails(t *testing.T) {
|
||||
f, manifests := catalogueMesh(t)
|
||||
v := gateJudged(t, f, aTree(t, withEdit(manifests, "album", "")))
|
||||
if v.Verdict != "fail" || !strings.Contains(v.Report(), "album is removed") {
|
||||
t.Fatalf("removing a module a machine runs passed:\n%s", v.Report())
|
||||
}
|
||||
}
|
||||
|
||||
// **Issues 278 and 280**: a file in no held module's directory read as shared code, and a merge rebuilt the
|
||||
// catalogue. A file is a module's only by being inside it — no build reads one outside — so it rebuilds
|
||||
// nothing, and the gate says why; a merge that does rebuild widely is warned of.
|
||||
func TestIssue278AWideRebuildIsSaidBeforeTheMerge(t *testing.T) {
|
||||
f, manifests := catalogueMesh(t)
|
||||
was := wideRebuild
|
||||
wideRebuild = 2
|
||||
t.Cleanup(func() { wideRebuild = was })
|
||||
for _, file := range []string{"modules/showcase/index.ts", "merge-check.sh", "README.md"} {
|
||||
v := gateJudged(t, f, aTree(t, manifests), file)
|
||||
if v.Width == nil || len(v.Width.Modules) != 0 || len(v.Width.Unread) != 1 || v.Verdict != "pass" {
|
||||
t.Fatalf("%s, read by no build, reads %+v, %s", file, v.Width, v.Verdict)
|
||||
}
|
||||
if !strings.Contains(v.Report(), "read by no module's build") {
|
||||
t.Errorf("why %s rebuilds nothing is not said:\n%s", file, v.Report())
|
||||
}
|
||||
}
|
||||
v := gateJudged(t, f, aTree(t, manifests), "modules/album/x.ts", "modules/objects/x.go", "modules/resolver/x.go")
|
||||
if v.Width == nil || len(v.Width.Modules) < 3 || v.Verdict != "warning" {
|
||||
t.Fatalf("a rebuild wider than the bound is not warned of: %+v, %s", v.Width, v.Verdict)
|
||||
}
|
||||
// With the reference module's definition in the tree, its directory is a module, held or not.
|
||||
withShowcase := withEdit(manifests, "showcase", `{"module":"showcase","version":"1"}`)
|
||||
v = gateJudged(t, f, aTree(t, withShowcase), "modules/showcase/index.ts")
|
||||
if v.Width == nil || len(v.Width.Modules) != 0 || len(v.Width.Unread) != 0 {
|
||||
t.Fatalf("a file of a module nobody holds reads %+v", v.Width)
|
||||
}
|
||||
v = gateJudged(t, f, aTree(t, manifests), "modules/album/module.json")
|
||||
if v.Width == nil || strings.Join(v.Width.Modules, ",") != "album" || v.Verdict != "pass" {
|
||||
t.Fatalf("a change to one module's directory reads %+v, %s", v.Width, v.Verdict)
|
||||
}
|
||||
}
|
||||
|
||||
// **A delivery group is judged as one future state** (novox/hq ADR 0239): a catalogue change that needs a
|
||||
// provision only another repository's change adds fails alone and passes composed with it; and a group
|
||||
// whose other head breaks what the first needs fails, naming it.
|
||||
func TestADeliveryGroupsHeadsAreComposedTogether(t *testing.T) {
|
||||
f, manifests := catalogueMeshWithAnApp(t)
|
||||
needsTheApp := withEdit(manifests, "album", `{"module":"album","version":"1","requires":["s3-bucket","app-api"]}`)
|
||||
catTree := aTree(t, needsTheApp)
|
||||
alone := gateJudged(t, f, catTree, "modules/album/module.json")
|
||||
if alone.Verdict != "fail" {
|
||||
t.Fatalf("a requirement nothing provides passed alone:\n%s", alone.Report())
|
||||
}
|
||||
app := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(app, "module.json"), []byte(`{"module":"app","version":"1",
|
||||
"provides":[{"name":"app-api","scope":"mesh","identity":false}]}`), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
in := mergeCheckInput{facts: f, admin: os.Getenv("MESH_TEST_POSTGRES"), repository: "novox/mesh-catalog",
|
||||
tree: catTree, changed: []string{"modules/album/module.json"},
|
||||
group: []groupChange{{Repository: "novox/app", Tree: app, Changed: []string{"module.json"}}}}
|
||||
together, err := judgeChange(t.Context(), in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if together.Verdict == "fail" {
|
||||
t.Fatalf("the group composed together fails:\n%s", together.Report())
|
||||
}
|
||||
if together.Modules["app"] != "changed" || together.Modules["album"] != "changed" {
|
||||
t.Fatalf("the group's heads were not both laid over the mesh: %v", together.Modules)
|
||||
}
|
||||
}
|
||||
|
||||
// catalogueMeshWithAnApp is catalogueMesh with an application built from a repository of its own, at its
|
||||
// root, on the anchor.
|
||||
func catalogueMeshWithAnApp(t *testing.T) (snapshot.Facts, map[string]string) {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
manifests := map[string]string{
|
||||
"objects": `{"module":"objects","version":"1",
|
||||
"provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}],
|
||||
"receives":{"s3-bucket":"/var/lib/mesh/objects/mesh.json"}}`,
|
||||
"album": `{"module":"album","version":"1","requires":["s3-bucket"]}`,
|
||||
}
|
||||
for name, raw := range manifests {
|
||||
m, err := catalogue.ParseManifest([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog",
|
||||
Path: "modules/" + name, BuiltFrom: "c0ffee"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"},
|
||||
inventory.Source{Repository: "novox/app", BuiltFrom: "c0ffee"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "app"}, {"laptop", "album"}} {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
f, err := gatherFacts(ctx, open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return f, manifests
|
||||
}
|
||||
|
||||
// busMesh is aMesh with a module on the bus on the laptop, its account issued as `module issue` issues
|
||||
// one: minted, and its credential held as the module's own secret named broker.
|
||||
func busMesh(t *testing.T) snapshot.Facts {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
m, err := catalogue.ParseManifest([]byte(`{"module":"speaker","version":"1",
|
||||
"own-secrets":{"broker":"/var/lib/speaker/broker"}}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog",
|
||||
Path: "modules/speaker", BuiltFrom: "c0ffee"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "speaker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
user := broker.Principal{Kind: broker.KindModule, Node: "laptop", Module: "speaker"}.Username()
|
||||
password, err := open.inventory.MintBusPassword(ctx, inventory.BusUser{Username: user, Kind: inventory.BusModule,
|
||||
Node: "laptop", Module: "speaker"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "speaker", "broker",
|
||||
`{"user":"`+user+`","password":"`+password+`"}`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := gatherFacts(ctx, open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, mc := range f.Machines {
|
||||
if !mc.Declaration.Composes {
|
||||
t.Fatalf("the mesh itself does not compose: %+v", mc.Declaration)
|
||||
}
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// **Issue 285**: every machine running a module on the bus failed to compose in the gate's store, with
|
||||
// the change and without — the store held the module's credential and no account for it, which
|
||||
// composition refuses (issue 203) — and the gate passed every change, "0 of 4 compose". The account the
|
||||
// mesh issued is raised with its credential, so the machine composes in the gate as on the mesh.
|
||||
func TestIssue285AModuleOnTheBusComposesInTheGate(t *testing.T) {
|
||||
f := busMesh(t)
|
||||
v := gateJudged(t, f, "")
|
||||
if v.Verdict != "pass" {
|
||||
t.Fatalf("the mesh as it is does not pass:\n%s", v.Report())
|
||||
}
|
||||
for _, m := range v.Machines {
|
||||
if !m.Base.Composes {
|
||||
t.Errorf("%s composes on the mesh and not in the gate: %v", m.Described, m.Base.Problems)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(v.Summary, "2 of 2 compose") {
|
||||
t.Errorf("the summary does not say every machine composes: %s", v.Summary)
|
||||
}
|
||||
}
|
||||
|
||||
// **Issue 285**: a machine the mesh composes that the gate cannot raise as it is leaves the change judged
|
||||
// against a machine that is not the mesh's — broken against broken, which passes whatever the change does.
|
||||
// That is an error, never a pass.
|
||||
func TestIssue285AMachineTheGateCannotRaiseIsAnErrorNeverAPass(t *testing.T) {
|
||||
f := busMesh(t)
|
||||
for i := range f.Machines {
|
||||
// A fact the snapshot does not carry: the module's credential, gone from the laptop's.
|
||||
var kept []snapshot.Accepted
|
||||
for _, a := range f.Machines[i].Accepted {
|
||||
if a.Name != "broker" {
|
||||
kept = append(kept, a)
|
||||
}
|
||||
}
|
||||
f.Machines[i].Accepted = kept
|
||||
}
|
||||
v := gateJudged(t, f, "")
|
||||
if v.Verdict != "error" {
|
||||
t.Fatalf("a gate whose mesh does not compose said %s:\n%s", v.Verdict, v.Report())
|
||||
}
|
||||
if len(v.Errors) != 1 || !strings.Contains(v.Errors[0], "speaker") {
|
||||
t.Errorf("the error does not name what could not be raised: %v", v.Errors)
|
||||
}
|
||||
}
|
||||
|
||||
// A path the snapshot withheld is given a path of its own where an access or a place needs one: a bare
|
||||
// "withheld" is no absolute path, and a machine whose setting composes on the mesh would not in the gate.
|
||||
func TestAWithheldPathIsStoodInForByAPath(t *testing.T) {
|
||||
got := standInPaths(map[string]any{
|
||||
"accesses": map[string]any{"races": "withheld", "films": "/media/films", "shows": "/withheld"},
|
||||
"places": map[string]any{"config": map[string]any{"path": "withheld", "owner": "1000:1000"}},
|
||||
"puid": 1000,
|
||||
})
|
||||
accesses := got["accesses"].(map[string]any)
|
||||
if accesses["races"] == accesses["shows"] || !strings.HasPrefix(accesses["races"].(string), "/") ||
|
||||
!strings.HasPrefix(accesses["shows"].(string), "/") || accesses["films"] != "/media/films" {
|
||||
t.Errorf("accesses: %v", accesses)
|
||||
}
|
||||
place := got["places"].(map[string]any)["config"].(map[string]any)
|
||||
if !strings.HasPrefix(place["path"].(string), "/") || place["owner"] != "1000:1000" || got["puid"] != 1000 {
|
||||
t.Errorf("places: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **Issue 286**: a check run by hand clones beside a change what the seat clones — one rule, read by the
|
||||
// controller's ask and by the facts — and finds the repository it checks from its origin.
|
||||
func TestACheckByHandClonesWhatTheSeatClones(t *testing.T) {
|
||||
for dir, refs := range map[string]map[string]string{
|
||||
"mesh-catalog": {"mesh-catalog": "main"},
|
||||
"mesh-host": {"mesh-host": "c0ffee"},
|
||||
"mesh-controller": {"mesh-controller": "c0ffee", "mesh-controller-main": "main", "mesh-lab": "main"},
|
||||
} {
|
||||
got := besideRefs(dir, "c0ffee")
|
||||
for d, ref := range refs {
|
||||
if got[d] != ref {
|
||||
t.Errorf("beside %s, %s is cloned at %q, not %q", dir, d, got[d], ref)
|
||||
}
|
||||
}
|
||||
}
|
||||
for owner, want := range map[string][3]string{
|
||||
"ssh://git@git.example:222/novox/mesh-host.git": {"novox", "mesh-host", "ssh://git@git.example:222/novox"},
|
||||
"git@git.example:novox/mesh-tools.git": {"novox", "mesh-tools", "git@git.example:novox"},
|
||||
"http://git.example/novox/hq": {"novox", "hq", "http://git.example/novox"},
|
||||
} {
|
||||
o, r, p := ownerRepoOf(owner)
|
||||
if [3]string{o, r, p} != want {
|
||||
t.Errorf("%s reads as %s %s %s", owner, o, r, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,271 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The operator's command, `mesh-cli` (alias `nox`), answered here (novox/hq ADR 0272).
|
||||
//
|
||||
// mesh-cli asks the node-engine on its own machine; the engine reads the asking account from the kernel and asks
|
||||
// this controller on its own node's subject. Here it is judged — the controller's terminal, an ordinary call, or
|
||||
// nothing — and run as every verb's line is: a fresh process of this binary, with this process's environment.
|
||||
//
|
||||
// **The terminal** is a line from a node's operator account, on the control-node (§4). Phase 2 adds a node whose
|
||||
// agents run under an account of their own, judged unable to become root (ADR 0266, not built yet); until then no
|
||||
// other node is the terminal, because agents there run as its operator. **An ordinary call** is a line from that
|
||||
// account elsewhere: it meets every refusal of the generic `command` verb and runs with `MESH_VERB=mesh-cli`, so a
|
||||
// terminal-only change is refused with its reason. **Any other account is refused**, root included: those two
|
||||
// accounts already reach the mesh's verbs through the mesh MCP server, and no other account gains anything here.
|
||||
|
||||
// cliVerb is what a line from mesh-cli that is not the terminal runs as: the verb its process names, so every
|
||||
// terminal-only refusal applies and says it came through mesh-cli.
|
||||
const cliVerb = "mesh-cli"
|
||||
|
||||
// cliServers are commands that serve until stopped. Run for mesh-cli they would hold a second server under this
|
||||
// one until the call's bound killed it.
|
||||
var cliServers = map[string]bool{"serve": true, "api": true, "board": true}
|
||||
|
||||
// cliVerdict is how a line is run, or why it is not.
|
||||
type cliVerdict struct {
|
||||
terminal bool
|
||||
// why says why the line is or is not the terminal, in words the operator reads under the answer.
|
||||
why string
|
||||
// refused says why nothing runs.
|
||||
refused string
|
||||
}
|
||||
|
||||
// judgeCLI decides how a line from mesh-cli runs (ADR 0272 §4). nodes is every node the mesh knows; control is
|
||||
// every node the controller's module is assigned to, which is exactly one in a mesh that is well.
|
||||
func judgeCLI(node string, asked link.CLIAsked, nodes []inventory.Node, control []string) cliVerdict {
|
||||
var record *inventory.Node
|
||||
for i := range nodes {
|
||||
if nodes[i].Name == node {
|
||||
record = &nodes[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case record == nil:
|
||||
return cliVerdict{refused: fmt.Sprintf("%s is not a node this mesh knows, so nothing ran", node)}
|
||||
case asked.UID == 0 || asked.Account == "root":
|
||||
return cliVerdict{refused: "mesh-cli answers the operator's own account, never root: run it as yourself, " +
|
||||
"not under sudo. Nothing ran"}
|
||||
case record.Account == "":
|
||||
return cliVerdict{refused: fmt.Sprintf("the mesh does not know %s's operator account (`node account <node> <login>`), "+
|
||||
"so no account there is answered. Nothing ran", node)}
|
||||
case asked.Account != record.Account:
|
||||
return cliVerdict{refused: fmt.Sprintf("mesh-cli answers %s's operator account (%s) only, and this line came "+
|
||||
"from %s. Nothing ran", node, record.Account, asked.Account)}
|
||||
}
|
||||
if len(control) != 1 {
|
||||
return cliVerdict{why: fmt.Sprintf("not the controller's terminal: the mesh names %d control-nodes, and the "+
|
||||
"terminal is the one control-node's operator account", len(control))}
|
||||
}
|
||||
if control[0] == node {
|
||||
// **A person at a terminal, not a service of the operator's** (review of ADR 0272): the tool runner and the
|
||||
// account's user units run as the operator too, and only a login session is the terminal.
|
||||
if asked.Session == "" {
|
||||
return cliVerdict{why: fmt.Sprintf("not the controller's terminal: %s on %s asked from no login session — a "+
|
||||
"service or a user unit running as the operator, not a person at a terminal", asked.Account, node)}
|
||||
}
|
||||
return cliVerdict{terminal: true, why: fmt.Sprintf("the controller's terminal: %s on the control-node %s, "+
|
||||
"login session %s", asked.Account, node, asked.Session)}
|
||||
}
|
||||
return cliVerdict{why: fmt.Sprintf("not the controller's terminal: agents on %s may run as %s, so a "+
|
||||
"terminal-only change is made from the control-node %s (novox/hq ADR 0272)", node, asked.Account, control[0])}
|
||||
}
|
||||
|
||||
// controlNodes is every node the controller's module is assigned to.
|
||||
func controlNodes(ctx context.Context, inv *inventory.Inventory, nodes []inventory.Node) ([]string, error) {
|
||||
var out []string
|
||||
for _, n := range nodes {
|
||||
modules, err := inv.Assigned(ctx, n.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if slices.Contains(modules, controllerModule) {
|
||||
out = append(out, n.Name)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// controllerModule is the module that runs the controller, and so marks the control-node.
|
||||
const controllerModule = "mesh-controller"
|
||||
|
||||
// answerMeshCLI is the serving controller's answer to mesh-cli.
|
||||
func answerMeshCLI(inv *inventory.Inventory) link.CLIHandler {
|
||||
return func(ctx context.Context, node string, asked link.CLIAsked) link.CLIAnswer {
|
||||
if handingOver.Load() {
|
||||
return link.CLIRefusal("this controller is stopping and runs no new command; ask again in a moment. Nothing ran")
|
||||
}
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return link.CLIRefusal("the mesh's nodes cannot be read, so nothing ran: " + err.Error())
|
||||
}
|
||||
control, err := controlNodes(ctx, inv, nodes)
|
||||
if err != nil {
|
||||
return link.CLIRefusal("which node is the control-node cannot be read, so nothing ran: " + err.Error())
|
||||
}
|
||||
return runForMeshCLI(ctx, node, asked, judgeCLI(node, asked, nodes, control))
|
||||
}
|
||||
}
|
||||
|
||||
// cliJournal says one line in the controller's journal (its standard output); a variable so a test can read it.
|
||||
var cliJournal = func(line string) { fmt.Println(line) }
|
||||
|
||||
// runForMeshCLI runs a judged line and answers what it said. Every line is said in the journal first: its call,
|
||||
// who asked on which node, its command word only, and how it runs (review of ADR 0272).
|
||||
func runForMeshCLI(ctx context.Context, node string, asked link.CLIAsked, v cliVerdict) link.CLIAnswer {
|
||||
how := "as an ordinary call"
|
||||
switch {
|
||||
case v.refused != "":
|
||||
how = "refused: " + v.refused
|
||||
case v.terminal:
|
||||
how = "as the controller's terminal"
|
||||
}
|
||||
call := link.CallIDIn(ctx)
|
||||
if call == "" {
|
||||
call = "(no call)"
|
||||
}
|
||||
cliJournal(fmt.Sprintf("mesh-cli %s: %s on %s asked %q, %s", call, asked.Account, node, asked.Line[0], how))
|
||||
if v.refused != "" {
|
||||
return link.CLIRefusal(v.refused)
|
||||
}
|
||||
// Standard input is the terminal's alone: a secret given at the terminal reaches `secret accept`, and no ordinary
|
||||
// call is handed what the asker's standard input held (novox/hq ADR 0259 §10, ADR 0272).
|
||||
if len(asked.Stdin) > 0 && !v.terminal {
|
||||
return link.CLIAnswer{Exit: 1, Why: v.why, Refused: "standard input is given to a line that runs as the " +
|
||||
"controller's terminal alone, and this one does not. Nothing ran"}
|
||||
}
|
||||
if cliServers[asked.Line[0]] {
|
||||
return link.CLIAnswer{Exit: 1, Why: v.why, Refused: fmt.Sprintf("%s serves until stopped, and is not a "+
|
||||
"command line mesh-cli runs. Nothing ran", asked.Line[0])}
|
||||
}
|
||||
verb, line := "", asked.Line
|
||||
if !v.terminal {
|
||||
composed, err := ordinaryLine(asked.Line)
|
||||
if err != nil {
|
||||
return link.CLIAnswer{Exit: 1, Why: v.why, Refused: err.Error()}
|
||||
}
|
||||
verb, line = cliVerb, composed
|
||||
}
|
||||
cmd := selfCommand(ctx, line)
|
||||
cmd.Env = commandEnvironment(fmt.Sprintf("%s through mesh-cli on %s", asked.Account, node), verb, v.terminal)
|
||||
// No standard input unless mesh-cli carried one for a terminal line: a command that reads one gets nothing, and
|
||||
// fails saying so (ADR 0272 §5).
|
||||
cmd.Stdin = nil
|
||||
if len(asked.Stdin) > 0 {
|
||||
cmd.Stdin = bytes.NewReader(asked.Stdin)
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout, cmd.Stderr = &stdout, &stderr
|
||||
err := cmd.Run()
|
||||
answer := link.CLIAnswer{Stdout: stdout.Bytes(), Stderr: stderr.Bytes(), Terminal: v.terminal, Why: v.why}
|
||||
var exit *exec.ExitError
|
||||
switch {
|
||||
case err == nil:
|
||||
case errors.As(err, &exit):
|
||||
answer.Exit = exit.ExitCode()
|
||||
if answer.Exit < 0 {
|
||||
// Killed: by the call's bound, or by this controller stopping.
|
||||
answer.Exit = 1
|
||||
answer.Stderr = append(answer.Stderr, []byte(fmt.Sprintf("\nmesh-cli: the command was stopped (%v)\n",
|
||||
exit))...)
|
||||
}
|
||||
default:
|
||||
answer.Exit = 1
|
||||
answer.Refused = fmt.Sprintf("could not run %s from this controller's own build: %v", strings.Join(asked.Line, " "), err)
|
||||
}
|
||||
return answer
|
||||
}
|
||||
|
||||
// settingsForms is what an ordinary `settings` line may say: the settings verb's own forms.
|
||||
const settingsForms = "settings set <module> <values> [--replace] [--node <node>], settings clear <module> " +
|
||||
"[--node <node>], settings show <module> [--history] [--node <node>], or settings preferences [<module>] " +
|
||||
"[--node <node>]"
|
||||
|
||||
// ordinaryLine is the command line an ordinary call runs (ADR 0272 §4): a `settings` line composed exactly as the
|
||||
// settings verb composes its own, so its refusals — the terminal-only keys among them — are that verb's (review of
|
||||
// ADR 0272); any other line as the generic `command` verb takes it, with its refusals.
|
||||
func ordinaryLine(argv []string) ([]string, error) {
|
||||
if len(argv) == 0 || argv[0] != "settings" {
|
||||
// What the generic verb runs, and nothing it would refuse: its reading forms only, and never a command that
|
||||
// is the terminal's alone (novox/hq ADR 0266) — the two checks argvFor makes of the `command` verb's line,
|
||||
// made of the words as given rather than re-split from one string.
|
||||
if err := refusedAsTheGenericCommand(argv); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := terminalOnly(argv); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
args := map[string]any{}
|
||||
var words []string
|
||||
rest := argv[1:]
|
||||
for i := 0; i < len(rest); i++ {
|
||||
w := rest[i]
|
||||
switch {
|
||||
case w == "--replace" || w == "-replace":
|
||||
args["replace"] = "true"
|
||||
case w == "--history" || w == "-history":
|
||||
args["history"] = "true"
|
||||
case w == "--node" || w == "-node":
|
||||
if i+1 >= len(rest) {
|
||||
return nil, fmt.Errorf("--node names no node; settings takes %s. Nothing ran", settingsForms)
|
||||
}
|
||||
i++
|
||||
args["node"] = rest[i]
|
||||
case strings.HasPrefix(w, "--node=") || strings.HasPrefix(w, "-node="):
|
||||
_, args["node"], _ = strings.Cut(w, "=")
|
||||
case strings.HasPrefix(w, "-"):
|
||||
return nil, fmt.Errorf("settings takes no %s through mesh-cli outside the terminal; it takes %s. "+
|
||||
"Nothing ran", w, settingsForms)
|
||||
default:
|
||||
words = append(words, w)
|
||||
}
|
||||
}
|
||||
wrong := fmt.Errorf("through mesh-cli outside the terminal, settings takes the settings verb's forms: %s. "+
|
||||
"Nothing ran", settingsForms)
|
||||
if len(words) == 0 {
|
||||
return nil, wrong
|
||||
}
|
||||
switch words[0] {
|
||||
case "set":
|
||||
if len(words) != 3 {
|
||||
return nil, wrong
|
||||
}
|
||||
args["module"], args["values"] = words[1], words[2]
|
||||
case "clear":
|
||||
if len(words) != 2 {
|
||||
return nil, wrong
|
||||
}
|
||||
args["module"], args["clear"] = words[1], "true"
|
||||
case "show":
|
||||
if len(words) != 2 {
|
||||
return nil, wrong
|
||||
}
|
||||
args["module"] = words[1]
|
||||
case "preferences":
|
||||
if len(words) > 2 {
|
||||
return nil, wrong
|
||||
}
|
||||
args["list"] = "preferences"
|
||||
if len(words) == 2 {
|
||||
args["module"] = words[1]
|
||||
}
|
||||
default:
|
||||
return nil, wrong
|
||||
}
|
||||
return argvFor("settings", args)
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// secretAcceptWants makes the test binary, run as a command line, read a secret as `secret accept` reads it and
|
||||
// say whether it is the value whose SHA-256 the variable names (TestMain).
|
||||
const secretAcceptWants = "MESH_TEST_SECRET_ACCEPT_WANTS"
|
||||
|
||||
// readAsSecretAccept is that process: `secret accept <node> <module> <name> [--from -]`, the value read by
|
||||
// valueFor, compared by digest, and only the verdict printed.
|
||||
func readAsSecretAccept(want string, argv []string) int {
|
||||
if len(argv) < 5 || argv[0] != "secret" || argv[1] != "accept" {
|
||||
fmt.Printf("not a secret accept line: %q\n", argv)
|
||||
return 2
|
||||
}
|
||||
from := ""
|
||||
if len(argv) == 7 && argv[5] == "--from" {
|
||||
from = argv[6]
|
||||
}
|
||||
value, err := valueFor(argv[2], argv[3], argv[4], from)
|
||||
if err != nil {
|
||||
fmt.Printf("secret accept read nothing: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
sum := sha256.Sum256([]byte(asSupplied(value)))
|
||||
if hex.EncodeToString(sum[:]) != want {
|
||||
fmt.Printf("secret accept read something else (%d bytes)\n", len(value))
|
||||
return 1
|
||||
}
|
||||
fmt.Println("secret accept read the value it was given")
|
||||
return 0
|
||||
}
|
||||
|
||||
// novox/hq ADR 0259 §10, ADR 0272: what mesh-cli's standard input held reaches `secret accept` on a line that runs
|
||||
// as the controller's terminal, and appears nowhere else — not in the answer, not in the journal, not in the calls
|
||||
// record; an ordinary line carrying it is refused and nothing runs.
|
||||
func TestStandardInputReachesSecretAcceptAtTheTerminalAndNowhereElse(t *testing.T) {
|
||||
token := "123456789:AAEhBOweik6ad9r_QxGivenAtTheTerminal"
|
||||
sum := sha256.Sum256([]byte(token))
|
||||
t.Setenv(secretAcceptWants, hex.EncodeToString(sum[:]))
|
||||
var journal []string
|
||||
var mu sync.Mutex
|
||||
was := cliJournal
|
||||
cliJournal = func(line string) { mu.Lock(); journal = append(journal, line); mu.Unlock() }
|
||||
t.Cleanup(func() { cliJournal = was })
|
||||
ctx := context.Background()
|
||||
|
||||
for _, line := range [][]string{
|
||||
{"secret", "accept", "anchor", "telegram", "telegram-token", "--from", "-"},
|
||||
{"secret", "accept", "anchor", "telegram", "telegram-token"}, // the prompt's path, a line on standard input
|
||||
} {
|
||||
asked := cliAsked("operator", 1000, line...)
|
||||
asked.Stdin = []byte(token + "\n")
|
||||
a := runForMeshCLI(ctx, "control", asked, cliVerdict{terminal: true, why: "the terminal"})
|
||||
if a.Exit != 0 || !strings.Contains(string(a.Stdout), "read the value it was given") {
|
||||
t.Fatalf("%q: secret accept did not read what mesh-cli carried: %+v (%s)", line, a, a.Stdout)
|
||||
}
|
||||
if body, _ := json.Marshal(a); strings.Contains(string(body), "AAEh") || strings.Contains(string(body), base64.StdEncoding.EncodeToString([]byte(token))) {
|
||||
t.Fatalf("the answer carries the secret: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
// Without standard input, the line reads nothing, as before.
|
||||
a := runForMeshCLI(ctx, "control", cliAsked("operator", 1000, "secret", "accept", "anchor", "telegram",
|
||||
"telegram-token", "--from", "-"), cliVerdict{terminal: true, why: "the terminal"})
|
||||
if a.Exit == 0 {
|
||||
t.Fatalf("a line with no standard input read a value: %+v", a)
|
||||
}
|
||||
|
||||
// An ordinary call is never handed it: refused, and nothing ran.
|
||||
asked := cliAsked("operator", 1000, "status")
|
||||
asked.Stdin = []byte(token)
|
||||
a = runForMeshCLI(ctx, "laptop", asked, cliVerdict{why: "not the terminal"})
|
||||
if a.Exit == 0 || !strings.Contains(a.Refused, "terminal alone") || len(a.Stdout) != 0 {
|
||||
t.Fatalf("an ordinary line was given standard input: %+v", a)
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
for _, l := range journal {
|
||||
if strings.Contains(l, "AAEh") {
|
||||
t.Fatalf("the journal says the secret: %s", l)
|
||||
}
|
||||
}
|
||||
if len(journal) == 0 {
|
||||
t.Fatal("the lines were not said in the journal at all")
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// echoEnvironment makes the test binary, run as a command line, say the verb and caller it was given (TestMain).
|
||||
const echoEnvironment = "MESH_TEST_ECHO_ENVIRONMENT"
|
||||
|
||||
var cliNodes = []inventory.Node{
|
||||
{Name: "control", Account: "operator"},
|
||||
{Name: "laptop", Account: "operator"},
|
||||
{Name: "unnamed"},
|
||||
}
|
||||
|
||||
func cliAsked(account string, uid uint32, line ...string) link.CLIAsked {
|
||||
return link.CLIAsked{Line: line, Account: account, UID: uid, Session: "session-1.scope"}
|
||||
}
|
||||
|
||||
// Who is the controller's terminal, and who is an ordinary call or refused (novox/hq ADR 0272 §4).
|
||||
func TestMeshCLIIsTheTerminalOnlyForTheControlNodesOperator(t *testing.T) {
|
||||
control := []string{"control"}
|
||||
cases := []struct {
|
||||
name, node string
|
||||
asked link.CLIAsked
|
||||
control []string
|
||||
terminal bool
|
||||
refused string
|
||||
why string
|
||||
}{
|
||||
{"the control-node's operator", "control", cliAsked("operator", 1000, "status"), control, true, "", "the controller's terminal"},
|
||||
{"another node's operator", "laptop", cliAsked("operator", 1000, "status"), control, false, "", "agents on laptop may run as operator"},
|
||||
{"another account", "control", cliAsked("agent", 1001, "status"), control, false, "operator account (operator) only", ""},
|
||||
{"root", "control", cliAsked("root", 0, "status"), control, false, "never root", ""},
|
||||
{"a node with no operator account", "unnamed", cliAsked("operator", 1000, "status"), control, false, "does not know unnamed's operator account", ""},
|
||||
{"a node the mesh does not know", "elsewhere", cliAsked("operator", 1000, "status"), control, false, "not a node this mesh knows", ""},
|
||||
{"two control-nodes", "control", cliAsked("operator", 1000, "status"), []string{"control", "laptop"}, false, "", "2 control-nodes"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
v := judgeCLI(c.node, c.asked, cliNodes, c.control)
|
||||
if v.terminal != c.terminal {
|
||||
t.Errorf("%s: terminal %v, want %v (%+v)", c.name, v.terminal, c.terminal, v)
|
||||
}
|
||||
if c.refused == "" && v.refused != "" || c.refused != "" && !strings.Contains(v.refused, c.refused) {
|
||||
t.Errorf("%s: refused %q, want %q", c.name, v.refused, c.refused)
|
||||
}
|
||||
if c.why != "" && !strings.Contains(v.why, c.why) {
|
||||
t.Errorf("%s: why %q, want %q", c.name, v.why, c.why)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The terminal runs its line without MESH_VERB, even where this process carries one; an ordinary call names
|
||||
// mesh-cli; both record who asked through mesh-cli.
|
||||
func TestTheTerminalRunsWithoutAVerbAndAnOrdinaryCallNamesMeshCLI(t *testing.T) {
|
||||
t.Setenv(echoEnvironment, "1")
|
||||
t.Setenv("MESH_VERB", "leaked-from-the-serving-process")
|
||||
// The serving controller marks itself (ADR 0266); its terminal line for mesh-cli is still the terminal's.
|
||||
t.Setenv(servedVar, "1")
|
||||
ctx := context.Background()
|
||||
|
||||
a := runForMeshCLI(ctx, "control", cliAsked("operator", 1000, "status"), cliVerdict{terminal: true, why: "the terminal"})
|
||||
if a.Exit != 0 || a.Refused != "" || !a.Terminal {
|
||||
t.Fatalf("the terminal's line did not run: %+v", a)
|
||||
}
|
||||
if got := string(a.Stdout); !strings.Contains(got, `verb=""`) || !strings.Contains(got, "operator through mesh-cli on control") ||
|
||||
!strings.Contains(got, "terminal=true") {
|
||||
t.Fatalf("the terminal's line ran with %s", got)
|
||||
}
|
||||
|
||||
a = runForMeshCLI(ctx, "laptop", cliAsked("operator", 1000, "status"), cliVerdict{why: "not the terminal"})
|
||||
if a.Exit != 0 || a.Terminal || a.Why != "not the terminal" {
|
||||
t.Fatalf("an ordinary line did not run as one: %+v", a)
|
||||
}
|
||||
if got := string(a.Stdout); !strings.Contains(got, `verb="mesh-cli"`) || !strings.Contains(got, "terminal=false") {
|
||||
t.Fatalf("an ordinary line ran with %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// An ordinary call meets every refusal of the generic command verb, and nothing runs; a server is never run.
|
||||
func TestAnOrdinaryCallMeetsTheCommandVerbsRefusals(t *testing.T) {
|
||||
t.Setenv(echoEnvironment, "1")
|
||||
ctx := context.Background()
|
||||
ordinary := cliVerdict{why: "not the terminal"}
|
||||
a := runForMeshCLI(ctx, "laptop", cliAsked("operator", 1000, "cleanup", "delete", "x"), ordinary)
|
||||
if a.Refused == "" || len(a.Stdout) != 0 || a.Exit != 1 || a.Why != "not the terminal" {
|
||||
t.Fatalf("a repair without --why ran as an ordinary call: %+v", a)
|
||||
}
|
||||
a = runForMeshCLI(ctx, "laptop", cliAsked("operator", 1000, "settings", "set", "claude-code", "{}"), ordinary)
|
||||
if a.Refused != "" || !strings.Contains(string(a.Stdout), `verb="mesh-cli"`) {
|
||||
t.Fatalf("an ordinary settings set did not run through the settings verb's path with MESH_VERB set: %+v", a)
|
||||
}
|
||||
for _, server := range []string{"serve", "api", "board"} {
|
||||
a := runForMeshCLI(ctx, "control", cliAsked("operator", 1000, server), cliVerdict{terminal: true})
|
||||
if a.Refused == "" || len(a.Stdout) != 0 {
|
||||
t.Fatalf("%s was run for mesh-cli: %+v", server, a)
|
||||
}
|
||||
}
|
||||
a = runForMeshCLI(ctx, "control", cliAsked("agent", 1001, "status"), cliVerdict{refused: "agent is not answered"})
|
||||
if a.Refused != "agent is not answered" || len(a.Stdout) != 0 {
|
||||
t.Fatalf("a refused line ran: %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
// **Which node is the terminal does not follow a verb** (review of ADR 0272): the controller's module is assigned
|
||||
// and unassigned at the terminal alone, so no caller of `assign` can move the terminal to a node of its choosing.
|
||||
// Asked through the acts themselves, as the verbs ask them; refused before any store is touched (none is given).
|
||||
func TestTheControllersModuleIsMovedAtTheTerminalAlone(t *testing.T) {
|
||||
t.Setenv("MESH_VERB", "assign")
|
||||
ctx := context.Background()
|
||||
if _, err := assignWith(ctx, nil, "laptop", assignOptions{}, "zsh", "mesh-controller"); err == nil ||
|
||||
!strings.Contains(err.Error(), "terminal") {
|
||||
t.Fatalf("assigning the controller through a verb was not refused: %v", err)
|
||||
}
|
||||
if _, err := assign(ctx, nil, "laptop", "mesh-controller"); err == nil || !strings.Contains(err.Error(), "terminal") {
|
||||
t.Fatalf("assigning the controller through a verb was not refused: %v", err)
|
||||
}
|
||||
t.Setenv("MESH_VERB", "unassign")
|
||||
if _, err := unassign(ctx, nil, "control", "mesh-controller"); err == nil || !strings.Contains(err.Error(), "terminal") {
|
||||
t.Fatalf("unassigning the controller through a verb was not refused: %v", err)
|
||||
}
|
||||
// Another module through a verb, and the controller's at the terminal, are not refused for it.
|
||||
if err := refusedMovingTheController([]string{"zsh"}); err != nil {
|
||||
t.Fatalf("another module was refused: %v", err)
|
||||
}
|
||||
t.Setenv("MESH_VERB", "")
|
||||
if err := refusedMovingTheController([]string{"mesh-controller"}); err != nil {
|
||||
t.Fatalf("the terminal was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// An ordinary `settings set|clear` goes down the settings verb's own path: composed as that verb composes it, and
|
||||
// run with MESH_VERB set, so its refusals — the terminal-only keys among them — are the settings command's own,
|
||||
// not a blanket refusal of the generic command (review of ADR 0272).
|
||||
func TestAnOrdinarySettingsLineTakesTheSettingsVerbsPath(t *testing.T) {
|
||||
cases := []struct {
|
||||
line []string
|
||||
want string
|
||||
err string
|
||||
}{
|
||||
{[]string{"settings", "set", "zsh", `{"execute":"withhold"}`, "--node", "laptop"}, `settings set zsh {"execute":"withhold"} --node laptop`, ""},
|
||||
{[]string{"settings", "set", "zsh", "{}", "--replace"}, "settings set zsh {} --replace", ""},
|
||||
{[]string{"settings", "clear", "zsh", "--node", "laptop"}, "settings clear zsh --node laptop", ""},
|
||||
{[]string{"settings", "show", "zsh", "--history"}, "settings show zsh --history", ""},
|
||||
{[]string{"settings", "preferences"}, "settings preferences", ""},
|
||||
{[]string{"settings", "set", "zsh"}, "", "settings"},
|
||||
{[]string{"settings", "set", "zsh", "{}", "--sideways"}, "", "--sideways"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
argv, err := ordinaryLine(c.line)
|
||||
if c.err != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), c.err) {
|
||||
t.Errorf("%q: refused with %v, want %q", c.line, err, c.err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil || strings.Join(argv, " ") != c.want {
|
||||
t.Errorf("%q: composed %q (%v), want %q", c.line, argv, err, c.want)
|
||||
}
|
||||
}
|
||||
// Anything else still meets the generic command verb's refusals.
|
||||
if _, err := ordinaryLine([]string{"retire", "delete", "x"}); err == nil {
|
||||
t.Error("a repair composed as an ordinary line")
|
||||
}
|
||||
}
|
||||
|
||||
// Every line is said in the controller's journal, with its call, who asked where and how it ran — its command word
|
||||
// only, never the rest of the line (review of ADR 0272).
|
||||
func TestEveryMeshCLILineIsSaidInTheJournal(t *testing.T) {
|
||||
t.Setenv(echoEnvironment, "1")
|
||||
var said []string
|
||||
was := cliJournal
|
||||
cliJournal = func(line string) { said = append(said, line) }
|
||||
t.Cleanup(func() { cliJournal = was })
|
||||
ctx := link.WithCallID(context.Background(), "call-1")
|
||||
runForMeshCLI(ctx, "control", cliAsked("operator", 1000, "settings", "set", "x", `{"password":"s3cret"}`),
|
||||
cliVerdict{terminal: true, why: "the terminal"})
|
||||
runForMeshCLI(ctx, "control", cliAsked("agent", 1001, "status"), cliVerdict{refused: "agent is not answered"})
|
||||
all := strings.Join(said, "\n")
|
||||
if len(said) != 2 || !strings.Contains(all, "call-1") || !strings.Contains(all, "operator on control") ||
|
||||
!strings.Contains(all, "as the controller's terminal") || !strings.Contains(all, "refused") {
|
||||
t.Fatalf("the journal said %q", said)
|
||||
}
|
||||
if strings.Contains(all, "s3cret") {
|
||||
t.Fatalf("the journal carries the line's values: %q", said)
|
||||
}
|
||||
}
|
||||
|
||||
// **An ordinary line runs only what the generic command verb would** (review of ADR 0272, ADR 0266): its reading
|
||||
// forms, and never a command that is the terminal's alone. Each of these, from another node's operator, is refused
|
||||
// and runs nothing.
|
||||
func TestAnOrdinaryLineRunsNothingTheCommandVerbWouldRefuse(t *testing.T) {
|
||||
t.Setenv(echoEnvironment, "1")
|
||||
for _, line := range [][]string{
|
||||
{"node", "account", "control", "x"},
|
||||
{"node", "agent-account", "control", "x", "--clear"},
|
||||
{"token", "issue", "laptop"},
|
||||
{"secret", "export", "x"},
|
||||
{"operator", "key", "set", "x"},
|
||||
{"assign", "laptop", "zsh"},
|
||||
} {
|
||||
if _, err := ordinaryLine(line); err == nil {
|
||||
t.Errorf("%q composed as an ordinary line", line)
|
||||
}
|
||||
a := runForMeshCLI(context.Background(), "laptop", cliAsked("operator", 1000, line...), cliVerdict{why: "not the terminal"})
|
||||
if a.Refused == "" || len(a.Stdout) != 0 {
|
||||
t.Errorf("%q ran as an ordinary line: %+v", line, a)
|
||||
}
|
||||
}
|
||||
if argv, err := ordinaryLine([]string{"status"}); err != nil || argv[0] != "status" {
|
||||
t.Fatalf("a read was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// **`calls` never shows a mesh-cli line's answer** (review of ADR 0272): the verb's own answer is read for a line
|
||||
// served over a bus, and neither the answer's text nor the base64 JSON writes its bytes in is there.
|
||||
func TestTheCallsVerbNeverShowsAMeshCLILinesAnswer(t *testing.T) {
|
||||
conn, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
stop, err := link.OverNATS{Conn: conn}.ServeCLI(func(context.Context, string, link.CLIAsked) link.CLIAnswer {
|
||||
return link.CLIAnswer{Stdout: []byte("s3cret-join")}
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
body, _ := json.Marshal(link.CLIAsked{Line: []string{"token", "issue", "x"}, Account: "operator"})
|
||||
msg, err := conn.Request(link.CLISubject("control"), body, 5*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(msg.Data), base64.StdEncoding.EncodeToString([]byte("s3cret-join"))) {
|
||||
t.Fatalf("the asker was not given its answer: %s", msg.Data)
|
||||
}
|
||||
recent, _ := link.Calls.Recent()
|
||||
var id string
|
||||
for _, c := range recent {
|
||||
if c.Seat == link.CLISeat {
|
||||
id = c.ID
|
||||
break
|
||||
}
|
||||
}
|
||||
shown, err := callsAnswer(link.Calls, id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said, _ := json.Marshal(shown)
|
||||
if strings.Contains(string(said), "s3cret-join") ||
|
||||
strings.Contains(string(said), base64.StdEncoding.EncodeToString([]byte("s3cret-join"))) {
|
||||
t.Fatalf("calls showed a mesh-cli line's answer: %s", said)
|
||||
}
|
||||
}
|
||||
|
||||
// **Only the terminal's line carries the terminal's mark** (review of ADR 0272): a mark the serving controller's
|
||||
// environment holds — leaked, or set by anything — is stripped from every other command line it runs, a verb's and
|
||||
// an ordinary mesh-cli line alike, so neither reads as the terminal.
|
||||
func TestTheTerminalsMarkIsStrippedFromEveryOtherLine(t *testing.T) {
|
||||
t.Setenv(echoEnvironment, "1")
|
||||
t.Setenv(cliTerminalVar, "1")
|
||||
t.Setenv(servedVar, "1")
|
||||
for _, env := range [][]string{commandEnvironment("someone", "status", false)} {
|
||||
for _, kv := range env {
|
||||
if strings.HasPrefix(kv, cliTerminalVar+"=") {
|
||||
t.Fatalf("a verb's line carries the terminal's mark: %s", kv)
|
||||
}
|
||||
}
|
||||
}
|
||||
a := runForMeshCLI(context.Background(), "laptop", cliAsked("operator", 1000, "status"), cliVerdict{why: "not the terminal"})
|
||||
if got := string(a.Stdout); !strings.Contains(got, "terminal=false") || !strings.Contains(got, `verb="mesh-cli"`) {
|
||||
t.Fatalf("an ordinary line with the mark in the serving environment ran as %s", got)
|
||||
}
|
||||
a = runForMeshCLI(context.Background(), "control", cliAsked("operator", 1000, "status"), cliVerdict{terminal: true})
|
||||
if got := string(a.Stdout); !strings.Contains(got, "terminal=true") {
|
||||
t.Fatalf("the terminal's line ran as %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **Only a login session is the terminal** (review of ADR 0272): the operator's account on the control-node, asking
|
||||
// from a service — the tool runner, a user unit — and not a login session, is an ordinary call.
|
||||
func TestOnlyALoginSessionIsTheTerminal(t *testing.T) {
|
||||
control := []string{"control"}
|
||||
v := judgeCLI("control", link.CLIAsked{Line: []string{"status"}, Account: "operator", UID: 1000}, cliNodes, control)
|
||||
if v.terminal || v.refused != "" || !strings.Contains(v.why, "login session") {
|
||||
t.Fatalf("a line from no login session reads %+v", v)
|
||||
}
|
||||
v = judgeCLI("control", link.CLIAsked{Line: []string{"status"}, Account: "operator", UID: 1000, Session: "session-3.scope"},
|
||||
cliNodes, control)
|
||||
if !v.terminal {
|
||||
t.Fatalf("a line from a login session on the control-node reads %+v", v)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The bases the mesh copied into its artifact store, and the copies no record names (novox/hq ADR 0257).
|
||||
//
|
||||
// A build that stands on an image published elsewhere copies it in first (ADR 0096, 0097). Since ADR 0257
|
||||
// each copy is recorded where it is made, and a copy is kept while a kept build stood on it. Copies made
|
||||
// before then by a build that failed — or before the records kept what a build stood on — are in the store
|
||||
// with no record naming them, and ADR 0189 §3 keeps the sweep away from anything no record names. This
|
||||
// verb is how a person says such a copy is the mesh's: it records it, and the sweep then decides as it
|
||||
// does for every other copy. It deletes nothing.
|
||||
|
||||
// mirrorRepository is a repository only the mirror writes: one image's repository, or a module's own
|
||||
// repository of a base from before ADR 0257.
|
||||
var mirrorRepository = regexp.MustCompile(`^(upstream/[a-z0-9][a-z0-9._/-]*|[a-z0-9][a-z0-9._-]*/on-[a-z0-9_]+)$`)
|
||||
|
||||
// mirrorReference reads a reference a person gave into its recorded form, refusing anything that is not
|
||||
// a manifest in a repository the mirror writes.
|
||||
func mirrorReference(given string) (string, error) {
|
||||
reference := catalogue.Recorded(strings.TrimSpace(given))
|
||||
path, ours := catalogue.InArtifactStore(reference)
|
||||
if !ours {
|
||||
return "", fmt.Errorf("%q is not a reference into the artifact store (artifact-store://<repository>@sha256:<hex>)", given)
|
||||
}
|
||||
repository, digest, ok := strings.Cut(path, "@sha256:")
|
||||
if !ok || len(digest) != 64 || strings.Trim(digest, "0123456789abcdef") != "" {
|
||||
return "", fmt.Errorf("%q names no manifest by digest", given)
|
||||
}
|
||||
if !mirrorRepository.MatchString(repository) {
|
||||
return "", fmt.Errorf("%q is in %s, which is not a repository the mirror writes "+
|
||||
"(upstream/<host>/<path>, or <module>/on-<argument>)", given, repository)
|
||||
}
|
||||
return reference, nil
|
||||
}
|
||||
|
||||
type mirrorEntry struct {
|
||||
Reference string `json:"reference"`
|
||||
State string `json:"state"`
|
||||
Why []string `json:"why,omitempty"`
|
||||
}
|
||||
|
||||
type mirrorsAnswer struct {
|
||||
DryRun bool `json:"dry_run,omitempty"`
|
||||
// Mirrors is every copy the records hold that is not yet let go of.
|
||||
Mirrors []mirrorEntry `json:"mirrors"`
|
||||
Counts struct {
|
||||
Kept int `json:"kept"`
|
||||
Eligible int `json:"eligible"`
|
||||
Collected int `json:"collected"`
|
||||
} `json:"counts"`
|
||||
// Recorded, AlreadyRecorded and Refused answer a record: what was (or, a dry run, would be)
|
||||
// recorded, what the records already held, and what was refused, with why.
|
||||
Recorded []string `json:"recorded,omitempty"`
|
||||
// Then says what recording does: a recorded copy is eligible, and the next sweep lets it go.
|
||||
Then string `json:"then,omitempty"`
|
||||
AlreadyRecorded []string `json:"already_recorded,omitempty"`
|
||||
Refused map[string]string `json:"refused,omitempty"`
|
||||
}
|
||||
|
||||
// mirrorsOf is the copies among the recorded states.
|
||||
func mirrorsOf(states []inventory.ArtifactState, mirrored map[string]bool) mirrorsAnswer {
|
||||
a := mirrorsAnswer{Mirrors: []mirrorEntry{}}
|
||||
for _, s := range states {
|
||||
if !mirrored[s.Reference] {
|
||||
continue
|
||||
}
|
||||
switch s.State {
|
||||
case inventory.ArtifactKept:
|
||||
a.Counts.Kept++
|
||||
case inventory.ArtifactEligible:
|
||||
a.Counts.Eligible++
|
||||
case inventory.ArtifactCollected:
|
||||
a.Counts.Collected++
|
||||
continue
|
||||
}
|
||||
a.Mirrors = append(a.Mirrors, mirrorEntry{Reference: s.Reference, State: s.State, Why: s.Why})
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// splitReferences reads references separated by spaces or commas.
|
||||
func splitReferences(given string) []string {
|
||||
return strings.FieldsFunc(given, func(r rune) bool { return r == ',' || r == ' ' || r == '\n' || r == '\t' })
|
||||
}
|
||||
|
||||
// recordMirrors decides, for each reference given, whether it may be recorded: in a repository the
|
||||
// mirror writes, not already recorded, and held by the store. A real run records those.
|
||||
func recordMirrors(ctx context.Context, inv *inventory.Inventory, store artifacts.Store, given []string,
|
||||
known map[string]bool, why string, real bool) (mirrorsAnswer, error) {
|
||||
a := mirrorsAnswer{DryRun: !real, Mirrors: []mirrorEntry{}, Refused: map[string]string{}}
|
||||
var record []string
|
||||
seen := map[string]bool{}
|
||||
for _, g := range given {
|
||||
reference, err := mirrorReference(g)
|
||||
if err != nil {
|
||||
a.Refused[g] = err.Error()
|
||||
continue
|
||||
}
|
||||
if seen[reference] {
|
||||
continue
|
||||
}
|
||||
seen[reference] = true
|
||||
if known[reference] {
|
||||
a.AlreadyRecorded = append(a.AlreadyRecorded, reference)
|
||||
continue
|
||||
}
|
||||
if store.Address == "" {
|
||||
a.Refused[g] = "this mesh has no artifact store on its network to ask whether it holds this"
|
||||
continue
|
||||
}
|
||||
held, err := store.HoldsManifest(ctx, reference)
|
||||
switch {
|
||||
case err != nil:
|
||||
a.Refused[g] = err.Error()
|
||||
continue
|
||||
case !held:
|
||||
a.Refused[g] = "the artifact store does not hold it"
|
||||
continue
|
||||
}
|
||||
record = append(record, reference)
|
||||
}
|
||||
a.Recorded = record
|
||||
if len(record) > 0 {
|
||||
verb := "would become"
|
||||
if real {
|
||||
verb = "became"
|
||||
}
|
||||
a.Then = fmt.Sprintf("%d cop%s %s eligible: the next sweep (after any build, or collect) lets each go "+
|
||||
"unless one of the five kept builds of a module the mesh holds stood on it", len(record),
|
||||
map[bool]string{true: "y", false: "ies"}[len(record) == 1], verb)
|
||||
}
|
||||
if real && len(record) > 0 {
|
||||
if err := inv.RecordMirrored(ctx, "", why, record); err != nil {
|
||||
return a, fmt.Errorf("recording %d copies: %w", len(record), err)
|
||||
}
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
func mirrorsCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("mirrors", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "answer as JSON")
|
||||
record := set.String("record", "", "references of copies no record names, separated by spaces or commas, to record as the mesh's")
|
||||
confirm := set.Bool("confirm", false, "record them, rather than only say what would be recorded")
|
||||
f := addHandActFlags(set)
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 0 {
|
||||
return errors.New("mirrors [--json] [--record <references> [--confirm --why <text>]]")
|
||||
}
|
||||
if *confirm {
|
||||
if strings.TrimSpace(*record) == "" {
|
||||
return errors.New("mirrors: --confirm records what --record names, and it names nothing. Nothing was done")
|
||||
}
|
||||
if err := f.require("mirrors"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
states, err := inv.Artifacts(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
known, err := inv.Mirrored(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var answer mirrorsAnswer
|
||||
if strings.TrimSpace(*record) == "" {
|
||||
answer = mirrorsOf(states, known)
|
||||
} else {
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
address, err := artifactStoreAddress(ctx, inv, shelf, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
answer, err = recordMirrors(ctx, inv, artifacts.Store{Address: address}, splitReferences(*record), known,
|
||||
strings.TrimSpace(*f.why), *confirm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The hand act is logged once the records say what it did, never before: an act that
|
||||
// failed half-way is not logged as done.
|
||||
if *confirm && len(answer.Recorded) > 0 {
|
||||
f.record(ctx, "mirrors", append([]string{"--record"}, answer.Recorded...))
|
||||
}
|
||||
}
|
||||
if *asJSON {
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
encoder.SetIndent("", " ")
|
||||
return encoder.Encode(answer)
|
||||
}
|
||||
if answer.DryRun && len(answer.Recorded) > 0 {
|
||||
fmt.Println("a dry run: nothing was recorded (--confirm --why <text> to record)")
|
||||
}
|
||||
if answer.Then != "" {
|
||||
fmt.Println(answer.Then)
|
||||
}
|
||||
for _, r := range answer.Recorded {
|
||||
fmt.Printf(" record %s\n", r)
|
||||
}
|
||||
for _, r := range answer.AlreadyRecorded {
|
||||
fmt.Printf(" already %s\n", r)
|
||||
}
|
||||
for g, why := range answer.Refused {
|
||||
fmt.Printf(" refused %s: %s\n", g, why)
|
||||
}
|
||||
for _, m := range answer.Mirrors {
|
||||
fmt.Printf(" %-9s %s %s\n", m.State, m.Reference, strings.Join(m.Why, ", "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,298 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The bases the mesh copied in, and recording a copy no record names (novox/hq ADR 0257).
|
||||
|
||||
func TestTheMirrorsVerbComposesItsCommandAndRefusesWhatItDoesNotTake(t *testing.T) {
|
||||
r := ref("route-proxy", "on-go_base", 1)
|
||||
for _, c := range []struct {
|
||||
args map[string]any
|
||||
want string
|
||||
}{
|
||||
{map[string]any{}, "mirrors --json"},
|
||||
{map[string]any{"record": r}, "mirrors --json --record " + r},
|
||||
{map[string]any{"record": r, "confirm": "true", "why": "copied before copies were recorded"},
|
||||
"mirrors --json --record " + r + " --confirm --why copied before copies were recorded"},
|
||||
} {
|
||||
argv, err := argvFor("mirrors", c.args)
|
||||
if err != nil || strings.Join(argv, " ") != c.want {
|
||||
t.Errorf("mirrors %v: %q %v, want %q", c.args, argv, err, c.want)
|
||||
}
|
||||
}
|
||||
for _, args := range []map[string]any{
|
||||
{"record": r, "confirm": "true"}, // recorded without a why
|
||||
{"record": r, "why": "x"}, // a why for a dry run
|
||||
{"confirm": "true", "why": "x"}, // nothing to record
|
||||
{"record": r, "confirm": "yes", "why": "x"}, // a switch is true or false
|
||||
{"delete": "true"},
|
||||
} {
|
||||
if argv, err := argvFor("mirrors", args); err == nil {
|
||||
t.Errorf("mirrors %v was composed as %q", args, argv)
|
||||
}
|
||||
}
|
||||
if repairingCommand([]string{"mirrors", "--json", "--record", r, "--confirm", "--why", "x"}) != "mirrors" {
|
||||
t.Error("recording a copy through the generic verb would go unrecorded")
|
||||
}
|
||||
if repairingCommand([]string{"mirrors", "--json", "--record", r}) != "" {
|
||||
t.Error("a dry run was taken for an act by hand")
|
||||
}
|
||||
if !personsDecision(link.HandAct{Verb: "mirrors"}) {
|
||||
t.Error("recording a copy would count toward a healer the mesh lacks")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyACopyInARepositoryTheMirrorWritesIsTaken(t *testing.T) {
|
||||
for given, ok := range map[string]bool{
|
||||
ref("route-proxy", "on-go_base", 1): true,
|
||||
catalogue.ArtifactStoreScheme + "upstream/docker.io/library/golang@sha256:" + strings.Repeat("a", 64): true,
|
||||
ref("route-proxy", "server", 1): false, // a module's own image
|
||||
catalogue.ArtifactStoreScheme + "novox/invoicing-api@sha256:" + strings.Repeat("a", 64): false,
|
||||
archiveRef("web", "on-tools", 1): false, // a blob
|
||||
catalogue.ArtifactStoreScheme + "web/on-x@sha256:abc": false, // not a whole digest
|
||||
"docker.io/library/golang@sha256:" + strings.Repeat("a", 64): false,
|
||||
} {
|
||||
if _, err := mirrorReference(given); (err == nil) != ok {
|
||||
t.Errorf("%s: taken %v, want %v (%v)", given, err == nil, ok, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// heldStore answers a manifest HEAD with 200 for the digests it holds, and records every request.
|
||||
func heldStore(t *testing.T, holds ...string) (artifacts.Store, *[]string) {
|
||||
t.Helper()
|
||||
var asked []string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
asked = append(asked, r.Method+" "+r.URL.Path)
|
||||
if r.Method == http.MethodHead && slices.ContainsFunc(holds, func(d string) bool { return strings.HasSuffix(r.URL.Path, d) }) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
return artifacts.Store{Address: strings.TrimPrefix(server.URL, "http://")}, &asked
|
||||
}
|
||||
|
||||
func TestRecordingACopyTakesWhatTheStoreHoldsAndDeletesNothing(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
held := ref("route-proxy", "on-go_base", 1)
|
||||
absent := ref("route-proxy", "on-go_base", 2)
|
||||
known := ref("nats", "on-nats_base", 3)
|
||||
notACopy := ref("route-proxy", "server", 4)
|
||||
if err := inv.RecordMirrored(t.Context(), "b1", "", []string{known}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store, asked := heldStore(t, fmt.Sprintf("%064x", 1))
|
||||
given := []string{held, absent, known, notACopy}
|
||||
mirrored, err := inv.Mirrored(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A dry run says, and records nothing.
|
||||
dry, err := recordMirrors(t.Context(), inv, store, given, mirrored, "", false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !dry.DryRun || !slices.Equal(dry.Recorded, []string{held}) || !slices.Equal(dry.AlreadyRecorded, []string{known}) ||
|
||||
len(dry.Refused) != 2 || dry.Refused[absent] == "" || dry.Refused[notACopy] == "" {
|
||||
t.Fatalf("a dry run answered %+v", dry)
|
||||
}
|
||||
if !strings.Contains(dry.Then, "would become eligible") || !strings.Contains(dry.Then, "next sweep") {
|
||||
t.Fatalf("a dry run did not say what recording does: %q", dry.Then)
|
||||
}
|
||||
if again, _ := inv.Mirrored(t.Context()); again[held] {
|
||||
t.Fatal("a dry run recorded a copy")
|
||||
}
|
||||
|
||||
// A real one records what the store holds, with the person's why, and the sweep may now decide.
|
||||
real, err := recordMirrors(t.Context(), inv, store, given, mirrored, "copied before copies were recorded", true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Equal(real.Recorded, []string{held}) {
|
||||
t.Fatalf("recorded %v", real.Recorded)
|
||||
}
|
||||
left, err := inv.ToCollect(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Contains(left, held) {
|
||||
t.Fatalf("a recorded copy no kept build stood on is not offered to collect: %v", left)
|
||||
}
|
||||
for _, r := range *asked {
|
||||
if !strings.HasPrefix(r, "HEAD ") {
|
||||
t.Fatalf("recording a copy asked the store %s", r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// indexRegistry holds indexes and platforms of one image's repository, answers GETs with their
|
||||
// documents, refuses GETs for the digests in fail, and records every delete.
|
||||
type indexRegistry struct {
|
||||
mu sync.Mutex
|
||||
indexes map[string][]string
|
||||
held map[string]bool
|
||||
fail map[string]bool
|
||||
deleted []string
|
||||
}
|
||||
|
||||
func (f *indexRegistry) serve(t *testing.T) artifacts.Store {
|
||||
t.Helper()
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
digest := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:]
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
if f.fail[digest] {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if children, ok := f.indexes[digest]; ok && f.held[digest] {
|
||||
var named []string
|
||||
for _, c := range children {
|
||||
named = append(named, `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"`+c+`"}`)
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[` +
|
||||
strings.Join(named, ",") + `]}`))
|
||||
return
|
||||
}
|
||||
if f.held[digest] {
|
||||
_, _ = w.Write([]byte(`{"schemaVersion":2,"layers":[]}`))
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
case http.MethodHead:
|
||||
if f.held[digest] {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
case http.MethodDelete:
|
||||
if !f.held[digest] {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
delete(f.held, digest)
|
||||
f.deleted = append(f.deleted, digest)
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
default:
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
return artifacts.Store{Address: strings.TrimPrefix(server.URL, "http://")}
|
||||
}
|
||||
|
||||
func copyDigest(n int) string { return fmt.Sprintf("sha256:%064x", n) }
|
||||
|
||||
// twoCopies records, in one image's repository, a kept copy (stood on by a held module's build) naming
|
||||
// platforms 11 and 12, and an eligible one (a failed build's) naming 12 and 13.
|
||||
func twoCopies(t *testing.T, inv *inventory.Inventory) (kept, eligible string, reg *indexRegistry) {
|
||||
t.Helper()
|
||||
const repository = "upstream/docker.io/library/golang"
|
||||
kept = catalogue.ArtifactStoreScheme + repository + "@" + copyDigest(1)
|
||||
eligible = catalogue.ArtifactStoreScheme + repository + "@" + copyDigest(2)
|
||||
if err := inv.RegisterModule(t.Context(), catalogue.Manifest{Module: "proxy", Version: "1"},
|
||||
inventory.Source{Repository: "https://forge.invalid/proxy.git"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failed := inventory.Build{ID: "f1", Repository: "https://forge.invalid/proxy.git", On: "a-build-machine",
|
||||
Failed: "a recipe refused", Mirrored: []string{eligible}}
|
||||
if err := inv.RecordBuild(t.Context(), failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ok := inventory.Build{ID: "b1", Repository: "https://forge.invalid/proxy.git", Module: "proxy", On: "a-build-machine",
|
||||
Commit: "c0ffee", Made: []inventory.Artifact{{Name: "app", Kind: "image", Reference: ref("proxy", "app", 7)}},
|
||||
Against: []string{kept}, Mirrored: []string{kept}}
|
||||
if err := inv.RecordBuild(t.Context(), ok); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reg = &indexRegistry{
|
||||
indexes: map[string][]string{copyDigest(1): {copyDigest(11), copyDigest(12)}, copyDigest(2): {copyDigest(12), copyDigest(13)}},
|
||||
held: map[string]bool{copyDigest(1): true, copyDigest(2): true, copyDigest(11): true, copyDigest(12): true, copyDigest(13): true},
|
||||
fail: map[string]bool{},
|
||||
}
|
||||
return kept, eligible, reg
|
||||
}
|
||||
|
||||
// Through the records: a confirmed collect lets the eligible index go with the platform only it names,
|
||||
// and leaves the platform the kept index names.
|
||||
func TestAConfirmedCollectLetsAnIndexGoWithItsOwnPlatformsOnly(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
_, eligible, reg := twoCopies(t, inv)
|
||||
store := reg.serve(t)
|
||||
references, err := inv.ToCollect(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Equal(references, []string{eligible}) {
|
||||
t.Fatalf("offered %v, want the failed build's copy", references)
|
||||
}
|
||||
a := runCollect(t.Context(), inv, store, references, nil, true,
|
||||
sweepBounds{most: 10, budget: 5 * time.Second, platforms: true})
|
||||
if !slices.Equal(a.LetGo, []string{eligible}) || a.Stopped != "" {
|
||||
t.Fatalf("let go %v, stopped %q", a.LetGo, a.Stopped)
|
||||
}
|
||||
if !slices.Equal(reg.deleted, []string{copyDigest(13), copyDigest(2)}) {
|
||||
t.Fatalf("deleted %v; want its own platform, then the index", reg.deleted)
|
||||
}
|
||||
}
|
||||
|
||||
// The sweep after a build leaves an eligible index in place and eligible: let go of alone, its
|
||||
// platforms would stay for ever under a record that says collected. A later collect a person confirms
|
||||
// takes it with the platform only it names. An image the same sweep reaches is let go of as before.
|
||||
func TestTheSweepAfterABuildLeavesAnIndexForAConfirmedCollect(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
_, eligible, reg := twoCopies(t, inv)
|
||||
reg.held[copyDigest(5)] = true
|
||||
image := catalogue.ArtifactStoreScheme + "upstream/docker.io/library/golang@" + copyDigest(5)
|
||||
store := reg.serve(t)
|
||||
r := sweep(t.Context(), inv, store, []string{eligible, image}, nil, afterBuild)
|
||||
if r.Indexes != 1 || !slices.Equal(r.LetGo, []string{image}) || !slices.Equal(reg.deleted, []string{copyDigest(5)}) {
|
||||
t.Fatalf("after a build: %d indexes left, let go %v, deleted %v; want the index left and the image gone",
|
||||
r.Indexes, r.LetGo, reg.deleted)
|
||||
}
|
||||
left, err := inv.ToCollect(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Equal(left, []string{eligible}) {
|
||||
t.Fatalf("after the sweep the records offer %v; want the index still eligible", left)
|
||||
}
|
||||
a := runCollect(t.Context(), inv, store, left, nil, true,
|
||||
sweepBounds{most: 10, budget: 5 * time.Second, platforms: true})
|
||||
if !slices.Equal(a.LetGo, []string{eligible}) ||
|
||||
!slices.Equal(reg.deleted, []string{copyDigest(5), copyDigest(13), copyDigest(2)}) {
|
||||
t.Fatalf("a confirmed collect let go %v, deleted %v; want the index with its own platform", a.LetGo, reg.deleted)
|
||||
}
|
||||
}
|
||||
|
||||
// A kept index the store will not answer for stops a confirmed collect before its first delete.
|
||||
func TestASpareListThatCannotBeReadStopsTheSweepBeforeAnyDelete(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
_, eligible, reg := twoCopies(t, inv)
|
||||
reg.fail[copyDigest(1)] = true
|
||||
store := reg.serve(t)
|
||||
a := runCollect(t.Context(), inv, store, []string{eligible}, nil, true,
|
||||
sweepBounds{most: 10, budget: 5 * time.Second, platforms: true})
|
||||
if len(a.LetGo) != 0 || len(reg.deleted) != 0 || !strings.Contains(a.Stopped, "nothing was let go") {
|
||||
t.Fatalf("let go %v, deleted %v, stopped %q", a.LetGo, reg.deleted, a.Stopped)
|
||||
}
|
||||
}
|
||||
@@ -40,6 +40,50 @@ type merges interface {
|
||||
AnnouncedMerges(ctx context.Context, since time.Time) ([]link.AnnouncedMerge, error)
|
||||
}
|
||||
|
||||
// unheardMerges is the announcements of merges the controller has not heard (novox/hq ADR 0276): a merge kept
|
||||
// in a batch is not acted on until its batch is cut, which can be past mergeGrace behind a long walk, and is
|
||||
// not missed for that.
|
||||
type unheardMerges struct {
|
||||
merges
|
||||
inv *inventory.Inventory
|
||||
}
|
||||
|
||||
func (u unheardMerges) AnnouncedMerges(ctx context.Context, since time.Time) ([]link.AnnouncedMerge, error) {
|
||||
all, err := u.merges.AnnouncedMerges(ctx, since)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []link.AnnouncedMerge
|
||||
for _, a := range all {
|
||||
_, kept, err := u.inv.MergeOf(ctx, a.Owner+"/"+a.Repo, a.Commit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !kept {
|
||||
out = append(out, a)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// owedToTheRecord says a merge read as history is still owed to the record (novox/hq ADR 0276): the merges
|
||||
// reader knows, when it keeps them (unheardMerges).
|
||||
func owedToTheRecord(ctx context.Context, announced merges, m link.SourceMoved, entries []inventory.Entry,
|
||||
read map[string][]inventory.ReadRepository) bool {
|
||||
u, ok := announced.(unheardMerges)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
merged, err := time.Parse(time.RFC3339Nano, m.MergedAt)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
raw := m
|
||||
raw.MergedAt = ""
|
||||
owed, err := owedLate(ctx, u.inv, m, merged, wouldMove(raw, entries, read))
|
||||
return err == nil && owed
|
||||
}
|
||||
|
||||
// catchingUpOnMerges reads back the forge's announcements on a timer, until the context ends.
|
||||
func catchingUpOnMerges(ctx context.Context, open *stores, announced merges) {
|
||||
f := following{open}
|
||||
@@ -48,7 +92,7 @@ func catchingUpOnMerges(ctx context.Context, open *stores, announced merges) {
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
read, err := open.inventory.ReadRepositories(ctx)
|
||||
read, err := readForPlanning(ctx, open.inventory)
|
||||
return entries, read, err
|
||||
}
|
||||
failing := ""
|
||||
@@ -61,7 +105,7 @@ func catchingUpOnMerges(ctx context.Context, open *stores, announced merges) {
|
||||
case <-tick.C:
|
||||
}
|
||||
watchedMerges.begin()
|
||||
err := catchUpOnMerges(ctx, time.Now(), announced, catalogued, f.SourceMoved, func(format string, args ...any) {
|
||||
err := catchUpOnMerges(ctx, time.Now(), unheardMerges{announced, open.inventory}, catalogued, f.SourceMoved, func(format string, args ...any) {
|
||||
fmt.Printf(format+"\n", args...)
|
||||
})
|
||||
// What the pass found is what S5 says (novox/hq to-be 45 §3); a pass that could not read
|
||||
@@ -102,10 +146,15 @@ func catchUpOnMerges(ctx context.Context, now time.Time, announced merges,
|
||||
return err
|
||||
}
|
||||
for _, a := range all {
|
||||
// A merge the forge said no time of is dated by its announcement, so a packaging module's look can
|
||||
// make it history once acted on, and the catch-up does not act on it again every pass (ADR 0267).
|
||||
if a.SourceMoved.MergedAt == "" && !a.At.IsZero() {
|
||||
a.SourceMoved.MergedAt = a.At.UTC().Format(time.RFC3339)
|
||||
}
|
||||
if now.Sub(a.At) < mergeGrace {
|
||||
continue
|
||||
}
|
||||
if len(wouldMove(a.SourceMoved, entries, read)) == 0 {
|
||||
if len(wouldMove(a.SourceMoved, entries, read)) == 0 && !owedToTheRecord(ctx, announced, a.SourceMoved, entries, read) {
|
||||
continue
|
||||
}
|
||||
if entries, read, err = catalogued(ctx); err != nil {
|
||||
@@ -113,7 +162,12 @@ func catchUpOnMerges(ctx context.Context, now time.Time, announced merges,
|
||||
}
|
||||
moves := wouldMove(a.SourceMoved, entries, read)
|
||||
if len(moves) == 0 {
|
||||
continue
|
||||
if !owedToTheRecord(ctx, announced, a.SourceMoved, entries, read) {
|
||||
continue
|
||||
}
|
||||
raw := a.SourceMoved
|
||||
raw.MergedAt = ""
|
||||
moves = wouldMove(raw, entries, read)
|
||||
}
|
||||
var names []string
|
||||
for _, e := range moves {
|
||||
|
||||
@@ -0,0 +1,687 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A module says how it is healthy, and the node-engine judges it (novox/hq ADR 0240, to-be 48 §4 and §5,
|
||||
// Phase A).
|
||||
//
|
||||
// **The node-engine owns every verdict; the controller keeps the last word and raises the condition.** A
|
||||
// machine states, in every report and as an event between reports, the state of every long-running
|
||||
// resource it runs for a module. The controller keeps the newest statement per machine (node_health), and
|
||||
// raises `module.<module>.<machine>.unhealthy` when two statements in a row say a resource of the module
|
||||
// is unhealthy — one statement is listed as unconfirmed, as the self-check does a finding one look can be
|
||||
// wrong about (to-be 45 §4, issue 277) — and clears it on the first that does not. The release gate reads
|
||||
// the stated health: a judging passes a module only when every long-running resource of it on that
|
||||
// machine is stated healthy, so a resource still starting is not yet a pass.
|
||||
//
|
||||
// **An engine older than the judging states nothing**, and its machine's health is not known: never
|
||||
// healthy, never a reason to raise anything, and the gate judges it as it did before.
|
||||
|
||||
// The condition a module's health raises.
|
||||
const (
|
||||
kindModuleUnhealthy = "module-unhealthy"
|
||||
// sourceHealth is what raised it: the machine's own statement.
|
||||
sourceHealth = "health"
|
||||
// moduleUnhealthyUrgentAfter is how long it stands before it is urgent (to-be 48 §4).
|
||||
moduleUnhealthyUrgentAfter = 4 * time.Hour
|
||||
// moduleUnhealthyAfter is how many statements in a row raise it.
|
||||
moduleUnhealthyAfter = 2
|
||||
)
|
||||
|
||||
// healthRefused counts the statements refused as older than the one kept, for the log and a test.
|
||||
var healthRefused atomic.Int64
|
||||
|
||||
// moduleHealth keeps what the machines state, for the link (link.Healths).
|
||||
type moduleHealth struct {
|
||||
inv *inventory.Inventory
|
||||
keeper func() *conditions.Keeper
|
||||
}
|
||||
|
||||
func (m moduleHealth) Stated(ctx context.Context, node string, h link.Health) error {
|
||||
return stateHealth(ctx, m.inv, m.keeper(), node, h, time.Now())
|
||||
}
|
||||
|
||||
// stateHealth keeps one machine's statement and raises or clears its modules' conditions from it. An
|
||||
// older statement than the one kept is refused, by when the engine looked.
|
||||
func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, node string, h link.Health,
|
||||
now time.Time) error {
|
||||
if h.Contract == 0 {
|
||||
return nil
|
||||
}
|
||||
prev, had, err := inv.HealthOf(ctx, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if had && h.At.Before(prev.SaidAt) {
|
||||
healthRefused.Add(1)
|
||||
return nil
|
||||
}
|
||||
unhealthy := map[string][]inventory.ResourceHealth{}
|
||||
resources := make([]inventory.ResourceHealth, 0, len(h.Resources))
|
||||
for _, r := range h.Resources {
|
||||
kept := inventory.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target,
|
||||
State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts,
|
||||
Check: r.Check, Needs: r.Needs, Account: r.Account, Root: r.Root}
|
||||
resources = append(resources, kept)
|
||||
if r.State == link.StateUnhealthy && r.Module != "" {
|
||||
unhealthy[r.Module] = append(unhealthy[r.Module], kept)
|
||||
}
|
||||
}
|
||||
streaks := map[string]int{}
|
||||
for module := range unhealthy {
|
||||
streaks[module] = prev.Streaks[module] + 1
|
||||
}
|
||||
var network *inventory.NetworkHealth
|
||||
if h.Network != nil {
|
||||
network = &inventory.NetworkHealth{State: h.Network.State, Since: h.Network.Since, Parts: []inventory.NetworkPart{}}
|
||||
for _, p := range h.Network.Parts {
|
||||
network.Parts = append(network.Parts, inventory.NetworkPart{Part: p.Part, State: p.State, Reason: p.Reason,
|
||||
Said: p.Said, Writer: p.Writer, Owner: p.Owner, Toward: p.Toward, Since: p.Since, Streak: p.Streak})
|
||||
}
|
||||
}
|
||||
stored, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: node, Contract: h.Contract, SaidAt: h.At,
|
||||
HeardAt: now, Resources: resources, Streaks: streaks, Network: network, Units: unitsKept(h.Units)})
|
||||
if err != nil || !stored {
|
||||
if err == nil {
|
||||
healthRefused.Add(1)
|
||||
}
|
||||
return err
|
||||
}
|
||||
if k == nil {
|
||||
return nil
|
||||
}
|
||||
err = judgeModuleHealth(ctx, inv, k, node, unhealthy, streaks, now)
|
||||
// And the machine's own failed units (issue 315): what no module places, one finding for the machine.
|
||||
if uerr := judgeUnits(ctx, k, node, h.Units); uerr != nil {
|
||||
if err == nil {
|
||||
err = uerr
|
||||
} else {
|
||||
err = fmt.Errorf("%w; %v", err, uerr)
|
||||
}
|
||||
}
|
||||
// And every machine's network, from every machine's newest statement (ADR 0241): a statement about one
|
||||
// machine can hold another's finding, or release it.
|
||||
if nerr := judgeNetworks(ctx, inv, k, now); nerr != nil {
|
||||
if err == nil {
|
||||
return nerr
|
||||
}
|
||||
return fmt.Errorf("%w; %v", err, nerr)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// judgeModuleHealth raises a module's condition on a machine on the second statement in a row that says a
|
||||
// resource of it is unhealthy — or on the first while it is already open — and clears every one this
|
||||
// statement no longer says. **A consumer whose findings wait on an unhealthy provider is held** (to-be 48
|
||||
// §6): raised as nothing of its own, listed at the provider's condition, which is urgent while anyone
|
||||
// waits on it.
|
||||
func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, node string,
|
||||
unhealthy map[string][]inventory.ResourceHealth, streaks map[string]int, now time.Time) error {
|
||||
open, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
standing := map[string]conditions.Condition{}
|
||||
for _, c := range open {
|
||||
if (c.Kind == kindModuleUnhealthy || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound) &&
|
||||
c.Subject.Machine == node {
|
||||
standing[c.Key] = c
|
||||
}
|
||||
}
|
||||
var hold *holding
|
||||
if inv != nil {
|
||||
if hold, err = readHolding(ctx, inv, open); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
var problems []string
|
||||
modules := make([]string, 0, len(unhealthy))
|
||||
for m := range unhealthy {
|
||||
modules = append(modules, m)
|
||||
}
|
||||
sort.Strings(modules)
|
||||
seen := map[string]bool{}
|
||||
// became is, per module, the kind of condition this statement says of it: what a standing one of the
|
||||
// other kind turned into.
|
||||
became := map[string]string{}
|
||||
heldOn := map[string]string{}
|
||||
providers := map[catalogue.Chosen]bool{}
|
||||
for _, m := range modules {
|
||||
// **A directory used as found is said as that** (novox/hq issue 339): the operator's to hand over at the
|
||||
// machine, never urgent — nothing is broken by the wait that a person was not told of — and its own kind,
|
||||
// so the gate never reads it as a fault of the build that happened to be sent beside it.
|
||||
if said, waits := foundWait(m, node, unhealthy[m]); waits {
|
||||
o := usedAsFoundObservation(m, node, said, unhealthy[m])
|
||||
seen[o.Key()] = true
|
||||
became[m] = kindUsedAsFound
|
||||
if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen {
|
||||
continue
|
||||
}
|
||||
if _, err := k.Observe(ctx, o); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
continue
|
||||
}
|
||||
// **A wait for a person's new login is said as that** (novox/hq ADR 0254): one plain sentence to the
|
||||
// operator, never urgent, cleared on the first statement that no longer says it.
|
||||
if said, waits := personWait(m, node, unhealthy[m]); waits {
|
||||
o := reloginObservation(m, node, said, operatorOn(ctx, inv, node), unhealthy[m])
|
||||
// **A provider waiting for a login says who waits on it** (novox/hq issue 318 review): its
|
||||
// consumers are held under it, and its own condition is where they are said.
|
||||
if hold != nil {
|
||||
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
|
||||
}
|
||||
seen[o.Key()] = true
|
||||
became[m] = kindReloginNeeded
|
||||
if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen {
|
||||
continue
|
||||
}
|
||||
if _, err := k.Observe(ctx, o); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
continue
|
||||
}
|
||||
o := moduleUnhealthyObservation(m, node, unhealthy[m])
|
||||
if hold != nil {
|
||||
if p, held := hold.heldUnder(node, m, unhealthy[m]); held {
|
||||
// Held under the provider's condition: nothing of its own, and the provider's says it waits.
|
||||
heldOn[o.Key()] = p.Module + " on " + p.Node
|
||||
providers[p] = true
|
||||
continue
|
||||
}
|
||||
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
|
||||
}
|
||||
seen[o.Key()] = true
|
||||
became[m] = kindModuleUnhealthy
|
||||
c, isOpen := standing[o.Key()]
|
||||
if streaks[m] < moduleUnhealthyAfter && !isOpen {
|
||||
continue // unconfirmed: one statement can be wrong; `node show` lists it
|
||||
}
|
||||
if isOpen && now.Sub(c.Raised) >= moduleUnhealthyUrgentAfter {
|
||||
o.Severity = conditions.Urgent
|
||||
}
|
||||
if _, err := k.Observe(ctx, o); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
}
|
||||
for key, c := range standing {
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
module := strings.TrimSuffix(c.Subject.ID, "."+node)
|
||||
why := fmt.Sprintf("%s says no resource of %s is unhealthy", node, module)
|
||||
if c.Kind == kindReloginNeeded {
|
||||
why = fmt.Sprintf("%s says %s no longer waits for a new login", node, module)
|
||||
}
|
||||
if c.Kind == kindUsedAsFound {
|
||||
why = fmt.Sprintf("%s says no directory of %s is used as found any more", node, module)
|
||||
}
|
||||
if on, held := heldOn[key]; held {
|
||||
why = fmt.Sprintf("what %s finds on %s waits on %s, which is unhealthy: held under its condition", module, node, on)
|
||||
}
|
||||
// **A condition that became the other kind** is not "working again" (issue 318 review): its clearing
|
||||
// line says what it became.
|
||||
resolved := ""
|
||||
switch {
|
||||
case c.Kind == kindModuleUnhealthy && became[module] == kindReloginNeeded:
|
||||
why = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
|
||||
resolved = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
|
||||
case c.Kind == kindReloginNeeded && became[module] == kindModuleUnhealthy:
|
||||
why = fmt.Sprintf("%s on %s no longer waits for a new login, and is not healthy", module, node)
|
||||
resolved = fmt.Sprintf("The new login on %s is done, and %s still does not work", node, module)
|
||||
}
|
||||
if _, err := k.ClearSaying(ctx, key, why, resolved); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
}
|
||||
// And each provider a consumer here now waits on, when its own condition is open: said again with who
|
||||
// waits on it, so the wait is listed at the provider whichever machine's statement arrived first.
|
||||
for p := range providers {
|
||||
if err := sayWaiters(ctx, k, hold, p, now); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
return fmt.Errorf("%s", strings.Join(problems, "; "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// sayWaiters observes a provider's open condition again, with who waits on it, from its machine's newest
|
||||
// statement. Nothing when its condition is not open: it is raised by its own statements, on its own looks.
|
||||
func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p catalogue.Chosen, now time.Time) error {
|
||||
var raisedAt *conditions.Condition
|
||||
for i, c := range hold.open {
|
||||
if c.Key == moduleUnhealthyKey(p.Module, p.Node) || c.Key == reloginKey(p.Module, p.Node) {
|
||||
raisedAt = &hold.open[i]
|
||||
}
|
||||
}
|
||||
if raisedAt == nil {
|
||||
return nil
|
||||
}
|
||||
var rs []inventory.ResourceHealth
|
||||
for _, r := range hold.healths[p.Node].Resources {
|
||||
if r.Module == p.Module && r.State == link.StateUnhealthy {
|
||||
rs = append(rs, r)
|
||||
}
|
||||
}
|
||||
if len(rs) == 0 {
|
||||
return nil
|
||||
}
|
||||
o := moduleUnhealthyObservation(p.Module, p.Node, rs)
|
||||
if said, waits := personWait(p.Module, p.Node, rs); waits {
|
||||
o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rs)
|
||||
}
|
||||
if o.Key() != raisedAt.Key {
|
||||
return nil // its own statement says it next
|
||||
}
|
||||
sayWaitingOn(&o, hold.waitersOn(p))
|
||||
_, err := k.Observe(ctx, o)
|
||||
return err
|
||||
}
|
||||
|
||||
// reloginKey is a module's relogin-needed condition on a machine.
|
||||
func reloginKey(module, node string) string {
|
||||
return conditions.Key(conditions.ScopeModule, module+"."+node, kindReloginNeeded)
|
||||
}
|
||||
|
||||
// operatorOn is the operator's account on a machine, or "" when it is not known (to-be 29).
|
||||
func operatorOn(ctx context.Context, inv *inventory.Inventory, node string) string {
|
||||
if inv == nil {
|
||||
return ""
|
||||
}
|
||||
n, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return n.Account
|
||||
}
|
||||
|
||||
// reloginObservation is a module waiting for a person's new login on a machine (novox/hq ADR 0254): the
|
||||
// operator's, a warning, its summary the one sentence that says what to do; the resources are evidence. Its
|
||||
// plain words (ADR 0253) are the kind's: it needs the operator, and offers no answer — no verb can log a
|
||||
// person in again, and a restart of the module's service would start it in the same session. operator is
|
||||
// the machine's operator account, when known: the words say "your account" only for it.
|
||||
func reloginObservation(module, node, said, operator string, rs []inventory.ResourceHealth) conditions.Observation {
|
||||
o := moduleUnhealthyObservation(module, node, rs)
|
||||
o.Token, o.Kind, o.Resolver, o.Summary = kindReloginNeeded, kindReloginNeeded, conditions.ResolverOperator, said
|
||||
accounts := waitingAccounts(module, rs)
|
||||
yours := operator != "" && len(accounts) > 0
|
||||
for _, a := range accounts {
|
||||
yours = yours && a == operator
|
||||
}
|
||||
w := reloginWords(module, node, yours)
|
||||
o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions = w.Headline, w.Explanation, w.Resolved, w.Needs, nil
|
||||
return o
|
||||
}
|
||||
|
||||
// reloginWords is what the operator reads of a module waiting for a new login on a machine (ADR 0253,
|
||||
// ADR 0254): never quiet, since only a person can do it, and no button, since nothing else can. yours says
|
||||
// the account waiting is the operator's own on that machine; otherwise the words do not claim it is.
|
||||
func reloginWords(module, node string, yours bool) words {
|
||||
if yours {
|
||||
return words{Headline: fmt.Sprintf("%s waits for a new login on %s", module, node),
|
||||
Needs: fmt.Sprintf("log out of %s completely and log in again, or restart it.", node),
|
||||
Explanation: fmt.Sprintf("%s put your account in a group it needs. You logged in before that, so %s "+
|
||||
"cannot run until you log in again. Its update is in place and nothing was undone.",
|
||||
conditions.Capital(module), module),
|
||||
Resolved: fmt.Sprintf("%s runs on %s after your new login", module, node)}
|
||||
}
|
||||
return words{Headline: fmt.Sprintf("%s waits for a new login on %s", module, node),
|
||||
Needs: fmt.Sprintf("have the account it names log out of %s completely and log in again, or restart %s.", node, node),
|
||||
Explanation: fmt.Sprintf("%s put an account on %s in a group it needs. That account logged in before that, "+
|
||||
"so %s cannot run until it logs in again. Its update is in place and nothing was undone.",
|
||||
conditions.Capital(module), node, module),
|
||||
Resolved: fmt.Sprintf("%s runs on %s after the new login", module, node)}
|
||||
}
|
||||
|
||||
// waitingAccounts is every account of a module whose resource says it waits for a new login, sorted.
|
||||
func waitingAccounts(module string, rs []inventory.ResourceHealth) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, r := range rs {
|
||||
if r.Module == module && r.Kind == link.KindAccount && r.State == link.StateUnhealthy &&
|
||||
strings.HasPrefix(r.Reason, link.ReasonRelogin) && accountOf(r) != "" && !seen[accountOf(r)] {
|
||||
seen[accountOf(r)] = true
|
||||
out = append(out, accountOf(r))
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// sayWaitingOn adds to a module's condition the consumers held under it (to-be 48 §6): urgent while anyone
|
||||
// waits on it, whether it is not working or waits for a new login.
|
||||
func sayWaitingOn(o *conditions.Observation, waiters []string) {
|
||||
if len(waiters) == 0 {
|
||||
return
|
||||
}
|
||||
o.Severity = conditions.Urgent
|
||||
o.Said += "; " + waitingWords(waiters)
|
||||
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
|
||||
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
|
||||
}
|
||||
|
||||
// moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module,
|
||||
// the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence.
|
||||
func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {
|
||||
var words, said, plain []string
|
||||
needs := moduleNeeds(node, rs)
|
||||
for _, r := range rs {
|
||||
plain = append(plain, resourcePlainWords(r))
|
||||
if r.Kind == link.KindUnit {
|
||||
// A failed unit is named by the unit, which is what a person looks for (issue 315); the
|
||||
// resource that places it — a package, a file — is evidence.
|
||||
words = append(words, fmt.Sprintf("its unit %s %s", r.Target, r.Reason))
|
||||
said = append(said, fmt.Sprintf("%s (unit %s, placed by %s): %s, since %s", r.Target, r.Target, r.Resource,
|
||||
orNotSaid(r.Reason), r.Since.UTC().Format("2006-01-02 15:04:05 MST")))
|
||||
continue
|
||||
}
|
||||
words = append(words, fmt.Sprintf("its %s %s %s", r.Kind, r.Resource, reasonWords(r)))
|
||||
said = append(said, fmt.Sprintf("%s (%s %s): %s, %d look(s) in a row, %d restart(s) counted, since %s",
|
||||
r.Resource, r.Kind, r.Target, orNotSaid(r.Reason), r.Streak, r.Restarts,
|
||||
r.Since.UTC().Format("2006-01-02 15:04:05 MST")))
|
||||
}
|
||||
return conditions.Observation{Scope: conditions.ScopeModule, ID: module + "." + node, Token: "unhealthy",
|
||||
Kind: kindModuleUnhealthy, Machine: node, Severity: conditions.Warning, Source: sourceHealth,
|
||||
Summary: fmt.Sprintf("%s on %s is not healthy: %s", module, node, strings.Join(words, "; ")),
|
||||
Said: strings.Join(said, "; "),
|
||||
Headline: fmt.Sprintf("%s not working on %s", module, node),
|
||||
Explanation: fmt.Sprintf("%s on %s is not healthy: %s. It clears as soon as it runs again.", module, node,
|
||||
namesWords(plain, 3)),
|
||||
Needs: needs,
|
||||
Actions: moduleActions(node, rs),
|
||||
Resolved: fmt.Sprintf("%s works again on %s", module, node)}
|
||||
}
|
||||
|
||||
// reasonWords is why a resource is unhealthy, as a person reads it.
|
||||
func reasonWords(r inventory.ResourceHealth) string {
|
||||
// An account waiting for a new login (ADR 0252) is said in the mesh's words, not the engine's.
|
||||
if r.Kind == link.KindAccount && strings.HasPrefix(r.Reason, link.ReasonRelogin) {
|
||||
return fmt.Sprintf("waits for a new login of %s, which is in the group and logged in before it was", accountOf(r))
|
||||
}
|
||||
switch r.Reason {
|
||||
case "restarting":
|
||||
return fmt.Sprintf("keeps restarting (%d restart(s) counted)", r.Restarts)
|
||||
case "down":
|
||||
return "is not running"
|
||||
case "":
|
||||
return "is unhealthy"
|
||||
}
|
||||
// What a declared check found says an endpoint, a path or an address: evidence, never the summary the
|
||||
// operator's channel carries (ADR 0234 §6). The summary names the check.
|
||||
if r.Check != "" {
|
||||
return "fails its " + r.Check + " check"
|
||||
}
|
||||
return "is unhealthy: " + r.Reason
|
||||
}
|
||||
|
||||
func orNotSaid(s string) string {
|
||||
if s == "" {
|
||||
return "no reason said"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// kindReloginNeeded is a module's condition while it waits for a person's new login on a machine (novox/hq
|
||||
// ADR 0254): its own kind, so that neither the gate nor anyone reading the conditions takes it for a fault.
|
||||
const kindReloginNeeded = "relogin-needed"
|
||||
|
||||
// personWait is whether everything unhealthy of a module on a machine waits for one person's new login, and
|
||||
// that wait in one plain sentence (novox/hq ADR 0254, issue 318). Narrow on purpose, so that a fault is never
|
||||
// excused:
|
||||
//
|
||||
// - at least one of the module's resources is its account (kind account), unhealthy with a reason that
|
||||
// starts "relogin needed" (ADR 0252): the database lists the account in the group, and the session
|
||||
// running began before;
|
||||
// - every other unhealthy resource of the module runs in the own service manager of one of those very
|
||||
// accounts (Account names it): the manager that began before the group and does not hold it.
|
||||
//
|
||||
// Anything else unhealthy of the module — a container, a unit of the machine's own manager, a unit in
|
||||
// another account's manager, a resource whose engine does not say whose manager it is in, an account
|
||||
// not in its group or that could not be read — is not a wait, and the module is judged as before. A
|
||||
// resource still starting is not unhealthy and does not make a wait either. Pure.
|
||||
func personWait(module, machine string, rs []inventory.ResourceHealth) (string, bool) {
|
||||
accounts := waitingAccounts(module, rs)
|
||||
if len(accounts) == 0 {
|
||||
return "", false
|
||||
}
|
||||
waiting := map[string]bool{}
|
||||
for _, a := range accounts {
|
||||
waiting[a] = true
|
||||
}
|
||||
var units []string
|
||||
for _, r := range rs {
|
||||
if r.Module != module || r.State != link.StateUnhealthy {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case r.Kind == link.KindAccount && strings.HasPrefix(r.Reason, link.ReasonRelogin) && waiting[accountOf(r)]:
|
||||
case r.Kind != link.KindAccount && r.Account != "" && waiting[r.Account]:
|
||||
units = append(units, r.Target)
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
said := fmt.Sprintf("relogin needed on %s: %s waits for a new login of %s, which is in its group and whose "+
|
||||
"login began before it was; log out of %[1]s completely and log in again, or restart it", machine, module,
|
||||
strings.Join(accounts, ", "))
|
||||
if len(units) > 0 {
|
||||
sort.Strings(units)
|
||||
said += fmt.Sprintf(" (until then %s cannot run)", strings.Join(units, ", "))
|
||||
}
|
||||
return said, true
|
||||
}
|
||||
|
||||
// accountOf is the account a resource of kind account is: named by the engine, or its target.
|
||||
func accountOf(r inventory.ResourceHealth) string {
|
||||
if r.Account != "" {
|
||||
return r.Account
|
||||
}
|
||||
return r.Target
|
||||
}
|
||||
|
||||
// moduleHealthWord is the gate's reading of a module's stated health on a machine (ADR 0240 §4, ADR 0236
|
||||
// §2 as amended): good when every long-running resource of it is stated healthy in a statement heard since
|
||||
// the send; not yet otherwise, saying which. A machine that never stated health is judged as before.
|
||||
//
|
||||
// **A wait for a person is its own reading** (novox/hq ADR 0254): when everything unhealthy of the module
|
||||
// waits for one person's new login (personWait), the reading is healthPerson — not a fault of the build,
|
||||
// and not something a machine can meet within a bound.
|
||||
func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (health, string) {
|
||||
if f.healthErr != nil {
|
||||
return healthNotYet, "what " + machine + " says of its resources' health cannot be read: " + firstLine(f.healthErr.Error())
|
||||
}
|
||||
h, states := f.health[machine]
|
||||
if !states {
|
||||
return healthGood, ""
|
||||
}
|
||||
if h.HeardAt.Before(since) {
|
||||
return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module)
|
||||
}
|
||||
wait, waits := personWait(module, machine, h.Resources)
|
||||
// **Only a build whose own send put the account in a new group is excused** (issue 318 review): read from
|
||||
// what the controller sent, never from when the machine says the wait began — that time is the engine's
|
||||
// memory, reset by its restart and moved by a change of words. A build that adds no account group cannot
|
||||
// have brought a wait, so a later build sent while the login is still owed is judged as before.
|
||||
if waits && !f.groupsAdded[module] {
|
||||
waits = false
|
||||
}
|
||||
var found []string
|
||||
for _, r := range h.Resources {
|
||||
if r.Module != module {
|
||||
continue
|
||||
}
|
||||
if waits && r.State == link.StateUnhealthy {
|
||||
continue
|
||||
}
|
||||
// **A directory used as found before this send waits for a person** (novox/hq issue 339): the node-engine
|
||||
// left its owner and mode, and only someone at the machine can hand it over. It is no fault of this
|
||||
// build, so it does not hold the module's walk — an urgent fix still goes through — and the verdict
|
||||
// carries the wait. Found by this very send, the send brought it, and it is judged as unhealthy.
|
||||
if usedAsFound(r) && r.Since.Before(since) {
|
||||
found = append(found, r.Resource)
|
||||
continue
|
||||
}
|
||||
switch r.State {
|
||||
case link.StateHealthy:
|
||||
case link.StateStarting:
|
||||
return healthNotYet, fmt.Sprintf("its %s %s on %s is still starting", r.Kind, r.Resource, machine)
|
||||
case link.StateUnhealthy:
|
||||
if on, held := f.heldOn[module+"@"+machine]; held {
|
||||
return healthWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which is unhealthy", r.Kind,
|
||||
r.Resource, machine, on)
|
||||
}
|
||||
return healthNotYet, fmt.Sprintf("its %s %s on %s %s", r.Kind, r.Resource, machine, reasonWords(r))
|
||||
default:
|
||||
return healthNotYet, fmt.Sprintf("its %s %s on %s is %s%s", r.Kind, r.Resource, machine, r.State,
|
||||
reasonAfter(r.Reason))
|
||||
}
|
||||
}
|
||||
if waits || len(found) > 0 {
|
||||
var said []string
|
||||
if waits {
|
||||
said = append(said, wait)
|
||||
}
|
||||
if len(found) > 0 {
|
||||
said = append(said, fmt.Sprintf("on %s, %s uses %s as found and waits for the operator to hand it over "+
|
||||
"(`nox node hand-over %s <directory>` on the control-node)", machine, module, strings.Join(found, ", "), machine))
|
||||
}
|
||||
return healthPerson, strings.Join(said, "; ")
|
||||
}
|
||||
return healthGood, ""
|
||||
}
|
||||
|
||||
// usedAsFound is a directory the node-engine states it uses as found (novox/hq issue 339).
|
||||
func usedAsFound(r inventory.ResourceHealth) bool {
|
||||
return r.Kind == link.KindDirectory && r.State == link.StateUnhealthy && strings.HasPrefix(r.Reason, link.ReasonUsedAsFound)
|
||||
}
|
||||
|
||||
func reasonAfter(s string) string {
|
||||
if s == "" {
|
||||
return ""
|
||||
}
|
||||
return ": " + s
|
||||
}
|
||||
|
||||
// healthLines is what `node show` says of a machine's long-running resources: each with its state and
|
||||
// since when, an unhealthy one said once marked unconfirmed.
|
||||
func healthLines(h inventory.NodeHealth, had bool, now time.Time) []string {
|
||||
if !had {
|
||||
return []string{" its node-engine does not say how what it runs is — it is older than the judging (ADR 0240)"}
|
||||
}
|
||||
if len(h.Resources) == 0 {
|
||||
return []string{fmt.Sprintf(" it runs nothing long-lived for a module (said %s ago)", roughly(now.Sub(h.HeardAt)))}
|
||||
}
|
||||
out := []string{fmt.Sprintf(" what it runs, as it said %s ago:", roughly(now.Sub(h.HeardAt)))}
|
||||
for _, r := range h.Resources {
|
||||
line := fmt.Sprintf(" %-10s %-34s %s %s, since %s", r.State, r.Resource, r.Kind, r.Target,
|
||||
r.Since.Local().Format("2006-01-02 15:04"))
|
||||
if r.Reason != "" {
|
||||
line += " — " + r.Reason
|
||||
}
|
||||
if r.Restarts > 0 {
|
||||
line += fmt.Sprintf(", %d restart(s) counted", r.Restarts)
|
||||
}
|
||||
if r.State == link.StateUnhealthy && h.Streaks[r.Module] < moduleUnhealthyAfter {
|
||||
line += " (unconfirmed: said once)"
|
||||
}
|
||||
out = append(out, line)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// accountGroups is every account group a manifest declares, as "<account>/<group>": a user resource's
|
||||
// groups (ADR 0252).
|
||||
func accountGroups(m catalogue.Manifest) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, r := range m.Resources {
|
||||
if t, _ := r["type"].(string); t != "user" {
|
||||
continue
|
||||
}
|
||||
name, _ := r["name"].(string)
|
||||
groups, _ := r["groups"].([]any)
|
||||
for _, g := range groups {
|
||||
if group, ok := g.(string); ok && group != "" {
|
||||
out[name+"/"+group] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// addsAccountGroups is whether a move from one manifest of a module to another puts an account in a group the
|
||||
// earlier one did not (issue 318 review): the only send that can bring a wait for a new login. A module new to
|
||||
// the machine (no earlier manifest) adds every group it declares.
|
||||
func addsAccountGroups(from catalogue.Manifest, hadFrom bool, to catalogue.Manifest) bool {
|
||||
before := map[string]bool{}
|
||||
if hadFrom {
|
||||
before = accountGroups(from)
|
||||
}
|
||||
for g := range accountGroups(to) {
|
||||
if !before[g] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// kindUsedAsFound is a module's condition while the node-engine uses one of its directories as found (novox/hq
|
||||
// issue 339): its own kind, the operator's, never urgent, and never read by the gate as a fault of a build.
|
||||
const kindUsedAsFound = "directory-used-as-found"
|
||||
|
||||
// usedAsFoundKey is a module's used-as-found condition on a machine.
|
||||
func usedAsFoundKey(module, node string) string {
|
||||
return conditions.Key(conditions.ScopeModule, module+"."+node, kindUsedAsFound)
|
||||
}
|
||||
|
||||
// foundWait is whether everything unhealthy of a module on a machine is a directory used as found, and that in
|
||||
// one sentence. Anything else unhealthy beside it is judged as a fault, with the directory among its resources.
|
||||
func foundWait(module, node string, rs []inventory.ResourceHealth) (string, bool) {
|
||||
var ids, why []string
|
||||
for _, r := range rs {
|
||||
if r.Module != module || r.State != link.StateUnhealthy {
|
||||
continue
|
||||
}
|
||||
if !usedAsFound(r) {
|
||||
return "", false
|
||||
}
|
||||
ids = append(ids, r.Resource)
|
||||
why = append(why, strings.TrimSpace(strings.TrimPrefix(r.Reason, link.ReasonUsedAsFound)))
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
return "", false
|
||||
}
|
||||
return fmt.Sprintf("%s on %s uses %s as found: %s", module, node, strings.Join(ids, ", "),
|
||||
strings.Join(why, "; ")), true
|
||||
}
|
||||
|
||||
// usedAsFoundObservation is a module whose directory the node-engine uses as found, in words: the operator's, a
|
||||
// warning however long it stays, its summary naming the directories and their owners; the paths are evidence.
|
||||
func usedAsFoundObservation(module, node, said string, rs []inventory.ResourceHealth) conditions.Observation {
|
||||
o := moduleUnhealthyObservation(module, node, rs)
|
||||
o.Token, o.Kind, o.Resolver, o.Severity, o.Summary = kindUsedAsFound, kindUsedAsFound, conditions.ResolverOperator,
|
||||
conditions.Warning, said
|
||||
o.Headline = fmt.Sprintf("%s waits for a directory on %s", module, node)
|
||||
o.Explanation = fmt.Sprintf("A directory of %s was already on %s, with another owner or mode than %s declares. "+
|
||||
"The mesh left it as it was rather than hand it to an account, so %s may not be able to use it.",
|
||||
module, node, module, module)
|
||||
// Plain words (ADR 0253): the line itself — `nox node hand-over <node> <path>` on the control-node (ADR 0272,
|
||||
// issue 356) — is in the summary and the evidence, which name the directory; a path is never in these.
|
||||
o.Needs = "hand the directory over from the control-node, as the operator; the details name it and the line to type."
|
||||
o.Resolved = fmt.Sprintf("%s's directory on %s is the mesh's", module, node)
|
||||
o.Actions = nil
|
||||
return o
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A module's stated health, as the controller keeps it and raises from it (novox/hq ADR 0240, "how it is
|
||||
// checked", rule 4): one unhealthy statement raises nothing and is listed unconfirmed; two raise; a
|
||||
// healthy one clears; a condition from before the send clears at the new build's start; an older
|
||||
// statement is refused; and an engine that states nothing raises nothing.
|
||||
|
||||
var h0 = time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
func aStatement(at time.Time, states ...string) link.Health {
|
||||
h := link.Health{Contract: link.LivenessContract, At: at}
|
||||
for i, s := range states {
|
||||
r := link.ResourceHealth{Module: "letta", Resource: "letta.server", Kind: "container", Target: "letta-server",
|
||||
State: s, Since: at}
|
||||
if i > 0 {
|
||||
r.Module, r.Resource, r.Target = "mqtt", "mqtt.broker", "mosquitto.service"
|
||||
}
|
||||
if s == link.StateUnhealthy {
|
||||
r.Reason, r.Restarts, r.Streak = "restarting", 4, 2
|
||||
}
|
||||
h.Resources = append(h.Resources, r)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
func TestTwoUnhealthyStatementsRaiseTheModulesConditionAndAHealthyOneClearsIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
k := conditionsFrom
|
||||
const key = "module.letta.anchor.unhealthy"
|
||||
openKeys := func() []string {
|
||||
t.Helper()
|
||||
list, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keys []string
|
||||
for _, c := range list {
|
||||
keys = append(keys, c.Key)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0, link.StateHealthy, link.StateHealthy), h0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// One statement: nothing raised, and `node show` lists it as unconfirmed.
|
||||
if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(time.Minute), link.StateUnhealthy, link.StateHealthy),
|
||||
h0.Add(time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if keys := openKeys(); len(keys) != 0 {
|
||||
t.Fatalf("one statement raised %v", keys)
|
||||
}
|
||||
kept, had, err := inv.HealthOf(ctx, "anchor")
|
||||
if err != nil || !had {
|
||||
t.Fatalf("the statement was not kept: %v %v", had, err)
|
||||
}
|
||||
if lines := strings.Join(healthLines(kept, had, h0.Add(time.Minute)), "\n"); !strings.Contains(lines, "unconfirmed") ||
|
||||
!strings.Contains(lines, "letta.server") {
|
||||
t.Fatalf("node show does not list the first statement as unconfirmed:\n%s", lines)
|
||||
}
|
||||
|
||||
// The second in a row raises it — the module's own, never the other module's on the machine.
|
||||
if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(2*time.Minute), link.StateUnhealthy, link.StateHealthy),
|
||||
h0.Add(2*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if keys := openKeys(); len(keys) != 1 || keys[0] != key {
|
||||
t.Fatalf("two statements raised %v, not %s", keys, key)
|
||||
}
|
||||
c, _, _ := k.Get(ctx, key)
|
||||
if c.Severity != conditions.Warning || c.Resolver != conditions.ResolverSelf || c.Subject.Machine != "anchor" ||
|
||||
!strings.Contains(c.Summary, "letta on anchor") || !strings.Contains(c.Summary, "keeps restarting") ||
|
||||
!strings.Contains(c.Evidence[0].Said, "letta-server") {
|
||||
t.Fatalf("the condition does not say it in words with its evidence: %+v", c)
|
||||
}
|
||||
|
||||
// Standing four hours, it is urgent.
|
||||
if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(5*time.Hour), link.StateUnhealthy, link.StateHealthy),
|
||||
time.Now().Add(5*time.Hour)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c, _, _ := k.Get(ctx, key); c.Severity != conditions.Urgent {
|
||||
t.Fatalf("unhealthy for four hours is still %s", c.Severity)
|
||||
}
|
||||
|
||||
// A new build's start — every start begins in `starting` — clears it: what follows is the new build's.
|
||||
if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(6*time.Hour), link.StateStarting, link.StateHealthy),
|
||||
h0.Add(6*time.Hour)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if keys := openKeys(); len(keys) != 0 {
|
||||
t.Fatalf("the first statement that says no resource is unhealthy did not clear it: %v", keys)
|
||||
}
|
||||
// And the streak starts again: one unhealthy statement after it raises nothing.
|
||||
if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(7*time.Hour), link.StateUnhealthy), h0.Add(7*time.Hour)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if keys := openKeys(); len(keys) != 0 {
|
||||
t.Fatalf("one statement after a clearing raised %v", keys)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOlderHealthStatementIsRefused(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
before := healthRefused.Load()
|
||||
if err := stateHealth(ctx, inv, nil, "anchor", aStatement(h0.Add(time.Minute), link.StateHealthy), h0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// An event said before the report that overtook it arrives late.
|
||||
if err := stateHealth(ctx, inv, nil, "anchor", aStatement(h0, link.StateUnhealthy), h0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kept, _, err := inv.HealthOf(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !kept.SaidAt.Equal(h0.Add(time.Minute)) || kept.Resources[0].State != link.StateHealthy {
|
||||
t.Fatalf("the older statement replaced the newer: %+v", kept)
|
||||
}
|
||||
if healthRefused.Load() != before+1 {
|
||||
t.Fatalf("the refusal was not counted")
|
||||
}
|
||||
}
|
||||
|
||||
// **An engine older than the judging states nothing**: its reports raise nothing, keep nothing, and the
|
||||
// gate judges its machine as before — never healthy for having said nothing, never unhealthy.
|
||||
func TestAReportWithNoHealthRaisesAndKeepsNothing(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
l := nudgingListener{Enrolment: link.Enrolment{Inventory: open.inventory}}
|
||||
if _, err := l.Heard(ctx, link.Report{Node: "anchor", Declared: "d1", Applied: []string{"letta.server"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, had, err := open.inventory.HealthOf(ctx, "anchor"); err != nil || had {
|
||||
t.Fatalf("health kept for an engine that said none: %v %v", had, err)
|
||||
}
|
||||
if lines := healthLines(inventory.NodeHealth{}, false, time.Now()); !strings.Contains(lines[0], "older than the judging") {
|
||||
t.Fatalf("node show: %v", lines)
|
||||
}
|
||||
// And one that does, through the report, is kept.
|
||||
h := aStatement(time.Now().UTC(), link.StateHealthy)
|
||||
if _, err := l.Heard(ctx, link.Report{Node: "anchor", Declared: "d1", Health: &h}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if kept, had, err := open.inventory.HealthOf(ctx, "anchor"); err != nil || !had || len(kept.Resources) != 1 {
|
||||
t.Fatalf("the report's health was not kept: %+v %v %v", kept, had, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A declared `health` is sent only to an engine whose own statement says it reads it (novox/hq ADR 0240
|
||||
// Phase B): an older engine is strict and would refuse the whole declaration for the field.
|
||||
func TestHealthIsSentOnlyToAnEngineThatSaysItReadsIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
reads := func() bool {
|
||||
t.Helper()
|
||||
got, err := engineReadsHealth(ctx, inv, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
if reads() {
|
||||
t.Fatal("an engine that never stated anything is sent health")
|
||||
}
|
||||
if err := stateHealth(ctx, inv, nil, "anchor", aStatement(h0, link.StateHealthy), h0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if reads() {
|
||||
t.Fatal("an engine judging liveness alone is sent health")
|
||||
}
|
||||
later := aStatement(h0.Add(time.Minute), link.StateHealthy)
|
||||
later.Contract = link.ReadinessContract
|
||||
if err := stateHealth(ctx, inv, nil, "anchor", later, h0.Add(time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reads() {
|
||||
t.Fatal("an engine that reads health is not sent it")
|
||||
}
|
||||
}
|
||||
+399
-17
@@ -10,10 +10,12 @@ import (
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
@@ -40,7 +42,12 @@ func providedModules() []catalogue.Manifest {
|
||||
// and neither could be swapped for anything, which is the test of whether a thing is a
|
||||
// module at all (novox/hq ADR 0040). They existed because computed output needed somewhere
|
||||
// to live, and now a module says where it wants it — `facts` in its own manifest.
|
||||
overlay.Manifest(), overlay.DomainManifest(),
|
||||
//
|
||||
// **And the bundle that required it is gone** (novox/hq ADR 0226): `networking` named this
|
||||
// module's requirement and nothing else, so every machine carried two modules for one
|
||||
// network. A machine is assigned the private network itself; what a release stops shipping
|
||||
// is retired at the next start (stores.go, Inventory.RetireUnshipped).
|
||||
overlay.Manifest(),
|
||||
} {
|
||||
var m catalogue.Manifest
|
||||
b, _ := json.Marshal(raw)
|
||||
@@ -177,6 +184,9 @@ func moduleCommand(ctx context.Context, args []string) error {
|
||||
Requires []string `json:"requires,omitempty"`
|
||||
Claims []string `json:"claims,omitempty"`
|
||||
Capabilities []string `json:"capabilities,omitempty"`
|
||||
// Replaces is what each of its own tools replaces (novox/hq ADR 0245), for the console's
|
||||
// search and the agent's instructions.
|
||||
Replaces map[string][]string `json:"replaces,omitempty"`
|
||||
}
|
||||
out := make([]listed, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
@@ -184,7 +194,7 @@ func moduleCommand(ctx context.Context, args []string) error {
|
||||
l := listed{Module: m.Module, Version: m.Version, Built: e.Source.BuiltFrom, Head: e.Source.Head,
|
||||
Current: e.Provided || e.Source.Repository == "" || e.Source.Current(), Provided: e.Provided,
|
||||
On: append([]string{}, e.On...), Provides: m.Offers(), Requires: m.Requires,
|
||||
Capabilities: m.Capabilities, Tools: declaresTools(m)}
|
||||
Capabilities: m.Capabilities, Tools: declaresTools(m), Replaces: m.Replaces}
|
||||
for _, c := range m.Claims {
|
||||
l.Claims = append(l.Claims, c.At()+"/"+c.Name)
|
||||
}
|
||||
@@ -347,9 +357,20 @@ func moduleCommand(ctx context.Context, args []string) error {
|
||||
func assignCommand(ctx context.Context, verb string, args []string) error {
|
||||
// Several modules in one act (novox/hq ADR 0207): holders that depend on each other — the
|
||||
// service manager and the package manager — can only go on, or come off, together.
|
||||
set := flag.NewFlagSet(verb, flag.ContinueOnError)
|
||||
// A module known and not built: ask for its build, and keep the assignment pending on it (novox/hq
|
||||
// issue 325). Only assign reads it.
|
||||
build := set.Bool("build", false, "for a module known and not built: ask for its build, and assign it when it registers")
|
||||
args, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(args) < 2 {
|
||||
return fmt.Errorf("%s <node> <module> [<module>…]", verb)
|
||||
}
|
||||
if *build && verb == "unassign" {
|
||||
return errors.New("unassign takes no --build")
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -358,7 +379,9 @@ func assignCommand(ctx context.Context, verb string, args []string) error {
|
||||
|
||||
// The act itself is in acts.go, so the command API refuses exactly what this refuses
|
||||
// (novox/hq ADR 0035). What differs between the surfaces is how the answer is printed.
|
||||
act := assign
|
||||
act := func(ctx context.Context, open *stores, node string, modules ...string) (string, error) {
|
||||
return assignWith(ctx, open, node, assignOptions{Build: *build}, modules...)
|
||||
}
|
||||
if verb == "unassign" {
|
||||
act = unassign
|
||||
}
|
||||
@@ -375,7 +398,10 @@ func assignCommand(ctx context.Context, verb string, args []string) error {
|
||||
|
||||
func settingsCommand(ctx context.Context, args []string) error {
|
||||
if len(args) == 0 {
|
||||
return errors.New("settings set <module> <file> [--node <node>], or settings clear <module> [--node <node>]")
|
||||
return errors.New("settings show <module> [--node <node>] [--history], settings set <module> <file> " +
|
||||
"[--node <node>] [--replace], settings clear <module> [--node <node>], settings preferences " +
|
||||
"[<module>] [--node <node>], settings propose <module> <file | --clear> [--node <node>] [--replace], " +
|
||||
"or settings proposals [<id>]")
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
@@ -386,10 +412,41 @@ func settingsCommand(ctx context.Context, args []string) error {
|
||||
|
||||
set := flag.NewFlagSet("settings", flag.ContinueOnError)
|
||||
node := set.String("node", "", "one machine, rather than the whole mesh")
|
||||
// **What a set removes is refused unless meant** (novox/hq ADR 0217). A layer is replaced whole,
|
||||
// and on 2026-10-05 setting one placement dropped a machine's whole layer for a module without a
|
||||
// word (novox/hq issue 304). Adding and changing keys needs nothing; removing one needs this.
|
||||
replace := set.Bool("replace", false, "for set and propose: remove the keys the new layer does not name")
|
||||
history := set.Bool("history", false, "for show: the layers this one replaced, the latest first")
|
||||
clear := set.Bool("clear", false, "for propose: propose that the layer be removed")
|
||||
positionals, err := parseAround(set, args[1:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch args[0] {
|
||||
case "propose":
|
||||
// A trusted setting, proposed by anyone and set only on the operator's warrant (novox/hq ADR 0277):
|
||||
// the stores are opened there, so the proposal and the verb's refusals below never meet.
|
||||
if len(positionals) < 1 || len(positionals) > 2 {
|
||||
return errors.New("settings propose <module> <settings.json | {…}> [--node <node>] [--replace], or settings propose <module> --clear [--node <node>]")
|
||||
}
|
||||
values := ""
|
||||
if len(positionals) == 2 {
|
||||
values = positionals[1]
|
||||
}
|
||||
return proposeCommand(ctx, positionals[0], *node, values, *clear, *replace)
|
||||
case "proposals":
|
||||
if len(positionals) > 1 || *node != "" || *clear || *replace || *history {
|
||||
return errors.New("settings proposals [<id>]")
|
||||
}
|
||||
id := ""
|
||||
if len(positionals) == 1 {
|
||||
id = positionals[0]
|
||||
}
|
||||
return proposalsCommand(ctx, id)
|
||||
}
|
||||
if *clear {
|
||||
return errors.New("--clear is for settings propose; a layer is cleared with settings clear")
|
||||
}
|
||||
|
||||
where := "the whole mesh"
|
||||
if *node != "" {
|
||||
@@ -413,34 +470,288 @@ func settingsCommand(ctx context.Context, args []string) error {
|
||||
if err := json.Unmarshal(raw, &values); err != nil {
|
||||
return fmt.Errorf("%s is not a settings file: %w", positionals[1], err)
|
||||
}
|
||||
if err := inv.SetSettings(ctx, *node, positionals[0], values); err != nil {
|
||||
before, _, err := inv.Layer(ctx, *node, positionals[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var keys []string
|
||||
for k := range values {
|
||||
keys = append(keys, k)
|
||||
if err := refuseTerminalSettingsThroughAVerb(ctx, inv, before, values, positionals[0], where); err != nil {
|
||||
return err
|
||||
}
|
||||
sort.Strings(keys)
|
||||
fmt.Printf("%s on %s: %s\n", positionals[0], where, strings.Join(keys, ", "))
|
||||
fmt.Println(" run `push` to send it")
|
||||
added, changed, removed := settingsChange(before, values)
|
||||
if len(removed) > 0 && !*replace {
|
||||
return fmt.Errorf("%s on %s: this layer would no longer set %s. A layer is replaced whole; "+
|
||||
"read it with `settings show %s%s` and include what should stay, or add --replace if the "+
|
||||
"removal is meant (novox/hq ADR 0217). Nothing was changed",
|
||||
positionals[0], where, strings.Join(removed, ", "), positionals[0], nodeFlag(*node))
|
||||
}
|
||||
if err := inv.SetSettingsBy(ctx, *node, positionals[0], values, setByWords()); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s on %s:\n", positionals[0], where)
|
||||
if len(added)+len(changed)+len(removed) == 0 {
|
||||
fmt.Println(" nothing changed")
|
||||
}
|
||||
for _, p := range added {
|
||||
fmt.Printf(" + %s\n", p)
|
||||
}
|
||||
for _, p := range changed {
|
||||
fmt.Printf(" ~ %s\n", p)
|
||||
}
|
||||
for _, p := range removed {
|
||||
fmt.Printf(" - %s\n", p)
|
||||
}
|
||||
if before != nil {
|
||||
fmt.Printf(" the layer it replaced is kept: settings show %s%s --history\n", positionals[0], nodeFlag(*node))
|
||||
}
|
||||
if *node != "" {
|
||||
fmt.Printf(" run `plan %s --diff` to see what it changes, `push %s` to send it\n", *node, *node)
|
||||
} else {
|
||||
fmt.Println(" run `push --behind` to send it to the machines running it")
|
||||
}
|
||||
return nil
|
||||
|
||||
case "show":
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("settings show <module> [--node <node>] [--history]")
|
||||
}
|
||||
if *history {
|
||||
past, err := inv.SettingsHistory(ctx, *node, positionals[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(past) == 0 {
|
||||
fmt.Printf("%s on %s: no layer has been replaced\n", positionals[0], where)
|
||||
return nil
|
||||
}
|
||||
for _, p := range past {
|
||||
shown, _ := json.MarshalIndent(p.Values, " ", " ")
|
||||
by := ""
|
||||
if p.SetBy != "" {
|
||||
by = "; " + p.SetBy
|
||||
}
|
||||
fmt.Printf("%s on %s, until %s (%s%s):\n %s\n", positionals[0], where,
|
||||
p.ReplacedAt.Local().Format("2006-01-02 15:04:05"), p.ReplacedBy, by, shown)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
values, has, err := inv.Layer(ctx, *node, positionals[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !has {
|
||||
fmt.Printf("%s on %s: no layer — the module's definition says\n", positionals[0], where)
|
||||
} else {
|
||||
shown, err := json.MarshalIndent(values, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(shown))
|
||||
// And who set it (novox/hq ADR 0277): a layer the operator approved on their phone says so.
|
||||
if setBy, setAt, has, err := inv.LayerOrigin(ctx, *node, positionals[0]); err != nil {
|
||||
return err
|
||||
} else if has && setBy != "" {
|
||||
fmt.Printf(" %s\n", setBy)
|
||||
} else if has {
|
||||
fmt.Printf(" set at %s; who set it was not kept\n", setAt.Local().Format("2006-01-02 15:04"))
|
||||
}
|
||||
}
|
||||
// Every value the module gives a default or a layer sets, and where it came from (novox/hq
|
||||
// ADR 0262): the default, the mesh's layer, or this node's. Said after the layer, which stays
|
||||
// the first thing printed because a caller reads it before replacing it (ADR 0217).
|
||||
known, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m, ok := known[positionals[0]]
|
||||
if !ok || len(m.Settings) == 0 {
|
||||
return nil
|
||||
}
|
||||
var layers []catalogue.Layer
|
||||
if *node != "" {
|
||||
if mesh, has, err := inv.Layer(ctx, "", positionals[0]); err != nil {
|
||||
return err
|
||||
} else if has {
|
||||
layers = append(layers, catalogue.Layer{From: catalogue.MeshWideLayer, Values: mesh})
|
||||
}
|
||||
if has {
|
||||
layers = append(layers, catalogue.Layer{From: *node, Values: values})
|
||||
}
|
||||
} else if has {
|
||||
layers = append(layers, catalogue.Layer{From: catalogue.MeshWideLayer, Values: values})
|
||||
}
|
||||
fmt.Print(describeEffective(positionals[0], where, catalogue.Effective(m, layers)))
|
||||
return nil
|
||||
|
||||
case "preferences":
|
||||
// Every module's preferences, and each machine's value with its source (novox/hq ADR 0262).
|
||||
if len(positionals) > 1 {
|
||||
return errors.New("settings preferences [<module>] [--node <node>]")
|
||||
}
|
||||
only := ""
|
||||
if len(positionals) == 1 {
|
||||
only = positionals[0]
|
||||
}
|
||||
if *node != "" {
|
||||
// A machine the mesh does not know is refused, never answered with an empty listing.
|
||||
if _, err := inv.NodeByName(ctx, *node); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var listed []preferencesOf
|
||||
for _, e := range entries {
|
||||
m := e.Manifest
|
||||
if len(m.Settings) == 0 || (only != "" && m.Module != only) {
|
||||
continue
|
||||
}
|
||||
if *node != "" && !containsString(e.On, *node) {
|
||||
// Asked for one machine: a module not on it has no value there to say.
|
||||
continue
|
||||
}
|
||||
p := preferencesOf{Manifest: m, On: map[string][]catalogue.SettingSource{}}
|
||||
for _, n := range e.On {
|
||||
if *node != "" && n != *node {
|
||||
continue
|
||||
}
|
||||
p.Nodes = append(p.Nodes, n)
|
||||
layers, err := inv.SettingsFor(ctx, n, m.Module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p.On[n] = catalogue.Effective(m, layers)
|
||||
}
|
||||
listed = append(listed, p)
|
||||
}
|
||||
if only != "" && len(listed) == 0 {
|
||||
fmt.Printf("%s declares no preferences%s\n", only, onNode(*node))
|
||||
return nil
|
||||
}
|
||||
if len(listed) == 0 && *node != "" {
|
||||
fmt.Printf("no module on %s declares a preference\n", *node)
|
||||
return nil
|
||||
}
|
||||
fmt.Print(describePreferences(listed))
|
||||
return nil
|
||||
|
||||
case "clear":
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("settings clear <module> [--node <node>]")
|
||||
}
|
||||
if err := inv.ClearSettings(ctx, *node, positionals[0]); err != nil {
|
||||
before, _, err := inv.Layer(ctx, *node, positionals[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := refuseTerminalSettingsThroughAVerb(ctx, inv, before, nil, positionals[0], where); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inv.ClearSettingsBy(ctx, *node, positionals[0], setByWords()); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s on %s is back to what the module says\n", positionals[0], where)
|
||||
return nil
|
||||
|
||||
default:
|
||||
return fmt.Errorf("settings has no %q; it has set and clear", args[0])
|
||||
return fmt.Errorf("settings has no %q; it has show, set, clear, preferences, propose and proposals", args[0])
|
||||
}
|
||||
}
|
||||
|
||||
// setByWords is who sets a layer from this process, as the layer keeps it (novox/hq ADR 0277): the caller at the
|
||||
// controller's terminal, or through which verb.
|
||||
func setByWords() string {
|
||||
if verb, through := throughAVerb(); through {
|
||||
return "set by " + link.Caller() + " through " + verb + " at " + time.Now().Local().Format("2006-01-02 15:04")
|
||||
}
|
||||
return "set at the controller's terminal by " + link.Caller() + " at " + time.Now().Local().Format("2006-01-02 15:04")
|
||||
}
|
||||
|
||||
// describeEffective says each setting's value on a machine or the whole mesh, where it came from, and
|
||||
// the module's default when a layer overrides it.
|
||||
func describeEffective(module, where string, values []catalogue.SettingSource) string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "%s on %s, every value and where it comes from:\n", module, where)
|
||||
for _, v := range values {
|
||||
value, _ := json.Marshal(v.Value)
|
||||
fmt.Fprintf(&b, " %s = %s (%s", v.Key, value, v.From)
|
||||
if v.HasDefault && !v.FromDefault {
|
||||
d, _ := json.Marshal(v.Default)
|
||||
fmt.Fprintf(&b, "; the default is %s", d)
|
||||
}
|
||||
b.WriteString(")\n")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// onNode is ` on <node>` for one machine, nothing for the whole mesh.
|
||||
func onNode(node string) string {
|
||||
if node == "" {
|
||||
return ""
|
||||
}
|
||||
return " on " + node
|
||||
}
|
||||
|
||||
// preferencesOf is one module's preferences and its value on each machine it is assigned to.
|
||||
type preferencesOf struct {
|
||||
Manifest catalogue.Manifest
|
||||
Nodes []string
|
||||
On map[string][]catalogue.SettingSource
|
||||
}
|
||||
|
||||
// describePreferences lists each module's preferences — key, default and why — and, per machine it is
|
||||
// assigned to, the value and where it comes from (novox/hq ADR 0262).
|
||||
func describePreferences(modules []preferencesOf) string {
|
||||
if len(modules) == 0 {
|
||||
return "no module declares a preference\n"
|
||||
}
|
||||
var b strings.Builder
|
||||
for i, p := range modules {
|
||||
if i > 0 {
|
||||
b.WriteString("\n")
|
||||
}
|
||||
on := "assigned nowhere"
|
||||
if len(p.Nodes) > 0 {
|
||||
on = "on " + strings.Join(p.Nodes, ", ")
|
||||
}
|
||||
fmt.Fprintf(&b, "%s (%s)\n", p.Manifest.Module, on)
|
||||
keys := make([]string, 0, len(p.Manifest.Settings))
|
||||
for k := range p.Manifest.Settings {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
d := p.Manifest.Settings[k]
|
||||
def, _ := json.Marshal(d.Default)
|
||||
fmt.Fprintf(&b, " %s, default %s: %s\n", k, def, d.Why)
|
||||
for _, n := range p.Nodes {
|
||||
for _, s := range p.On[n] {
|
||||
if s.Key != k {
|
||||
continue
|
||||
}
|
||||
v, _ := json.Marshal(s.Value)
|
||||
from := s.From
|
||||
if s.FromDefault {
|
||||
from = catalogue.DefaultLayer
|
||||
} else if s.From != catalogue.MeshWideLayer {
|
||||
from = "the node"
|
||||
}
|
||||
fmt.Fprintf(&b, " %s: %s (%s)\n", n, v, from)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// nodeFlag is ` --node <node>` for a machine's layer, nothing for the whole mesh's.
|
||||
func nodeFlag(node string) string {
|
||||
if node == "" {
|
||||
return ""
|
||||
}
|
||||
return " --node " + node
|
||||
}
|
||||
|
||||
// describeOffers says what a module provides, and marks the ones answered from anywhere in the
|
||||
// mesh — because "provides a database" and "provides a shell" are read the same way and mean
|
||||
// entirely different things about where the answer has to be.
|
||||
@@ -505,8 +816,8 @@ const theBrokerSeat = "mesh-broker"
|
||||
// says. Only when nothing holds the seat yet (genesis raised the broker as plumbing and no module
|
||||
// has adopted it) does the hub stand in, which is where the foundation is by convention.
|
||||
//
|
||||
// "On the overlay" is what `whereEveryoneIs` answers — a machine that RESOLVED the networking
|
||||
// module — not "has an address", which is true of every placed machine and says nothing about
|
||||
// "On the overlay" is what `whereEveryoneIs` answers — a machine that RESOLVED the private network
|
||||
// — not "has an address", which is true of every placed machine and says nothing about
|
||||
// whether anything can reach it (novox/hq issue 059). A node not on the overlay — at genesis,
|
||||
// before any `overlay place`, which is when the builder's account is issued — keeps the genesis
|
||||
// address, so nothing about bring-up changes. This is issue 055, corrected by 059.
|
||||
@@ -675,7 +986,7 @@ func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manife
|
||||
Emits: m.EmitsAll(), Consumes: m.Consumes, Serves: m.Tools,
|
||||
}); needed {
|
||||
if busAddress == "" {
|
||||
fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push`, then "+
|
||||
fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push --behind`, then "+
|
||||
"`rollout mint` again is harmless)\n", m.Module)
|
||||
} else {
|
||||
js, err := broker.Dial(busAddress)
|
||||
@@ -737,6 +1048,9 @@ func whereItComesFrom(repository, ref, commit, path string, self bool) (inventor
|
||||
// definition that got past the check — written elsewhere, or checked by nobody — is refused here
|
||||
// in the same words. A name meant on purpose is declared with its reason and passes.
|
||||
func namesNoInstallation(m catalogue.Manifest) error {
|
||||
if problems := catalogue.TrustProblems(m); len(problems) > 0 {
|
||||
return fmt.Errorf("%s", strings.Join(problems, "; "))
|
||||
}
|
||||
named := catalogue.InstallationProblems(m)
|
||||
if len(named) == 0 {
|
||||
return nil
|
||||
@@ -800,3 +1114,71 @@ func declaresTools(m catalogue.Manifest) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// The keys no verb may change are catalogue.TerminalKeys (novox/hq issue 339). `places` says where the
|
||||
// node-engine creates and, as root, owns a module's directories, with an owner the setting names; `accesses` says
|
||||
// which of the machine's paths are mounted into a module's container; a provider's trust anchors say what every
|
||||
// consumer trusts. Set through a verb, any of them lets any caller of the mesh's verbs — an agent among them —
|
||||
// have root hand it a directory, mount one of the machine's into a container it reaches, or have the mesh trust
|
||||
// an authority of its own. They are the operator's, typed at the controller's terminal.
|
||||
|
||||
// throughAVerb says whether this process runs a seat verb's command line: the serving controller names the
|
||||
// verb in the environment of every command it runs for one (runVerb), and a person at the terminal runs none.
|
||||
// Every verb route reaches a command through runVerb — the named verbs and the generic `command` alike — so
|
||||
// this is the one place that knows, whatever line the verb composed.
|
||||
func throughAVerb() (string, bool) {
|
||||
verb := os.Getenv(verbVar)
|
||||
return verb, verb != ""
|
||||
}
|
||||
|
||||
// sameLayer says whether two layers hold the same values, an absent layer and an empty one alike.
|
||||
func sameLayer(a, b map[string]any) bool {
|
||||
if len(a) == 0 && len(b) == 0 {
|
||||
return true
|
||||
}
|
||||
ra, _ := json.Marshal(a)
|
||||
rb, _ := json.Marshal(b)
|
||||
return string(ra) == string(rb)
|
||||
}
|
||||
|
||||
// refuseTerminalSettingsThroughAVerb refuses a layer change through a verb that would add, change or remove
|
||||
// places or accesses; a change that leaves both as they were is not refused.
|
||||
func refuseTerminalSettingsThroughAVerb(ctx context.Context, inv *inventory.Inventory, before, after map[string]any,
|
||||
module, where string) error {
|
||||
verb, through := throughAVerb()
|
||||
if !through {
|
||||
return nil
|
||||
}
|
||||
// Judged against the module's definition as the catalogue holds it: what it serves and which of its files
|
||||
// are trusted. A catalogue that cannot be read refuses rather than judging against nothing.
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("which settings of %s are the terminal's cannot be read, so nothing was changed: %w", module, err)
|
||||
}
|
||||
// A trusted mergeable file takes any key, so its module's whole layer is the terminal's (novox/hq issue 340).
|
||||
if files := catalogue.TrustedMergeable(shelf[module]); len(files) > 0 && !sameLayer(before, after) {
|
||||
return fmt.Errorf("the settings of %s on %s are set at the controller's terminal (`mesh-cli` on the control-node) or on "+
|
||||
"the operator's warrant, never through a verb alone (this line came through %q): %s merges whatever key a "+
|
||||
"layer sets into a file root or a consumer trusts, so any key could point the module at a listener of the "+
|
||||
"caller's, and whoever may call a verb includes agents (novox/hq issue 340; a file nothing trusts says "+
|
||||
"\"trusted\": false). Propose it instead: the settings verb with propose puts the exact values to the "+
|
||||
"operator on a channel that proves who answers, and the layer is set on their Approve (novox/hq ADR 0277). "+
|
||||
"Nothing was changed",
|
||||
module, where, verb, strings.Join(files, ", "))
|
||||
}
|
||||
for _, key := range catalogue.TerminalKeys(shelf[module]) {
|
||||
was, _ := json.Marshal(before[key])
|
||||
now, _ := json.Marshal(after[key])
|
||||
if string(was) == string(now) {
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf("%s of %s on %s is set at the controller's terminal (`mesh-cli` on the control-node) or on the "+
|
||||
"operator's warrant, never through a verb alone (this line came through %q): it says where root creates and "+
|
||||
"owns a module's directories, which of the machine's paths are mounted into its container, what the mesh's "+
|
||||
"consumers trust, or what a file root or a person's session obeys takes, and whoever may call a verb "+
|
||||
"includes agents (novox/hq issue 339; issue 340 for a mergeable file's own keys). Propose it instead: the "+
|
||||
"settings verb with propose puts the exact values to the operator on a channel that proves who answers, and "+
|
||||
"the layer is set on their Approve (novox/hq ADR 0277). Nothing was changed", key, module, where, verb)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -232,9 +232,22 @@ func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool,
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// **A machine joining is on the network before it is anything else** (novox/hq ADR 0169). Its
|
||||
// token was issued for its tunnel key and gave it an address, so while that token can still be
|
||||
// used the hub carries it as a peer: it brings its tunnel up from the token and enrols over it.
|
||||
// When the token is spent the machine is on the network by what it runs, as every other is; when
|
||||
// it expires unused, the peer goes with it at the hub's next composition.
|
||||
joining, err := inv.NodesWithALiveToken(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
isJoining := map[string]bool{}
|
||||
for _, name := range joining {
|
||||
isJoining[name] = true
|
||||
}
|
||||
nodes := make([]overlay.Node, 0, len(places))
|
||||
for _, p := range places {
|
||||
if !on[p.Name] {
|
||||
if !on[p.Name] && !(isJoining[p.Name] && p.Key != "" && p.Address != "") {
|
||||
continue
|
||||
}
|
||||
n := overlay.Node{
|
||||
@@ -413,10 +426,10 @@ func overlayShow(ctx context.Context, open *stores) error {
|
||||
tunnel.Interface, tunnel.Range, tunnel.Port)
|
||||
case n.Hub && hubName == n.Name && tunnel.Interface != "":
|
||||
fmt.Printf(" hub — found a tunnel on %s and did NOT take it over: its key is not the tunnel's; "+
|
||||
"`mesh-host overlay take --tunnel %s` on the machine takes it", tunnel.Interface, tunnel.Interface)
|
||||
"`nox-mesh-host overlay take --tunnel %s` on the machine takes it", tunnel.Interface, tunnel.Interface)
|
||||
case n.Hub:
|
||||
fmt.Print(" hub — found no tunnel; if the machine runs the predecessor's, " +
|
||||
"`mesh-host overlay take --tunnel <iface>` there adopts it (novox/hq ADR 0105)")
|
||||
"`nox-mesh-host overlay take --tunnel <iface>` there adopts it (novox/hq ADR 0105)")
|
||||
case !n.Reachable():
|
||||
fmt.Print(" not dialable")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// asksWithPaths records every build a merge asks for, as repository, path and ref.
|
||||
func asksWithPaths(t *testing.T) *[][3]string {
|
||||
t.Helper()
|
||||
var asked [][3]string
|
||||
was := askABuild
|
||||
askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) {
|
||||
asked = append(asked, [3]string{source.Repository, path, ref})
|
||||
return "b-" + path, nil
|
||||
}
|
||||
t.Cleanup(func() { askABuild = was })
|
||||
return &asked
|
||||
}
|
||||
|
||||
// The live case of 2026-10-07 (novox/hq issue 300), replayed: a merge of the catalogue adds one module's
|
||||
// directory and touches nothing else the mesh holds. Its delivery plan said "builds new: …, sent nowhere";
|
||||
// the merge said it changed nothing any module the mesh holds is built from, and built nothing, so the
|
||||
// module was never registered and `assign` refused it. The merge asks for its build, at the branch merged
|
||||
// into, from the repository as the mesh spells it — and opens no plan, since nothing it holds moved.
|
||||
func TestAMergeBuildsTheNewModuleItAdds(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asked := asksWithPaths(t)
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "networkmanager", Version: "1"},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/networkmanager", Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "3da80a4b00",
|
||||
Paths: []string{"modules/systemd-resolved/module.json", "modules/systemd-resolved/cmd/main.go"},
|
||||
ModuleDirs: []string{"modules/systemd-resolved"}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := [3]string{"novox/mesh-catalog", "modules/systemd-resolved", "main"}
|
||||
if len(*asked) != 1 || (*asked)[0] != want {
|
||||
t.Fatalf("the merge adding modules/systemd-resolved asked for %v, not %v", *asked, want)
|
||||
}
|
||||
if plans, err := inv.OpenPlans(ctx); err != nil || len(plans) != 0 {
|
||||
t.Fatalf("a merge moving nothing the mesh holds opened a plan: %+v %v", plans, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge that changes a held module and adds a new one does both: the held one walks its plan, the new
|
||||
// one is asked for beside it. A merge adding nothing builds nothing new.
|
||||
func TestAMergeBuildsItsNewModuleBesideItsPlan(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asked := asksWithPaths(t)
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1aaaaaaaa",
|
||||
Paths: []string{"modules/app/index.ts", "modules/fresh/module.json"},
|
||||
ModuleDirs: []string{"modules/app", "modules/fresh"}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
paths := map[string]bool{}
|
||||
for _, a := range *asked {
|
||||
paths[a[1]] = true
|
||||
}
|
||||
if !paths["modules/fresh"] || !paths["modules/app"] || len(*asked) != 2 {
|
||||
t.Fatalf("asked %v; want the new module and the plan's first tier", *asked)
|
||||
}
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil || len(plans) != 1 || plans[0].Modules["fresh"] != nil || plans[0].Modules["app"] == nil {
|
||||
t.Fatalf("the plan should walk app alone: %+v %v", plans, err)
|
||||
}
|
||||
|
||||
*asked = nil
|
||||
m2 := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c2aaaaaaaa",
|
||||
Paths: []string{"README.md"}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(*asked) != 0 {
|
||||
t.Fatalf("a merge adding no module asked for %v", *asked)
|
||||
}
|
||||
}
|
||||
@@ -2,16 +2,23 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/token"
|
||||
)
|
||||
|
||||
@@ -23,7 +30,7 @@ import (
|
||||
|
||||
func nodeCommand(ctx context.Context, args []string) error {
|
||||
if len(args) == 0 {
|
||||
return errors.New("node add <name>, node list, node show <name>, or " + publicDomainUsage)
|
||||
return errors.New("node add <name>, node list, node show <name>, " + publicDomainUsage + ", or " + handOverUsage)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
@@ -95,17 +102,175 @@ func nodeCommand(ctx context.Context, args []string) error {
|
||||
"containers reaching outward. The machine reports which of its links face outside; see " +
|
||||
"`node show <name>`")
|
||||
|
||||
case "agent-account":
|
||||
// The account agents run as on this machine, when it is not the operator's (novox/hq ADR 0266). Here,
|
||||
// at the controller's terminal, and nowhere else: no verb and no setting names it, so no agent can
|
||||
// name itself another account.
|
||||
return nodeAgentAccount(ctx, inv, args[1:])
|
||||
|
||||
case "account":
|
||||
// The operator's login on this machine (novox/hq to-be 29): what a home-scoped file is
|
||||
// owned by and which account `ssh <node>` uses. Reports with no argument; sets with one;
|
||||
// an optional second argument is the home when it is not /home/<account>.
|
||||
return nodeAccount(ctx, inv, args[1:])
|
||||
|
||||
case "hand-over":
|
||||
// A directory the node-engine uses as found, handed to the mesh (novox/hq issue 356, issue 339). Here, at
|
||||
// the controller's terminal, and nowhere else: at the next apply root gives the directory to the account
|
||||
// the module declares, and whoever may call a verb includes agents.
|
||||
return nodeHandOver(ctx, open, args[1:])
|
||||
|
||||
case "setuid-search":
|
||||
// A fresh search for setuid programs on a node (novox/hq issue 361), after the operator changed by hand
|
||||
// what the last one found. Here, at the controller's terminal, and nowhere else: a search never makes a
|
||||
// machine free wrongly, but asked again and again it would keep the machine unjudged and its disks busy,
|
||||
// and whoever may call a verb includes agents.
|
||||
return nodeSetuidSearch(ctx, open, args[1:])
|
||||
|
||||
default:
|
||||
return fmt.Errorf("node has no %q; it has add, list, show, public-domain and account", args[0])
|
||||
return fmt.Errorf("node has no %q; it has add, list, show, public-domain, account, agent-account, hand-over "+
|
||||
"and setuid-search", args[0])
|
||||
}
|
||||
}
|
||||
|
||||
const handOverUsage = "node hand-over <node> <directory> — hand a directory the node-engine on <node> uses as found " +
|
||||
"to the mesh: its next apply gives it the declared owner and mode. The directory's absolute path, as the module's " +
|
||||
"condition names it"
|
||||
|
||||
// handOverLine reads a hand-over's line: the node and the directory's absolute path, exactly as the engine states
|
||||
// it. Judged before anything is asked, and judged again by the engine, which is the one that acts.
|
||||
func handOverLine(args []string) (node, path string, err error) {
|
||||
if len(args) != 2 {
|
||||
return "", "", errors.New(handOverUsage)
|
||||
}
|
||||
node, path = args[0], args[1]
|
||||
if node == "" || strings.HasPrefix(node, "-") {
|
||||
return "", "", fmt.Errorf("%q is not a node's name; %s", node, handOverUsage)
|
||||
}
|
||||
if !filepath.IsAbs(path) {
|
||||
return "", "", fmt.Errorf("%q is not an absolute path; %s", path, handOverUsage)
|
||||
}
|
||||
if filepath.Clean(path) != path {
|
||||
return "", "", fmt.Errorf("%q is not the directory's path as the engine states it (no `..`, no doubled or "+
|
||||
"trailing separator); %s", path, handOverUsage)
|
||||
}
|
||||
return node, path, nil
|
||||
}
|
||||
|
||||
// handOverBy is who hands the directory over, in the words a verb's caller is recorded in: the operator through
|
||||
// mesh-cli on the control-node, or whoever runs this controller's binary at its terminal.
|
||||
func handOverBy() string {
|
||||
if by := strings.TrimSpace(os.Getenv(link.CallerVar)); by != "" {
|
||||
return by
|
||||
}
|
||||
return "the controller's terminal"
|
||||
}
|
||||
|
||||
// nodeHandOver asks the node's engine to take a directory it uses as found as the mesh's, and says what came of
|
||||
// it. The engine records the hand-over or refuses; nothing is recorded here, because the directory is the
|
||||
// machine's and the engine is the one that reads it. The ask is signed with the mesh's key (issue 356's review).
|
||||
func nodeHandOver(ctx context.Context, open *stores, args []string) error {
|
||||
known := func(node string) error {
|
||||
_, err := open.inventory.NodeByName(ctx, node)
|
||||
return err
|
||||
}
|
||||
ask := func(node, path, by string) (link.HandOverAnswer, error) {
|
||||
ident, err := open.Identity(ctx)
|
||||
if err != nil {
|
||||
return link.HandOverAnswer{}, fmt.Errorf("the mesh's signing key cannot be read, so nothing was asked of %s: %w",
|
||||
node, err)
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return link.HandOverAnswer{}, err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return link.HandOverAnswer{}, fmt.Errorf("cannot reach the bus, so nothing was asked of %s: %w", node, err)
|
||||
}
|
||||
defer js.Close()
|
||||
return link.AskHandOver(ctx, js.Conn(), ident, node, path, by, link.HandOverWithin)
|
||||
}
|
||||
return handOverAsked(args, known, ask, os.Stdout)
|
||||
}
|
||||
|
||||
// handOverAsked is the hand-over's line with its two acts given: whether the mesh knows the node, and the ask.
|
||||
// Nothing is asked of a line or a node that is refused, and the engine's refusal is this command's failure —
|
||||
// never a success with the refusal printed.
|
||||
func handOverAsked(args []string, known func(node string) error,
|
||||
ask func(node, path, by string) (link.HandOverAnswer, error), out io.Writer) error {
|
||||
node, path, err := handOverLine(args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := known(node); err != nil {
|
||||
return fmt.Errorf("nothing was asked: %w", err)
|
||||
}
|
||||
answer, err := ask(node, path, handOverBy())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Refused != "" {
|
||||
return fmt.Errorf("%s refused: %s", node, answer.Refused)
|
||||
}
|
||||
fmt.Fprintln(out, answer.Said)
|
||||
fmt.Fprintf(out, " the module's condition clears once %s applies; `nox push %s` applies it now\n", node, node)
|
||||
return nil
|
||||
}
|
||||
|
||||
const setuidSearchUsage = "node setuid-search <node> — throw away the node-engine's last search for setuid " +
|
||||
"programs on <node> and start a full one: after a setuid-root program it found was removed by hand. Until it " +
|
||||
"completes, root-free says the node is not judged yet"
|
||||
|
||||
// nodeSetuidSearch asks the node's engine for a fresh search, signed with the mesh's key as a hand-over is.
|
||||
func nodeSetuidSearch(ctx context.Context, open *stores, args []string) error {
|
||||
known := func(node string) error {
|
||||
_, err := open.inventory.NodeByName(ctx, node)
|
||||
return err
|
||||
}
|
||||
ask := func(node, by string) (link.HandOverAnswer, error) {
|
||||
ident, err := open.Identity(ctx)
|
||||
if err != nil {
|
||||
return link.HandOverAnswer{}, fmt.Errorf("the mesh's signing key cannot be read, so nothing was asked of %s: %w",
|
||||
node, err)
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return link.HandOverAnswer{}, err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return link.HandOverAnswer{}, fmt.Errorf("cannot reach the bus, so nothing was asked of %s: %w", node, err)
|
||||
}
|
||||
defer js.Close()
|
||||
return link.AskSetuidSearch(ctx, js.Conn(), ident, node, by, link.HandOverWithin)
|
||||
}
|
||||
return setuidSearchAsked(args, known, ask, os.Stdout)
|
||||
}
|
||||
|
||||
// setuidSearchAsked is the line with its two acts given: whether the mesh knows the node, and the ask. The
|
||||
// engine's refusal is this command's failure.
|
||||
func setuidSearchAsked(args []string, known func(node string) error,
|
||||
ask func(node, by string) (link.HandOverAnswer, error), out io.Writer) error {
|
||||
if len(args) != 1 || args[0] == "" || strings.HasPrefix(args[0], "-") {
|
||||
return errors.New(setuidSearchUsage)
|
||||
}
|
||||
node := args[0]
|
||||
if err := known(node); err != nil {
|
||||
return fmt.Errorf("nothing was asked: %w", err)
|
||||
}
|
||||
answer, err := ask(node, handOverBy())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Refused != "" {
|
||||
return fmt.Errorf("%s refused: %s", node, answer.Refused)
|
||||
}
|
||||
fmt.Fprintln(out, answer.Said)
|
||||
fmt.Fprintf(out, " the controller's root-free verb shows the search's progress until it completes\n")
|
||||
return nil
|
||||
}
|
||||
|
||||
// addNode creates a node record, adopted when the operator says so (novox/hq ADR 0100).
|
||||
func addNode(ctx context.Context, inv *inventory.Inventory, args []string) error {
|
||||
set := flag.NewFlagSet("node add", flag.ContinueOnError)
|
||||
@@ -173,6 +338,57 @@ func nodeAccount(ctx context.Context, inv *inventory.Inventory, positionals []st
|
||||
return nil
|
||||
}
|
||||
|
||||
const agentAccountUsage = "node agent-account <name> — what it is now; " +
|
||||
"<name> <account> [home] to name the account agents run as (home defaults to /home/<account>); " +
|
||||
"<name> --clear to have them run as the operator account again"
|
||||
|
||||
// nodeAgentAccount reports, names or clears the account agents run as on a node (novox/hq ADR 0266). Read-
|
||||
// shaped with no account, like public-domain; clearing is asked for by name.
|
||||
func nodeAgentAccount(ctx context.Context, inv *inventory.Inventory, args []string) error {
|
||||
set := flag.NewFlagSet("node agent-account", flag.ContinueOnError)
|
||||
clear := set.Bool("clear", false, "agents run as the operator account again")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) == 0 || len(positionals) > 3 {
|
||||
return errors.New(agentAccountUsage)
|
||||
}
|
||||
node := positionals[0]
|
||||
switch {
|
||||
case *clear && len(positionals) > 1:
|
||||
return fmt.Errorf("name an agent account for %s or --clear, not both", node)
|
||||
case *clear:
|
||||
if err := inv.SetAgentAccount(ctx, node, "", ""); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("agents on %s run as the operator account again\n", node)
|
||||
fmt.Printf(" run `push %s` to send it; the agent account itself is kept (the mesh never deletes a login)\n", node)
|
||||
return nil
|
||||
case len(positionals) >= 2:
|
||||
home := ""
|
||||
if len(positionals) == 3 {
|
||||
home = positionals[2]
|
||||
}
|
||||
if err := inv.SetAgentAccount(ctx, node, positionals[1], home); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("agents on %s run as %s\n", node, positionals[1])
|
||||
fmt.Printf(" run `push %s` to send it; the self-check says once its node-engine has judged it "+
|
||||
"unable to become root\n", node)
|
||||
return nil
|
||||
default:
|
||||
n, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, line := range agentAccountLines(ctx, inv, n) {
|
||||
fmt.Println(strings.TrimPrefix(line, " "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
const publicDomainUsage = "node public-domain <name> — what it is now; " +
|
||||
"<name> <domain> to set it; <name> --clear to take it away"
|
||||
|
||||
@@ -247,6 +463,8 @@ func tokenCommand(ctx context.Context, args []string) error {
|
||||
validFor := set.Duration("for", time.Hour, "how long the token may be used")
|
||||
adopted := set.Bool("adopted", false,
|
||||
"the machine joining is in use: it is adopted, and keeps what is found on it")
|
||||
tunnelKey := set.String("overlay-key", "",
|
||||
"the public half of the tunnel key the machine made (`nox-mesh-host key`): it joins through the tunnel")
|
||||
if err := set.Parse(args[1:]); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -300,6 +518,14 @@ func tokenCommand(ctx context.Context, args []string) error {
|
||||
default:
|
||||
return err
|
||||
}
|
||||
// **Through the tunnel** (novox/hq ADR 0169): the machine's key recorded, its address given, the
|
||||
// hub sent it as a peer — all before the token is shown, so the tunnel answers the first time the
|
||||
// machine knocks. The bus is then reached at its address on the private network.
|
||||
if *tunnelKey != "" {
|
||||
if made.Tunnel, made.Broker, err = throughTheTunnel(ctx, open, issued.Node, *tunnelKey, made.Broker); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
encoded, err := made.Encode()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -324,6 +550,78 @@ func tokenCommand(ctx context.Context, args []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// throughTheTunnel makes a machine a peer of the hub for its token, and says what the token carries
|
||||
// for it: its first tunnel, and the bus at its address on the private network (novox/hq ADR 0169).
|
||||
//
|
||||
// The hub is pushed here, before the token is shown. A token shown before the hub knew the key is a
|
||||
// tunnel that does not answer, and a machine that cannot tell that from a bus that is down.
|
||||
func throughTheTunnel(ctx context.Context, open *stores, node inventory.Node, key, busAt string) (
|
||||
*token.Tunnel, string, error) {
|
||||
inv := open.inventory
|
||||
key = strings.TrimSpace(key)
|
||||
if raw, err := base64.StdEncoding.DecodeString(key); err != nil || len(raw) != 32 {
|
||||
return nil, "", fmt.Errorf("%q is not a tunnel public key: it is 32 bytes in base64, as "+
|
||||
"`nox-mesh-host key` prints it", key)
|
||||
}
|
||||
// The bus on the private network is its holder's address at the bus's own port, so the port must
|
||||
// be known before anything is recorded.
|
||||
_, port, err := net.SplitHostPort(busAt)
|
||||
if err != nil || port == "" {
|
||||
return nil, "", fmt.Errorf("the bus's address %q has no port to reach it on", busAt)
|
||||
}
|
||||
places, err := inv.Overlays(ctx)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
var hub *inventory.Overlay
|
||||
for i := range places {
|
||||
if places[i].Hub {
|
||||
hub = &places[i]
|
||||
}
|
||||
}
|
||||
if hub == nil || hub.Key == "" || hub.Endpoint == "" || hub.Address == "" {
|
||||
return nil, "", errors.New("this mesh has no hub with a key, an address and an endpoint to " +
|
||||
"dial, so there is no tunnel to join through: place one (`overlay place <node> --hub " +
|
||||
"--endpoint <host>:<port>`), or issue the token without --overlay-key")
|
||||
}
|
||||
if err := inv.RecordOverlayKey(ctx, node.ID, key); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if err := inv.BindTokenToKey(ctx, node.ID, key); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
cidr, err := overlayRange(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
address, err := inv.AssignAddress(ctx, node.ID, cidr)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
// The bus at its holder's address on the private network, reached through the hub like the rest
|
||||
// of the range; the hub's own when no machine is recorded as holding it yet.
|
||||
busAddress := hub.Address
|
||||
if holders, err := seatHolders(ctx, inv); err != nil {
|
||||
return nil, "", err
|
||||
} else if h, held := holders[catalogue.BrokerSeat]; held {
|
||||
for _, p := range places {
|
||||
if p.Name == h.Node && p.Address != "" {
|
||||
busAddress = p.Address
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := sendTo(ctx, open, []string{hub.Name}); err != nil {
|
||||
return nil, "", fmt.Errorf("%s was made a peer of the hub, and the hub could not be sent "+
|
||||
"it, so the tunnel would not answer — the token is not shown; issue it again once %s "+
|
||||
"can be pushed: %w", node.Name, hub.Name, err)
|
||||
}
|
||||
// An address, not a name — nothing resolves before the machine has joined (novox/hq ADR 0004).
|
||||
return &token.Tunnel{
|
||||
Key: key, Address: address + "/32", Range: cidr,
|
||||
HubKey: hub.Key, HubEndpoint: hub.Endpoint,
|
||||
}, net.JoinHostPort(busAddress, port), nil
|
||||
}
|
||||
|
||||
// issueFor is the inventory's half of issuing a token: the record, made when it is new, adopted
|
||||
// when the operator says so, and the one-time secret for it. The node in what it returns carries
|
||||
// its mode, which is what the token says.
|
||||
@@ -505,6 +803,10 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
|
||||
if err := showMode(ctx, inv, node); err != nil {
|
||||
return err
|
||||
}
|
||||
// Whom agents run as here, and whether that account can become root without a person (ADR 0266).
|
||||
for _, line := range agentAccountLines(ctx, inv, node) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
|
||||
// The domain its routed names are composed under, when it has one (novox/hq ADR 0066). Shown
|
||||
// only when set: a machine that serves nothing to the outside has no domain, and saying so of
|
||||
@@ -537,6 +839,23 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
|
||||
}
|
||||
}
|
||||
|
||||
// What it runs for its modules, and how each is (novox/hq ADR 0240): "is it working" answered without
|
||||
// a terminal on the machine.
|
||||
if h, had, err := inv.HealthOf(ctx, name); err != nil {
|
||||
fmt.Printf("\n what it says of what it runs could NOT be read: %v\n", err)
|
||||
} else {
|
||||
fmt.Println()
|
||||
for _, line := range healthLines(h, had, time.Now()) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
for _, line := range networkLines(h, had, time.Now()) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
for _, line := range unitsLines(h, had, time.Now()) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
}
|
||||
|
||||
held, err := inv.Profile(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -582,11 +901,14 @@ func orNotReported(s string) string {
|
||||
}
|
||||
|
||||
// printJSON prints a value as indented JSON, the shape every `--json` answers in.
|
||||
func printJSON(v any) error {
|
||||
func printJSON(v any) error { return printJSONTo(os.Stdout, v) }
|
||||
|
||||
// printJSONTo is printJSON to a writer of the caller's.
|
||||
func printJSONTo(w io.Writer, v any) error {
|
||||
body, err := json.MarshalIndent(v, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
fmt.Fprintln(w, string(body))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -143,7 +144,7 @@ func TestAMergeRebuildsTheModulesItChanged(t *testing.T) {
|
||||
}{
|
||||
{"one module's own files", merge([]string{"modules/gitea/index.ts", "modules/gitea/client.ts"}, false), "gitea"},
|
||||
{"two modules' files", merge([]string{"modules/gitea/index.ts", "modules/keycloak/module.json"}, false), "gitea,keycloak"},
|
||||
{"a file they share", merge([]string{"tsconfig.json"}, false), "gitea,keycloak"},
|
||||
{"a file at the root no build reads (issue 280)", merge([]string{"tsconfig.json"}, false), ""},
|
||||
{"a module the mesh does not hold", merge([]string{"modules/plex/index.ts"}, false), ""},
|
||||
{"nothing said about the files", merge(nil, false), "gitea,keycloak"},
|
||||
{"more files than were listed", merge([]string{"modules/gitea/index.ts"}, true), "gitea,keycloak"},
|
||||
@@ -152,11 +153,11 @@ func TestAMergeRebuildsTheModulesItChanged(t *testing.T) {
|
||||
{"a new module beside a held one", merge([]string{"modules/gitea/x", "modules/newmod/module.json"}, false), "gitea"},
|
||||
{"a new module's other files", merge([]string{"modules/newmod/index.ts", "modules/newmod/module.json"}, false), ""},
|
||||
{"a module removed", merge([]string{"modules/gone/module.json"}, false), ""},
|
||||
{"a directory with no manifest", merge([]string{"modules/lib/x.go"}, false), "gitea,keycloak"},
|
||||
{"a file directly among the modules", merge([]string{"modules/README.md"}, false), "gitea,keycloak"},
|
||||
{"the root's files still", merge([]string{"tsconfig.json"}, false), "gitea,keycloak"},
|
||||
{"a directory with no manifest", merge([]string{"modules/lib/x.go"}, false), ""},
|
||||
{"a file directly among the modules", merge([]string{"modules/README.md"}, false), ""},
|
||||
{"a root file beside a module's", merge([]string{"merge-check.sh", "modules/keycloak/x.ts"}, false), "keycloak"},
|
||||
} {
|
||||
if got := named(whatTheMergeTouched(candidates, known, c.m)); got != c.want {
|
||||
if got := named(whatTheMergeTouched(candidates, known, c.m, nil)); got != c.want {
|
||||
t.Errorf("%s: rebuilt %q, wanted %q", c.what, got, c.want)
|
||||
}
|
||||
}
|
||||
@@ -243,3 +244,69 @@ func TestAModuleBuiltAtACommitStillFollowsItsBranch(t *testing.T) {
|
||||
t.Error("a module following another branch was matched")
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq issues 278 and 280: a merge touching the code of a module the mesh does not hold — the
|
||||
// catalogue's reference module, whose manifest it left alone — read as shared and rebuilt every module
|
||||
// built from the repository (103 of them on 2026-10-06, 88 byte-identical); so did a merge-check.sh added at
|
||||
// the root. No build reads a file outside its module's directory, so neither rebuilds anything, said by the
|
||||
// announcer or not.
|
||||
func TestAChangeInsideAModuleIsThatModulesHeldOrNot(t *testing.T) {
|
||||
const repo = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||
gitea := fromRepo("gitea", repo, "modules/gitea")
|
||||
keycloak := fromRepo("keycloak", repo, "modules/keycloak")
|
||||
known := []inventory.Entry{gitea, keycloak}
|
||||
candidates := []inventory.Entry{gitea, keycloak}
|
||||
merge := func(paths, modules []string, said bool) link.SourceMoved {
|
||||
return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Paths: paths,
|
||||
ModuleDirs: modules, ModuleDirsSaid: said}
|
||||
}
|
||||
named := func(entries []inventory.Entry) string {
|
||||
var names []string
|
||||
for _, e := range entries {
|
||||
names = append(names, e.Manifest.Module)
|
||||
}
|
||||
return strings.Join(names, ",")
|
||||
}
|
||||
showcase := []string{"modules/showcase/index.ts"}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
m link.SourceMoved
|
||||
want string
|
||||
}{
|
||||
{"a module held by none, said", merge(showcase, []string{"modules/showcase"}, true), ""},
|
||||
{"beside a held one's change", merge(append([]string{"modules/gitea/index.ts"}, showcase...),
|
||||
[]string{"modules/gitea", "modules/showcase"}, true), "gitea"},
|
||||
{"deeper inside it", merge([]string{"modules/showcase/daemon/index.ts"}, []string{"modules/showcase"}, true), ""},
|
||||
{"a directory holding no manifest, read by no build", merge([]string{"modules/lib/x.go"}, nil, true), ""},
|
||||
{"one of two files in no module", merge([]string{"modules/showcase/index.ts", "modules/lib/x.go"},
|
||||
[]string{"modules/showcase"}, true), ""},
|
||||
{"the root is never a module directory", merge([]string{"tsconfig.json"}, []string{"", "/", "."}, true), ""},
|
||||
{"not said: still no build reads it", merge(showcase, []string{"modules/showcase"}, false), ""},
|
||||
{"an old announcer saying nothing", merge(showcase, nil, false), ""},
|
||||
{"a manifest the merge removed, said or not", merge([]string{"modules/gone/module.json", "modules/gone/x.ts"}, nil, true), ""},
|
||||
} {
|
||||
if got := named(whatTheMergeTouched(candidates, known, c.m, nil)); got != c.want {
|
||||
t.Errorf("%s: rebuilt %q, wanted %q", c.what, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// What the announcer says reaches the controller as it is sent: the two fields, by their names on the
|
||||
// wire, and an older announcement without them reads as not said.
|
||||
func TestTheAnnouncerSaysWhichDirectoriesAreModules(t *testing.T) {
|
||||
var m link.SourceMoved
|
||||
if err := json.Unmarshal([]byte(`{"owner":"novox","repo":"mesh-catalog","merge_commit_sha":"abc",`+
|
||||
`"paths":["modules/showcase/index.ts"],"module_dirs":["modules/showcase"],"module_dirs_said":true}`), &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !m.ModuleDirsSaid || !reflect.DeepEqual(m.ModuleDirs, []string{"modules/showcase"}) {
|
||||
t.Fatalf("the announcer's word was lost: %+v", m)
|
||||
}
|
||||
var old link.SourceMoved
|
||||
if err := json.Unmarshal([]byte(`{"owner":"novox","repo":"mesh-catalog","merge_commit_sha":"abc","paths":["x"]}`), &old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if old.ModuleDirsSaid || old.ModuleDirs != nil {
|
||||
t.Fatalf("an older announcement says nothing about module directories: %+v", old)
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user