Compare commits
165
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
002f203079 | ||
|
|
d77e4062dd | ||
|
|
f024be6e04 | ||
|
|
b053dc66bf | ||
|
|
c0b1225f6d | ||
|
|
5ba88c69fa | ||
|
|
8635560993 | ||
|
|
f9be71b9ad | ||
|
|
307dd23d6d | ||
|
|
dcaf843b8a | ||
|
|
7ed7da1398 | ||
|
|
1316a405a6 | ||
|
|
a0754ab06d | ||
|
|
adaf525a88 | ||
|
|
a8d86bb10b | ||
|
|
9deefc6370 | ||
|
|
d7098c4b27 | ||
|
|
d21949c041 | ||
|
|
7806a42c45 | ||
|
|
f71250c3dd | ||
|
|
ba2806b985 | ||
|
|
9497dda929 | ||
|
|
3b4ba29bec | ||
|
|
3eb48c949f | ||
|
|
5f13bf0aff | ||
|
|
73800225c3 | ||
|
|
becff66e0a | ||
|
|
2aebf8f40e | ||
|
|
0ca7a83e0b | ||
|
|
d16c3db9d2 | ||
|
|
39ec294853 | ||
|
|
6ba65a12f2 | ||
|
|
ea24e7890f | ||
|
|
0714f7a509 | ||
|
|
d03f87b00d | ||
|
|
51fa3a73ab | ||
|
|
383176a7ea | ||
|
|
671d56a08a | ||
|
|
b7f46559f0 | ||
|
|
abc022eece | ||
|
|
87eca0c9a9 | ||
|
|
e5ccf48468 | ||
|
|
c7c7268db0 | ||
|
|
d3ef6fa44c | ||
|
|
1af6efb356 | ||
|
|
782aa9f737 | ||
|
|
a96226f228 | ||
|
|
5c1c6f56c9 | ||
|
|
3c655d83e1 | ||
|
|
e63459dd70 | ||
|
|
24b833d19d | ||
|
|
79108a72a0 | ||
|
|
930b13ff47 | ||
|
|
39531a9972 | ||
|
|
a97ba661d9 | ||
|
|
bd33abf3dd | ||
|
|
1f008aeb18 | ||
|
|
26a0f45269 | ||
|
|
9bd51f3c6c | ||
|
|
8c78bcf9ce | ||
|
|
858b15a1a3 | ||
|
|
b3d6d65db1 | ||
|
|
c48d68ac92 | ||
|
|
af0a32a780 | ||
|
|
95736b0eab | ||
|
|
037bc32a6f | ||
|
|
22043a3c86 | ||
|
|
719ca7ae0b | ||
|
|
bdb6efa8c6 | ||
|
|
a10aeb9843 | ||
|
|
5ebb3520ec | ||
|
|
70bc7ad3db | ||
|
|
086a5eb7a0 | ||
|
|
01a541cadf | ||
|
|
569a9e7ae8 | ||
|
|
a076ccfe51 | ||
|
|
8d24e7bbb5 | ||
|
|
3a486014fc | ||
|
|
d9173d49b8 | ||
|
|
fb5bb1e3de | ||
|
|
aaca73afb2 | ||
|
|
06fa70d081 | ||
|
|
5981f0ae85 | ||
|
|
c9582ce041 | ||
|
|
e99c2b22cb | ||
|
|
617f95f2bf | ||
|
|
06e86ccd05 | ||
|
|
386e306642 | ||
|
|
f91164863b | ||
|
|
54db7c3458 | ||
|
|
96b05fbd85 | ||
|
|
3a3317b487 | ||
|
|
24bc736393 | ||
|
|
33e63ee5b4 | ||
|
|
5c128c1c18 | ||
|
|
c02a8ec4c8 | ||
|
|
983fd839eb | ||
|
|
1455d9560e | ||
|
|
65ae6a9f89 | ||
|
|
596f0b4532 | ||
|
|
d452a47be0 | ||
|
|
99923bed12 | ||
|
|
434bac93ce | ||
|
|
1e371cd829 | ||
|
|
157bb18fa8 | ||
|
|
6fd0683425 | ||
|
|
193e669bf1 | ||
|
|
20bdf96b31 | ||
|
|
8368516253 | ||
|
|
c76c64a4b5 | ||
|
|
179f73c18f | ||
|
|
505db85539 | ||
|
|
09d347a045 | ||
|
|
e80a4b1642 | ||
|
|
35babebbd1 | ||
|
|
4b7ad9a387 | ||
|
|
8fdbf9ca90 | ||
|
|
db0069f262 | ||
|
|
960539066e | ||
|
|
d2c6e8fc5a | ||
|
|
d0b0c7b02a | ||
|
|
b75a17e51b | ||
|
|
c4f57432e1 | ||
|
|
a9c3bd8e6d | ||
|
|
74c59661b1 | ||
|
|
a7903562b4 | ||
|
|
14246bcf01 | ||
|
|
2b489783ec | ||
|
|
3d871c992d | ||
|
|
a1a03f9880 | ||
|
|
eecbfd6693 | ||
|
|
d93f1e4eab | ||
|
|
69e1215424 | ||
|
|
5e3ae7b835 | ||
|
|
bc3709a66a | ||
|
|
6b44d7c414 | ||
|
|
8962454530 | ||
|
|
70f62ee17b | ||
|
|
a4f6bb8ce0 | ||
|
|
a734d427c0 | ||
|
|
f2d6ab3bf9 | ||
|
|
f94ee2dd0e | ||
|
|
62a02d7e94 | ||
|
|
08e3aa04e7 | ||
|
|
d93dc2628f | ||
|
|
23ca9cadad | ||
|
|
204a226e36 | ||
|
|
8d9e4bdb77 | ||
|
|
3f224c2876 | ||
|
|
a53dc8c586 | ||
|
|
bb746505dc | ||
|
|
514f7a46cd | ||
|
|
c9fd6d7887 | ||
|
|
4d2ec6e6a8 | ||
|
|
dba95f7e27 | ||
|
|
3ce66e8369 | ||
|
|
3f71b91fb8 | ||
|
|
2dfffd6dac | ||
|
|
36f2fd1c2c | ||
|
|
7a5d670e4e | ||
|
|
8a85e2d02e | ||
|
|
146d7da9ef | ||
|
|
10cfbd094a | ||
|
|
353cf88a4b | ||
|
|
9eaa3a623d |
@@ -1 +1,2 @@
|
||||
node_modules/
|
||||
node_modules
|
||||
|
||||
@@ -45,6 +45,10 @@ mesh-lab diagram --live <instance> draw what is actually standing
|
||||
correctly and snapshots roughly 76× slower — which does not make the lab slow, it makes it
|
||||
unused, and a warning about that is read once and ignored forever.
|
||||
|
||||
Behind a VPN client that routes every private range, incus finds no free range for the uplink a
|
||||
machine fetches its images over, and nothing can be raised. Name one the host does not route:
|
||||
`MESH_LAB_UPLINK_V4=192.168.231.1/24`.
|
||||
|
||||
If the incus socket is not reachable as your user — the group was granted to a session that
|
||||
already existed — set `MESH_LAB_INCUS="sudo -n incus"`.
|
||||
|
||||
@@ -139,7 +143,14 @@ nothing. That is `novox/hq` 04-ISSUES/005 exactly, and it has now been rediscove
|
||||
is written down here rather than reconstructed a third time.
|
||||
|
||||
```sh
|
||||
# Built with `make SYSTEM=arch build`, NOT with a bare `go build`. The system a host was built for
|
||||
# is a link-time value, and one built without it refuses everything it is given with "this host was
|
||||
# built for \"\", which is not a system it knows" — which reads like a broken bundle and is not
|
||||
# (novox/hq 04-ISSUES/146).
|
||||
export MESH_LAB_HOST_BINARY=<mesh-host>/mesh-host
|
||||
# The one genesis bundle, on the bus the mesh runs on (novox/hq ADR 0331). The bundle that raised the
|
||||
# predecessor's broker, and a control plane that crash-looped on a missing MESH_BUS_NATS, is gone
|
||||
# (novox/hq issue 146).
|
||||
export MESH_LAB_BUNDLE=<mesh-host>/examples/foundation-first-node.lock
|
||||
export MESH_LAB_MODULES=<mesh-controller>/examples/modules
|
||||
export MESH_LAB_BUILDER=<mesh-controller>/build/mesh-builder # build/, which is git-ignored
|
||||
@@ -163,6 +174,22 @@ export MESH_LAB_OBJECTSTORE_PROVISIONER=<somewhere>/objectstore-provisioner
|
||||
export MESH_LAB_ROUTE_PROXY=<somewhere>/route-proxy
|
||||
```
|
||||
|
||||
### When a bed fails, before anything else
|
||||
|
||||
Three habits, each of which cost a run on 2026-09-29 before it was adopted (novox/hq
|
||||
04-ISSUES/146):
|
||||
|
||||
- **`MESH_LAB_KEEP=1` leaves the machine standing.** Every answer in that sequence came from
|
||||
shelling into it afterwards — the host's log, the bus's log, the file the bus was actually given,
|
||||
the identity the node actually stored. The test output said only that nothing had converged.
|
||||
- **`MESH_LAB_WARM=1` between attempts.** A fault found on the fifth run is a fault the first four
|
||||
runs paid full price for; warm restores a snapshot instead of raising a machine, and a commit
|
||||
invalidates it, so it is safe to leave on while iterating and off for the run that counts.
|
||||
- **Rebuild what the bed places, not just what you changed.** The host binary, the control-plane
|
||||
image and the bundle are three repositories, and a bed testing yesterday's binary reports on code
|
||||
nobody is looking at (04-ISSUES/005). `make image` in mesh-controller reads its base from the
|
||||
manifest, so it no longer needs a digest found by hand.
|
||||
|
||||
`MESH_LAB_HOST_BINARY` and `MESH_LAB_MODULES` do double duty: the repository each sits in is what
|
||||
`suite` rebuilds and what the receipt claims. Point the run at a repository and it is built and
|
||||
claimed; leave it out and it is neither. `MESH_LAB_CATALOG` is claimed without being built as a
|
||||
@@ -171,8 +198,12 @@ runs, so the receipt names the catalogue's commit too, and a run taken before a
|
||||
says so (novox/hq 04-ISSUES/073). What IS built from it are the module runtimes the named beds'
|
||||
scenarios stock (`mesh-runtime-<module>:development`): each is compared against the module's
|
||||
source and the tool runtime and SDK it is built on (`MESH_TOOLS`, `MESH_SDK`, or the checkouts
|
||||
beside this one — they need their `node_modules`), and rebuilt by `scripts/build-module-runtime.sh`
|
||||
where the image is older, missing, or the source is uncommitted (novox/hq 04-ISSUES/075).
|
||||
beside this one), and rebuilt by `scripts/build-module-runtime.sh` where the image is older, missing,
|
||||
or the source is uncommitted (novox/hq 04-ISSUES/075). The image is today's tool runtime: `node-tools`,
|
||||
built with Go from mesh-tools' `node-tools/`, launching the module's compiled bundle through the
|
||||
launcher the mesh's builder writes beside each entrypoint (novox/hq ADR 0193) — so building one needs
|
||||
`go`, and `npm` for the SDK's compiler. It serves on the node's runtime credential or a
|
||||
`MESH_BROKER_URL`, never the served module's own: the runtime refuses that.
|
||||
`--no-build` skips this too, and then the bed runs whatever image the store holds.
|
||||
|
||||
Check before running a long suite — it says which of these are missing rather than skipping
|
||||
@@ -383,3 +414,20 @@ That suite earned itself on its first run: it found that a snapshot of a running
|
||||
miss a file written seconds earlier — not stale, **absent** — because the write was still in
|
||||
the guest's page cache. The design had listed that as an open question. The test answered it,
|
||||
and `snapshot` now flushes first.
|
||||
|
||||
## The replays (`replays/`)
|
||||
|
||||
Every core incident, replayed (novox/hq to-be 45 §9, M9) — in Go, a module of its own. `register.go`
|
||||
names each replay, its issue, its fix and how it is run; a replay of one component's logic is a test in
|
||||
that component's repository (the controller's `replays_test.go`), and a replay of what the mesh runs —
|
||||
the bus server's release, the resolver the catalogue configures under musl and glibc — is here.
|
||||
|
||||
```
|
||||
cd replays
|
||||
MESH_TEST_NATS=nats://… go test ./... the bus and resolver replays (the resolver's needs a container runtime)
|
||||
go run ./cmd/prove [R262 …] each replay on the commit before its fix (must fail) and on it (must pass)
|
||||
```
|
||||
|
||||
The prover finds the core repositories beside this one, as the beds do. The build seat runs the bus and
|
||||
resolver replays before every core and catalogue merge, against the bus of the release the mesh runs and
|
||||
the change's own catalogue.
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
package asks
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/asker"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// labAsker is the module `lab-asker` asking the operator, as any module does: the SDK's asker client
|
||||
// (mesh-sdk go/asker) over the lab's bus, on that module's own composed credential. What it performs on a
|
||||
// warrant is one switch flipped, bound when it asked, and counted: the proof that an approval does exactly
|
||||
// that one thing, once.
|
||||
type labAsker struct {
|
||||
t *testing.T
|
||||
client *asker.Client
|
||||
nc *nats.Conn
|
||||
|
||||
mu sync.Mutex
|
||||
warrants []asks.Warrant
|
||||
handled []asker.Handled
|
||||
acts []map[string]string // what was performed
|
||||
}
|
||||
|
||||
// flip is the act an approve option binds.
|
||||
func flip(which string) asks.Act {
|
||||
return asks.Act{"verb": "lab.flip", "arg.switch": which}
|
||||
}
|
||||
|
||||
func newLabAsker(t *testing.T, l *lab) *labAsker {
|
||||
nc, _ := l.as(machine + ".lab-asker")
|
||||
js, err := nc.JetStream()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := &labAsker{t: t, nc: nc}
|
||||
a.client = &asker.Client{Module: "lab-asker", State: &memState{vals: map[string]json.RawMessage{}, revs: map[string]uint64{}},
|
||||
Now: time.Now,
|
||||
Publish: func(subject string, body any, id string) error {
|
||||
raw, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = js.Publish(subject, raw, nats.MsgId("lab-asker."+id))
|
||||
return err
|
||||
}}
|
||||
// Its warrants, on the one subject the bus lets it hear them on.
|
||||
sub, err := nc.Subscribe(asks.DecidedSubject("lab-asker"), func(m *nats.Msg) {
|
||||
var w asks.Warrant
|
||||
if err := json.Unmarshal(m.Data, &w); err != nil {
|
||||
t.Errorf("a word on the asker's decided subject is not a warrant: %v", err)
|
||||
return
|
||||
}
|
||||
a.take(w)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
return a
|
||||
}
|
||||
|
||||
// take hands one warrant to the SDK's Take, which performs the option once, and only the act it bound.
|
||||
func (a *labAsker) take(w asks.Warrant) asker.Handled {
|
||||
h, err := a.client.Take(w, func(o asks.Option, w asks.Warrant) error {
|
||||
if o.Level == asks.Acknowledge {
|
||||
return nil // decline: nothing is performed
|
||||
}
|
||||
act := flip(o.ID)
|
||||
if err := o.Performs(act); err != nil {
|
||||
return err
|
||||
}
|
||||
a.mu.Lock()
|
||||
a.acts = append(a.acts, act)
|
||||
a.mu.Unlock()
|
||||
return nil
|
||||
})
|
||||
a.mu.Lock()
|
||||
a.warrants = append(a.warrants, w)
|
||||
a.handled = append(a.handled, h)
|
||||
a.mu.Unlock()
|
||||
if err != nil {
|
||||
a.t.Logf("the asker refused a warrant for %s: %v", w.Ask, err)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// ask asks the operator: one approve option binding a switch, one decline that acknowledges.
|
||||
func (a *labAsker) ask(id, headline string, expires time.Duration) asks.Ask {
|
||||
a.t.Helper()
|
||||
binds, err := asks.ActDigest(flip("approve"))
|
||||
if err != nil {
|
||||
a.t.Fatal(err)
|
||||
}
|
||||
q := asks.Ask{ID: id, Headline: headline,
|
||||
Explanation: "Needs you: approve or decline. The lab asks whether to flip one switch.",
|
||||
Who: asks.Operator, Expires: time.Now().Add(expires), OnExpiry: "nothing is flipped",
|
||||
Options: []asks.Option{
|
||||
{ID: "approve", Label: "Approve", Does: "the switch is flipped, once", Level: asks.Approve, Binds: binds},
|
||||
{ID: "decline", Label: "Decline", Does: "nothing is done", Level: asks.Acknowledge},
|
||||
}}
|
||||
if err := a.client.Ask(q); err != nil {
|
||||
a.t.Fatalf("the asker could not ask %s: %v", id, err)
|
||||
}
|
||||
return q
|
||||
}
|
||||
|
||||
// warrantFor waits for the router's word on one ask.
|
||||
func (a *labAsker) warrantFor(id string, within time.Duration) asks.Warrant {
|
||||
a.t.Helper()
|
||||
deadline := time.Now().Add(within)
|
||||
for time.Now().Before(deadline) {
|
||||
a.mu.Lock()
|
||||
for _, w := range a.warrants {
|
||||
if w.Ask == id {
|
||||
a.mu.Unlock()
|
||||
return w
|
||||
}
|
||||
}
|
||||
a.mu.Unlock()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
a.t.Fatalf("no word on the ask %s within %s", id, within)
|
||||
return asks.Warrant{}
|
||||
}
|
||||
|
||||
// noWordOn says the router said nothing on an ask for a while.
|
||||
func (a *labAsker) noWordOn(id string, wait time.Duration) bool {
|
||||
time.Sleep(wait)
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
for _, w := range a.warrants {
|
||||
if w.Ask == id {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (a *labAsker) performed() []map[string]string {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
return append([]map[string]string(nil), a.acts...)
|
||||
}
|
||||
|
||||
func (a *labAsker) wordsOn(id string) int {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
n := 0
|
||||
for _, w := range a.warrants {
|
||||
if w.Ask == id {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// memState is the asker's own record, kept in memory: the lab's asker restarts never.
|
||||
type memState struct {
|
||||
mu sync.Mutex
|
||||
vals map[string]json.RawMessage
|
||||
revs map[string]uint64
|
||||
n uint64
|
||||
}
|
||||
|
||||
func (m *memState) Get(k string) (*stdio.StateEntry, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
v, ok := m.vals[k]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
return &stdio.StateEntry{Key: k, Value: v, Revision: m.revs[k]}, nil
|
||||
}
|
||||
|
||||
func (m *memState) Create(k string, v any) (uint64, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if _, ok := m.vals[k]; ok {
|
||||
return 0, stdio.ErrChanged
|
||||
}
|
||||
return m.put(k, v)
|
||||
}
|
||||
|
||||
func (m *memState) Update(k string, v any, rev uint64) (uint64, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if rev == 0 || m.revs[k] != rev {
|
||||
return 0, stdio.ErrChanged
|
||||
}
|
||||
return m.put(k, v)
|
||||
}
|
||||
|
||||
func (m *memState) put(k string, v any) (uint64, error) {
|
||||
raw, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
m.n++
|
||||
m.vals[k], m.revs[k] = raw, m.n
|
||||
return m.n, nil
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
// Package asks is the lab's proof of the operator's answers (novox/hq ADR 0259): the whole flow, from an ask
|
||||
// to the act its approval authorises, on a real bus composed by the controller, with a fake Telegram. The
|
||||
// proof is TestTheOperatorsAnswerEndToEnd; the live acceptance after rollout is live-acceptance.sh.
|
||||
package asks
|
||||
@@ -0,0 +1,351 @@
|
||||
package asks
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"html"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fakeTelegram is the Bot API as the channel uses it, and nothing more: the lab's phone. It holds the updates
|
||||
// the lab makes happen (a message typed, a button tapped) for the channel's long poll, and keeps every message
|
||||
// the bot sends or edits, by chat, so the lab reads what the operator would see. It never reaches Telegram.
|
||||
type fakeTelegram struct {
|
||||
t *testing.T
|
||||
server *httptest.Server
|
||||
token string
|
||||
|
||||
mu sync.Mutex
|
||||
updates []map[string]any
|
||||
next int64
|
||||
arrived chan struct{}
|
||||
sent []tgMessage
|
||||
msgID int64
|
||||
taps int // answerCallbackQuery calls
|
||||
deleted int
|
||||
// toasts are the words a tap's acknowledgement showed over the chat (answerCallbackQuery), by the chat the
|
||||
// tap came from: what the phone shows the tapper and never puts in the chat. Kept apart from sent, so a
|
||||
// chat's messages stay what the chat holds.
|
||||
toasts []tgMessage
|
||||
tappedIn map[string]int64 // a tap's callback id to the chat it came from
|
||||
}
|
||||
|
||||
// tgMessage is one message the bot sent or edited: its chat, its id, its words and its buttons (label →
|
||||
// callback data).
|
||||
type tgMessage struct {
|
||||
Chat int64
|
||||
ID int64
|
||||
Text string
|
||||
Buttons [][2]string
|
||||
Edited bool
|
||||
At time.Time
|
||||
}
|
||||
|
||||
func newFakeTelegram(t *testing.T, token string) *fakeTelegram {
|
||||
f := &fakeTelegram{t: t, token: token, next: 1, arrived: make(chan struct{}, 1)}
|
||||
f.server = httptest.NewServer(http.HandlerFunc(f.serve))
|
||||
t.Cleanup(f.server.Close)
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *fakeTelegram) URL() string { return f.server.URL }
|
||||
|
||||
func (f *fakeTelegram) serve(w http.ResponseWriter, r *http.Request) {
|
||||
prefix := "/bot" + f.token + "/"
|
||||
if !strings.HasPrefix(r.URL.Path, prefix) {
|
||||
// A wrong token is what the real API answers 401 to.
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"ok": false, "error_code": 401, "description": "Unauthorized"})
|
||||
return
|
||||
}
|
||||
method := strings.TrimPrefix(r.URL.Path, prefix)
|
||||
var body map[string]any
|
||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||
ok := func(result any) { _ = json.NewEncoder(w).Encode(map[string]any{"ok": true, "result": result}) }
|
||||
switch method {
|
||||
case "getMe":
|
||||
ok(map[string]any{"id": 1000, "is_bot": true, "first_name": "Lab", "username": "lab_mesh_bot"})
|
||||
case "getUpdates":
|
||||
offset := int64(num(body["offset"]))
|
||||
wait := time.Duration(num(body["timeout"])) * time.Second
|
||||
if wait > 2*time.Second {
|
||||
wait = 2 * time.Second // the lab's phone answers quickly; the channel polls again at once
|
||||
}
|
||||
deadline := time.Now().Add(wait)
|
||||
for {
|
||||
f.mu.Lock()
|
||||
var out []map[string]any
|
||||
for _, u := range f.updates {
|
||||
if int64(num(u["update_id"])) >= offset {
|
||||
out = append(out, u)
|
||||
}
|
||||
}
|
||||
f.mu.Unlock()
|
||||
if len(out) > 0 {
|
||||
f.t.Logf("phone: getUpdates from %d hands %d update(s)", offset, len(out))
|
||||
}
|
||||
if len(out) > 0 || time.Now().After(deadline) {
|
||||
ok(orEmpty(out))
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-f.arrived:
|
||||
case <-time.After(time.Until(deadline)):
|
||||
}
|
||||
}
|
||||
case "sendMessage", "editMessageText":
|
||||
chat := int64(num(body["chat_id"]))
|
||||
text, _ := body["text"].(string)
|
||||
if mode, _ := body["parse_mode"].(string); mode == "HTML" {
|
||||
// What a message shows, and what a tap on it carries back, is its text with the markup read: tags gone,
|
||||
// entities unescaped — as the service gives it (novox/hq issue 692).
|
||||
text = shownOfHTML(text)
|
||||
}
|
||||
m := tgMessage{Chat: chat, Text: text, Buttons: buttonsOf(body["reply_markup"]), At: time.Now()}
|
||||
f.mu.Lock()
|
||||
if method == "sendMessage" {
|
||||
f.msgID++
|
||||
m.ID = f.msgID
|
||||
} else {
|
||||
m.ID, m.Edited = int64(num(body["message_id"])), true
|
||||
}
|
||||
f.sent = append(f.sent, m)
|
||||
f.mu.Unlock()
|
||||
f.t.Logf("phone: %s to %d (#%d): %q %v", method, chat, m.ID, firstLine(text), m.Buttons)
|
||||
ok(map[string]any{"message_id": m.ID, "chat": map[string]any{"id": chat, "type": "private"}, "text": text})
|
||||
case "answerCallbackQuery":
|
||||
id, _ := body["callback_query_id"].(string)
|
||||
text, _ := body["text"].(string)
|
||||
f.mu.Lock()
|
||||
f.taps++
|
||||
chat, known := f.tappedIn[id]
|
||||
if known && text != "" {
|
||||
f.toasts = append(f.toasts, tgMessage{Chat: chat, Text: text, At: time.Now()})
|
||||
}
|
||||
f.mu.Unlock()
|
||||
if text != "" {
|
||||
f.t.Logf("phone: answerCallbackQuery %s in %d: %q", id, chat, firstLine(text))
|
||||
}
|
||||
ok(true)
|
||||
case "deleteMessage":
|
||||
f.mu.Lock()
|
||||
f.deleted++
|
||||
f.mu.Unlock()
|
||||
ok(true)
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"ok": false, "error_code": 404, "description": "Not Found: " + method})
|
||||
}
|
||||
}
|
||||
|
||||
// push makes one thing happen on the phone, for the channel's next poll.
|
||||
func (f *fakeTelegram) push(u map[string]any) {
|
||||
f.mu.Lock()
|
||||
u["update_id"] = f.next
|
||||
f.next++
|
||||
f.updates = append(f.updates, u)
|
||||
f.mu.Unlock()
|
||||
select {
|
||||
case f.arrived <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// typed is a message an account writes in a chat with the bot; private when the chat is the account's own.
|
||||
func (f *fakeTelegram) typed(from int64, chat int64, chatType, text string) {
|
||||
f.push(map[string]any{"message": map[string]any{"message_id": 9000 + f.nextID(), "text": text,
|
||||
"from": map[string]any{"id": from, "is_bot": false, "first_name": fmt.Sprintf("Account %d", from)},
|
||||
"chat": map[string]any{"id": chat, "type": chatType}}})
|
||||
}
|
||||
|
||||
// tapped is a button tapped on a message the bot sent, carrying the button's data and the message's words as
|
||||
// the service gives them: the words the message shows now (its last send or edit), as on the phone.
|
||||
func (f *fakeTelegram) tapped(from int64, chat int64, chatType string, onMessage int64, data string) {
|
||||
f.tappedOn(from, chat, chatType, onMessage, data, f.textOf(onMessage))
|
||||
}
|
||||
|
||||
// tappedOn is a tap on a message showing the words given: what a message changed under the operator's thumb,
|
||||
// or forwarded and edited elsewhere, carries.
|
||||
func (f *fakeTelegram) tappedOn(from int64, chat int64, chatType string, onMessage int64, data, text string) {
|
||||
id := fmt.Sprintf("cb-%d", f.nextID())
|
||||
f.mu.Lock()
|
||||
if f.tappedIn == nil {
|
||||
f.tappedIn = map[string]int64{}
|
||||
}
|
||||
f.tappedIn[id] = chat
|
||||
f.mu.Unlock()
|
||||
f.push(map[string]any{"callback_query": map[string]any{"id": id, "data": data,
|
||||
"from": map[string]any{"id": from, "is_bot": false, "first_name": fmt.Sprintf("Account %d", from)},
|
||||
"message": map[string]any{"message_id": onMessage, "text": text,
|
||||
"chat": map[string]any{"id": chat, "type": chatType}}}})
|
||||
}
|
||||
|
||||
// textOf is what a message the bot sent shows now.
|
||||
func (f *fakeTelegram) textOf(id int64) string {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
text := ""
|
||||
for _, m := range f.sent {
|
||||
if m.ID == id {
|
||||
text = m.Text
|
||||
}
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
func (f *fakeTelegram) nextID() int64 {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.msgID++
|
||||
return f.msgID
|
||||
}
|
||||
|
||||
// waitFor waits for a message to a chat that satisfies match, sent after since.
|
||||
func (f *fakeTelegram) waitFor(what string, chat int64, since time.Time, within time.Duration, match func(tgMessage) bool) tgMessage {
|
||||
f.t.Helper()
|
||||
deadline := time.Now().Add(within)
|
||||
for time.Now().Before(deadline) {
|
||||
f.mu.Lock()
|
||||
for _, m := range f.sent {
|
||||
if m.Chat == chat && !m.At.Before(since) && match(m) {
|
||||
f.mu.Unlock()
|
||||
return m
|
||||
}
|
||||
}
|
||||
f.mu.Unlock()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
f.t.Fatalf("the phone never showed %s (chat %d) within %s; it showed:\n%s", what, chat, within, f.transcript())
|
||||
return tgMessage{}
|
||||
}
|
||||
|
||||
// toldTapper waits for words the phone showed in a chat at or after a time, as a message in it or as a tap's
|
||||
// acknowledgement over it: where the channel says the router's word on a tap (issue 372).
|
||||
func (f *fakeTelegram) toldTapper(what string, chat int64, since time.Time, within time.Duration, match func(string) bool) string {
|
||||
f.t.Helper()
|
||||
deadline := time.Now().Add(within)
|
||||
for time.Now().Before(deadline) {
|
||||
f.mu.Lock()
|
||||
for _, m := range append(append([]tgMessage{}, f.sent...), f.toasts...) {
|
||||
if m.Chat == chat && !m.At.Before(since) && match(m.Text) {
|
||||
f.mu.Unlock()
|
||||
return m.Text
|
||||
}
|
||||
}
|
||||
f.mu.Unlock()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
f.t.Fatalf("the phone never showed %s (chat %d) within %s; it showed:\n%s", what, chat, within, f.transcript())
|
||||
return ""
|
||||
}
|
||||
|
||||
// since is every message to a chat sent at or after a time.
|
||||
func (f *fakeTelegram) since(chat int64, at time.Time) []tgMessage {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var out []tgMessage
|
||||
for _, m := range f.sent {
|
||||
if m.Chat == chat && !m.At.Before(at) {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (f *fakeTelegram) transcript() string {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var b strings.Builder
|
||||
for _, m := range f.sent {
|
||||
fmt.Fprintf(&b, " %s to %d #%d edited=%v %q %v\n", m.At.Format("15:04:05.000"), m.Chat, m.ID, m.Edited,
|
||||
firstLine(m.Text), m.Buttons)
|
||||
}
|
||||
for _, m := range f.toasts {
|
||||
fmt.Fprintf(&b, " %s to %d tap acknowledged %q\n", m.At.Format("15:04:05.000"), m.Chat, firstLine(m.Text))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// button is the data of the button labelled so on a message.
|
||||
func button(m tgMessage, label string) string {
|
||||
for _, b := range m.Buttons {
|
||||
if b[0] == label {
|
||||
return b[1]
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func labels(m tgMessage) []string {
|
||||
var out []string
|
||||
for _, b := range m.Buttons {
|
||||
out = append(out, b[0])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func buttonsOf(markup any) [][2]string {
|
||||
var out [][2]string
|
||||
m, _ := markup.(map[string]any)
|
||||
rows, _ := m["inline_keyboard"].([]any)
|
||||
for _, row := range rows {
|
||||
cells, _ := row.([]any)
|
||||
for _, c := range cells {
|
||||
cell, _ := c.(map[string]any)
|
||||
text, _ := cell["text"].(string)
|
||||
data, _ := cell["callback_data"].(string)
|
||||
out = append(out, [2]string{text, data})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func num(v any) float64 {
|
||||
switch n := v.(type) {
|
||||
case float64:
|
||||
return n
|
||||
case int64:
|
||||
return float64(n)
|
||||
case int:
|
||||
return float64(n)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func orEmpty(in []map[string]any) []map[string]any {
|
||||
if in == nil {
|
||||
return []map[string]any{}
|
||||
}
|
||||
return in
|
||||
}
|
||||
|
||||
func firstLine(s string) string {
|
||||
if i := strings.IndexByte(s, '\n'); i >= 0 {
|
||||
return s[:i] + " …"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
var htmlTag = regexp.MustCompile(`</?[a-zA-Z][^>]*>`)
|
||||
|
||||
// shownOfHTML is a text sent in Telegram's HTML as the service shows it and gives it back: its tags read and taken
|
||||
// out, its entities written as the characters they stand for.
|
||||
func shownOfHTML(text string) string {
|
||||
return html.UnescapeString(htmlTag.ReplaceAllString(text, ""))
|
||||
}
|
||||
|
||||
// The fake phone shows a message sent in HTML as Telegram does (novox/hq issue 692): the words without their
|
||||
// markup, so a tap carries back what the channel digests, as on the real service.
|
||||
func TestTheFakePhoneShowsHTMLAsTelegramDoes(t *testing.T) {
|
||||
in := "<b>Flip the lab's switch?</b>\nA <b> & c\n<code>+ k: v</code>\n\n<blockquote expandable>Asked by lab-asker.</blockquote>"
|
||||
want := "Flip the lab's switch?\nA <b> & c\n+ k: v\n\nAsked by lab-asker."
|
||||
if got := shownOfHTML(in); got != want {
|
||||
t.Errorf("%q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
module github.com/novox/mesh-lab/asks
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.10
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
)
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.10 h1:EHhnPRrkw7/CSQAqRsxYnTvlTZ3dI02IPO6WSmVX80o=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.10/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
@@ -0,0 +1,537 @@
|
||||
package asks
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// machine is the lab's one machine, as the controller's fixture names it.
|
||||
const machine = "anchor"
|
||||
|
||||
// credential is what a lab process connects as (the controller fixture's credentials.json, mesh-tools'
|
||||
// bus.Credential).
|
||||
type credential struct {
|
||||
URL string `json:"url"`
|
||||
Node string `json:"node,omitempty"`
|
||||
Module string `json:"module,omitempty"`
|
||||
User string `json:"user"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
|
||||
// lab is one run of the proof: the repositories it builds from, its directory, its bus, its phone.
|
||||
type lab struct {
|
||||
t *testing.T
|
||||
repos clones
|
||||
dir string
|
||||
url string
|
||||
creds map[string]credential
|
||||
server *server.Server
|
||||
// observer is the lab's own user on the bus, outside the mesh's composition: it plays the controller's
|
||||
// conditions verb and reads the router's state, and is never used to show that something is allowed.
|
||||
observer *nats.Conn
|
||||
tg *fakeTelegram
|
||||
|
||||
mu sync.Mutex
|
||||
conditions []map[string]any
|
||||
// notFree are the machines the controller's root-free verb answers as not root-free.
|
||||
notFree map[string]string
|
||||
rootAsked int
|
||||
desk []string // what the desk was shown
|
||||
procs []*exec.Cmd
|
||||
logs map[string]*bytes.Buffer
|
||||
}
|
||||
|
||||
// clones is where each core repository the proof builds is checked out: side by side in MESH_LAB_REPOS (beside
|
||||
// mesh-lab by default), or one by one in MESH_LAB_CONTROLLER, MESH_LAB_TOOLS and MESH_LAB_CATALOGUE — a branch
|
||||
// under review is often a worktree of its own.
|
||||
type clones struct{ controller, tools, catalogue string }
|
||||
|
||||
func clonesFor(t *testing.T) clones {
|
||||
repos := os.Getenv("MESH_LAB_REPOS")
|
||||
if repos == "" {
|
||||
repos = filepath.Join("..", "..")
|
||||
}
|
||||
at := func(env, name string) string {
|
||||
dir := os.Getenv(env)
|
||||
if dir == "" {
|
||||
dir = filepath.Join(repos, name)
|
||||
}
|
||||
abs, err := filepath.Abs(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return abs
|
||||
}
|
||||
c := clones{controller: at("MESH_LAB_CONTROLLER", "mesh-controller"), tools: at("MESH_LAB_TOOLS", "mesh-tools"),
|
||||
catalogue: at("MESH_LAB_CATALOGUE", "mesh-catalog")}
|
||||
for _, want := range []string{filepath.Join(c.controller, "go.mod"), filepath.Join(c.tools, "node-tools", "go.mod"),
|
||||
filepath.Join(c.catalogue, "modules", "messenger", "go.mod"), filepath.Join(c.catalogue, "modules", "telegram", "go.mod")} {
|
||||
if _, err := os.Stat(want); err != nil {
|
||||
t.Skipf("NOT RUN: the proof builds the controller, the runtime, the router and the Telegram channel from "+
|
||||
"their clones, and there is no %s (set MESH_LAB_REPOS, or MESH_LAB_CONTROLLER, MESH_LAB_TOOLS and "+
|
||||
"MESH_LAB_CATALOGUE)", want)
|
||||
}
|
||||
}
|
||||
// The controller beside is, on the build seat, the one the mesh runs: until it carries the proof's fixture
|
||||
// (mesh-controller's TestTheAsksLabBus), the proof cannot compose its bus, and says so rather than fail
|
||||
// every pull request of the lab for a merge not yet made.
|
||||
fixtures, _ := filepath.Glob(filepath.Join(c.controller, "internal", "inventory", "*_test.go"))
|
||||
has := false
|
||||
for _, f := range fixtures {
|
||||
if body, err := os.ReadFile(f); err == nil && strings.Contains(string(body), "func TestTheAsksLabBus(") {
|
||||
has = true
|
||||
}
|
||||
}
|
||||
if !has {
|
||||
t.Skipf("NOT RUN: the controller at %s has no TestTheAsksLabBus, the fixture that composes the proof's bus "+
|
||||
"(mesh-controller #157): the proof runs once the controller it is given carries it", c.controller)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// run runs a command in a directory and fails the test with what it said.
|
||||
func (l *lab) run(dir string, env []string, name string, args ...string) string {
|
||||
l.t.Helper()
|
||||
cmd := exec.Command(name, args...)
|
||||
cmd.Dir = dir
|
||||
cmd.Env = append(os.Environ(), env...)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
l.t.Fatalf("%s %s in %s: %v\n%s", name, strings.Join(args, " "), dir, err, out)
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
|
||||
// compose asks the controller at its commit for the bus it would compose, then (bus set) to raise it.
|
||||
func (l *lab) compose(bus string) {
|
||||
// The anchor composed as a machine the controller measured root-free: verified-sender reaches the router
|
||||
// only then (ADR 0259 §8). The not-free case is played live, through the root-free verb, in the proof.
|
||||
env := []string{"GOFLAGS=-mod=vendor", "GOPROXY=off", "MESH_LAB_ASKS_OUT=" + l.dir, "MESH_LAB_ASKS_ROOT_FREE=true",
|
||||
"MESH_LAB_ASKS_CATALOGUE=" + filepath.Join(l.repos.catalogue, "modules"),
|
||||
"MESH_LAB_ASKS_RUNTIME=" + filepath.Join(l.repos.tools, "node-tools", "module.json")}
|
||||
if bus != "" {
|
||||
env = append(env, "MESH_LAB_ASKS_BUS="+bus)
|
||||
}
|
||||
out := l.run(l.repos.controller, env, "go", "test", "-count=1", "-v", "-run",
|
||||
"^TestTheAsksLabBus$", "./internal/inventory/")
|
||||
if !strings.Contains(out, "--- PASS: TestTheAsksLabBus") {
|
||||
l.t.Fatalf("the controller did not compose the lab's bus (is its fixture on this commit?):\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
// startBus starts a server of the mesh's release with the controller's composed accounts, and the lab's
|
||||
// observer beside them.
|
||||
func (l *lab) startBus() {
|
||||
l.t.Helper()
|
||||
accounts, err := os.ReadFile(filepath.Join(l.dir, "accounts.conf"))
|
||||
if err != nil {
|
||||
l.t.Fatal(err)
|
||||
}
|
||||
const usersOpen = " users = [\n"
|
||||
if !bytes.Contains(accounts, []byte(usersOpen)) {
|
||||
l.t.Fatalf("the composed accounts block has no users list:\n%s", accounts)
|
||||
}
|
||||
observerPassword := fmt.Sprintf("observer-%d", time.Now().UnixNano())
|
||||
accounts = bytes.Replace(accounts, []byte(usersOpen),
|
||||
[]byte(usersOpen+fmt.Sprintf(" { user: \"lab-observer\", password: %q }\n", observerPassword)), 1)
|
||||
port := freePort(l.t)
|
||||
conf := fmt.Sprintf("listen: \"127.0.0.1:%d\"\njetstream {\n store_dir: %q\n}\n\n%s", port,
|
||||
filepath.Join(l.dir, "jetstream"), accounts)
|
||||
path := filepath.Join(l.dir, "bus.conf")
|
||||
if err := os.WriteFile(path, []byte(conf), 0o600); err != nil {
|
||||
l.t.Fatal(err)
|
||||
}
|
||||
opts, err := server.ProcessConfigFile(path)
|
||||
if err != nil {
|
||||
l.t.Fatalf("the composed configuration is not one the server reads: %v", err)
|
||||
}
|
||||
// The server's own log, in the lab's directory: a refusal it says is evidence, kept with the run.
|
||||
opts.NoSigs, opts.LogFile = true, filepath.Join(l.dir, "bus.log")
|
||||
opts.Trace = os.Getenv("MESH_LAB_ASKS_TRACE") != ""
|
||||
s, err := server.NewServer(opts)
|
||||
if err != nil {
|
||||
l.t.Fatal(err)
|
||||
}
|
||||
s.ConfigureLogger()
|
||||
go s.Start()
|
||||
if !s.ReadyForConnections(30 * time.Second) {
|
||||
l.t.Fatal("the lab's bus did not come up")
|
||||
}
|
||||
l.server, l.url = s, s.ClientURL()
|
||||
l.t.Cleanup(func() { s.Shutdown(); s.WaitForShutdown() })
|
||||
raw, err := os.ReadFile(filepath.Join(l.dir, "credentials.json"))
|
||||
if err != nil {
|
||||
l.t.Fatal(err)
|
||||
}
|
||||
if err := json.Unmarshal(raw, &l.creds); err != nil {
|
||||
l.t.Fatal(err)
|
||||
}
|
||||
l.observer, err = nats.Connect(l.url, nats.UserInfo("lab-observer", observerPassword))
|
||||
if err != nil {
|
||||
l.t.Fatal(err)
|
||||
}
|
||||
l.t.Cleanup(l.observer.Close)
|
||||
}
|
||||
|
||||
func freePort(t *testing.T) int {
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer ln.Close()
|
||||
return ln.Addr().(*net.TCPAddr).Port
|
||||
}
|
||||
|
||||
// as connects as one of the composed users. refused collects what the server refused it.
|
||||
func (l *lab) as(user string) (*nats.Conn, *refusals) {
|
||||
l.t.Helper()
|
||||
c, ok := l.creds[user]
|
||||
if !ok {
|
||||
l.t.Fatalf("the controller composed no user %s", user)
|
||||
}
|
||||
r := &refusals{}
|
||||
nc, err := nats.Connect(l.url, nats.UserInfo(c.User, c.Password), nats.CustomInboxPrefix("_INBOX."+c.User),
|
||||
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { r.add(err) }))
|
||||
if err != nil {
|
||||
l.t.Fatalf("connecting as %s: %v", user, err)
|
||||
}
|
||||
l.t.Cleanup(nc.Close)
|
||||
return nc, r
|
||||
}
|
||||
|
||||
// refusals are the permission violations the server said to one connection.
|
||||
type refusals struct {
|
||||
mu sync.Mutex
|
||||
said []string
|
||||
}
|
||||
|
||||
func (r *refusals) add(err error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.said = append(r.said, err.Error())
|
||||
}
|
||||
|
||||
func (r *refusals) of(subject string) bool {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
for _, s := range r.said {
|
||||
if strings.Contains(strings.ToLower(s), "permissions violation") && strings.Contains(s, subject) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// refusedToPublish publishes once as a user and says whether the server refused it.
|
||||
func (l *lab) refusedToPublish(user, subject string, body []byte) bool {
|
||||
l.t.Helper()
|
||||
nc, r := l.as(user)
|
||||
if err := nc.Publish(subject, body); err != nil {
|
||||
return true
|
||||
}
|
||||
_ = nc.Flush()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if r.of(subject) {
|
||||
return true
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// build builds one Go program from a directory into the lab's bin.
|
||||
func (l *lab) build(dir, pkg, name string, ldflags string) string {
|
||||
out := filepath.Join(l.dir, "bin", name)
|
||||
args := []string{"build", "-o", out}
|
||||
if ldflags != "" {
|
||||
args = append(args, "-ldflags", ldflags)
|
||||
}
|
||||
l.run(dir, []string{"GOPRIVATE=git.novox.be", "GOFLAGS=-mod=mod", "CGO_ENABLED=0"}, "go", append(args, pkg)...)
|
||||
return out
|
||||
}
|
||||
|
||||
// runtime starts the mesh's runtime on a module's own credential, serving that module's bundle alone: how
|
||||
// the mesh runs a module that `runs-as` an account of its own.
|
||||
func (l *lab) runtime(module, runtimeBin, bundle string, env map[string]string) {
|
||||
l.t.Helper()
|
||||
user := machine + "." + module
|
||||
c := l.creds[user]
|
||||
c.URL, c.Node, c.Module = l.url, machine, module
|
||||
credPath := filepath.Join(l.dir, module, "broker")
|
||||
if err := os.MkdirAll(filepath.Dir(credPath), 0o700); err != nil {
|
||||
l.t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(c)
|
||||
if err := os.WriteFile(credPath, raw, 0o600); err != nil {
|
||||
l.t.Fatal(err)
|
||||
}
|
||||
toolEnv, _ := json.Marshal(map[string]map[string]string{module: env})
|
||||
cmd := exec.Command(runtimeBin, "serve")
|
||||
cmd.Env = append(os.Environ(), "MESH_BROKER_FILE="+credPath, "MESH_TOOL_MODULES="+module+"="+bundle,
|
||||
"MESH_TOOL_ENV="+string(toolEnv))
|
||||
logs := &bytes.Buffer{}
|
||||
cmd.Stdout, cmd.Stderr = &lockedWriter{w: logs, mu: &l.mu}, &lockedWriter{w: logs, mu: &l.mu}
|
||||
if err := cmd.Start(); err != nil {
|
||||
l.t.Fatal(err)
|
||||
}
|
||||
l.mu.Lock()
|
||||
l.procs = append(l.procs, cmd)
|
||||
if l.logs == nil {
|
||||
l.logs = map[string]*bytes.Buffer{}
|
||||
}
|
||||
l.logs[module] = logs
|
||||
l.mu.Unlock()
|
||||
l.t.Cleanup(func() {
|
||||
_ = cmd.Process.Signal(os.Interrupt)
|
||||
done := make(chan struct{})
|
||||
go func() { _ = cmd.Wait(); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
_ = cmd.Process.Kill()
|
||||
}
|
||||
if l.t.Failed() || os.Getenv("MESH_LAB_ASKS_SAY") != "" {
|
||||
l.mu.Lock()
|
||||
l.t.Logf("--- what the %s runtime said:\n%s", module, logs.String())
|
||||
l.mu.Unlock()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// said is what a runtime said so far.
|
||||
func (l *lab) said(module string) string {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
if b := l.logs[module]; b != nil {
|
||||
return b.String()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// waitSaid waits for a runtime to say something.
|
||||
func (l *lab) waitSaid(module, what string, within time.Duration) {
|
||||
l.t.Helper()
|
||||
deadline := time.Now().Add(within)
|
||||
for time.Now().Before(deadline) {
|
||||
if strings.Contains(l.said(module), what) {
|
||||
return
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
l.t.Fatalf("the %s runtime never said %q within %s:\n%s", module, what, within, l.said(module))
|
||||
}
|
||||
|
||||
type lockedWriter struct {
|
||||
w *bytes.Buffer
|
||||
mu *sync.Mutex
|
||||
}
|
||||
|
||||
func (w *lockedWriter) Write(p []byte) (int, error) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.w.Write(p)
|
||||
}
|
||||
|
||||
// playController answers the controller's `conditions` verb from the lab's list, and its `root-free` verb from
|
||||
// the lab's word on each machine (free unless notFree names it), as the router asks them. That a machine is
|
||||
// root-free is the controller's own judgement, proven in its repository (probe_agent_root_test.go); here the
|
||||
// lab says it, so the proof can show what the router does with each answer.
|
||||
func (l *lab) playController() {
|
||||
l.t.Helper()
|
||||
free, err := l.observer.Subscribe("mesh.seat.mesh-controller.tool.root-free", func(m *nats.Msg) {
|
||||
var args struct {
|
||||
Machines []string `json:"machines"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Data, &args)
|
||||
var out []map[string]any
|
||||
l.mu.Lock()
|
||||
for _, name := range args.Machines {
|
||||
why, not := l.notFree[name]
|
||||
if !not {
|
||||
why = "the lab says no agent can become root here"
|
||||
}
|
||||
out = append(out, map[string]any{"machine": name, "free": !not, "why": why,
|
||||
"judged": time.Now().UTC().Format(time.RFC3339Nano)})
|
||||
}
|
||||
l.rootAsked++
|
||||
l.mu.Unlock()
|
||||
raw, _ := json.Marshal(map[string]any{"result": map[string]any{"machines": out}, "node": machine})
|
||||
_ = m.Respond(raw)
|
||||
})
|
||||
if err != nil {
|
||||
l.t.Fatal(err)
|
||||
}
|
||||
l.t.Cleanup(func() { _ = free.Unsubscribe() })
|
||||
sub, err := l.observer.Subscribe("mesh.seat.mesh-controller.tool.conditions", func(m *nats.Msg) {
|
||||
l.mu.Lock()
|
||||
list := append([]map[string]any{}, l.conditions...)
|
||||
l.mu.Unlock()
|
||||
raw, _ := json.Marshal(map[string]any{"result": map[string]any{"conditions": list}, "node": machine})
|
||||
_ = m.Respond(raw)
|
||||
})
|
||||
if err != nil {
|
||||
l.t.Fatal(err)
|
||||
}
|
||||
l.t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
}
|
||||
|
||||
// raise says a condition as the controller does: on its event, and in its conditions verb from then on.
|
||||
func (l *lab) raise(c map[string]any) {
|
||||
l.t.Helper()
|
||||
l.mu.Lock()
|
||||
l.conditions = append(l.conditions, c)
|
||||
l.mu.Unlock()
|
||||
l.sayCondition("condition-raised", c)
|
||||
}
|
||||
|
||||
// clear ends a condition as the controller does.
|
||||
func (l *lab) clear(key string) {
|
||||
l.t.Helper()
|
||||
l.mu.Lock()
|
||||
var kept []map[string]any
|
||||
var gone map[string]any
|
||||
for _, c := range l.conditions {
|
||||
if c["key"] == key {
|
||||
gone = c
|
||||
continue
|
||||
}
|
||||
kept = append(kept, c)
|
||||
}
|
||||
l.conditions = kept
|
||||
l.mu.Unlock()
|
||||
if gone != nil {
|
||||
gone["cleared"] = time.Now().UTC().Format(time.RFC3339Nano)
|
||||
l.sayCondition("condition-cleared", gone)
|
||||
}
|
||||
}
|
||||
|
||||
func (l *lab) sayCondition(event string, c map[string]any) {
|
||||
l.t.Helper()
|
||||
nc, _ := l.as("controller")
|
||||
js, err := nc.JetStream()
|
||||
if err != nil {
|
||||
l.t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(c)
|
||||
msg := nats.NewMsg("mesh.seat.mesh-controller.event." + event)
|
||||
msg.Data = raw
|
||||
msg.Header.Set("x-event-id", fmt.Sprintf("%s.%v.%d", event, c["key"], time.Now().UnixNano()))
|
||||
if _, err := js.PublishMsg(msg); err != nil {
|
||||
l.t.Fatalf("saying %s as the controller: %v", event, err)
|
||||
}
|
||||
}
|
||||
|
||||
// playDesk is the desk channel (kind `desktop`) at the bus, on the machine's runtime's credential, which
|
||||
// carries the desk channel: it says it is ready, and keeps what the router shows it — the link's code.
|
||||
func (l *lab) playDesk(ctx context.Context) {
|
||||
l.t.Helper()
|
||||
// The desk channel runs as an account of its own (it carries a link's code), on its own credential.
|
||||
nc, _ := l.as(machine + ".desk-channel")
|
||||
stand := func() {
|
||||
raw, _ := json.Marshal(map[string]any{"ready": true, "at": time.Now().UTC()})
|
||||
js, _ := nc.JetStream()
|
||||
msg := nats.NewMsg("mesh.seat.intake.event.standing.desktop")
|
||||
msg.Data = raw
|
||||
msg.Header.Set("x-event-id", fmt.Sprintf("standing.desktop.%d", time.Now().UnixNano()))
|
||||
if _, err := js.PublishMsg(msg); err != nil {
|
||||
l.t.Logf("the desk could not say it is ready: %v", err)
|
||||
}
|
||||
}
|
||||
stand()
|
||||
go func() {
|
||||
tick := time.NewTicker(20 * time.Second)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
stand()
|
||||
}
|
||||
}
|
||||
}()
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
l.t.Fatal(err)
|
||||
}
|
||||
bind, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
worker, err := js.Consumer(bind, "SEAT_CHANNEL", "SEAT_CHANNEL_DESKTOP_worker")
|
||||
if err != nil {
|
||||
l.t.Fatalf("the desk's worker: %v", err)
|
||||
}
|
||||
consuming, err := worker.Consume(func(m jetstream.Msg) {
|
||||
var shown struct {
|
||||
Title string `json:"title"`
|
||||
Body string `json:"body"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Data(), &shown)
|
||||
l.mu.Lock()
|
||||
l.desk = append(l.desk, shown.Title+": "+shown.Body)
|
||||
l.mu.Unlock()
|
||||
_ = m.Ack()
|
||||
})
|
||||
if err != nil {
|
||||
l.t.Fatal(err)
|
||||
}
|
||||
l.t.Cleanup(consuming.Stop)
|
||||
}
|
||||
|
||||
var linkCode = regexp.MustCompile(`(\d{3}) (\d{3})\.`)
|
||||
|
||||
// codeOnTheDesk waits for the link's code the router shows on the desk.
|
||||
func (l *lab) codeOnTheDesk(within time.Duration) string {
|
||||
l.t.Helper()
|
||||
deadline := time.Now().Add(within)
|
||||
for time.Now().Before(deadline) {
|
||||
l.mu.Lock()
|
||||
for _, s := range l.desk {
|
||||
if m := linkCode.FindStringSubmatch(s); m != nil {
|
||||
l.mu.Unlock()
|
||||
return m[1] + m[2]
|
||||
}
|
||||
}
|
||||
l.mu.Unlock()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
l.t.Fatalf("the desk was never shown a code; it was shown %q\nthe router said:\n%s", l.desk, l.logs["messenger"])
|
||||
return ""
|
||||
}
|
||||
|
||||
// routerRecord reads one key of the router's state, as the lab's observer.
|
||||
func (l *lab) routerRecord(bucket, key string) (jetstream.KeyValueEntry, jetstream.KeyValue) {
|
||||
l.t.Helper()
|
||||
js, err := jetstream.New(l.observer)
|
||||
if err != nil {
|
||||
l.t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
kv, err := js.KeyValue(ctx, bucket)
|
||||
if err != nil {
|
||||
l.t.Fatalf("the router's bucket %s: %v", bucket, err)
|
||||
}
|
||||
e, err := kv.Get(ctx, key)
|
||||
if err != nil {
|
||||
l.t.Fatalf("the router's %s %s: %v", bucket, key, err)
|
||||
}
|
||||
return e, kv
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
#!/bin/sh
|
||||
# The live acceptance of the operator's answers (novox/hq ADR 0259), after rollout, at the controller's
|
||||
# terminal on the control node. The lab's proof (TestTheOperatorsAnswerEndToEnd) ran the same flow with a fake
|
||||
# Telegram; this runs it once for real, with a question that changes nothing.
|
||||
#
|
||||
# sh live-acceptance.sh check, ask the rehearsal, wait for the answer, read the record
|
||||
#
|
||||
# It reads and asks only: no setting, assignment or push. The one thing it starts is a rehearsal, a question the
|
||||
# operator answers on the phone; approving it performs nothing and is recorded as the operator's decision.
|
||||
#
|
||||
# It stops at the first check that fails and says what to do. Exit 0 means: the question reached the phone
|
||||
# with Approve and Decline, the answer came back as a warrant, and the hand-act log names who answered,
|
||||
# through which channel, with which proofs.
|
||||
set -eu
|
||||
|
||||
# mesh-cli asks the controller; on the control node, from the operator's own account, a line is the
|
||||
# controller's terminal (ADR 0272). `nox` is its interactive alias; a script says mesh-cli.
|
||||
mc=${MESH_CLI:-mesh-cli}
|
||||
stop() { echo "NOT ACCEPTED: $*"; exit 1; }
|
||||
|
||||
echo "1. Nothing open says the router's or Telegram's machine is not root-free (root-not-free, agent-root)."
|
||||
if $mc conditions | grep -q 'root-not-free\|agent-root'; then
|
||||
$mc conditions | grep 'root-not-free\|agent-root'
|
||||
stop "while an agent can become root there, Telegram only acknowledges; close it first (ADR 0266, ADR 0268)"
|
||||
fi
|
||||
|
||||
echo "2. The Telegram channel holds a working bot token and its updates arrive."
|
||||
tg=$($mc ask telegram telegram_status) || stop "the telegram module does not answer"
|
||||
echo "$tg" | grep -q '"can_send": *true' || { echo "$tg"; stop "Telegram cannot send now (why_not above): give the bot token again at the controller's terminal (hq ADR 0259, the operator's steps), then push the control node"; }
|
||||
|
||||
echo "3. One Telegram account is linked as the operator, and its hour has passed."
|
||||
ids=$($mc ask messenger messenger_identities) || stop "the router does not answer"
|
||||
echo "$ids"
|
||||
echo "$ids" | grep -q '"telegram"' || stop "no Telegram account is linked: send /start to the bot, then the code from the desk"
|
||||
# The router answers approvals from a new link only an hour after it was made (ADR 0259 §7): said here, with the
|
||||
# time, rather than seen as a rehearsal nobody's approval reaches.
|
||||
from=$(echo "$ids" | tr -d '\n ' | grep -o '"kind":"telegram"[^}]*' | grep -o '"approve-from":"[^"]*"' | cut -d'"' -f4 | head -n 1)
|
||||
[ -n "$from" ] || stop "the router's word on the Telegram account says no time it can approve from"
|
||||
starts=$(date -d "$from" +%s 2>/dev/null) || stop "the time the Telegram account can approve from is not a time: $from"
|
||||
if [ "$(date +%s)" -lt "$starts" ]; then
|
||||
stop "the Telegram account was linked less than an hour ago — approvals start at $(date -d "$from" '+%H:%M') (local time); run this again then"
|
||||
fi
|
||||
|
||||
echo "4. The rehearsal: a question on your phone. Approve it there within 15 minutes."
|
||||
said=$($mc rehearse --for 15m) || stop "the rehearsal could not be asked"
|
||||
echo "$said"
|
||||
id=$(echo "$said" | sed -n 's/^rehearsal \([A-Za-z0-9_-]*\) asked.*/\1/p')
|
||||
[ -n "$id" ] || stop "the rehearsal did not say its id"
|
||||
|
||||
echo "5. Waiting for your answer in the hand-act log (every 10 s, at most 16 minutes)."
|
||||
i=0
|
||||
while [ $i -lt 96 ]; do
|
||||
acts=$($mc hand-acts --days 1 --json 2>/dev/null || true)
|
||||
mine=$(echo "$acts" | tr -d '\n' | grep -o "{[^{}]*\"ask\": *\"$id\"[^{}]*}" || true)
|
||||
if [ -n "$mine" ]; then
|
||||
echo "$mine"
|
||||
echo "$mine" | grep -q 'rehearsal=decline' && stop "the rehearsal was declined: run it again and choose Approve to see an approval recorded"
|
||||
echo "$mine" | grep -q 'rehearsal=approve' || stop "the rehearsal's record names no approval"
|
||||
echo "$mine" | grep -q '"via": *"telegram (telegram), user id verified"' || stop "the record does not say the answer came through Telegram from a verified account"
|
||||
echo "$mine" | grep -q '"P1"' || stop "the record does not carry the proof"
|
||||
echo "$mine" | grep -q '"outcome": *"done"' || stop "the rehearsal's act did not end done"
|
||||
echo "ACCEPTED: the rehearsal was approved on Telegram, acted on once, and recorded as your decision."
|
||||
exit 0
|
||||
fi
|
||||
i=$((i + 1))
|
||||
sleep 10
|
||||
done
|
||||
stop "no answer was recorded within 16 minutes: check messenger_status and telegram_status"
|
||||
@@ -0,0 +1,37 @@
|
||||
#!/bin/sh
|
||||
# mesh-check-toolchain: go
|
||||
#
|
||||
# The proof of the operator's answers (novox/hq ADR 0259), the Go part of the lab's own check: run by the
|
||||
# build seat in the mesh's Go toolchain, its own container, because the lab's merge-check.sh declares it
|
||||
# (`# mesh-check-also: go asks/merge-check.sh`) — and by hand: `sh asks/merge-check.sh`.
|
||||
#
|
||||
# Formatted, vetted, and run against the clones beside the lab: on the build seat those are the controller,
|
||||
# the runtime and the catalogue **the mesh runs**, so every pull request of the lab proves the answers the
|
||||
# operator would get today. By hand, MESH_LAB_REPOS, or MESH_LAB_CONTROLLER, MESH_LAB_TOOLS and
|
||||
# MESH_LAB_CATALOGUE, point it at branches under review. It takes about four minutes when it runs.
|
||||
#
|
||||
# **Said, never passed silently**: where a clone is missing, or the controller predates the proof's fixture,
|
||||
# it says NOT RUN and why, in capitals, and passes; a proof that runs and fails, fails the check.
|
||||
set -eu
|
||||
cd "$(dirname "$0")"
|
||||
export GOPRIVATE=git.novox.be
|
||||
|
||||
unformatted=$(gofmt -l .)
|
||||
if [ -n "$unformatted" ]; then
|
||||
echo "not gofmt'd:"
|
||||
echo "$unformatted"
|
||||
exit 1
|
||||
fi
|
||||
sh -n live-acceptance.sh
|
||||
go vet ./...
|
||||
out=$(mktemp)
|
||||
status=0
|
||||
go test -count=1 -timeout 25m -v -run '^(TestTheOperatorsAnswerEndToEnd|TestTheFakePhoneShowsHTMLAsTelegramDoes)$' . >"$out" 2>&1 || status=$?
|
||||
grep -E '^(--- | --- |ok|FAIL|PASS)' "$out" || true
|
||||
if grep -q 'NOT RUN' "$out"; then
|
||||
echo "NOT RUN: THE PROOF OF THE OPERATOR'S ANSWERS DID NOT RUN HERE:"
|
||||
grep -o 'NOT RUN: .*' "$out" | head -3
|
||||
fi
|
||||
[ "$status" -eq 0 ] || tail -60 "$out"
|
||||
rm -f "$out"
|
||||
exit "$status"
|
||||
@@ -0,0 +1,351 @@
|
||||
package asks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asker"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
)
|
||||
|
||||
// The operator's own Telegram account in the lab, and somebody else's.
|
||||
const (
|
||||
operatorAccount = 42
|
||||
strangerAccount = 99
|
||||
groupChat = -1001
|
||||
)
|
||||
|
||||
// TestTheOperatorsAnswerEndToEnd is the operator's acceptance of novox/hq ADR 0259, run before anybody uses it:
|
||||
//
|
||||
// "I get a Telegram message from the mesh with what is asked, why, by whom, and Approve/Decline buttons;
|
||||
// tapping Approve makes the mesh do exactly that one thing once; Decline or no answer does nothing;
|
||||
// nobody else (no agent, no other Telegram user, no replay of an old message) can produce an approval;
|
||||
// and I can see in the record who approved what."
|
||||
//
|
||||
// What is real: the bus, of the release the mesh runs, with the accounts the controller at its commit composes
|
||||
// for one machine and raised by that controller's own code (its fixture TestTheAsksLabBus); the mesh's runtime
|
||||
// (mesh-tools node-tools) serving the router (messenger) and the Telegram channel (telegram), each on its own
|
||||
// credential, as `runs-as` places them; the router's and the channel's code at their commits; the SDK's asker.
|
||||
//
|
||||
// What is the lab's: Telegram itself (a fake Bot API, the operator's phone), the desk (the desktop kind at the
|
||||
// bus, which shows the link's code), the controller's conditions verb (a list the lab keeps), and the asker
|
||||
// (the module `lab-asker`, the SDK's client in this process). The controller as an asker — that it acts on a
|
||||
// warrant once, through the verb the action names, and records it — is its own repository's test
|
||||
// (asker_test.go); the live acceptance after rollout runs it for real.
|
||||
func TestTheOperatorsAnswerEndToEnd(t *testing.T) {
|
||||
repos := clonesFor(t)
|
||||
dir := os.Getenv("MESH_LAB_ASKS_KEEP")
|
||||
if dir == "" {
|
||||
dir = t.TempDir()
|
||||
} else if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
l := &lab{t: t, repos: repos, dir: dir}
|
||||
token := "123456789:" + strings.Repeat("A", 35)
|
||||
l.tg = newFakeTelegram(t, token)
|
||||
|
||||
// The bus, as the controller composes and raises it.
|
||||
l.compose("")
|
||||
l.startBus()
|
||||
l.compose(l.url)
|
||||
l.playController()
|
||||
|
||||
// The programs, at their commits. The channel is built to call the lab's phone in place of Telegram.
|
||||
runtimeBin := l.build(filepath.Join(repos.tools, "node-tools"), "./cmd/node-tools", "node-tools", "")
|
||||
routerBin := l.build(filepath.Join(repos.catalogue, "modules", "messenger"), "./cmd/messenger", "messenger", "")
|
||||
channelBin := l.build(filepath.Join(repos.catalogue, "modules", "telegram"), "./cmd/telegram", "telegram",
|
||||
"-X main.API="+l.tg.URL())
|
||||
|
||||
routerState := filepath.Join(dir, "messenger")
|
||||
if err := os.MkdirAll(routerState, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(routerState, "settings.json"), []byte(`{"time-zone": "UTC"}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tokenFile := filepath.Join(dir, "telegram", "telegram-token")
|
||||
if err := os.MkdirAll(filepath.Dir(tokenFile), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(tokenFile, []byte(token+"\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
t.Cleanup(stop)
|
||||
l.playDesk(ctx)
|
||||
l.runtime("messenger", runtimeBin, routerBin, map[string]string{
|
||||
"MESH_MESSENGER_SETTINGS": filepath.Join(routerState, "settings.json"), "MESH_MESSENGER_STATE": routerState})
|
||||
l.runtime("telegram", runtimeBin, channelBin, map[string]string{"MESH_TELEGRAM_TOKEN_FILE": tokenFile})
|
||||
l.waitSaid("messenger", "taking the asks submitted", time.Minute)
|
||||
l.waitSaid("telegram", "taking the router's messages for this kind", time.Minute)
|
||||
a := newLabAsker(t, l)
|
||||
|
||||
// --- 1. The operator links their account: /start on the phone, the code from the desk.
|
||||
t.Run("the operator links their account with the code shown on the desk", func(t *testing.T) {
|
||||
since := time.Now()
|
||||
l.tg.typed(operatorAccount, operatorAccount, "private", "/start")
|
||||
code := l.codeOnTheDesk(90 * time.Second)
|
||||
l.tg.typed(operatorAccount, operatorAccount, "private", code[:3]+" "+code[3:])
|
||||
l.tg.waitFor("the link said done", operatorAccount, since, 90*time.Second, func(m tgMessage) bool {
|
||||
return strings.Contains(m.Text, "Linked")
|
||||
})
|
||||
})
|
||||
if t.Failed() {
|
||||
return
|
||||
}
|
||||
|
||||
// --- 2. Within the hour after a link, an approval is refused; the button is not spent.
|
||||
var first asks.Ask
|
||||
var firstMessage tgMessage
|
||||
t.Run("an approval in the hour after linking is refused and nothing is done", func(t *testing.T) {
|
||||
since := time.Now()
|
||||
first = a.ask("a1", "Flip the lab's switch?", time.Hour)
|
||||
firstMessage = l.tg.waitFor("the first question with its buttons", operatorAccount, since, 90*time.Second,
|
||||
func(m tgMessage) bool { return !m.Edited && button(m, "Approve") != "" && button(m, "Decline") != "" })
|
||||
for _, says := range []string{"Flip the lab's switch?", "approve or decline", "lab-asker",
|
||||
"the switch is flipped, once", "nothing is flipped"} {
|
||||
if !strings.Contains(firstMessage.Text, says) {
|
||||
t.Errorf("the question does not say %q (what is asked, why, by whom, what each answer does, what "+
|
||||
"no answer does): %q", says, firstMessage.Text)
|
||||
}
|
||||
}
|
||||
tapped := time.Now()
|
||||
l.tg.tapped(operatorAccount, operatorAccount, "private", firstMessage.ID, button(firstMessage, "Approve"))
|
||||
l.tg.waitFor("the refusal of an approval inside the hour", operatorAccount, tapped, 90*time.Second,
|
||||
func(m tgMessage) bool {
|
||||
return strings.Contains(m.Text, "That answer was not taken") && strings.Contains(m.Text, "linked at")
|
||||
})
|
||||
if !a.noWordOn("a1", 2*time.Second) || len(a.performed()) != 0 {
|
||||
t.Fatalf("an approval inside the hour was a warrant: %v", a.performed())
|
||||
}
|
||||
})
|
||||
|
||||
// The hour passes: the lab ages the link in the router's own record (the hour itself is the router's
|
||||
// unit test: an approve at 59 minutes refused, at 60 taken).
|
||||
t.Run("the hour after the link passes", func(t *testing.T) {
|
||||
e, kv := l.routerRecord("messenger_identities", fmt.Sprintf("telegram.%d", operatorAccount))
|
||||
var id map[string]any
|
||||
if err := json.Unmarshal(e.Value(), &id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
past := time.Now().Add(-61 * time.Minute).UTC()
|
||||
id["linked"], id["approve-from"] = past.Format(time.RFC3339Nano), past.Add(time.Hour).Format(time.RFC3339Nano)
|
||||
raw, _ := json.Marshal(id)
|
||||
if _, err := kv.Update(context.Background(), e.Key(), raw, e.Revision()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
|
||||
// --- 3. Approve: the warrant, for this ask alone, performed once.
|
||||
t.Run("approve does exactly the one thing, once, and says who approved", func(t *testing.T) {
|
||||
tapped := time.Now()
|
||||
l.tg.tapped(operatorAccount, operatorAccount, "private", firstMessage.ID, button(firstMessage, "Approve"))
|
||||
w := a.warrantFor("a1", 30*time.Second)
|
||||
if w.Outcome != asks.OutcomeChosen || w.Option != "approve" || w.Level != asks.Approve {
|
||||
t.Fatalf("the warrant: %+v", w)
|
||||
}
|
||||
if w.By == nil || w.By.Kind != "telegram" || w.By.Identity != fmt.Sprint(operatorAccount) ||
|
||||
w.By.Verified != "user id verified" || w.Channel != "telegram" || strings.Join(w.Proofs, ",") != "P1" {
|
||||
t.Errorf("the warrant does not say who approved, how and where: %+v by %+v", w, w.By)
|
||||
}
|
||||
if w.AskDigest != first.Digest() {
|
||||
t.Errorf("the warrant names the digest %s, not the ask's %s", w.AskDigest, first.Digest())
|
||||
}
|
||||
if got := a.performed(); len(got) != 1 || got[0]["arg.switch"] != "approve" {
|
||||
t.Fatalf("performed %v, want the one bound act once", got)
|
||||
}
|
||||
// Every copy says how it ended, its buttons gone.
|
||||
l.tg.waitFor("the question edited to its outcome", operatorAccount, tapped, 90*time.Second,
|
||||
func(m tgMessage) bool { return m.Edited && m.ID == firstMessage.ID && len(m.Buttons) == 0 })
|
||||
// And the router's record holds who approved what.
|
||||
e, _ := l.routerRecord("messenger_asks", "lab-asker.a1")
|
||||
var rec struct {
|
||||
State string `json:"state"`
|
||||
Warrant *asks.Warrant `json:"warrant"`
|
||||
}
|
||||
_ = json.Unmarshal(e.Value(), &rec)
|
||||
if rec.State != "chosen" || rec.Warrant == nil || rec.Warrant.By == nil || rec.Warrant.By.Identity != fmt.Sprint(operatorAccount) {
|
||||
t.Errorf("the router's record: %s", e.Value())
|
||||
}
|
||||
t.Logf("the record reads: %s", w.Says())
|
||||
})
|
||||
|
||||
// --- 4. A replayed tap, and a redelivered warrant, do nothing more.
|
||||
// The replay is the operator's own answer again: the router says it is the one already recorded (on the tap's
|
||||
// acknowledgement, issue 372), not refused as if the approval failed, and it is no second warrant or act.
|
||||
t.Run("a replayed tap and a redelivered warrant do nothing more", func(t *testing.T) {
|
||||
tapped := time.Now()
|
||||
l.tg.tapped(operatorAccount, operatorAccount, "private", firstMessage.ID, button(firstMessage, "Approve"))
|
||||
l.tg.toldTapper("that a replayed tap is the answer already recorded", operatorAccount, tapped, 90*time.Second,
|
||||
func(text string) bool {
|
||||
return strings.Contains(text, "You already chose Approve") && strings.Contains(text, "recorded")
|
||||
})
|
||||
time.Sleep(2 * time.Second)
|
||||
if n := a.wordsOn("a1"); n != 1 {
|
||||
t.Errorf("the router said %d words on one ask", n)
|
||||
}
|
||||
w := a.warrantFor("a1", time.Second)
|
||||
if h := a.take(w); h != asker.Done {
|
||||
t.Errorf("a warrant heard again was %s", h)
|
||||
}
|
||||
if got := a.performed(); len(got) != 1 {
|
||||
t.Fatalf("performed %v after a replay", got)
|
||||
}
|
||||
})
|
||||
|
||||
// --- 5. Somebody else's account, and a group, cannot answer; Decline does nothing.
|
||||
t.Run("another account and a group cannot answer, and decline does nothing", func(t *testing.T) {
|
||||
since := time.Now()
|
||||
a.ask("a2", "Flip it once more?", time.Hour)
|
||||
m := l.tg.waitFor("the second question", operatorAccount, since, 90*time.Second,
|
||||
func(m tgMessage) bool { return !m.Edited && button(m, "Approve") != "" })
|
||||
// The buttons' data reached a stranger (a forwarded question), who taps in their own chat with the bot:
|
||||
// the channel never showed that message there, so the tap goes nowhere, and the stranger is told
|
||||
// nothing. (A stranger's answer that does reach the router is refused there and told to the operator by
|
||||
// name: the router's own test.)
|
||||
l.tg.tapped(strangerAccount, strangerAccount, "private", m.ID, button(m, "Approve"))
|
||||
// The operator taps a message whose words are not the ones the channel put there.
|
||||
l.tg.tappedOn(operatorAccount, operatorAccount, "private", m.ID, button(m, "Approve"), "Flip ALL the switches?")
|
||||
// The operator taps in a group the bot is in: not their own chat, dropped by the channel.
|
||||
l.tg.tapped(operatorAccount, groupChat, "supergroup", m.ID, button(m, "Approve"))
|
||||
if !a.noWordOn("a2", 3*time.Second) {
|
||||
t.Fatal("a tap from another account, on changed words, or from a group, was a warrant")
|
||||
}
|
||||
if got := l.tg.since(strangerAccount, since); len(got) != 0 {
|
||||
t.Errorf("the bot answered the stranger: %+v", got)
|
||||
}
|
||||
if got := l.tg.since(groupChat, since); len(got) != 0 {
|
||||
t.Errorf("the bot answered in a group: %+v", got)
|
||||
}
|
||||
l.tg.tapped(operatorAccount, operatorAccount, "private", m.ID, button(m, "Decline"))
|
||||
w := a.warrantFor("a2", 30*time.Second)
|
||||
if w.Option != "decline" || w.Level != asks.Acknowledge {
|
||||
t.Errorf("decline's warrant: %+v", w)
|
||||
}
|
||||
if got := a.performed(); len(got) != 1 {
|
||||
t.Fatalf("decline performed something: %v", got)
|
||||
}
|
||||
})
|
||||
|
||||
// --- 6. No agent can produce an approval: the bus refuses every forgery, as the controller composed it.
|
||||
t.Run("no agent, channel or module but the router can say a warrant, and none asks in another's name", func(t *testing.T) {
|
||||
warrant, _ := json.Marshal(asks.Warrant{Ask: "a9", Asker: "lab-asker", Outcome: asks.OutcomeChosen,
|
||||
Option: "approve", Level: asks.Approve, AskDigest: first.Digest(),
|
||||
By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}})
|
||||
decided := asks.DecidedSubject("lab-asker")
|
||||
// The control: what is granted is not refused, so a refusal below is the bus's word and not the lab's.
|
||||
// The channel's control is a true standing, now: a false one would hold the router's messages.
|
||||
standing, _ := json.Marshal(asks.Standing{Ready: true, EditsSilently: true, At: time.Now().UTC()})
|
||||
for _, try := range []struct {
|
||||
user, subject string
|
||||
body []byte
|
||||
}{
|
||||
{machine + ".lab-asker", "mesh.seat.operator-channel.accept.cancel.lab-asker", []byte(`{"id":"none"}`)},
|
||||
{machine + ".telegram", asks.IntakeSubject(asks.StandingWhat, "telegram"), standing},
|
||||
} {
|
||||
if l.refusedToPublish(try.user, try.subject, try.body) {
|
||||
t.Fatalf("the control failed: %s was refused %s, which the controller grants it", try.user, try.subject)
|
||||
}
|
||||
}
|
||||
for _, user := range []string{
|
||||
machine + ".node-tools", // the machine's runtime: every agent's tool calls go through one
|
||||
machine + ".lab-asker", // the asker itself
|
||||
machine + ".lab-bystander", // any other module
|
||||
machine + ".telegram", // the channel
|
||||
"node." + machine, // the machine's node-engine
|
||||
"controller", // the controller asks; it does not decide
|
||||
} {
|
||||
if !l.refusedToPublish(user, decided, warrant) {
|
||||
t.Errorf("%s may say a warrant on %s", user, decided)
|
||||
}
|
||||
}
|
||||
for _, try := range []struct{ user, subject, what string }{
|
||||
{machine + ".lab-asker", asks.AskSubject("mesh-controller"), "an ask in another asker's name"},
|
||||
{machine + ".lab-bystander", asks.AskSubject("lab-bystander"), "an ask from a module that may not ask"},
|
||||
{machine + ".node-tools", asks.AskSubject("mesh-controller"), "an agent asking as the controller through the runtime"},
|
||||
{machine + ".node-tools", asks.IntakeSubject(asks.Chosen, "telegram"), "an agent saying a Telegram tap"},
|
||||
{machine + ".lab-asker", asks.IntakeSubject(asks.Chosen, "telegram"), "an asker saying a Telegram tap"},
|
||||
{machine + ".node-tools", "$KV.messenger_identities.telegram.666", "an agent linking an account of its own"},
|
||||
{machine + ".lab-asker", "$KV.messenger_asks.lab-asker.a1", "an asker rewriting the router's record"},
|
||||
{machine + ".telegram", asks.ProofSubject(asks.CodeProof, "desktop"), "a channel speaking for another kind"},
|
||||
} {
|
||||
if !l.refusedToPublish(try.user, try.subject, []byte(`{}`)) {
|
||||
t.Errorf("%s: %s may publish %s", try.what, try.user, try.subject)
|
||||
}
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
if n := a.wordsOn("a9"); n != 0 || len(a.performed()) != 1 {
|
||||
t.Fatalf("a forged warrant reached the asker: %d words, performed %v", n, a.performed())
|
||||
}
|
||||
})
|
||||
|
||||
// --- 7. While an agent can become root where the router runs, nothing proven there approves.
|
||||
t.Run("while the controller cannot say root is free where the router runs, nothing approves", func(t *testing.T) {
|
||||
// An approval the operator already sees on the phone, then root stops being free.
|
||||
since := time.Now()
|
||||
a.ask("a5", "Flip it while root is checked?", time.Hour)
|
||||
m := l.tg.waitFor("the question asked while root was free", operatorAccount, since, 90*time.Second,
|
||||
func(m tgMessage) bool { return !m.Edited && button(m, "Approve") != "" })
|
||||
l.mu.Lock()
|
||||
l.notFree = map[string]string{machine: "an agent can become root without a person"}
|
||||
asked := l.rootAsked
|
||||
l.mu.Unlock()
|
||||
tapped := time.Now()
|
||||
l.tg.tapped(operatorAccount, operatorAccount, "private", m.ID, button(m, "Approve"))
|
||||
l.tg.waitFor("the refusal of an approval while root is not free", operatorAccount, tapped, 90*time.Second,
|
||||
func(m tgMessage) bool { return strings.Contains(m.Text, "That answer was not taken") })
|
||||
l.mu.Lock()
|
||||
askedAgain := l.rootAsked > asked
|
||||
l.mu.Unlock()
|
||||
if !askedAgain {
|
||||
t.Error("the router judged an approval without asking the controller whether root is free")
|
||||
}
|
||||
if !a.noWordOn("a5", 2*time.Second) {
|
||||
t.Fatal("an approval was a warrant while root was not free")
|
||||
}
|
||||
// A new ask that needs an approval is refused to its asker, in words: no channel can carry it now.
|
||||
a.ask("a3", "Flip it while root is not free?", time.Hour)
|
||||
w := a.warrantFor("a3", 30*time.Second)
|
||||
if w.Outcome != asks.OutcomeRefused || w.By != nil {
|
||||
t.Errorf("an approving ask while root is not free: %+v", w)
|
||||
}
|
||||
if got := a.performed(); len(got) != 1 {
|
||||
t.Fatalf("performed %v while root was not free", got)
|
||||
}
|
||||
l.mu.Lock()
|
||||
l.notFree = nil
|
||||
l.mu.Unlock()
|
||||
if err := a.client.Cancel("a5"); err != nil {
|
||||
t.Errorf("taking the ask back: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
// --- 8. No answer does nothing: the ask expires.
|
||||
t.Run("an ask nobody answers expires and nothing is done", func(t *testing.T) {
|
||||
since := time.Now()
|
||||
a.ask("a4", "Flip it if you answer in time?", 20*time.Second)
|
||||
m := l.tg.waitFor("the fourth question", operatorAccount, since, 90*time.Second,
|
||||
func(m tgMessage) bool { return !m.Edited && button(m, "Approve") != "" })
|
||||
w := a.warrantFor("a4", 2*time.Minute)
|
||||
if w.Outcome != asks.OutcomeExpired || w.By != nil {
|
||||
t.Errorf("the end of an unanswered ask: %+v", w)
|
||||
}
|
||||
late := time.Now()
|
||||
l.tg.tapped(operatorAccount, operatorAccount, "private", m.ID, button(m, "Approve"))
|
||||
l.tg.waitFor("the refusal of a late tap", operatorAccount, late, 90*time.Second,
|
||||
func(m tgMessage) bool { return strings.Contains(m.Text, "That answer was not taken") })
|
||||
time.Sleep(2 * time.Second)
|
||||
if got := a.performed(); len(got) != 1 {
|
||||
t.Fatalf("an expired ask performed something: %v", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Logf("performed, over the whole run: %v (one approval, one act)", a.performed())
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
#!/bin/sh
|
||||
# mesh-check-toolchain: typescript
|
||||
# mesh-check-also: go replays/merge-check.sh
|
||||
# mesh-check-also: go asks/merge-check.sh
|
||||
#
|
||||
# The lab's own check (novox/hq ADR 0237 as amended): the second layer of a pull request's merge check,
|
||||
# `mesh/repo-check`, run by the build seat — and by hand.
|
||||
#
|
||||
# The mesh builds no module from the lab, so no gate runs for its pull requests (`mesh/merge-gate` says the
|
||||
# change touches no module of the graph); this is all that is checked before a change to it merges. The lab
|
||||
# is in two languages, and each part runs in its own toolchain's container (novox/hq issue 302):
|
||||
#
|
||||
# - this script, in the TypeScript toolchain: installed from the lock file, type-checked, sources and
|
||||
# tests, and the unit suite;
|
||||
# - replays/merge-check.sh, in the Go toolchain, declared above: the replays register and its prover
|
||||
# formatted, vetted and compiled, and the register's own tests;
|
||||
# - asks/merge-check.sh, in the Go toolchain, declared above: the proof of the operator's answers, run
|
||||
# against the controller, runtime and catalogue cloned beside the lab (novox/hq ADR 0259).
|
||||
#
|
||||
# **Said, never passed silently**: the integration suite raises a lab on a workstation, and the replays
|
||||
# raise containers through the container runtime — neither is given to code nobody has approved. The
|
||||
# replays run from the lab's main, reviewed, in every gate of a core change; a change to them is proven by
|
||||
# `go run ./replays/cmd/prove` by hand before it merges.
|
||||
set -eu
|
||||
|
||||
npm ci --no-audit --no-fund --loglevel=error
|
||||
npm run typecheck
|
||||
npm test
|
||||
|
||||
# By hand, with Go at hand, the Go part runs here too; on the build seat it runs in the Go toolchain.
|
||||
if command -v go >/dev/null 2>&1; then
|
||||
sh replays/merge-check.sh
|
||||
sh asks/merge-check.sh
|
||||
else
|
||||
echo "replays/ and asks/ (Go) are checked by their merge-check.sh, which the build seat runs in the Go toolchain"
|
||||
fi
|
||||
echo "NOT RUN HERE: the integration suite and the replays — they need a lab and the container runtime"
|
||||
@@ -1 +0,0 @@
|
||||
/home/jochen/projects/novox/mesh-lab/node_modules
|
||||
+410
@@ -0,0 +1,410 @@
|
||||
package replays
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The bed (novox/hq ADR 0240 rule 7, to-be 48 §8, Phase D): **a declaration is proved before it is trusted.**
|
||||
//
|
||||
// Two of the nineteen image checks the catalogue's containers ship read *unhealthy* while working in the
|
||||
// mesh's hands — the studio's asked an address its program does not bind, the flow editor's read settings
|
||||
// from a path the module mounted elsewhere. Read without proof, they would have put back two good builds.
|
||||
// So the catalogue's merge check starts every resource whose declared check or image the change touches,
|
||||
// **alone**, on a throwaway runtime, and requires its check to see the program within its grace.
|
||||
//
|
||||
// **What it proves is the check's wiring**: that it can see the program working — a property of the image
|
||||
// and the declaration, not of the mesh. A resource is started with what the module declares and nothing of
|
||||
// the mesh's: its literal environment, its arguments, its files where the module writes them in full, and an
|
||||
// empty directory for every other place it mounts; no secret, no binding, no provider. So:
|
||||
//
|
||||
// - a check naming a provision in `needs` gets no provider here, and must only reach the program — an
|
||||
// http answer of any status, a connect;
|
||||
// - a program that does not stay up alone — it needs what the mesh gives it — is said as not proved here,
|
||||
// and judged on the first machine's gate, never passed as proved and never failed;
|
||||
// - a program that stays up while its check does not see it **fails**: that is the studio's fault.
|
||||
//
|
||||
// A tool check and a unit's own readiness need the mesh and a machine; they are said, not proved.
|
||||
|
||||
// BedResource is one resource the bed proves: its module, its id, and the resource as the manifest says it.
|
||||
type BedResource struct {
|
||||
Module string
|
||||
ID string
|
||||
Resource map[string]any
|
||||
// Listens are the module's endpoints, by name: the container's own port for each.
|
||||
Listens map[string]int
|
||||
}
|
||||
|
||||
// BedVerdict is what the bed found of one resource.
|
||||
type BedVerdict struct {
|
||||
Proved bool
|
||||
// Failed says the program stayed up and its check did not see it: the change's fault.
|
||||
Failed bool
|
||||
Said string
|
||||
}
|
||||
|
||||
// BedLook is how often the bed looks: often, because it proves the wiring, not the timing.
|
||||
var BedLook = 2 * time.Second
|
||||
|
||||
// BedFloor is the least the bed waits for a check to pass, whatever the grace: a program needs a moment to
|
||||
// listen even when the module declared no grace.
|
||||
var BedFloor = 30 * time.Second
|
||||
|
||||
// ChangedHealth is every long-running container resource of the catalogue at root that declares `health`
|
||||
// and whose resource changed against base (a git ref in that checkout): its declaration, its image, or
|
||||
// anything else of it. A manifest new on this branch counts whole. Read through git; an error when the
|
||||
// base cannot be read.
|
||||
func ChangedHealth(root, base string, show func(ref, path string) ([]byte, error)) ([]BedResource, error) {
|
||||
paths, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []BedResource
|
||||
for _, p := range paths {
|
||||
raw, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rel, _ := filepath.Rel(root, p)
|
||||
was := map[string]string{}
|
||||
if before, err := show(base, rel); err == nil {
|
||||
for _, r := range resourcesOf(before) {
|
||||
was[fmt.Sprint(r["id"])] = canonical(r)
|
||||
}
|
||||
}
|
||||
module, listens, resources := manifestParts(raw)
|
||||
for _, r := range resources {
|
||||
if _, declared := r["health"]; !declared || fmt.Sprint(r["type"]) != "container" || !stays(r) {
|
||||
continue
|
||||
}
|
||||
if was[fmt.Sprint(r["id"])] == canonical(r) {
|
||||
continue
|
||||
}
|
||||
out = append(out, BedResource{Module: module, ID: fmt.Sprint(r["id"]), Resource: r, Listens: listens})
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(a, b int) bool { return out[a].Module+"."+out[a].ID < out[b].Module+"."+out[b].ID })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func stays(r map[string]any) bool {
|
||||
once, _ := r["run-once"].(bool)
|
||||
return !once && r["schedule"] == nil
|
||||
}
|
||||
|
||||
func canonical(r map[string]any) string {
|
||||
raw, _ := json.Marshal(r)
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
func resourcesOf(raw []byte) []map[string]any {
|
||||
_, _, rs := manifestParts(raw)
|
||||
return rs
|
||||
}
|
||||
|
||||
func manifestParts(raw []byte) (string, map[string]int, []map[string]any) {
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Listens []struct {
|
||||
Name string `json:"name"`
|
||||
Port int `json:"port"`
|
||||
} `json:"listens"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &m)
|
||||
listens := map[string]int{}
|
||||
for _, l := range m.Listens {
|
||||
if l.Name != "" {
|
||||
listens[l.Name] = l.Port
|
||||
}
|
||||
}
|
||||
return m.Module, listens, m.Resources
|
||||
}
|
||||
|
||||
// placeholder is anything the mesh fills in on a machine: a value holding one is not the module's alone.
|
||||
var placeholder = regexp.MustCompile(`\$\{[^}]*\}`)
|
||||
|
||||
// Prove starts one resource alone on the runtime and looks at it with its declared check until the check
|
||||
// sees it, the program stops, or its grace (at least BedFloor) runs out. Everything it made is removed.
|
||||
func (d *Docker) Prove(ctx context.Context, r BedResource, files map[string]string) BedVerdict {
|
||||
h, _ := r.Resource["health"].(map[string]any)
|
||||
kind := fmt.Sprint(h["kind"])
|
||||
switch kind {
|
||||
case "tool", "unit":
|
||||
return BedVerdict{Said: fmt.Sprintf("%s.%s: a %s check needs the mesh and a machine; judged on the first "+
|
||||
"machine's gate", r.Module, r.ID, kind)}
|
||||
}
|
||||
image, _ := r.Resource["image"].(string)
|
||||
if image == "" || placeholder.MatchString(image) {
|
||||
return BedVerdict{Said: fmt.Sprintf("%s.%s: its image is built by the mesh (%v); judged on the first machine's gate",
|
||||
r.Module, r.ID, r.Resource["artifact"])}
|
||||
}
|
||||
if err := d.Pull(ctx, image); err != nil {
|
||||
return BedVerdict{Said: fmt.Sprintf("%s.%s: its image could not be fetched here: %v", r.Module, r.ID, oneLine(err.Error()))}
|
||||
}
|
||||
scratch, err := os.MkdirTemp("", "mesh-bed-")
|
||||
if err != nil {
|
||||
return BedVerdict{Said: err.Error()}
|
||||
}
|
||||
defer os.RemoveAll(scratch)
|
||||
// Its literal environment, and every env-file the module writes in full.
|
||||
var env []string
|
||||
if e, ok := r.Resource["env"].(map[string]any); ok {
|
||||
for k, v := range e {
|
||||
if s := fmt.Sprint(v); !placeholder.MatchString(s) {
|
||||
env = append(env, k+"="+s)
|
||||
}
|
||||
}
|
||||
}
|
||||
if list, ok := r.Resource["env-file"].([]any); ok {
|
||||
for _, f := range list {
|
||||
if content, ok := files[fmt.Sprint(f)]; ok {
|
||||
env = append(env, envLines(content)...)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(env)
|
||||
// Its mounts: a file the module writes in full, or an empty place of its own.
|
||||
var binds []string
|
||||
if list, ok := r.Resource["volumes"].([]any); ok {
|
||||
for i, v := range list {
|
||||
src, dst, rest := splitMount(fmt.Sprint(v))
|
||||
if dst == "" || !strings.HasPrefix(dst, "/") {
|
||||
continue
|
||||
}
|
||||
local := filepath.Join(scratch, "m"+strconv.Itoa(i))
|
||||
if content, ok := files[src]; ok {
|
||||
if err := os.WriteFile(local, []byte(content), 0o644); err != nil {
|
||||
continue
|
||||
}
|
||||
} else if err := os.MkdirAll(local, 0o777); err != nil {
|
||||
continue
|
||||
}
|
||||
_ = os.Chmod(local, 0o777)
|
||||
binds = append(binds, local+":"+dst+rest)
|
||||
}
|
||||
}
|
||||
var args []string
|
||||
if list, ok := r.Resource["args"].([]any); ok {
|
||||
for _, a := range list {
|
||||
args = append(args, fmt.Sprint(a))
|
||||
}
|
||||
}
|
||||
network := fmt.Sprintf("mesh-bed-%d", time.Now().UnixNano())
|
||||
netID, err := d.Network(ctx, network)
|
||||
if err != nil {
|
||||
return BedVerdict{Said: "the bed's network could not be made: " + err.Error()}
|
||||
}
|
||||
defer d.RemoveNetwork(context.Background(), netID)
|
||||
|
||||
grace, _ := time.ParseDuration(fmt.Sprint(h["grace"]))
|
||||
if _, said := h["grace"]; !said {
|
||||
grace = 60 * time.Second
|
||||
}
|
||||
if grace < BedFloor {
|
||||
grace = BedFloor
|
||||
}
|
||||
config := map[string]any{"Image": image, "Env": env, "Labels": map[string]string{d.Label: "bed"}}
|
||||
if len(args) > 0 {
|
||||
config["Cmd"] = args
|
||||
}
|
||||
switch kind {
|
||||
case "exec":
|
||||
config["Healthcheck"] = map[string]any{"Test": []string{"CMD-SHELL", fmt.Sprint(h["command"])},
|
||||
"Interval": BedLook.Nanoseconds(), "Timeout": BedLook.Nanoseconds() - 1, "Retries": 1}
|
||||
case "runtime":
|
||||
// The image's own command, adopted by name: an empty Test keeps it.
|
||||
config["Healthcheck"] = map[string]any{"Interval": BedLook.Nanoseconds(), "Timeout": BedLook.Nanoseconds() - 1,
|
||||
"Retries": 1}
|
||||
}
|
||||
config["HostConfig"] = map[string]any{"Binds": binds, "NetworkMode": network}
|
||||
name := fmt.Sprintf("mesh-bed-%s-%s-%d", r.Module, r.ID, time.Now().UnixNano())
|
||||
var made struct{ ID string }
|
||||
if err := d.call(ctx, http.MethodPost, "/containers/create?name="+name, config, &made); err != nil {
|
||||
return BedVerdict{Said: fmt.Sprintf("%s.%s could not be made here: %v", r.Module, r.ID, oneLine(err.Error()))}
|
||||
}
|
||||
defer d.Remove(context.Background(), made.ID)
|
||||
if err := d.call(ctx, http.MethodPost, "/containers/"+made.ID+"/start", nil, nil); err != nil {
|
||||
return BedVerdict{Said: fmt.Sprintf("%s.%s could not be started here: %v", r.Module, r.ID, oneLine(err.Error()))}
|
||||
}
|
||||
|
||||
port := r.Listens[fmt.Sprint(h["endpoint"])]
|
||||
needs := fmt.Sprint(h["needs"]) != "" && h["needs"] != nil
|
||||
deadline := time.Now().Add(grace)
|
||||
last := "it was never looked at"
|
||||
for {
|
||||
var seen struct {
|
||||
State struct {
|
||||
Running bool
|
||||
Health *struct {
|
||||
Status string
|
||||
Log []struct{ Output string }
|
||||
}
|
||||
}
|
||||
NetworkSettings struct {
|
||||
Networks map[string]struct{ IPAddress string }
|
||||
}
|
||||
}
|
||||
if err := d.call(ctx, http.MethodGet, "/containers/"+made.ID+"/json", nil, &seen); err != nil {
|
||||
return BedVerdict{Said: err.Error()}
|
||||
}
|
||||
if !seen.State.Running {
|
||||
return BedVerdict{Said: fmt.Sprintf("%s.%s does not stay up alone — it needs what the mesh gives it — so its "+
|
||||
"check is judged on the first machine's gate: %s", r.Module, r.ID, oneLine(lastLine(d.Logs(ctx, made.ID))))}
|
||||
}
|
||||
ok := false
|
||||
switch kind {
|
||||
case "exec", "runtime":
|
||||
hs := seen.State.Health
|
||||
switch {
|
||||
case hs == nil:
|
||||
last = "the container carries no check: its image ships none to adopt"
|
||||
case hs.Status == "healthy":
|
||||
ok = true
|
||||
default:
|
||||
last = "the runtime says " + hs.Status
|
||||
if n := len(hs.Log); n > 0 {
|
||||
last += ": " + oneLine(hs.Log[n-1].Output)
|
||||
}
|
||||
}
|
||||
case "http", "tcp":
|
||||
address := ""
|
||||
for _, n := range seen.NetworkSettings.Networks {
|
||||
address = n.IPAddress
|
||||
}
|
||||
ok, last = look(ctx, kind, address, port, h, needs)
|
||||
default:
|
||||
return BedVerdict{Said: fmt.Sprintf("%s.%s declares a %s check the bed does not know", r.Module, r.ID, kind)}
|
||||
}
|
||||
if ok {
|
||||
return BedVerdict{Proved: true, Said: fmt.Sprintf("%s.%s: its %s check sees the program", r.Module, r.ID, kind)}
|
||||
}
|
||||
if time.Now().After(deadline) || ctx.Err() != nil {
|
||||
return BedVerdict{Failed: true, Said: fmt.Sprintf("%s.%s stays up and its %s check does not see it within %s: %s",
|
||||
r.Module, r.ID, kind, grace, last)}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(BedLook):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// look is one http or tcp look at the program, from outside it, as the node-engine makes it. A check that
|
||||
// needs a provider only has to reach the program: any answer, a connect.
|
||||
func look(ctx context.Context, kind, address string, port int, h map[string]any, needs bool) (bool, string) {
|
||||
if address == "" || port == 0 {
|
||||
return false, "it has no address on the bed's network yet"
|
||||
}
|
||||
at := net.JoinHostPort(address, strconv.Itoa(port))
|
||||
ctx, cancel := context.WithTimeout(ctx, BedLook)
|
||||
defer cancel()
|
||||
if kind == "tcp" {
|
||||
c, err := (&net.Dialer{}).DialContext(ctx, "tcp", at)
|
||||
if err != nil {
|
||||
return false, oneLine(err.Error())
|
||||
}
|
||||
c.Close()
|
||||
return true, ""
|
||||
}
|
||||
scheme, _ := h["scheme"].(string)
|
||||
if scheme == "" {
|
||||
scheme = "http"
|
||||
}
|
||||
path, _ := h["path"].(string)
|
||||
if path == "" {
|
||||
path = "/"
|
||||
}
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, scheme+"://"+at+path, nil)
|
||||
res, err := (&http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
|
||||
Transport: insecure()}).Do(req)
|
||||
if err != nil {
|
||||
return false, oneLine(err.Error())
|
||||
}
|
||||
res.Body.Close()
|
||||
if needs {
|
||||
return true, ""
|
||||
}
|
||||
want, _ := h["status"].(float64)
|
||||
switch {
|
||||
case want != 0 && res.StatusCode != int(want):
|
||||
return false, fmt.Sprintf("answered %d, expected %d", res.StatusCode, int(want))
|
||||
case want == 0 && res.StatusCode >= 400:
|
||||
return false, fmt.Sprintf("answered %d", res.StatusCode)
|
||||
}
|
||||
return true, ""
|
||||
}
|
||||
|
||||
func splitMount(v string) (src, dst, rest string) {
|
||||
parts := strings.SplitN(v, ":", 3)
|
||||
if len(parts) < 2 {
|
||||
return "", "", ""
|
||||
}
|
||||
if len(parts) == 3 {
|
||||
rest = ":" + parts[2]
|
||||
}
|
||||
return parts[0], parts[1], rest
|
||||
}
|
||||
|
||||
func envLines(content string) []string {
|
||||
var out []string
|
||||
s := bufio.NewScanner(strings.NewReader(content))
|
||||
for s.Scan() {
|
||||
line := strings.TrimSpace(s.Text())
|
||||
if line == "" || strings.HasPrefix(line, "#") || !strings.Contains(line, "=") || placeholder.MatchString(line) {
|
||||
continue
|
||||
}
|
||||
out = append(out, line)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func lastLine(s string) string {
|
||||
lines := strings.Split(strings.TrimSpace(s), "\n")
|
||||
return lines[len(lines)-1]
|
||||
}
|
||||
|
||||
// FilesOf is every file a module writes in full — its content declared, nothing the mesh fills in — by the
|
||||
// path it writes it at, as the module names it: what the bed gives a container that mounts or reads it.
|
||||
func FilesOf(manifest []byte) map[string]string {
|
||||
_, _, rs := manifestParts(manifest)
|
||||
out := map[string]string{}
|
||||
for _, r := range rs {
|
||||
if fmt.Sprint(r["type"]) != "file" {
|
||||
continue
|
||||
}
|
||||
path, _ := r["path"].(string)
|
||||
content, ok := r["content"].(string)
|
||||
if path == "" || !ok || placeholder.MatchString(content) {
|
||||
continue
|
||||
}
|
||||
out[path] = content
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// insecure is a transport that asks whether a program answers, not whom to trust.
|
||||
func insecure() *http.Transport {
|
||||
return &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, DisableKeepAlives: true}
|
||||
}
|
||||
|
||||
// oneLine is the first line of what was said, short.
|
||||
func oneLine(s string) string {
|
||||
line, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
|
||||
if len(line) > 300 {
|
||||
line = line[:300] + "…"
|
||||
}
|
||||
return line
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
package replays
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// **The catalogue's bed** (novox/hq ADR 0240 rule 7, to-be 48 §8, Phase D): every long-running resource whose
|
||||
// declared `health` or image the change touches is started alone and its check must see the program within
|
||||
// its grace. Run by every merge check of the catalogue on the build seat, with the change's catalogue at
|
||||
// MESH_REPLAY_CATALOGUE; by hand against the catalogue beside the lab. What changed is against
|
||||
// MESH_BED_BASE (origin/main by default) in that checkout.
|
||||
func TestBedProvesEveryChangedHealthCheck(t *testing.T) {
|
||||
root := orDefault(os.Getenv("MESH_REPLAY_CATALOGUE"), filepath.Join("..", "..", "mesh-catalog"))
|
||||
if _, err := os.Stat(filepath.Join(root, "modules")); err != nil {
|
||||
t.Skipf("no catalogue at %s (MESH_REPLAY_CATALOGUE names it)", root)
|
||||
}
|
||||
base := orDefault(os.Getenv("MESH_BED_BASE"), "origin/main")
|
||||
show := func(ref, path string) ([]byte, error) {
|
||||
return exec.Command("git", "-c", "safe.directory=*", "-C", root, "show", ref+":"+path).Output()
|
||||
}
|
||||
if _, err := exec.Command("git", "-c", "safe.directory=*", "-C", root, "rev-parse", "--verify", base).Output(); err != nil {
|
||||
// What changed cannot be told, so every declared check is proved: the bed never passes what it did
|
||||
// not look at.
|
||||
t.Logf("%s is not in the catalogue at %s (%v): every declared check is proved", base, root, err)
|
||||
}
|
||||
changed, err := ChangedHealth(root, base, show)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(changed) == 0 {
|
||||
t.Logf("no declared check or image of a long-running resource changed against %s: nothing to prove", base)
|
||||
return
|
||||
}
|
||||
docker, ok := DockerFromEnv()
|
||||
if !ok {
|
||||
t.Fatalf("%d changed check(s) and no container runtime to prove them on", len(changed))
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Minute)
|
||||
defer cancel()
|
||||
for _, r := range changed {
|
||||
manifest, _ := os.ReadFile(filepath.Join(root, "modules", r.Module, "module.json"))
|
||||
v := docker.Prove(ctx, r, FilesOf(manifest))
|
||||
switch {
|
||||
case v.Failed:
|
||||
t.Errorf("FAILS ON THE BED: %s", v.Said)
|
||||
case v.Proved:
|
||||
t.Logf("proved: %s", v.Said)
|
||||
default:
|
||||
t.Logf("NOT PROVED HERE: %s", v.Said)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **R-studio — a check that asks an address the program does not bind fails the bed, not a machine**
|
||||
// (novox/hq ADR 0240 Phase D, done when). The hosted database suite's studio binds the address the runtime
|
||||
// puts in HOSTNAME, so it answered only on its network address while its image's own check asked localhost:
|
||||
// unhealthy for ever while working, until the module set HOSTNAME=0.0.0.0. Adopted by name without proof, it
|
||||
// would have put back a good build.
|
||||
//
|
||||
// The replay is that image in miniature: a web server that binds $HOSTNAME's address, and an image check that
|
||||
// asks localhost. Adopted as the module declared it before the fix, the bed fails it; with the fix, it proves
|
||||
// it. The image is built here and removed after, with everything the bed made.
|
||||
func TestBedFailsTheStudiosFalseUnhealthy(t *testing.T) {
|
||||
docker, ok := DockerFromEnv()
|
||||
if !ok {
|
||||
t.Skip("no container runtime: the studio is a container")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Minute)
|
||||
defer cancel()
|
||||
if err := docker.Pull(ctx, "busybox:1.36"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tag := fmt.Sprintf("mesh-replay-studio:%d", time.Now().UnixNano())
|
||||
if err := docker.Build(ctx, tag, StudioDockerfile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer docker.RemoveImage(context.Background(), tag)
|
||||
was := BedFloor
|
||||
BedFloor = 15 * time.Second
|
||||
defer func() { BedFloor = was }()
|
||||
|
||||
studio := func(env map[string]any) BedResource {
|
||||
r := map[string]any{"id": "studio", "type": "container", "name": "supabase-studio", "image": tag,
|
||||
"health": map[string]any{"kind": "runtime", "grace": "10s"}}
|
||||
if env != nil {
|
||||
r["env"] = env
|
||||
}
|
||||
return BedResource{Module: "supabase", ID: "studio", Resource: r, Listens: map[string]int{}}
|
||||
}
|
||||
before := docker.Prove(ctx, studio(nil), nil)
|
||||
if !before.Failed || !strings.Contains(before.Said, "does not see it") {
|
||||
t.Fatalf("the studio's false unhealthy was not failed on the bed: %+v", before)
|
||||
}
|
||||
t.Logf("before the fix: %s", before.Said)
|
||||
after := docker.Prove(ctx, studio(map[string]any{"HOSTNAME": "0.0.0.0"}), nil)
|
||||
if !after.Proved {
|
||||
t.Fatalf("the studio with HOSTNAME=0.0.0.0 was not proved: %+v", after)
|
||||
}
|
||||
t.Logf("with the fix: %s", after.Said)
|
||||
}
|
||||
|
||||
// StudioDockerfile is the studio in miniature: it serves on the address HOSTNAME names, and its own check
|
||||
// asks localhost, as the studio's image does.
|
||||
const StudioDockerfile = `FROM busybox:1.36
|
||||
RUN mkdir -p /www && echo studio > /www/index.html
|
||||
HEALTHCHECK --interval=5s --timeout=2s --retries=2 CMD wget -q -O /dev/null http://localhost:3000/ || exit 1
|
||||
CMD addr=$(grep -w "$HOSTNAME" /etc/hosts | head -n 1 | cut -f 1); exec httpd -f -p "${addr:-$HOSTNAME}:3000" -h /www
|
||||
`
|
||||
|
||||
// What the bed proves is what the change touches: a long-running container whose declared check or image
|
||||
// changed, or that is new; never a step, never one left as it was.
|
||||
func TestTheBedProvesWhatTheChangeTouches(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
git := func(args ...string) {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", append([]string{"-C", root, "-c", "user.email=lab@example", "-c", "user.name=lab"}, args...)...)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
t.Fatalf("git %v: %v %s", args, err, out)
|
||||
}
|
||||
}
|
||||
write := func(module, body string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Join(root, "modules", module), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "modules", module, "module.json"), []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
git("init", "-q", "-b", "main")
|
||||
write("web", `{"module":"web","resources":[{"id":"server","type":"container","image":"web@sha256:a","health":{"kind":"runtime"}},
|
||||
{"id":"seed","type":"container","image":"web@sha256:a","run-once":true,"health":{"kind":"runtime"}}]}`)
|
||||
write("db", `{"module":"db","resources":[{"id":"server","type":"container","image":"db@sha256:a","health":{"kind":"runtime"}}]}`)
|
||||
git("add", "-A")
|
||||
git("commit", "-qm", "base")
|
||||
write("web", `{"module":"web","resources":[{"id":"server","type":"container","image":"web@sha256:b","health":{"kind":"runtime"}},
|
||||
{"id":"seed","type":"container","image":"web@sha256:b","run-once":true,"health":{"kind":"runtime"}}]}`)
|
||||
write("cache", `{"module":"cache","resources":[{"id":"server","type":"container","image":"cache@sha256:a","health":{"kind":"tcp","endpoint":"redis"}},
|
||||
{"id":"plain","type":"container","image":"cache@sha256:a"}]}`)
|
||||
show := func(ref, path string) ([]byte, error) {
|
||||
return exec.Command("git", "-C", root, "show", ref+":"+path).Output()
|
||||
}
|
||||
changed, err := ChangedHealth(root, "main", show)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got []string
|
||||
for _, r := range changed {
|
||||
got = append(got, r.Module+"."+r.ID)
|
||||
}
|
||||
if strings.Join(got, ",") != "cache.server,web.server" {
|
||||
t.Fatalf("the bed would prove %v; want the new module's checked container and the one whose image moved", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
package replays
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// **R266 — a consumer with several filters is handed every message** (novox/hq issue 266).
|
||||
//
|
||||
// The controller follows the forge's merges, build outcomes and providers' standings on one durable
|
||||
// consumer with several filters. On nats 2.10.29 such a consumer was moved past a message now and then
|
||||
// without handing it over — nothing pending, nothing redelivered — and on 2026-10-06 that message was a
|
||||
// merge nobody acted on. This is that consumer under traffic shaped like the mesh's, against the bus
|
||||
// MESH_TEST_NATS names: in a merge check, a throwaway of the release the mesh runs; for the prover, the
|
||||
// release the catalogue pinned at the commit it proves. Fails on 2.10.29, passes on 2.11.17.
|
||||
//
|
||||
// Everything it makes is under a prefix of its own, so it can run on a bus other things use.
|
||||
func TestReplay266AConsumerWithSeveralFiltersIsHandedEveryMessage(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS names no bus to replay against")
|
||||
}
|
||||
p := fmt.Sprintf("r266x%d", time.Now().UnixNano())
|
||||
admin, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer admin.Close()
|
||||
js, err := admin.JetStream()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stream := "R266_" + p
|
||||
if _, err := js.AddStream(&nats.StreamConfig{Name: stream, Subjects: []string{p + ".mod.*.event.>", p + ".seat.*.event.>"},
|
||||
MaxAge: time.Hour, MaxMsgsPerSubject: 10000, Storage: nats.FileStorage}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = js.DeleteStream(stream) }()
|
||||
// The controller's consumer, as the mesh declared it on the day: seven filters, one at a time.
|
||||
if _, err := js.AddConsumer(stream, &nats.ConsumerConfig{Durable: "controller", DeliverSubject: "_DELIVER." + p,
|
||||
FilterSubjects: []string{
|
||||
p + ".mod.mesh-catalog.event.upgraded",
|
||||
p + ".mod.mesh-catalog.event.catching-up",
|
||||
p + ".seat.node-build-agent.event.built",
|
||||
p + ".mod.gitea.event.pull.merged",
|
||||
p + ".seat.mesh-build-machine.event.built",
|
||||
p + ".mod.*.event.provisioner.failing",
|
||||
p + ".mod.*.event.provisioner.recovered",
|
||||
},
|
||||
AckPolicy: nats.AckExplicitPolicy, AckWait: 30 * time.Second, MaxDeliver: 5, MaxAckPending: 1,
|
||||
DeliverPolicy: nats.DeliverNewPolicy}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
controller, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer controller.Close()
|
||||
cjs, _ := controller.JetStream()
|
||||
handed := make(chan *nats.Msg, 64)
|
||||
sub, err := cjs.ChanSubscribe("", handed, nats.Bind(stream, "controller"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
var got sync.Map
|
||||
go func() {
|
||||
for m := range handed {
|
||||
if meta, err := m.Metadata(); err == nil {
|
||||
got.Store(meta.Sequence.Stream, true)
|
||||
}
|
||||
time.Sleep(time.Duration(rand.Intn(20)) * time.Millisecond)
|
||||
_ = m.Ack()
|
||||
}
|
||||
}()
|
||||
|
||||
followed := []string{p + ".mod.gitea.event.pull.merged", p + ".seat.node-build-agent.event.built",
|
||||
p + ".mod.postgres.event.provisioner.recovered", p + ".mod.mesh-catalog.event.catching-up"}
|
||||
var mu sync.Mutex
|
||||
var sent []uint64
|
||||
stop := time.Now().Add(6 * time.Second)
|
||||
var wg sync.WaitGroup
|
||||
for w := 0; w < 4; w++ {
|
||||
wg.Add(1)
|
||||
go func(w int) {
|
||||
defer wg.Done()
|
||||
nc, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
defer nc.Close()
|
||||
pjs, _ := nc.JetStream()
|
||||
for i := 0; time.Now().Before(stop); i++ {
|
||||
if rand.Intn(40) == 0 {
|
||||
if ack, err := pjs.Publish(followed[rand.Intn(len(followed))], []byte(`{}`)); err == nil {
|
||||
mu.Lock()
|
||||
sent = append(sent, ack.Sequence)
|
||||
mu.Unlock()
|
||||
}
|
||||
} else {
|
||||
// A build's log, on a subject of its own: what the stream mostly holds.
|
||||
_, _ = pjs.Publish(fmt.Sprintf("%s.seat.node-build-agent.event.log.build-%d-%d", p, w, i/50), make([]byte, 200))
|
||||
}
|
||||
if rand.Intn(100) == 0 {
|
||||
time.Sleep(time.Duration(rand.Intn(300)) * time.Millisecond)
|
||||
}
|
||||
}
|
||||
}(w)
|
||||
}
|
||||
wg.Wait()
|
||||
if len(sent) == 0 {
|
||||
t.Fatal("no followed event was published")
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(30 * time.Second)
|
||||
for {
|
||||
var missing []uint64
|
||||
for _, seq := range sent {
|
||||
if _, ok := got.Load(seq); !ok {
|
||||
missing = append(missing, seq)
|
||||
}
|
||||
}
|
||||
if len(missing) == 0 {
|
||||
return
|
||||
}
|
||||
info, err := js.ConsumerInfo(stream, "controller")
|
||||
if time.Now().After(deadline) || (err == nil && info.NumPending == 0 && info.NumAckPending == 0) {
|
||||
t.Fatalf("%d of %d followed events were never handed to the consumer (first at stream sequence %d), "+
|
||||
"and it has nothing pending: the server moved past them — issue 266, on %s", len(missing), len(sent),
|
||||
missing[0], admin.ConnectedServerVersion())
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,402 @@
|
||||
// prove runs each replay on the commit before its fix and on its fix, and says whether it fails before
|
||||
// and passes after (novox/hq to-be 45 Phase 5, "done when"). A replay that passes before its fix proves
|
||||
// nothing about it, and one that fails after it is not a replay of that fix.
|
||||
//
|
||||
// go run ./cmd/prove [--repos <dir holding the clones, ../.. by default>] [R262 R266 …]
|
||||
//
|
||||
// The clones are the core repositories, side by side as the lab expects them. A replay in its repository
|
||||
// is laid over a worktree of each commit and run with go test; the bus replay runs against a server of
|
||||
// the release the catalogue pinned at each commit, raised for it; the resolver replay with the catalogue's
|
||||
// resolver at each commit; a bed of the lab is raised on its hypervisor with the node-engine and its
|
||||
// installer built at each commit (a workstation where `mesh-lab check` passes). MESH_TEST_POSTGRES is passed
|
||||
// through to the controller's.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-lab/replays"
|
||||
)
|
||||
|
||||
func main() {
|
||||
repos := flag.String("repos", filepath.Join("..", ".."), "the directory the core repositories are cloned side by side in")
|
||||
flag.Parse()
|
||||
if abs, err := filepath.Abs(*repos); err == nil {
|
||||
*repos = abs
|
||||
}
|
||||
want := flag.Args()
|
||||
failed := 0
|
||||
for _, r := range replays.Register {
|
||||
if len(want) > 0 && !contains(want, r.ID) {
|
||||
continue
|
||||
}
|
||||
at := r.Fix + "^1"
|
||||
if r.Before != "" {
|
||||
at = r.Before
|
||||
}
|
||||
before, beforeSaid := run(r, *repos, at, false)
|
||||
after, afterSaid := run(r, *repos, r.Fix, true)
|
||||
verdict := "PROVED"
|
||||
if before != "fails" && before != "absent" || after != "passes" {
|
||||
verdict = "NOT PROVED"
|
||||
failed++
|
||||
}
|
||||
fmt.Printf("%s (issue %d): %s — before its fix %s, on its fix %s\n", r.ID, r.Issue, verdict, before, after)
|
||||
if verdict != "PROVED" {
|
||||
fmt.Printf(" before: %s\n after: %s\n", beforeSaid, afterSaid)
|
||||
} else {
|
||||
fmt.Printf(" before: %s\n", firstLine(beforeSaid))
|
||||
}
|
||||
}
|
||||
if failed > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// run is the replay at one commit: passes, fails, or absent (the check it replays did not exist).
|
||||
func run(r replays.Replay, repos, at string, fixed bool) (string, string) {
|
||||
limit := 15 * time.Minute
|
||||
if r.Kind == replays.Bed {
|
||||
// Two machines raised, a foundation pulled and a control plane built inside one of them.
|
||||
limit = 90 * time.Minute
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), limit)
|
||||
defer cancel()
|
||||
repo := filepath.Join(repos, r.Repository)
|
||||
tree, err := os.MkdirTemp("", "prove-"+r.ID+"-")
|
||||
if err != nil {
|
||||
return "error", err.Error()
|
||||
}
|
||||
defer os.RemoveAll(tree)
|
||||
if out, err := exec.CommandContext(ctx, "git", "-C", repo, "worktree", "add", "--quiet", "--detach", tree, at).CombinedOutput(); err != nil {
|
||||
return "error", string(out)
|
||||
}
|
||||
defer exec.Command("git", "-C", repo, "worktree", "remove", "--force", tree).Run()
|
||||
|
||||
switch r.Kind {
|
||||
case replays.Bed:
|
||||
return aBed(ctx, r, repos, tree, fixed)
|
||||
case replays.InRepository, replays.Gate, replays.Liveness:
|
||||
home := filepath.Join(repos, r.Repository)
|
||||
if r.Home != "" {
|
||||
home = filepath.Join(repos, r.Home)
|
||||
}
|
||||
ref := r.HomeRef
|
||||
if ref == "" {
|
||||
ref = r.Fix
|
||||
}
|
||||
for _, f := range r.Files {
|
||||
body, err := exec.Command("git", "-C", home, "show", ref+":"+f).Output()
|
||||
if err != nil {
|
||||
return "error", fmt.Sprintf("%s has no %s: %v", ref, f, err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(tree, f), body, 0o644); err != nil {
|
||||
return "error", err.Error()
|
||||
}
|
||||
}
|
||||
if r.Kind == replays.Liveness {
|
||||
// The other repositories at their own commits on the same side, and the replay here run
|
||||
// against all of them.
|
||||
env := []string{"MESH_REPLAY_CONTROLLER=" + tree}
|
||||
for _, also := range r.With {
|
||||
ref := also.Fix
|
||||
if !fixed {
|
||||
ref = also.Fix + "^1"
|
||||
if also.Before != "" {
|
||||
ref = also.Before
|
||||
}
|
||||
}
|
||||
other, err := os.MkdirTemp("", "prove-"+r.ID+"-"+also.Repository+"-")
|
||||
if err != nil {
|
||||
return "error", err.Error()
|
||||
}
|
||||
defer os.RemoveAll(other)
|
||||
clone := filepath.Join(repos, also.Repository)
|
||||
if out, err := exec.CommandContext(ctx, "git", "-C", clone, "worktree", "add", "--quiet", "--detach", other,
|
||||
ref).CombinedOutput(); err != nil {
|
||||
return "error", string(out)
|
||||
}
|
||||
defer exec.Command("git", "-C", clone, "worktree", "remove", "--force", other).Run()
|
||||
if also.Repository == "mesh-host" {
|
||||
env = append(env, "MESH_REPLAY_HOST="+other)
|
||||
}
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^"+r.Test+"$", r.Package)
|
||||
cmd.Env = append(os.Environ(), env...)
|
||||
out, err := cmd.CombinedOutput()
|
||||
return verdictOf(string(out), err, false)
|
||||
}
|
||||
mode := "-mod=vendor"
|
||||
if _, err := os.Stat(filepath.Join(tree, r.Module, "vendor")); err != nil {
|
||||
mode = "-mod=mod"
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^"+r.Test, r.Package)
|
||||
cmd.Dir = filepath.Join(tree, r.Module)
|
||||
cmd.Env = append(os.Environ(), "GOFLAGS="+mode, "GOPRIVATE=git.novox.be")
|
||||
out, err := cmd.CombinedOutput()
|
||||
return verdictOf(string(out), err, r.Kind == replays.Gate)
|
||||
case replays.Bus:
|
||||
// **The very image the catalogue pinned at the commit**, by its digest: the release a digest is was
|
||||
// not said beside it before the fix, and the digest is what ran.
|
||||
image, err := pinnedBus(tree)
|
||||
if err != nil {
|
||||
return "error", err.Error()
|
||||
}
|
||||
url, stop, err := aBus(ctx, image)
|
||||
if err != nil {
|
||||
return "error", err.Error()
|
||||
}
|
||||
defer stop()
|
||||
return replayHere(ctx, r, "MESH_TEST_NATS="+url)
|
||||
case replays.Resolver:
|
||||
return replayHere(ctx, r, "MESH_REPLAY_CATALOGUE="+tree)
|
||||
}
|
||||
return "error", "no such kind"
|
||||
}
|
||||
|
||||
// aBed raises the lab's bed at one commit of the fixed repository (novox/hq issue 92): the node-engine and its
|
||||
// installer built from it, the installer carrying a builder built from the controller the replay pins, and
|
||||
// the bed run with them, the pinned controller to build from and the pinned catalogue to read.
|
||||
func aBed(ctx context.Context, r replays.Replay, repos, tree string, fixed bool) (string, string) {
|
||||
beside := map[string]string{}
|
||||
for _, also := range r.With {
|
||||
ref := also.Fix
|
||||
if !fixed {
|
||||
ref = also.Fix + "^1"
|
||||
if also.Before != "" {
|
||||
ref = also.Before
|
||||
}
|
||||
}
|
||||
other, err := os.MkdirTemp("", "prove-"+r.ID+"-"+also.Repository+"-")
|
||||
if err != nil {
|
||||
return "error", err.Error()
|
||||
}
|
||||
defer os.RemoveAll(other)
|
||||
clone := filepath.Join(repos, also.Repository)
|
||||
if out, err := exec.CommandContext(ctx, "git", "-C", clone, "worktree", "add", "--quiet", "--detach", other,
|
||||
ref).CombinedOutput(); err != nil {
|
||||
return "error", string(out)
|
||||
}
|
||||
defer exec.Command("git", "-C", clone, "worktree", "remove", "--force", other).Run()
|
||||
beside[also.Repository] = other
|
||||
}
|
||||
controller, catalogue := beside["mesh-controller"], beside["mesh-catalog"]
|
||||
if controller == "" || catalogue == "" {
|
||||
return "error", "a bed is raised with the controller and the catalogue it pins, and this one pins " +
|
||||
"not both"
|
||||
}
|
||||
builder, err := builderImage(ctx, controller)
|
||||
if err != nil {
|
||||
return "error", err.Error()
|
||||
}
|
||||
host, installer := filepath.Join(tree, "mesh-host"), filepath.Join(tree, "mesh-bootstrap")
|
||||
for _, step := range [][]string{
|
||||
{"make", "build", "SYSTEM=arch"},
|
||||
{"make", "bootstrap", "IMAGE=" + builder, "BOOTSTRAP_OUT=" + installer},
|
||||
} {
|
||||
cmd := exec.CommandContext(ctx, step[0], step[1:]...)
|
||||
cmd.Dir = tree
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
return "error", fmt.Sprintf("%s: %v %s", strings.Join(step, " "), err, lastLines(string(out), 6))
|
||||
}
|
||||
}
|
||||
lab, err := filepath.Abs("..")
|
||||
if err != nil {
|
||||
return "error", err.Error()
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "node", "--test", "--test-reporter=tap", "--experimental-strip-types", r.Test)
|
||||
cmd.Dir = lab
|
||||
cmd.Env = append(os.Environ(),
|
||||
"MESH_LAB_HOST_BINARY="+host,
|
||||
"MESH_LAB_BOOTSTRAP_BINARY="+installer,
|
||||
// The template the installer knows how to make safe: it rotates the store's and the broker's
|
||||
// well-known credentials, and refuses a template naming neither (the bus one names no broker).
|
||||
"MESH_LAB_BUNDLE="+filepath.Join(tree, "examples", "foundation-first-node.lock"),
|
||||
"MESH_LAB_CATALOG="+filepath.Join(catalogue, "modules"),
|
||||
"MESH_REPLAY_CONTROLLER="+controller)
|
||||
out, err := cmd.CombinedOutput()
|
||||
return bedVerdict(string(out), err)
|
||||
}
|
||||
|
||||
// builderImage is the builder the installer carries, built from the controller at the commit the replay
|
||||
// pins, on the Go release its go.mod asks for; built once and kept under a tag naming the commit.
|
||||
func builderImage(ctx context.Context, controller string) (string, error) {
|
||||
head, err := exec.CommandContext(ctx, "git", "-C", controller, "rev-parse", "--short=12", "HEAD").Output()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
tag := "mesh-builder:prove-" + strings.TrimSpace(string(head))
|
||||
if exec.CommandContext(ctx, "docker", "image", "inspect", tag).Run() == nil {
|
||||
return tag, nil
|
||||
}
|
||||
mod, err := os.ReadFile(filepath.Join(controller, "go.mod"))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
release := ""
|
||||
for _, line := range strings.Split(string(mod), "\n") {
|
||||
if v, ok := strings.CutPrefix(line, "go "); ok {
|
||||
parts := strings.Split(strings.TrimSpace(v), ".")
|
||||
if len(parts) >= 2 {
|
||||
release = parts[0] + "." + parts[1]
|
||||
}
|
||||
}
|
||||
}
|
||||
if release == "" {
|
||||
return "", errors.New("the controller's go.mod names no Go release to build its builder on")
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "docker", "build", "-q", "--build-arg", "GO_BASE=golang:"+release+"-alpine",
|
||||
"-f", "cmd/mesh-builder/Dockerfile", "-t", tag, ".")
|
||||
cmd.Dir = controller
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
return "", fmt.Errorf("the builder could not be built from %s: %v %s", controller, err, lastLines(string(out), 6))
|
||||
}
|
||||
return tag, nil
|
||||
}
|
||||
|
||||
// bedVerdict reads node's test runner: a skipped bed is an error — it never reached what it replays, or was
|
||||
// not given what it raises — and never the fault reproduced.
|
||||
func bedVerdict(out string, err error) (string, string) {
|
||||
count := func(what string) int {
|
||||
m := regexp.MustCompile(`(?m)^# ` + what + ` (\d+)$`).FindStringSubmatch(out)
|
||||
if m == nil {
|
||||
return -1
|
||||
}
|
||||
n, _ := strconv.Atoi(m[1])
|
||||
return n
|
||||
}
|
||||
pass, fail, skipped := count("pass"), count("fail"), count("skipped")
|
||||
switch {
|
||||
case skipped > 0:
|
||||
said := regexp.MustCompile(`(?m)# SKIP (.*)$`).FindStringSubmatch(out)
|
||||
if said != nil {
|
||||
return "error", "the bed skipped: " + said[1]
|
||||
}
|
||||
return "error", "the bed skipped: " + lastLines(out, 6)
|
||||
case fail > 0:
|
||||
if m := regexp.MustCompile(`(?m)^\s+error: [|>]?-?\s*\n?\s*(.+)$`).FindStringSubmatch(out); m != nil {
|
||||
return "fails", strings.TrimSpace(m[1])
|
||||
}
|
||||
return "fails", lastLines(out, 6)
|
||||
case pass > 0 && err == nil:
|
||||
return "passes", lastLines(out, 3)
|
||||
}
|
||||
return "error", lastLines(out, 6)
|
||||
}
|
||||
|
||||
func replayHere(ctx context.Context, r replays.Replay, env string) (string, string) {
|
||||
cmd := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^"+r.Test, r.Package)
|
||||
cmd.Env = append(os.Environ(), env)
|
||||
out, err := cmd.CombinedOutput()
|
||||
return verdictOf(string(out), err, false)
|
||||
}
|
||||
|
||||
// verdictOf reads go test's word: a replay that does not build where the check did not exist is absent.
|
||||
func verdictOf(out string, err error, gate bool) (string, string) {
|
||||
switch {
|
||||
case err == nil && strings.Contains(out, "no tests to run"):
|
||||
return "absent", "no test of that name at this commit"
|
||||
case err == nil:
|
||||
return "passes", lastLines(out, 3)
|
||||
case strings.Contains(out, "[build failed]") || strings.Contains(out, "[setup failed]"):
|
||||
if gate {
|
||||
return "absent", "the check did not exist: " + lastLines(out, 3)
|
||||
}
|
||||
return "error", "does not build: " + lastLines(out, 6)
|
||||
case strings.Contains(out, "--- FAIL"):
|
||||
return "fails", failure(out)
|
||||
}
|
||||
return "error", lastLines(out, 6)
|
||||
}
|
||||
|
||||
// pinnedBus is the bus image the catalogue's nats module is built on at a commit.
|
||||
func pinnedBus(tree string) (string, error) {
|
||||
raw, err := os.ReadFile(filepath.Join(tree, "modules", "nats", "module.json"))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var m struct {
|
||||
Build struct {
|
||||
On []struct{ Arg, Image string } `json:"on"`
|
||||
} `json:"build"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, on := range m.Build.On {
|
||||
if on.Arg == "NATS_BASE" && on.Image != "" {
|
||||
return on.Image, nil
|
||||
}
|
||||
}
|
||||
return "", errors.New("the catalogue's nats module names no NATS_BASE image")
|
||||
}
|
||||
|
||||
// aBus is a server of an image, on loopback, for the length of one replay.
|
||||
func aBus(ctx context.Context, image string) (string, func(), error) {
|
||||
name := fmt.Sprintf("prove-bus-%d", time.Now().UnixNano())
|
||||
if out, err := exec.CommandContext(ctx, "docker", "run", "-d", "--rm", "--name", name, "-p", "127.0.0.1::4222",
|
||||
image, "-js").CombinedOutput(); err != nil {
|
||||
return "", nil, fmt.Errorf("%s: %v %s", image, err, out)
|
||||
}
|
||||
stop := func() { _ = exec.Command("docker", "rm", "-f", name).Run() }
|
||||
out, err := exec.CommandContext(ctx, "docker", "port", name, "4222/tcp").Output()
|
||||
if err != nil {
|
||||
stop()
|
||||
return "", nil, err
|
||||
}
|
||||
address := strings.TrimSpace(strings.Split(string(out), "\n")[0])
|
||||
for i := 0; i < 50; i++ {
|
||||
if c, err := net.Dial("tcp", address); err == nil {
|
||||
c.Close()
|
||||
break
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
return "nats://" + address, stop, nil
|
||||
}
|
||||
|
||||
func failure(out string) string {
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
if strings.Contains(line, "_test.go:") {
|
||||
return strings.TrimSpace(line)
|
||||
}
|
||||
}
|
||||
return lastLines(out, 3)
|
||||
}
|
||||
|
||||
func lastLines(s string, n int) string {
|
||||
lines := strings.Split(strings.TrimSpace(s), "\n")
|
||||
if len(lines) > n {
|
||||
lines = lines[len(lines)-n:]
|
||||
}
|
||||
return strings.Join(lines, " | ")
|
||||
}
|
||||
|
||||
func firstLine(s string) string {
|
||||
line, _, _ := strings.Cut(s, "\n")
|
||||
if len(line) > 300 {
|
||||
line = line[:300] + "…"
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
func contains(list []string, s string) bool {
|
||||
for _, x := range list {
|
||||
if x == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package replays
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// **R-crashloop — a module's container that crash-loops fails its gate on the first machine** (novox/hq
|
||||
// ADR 0240, "how it is checked", rule 1; to-be 48 Phase A, done when).
|
||||
//
|
||||
// On the home server the agent server restarted about a hundred times in a crash loop while the mesh read
|
||||
// it applied, its tools served and no condition raised; a person found it reading its log for another
|
||||
// reason (issue 268, research 032 §6). The gate judged a module by what the mesh saw from outside.
|
||||
//
|
||||
// The replay is the whole chain, each half at its commit:
|
||||
//
|
||||
// 1. a container whose program exits at start is raised here as the node-engine raises a module's — the
|
||||
// runtime restarting it unless stopped, labelled with its resource;
|
||||
// 2. the node-engine at MESH_REPLAY_HOST judges it through the runtime, as it judges every long-running
|
||||
// resource, and writes what it states (its TestReplayCrashLoopIsSaidUnhealthy) — an engine older than
|
||||
// the judging has nothing to state, and its reports carry no health;
|
||||
// 3. the controller at MESH_REPLAY_CONTROLLER rolls a build of the module out, hears its first machine say
|
||||
// that, and judges the gate (its TestReplayCrashLoopFailsItsGateOnTheFirstMachine).
|
||||
//
|
||||
// The assertion is the controller's: the build fails its gate on the first machine and is put back there.
|
||||
// Needs a container runtime, Go, and MESH_TEST_POSTGRES for the controller's store; the container is
|
||||
// removed after, whatever happened.
|
||||
func TestReplayCrashLoop(t *testing.T) {
|
||||
host := orDefault(os.Getenv("MESH_REPLAY_HOST"), filepath.Join("..", "..", "mesh-host"))
|
||||
controller := orDefault(os.Getenv("MESH_REPLAY_CONTROLLER"), filepath.Join("..", "..", "mesh-controller"))
|
||||
for _, dir := range []string{host, controller} {
|
||||
if _, err := os.Stat(filepath.Join(dir, "go.mod")); err != nil {
|
||||
t.Skipf("no checkout at %s (MESH_REPLAY_HOST and MESH_REPLAY_CONTROLLER name them)", dir)
|
||||
}
|
||||
}
|
||||
if os.Getenv("MESH_TEST_POSTGRES") == "" {
|
||||
t.Skip("no MESH_TEST_POSTGRES: the controller's half judges its gate against a store")
|
||||
}
|
||||
docker, ok := DockerFromEnv()
|
||||
if !ok {
|
||||
t.Skip("no container runtime: the crash loop is a container")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 10*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
// 1. The crash loop, as the node-engine raises a module's container.
|
||||
const image = "alpine:3.20"
|
||||
if err := docker.Pull(ctx, image); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
name := fmt.Sprintf("mesh-replay-crashloop-%d", time.Now().UnixNano())
|
||||
id, _, err := docker.Start(ctx, Run{Image: image, Cmd: []string{"sh", "-c", "echo starting; exit 3"}, Name: name,
|
||||
Restart: "unless-stopped", Labels: map[string]string{"mesh-host.id": "app.server"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer docker.Remove(context.Background(), id)
|
||||
|
||||
// 2. The node-engine at its commit judges it, and says what it states.
|
||||
statement := filepath.Join(t.TempDir(), "statement.json")
|
||||
engine := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^TestReplayCrashLoopIsSaidUnhealthy$",
|
||||
"./internal/liveness/")
|
||||
engine.Dir = host
|
||||
engine.Env = append(os.Environ(), "MESH_REPLAY_CONTAINER="+name, "MESH_REPLAY_STATEMENT="+statement, "GOFLAGS=")
|
||||
out, err := engine.CombinedOutput()
|
||||
switch {
|
||||
case err == nil && !strings.Contains(string(out), "no tests to run"):
|
||||
t.Logf("the node-engine said: %s", firstLineOf(statementOf(statement)))
|
||||
case judgesNothing(string(out)):
|
||||
// An engine older than the judging: its reports carry no health.
|
||||
t.Logf("the node-engine at this commit judges nothing it runs")
|
||||
if err := os.WriteFile(statement, []byte("null"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
default:
|
||||
t.Fatalf("the node-engine did not say the crash loop unhealthy: %v\n%s", err, lastOf(string(out), 12))
|
||||
}
|
||||
|
||||
// 3. The controller at its commit judges the gate from what the engine said.
|
||||
gate := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^TestReplayCrashLoopFailsItsGateOnTheFirstMachine$",
|
||||
"./cmd/mesh-controller/")
|
||||
gate.Dir = controller
|
||||
gate.Env = append(os.Environ(), "MESH_REPLAY_STATEMENT="+statement, "GOFLAGS=-mod=vendor", "GOPROXY=off")
|
||||
out, err = gate.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("the gate did not fail the crash loop on its first machine: %v\n%s", err, lastOf(string(out), 12))
|
||||
}
|
||||
if strings.Contains(string(out), "no tests to run") {
|
||||
t.Fatal("the controller at this commit has no half of the replay to run")
|
||||
}
|
||||
}
|
||||
|
||||
// judgesNothing is go test's word for a package that is not there: an engine older than the judging.
|
||||
func judgesNothing(out string) bool {
|
||||
return strings.Contains(out, "no tests to run") || strings.Contains(out, "directory not found") ||
|
||||
strings.Contains(out, "no Go files") || strings.Contains(out, "cannot find package") ||
|
||||
strings.Contains(out, "matched no packages") || strings.Contains(out, "is not in std")
|
||||
}
|
||||
|
||||
func statementOf(path string) string {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "(nothing written: " + err.Error() + ")"
|
||||
}
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
func orDefault(s, fallback string) string {
|
||||
if s == "" {
|
||||
return fallback
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func firstLineOf(s string) string {
|
||||
line, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
|
||||
if len(line) > 400 {
|
||||
line = line[:400] + "…"
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
func lastOf(s string, n int) string {
|
||||
lines := strings.Split(strings.TrimSpace(s), "\n")
|
||||
if len(lines) > n {
|
||||
lines = lines[len(lines)-n:]
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
package replays
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Docker is the container runtime's own API, over its socket: what a replay needs to raise a resolver and
|
||||
// ask it from programs under two C libraries — pull, network, run, wait, logs, remove — and nothing else.
|
||||
// Spoken directly rather than through the command line, because the Go toolchain a check runs in carries
|
||||
// no docker client, and a replay that needed one would be a replay that runs only on a workstation.
|
||||
type Docker struct {
|
||||
http *http.Client
|
||||
// Label marks everything it makes, so a replay that dies is cleaned up by it.
|
||||
Label string
|
||||
}
|
||||
|
||||
// DockerFromEnv is the runtime at DOCKER_HOST's socket, or the usual one; ok false when there is none.
|
||||
func DockerFromEnv() (*Docker, bool) {
|
||||
sock := "/var/run/docker.sock"
|
||||
if h := os.Getenv("DOCKER_HOST"); strings.HasPrefix(h, "unix://") {
|
||||
sock = strings.TrimPrefix(h, "unix://")
|
||||
}
|
||||
if _, err := os.Stat(sock); err != nil {
|
||||
return nil, false
|
||||
}
|
||||
return &Docker{Label: "mesh.replay", http: &http.Client{Transport: &http.Transport{
|
||||
DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
|
||||
return (&net.Dialer{}).DialContext(ctx, "unix", sock)
|
||||
}}}}, true
|
||||
}
|
||||
|
||||
func (d *Docker) call(ctx context.Context, method, path string, body any, out any) error {
|
||||
var r io.Reader
|
||||
if body != nil {
|
||||
raw, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r = bytes.NewReader(raw)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, "http://docker"+path, r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
res, err := d.http.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
raw, _ := io.ReadAll(res.Body)
|
||||
if res.StatusCode >= 300 {
|
||||
return fmt.Errorf("docker %s %s: %s %s", method, path, res.Status, strings.TrimSpace(string(raw)))
|
||||
}
|
||||
if out != nil && len(raw) > 0 {
|
||||
return json.Unmarshal(raw, out)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Pull fetches an image unless the runtime has it.
|
||||
func (d *Docker) Pull(ctx context.Context, image string) error {
|
||||
if err := d.call(ctx, http.MethodGet, "/images/"+url.PathEscape(image)+"/json", nil, nil); err == nil {
|
||||
return nil
|
||||
}
|
||||
// A digest is the tag the runtime is asked for, as its own client asks: `name@sha256:…` cut at the
|
||||
// `@`; otherwise the tag after the last `:` that is not part of a registry's port.
|
||||
name, tag, digested := strings.Cut(image, "@")
|
||||
if !digested {
|
||||
name, tag = image, "latest"
|
||||
if at := strings.LastIndex(image, ":"); at > strings.LastIndex(image, "/") {
|
||||
name, tag = image[:at], image[at+1:]
|
||||
}
|
||||
}
|
||||
err := d.call(ctx, http.MethodPost, "/images/create?fromImage="+url.QueryEscape(name)+"&tag="+url.QueryEscape(tag), nil, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The runtime answers a pull it could not make with a stream that ends in an error, not a status.
|
||||
return d.call(ctx, http.MethodGet, "/images/"+url.PathEscape(image)+"/json", nil, nil)
|
||||
}
|
||||
|
||||
// Network makes a network of its own and answers its id.
|
||||
func (d *Docker) Network(ctx context.Context, name string) (string, error) {
|
||||
var made struct{ ID string }
|
||||
err := d.call(ctx, http.MethodPost, "/networks/create", map[string]any{"Name": name,
|
||||
"Labels": map[string]string{d.Label: name}}, &made)
|
||||
return made.ID, err
|
||||
}
|
||||
|
||||
// RemoveNetwork removes a network.
|
||||
func (d *Docker) RemoveNetwork(ctx context.Context, id string) {
|
||||
_ = d.call(ctx, http.MethodDelete, "/networks/"+id, nil, nil)
|
||||
}
|
||||
|
||||
// Run is one container: an image, a command, the files it is given, its network and resolver.
|
||||
type Run struct {
|
||||
Image string
|
||||
Cmd []string
|
||||
Env []string
|
||||
Binds []string
|
||||
Network string
|
||||
DNS []string
|
||||
// Name, Labels and Restart are the container's name, its labels beside the replay's own, and the
|
||||
// runtime's restart policy — what the node-engine creates a module's container with.
|
||||
Name string
|
||||
Labels map[string]string
|
||||
Restart string
|
||||
}
|
||||
|
||||
// Start creates and starts a container and answers its id and its address on its network.
|
||||
func (d *Docker) Start(ctx context.Context, r Run) (string, string, error) {
|
||||
var made struct{ ID string }
|
||||
host := map[string]any{"Binds": r.Binds, "NetworkMode": r.Network}
|
||||
if len(r.DNS) > 0 {
|
||||
host["Dns"] = r.DNS
|
||||
}
|
||||
if r.Restart != "" {
|
||||
host["RestartPolicy"] = map[string]any{"Name": r.Restart}
|
||||
}
|
||||
labels := map[string]string{d.Label: "1"}
|
||||
for k, v := range r.Labels {
|
||||
labels[k] = v
|
||||
}
|
||||
path := "/containers/create"
|
||||
if r.Name != "" {
|
||||
path += "?name=" + url.QueryEscape(r.Name)
|
||||
}
|
||||
if err := d.call(ctx, http.MethodPost, path, map[string]any{"Image": r.Image, "Cmd": r.Cmd,
|
||||
"Env": r.Env, "Labels": labels, "HostConfig": host}, &made); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if err := d.call(ctx, http.MethodPost, "/containers/"+made.ID+"/start", nil, nil); err != nil {
|
||||
d.Remove(ctx, made.ID)
|
||||
return "", "", err
|
||||
}
|
||||
var seen struct {
|
||||
NetworkSettings struct {
|
||||
Networks map[string]struct{ IPAddress string }
|
||||
}
|
||||
}
|
||||
if err := d.call(ctx, http.MethodGet, "/containers/"+made.ID+"/json", nil, &seen); err != nil {
|
||||
return made.ID, "", err
|
||||
}
|
||||
for _, n := range seen.NetworkSettings.Networks {
|
||||
return made.ID, n.IPAddress, nil
|
||||
}
|
||||
return made.ID, "", nil
|
||||
}
|
||||
|
||||
// Wait waits for a container to end, and answers its exit code and what it said.
|
||||
func (d *Docker) Wait(ctx context.Context, id string) (int, string, error) {
|
||||
var ended struct{ StatusCode int }
|
||||
if err := d.call(ctx, http.MethodPost, "/containers/"+id+"/wait", nil, &ended); err != nil {
|
||||
return -1, "", err
|
||||
}
|
||||
return ended.StatusCode, d.Logs(ctx, id), nil
|
||||
}
|
||||
|
||||
// Logs is what a container said, both streams.
|
||||
func (d *Docker) Logs(ctx context.Context, id string) string {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://docker/containers/"+id+"/logs?stdout=1&stderr=1", nil)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
res, err := d.http.Do(req)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
defer res.Body.Close()
|
||||
var b strings.Builder
|
||||
head := make([]byte, 8)
|
||||
for {
|
||||
if _, err := io.ReadFull(res.Body, head); err != nil {
|
||||
return b.String()
|
||||
}
|
||||
n := binary.BigEndian.Uint32(head[4:])
|
||||
if _, err := io.CopyN(&b, res.Body, int64(n)); err != nil {
|
||||
return b.String()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Remove removes a container, running or not.
|
||||
func (d *Docker) Remove(ctx context.Context, id string) {
|
||||
_ = d.call(ctx, http.MethodDelete, "/containers/"+id+"?force=1", nil, nil)
|
||||
}
|
||||
|
||||
// Exec runs a command in a running container and answers its exit code.
|
||||
func (d *Docker) Exec(ctx context.Context, id string, cmd ...string) (int, error) {
|
||||
var made struct{ ID string }
|
||||
if err := d.call(ctx, http.MethodPost, "/containers/"+id+"/exec", map[string]any{"Cmd": cmd}, &made); err != nil {
|
||||
return -1, err
|
||||
}
|
||||
if err := d.call(ctx, http.MethodPost, "/exec/"+made.ID+"/start", map[string]any{"Detach": true}, nil); err != nil {
|
||||
return -1, err
|
||||
}
|
||||
for i := 0; i < 600; i++ {
|
||||
var seen struct {
|
||||
Running bool
|
||||
ExitCode int
|
||||
}
|
||||
if err := d.call(ctx, http.MethodGet, "/exec/"+made.ID+"/json", nil, &seen); err != nil {
|
||||
return -1, err
|
||||
}
|
||||
if !seen.Running {
|
||||
return seen.ExitCode, nil
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
return -1, fmt.Errorf("%v did not end", cmd)
|
||||
}
|
||||
|
||||
// Build builds an image from a Dockerfile alone, tagged as given, and labelled so a replay that dies is
|
||||
// cleaned up by it.
|
||||
func (d *Docker) Build(ctx context.Context, tag, dockerfile string) error {
|
||||
var archive bytes.Buffer
|
||||
tw := tar.NewWriter(&archive)
|
||||
if err := tw.WriteHeader(&tar.Header{Name: "Dockerfile", Mode: 0o644, Size: int64(len(dockerfile))}); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tw.Write([]byte(dockerfile)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
labels, _ := json.Marshal(map[string]string{d.Label: "1"})
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://docker/build?t="+url.QueryEscape(tag)+
|
||||
"&labels="+url.QueryEscape(string(labels))+"&rm=1", &archive)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-tar")
|
||||
res, err := d.http.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
said, _ := io.ReadAll(res.Body)
|
||||
if res.StatusCode >= 300 || bytes.Contains(said, []byte(`"error"`)) {
|
||||
return fmt.Errorf("building %s: %s %s", tag, res.Status, strings.TrimSpace(string(said)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemoveImage removes an image.
|
||||
func (d *Docker) RemoveImage(ctx context.Context, ref string) {
|
||||
_ = d.call(ctx, http.MethodDelete, "/images/"+url.PathEscape(ref)+"?force=1", nil, nil)
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
module github.com/novox/mesh-lab/replays
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require github.com/nats-io/nats.go v1.54.0
|
||||
|
||||
require (
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,12 @@
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
@@ -0,0 +1,28 @@
|
||||
#!/bin/sh
|
||||
# mesh-check-toolchain: go
|
||||
#
|
||||
# The Go part of the lab's own check (novox/hq issue 302): the replays register and its prover, run by the
|
||||
# build seat in the mesh's Go toolchain, its own container, because the lab's merge-check.sh declares it
|
||||
# (`# mesh-check-also: go replays/merge-check.sh`) — and by hand, from anywhere:
|
||||
# `sh replays/merge-check.sh`.
|
||||
#
|
||||
# 1. formatted and vetted — the register, the replays and the prover compile, tests included;
|
||||
# 2. the tests that need nothing but Go and git: every replay registered whole (TestEveryReplayIsWhole),
|
||||
# and the bed's reading of what a change touches.
|
||||
#
|
||||
# **Said, never passed silently**: the replays themselves raise containers and a bus of the release the mesh
|
||||
# runs; this container holds no container runtime, so they are not run here. They run from the lab's main,
|
||||
# reviewed, in the gate of every core change, and a change to one is proven by `go run ./cmd/prove` before
|
||||
# it merges.
|
||||
set -eu
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
unformatted=$(gofmt -l .)
|
||||
if [ -n "$unformatted" ]; then
|
||||
echo "not gofmt'd:"
|
||||
echo "$unformatted"
|
||||
exit 1
|
||||
fi
|
||||
go vet ./...
|
||||
go test -count=1 -run '^(TestEveryReplayIsWhole|TestTheBedProvesWhatTheChangeTouches)$' .
|
||||
echo "NOT RUN HERE: the replays themselves — they raise containers, and this container holds no container runtime"
|
||||
@@ -0,0 +1,620 @@
|
||||
// Package replays is the lab's replay of every core incident (novox/hq to-be 45 §9, M9): a scripted
|
||||
// replay of what happened, asserting the rule's outcome, run before every merge of a core repository —
|
||||
// and proved, once, to fail on the commit before its fix and to pass on the fix.
|
||||
//
|
||||
// **Where a replay lives.** A replay of one component's logic is a test in that component's repository,
|
||||
// written only with what the component had before its fix, so it can be laid over the older commit
|
||||
// (the controller's replays_test.go). A replay of what the mesh runs rather than what it wrote — the bus
|
||||
// server's release, the resolver the catalogue configures, under both C libraries — lives here, where a
|
||||
// container runtime and a bus of a given release are at hand. Every one is in Register, with the issue,
|
||||
// the fix, and how the prover runs it at a commit.
|
||||
//
|
||||
// **A new core issue resolves with its replay added here, or with a stated reason none is possible**
|
||||
// (hq 00-META/checks/cycle.py holds the issue to it: `replay:` names an entry here, or `replay-none:`
|
||||
// says why).
|
||||
package replays
|
||||
|
||||
// Kind is how a replay is run at a commit.
|
||||
type Kind string
|
||||
|
||||
const (
|
||||
// InRepository is a test in the fixed repository, laid over the commit and run there with go test.
|
||||
InRepository Kind = "test"
|
||||
// Bus is the bus replay here, run against a server of the release the catalogue pinned at the commit.
|
||||
Bus Kind = "bus"
|
||||
// Resolver is the resolver replay here, run with the catalogue's resolver configuration at the commit,
|
||||
// asked by a program under musl and one under glibc.
|
||||
Resolver Kind = "resolver"
|
||||
// Gate is a test of the merge gate itself: before its fix there was no check to fail, so the commit
|
||||
// before is held to fail by not having it.
|
||||
Gate Kind = "gate"
|
||||
// Liveness is the crash-loop replay here: a container that exits at start, raised on the runtime here,
|
||||
// judged by the node-engine at the commit, and the controller at the commit judging its gate from what
|
||||
// the engine said (novox/hq ADR 0240). Two repositories move together: Repository and With.
|
||||
Liveness Kind = "liveness"
|
||||
// Bed is a bed of the lab, raised on its hypervisor: Test is its file under test/integration, run with
|
||||
// the node-engine and its installer built from Repository at the commit, beside the With repositories at
|
||||
// theirs (novox/hq issue 92). A bed that could not reach the step it replays skips, and the prover reads
|
||||
// a skip as an error, never as the fault reproduced.
|
||||
Bed Kind = "bed"
|
||||
)
|
||||
|
||||
// Also is another repository a replay runs at its own commits beside Repository's: before its fix and on
|
||||
// it, as Repository is.
|
||||
type Also struct {
|
||||
Repository string
|
||||
Fix string
|
||||
// Before is the commit the replay runs this repository at before the fix, when that is not Fix's
|
||||
// first parent.
|
||||
Before string
|
||||
}
|
||||
|
||||
// Replay is one incident, replayed.
|
||||
type Replay struct {
|
||||
ID string
|
||||
Issue int
|
||||
// What is the incident in a line, and Asserts the rule's outcome the replay holds.
|
||||
What, Asserts string
|
||||
Kind Kind
|
||||
// Repository is where the fix is, and Fix the fix's commit there: the replay must fail on Fix's first
|
||||
// parent and pass on Fix.
|
||||
Repository string
|
||||
Fix string
|
||||
// Before is the commit the replay must fail on, when that is not Fix's first parent: a fix still on its
|
||||
// branch is proved against the main it branched from. Set to "" once the fix is merged.
|
||||
Before string
|
||||
// Package and Test are where the replay is run: a package of Repository for InRepository and Gate,
|
||||
// of this module otherwise; Files are the files laid over the commit, from where the replay lives.
|
||||
Package string
|
||||
Test string
|
||||
Files []string
|
||||
// Home is where the replay's files live, when that is not the fix: the repository and ref.
|
||||
Home, HomeRef string
|
||||
// With is the other repositories the replay runs at their commits beside Repository's (Liveness).
|
||||
With []Also
|
||||
// Module is the directory of the Go module the replay runs in, within Repository, when that is not its
|
||||
// root: a catalogue module is a Go module of its own. Package is relative to it; Files are not.
|
||||
Module string
|
||||
}
|
||||
|
||||
// Register is every replay, by incident.
|
||||
var Register = []Replay{
|
||||
{ID: "R236", Issue: 236, Kind: Gate, Repository: "mesh-controller", Fix: "feat/merge-gate",
|
||||
Before: "origin/main",
|
||||
What: "a login manager's service passed the catalogue check and was refused whole by the node-engine",
|
||||
Asserts: "a manifest the node-engine refuses fails its pull request, naming the machine and the module",
|
||||
Package: "./cmd/mesh-controller", Test: "TestIssue236", Files: []string{"cmd/mesh-controller/merge_gate_test.go"}},
|
||||
{ID: "R262", Issue: 262, Kind: Resolver, Repository: "mesh-catalog", Fix: "20603b63e67323f2c725b242c0ea87281d4dee93",
|
||||
What: "an Alpine container could not find a machine by its mesh name: NXDOMAIN for IPv6 is final to musl",
|
||||
Asserts: "every machine's name resolves through the catalogue's resolver under musl and under glibc",
|
||||
Package: ".", Test: "TestReplay262"},
|
||||
{ID: "R263", Issue: 263, Kind: InRepository, Repository: "mesh-controller", Fix: "6d620f77c3f3f967be953ff760823c6a51b7b8d3",
|
||||
What: "a consumer's 26-character identity refused the resolver holder's whole declaration",
|
||||
Asserts: "a provider's machine composes whatever its consumers are called",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay263", Files: []string{"cmd/mesh-controller/replays_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "feat/replays"},
|
||||
{ID: "R266", Issue: 266, Kind: Bus, Repository: "mesh-catalog", Fix: "0275c2e",
|
||||
What: "a merge on the bus was never handed to the controller: 2.10 skipped messages on a consumer with several filters",
|
||||
Asserts: "a consumer filtered like the controller's is handed every followed message under the mesh's traffic",
|
||||
Package: ".", Test: "TestReplay266"},
|
||||
{ID: "R273", Issue: 273, Kind: InRepository, Repository: "mesh-controller", Fix: "8bfaf1523ed2dddde80daa0994f1044e59db44b1",
|
||||
What: "a rule for the resolver re-bound a machine's database consumers to empty databases elsewhere",
|
||||
Asserts: "a consumer beside its store stays bound to it while another machine holds the store's seat",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay273", Files: []string{"cmd/mesh-controller/replays_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "feat/replays"},
|
||||
// The controller's incidents of 2026-10-07, each a test laid over the merge before its fix. Home is
|
||||
// the controller's commit that added them; it stays reachable once that pull request merges.
|
||||
{ID: "R296", Issue: 296, Kind: InRepository, Repository: "mesh-controller", Fix: "3808634",
|
||||
What: "a plan's clock restarted at every save: a build queued for minutes read \"building for 1s\", " +
|
||||
"then 4s, then 9s, and was never LATE",
|
||||
Asserts: "a plan's line counts from when it entered its tier, and a plan standing still in its tier is not " +
|
||||
"saved again",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay296", Files: []string{"cmd/mesh-controller/replays_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "7597294ff86383c171b8bec3920b5472fbce00e9"},
|
||||
{ID: "R299", Issue: 299, Kind: InRepository, Repository: "mesh-controller", Fix: "c37d774",
|
||||
What: "a seat's first verbs, all optional, made the self-check say its holder silent on every machine",
|
||||
Asserts: "a holder of a seat whose verbs are all optional is not said silent when it answers nothing",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay299", Files: []string{"cmd/mesh-controller/replays_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "7597294ff86383c171b8bec3920b5472fbce00e9"},
|
||||
{ID: "R300", Issue: 300, Kind: InRepository, Repository: "mesh-controller", Fix: "e7dca6c",
|
||||
What: "a merge that added a module said it changed nothing the mesh holds, and built nothing, so the " +
|
||||
"module was never registered",
|
||||
Asserts: "a merge adding a module asks the build seat for it, at the branch merged into, and opens no plan",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay300", Files: []string{"cmd/mesh-controller/replays_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "7597294ff86383c171b8bec3920b5472fbce00e9"},
|
||||
// A drill counted as a repair (2026-10-07): S15 read two deliberate drills, recorded through hand-act
|
||||
// record for want of a verb, as a cause repaired twice. In a file of its own, since replays_test.go holds
|
||||
// replays of later commits that do not build at the commit before this fix. Home is the controller's
|
||||
// commit that added it; it stays reachable once that pull request merges.
|
||||
{ID: "R292", Issue: 292, Kind: InRepository, Repository: "mesh-controller", Fix: "863ebd4",
|
||||
What: "two drills of ADR 0240, each with the operator's word, recorded through hand-act record --cause drill, " +
|
||||
"raised mesh.hand-acts.drill.healer-wanted",
|
||||
Asserts: "a drill recorded in the hand-act log wants no healer, however it was recorded",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay292", Files: []string{"cmd/mesh-controller/replay292_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "80f9d26e6d4c60ec7d137110b9d84c841072f04e"},
|
||||
// A seat's new verb deadlocked across two repositories (2026-10-07): the controller promising checks
|
||||
// refused the delivery seat's holder that did not serve it, and a holder serving it was refused by the
|
||||
// controller that did not promise it.
|
||||
{ID: "R298", Issue: 298, Kind: InRepository, Repository: "mesh-controller", Fix: "f6aea4b",
|
||||
What: "the delivery seat's new verb checks could land in neither repository first: each side's claim " +
|
||||
"check refused the other's holder",
|
||||
Asserts: "the delivery seat's holder holds it both before and after it serves checks",
|
||||
Package: "./internal/catalogue", Test: "TestReplay298", Files: []string{"internal/catalogue/replays_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "80f9d26e6d4c60ec7d137110b9d84c841072f04e"},
|
||||
// The push a recorded build waits for, counted as a repair (2026-10-07): a test in the controller, in
|
||||
// the fix's own commit, recording two such pushes as the seat's push records them.
|
||||
{ID: "R301", Issue: 301, Kind: InRepository, Repository: "mesh-controller", Fix: "e92a3fe",
|
||||
What: "the operator's pushes of new builds whose upgrade policy is record — the resolver, the network " +
|
||||
"managers, the packet filter — were counted as repairs, and S15 wanted a healer for each cause",
|
||||
Asserts: "a push that moves only recorded builds, recorded as the push records it, wants no healer " +
|
||||
"however often its cause is given",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay301", Files: []string{"cmd/mesh-controller/replays_test.go"}},
|
||||
// The lab's Go replays never compiled before a lab merge (2026-10-07): the build agent ran a repository's
|
||||
// check in one toolchain. Before the fix there was no part to run, so the commit before is held to fail
|
||||
// by not having it.
|
||||
{ID: "R302", Issue: 302, Kind: Gate, Repository: "mesh-controller", Fix: "98ef7ba",
|
||||
What: "mesh-lab's replays register merged with NOT CHECKED HERE: its check ran in the TypeScript " +
|
||||
"toolchain, which holds no Go compiler",
|
||||
Asserts: "a repository in two languages has each declared part of its own check run in that part's " +
|
||||
"toolchain, and the layer passes only when every part does",
|
||||
Package: "./internal/builder", Test: "TestARepositoryInTwoLanguagesIsCheckedInBoth",
|
||||
Files: []string{"internal/builder/check_test.go"}},
|
||||
// A recheck left the old verdict standing (2026-10-07): the forge keeps the newest status of each context
|
||||
// on a head, and a recheck asks of the same head, so the old green let the pull request merge while the
|
||||
// fresh check ran. A test in mesh-delivery, which asks the forge's holder (core) to put the merge check
|
||||
// back to pending; the holder's own half is its TypeScript test.
|
||||
{ID: "R305", Issue: 305, Kind: InRepository, Repository: "mesh-catalog", Fix: "74f912e",
|
||||
What: "a pull request rechecked at 23:04 kept mesh/merge-gate and mesh/repo-check success from 16:59, " +
|
||||
"and the forge would have merged it while its fresh check ran",
|
||||
Asserts: "a recheck puts the head's merge check statuses back to pending before it asks the check, and " +
|
||||
"is refused whole when the forge cannot be told",
|
||||
Module: "modules/mesh-delivery", Package: "./cmd/mesh-delivery", Test: "TestARecheck",
|
||||
Files: []string{"modules/mesh-delivery/cmd/mesh-delivery/recheck_test.go",
|
||||
"modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go"}},
|
||||
// A pull request ran its own copy of the repository's check (2026-10-08): the build agent read
|
||||
// merge-check.sh from the change's head, so a branch cut before the script, or one that deleted or
|
||||
// gutted it, merged with none of its tests run. Written with only what the agent had before the fix, so
|
||||
// it fails on the commit before by answering the warning. The fix is still on its branch: proved
|
||||
// against the main it branched from.
|
||||
{ID: "R310", Issue: 310, Kind: InRepository, Repository: "mesh-controller", Fix: "cb0327d",
|
||||
Before: "1a50d6e",
|
||||
What: "the media catalogue's Lidarr pull request, cut before its repository had a merge-check.sh, was " +
|
||||
"answered no repository check defined as a warning and merged with none of its tests run",
|
||||
Asserts: "where the base branch holds a merge-check.sh, its script judges the change's tree, and a head " +
|
||||
"without one is never answered the warning",
|
||||
Package: "./internal/builder", Test: "TestReplay310", Files: []string{"internal/builder/replay310_test.go"}},
|
||||
// Two order rules ordered one pair both ways (2026-10-08): the controller's member moved the build agent,
|
||||
// which builds the node-engine, and the node-engine goes before the controller, so the group
|
||||
// feat/a-machine-joins-through-the-tunnel was refused as a cycle and merged by hand. A test in the
|
||||
// controller, using only what orderOf had before its fix, laid over the commit before it.
|
||||
{ID: "R309", Issue: 309, Kind: InRepository, Repository: "mesh-controller", Fix: "c5a2edf",
|
||||
What: "a delivery group of the controller, the node-engine and the lab, each ready, was rejected: built by " +
|
||||
"put the controller's member first and engine before controller put the node-engine's first",
|
||||
Asserts: "rules that order one pair both ways are resolved by precedence — built by over engine before " +
|
||||
"controller — and the group is ordered, the controller's member first",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay309", Files: []string{"cmd/mesh-controller/replay309_test.go"}},
|
||||
// An answer larger than one message of the bus (2026-10-08): the controller's conditions and status
|
||||
// outgrew the bus's max_payload once 689 stalled deliveries were open conditions, the client library
|
||||
// refused every reply inside the controller, and every caller timed out while calls said answered.
|
||||
{ID: "R314", Issue: 314, Kind: InRepository, Repository: "mesh-controller", Fix: "175b28e",
|
||||
What: "conditions and status answered three times what one message of the bus carries; the reply was " +
|
||||
"refused inside the controller and every caller waited 30 s and read that it did not answer",
|
||||
Asserts: "an answer larger than one message arrives whole to a caller that pages, and one that does not " +
|
||||
"is told so in a message that fits — never a timeout",
|
||||
Package: "./internal/link", Test: "TestReplay314", Files: []string{"internal/link/replay314_test.go"}},
|
||||
// One module's wait for a new login held every other module's walk (2026-10-08): the walk of the builds
|
||||
// waiting for a gate sent the laptop three moves in one send, the lighting module's account said relogin
|
||||
// needed and its daemon's unit failed in that account's manager, and the gate failed the send at its bound,
|
||||
// put the module back (taking its account group back) and stopped every walk behind it. A test in the
|
||||
// controller, reading the statements as the node-engine's JSON, so it is laid over the commit before.
|
||||
// Fix is the head of the fix: e3b5c22 alone raises the new condition kind without the plain words the
|
||||
// controller's suite requires since hq ADR 0253, which 51d7bbf gives it. Before is the main both stood on.
|
||||
{ID: "R318", Issue: 318, Kind: InRepository, Repository: "mesh-controller", Fix: "51d7bbf", Before: "6df30b6",
|
||||
What: "a send of three modules to the laptop failed its gate after ten minutes because one module's account " +
|
||||
"needed a new login; the module was put back and the walks of the other modules stopped behind it",
|
||||
Asserts: "a wait for a person's new login passes the gate with the wait carried and said as the module's " +
|
||||
"relogin-needed condition; nothing is put back, and the walk goes on to the next machine",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay318", Files: []string{"cmd/mesh-controller/replay318_test.go"}},
|
||||
// An assignment answered as unknown while its module's build ran (2026-10-08): the merge adding `sensors`
|
||||
// asked for its build (issue 300), and `assign g14 sensors` a minute later said "no module of that name",
|
||||
// which read as a module nobody registered. A test in the controller, written with only what it had
|
||||
// before the fix, laid over the commit before. The fix is still on its branch: Fix is its pull request's
|
||||
// head, proved against the main it is rebased on.
|
||||
{ID: "R325", Issue: 325, Kind: InRepository, Repository: "mesh-controller", Fix: "8adb7f1", Before: "fe00fec",
|
||||
What: "a minute after the merge that added sensors asked for its build, assign g14 sensors answered no " +
|
||||
"module of that name, and the same call worked minutes later",
|
||||
Asserts: "an assignment made while its module's build runs says the build, is kept, and is made when the " +
|
||||
"build registers the module",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay325", Files: []string{"cmd/mesh-controller/replay325_test.go"}},
|
||||
// A declared tool check never asked on a live machine (2026-10-08): the queue reached the bus through an
|
||||
// assertion only OverNATS met, and the NATS link the node-engine attaches had no Ask, so every tool check
|
||||
// read "no link to the bus is open" while the link was up. A test in the node-engine, written with only
|
||||
// what it had before the fix; Home is its commit that added it.
|
||||
{ID: "R331", Issue: 331, Kind: InRepository, Repository: "mesh-host", Fix: "824911d",
|
||||
What: "the sensors module's tool check on the laptop read no link to the bus is open for hours while the " +
|
||||
"node-engine's link was up and the same tool answered through the mesh MCP server",
|
||||
Asserts: "a tool check on the NATS link the node-engine attaches reaches that link, and is not refused as no link",
|
||||
Package: "./internal/link", Test: "TestReplay331", Files: []string{"internal/link/replay331_test.go"},
|
||||
Home: "mesh-host", HomeRef: "5684a4579d92"},
|
||||
// Messages a consumer gave up on were kept nowhere (2026-10-06): a media server's consumer gave up on
|
||||
// several downloads after five deliveries each, the controller's condition cleared within minutes, and
|
||||
// EVENTS dropped them a week later; the dead-letter stream design 25 promised did not exist. A test in
|
||||
// the controller, written with only what the link had before the fix, laid over the commit before. Fix is
|
||||
// the merge of its pull request, proved against its first parent.
|
||||
{ID: "R330", Issue: 330, Kind: InRepository, Repository: "mesh-controller", Fix: "ca09a07",
|
||||
What: "a media server's event consumer gave up on several messages after five deliveries each; the condition " +
|
||||
"cleared within minutes and nothing kept which messages they were",
|
||||
Asserts: "a consumer that never acknowledges an event gives it up, and the event is kept on the bus with " +
|
||||
"its consumer, stream, subject, deliveries and when it was given up",
|
||||
Package: "./internal/link", Test: "TestReplay330", Files: []string{"internal/link/replay330_test.go"}},
|
||||
// Every verb opened a bus connection of its own (2026-10-08): the server's connection total rose by 5.2 a
|
||||
// minute while the same 16 stayed open, and three conditions calls in one second added three, each a
|
||||
// process of the controller's binary that dialled, logged in and left. A test in the controller, written
|
||||
// with only what it had before the fix, laid over the commit before. Fix is the merge of its pull
|
||||
// request, proved against its first parent.
|
||||
{ID: "R327", Issue: 327, Kind: InRepository, Repository: "mesh-controller", Fix: "efcdd5d",
|
||||
What: "three calls of conditions in one second added three connections to the bus: each verb ran as a " +
|
||||
"process that dialled it, and hundreds of such connections an hour hid any client reconnecting",
|
||||
Asserts: "a verb that only reads, served by the serving controller, opens no connection of its own",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay327", Files: []string{"cmd/mesh-controller/replay327_test.go"}},
|
||||
// A passed build put back half an hour after its pass (2026-10-08): dunst passed its gate on the laptop,
|
||||
// its send to the rest waited on another walk, that walk sent the laptop a declaration it did not report
|
||||
// on, and the plan read the silence as the passed build never applied: failed at its gate with the pass's
|
||||
// own words, and put back. A test in the controller, written with only what it had before the fix, laid
|
||||
// over the commit before. Fix is the merge of the fix.
|
||||
{ID: "R335", Issue: 335, Kind: InRepository, Repository: "mesh-controller", Fix: "8170fc5",
|
||||
What: "a build that passed its gate on the laptop (healthy 3 times over 2m5s) was marked failed at its " +
|
||||
"gate 28 minutes later, with those words as the reason, and put back",
|
||||
Asserts: "once a build passed its gate, the first machine's later reports neither fail it nor hold the rest, " +
|
||||
"and a walk stopped after the pass keeps it",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay335", Files: []string{"cmd/mesh-controller/replay335_test.go"}},
|
||||
// A send held for the bus's planned step said nothing (2026-10-08): a catalogue merge built a new bus for the
|
||||
// control-node, every send to that machine was refused for 28 minutes for a step only a person starts, and
|
||||
// nothing but the walks' notes said so. A test in the controller, written with only what it had before the
|
||||
// fix, laid over the commit before. Fix is the merge of its pull request (mesh-controller #173), proved
|
||||
// against its first parent.
|
||||
{ID: "R336", Issue: 336, Kind: InRepository, Repository: "mesh-controller", Fix: "b13a0f7",
|
||||
What: "for 28 minutes every send to the control-node was refused for a new bus build waiting for a " +
|
||||
"person's bus upgrade, and no condition said so",
|
||||
Asserts: "the first watchdog tick after a send is refused for the bus's planned step opens a condition for " +
|
||||
"the operator naming what waits and the call that ends it, and it clears once the machine is sent the " +
|
||||
"new bus",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay336", Files: []string{"cmd/mesh-controller/replay336_test.go"}},
|
||||
// Two merges seconds apart left their delivery waiting for a person (2026-10-10): two catalogue merges 18 s
|
||||
// apart were walked by one walk at the later commit, which superseded the first; mesh-delivery tied it to the
|
||||
// delivery the supersession had ended, so nothing gave it its word for 58 minutes. A test in mesh-delivery,
|
||||
// written with only what it had before the fix, as the controller of that night said its walks. The fix is
|
||||
// still on its branch (mesh-catalog #171): proved against the main it branched from.
|
||||
{ID: "R362", Issue: 362, Kind: InRepository, Repository: "mesh-catalog", Fix: "13ffae7",
|
||||
Before: "52f34d3",
|
||||
What: "claude-code and dunst merged 18 s apart; the one walk at the dunst commit was tied to the dunst " +
|
||||
"delivery its supersession had ended, never let go, and claude-code was held for a person after ten minutes",
|
||||
Asserts: "the walk at the later commit is let go once and delivers both merges without a person — dunst " +
|
||||
"delivered, claude-code covered by it — whichever of the controller's two records arrives first",
|
||||
Module: "modules/mesh-delivery", Package: "./cmd/mesh-delivery", Test: "TestReplay362",
|
||||
Files: []string{"modules/mesh-delivery/cmd/mesh-delivery/replay362_test.go",
|
||||
"modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go"}},
|
||||
// A path change read as data loss (2026-10-10): the agent's home on the control-node moved from the
|
||||
// operator's own home to the agent account's (ADR 0266), and data-shrank was raised because the fresh
|
||||
// directory was read against the old one's size. A test in the controller, written with only what it had
|
||||
// before the fix, the incident replayed through the store into D13's findings. Fix is the merge of its
|
||||
// pull request (mesh-controller #204), proved against its first parent.
|
||||
{ID: "R368", Issue: 368, Kind: InRepository, Repository: "mesh-controller", Fix: "798738c",
|
||||
What: "agent-home of claude-code on the control-node was raised as shrunk to 490 B from 94.7 MB when its " +
|
||||
"path moved to the agent account's fresh home; nothing was lost",
|
||||
Asserts: "an item whose path moved raises no data-shrank, and a genuine shrink at its new path is still " +
|
||||
"raised, urgent for what is irreplaceable",
|
||||
Package: "./cmd/mesh-controller", Test: "TestAMovedPathIsNoShrinkAndAShrinkThereStillIs",
|
||||
Files: []string{"cmd/mesh-controller/data_test.go"}},
|
||||
// A quote that could not be said (2026-10-07): a machine's role, a JSON value in single quotes with an
|
||||
// apostrophe inside, was cut at the apostrophe by the controller's `command` verb, and the line was refused
|
||||
// as a bare unclosed quote, naming neither where nor how a quote is written. A test in the controller,
|
||||
// written with only what it had before the fix. Fix is the merge of its pull request (mesh-controller
|
||||
// #205), proved against its first parent.
|
||||
{ID: "R294", Issue: 294, Kind: InRepository, Repository: "mesh-controller", Fix: "690b75f",
|
||||
What: "settings set claude-code '{\"role\":\"the operator's laptop\"}' through `command` was cut at the " +
|
||||
"apostrophe and refused as an unclosed quote, with no position and no way to write a quote",
|
||||
Asserts: "a quote in a value reads back through both shell forms, and a line left with an open quote is " +
|
||||
"refused naming the character and how a quote is written, quoting none of the line",
|
||||
Package: "./cmd/mesh-controller", Test: "TestAQuotedValueCanHoldAQuote",
|
||||
Files: []string{"cmd/mesh-controller/seatverbs_test.go"}},
|
||||
// A mesh directory outside every home was changed by path (2026-10-12): the directory applier set a
|
||||
// directory's mode and owner by calling chmod and chown on its path, which follows a link anywhere in
|
||||
// the path, so an account able to swap a parent for a link between two applies could direct root's
|
||||
// chmod and chown at the link's target. Found by code reading, no known exploit. A test in the
|
||||
// node-engine, written only with what applyDirectory had before its fix, laid over the commit before
|
||||
// (mesh-host #87). Fix is the merge of its pull request.
|
||||
{ID: "R358", Issue: 358, Kind: InRepository, Repository: "mesh-host", Fix: "36704b5471cb7b4e585847a08e4fd7fd64aea357",
|
||||
What: "a mesh directory outside every home was given its mode and owner by path, so a parent swapped " +
|
||||
"for a link between two applies carried root's chmod and chown to the link's target",
|
||||
Asserts: "no apply changes the mode or owner of anything but the directory its declaration names, " +
|
||||
"however a parent of its path is swapped for a link between applies, and the refusal names the link " +
|
||||
"and the account that could have made it",
|
||||
Package: "./internal/apply", Test: "TestAParentSwappedForALinkLeavesItsTargetUntouched",
|
||||
Files: []string{"internal/apply/replay358_test.go"}},
|
||||
// assign said two things that could not both be true (2026-10-04): a workstation already ran a module
|
||||
// contributing node-hotkeys, and the module holding that seat was assigned to it; the answer said in
|
||||
// order that the seat was now held, then that the node was left out of the rest of the mesh for lack
|
||||
// of it, naming the assignment just made as the remedy. `plan` for the node right after resolved. A
|
||||
// test in the controller, written with only what it had before its fix — register, assign's command,
|
||||
// planFor — laid over the commit before. Needs a store: raised by inventory.ForTest against
|
||||
// MESH_TEST_POSTGRES, same as the controller's other store-backed replays. Fix is the merge of its
|
||||
// pull request (mesh-controller #253).
|
||||
{ID: "R237", Issue: 237, Kind: InRepository, Repository: "mesh-controller", Fix: "5facd8b08e245545fb7656dd4af11280d9306a08",
|
||||
What: "assign on a workstation already running a module that contributed node-hotkeys, after " +
|
||||
"assigning the module holding that seat, said the seat was now held and then that the node was " +
|
||||
"left out of the mesh for lack of it, naming the assignment just made as the remedy",
|
||||
Asserts: "every line of assign's answer is derived from one resolution of the node taken after the " +
|
||||
"write committed, and agrees with plan for the node run right after the act",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay237", Files: []string{"cmd/mesh-controller/replay237_test.go"}},
|
||||
// A host without its package manager's configuration refused the declaration that would restore it
|
||||
// (2026-10-04): a declaration that left the package manager's module out removed pacman's configuration
|
||||
// file; from then on pacman did not answer, and the node-engine refused every declaration whole, the
|
||||
// one that named the module again and would have written the file back included. A person restored the
|
||||
// file by hand an hour and a half later. A test in the node-engine, written only with what it had before
|
||||
// its fix, laid over the commit before (mesh-host #90). Fix is the merge of its pull request. The
|
||||
// controller's half of the fix (mesh-controller #263, db7e4869: a machine refused or applied only partly
|
||||
// past one apply interval raises a condition) is held by its own tests there, not by this replay.
|
||||
{ID: "R233", Issue: 233, Kind: InRepository, Repository: "mesh-host", Fix: "a1f9856b3957fa0b11c04294f6367bc6febf5eb0",
|
||||
What: "the package manager's configuration file was removed, pacman stopped answering, and the node-engine " +
|
||||
"refused every declaration whole, the one that would have written the file back included",
|
||||
Asserts: "a declaration that restores the package manager's configuration is applied while pacman does not " +
|
||||
"answer: only what needs pacman fails, with the probe's words, and the file is written and recorded",
|
||||
Package: "./cmd/mesh-host", Test: "TestReplay233",
|
||||
Files: []string{"cmd/mesh-host/replay233_test.go"}},
|
||||
// Genesis pushed to a registry its runtime did not yet trust (2026-09-22): the first real genesis raised the
|
||||
// mesh's registry and pushed the control plane's image to it by the address it was given, and the runtime
|
||||
// refused plain HTTP, because the trust was written by the docker module six steps after the push. The
|
||||
// lab never saw it: its base image trusts the documentation ranges before any bed starts. A bed here,
|
||||
// whose anchor has that trust taken off before the installer runs, raised with the node-engine and its
|
||||
// installer built at the commit; the control plane genesis builds and the catalogue it reads are pinned,
|
||||
// the same on both sides.
|
||||
//
|
||||
// **Pinned to today's genesis, not to the fix's merge** (hq issues 146, 590, 596, 597): at the merge of
|
||||
// the fix (mesh-host #91, dd9ab792) no genesis got past step 6, so neither side reached the push. Fix is
|
||||
// mesh-host main with every genesis fix since; Before is the same main with #91 reverted, kept on the
|
||||
// branch replay/r92-before and never merged.
|
||||
{ID: "R92", Issue: 92, Kind: Bed, Repository: "mesh-host", Fix: "0d11b4fc930d6723f3dc54dc173175f66fb6b3b3",
|
||||
Before: "9e233e1d87569273b4ed353c450b0bd27d21039b",
|
||||
What: "the first real genesis pushed the control plane's image to the registry it had just raised and was " +
|
||||
"refused, server gave HTTP response to HTTPS client: the runtime's trust of the registry was written " +
|
||||
"by the docker module, six steps after the push",
|
||||
Asserts: "on a runtime that trusts no registry but loopback, genesis given --registry <its own address>:5000 " +
|
||||
"pushes the control plane to the registry it raised, and the runtime trusts that registry by the " +
|
||||
"mesh's name too",
|
||||
Test: "test/integration/replay92.test.ts",
|
||||
With: []Also{
|
||||
{Repository: "mesh-controller", Fix: "49102e1645c920e42e453caf0aa0746d2b93ce90", Before: "49102e1645c920e42e453caf0aa0746d2b93ce90"},
|
||||
{Repository: "mesh-catalog", Fix: "f88e3452f8b8b285ed86e1fe93e1a153f71faada", Before: "f88e3452f8b8b285ed86e1fe93e1a153f71faada"},
|
||||
}},
|
||||
// Not a core incident, and the first of its kind: a module's. The crash loop was found by a person
|
||||
// reading the agent server's log for another reason (issue 268); research 032 measured it, and ADR
|
||||
// 0240 makes the node-engine judge it and the gate fail it.
|
||||
{ID: "R-crashloop", Issue: 268, Kind: Liveness, Repository: "mesh-controller",
|
||||
Fix: "9d15f3a",
|
||||
With: []Also{{Repository: "mesh-host", Fix: "03031a4"}},
|
||||
What: "the agent server crash-looped about a hundred times while the mesh read it applied, its tools served " +
|
||||
"and nothing raised: the gate judged a module by what the mesh saw from outside",
|
||||
Asserts: "a container whose program exits at start is said unhealthy by the node-engine, and its build fails " +
|
||||
"its gate on the first machine and is put back there",
|
||||
Package: ".", Test: "TestReplayCrashLoop", Files: []string{"cmd/mesh-controller/replays_test.go"}},
|
||||
// A module's again: the silent web application, which no liveness and no TCP check sees, raised by the
|
||||
// HTTP check its module declares (ADR 0240 Phase B).
|
||||
{ID: "R145", Issue: 145, Kind: Liveness, Repository: "mesh-controller",
|
||||
Fix: "7f25666",
|
||||
With: []Also{{Repository: "mesh-host", Fix: "56e2ebe"}},
|
||||
What: "a web application accepted TCP and answered nothing for eleven hours while the mesh said its machine " +
|
||||
"healthy: its port was open and its program ran",
|
||||
Asserts: "a web server whose application never answers is said unhealthy by its declared HTTP check within two " +
|
||||
"looks, and the controller raises the module's condition on the second",
|
||||
Package: ".", Test: "TestReplaySilentWebApp", Files: []string{"cmd/mesh-controller/replays_test.go"}},
|
||||
// A merge check judged the head alone (2026-10-09): node-engine #66 was checked green at its head after #64/#65
|
||||
// had moved the trunk, both adding a field under the JSON key `declared`; git merged the two edits cleanly and the
|
||||
// trunk no longer compiled. A test in the controller's builder, written with only what it had before the fix, run
|
||||
// with a container runtime that runs each container's command on the machine. The fix is still on its branch:
|
||||
// Fix is its pull request's head, proved against the main it branched from (novox/hq ADR 0323).
|
||||
{ID: "R341", Issue: 341, Kind: InRepository, Repository: "mesh-controller", Fix: "83a9cf6", Before: "3cfc121",
|
||||
What: "node-engine #66 passed both statuses at its head and merged; merged into the trunk #65 had moved, it did " +
|
||||
"not compile: Declared redeclared",
|
||||
Asserts: "a check judges the head merged into the trunk's head, and fails a head whose merge does not compile, " +
|
||||
"saying so",
|
||||
Package: "./internal/builder", Test: "TestReplay341", Files: []string{"internal/builder/replay341_test.go"}},
|
||||
// The mesh made a value for a secret only an outside party issues (2026-10-10): nobody had given the mounts
|
||||
// module's smb-credentials, the send made a random value for it and placed it, and the module read it as a
|
||||
// credential given and wrong (ADR 0228: such a secret is never made). The fix's own test reads what the fix added,
|
||||
// so the replay is a test of its own in the controller, written with only what it had before the fix — register,
|
||||
// assign, planFor, declarationFor, the inventory's secret reads — laid over the commit before. Needs a store:
|
||||
// MESH_TEST_POSTGRES, as R237. Fix is the merge of its pull request (mesh-controller #262); Home is the commit that
|
||||
// added the replay (mesh-controller #274), reachable once that pull request merges.
|
||||
{ID: "R401", Issue: 401, Kind: InRepository, Repository: "mesh-controller", Fix: "1e28342edec170d59570e8127ddead7b19a70c20",
|
||||
What: "the send made a random value for an own secret issued outside the mesh that nobody gave, and placed it: " +
|
||||
"the module read it as a credential given and wrong",
|
||||
Asserts: "the send makes no value for an own secret issued outside the mesh that nobody gave and places none, " +
|
||||
"places no value the mesh made for it before, and places a value once a person gives one",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay401", Files: []string{"cmd/mesh-controller/replay401_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "f3c15e00af5004d053bc424fe01c7f9a24656d59"},
|
||||
// A plan's id was pasted into its query (2026-10-12): an id given to the plans verb holding a quote answered
|
||||
// another plan, or failed as a syntax error. The fix's own test is a file of its own using only what the
|
||||
// inventory had before (ForTest, SavePlan, PlanByID, RecentPlans), so it is laid over the commit before. Needs a
|
||||
// store: MESH_TEST_POSTGRES. Fix is the merge of its pull request (mesh-controller #268).
|
||||
{ID: "R561", Issue: 561, Kind: InRepository, Repository: "mesh-controller", Fix: "f8daec01d3f085ad30aa7fb584055d8d451c79dc",
|
||||
What: "a plan's id was pasted into the SQL that reads it: an id holding a quote answered another plan, or " +
|
||||
"failed as a syntax error",
|
||||
Asserts: "an id holding a quote is read as a plain id: no plan answers it but one of that very id, and " +
|
||||
"RecentPlans takes its limit as a parameter",
|
||||
Package: "./internal/inventory", Test: "TestAPlanIdHoldingAQuoteIsReadAsAPlainId",
|
||||
Files: []string{"internal/inventory/plan_id_test.go"}},
|
||||
// A plan sent the control node a controller older than the one running there (2026-10-02): two merges a minute
|
||||
// apart made two plans; the controller's plan rolled the newer controller, which widened its seat's row, and the
|
||||
// node-engine's plan then sent the build before it, whose binary refused the row. The fix's own TestReplay201
|
||||
// reads what the fix added, so the replay is a test of its own in the controller, written with only what it had
|
||||
// before — the inventory's build, gate, assignment and send records, keepingRecorded, planFor, declarationFor —
|
||||
// laid over the commit before. Needs a store: MESH_TEST_POSTGRES. Fix is the merge of its pull request
|
||||
// (mesh-controller #273); Home is the commit that added the replay (mesh-controller #275), reachable once that
|
||||
// pull request merges.
|
||||
{ID: "R201", Issue: 201, Kind: InRepository, Repository: "mesh-controller", Fix: "84deb530ee5afe2ffdf4d2cc65cdc355b12f8ccb",
|
||||
What: "a plan sent the control node a controller older than the one running there, whose seat row the newer " +
|
||||
"one had widened: the older binary refused the row and the mesh went quiet",
|
||||
Asserts: "a send that is no person's push carries the controller at the build its machine was last sent, not " +
|
||||
"an older record; a person's push carries the record",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay201ThePlansSend",
|
||||
Files: []string{"cmd/mesh-controller/replay201_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "45a1783ba9a43fd466a518ee85673c701223b517"},
|
||||
// A requirement could not be optional (issue 166): every soft dependency had become a hard requirement, so a
|
||||
// manifest could not say grafana runs without a data source — a mesh without influxdb could not register it,
|
||||
// and with influxdb only in the catalogue, assigning grafana installed influxdb for it. A test of its own in
|
||||
// the controller, written with only what it had before — ParseManifest, Resolve, the resolve tests' helpers —
|
||||
// laid over the commit before. Fix is the merge of its pull request (mesh-controller #281); Home is the commit
|
||||
// that added the replay (mesh-controller #287), reachable once that pull request merges.
|
||||
{ID: "R166", Issue: 166, Kind: InRepository, Repository: "mesh-controller", Fix: "4473bc7d7315bb462dcce7d6a8927bff37ecdc59",
|
||||
What: "a manifest could not say a requirement is optional: grafana, which runs without a data source, could " +
|
||||
"not be assigned on a mesh without influxdb, and influxdb only in the catalogue was installed for it",
|
||||
Asserts: "a requirement marked optional parses; with no provider running, assigning its consumer is not refused " +
|
||||
"and installs nothing for it; with a provider running elsewhere it binds",
|
||||
Package: "./internal/catalogue", Test: "TestReplay166", Files: []string{"internal/catalogue/replay166_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "b1baa8d8baf0289b76e8082b3b95d146af6a973e"},
|
||||
// A folder could not be owned by the account its module runs as (2026-10-06): the media managers' recycle-bin
|
||||
// folders were declared owned by a literal user and group, and on a machine whose puid and pgid named another
|
||||
// account the apps refused them as not writable. The fix's own tests read what the fix added, so the replay is
|
||||
// a test of its own in the controller, written with only what it had before — Compose, a setting layer, the
|
||||
// adoption tests' anchor — laid over the commit before. Fix is the merge of its pull request (mesh-controller
|
||||
// #260); Home is the commit that added the replay, reachable once its pull request merges.
|
||||
{ID: "R279", Issue: 279, Kind: InRepository, Repository: "mesh-controller", Fix: "9eddf36503f2a9af7ac22f6dffd4ab4462592b27",
|
||||
What: "the TV and film managers refused their recycle-bin folders as not writable: a declared directory's owner " +
|
||||
"was literal, and could not follow the puid and pgid the module runs as on that machine",
|
||||
Asserts: "a directory owned by ${setting:puid}:${setting:pgid} reaches the node-engine owned by the numbers the " +
|
||||
"machine's settings give",
|
||||
Package: "./internal/catalogue", Test: "TestReplay279", Files: []string{"internal/catalogue/replay279_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "3bdab29a5a0ed0f15cbfdb4738a321f968ac9467"},
|
||||
// A seat did not name the paths its holder owns (2026-10-10): who owns /etc/hosts was enforced only when two
|
||||
// modules on one machine declared it, so a module not claiming node-hostname could declare it, and a holder of
|
||||
// node-hostname could declare none of its paths. A test of its own in the controller, written with only what it
|
||||
// had before — CatalogueProblems, SeatNamed, CanHold — laid over the commit before. Fix is the merge of its pull
|
||||
// request (mesh-controller #267); Home as R279.
|
||||
{ID: "R412", Issue: 412, Kind: InRepository, Repository: "mesh-controller", Fix: "4dbd4066ca54681bd92169502379ec42ff7599b6",
|
||||
What: "a module not claiming node-hostname could declare /etc/hosts, and a module could claim node-hostname " +
|
||||
"declaring neither /etc/hosts nor /etc/hostname: nothing refused either",
|
||||
Asserts: "a module declaring a seat's path without claiming the seat is refused naming the seat, and a claim " +
|
||||
"whose module does not declare the seat's paths is refused naming each",
|
||||
Package: "./internal/catalogue", Test: "TestReplay412", Files: []string{"internal/catalogue/replay412_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "3bdab29a5a0ed0f15cbfdb4738a321f968ac9467"},
|
||||
// A verb set what every agent session obeys (2026-10-09): the agent's module keeps the managed settings and tool
|
||||
// servers every Claude Code session on a node obeys in a mergeable file, and the settings verb took any key for it
|
||||
// from any caller, so an agent could plant a SessionStart hook in the operator's sessions on every node; an
|
||||
// identity provider's empty runtime configuration took a url of the caller's the same way. The fix's own tests
|
||||
// read what the fix added, so the replay is a test of its own in the controller, written with only what it had
|
||||
// before — a mesh, a registered manifest, an assignment, the settings verb run as a verb and as the terminal runs
|
||||
// it, the inventory's layer — laid over the commit before. Needs a store: MESH_TEST_POSTGRES. Fix is the merge of
|
||||
// its pull request (mesh-controller #172); Home is the commit that added the replay, reachable once its pull
|
||||
// request merges.
|
||||
{ID: "R340", Issue: 340, Kind: InRepository, Repository: "mesh-controller", Fix: "4d60628f37530c23c072011a697e6014de14b3f6",
|
||||
What: "the settings verb took a SessionStart hook in the agent module's managed settings, a tool server of the " +
|
||||
"caller's, and a url for an identity provider's empty runtime configuration, from any caller of the mesh's verbs",
|
||||
Asserts: "through a verb, any change to the layer of a module with a mergeable file not marked trusted: false is " +
|
||||
"refused as the controller's terminal's and leaves the layer as it was; at the terminal it is taken",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay340", Files: []string{"cmd/mesh-controller/replay340_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "e44c726a38125ade30a6d70cee9873a4e2ea6ac2"},
|
||||
// A merge acted on late superseded the merge after it (2026-10-09): two catalogue merges were acted on in reverse
|
||||
// order. The later one, a security fix to the forge's module, was planned first; the earlier one, handed over late,
|
||||
// was planned after it, superseded its plan at tier 0 and planned the fixed module again at the commit without the
|
||||
// fix, because a plan's age was read from when it was made, never from the branch's order. The fix (mesh-controller
|
||||
// #179) ordered a branch's plans by their merge times and planned a late merge at the newer commit; ADR 0276
|
||||
// (mesh-controller #197) later replaced that with batches, and the replay holds the outcome under both. A test of its
|
||||
// own in the controller, written with only what it had before — a mesh, registered modules, following.SourceMoved,
|
||||
// the plans the inventory keeps — laid over the commit before. Needs a store: MESH_TEST_POSTGRES. Fix is the merge of
|
||||
// its pull request; Home is the commit that added the replay, reachable once its pull request merges.
|
||||
{ID: "R349", Issue: 349, Kind: InRepository, Repository: "mesh-controller", Fix: "9ca7952d5e0a5ed1e23bdcf5aea36fe967e62d14",
|
||||
What: "a merge handed over late superseded the plan of the merge made after it, and planned the module that " +
|
||||
"merge fixed again at the earlier commit, without the fix",
|
||||
Asserts: "with the later merge acted on first and the earlier one after it, every plan left standing that plans " +
|
||||
"the fixed module plans it at the later merge's commit, and one does",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay349", Files: []string{"cmd/mesh-controller/replay349_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "5740874ce3b99b47c27403f33e6c6d2a334ad3d7"},
|
||||
// A resolver that stopped answering ran on unnoticed, and builds sent after it were blamed for it (2026-10-09):
|
||||
// the control node's resolver refused every question on its private address from 10:57:57 UTC and the machine's
|
||||
// network condition was raised at 10:58:45. The controller and the node-engine sent at 10:59:34 failed their gate
|
||||
// with "raised since it was sent", because the restarted engine's first, undecided statement cleared the condition
|
||||
// and its next look raised it anew after the send. A test of its own in the controller, written with only what it
|
||||
// had before — a mesh, stateHealth, the keeper's open conditions, aboutTheMachine — laid over the commit before.
|
||||
// Needs a store: MESH_TEST_POSTGRES. Fix is the merge of its pull request (mesh-controller #179); Home is the
|
||||
// commit that added the replay, reachable once its pull request merges. The resolver's own half — it binds each
|
||||
// address once and its watch asks it a real question and restarts it (mesh-catalog #161, 105cc7c0) — has no
|
||||
// replay here: the module had no Go before that fix, and the binding's loss was never reproduced past the burst
|
||||
// that caused it (hq issue 348's diagnosis); dnsmasq's own tests and the controller's manifest test hold it.
|
||||
{ID: "R348", Issue: 348, Kind: InRepository, Repository: "mesh-controller", Fix: "9ca7952d5e0a5ed1e23bdcf5aea36fe967e62d14",
|
||||
What: "a network fault raised before a send was cleared by a restarted node-engine's undecided statement and " +
|
||||
"raised again after it, and the gate put back the builds sent after the fault began",
|
||||
Asserts: "a build sent while the machine's network condition is open, which flaps after the send, is not held " +
|
||||
"for it, and an undecided statement does not clear the condition",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay348AFaultFromBeforeTheSendFailsNoGate",
|
||||
Files: []string{"cmd/mesh-controller/replay348_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "6eea92c323f4862d4162f9d977c8783d8a62efc3"},
|
||||
// A credential only a person can give was made anyway (found provisioning the media machine, re-checked
|
||||
// 2026-10-08): a Servarr app's API key, an indexer's key and a downloader's password are held by the provider's
|
||||
// software, and for each consumer of them the send made a random pair credential, sealed it to both ends and
|
||||
// placed it; the provider refused it, and a consumer retrying it could be banned (ADR 0337: such a credential is
|
||||
// never made). A test of its own in the controller, written with only what it had before the fix — register,
|
||||
// assign, planFor, declarationFor, AcceptSecretForPair, SecretsOf — laid over the commit before. Needs a store:
|
||||
// MESH_TEST_POSTGRES. Fix is the merge of its pull request (mesh-controller #298), which also added the replay.
|
||||
{ID: "R164", Issue: 164, Kind: InRepository, Repository: "mesh-controller", Fix: "584aa2d7d7a5a68b5087970feb93d0b1f1d4f503",
|
||||
What: "the send made a random pair credential for a provision whose credential only a person can give, and " +
|
||||
"placed it: the provider refused it",
|
||||
Asserts: "for a provision whose offer says its credential is issued outside the mesh, the send makes no value " +
|
||||
"and places none, the consumer's machine still composes, and a value a person gave for the pair is placed",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay164", Files: []string{"cmd/mesh-controller/replay164_test.go"}},
|
||||
// A README-only merge made a new digest of an unchanged Go program (2026-10-13): a module built from its
|
||||
// repository's root is compiled with the clone's .git in view, and Go stamped the commit into the binary — the
|
||||
// node-engine on the laptop carried vcs.revision=7a41e120… — so a rebuild of an unchanged build source was a new
|
||||
// program (novox/hq ADR 0267 rule 6). The builder run whole, Build to the packed bundle, at two commits differing
|
||||
// only in README.md, with the toolchain's command run on this machine; written with only what the builder had
|
||||
// before. Fix is the merge of its pull request (mesh-controller #292); Home is the commit that added the replay,
|
||||
// reachable once its pull request merges.
|
||||
{ID: "R363", Issue: 363, Kind: InRepository, Repository: "mesh-controller", Fix: "cd6757f1e3c818bc6043178bfb4edb42b8f3c63d",
|
||||
What: "two builds of a Go module built from its repository's root, at commits differing only in its README, " +
|
||||
"made two digests: the commit was stamped into the binary",
|
||||
Asserts: "two builds of one Go module whose sources differ only outside its build source give one digest",
|
||||
Package: "./internal/builder", Test: "TestReplay363", Files: []string{"internal/builder/replay363_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "94c928b1f5af099c328eb6950550b6380df24b31"},
|
||||
// A module's name was taken over by another repository (2026-10-04): the photo app's module was registered from
|
||||
// the app's own repository, and rebuilding it "to main" built the catalogue's stub of the same name instead. The
|
||||
// controller registered the outcome, recorded the catalogue as the module's source, and the next push replaced the
|
||||
// app with the stub: two sites' containers and five routes went. The fix (mesh-controller #174, hq ADR 0266 §7)
|
||||
// registers an outcome only from the repository the module is registered from, unless asked at the controller's
|
||||
// terminal. A test of its own in the controller, written with only what it had before — a mesh, RegisterModule,
|
||||
// SourceOf, a kept build request, takeIn — laid over the commit before. Needs a store: MESH_TEST_POSTGRES. Fix is
|
||||
// the merge of its pull request; Home is the commit that added the replay, reachable once its pull request merges.
|
||||
{ID: "R239", Issue: 239, Kind: InRepository, Repository: "mesh-controller", Fix: "eb7a15808641f2075d9fd1660fa0e949b9cd7230",
|
||||
What: "a build of a module's name from another repository was registered over the module's own, and the next " +
|
||||
"push replaced the photo app with the catalogue's stub",
|
||||
Asserts: "a build asked through a verb, of a module's name from a repository other than the one it is registered " +
|
||||
"from, is refused naming both sources, and the module keeps its source and version",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay239AModuleNameIsNotTakenOverByAnotherRepository",
|
||||
Files: []string{"cmd/mesh-controller/replay239_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "95556b8def3f3f21e3d037617d73f97606e6ea1f"},
|
||||
// A new SDK rebuilt neither its toolchain nor the bundles compiled in it (2026-10-13): mesh-sdk 0.1.14 planned
|
||||
// "builds mesh-sdk, sent nowhere". A built manifest is registered without its `build`, so mesh-tools' `build.on`
|
||||
// naming the SDK's package and every bundle's toolchain were edges only before a module's first build — the
|
||||
// fixes of issues 211 and 212 lost once modules were built. The fix (mesh-controller #296) reads them from what is
|
||||
// kept: the package's name@version a build stood on, and the bundles' languages. A test of its own in the
|
||||
// controller, written with only what it had before — RegisterModule, RecordBuild, theGraph, pullScope, reasoned
|
||||
// as `delivery-plan` reasons — laid over main, where it plans [[mesh-sdk]]. Needs a store: MESH_TEST_POSTGRES.
|
||||
// Fix is the merge of its pull request.
|
||||
{ID: "R614", Issue: 614, Kind: InRepository, Repository: "mesh-controller",
|
||||
Fix: "404b4409e70ed7365bb8a53862e4b197deb0bdc6",
|
||||
What: "a new SDK was planned alone: neither the toolchain built on its package nor the TypeScript bundles " +
|
||||
"compiled in that toolchain were rebuilt, so the SDK's fix reached no machine",
|
||||
Asserts: "a change to the SDK plans the SDK, then the toolchain, then every bundle compiled in it, from modules " +
|
||||
"registered as built and builds recorded with what they stood on",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay614ANewSDKPlansTheToolchainAndThenItsBundles",
|
||||
Files: []string{"cmd/mesh-controller/replay614_test.go"}},
|
||||
}
|
||||
|
||||
// Find is the replay of that id, or of that issue.
|
||||
func Find(id string) (Replay, bool) {
|
||||
for _, r := range Register {
|
||||
if r.ID == id || r.ID == "R"+id {
|
||||
return r, true
|
||||
}
|
||||
}
|
||||
return Replay{}, false
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package replays
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Every replay says what happened, what it asserts, where its fix is, and how it is run.
|
||||
func TestEveryReplayIsWhole(t *testing.T) {
|
||||
seen := map[string]bool{}
|
||||
for _, r := range Register {
|
||||
if seen[r.ID] {
|
||||
t.Errorf("%s is registered twice", r.ID)
|
||||
}
|
||||
seen[r.ID] = true
|
||||
if r.Issue == 0 || r.What == "" || r.Asserts == "" || r.Repository == "" || r.Fix == "" || r.Test == "" {
|
||||
t.Errorf("%s does not say its incident, its outcome, its fix and its test: %+v", r.ID, r)
|
||||
}
|
||||
// By its issue, or — for a replay that is not one core incident's — by a name of its own.
|
||||
if !regexp.MustCompile(`^R(\d+|-[a-z][a-z0-9-]*)$`).MatchString(r.ID) {
|
||||
t.Errorf("%q is not a replay's id", r.ID)
|
||||
}
|
||||
switch r.Kind {
|
||||
case InRepository, Gate:
|
||||
if len(r.Files) == 0 || r.Package == "" {
|
||||
t.Errorf("%s runs in its repository and lays no files over the commit", r.ID)
|
||||
}
|
||||
case Liveness:
|
||||
if len(r.Files) == 0 || len(r.With) == 0 {
|
||||
t.Errorf("%s runs two repositories and lays the controller's half over its commit: %+v", r.ID, r)
|
||||
}
|
||||
case Bed:
|
||||
if !regexp.MustCompile(`^test/integration/[a-z0-9-]+\.test\.ts$`).MatchString(r.Test) {
|
||||
t.Errorf("%s is a bed and names no bed of the lab to raise: %q", r.ID, r.Test)
|
||||
}
|
||||
if !slices.ContainsFunc(r.With, func(a Also) bool { return a.Repository == "mesh-controller" }) ||
|
||||
!slices.ContainsFunc(r.With, func(a Also) bool { return a.Repository == "mesh-catalog" }) {
|
||||
t.Errorf("%s is a bed and does not pin the controller and the catalogue it raises: %+v", r.ID, r)
|
||||
}
|
||||
case Bus, Resolver:
|
||||
default:
|
||||
t.Errorf("%s is of no kind the prover runs: %q", r.ID, r.Kind)
|
||||
}
|
||||
}
|
||||
for _, issue := range []string{"236", "262", "263", "266"} {
|
||||
if _, ok := Find(issue); !ok {
|
||||
t.Errorf("to-be 45 Phase 5 is done when the replay of %s fails before its fix and passes after; it has none", issue)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
package replays
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"text/template"
|
||||
"time"
|
||||
)
|
||||
|
||||
// **R262 — every machine's name resolves under musl and under glibc** (novox/hq issue 262; to-be 45 §9,
|
||||
// "the resolver module's tests run under musl and glibc").
|
||||
//
|
||||
// After the mesh moved to one resolver, an Alpine container crash-looped on `getaddrinfo ENOTFOUND` for
|
||||
// a machine's mesh name that `getent hosts` found: the resolver answered the name only through a
|
||||
// wildcard, so asked for its IPv6 address it said there is no such name, and musl — which asks for both
|
||||
// and takes NXDOMAIN as final — gave up; glibc did not. The replay renders the machine list the
|
||||
// catalogue's resolver module reads (its `node-zones` fact, from the manifest at MESH_REPLAY_CATALOGUE),
|
||||
// runs the resolver with it, and asks for each machine by `getaddrinfo` from a program under each C
|
||||
// library, on the runtime's default bridge as the incident's container was. Needs a container runtime
|
||||
// (its socket); its images come from the public registry.
|
||||
var libcs = map[string]string{"musl": "python:3.13-alpine", "glibc": "python:3.13-slim"}
|
||||
|
||||
// resolverImage is where the resolver runs: the distribution's own dnsmasq, installed as the module
|
||||
// installs it on a machine.
|
||||
const resolverImage = "alpine:3.20"
|
||||
|
||||
func TestReplay262EveryMachineResolvesUnderMuslAndGlibc(t *testing.T) {
|
||||
catalogue := os.Getenv("MESH_REPLAY_CATALOGUE")
|
||||
if catalogue == "" {
|
||||
catalogue = filepath.Join("..", "..", "mesh-catalog")
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(catalogue, "modules", "dnsmasq", "module.json"))
|
||||
if err != nil {
|
||||
t.Skipf("no catalogue to read the resolver's configuration from (%v); MESH_REPLAY_CATALOGUE names one", err)
|
||||
}
|
||||
docker, ok := DockerFromEnv()
|
||||
if !ok {
|
||||
t.Skip("no container runtime: the resolver and the two C libraries run in containers")
|
||||
}
|
||||
var manifest struct {
|
||||
Facts map[string]struct {
|
||||
Template string `json:"template"`
|
||||
} `json:"facts"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &manifest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
zones, ok := manifest.Facts["node-zones"]
|
||||
if !ok {
|
||||
t.Fatal("the resolver module renders no machine list (`facts.node-zones`)")
|
||||
}
|
||||
type entry struct{ Name, FQDN, Address, Account string }
|
||||
machines := []entry{{Name: "anchor", FQDN: "anchor.internal", Address: "10.88.0.1"},
|
||||
{Name: "homeserver", FQDN: "homeserver.internal", Address: "10.88.0.2"}}
|
||||
var rendered bytes.Buffer
|
||||
tpl, err := template.New("node-zones").Parse(zones.Template)
|
||||
if err == nil {
|
||||
err = tpl.Execute(&rendered, map[string]any{"Node": "anchor", "Suffix": "internal", "Names": machines,
|
||||
"Machines": machines, "Zones": nil, "Holders": map[string][]entry{}})
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("the machine list does not render: %v", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Minute)
|
||||
defer cancel()
|
||||
for _, image := range append([]string{resolverImage}, libcs["musl"], libcs["glibc"]) {
|
||||
if err := docker.Pull(ctx, image); err != nil {
|
||||
t.Fatalf("cannot fetch %s: %v", image, err)
|
||||
}
|
||||
}
|
||||
// The resolver: nothing but the machine list, answering on its address on the bridge.
|
||||
conf := base64.StdEncoding.EncodeToString(rendered.Bytes())
|
||||
script := fmt.Sprintf(`apk add -q --no-cache dnsmasq >/dev/null && echo %s | base64 -d > /etc/nodes.conf && `+
|
||||
`exec dnsmasq -k --no-resolv --no-hosts --log-queries --log-facility=- --conf-file=/etc/nodes.conf`, conf)
|
||||
resolver, address, err := docker.Start(ctx, Run{Image: resolverImage, Cmd: []string{"sh", "-c", script}, Network: "bridge"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer docker.Remove(context.Background(), resolver)
|
||||
if address == "" {
|
||||
t.Fatal("the resolver has no address on the bridge")
|
||||
}
|
||||
|
||||
// Each machine, asked by getaddrinfo as a program does — both families, as musl asks — a few times
|
||||
// while the resolver comes up.
|
||||
ask := fmt.Sprintf(`
|
||||
import socket, sys, time
|
||||
names = %q.split()
|
||||
for attempt in range(30):
|
||||
failed = []
|
||||
for n in names:
|
||||
try:
|
||||
socket.getaddrinfo(n, 80)
|
||||
except socket.gaierror as e:
|
||||
failed.append("%%s: %%s" %% (n, e))
|
||||
if not failed:
|
||||
print("resolved", " ".join(names)); sys.exit(0)
|
||||
time.sleep(1)
|
||||
print("\n".join(failed)); sys.exit(1)
|
||||
`, machines[0].FQDN+" "+machines[1].FQDN)
|
||||
for libc, image := range libcs {
|
||||
id, _, err := docker.Start(ctx, Run{Image: image, Cmd: []string{"python3", "-c", ask}, Network: "bridge",
|
||||
DNS: []string{address}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, said, err := docker.Wait(ctx, id)
|
||||
docker.Remove(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if code != 0 {
|
||||
t.Errorf("under %s (%s) a machine's mesh name does not resolve through the catalogue's resolver — "+
|
||||
"issue 262:\n%s\nthe resolver said:\n%s", libc, image, strings.TrimSpace(said), tail(docker.Logs(ctx, resolver), 12))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func tail(s string, n int) string {
|
||||
lines := strings.Split(strings.TrimSpace(s), "\n")
|
||||
if len(lines) > n {
|
||||
lines = lines[len(lines)-n:]
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package replays
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// **R145 — a web application that accepts TCP and answers nothing is raised within two looks** (novox/hq
|
||||
// ADR 0240 Phase B, "how it is checked", rule 4; issue 145).
|
||||
//
|
||||
// For eleven hours a web application's port was open and its program ran while every request hung, and the
|
||||
// mesh said its machine was healthy; a person found it. Liveness cannot see it and a TCP check cannot either.
|
||||
// The replay is the whole chain, each half at its commit:
|
||||
//
|
||||
// 1. a web server whose application never answers is raised here, as the node-engine raises a module's;
|
||||
// 2. the node-engine at MESH_REPLAY_HOST looks at it with the HTTP check its module declares, two looks,
|
||||
// and writes what it states (its TestReplaySilentWebAppIsSaidUnhealthy) — an engine older than the
|
||||
// declared check states it alive and nothing more;
|
||||
// 3. the controller at MESH_REPLAY_CONTROLLER hears that statement on two looks in a row and raises the
|
||||
// module's condition on the second (its TestReplaySilentWebAppIsRaisedWithinTwoLooks).
|
||||
//
|
||||
// Needs a container runtime, Go, and MESH_TEST_POSTGRES for the controller's store; the container is removed
|
||||
// after, whatever happened.
|
||||
func TestReplaySilentWebApp(t *testing.T) {
|
||||
host := orDefault(os.Getenv("MESH_REPLAY_HOST"), filepath.Join("..", "..", "mesh-host"))
|
||||
controller := orDefault(os.Getenv("MESH_REPLAY_CONTROLLER"), filepath.Join("..", "..", "mesh-controller"))
|
||||
for _, dir := range []string{host, controller} {
|
||||
if _, err := os.Stat(filepath.Join(dir, "go.mod")); err != nil {
|
||||
t.Skipf("no checkout at %s (MESH_REPLAY_HOST and MESH_REPLAY_CONTROLLER name them)", dir)
|
||||
}
|
||||
}
|
||||
if os.Getenv("MESH_TEST_POSTGRES") == "" {
|
||||
t.Skip("no MESH_TEST_POSTGRES: the controller's half raises its condition in a store")
|
||||
}
|
||||
docker, ok := DockerFromEnv()
|
||||
if !ok {
|
||||
t.Skip("no container runtime: the web application is a container")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 10*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
// 1. The silent web application (mesh-host internal/liveness SilentWebImage, SilentWebProgram).
|
||||
const image = "busybox:1.36"
|
||||
if err := docker.Pull(ctx, image); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
name := fmt.Sprintf("mesh-replay-silent-web-%d", time.Now().UnixNano())
|
||||
id, address, err := docker.Start(ctx, Run{Image: image, Name: name, Restart: "unless-stopped",
|
||||
Labels: map[string]string{"mesh-host.id": "app.server"},
|
||||
Cmd: []string{"sh", "-c", "mkdir -p /www/cgi-bin && printf '#!/bin/sh\\nsleep 3600\\n' > /www/cgi-bin/app && " +
|
||||
"chmod +x /www/cgi-bin/app && exec httpd -f -p 8080 -h /www"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer docker.Remove(context.Background(), id)
|
||||
|
||||
// 2. The node-engine at its commit looks at it, and says what it states.
|
||||
statement := filepath.Join(t.TempDir(), "statement.json")
|
||||
engine := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^TestReplaySilentWebAppIsSaidUnhealthy$",
|
||||
"./internal/liveness/")
|
||||
engine.Dir = host
|
||||
engine.Env = append(os.Environ(), "MESH_REPLAY_CONTAINER="+name, "MESH_REPLAY_ADDRESS="+address,
|
||||
"MESH_REPLAY_STATEMENT="+statement, "GOFLAGS=")
|
||||
out, err := engine.CombinedOutput()
|
||||
switch {
|
||||
case err == nil && !strings.Contains(string(out), "no tests to run"):
|
||||
t.Logf("the node-engine said: %s", firstLineOf(statementOf(statement)))
|
||||
case judgesNothing(string(out)) || strings.Contains(string(out), "no tests to run"):
|
||||
// An engine older than the declared check: it states the application alive, and nothing more.
|
||||
t.Fatalf("the node-engine at this commit does not look at a declared HTTP check")
|
||||
default:
|
||||
t.Fatalf("the node-engine did not say the silent web application unhealthy: %v\n%s", err, lastOf(string(out), 12))
|
||||
}
|
||||
|
||||
// 3. The controller at its commit raises the module's condition from what the engine said.
|
||||
raise := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^TestReplaySilentWebAppIsRaisedWithinTwoLooks$",
|
||||
"./cmd/mesh-controller/")
|
||||
raise.Dir = controller
|
||||
raise.Env = append(os.Environ(), "MESH_REPLAY_STATEMENT="+statement, "GOFLAGS=-mod=vendor", "GOPROXY=off")
|
||||
out, err = raise.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("the controller did not raise the silent web application within two looks: %v\n%s", err, lastOf(string(out), 12))
|
||||
}
|
||||
if strings.Contains(string(out), "no tests to run") {
|
||||
t.Fatal("the controller at this commit has no half of the replay to run")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
package replays
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// THE STORE GENESIS RAISES INITIALISES WHERE THE POSTGRES MODULE KEEPS IT (novox/hq ADR 0338 §5, issue 609).
|
||||
//
|
||||
// Genesis raises the store as the catalogue's postgres module declares its server: the module's image, its
|
||||
// data in a directory made before the store starts and owned 999:70 as the module declares it, and its
|
||||
// superuser password read from a file. The measurement the decision names: the module's image, started on
|
||||
// such a directory, empty, initialises a store that answers with that password. The image is read from the
|
||||
// catalogue's manifest, so the measurement moves with the module's pin.
|
||||
//
|
||||
// Needs a container runtime (its socket) and the catalogue (MESH_REPLAY_CATALOGUE, or the checkout beside
|
||||
// the lab); skipped aloud without either.
|
||||
func TestTheStoreGenesisRaisesInitialisesOnTheDirectoryGenesisMakes(t *testing.T) {
|
||||
catalogue := os.Getenv("MESH_REPLAY_CATALOGUE")
|
||||
if catalogue == "" {
|
||||
catalogue = filepath.Join("..", "..", "mesh-catalog")
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(catalogue, "modules", "postgres", "module.json"))
|
||||
if err != nil {
|
||||
t.Skipf("no catalogue to read the store's image from (%v); MESH_REPLAY_CATALOGUE names one", err)
|
||||
}
|
||||
docker, ok := DockerFromEnv()
|
||||
if !ok {
|
||||
t.Skip("no container runtime: the store runs in a container")
|
||||
}
|
||||
var m struct {
|
||||
Resources []struct {
|
||||
ID, Type, Path, Owner, Mode, Image string
|
||||
Env map[string]string
|
||||
} `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var image, pgdata, owner, mode string
|
||||
for _, r := range m.Resources {
|
||||
switch {
|
||||
case r.Type == "container" && r.ID == "server":
|
||||
image, pgdata = r.Image, r.Env["PGDATA"]
|
||||
case r.Type == "directory" && r.ID == "store-data":
|
||||
owner, mode = r.Owner, r.Mode
|
||||
}
|
||||
}
|
||||
if image == "" || pgdata == "" || owner == "" {
|
||||
t.Fatalf("the postgres module names no server image (%q), PGDATA (%q) or data directory owner (%q)", image, pgdata, owner)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
||||
defer cancel()
|
||||
// Under the working directory: a check's workspace is mounted into its containers at the same path,
|
||||
// so the runtime binds what this test made.
|
||||
here, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dir, err := os.MkdirTemp(here, "store-measure-")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, img := range []string{"alpine:3.20", image} {
|
||||
if err := docker.Pull(ctx, img); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// The directory as genesis makes it, and the password file as genesis writes it: as root, a helper
|
||||
// container doing what the node-engine does on the machine.
|
||||
prep, _, err := docker.Start(ctx, Run{Image: "alpine:3.20", Binds: []string{dir + ":/w"}, Cmd: []string{"sh", "-c",
|
||||
"mkdir -p /w/store /w/state && chown " + owner + " /w/store && chmod " + orMode(mode) + " /w/store && " +
|
||||
"printf measured-pw > /w/state/superuser.secret && chmod 0600 /w/state/superuser.secret"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if code, said, err := docker.Wait(ctx, prep); err != nil || code != 0 {
|
||||
t.Fatalf("the directory could not be made as genesis makes it: %v %d %s", err, code, said)
|
||||
}
|
||||
docker.Remove(ctx, prep)
|
||||
t.Cleanup(func() {
|
||||
clean, _, err := docker.Start(context.Background(), Run{Image: "alpine:3.20", Binds: []string{dir + ":/w"},
|
||||
Cmd: []string{"sh", "-c", "rm -rf /w/store /w/state"}})
|
||||
if err == nil {
|
||||
_, _, _ = docker.Wait(context.Background(), clean)
|
||||
docker.Remove(context.Background(), clean)
|
||||
}
|
||||
_ = os.RemoveAll(dir)
|
||||
})
|
||||
|
||||
store, _, err := docker.Start(ctx, Run{Image: image,
|
||||
Env: []string{"POSTGRES_PASSWORD_FILE=/run/secrets/superuser", "PGDATA=" + pgdata},
|
||||
Binds: []string{dir + "/store:/var/lib/postgresql/data", dir + "/state/superuser.secret:/run/secrets/superuser:ro"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer docker.Remove(context.Background(), store)
|
||||
answered := false
|
||||
for i := 0; i < 90 && !answered; i++ {
|
||||
// Over TCP, and with the password: the temporary server initdb runs listens on the socket only.
|
||||
code, err := docker.Exec(ctx, store, "sh", "-c",
|
||||
"PGPASSWORD=measured-pw psql -h 127.0.0.1 -U postgres -tAc 'select 1' | grep -qx 1")
|
||||
answered = err == nil && code == 0
|
||||
if !answered {
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
}
|
||||
if !answered {
|
||||
t.Fatalf("%s did not initialise a store that answers on a directory made %s %s, as genesis makes it:\n%s",
|
||||
image, owner, orMode(mode), lastLinesOf(docker.Logs(ctx, store), 15))
|
||||
}
|
||||
}
|
||||
|
||||
func orMode(m string) string {
|
||||
if m == "" {
|
||||
return "0700"
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func lastLinesOf(s string, n int) string {
|
||||
lines := strings.Split(strings.TrimRight(s, "\n"), "\n")
|
||||
if len(lines) > n {
|
||||
lines = lines[len(lines)-n:]
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
# A MACHINE IN USE, ADOPTED — and then converged, and returned (novox/hq ADR 0100, ADR 0101).
|
||||
#
|
||||
# The mesh replaces a predecessor that is running on the same machines. The anchor here is
|
||||
# prepared the way the predecessor leaves one: its own firewall (ufw) allowing a served port and
|
||||
# denying the rest, a service container on that port under a name a catalogue module also uses, a
|
||||
# file at a path that module declares, a stand-in for the predecessor's configuration sync that
|
||||
# rewrites the file, and a container holding the registry's port. The bed then raises the mesh on
|
||||
# it, adopted, and walks the migration the record decides.
|
||||
#
|
||||
# hosting (public)
|
||||
# anchor 192.0.2.10 the machine in use: the predecessor, then the mesh adopted on it
|
||||
# joiner 192.0.2.20 a fresh machine: the "second machine" that reaches the service and
|
||||
# enrols through the found firewall; also where a converged genesis on a
|
||||
# FRESH machine is asked (ADR 0101)
|
||||
# outsider 192.0.2.30 never enrolled, never on the private network: the probe from outside,
|
||||
# and the lab's forge — the bed serves the checkouts under test to the
|
||||
# anchor's builder from here, so nothing on the workstation listens
|
||||
#
|
||||
# inbound: allow on every machine — the anchor's firewall is the predecessor's, installed by the
|
||||
# bed; `inbound: deny` would load the lab's own table beside it and make that the thing under test.
|
||||
scenario: adoption
|
||||
|
||||
segments:
|
||||
hosting:
|
||||
kind: public
|
||||
cidr: [192.0.2.0/24]
|
||||
|
||||
machines:
|
||||
# Sized like the one-node bed's anchor: genesis builds the control plane, the base and the
|
||||
# catalogue's modules here, beside the predecessor's two containers.
|
||||
anchor:
|
||||
at: { segment: hosting, address: [192.0.2.10] }
|
||||
egress: true
|
||||
inbound: allow
|
||||
memory: 12GiB
|
||||
cpus: 6
|
||||
disk: 60GiB
|
||||
|
||||
joiner:
|
||||
at: { segment: hosting, address: [192.0.2.20] }
|
||||
egress: true
|
||||
inbound: allow
|
||||
memory: 3GiB
|
||||
cpus: 2
|
||||
disk: 20GiB
|
||||
|
||||
outsider:
|
||||
at: { segment: hosting, address: [192.0.2.30] }
|
||||
egress: true
|
||||
inbound: allow
|
||||
memory: 2GiB
|
||||
cpus: 2
|
||||
disk: 15GiB
|
||||
|
||||
place:
|
||||
all: [host, runtime]
|
||||
@@ -0,0 +1,41 @@
|
||||
# GENESIS PAST ITS BROKER STEP: the bed of novox/hq ADR 0333 §5 (issue 590).
|
||||
#
|
||||
# The installer raises the bus from its bundle, then adopts it as the `nats` module at its broker step (step
|
||||
# 18 of 24). Before ADR 0333 the bundle raised the bus under another name and on other paths than the module
|
||||
# keeps it, so adopting it would have raised a second bus beside the first, and genesis stopped there. The
|
||||
# bed runs the installer from bare to that step, looks at the bus, runs it on past the step, and looks again.
|
||||
#
|
||||
# Two machines on one public segment, both with a way out — the installer pulls the store, the bus and the
|
||||
# toolchains over it, as a bare machine does:
|
||||
# - anchor: the machine genesis raises;
|
||||
# - forge: serves the control plane's, the base's, the SDK's and the catalogue's repositories at the
|
||||
# commits under test over git's own protocol, so the installer builds exactly those and nothing on the
|
||||
# workstation listens for the lab. On the anchor itself it would be a listener, and genesis refuses a
|
||||
# machine something already listens on.
|
||||
scenario: genesis-keeps-its-bus
|
||||
|
||||
segments:
|
||||
hosting:
|
||||
kind: public
|
||||
cidr: [192.0.2.0/24]
|
||||
|
||||
machines:
|
||||
anchor:
|
||||
at: { segment: hosting, address: [192.0.2.10] }
|
||||
egress: true
|
||||
inbound: allow
|
||||
# The foundation, the registry, the forge genesis raises, the base and the modules it builds.
|
||||
memory: 8GiB
|
||||
cpus: 4
|
||||
disk: 60GiB
|
||||
|
||||
forge:
|
||||
at: { segment: hosting, address: [192.0.2.30] }
|
||||
egress: true
|
||||
inbound: allow
|
||||
memory: 1GiB
|
||||
cpus: 1
|
||||
disk: 10GiB
|
||||
|
||||
place:
|
||||
all: [host, runtime]
|
||||
@@ -0,0 +1,42 @@
|
||||
# GENESIS on a runtime nobody told to trust a registry: the bed of replay R92 (novox/hq issue 92).
|
||||
#
|
||||
# The first real genesis pushed the control plane's image to the registry it had just raised, and the
|
||||
# container runtime refused: "server gave HTTP response to HTTPS client". The mesh's registry speaks plain
|
||||
# HTTP on purpose, and the trust that lets a runtime speak to it was written six steps after the push. The
|
||||
# lab never saw it, because its base image trusts the documentation ranges as plain-HTTP registries before
|
||||
# any bed starts (src/lifecycle/base.ts). The bed takes that trust back off the anchor before genesis runs,
|
||||
# so the anchor's runtime is a bare machine's: it trusts loopback and nothing else.
|
||||
#
|
||||
# Two machines on one public segment, both with a way out — the installer pulls the store, the bus and the
|
||||
# registry over it, as a bare machine does:
|
||||
# - anchor: the machine genesis raises, given its registry at its own address, 192.0.2.10:5000;
|
||||
# - forge: serves the control plane's repository at the commit under test over git's own protocol, so the
|
||||
# installer builds exactly that and nothing on the workstation listens for the lab. On the anchor
|
||||
# itself it would be a listener, and genesis refuses a machine something already listens on.
|
||||
scenario: genesis-untrusted
|
||||
|
||||
segments:
|
||||
hosting:
|
||||
kind: public
|
||||
cidr: [192.0.2.0/24]
|
||||
|
||||
machines:
|
||||
anchor:
|
||||
at: { segment: hosting, address: [192.0.2.10] }
|
||||
egress: true
|
||||
inbound: allow
|
||||
# Genesis-single's anchor: the foundation, the registry, and two control planes during the pivot.
|
||||
memory: 8GiB
|
||||
cpus: 4
|
||||
disk: 40GiB
|
||||
|
||||
forge:
|
||||
at: { segment: hosting, address: [192.0.2.30] }
|
||||
egress: true
|
||||
inbound: allow
|
||||
memory: 2GiB
|
||||
cpus: 2
|
||||
disk: 15GiB
|
||||
|
||||
place:
|
||||
all: [host, runtime]
|
||||
@@ -0,0 +1,33 @@
|
||||
# A machine joins through the tunnel, with the bus closed to it (novox/hq ADR 0169).
|
||||
#
|
||||
# The anchor raises the foundation from the bundle, joins over its own loopback and becomes the hub.
|
||||
# The joiner has a host and nothing else: it makes its tunnel key, a token is issued for that key,
|
||||
# and it enrols. The bus is closed to the joiner's own address on the anchor, so the only way its
|
||||
# enrolment can arrive is over the tunnel the token gave it.
|
||||
scenario: joins-through-the-tunnel
|
||||
|
||||
segments:
|
||||
hosting:
|
||||
kind: public
|
||||
cidr: [192.0.2.0/24]
|
||||
|
||||
machines:
|
||||
anchor:
|
||||
at: { segment: hosting, address: [192.0.2.10] }
|
||||
egress: true
|
||||
inbound: allow
|
||||
memory: 3GiB
|
||||
cpus: 2
|
||||
joiner:
|
||||
at: { segment: hosting, address: [192.0.2.30] }
|
||||
egress: true
|
||||
inbound: allow
|
||||
memory: 1GiB
|
||||
# It carries none of the images: it only has to join.
|
||||
images: []
|
||||
|
||||
images:
|
||||
- mesh-controller:development
|
||||
|
||||
place:
|
||||
all: [host, runtime]
|
||||
@@ -0,0 +1,29 @@
|
||||
# Research 053 question 8: three machines for the signed-login bus.
|
||||
# a = today's style server (static users, JetStream), b and c = a cluster in operator mode with a
|
||||
# full resolver, a joined to b as a leaf. No overlay and no mesh software: the bed brings its own
|
||||
# nats-server (the program test/bed-programs/signed-logins embeds 2.11.17).
|
||||
scenario: signed-logins
|
||||
|
||||
segments:
|
||||
hosting:
|
||||
kind: public
|
||||
cidr: [192.0.2.0/24]
|
||||
|
||||
machines:
|
||||
a:
|
||||
at: { segment: hosting, address: [192.0.2.11] }
|
||||
inbound: allow
|
||||
memory: 2GiB
|
||||
cpus: 2
|
||||
b:
|
||||
at: { segment: hosting, address: [192.0.2.12] }
|
||||
inbound: allow
|
||||
memory: 2GiB
|
||||
cpus: 2
|
||||
c:
|
||||
at: { segment: hosting, address: [192.0.2.13] }
|
||||
inbound: allow
|
||||
memory: 2GiB
|
||||
cpus: 2
|
||||
|
||||
snapshot: raised
|
||||
@@ -0,0 +1,32 @@
|
||||
# The control plane's store restarting while a machine joins (novox/hq issue 083).
|
||||
#
|
||||
# Adopting the foundation's store — the first thing a control-node does, and the first thing a
|
||||
# migration does — recreates it, and for those seconds the control plane cannot write. A machine
|
||||
# enrolling then used to be refused, or worse, left with its token spent and no identity. This
|
||||
# raises the foundation on `anchor`, takes its store away, has `laptop` enrol into the gap and
|
||||
# brings the store back: the enrolment must complete on its own.
|
||||
scenario: store-window
|
||||
|
||||
segments:
|
||||
hosting:
|
||||
kind: public
|
||||
cidr: [192.0.2.0/24]
|
||||
|
||||
machines:
|
||||
anchor:
|
||||
at: { segment: hosting, address: [192.0.2.10] }
|
||||
egress: true
|
||||
inbound: allow
|
||||
memory: 3GiB
|
||||
cpus: 2
|
||||
laptop:
|
||||
at: { segment: hosting, address: [192.0.2.20] }
|
||||
egress: true
|
||||
inbound: allow
|
||||
memory: 1GiB
|
||||
|
||||
images:
|
||||
- mesh-controller:development
|
||||
|
||||
place:
|
||||
all: [host, runtime]
|
||||
@@ -0,0 +1,40 @@
|
||||
# One machine that becomes a mesh, runs the mesh's own certificate authority, and is then given the
|
||||
# module that makes it trust it — the bed for novox/hq ADR 0147 and issue 129.
|
||||
#
|
||||
# The question is narrow and the bed is shaped to answer only it: does a machine holding `ca-trust`
|
||||
# verify a certificate from the mesh's own authority with no bundle argument and no `-k`, and does
|
||||
# it stop verifying it when the module is taken away? The authority itself is what is dialled —
|
||||
# step-ca serves its own API with a leaf it issued — so nothing else has to be right for the answer
|
||||
# to mean something. No proxy, no routed name, no public issuance: those are the certificates and
|
||||
# route-forwarding beds, and a trust bed that leaned on them would pass for their reasons.
|
||||
#
|
||||
# The negative half is not optional. It is asserted BEFORE the module is assigned and again AFTER it
|
||||
# is unassigned, because an anchor bed that only ever checks the success is one that would pass on a
|
||||
# machine that already trusted everything.
|
||||
#
|
||||
# MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||
# MESH_LAB_CATALOG=.../mesh-catalog/modules
|
||||
# step-ca's image is upstream and pinned by the catalogue; the machine pulls it over its uplink.
|
||||
# ca-trust carries no image at all — a script, a unit, and the machine's own systemd.
|
||||
scenario: trust-anchor
|
||||
|
||||
segments:
|
||||
hosting:
|
||||
kind: public
|
||||
cidr: [192.0.2.0/24]
|
||||
|
||||
machines:
|
||||
anchor:
|
||||
at: { segment: hosting, address: [192.0.2.10] }
|
||||
egress: true
|
||||
inbound: allow
|
||||
memory: 3GiB
|
||||
cpus: 2
|
||||
|
||||
images:
|
||||
- mesh-controller:development
|
||||
|
||||
place:
|
||||
# Only the host. The authority's image comes from the internet over the machine's uplink, and the
|
||||
# trust module has nothing to place.
|
||||
all: [host]
|
||||
@@ -3,11 +3,11 @@
|
||||
# The app-postgres provider and the mesh's own foundation store both want host port 5432, so they
|
||||
# cannot share a machine — the collision that blocked this chain single-node. Here the foundation
|
||||
# (store, broker, control) lives on `anchor` and NOTHING else; `laptop` runs the whole chain —
|
||||
# postgres and redis PROVIDERS plus the baserow and letta CONSUMERS that require them. Both
|
||||
# the baserow and letta CONSUMERS of the one store (baserow keeps its cache inside its own container,
|
||||
# novox/hq 081). Both
|
||||
# machines sit on one shared segment and enrol into the one mesh; only enrolment crosses to anchor,
|
||||
# over the underlay both machines already share. Provider and consumers are co-located on laptop, so
|
||||
# no cross-node module comms and no overlay are needed — and the 5432-vs-foundation conflict is gone
|
||||
# because the foundation store is on the OTHER node.
|
||||
# over the underlay both machines already share. The consumers' databases are minted on the one
|
||||
# foundation store on anchor and reached over the overlay; laptop runs no provider of its own.
|
||||
scenario: two-node-db
|
||||
|
||||
segments:
|
||||
@@ -25,8 +25,7 @@ machines:
|
||||
inbound: allow
|
||||
memory: 4GiB
|
||||
cpus: 4
|
||||
# The whole DB-consumer chain: postgres + redis providers, each a server and a broker-bound
|
||||
# runtime, plus the baserow and letta consumer services and their tools runtimes — a dozen
|
||||
# The DB consumers: the baserow and letta services and their tools runtimes — a handful of
|
||||
# containers, two of them memory-hungry app servers (the Baserow all-in-one and the Letta server).
|
||||
# At the 2GiB the two-nodes bed gives this machine it would thrash — its own anchor comment says
|
||||
# so — and convergence would present as "the mesh hangs". Six gigabytes gives it room.
|
||||
@@ -49,7 +48,6 @@ images:
|
||||
# provisioner (ADR 0048).
|
||||
- mesh-runtime-postgres:development
|
||||
- mesh-runtime-lavinmq:development
|
||||
- mesh-runtime-redis:development
|
||||
- mesh-runtime-baserow:development
|
||||
- mesh-runtime-letta:development
|
||||
|
||||
|
||||
@@ -30,6 +30,9 @@ machines:
|
||||
memory: 2GiB
|
||||
|
||||
images:
|
||||
# No packet-filter runtime: the filter module's verbs are a tools bundle the node's runtime launches
|
||||
# (novox/hq ADR 0170, 0193), not a container of its own, and what this walk needs of the module is
|
||||
# the ruleset it loads — which its resources do. Stocking one built an image nothing ran.
|
||||
- mesh-controller:development
|
||||
# And the builder, because it is a module the mesh assigns rather than a program somebody
|
||||
# starts by hand — which is the only way its credential can be one the mesh delivered.
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# scenarios/whole-mesh-novox.yml; same topology, a different (larger, media-heavy) module set.
|
||||
#
|
||||
# Foundation (store, broker, control) rides `anchor` and NOTHING else; ALL of ace's services ride the
|
||||
# `ace` node. An overlay is placed so the two DB consumers (baserow, letta) reach the postgres/redis
|
||||
# `ace` node. An overlay is placed so the two DB consumers (baserow, letta) reach the postgres
|
||||
# providers co-located with them. The media stack (sonarr/radarr/lidarr/plex/bazarr/nzbget/
|
||||
# qbittorrent/bookshelf) shares the operator-owned library directories under /services/media (ADR
|
||||
# 0051 `accesses`); the test pre-creates them on the node, as the operator would, before the push —
|
||||
|
||||
+115
-57
@@ -1,10 +1,38 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build a per-module runtime image (novox/hq ADR 0052): the tool runtime carrying ONE module's
|
||||
# compiled code, which serves that module's tools and runs its events/provisioner under the module's
|
||||
# own scoped broker account. Generalises build-runtime-image.sh from the audit-logger to any module.
|
||||
# Build a per-module runtime image: the node's tool runtime serving ONE module's tools bundle, built
|
||||
# the way the mesh builds and serves it today (novox/hq ADR 0175, 0188, 0193).
|
||||
#
|
||||
# build-module-runtime.sh <module> <output.tar>
|
||||
# -> tags mesh-runtime-<module>:development and saves it to <output.tar>
|
||||
#
|
||||
# **What replaced mesh-tools' npm package.** The tool runtime was a TypeScript program in mesh-tools that
|
||||
# imported every module's compiled code into one process, and this script copied its dist and its
|
||||
# node_modules into an image. The runtime is now `node-tools` — a Go binary in mesh-tools' `node-tools/`
|
||||
# (built by the mesh as that module's bundle) — which imports nothing: it LAUNCHES each bundle as a child
|
||||
# and speaks MCP over stdio to it. A TypeScript bundle is made launchable by the builder, which writes
|
||||
# beside each compiled entrypoint an executable `<entry>.serve.mjs` that imports it and serves what it
|
||||
# registered through the SDK's `@novox/mesh-sdk/stdio`. The repository root has had no package.json since,
|
||||
# so the old script failed at its first step.
|
||||
#
|
||||
# This does what the mesh's builder does for a TypeScript bundle, on the workstation:
|
||||
# 1. compiles the module's declared entrypoints against the sibling SDK (rooted at the module, so an
|
||||
# entrypoint lands where it is named);
|
||||
# 2. writes each entrypoint's launcher, executable;
|
||||
# 3. stages the SDK (it has no runtime dependencies) and the module's own third-party dependencies;
|
||||
# 4. builds node-tools from mesh-tools' Go module, and makes it the image's entrypoint, serving the
|
||||
# module's tools, events and provisioner entrypoints, whichever exist.
|
||||
#
|
||||
# The builder also bundles each file into one with esbuild; that is a size optimisation, not a behaviour,
|
||||
# and is not repeated here.
|
||||
#
|
||||
# **Which credential the image runs on.** node-tools serves the modules it is GIVEN, on the credential it
|
||||
# holds, and refuses a module that is the credential's own (ADR 0193: the runtime launches bundles of
|
||||
# other modules, it is not one). So the container is given the node's runtime credential (the node-tools
|
||||
# module's, as the mesh issues it) in MESH_BROKER_FILE, or a plain MESH_BROKER_URL — never the served
|
||||
# module's own.
|
||||
#
|
||||
# Needs: go, node and npm; MESH_SDK with its node_modules (for the compiler); MESH_TOOLS and
|
||||
# MESH_CATALOG checkouts. Each defaults to the sibling of this repository.
|
||||
set -euo pipefail
|
||||
|
||||
MODULE="${1:?usage: build-module-runtime.sh <module> <output.tar>}"
|
||||
@@ -13,76 +41,107 @@ HERE="$(cd "$(dirname "$0")/.." && pwd)"; ROOT="$(cd "$HERE/.." && pwd)"
|
||||
MESH_TOOLS="${MESH_TOOLS:-$ROOT/mesh-tools}"
|
||||
MESH_SDK="${MESH_SDK:-$ROOT/mesh-sdk}"
|
||||
MESH_CATALOG="${MESH_CATALOG:-$ROOT/mesh-catalog}"
|
||||
MOD="$MESH_CATALOG/modules/$MODULE"
|
||||
# Absolute, because the steps below run from inside the module.
|
||||
for v in MESH_TOOLS MESH_SDK MESH_CATALOG; do
|
||||
[ -d "${!v}" ] || { echo "$v=${!v} is not a checkout" >&2; exit 1; }
|
||||
printf -v "$v" '%s' "$(cd "${!v}" && pwd)"
|
||||
done
|
||||
# A catalogue checkout or its modules/ directory, as MESH_LAB_CATALOG may name either.
|
||||
if [ -d "$MESH_CATALOG/modules" ]; then MESH_CATALOG="$MESH_CATALOG/modules"; fi
|
||||
MOD="$MESH_CATALOG/$MODULE"
|
||||
TAG="${RUNTIME_TAG:-mesh-runtime-$MODULE:development}"
|
||||
BASE="${RUNTIME_BASE:-node:22-bookworm-slim}"
|
||||
RUNTIME_SRC="$MESH_TOOLS/node-tools"
|
||||
[ -d "$MOD" ] || { echo "no module $MODULE at $MOD" >&2; exit 1; }
|
||||
[ -f "$RUNTIME_SRC/go.mod" ] || { echo "no node-tools Go module at $RUNTIME_SRC (MESH_TOOLS=$MESH_TOOLS)" >&2; exit 1; }
|
||||
command -v go >/dev/null || { echo "go is needed to build node-tools, the runtime the image runs" >&2; exit 1; }
|
||||
|
||||
# The SDK, built: the module compiles against its types and the launchers import its stdio loop.
|
||||
[ -d "$MESH_SDK/node_modules" ] || ( cd "$MESH_SDK" && npm ci --no-audit --no-fund --silent )
|
||||
( cd "$MESH_SDK" && npm run build >/dev/null )
|
||||
( cd "$MESH_TOOLS" && npm run build >/dev/null )
|
||||
# Compile whichever of the module's entrypoints exist. Besides the serve-time entrypoints (tools,
|
||||
# events, provisioner) and the run-once bootstrap, a module may carry scheduled/one-shot entrypoints
|
||||
# it names in a `schedule`/`run-once` container's args (novox/hq ADR 0052/0053) — refresh/apply/usage
|
||||
# for the anthropic model-access modules, migrate for model-usage's run-once schema step. tsc pulls
|
||||
# in their imports, so leaf files they use are compiled with them. A module may also carry an ambient
|
||||
# `.d.ts` typing a third-party dep it default-imports (model-usage's pg.d.ts) — listed here so the
|
||||
# ambient declaration is in the program even though the dep is only installed into the image below.
|
||||
SRCS=(); for f in \
|
||||
client.ts index.ts tools/index.ts provisioner/index.ts bootstrap/index.ts \
|
||||
adopt/index.ts refresh/index.ts apply/index.ts usage/index.ts migrate/index.ts \
|
||||
pg.d.ts; do
|
||||
[ -f "$MOD/$f" ] && SRCS+=("$f")
|
||||
|
||||
# The entrypoints the module declares for its TypeScript bundle (build.artifacts[].entrypoints), as the
|
||||
# builder reads them; a module declaring none falls back to the ones the runtime serves.
|
||||
ENTRIES_JS="$(node -e '
|
||||
const m = require(process.argv[1]);
|
||||
const out = new Set();
|
||||
for (const a of (m.build && m.build.artifacts) || [])
|
||||
if (a.language === "typescript") for (const e of a.entrypoints || []) if (e.endsWith(".js")) out.add(e);
|
||||
process.stdout.write([...out].join(" "));
|
||||
' "$MOD/module.json")"
|
||||
if [ -z "$ENTRIES_JS" ]; then
|
||||
for e in tools/index.js index.js provisioner/index.js; do
|
||||
[ -f "$MOD/${e%.js}.ts" ] && ENTRIES_JS="$ENTRIES_JS $e"
|
||||
done
|
||||
fi
|
||||
SRCS=(); for e in $ENTRIES_JS; do
|
||||
[ -f "$MOD/${e%.js}.ts" ] || { echo "$MODULE declares $e and has no ${e%.js}.ts" >&2; exit 1; }
|
||||
SRCS+=("${e%.js}.ts")
|
||||
done
|
||||
[ "${#SRCS[@]}" -gt 0 ] || { echo "$MODULE has no TypeScript entrypoint to serve" >&2; exit 1; }
|
||||
# An ambient `.d.ts` at the module's root types a third-party dep it default-imports (model-usage's
|
||||
# pg.d.ts); in the program so the compile sees it, though the dep is installed only into the image.
|
||||
for d in "$MOD"/*.d.ts; do [ -f "$d" ] && SRCS+=("$(basename "$d")"); done
|
||||
|
||||
# The module compiles against the SDK, which its package.json names and nothing installs: a module
|
||||
# never built on this workstation has no node_modules, and tsc fails on the first import. Installed
|
||||
# as a package copy from the sibling checkout (never a link) when absent — the compile needs only
|
||||
# the types; the image takes the SDK from MESH_SDK below.
|
||||
# as a package copy from the sibling checkout (never a link) when absent.
|
||||
if [ ! -e "$MOD/node_modules/@novox/mesh-sdk" ]; then
|
||||
( cd "$MOD" && npm install --no-save --install-links --no-package-lock --ignore-scripts --silent "$MESH_SDK" ) \
|
||||
|| { echo "cannot install the SDK into $MOD for the compile" >&2; exit 1; }
|
||||
fi
|
||||
# Output kept: a compile error hidden behind /dev/null is a build that fails saying nothing.
|
||||
TSC="$MESH_SDK/node_modules/.bin/tsc"; ( cd "$MOD" && "$TSC" "${SRCS[@]}" --module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist 1>&2 )
|
||||
|
||||
STAGE="$(mktemp -d)"; trap 'rm -rf "$STAGE"' EXIT
|
||||
cp -r "$MESH_TOOLS/dist" "$STAGE/dist"
|
||||
cp -rL "$MESH_TOOLS/node_modules" "$STAGE/node_modules"
|
||||
# And the sdk, from the sibling this script just built, whatever form the installed tree holds it
|
||||
# in. It used to be relied on being a symlink into that sibling, which `-L` above materialised —
|
||||
# true only on a workstation where somebody had linked them, and false the moment the runtime's
|
||||
# dependencies are installed the ordinary way, which now fetches the sdk as sources with nothing
|
||||
# compiled in it. The image built then looked fine and every entry point inside it pointed at
|
||||
# nothing.
|
||||
rm -rf "$STAGE/node_modules/@novox/mesh-sdk"
|
||||
mkdir -p "$STAGE/node_modules/@novox"
|
||||
cp -rL "$MESH_SDK" "$STAGE/node_modules/@novox/mesh-sdk"
|
||||
rm -rf "$STAGE/node_modules/@novox/mesh-sdk/node_modules"
|
||||
# **The image runs compiled code and never compiles any**, so it does not need a compiler. The
|
||||
# tree copied above is the runtime's full install, development dependencies and all — and the
|
||||
# compiler alone is 23 of its 28 MB. Every module image carried one, on every machine, for nothing:
|
||||
# tsc runs on the workstation a few lines above, not in here.
|
||||
#
|
||||
# Removed by name rather than by `npm prune --omit=dev`, which would re-resolve dependencies — one
|
||||
# of them a git URL with no registry behind it — and could drop something the image needs.
|
||||
rm -rf "$STAGE/node_modules/typescript" "$STAGE/node_modules/@types"
|
||||
mkdir -p "$STAGE/modules/$MODULE"; cp -r "$MOD/dist" "$STAGE/modules/$MODULE/dist"
|
||||
cp "$MESH_TOOLS/package.json" "$STAGE/package.json"
|
||||
DIST="$STAGE/modules/$MODULE/dist"
|
||||
# Output kept: a compile error hidden behind /dev/null is a build that fails saying nothing. Rooted at
|
||||
# the module, as the builder compiles, so tools/index.ts lands at tools/index.js.
|
||||
TSC="$MESH_SDK/node_modules/.bin/tsc"
|
||||
( cd "$MOD" && "$TSC" "${SRCS[@]}" --module NodeNext --moduleResolution NodeNext --target ES2022 \
|
||||
--rootDir . --outDir "$DIST" 1>&2 )
|
||||
# The compiled files are ES modules; said where Node looks for it, as the builder's bundle does.
|
||||
printf '{"type":"module","private":true}\n' > "$DIST/package.json"
|
||||
|
||||
# A launcher beside every entrypoint, exactly the builder's (mesh-controller internal/builder,
|
||||
# writeLaunchers): node-tools starts it and it serves what the entrypoint registered over stdio.
|
||||
for e in $ENTRIES_JS; do
|
||||
[ -f "$DIST/$e" ] || { echo "the compile wrote no $e" >&2; exit 1; }
|
||||
launcher="$DIST/${e%.js}.serve.mjs"
|
||||
cat > "$launcher" <<LAUNCHER
|
||||
#!/usr/bin/env node
|
||||
// Written as the mesh's builder writes it (novox/hq ADR 0193): serve what $e registers,
|
||||
// over MCP on stdio, as the module the node's runtime names in MESH_SERVED_MODULE.
|
||||
import { serveRegisteredOverStdio } from "@novox/mesh-sdk/stdio";
|
||||
await import("./$(basename "$e")");
|
||||
await serveRegisteredOverStdio();
|
||||
LAUNCHER
|
||||
chmod 0755 "$launcher"
|
||||
done
|
||||
|
||||
# The SDK, from the sibling just built, as a package copy: its package.json and its dist. It has no
|
||||
# runtime dependencies of its own.
|
||||
mkdir -p "$STAGE/node_modules/@novox/mesh-sdk"
|
||||
cp "$MESH_SDK/package.json" "$STAGE/node_modules/@novox/mesh-sdk/"
|
||||
cp -r "$MESH_SDK/dist" "$STAGE/node_modules/@novox/mesh-sdk/dist"
|
||||
|
||||
# A module may declare its own third-party runtime deps (the anthropic-manager seals with
|
||||
# tweetnacl-sealedbox-js). The shared node_modules copied above carries the common packages and
|
||||
# @novox/* — but not a module's private deps. Install those under the module itself, so Node
|
||||
# resolves them from /app/modules/<module>/node_modules and still falls back to the shared tree
|
||||
# at /app/node_modules for @novox/* and everything common. Modules with no non-@novox deps are a
|
||||
# no-op. (@novox/* are workspace deps with no registry to fetch from, so they are excluded here.)
|
||||
MOD_DEPS="$(node -e 'const d=(require("'"$MOD"'/package.json").dependencies)||{};process.stdout.write(Object.keys(d).filter(k=>!k.startsWith("@novox/")).map(k=>k+"@"+d[k]).join(" "))')"
|
||||
# tweetnacl-sealedbox-js). Installed under the module, so Node resolves them from
|
||||
# /app/modules/<module>/node_modules and still falls back to /app/node_modules for the SDK. @novox/* are
|
||||
# excluded: there is no public registry for them, and the SDK is staged above.
|
||||
MOD_DEPS="$(node -e 'const d=(require(process.argv[1]).dependencies)||{};process.stdout.write(Object.keys(d).filter(k=>!k.startsWith("@novox/")).map(k=>k+"@"+d[k]).join(" "))' "$MOD/package.json")"
|
||||
if [ -n "$MOD_DEPS" ]; then
|
||||
# shellcheck disable=SC2086
|
||||
npm install --prefix "$STAGE/modules/$MODULE" --omit=dev --no-save --no-package-lock --ignore-scripts $MOD_DEPS >/dev/null
|
||||
fi
|
||||
|
||||
# The entrypoints the runtime loads: tools, events and (a provider's) provisioner, whichever exist.
|
||||
ENTRIES=""; for e in tools/index.js index.js provisioner/index.js; do
|
||||
[ -f "$STAGE/modules/$MODULE/dist/$e" ] && ENTRIES="${ENTRIES:+$ENTRIES,}/app/modules/$MODULE/dist/$e"
|
||||
# The runtime itself: node-tools, static, from mesh-tools' Go module.
|
||||
( cd "$RUNTIME_SRC" && CGO_ENABLED=0 go build -trimpath -o "$STAGE/node-tools" ./cmd/node-tools )
|
||||
|
||||
# What the runtime serves: the tools, events and (a provider's) provisioner entrypoints, whichever the
|
||||
# module has — each by its launcher, as <module>=<path>. The others (bootstrap, prepare, apply, …) are
|
||||
# run once by name, as the mesh runs them, and carry launchers only because the builder writes one for
|
||||
# every entrypoint.
|
||||
SERVED=""; for e in tools/index.js index.js provisioner/index.js; do
|
||||
[ -f "$DIST/${e%.js}.serve.mjs" ] && SERVED="${SERVED:+$SERVED,}$MODULE=/app/modules/$MODULE/dist/${e%.js}.serve.mjs"
|
||||
done
|
||||
|
||||
# A module whose code drives a CLI needs that CLI in the image — postgres shells out to `psql`, minio
|
||||
@@ -105,13 +164,12 @@ cat > "$STAGE/Dockerfile" <<DOCKER
|
||||
FROM $BASE
|
||||
WORKDIR /app
|
||||
$EXTRA
|
||||
COPY package.json ./
|
||||
COPY node-tools /usr/local/bin/node-tools
|
||||
COPY node_modules ./node_modules
|
||||
COPY dist ./dist
|
||||
COPY modules ./modules
|
||||
ENV MESH_TOOL_MODULES=$ENTRIES
|
||||
ENTRYPOINT ["node", "dist/main.js"]
|
||||
ENV MESH_TOOL_MODULES=$SERVED
|
||||
ENTRYPOINT ["/usr/local/bin/node-tools", "serve"]
|
||||
DOCKER
|
||||
docker build -t "$TAG" "$STAGE"
|
||||
docker save -o "$OUT" "$TAG"
|
||||
echo "built $TAG (entrypoints: $ENTRIES) -> $OUT"
|
||||
echo "built $TAG (serving: ${SERVED:-nothing}) -> $OUT"
|
||||
|
||||
+18
-1
@@ -20,7 +20,24 @@ import { around, log, shorten } from "../log.ts";
|
||||
* predates the group grant cannot reach it — which is a real thing that happens on the
|
||||
* machine that just installed it.
|
||||
*/
|
||||
const INCUS = (process.env["MESH_LAB_INCUS"] ?? "incus").split(" ").filter(Boolean);
|
||||
let INCUS = (process.env["MESH_LAB_INCUS"] ?? "incus").split(" ").filter(Boolean);
|
||||
|
||||
/**
|
||||
* Point every call at another program, and say what it was pointed at before.
|
||||
*
|
||||
* **For a test about how the lab reads a failure, never about the hypervisor.** The command was
|
||||
* read from `MESH_LAB_INCUS` once, when this module loaded — and a test file setting that variable
|
||||
* in its body set it too late, because an ES module's imports are evaluated before its body. So the
|
||||
* enumeration tests asked the real incus wherever one was installed and reachable: they passed on a
|
||||
* machine without incus (the spawn fails) and failed on the workstation that runs the lab, where
|
||||
* listing succeeds. Injected here, a test names the failing program and gets it, on every machine.
|
||||
*/
|
||||
export function useIncusCommand(argv: string[]): string[] {
|
||||
if (argv.length === 0 || !argv[0]) throw new Error("an incus command needs a program to run");
|
||||
const was = INCUS;
|
||||
INCUS = [...argv];
|
||||
return was;
|
||||
}
|
||||
|
||||
export interface IncusResult {
|
||||
stdout: string;
|
||||
|
||||
+19
-3
@@ -24,6 +24,7 @@ import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { homedir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
|
||||
import type { Outcome, Skip } from "./outcomes.ts";
|
||||
import { repositories } from "./repos.ts";
|
||||
|
||||
/** What a run was taken against, per repository. */
|
||||
@@ -38,6 +39,11 @@ export interface Receipt {
|
||||
against: Against;
|
||||
/** The test files this run was pointed at. See {@link endToEnd}. */
|
||||
ran: string[];
|
||||
/** How many tests skipped, and each with its reason (novox/hq issue 693). Absent on older receipts. */
|
||||
skipped?: number;
|
||||
skips?: Skip[];
|
||||
/** Why the run did not pass although nothing failed — no test passed, or one was not reached. */
|
||||
notPassed?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -104,6 +110,7 @@ export function record(
|
||||
ran: string[],
|
||||
env = process.env,
|
||||
builtAgainst?: Against,
|
||||
outcome?: Outcome,
|
||||
): Receipt {
|
||||
const receipt: Receipt = {
|
||||
at: new Date().toISOString(),
|
||||
@@ -115,6 +122,8 @@ export function record(
|
||||
// never in the binaries. A verdict is only worth something attributed to one exact state.
|
||||
against: builtAgainst ?? whatWasTested(env),
|
||||
ran,
|
||||
...(outcome ? { skipped: outcome.skipped, skips: outcome.skips } : {}),
|
||||
...(outcome?.notPassed ? { notPassed: outcome.notPassed } : {}),
|
||||
};
|
||||
const path = receiptPath();
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
@@ -166,9 +175,14 @@ export function judge(
|
||||
|
||||
const days = (now.getTime() - Date.parse(receipt.at)) / 86_400_000;
|
||||
const lines: string[] = [];
|
||||
// A run that proved nothing did not pass, though nothing failed (novox/hq issue 693): no test passed,
|
||||
// or one skipped as not reached. A receipt from before the field is judged by its count alone.
|
||||
const notPassed = receipt.failed > 0 ? "" : receipt.notPassed ?? (receipt.passed === 0 ? "no test passed" : "");
|
||||
const outcome = receipt.failed > 0
|
||||
? `last ran ${ago(days)} and ${receipt.failed} test(s) failed`
|
||||
: `last passed ${ago(days)}, ${receipt.passed} test(s)`;
|
||||
: notPassed
|
||||
? `last ran ${ago(days)} and did not pass: ${notPassed}`
|
||||
: `last passed ${ago(days)}, ${receipt.passed} test(s)`;
|
||||
lines.push(`the end-to-end suite ${outcome}`);
|
||||
|
||||
let moved = false;
|
||||
@@ -192,7 +206,9 @@ export function judge(
|
||||
lines.push(` That run did not include ${endToEnd}, so it raised no machines.`);
|
||||
}
|
||||
|
||||
if (receipt.failed > 0) {
|
||||
for (const s of receipt.skips ?? []) lines.push(` skipped: ${s.test} — ${s.reason || "no reason given"}`);
|
||||
|
||||
if (receipt.failed > 0 || notPassed) {
|
||||
lines.push(" Nothing has been proven end to end since.");
|
||||
} else if (moved) {
|
||||
lines.push(" What it proved was proven about code that has since changed.");
|
||||
@@ -201,7 +217,7 @@ export function judge(
|
||||
}
|
||||
|
||||
return {
|
||||
current: endToEndRan && receipt.failed === 0 && !moved && days <= staleAfterDays,
|
||||
current: endToEndRan && receipt.failed === 0 && !notPassed && !moved && days <= staleAfterDays,
|
||||
lines,
|
||||
};
|
||||
}
|
||||
|
||||
+102
-9
@@ -23,6 +23,88 @@ export const UPSTREAM_IMAGE = "images:archlinux/current";
|
||||
|
||||
const BUILDER = "mesh-lab-base-builder";
|
||||
|
||||
/**
|
||||
* **The mirrors the builder installs from, several and in order** (novox/hq issue 700).
|
||||
*
|
||||
* The stock image names one mirror, mirrors.kernel.org, and a base build failed twice on it in one morning:
|
||||
* timeouts while installing the runtime, then a 500 on wireguard-tools. pacman tries the next server in the
|
||||
* list for each file it could not fetch, so a list of several turns one mirror's bad minute into a slower
|
||||
* download instead of a failed build.
|
||||
*
|
||||
* A fixed list rather than ranking at build time: ranking (rankmirrors, reflector) needs a package installed
|
||||
* first, from the one mirror that is the problem, and gives a different list on every build. These are
|
||||
* Arch's own geo-routed mirror first, then well-kept European mirrors near the lab's machine, then the
|
||||
* stock image's own. Each answered for core.db on 2026-10-14. A mirror that goes away costs a fallback,
|
||||
* not a build, and is replaced here.
|
||||
*/
|
||||
export const MIRRORS = [
|
||||
"https://geo.mirror.pkgbuild.com/$repo/os/$arch",
|
||||
"https://mirror.netcologne.de/archlinux/$repo/os/$arch",
|
||||
"https://ftp.fau.de/archlinux/$repo/os/$arch",
|
||||
"https://archlinux.mirror.liteserver.nl/$repo/os/$arch",
|
||||
"https://mirror.leaseweb.net/archlinux/$repo/os/$arch",
|
||||
"https://mirrors.kernel.org/archlinux/$repo/os/$arch",
|
||||
];
|
||||
|
||||
/** The mirrorlist the builder is given: every mirror, in order. */
|
||||
export function mirrorlist(): string {
|
||||
return "# Written by the mesh-lab base build (novox/hq issue 700): several mirrors, tried in order.\n" +
|
||||
MIRRORS.map((m) => `Server = ${m}`).join("\n") + "\n";
|
||||
}
|
||||
|
||||
/** The mirrors a failed pacman step names: "from <host>" in what it said, each once. */
|
||||
export function failedMirrors(said: string): string[] {
|
||||
const hosts = [...said.matchAll(/\bfrom ([a-z0-9.-]+\.[a-z]{2,})\b/gi)].map((m) => m[1]!.toLowerCase());
|
||||
return [...new Set(hosts)];
|
||||
}
|
||||
|
||||
/** How long to wait before each retry of a failed pacman step: three retries, backing off. */
|
||||
export const PACMAN_BACKOFF_MS = [10_000, 30_000, 90_000];
|
||||
|
||||
/** What a base build runs through, so a test can stand in for the hypervisor and the clock. */
|
||||
export interface BaseDeps {
|
||||
incus: (args: string[], timeoutMs?: number) => Promise<unknown>;
|
||||
incusOk: (args: string[], timeoutMs?: number) => Promise<string | null>;
|
||||
succeeds: (args: string[], timeoutMs?: number) => Promise<boolean>;
|
||||
sleep: (ms: number) => Promise<void>;
|
||||
}
|
||||
|
||||
const realDeps: BaseDeps = {
|
||||
incus, incusOk, succeeds, sleep: (ms) => new Promise((r) => setTimeout(r, ms)),
|
||||
};
|
||||
|
||||
/**
|
||||
* pacmanIn runs one pacman step in the builder, and **runs it again after a pause when it fails**, up to
|
||||
* three more times, saying each time which mirror failed and what it said. Installing is idempotent — what is
|
||||
* installed stays installed — so a retry repeats only what did not happen. The last failure is thrown, naming
|
||||
* every mirror that failed along the way.
|
||||
*/
|
||||
export async function pacmanIn(name: string, args: string[], log: (m: string) => void, deps: BaseDeps = realDeps):
|
||||
Promise<void> {
|
||||
const failed: string[] = [];
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
try {
|
||||
await deps.incus(["exec", name, "--", "pacman", ...args], 600_000);
|
||||
return;
|
||||
} catch (err) {
|
||||
// What pacman said, where incus kept it apart; otherwise the whole message.
|
||||
const said = (err as { stderr?: string }).stderr?.trim() || (err as Error).message;
|
||||
const mirrors = failedMirrors(said);
|
||||
for (const m of mirrors) if (!failed.includes(m)) failed.push(m);
|
||||
const where = mirrors.length > 0 ? `from ${mirrors.join(", ")}` : "naming no mirror";
|
||||
const first = said.split("\n").find((l) => /error/i.test(l))?.trim() ?? said.split("\n")[0]?.trim() ?? "";
|
||||
if (attempt >= PACMAN_BACKOFF_MS.length) {
|
||||
throw new BaseImageError(`pacman ${args.join(" ")} failed ${attempt + 1} times; the mirrors that failed: ` +
|
||||
`${failed.join(", ") || "none named"}. Last: ${said}`);
|
||||
}
|
||||
const wait = PACMAN_BACKOFF_MS[attempt]!;
|
||||
log(` pacman ${args.join(" ")} failed ${where} (${first}); trying again in ${wait / 1000}s ` +
|
||||
`(attempt ${attempt + 2} of ${PACMAN_BACKOFF_MS.length + 1})`);
|
||||
await deps.sleep(wait);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class BaseImageError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
@@ -38,7 +120,9 @@ export class BaseImageError extends Error {
|
||||
*/
|
||||
export async function buildBaseImage(
|
||||
log: (message: string) => void = () => {},
|
||||
deps: BaseDeps = realDeps,
|
||||
): Promise<{ alias: string; runtime: string }> {
|
||||
const { incus, incusOk, succeeds } = deps;
|
||||
await succeeds(["delete", "-f", BUILDER], 120_000);
|
||||
|
||||
log(` launching ${BUILDER} from ${UPSTREAM_IMAGE}, with a network`);
|
||||
@@ -50,10 +134,19 @@ export async function buildBaseImage(
|
||||
], 300_000);
|
||||
|
||||
try {
|
||||
await waitForAgent(BUILDER);
|
||||
await waitForAgent(BUILDER, deps);
|
||||
|
||||
// Several mirrors before anything is fetched (MIRRORS), and read back: a list that was not written is
|
||||
// the one mirror again, and the failure would look like the one this prevents.
|
||||
log(` giving pacman ${MIRRORS.length} mirrors`);
|
||||
await incus(["exec", BUILDER, "--", "sh", "-c", `printf '%s' '${mirrorlist()}' > /etc/pacman.d/mirrorlist`], 60_000);
|
||||
const written = await incusOk(["exec", BUILDER, "--", "cat", "/etc/pacman.d/mirrorlist"], 60_000);
|
||||
if (written?.trim() !== mirrorlist().trim()) {
|
||||
throw new BaseImageError(`the mirrorlist written into ${BUILDER} reads back as ${JSON.stringify(written)}`);
|
||||
}
|
||||
|
||||
log(" installing a container runtime");
|
||||
await incus(["exec", BUILDER, "--", "pacman", "-Sy", "--noconfirm", "docker"], 600_000);
|
||||
await pacmanIn(BUILDER, ["-Sy", "--noconfirm", "docker"], log, deps);
|
||||
|
||||
// And the tools for the private network, for the same reason as the runtime: a sealed
|
||||
// scenario cannot install them, so a lab that omits them cannot test connectivity at all —
|
||||
@@ -63,27 +156,27 @@ export async function buildBaseImage(
|
||||
// declaration; a lab that brought the interface up itself would be testing its own setup
|
||||
// rather than the mesh's.
|
||||
log(" installing the tools for the private network");
|
||||
await incus(["exec", BUILDER, "--", "pacman", "-S", "--noconfirm", "wireguard-tools"], 600_000);
|
||||
await pacmanIn(BUILDER, ["-S", "--noconfirm", "wireguard-tools"], log, deps);
|
||||
|
||||
// And git, for the same reason again: a machine that builds modules clones them, and a sealed
|
||||
// scenario cannot install it. On a real build machine the mesh installs it as a package like
|
||||
// anything else — the lab is the special case, because its machines reach no mirror.
|
||||
log(" installing git, so a machine can build modules");
|
||||
await incus(["exec", BUILDER, "--", "pacman", "-S", "--noconfirm", "git"], 600_000);
|
||||
await pacmanIn(BUILDER, ["-S", "--noconfirm", "git"], log, deps);
|
||||
|
||||
// And nftables, because the mesh computes a machine's filtering and delivers it as a file
|
||||
// that a service reflects — and neither the file nor the service can install what loads it.
|
||||
// Installed and NOT enabled: whether a machine filters is the mesh's decision, and a lab that
|
||||
// turned it on itself would be testing its own setup.
|
||||
log(" installing nftables, so a machine can enforce what the mesh computed");
|
||||
await incus(["exec", BUILDER, "--", "pacman", "-S", "--noconfirm", "nftables"], 600_000);
|
||||
await pacmanIn(BUILDER, ["-S", "--noconfirm", "nftables"], log, deps);
|
||||
|
||||
// And dnsmasq, because a service is named under the machine it runs on — postgres.novox.internal
|
||||
// — and only a resolver can answer a name the mesh was never told about. Installed and NOT
|
||||
// started: whether a machine resolves for the mesh is the mesh's decision, and a lab that
|
||||
// turned it on itself would be testing its own setup.
|
||||
log(" installing dnsmasq, so a machine can answer names under another machine");
|
||||
await incus(["exec", BUILDER, "--", "pacman", "-S", "--noconfirm", "dnsmasq"], 600_000);
|
||||
await pacmanIn(BUILDER, ["-S", "--noconfirm", "dnsmasq"], log, deps);
|
||||
|
||||
// Trust the documentation ranges as plain-HTTP registries.
|
||||
//
|
||||
@@ -216,10 +309,10 @@ export async function baseImageExists(): Promise<boolean> {
|
||||
* Wait for the guest agent, because `launch` returning means the VM started, not that anything
|
||||
* inside it will answer.
|
||||
*/
|
||||
async function waitForAgent(name: string): Promise<void> {
|
||||
async function waitForAgent(name: string, deps: BaseDeps = realDeps): Promise<void> {
|
||||
for (let i = 0; i < 90; i++) {
|
||||
if (await succeeds(["exec", name, "--", "true"], 10_000)) return;
|
||||
await new Promise((r) => setTimeout(r, 2_000));
|
||||
if (await deps.succeeds(["exec", name, "--", "true"], 10_000)) return;
|
||||
await deps.sleep(2_000);
|
||||
}
|
||||
throw new BaseImageError(`${name} started and its agent never answered.`);
|
||||
}
|
||||
|
||||
+13
-1
@@ -135,6 +135,18 @@ async function createNetwork(
|
||||
return name;
|
||||
}
|
||||
|
||||
/**
|
||||
* The uplink's own address and prefix: incus picks a free one, unless `MESH_LAB_UPLINK_V4` names it.
|
||||
*
|
||||
* **A workstation behind a VPN client may have no free range left to pick.** A corporate client that
|
||||
* routes all of 10.0.0.0/8 and 172.16.0.0/12 leaves incus refusing with "failed to automatically find
|
||||
* an unused IPv4 subnet", and no scenario can be raised at all. Naming one the host does not route —
|
||||
* `192.168.231.1/24` — is the way past it; it is the host's fact, never the scenario's.
|
||||
*/
|
||||
export function uplinkAddress(env: NodeJS.ProcessEnv = process.env): string {
|
||||
return env["MESH_LAB_UPLINK_V4"]?.trim() || "auto";
|
||||
}
|
||||
|
||||
/**
|
||||
* The one network the lab supplies rather than the declaration.
|
||||
*
|
||||
@@ -153,7 +165,7 @@ async function createUplink(instanceId: string): Promise<string> {
|
||||
if (await succeeds(["network", "show", name], 15_000)) return name;
|
||||
await incus([
|
||||
"network", "create", name,
|
||||
"ipv4.address=auto",
|
||||
`ipv4.address=${uplinkAddress()}`,
|
||||
"ipv4.nat=true",
|
||||
"ipv6.address=none",
|
||||
`user.mesh-lab.instance=${instanceId}`,
|
||||
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
/**
|
||||
* What a suite's tests did, beyond the runner's totals: which were skipped and why, and whether the run
|
||||
* proved anything (novox/hq issue 693).
|
||||
*
|
||||
* **A run in which no test passed has proven nothing, whatever its exit code.** The runner exits 0 when
|
||||
* every test skipped, and the lab recorded such a run as passed: the genesis bed stopped before its
|
||||
* subject, skipped its only test as "NOT REACHED", and the run read "0 passed, 0 failed", passed.
|
||||
* Failure must be loud, and a green run that verified nothing is the quietest failure there is.
|
||||
*
|
||||
* The default export is a reporter for node's test runner (`--test-reporter`): it writes one JSON line per
|
||||
* skipped test, with its file, its name and the reason it gave, word for word. The reasons come from the
|
||||
* runner's events rather than from the spec reporter's text, because a reason runs over several lines and a
|
||||
* later test's output lands between them.
|
||||
*/
|
||||
|
||||
import { readFileSync } from "node:fs";
|
||||
|
||||
/** One skipped test and the reason it gave ("" when it gave none). */
|
||||
export interface Skip {
|
||||
file: string;
|
||||
test: string;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
/** What a receipt keeps beside the totals: the skips, and why the run did not pass when it did not. */
|
||||
export interface Outcome {
|
||||
skipped: number;
|
||||
skips: Skip[];
|
||||
notPassed?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The words a bed's test skips with when the run stopped before its subject: the subject was not reached,
|
||||
* so nothing about it was proven (the beds say "<what> NOT REACHED: <where it stopped>").
|
||||
*/
|
||||
export const NOT_REACHED = "NOT REACHED";
|
||||
|
||||
/**
|
||||
* Whether a run passed, and if not, why — the design limit of issue 693, in one place:
|
||||
*
|
||||
* - a test failed: not passed (the runner already says so);
|
||||
* - **no test passed**: not passed, whatever was skipped — a run that skipped everything, or ran
|
||||
* nothing, verified nothing;
|
||||
* - **a test skipped as NOT REACHED**: not passed, even beside tests that passed — its subject was not
|
||||
* reached, which is the bed failing to get where it was sent, not a skip anyone declared expected;
|
||||
* - otherwise passed, its other skips listed with their reasons: a test skipped for a reason its file
|
||||
* declares (no hypervisor, an optional bed not asked for) beside tests that passed is an expected
|
||||
* skip, and the run still proved what passed.
|
||||
*
|
||||
* Counts the runner did not report (null) decide nothing here: that is reportOn's "no receipt" case.
|
||||
*/
|
||||
export function judgeRun(
|
||||
counts: { passed: number | null; failed: number | null },
|
||||
skips: Skip[],
|
||||
): { passed: boolean; why: string } {
|
||||
if (counts.failed !== null && counts.failed > 0) return { passed: false, why: `${counts.failed} test(s) failed` };
|
||||
// In any case: "not reached" says the same as "NOT REACHED".
|
||||
const unreached = skips.filter((s) => s.reason.toUpperCase().includes(NOT_REACHED));
|
||||
if (counts.passed === 0) {
|
||||
return {
|
||||
passed: false,
|
||||
why: skips.length > 0
|
||||
? `no test passed: ${skips.length} skipped${unreached.length > 0 ? `, ${unreached.length} of them as ${NOT_REACHED}` : ""}`
|
||||
: "no test passed: none ran",
|
||||
};
|
||||
}
|
||||
if (unreached.length > 0) {
|
||||
return { passed: false, why: `${unreached.length} test(s) skipped as ${NOT_REACHED}: their subject was not reached` };
|
||||
}
|
||||
return { passed: true, why: "" };
|
||||
}
|
||||
|
||||
/** A reason on one line, for a log line and a record: its whitespace folded, and at most 500 characters. */
|
||||
export function oneLine(reason: string): string {
|
||||
const flat = reason.replace(/\s+/g, " ").trim();
|
||||
return flat.length > 500 ? `${flat.slice(0, 499)}…` : flat;
|
||||
}
|
||||
|
||||
/** The skips the reporter wrote to a file; none when it wrote nothing or the file cannot be read. */
|
||||
export function readSkips(path: string): Skip[] {
|
||||
let text: string;
|
||||
try {
|
||||
text = readFileSync(path, "utf8");
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
const skips: Skip[] = [];
|
||||
for (const line of text.split("\n")) {
|
||||
if (!line.trim()) continue;
|
||||
try {
|
||||
const s = JSON.parse(line) as Partial<Skip>;
|
||||
skips.push({ file: String(s.file ?? ""), test: String(s.test ?? ""), reason: String(s.reason ?? "") });
|
||||
} catch {
|
||||
// A torn last line, when the runner was killed mid-write: the lines before it still stand.
|
||||
}
|
||||
}
|
||||
return skips;
|
||||
}
|
||||
|
||||
interface Event {
|
||||
type: string;
|
||||
data: { name?: string; file?: string; skip?: string | boolean; details?: { type?: string } };
|
||||
}
|
||||
|
||||
/** The reporter: one JSON line per skipped test (not per skipped suite: its tests are reported each). */
|
||||
export default async function* skipsReporter(source: AsyncIterable<Event>): AsyncGenerator<string> {
|
||||
for await (const event of source) {
|
||||
if (event.type !== "test:pass" && event.type !== "test:fail") continue;
|
||||
const { skip, details } = event.data;
|
||||
if (skip === undefined || skip === false || details?.type === "suite") continue;
|
||||
const said: Skip = { file: event.data.file ?? "", test: event.data.name ?? "", reason: typeof skip === "string" ? skip : "" };
|
||||
yield `${JSON.stringify(said)}\n`;
|
||||
}
|
||||
}
|
||||
+70
-16
@@ -10,11 +10,16 @@
|
||||
*/
|
||||
|
||||
import { spawn } from "node:child_process";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { isAbsolute, join, relative } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { endToEnd, record, whatWasTested } from "./lastrun.ts";
|
||||
import { judgeRun, oneLine, readSkips, type Outcome, type Skip } from "./outcomes.ts";
|
||||
import { rebuild } from "./rebuild.ts";
|
||||
|
||||
/** counted is what the runner said, or nulls when it said nothing recognisable. */
|
||||
export function counted(output: string): { passed: number | null; failed: number | null } {
|
||||
export function counted(output: string): { passed: number | null; failed: number | null; skipped: number | null } {
|
||||
// The runner's own summary lines, each on a line of its own. Anchored, so a test *named*
|
||||
// "pass 3" cannot be mistaken for the total — which is not a hypothetical worry in a suite whose
|
||||
// tests are named in sentences.
|
||||
@@ -23,13 +28,18 @@ export function counted(output: string): { passed: number | null; failed: number
|
||||
// running this against the real runner — the fixture it was first written against was output I
|
||||
// had imagined, which is a test that agrees with the mistake it was written beside.
|
||||
const plain = output.replace(/\u001b\[[0-9;]*m/g, "");
|
||||
// **The last such line, never the first** (novox/hq issue 693): the runner prints its summary after
|
||||
// everything a test printed, so a test that prints "ℹ pass 5" before it skips would otherwise be read
|
||||
// as five passes, and a run that proved nothing as passed.
|
||||
const total = (what: RegExp) => {
|
||||
const found = plain.match(what);
|
||||
return found ? Number(found[1]) : null;
|
||||
const found = [...plain.matchAll(what)];
|
||||
const last = found[found.length - 1];
|
||||
return last ? Number(last[1]) : null;
|
||||
};
|
||||
return {
|
||||
passed: total(/^\s*(?:\u2139|#)\s*pass\s+(\d+)\s*$/m),
|
||||
failed: total(/^\s*(?:\u2139|#)\s*fail\s+(\d+)\s*$/m),
|
||||
passed: total(/^\s*(?:\u2139|#)\s*pass\s+(\d+)\s*$/gm),
|
||||
failed: total(/^\s*(?:\u2139|#)\s*fail\s+(\d+)\s*$/gm),
|
||||
skipped: total(/^\s*(?:\u2139|#)\s*skipped\s+(\d+)\s*$/gm),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -57,16 +67,38 @@ export async function runSuite(args: string[]): Promise<number> {
|
||||
// `test/integration/canary.test.ts` is still there and still runs when it is named; it is no
|
||||
// longer raised on the way to everything else.
|
||||
|
||||
const { code, seen } = await runFiles(files);
|
||||
console.log("\n" + reportOn(counted(seen), (p, f) => record(p, f, files, process.env, against)));
|
||||
return code;
|
||||
const { code, seen, skips } = await runFiles(files);
|
||||
const counts = counted(seen);
|
||||
console.log("\n" + reportOn(counts, skips, (p, f, outcome) => record(p, f, files, process.env, against, outcome)));
|
||||
return exitOf(code, counts, skips);
|
||||
}
|
||||
|
||||
/** Run some test files, passing their output through as it arrives. */
|
||||
async function runFiles(files: string[]): Promise<{ code: number; seen: string }> {
|
||||
/**
|
||||
* The suite's exit: the runner's, unless it says 0 for a run that did not pass (novox/hq issue 693). The
|
||||
* runner exits 0 when every test skipped; the lab takes this exit as the run's verdict, so a run that
|
||||
* proved nothing must not end in 0.
|
||||
*/
|
||||
export function exitOf(code: number, counts: { passed: number | null; failed: number | null }, skips: Skip[]): number {
|
||||
if (code !== 0) return code;
|
||||
if (counts.passed === null || counts.failed === null) return code;
|
||||
return judgeRun(counts, skips).passed ? 0 : 1;
|
||||
}
|
||||
|
||||
/** The reporter that writes each skipped test and its reason (outcomes.ts), by its path. */
|
||||
const skipsReporter = fileURLToPath(new URL("./outcomes.ts", import.meta.url));
|
||||
|
||||
/**
|
||||
* Run some test files, passing their output through as it arrives, and collect what was skipped and why:
|
||||
* the spec reporter to stdout as before, and the skips reporter beside it into a file of the run's own.
|
||||
*/
|
||||
async function runFiles(files: string[]): Promise<{ code: number; seen: string; skips: Skip[] }> {
|
||||
const scratch = mkdtempSync(join(tmpdir(), "mesh-lab-suite-"));
|
||||
const skipsFile = join(scratch, "skips.jsonl");
|
||||
const running = spawn(
|
||||
process.execPath,
|
||||
["--test", "--test-concurrency=1", "--experimental-strip-types",
|
||||
"--test-reporter=spec", "--test-reporter-destination=stdout",
|
||||
`--test-reporter=${skipsReporter}`, `--test-reporter-destination=${skipsFile}`,
|
||||
...files],
|
||||
{ stdio: ["inherit", "pipe", "inherit"] },
|
||||
);
|
||||
@@ -81,7 +113,14 @@ async function runFiles(files: string[]): Promise<{ code: number; seen: string }
|
||||
const code: number = await new Promise((resolve) => {
|
||||
running.on("close", (c) => resolve(c ?? 1));
|
||||
});
|
||||
return { code, seen };
|
||||
// Named as the files were given, relative to where the suite runs, rather than as the runner's absolute paths.
|
||||
const named = (file: string) => {
|
||||
const rel = isAbsolute(file) ? relative(process.cwd(), file) : file;
|
||||
return rel.startsWith("..") ? file : rel;
|
||||
};
|
||||
const skips = readSkips(skipsFile).map((s) => ({ ...s, file: named(s.file) }));
|
||||
rmSync(scratch, { recursive: true, force: true });
|
||||
return { code, seen, skips };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -92,16 +131,31 @@ async function runFiles(files: string[]): Promise<{ code: number; seen: string }
|
||||
* could check it — which is 04-ISSUES/005 in miniature, inside the fix for it.
|
||||
*/
|
||||
export function reportOn(
|
||||
counts: { passed: number | null; failed: number | null },
|
||||
write: (passed: number, failed: number) => { against: Record<string, string>; ran: string[] },
|
||||
counts: { passed: number | null; failed: number | null; skipped?: number | null },
|
||||
skips: Skip[],
|
||||
write: (passed: number, failed: number, outcome: Outcome) => { against: Record<string, string>; ran: string[] },
|
||||
): string {
|
||||
if (counts.passed === null || counts.failed === null) {
|
||||
// A run whose result could not be read is a run nobody can say anything about. Writing
|
||||
// "0 failed" because nothing said otherwise is how a green record comes to mean nothing.
|
||||
return "could not read what the runner reported; no receipt written";
|
||||
}
|
||||
const receipt = write(counts.passed, counts.failed);
|
||||
// **A skip is said, with its reason, and a run that proved nothing says NOT PASSED** (novox/hq issue
|
||||
// 693): "0 passed, 0 failed" read as a clean run. The lab module reads these lines into its record.
|
||||
const skipped = counts.skipped ?? skips.length;
|
||||
const judged = judgeRun({ passed: counts.passed, failed: counts.failed }, skips);
|
||||
const outcome: Outcome = {
|
||||
skipped,
|
||||
skips: skips.map((s) => ({ ...s, reason: oneLine(s.reason) })),
|
||||
...(judged.passed ? {} : { notPassed: judged.why }),
|
||||
};
|
||||
const receipt = write(counts.passed, counts.failed, outcome);
|
||||
const against = Object.entries(receipt.against).map(([n, c]) => `${n} ${c}`).join(", ");
|
||||
return `recorded: ${receipt.ran.join(", ")} — ${counts.passed} passed, ` +
|
||||
`${counts.failed} failed, against ${against || "nothing in git"}`;
|
||||
const lines = [`recorded: ${receipt.ran.join(", ")} — ${counts.passed} passed, ` +
|
||||
`${counts.failed} failed, ${skipped} skipped, against ${against || "nothing in git"}`];
|
||||
if (!judged.passed) lines.push(`NOT PASSED: ${judged.why}`);
|
||||
// As the runner's own spec line has it, "<test> (<file>) # <reason>": a test's name may hold any
|
||||
// punctuation, so the file in brackets is what the lab module reads the line apart by.
|
||||
for (const s of outcome.skips) lines.push(`skipped: ${s.test} (${s.file}) # ${s.reason || "no reason given"}`);
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { IncusError } from "../src/incus/client.ts";
|
||||
import {
|
||||
buildBaseImage, failedMirrors, MIRRORS, mirrorlist, PACMAN_BACKOFF_MS, pacmanIn, type BaseDeps,
|
||||
} from "../src/lifecycle/base.ts";
|
||||
import { BASE_IMAGE_ALIAS } from "../src/lifecycle/place.ts";
|
||||
|
||||
/**
|
||||
* Defends novox/hq issue 700: the base build installed from the one mirror the stock image names,
|
||||
* mirrors.kernel.org, and failed twice in a morning when that mirror timed out and then answered 500.
|
||||
*/
|
||||
|
||||
const KERNEL_500 = "error: failed retrieving file 'wireguard-tools-1.0.20260223-1-x86_64.pkg.tar.zst' from " +
|
||||
"mirrors.kernel.org : The requested URL returned error: 500\nwarning: failed to retrieve some files\n" +
|
||||
"error: failed to commit transaction (failed to retrieve some files)";
|
||||
|
||||
/** A hypervisor for one base build: pacman fails `fails` times per step, then installs. */
|
||||
function aBuilder(fails: number, said = KERNEL_500) {
|
||||
const ran: string[] = [];
|
||||
const slept: number[] = [];
|
||||
let mirrorlistWritten = "";
|
||||
const tries = new Map<string, number>();
|
||||
const deps: BaseDeps = {
|
||||
incus: async (args) => {
|
||||
ran.push(args.join(" "));
|
||||
const at = args.indexOf("pacman");
|
||||
if (at >= 0) {
|
||||
const step = args.slice(at).join(" ");
|
||||
const n = (tries.get(step) ?? 0) + 1;
|
||||
tries.set(step, n);
|
||||
if (n <= fails) throw new IncusError(args, said, 1);
|
||||
}
|
||||
const printf = args.join(" ").match(/printf '%s' '([\s\S]*)' > \/etc\/pacman.d\/mirrorlist/);
|
||||
if (printf) mirrorlistWritten = printf[1]!;
|
||||
return { stdout: "", stderr: "" };
|
||||
},
|
||||
incusOk: async (args) => {
|
||||
const c = args.join(" ");
|
||||
if (c.endsWith("cat /etc/pacman.d/mirrorlist")) return mirrorlistWritten;
|
||||
if (c.includes("{{.ServerVersion}}")) return "29.8.2";
|
||||
if (c.includes("InsecureRegistryCIDRs")) return "[127.0.0.0/8 192.0.2.0/24]";
|
||||
if (c.startsWith("image list")) return BASE_IMAGE_ALIAS;
|
||||
return "tool 1.0";
|
||||
},
|
||||
succeeds: async () => true,
|
||||
sleep: async (ms) => { slept.push(ms); },
|
||||
};
|
||||
return { deps, ran, slept };
|
||||
}
|
||||
|
||||
test("the builder is given several mirrors, the stock one last, before anything is installed", async () => {
|
||||
const hosts = MIRRORS.map((m) => new URL(m.replace("$repo", "core").replace("$arch", "x86_64")).host);
|
||||
assert.ok(new Set(hosts).size >= 4, `too few mirrors: ${hosts.join(", ")}`);
|
||||
assert.equal(hosts.at(-1), "mirrors.kernel.org");
|
||||
assert.ok(!mirrorlist().includes("'"), "the mirrorlist is written inside single quotes and holds one");
|
||||
assert.equal(mirrorlist().split("\n").filter((l) => l.startsWith("Server = ")).length, MIRRORS.length);
|
||||
|
||||
const b = aBuilder(0);
|
||||
await buildBaseImage(() => {}, b.deps);
|
||||
const wrote = b.ran.findIndex((c) => c.includes("/etc/pacman.d/mirrorlist"));
|
||||
const firstPacman = b.ran.findIndex((c) => c.includes(" pacman "));
|
||||
assert.ok(wrote >= 0 && wrote < firstPacman, "pacman ran before the mirrorlist was written");
|
||||
});
|
||||
|
||||
test("a pacman step that fails is tried again after a pause, saying which mirror failed", async () => {
|
||||
const said: string[] = [];
|
||||
const b = aBuilder(2);
|
||||
await pacmanIn("builder", ["-S", "--noconfirm", "wireguard-tools"], (m) => said.push(m), b.deps);
|
||||
assert.equal(b.ran.length, 3);
|
||||
assert.deepEqual(b.slept, PACMAN_BACKOFF_MS.slice(0, 2));
|
||||
assert.ok(PACMAN_BACKOFF_MS[1]! > PACMAN_BACKOFF_MS[0]!, "no backoff");
|
||||
assert.equal(said.length, 2);
|
||||
assert.match(said[0]!, /failed from mirrors\.kernel\.org \(error: failed retrieving file 'wireguard-tools/);
|
||||
assert.match(said[0]!, /trying again in 10s \(attempt 2 of 4\)/);
|
||||
});
|
||||
|
||||
test("a step that keeps failing stops the build, naming every mirror that failed", async () => {
|
||||
const b = aBuilder(99, "error: too many errors from mirrors.kernel.org\nerror: failed retrieving file 'x' from ftp.fau.de : 500");
|
||||
await assert.rejects(
|
||||
pacmanIn("builder", ["-Sy", "--noconfirm", "docker"], () => {}, b.deps),
|
||||
/failed 4 times; the mirrors that failed: mirrors\.kernel\.org, ftp\.fau\.de/,
|
||||
);
|
||||
assert.equal(b.ran.length, PACMAN_BACKOFF_MS.length + 1);
|
||||
});
|
||||
|
||||
test("a whole base build rides out a mirror's bad minute on every package", async () => {
|
||||
const b = aBuilder(1);
|
||||
const built = await buildBaseImage(() => {}, b.deps);
|
||||
assert.equal(built.alias, BASE_IMAGE_ALIAS);
|
||||
const pacman = b.ran.filter((c) => c.includes(" pacman "));
|
||||
assert.equal(pacman.length, 10, pacman.join("\n"));
|
||||
});
|
||||
|
||||
test("the mirrors a failure names are read from what pacman said, each once", () => {
|
||||
assert.deepEqual(failedMirrors(KERNEL_500), ["mirrors.kernel.org"]);
|
||||
assert.deepEqual(failedMirrors("Connection timed out after 10000 milliseconds\nerror: too many errors from mirrors.kernel.org"),
|
||||
["mirrors.kernel.org"]);
|
||||
assert.deepEqual(failedMirrors("error: target not found: dockr"), []);
|
||||
});
|
||||
@@ -0,0 +1,584 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
type env struct {
|
||||
k Keys
|
||||
op, sys, acc nkeys.KeyPair
|
||||
sig nkeys.KeyPair
|
||||
revoked map[string]time.Time
|
||||
lastIat int64
|
||||
accountDoneAt time.Time
|
||||
}
|
||||
|
||||
func (e *env) url(ip string) string { return "nats://" + ip + ":4222" }
|
||||
|
||||
func say(format string, a ...any) {
|
||||
fmt.Printf("%s %s\n", time.Now().Format("15:04:05.000"), fmt.Sprintf(format, a...))
|
||||
}
|
||||
|
||||
func ms(d time.Duration) string { return fmt.Sprintf("%.1f ms", float64(d.Microseconds())/1000) }
|
||||
|
||||
type user struct {
|
||||
kp nkeys.KeyPair
|
||||
opt nats.Option
|
||||
}
|
||||
|
||||
func (e *env) newUser(name string) user {
|
||||
kp := must(nkeys.CreateUser())
|
||||
j := userJWT(e.acc, e.sig, kp, name)
|
||||
return user{kp, nats.UserJWT(func() (string, error) { return j, nil }, func(n []byte) ([]byte, error) { return kp.Sign(n) })}
|
||||
}
|
||||
|
||||
func (e *env) sysConn(ip string) *nats.Conn {
|
||||
kp := must(nkeys.CreateUser())
|
||||
uc := newSysUser(e.sys, kp)
|
||||
nc, err := nats.Connect(e.url(ip), nats.UserJWT(func() (string, error) { return uc, nil }, func(n []byte) ([]byte, error) { return kp.Sign(n) }), nats.Timeout(5*time.Second))
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("system connection to %s: %v", ip, err))
|
||||
}
|
||||
return nc
|
||||
}
|
||||
|
||||
// probe is one login attempt from this machine across the network.
|
||||
func (e *env) probe(ip string, u user) error {
|
||||
nc, err := nats.Connect(e.url(ip), u.opt, nats.NoReconnect(), nats.Timeout(3*time.Second))
|
||||
if err == nil {
|
||||
nc.Close()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
type open struct {
|
||||
nc *nats.Conn
|
||||
closed atomic.Int64
|
||||
}
|
||||
|
||||
func (e *env) openConn(ip string, u user) *open {
|
||||
o := &open{}
|
||||
nc, err := nats.Connect(e.url(ip), u.opt, nats.NoReconnect(), nats.Timeout(3*time.Second),
|
||||
nats.ClosedHandler(func(*nats.Conn) { o.closed.Store(time.Now().UnixNano()) }))
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("open connection to %s: %v", ip, err))
|
||||
}
|
||||
nc.Flush()
|
||||
o.nc = nc
|
||||
return o
|
||||
}
|
||||
|
||||
// push sends an account JWT to one server's resolver over the system account. The reply is not proof it applied.
|
||||
func (e *env) push(sys *nats.Conn, j string) (string, time.Duration) {
|
||||
t0 := time.Now()
|
||||
m, err := sys.Request("$SYS.REQ.ACCOUNT."+pubOf(e.acc)+".CLAIMS.UPDATE", []byte(j), 5*time.Second)
|
||||
if err != nil {
|
||||
return "error: " + err.Error(), time.Since(t0)
|
||||
}
|
||||
return string(m.Data), time.Since(t0)
|
||||
}
|
||||
|
||||
// nextAccountJWT builds the account JWT with the revocation list; its iat has to be a later second than the last one.
|
||||
func (e *env) nextAccountJWT() string {
|
||||
for time.Now().Unix() <= e.lastIat {
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
e.lastIat = time.Now().Unix()
|
||||
return accountJWT(e.op, e.acc, e.sig, e.revoked)
|
||||
}
|
||||
|
||||
func (e *env) refusedAfter(ip string, u user, t0 time.Time, limit time.Duration) (time.Duration, int) {
|
||||
n := 0
|
||||
for time.Since(t0) < limit {
|
||||
if err := e.probe(ip, u); err != nil {
|
||||
return time.Since(t0), n
|
||||
}
|
||||
n++
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
return -1, n
|
||||
}
|
||||
|
||||
func newSysUser(sys, ukp nkeys.KeyPair) string { return userClaimsFor(sys, ukp) }
|
||||
|
||||
func drive(args []string) {
|
||||
fs := flag.NewFlagSet("drive", flag.ExitOnError)
|
||||
keysFile := fs.String("keys", "/root/q8/keys.json", "keys")
|
||||
only := fs.String("only", "2,4,3", "which parts, in order")
|
||||
trials := fs.Int("trials", 3, "trials per part")
|
||||
fs.Parse(args)
|
||||
raw, err := os.ReadFile(*keysFile)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
e := &env{revoked: map[string]time.Time{}}
|
||||
json.Unmarshal(raw, &e.k)
|
||||
e.op, e.sys, e.acc, e.sig = fromSeed(e.k.Op), fromSeed(e.k.Sys), fromSeed(e.k.Acc), fromSeed(e.k.Sig)
|
||||
e.lastIat = time.Now().Unix()
|
||||
// the account JWT the servers were preloaded with has an iat from gen time; ours must be later
|
||||
time.Sleep(1100 * time.Millisecond)
|
||||
if d := waitRoutes(e, 1, 90*time.Second); d < 0 {
|
||||
say("the route between B and C did not come up")
|
||||
} else {
|
||||
say("cluster formed: route B-C up after %s", ms(d))
|
||||
}
|
||||
for _, p := range strings.Split(*only, ",") {
|
||||
switch p {
|
||||
case "2":
|
||||
part2(e, *trials)
|
||||
case "3":
|
||||
part3(e, *trials)
|
||||
case "4":
|
||||
part4(e)
|
||||
}
|
||||
}
|
||||
say("DRIVE-DONE")
|
||||
}
|
||||
|
||||
func routes(ip string) int {
|
||||
r, err := http.Get("http://" + ip + ":8222/routez")
|
||||
if err != nil {
|
||||
return -1
|
||||
}
|
||||
defer r.Body.Close()
|
||||
var v struct {
|
||||
NumRoutes int `json:"num_routes"`
|
||||
}
|
||||
json.NewDecoder(r.Body).Decode(&v)
|
||||
return v.NumRoutes
|
||||
}
|
||||
|
||||
func waitRoutes(e *env, want int, limit time.Duration) time.Duration {
|
||||
t0 := time.Now()
|
||||
for time.Since(t0) < limit {
|
||||
if rb, rc := routes(e.k.B), routes(e.k.C); (want == 0 && rb == 0 && rc == 0) || (want > 0 && rb >= want && rc >= want) {
|
||||
return time.Since(t0)
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func agent(e *env, what string) (string, time.Time) {
|
||||
r, err := http.Get("http://" + e.k.C + ":9099/" + what)
|
||||
t := time.Now()
|
||||
if err != nil {
|
||||
return "error: " + err.Error(), t
|
||||
}
|
||||
defer r.Body.Close()
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
return string(b), t
|
||||
}
|
||||
|
||||
// ---- part 2: revocation across the network ----
|
||||
|
||||
func part2(e *env, n int) {
|
||||
say("=== PART 2: revocation over the network (B and C, pushed to B from A)")
|
||||
sysB := e.sysConn(e.k.B)
|
||||
defer sysB.Close()
|
||||
for run := 1; run <= n; run++ {
|
||||
u := e.newUser(fmt.Sprintf("p2-%d", run))
|
||||
ctl := e.newUser(fmt.Sprintf("p2-ctl-%d", run))
|
||||
ob, oc := e.openConn(e.k.B, u), e.openConn(e.k.C, u)
|
||||
cb, cc := e.openConn(e.k.B, ctl), e.openConn(e.k.C, ctl)
|
||||
e.revoked[pubOf(u.kp)] = time.Now()
|
||||
j := e.nextAccountJWT()
|
||||
t0 := time.Now()
|
||||
reply, rtt := e.push(sysB, j)
|
||||
type res struct {
|
||||
ip string
|
||||
refused time.Duration
|
||||
acc int
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
out := make([]res, 2)
|
||||
for i, ip := range []string{e.k.B, e.k.C} {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
d, a := e.refusedAfter(ip, u, t0, 30*time.Second)
|
||||
out[i] = res{ip, d, a}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
cl := func(o *open) string {
|
||||
if v := o.closed.Load(); v != 0 {
|
||||
return ms(time.Duration(v - t0.UnixNano()))
|
||||
}
|
||||
return "NOT closed"
|
||||
}
|
||||
alive := func(o *open) bool { return o.nc.IsConnected() }
|
||||
say("RESULT part2 run %d: push reply %q rtt %s; new login refused after: B %s (%d accepted before), C %s (%d accepted before); open connection closed after: B %s, C %s; control user still connected: B %v, C %v",
|
||||
run, strings.TrimSpace(reply)[:min(len(strings.TrimSpace(reply)), 60)], ms(rtt), ms(out[0].refused), out[0].acc, ms(out[1].refused), out[1].acc, cl(ob), cl(oc), alive(cb), alive(cc))
|
||||
for _, o := range []*open{ob, oc, cb, cc} {
|
||||
o.nc.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- part 3: C cut off while a user is revoked ----
|
||||
|
||||
type seen struct {
|
||||
at time.Time
|
||||
subject string
|
||||
data []byte
|
||||
}
|
||||
|
||||
func part3(e *env, n int) {
|
||||
say("=== PART 3: C cut off from B while a user is revoked; modes a (periodic sync only) and b (re-push on C's return)")
|
||||
for _, mode := range []string{"a", "b"} {
|
||||
for run := 1; run <= n; run++ {
|
||||
trial3(e, mode, run)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func trial3(e *env, mode string, run int) {
|
||||
say("--- trial %s%d", mode, run)
|
||||
if d := waitRoutes(e, 1, 90*time.Second); d < 0 {
|
||||
say("RESULT part3 %s%d: the route B-C did not come up (routes B=%d C=%d); trial skipped", mode, run, routes(e.k.B), routes(e.k.C))
|
||||
return
|
||||
}
|
||||
time.Sleep(3 * time.Second)
|
||||
sysB, sysC := e.sysConn(e.k.B), e.sysConn(e.k.C)
|
||||
defer sysB.Close()
|
||||
defer sysC.Close()
|
||||
u, ctl := e.newUser(fmt.Sprintf("p3-%s%d", mode, run)), e.newUser(fmt.Sprintf("p3-ctl-%s%d", mode, run))
|
||||
oc, cc := e.openConn(e.k.C, u), e.openConn(e.k.C, ctl)
|
||||
defer oc.nc.Close()
|
||||
defer cc.nc.Close()
|
||||
var mu sync.Mutex
|
||||
var events []seen
|
||||
sysB.Subscribe("$SYS.>", func(m *nats.Msg) {
|
||||
mu.Lock()
|
||||
events = append(events, seen{time.Now(), m.Subject, m.Data})
|
||||
mu.Unlock()
|
||||
})
|
||||
sysB.Flush()
|
||||
|
||||
r, tCut := agent(e, "block")
|
||||
say("cut: agent said %q", r)
|
||||
dr := waitRoutes(e, 0, 60*time.Second)
|
||||
say("routes gone on B and C after %s (ping interval 2s, 2 missed pings)", ms(dr))
|
||||
time.Sleep(1100 * time.Millisecond)
|
||||
e.revoked[pubOf(u.kp)] = time.Now()
|
||||
j := e.nextAccountJWT()
|
||||
tPush := time.Now()
|
||||
reply, rtt := e.push(sysB, j)
|
||||
dB, _ := e.refusedAfter(e.k.B, u, tPush, 10*time.Second)
|
||||
time.Sleep(4 * time.Second)
|
||||
accC := e.probe(e.k.C, u)
|
||||
say("partitioned: push to B reply %q rtt %s; B refused the user after %s; C still accepts the user: %v (cut lasted %s so far)", strings.TrimSpace(reply)[:min(len(strings.TrimSpace(reply)), 40)], ms(rtt), ms(dB), accC == nil, time.Since(tCut).Round(time.Second))
|
||||
|
||||
mu.Lock()
|
||||
events = nil
|
||||
mu.Unlock()
|
||||
r, tHeal := agent(e, "unblock")
|
||||
say("healed: agent said %q", r)
|
||||
|
||||
var trigger atomic.Int64 // offset ns of the event used as trigger
|
||||
var triggerSubj atomic.Value
|
||||
var pushedRTT atomic.Int64
|
||||
var pushedReply atomic.Value
|
||||
if mode == "b" {
|
||||
go func() {
|
||||
deadline := time.Now().Add(150 * time.Second)
|
||||
for time.Now().Before(deadline) && trigger.Load() == 0 {
|
||||
mu.Lock()
|
||||
var hit *seen
|
||||
for i := range events {
|
||||
ev := events[i]
|
||||
if strings.HasPrefix(ev.subject, "$SYS.SERVER.") && strings.HasSuffix(ev.subject, ".STATSZ") && ev.at.After(tHeal) {
|
||||
var v struct {
|
||||
Server struct{ Name string } `json:"server"`
|
||||
}
|
||||
if json.Unmarshal(ev.data, &v) == nil && v.Server.Name == "C" {
|
||||
hit = &ev
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
mu.Unlock()
|
||||
if hit != nil {
|
||||
trigger.Store(int64(hit.at.Sub(tHeal)))
|
||||
triggerSubj.Store(hit.subject)
|
||||
rep, rt := e.push(sysC, j)
|
||||
pushedRTT.Store(int64(rt))
|
||||
pushedReply.Store(rep)
|
||||
return
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
}()
|
||||
}
|
||||
refused, accepted := e.refusedAfter(e.k.C, u, tHeal, 150*time.Second)
|
||||
time.Sleep(700 * time.Millisecond)
|
||||
closedAt := "NOT closed"
|
||||
if v := oc.closed.Load(); v != 0 {
|
||||
closedAt = ms(time.Duration(v - tHeal.UnixNano()))
|
||||
}
|
||||
line := fmt.Sprintf("RESULT part3 %s%d: after the link came back C accepted the revoked user for %s (%d logins accepted), open connection closed after %s, control user still connected %v",
|
||||
mode, run, ms(refused), accepted, closedAt, cc.nc.IsConnected())
|
||||
if mode == "b" {
|
||||
if t := trigger.Load(); t != 0 {
|
||||
subj, _ := triggerSubj.Load().(string)
|
||||
rep, _ := pushedReply.Load().(string)
|
||||
line += fmt.Sprintf("; trigger: first STATSZ from server C arrived on B's system account %s after the link came back (%s), push to C reply %q rtt %s",
|
||||
ms(time.Duration(t)), strings.ReplaceAll(subj, "$SYS.SERVER.", "$SYS.SERVER.<id>."), strings.TrimSpace(rep)[:min(len(strings.TrimSpace(rep)), 30)], ms(time.Duration(pushedRTT.Load())))
|
||||
} else {
|
||||
line += "; trigger NEVER fired"
|
||||
}
|
||||
}
|
||||
say("%s", line)
|
||||
if run == 1 {
|
||||
mu.Lock()
|
||||
first := map[string]time.Duration{}
|
||||
count := map[string]int{}
|
||||
for _, ev := range events {
|
||||
k := normalise(ev.subject)
|
||||
if _, ok := first[k]; !ok {
|
||||
first[k] = ev.at.Sub(tHeal)
|
||||
}
|
||||
count[k]++
|
||||
}
|
||||
mu.Unlock()
|
||||
var ks []string
|
||||
for k := range first {
|
||||
ks = append(ks, k)
|
||||
}
|
||||
sort.Slice(ks, func(i, j int) bool { return first[ks[i]] < first[ks[j]] })
|
||||
say("system-account subjects seen on B after the link came back (trial %s%d), first-seen offset, count:", mode, run)
|
||||
for _, k := range ks {
|
||||
say(" %9s x%-3d %s", ms(first[k]), count[k], k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func normalise(s string) string {
|
||||
parts := strings.Split(s, ".")
|
||||
for i, p := range parts {
|
||||
if len(p) >= 40 && strings.ToUpper(p) == p {
|
||||
parts[i] = "<id>"
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, ".")
|
||||
}
|
||||
|
||||
// ---- part 4: moving a stream and a KV bucket from A to B over the leaf link ----
|
||||
|
||||
func part4(e *env) {
|
||||
say("=== PART 4: stream S (10000 msgs) and KV bucket cfg moved from A (legacy) to B/C (hub) over the leaf link, writer running")
|
||||
ctx := context.Background()
|
||||
lc, err := nats.Connect(e.url(e.k.A), nats.UserInfo("w", "pw"))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer lc.Close()
|
||||
ljs := must(jetstream.New(lc))
|
||||
must(ljs.CreateStream(ctx, jetstream.StreamConfig{Name: "S", Subjects: []string{"mesh.s.>"}}))
|
||||
t0 := time.Now()
|
||||
for i := 0; i < 10000; i++ {
|
||||
if _, err := ljs.Publish(ctx, fmt.Sprintf("mesh.s.%d", i%50), []byte(fmt.Sprintf("seed-%d", i)), jetstream.WithMsgID(fmt.Sprintf("seed-%d", i))); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
say("seeded S with 10000 messages on A in %s", time.Since(t0).Round(time.Millisecond))
|
||||
kv := must(ljs.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "cfg", History: 5}))
|
||||
for i := 0; i < 1000; i++ {
|
||||
must(kv.Put(ctx, fmt.Sprintf("k%d", i), []byte(fmt.Sprintf("v%d", i))))
|
||||
}
|
||||
for i := 0; i < 200; i++ {
|
||||
for r := 0; r < 3; r++ {
|
||||
must(kv.Put(ctx, fmt.Sprintf("k%d", i), []byte(fmt.Sprintf("v%d-r%d", i, r))))
|
||||
}
|
||||
}
|
||||
kv.Delete(ctx, "k999")
|
||||
say("seeded KV cfg on A")
|
||||
|
||||
for t := time.Now(); time.Since(t) < 60*time.Second; time.Sleep(500 * time.Millisecond) {
|
||||
r, err := http.Get("http://" + e.k.B + ":8222/leafz")
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var v struct {
|
||||
Leafs int `json:"leafnodes"`
|
||||
}
|
||||
json.NewDecoder(r.Body).Decode(&v)
|
||||
r.Body.Close()
|
||||
if v.Leafs >= 2 {
|
||||
say("leaf links from A to B up (%d connections: MESH and SYS)", v.Leafs)
|
||||
break
|
||||
}
|
||||
}
|
||||
u := e.newUser("p4")
|
||||
hc, err := nats.Connect(e.url(e.k.B), u.opt)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer hc.Close()
|
||||
hjs := must(jetstream.New(hc))
|
||||
ready := time.Now()
|
||||
for time.Since(ready) < 60*time.Second {
|
||||
if _, err := hjs.AccountInfo(ctx); err == nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
say("hub JetStream answered after %s", time.Since(ready).Round(time.Millisecond))
|
||||
|
||||
var stop atomic.Bool
|
||||
var acked, kvacked, failed atomic.Int64
|
||||
done := make(chan bool)
|
||||
go func() {
|
||||
for i := 0; !stop.Load(); i++ {
|
||||
if _, err := ljs.Publish(ctx, fmt.Sprintf("mesh.s.%d", i%50), []byte(fmt.Sprintf("live-%d", i)), jetstream.WithMsgID(fmt.Sprintf("live-%d", i))); err == nil {
|
||||
acked.Add(1)
|
||||
} else {
|
||||
failed.Add(1)
|
||||
}
|
||||
if i%10 == 0 {
|
||||
if _, err := kv.Put(ctx, fmt.Sprintf("live%d", i%20), []byte(fmt.Sprintf("x%d", i))); err == nil {
|
||||
kvacked.Add(1)
|
||||
}
|
||||
}
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
}
|
||||
close(done)
|
||||
}()
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
|
||||
tm := time.Now()
|
||||
for _, n := range []string{"S", "KV_cfg"} {
|
||||
l := must(ljs.Stream(ctx, n))
|
||||
cfg := l.CachedInfo().Config
|
||||
cfg.Subjects = nil
|
||||
cfg.Sources = []*jetstream.StreamSource{{Name: n, External: &jetstream.ExternalStream{APIPrefix: "$JS.legacy.API"}}}
|
||||
if _, err := hjs.CreateStream(ctx, cfg); err != nil {
|
||||
panic(fmt.Sprintf("create source stream %s: %v", n, err))
|
||||
}
|
||||
}
|
||||
msgs := func(js jetstream.JetStream, n string) uint64 {
|
||||
s, err := js.Stream(ctx, n)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
i, err := s.Info(ctx)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return i.State.Msgs
|
||||
}
|
||||
waitEq := func(n string, limit time.Duration) time.Duration {
|
||||
t := time.Now()
|
||||
for time.Since(t) < limit {
|
||||
if msgs(hjs, n) == msgs(ljs, n) && msgs(hjs, n) > 0 {
|
||||
return time.Since(tm)
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
return -1
|
||||
}
|
||||
say("catch-up under writes, counts equal after: S %s, KV_cfg %s (the writer keeps going)", ms(waitEq("S", 60*time.Second)), ms(waitEq("KV_cfg", 60*time.Second)))
|
||||
time.Sleep(3 * time.Second)
|
||||
stop.Store(true)
|
||||
<-done
|
||||
say("writer stopped: S acked %d (failed %d), KV puts acked %d", acked.Load(), failed.Load(), kvacked.Load())
|
||||
tq := time.Now()
|
||||
for _, n := range []string{"S", "KV_cfg"} {
|
||||
for time.Since(tq) < 30*time.Second && msgs(hjs, n) != msgs(ljs, n) {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
say("quiesced: equal after %s; counts S legacy %d hub %d (expect %d); KV_cfg legacy %d hub %d", time.Since(tq).Round(time.Millisecond),
|
||||
msgs(ljs, "S"), msgs(hjs, "S"), 10000+acked.Load(), msgs(ljs, "KV_cfg"), msgs(hjs, "KV_cfg"))
|
||||
|
||||
// content: every message once on both sides
|
||||
readAll := func(js jetstream.JetStream) (map[string]int, int) {
|
||||
s := must(js.Stream(ctx, "S"))
|
||||
info := must(s.Info(ctx))
|
||||
set := map[string]int{}
|
||||
total := 0
|
||||
for seq := info.State.FirstSeq; seq <= info.State.LastSeq; seq++ {
|
||||
m, err := s.GetMsg(ctx, seq)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
set[string(m.Data)]++
|
||||
total++
|
||||
}
|
||||
return set, total
|
||||
}
|
||||
ls, lt := readAll(ljs)
|
||||
hs, ht := readAll(hjs)
|
||||
dup, missing, extra := 0, 0, 0
|
||||
for k, c := range hs {
|
||||
if c > 1 {
|
||||
dup += c - 1
|
||||
}
|
||||
if _, ok := ls[k]; !ok {
|
||||
extra++
|
||||
}
|
||||
}
|
||||
for k := range ls {
|
||||
if _, ok := hs[k]; !ok {
|
||||
missing++
|
||||
}
|
||||
}
|
||||
say("RESULT part4 stream S: messages legacy %d (distinct %d) hub %d (distinct %d); on hub: duplicates %d, missing vs legacy %d, not on legacy %d; expected %d = 10000 seed + %d acked live",
|
||||
lt, len(ls), ht, len(hs), dup, missing, extra, 10000+acked.Load(), acked.Load())
|
||||
|
||||
hkv := must(hjs.KeyValue(ctx, "cfg"))
|
||||
lk := must(kv.Keys(ctx))
|
||||
hk := must(hkv.Keys(ctx))
|
||||
diff := 0
|
||||
for _, k := range lk {
|
||||
a, err1 := kv.Get(ctx, k)
|
||||
b, err2 := hkv.Get(ctx, k)
|
||||
if err1 != nil || err2 != nil || string(a.Value()) != string(b.Value()) {
|
||||
diff++
|
||||
}
|
||||
}
|
||||
say("RESULT part4 KV cfg: keys legacy %d hub %d; keys whose latest value differs or is missing on hub: %d", len(lk), len(hk), diff)
|
||||
|
||||
// cut-over: the hub stream stops sourcing and takes over the subjects
|
||||
tc := time.Now()
|
||||
subj := map[string][]string{"S": {"mesh.s.>"}, "KV_cfg": {"$KV.cfg.>"}}
|
||||
for n, ss := range subj {
|
||||
h := must(hjs.Stream(ctx, n))
|
||||
cfg := h.CachedInfo().Config
|
||||
cfg.Sources = nil
|
||||
cfg.Subjects = ss
|
||||
if _, err := hjs.UpdateStream(ctx, cfg); err != nil {
|
||||
say("cut-over %s FAILED: %v", n, err)
|
||||
}
|
||||
}
|
||||
say("cut-over update (sources removed, subjects added, 2 streams): %s", time.Since(tc).Round(time.Millisecond))
|
||||
before := msgs(hjs, "S")
|
||||
if _, err := hjs.Publish(ctx, "mesh.s.1", []byte("after-cutover")); err != nil {
|
||||
say("RESULT part4 publish on hub after cut-over FAILED: %v", err)
|
||||
} else {
|
||||
say("RESULT part4 after cut-over a publish on the hub lands in the hub stream: %d -> %d messages (legacy stays %d)", before, msgs(hjs, "S"), msgs(ljs, "S"))
|
||||
}
|
||||
if _, err := hkv.Put(ctx, "after", []byte("1")); err != nil {
|
||||
say("RESULT part4 KV put on hub after cut-over FAILED: %v", err)
|
||||
} else {
|
||||
g, err := hkv.Get(ctx, "after")
|
||||
say("RESULT part4 KV put on hub after cut-over: readable %v", err == nil && string(g.Value()) == "1")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/jwt/v2"
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
// Keys is what the driver on machine A needs to act as the operator, the identity seat and a system user.
|
||||
type Keys struct {
|
||||
Op, Sys, Acc, Sig string // seeds
|
||||
A, B, C string // addresses
|
||||
}
|
||||
|
||||
func must[T any](v T, err error) T {
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func pubOf(kp nkeys.KeyPair) string { return must(kp.PublicKey()) }
|
||||
|
||||
func fromSeed(s string) nkeys.KeyPair { return must(nkeys.FromSeed([]byte(s))) }
|
||||
|
||||
func seedOf(kp nkeys.KeyPair) string { return string(must(kp.Seed())) }
|
||||
|
||||
// accountJWT is the MESH account: JetStream limits, one signing key, a revocation list.
|
||||
func accountJWT(op, acc, sig nkeys.KeyPair, revoked map[string]time.Time) string {
|
||||
ac := jwt.NewAccountClaims(pubOf(acc))
|
||||
ac.Name = "MESH"
|
||||
ac.Limits.JetStreamLimits = jwt.JetStreamLimits{MemoryStorage: 64 << 20, DiskStorage: 1 << 30, Streams: 20, Consumer: 50}
|
||||
ac.SigningKeys.Add(pubOf(sig))
|
||||
if len(revoked) > 0 {
|
||||
ac.Revocations = jwt.RevocationList{}
|
||||
for k, t := range revoked {
|
||||
ac.Revocations.Revoke(k, t)
|
||||
}
|
||||
}
|
||||
return must(ac.Encode(op))
|
||||
}
|
||||
|
||||
// userJWT: a user of the MESH account signed by the account's signing key, allowed everything.
|
||||
func userJWT(acc, sig, ukp nkeys.KeyPair, name string) string {
|
||||
uc := jwt.NewUserClaims(pubOf(ukp))
|
||||
uc.Name = name
|
||||
uc.IssuerAccount = pubOf(acc)
|
||||
uc.Pub.Allow.Add(">")
|
||||
uc.Sub.Allow.Add(">")
|
||||
return must(uc.Encode(sig))
|
||||
}
|
||||
|
||||
func creds(token string, ukp nkeys.KeyPair) []byte {
|
||||
return must(jwt.FormatUserConfig(token, []byte(seedOf(ukp))))
|
||||
}
|
||||
|
||||
func gen(args []string) {
|
||||
fs := flag.NewFlagSet("gen", flag.ExitOnError)
|
||||
out := fs.String("out", "", "output directory")
|
||||
a := fs.String("a", "", "address of A")
|
||||
b := fs.String("b", "", "address of B")
|
||||
c := fs.String("c", "", "address of C")
|
||||
fs.Parse(args)
|
||||
os.MkdirAll(*out, 0o755)
|
||||
op := must(nkeys.CreateOperator())
|
||||
sys := must(nkeys.CreateAccount())
|
||||
acc := must(nkeys.CreateAccount())
|
||||
sig := must(nkeys.CreateAccount())
|
||||
oc := jwt.NewOperatorClaims(pubOf(op))
|
||||
oc.Name = "op"
|
||||
oc.SystemAccount = pubOf(sys)
|
||||
opJWT := must(oc.Encode(op))
|
||||
sc := jwt.NewAccountClaims(pubOf(sys))
|
||||
sc.Name = "SYS"
|
||||
sysJWT := must(sc.Encode(op))
|
||||
accJ := accountJWT(op, acc, sig, nil)
|
||||
|
||||
keys := Keys{seedOf(op), seedOf(sys), seedOf(acc), seedOf(sig), *a, *b, *c}
|
||||
kj, _ := json.MarshalIndent(keys, "", " ")
|
||||
w := func(name string, data []byte) { must(0, os.WriteFile(filepath.Join(*out, name), data, 0o600)) }
|
||||
w("keys.json", kj)
|
||||
w("op.jwt", []byte(opJWT))
|
||||
|
||||
// the legacy server joins as a leaf with one user in each of the two accounts
|
||||
lk := must(nkeys.CreateUser())
|
||||
w("leaf-mesh.creds", creds(userJWT(acc, sig, lk, "legacy-leaf"), lk))
|
||||
sk := must(nkeys.CreateUser())
|
||||
suc := jwt.NewUserClaims(pubOf(sk))
|
||||
suc.Name = "legacy-leaf-sys"
|
||||
w("leaf-sys.creds", creds(must(suc.Encode(sys)), sk))
|
||||
|
||||
common := func(name, domain string) string {
|
||||
return fmt.Sprintf(`listen: 0.0.0.0:4222
|
||||
server_name: %s
|
||||
http_port: 8222
|
||||
ping_interval: "2s"
|
||||
ping_max: 2
|
||||
log_file: "/var/log/q8-%s.log"
|
||||
jetstream: {store_dir: "/var/lib/q8/%s/js", domain: %s}
|
||||
`, name, name, name, domain)
|
||||
}
|
||||
hub := func(name, peer string) string {
|
||||
return common(name, "hub") + fmt.Sprintf(`operator: "/root/q8/op.jwt"
|
||||
system_account: %s
|
||||
resolver: {type: full, dir: "/var/lib/q8/%s/accts", allow_delete: false}
|
||||
resolver_preload: {%s: %q, %s: %q}
|
||||
cluster { name: HUB, listen: 0.0.0.0:6222, routes: ["nats-route://%s:6222"] }
|
||||
leafnodes { listen: 0.0.0.0:7422 }
|
||||
`, pubOf(sys), name, pubOf(sys), sysJWT, pubOf(acc), accJ, peer)
|
||||
}
|
||||
w("B.conf", []byte(hub("B", *c)))
|
||||
w("C.conf", []byte(hub("C", *b)))
|
||||
w("A.conf", []byte(common("A", "legacy")+fmt.Sprintf(`accounts {
|
||||
MESH { jetstream: enabled
|
||||
users: [ {user: w, password: pw} ] }
|
||||
SYS { users: [ {user: sys, password: sys} ] }
|
||||
}
|
||||
system_account: SYS
|
||||
leafnodes { remotes [
|
||||
{url: "nats-leaf://%s:7422", account: MESH, credentials: "/root/q8/leaf-mesh.creds"},
|
||||
{url: "nats-leaf://%s:7422", account: SYS, credentials: "/root/q8/leaf-sys.creds"}
|
||||
] }
|
||||
`, *b, *b)))
|
||||
}
|
||||
|
||||
func userClaimsFor(sys, ukp nkeys.KeyPair) string {
|
||||
uc := jwt.NewUserClaims(pubOf(ukp))
|
||||
uc.Name = "sys"
|
||||
return must(uc.Encode(sys))
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
module signedlogins
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
github.com/nats-io/jwt/v2 v2.8.1
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
github.com/nats-io/nkeys v0.4.16
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,25 @@
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
@@ -0,0 +1,36 @@
|
||||
// Command q8lab is the program behind test/integration/signed-logins.test.ts (research 053, question 8).
|
||||
//
|
||||
// q8lab gen -out DIR -a IP -b IP -c IP writes keys and the three servers' configs (runs on the workstation)
|
||||
// q8lab server -conf FILE runs one nats-server 2.11.17 (embedded) until killed
|
||||
// q8lab agent -peer IP on machine C: an HTTP switch that cuts and restores traffic to the peer
|
||||
// q8lab wait -url URL waits until the URL answers 200
|
||||
// q8lab drive -keys FILE on machine A: runs the measurements and prints them
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 2 {
|
||||
fmt.Println("usage: q8lab gen|server|agent|wait|drive")
|
||||
os.Exit(2)
|
||||
}
|
||||
args := os.Args[2:]
|
||||
switch os.Args[1] {
|
||||
case "gen":
|
||||
gen(args)
|
||||
case "server":
|
||||
runServer(args)
|
||||
case "agent":
|
||||
runAgent(args)
|
||||
case "wait":
|
||||
runWait(args)
|
||||
case "drive":
|
||||
drive(args)
|
||||
default:
|
||||
fmt.Println("unknown command", os.Args[1])
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
)
|
||||
|
||||
func runServer(args []string) {
|
||||
fs := flag.NewFlagSet("server", flag.ExitOnError)
|
||||
conf := fs.String("conf", "", "config file")
|
||||
fs.Parse(args)
|
||||
opts, err := server.ProcessConfigFile(*conf)
|
||||
if err != nil {
|
||||
fmt.Println("config:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
opts.NoSigs = true
|
||||
opts.Debug = os.Getenv("Q8DEBUG") != ""
|
||||
s, err := server.NewServer(opts)
|
||||
if err != nil {
|
||||
fmt.Println("server:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
s.ConfigureLogger()
|
||||
go s.Start()
|
||||
if !s.ReadyForConnections(30 * time.Second) {
|
||||
fmt.Println("not ready")
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("ready", server.VERSION)
|
||||
ch := make(chan os.Signal, 1)
|
||||
signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
|
||||
<-ch
|
||||
s.Shutdown()
|
||||
}
|
||||
|
||||
// runAgent is a switch for machine C: it drops every packet to and from the peer, and takes the rule away again.
|
||||
func runAgent(args []string) {
|
||||
fs := flag.NewFlagSet("agent", flag.ExitOnError)
|
||||
peer := fs.String("peer", "", "address to cut off")
|
||||
fs.Parse(args)
|
||||
nft := func(script string) string {
|
||||
cmd := exec.Command("nft", "-f", "-")
|
||||
cmd.Stdin = strings.NewReader(script)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Sprintf("error: %v: %s", err, out)
|
||||
}
|
||||
return "ok"
|
||||
}
|
||||
http.HandleFunc("/block", func(w http.ResponseWriter, r *http.Request) {
|
||||
nft("delete table inet q8cut\n") // ignore: it may not exist
|
||||
io.WriteString(w, nft(fmt.Sprintf(`add table inet q8cut
|
||||
add chain inet q8cut in { type filter hook input priority -10; policy accept; }
|
||||
add chain inet q8cut out { type filter hook output priority -10; policy accept; }
|
||||
add rule inet q8cut in ip saddr %s drop
|
||||
add rule inet q8cut out ip daddr %s drop
|
||||
`, *peer, *peer)))
|
||||
})
|
||||
http.HandleFunc("/unblock", func(w http.ResponseWriter, r *http.Request) {
|
||||
io.WriteString(w, nft("delete table inet q8cut\n"))
|
||||
})
|
||||
fmt.Println("agent listening on :9099 for peer", *peer)
|
||||
fmt.Println(http.ListenAndServe(":9099", nil))
|
||||
}
|
||||
|
||||
func runWait(args []string) {
|
||||
fs := flag.NewFlagSet("wait", flag.ExitOnError)
|
||||
url := fs.String("url", "", "url")
|
||||
secs := fs.Int("s", 60, "seconds")
|
||||
fs.Parse(args)
|
||||
end := time.Now().Add(time.Duration(*secs) * time.Second)
|
||||
for time.Now().Before(end) {
|
||||
if r, err := http.Get(*url); err == nil {
|
||||
r.Body.Close()
|
||||
if r.StatusCode == 200 {
|
||||
fmt.Println("up")
|
||||
return
|
||||
}
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
fmt.Println("timeout")
|
||||
os.Exit(1)
|
||||
}
|
||||
@@ -50,8 +50,8 @@ const STILL_CARRIED: Record<string, { modules: string[]; why: string }> = {
|
||||
"assigned-catalogue-small.test.ts": { modules: ["postgres", "minio", "redis", "plex"],
|
||||
why: "BESIDE (postgres); DIFFERS (minio's root password by env-file, redis minting its own secret instead of the vault's, plex without its server)" },
|
||||
"assigned-model-usage.test.ts": { modules: ["postgres"], why: "BESIDE" },
|
||||
"assigned-two-node-db.test.ts": { modules: ["redis", "baserow", "letta"],
|
||||
why: "DIFFERS: redis mints its own secret, baserow drops its route requirement, letta drops its ports" },
|
||||
"assigned-two-node-db.test.ts": { modules: ["baserow", "letta"],
|
||||
why: "DIFFERS: baserow drops its route requirement, letta drops its ports" },
|
||||
"lavinmq-bed.test.ts": { modules: ["lavinmq", "amqp-ping"],
|
||||
why: "BESIDE (lavinmq, with a bootstrap step and a data directory the catalogue has not got); DIFFERS (amqp-ping names its entrypoint)" },
|
||||
};
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import { busNotKept, runningFrom, streamsFrom, written, type Seen } from "./integration/bus-kept.ts";
|
||||
|
||||
/**
|
||||
* The judgement of the bed that raises genesis past its broker step (novox/hq ADR 0333 §5, issue 590):
|
||||
* exactly one bus container, it is mesh-broker-nats, a stream written before the step is still there, and
|
||||
* the first node is still connected. Held here without a lab, so the bed's verdict is the judgement's and
|
||||
* not whatever a bed happened to compare.
|
||||
*/
|
||||
|
||||
const UPSTREAM = "nats@sha256:" + "b".repeat(64);
|
||||
const MODULES = "192.0.2.10:5000/nats@sha256:" + "c".repeat(64);
|
||||
const PORTS = "127.0.0.1:8222->8222/tcp, 0.0.0.0:5671->4222/tcp";
|
||||
|
||||
function jsz(streams: Array<[string, string, number, number]>): string {
|
||||
return JSON.stringify({ account_details: [{ name: "MESH", stream_detail: streams.map(([name, created, messages, last]) =>
|
||||
({ name, created, state: { messages, last_seq: last } })) }] });
|
||||
}
|
||||
|
||||
const before: Seen = {
|
||||
running: runningFrom(`mesh-store\tpostgres@sha256:aa\t0.0.0.0:5432->5432/tcp\nmesh-broker-nats\t${UPSTREAM}\t${PORTS}\n` +
|
||||
"mesh-controller\t192.0.2.10:5000/mesh-controller@sha256:dd\t\n"),
|
||||
streams: streamsFrom(jsz([["KV_mesh-controller_lease", "2026-10-13T00:10:00Z", 3, 7], ["EMPTY", "2026-10-13T00:10:01Z", 0, 0]])),
|
||||
};
|
||||
|
||||
const kept: Seen = {
|
||||
running: runningFrom(`mesh-store\tpostgres@sha256:aa\t0.0.0.0:5432->5432/tcp\nmesh-broker-nats\t${MODULES}\t${PORTS}\n` +
|
||||
"mesh-controller\t192.0.2.10:5000/mesh-controller@sha256:dd\t\n"),
|
||||
streams: streamsFrom(jsz([["KV_mesh-controller_lease", "2026-10-13T00:10:00Z", 3, 9]])),
|
||||
};
|
||||
|
||||
const SAID = "broker — the foundation's bus, adopted as the nats module\n restarting mesh-broker-nats — once, onto the nats module's image";
|
||||
const HERE = "anchor here last 3s ago";
|
||||
|
||||
test("a bus adopted in place is kept: one bus, the module's image, its streams, the node connected", () => {
|
||||
assert.deepEqual(busNotKept(before, kept, HERE, SAID), []);
|
||||
});
|
||||
|
||||
test("only a stream holding something counts as written before the step", () => {
|
||||
assert.deepEqual(written(before).map((s) => s.name), ["KV_mesh-controller_lease"]);
|
||||
});
|
||||
|
||||
test("a second bus beside the first is not kept: the fault genesis had before ADR 0333", () => {
|
||||
const two: Seen = { ...kept, running: [...kept.running,
|
||||
{ name: "mesh-broker", image: UPSTREAM, ports: "0.0.0.0:5671->4222/tcp" }] };
|
||||
const wrong = busNotKept(before, two, HERE, SAID);
|
||||
assert.equal(wrong.length, 1);
|
||||
assert.match(wrong[0] as string, /2 bus containers.*mesh-broker \(/);
|
||||
});
|
||||
|
||||
test("a bus under another name, or none, is not kept", () => {
|
||||
const renamed: Seen = { ...kept, running: kept.running.map((r) => r.name === "mesh-broker-nats" ? { ...r, name: "nats-server" } : r) };
|
||||
assert.ok(busNotKept(before, renamed, HERE, SAID).some((w) => /mesh-broker-nats does not run/.test(w)));
|
||||
const none: Seen = { ...kept, running: kept.running.filter((r) => r.name !== "mesh-broker-nats") };
|
||||
assert.ok(busNotKept(before, none, HERE, SAID).some((w) => /0 bus containers/.test(w)));
|
||||
});
|
||||
|
||||
test("a bus still on the bundle's image was not adopted", () => {
|
||||
const same: Seen = { ...kept, running: before.running };
|
||||
assert.ok(busNotKept(before, same, HERE, SAID).some((w) => /still runs .* the image the bundle raised/.test(w)));
|
||||
assert.ok(busNotKept(before, kept, HERE, "broker — adopted").some((w) => /did not say the bus restarts/.test(w)));
|
||||
});
|
||||
|
||||
test("a stream gone or made again is not kept", () => {
|
||||
const gone: Seen = { ...kept, streams: [] };
|
||||
assert.ok(busNotKept(before, gone, HERE, SAID).some((w) => /KV_mesh-controller_lease .* is gone/.test(w)));
|
||||
const again: Seen = { ...kept, streams: streamsFrom(jsz([["KV_mesh-controller_lease", "2026-10-13T00:40:00Z", 1, 1]])) };
|
||||
assert.ok(busNotKept(before, again, HERE, SAID).some((w) => /was made again/.test(w)));
|
||||
});
|
||||
|
||||
test("a node the mesh has not heard from since the step is not connected", () => {
|
||||
assert.ok(busNotKept(before, kept, "anchor out of touch 4m", SAID).some((w) => /has not heard from the first node/.test(w)));
|
||||
assert.ok(busNotKept(before, kept, "", SAID).some((w) => /no record/.test(w)));
|
||||
});
|
||||
@@ -0,0 +1,75 @@
|
||||
/**
|
||||
* **A module that takes a shared-cache grant presents the login it was granted** (novox/hq 081).
|
||||
*
|
||||
* The cache provider scopes each consumer to an ACL user of its own, confined to keys under its
|
||||
* login (novox/hq 080). A consumer that hands its software the password and not the login logs in
|
||||
* as the server's default user: the grant is honoured by the provider and ignored by the consumer,
|
||||
* and nothing notices while the default user is open. The grant bed proves the provider's half
|
||||
* against a consumer written to the contract; this holds every catalogue module that asks for the
|
||||
* cache to its half — the login, as `${bound:redis-cache:as}`, somewhere it hands its software.
|
||||
*
|
||||
* What it cannot see is whether the software also keeps its keys under that login: that is the
|
||||
* software's, and a module whose software cannot (fixed key or channel names in its code) does not
|
||||
* take the shared cache at all — baserow runs its own, and n8n in its shipped mode needs none.
|
||||
*/
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { existsSync, readdirSync, readFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
import { catalogueIsPresent, catalogueDir } from "./integration/harness.ts";
|
||||
|
||||
const CACHE = "redis-cache";
|
||||
|
||||
/** What a module hands its software: every file it writes, and every container's environment and
|
||||
* arguments. A comment, a `why`, or a declared exception is not handed to anything. */
|
||||
function handedToSoftware(manifest: string): string[] {
|
||||
const m = JSON.parse(manifest) as { resources?: Record<string, unknown>[] };
|
||||
const out: string[] = [];
|
||||
for (const r of m.resources ?? []) {
|
||||
if (typeof r["content"] === "string") out.push(r["content"] as string);
|
||||
if (r["env"] && typeof r["env"] === "object") out.push(...Object.values(r["env"] as Record<string, string>).map(String));
|
||||
if (Array.isArray(r["args"])) out.push(...(r["args"] as unknown[]).map(String));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Whether a manifest hands its software the login it is granted for the cache. */
|
||||
function presentsTheLogin(manifest: string): boolean {
|
||||
const login = `\${bound:${CACHE}:as}`;
|
||||
return handedToSoftware(manifest).some((given) => given.includes(login));
|
||||
}
|
||||
|
||||
test("the check sees a module that hands its software the password and not the login", () => {
|
||||
const passwordOnly = JSON.stringify({ module: "m", requires: [CACHE], resources: [
|
||||
{ id: "env", type: "file", path: "/x", content: `HOST=\${bound:${CACHE}:at}\nPASSWORD=\${secret:${CACHE}}\n` }] });
|
||||
const withLogin = JSON.stringify({ module: "m", requires: [CACHE], resources: [
|
||||
{ id: "env", type: "file", path: "/x", content: `USER=\${bound:${CACHE}:as}\nPASSWORD=\${secret:${CACHE}}\n` }] });
|
||||
assert.equal(presentsTheLogin(passwordOnly), false);
|
||||
assert.equal(presentsTheLogin(withLogin), true);
|
||||
// Named only where no software reads it — a declared reason — is not presenting it.
|
||||
const onlyInAReason = JSON.stringify({ module: "m", requires: [CACHE], resources: [
|
||||
{ id: "srv", type: "container", name: "s", image: "x", env: { PASSWORD: `\${secret:${CACHE}}` },
|
||||
"secrets-in-environment": `the login is \${bound:${CACHE}:as}` }] });
|
||||
assert.equal(presentsTheLogin(onlyInAReason), false);
|
||||
});
|
||||
|
||||
test("every catalogue module that takes the shared cache presents the login it was granted", (t) => {
|
||||
const absent = catalogueIsPresent();
|
||||
if (absent) {
|
||||
t.skip(`cannot check — ${absent}`);
|
||||
return;
|
||||
}
|
||||
const offences: string[] = [];
|
||||
for (const d of readdirSync(catalogueDir(), { withFileTypes: true })) {
|
||||
const file = resolve(catalogueDir(), d.name, "module.json");
|
||||
if (!d.isDirectory() || !existsSync(file)) continue;
|
||||
const text = readFileSync(file, "utf8");
|
||||
const m = JSON.parse(text) as { requires?: string[] };
|
||||
if (!(m.requires ?? []).includes(CACHE)) continue;
|
||||
if (!presentsTheLogin(text)) offences.push(d.name);
|
||||
}
|
||||
assert.deepEqual(offences, [],
|
||||
`these take the shared cache and never hand their software the login (\${bound:${CACHE}:as}), so they ` +
|
||||
`log in as the server's default user — present the login, or run a cache of their own:\n ${offences.join("\n ")}`);
|
||||
});
|
||||
@@ -1,11 +1,14 @@
|
||||
// Set before importing: the client reads MESH_LAB_INCUS once, at module load.
|
||||
// `false` is a real program that exits non-zero and prints nothing — which is also the worst
|
||||
// case, because an empty stderr is how a failure arrives with no explanation.
|
||||
process.env["MESH_LAB_INCUS"] = "false";
|
||||
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { instanceExists, taggedInstances, taggedNetworks } from "../src/incus/client.ts";
|
||||
import { instanceExists, taggedInstances, taggedNetworks, useIncusCommand } from "../src/incus/client.ts";
|
||||
|
||||
// `false` is a real program that exits non-zero and prints nothing — which is also the worst
|
||||
// case, because an empty stderr is how a failure arrives with no explanation.
|
||||
//
|
||||
// **Injected, never set in the environment here.** This file used to set MESH_LAB_INCUS in its body,
|
||||
// which runs after the imports above — so the client had already read the variable, and these tests
|
||||
// asked the real incus: green on a machine without one, red on the workstation that runs the lab.
|
||||
useIncusCommand(["false"]);
|
||||
|
||||
/**
|
||||
* "I cannot see" must never be answered as "there is nothing there."
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { carry } from "./integration/forge.ts";
|
||||
|
||||
/**
|
||||
* A bed's forge takes the lab's checkouts as they are: shallow (novox/hq issue 689). The lab module clones
|
||||
* every repository with `--depth`, and the genesis bed's forge refused the push ("shallow update not
|
||||
* allowed"), so the bed never reached its installer. Held here with a real git and no lab.
|
||||
*/
|
||||
|
||||
const git = (cwd: string, ...args: string[]) =>
|
||||
execFileSync("git", ["-C", cwd, ...args], { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"],
|
||||
env: { ...process.env, GIT_AUTHOR_NAME: "lab", GIT_AUTHOR_EMAIL: "lab@example.org",
|
||||
GIT_COMMITTER_NAME: "lab", GIT_COMMITTER_EMAIL: "lab@example.org" } }).trim();
|
||||
|
||||
test("a shallow checkout, as the lab clones it, is carried into the bed's forge and served from it", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mesh-lab-689-"));
|
||||
try {
|
||||
const origin = join(dir, "origin");
|
||||
execFileSync("git", ["init", "-q", "-b", "main", origin]);
|
||||
for (let i = 0; i < 5; i++) {
|
||||
writeFileSync(join(origin, "file"), `${i}\n`);
|
||||
git(origin, "add", "file");
|
||||
git(origin, "commit", "-q", "-m", `commit ${i}`);
|
||||
}
|
||||
// As the lab module clones: shallow, over a transport that honours the depth.
|
||||
const checkout = join(dir, "checkout");
|
||||
execFileSync("git", ["clone", "-q", "--depth", "2", "--branch", "main", `file://${origin}`, checkout]);
|
||||
assert.equal(git(checkout, "rev-parse", "--is-shallow-repository"), "true");
|
||||
|
||||
const bare = join(dir, "forge", "repo.git");
|
||||
const head = carry(checkout, bare, ["main", "lab"]);
|
||||
assert.equal(head, git(origin, "rev-parse", "HEAD"));
|
||||
assert.equal(git(bare, "rev-parse", "refs/heads/main"), head);
|
||||
assert.equal(git(bare, "rev-parse", "refs/heads/lab"), head);
|
||||
|
||||
// What the bed's machines do with it: clone over git's protocol, and find the commit under test.
|
||||
const served = join(dir, "served");
|
||||
execFileSync("git", ["clone", "-q", "--branch", "main", `file://${bare}`, served]);
|
||||
assert.equal(git(served, "rev-parse", "HEAD"), head);
|
||||
assert.equal(git(served, "show", "HEAD:file"), "4");
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,973 @@
|
||||
/**
|
||||
* A MACHINE IN USE IS ADOPTED BEFORE IT IS CONVERGED (novox/hq ADR 0100, and ADR 0101 on what
|
||||
* "in use" ignores).
|
||||
*
|
||||
* The mesh replaces a predecessor running on the same machines. This bed prepares a machine the
|
||||
* way the predecessor leaves one — the record's own words, "How it is checked":
|
||||
*
|
||||
* - its firewall (ufw) allowing a served port and denying the rest, incoming and routed, with the
|
||||
* predecessor's published container ports filtered through it (the ufw-docker arrangement);
|
||||
* - a service container, `hello-web`, listening on that port under a name the catalogue's
|
||||
* `hello-web` module also uses, and a file at a path that module declares;
|
||||
* - a stand-in for the predecessor's control that rewrites that file, stopped by the operator
|
||||
* before adoption as the record prescribes, and started again later to play one forgotten;
|
||||
* - a container holding the registry's port.
|
||||
*
|
||||
* Then it asks, in the record's order: a converged genesis refuses; an adopted one refuses the held
|
||||
* registry port and comes up on another; nothing that serves changed; the store is unreachable
|
||||
* from outside and reachable where it must be; the mesh works through the found firewall, across a
|
||||
* reload and a reboot; a predecessor still writing is caught; assigning prepares and taking cuts
|
||||
* over; converging previews, refuses while a found container is held, flips, and returns.
|
||||
*
|
||||
* **The module under migration is the catalogue's `hello-web` with its route requirement taken
|
||||
* off**, read from the catalogue (never a copy): the route needs a proxy module and a public name,
|
||||
* neither of which is what adoption is about. Its names — the container, the file, the port — are
|
||||
* the catalogue's, which is the point: they are what the predecessor also uses.
|
||||
*
|
||||
* **The forge is in the lab.** Genesis builds the control plane and the catalogue from a
|
||||
* repository and a commit; this bed serves the checkouts it was pointed at (their HEADs) from the
|
||||
* `outsider` machine, so the run builds exactly the code under test and nothing on the workstation
|
||||
* listens for the lab.
|
||||
*
|
||||
* Each step is recorded rather than allowed to throw; a step whose dependency failed is not
|
||||
* attempted, and the report says which.
|
||||
*
|
||||
* MESH_LAB_INCUS='sudo -n incus'
|
||||
* MESH_LAB_HOST_BINARY=<mesh-host>/mesh-host MESH_LAB_BOOTSTRAP_BINARY=<mesh-host>/mesh-bootstrap
|
||||
* MESH_LAB_BUNDLE=<mesh-host>/examples/foundation-first-node.lock
|
||||
* MESH_LAB_CATALOG=<mesh-catalog>/modules MESH_LAB_MODULES=<mesh-controller>/examples/modules
|
||||
* MESH_TOOLS=<mesh-tools> MESH_SDK=<mesh-sdk> (default: the checkouts beside this one)
|
||||
* MESH_LAB_KEEP=1 leave it standing MESH_LAB_WARM=1 iterate from the adopted foundation
|
||||
*/
|
||||
import { test, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { loadScenario } from "../../src/declaration/parse.ts";
|
||||
import { raise } from "../../src/lifecycle/raise.ts";
|
||||
import { destroy, exec, push, instanceNameOf } from "../../src/lifecycle/operate.ts";
|
||||
import { bootstrapBinaryPath, hostBinaryPath, placeBootstrap, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||||
import { waitUntilAllUsable } from "../../src/lifecycle/ready.ts";
|
||||
import { incus } from "../../src/incus/client.ts";
|
||||
import { catalogueRoot } from "../../src/repos.ts";
|
||||
import { ready, returnTo, keep } from "../../src/warm.ts";
|
||||
import { labIsUsable, destroyAll, foundationBundle, catalogueModule, catalogueManifest } from "./harness.ts";
|
||||
import { genesis, type GenesisOptions } from "./genesis.ts";
|
||||
import { carry } from "./forge.ts";
|
||||
|
||||
const SCENARIO = "adoption";
|
||||
const CONTROL = "anchor";
|
||||
const JOINER = "joiner";
|
||||
const OUTSIDER = "outsider";
|
||||
const ANCHOR = "192.0.2.10";
|
||||
const JOINER_ADDRESS = "192.0.2.20";
|
||||
const FORGE = "192.0.2.30";
|
||||
|
||||
/** The port the predecessor serves, and the module that also names its container and file. */
|
||||
const SERVED = 8080;
|
||||
const SERVICE = "hello-web";
|
||||
const SERVICE_FILE = "/var/lib/hello-web/index.html";
|
||||
const PREDECESSOR_PAGE = "hello from the predecessor\n";
|
||||
/** The registry's port, which the predecessor holds — and the one the mesh is given instead. */
|
||||
const HELD_REGISTRY = 5000;
|
||||
const REGISTRY_PORT = 5100;
|
||||
const STORE_PORT = 5432;
|
||||
const BUS_PORT = 5671;
|
||||
const HUB_PORT = 51820;
|
||||
const NETWORK_MODULE = "networking";
|
||||
const FILTER_MODULE = "nftables";
|
||||
|
||||
/** Upstream images, pinned as the catalogue pins them (the harness's table). */
|
||||
const ALPINE = "alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b";
|
||||
const REGISTRY_IMAGE = "registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373";
|
||||
|
||||
const capability = await labIsUsable();
|
||||
const binary = hostBinaryPath();
|
||||
const installer = bootstrapBinaryPath();
|
||||
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
||||
const catalogDir = process.env["MESH_LAB_CATALOG"] ?? "";
|
||||
const modulesDir = process.env["MESH_LAB_MODULES"] ?? "";
|
||||
const KEEP = !!process.env["MESH_LAB_KEEP"];
|
||||
const WARM = !!process.env["MESH_LAB_WARM"];
|
||||
const FIXED_ID = process.env["MESH_LAB_INSTANCE_ID"] ?? (KEEP ? "adoption-live" : undefined);
|
||||
|
||||
/** The checkouts this run builds from, served by the lab's forge. */
|
||||
const REPOS: Record<string, string> = {
|
||||
"mesh-controller": modulesDir ? dirname(dirname(modulesDir)) : "",
|
||||
"mesh-catalog": catalogDir ? catalogueRoot(catalogDir) : "",
|
||||
"mesh-tools": process.env["MESH_TOOLS"] ?? resolve(process.cwd(), "..", "mesh-tools"),
|
||||
"mesh-sdk": process.env["MESH_SDK"] ?? resolve(process.cwd(), "..", "mesh-sdk"),
|
||||
};
|
||||
|
||||
const skip =
|
||||
!capability.usable ? capability.why :
|
||||
!binary ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" :
|
||||
!installer ? "MESH_LAB_BOOTSTRAP_BINARY is not set to a built mesh-bootstrap" :
|
||||
!bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation template" :
|
||||
!catalogDir || !existsSync(catalogDir) ? "MESH_LAB_CATALOG is not set to mesh-catalog/modules" :
|
||||
!modulesDir ? "MESH_LAB_MODULES is not set, so there is no control-plane checkout to build from" :
|
||||
Object.entries(REPOS).find(([, p]) => !p || !existsSync(resolve(p, ".git")))
|
||||
?.map((x) => `no checkout of ${x[0]} at ${x[1]}`)[0] ?? false;
|
||||
|
||||
let instanceId = "";
|
||||
|
||||
// ---- talking to the machines ------------------------------------------------------------------
|
||||
|
||||
function quote(s: string): string {
|
||||
return `'${s.replaceAll("'", `'\\''`)}'`;
|
||||
}
|
||||
async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
||||
const { stdout } = await exec(instanceId, machine, [
|
||||
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
|
||||
], timeoutMs);
|
||||
const marker = stdout.lastIndexOf("__exit=");
|
||||
if (marker < 0) return { out: stdout, ok: false };
|
||||
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
||||
}
|
||||
async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
|
||||
const { out, ok } = await on(machine, command, timeoutMs);
|
||||
if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
|
||||
return out;
|
||||
}
|
||||
/** The control plane, retried across the brief windows in which the mesh recreates it. */
|
||||
async function meshSays(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
||||
const deadline = Date.now() + (timeoutMs ?? 120_000);
|
||||
for (;;) {
|
||||
const r = await on(CONTROL, `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
|
||||
if (r.ok) return r;
|
||||
if (/is not running|No such container|No such exec instance|Cannot connect to the Docker daemon|is restarting/i.test(r.out) &&
|
||||
Date.now() < deadline) {
|
||||
await sleep(2_000);
|
||||
continue;
|
||||
}
|
||||
return r;
|
||||
}
|
||||
}
|
||||
async function mesh(command: string, timeoutMs?: number): Promise<string> {
|
||||
const r = await meshSays(command, timeoutMs);
|
||||
if (!r.ok) throw new Error(`${CONTROL}: mesh-controller ${command}\n${r.out}`);
|
||||
return r.out;
|
||||
}
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((r) => setTimeout(r, ms));
|
||||
}
|
||||
/** Poll until `probe` returns a value, or fail naming the last thing it saw. */
|
||||
async function until<T>(what: string, seconds: number, probe: () => Promise<T | null>, last: () => string): Promise<T> {
|
||||
const deadline = Date.now() + seconds * 1000;
|
||||
for (;;) {
|
||||
const got = await probe();
|
||||
if (got !== null) return got;
|
||||
if (Date.now() > deadline) throw new Error(`${what} — not within ${seconds}s. Last:\n${last()}`);
|
||||
await sleep(5_000);
|
||||
}
|
||||
}
|
||||
/** Whether a TCP connection from `machine` to address:port opens within three seconds. */
|
||||
async function connects(machine: string, address: string, port: number): Promise<boolean> {
|
||||
return (await on(machine, `timeout 4 bash -c ${quote(`</dev/tcp/${address}/${port}`)}`)).ok;
|
||||
}
|
||||
/** Register a module with the control plane from a manifest's text. */
|
||||
async function registerModule(module: string, manifest: string): Promise<string> {
|
||||
const local = join(tmpdir(), `mesh-lab-adoption-${process.pid}-${module}.json`);
|
||||
writeFileSync(local, manifest);
|
||||
await push(instanceId, CONTROL, local, `/tmp/${module}.json`);
|
||||
await must(CONTROL, `docker cp /tmp/${module}.json mesh-controller:/${module}.json`);
|
||||
return mesh(`module add /${module}.json`);
|
||||
}
|
||||
async function nodeShow(node: string): Promise<string> {
|
||||
return mesh(`node show ${node}`);
|
||||
}
|
||||
/** The anchor's address on the private network. */
|
||||
async function meshAddressOf(machine: string): Promise<string> {
|
||||
const out = await must(machine, `ip -4 -o addr show dev mesh0`);
|
||||
const found = out.match(/inet (\d+\.\d+\.\d+\.\d+)\//)?.[1];
|
||||
assert.ok(found, `${machine} has no address on mesh0:\n${out}`);
|
||||
return found;
|
||||
}
|
||||
/** The rules ufw was given, one per line, as `ufw show added` prints them. */
|
||||
async function ufwAdded(): Promise<string[]> {
|
||||
const out = await must(CONTROL, `ufw show added`);
|
||||
return out.split("\n").map((l) => l.trim()).filter((l) => l.startsWith("ufw "));
|
||||
}
|
||||
function marked(rule: string): boolean {
|
||||
return /comment 'mesh-host /.test(rule);
|
||||
}
|
||||
async function restartMachine(machine: string): Promise<void> {
|
||||
const name = await instanceNameOf(instanceId, machine);
|
||||
await incus(["restart", name], 180_000);
|
||||
await waitUntilAllUsable([name], 300, (m) => console.log(` restart: ${m}`));
|
||||
}
|
||||
/** Until the anchor reports what it was last sent as applied and current. */
|
||||
async function settled(node = CONTROL, withinMs = 300_000): Promise<void> {
|
||||
let last = "";
|
||||
await until(`${node} reports the declaration it was sent as applied and current`, withinMs / 1000, async () => {
|
||||
const asked = await meshSays(`status --json`);
|
||||
last = asked.out;
|
||||
if (!asked.ok) return null;
|
||||
try {
|
||||
const state = JSON.parse(asked.out) as {
|
||||
wrong: { node: string; outcome: string }[];
|
||||
waiting: { node: string }[];
|
||||
reported: { node: string; outcome: string; current: boolean }[];
|
||||
};
|
||||
const bad = state.wrong.find((w) => w.node === node);
|
||||
if (bad) throw new Error(`${node} did not apply what it was sent: ${bad.outcome}\n${asked.out}`);
|
||||
const word = state.reported.find((r) => r.node === node);
|
||||
return !state.waiting.some((w) => w.node === node) && word?.outcome === "applied" && word.current ? true : null;
|
||||
} catch (err) {
|
||||
if (err instanceof Error && err.message.includes("did not apply")) throw err;
|
||||
return null;
|
||||
}
|
||||
}, () => last);
|
||||
}
|
||||
/** Send a node what it should be, and wait until it says it applied it. */
|
||||
async function pushAndSettle(node: string): Promise<string> {
|
||||
const said = await mesh(`push ${node}`, 600_000);
|
||||
await settled(node);
|
||||
return said;
|
||||
}
|
||||
function tokenFrom(said: string): string {
|
||||
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
|
||||
assert.ok(found, `no token in:\n${said}`);
|
||||
return found;
|
||||
}
|
||||
|
||||
// ---- the lab's forge ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Serve the checkouts under test from the outsider machine, over git's own protocol.
|
||||
*
|
||||
* Each checkout's HEAD is pushed into a bare repository as `main` and `lab`, the lot is carried in,
|
||||
* and a git daemon answers on the outsider's scenario address — which the anchor's builder reaches
|
||||
* over the hosting segment. Returns the commit each repository is served at.
|
||||
*/
|
||||
async function raiseForge(): Promise<Record<string, string>> {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mesh-lab-forge-"));
|
||||
const heads: Record<string, string> = {};
|
||||
try {
|
||||
for (const [name, checkout] of Object.entries(REPOS)) {
|
||||
heads[name] = carry(checkout, join(dir, `${name}.git`), ["main", "lab"]);
|
||||
}
|
||||
const tar = join(tmpdir(), `mesh-lab-forge-${process.pid}.tar`);
|
||||
execFileSync("tar", ["-cf", tar, "-C", dir, "."]);
|
||||
await push(instanceId, OUTSIDER, tar, "/tmp/forge.tar");
|
||||
rmSync(tar, { force: true });
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
await must(OUTSIDER, `command -v git >/dev/null || pacman -S --noconfirm --needed git`, 600_000);
|
||||
// Owned by the daemon's user: extracted as the workstation's uid, git refuses to serve a
|
||||
// repository someone else owns ("dubious ownership"), and the clone fails with no reason given.
|
||||
await must(OUTSIDER, `mkdir -p /srv/git && tar --no-same-owner -xf /tmp/forge.tar -C /srv/git && chown -R root:root /srv/git && ` +
|
||||
`git daemon --base-path=/srv/git --export-all --reuseaddr --detach --listen=${FORGE} --pid-file=/run/git-daemon.pid`);
|
||||
await must(OUTSIDER, `git ls-remote git://${FORGE}/mesh-controller.git lab`);
|
||||
await until("the lab's forge answers the anchor", 60, async () =>
|
||||
(await connects(CONTROL, FORGE, 9418)) ? true : null, () => "no connection to 9418");
|
||||
return heads;
|
||||
}
|
||||
const forgeUrl = (repo: string) => `git://${FORGE}/${repo}.git`;
|
||||
|
||||
// ---- the predecessor ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* The ufw-docker arrangement: published container ports pass through ufw's route rules, and
|
||||
* traffic from private ranges is let through. It is how a ufw machine filters what docker publishes
|
||||
* at all — without it docker's own rules bypass ufw entirely (novox/hq research 012 measured 52
|
||||
* forwarding rules on the control-node, one per served port).
|
||||
*/
|
||||
const UFW_DOCKER = `
|
||||
# BEGIN UFW AND DOCKER
|
||||
*filter
|
||||
:ufw-user-forward - [0:0]
|
||||
:ufw-docker-logging-deny - [0:0]
|
||||
:DOCKER-USER - [0:0]
|
||||
-A DOCKER-USER -j ufw-user-forward
|
||||
-A DOCKER-USER -j RETURN -s 10.0.0.0/8
|
||||
-A DOCKER-USER -j RETURN -s 172.16.0.0/12
|
||||
-A DOCKER-USER -j RETURN -s 192.168.0.0/16
|
||||
-A DOCKER-USER -p udp -m udp --sport 53 --dport 1024:65535 -j RETURN
|
||||
-A DOCKER-USER -j ufw-docker-logging-deny -p tcp -m tcp --tcp-flags FIN,SYN,RST,ACK SYN -d 192.168.0.0/16
|
||||
-A DOCKER-USER -j ufw-docker-logging-deny -p tcp -m tcp --tcp-flags FIN,SYN,RST,ACK SYN -d 10.0.0.0/8
|
||||
-A DOCKER-USER -j ufw-docker-logging-deny -p tcp -m tcp --tcp-flags FIN,SYN,RST,ACK SYN -d 172.16.0.0/12
|
||||
-A DOCKER-USER -j ufw-docker-logging-deny -p udp -m udp --dport 0:32767 -d 192.168.0.0/16
|
||||
-A DOCKER-USER -j ufw-docker-logging-deny -p udp -m udp --dport 0:32767 -d 10.0.0.0/8
|
||||
-A DOCKER-USER -j ufw-docker-logging-deny -p udp -m udp --dport 0:32767 -d 172.16.0.0/12
|
||||
-A DOCKER-USER -j RETURN
|
||||
-A ufw-docker-logging-deny -j DROP
|
||||
COMMIT
|
||||
# END UFW AND DOCKER
|
||||
`;
|
||||
|
||||
/** The stand-in for the predecessor's configuration sync: it rewrites the file every ten seconds. */
|
||||
const STAND_IN = `[Unit]
|
||||
Description=Stand-in for the predecessor's configuration sync: rewrites a file a catalogue module declares
|
||||
|
||||
[Service]
|
||||
ExecStart=/bin/sh -c 'while true; do printf "hello from the predecessor, synced %%s\\\\n" "$(date +%%s)" > ${SERVICE_FILE}; sleep 10; done'
|
||||
`;
|
||||
|
||||
/** The page the predecessor's service loop serves — the catalogue module's own loop, verbatim. */
|
||||
const SERVE_LOOP =
|
||||
"while true; do { printf 'HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\nConnection: close\\r\\n\\r\\n'; cat /www/index.html; } | nc -l -p 8080; done";
|
||||
|
||||
/** Where the bed keeps what the machine looked like before the mesh arrived — on the machine, so a warm restore keeps it. */
|
||||
const BEFORE = "/root/predecessor";
|
||||
|
||||
/** A predecessor container with no restart policy: a runtime restart would lose it. */
|
||||
const NO_POLICY = "predecessor-nopolicy";
|
||||
|
||||
/** The broker's plaintext port: published on every interface, and the filter admits it from the mesh only. */
|
||||
const AMQP_PORT = 5672;
|
||||
|
||||
async function preparePredecessor(): Promise<string> {
|
||||
const said: string[] = [];
|
||||
await must(CONTROL, `pacman -S --noconfirm --needed ufw`, 600_000);
|
||||
const rules = join(tmpdir(), `mesh-lab-ufw-docker-${process.pid}`);
|
||||
writeFileSync(rules, UFW_DOCKER);
|
||||
await push(instanceId, CONTROL, rules, "/tmp/ufw-docker.rules");
|
||||
await must(CONTROL, [
|
||||
`grep -q 'BEGIN UFW AND DOCKER' /etc/ufw/after.rules || cat /tmp/ufw-docker.rules >> /etc/ufw/after.rules`,
|
||||
`ufw default deny incoming`,
|
||||
`ufw default allow outgoing`,
|
||||
`ufw default deny routed`,
|
||||
`ufw allow 22/tcp`,
|
||||
`ufw allow ${SERVED}/tcp`,
|
||||
`ufw route allow proto tcp from any to any port ${SERVED}`,
|
||||
`ufw --force enable`,
|
||||
`systemctl enable ufw`,
|
||||
].join(" && "));
|
||||
said.push(` firewall ufw: deny incoming and routed; allow 22 and ${SERVED}; ufw-docker after.rules`);
|
||||
|
||||
await must(CONTROL, `mkdir -p /var/lib/hello-web && printf %s ${quote(PREDECESSOR_PAGE)} > ${SERVICE_FILE}`);
|
||||
await must(CONTROL,
|
||||
`docker run -d --name ${SERVICE} --restart unless-stopped -p ${SERVED}:${SERVED} ` +
|
||||
`-v ${SERVICE_FILE}:/www/index.html:ro ${ALPINE} sh -c ${quote(SERVE_LOOP)}`, 600_000);
|
||||
await must(CONTROL,
|
||||
`docker run -d --name predecessor-registry --restart unless-stopped -p ${HELD_REGISTRY}:5000 ${REGISTRY_IMAGE}`, 600_000);
|
||||
// A container the predecessor left running with no restart policy: a restart of the runtime
|
||||
// would not bring it back, which is what ADR 0102 forbids the mesh from causing.
|
||||
await must(CONTROL, `docker run -d --name ${NO_POLICY} ${ALPINE} sleep infinity`, 600_000);
|
||||
// And a setting of the machine's own in the runtime's file, beside the registries it ships with.
|
||||
await must(CONTROL,
|
||||
`python3 - <<'EOF'
|
||||
import json
|
||||
f="/etc/docker/daemon.json"
|
||||
d=json.load(open(f))
|
||||
d["log-opts"]={"max-size":"7m"}
|
||||
json.dump(d,open(f,"w"),indent=2)
|
||||
EOF
|
||||
systemctl reload docker`);
|
||||
said.push(` containers ${SERVICE} on ${SERVED}, predecessor-registry on ${HELD_REGISTRY}`);
|
||||
|
||||
// The predecessor's control, which the operator stops before adopting (the record's words).
|
||||
const unit = join(tmpdir(), `mesh-lab-stand-in-${process.pid}`);
|
||||
writeFileSync(unit, STAND_IN);
|
||||
await push(instanceId, CONTROL, unit, "/etc/systemd/system/predecessor-sync.service");
|
||||
await must(CONTROL, `systemctl daemon-reload && systemctl start predecessor-sync && sleep 12 && systemctl stop predecessor-sync`);
|
||||
said.push(` stand-in predecessor-sync rewrote ${SERVICE_FILE}, then the operator stopped it`);
|
||||
|
||||
// What the machine was, recorded ON the machine so a restored warm instance still has it.
|
||||
await must(CONTROL, [
|
||||
`mkdir -p ${BEFORE}`,
|
||||
`sha256sum ${SERVICE_FILE} | cut -d' ' -f1 > ${BEFORE}/file.sha256`,
|
||||
`cp ${SERVICE_FILE} ${BEFORE}/file`,
|
||||
`docker inspect -f '{{.Id}}' ${SERVICE} > ${BEFORE}/container.id`,
|
||||
`docker inspect -f '{{.State.Running}} {{.Id}}' ${NO_POLICY} > ${BEFORE}/nopolicy.id`,
|
||||
`ufw show added > ${BEFORE}/ufw-added.txt`,
|
||||
].join(" && "));
|
||||
await until(`the predecessor's ${SERVICE} answers the joiner`, 60, async () =>
|
||||
(await on(JOINER, `curl -s --max-time 3 http://${ANCHOR}:${SERVED}/`)).out.includes("hello from the predecessor") ? true : null,
|
||||
() => "no answer");
|
||||
said.push((await must(CONTROL, `ufw status verbose`)).trim());
|
||||
said.push((await must(CONTROL, `docker ps --format '{{.Names}}\t{{.Ports}}'`)).trim());
|
||||
return said.join("\n");
|
||||
}
|
||||
|
||||
// ---- steps, recorded rather than thrown --------------------------------------------------------
|
||||
|
||||
interface Step { code: string; title: string; ok: boolean; why: string; said: string; seconds: number; warm?: boolean }
|
||||
const steps = new Map<string, Step>();
|
||||
|
||||
async function step(code: string, needs: string[], fn: () => Promise<string>): Promise<void> {
|
||||
const title = TITLE[code]!;
|
||||
const missing = needs.filter((n) => !steps.get(n)?.ok);
|
||||
if (missing.length) {
|
||||
steps.set(code, { code, title, ok: false, seconds: 0, said: "",
|
||||
why: `not attempted — ${missing.join(", ")} did not succeed` });
|
||||
console.log(`[${code}] SKIP ${title}`);
|
||||
return;
|
||||
}
|
||||
console.log(`\n[${code}] ---- ${title} ----`);
|
||||
const began = Date.now();
|
||||
const took = () => Math.round((Date.now() - began) / 1000);
|
||||
try {
|
||||
const said = await fn();
|
||||
steps.set(code, { code, title, ok: true, why: "", said, seconds: took() });
|
||||
console.log(`${said}\n[${code}] PASS ${title} (${took()}s)`);
|
||||
} catch (err) {
|
||||
const why = (err as Error).message;
|
||||
steps.set(code, { code, title, ok: false, why, said: "", seconds: took() });
|
||||
console.log(`[${code}] FAIL ${title} (${took()}s)\n${why.split("\n").slice(0, 40).join("\n")}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** The plan, in the record's order. Codes are stable; titles may be reworded. */
|
||||
const TITLE: Record<string, string> = {
|
||||
P0: "the machine is prepared the way the predecessor leaves one",
|
||||
F0: "a converged genesis on a FRESH machine is not refused — its own resolver does not make it in use (ADR 0101)",
|
||||
A1: "a converged genesis refuses the machine in use, naming every container and listener it counted",
|
||||
A2: "an adopted genesis refuses the registry's held port, naming what holds it",
|
||||
A3: "given another registry port, the adopted foundation comes up — and stays on that port as modules",
|
||||
B1: "nothing that serves changed: the service answers, its file and container are untouched, the firewall gained only the mesh's marked rules",
|
||||
B2: "the store is unreachable from outside, before and after the found firewall reloads; the bus answers a machine not yet enrolled",
|
||||
B4: "the guard lets the machine's own containers reach the store (with the found firewall admitting them)",
|
||||
C1: "a second machine enrols through the found firewall and joins the private network",
|
||||
B3: "the store is reachable over the private network",
|
||||
C2: "after the found firewall reloads, the openings are there and the mesh still works",
|
||||
C3: "after the machine reboots, the openings are there and the mesh still works",
|
||||
D1: "assigning prepares: the module holds the found container and file, and neither changes",
|
||||
D2: "a predecessor still writing is caught: the held file's change is reported, and not reverted",
|
||||
D3: "converging refuses while the service's module holds its found container",
|
||||
D4: "taking cuts over: the found container and file are replaced, the original kept, the port reachable",
|
||||
E1: "converging previews: the service's port, a published port no rule mentions, and the modules it takes",
|
||||
E2: "the flip: the derived filter loaded, the found firewall disabled with its configuration on disk, the declared port open and the undeclared closed",
|
||||
E3: "returned to adopted: the found firewall enabled again, the derived filter gone, the openings back",
|
||||
F1: "unassigning takes the mesh's opening away and leaves the operator's own rule",
|
||||
};
|
||||
|
||||
function stateOutcome(): void {
|
||||
console.log(`\n================ ADOPTION: WHAT WAS ESTABLISHED ================`);
|
||||
let established = 0;
|
||||
for (const code of Object.keys(TITLE)) {
|
||||
const s = steps.get(code);
|
||||
const mark = !s ? "NEVER" : s.warm ? "WARM" : s.ok ? "PASS" : s.why.startsWith("not attempted") ? "SKIP" : "FAIL";
|
||||
if (s?.ok) established++;
|
||||
console.log(` ${code.padEnd(3)} ${mark.padEnd(5)} ${TITLE[code]}${s?.seconds ? ` (${s.seconds}s)` : ""}`);
|
||||
}
|
||||
console.log(` ${established}/${Object.keys(TITLE).length} established.`);
|
||||
}
|
||||
|
||||
// ---- the run -----------------------------------------------------------------------------------
|
||||
|
||||
before(async () => {
|
||||
if (skip) return;
|
||||
console.log(`\n================ THE PLAN ================`);
|
||||
for (const [code, title] of Object.entries(TITLE)) console.log(` ${code.padEnd(3)} ${title}`);
|
||||
|
||||
const verdict = WARM ? await ready(SCENARIO) : { use: "raise" as const, why: "not asked to be warm" };
|
||||
let restored = false;
|
||||
if (verdict.use === "restore") {
|
||||
instanceId = verdict.instanceId;
|
||||
const seconds = await returnTo(instanceId, (m) => console.log(`warm: ${m}`));
|
||||
console.log(`WARM: restored ${instanceId} to the adopted foundation in ${seconds}s`);
|
||||
restored = true;
|
||||
for (const code of ["P0", "F0", "A1", "A2", "A3"]) {
|
||||
steps.set(code, { code, title: TITLE[code]!, ok: true, warm: true, seconds: 0, why: "",
|
||||
said: "restored from the warm snapshot — not re-run; only a fresh run proves it" });
|
||||
}
|
||||
} else {
|
||||
if (WARM) console.log(`WARM: raising — ${verdict.why}`);
|
||||
const bed = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
||||
onProgress: (m) => console.log(`raise: ${m}`),
|
||||
...(FIXED_ID ? { instanceId: FIXED_ID } : {}),
|
||||
});
|
||||
instanceId = bed.instanceId;
|
||||
}
|
||||
console.log(`INSTANCE ${instanceId}${KEEP ? " (KEEP — will be left standing)" : ""}`);
|
||||
|
||||
let heads: Record<string, string> = {};
|
||||
const genesisOn = (node: string, o: Partial<GenesisOptions>): Promise<ReturnType<typeof genesis> extends Promise<infer R> ? R : never> =>
|
||||
genesis({
|
||||
instanceId, node, installer: installer as string, catalogDir,
|
||||
bundleTemplate: foundationBundle(bundle, []),
|
||||
registry: `${ANCHOR}:${HELD_REGISTRY}`,
|
||||
source: forgeUrl("mesh-controller"), sourceRef: heads["mesh-controller"] ?? "",
|
||||
toolsSource: forgeUrl("mesh-tools"), toolsRef: "lab",
|
||||
catalogSource: forgeUrl("mesh-catalog"), catalogRef: "lab",
|
||||
sdkSource: forgeUrl("mesh-sdk"), sdkRef: "lab",
|
||||
site: "hosting",
|
||||
// The service, so a machine that reboots comes back holding itself (the bed reboots one).
|
||||
// F0 asks for a dry run on a machine that must stay fresh, and passes false for itself.
|
||||
hostService: true,
|
||||
...(binary ? { hostBinary: binary } : {}),
|
||||
log: (m) => console.log(m),
|
||||
...o,
|
||||
});
|
||||
|
||||
if (!restored) {
|
||||
await step("P0", [], async () => {
|
||||
heads = await raiseForge();
|
||||
const said = [` forge git://${FORGE}/ serving ${Object.entries(heads).map(([n, h]) => `${n}@${h.slice(0, 8)}`).join(", ")}`];
|
||||
said.push(await preparePredecessor());
|
||||
return said.join("\n");
|
||||
});
|
||||
|
||||
// ADR 0101: the joiner is a freshly installed machine — nothing but its operating system, a
|
||||
// container runtime and the host binary. A converged genesis there must not be refused. Asked
|
||||
// as a dry run: the question is the preflight's, and a real raise would make it a second mesh.
|
||||
await step("F0", ["P0"], async () => {
|
||||
const ran = await genesisOn(JOINER, { flags: ["--dry-run"], attempts: 1, verify: false, hostService: false });
|
||||
assert.doesNotMatch(ran.said, /this machine is in use/,
|
||||
`a converged genesis refused a fresh machine as in use:\n${ran.said}`);
|
||||
assert.match(ran.said, /in use\s+no: no container runs and nothing listens beyond ssh/,
|
||||
`the installer never said the fresh machine is not in use:\n${ran.said}`);
|
||||
const listening = (await must(JOINER, `ss -Hltunp`)).trim();
|
||||
return ` in use no — the installer went on (${ran.ok ? "dry run finished" : `dry run stopped later, at ${ran.step}`})\n` +
|
||||
` what listens on the fresh machine:\n${listening.split("\n").map((l) => ` ${l}`).join("\n")}`;
|
||||
});
|
||||
|
||||
await step("A1", ["P0"], async () => {
|
||||
const ran = await genesisOn(CONTROL, { attempts: 1, verify: false });
|
||||
assert.ok(!ran.ok, `a converged genesis went ahead on a machine in use:\n${ran.said}`);
|
||||
assert.match(ran.step, /preflight/, `it was refused, but not before changing anything (at ${ran.step}):\n${ran.said}`);
|
||||
for (const want of [/container hello-web/, /container predecessor-registry/,
|
||||
new RegExp(`tcp \\S+:${SERVED} by`), new RegExp(`tcp \\S+:${HELD_REGISTRY} by`)]) {
|
||||
assert.match(ran.said, want, `the refusal does not name ${want}:\n${ran.said}`);
|
||||
}
|
||||
assert.match(ran.said, /--adopted/, `the refusal does not say how to raise it adopted`);
|
||||
// And nothing was changed: the predecessor as it was, no table of the mesh's.
|
||||
const ps = await must(CONTROL, `docker ps --format '{{.Names}}'`);
|
||||
assert.deepEqual(ps.trim().split("\n").sort(), [SERVICE, NO_POLICY, "predecessor-registry"].sort(), `containers changed:\n${ps}`);
|
||||
assert.match(await must(CONTROL, `ufw status`), /Status: active/);
|
||||
assert.ok(!(await on(CONTROL, `nft list table inet mesh`)).ok, `a mesh table was loaded`);
|
||||
return ran.said.split("\n").filter((l) => /in use|^\s+- /.test(l)).join("\n");
|
||||
});
|
||||
|
||||
await step("A2", ["A1"], async () => {
|
||||
const ran = await genesisOn(CONTROL, { adopted: true, attempts: 1, verify: false });
|
||||
assert.ok(!ran.ok, `an adopted genesis went ahead with the registry's port held:\n${ran.said}`);
|
||||
assert.match(ran.said, new RegExp(`registry's port tcp/${HELD_REGISTRY} is held by [^\\n]*predecessor-registry`),
|
||||
`the refusal does not name what holds the registry's port:\n${ran.said.split("\n").slice(-15).join("\n")}`);
|
||||
assert.match(ran.said, /--registry-port/, `the refusal does not say which flag gives another port`);
|
||||
const id = (await must(CONTROL, `docker inspect -f '{{.Id}}' ${SERVICE}`)).trim();
|
||||
assert.equal(id, (await must(CONTROL, `cat ${BEFORE}/container.id`)).trim(), `the predecessor's container was replaced`);
|
||||
assert.match(await must(CONTROL, `ufw status`), /Status: active/);
|
||||
assert.ok(!(await on(CONTROL, `nft list table inet mesh`)).ok, `a mesh table was loaded`);
|
||||
return ` refused at ${ran.step}\n` + ran.said.split("\n").filter((l) => /held by|port|adopted/.test(l)).slice(-8).join("\n");
|
||||
});
|
||||
|
||||
await step("A3", ["A2"], async () => {
|
||||
const ran = await genesisOn(CONTROL, { adopted: true, registry: `${ANCHOR}:${REGISTRY_PORT}` });
|
||||
if (!ran.ok) throw new Error(`${ran.step}: ${ran.why}\n\n${ran.report.join("\n")}`);
|
||||
await settled();
|
||||
const said = [ran.report.join("\n")];
|
||||
const bindings = await must(CONTROL, `docker inspect -f '{{json .HostConfig.PortBindings}}' mesh-registry`);
|
||||
assert.match(bindings, new RegExp(`"HostPort":"${REGISTRY_PORT}"`), `the registry is not on ${REGISTRY_PORT}: ${bindings}`);
|
||||
assert.match(await must(CONTROL, `docker inspect -f '{{index .Config.Labels "mesh-host.spec"}}' mesh-registry`), /\S/,
|
||||
`mesh-registry is not the host's: the foundation was not adopted as a module`);
|
||||
assert.match(await mesh(`module list`), /^distribution\b/m, `the registry is not a module the mesh holds`);
|
||||
// The node's own port, in what the mesh would send it — not the catalogue's default.
|
||||
const plan = await mesh(`plan ${CONTROL} --json`);
|
||||
assert.match(plan, new RegExp(`"${REGISTRY_PORT}:5000"`), `the registry's module does not publish ${REGISTRY_PORT}`);
|
||||
assert.doesNotMatch(plan, /"5000:5000"/, `the plan still publishes the catalogue's 5000`);
|
||||
said.push(` registry on ${REGISTRY_PORT}, as the module the mesh holds: ${bindings.trim()}`);
|
||||
const show = await nodeShow(CONTROL);
|
||||
assert.match(show, /mode\s+adopted since/, `the anchor is not reported adopted:\n${show}`);
|
||||
assert.ok(!(await on(CONTROL, `nft list table inet mesh`)).ok, `the foundation's dropping table was loaded on an adopted node`);
|
||||
const guard = await must(CONTROL, `nft list table inet mesh_guard`);
|
||||
// The guard's port set is the controller's to derive; what the record fixes is that it refuses
|
||||
// the store's port from outside and holds nothing but refusals.
|
||||
assert.match(guard, new RegExp(`dport (\\{[^}]*\\b${STORE_PORT}\\b[^}]*\\}|${STORE_PORT}) drop`), `the guard does not refuse the store's port:\n${guard}`);
|
||||
assert.doesNotMatch(guard, /accept\s*$/m, `the guard holds an accept:\n${guard}`);
|
||||
assert.match(await must(CONTROL, `ufw status`), /Status: active/, `the found firewall is not in force`);
|
||||
assert.match(await must(CONTROL, `curl -s -o /dev/null -w '%{http_code}' --max-time 5 http://127.0.0.1:${HELD_REGISTRY}/v2/`), /200/,
|
||||
`the predecessor's registry stopped answering`);
|
||||
said.push(show.trim(), guard.trim());
|
||||
return said.join("\n");
|
||||
});
|
||||
|
||||
if (WARM && steps.get("A3")?.ok) {
|
||||
await keep(SCENARIO, instanceId);
|
||||
console.log(`WARM: kept ${instanceId} at the adopted foundation`);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- nothing that serves changed -------------------------------------------------------------
|
||||
await step("B1", ["A3"], async () => {
|
||||
const said: string[] = [];
|
||||
const want = await must(CONTROL, `cat ${BEFORE}/file`);
|
||||
let page = "";
|
||||
await until(`the service answers the joiner as it did`, 120, async () => {
|
||||
page = (await on(JOINER, `curl -s --max-time 5 http://${ANCHOR}:${SERVED}/`)).out;
|
||||
return page === want ? true : null;
|
||||
}, () => page);
|
||||
said.push(` ${SERVICE} answers the joiner on ${SERVED} with the predecessor's page`);
|
||||
assert.equal((await must(CONTROL, `sha256sum ${SERVICE_FILE} | cut -d' ' -f1`)).trim(),
|
||||
(await must(CONTROL, `cat ${BEFORE}/file.sha256`)).trim(), `${SERVICE_FILE} changed`);
|
||||
assert.equal((await must(CONTROL, `docker inspect -f '{{.Id}}' ${SERVICE}`)).trim(),
|
||||
(await must(CONTROL, `cat ${BEFORE}/container.id`)).trim(), `the ${SERVICE} container was replaced`);
|
||||
said.push(` file, container byte for byte / the same id as before the mesh`);
|
||||
const was = (await must(CONTROL, `cat ${BEFORE}/ufw-added.txt`)).split("\n").map((l) => l.trim()).filter((l) => l.startsWith("ufw "));
|
||||
const now = await ufwAdded();
|
||||
const lost = was.filter((r) => !now.includes(r));
|
||||
const added = now.filter((r) => !was.includes(r));
|
||||
assert.deepEqual(lost, [], `the found firewall lost rules:\n${lost.join("\n")}`);
|
||||
const unmarked = added.filter((r) => !marked(r));
|
||||
assert.deepEqual(unmarked, [], `the found firewall gained rules not marked as the mesh's:\n${unmarked.join("\n")}`);
|
||||
assert.ok(added.length > 0, `the mesh opened nothing through the found firewall`);
|
||||
// ADR 0102: the runtime's file is written into, never over, and the runtime is reloaded.
|
||||
const daemon = JSON.parse(await must(CONTROL, `cat /etc/docker/daemon.json`)) as
|
||||
{ "log-opts"?: Record<string, string>; "insecure-registries"?: string[] };
|
||||
assert.equal(daemon["log-opts"]?.["max-size"], "7m", `the machine's own runtime setting was replaced: ${JSON.stringify(daemon)}`);
|
||||
assert.ok((daemon["insecure-registries"] ?? []).some((r) => r.includes(":")),
|
||||
`the mesh's registry trust is not in the runtime's file: ${JSON.stringify(daemon)}`);
|
||||
assert.ok((daemon["insecure-registries"] ?? []).length > 1,
|
||||
`the machine's own registries were replaced rather than added to: ${JSON.stringify(daemon)}`);
|
||||
const stillUp = (await must(CONTROL, `docker inspect -f '{{.State.Running}} {{.Id}}' ${NO_POLICY}`)).trim();
|
||||
assert.match(stillUp, /^true /, `the container with no restart policy is not running: ${stillUp}`);
|
||||
assert.equal(stillUp, (await must(CONTROL, `cat ${BEFORE}/nopolicy.id`)).trim(),
|
||||
`the container with no restart policy was restarted or replaced`);
|
||||
said.push(` runtime file the machine's log-opts kept, the mesh's registry added; ${NO_POLICY} still up`);
|
||||
said.push(` firewall ${was.length} rule(s) kept, ${added.length} added, every one marked:`, ...added.map((r) => ` ${r}`));
|
||||
return said.join("\n");
|
||||
});
|
||||
|
||||
await step("B2", ["A3"], async () => {
|
||||
const said: string[] = [];
|
||||
assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers a machine off the private network`);
|
||||
await must(CONTROL, `ufw reload`);
|
||||
await sleep(3_000);
|
||||
assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers from outside after the found firewall reloaded`);
|
||||
said.push(` outsider -> ${STORE_PORT} refused, before and after \`ufw reload\``);
|
||||
assert.ok(await connects(OUTSIDER, ANCHOR, BUS_PORT), `the bus does not answer a machine that has not enrolled`);
|
||||
said.push(` outsider -> ${BUS_PORT} the bus answers`);
|
||||
|
||||
// **What the guard is for** (ADR 0100, 0103): the store must be unreachable from outside
|
||||
// *whatever the found firewall does*. With ufw admitting both ports, only the guard is left
|
||||
// between the outsider and the store — and with the guard gone they are reachable, which is
|
||||
// what makes this a test of the guard rather than of ufw.
|
||||
const admit = [STORE_PORT, AMQP_PORT].map((p) =>
|
||||
`ufw route allow proto tcp to any port ${p} comment 'bed-probe-only ${p}'`);
|
||||
try {
|
||||
await must(CONTROL, admit.join(" && "));
|
||||
await sleep(2_000);
|
||||
assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)),
|
||||
`the store answers from outside once the found firewall admits it — the guard refuses nothing`);
|
||||
assert.ok(!(await connects(OUTSIDER, ANCHOR, AMQP_PORT)),
|
||||
`the broker's plaintext port answers from outside once the found firewall admits it`);
|
||||
said.push(` outsider -> ${STORE_PORT}, ${AMQP_PORT} still refused with the found firewall admitting both — the guard's own refusal`);
|
||||
|
||||
// The positive control: without the guard, both answer. Anything else would mean the probe
|
||||
// could not have failed.
|
||||
await must(CONTROL, `nft delete table inet mesh_guard`);
|
||||
await sleep(2_000);
|
||||
const openNow = (await connects(OUTSIDER, ANCHOR, STORE_PORT)) || (await connects(OUTSIDER, ANCHOR, AMQP_PORT));
|
||||
await must(CONTROL, `systemctl reload mesh-guard.service || systemctl restart mesh-guard.service`);
|
||||
await sleep(2_000);
|
||||
assert.ok(openNow, `neither port answered with the guard deleted, so the guard is not what refuses them`);
|
||||
assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store stayed open after the guard was loaded again`);
|
||||
said.push(` guard deleted both answered; loaded again, both refused`);
|
||||
} finally {
|
||||
await on(CONTROL, [STORE_PORT, AMQP_PORT].map((p) =>
|
||||
`ufw route delete allow proto tcp to any port ${p} comment 'bed-probe-only ${p}'`).join("; "));
|
||||
}
|
||||
return said.join("\n");
|
||||
});
|
||||
|
||||
// Its own step: it asks something different of the found firewall — a container on the machine
|
||||
// reaches a published port through the runtime's proxy, on the incoming path, not the forwarded.
|
||||
await step("B4", ["A3"], async () => {
|
||||
const said: string[] = [];
|
||||
// What this asks is the guard's promise: it never refuses the machine's own containers. The
|
||||
// found firewall stays in force (ADR 0100) and denies inbound by default, and a container on
|
||||
// the store's own network reaches its published port through the runtime's proxy — inbound,
|
||||
// not forwarded — so the operator's firewall has to admit the container interface for any
|
||||
// container to get there, on an adopted node as on a converged one. The probe admits it for
|
||||
// itself alone, and takes the rule away again.
|
||||
await must(CONTROL, `ufw allow in on docker0 to any port ${STORE_PORT} proto tcp comment bed-probe-only`);
|
||||
let fromContainer: { ok: boolean; out: string };
|
||||
try {
|
||||
fromContainer = await on(CONTROL, `docker run --rm ${ALPINE} nc -zv -w 3 ${ANCHOR} ${STORE_PORT} 2>&1`, 180_000);
|
||||
} finally {
|
||||
await on(CONTROL, `ufw delete allow in on docker0 to any port ${STORE_PORT} proto tcp comment bed-probe-only`);
|
||||
}
|
||||
if (!fromContainer.ok) {
|
||||
// Evidence, so the cause can be read from this run rather than guessed at the next one.
|
||||
const evidence = await on(CONTROL, [
|
||||
`echo '--- published'; docker ps --format '{{.Names}} {{.Ports}}' | grep -i ${STORE_PORT}`,
|
||||
`echo '--- from the host'; nc -zv -w 3 ${ANCHOR} ${STORE_PORT} 2>&1`,
|
||||
`echo '--- from the host network'; docker run --rm --network host ${ALPINE} nc -zv -w 3 ${ANCHOR} ${STORE_PORT} 2>&1`,
|
||||
`echo '--- iptables FORWARD, DOCKER-USER, isolation'; iptables -S FORWARD; iptables -S DOCKER-USER; iptables -S | grep -i isolation`,
|
||||
`echo '--- the guard'; nft list table inet mesh_guard`,
|
||||
`echo '--- nat for the port'; iptables -t nat -S | grep ${STORE_PORT}`,
|
||||
].join("; "), 120_000);
|
||||
assert.fail(`a container on the node cannot reach the store:\n${fromContainer.out}\n${evidence.out}`);
|
||||
}
|
||||
said.push(` container -> ${STORE_PORT} reachable from a container on the node itself`);
|
||||
return said.join("\n");
|
||||
});
|
||||
|
||||
// ---- the mesh works through the found firewall -----------------------------------------------
|
||||
let anchorOnMesh = "";
|
||||
await step("C1", ["B2"], async () => {
|
||||
const said: string[] = [];
|
||||
await mesh(`node add ${JOINER}`);
|
||||
const token = tokenFrom(await mesh(`token issue --node ${JOINER}`));
|
||||
const out = await must(JOINER, `${HOST_PATH} enrol --token ${quote(token)}`, 180_000);
|
||||
assert.match(out, new RegExp(`enrolled as ${JOINER}`), out);
|
||||
await must(JOINER, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
|
||||
said.push(` ${JOINER} enrolled over the bus, through the anchor's ufw`);
|
||||
await mesh(`overlay place ${JOINER} --site hosting`);
|
||||
await mesh(`assign ${JOINER} ${NETWORK_MODULE}`);
|
||||
await pushAndSettle(JOINER);
|
||||
// The hub's peer list changed: the anchor has to be sent it too.
|
||||
await pushAndSettle(CONTROL);
|
||||
anchorOnMesh = await meshAddressOf(CONTROL);
|
||||
await until(`the joiner reaches the anchor over mesh0`, 180, async () =>
|
||||
(await on(JOINER, `ping -c1 -W2 ${anchorOnMesh}`)).ok ? true : null,
|
||||
() => "no ping reply");
|
||||
said.push(` private network the joiner reaches the anchor at ${anchorOnMesh}`);
|
||||
said.push((await must(CONTROL, `wg show mesh0 latest-handshakes`)).trim());
|
||||
return said.join("\n");
|
||||
});
|
||||
|
||||
await step("B3", ["C1"], async () => {
|
||||
assert.ok(await connects(JOINER, anchorOnMesh, STORE_PORT), `the store does not answer over the private network`);
|
||||
return ` joiner -> ${anchorOnMesh}:${STORE_PORT} reachable over mesh0`;
|
||||
});
|
||||
|
||||
/** The mesh's own rules in the found firewall. */
|
||||
const openings = async (): Promise<string[]> => (await ufwAdded()).filter(marked);
|
||||
const assertOpenings = (rules: string[]) => {
|
||||
const text = rules.join("\n");
|
||||
// By the opening's id, which names the machine's port: a forwarded rule names the CONTAINER's
|
||||
// port (ufw's route rules match after the runtime's translation), so the text may say another.
|
||||
for (const port of [BUS_PORT, REGISTRY_PORT, HUB_PORT, STORE_PORT]) {
|
||||
assert.match(text, new RegExp(`opening-(tcp|udp)-${port}-`), `no opening for ${port} among the mesh's rules:\n${text}`);
|
||||
}
|
||||
};
|
||||
await step("C2", ["B3"], async () => {
|
||||
const before = await openings();
|
||||
assertOpenings(before);
|
||||
await must(CONTROL, `ufw reload`);
|
||||
await sleep(3_000);
|
||||
const after = await openings();
|
||||
assert.deepEqual(after.sort(), before.sort(), `the openings changed across a reload`);
|
||||
assert.ok(await connects(JOINER, anchorOnMesh, STORE_PORT), `the store stopped answering over mesh0 after the reload`);
|
||||
assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers from outside after the reload`);
|
||||
await pushAndSettle(JOINER);
|
||||
return ` after ufw reload ${after.length} opening(s) in place; the joiner reaches the store over mesh0 and takes a push\n` +
|
||||
after.map((r) => ` ${r}`).join("\n");
|
||||
});
|
||||
|
||||
await step("C3", ["C2"], async () => {
|
||||
const before = await openings();
|
||||
await restartMachine(CONTROL);
|
||||
await until(`the control plane answers after the reboot`, 300, async () =>
|
||||
(await meshSays(`status`)).ok ? true : null, () => "no answer");
|
||||
assert.match(await must(CONTROL, `ufw status`), /Status: active/, `the found firewall is not in force after the reboot`);
|
||||
assert.match(await must(CONTROL, `nft list table inet mesh_guard`), /drop/, `the guard did not come back`);
|
||||
const after = await until(`the openings are there again`, 420, async () => {
|
||||
const now = await openings();
|
||||
return before.every((r) => now.includes(r)) ? now : null;
|
||||
}, () => "not all openings");
|
||||
assertOpenings(after);
|
||||
assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers from outside after the reboot`);
|
||||
await until(`the joiner reaches the store over mesh0 again`, 300, async () =>
|
||||
(await connects(JOINER, anchorOnMesh, STORE_PORT)) ? true : null, () => "no connection");
|
||||
await pushAndSettle(JOINER);
|
||||
const page = await must(JOINER, `curl -s --max-time 5 http://${ANCHOR}:${SERVED}/`);
|
||||
assert.match(page, /hello from the predecessor/, `the predecessor's service did not come back: ${page}`);
|
||||
return ` after a reboot ufw active, the guard loaded, ${after.length} opening(s); the joiner reaches the store and takes a push`;
|
||||
});
|
||||
|
||||
// ---- assigning prepares, taking cuts over ----------------------------------------------------
|
||||
let kept = "";
|
||||
await step("D1", ["B1"], async () => {
|
||||
const said: string[] = [];
|
||||
// The catalogue's module, read from the catalogue, with the route requirement taken off: the
|
||||
// route needs a proxy module and a public name, and neither is what adoption is about.
|
||||
const manifest = JSON.parse(catalogueModule(SERVICE, [])) as Record<string, unknown>;
|
||||
delete manifest["requires"]; delete manifest["contributes"]; delete manifest["binds"];
|
||||
await registerModule(SERVICE, JSON.stringify(manifest));
|
||||
await mesh(`assign ${CONTROL} ${SERVICE}`);
|
||||
await pushAndSettle(CONTROL);
|
||||
const show = await until(`the anchor reports holding ${SERVICE}'s container and file`, 120, async () => {
|
||||
const s = await nodeShow(CONTROL);
|
||||
return /holds container\s+hello-web\b/.test(s) && /holds file\s+\/var\/lib\/hello-web\/index\.html/.test(s) ? s : null;
|
||||
}, () => "");
|
||||
kept = show.match(/holds file\s+\/var\/lib\/hello-web\/index\.html[^\n]*\n\s*original kept at (\S+)/)?.[1] ?? "";
|
||||
assert.ok(kept, `the held file's original is not reported kept:\n${show}`);
|
||||
assert.equal((await must(CONTROL, `sha256sum ${SERVICE_FILE} | cut -d' ' -f1`)).trim(),
|
||||
(await must(CONTROL, `cat ${BEFORE}/file.sha256`)).trim(), `assigning changed ${SERVICE_FILE}`);
|
||||
assert.equal((await must(CONTROL, `docker inspect -f '{{.Id}}' ${SERVICE}`)).trim(),
|
||||
(await must(CONTROL, `cat ${BEFORE}/container.id`)).trim(), `assigning replaced the ${SERVICE} container`);
|
||||
assert.equal(await must(CONTROL, `cat ${kept}`), await must(CONTROL, `cat ${BEFORE}/file`), `the kept original is not the file as found`);
|
||||
said.push(show.trim());
|
||||
return said.join("\n");
|
||||
});
|
||||
|
||||
await step("D2", ["D1"], async () => {
|
||||
await must(CONTROL, `systemctl start predecessor-sync`);
|
||||
try {
|
||||
await sleep(15_000);
|
||||
await pushAndSettle(CONTROL);
|
||||
const show = await until(`the anchor reports the held file changed`, 120, async () => {
|
||||
const s = await nodeShow(CONTROL);
|
||||
return /hello-web\/index\.html[^\n]*REWRITTEN/i.test(s) ? s : null;
|
||||
}, () => "");
|
||||
// Stop the writer first, then hash: while it rewrites every ten seconds, a file the host
|
||||
// reverted in between would still read as the predecessor's a moment later.
|
||||
await must(CONTROL, `systemctl stop predecessor-sync`);
|
||||
const was = await must(CONTROL, `sha256sum ${SERVICE_FILE} | cut -d' ' -f1`);
|
||||
await pushAndSettle(CONTROL);
|
||||
await sleep(5_000);
|
||||
const now = await must(CONTROL, `cat ${SERVICE_FILE}`);
|
||||
assert.match(now, /synced \d+/, `the host reverted what the predecessor wrote:\n${now}`);
|
||||
assert.equal(await must(CONTROL, `sha256sum ${SERVICE_FILE} | cut -d' ' -f1`), was,
|
||||
`the held file changed under a push after the predecessor stopped writing`);
|
||||
return show.trim();
|
||||
} finally {
|
||||
await on(CONTROL, `systemctl stop predecessor-sync`);
|
||||
}
|
||||
});
|
||||
|
||||
await step("D3", ["D1"], async () => {
|
||||
await pushAndSettle(CONTROL);
|
||||
const r = await meshSays(`converge ${CONTROL}`);
|
||||
assert.ok(!r.ok, `converge went ahead while ${SERVICE} holds its found container:\n${r.out}`);
|
||||
assert.match(r.out, new RegExp(`hello-web holds the found container hello-web`), `the refusal does not name the held container:\n${r.out}`);
|
||||
assert.match(r.out, new RegExp(`take ${CONTROL} hello-web`), `the refusal does not say what to do:\n${r.out}`);
|
||||
return r.out.trim();
|
||||
});
|
||||
|
||||
await step("D4", ["D1"], async () => {
|
||||
const said: string[] = [];
|
||||
said.push((await mesh(`take ${CONTROL} ${SERVICE}`)).trim());
|
||||
await pushAndSettle(CONTROL);
|
||||
const label = await until(`the ${SERVICE} container is the mesh's`, 180, async () => {
|
||||
const l = (await on(CONTROL, `docker inspect -f '{{index .Config.Labels "mesh-host.spec"}}' ${SERVICE}`)).out.trim();
|
||||
return l && l !== "<no value>" ? l : null;
|
||||
}, () => "");
|
||||
assert.notEqual((await must(CONTROL, `docker inspect -f '{{.Id}}' ${SERVICE}`)).trim(),
|
||||
(await must(CONTROL, `cat ${BEFORE}/container.id`)).trim(), `the found container was not replaced`);
|
||||
const catalogue = (JSON.parse(catalogueModule(SERVICE, [])) as { resources: { id: string; content?: string }[] })
|
||||
.resources.find((r) => r.id === "page")?.content ?? "";
|
||||
assert.equal(await must(CONTROL, `cat ${SERVICE_FILE}`), catalogue, `the found file was not replaced with the module's`);
|
||||
assert.equal(await must(CONTROL, `cat ${kept}`), await must(CONTROL, `cat ${BEFORE}/file`), `the original was not kept`);
|
||||
said.push(` replaced container (spec ${label.slice(0, 12)}…) and ${SERVICE_FILE}; original still at ${kept}`);
|
||||
// Either the mesh opened the port, or the predecessor's own rule already admits it — then the
|
||||
// mesh adds nothing and will remove nothing (ADR 0103), and the operator's rule stays theirs.
|
||||
const opened = (await openings()).filter((r) => new RegExp(`opening-tcp-${SERVED}-`).test(r));
|
||||
const operators = (await ufwAdded()).filter((r) => !marked(r) && new RegExp(`\\b${SERVED}\\b`).test(r));
|
||||
assert.ok(opened.length > 0 || operators.length > 0,
|
||||
`no opening for ${SERVED} once ${SERVICE} was taken, and no rule of the operator's admits it:\n${(await ufwAdded()).join("\n")}`);
|
||||
assert.ok(operators.length > 0, `the operator's own rule for ${SERVED} is gone:\n${(await ufwAdded()).join("\n")}`);
|
||||
said.push(...opened.map((r) => ` opened ${r}`));
|
||||
if (opened.length === 0) said.push(` opening ${SERVED} satisfied by the operator's own rule: ${operators.join(" | ")}`);
|
||||
const page = await until(`the taken ${SERVICE} answers over the private network`, 120, async () => {
|
||||
const p = await on(JOINER, `curl -s --max-time 3 http://${anchorOnMesh}:${SERVED}/`);
|
||||
return p.ok && p.out === catalogue ? p.out : null;
|
||||
}, () => "");
|
||||
said.push(` joiner -> ${SERVED} ${page.trim()}`);
|
||||
const show = await nodeShow(CONTROL);
|
||||
assert.doesNotMatch(show, /holds (container|file)\s+\S*hello-web/, `the anchor still holds what was taken:\n${show}`);
|
||||
return said.join("\n");
|
||||
});
|
||||
|
||||
// ---- converging previews, then changes -------------------------------------------------------
|
||||
await step("E1", ["D4"], async () => {
|
||||
if (!/^nftables\b/m.test(await mesh(`module list`))) {
|
||||
await registerModule(FILTER_MODULE, JSON.stringify(JSON.parse(catalogueModule(FILTER_MODULE, []))));
|
||||
}
|
||||
// What the flip will close must be reachable now, or its closing proves nothing.
|
||||
assert.ok(await connects(JOINER, anchorOnMesh, HELD_REGISTRY),
|
||||
`the predecessor's published ${HELD_REGISTRY} is not reachable over the private network before the flip`);
|
||||
await pushAndSettle(CONTROL);
|
||||
const preview = await mesh(`converge ${CONTROL}`);
|
||||
assert.match(preview, new RegExp(`tcp/${SERVED}\\b[^\\n]*declared by hello-web`), `the preview does not name the service's port as declared:\n${preview}`);
|
||||
assert.match(preview, new RegExp(`tcp/${HELD_REGISTRY}\\b[^\\n]*published[^\\n]*WILL CLOSE`), `the preview does not name the published ${HELD_REGISTRY} as closing:\n${preview}`);
|
||||
assert.match(preview, /the flip takes:\n(\s{4}\S+\n?)+/, `the preview names no module the flip takes:\n${preview}`);
|
||||
assert.match(preview, new RegExp(`\\n\\s{4}${NETWORK_MODULE}\\b`), `the preview does not say the flip takes ${NETWORK_MODULE}:\n${preview}`);
|
||||
assert.match(preview, /Nothing has changed/, preview);
|
||||
assert.match(await must(CONTROL, `ufw status`), /Status: active/, `previewing changed the firewall`);
|
||||
return preview.trim();
|
||||
});
|
||||
|
||||
await step("E2", ["E1"], async () => {
|
||||
const said: string[] = [];
|
||||
// The flip as the preview says to make it — whatever the preview binds `--yes` to.
|
||||
const preview = await mesh(`converge ${CONTROL}`);
|
||||
const flip = preview.match(new RegExp(`\`(converge ${CONTROL} --yes[^\`]*)\``))?.[1];
|
||||
assert.ok(flip, `the preview does not say how to make the flip:\n${preview}`);
|
||||
said.push((await mesh(flip, 300_000)).trim().split("\n").slice(-3).join("\n"));
|
||||
await settled();
|
||||
const table = await until(`the derived filter is loaded`, 300, async () => {
|
||||
const t = await on(CONTROL, `nft list table inet mesh`);
|
||||
return t.ok && /policy drop/.test(t.out) ? t.out : null;
|
||||
}, () => "");
|
||||
const status = await until(`the found firewall is disabled`, 180, async () => {
|
||||
const s = (await on(CONTROL, `ufw status`)).out;
|
||||
return /Status: inactive/.test(s) ? s : null;
|
||||
}, () => "");
|
||||
assert.ok((await on(CONTROL, `test -s /etc/ufw/user.rules && grep -q 'BEGIN UFW AND DOCKER' /etc/ufw/after.rules`)).ok,
|
||||
`the found firewall's configuration is not on disk any more`);
|
||||
assert.match(await nodeShow(CONTROL), /mode\s+converged/, `the anchor is not reported converged`);
|
||||
said.push(` ufw ${status.trim()} — /etc/ufw/user.rules and after.rules still on disk`);
|
||||
await until(`the declared ${SERVED} answers over the private network`, 120, async () =>
|
||||
(await connects(JOINER, anchorOnMesh, SERVED)) ? true : null, () => "");
|
||||
assert.ok(!(await connects(JOINER, anchorOnMesh, HELD_REGISTRY)), `the undeclared ${HELD_REGISTRY} is still open`);
|
||||
assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers from outside once converged`);
|
||||
said.push(` ports ${SERVED} open over the private network; ${HELD_REGISTRY} closed; the store still closed from outside`);
|
||||
said.push(table.split("\n").slice(0, 30).join("\n"));
|
||||
return said.join("\n");
|
||||
});
|
||||
|
||||
await step("E3", ["E2"], async () => {
|
||||
const said = (await mesh(`adopt ${CONTROL}`, 300_000)).trim();
|
||||
await settled();
|
||||
await until(`the found firewall is enabled again`, 180, async () =>
|
||||
/Status: active/.test((await on(CONTROL, `ufw status`)).out) ? true : null, () => "");
|
||||
await until(`the derived filter is gone`, 180, async () =>
|
||||
!(await on(CONTROL, `nft list table inet mesh`)).ok ? true : null, () => "");
|
||||
assert.match(await must(CONTROL, `nft list table inet mesh_guard`), /drop/, `the guard is not restored`);
|
||||
assertOpenings(await until(`the openings are back`, 180, async () => {
|
||||
const o = await openings();
|
||||
return o.length ? o : null;
|
||||
}, () => ""));
|
||||
assert.match(await nodeShow(CONTROL), /mode\s+adopted since/, `the anchor is not reported adopted`);
|
||||
assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers from outside once adopted again`);
|
||||
return `${said}\n ufw active, table inet mesh gone, the guard and the openings back`;
|
||||
});
|
||||
|
||||
// ---- what the mesh added, it takes back; what it found, it leaves ---------------------------
|
||||
await step("F1", ["E3"], async () => {
|
||||
const said: string[] = [];
|
||||
const before = await ufwAdded();
|
||||
const operators = before.filter((r) => !marked(r) && new RegExp(`\\b${SERVED}\\b`).test(r));
|
||||
assert.ok(operators.length > 0, `the operator's own rules for ${SERVED} are already gone:\n${before.join("\n")}`);
|
||||
await mesh(`unassign ${CONTROL} ${SERVICE}`);
|
||||
await pushAndSettle(CONTROL);
|
||||
let lastRules: string[] = before;
|
||||
const after = await until(`the mesh's own rules for ${SERVED} are gone`, 180, async () => {
|
||||
const rules = await ufwAdded();
|
||||
lastRules = rules;
|
||||
return rules.some((r) => marked(r) && new RegExp(`opening-tcp-${SERVED}-`).test(r)) ? null : rules;
|
||||
}, () => lastRules.join("\n"));
|
||||
for (const rule of operators) {
|
||||
assert.ok(after.includes(rule), `the operator's own rule went with the mesh's opening: ${rule}\n${after.join("\n")}`);
|
||||
}
|
||||
said.push(` kept ${operators.length} rule(s) of the operator's, unmarked, after the module was unassigned`);
|
||||
said.push(...operators.map((r) => ` ${r}`));
|
||||
return said.join("\n");
|
||||
});
|
||||
|
||||
stateOutcome();
|
||||
}, { timeout: 10_800_000 });
|
||||
|
||||
after(async () => {
|
||||
if (KEEP || WARM) {
|
||||
console.log(`\nLEFT STANDING: ${instanceId} — not destroyed (${KEEP ? "MESH_LAB_KEEP" : "MESH_LAB_WARM"}).`);
|
||||
return;
|
||||
}
|
||||
if (instanceId) await destroy(instanceId);
|
||||
await destroyAll(`${SCENARIO}-`);
|
||||
}, { timeout: 900_000 });
|
||||
|
||||
for (const [code, title] of Object.entries(TITLE)) {
|
||||
test(`${code} ${title}`, { skip, timeout: 60_000 }, () => {
|
||||
const s = steps.get(code);
|
||||
assert.ok(s?.ok, s ? `${s.why}` : `${code} never ran`);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
/**
|
||||
* A machine trusts the mesh's own authority because a module put its root there — and stops when
|
||||
* the module is taken away (novox/hq ADR 0147, 04-ISSUES/129).
|
||||
*
|
||||
* What is dialled is the authority itself. step-ca serves its own API with a leaf it issued, so a
|
||||
* plain client verifying that handshake — no `-k`, no `--cacert` — is verifying exactly one thing:
|
||||
* that this machine's trust store now contains the mesh's root. No proxy, no routed name, no public
|
||||
* issuance. A trust bed leaning on those would pass for their reasons, which is the failure this
|
||||
* whole sequence keeps producing.
|
||||
*
|
||||
* The negative half runs twice, before the module is assigned and after it is unassigned. An anchor
|
||||
* bed that only checks the success would pass on a machine that already trusted everything, and
|
||||
* would say nothing at all about removal — which is the half issue 129 asked for by name.
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host
|
||||
* MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||
* MESH_LAB_CATALOG=.../mesh-catalog/modules
|
||||
* step-ca's image is upstream, pinned by the catalogue, pulled by the machine over its uplink.
|
||||
* ca-trust carries no image: a script, a unit, and the machine's own systemd.
|
||||
*/
|
||||
|
||||
import { test, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { existsSync } from "node:fs";
|
||||
import { loadScenario } from "../../src/declaration/parse.ts";
|
||||
import { raise } from "../../src/lifecycle/raise.ts";
|
||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||||
import { labIsUsable, destroyAll, foundationBundle, catalogueModule, catalogueIsPresent } from "./harness.ts";
|
||||
import type { HeldImage } from "../../src/pinning.ts";
|
||||
|
||||
const capability = await labIsUsable();
|
||||
const binary = hostBinaryPath();
|
||||
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
||||
|
||||
const skip = !capability.usable
|
||||
? `lab not usable: ${capability.why}`
|
||||
: !binary || !existsSync(binary)
|
||||
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
||||
: !bundle || !existsSync(bundle)
|
||||
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
||||
: catalogueIsPresent();
|
||||
|
||||
const SCENARIO = "trust-anchor";
|
||||
const MACHINE = "anchor";
|
||||
/** The authority's own API, which it serves with a certificate it issued itself. */
|
||||
const AUTHORITY = "https://127.0.0.1:9000/health";
|
||||
/** Where the module puts the root, and therefore what removal must take away. */
|
||||
const ANCHOR = "/etc/ca-certificates/trust-source/anchors/mesh-internal-ca.crt";
|
||||
|
||||
let instanceId = "";
|
||||
let held: HeldImage[] = [];
|
||||
|
||||
function quote(s: string): string {
|
||||
return `'${s.replaceAll("'", `'\\''`)}'`;
|
||||
}
|
||||
|
||||
async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
||||
const { stdout } = await exec(instanceId, MACHINE, [
|
||||
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
|
||||
], timeoutMs);
|
||||
const marker = stdout.lastIndexOf("__exit=");
|
||||
if (marker < 0) return { out: stdout, ok: false };
|
||||
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
||||
}
|
||||
|
||||
async function must(command: string, timeoutMs?: number): Promise<string> {
|
||||
const { out, ok } = await on(command, timeoutMs);
|
||||
if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The control plane, a container on the node. */
|
||||
async function mesh(command: string, timeoutMs?: number): Promise<string> {
|
||||
return must(`docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
|
||||
}
|
||||
|
||||
function tokenFrom(said: string): string {
|
||||
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
|
||||
assert.ok(found, `no token in:\n${said}`);
|
||||
return found;
|
||||
}
|
||||
|
||||
async function settled(withinMs = 480_000): Promise<void> {
|
||||
const until = Date.now() + withinMs;
|
||||
let last = "";
|
||||
while (Date.now() < until) {
|
||||
const asked = await on(`docker exec mesh-controller /mesh-controller status --json`);
|
||||
if (asked.ok) {
|
||||
try {
|
||||
const state = JSON.parse(asked.out) as {
|
||||
wrong: { node: string; outcome: string }[];
|
||||
waiting: { node: string }[];
|
||||
reported: { node: string; outcome: string; current: boolean }[];
|
||||
};
|
||||
const bad = state.wrong.find((w) => w.node === MACHINE);
|
||||
if (bad) throw new Error(`${MACHINE} did not apply what it was sent: ${bad.outcome}\n${asked.out}`);
|
||||
const word = state.reported.find((r) => r.node === MACHINE);
|
||||
if (!state.waiting.some((w) => w.node === MACHINE) && word?.outcome === "applied" && word.current) return;
|
||||
last = asked.out;
|
||||
} catch (err) {
|
||||
if (err instanceof Error && err.message.includes("did not apply")) throw err;
|
||||
last = asked.out;
|
||||
}
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 5000));
|
||||
}
|
||||
throw new Error(`${MACHINE} never caught up within ${Math.round(withinMs / 1000)}s. Last:\n${last}`);
|
||||
}
|
||||
|
||||
/** Install a catalogue module's manifest into the control plane, read from the catalogue. */
|
||||
async function register(module: string): Promise<void> {
|
||||
const manifest = catalogueModule(module, held);
|
||||
await must(
|
||||
`printf %s ${quote(manifest)} > /tmp/${module}.json && ` +
|
||||
`docker cp /tmp/${module}.json mesh-controller:/${module}.json`,
|
||||
);
|
||||
await mesh(`module add /${module}.json`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Put the mesh's composed user list where this machine's bus reads it, and make it re-read.
|
||||
*
|
||||
* **What genesis has to do by hand, and only genesis** (novox/hq 04-ISSUES/146). Every account on
|
||||
* the bus reaches it in the declaration of the machine that runs it — which requires that machine
|
||||
* to be an enrolled node, and at genesis it is not. So until the bus is a module, the composition
|
||||
* is placed by whoever is raising the machine: the control plane says what it composed, and this
|
||||
* writes it beside the bus's configuration. Twice, because two accounts come into existence at
|
||||
* different moments — the enrolment when the token is issued, and the node's own when it enrols.
|
||||
*/
|
||||
async function composeTheBusUsers(): Promise<void> {
|
||||
await must(
|
||||
`docker exec mesh-controller /mesh-controller broker accounts > /var/lib/mesh-bus-conf/accounts.conf && ` +
|
||||
`docker kill -s HUP mesh-broker`,
|
||||
);
|
||||
}
|
||||
|
||||
/** Dial the authority the way anything on this machine would: verifying, with nothing handed to it. */
|
||||
async function verifying(): Promise<{ out: string; ok: boolean }> {
|
||||
return on(`curl --silent --show-error --max-time 10 ${AUTHORITY}`);
|
||||
}
|
||||
|
||||
before(async () => {
|
||||
if (skip) return;
|
||||
|
||||
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
||||
onProgress: (m) => console.log(`raise: ${m}`),
|
||||
});
|
||||
instanceId = raised.instanceId;
|
||||
held = raised.images;
|
||||
|
||||
await must(`cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${foundationBundle(bundle, raised.images)}\nMESHBUNDLE`);
|
||||
await must(`${HOST_PATH} apply /tmp/foundation.lock`, 600_000);
|
||||
const up = await must(`docker ps --format '{{.Names}}'`);
|
||||
for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) {
|
||||
assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`);
|
||||
}
|
||||
|
||||
// The control plane is a container, and a container that is restarting answers nothing. Asked
|
||||
// until it answers rather than once, so a crash-looping control plane is reported as itself
|
||||
// instead of as whatever command happened to be sent first.
|
||||
let control = { out: "", ok: false };
|
||||
const answering = Date.now() + 120_000;
|
||||
while (Date.now() < answering) {
|
||||
control = await on(`docker exec mesh-controller /mesh-controller status`);
|
||||
if (control.ok) break;
|
||||
await new Promise((r) => setTimeout(r, 3000));
|
||||
}
|
||||
assert.ok(control.ok,
|
||||
`the control plane never answered:\n${control.out}\n` +
|
||||
`${(await on(`docker logs mesh-controller 2>&1 | tail -40`)).out}`);
|
||||
|
||||
await mesh(`node add ${MACHINE}`);
|
||||
const token = tokenFrom(await mesh(`token issue --node ${MACHINE}`));
|
||||
// The account that token is the password of exists in the mesh's records now; this is what puts
|
||||
// it on the bus, because nothing else can until this machine is a node.
|
||||
await composeTheBusUsers();
|
||||
await must(`${HOST_PATH} enrol --token ${quote(token)}`);
|
||||
// And again for the credential enrolment just minted, which the machine's own link connects with.
|
||||
await composeTheBusUsers();
|
||||
await must(`nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
|
||||
}, { timeout: 1_800_000 });
|
||||
|
||||
after(async () => {
|
||||
if (process.env["MESH_LAB_KEEP"]) {
|
||||
console.log(`MESH_LAB_KEEP set — leaving ${instanceId} standing`);
|
||||
return;
|
||||
}
|
||||
if (instanceId) await destroy(instanceId);
|
||||
await destroyAll(`${SCENARIO}-`);
|
||||
}, { timeout: 600_000 });
|
||||
|
||||
test("the mesh's authority is verified on a machine holding ca-trust, and not on one that is not", {
|
||||
skip, timeout: 1_800_000,
|
||||
}, async () => {
|
||||
// The private network first. The authority certifies itself for the address it holds there
|
||||
// (`${machine:at}`), which a machine with no overlay has not got — so this is what the bed needs
|
||||
// it for, and nothing else.
|
||||
await mesh(`overlay place ${MACHINE} --hub --endpoint 192.0.2.10:51820 --site lab`);
|
||||
// The control plane ships this one — its resources are computed per node, so there is no
|
||||
// manifest to register.
|
||||
await mesh(`assign ${MACHINE} networking`);
|
||||
await mesh(`push ${MACHINE}`);
|
||||
await settled();
|
||||
|
||||
// The authority next, on its own. Nothing about trust yet.
|
||||
await register("step-ca");
|
||||
// No `module issue`: that delivers a bus account, and the authority declares none — its own
|
||||
// secret is the password it initialises itself with, which the mesh mints at assignment.
|
||||
await mesh(`assign ${MACHINE} step-ca`);
|
||||
await mesh(`push ${MACHINE}`);
|
||||
await settled();
|
||||
|
||||
// It is up and answering — asked the way nothing else in this bed is allowed to ask, with
|
||||
// verification off, because at this point in the bed no machine could verify it.
|
||||
let answering = false;
|
||||
const until = Date.now() + 180_000;
|
||||
while (Date.now() < until && !answering) {
|
||||
answering = (await on(`curl --silent --insecure --max-time 5 ${AUTHORITY}`)).ok;
|
||||
if (!answering) await new Promise((r) => setTimeout(r, 3000));
|
||||
}
|
||||
assert.ok(answering, `the authority never answered:\n${(await on(`docker logs step-ca 2>&1 | tail -30`)).out}`);
|
||||
|
||||
// **The negative half, before anything is assigned.** If this passes, the bed is measuring
|
||||
// something already in the machine's trust store and everything below it is worthless.
|
||||
const before = await verifying();
|
||||
assert.equal(before.ok, false, `the authority verified before anything anchored its root:\n${before.out}`);
|
||||
assert.match(before.out, /certificate|issuer/i, `it failed for some other reason:\n${before.out}`);
|
||||
|
||||
// Now the module.
|
||||
await register("ca-trust");
|
||||
await mesh(`assign ${MACHINE} ca-trust`);
|
||||
await mesh(`push ${MACHINE}`);
|
||||
await settled();
|
||||
|
||||
const unit = await on(`systemctl is-active mesh-ca-trust.service`);
|
||||
assert.ok(unit.ok, `the unit is ${unit.out.trim()}:\n${(await on(`journalctl -u mesh-ca-trust --no-pager | tail -30`)).out}`);
|
||||
await must(`test -s ${ANCHOR}`);
|
||||
const anchors = await must(`trust list | grep -A2 -i 'Mesh Internal CA' || trust list`);
|
||||
assert.match(anchors, /Mesh Internal CA/, `the mesh's authority is not among the machine's anchors:\n${anchors}`);
|
||||
|
||||
// **The whole claim, in one command.** No -k, no --cacert: the machine's own trust store, and a
|
||||
// certificate the mesh's authority issued.
|
||||
const after = await verifying();
|
||||
assert.ok(after.ok, `the authority still does not verify with the module assigned:\n${after.out}`);
|
||||
|
||||
// **And removal is the same unit's business.** Unassigned, the host stops it; stopping it takes
|
||||
// the anchor away and refreshes the bundles, so the machine stops trusting the mesh.
|
||||
await mesh(`unassign ${MACHINE} ca-trust`);
|
||||
await mesh(`push ${MACHINE}`);
|
||||
await settled();
|
||||
|
||||
const gone = await on(`test -e ${ANCHOR}`);
|
||||
assert.equal(gone.ok, false, "the anchor is still on the machine after the module was unassigned");
|
||||
const afterwards = await verifying();
|
||||
assert.equal(afterwards.ok, false, `the machine still verifies the mesh's authority with nothing anchoring it:\n${afterwards.out}`);
|
||||
assert.match(afterwards.out, /certificate|issuer/i, `it failed for some other reason:\n${afterwards.out}`);
|
||||
});
|
||||
@@ -8,15 +8,14 @@
|
||||
*
|
||||
* This bed proves that across two machines. `anchor` is the control-node: it raises the foundation
|
||||
* and adopts `postgres` there, so `mesh-store` is the one store and `mesh-postgres` its provisioner.
|
||||
* `laptop` joins and runs the CONSUMERS — baserow and letta, which require `postgres-database` — plus
|
||||
* a co-located `redis` (which holds no seat). Each consumer's database is minted on the store on
|
||||
* anchor and reached over the overlay: their bindings name `anchor.internal`, and their minted logins
|
||||
* authenticate against the store. redis stays co-located on laptop for baserow's cache.
|
||||
* `laptop` joins and runs the CONSUMERS — baserow and letta, which require `postgres-database`. Each
|
||||
* consumer's database is minted on the store on anchor and reached over the overlay: their bindings
|
||||
* name `anchor.internal`, and their minted logins authenticate against the store. baserow's cache is
|
||||
* its own, inside its container (novox/hq 081).
|
||||
*
|
||||
* The four manifests are the committed catalogue shapes (novox/hq ADR 0039/0047/0048), verbatim
|
||||
* from the catalogue-broad bed — postgres publishes 5432 so its consumers connect, redis runs on
|
||||
* the host network with a lab-local seal key, and baserow/letta wire their servers to the grant the
|
||||
* mesh writes. They are added, each issued a scoped broker account, assigned to laptop, and pushed
|
||||
* The three manifests are the committed catalogue shapes (novox/hq ADR 0039/0047/0048), verbatim
|
||||
* from the catalogue-broad bed — postgres publishes 5432 so its consumers connect, and baserow/letta
|
||||
* wire their servers to the grant the mesh writes. They are added, each issued a scoped broker account, assigned to laptop, and pushed
|
||||
* ONCE; laptop converges once with every one up, and the two consumers are provisioned against the
|
||||
* database the provider on their own node gave them.
|
||||
*
|
||||
@@ -25,12 +24,11 @@
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host
|
||||
* MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||
*
|
||||
* HELPER — stock the four runtimes into the local daemon before the run (some may already be there):
|
||||
* HELPER — stock the three runtimes into the local daemon before the run (some may already be there):
|
||||
* scripts/build-module-runtime.sh postgres /tmp/postgres.tar
|
||||
* scripts/build-module-runtime.sh redis /tmp/redis.tar
|
||||
* scripts/build-module-runtime.sh baserow /tmp/baserow.tar
|
||||
* scripts/build-module-runtime.sh letta /tmp/letta.tar
|
||||
* The service images (postgres:17-alpine, redis:7-alpine, baserow/baserow:latest, letta/letta:latest)
|
||||
* The service images (postgres, baserow, letta, each pinned by digest)
|
||||
* must be in the local daemon too; scenarios/two-node-db.yml stocks all of them, and each node pulls
|
||||
* what it runs from the scenario's own registry by digest.
|
||||
*/
|
||||
@@ -152,12 +150,13 @@ async function settled(node: string, withinMs = 1_200_000): Promise<void> {
|
||||
last = said;
|
||||
await new Promise((r) => setTimeout(r, 5000));
|
||||
}
|
||||
// Timed out — capture what the node is actually doing so the failure is diagnosable.
|
||||
const ps = (await on(NODE, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out;
|
||||
const hostLog = (await on(NODE, `tail -80 /var/log/mesh-host.log`)).out;
|
||||
// Timed out — capture what the node that did not answer is doing, so the failure is diagnosable.
|
||||
// Its own machine, not the second node's: the anchor timing out used to print the laptop's log.
|
||||
const ps = (await on(node, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out;
|
||||
const hostLog = (await on(node, `tail -80 /var/log/mesh-host.log`)).out;
|
||||
throw new Error(
|
||||
`${node} never caught up within ${Math.round(withinMs / 1000)}s.\nLast status:\n${last}\n` +
|
||||
`--- ${NODE} docker ps -a ---\n${ps}\n--- ${NODE} mesh-host.log tail ---\n${hostLog}`);
|
||||
`--- ${node} docker ps -a ---\n${ps}\n--- ${node} mesh-host.log tail ---\n${hostLog}`);
|
||||
}
|
||||
|
||||
before(async () => {
|
||||
@@ -205,82 +204,38 @@ test("consumers on a joined node get their databases from the one foundation sto
|
||||
// they land on changes.
|
||||
// ================================================================================================
|
||||
|
||||
// --- redis: a cache provider on the host network (127.0.0.1:6379). No seal key: the provider
|
||||
// is handed the minted credential already unsealed by the host (ADR 0048). It carries
|
||||
// the committed provides/serves/receives/grants so baserow's redis-cache requirement resolves. ----
|
||||
const redisManifest = JSON.stringify({
|
||||
module: "redis",
|
||||
version: "1",
|
||||
provides: [{ name: "redis-cache", scope: "mesh" }],
|
||||
serves: { "redis-cache": { port: 6379 } },
|
||||
emits: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"],
|
||||
consumes: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"],
|
||||
receives: { "redis-cache": "/var/lib/redis-module/grants/mesh.json" },
|
||||
grants: { "redis-cache": "/var/lib/redis-module/grants" },
|
||||
"own-secrets": { default: "/var/lib/redis-module/default.secret", broker: "/var/lib/mesh/redis/broker" },
|
||||
resources: [
|
||||
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/redis", mode: "0700" },
|
||||
{ id: "state", type: "directory", path: "/var/lib/redis-module", mode: "0700" },
|
||||
{ id: "grants-dir", type: "directory", path: "/var/lib/redis-module/grants", mode: "0700" },
|
||||
{ id: "data", type: "directory", path: "/services/redis/data", mode: "0700", owner: "999:999" },
|
||||
{
|
||||
id: "server-conf", type: "file", path: "/var/lib/redis-module/redis.conf", mode: "0644",
|
||||
content: "requirepass ${secret:default}\nappendonly no\ndir /data\n",
|
||||
},
|
||||
{
|
||||
id: "server", type: "container", name: "redis", image: pinned("redis"), network: "host",
|
||||
volumes: [
|
||||
"/services/redis/data:/data",
|
||||
"/var/lib/redis-module/redis.conf:/etc/redis/redis.conf:ro",
|
||||
],
|
||||
args: ["/etc/redis/redis.conf"],
|
||||
},
|
||||
{
|
||||
id: "runtime", type: "container", name: "mesh-redis", image: pinned("mesh-runtime-redis"),
|
||||
network: "host",
|
||||
volumes: [
|
||||
"/var/lib/mesh/redis/broker:/run/secrets/broker:ro",
|
||||
"/var/lib/redis-module/grants:/var/lib/redis-module/grants",
|
||||
"/var/lib/redis-module/default.secret:/run/secrets/default:ro",
|
||||
],
|
||||
env: {
|
||||
MESH_BROKER_FILE: "/run/secrets/broker",
|
||||
MESH_RECEIVES: "/var/lib/redis-module/grants/mesh.json",
|
||||
GRANTS: "/var/lib/redis-module/grants",
|
||||
MESH_PROVISION_REDIS: "127.0.0.1:6379",
|
||||
MESH_PROVISION_PASSWORD_FILE: "/run/secrets/default",
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
// --- baserow: a consumer that requires postgres-database AND redis-cache; server wired to both
|
||||
// from the grants the mesh writes, plus a runtime that serves baserow's tools. --------------------
|
||||
// --- baserow: a consumer that requires postgres-database, its server wired to the grant the mesh
|
||||
// writes, plus a runtime that serves baserow's tools. Its cache is its own — the image runs one when
|
||||
// no REDIS_HOST is given — because baserow keeps keys and channels under fixed names a shared
|
||||
// cache's per-consumer grant cannot confine (novox/hq 081). --------------------------------------
|
||||
const baserowManifest = JSON.stringify({
|
||||
module: "baserow",
|
||||
version: "1",
|
||||
requires: ["postgres-database", "redis-cache"],
|
||||
requires: ["postgres-database"],
|
||||
contributes: { "postgres-database": { name: "baserow" } },
|
||||
binds: { "postgres-database": "/var/lib/baserow/database.json", "redis-cache": "/var/lib/baserow/redis.json" },
|
||||
secrets: { "postgres-database": "/var/lib/baserow/database.secret", "redis-cache": "/var/lib/baserow/redis.secret" },
|
||||
binds: { "postgres-database": "/var/lib/baserow/database.json" },
|
||||
secrets: { "postgres-database": "/var/lib/baserow/database.secret" },
|
||||
"own-secrets": { "secret-key": "/var/lib/baserow/secret-key.secret", broker: "/var/lib/mesh/baserow/broker" },
|
||||
resources: [
|
||||
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/baserow", mode: "0700" },
|
||||
{ id: "state", type: "directory", path: "/var/lib/baserow", mode: "0700" },
|
||||
{ id: "data", type: "directory", path: "/services/baserow/data", mode: "0700", owner: "9999:9999" },
|
||||
// 0755, as the image ships it: the cache it runs for itself does so as another user, who
|
||||
// must be able to reach its own directory under this one (novox/hq 081).
|
||||
{ id: "data", type: "directory", path: "/services/baserow/data", mode: "0755", owner: "9999:9999" },
|
||||
{
|
||||
id: "server-env", type: "file", path: "/var/lib/baserow/server.env", mode: "0600",
|
||||
content:
|
||||
"DATABASE_HOST=${bound:postgres-database:at}\nDATABASE_PORT=${bound:postgres-database:port}\n" +
|
||||
"DATABASE_NAME=${bound:postgres-database:as}\nDATABASE_USER=${bound:postgres-database:as}\n" +
|
||||
"DATABASE_PASSWORD=${secret:postgres-database}\nREDIS_HOST=${bound:redis-cache:at}\n" +
|
||||
"REDIS_PORT=${bound:redis-cache:port}\nREDIS_PROTOCOL=redis\nREDIS_PASSWORD=${secret:redis-cache}\n" +
|
||||
"DATABASE_PASSWORD=${secret:postgres-database}\n" +
|
||||
"SECRET_KEY=${secret:secret-key}\nBASEROW_PUBLIC_URL=http://localhost\n",
|
||||
},
|
||||
{ id: "net", type: "network", name: "baserow" },
|
||||
{
|
||||
id: "server", type: "container", name: "baserow", image: pinned("baserow/baserow"), network: "baserow",
|
||||
"env-file": ["/var/lib/baserow/server.env"],
|
||||
// Declared as the catalogue declares it (novox/hq ADR 0086, issue 041).
|
||||
"secrets-in-environment": "baserow reads DATABASE_PASSWORD and SECRET_KEY with os.getenv and has no _FILE twin (settings/base.py); not convertible",
|
||||
volumes: ["/services/baserow/data:/baserow/data"],
|
||||
},
|
||||
{ id: "runtime-config", type: "file", path: "/var/lib/mesh/baserow/config.json", mode: "0600", content: "{}\n", merge: "json" },
|
||||
@@ -325,6 +280,8 @@ test("consumers on a joined node get their databases from the one foundation sto
|
||||
{
|
||||
id: "server", type: "container", name: "letta", image: pinned("letta/letta"), network: "letta",
|
||||
"env-file": ["/var/lib/letta/server.env"],
|
||||
// Declared as the catalogue declares it (novox/hq ADR 0086, issue 041).
|
||||
"secrets-in-environment": "the letta image is env-driven and its file-source support could not be verified; the mesh runtime can take its password from config.json (client.ts) \u2014 not yet converted",
|
||||
},
|
||||
{ id: "runtime-config", type: "file", path: "/var/lib/mesh/letta/config.json", mode: "0600", content: "{}\n", merge: "json" },
|
||||
{ id: "runtime-env", type: "file", path: "/var/lib/letta/runtime.env", mode: "0600", content: "MESH_LETTA_PASSWORD=${secret:server-password}\n" },
|
||||
@@ -341,6 +298,8 @@ test("consumers on a joined node get their databases from the one foundation sto
|
||||
MESH_LETTA_CONFIG_FILE: "/run/config/config.json",
|
||||
},
|
||||
"env-file": ["/var/lib/letta/runtime.env"],
|
||||
// Declared as the catalogue declares it (novox/hq ADR 0086, issue 041).
|
||||
"secrets-in-environment": "the letta image is env-driven and its file-source support could not be verified; the mesh runtime can take its password from config.json (client.ts) \u2014 not yet converted",
|
||||
"restart-on": ["runtime-config"],
|
||||
},
|
||||
],
|
||||
@@ -401,8 +360,7 @@ test("consumers on a joined node get their databases from the one foundation sto
|
||||
await mesh(`push anchor`, 600_000);
|
||||
await settled("anchor");
|
||||
|
||||
// The consumers ride laptop; redis is an ordinary co-located provider (it holds no seat).
|
||||
await addIssueAssign("redis", redisManifest);
|
||||
// The consumers ride laptop.
|
||||
await addIssueAssign("baserow", baserowManifest);
|
||||
await addIssueAssign("letta", lettaManifest);
|
||||
await mesh(`push ${NODE}`);
|
||||
@@ -429,7 +387,6 @@ test("consumers on a joined node get their databases from the one foundation sto
|
||||
// THE co-residence proof — every module's containers up and stable on the second node.
|
||||
// ================================================================================================
|
||||
const expected = [
|
||||
"redis", "mesh-redis",
|
||||
"baserow", "mesh-baserow",
|
||||
"letta", "mesh-letta",
|
||||
];
|
||||
@@ -505,7 +462,7 @@ test("consumers on a joined node get their databases from the one foundation sto
|
||||
// reached from laptop over the shared segment) — named for the node that runs it and the module.
|
||||
// ================================================================================================
|
||||
const users = await must("anchor", `docker exec mesh-broker lavinmqctl list_users 2>&1`);
|
||||
for (const acct of ["anchor-postgres", "laptop-redis", "laptop-baserow", "laptop-letta"]) {
|
||||
for (const acct of ["anchor-postgres", "laptop-baserow", "laptop-letta"]) {
|
||||
assert.match(users, new RegExp(acct), `the scoped account ${acct} is not on the broker:\n${users}`);
|
||||
}
|
||||
|
||||
@@ -537,14 +494,26 @@ test("consumers on a joined node get their databases from the one foundation sto
|
||||
assert.match(pg.out, /^1$/m, `${mod} could not connect to its granted postgres database as ${bound.as}:\n${pg.out}`);
|
||||
}
|
||||
|
||||
// baserow also got its redis-cache binding: the mesh wrote the binding and unsealed the secret,
|
||||
// and both arrived on the second node (a live redis AUTH is left to the redis single-module bed
|
||||
// and the open provider-seal-key work).
|
||||
const redisBound = await waitForBinding("/var/lib/baserow/redis.json");
|
||||
assert.equal(redisBound.provision, "redis-cache", `baserow's redis binding is the wrong provision: ${redisBound.provision}`);
|
||||
assert.ok(redisBound.as, `baserow's redis binding carries no login:\n${JSON.stringify(redisBound)}`);
|
||||
const redisSecret = (await must(NODE, `cat /var/lib/baserow/redis.secret`)).trim();
|
||||
assert.ok(redisSecret.length > 0, "baserow's redis secret was not delivered");
|
||||
// baserow's cache is its own: with no REDIS_HOST the image runs one inside the container, under a
|
||||
// password it makes itself, and the mesh grants nothing (novox/hq 081). Three things say so:
|
||||
// baserow said it chose its own; the cache answers on loopback with the challenge for that password
|
||||
// (PONG would be a cache anyone can use, and fails); and nothing was refused a login.
|
||||
const baserowLog = async () => (await on(NODE, `docker logs baserow 2>&1`)).out;
|
||||
let chose = "";
|
||||
let cache = { out: "", ok: false };
|
||||
const untilCache = Date.now() + 240_000;
|
||||
while (Date.now() < untilCache) {
|
||||
chose = await baserowLog();
|
||||
cache = await on(NODE, `docker exec baserow redis-cli -h 127.0.0.1 ping 2>&1`);
|
||||
if (/Using embedded baserow redis/.test(chose) && /NOAUTH/.test(cache.out)) break;
|
||||
await new Promise((r) => setTimeout(r, 5000));
|
||||
}
|
||||
assert.match(chose, /Using embedded baserow redis/,
|
||||
`baserow did not start its own cache:\n${chose.split("\n").filter((l) => /redis/i.test(l)).slice(-15).join("\n")}`);
|
||||
assert.match(cache.out, /NOAUTH/,
|
||||
`baserow's own cache does not answer with its password challenge inside the container:\n${cache.out}`);
|
||||
assert.doesNotMatch(chose, /WRONGPASS|NOPERM/,
|
||||
`baserow was refused by its cache:\n${chose.split("\n").filter((l) => /WRONGPASS|NOPERM/.test(l)).slice(-15).join("\n")}`);
|
||||
|
||||
// Helper: wait for the mesh to write a consumer's bound file with an `as`, and parse it.
|
||||
async function waitForBinding(path: string): Promise<{ as: string; provision: string }> {
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
/**
|
||||
* Whether genesis kept its bus through the broker step (novox/hq ADR 0333 §5, issue 590).
|
||||
*
|
||||
* Pure, so the judgement is tested without a lab (test/bus-kept.test.ts) and the bed only gathers what it
|
||||
* judges: the containers the anchor runs, the streams its bus holds, and what the mesh says of the node,
|
||||
* once before the broker step and once after it.
|
||||
*/
|
||||
|
||||
/** One running container, as `docker ps --format '{{.Names}}\t{{.Image}}\t{{.Ports}}'` says it. */
|
||||
export interface Running {
|
||||
name: string;
|
||||
image: string;
|
||||
ports: string;
|
||||
}
|
||||
|
||||
/** One stream on the bus, as the server's monitor says it (`/jsz?accounts=true&streams=true`). */
|
||||
export interface Stream {
|
||||
account: string;
|
||||
name: string;
|
||||
created: string;
|
||||
messages: number;
|
||||
lastSeq: number;
|
||||
}
|
||||
|
||||
/** What the anchor showed at one moment. */
|
||||
export interface Seen {
|
||||
running: Running[];
|
||||
streams: Stream[];
|
||||
}
|
||||
|
||||
/** The container the bundle raises the bus as, and the nats module keeps it as (ADR 0333 §1). */
|
||||
export const BUS = "mesh-broker-nats";
|
||||
|
||||
/** `docker ps --format '{{.Names}}\t{{.Image}}\t{{.Ports}}'`, read. */
|
||||
export function runningFrom(ps: string): Running[] {
|
||||
return ps.split("\n").map((l) => l.trim()).filter(Boolean).map((l) => {
|
||||
const [name = "", image = "", ports = ""] = l.split("\t");
|
||||
return { name, image, ports };
|
||||
});
|
||||
}
|
||||
|
||||
/** The server's `/jsz?accounts=true&streams=true`, read: every stream of every account. */
|
||||
export function streamsFrom(jsz: string): Stream[] {
|
||||
const parsed = JSON.parse(jsz) as {
|
||||
account_details?: Array<{
|
||||
name?: string;
|
||||
stream_detail?: Array<{ name?: string; created?: string; state?: { messages?: number; last_seq?: number } }>;
|
||||
}>;
|
||||
};
|
||||
const out: Stream[] = [];
|
||||
for (const account of parsed.account_details ?? []) {
|
||||
for (const s of account.stream_detail ?? []) {
|
||||
out.push({
|
||||
account: account.name ?? "", name: s.name ?? "", created: s.created ?? "",
|
||||
messages: s.state?.messages ?? 0, lastSeq: s.state?.last_seq ?? 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* A bus container: one that serves the bus's port inside it, or carries a bus's name. Both, because a
|
||||
* second bus raised by a module under another name is exactly what this looks for, and so is one raised on
|
||||
* another port under the bus's name.
|
||||
*/
|
||||
export function busContainers(running: Running[]): Running[] {
|
||||
return running.filter((r) => /->4222\/tcp/.test(r.ports) || r.name.startsWith("mesh-broker"));
|
||||
}
|
||||
|
||||
/** The streams written before the step: the ones holding anything. */
|
||||
export function written(seen: Seen): Stream[] {
|
||||
return seen.streams.filter((s) => s.messages > 0 || s.lastSeq > 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Every way the bus was not kept, in sentences; empty when it was. `node` is the mesh's line for the first
|
||||
* node from `node list`; `said` is what the installer printed on the run that crossed the step.
|
||||
*/
|
||||
export function busNotKept(before: Seen, after: Seen, node: string, said: string): string[] {
|
||||
const wrong: string[] = [];
|
||||
const buses = busContainers(after.running);
|
||||
if (buses.length !== 1) {
|
||||
wrong.push(`${buses.length} bus containers run after the broker step, not one: ` +
|
||||
(buses.map((b) => `${b.name} (${b.image}, ${b.ports || "nothing published"})`).join("; ") || "none"));
|
||||
}
|
||||
const bus = after.running.find((r) => r.name === BUS);
|
||||
if (!bus) {
|
||||
wrong.push(`${BUS} does not run after the broker step`);
|
||||
}
|
||||
const was = before.running.find((r) => r.name === BUS);
|
||||
if (bus && was && bus.image === was.image) {
|
||||
wrong.push(`${BUS} still runs ${bus.image}, the image the bundle raised it from: the module's image never ` +
|
||||
`took its place (ADR 0333 §3)`);
|
||||
}
|
||||
if (!/restarting\s+mesh-broker-nats/.test(said)) {
|
||||
wrong.push("the installer did not say the bus restarts onto the module's image (ADR 0333 §3)");
|
||||
}
|
||||
for (const s of written(before)) {
|
||||
const now = after.streams.find((a) => a.account === s.account && a.name === s.name);
|
||||
if (!now) {
|
||||
wrong.push(`the stream ${s.name} (${s.account}), holding ${s.messages} message(s) before the step, is gone`);
|
||||
} else if (now.created !== s.created || now.lastSeq < s.lastSeq) {
|
||||
wrong.push(`the stream ${s.name} (${s.account}) was made again: created ${s.created}, last ${s.lastSeq} ` +
|
||||
`before the step; created ${now.created}, last ${now.lastSeq} after it`);
|
||||
}
|
||||
}
|
||||
if (!/^\S+\s+here\b/.test(node.trim())) {
|
||||
wrong.push(`the mesh has not heard from the first node since the step: ${node.trim() || "it has no record of it"}`);
|
||||
}
|
||||
return wrong;
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
/**
|
||||
* A bed's forge: a checkout's HEAD carried into a fresh bare repository, which a git daemon then serves.
|
||||
*
|
||||
* **A shallow checkout is carried as it is** (novox/hq issue 689). The lab module checks every repository
|
||||
* out shallow, and a fresh bare repository refuses a push whose history ends in a cut ("shallow update not
|
||||
* allowed"), so the genesis bed raised its anchor and forge and then skipped. The bare repository accepts
|
||||
* the cut (`receive.shallowUpdate`) and serves the history the checkout holds: the bed serves the commit
|
||||
* under test, and nothing it raises reads further back than the run's checkout does. Unshallowing first is
|
||||
* not an option in a lab run: the forge credential is gone once the run's clones are done, so a private
|
||||
* repository could not be fetched again.
|
||||
*/
|
||||
|
||||
import { execFileSync } from "node:child_process";
|
||||
|
||||
/**
|
||||
* Initialise `bare` and push `checkout`'s HEAD into it under each of `branches`. Returns the commit served.
|
||||
*/
|
||||
export function carry(checkout: string, bare: string, branches: string[] = ["main"]): string {
|
||||
execFileSync("git", ["init", "-q", "--bare", bare]);
|
||||
// No collection after the push: one run in the background rewrites the objects while they are carried.
|
||||
execFileSync("git", ["-C", bare, "config", "receive.autogc", "false"]);
|
||||
// The lab's checkouts are shallow, and a bare repository refuses a shallow push without this.
|
||||
execFileSync("git", ["-C", bare, "config", "receive.shallowUpdate", "true"]);
|
||||
const [first, ...rest] = branches;
|
||||
execFileSync("git", ["-C", checkout, "push", "-q", "--force", bare, `HEAD:refs/heads/${first}`], { stdio: "pipe" });
|
||||
const head = execFileSync("git", ["-C", checkout, "rev-parse", "HEAD"], { encoding: "utf8" }).trim();
|
||||
// The other branches are set in the bare repository itself: a shallow push of several refs at once has
|
||||
// receive-pack fail every ref after the first ("remote failed to report status").
|
||||
for (const branch of rest) execFileSync("git", ["-C", bare, "update-ref", `refs/heads/${branch}`, head]);
|
||||
return head;
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
/**
|
||||
* GENESIS KEEPS ITS BUS through the broker step (novox/hq ADR 0333 §5, issue 590).
|
||||
*
|
||||
* The installer raises the bus from its bundle and, at its broker step (step 18 of 24), adopts it as the
|
||||
* catalogue's `nats` module. Until ADR 0333 the bundle raised the bus as `mesh-broker` on docker volumes,
|
||||
* and the module keeps it as `mesh-broker-nats` on directories of the machine: adopting it would have raised
|
||||
* a second bus beside the first, so the installer refused there and genesis stopped. ADR 0333 has the bundle
|
||||
* raise the bus where the module keeps it, so the adoption replaces the server's image once and keeps
|
||||
* everything else.
|
||||
*
|
||||
* The bed runs the installer from bare twice. The first run is given a catalogue without the `nats` module,
|
||||
* so it stops at the broker step by name, before anything is registered — the moment just before the step,
|
||||
* with everything genesis wrote on the bus by then. The bed reads the bus there: its containers, and every
|
||||
* stream holding anything. The second run, given the whole catalogue, finds the steps before done and goes
|
||||
* on through the broker step. The bed reads the bus again, and passes only when (ADR 0333 §5):
|
||||
*
|
||||
* - exactly one bus container runs, and it is mesh-broker-nats, on the module's image now;
|
||||
* - every stream that held something before the step is still there, the same stream, as long or longer;
|
||||
* - the mesh has heard from the first node since.
|
||||
*
|
||||
* The judgement is test/integration/bus-kept.ts, held without a lab by test/bus-kept.test.ts.
|
||||
*
|
||||
* **A bed that could not reach the step says so and does not fail**: a first run that stopped earlier than
|
||||
* the broker step — a pull the internet refused, a build that broke — proves nothing about the step, and the
|
||||
* test is skipped with the reason. A second run that stops AT the broker step is the fault, and fails. One
|
||||
* that stops after it has crossed it: what it stopped on is reported, and the bus is judged.
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=<mesh-host>/mesh-host MESH_LAB_BOOTSTRAP_BINARY=<mesh-host>/mesh-bootstrap
|
||||
* MESH_LAB_BUNDLE=<mesh-host>/examples/foundation-first-node.lock
|
||||
* MESH_LAB_CATALOG=<mesh-catalog>/modules
|
||||
* MESH_LAB_REPOS=<a directory holding mesh-controller, mesh-tools, mesh-tools-go, mesh-sdk and mesh-catalog checkouts>
|
||||
* — served by the forge machine at each checkout's HEAD; the catalogue's is the one built from
|
||||
* MESH_LAB_KEEP=1 leave it standing
|
||||
*/
|
||||
import { test, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { cpSync, existsSync, mkdtempSync, rmSync } from "node:fs";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { loadScenario } from "../../src/declaration/parse.ts";
|
||||
import { raise } from "../../src/lifecycle/raise.ts";
|
||||
import { destroy, exec, push } from "../../src/lifecycle/operate.ts";
|
||||
import { bootstrapBinaryPath, hostBinaryPath } from "../../src/lifecycle/place.ts";
|
||||
import { labIsUsable, destroyAll, foundationBundle } from "./harness.ts";
|
||||
import { genesis, type GenesisResult } from "./genesis.ts";
|
||||
import { carry } from "./forge.ts";
|
||||
import { BUS, busContainers, busNotKept, runningFrom, streamsFrom, written, type Seen } from "./bus-kept.ts";
|
||||
|
||||
const SCENARIO = "genesis-keeps-its-bus";
|
||||
const ANCHOR = "anchor";
|
||||
const FORGE = "forge";
|
||||
const FORGE_ADDRESS = "192.0.2.30";
|
||||
const REGISTRY = "192.0.2.10:5000";
|
||||
/** The repositories the installer clones, served by the forge at each checkout's HEAD. */
|
||||
// mesh-tools-go is found by the installer beside mesh-tools (hq issue 616): genesis builds the Go toolchain too.
|
||||
const REPOS = ["mesh-controller", "mesh-tools", "mesh-tools-go", "mesh-sdk", "mesh-catalog"] as const;
|
||||
/** The step this bed is about, as the installer names it. */
|
||||
const BROKER = "broker";
|
||||
/** The installer's steps before it, in its order: a first run stopped at one of these never reached it. */
|
||||
const BEFORE_BROKER = ["preflight", "load", "build", "bundle", "apply", "verify", "enrol", "operator", "registry",
|
||||
"publish", "control-plane", "retire", "builder", "packages", "sdk", "base", "store"];
|
||||
|
||||
const capability = await labIsUsable();
|
||||
const binary = hostBinaryPath();
|
||||
const installer = bootstrapBinaryPath();
|
||||
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
||||
const catalogDir = process.env["MESH_LAB_CATALOG"] ?? "";
|
||||
const repos = process.env["MESH_LAB_REPOS"] ?? "";
|
||||
const KEEP = !!process.env["MESH_LAB_KEEP"];
|
||||
const FIXED_ID = process.env["MESH_LAB_INSTANCE_ID"] ?? (KEEP ? "genesis-keeps-its-bus-live" : undefined);
|
||||
|
||||
const missingRepo = REPOS.find((r) => !repos || !existsSync(resolve(repos, r, ".git")));
|
||||
const skip =
|
||||
!capability.usable ? capability.why :
|
||||
!binary ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" :
|
||||
!installer ? "MESH_LAB_BOOTSTRAP_BINARY is not set to a built mesh-bootstrap" :
|
||||
!bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to the genesis bundle" :
|
||||
!catalogDir || !existsSync(join(catalogDir, "nats", "module.json")) ? "MESH_LAB_CATALOG is not set to mesh-catalog/modules" :
|
||||
missingRepo ? `MESH_LAB_REPOS holds no ${missingRepo} checkout` :
|
||||
false;
|
||||
|
||||
let instanceId = "";
|
||||
/** Why the bed never reached the step, or "" when it did. */
|
||||
let unreached = "";
|
||||
let first: GenesisResult | undefined;
|
||||
let second: GenesisResult | undefined;
|
||||
let seenBefore: Seen | undefined;
|
||||
let seenAfter: Seen | undefined;
|
||||
let nodeLine = "";
|
||||
const heads: Record<string, string> = {};
|
||||
const report: string[] = [];
|
||||
|
||||
async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
||||
const { stdout } = await exec(instanceId, machine, ["sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`], timeoutMs);
|
||||
const marker = stdout.lastIndexOf("__exit=");
|
||||
if (marker < 0) return { out: stdout, ok: false };
|
||||
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
||||
}
|
||||
async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
|
||||
const { out, ok } = await on(machine, command, timeoutMs);
|
||||
if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Each repository at its checkout's HEAD, served from the forge machine over git's protocol as `main`. */
|
||||
async function raiseForge(): Promise<void> {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mesh-lab-590-forge-"));
|
||||
try {
|
||||
for (const repo of REPOS) {
|
||||
heads[repo] = carry(resolve(repos, repo), join(dir, `${repo}.git`));
|
||||
}
|
||||
const tar = join(tmpdir(), `mesh-lab-590-forge-${process.pid}.tar`);
|
||||
execFileSync("tar", ["-cf", tar, "-C", dir, "."]);
|
||||
await push(instanceId, FORGE, tar, "/tmp/forge.tar");
|
||||
rmSync(tar, { force: true });
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
// Owned by the daemon's user, or git refuses to serve it ("dubious ownership") and says nothing.
|
||||
await must(FORGE, `mkdir -p /srv/git && tar --no-same-owner -xf /tmp/forge.tar -C /srv/git && chown -R root:root /srv/git && ` +
|
||||
`git daemon --base-path=/srv/git --export-all --reuseaddr --detach --listen=${FORGE_ADDRESS} --pid-file=/run/git-daemon.pid`);
|
||||
for (let i = 0; ; i++) {
|
||||
if ((await on(ANCHOR, `git ls-remote git://${FORGE_ADDRESS}/mesh-catalog.git main`)).ok) break;
|
||||
if (i >= 30) throw new Error(`the anchor cannot reach the forge at git://${FORGE_ADDRESS}/`);
|
||||
await new Promise((r) => setTimeout(r, 2_000));
|
||||
}
|
||||
}
|
||||
|
||||
/** What the anchor shows of its bus: the containers it runs, and every stream the bus holds. */
|
||||
async function look(): Promise<Seen> {
|
||||
const ps = await must(ANCHOR, `docker ps --format '{{.Names}}\t{{.Image}}\t{{.Ports}}'`);
|
||||
// The monitor, published on the anchor's loopback by the bundle and by the module alike.
|
||||
let jsz = "";
|
||||
for (let i = 0; ; i++) {
|
||||
const asked = await on(ANCHOR, `curl -sf --max-time 10 'http://127.0.0.1:8222/jsz?accounts=true&streams=true'`);
|
||||
if (asked.ok) { jsz = asked.out; break; }
|
||||
if (i >= 30) throw new Error(`the bus's monitor does not answer on the anchor: ${asked.out}`);
|
||||
await new Promise((r) => setTimeout(r, 2_000));
|
||||
}
|
||||
return { running: runningFrom(ps), streams: streamsFrom(jsz) };
|
||||
}
|
||||
|
||||
function stoppedAt(r: GenesisResult): string {
|
||||
return r.step.replace(/^step \d+ of \d+, /, "");
|
||||
}
|
||||
|
||||
before(async () => {
|
||||
if (skip) return;
|
||||
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
||||
onProgress: (m) => console.log(`raise: ${m}`),
|
||||
...(FIXED_ID ? { instanceId: FIXED_ID } : {}),
|
||||
});
|
||||
instanceId = raised.instanceId;
|
||||
console.log(`INSTANCE ${instanceId}${KEEP ? " (KEEP — will be left standing)" : ""}`);
|
||||
|
||||
try {
|
||||
await raiseForge();
|
||||
report.push(` forge ${REPOS.map((r) => `${r} ${heads[r]?.slice(0, 8)}`).join(", ")}`);
|
||||
} catch (err) {
|
||||
unreached = `the forge was not raised, so the installer never ran: ${(err as Error).message}`;
|
||||
return;
|
||||
}
|
||||
|
||||
const forge = (repo: string) => `git://${FORGE_ADDRESS}/${repo}.git`;
|
||||
const run = (catalogue: string) => genesis({
|
||||
instanceId,
|
||||
node: ANCHOR,
|
||||
installer: installer as string,
|
||||
catalogDir: catalogue,
|
||||
catalogueModules: ["distribution", "build-agent", "postgres"],
|
||||
bundleTemplate: foundationBundle(bundle, []),
|
||||
registry: REGISTRY,
|
||||
source: forge("mesh-controller"),
|
||||
sourceRef: heads["mesh-controller"] as string,
|
||||
toolsSource: forge("mesh-tools"),
|
||||
sdkSource: forge("mesh-sdk"),
|
||||
catalogSource: forge("mesh-catalog"),
|
||||
// A refusal is the answer, and a re-run is the second half of this bed, not a retry.
|
||||
attempts: 1,
|
||||
verify: false,
|
||||
log: (m) => console.log(m),
|
||||
});
|
||||
|
||||
// 1. From bare to the broker step, and no further: the catalogue the installer reads has no nats module.
|
||||
const withoutTheBus = mkdtempSync(join(tmpdir(), "mesh-lab-590-catalogue-"));
|
||||
try {
|
||||
const nats = join(catalogDir, "nats");
|
||||
cpSync(catalogDir, withoutTheBus, { recursive: true, filter: (src) => src !== nats && !src.startsWith(nats + "/") });
|
||||
first = await run(withoutTheBus);
|
||||
} catch (err) {
|
||||
unreached = `the installer could not be run: ${(err as Error).message}`;
|
||||
return;
|
||||
} finally {
|
||||
rmSync(withoutTheBus, { recursive: true, force: true });
|
||||
}
|
||||
report.push(...first.report);
|
||||
if (first.ok || stoppedAt(first) !== BROKER) {
|
||||
unreached = first.ok
|
||||
? "the first run finished without the nats module, so the broker step never stopped it"
|
||||
: `the first run stopped at ${first.step || "no step it named"}, before the broker step:\n${first.why}`;
|
||||
return;
|
||||
}
|
||||
if (!/nats module's manifest could not be read/.test(first.said)) {
|
||||
unreached = `the first run stopped at the broker step for another reason than the missing module:\n${first.why}`;
|
||||
return;
|
||||
}
|
||||
seenBefore = await look();
|
||||
report.push(` before the step ${busContainers(seenBefore.running).map((b) => `${b.name} on ${b.image}`).join("; ")}`);
|
||||
report.push(` ${written(seenBefore).length} stream(s) holding something: ` +
|
||||
written(seenBefore).map((s) => `${s.name} (last ${s.lastSeq})`).join(", "));
|
||||
if (written(seenBefore).length === 0) {
|
||||
unreached = "nothing was written on the bus before the broker step, so whether its streams survive it says nothing";
|
||||
return;
|
||||
}
|
||||
|
||||
// 2. The whole catalogue: the steps before are done already, and the broker step adopts the bus.
|
||||
try {
|
||||
second = await run(catalogDir);
|
||||
} catch (err) {
|
||||
unreached = `the installer could not be run again: ${(err as Error).message}`;
|
||||
return;
|
||||
}
|
||||
report.push(...second.report);
|
||||
if (!second.ok && BEFORE_BROKER.includes(stoppedAt(second))) {
|
||||
unreached = `the second run stopped at ${second.step}, before the broker step it was run again for:\n${second.why}`;
|
||||
return;
|
||||
}
|
||||
seenAfter = await look();
|
||||
report.push(` after the step ${busContainers(seenAfter.running).map((b) => `${b.name} on ${b.image}`).join("; ") || "no bus"}`);
|
||||
const nodes = await on(ANCHOR, `docker exec mesh-controller /mesh-controller node list`, 60_000);
|
||||
nodeLine = nodes.out.split("\n").map((l) => l.trim()).find((l) => l.startsWith(`${ANCHOR} `)) ?? "";
|
||||
report.push(` node list ${nodeLine || nodes.out.trim()}`);
|
||||
}, { timeout: 7_200_000 });
|
||||
|
||||
after(async () => {
|
||||
if (KEEP) {
|
||||
console.log(`\nLEFT STANDING: ${instanceId} — not destroyed (MESH_LAB_KEEP).`);
|
||||
return;
|
||||
}
|
||||
if (instanceId) await destroy(instanceId);
|
||||
await destroyAll(`${SCENARIO}-`);
|
||||
}, { timeout: 900_000 });
|
||||
|
||||
test("genesis adopts its bus as the nats module in place: one bus, its streams kept, the node connected", {
|
||||
skip, timeout: 7_200_000,
|
||||
}, async (t) => {
|
||||
console.log(report.join("\n"));
|
||||
if (unreached) {
|
||||
t.skip(`ADR 0333 §5 NOT REACHED: ${unreached}`);
|
||||
return;
|
||||
}
|
||||
const ran = second as GenesisResult;
|
||||
// Stopped AT the broker step: the adoption itself refused or failed — the fault, not a missed bed.
|
||||
assert.ok(ran.ok || stoppedAt(ran) !== BROKER,
|
||||
`genesis stopped at its broker step and did not adopt the bus:\n${ran.why}`);
|
||||
if (!ran.ok) {
|
||||
console.log(`\nthe second run crossed the broker step and stopped later, at ${ran.step} — not this bed's ` +
|
||||
`question:\n${ran.why}`);
|
||||
}
|
||||
const wrong = busNotKept(seenBefore as Seen, seenAfter as Seen, nodeLine, ran.said);
|
||||
assert.deepEqual(wrong, [], `genesis did not keep its bus through the broker step:\n - ${wrong.join("\n - ")}`);
|
||||
console.log(`\nVERDICT: one bus, ${BUS}, on the module's image; ${written(seenBefore as Seen).length} stream(s) ` +
|
||||
`kept; ${ANCHOR} connected.`);
|
||||
});
|
||||
@@ -93,6 +93,7 @@ before(async () => {
|
||||
step: "getting the machine ready to be bootstrapped — the installer never ran",
|
||||
why: (err as Error).message,
|
||||
report: [],
|
||||
said: "",
|
||||
};
|
||||
}
|
||||
console.log(result.report.join("\n"));
|
||||
|
||||
@@ -26,6 +26,8 @@ export interface GenesisResult {
|
||||
step: string;
|
||||
why: string;
|
||||
report: string[];
|
||||
/** Everything the installer printed on its last attempt — what a bed asserts a refusal names. */
|
||||
said: string;
|
||||
}
|
||||
|
||||
export interface GenesisOptions {
|
||||
@@ -80,6 +82,23 @@ export interface GenesisOptions {
|
||||
hostBinary?: string;
|
||||
/** What of the catalogue to build. A branch under test is the usual reason this is not main. */
|
||||
catalogRef?: string;
|
||||
/**
|
||||
* Raise the machine adopted (novox/hq ADR 0100): what it runs and its firewall are kept. Without
|
||||
* it the installer raises a converged node, and refuses a machine in use.
|
||||
*/
|
||||
adopted?: boolean;
|
||||
/** Further installer flags, as the operator would type them — `--registry-port 5100`, `--dry-run`. */
|
||||
flags?: string[];
|
||||
/**
|
||||
* How many times to run the installer. Three by default, for a pull the internet rate-limited; a
|
||||
* bed that expects a REFUSAL runs it once, because a refusal is the answer, not a flake.
|
||||
*/
|
||||
attempts?: number;
|
||||
/**
|
||||
* Whether to ask the machine if it became a working mesh of one afterwards. Off for a run that is
|
||||
* not meant to raise one — a dry run, or a refusal the bed expects.
|
||||
*/
|
||||
verify?: boolean;
|
||||
log?: (m: string) => void;
|
||||
}
|
||||
|
||||
@@ -93,14 +112,15 @@ export async function genesis(o: GenesisOptions): Promise<GenesisResult> {
|
||||
const registry = o.registry ?? "127.0.0.1:5000";
|
||||
// The registry's and the builder's. Not the control plane's: its manifest lives in its own
|
||||
// repository and genesis takes it from the build it runs (novox/hq ADR 0069, 04-ISSUES/072).
|
||||
const modules = o.catalogueModules ?? ["distribution", "builder"];
|
||||
const modules = o.catalogueModules ?? ["distribution", "build-agent"];
|
||||
const catalogueOnMachine = o.catalogueOnMachine ?? "/opt/mesh-catalog";
|
||||
const log = o.log ?? (() => {});
|
||||
|
||||
const report: string[] = [`================ GENESIS: ${node} becomes a mesh of one ================`];
|
||||
let said = "";
|
||||
const stop = (step: string, why: string): GenesisResult => {
|
||||
report.push(`\nSTOPPED at ${step || "(no step named)"}: ${why}`);
|
||||
return { ok: false, step, why, report };
|
||||
return { ok: false, step, why, report, said };
|
||||
};
|
||||
|
||||
const on = async (command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> => {
|
||||
@@ -136,7 +156,9 @@ export async function genesis(o: GenesisOptions): Promise<GenesisResult> {
|
||||
// the lab stands in for that by copying the whole tree once.
|
||||
const bundleTar = join(tmpdir(), `mesh-lab-catalogue-${process.pid}-${node}.tar`);
|
||||
execFileSync("tar", ["-cf", bundleTar, "-C", o.catalogDir, "."]);
|
||||
await must(`mkdir -p ${catalogueOnMachine}/modules`);
|
||||
// Cleared first: a bed that runs genesis more than once (a refusal, then the raise) finds the last
|
||||
// run's staging files, and the machine refuses to open them for the push.
|
||||
await must(`rm -f /tmp/catalogue.tar /tmp/foundation-template.lock && mkdir -p ${catalogueOnMachine}/modules`);
|
||||
await push(o.instanceId, node, bundleTar, "/tmp/catalogue.tar");
|
||||
await must(`tar -xf /tmp/catalogue.tar -C ${catalogueOnMachine}/modules`);
|
||||
// The three genesis itself needs must be present, or the pivot cannot even begin — checked here
|
||||
@@ -184,6 +206,8 @@ export async function genesis(o: GenesisOptions): Promise<GenesisResult> {
|
||||
`--private-network wireguard`,
|
||||
`--packet-filter nftables`,
|
||||
`--host ${HOST_PATH}`,
|
||||
...(o.adopted ? [`--adopted`] : []),
|
||||
...(o.flags ?? []),
|
||||
// A service when the packaging was installed above (survives a reboot); otherwise the
|
||||
// background process, which does not — the installer refuses to invent a unit either way.
|
||||
...(o.hostService ? [] as string[] : [`--host-in-background`]),
|
||||
@@ -193,20 +217,24 @@ export async function genesis(o: GenesisOptions): Promise<GenesisResult> {
|
||||
// but because the installer is idempotent by design and says so, and because the one thing that
|
||||
// fails for a reason which goes away by itself is a pull: the store, broker and registry come
|
||||
// from the internet, and a rate-limited anonymous pull is not this mesh's fault.
|
||||
let said = "";
|
||||
let step = "";
|
||||
for (let attempt = 1; attempt <= 3; attempt++) {
|
||||
const attempts = o.attempts ?? 3;
|
||||
for (let attempt = 1; attempt <= attempts; attempt++) {
|
||||
const ran = await on(command, 2_400_000);
|
||||
said = ran.out;
|
||||
log(`\n---- mesh-bootstrap on ${node} (attempt ${attempt}) ----\n${said}`);
|
||||
if (ran.ok) { step = ""; break; }
|
||||
step = stepIn(said);
|
||||
if (attempt < 3) {
|
||||
step = stepIn(said) || "an unnamed step";
|
||||
if (attempt < attempts) {
|
||||
log(`genesis attempt ${attempt} stopped at ${step || "an unnamed step"}; re-running in 30s`);
|
||||
await new Promise((r) => setTimeout(r, 30_000));
|
||||
}
|
||||
}
|
||||
if (step) return stop(step, said.split("\n").filter(Boolean).slice(-6).join("\n"));
|
||||
if (o.verify === false) {
|
||||
report.push(`\nThe installer finished; not asked whether it raised a mesh (verify: false).`);
|
||||
return { ok: true, step: "", why: "", report, said };
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------------------
|
||||
// Is it a WORKING MESH OF ONE? Asked of the machine, never inferred from the installer exiting
|
||||
@@ -286,5 +314,5 @@ export async function genesis(o: GenesisOptions): Promise<GenesisResult> {
|
||||
}
|
||||
|
||||
report.push(`\nVERDICT: ${node} is a working mesh of one, bootstrapped through the installer.`);
|
||||
return { ok: true, step: "", why: "", report };
|
||||
return { ok: true, step: "", why: "", report, said };
|
||||
}
|
||||
|
||||
@@ -27,10 +27,10 @@ import { mustBeHandedOver, pinnedInto, referenceFor, repositoryOf, type HeldImag
|
||||
* pins `192.0.2.250:5000/…` because that is where the registry the lab used to raise served from.
|
||||
* That registry is gone, so those three references name nothing.
|
||||
*
|
||||
* Two of them are ordinary third-party images and belong to the internet. Rather than invent
|
||||
* digests here, they are the ones the mesh's own modules already pin — mesh-catalog's `postgres`
|
||||
* and `lavinmq` — so the foundation's store and broker are literally the images the mesh runs. The
|
||||
* third, mesh-controller, exists in no registry at all and becomes the ID the machine holds it under.
|
||||
* Two of them are ordinary third-party images and belong to the internet. The store's is the one
|
||||
* mesh-catalog's `postgres` module already pins, so the foundation's store is literally the image the
|
||||
* mesh runs; the bus's is below. The third, mesh-controller, exists in no registry at all and becomes
|
||||
* the ID the machine holds it under.
|
||||
*
|
||||
* **The bundle itself should be fixed in mesh-host**, and this substitution deleted with it. It is
|
||||
* here because the file lives in another repository and because a fixture that lies about where an
|
||||
@@ -38,8 +38,14 @@ import { mustBeHandedOver, pinnedInto, referenceFor, repositoryOf, type HeldImag
|
||||
*/
|
||||
const UPSTREAM_STORE =
|
||||
"postgres@sha256:7456ef82e5f5bc43d997f4781bbd7c0d6389bff397564649a356e206ba473aee";
|
||||
const UPSTREAM_BROKER =
|
||||
"cloudamqp/lavinmq@sha256:3eb54c12916d700a978c2ea86e6362cd4974b0e3189508718006d4e6d341246b";
|
||||
/**
|
||||
* And the bus, for `foundation-first-node.lock` — the one genesis bundle, which raises the bus the
|
||||
* mesh runs on since it stopped speaking AMQP (novox/hq ADR 0131, ADR 0331). The digest is the bundle's own: what the registry served was a
|
||||
* copy of the upstream image, so the same digest resolves on Docker Hub, and this is a prefix being
|
||||
* removed rather than a reference being replaced.
|
||||
*/
|
||||
const UPSTREAM_BUS =
|
||||
"nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927";
|
||||
|
||||
/**
|
||||
* The foundation bundle as a machine should receive it.
|
||||
@@ -51,8 +57,7 @@ const UPSTREAM_BROKER =
|
||||
export function foundationBundle(path: string, held: HeldImage[]): string {
|
||||
let text = readFileSync(path, "utf8");
|
||||
text = text.replaceAll(/[A-Za-z0-9_.:-]+\/postgres@sha256:[0-9a-f]{64}/g, UPSTREAM_STORE);
|
||||
text = text.replaceAll(
|
||||
/[A-Za-z0-9_.:-]+\/cloudamqp\/lavinmq@sha256:[0-9a-f]{64}/g, UPSTREAM_BROKER);
|
||||
text = text.replaceAll(/[A-Za-z0-9_.:-]+:[0-9]+\/nats@sha256:[0-9a-f]{64}/g, UPSTREAM_BUS);
|
||||
return pinnedInto(text, held);
|
||||
}
|
||||
|
||||
@@ -367,11 +372,16 @@ function shellQuote(s: string): string {
|
||||
*/
|
||||
export async function deriveTheFilterOn(o: {
|
||||
machine: string; node: string; hubPort: number;
|
||||
/** The images the machines hold. Given, the manifest is the lab's form of the catalogue's
|
||||
* (catalogueModule): its build section gone, since the lab builds nothing. The filter's verbs are a
|
||||
* tools bundle the node's runtime launches (novox/hq ADR 0170, 0193), so no container of the
|
||||
* module names an image and none need be stocked; what a bed needs of it is the ruleset it loads. */
|
||||
held?: HeldImage[];
|
||||
must: (machine: string, command: string, timeoutMs?: number) => Promise<string>;
|
||||
mesh: (command: string, timeoutMs?: number) => Promise<string>;
|
||||
on: (machine: string, command: string, timeoutMs?: number) => Promise<{ out: string; ok: boolean }>;
|
||||
}): Promise<string> {
|
||||
const manifest = readFileSync(catalogueManifest(FILTER_MODULE), "utf8");
|
||||
const manifest = o.held ? catalogueModule(FILTER_MODULE, o.held) : readFileSync(catalogueManifest(FILTER_MODULE), "utf8");
|
||||
await o.must(o.machine,
|
||||
`printf %s ${shellQuote(manifest)} > /tmp/${FILTER_MODULE}.json && docker cp /tmp/${FILTER_MODULE}.json mesh-controller:/${FILTER_MODULE}.json`);
|
||||
await o.mesh(`module add /${FILTER_MODULE}.json`);
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
/**
|
||||
* **A machine joins through the tunnel, and the bus is never public** (novox/hq ADR 0169).
|
||||
*
|
||||
* The anchor raises the foundation from the bundle, joins over its own loopback — it runs the bus,
|
||||
* so it needs no tunnel to reach it — and is placed as the hub. The joiner then does what a new
|
||||
* machine does: makes its tunnel key and prints the public half, is issued a token for that key,
|
||||
* and enrols with the token alone. The bus is closed to the joiner's own address on the anchor
|
||||
* before any of that, so the enrolment can arrive only over the tunnel the token gave it.
|
||||
*
|
||||
* It asserts the ADR's last row: *a new machine joins from outside the hub's network with the bus
|
||||
* closed to it.* The rows before it are the controller's own tests.
|
||||
*
|
||||
* It needs a host binary, the foundation bundle and the control plane's image:
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host
|
||||
* MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||
* (mesh-controller:development, from mesh-controller's `make image`)
|
||||
*/
|
||||
import { test, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { existsSync } from "node:fs";
|
||||
import { loadScenario } from "../../src/declaration/parse.ts";
|
||||
import { raise } from "../../src/lifecycle/raise.ts";
|
||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||||
import { labIsUsable, destroyAll, foundationBundle } from "./harness.ts";
|
||||
|
||||
const capability = await labIsUsable();
|
||||
const binary = hostBinaryPath();
|
||||
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
||||
|
||||
const skip = !capability.usable
|
||||
? `lab not usable: ${capability.why}`
|
||||
: !binary || !existsSync(binary)
|
||||
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
||||
: !bundle || !existsSync(bundle)
|
||||
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
||||
: false;
|
||||
|
||||
const SCENARIO = "joins-through-the-tunnel";
|
||||
/** The hub's tunnel port, and the bus's port as the bundle publishes it on the anchor. */
|
||||
const HUB_PORT = 51820;
|
||||
const BUS_PORT = 5671;
|
||||
let instanceId = "";
|
||||
|
||||
function quote(s: string): string {
|
||||
return `'${s.replaceAll("'", `'\\''`)}'`;
|
||||
}
|
||||
|
||||
async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
||||
const { stdout } = await exec(instanceId, machine, [
|
||||
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
|
||||
], timeoutMs);
|
||||
const marker = stdout.lastIndexOf("__exit=");
|
||||
if (marker < 0) return { out: stdout, ok: false };
|
||||
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
||||
}
|
||||
|
||||
async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
|
||||
const { out, ok } = await on(machine, command, timeoutMs);
|
||||
if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
|
||||
return out;
|
||||
}
|
||||
|
||||
async function mesh(command: string, timeoutMs?: number): Promise<string> {
|
||||
return must("anchor", `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
|
||||
}
|
||||
|
||||
function tokenFrom(said: string): string {
|
||||
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
|
||||
assert.ok(found, `no token in:\n${said}`);
|
||||
return found;
|
||||
}
|
||||
|
||||
async function until(what: string, within: number, check: () => Promise<boolean>, why: () => Promise<string>): Promise<void> {
|
||||
const end = Date.now() + within;
|
||||
while (Date.now() < end) {
|
||||
if (await check()) return;
|
||||
await new Promise((r) => setTimeout(r, 3000));
|
||||
}
|
||||
assert.fail(`${what} did not happen within ${Math.round(within / 1000)}s:\n${await why()}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Put the mesh's composed user list where the anchor's bus reads it, and make it re-read.
|
||||
*
|
||||
* **Genesis's step, done by hand because this bed raises genesis by hand** (novox/hq 04-ISSUES/146):
|
||||
* the bus here is the bundle's, so an account the mesh composes reaches it only if whoever raised it
|
||||
* places it — the enrolment's when a token is issued, the node's own once it enrols.
|
||||
*/
|
||||
async function placeTheBusUsers(): Promise<void> {
|
||||
await must("anchor",
|
||||
`docker exec mesh-controller /mesh-controller broker accounts > /var/lib/mesh-bus-conf/accounts.conf && ` +
|
||||
`docker kill -s HUP mesh-broker >/dev/null`);
|
||||
}
|
||||
|
||||
before(async () => {
|
||||
if (skip) return;
|
||||
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
||||
onProgress: (m) => console.log(`raise: ${m}`),
|
||||
});
|
||||
instanceId = raised.instanceId;
|
||||
|
||||
await must("anchor", `cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${foundationBundle(bundle, raised.images)}\nMESHBUNDLE`);
|
||||
await must("anchor", `${HOST_PATH} apply /tmp/foundation.lock`, 900_000);
|
||||
const up = await must("anchor", `docker ps --format '{{.Names}}'`);
|
||||
for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) {
|
||||
assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`);
|
||||
}
|
||||
}, { timeout: 1_800_000 });
|
||||
|
||||
after(async () => {
|
||||
if (process.env["MESH_LAB_KEEP"]) { console.log(`MESH_LAB_KEEP set — leaving ${instanceId} standing`); return; }
|
||||
if (instanceId) await destroy(instanceId);
|
||||
await destroyAll(`${SCENARIO}-`);
|
||||
}, { timeout: 600_000 });
|
||||
|
||||
test("the machine that runs the bus joins over its own loopback and becomes the hub", { skip, timeout: 900_000 }, async () => {
|
||||
await mesh("node add anchor");
|
||||
const token = tokenFrom(await mesh("token issue --node anchor"));
|
||||
await placeTheBusUsers();
|
||||
const said = await must("anchor", `${HOST_PATH} enrol --token ${quote(token)}`);
|
||||
assert.match(said, /enrolled as anchor/, said);
|
||||
assert.doesNotMatch(said, /tunnel to the hub/, `the machine running the bus went through a tunnel:\n${said}`);
|
||||
await placeTheBusUsers();
|
||||
await must("anchor", `pgrep -x mesh-host >/dev/null || (nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3)`);
|
||||
|
||||
await mesh(`overlay place anchor --hub --endpoint 192.0.2.10:${HUB_PORT} --site lab`);
|
||||
await mesh("assign anchor mesh-wireguard");
|
||||
await mesh("push anchor", 600_000);
|
||||
await until("the hub's tunnel coming up", 300_000,
|
||||
async () => (await on("anchor", `wg show mesh0 listen-port`)).out.trim() === String(HUB_PORT),
|
||||
async () => `--- anchor host ---\n${(await on("anchor", `tail -30 /var/log/mesh-host.log`)).out}`);
|
||||
// The hub's port, opened as the derived filter opens it — scenery here: the filter module is the
|
||||
// two-node walk's to prove, and the bundle's own filter admits only ssh, the bus and the registry.
|
||||
await must("anchor", `nft insert rule inet mesh input udp dport ${HUB_PORT} accept`);
|
||||
});
|
||||
|
||||
test("a machine joins through the tunnel, with the bus closed to it", { skip, timeout: 900_000 }, async () => {
|
||||
// **The bus closed to the joiner**, at the anchor's very first hook — before the container
|
||||
// runtime's forwarding — so nothing from its own address reaches the bus, by any path.
|
||||
await must("anchor", `nft add table ip lab_bus_closed && ` +
|
||||
`nft add chain ip lab_bus_closed pre '{ type filter hook prerouting priority -300; }' && ` +
|
||||
`nft add rule ip lab_bus_closed pre ip saddr 192.0.2.30 tcp dport ${BUS_PORT} drop`);
|
||||
const reached = await on("joiner", `timeout 5 bash -c '</dev/tcp/192.0.2.10/${BUS_PORT}'`);
|
||||
assert.ok(!reached.ok, "the bus is still reachable from the joiner's own address; the bed proves nothing");
|
||||
|
||||
await must("joiner", `pacman -Sy --noconfirm --needed wireguard-tools >/dev/null 2>&1`, 300_000);
|
||||
const key = (await must("joiner", `${HOST_PATH} key 2>/dev/null`)).trim();
|
||||
assert.match(key, /^[A-Za-z0-9+/]{43}=$/, `not a tunnel key: ${key}`);
|
||||
// Asked again, the same key: a token may already have been issued for it.
|
||||
assert.equal((await must("joiner", `${HOST_PATH} key 2>/dev/null`)).trim(), key);
|
||||
|
||||
// The token is issued for the key; the hub is sent the machine as a peer before it is shown.
|
||||
const issued = await mesh(`token issue --new joiner --overlay-key ${key}`, 600_000);
|
||||
const token = tokenFrom(issued);
|
||||
const carried = JSON.parse(Buffer.from(token, "base64url").toString("utf8")) as {
|
||||
broker: string; tunnel?: { key: string; address: string; hub_endpoint: string };
|
||||
};
|
||||
assert.equal(carried.tunnel?.key, key, `the token was not issued for the machine's key: ${JSON.stringify(carried)}`);
|
||||
assert.equal(carried.tunnel?.hub_endpoint, `192.0.2.10:${HUB_PORT}`);
|
||||
assert.doesNotMatch(carried.broker, /^192\.0\.2\./, `the token sends the machine to the bus's public address: ${carried.broker}`);
|
||||
await placeTheBusUsers();
|
||||
|
||||
const said = await must("joiner", `${HOST_PATH} enrol --token ${quote(token)}`, 300_000);
|
||||
assert.match(said, /the tunnel to the hub is up/, said);
|
||||
assert.match(said, /the hub answered the tunnel/, said);
|
||||
assert.match(said, /enrolled as joiner/, said);
|
||||
|
||||
const shakes = await must("joiner", `wg show mesh0 latest-handshakes`);
|
||||
assert.ok(/\s[1-9]\d*\s*$/m.test(shakes), `the tunnel never shook hands with the hub: ${shakes}`);
|
||||
// And the bus is still closed to it: it joined through the tunnel, not around it.
|
||||
assert.ok(!(await on("joiner", `timeout 5 bash -c '</dev/tcp/192.0.2.10/${BUS_PORT}'`)).ok,
|
||||
"the bus opened to the joiner's own address during the join");
|
||||
|
||||
const recorded = (await must("anchor", `docker exec mesh-store psql -U postgres -d inventory -qAt ` +
|
||||
`-c "select overlay_key from node where name = 'joiner'"`)).trim();
|
||||
assert.equal(recorded, key, "the mesh does not hold the tunnel key the machine made");
|
||||
});
|
||||
@@ -227,4 +227,17 @@ test("the mesh grants a consumer redis's cache, and the credential it delivers a
|
||||
// writing the contributions rather than the bed.
|
||||
const runtimeEnv = await must(`docker inspect mesh-redis --format '{{json .Config.Env}}'`);
|
||||
assert.doesNotMatch(runtimeEnv, /MESH_SEAL_KEY/, `a seal key was set after all — ADR 0048 is not what ran:\n${runtimeEnv}`);
|
||||
|
||||
// A grant means exactly the consumer's own keys — `<login>:*`, the keyspace redis's provisioner
|
||||
// scopes the ACL user to: under it the consumer reads and writes, outside it and on the server as
|
||||
// a whole it is refused. Carried over from the large mesh bed's retired cache-grant test —
|
||||
// without this a provisioner that granted everything would keep every bed green.
|
||||
const asConsumer = (command: string) =>
|
||||
on(`docker exec redis redis-cli --user ${quote(as)} --pass ${quote(password)} --no-auth-warning ${command} 2>&1`);
|
||||
assert.match((await asConsumer(`SET ${as}:proof yes`)).out, /OK/, "the consumer cannot write under its own login");
|
||||
assert.match((await asConsumer(`GET ${as}:proof`)).out, /yes/, "the consumer cannot read back what it wrote");
|
||||
assert.match((await asConsumer("SET other:proof no")).out, /NOPERM|no permissions/i,
|
||||
"the consumer wrote outside its own keys, so the grant means more than it says");
|
||||
assert.match((await asConsumer("FLUSHALL")).out, /NOPERM|no permissions/i,
|
||||
"the consumer flushed the whole server, so the grant means more than it says");
|
||||
});
|
||||
|
||||
+192
-423
@@ -18,13 +18,13 @@
|
||||
|
||||
import { test, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { existsSync } from "node:fs";
|
||||
import { loadScenario } from "../../src/declaration/parse.ts";
|
||||
import { raise } from "../../src/lifecycle/raise.ts";
|
||||
import { pinnedInto, stillUnpinned, type HeldImage } from "../../src/pinning.ts";
|
||||
import type { HeldImage } from "../../src/pinning.ts";
|
||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||||
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
|
||||
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, deriveTheFilterOn, catalogueIsPresent } from "./harness.ts";
|
||||
import { incus } from "../../src/incus/client.ts";
|
||||
import { machineName } from "../../src/lifecycle/names.ts";
|
||||
import { ready, returnTo, keep, rememberStock, warmStock } from "../../src/warm.ts";
|
||||
@@ -37,7 +37,6 @@ const binary = hostBinaryPath();
|
||||
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
||||
const builder = process.env["MESH_LAB_BUILDER"] ?? "";
|
||||
/** mesh-controller's `examples/modules`, so the manifests proven here are the ones that ship. */
|
||||
const moduleExamples = process.env["MESH_LAB_MODULES"] ?? "";
|
||||
|
||||
const skip = !capability.usable
|
||||
? `lab not usable: ${capability.why}`
|
||||
@@ -45,7 +44,8 @@ const skip = !capability.usable
|
||||
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
||||
: !bundle || !existsSync(bundle)
|
||||
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
||||
: false;
|
||||
// The anchor's filter and the resolvers are the catalogue's (ADR 0088, issue 074).
|
||||
: catalogueIsPresent() || false;
|
||||
|
||||
const SCENARIO = "two-nodes";
|
||||
let instanceId = "";
|
||||
@@ -193,6 +193,20 @@ function bundleFor(images: HeldImage[]): string {
|
||||
return foundationBundle(bundle, images);
|
||||
}
|
||||
|
||||
/**
|
||||
* Put the mesh's composed user list where the anchor's bus reads it, and make it re-read.
|
||||
*
|
||||
* **Genesis's step, done by hand because this bed raises genesis by hand** (novox/hq 04-ISSUES/146).
|
||||
* The bus here is the bundle's, not a module the mesh delivers, so an account the mesh composes —
|
||||
* the enrolment when a token is issued, the node's own when it enrols — reaches it only if whoever
|
||||
* raised it places it. Without this the first join is refused with an authorisation violation.
|
||||
*/
|
||||
async function composeTheBusUsers(): Promise<void> {
|
||||
await must("anchor",
|
||||
`docker exec mesh-controller /mesh-controller broker accounts > /var/lib/mesh-bus-conf/accounts.conf && ` +
|
||||
`docker kill -s HUP mesh-broker >/dev/null`);
|
||||
}
|
||||
|
||||
/** Take a token out of what `token issue` printed. It is the one base64url blob on its own line. */
|
||||
function tokenFrom(said: string): string {
|
||||
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
|
||||
@@ -200,6 +214,69 @@ function tokenFrom(said: string): string {
|
||||
return found;
|
||||
}
|
||||
|
||||
/**
|
||||
* The build machine's module: a holder of the node-build-agent seat, with the bus credential the mesh
|
||||
* seals to the machine and nothing else.
|
||||
*
|
||||
* **On the bus, the way today's build agent is** (novox/hq ADR 0106, 0127, 0131). The builder used to
|
||||
* be started against the old broker's guest account on loopback; the mesh moved to NATS, the builder
|
||||
* dials only a credential the mesh delivered (`MESH_BROKER_FILE`), and that account exists only as a
|
||||
* module's, composed into the bus's user list. The catalogue's `build-agent` is that module, but it
|
||||
* runs the builder as a container from an artifact the mesh builds and needs the artifact store and
|
||||
* package registry bound; this bed builds nothing before it has a builder, so it declares the same
|
||||
* claim and the same sealed credential and starts the binary it was pointed at itself.
|
||||
*/
|
||||
const BUILDER_MODULE = "lab-builder";
|
||||
const BUILDER_STATE = `/var/lib/mesh/${BUILDER_MODULE}`;
|
||||
const BUILDER_MANIFEST = JSON.stringify({
|
||||
module: BUILDER_MODULE, version: "1",
|
||||
claims: [{ name: "node-build-agent", scope: "node", serves: ["current", "kill", "pause", "resume"] }],
|
||||
"own-secrets": { broker: `${BUILDER_STATE}/broker` },
|
||||
resources: [{ id: "mesh-state", type: "directory", path: BUILDER_STATE, mode: "0700" }],
|
||||
});
|
||||
|
||||
/**
|
||||
* The builder, on the anchor, holding the build seat over the bus with the credential the mesh sealed
|
||||
* to the machine. Assigning the module issues the account (novox/hq issue 203); the bus here is the
|
||||
* bundle's, so its user list is placed by hand (composeTheBusUsers); the push writes the credential
|
||||
* where the module declared it. Idempotent: a warm restore or a second call starts only what is gone.
|
||||
*/
|
||||
async function startBuilder(): Promise<void> {
|
||||
// The binary is disk and survives a snapshot; a snapshot taken without it does not gain it on a
|
||||
// return, so it is pushed whenever the machine has none.
|
||||
if (!(await on("anchor", `test -x /usr/local/bin/mesh-builder`)).ok) {
|
||||
await incus([
|
||||
"file", "push", builder, `${machineName(instanceId, "anchor")}/usr/local/bin/mesh-builder`,
|
||||
"--mode", "0755",
|
||||
], 180_000);
|
||||
}
|
||||
if (!(await on("anchor", `test -s ${BUILDER_STATE}/broker`)).ok) {
|
||||
await must("anchor", `printf %s ${quote(BUILDER_MANIFEST)} > /tmp/${BUILDER_MODULE}.json && ` +
|
||||
`docker cp /tmp/${BUILDER_MODULE}.json mesh-controller:/${BUILDER_MODULE}.json`);
|
||||
await mesh(`module add /${BUILDER_MODULE}.json`);
|
||||
await mesh(`assign anchor ${BUILDER_MODULE}`);
|
||||
await composeTheBusUsers();
|
||||
await mesh("push anchor", 600_000);
|
||||
const by = Date.now() + 180_000;
|
||||
while (!(await on("anchor", `test -s ${BUILDER_STATE}/broker`)).ok) {
|
||||
if (Date.now() > by) {
|
||||
assert.fail(`the mesh never delivered the builder's bus credential to ${BUILDER_STATE}/broker:\n` +
|
||||
(await on("anchor", "tail -30 /var/log/mesh-host.log")).out);
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 3000));
|
||||
}
|
||||
}
|
||||
await must("anchor", `mkdir -p /var/lib/mesh-builder`);
|
||||
await must("anchor", `pgrep -x mesh-builder >/dev/null || ` +
|
||||
`(MESH_BROKER_FILE=${BUILDER_STATE}/broker MESH_NODE=anchor MESH_REGISTRY=${registry} ` +
|
||||
`MESH_WORKSPACE=/var/lib/mesh-builder ` +
|
||||
`nohup /usr/local/bin/mesh-builder > /var/log/mesh-builder.log 2>&1 & sleep 3)`);
|
||||
// Holding the seat, not merely started: a builder refused by the bus exits, and a build asked of a
|
||||
// seat nobody holds waits out its whole timeout before saying anything.
|
||||
const running = await on("anchor", `pgrep -x mesh-builder >/dev/null`);
|
||||
assert.ok(running.ok, `the builder did not stay up:\n${(await on("anchor", "tail -30 /var/log/mesh-builder.log")).out}`);
|
||||
}
|
||||
|
||||
before(async () => {
|
||||
if (skip) return;
|
||||
|
||||
@@ -232,6 +309,7 @@ before(async () => {
|
||||
const running = await on("anchor", `pgrep -x mesh-host >/dev/null && echo yes || echo no`);
|
||||
assert.equal(running.out.trim(), "yes",
|
||||
"the host did not come back after a restore, so nothing would apply anything");
|
||||
// The hand-started builder is memory too; the build test starts it again where it is needed.
|
||||
|
||||
console.log(`warm: returned ${instanceId} to its state in ${seconds.toFixed(1)}s, ` +
|
||||
`and started the host again`);
|
||||
@@ -256,19 +334,8 @@ before(async () => {
|
||||
await must("anchor", `cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
|
||||
await must("anchor", `${HOST_PATH} apply /tmp/foundation.lock`);
|
||||
|
||||
// A build machine, so anything here can ask the mesh to build something. Placed rather than
|
||||
// assumed: nothing else in this scenario would start one.
|
||||
if (builder) {
|
||||
await incus([
|
||||
"file", "push", builder, `${machineName(instanceId, "anchor")}/usr/local/bin/mesh-builder`,
|
||||
"--mode", "0755",
|
||||
], 180_000);
|
||||
await must("anchor", `mkdir -p /var/lib/mesh-builder`);
|
||||
await must("anchor",
|
||||
`MESH_BROKER_AMQP='amqp://guest:guest@127.0.0.1:5672/' MESH_REGISTRY=${registry} ` +
|
||||
`MESH_WORKSPACE=/var/lib/mesh-builder ` +
|
||||
`nohup /usr/local/bin/mesh-builder > /var/log/mesh-builder.log 2>&1 & sleep 3`);
|
||||
}
|
||||
// No build machine yet: the builder holds its seat on a credential the mesh delivers, so it is
|
||||
// started by the test that builds, once the anchor has joined (startBuilder).
|
||||
if (warming) {
|
||||
// Snapshotted only now, with everything up: a state worth returning to is the one after the
|
||||
// part nobody wants to repeat.
|
||||
@@ -298,15 +365,44 @@ test("a bare machine becomes a mesh", { skip, timeout: 600_000 }, async () => {
|
||||
});
|
||||
|
||||
test("both machines join it, and the token is all they need", { skip, timeout: 900_000 }, async () => {
|
||||
for (const [machine, node] of [["anchor", "anchor"], ["laptop", "laptop"]] as const) {
|
||||
await mesh(`node add ${node}`);
|
||||
const token = tokenFrom(await mesh(`token issue --node ${node}`));
|
||||
// No --name. The token says what the mesh calls the machine, which is the fault this walk
|
||||
// found the first time it was run.
|
||||
const said = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`);
|
||||
assert.match(said, new RegExp(`enrolled as ${node}`), said);
|
||||
assert.match(said, /sealing key/, "no sealing key was generated");
|
||||
// **The anchor runs the bus, so it joins over its own loopback.** Every other machine joins
|
||||
// through the tunnel (novox/hq ADR 0169): the anchor is the hub, and its tunnel comes up first.
|
||||
await mesh(`node add anchor`);
|
||||
const first = tokenFrom(await mesh(`token issue --node anchor`));
|
||||
await composeTheBusUsers();
|
||||
// No --name. The token says what the mesh calls the machine, which is the fault this walk
|
||||
// found the first time it was run.
|
||||
const anchorSaid = await must("anchor", `${HOST_PATH} enrol --token ${quote(first)}`);
|
||||
await composeTheBusUsers();
|
||||
assert.match(anchorSaid, /enrolled as anchor/, anchorSaid);
|
||||
assert.match(anchorSaid, /sealing key/, "no sealing key was generated");
|
||||
|
||||
await mesh("overlay place anchor --hub --endpoint 192.0.2.10:51820 --site lab");
|
||||
await mesh("assign anchor networking");
|
||||
await must("anchor", `pgrep -x mesh-host >/dev/null || (nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3)`);
|
||||
await mesh("push anchor");
|
||||
// The host reports which of its links face outside with every apply, and the filter is not
|
||||
// composed for a machine that has not — so its first apply is waited for, not slept past.
|
||||
const appliedBy = Date.now() + 180_000;
|
||||
while (!(await on("anchor", `grep -q 'applied [0-9]' /var/log/mesh-host.log`)).ok) {
|
||||
if (Date.now() > appliedBy) assert.fail(`the anchor never applied what it was sent:\n${(await on("anchor", "tail -30 /var/log/mesh-host.log")).out}`);
|
||||
await new Promise((r) => setTimeout(r, 3000));
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 5000));
|
||||
await deriveTheFilterOn({ machine: "anchor", node: "anchor", hubPort: 51820, held, must, mesh, on });
|
||||
|
||||
// **The laptop makes its tunnel key, and the token is issued for it.** The hub is told the key
|
||||
// before the token is shown, so the tunnel answers the first time the laptop knocks.
|
||||
await must("laptop", `pacman -Sy --noconfirm --needed wireguard-tools >/dev/null 2>&1`);
|
||||
const key = (await must("laptop", `${HOST_PATH} key 2>/dev/null`)).trim();
|
||||
await mesh(`node add laptop`);
|
||||
const second = tokenFrom(await mesh(`token issue --node laptop --overlay-key ${key}`));
|
||||
await composeTheBusUsers();
|
||||
const laptopSaid = await must("laptop", `${HOST_PATH} enrol --token ${quote(second)}`);
|
||||
await composeTheBusUsers();
|
||||
assert.match(laptopSaid, /the tunnel to the hub is up/, laptopSaid);
|
||||
assert.match(laptopSaid, /enrolled as laptop/, laptopSaid);
|
||||
assert.match(laptopSaid, /sealing key/, "no sealing key was generated");
|
||||
|
||||
const recorded = await must("anchor",
|
||||
`docker exec mesh-store psql -U postgres -d inventory -qAt ` +
|
||||
@@ -347,10 +443,16 @@ test("a credential reaches both ends and the mesh holds neither", { skip, timeou
|
||||
await mesh("assign laptop meshboard");
|
||||
|
||||
for (const machine of ["anchor", "laptop"]) {
|
||||
await must(machine, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
|
||||
// Once. On a warm return the host is already running (see `before`); a second one would
|
||||
// consume the same queue and apply the same declaration twice, concurrently.
|
||||
await must(machine, `pgrep -x mesh-host >/dev/null || (nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3)`);
|
||||
}
|
||||
await mesh("push");
|
||||
await new Promise((r) => setTimeout(r, 8000));
|
||||
// The anchor's derived filter, admitting the hub's port — what genesis does on the control-node,
|
||||
// and what a bed raised from the bundle must do itself (ADR 0088). Until it is, the base filter
|
||||
// keeps the hub closed and nothing on the laptop reaches anchor over the private network.
|
||||
await deriveTheFilterOn({ machine: "anchor", node: "anchor", hubPort: 51820, held, must, mesh, on });
|
||||
|
||||
const onConsumer = (await must("laptop", `cat /etc/meshboard/database.password`)).trim();
|
||||
// Named after the machine *and* the module, because a consumer is both (novox/hq
|
||||
@@ -492,11 +594,11 @@ test("a declaration waits for a machine that is switched off", { skip, timeout:
|
||||
// Everything needed to tell "the message was never queued" from "the host never read it".
|
||||
const log = await on("laptop", `tail -20 /var/log/mesh-host.log`);
|
||||
const queues = await on("anchor",
|
||||
`docker exec mesh-broker lavinmqctl list_queues name messages 2>&1 | head -10`);
|
||||
`curl -s 'http://127.0.0.1:8222/jsz?consumers=true' 2>&1 | head -c 4000`);
|
||||
const owned = await on("anchor",
|
||||
`docker exec mesh-store psql -U postgres -d inventory -qAt -c "select name, outcome from node_report r join node n on n.id=r.node"`);
|
||||
assert.fail(`a declaration sent to a switched-off machine was lost\n` +
|
||||
`--- the host's log ---\n${log.out}\n--- the broker's queues ---\n${queues.out}\n` +
|
||||
`--- the host's log ---\n${log.out}\n--- the bus's streams and consumers ---\n${queues.out}\n` +
|
||||
`--- what each machine last did ---\n${owned.out}`);
|
||||
}
|
||||
assert.equal((await must("laptop", `cat /etc/mesh-while-away`)).trim(), "waited");
|
||||
@@ -618,6 +720,7 @@ test("the mesh runs its own artifact store", { skip, timeout: 900_000 }, async (
|
||||
`"capabilities":["container-runtime"],` +
|
||||
`"claims":[{"name":"the-artifact-store","scope":"node"}],` +
|
||||
`"serves":{"artifact-store":{"port":5000}},` +
|
||||
`"listens":[{"port":5000,"from":"mesh","why":"every machine pulls what the mesh built"}],` +
|
||||
`"resources":[` +
|
||||
`{"id":"state","type":"directory","path":"/var/lib/mesh/registry","mode":"0700"},` +
|
||||
`{"id":"store","type":"container","name":"mesh-registry","image":"${ARTIFACT_STORE}",` +
|
||||
@@ -631,10 +734,15 @@ test("the mesh runs its own artifact store", { skip, timeout: 900_000 }, async (
|
||||
await mesh("module add /registry.json");
|
||||
await mesh("assign anchor registry");
|
||||
await mesh("push anchor");
|
||||
await new Promise((r) => setTimeout(r, 12_000));
|
||||
|
||||
// The store's image is pulled from upstream at apply, over the uplink; that takes what it takes.
|
||||
let names = "";
|
||||
for (let i = 0; i < 60 && !/mesh-registry/.test(names); i++) {
|
||||
await new Promise((r) => setTimeout(r, 3000));
|
||||
names = await must("anchor", `docker ps --format '{{.Names}}'`);
|
||||
}
|
||||
// Running, and answering — a container that is up is not a registry that replies.
|
||||
assert.match(await must("anchor", `docker ps --format '{{.Names}}'`), /mesh-registry/);
|
||||
assert.match(names, /mesh-registry/,
|
||||
`the mesh's registry never started:\n${names}\n--- host log ---\n${(await on("anchor", `tail -20 /var/log/mesh-host.log`)).out}`);
|
||||
let answers = false;
|
||||
for (let i = 0; i < 20 && !answers; i++) {
|
||||
answers = (await on("anchor", `curl -sf http://127.0.0.1:5000/v2/ -o /dev/null`)).ok;
|
||||
@@ -655,8 +763,11 @@ test("a machine serves its internal name with a certificate the mesh issued", {
|
||||
// The mesh's own authority certifies names only the mesh knows (novox/hq 08-connectivity).
|
||||
// Asserted with a real handshake: a certificate that parses and does not chain fails at the
|
||||
// moment something connects, which is the worst place to find out.
|
||||
// The port the handshake below is tried on, declared: the anchor filters what its modules
|
||||
// did not declare (ADR 0088), and a test server on an undeclared port proves only that.
|
||||
await must("anchor", `printf %s '{"module":"served","version":"1",` +
|
||||
`"certificate":{"into":"/etc/mesh/serving.crt","authority":"/etc/mesh/authority.crt"},` +
|
||||
`"listens":[{"port":8443,"from":"mesh","why":"a handshake against the certificate the mesh issued"}],` +
|
||||
`"resources":[{"id":"dir","type":"directory","path":"/etc/mesh","mode":"0755"}]}' ` +
|
||||
`> /tmp/served.json`);
|
||||
await must("anchor", `docker cp /tmp/served.json mesh-controller:/served.json`);
|
||||
@@ -967,7 +1078,7 @@ test("a route is a grant: a workload is reached by the name it asked for", {
|
||||
// they are different questions: one says who may reach it, the other says by what name — and
|
||||
// the earlier test left this machine filtering, so a module that asked for a route and not for
|
||||
// the port would be unreachable by the proxy it just asked for.
|
||||
await must("anchor", `printf %s '{"module":"storefront","version":"1",` +
|
||||
await must("anchor", `printf %s '{"module":"storefront","version":"1","slug":"shop",` +
|
||||
`"requires":["route"],"capabilities":["container-runtime"],` +
|
||||
`"contributes":{"route":{"name":"shop.mesh.test","port":8088}},` +
|
||||
`"binds":{"route":"/etc/storefront/route.json"},` +
|
||||
@@ -1131,6 +1242,9 @@ test("a new commit reaches a machine that is already running the old one", {
|
||||
// novox/hq ADR 0010 names the real risk of replacing a pipeline with a comparison: losing the
|
||||
// question "did my change go out?". This is that question, end to end — a commit, a build, a
|
||||
// catalogue, and a machine that ends up running what the source says.
|
||||
// The builder needs the anchor joined — its credential is the mesh's to deliver — so it is started
|
||||
// here, where a build is asked for. The mesh's own build agent is a module with a restart policy.
|
||||
await startBuilder();
|
||||
const repo = "/var/lib/mesh/builder/repositories/delivered";
|
||||
const write = async (what: string) =>
|
||||
await must("anchor", `mkdir -p ${repo} && printf %s '{"module":"delivered","version":"1",` +
|
||||
@@ -1258,79 +1372,11 @@ test("the board names the machine that is not doing what it was told", {
|
||||
});
|
||||
|
||||
// Defends novox/hq ADR 0007: filtering the hub must not cut the overlay it carries.
|
||||
test("the hub can be filtered without severing the mesh", {
|
||||
skip, timeout: 900_000,
|
||||
}, async () => {
|
||||
// The machine that most needs a firewall was the one that could not have one. A hub is dialled
|
||||
// by every node at other sites; a machine that is not a hub dials out and needs nothing open.
|
||||
// They are the same module, so a static `listens` cannot say it — and the machine it gets wrong
|
||||
// is the one facing the public internet.
|
||||
//
|
||||
// The failure this guards against is not subtle and is very hard to recover from: a rule set
|
||||
// that closes the hub's own port takes the private network down, and the mesh's way of fixing
|
||||
// anything is to send a declaration over it.
|
||||
// Its own directory. Another module on this machine already declares /etc/mesh, and the mesh
|
||||
// refuses two modules declaring one path rather than letting the second quietly win — which it
|
||||
// did here, correctly, the first time this ran.
|
||||
const rules = "/etc/mesh-hub/filter.nft";
|
||||
await must("anchor", `printf %s '{"module":"hubfilter","version":"1",` +
|
||||
`"capabilities":["firewall"],` +
|
||||
`"filtering":{"into":"${rules}"},` +
|
||||
`"resources":[{"id":"nftables","type":"package","package":"nftables"},` +
|
||||
`{"id":"dir","type":"directory","path":"/etc/mesh-hub","mode":"0755"},` +
|
||||
`{"id":"unit","type":"file","path":"/etc/systemd/system/hub-filter.service",` +
|
||||
`"mode":"0644","content":"[Unit]\\nDescription=What the mesh computed for the hub\\n` +
|
||||
`[Service]\\nType=oneshot\\nRemainAfterExit=yes\\n` +
|
||||
`ExecStart=/usr/bin/nft -f ${rules}\\n[Install]\\nWantedBy=multi-user.target\\n"},` +
|
||||
`{"id":"filter","type":"service","unit":"hub-filter.service","state":"running",` +
|
||||
`"boot":"enabled","restart-on":["filtering"]}]}' > /tmp/hubfilter.json`);
|
||||
await must("anchor", `docker cp /tmp/hubfilter.json mesh-controller:/hubfilter.json`);
|
||||
await mesh("module add /hubfilter.json");
|
||||
await mesh("assign anchor hubfilter");
|
||||
await mesh("push anchor");
|
||||
await new Promise((r) => setTimeout(r, 20_000));
|
||||
|
||||
// The hub's own way onto the private network is open, and derived — nothing in that manifest
|
||||
// mentions a port.
|
||||
const written = await must("anchor", `cat ${rules}`);
|
||||
assert.match(written, /udp dport 51820 accept/,
|
||||
`the hub's rule set closes the private network it is the way onto:\n${written}`);
|
||||
// The module that provides the private network, not the requirement it answers: `networking`
|
||||
// is the domain a module offers, and what caused a rule is the module itself.
|
||||
assert.match(written, /# mesh-wireguard — the private network/,
|
||||
`the rule does not name what caused it:\n${written}`);
|
||||
|
||||
// Loaded, and the mesh still works: a declaration reaches the other machine, which it cannot if
|
||||
// the overlay is severed. This is the assertion that matters — a rule file that looks right and
|
||||
// a mesh that has stopped are exactly what this is guarding against.
|
||||
assert.match(await must("anchor", `nft list table inet mesh`), /dport 51820/);
|
||||
|
||||
await must("laptop", `rm -f /etc/mesh-still-works`);
|
||||
await must("anchor", `printf %s '{"module":"stillworks","version":"1",` +
|
||||
`"resources":[{"id":"marker","type":"file","path":"/etc/mesh-still-works",` +
|
||||
`"content":"yes","mode":"0644"}]}' > /tmp/stillworks.json`);
|
||||
await must("anchor", `docker cp /tmp/stillworks.json mesh-controller:/stillworks.json`);
|
||||
await mesh("module add /stillworks.json");
|
||||
await mesh("assign laptop stillworks");
|
||||
await mesh("push laptop");
|
||||
|
||||
let arrived = false;
|
||||
for (let i = 0; i < 20 && !arrived; i++) {
|
||||
arrived = (await on("laptop", `test -f /etc/mesh-still-works`)).ok;
|
||||
if (!arrived) await new Promise((r) => setTimeout(r, 3000));
|
||||
}
|
||||
assert.ok(arrived,
|
||||
"the hub applied its own rule set and the mesh stopped reaching the other machine");
|
||||
|
||||
// And the other machine still reaches the hub over the private network, which is what the
|
||||
// opened port is for.
|
||||
assert.ok((await on("laptop", `ping -c 1 -W 5 anchor.internal`)).ok,
|
||||
"the private network is down after the hub filtered itself");
|
||||
|
||||
await mesh("unassign anchor hubfilter");
|
||||
await mesh("unassign laptop stillworks");
|
||||
await mesh("push");
|
||||
});
|
||||
// "The hub can be filtered without severing the mesh" lived here, with an inline filter module on
|
||||
// the anchor. Since ADR 0088 the hub IS filtered on every mesh — the base filter closes it until a
|
||||
// filter module derives the rules — so the credential test above assigns the catalogue's and
|
||||
// asserts the hub's port is admitted, and every cross-machine test after it is the proof the mesh
|
||||
// was not severed. Retired 2026-09-22.
|
||||
|
||||
test("a container reaches another machine by the name the mesh gave it", {
|
||||
skip, timeout: 900_000,
|
||||
@@ -1375,6 +1421,23 @@ test("a container reaches another machine by the name the mesh gave it", {
|
||||
|
||||
// Defends novox/hq ADR 0007: a name under a machine is that machine, without the mesh being
|
||||
// told each one.
|
||||
/** The catalogue's resolver modules on the control plane, added once; dnsmasq speaks on the bus so
|
||||
* it is issued once per machine. The mesh writes the resolver's data as a fact the module
|
||||
* declares (/etc/mesh-resolver/nodes.conf); no module of the mesh's own writes it any more. */
|
||||
const resolverIssued = new Set<string>();
|
||||
async function resolverModules(machines: string[]): Promise<void> {
|
||||
for (const name of ["dnsmasq", "resolved-split-dns"]) {
|
||||
const manifest = catalogueModule(name, held);
|
||||
await must("anchor", `printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-controller:/${name}.json`);
|
||||
await mesh(`module add /${name}.json`);
|
||||
}
|
||||
for (const machine of machines) {
|
||||
if (resolverIssued.has(machine)) continue;
|
||||
await mesh(`module issue dnsmasq --node ${machine}`);
|
||||
resolverIssued.add(machine);
|
||||
}
|
||||
}
|
||||
|
||||
test("every name under a machine resolves to that machine", {
|
||||
skip, timeout: 900_000,
|
||||
}, async () => {
|
||||
@@ -1385,9 +1448,11 @@ test("every name under a machine resolves to that machine", {
|
||||
//
|
||||
// The mesh writes the data and runs no daemon: a resolver is third-party software, and the
|
||||
// mesh has no business choosing one. So what is checked here is the mesh's half — that the
|
||||
// data is right, complete, and follows the machines.
|
||||
await mesh("assign anchor mesh-resolver");
|
||||
await mesh("assign laptop mesh-resolver");
|
||||
// data is right, complete, and follows the machines. The data is a fact the catalogue's dnsmasq
|
||||
// declares, so that module is what is assigned; what it runs is the next test's concern.
|
||||
await resolverModules(["anchor", "laptop"]);
|
||||
await mesh("assign anchor dnsmasq");
|
||||
await mesh("assign laptop dnsmasq");
|
||||
await mesh("push");
|
||||
await new Promise((r) => setTimeout(r, 15_000));
|
||||
|
||||
@@ -1413,10 +1478,10 @@ test("every name under a machine resolves to that machine", {
|
||||
// because a wildcard pointing at nothing resolves and then hangs — where an unresolvable name
|
||||
// fails at once and says which name it was.
|
||||
//
|
||||
// Both, and that is not tidiness: `mesh-resolver` requires name resolution, which requires the
|
||||
// network, so unassigning the domain module alone leaves the machine on the network — pulled
|
||||
// back by its own requirement. The mesh was right and this test was wrong the first time.
|
||||
await mesh("unassign laptop mesh-resolver");
|
||||
// Both: the resolver's data follows the private network, so the machine leaves the network as
|
||||
// well as the resolver, and what is asserted is that the machine that stayed is answered for and
|
||||
// the one that left is not.
|
||||
await mesh("unassign laptop dnsmasq");
|
||||
await mesh("unassign laptop networking");
|
||||
await mesh("push anchor");
|
||||
await new Promise((r) => setTimeout(r, 15_000));
|
||||
@@ -1428,14 +1493,13 @@ test("every name under a machine resolves to that machine", {
|
||||
`the machine that stayed lost its own name:\n${after}`);
|
||||
|
||||
await mesh("assign laptop networking");
|
||||
await mesh("unassign anchor mesh-resolver");
|
||||
await mesh("unassign anchor dnsmasq");
|
||||
await mesh("push");
|
||||
await new Promise((r) => setTimeout(r, 15_000));
|
||||
});
|
||||
|
||||
test("a service is reached by a name under the machine it runs on", {
|
||||
skip: skip || (!moduleExamples ? "set MESH_LAB_MODULES to mesh-controller's examples/modules" : false),
|
||||
timeout: 900_000,
|
||||
skip, timeout: 900_000,
|
||||
}, async () => {
|
||||
// postgres.novox.internal, plex.ace.internal — the first label is the service and the rest is
|
||||
// the node, so anything under a node's name must resolve to that node. What routes it once it
|
||||
@@ -1447,12 +1511,7 @@ test("a service is reached by a name under the machine it runs on", {
|
||||
// /etc/resolv.conf. The two claim the same thing precisely so that assigning the wrong one is a
|
||||
// refusal rather than a fight over the file — and picking the wrong one here would have been
|
||||
// testing that fight.
|
||||
for (const name of ["dnsmasq", "resolved-split-dns"]) {
|
||||
const manifest = readFileSync(`${moduleExamples}/${name}.json`, "utf8");
|
||||
await must("anchor", `cat > /tmp/${name}.json <<'MANIFEST'\n${manifest}\nMANIFEST`);
|
||||
await must("anchor", `docker cp /tmp/${name}.json mesh-controller:/${name}.json`);
|
||||
await mesh(`module add /${name}.json`);
|
||||
}
|
||||
await resolverModules(["anchor", "laptop"]);
|
||||
|
||||
// Both machines, because a node resolves from its own copy — the same rule as everything else
|
||||
// it holds. A mesh where one machine answers for all of them stops resolving when that machine
|
||||
@@ -1679,305 +1738,15 @@ test("a third-party workload is adopted, with the credential it already had", {
|
||||
// Running them needs their images stocked and two provisioners built, which is a separate and
|
||||
// larger job. This is the half that can be known now, and it is the half where a design fault
|
||||
// would live.
|
||||
test("the real modules resolve together, and compose a declaration a host accepts", {
|
||||
skip, timeout: 300_000,
|
||||
}, async (t) => {
|
||||
// Everything the catalogue holds, except two whose names this mesh is already running under:
|
||||
// `registry` is the artifact store the suite stood up, and `umami` is the adopted workload —
|
||||
// adding the catalogue's manifests would replace the records of modules that are live and
|
||||
// assigned, and the adopted umami would suddenly require a database it never asked for.
|
||||
const modules = ["postgres", "keycloak", "gitea", "minio", "mailu",
|
||||
"redis", "grafana", "nextcloud", "searxng", "influxdb", "verdaccio"];
|
||||
const planned: string[] = [];
|
||||
for (const name of modules) {
|
||||
const raw = readFileSync(
|
||||
`${process.env["MESH_LAB_MODULES"]}/${name}.json`, "utf8");
|
||||
// Pointed at this scenario's registry before being added, exactly as the forge is.
|
||||
//
|
||||
// **Not cosmetic.** Some of these name an image the mesh builds, whose digest does not exist
|
||||
// until it is built — so the file legitimately carries a placeholder, and composing a
|
||||
// declaration from it is refused (novox/hq 04-ISSUES/025). Planning what could never run is
|
||||
// what this test used to do.
|
||||
const pinned = pinnedInto(raw, held);
|
||||
// What this scenario does not serve cannot be redirected, and a module still naming a
|
||||
// placeholder cannot be planned — the refusal is the point (novox/hq 04-ISSUES/025). Skipped
|
||||
// and said, rather than silently dropped: a planning test quietly covering four modules
|
||||
// instead of five is the false coverage this suite exists to prevent.
|
||||
const left = stillUnpinned(pinned);
|
||||
if (left.length > 0) {
|
||||
console.log(`skipping ${name}: this scenario serves no ${left.join(", ")}`);
|
||||
continue;
|
||||
}
|
||||
planned.push(name);
|
||||
await must("anchor", `printf %s ${quote(pinned)} > /${name}.json`);
|
||||
await must("anchor", `docker cp /${name}.json mesh-controller:/${name}.json`);
|
||||
await mesh(`module add /${name}.json`);
|
||||
}
|
||||
// Three tests lived here that read the mesh's example modules, which moved to the catalogue.
|
||||
// Retired 2026-09-22 rather than rewritten into copies of the beds that stand where they stood
|
||||
// (novox/hq issue 074): "the real modules resolve together" — whole-mesh-novox installs the
|
||||
// catalogue's modules together and its gate is the composed declaration accepted and every core
|
||||
// container running; "the forge runs, on a database the mesh gave it" — the same bed, which gates
|
||||
// on gitea running but does not yet ask it to answer on its port with the credential it was given,
|
||||
// a gap that bed should close; "a consumer's cache grant means exactly its own keys" — its tenancy
|
||||
// assertions (a write outside the consumer's keys and a FLUSHALL are refused) moved into
|
||||
// mesh-grant-end-to-end, against the catalogue's redis.
|
||||
|
||||
// **Put the machine back whatever happens.** Tests here share one mesh, so what this one
|
||||
// assigns is what the next one inherits. Written at the end of the body once, it was skipped
|
||||
// the first time this test failed — and the next test's push was refused by a module this one
|
||||
// had left behind, which reads as a fault in the test that was actually working.
|
||||
t.after(async () => {
|
||||
for (const name of planned) await mesh(`unassign anchor ${name}`).catch(() => {});
|
||||
});
|
||||
|
||||
// Assigned one at a time, because assignment resolves the whole set and says so immediately.
|
||||
// A refusal here is the graph rejecting something, which is the point of asking.
|
||||
for (const name of planned) {
|
||||
await mesh(`assign anchor ${name}`);
|
||||
}
|
||||
|
||||
const plan = await mesh("plan anchor --json", 120_000);
|
||||
const declaration = JSON.parse(plan.slice(plan.indexOf("{")));
|
||||
const byId = new Map<string, any>(
|
||||
(declaration.resources as any[]).map((r) => [r.id, r]));
|
||||
const ids = [...byId.keys()];
|
||||
|
||||
// Every module's own network, which only exists because more than one container needs to reach
|
||||
// another by name.
|
||||
for (const id of ["postgres.net", "keycloak.net", "minio.net", "mailu.net"]) {
|
||||
assert.ok(byId.has(id), `${id} is missing; ${ids.length} resources: ${ids.join(", ")}`);
|
||||
assert.equal(byId.get(id).type, "network");
|
||||
}
|
||||
|
||||
// The cross-module edge: keycloak asked for a database and was told where it is and given a
|
||||
// credential. Neither file is anything keycloak's manifest could have written.
|
||||
const bound = [...byId.values()].find((r) =>
|
||||
r.type === "file" && r.path === "/var/lib/keycloak/database.json");
|
||||
assert.ok(bound, `keycloak was never told where its database is: ${ids.join(", ")}`);
|
||||
assert.match(JSON.stringify(bound), /postgres/,
|
||||
"keycloak's binding does not name what answered its requirement");
|
||||
|
||||
// The password, alone in a file and sealed. It is a password and nothing else, so nothing reads
|
||||
// it as configuration — novox/hq 04-ISSUES/023 and the playbook both turn on that distinction.
|
||||
const credential = [...byId.values()].find((r) =>
|
||||
r.type === "file" && r.path === "/var/lib/keycloak/database.secret");
|
||||
assert.ok(credential, `keycloak was given no credential for its database: ${ids.join(", ")}`);
|
||||
assert.ok(credential.sealed, "keycloak's credential is not sealed, so the mesh can read it");
|
||||
assert.ok(!credential.content, "a credential arrived as content rather than sealed");
|
||||
|
||||
// And the connection itself, which keycloak could not have written: the address and port come
|
||||
// from what the provider serves, and the user name from what the mesh decided both ends would
|
||||
// call this consumer (novox/hq 04-ISSUES/023).
|
||||
const connection = [...byId.values()].find((r) =>
|
||||
r.type === "file" && r.path === "/var/lib/keycloak/database.env");
|
||||
assert.ok(connection, "keycloak was given no database configuration");
|
||||
assert.match(connection.content, /KC_DB_USERNAME=mesh_[a-z0-9_]+_keycloak/,
|
||||
`keycloak was not told what name to present:\n${connection.content}`);
|
||||
assert.doesNotMatch(connection.content, /\$\{bound:/,
|
||||
`a placeholder reached the machine as a value:\n${connection.content}`);
|
||||
|
||||
// The password is the one hole left open, and the sealed value travels beside it. The mesh
|
||||
// discarded the plaintext, so the host is the only thing that can close it.
|
||||
assert.match(connection.content, /KC_DB_PASSWORD=\$\{secret:postgres-database\}/,
|
||||
`the password was not left for the host to fill:\n${connection.content}`);
|
||||
assert.ok(connection.secrets?.["postgres-database"],
|
||||
"the sealed credential did not travel with the file that needs it");
|
||||
assert.doesNotMatch(JSON.stringify(connection.content), /postgres-database":"[A-Za-z0-9+/]{24,}/,
|
||||
"the credential was written into the configuration in the clear");
|
||||
|
||||
// And the provider was told who asked, which is what its provisioner reconciles against.
|
||||
const grants = [...byId.values()].find((r) =>
|
||||
r.type === "file" && String(r.path).startsWith("/var/lib/postgres/grants"));
|
||||
assert.ok(grants, "postgres was never told which modules were granted a database");
|
||||
assert.match(JSON.stringify(grants), /keycloak|gitea/,
|
||||
"the grants file names neither module that asked for a database");
|
||||
|
||||
// Secrets reach containers as files, never as environment in the declaration.
|
||||
const containers = [...byId.values()].filter((r) => r.type === "container");
|
||||
assert.ok(containers.length >= 12,
|
||||
`only ${containers.length} containers; mailu alone is nine`);
|
||||
for (const c of containers) {
|
||||
for (const [key, value] of Object.entries(c.env ?? {})) {
|
||||
// **An absolute path is a reference to a secret, not a secret**, and naming one is the
|
||||
// whole design: the mesh delivers a credential as a file and a module says where.
|
||||
//
|
||||
// Excluded because `/` is in the base64 alphabet, so any path of 24 characters or more
|
||||
// matched — `MESH_BROKER_FILE=/var/lib/mesh/builder/broker` was reported as a credential
|
||||
// the broker would see. A check that fires on the right shape for the wrong reason is
|
||||
// worse than none: it is the one that gets suppressed, and then it is not there when it
|
||||
// is right.
|
||||
if (String(value).startsWith("/")) continue;
|
||||
assert.doesNotMatch(String(value), /^[A-Za-z0-9+/]{24,}={0,2}$/,
|
||||
`${c.name} carries something secret-shaped in env.${key}, which the broker would see`);
|
||||
}
|
||||
}
|
||||
|
||||
});
|
||||
|
||||
// The first of the real module descriptions to actually run.
|
||||
//
|
||||
// **Everything before this stopped at composing a declaration.** That proves the control plane and
|
||||
// the host agree, and proves nothing about whether the thing described works — which is how five
|
||||
// modules sat pinned to images that did not exist, parsing and resolving perfectly
|
||||
// (novox/hq 04-ISSUES/025).
|
||||
//
|
||||
// The forge is the one worth running first. It needs a database from another module, a password it
|
||||
// did not choose, and a connection string it could not have written itself: the address and port
|
||||
// come from what the database serves, and the user name from what the mesh decided both ends would
|
||||
// call it (04-ISSUES/022 and 023). If any of that is wrong it cannot start, and nothing else in
|
||||
// this file would notice.
|
||||
test("the forge runs, on a database the mesh gave it", { skip, timeout: 900_000 }, async () => {
|
||||
for (const name of ["postgres", "gitea"]) {
|
||||
const raw = readFileSync(`${process.env["MESH_LAB_MODULES"]}/${name}.json`, "utf8");
|
||||
// An image the mesh builds has no digest until it is built, and one it does not build belongs
|
||||
// to whichever registry served it. Only the first is rewritten; the second is pulled.
|
||||
const pinned = pinnedInto(raw, held);
|
||||
assert.deepEqual(stillUnpinned(pinned), [],
|
||||
`${name} still names an image nothing serves, so it could not start`);
|
||||
await must("anchor", `printf %s ${quote(pinned)} > /run-${name}.json`);
|
||||
await must("anchor", `docker cp /run-${name}.json mesh-controller:/run-${name}.json`);
|
||||
await mesh(`module add /run-${name}.json`);
|
||||
await mesh(`assign anchor ${name}`);
|
||||
}
|
||||
await mesh("push anchor", 300_000);
|
||||
|
||||
// **What the machine says it did, before asking what it produced.** This test pushed and then
|
||||
// waited for a database role, so when the containers were never created at all it reported "no
|
||||
// login was created" — true, and silent about the reason. A push that was accepted and an apply
|
||||
// that worked are different facts, and the second is the one this depends on.
|
||||
//
|
||||
// And waited for, because `push` sends without waiting. Reading `status` the instant it returns
|
||||
// describes the apply *before* this one, which is how this test came to report a missing
|
||||
// container while insisting the machine was fine.
|
||||
await settled("anchor");
|
||||
const running = (await on("anchor", `docker ps -a --format '{{.Names}} {{.Status}}'`)).out;
|
||||
// Named with what the mesh meant to send, not only with what the machine has. A container that
|
||||
// is absent because the mesh never asked for it and one that is absent because the machine could
|
||||
// not make it are the same sentence here and different faults entirely, and the plan is the only
|
||||
// thing that tells them apart.
|
||||
assert.match(running, /\bpostgres\b/,
|
||||
`the database module was pushed and no container for it exists:\n${running}\n\n` +
|
||||
`what the mesh would send anchor:\n${await mesh("plan anchor")}\n\n` +
|
||||
`${(await on("anchor", `tail -30 /var/log/mesh-host.log`)).out}`);
|
||||
|
||||
// The database first: until the provisioner has made the login, the forge has nothing to
|
||||
// connect to and its own start would prove only that it retries.
|
||||
const psql = async (q: string) =>
|
||||
(await on("anchor",
|
||||
`docker exec postgres psql -U postgres -qAt -c ${quote(q)}`, 60_000)).out.trim();
|
||||
|
||||
// Both halves in one poll. The provisioner makes the role and then the database, and a test
|
||||
// that waited for the first and checked the second once was racing the gap between two
|
||||
// statements — it lost, once, eighteen seconds into a run.
|
||||
let made = "";
|
||||
for (let i = 0; i < 40 && made !== "t"; i++) {
|
||||
made = await psql("select true from pg_roles where rolname = 'mesh_anchor_gitea'" +
|
||||
" and exists (select from pg_database where datname = 'gitea')");
|
||||
if (made !== "t") await new Promise((r) => setTimeout(r, 3000));
|
||||
}
|
||||
assert.equal(made, "t",
|
||||
`no login was created for the forge:\n${(await on("anchor", "docker logs mesh-provision-postgres 2>&1 | tail -20")).out}`);
|
||||
|
||||
// And the forge itself, answering. Not that its container exists — that it serves.
|
||||
//
|
||||
// On the port the mesh assigned, not the one the module declared (novox/hq ADR 0038): the
|
||||
// module says 3000 and the machine publishes wherever the mesh put it. Read from the plan,
|
||||
// because the plan is the same composition a push sends.
|
||||
const planned = await mesh("plan anchor --json", 120_000);
|
||||
const mapping = (JSON.parse(planned.slice(planned.indexOf("{"))).resources as any[])
|
||||
.find((r) => r.id === "gitea.server")?.ports
|
||||
?.map(String).find((p: string) => p.endsWith(":3000"));
|
||||
assert.ok(mapping, "the plan does not say where the machine publishes the forge");
|
||||
const at = mapping.split(":")[0];
|
||||
let answered = false;
|
||||
let said = { out: "", ok: false };
|
||||
for (let i = 0; i < 60 && !answered; i++) {
|
||||
said = await on("anchor", `curl -sf -o /dev/null -w '%{http_code}' --max-time 5 http://127.0.0.1:${at}/`, 30_000);
|
||||
answered = said.out.trim().startsWith("2") || said.out.trim() === "303";
|
||||
if (!answered) await new Promise((r) => setTimeout(r, 5000));
|
||||
}
|
||||
assert.ok(answered,
|
||||
`the forge never answered (last: ${said.out.trim()}):\n` +
|
||||
`${(await on("anchor", "docker logs gitea 2>&1 | tail -25")).out}`);
|
||||
|
||||
// **The credential actually worked.** A forge that started and could not reach its database
|
||||
// would still answer on its port, so the log is where the difference lives.
|
||||
const log = (await on("anchor", "docker logs gitea 2>&1 | tail -60")).out;
|
||||
assert.doesNotMatch(log, /password authentication failed|connection refused|does not exist/i,
|
||||
`the forge started and could not use the database it was given:\n${log}`);
|
||||
|
||||
await mesh("unassign anchor gitea");
|
||||
await mesh("unassign anchor postgres");
|
||||
await mesh("push anchor", 300_000);
|
||||
});
|
||||
|
||||
test("a consumer's cache grant means exactly its own keys", { skip, timeout: 600_000 }, async (t) => {
|
||||
// The third provision after a database and a bucket, and the first whose tenancy is enforced
|
||||
// by the store's own ACL rather than by separate namespaces: every consumer shares one
|
||||
// keyspace, so the grant is a pattern — and the test is that the pattern means what the
|
||||
// manifest said, in both directions.
|
||||
const raw = readFileSync(`${process.env["MESH_LAB_MODULES"]}/redis.json`, "utf8");
|
||||
const pinned = pinnedInto(raw, held);
|
||||
assert.deepEqual(stillUnpinned(pinned), [],
|
||||
"redis still names an image nothing serves, so it could not start");
|
||||
await must("anchor", `printf %s ${quote(pinned)} > /run-redis.json`);
|
||||
await must("anchor", `docker cp /run-redis.json mesh-controller:/run-redis.json`);
|
||||
await mesh("module add /run-redis.json");
|
||||
|
||||
// A consumer with no container: what is under test is the credential's reach, and files on the
|
||||
// machine are enough to prove it — the same reduction the first credential test makes.
|
||||
await must("anchor", `printf %s '{"module":"cachetest","version":"1",` +
|
||||
`"requires":["redis-cache"],` +
|
||||
`"contributes":{"redis-cache":{"prefix":"cachetest"}},` +
|
||||
`"binds":{"redis-cache":"/var/lib/cachetest/cache.json"},` +
|
||||
`"secrets":{"redis-cache":"/var/lib/cachetest/cache.secret"},` +
|
||||
`"resources":[{"id":"state","type":"directory","path":"/var/lib/cachetest","mode":"0700"}]}' ` +
|
||||
`> /cachetest.json`);
|
||||
await must("anchor", `docker cp /cachetest.json mesh-controller:/cachetest.json`);
|
||||
await mesh("module add /cachetest.json");
|
||||
|
||||
await mesh("assign anchor redis");
|
||||
await mesh("assign anchor cachetest");
|
||||
await mesh("push anchor", 300_000);
|
||||
await settled("anchor");
|
||||
|
||||
t.after(async () => {
|
||||
for (const name of ["cachetest", "redis"]) {
|
||||
await mesh(`unassign anchor ${name}`).catch(() => {});
|
||||
}
|
||||
await mesh("push anchor", 300_000).catch(() => {});
|
||||
});
|
||||
|
||||
// What the mesh told each end. The consumer's user name comes from its binding; the user's
|
||||
// password from the sealed file beside it — both written by the host, neither invented here.
|
||||
const bound = JSON.parse(await must("anchor", `cat /var/lib/cachetest/cache.json`));
|
||||
const user = bound.as;
|
||||
assert.ok(user?.startsWith("mesh_"), `the binding does not carry a usable user: ${user}`);
|
||||
const secret = (await must("anchor", `cat /var/lib/cachetest/cache.secret`)).trim();
|
||||
|
||||
// The provisioner has to have run before anything can authenticate. Waited for via the store
|
||||
// itself: the user list, asked with the server's own password, which the conf file the host
|
||||
// wrote holds on the machine.
|
||||
const admin = (await must("anchor",
|
||||
`awk '/^requirepass/ {print $2}' /var/lib/redis-module/redis.conf`)).trim();
|
||||
let granted = false;
|
||||
for (let i = 0; i < 40 && !granted; i++) {
|
||||
const users = (await on("anchor",
|
||||
`docker exec redis redis-cli --no-auth-warning -a ${quote(admin)} ACL USERS`)).out;
|
||||
granted = users.includes(user);
|
||||
if (!granted) await new Promise((r) => setTimeout(r, 3000));
|
||||
}
|
||||
assert.ok(granted, `no user was created for the consumer:
|
||||
` +
|
||||
`containers:\n${(await on("anchor", "docker ps -a --format '{{.Names}} {{.Status}}' | head -20")).out}\n` +
|
||||
`the store:\n${(await on("anchor", "docker logs redis 2>&1 | tail -15")).out}\n` +
|
||||
`the provisioner:\n${(await on("anchor", "docker logs mesh-provision-redis 2>&1 | tail -15")).out}`);
|
||||
|
||||
const asConsumer = (command: string) =>
|
||||
on("anchor", `docker exec redis redis-cli --no-auth-warning ` +
|
||||
`--user ${quote(user)} --pass ${quote(secret)} ${command}`);
|
||||
|
||||
// Its own keys: usable.
|
||||
assert.match((await asConsumer("SET cachetest:proof yes")).out, /OK/,
|
||||
"the consumer cannot write under the prefix it was granted");
|
||||
assert.match((await asConsumer("GET cachetest:proof")).out, /yes/,
|
||||
"the consumer cannot read back what it wrote");
|
||||
|
||||
// Anyone else's: refused by the store itself, which is the entire point of the grant.
|
||||
assert.match((await asConsumer("SET other:proof no")).out, /NOPERM|no permissions/i,
|
||||
"the consumer wrote outside its prefix — the grant means more than the manifest said");
|
||||
assert.match((await asConsumer("FLUSHALL")).out, /NOPERM|no permissions/i,
|
||||
"the consumer can flush the store, which no tenant may");
|
||||
});
|
||||
// A machine joining through the tunnel with the bus closed to it is its own bed:
|
||||
// joins-through-the-tunnel.test.ts (novox/hq ADR 0169).
|
||||
|
||||
@@ -109,7 +109,7 @@ const CONTROL_PLANE = { module: "mesh-controller", repo: "mesh-controller", path
|
||||
* the registry and the builder — are here too: they are carried in, and a mesh missing any of them
|
||||
* is not one.
|
||||
*/
|
||||
const MUST_HOLD = ["mesh-controller", "distribution", "builder", "mesh-tools", "postgres",
|
||||
const MUST_HOLD = ["mesh-controller", "distribution", "build-agent", "mesh-tools", "postgres",
|
||||
"mesh-catalog", "lavinmq", "mesh-vault", "amqp-ping"];
|
||||
const MUST_RUN = ["mesh-controller", "mesh-registry", "mesh-broker", "mesh-store",
|
||||
"mesh-postgres", "mesh-catalog", "mesh-lavinmq", "mesh-vault", "amqp-ping"];
|
||||
@@ -898,7 +898,7 @@ before(async () => {
|
||||
// does. novox/hq issue 050 — on a fresh mesh the modules built before the catalogue existed
|
||||
// are exactly the ones it needed in order to exist, so the hole is always the foundation.
|
||||
const missing = MUST_HOLD.filter((m) =>
|
||||
!["distribution", "builder"].includes(m) && !held.includes(m));
|
||||
!["distribution", "build-agent"].includes(m) && !held.includes(m));
|
||||
assert.deepEqual(missing, [],
|
||||
`the catalogue does not hold ${missing.join(", ")} — the mesh built them and its own ` +
|
||||
`record has no trace of it (novox/hq issue 050):\n${held}`);
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
/**
|
||||
* REPLAY R92 — genesis pushes to its own registry on a runtime nobody told to trust it (novox/hq issue 92).
|
||||
*
|
||||
* On the first real genesis the push of the control plane's image into the registry genesis had just
|
||||
* raised was refused: `Get "https://<address>/v2/": http: server gave HTTP response to HTTPS client`. The
|
||||
* registry speaks plain HTTP on purpose (ADR 0082), and the trust that lets a runtime speak to it was
|
||||
* written by the `docker` module, installed six steps after the push. The fix (mesh-host #91) has genesis
|
||||
* write that trust into the runtime's configuration itself, and confirm it through `docker info`, before
|
||||
* anything is pushed.
|
||||
*
|
||||
* **The lab hid it.** Its base image trusts the documentation ranges as plain-HTTP registries before any bed
|
||||
* starts (src/lifecycle/base.ts), so every genesis bed pushed to a registry its runtime already trusted. This
|
||||
* bed takes that trust back off the anchor — the base's file removed, the runtime restarted, and the runtime
|
||||
* asked that it now trusts loopback and nothing else — and then runs the installer as an operator does, with
|
||||
* `--registry 192.0.2.10:5000`.
|
||||
*
|
||||
* What it holds: genesis gets past its publish step, the registry it raised serves the control plane's image
|
||||
* it pushed by the address it was given, and the runtime trusts that registry by the mesh's name too, which
|
||||
* the builder pushes to later. Before the fix the push is refused, and the bed fails naming it.
|
||||
*
|
||||
* **A bed that could not reach the step it replays says so and does not fail**: a genesis stopped before
|
||||
* its registry step — a pull the internet refused, a build that broke — proves nothing about the order this
|
||||
* replays, and the test is skipped with the reason, which the prover reads as an error, never as the fault
|
||||
* reproduced. Genesis is run once and not asked to finish: the steps after the control plane read
|
||||
* repositories this bed does not serve, and are not what it replays.
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=<mesh-host>/mesh-host MESH_LAB_BOOTSTRAP_BINARY=<mesh-host>/mesh-bootstrap
|
||||
* MESH_LAB_BUNDLE=<mesh-host>/examples/foundation-first-node.lock (the installer refuses the bus one)
|
||||
* MESH_LAB_CATALOG=<mesh-catalog>/modules MESH_REPLAY_CONTROLLER=<a mesh-controller checkout>
|
||||
* MESH_LAB_KEEP=1 leave it standing
|
||||
*
|
||||
* Run by `go run ./replays/cmd/prove R92`, which builds all of that at the fix and at the commit before it.
|
||||
*/
|
||||
import { test, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { existsSync, mkdtempSync, rmSync } from "node:fs";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { loadScenario } from "../../src/declaration/parse.ts";
|
||||
import { raise } from "../../src/lifecycle/raise.ts";
|
||||
import { destroy, exec, push } from "../../src/lifecycle/operate.ts";
|
||||
import { bootstrapBinaryPath, hostBinaryPath } from "../../src/lifecycle/place.ts";
|
||||
import { labIsUsable, destroyAll, foundationBundle } from "./harness.ts";
|
||||
import { genesis, type GenesisResult } from "./genesis.ts";
|
||||
import { carry } from "./forge.ts";
|
||||
|
||||
const SCENARIO = "genesis-untrusted";
|
||||
const ANCHOR = "anchor";
|
||||
const FORGE = "forge";
|
||||
const ANCHOR_ADDRESS = "192.0.2.10";
|
||||
const FORGE_ADDRESS = "192.0.2.30";
|
||||
/** The registry as the operator gives it: the anchor's own address and the registry's port. */
|
||||
const REGISTRY = `${ANCHOR_ADDRESS}:5000`;
|
||||
/** And as the mesh names it: the node, the mesh's internal suffix, the same port. */
|
||||
const REGISTRY_BY_NAME = `${ANCHOR}.internal:5000`;
|
||||
/** The installer's steps up to and including the one this replays, in its order. */
|
||||
const UP_TO_PUBLISH = ["preflight", "load", "build", "bundle", "apply", "verify", "enrol", "operator", "registry", "publish"];
|
||||
/** The steps before the registry's: a stop there never reached what this replays. */
|
||||
const BEFORE_REGISTRY = UP_TO_PUBLISH.slice(0, UP_TO_PUBLISH.indexOf("registry"));
|
||||
|
||||
const capability = await labIsUsable();
|
||||
const binary = hostBinaryPath();
|
||||
const installer = bootstrapBinaryPath();
|
||||
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
||||
const catalogDir = process.env["MESH_LAB_CATALOG"] ?? "";
|
||||
const controller = process.env["MESH_REPLAY_CONTROLLER"] ?? "";
|
||||
const KEEP = !!process.env["MESH_LAB_KEEP"];
|
||||
const FIXED_ID = process.env["MESH_LAB_INSTANCE_ID"] ?? (KEEP ? "genesis-untrusted-live" : undefined);
|
||||
|
||||
const skip =
|
||||
!capability.usable ? capability.why :
|
||||
!binary ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" :
|
||||
!installer ? "MESH_LAB_BOOTSTRAP_BINARY is not set to a built mesh-bootstrap" :
|
||||
!bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation template" :
|
||||
!catalogDir || !existsSync(catalogDir) ? "MESH_LAB_CATALOG is not set to mesh-catalog/modules" :
|
||||
!controller || !existsSync(resolve(controller, ".git")) ? "MESH_REPLAY_CONTROLLER is not set to a mesh-controller checkout" :
|
||||
false;
|
||||
|
||||
let instanceId = "";
|
||||
/** Why the bed never reached the step it replays, or "" when it did. */
|
||||
let unreached = "";
|
||||
let result: GenesisResult | undefined;
|
||||
let untrusted = "";
|
||||
const report: string[] = [];
|
||||
|
||||
async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
||||
const { stdout } = await exec(instanceId, machine, ["sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`], timeoutMs);
|
||||
const marker = stdout.lastIndexOf("__exit=");
|
||||
if (marker < 0) return { out: stdout, ok: false };
|
||||
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
||||
}
|
||||
async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
|
||||
const { out, ok } = await on(machine, command, timeoutMs);
|
||||
if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The control plane's repository at the checkout's HEAD, served from the forge machine over git's protocol. */
|
||||
async function raiseForge(): Promise<string> {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mesh-lab-r92-forge-"));
|
||||
let head: string;
|
||||
try {
|
||||
head = carry(controller, join(dir, "mesh-controller.git"));
|
||||
const tar = join(tmpdir(), `mesh-lab-r92-forge-${process.pid}.tar`);
|
||||
execFileSync("tar", ["-cf", tar, "-C", dir, "."]);
|
||||
await push(instanceId, FORGE, tar, "/tmp/forge.tar");
|
||||
rmSync(tar, { force: true });
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
// Owned by the daemon's user, or git refuses to serve it ("dubious ownership") and says nothing.
|
||||
await must(FORGE, `mkdir -p /srv/git && tar --no-same-owner -xf /tmp/forge.tar -C /srv/git && chown -R root:root /srv/git && ` +
|
||||
`git daemon --base-path=/srv/git --export-all --reuseaddr --detach --listen=${FORGE_ADDRESS} --pid-file=/run/git-daemon.pid`);
|
||||
for (let i = 0; ; i++) {
|
||||
if ((await on(ANCHOR, `git ls-remote git://${FORGE_ADDRESS}/mesh-controller.git main`)).ok) break;
|
||||
if (i >= 30) throw new Error(`the anchor cannot reach the forge at git://${FORGE_ADDRESS}/`);
|
||||
await new Promise((r) => setTimeout(r, 2_000));
|
||||
}
|
||||
return head;
|
||||
}
|
||||
|
||||
/**
|
||||
* The anchor's runtime made a bare machine's: the base's trust of the documentation ranges taken off, and
|
||||
* the runtime asked that it trusts no registry but loopback. Before genesis, so nothing runs that a restart
|
||||
* would stop. Answers what the runtime said.
|
||||
*/
|
||||
async function untrust(): Promise<string> {
|
||||
await must(ANCHOR, `rm -f /etc/docker/daemon.json && systemctl restart docker`, 180_000);
|
||||
for (let i = 0; ; i++) {
|
||||
if ((await on(ANCHOR, `docker info >/dev/null`)).ok) break;
|
||||
if (i >= 60) throw new Error("the anchor's runtime did not answer after its trust was taken off");
|
||||
await new Promise((r) => setTimeout(r, 2_000));
|
||||
}
|
||||
const config = (await must(ANCHOR, `docker info --format '{{json .RegistryConfig}}'`)).trim();
|
||||
const parsed = JSON.parse(config) as { InsecureRegistryCIDRs?: string[]; IndexConfigs?: Record<string, { Secure?: boolean }> };
|
||||
const cidrs = parsed.InsecureRegistryCIDRs ?? [];
|
||||
const insecureNames = Object.entries(parsed.IndexConfigs ?? {}).filter(([, c]) => c.Secure === false).map(([n]) => n);
|
||||
if (cidrs.some((c) => c !== "127.0.0.0/8" && c !== "::1/128") || insecureNames.length > 0) {
|
||||
throw new Error(`the anchor's runtime still trusts more than loopback after the base's trust was taken off: ${config}`);
|
||||
}
|
||||
return `trusts ${cidrs.join(", ") || "nothing"} as plain-HTTP registries, and no registry by name`;
|
||||
}
|
||||
|
||||
before(async () => {
|
||||
if (skip) return;
|
||||
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
||||
onProgress: (m) => console.log(`raise: ${m}`),
|
||||
...(FIXED_ID ? { instanceId: FIXED_ID } : {}),
|
||||
});
|
||||
instanceId = raised.instanceId;
|
||||
console.log(`INSTANCE ${instanceId}${KEEP ? " (KEEP — will be left standing)" : ""}`);
|
||||
|
||||
let head = "";
|
||||
try {
|
||||
head = await raiseForge();
|
||||
report.push(` forge git://${FORGE_ADDRESS}/mesh-controller.git at ${head.slice(0, 8)}`);
|
||||
untrusted = await untrust();
|
||||
report.push(` anchor's runtime ${untrusted}`);
|
||||
} catch (err) {
|
||||
unreached = `the bed was not prepared, so the installer never ran: ${(err as Error).message}`;
|
||||
return;
|
||||
}
|
||||
|
||||
const forge = (repo: string) => `git://${FORGE_ADDRESS}/${repo}.git`;
|
||||
try {
|
||||
result = await genesis({
|
||||
instanceId,
|
||||
node: ANCHOR,
|
||||
installer: installer as string,
|
||||
catalogDir,
|
||||
// The registry's manifest is all the steps this replays read from the catalogue.
|
||||
catalogueModules: ["distribution"],
|
||||
bundleTemplate: foundationBundle(bundle, []),
|
||||
registry: REGISTRY,
|
||||
source: forge("mesh-controller"),
|
||||
sourceRef: head,
|
||||
// Named because the installer requires them; served by nobody, since the steps that clone them
|
||||
// come after the one replayed here.
|
||||
toolsSource: forge("mesh-tools"),
|
||||
sdkSource: forge("mesh-sdk"),
|
||||
catalogSource: forge("mesh-catalog"),
|
||||
attempts: 1,
|
||||
verify: false,
|
||||
log: (m) => console.log(m),
|
||||
});
|
||||
} catch (err) {
|
||||
unreached = `the installer could not be run: ${(err as Error).message}`;
|
||||
return;
|
||||
}
|
||||
report.push(...result.report);
|
||||
const stoppedAt = result.step.replace(/^step \d+ of \d+, /, "");
|
||||
if (!result.ok && (BEFORE_REGISTRY.includes(stoppedAt) || !result.step)) {
|
||||
unreached = `genesis stopped at ${result.step || "no step it named"}, before its registry step, so the order ` +
|
||||
`this replays was never reached:\n${result.why}`;
|
||||
}
|
||||
}, { timeout: 3_600_000 });
|
||||
|
||||
after(async () => {
|
||||
if (KEEP) {
|
||||
console.log(`\nLEFT STANDING: ${instanceId} — not destroyed (MESH_LAB_KEEP).`);
|
||||
return;
|
||||
}
|
||||
if (instanceId) await destroy(instanceId);
|
||||
await destroyAll(`${SCENARIO}-`);
|
||||
}, { timeout: 900_000 });
|
||||
|
||||
test("genesis pushes to the registry it raised on a runtime nobody told to trust it", {
|
||||
skip, timeout: 3_600_000,
|
||||
}, async (t) => {
|
||||
console.log(report.join("\n"));
|
||||
if (unreached) {
|
||||
t.skip(`R92 NOT REACHED: ${unreached}`);
|
||||
return;
|
||||
}
|
||||
const ran = result as GenesisResult;
|
||||
const stoppedAt = ran.step.replace(/^step \d+ of \d+, /, "");
|
||||
|
||||
// The incident's own words, and the step it stopped at.
|
||||
assert.doesNotMatch(ran.said, /server gave HTTP response to HTTPS client/,
|
||||
`the runtime refused to push to ${REGISTRY} over plain HTTP — the registry was not trusted before the push ` +
|
||||
`(stopped at ${ran.step || "no step"}):\n${ran.why}`);
|
||||
assert.ok(ran.ok || !UP_TO_PUBLISH.includes(stoppedAt),
|
||||
`genesis stopped at ${ran.step}, before its control plane was published to ${REGISTRY}:\n${ran.why}`);
|
||||
|
||||
// The registry agreeing: the image genesis pushed, by the address it was given.
|
||||
const tags = (await on(ANCHOR, `curl -s --max-time 10 http://${REGISTRY}/v2/mesh-controller/tags/list`)).out.trim();
|
||||
assert.match(tags, /"genesis"/, `${REGISTRY} does not serve the control plane genesis pushed: ${tags || "no answer"}`);
|
||||
|
||||
// And the runtime trusts the registry by the mesh's name as well: the builder pushes there later.
|
||||
const config = (await must(ANCHOR, `docker info --format '{{json .RegistryConfig.IndexConfigs}}'`)).trim();
|
||||
const byName = (JSON.parse(config) as Record<string, { Secure?: boolean }>)[REGISTRY_BY_NAME];
|
||||
assert.ok(byName && byName.Secure === false,
|
||||
`the runtime does not trust ${REGISTRY_BY_NAME}, the name the mesh pushes to its registry by: ${config}`);
|
||||
});
|
||||
@@ -0,0 +1,108 @@
|
||||
/**
|
||||
* Research 053, question 8: the bus on signed logins (NATS operator mode), over a real network.
|
||||
*
|
||||
* Three machines on one segment: `a` is today's style server (static users, an account with
|
||||
* JetStream), `b` and `c` are a two-server cluster in operator mode with a full resolver, and `a` is
|
||||
* joined to `b` as a leaf node, the system account bridged. The measurements run in
|
||||
* test/bed-programs/signed-logins (one Go program; it embeds nats-server 2.11.17, the mesh's
|
||||
* version, so nothing is downloaded into the machines). It runs on `a` and reaches `b` and `c`
|
||||
* across the segment:
|
||||
*
|
||||
* part 2 revocation: time for b and c to refuse a revoked user and close its open connection
|
||||
* part 4 moving a stream and a KV bucket from a to b over the leaf link, with a writer running
|
||||
* part 3 c cut off from b by a packet filter on c while a user is revoked, then restored:
|
||||
* how long c still accepts the user with the periodic sync alone, and with a re-push
|
||||
* triggered by the first system-account event that shows c is back
|
||||
*
|
||||
* The test asserts only that the measurements ran to the end; the numbers are the result and are
|
||||
* printed. It proves nothing about the mesh's own software, which it does not use.
|
||||
*/
|
||||
|
||||
import { test, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { loadScenario } from "../../src/declaration/parse.ts";
|
||||
import { raise } from "../../src/lifecycle/raise.ts";
|
||||
import { destroy, exec, push } from "../../src/lifecycle/operate.ts";
|
||||
import { labIsUsable } from "./harness.ts";
|
||||
|
||||
const capability = await labIsUsable();
|
||||
const skip = capability.usable ? false : `lab not usable: ${capability.why}`;
|
||||
|
||||
const ADDR = { a: "192.0.2.11", b: "192.0.2.12", c: "192.0.2.13" };
|
||||
let instanceId = "";
|
||||
|
||||
before(async () => {
|
||||
if (skip) return;
|
||||
instanceId = (await raise(loadScenario("scenarios/signed-logins.yml"), {})).instanceId;
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
if (instanceId && !process.env["MESH_LAB_KEEP"]) await destroy(instanceId);
|
||||
});
|
||||
|
||||
const sh = (machine: string, script: string, timeout = 120_000) =>
|
||||
exec(instanceId, machine, ["sh", "-c", script], timeout);
|
||||
|
||||
test("research 053 q8: signed-login bus over a real network", { skip, timeout: 3_000_000 }, async () => {
|
||||
const work = mkdtempSync(join(tmpdir(), "q8lab-"));
|
||||
const bin = join(work, "q8lab");
|
||||
execFileSync("go", ["build", "-o", bin, "."], {
|
||||
cwd: "test/bed-programs/signed-logins",
|
||||
env: { ...process.env, CGO_ENABLED: "0", GOOS: "linux", GOARCH: "amd64", GOFLAGS: "-mod=readonly" },
|
||||
stdio: "inherit",
|
||||
});
|
||||
const conf = join(work, "conf");
|
||||
mkdirSync(conf);
|
||||
execFileSync(bin, ["gen", "-out", conf, "-a", ADDR.a, "-b", ADDR.b, "-c", ADDR.c], { stdio: "inherit" });
|
||||
|
||||
for (const m of ["a", "b", "c"]) {
|
||||
await sh(m, "mkdir -p /root/q8 /var/lib/q8");
|
||||
await push(instanceId, m, bin, "/root/q8/q8lab", "0755");
|
||||
}
|
||||
const files: Record<string, string[]> = {
|
||||
a: ["A.conf", "leaf-mesh.creds", "leaf-sys.creds", "keys.json"],
|
||||
b: ["B.conf", "op.jwt"],
|
||||
c: ["C.conf", "op.jwt"],
|
||||
};
|
||||
for (const [m, list] of Object.entries(files)) {
|
||||
for (const f of list) await push(instanceId, m, join(conf, f), `/root/q8/${f}`, "0600");
|
||||
}
|
||||
|
||||
const start = (m: string, cmd: string, out: string) =>
|
||||
sh(m, `setsid nohup ${cmd} >/root/q8/${out} 2>&1 </dev/null & sleep 1; echo started`);
|
||||
await start("b", "/root/q8/q8lab server -conf /root/q8/B.conf", "B.out");
|
||||
await start("c", "/root/q8/q8lab server -conf /root/q8/C.conf", "C.out");
|
||||
for (const m of ["b", "c"]) {
|
||||
const r = await sh(m, "/root/q8/q8lab wait -url http://127.0.0.1:8222/healthz -s 90");
|
||||
assert.match(r.stdout, /up/, `${m} server did not come up`);
|
||||
}
|
||||
await start("a", "/root/q8/q8lab server -conf /root/q8/A.conf", "A.out");
|
||||
assert.match((await sh("a", "/root/q8/q8lab wait -url http://127.0.0.1:8222/healthz -s 60")).stdout, /up/);
|
||||
await start("c", `/root/q8/q8lab agent -peer ${ADDR.b}`, "agent.out");
|
||||
console.log("servers up:", (await sh("a", "cat /root/q8/A.out")).stdout.trim());
|
||||
|
||||
await sh("a", "setsid nohup sh -c '/root/q8/q8lab drive -keys /root/q8/keys.json > /root/q8/drive.log 2>&1; echo finished > /root/q8/drive.done' >/dev/null 2>&1 </dev/null & sleep 1; echo started");
|
||||
let offset = 0;
|
||||
let finished = false;
|
||||
let all = "";
|
||||
for (let i = 0; i < 400 && !finished; i++) {
|
||||
await new Promise((r) => setTimeout(r, 5000));
|
||||
const chunk = (await sh("a", `tail -c +${offset + 1} /root/q8/drive.log`)).stdout;
|
||||
offset += Buffer.byteLength(chunk);
|
||||
finished = (await sh("a", "test -f /root/q8/drive.done && echo yes; true")).stdout.includes("yes");
|
||||
if (chunk.trim()) { console.log(chunk.trimEnd()); all += chunk; }
|
||||
}
|
||||
for (const m of ["a", "b", "c"]) {
|
||||
const up = m.toUpperCase();
|
||||
const r = await sh(m, `grep -c '\\[ERR\\]' /var/log/q8-${up}.log; grep '\\[ERR\\]' /var/log/q8-${up}.log | cut -c1-220 | sort | uniq -c | sort -rn | head -8; true`);
|
||||
console.log(`--- server ${up} log, error lines:\n${r.stdout.trim()}`);
|
||||
}
|
||||
writeFileSync(join(work, "drive.log"), all);
|
||||
assert.ok(finished, "the measurements did not finish");
|
||||
assert.match(all, /DRIVE-DONE/);
|
||||
assert.doesNotMatch(all, /panic:/);
|
||||
});
|
||||
@@ -0,0 +1,198 @@
|
||||
/**
|
||||
* **Nothing a machine sends while the store restarts is lost** (novox/hq issues 082, 083).
|
||||
*
|
||||
* The foundation's store is recreated when it is adopted, and for those seconds the control plane
|
||||
* cannot write. This bed takes the store away on the control-node, has a second machine enrol into
|
||||
* the gap, and brings the store back after the control plane has had to say "not now":
|
||||
*
|
||||
* - a report arriving in the gap is held, and recorded when the store is back;
|
||||
* - the enrolment is answered "not now" while that report is held — not queued behind it — and
|
||||
* completes on its own when the store is back, with the keys it started with: the mesh holds
|
||||
* the very keys the machine generated;
|
||||
* - the machine then applies what it is pushed and is heard from.
|
||||
*
|
||||
* It needs a host binary and the foundation bundle:
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host
|
||||
* MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||
*/
|
||||
import { test, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { existsSync } from "node:fs";
|
||||
import { loadScenario } from "../../src/declaration/parse.ts";
|
||||
import { raise } from "../../src/lifecycle/raise.ts";
|
||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||||
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
|
||||
import type { HeldImage } from "../../src/pinning.ts";
|
||||
|
||||
const capability = await labIsUsable();
|
||||
const binary = hostBinaryPath();
|
||||
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
||||
|
||||
const skip = !capability.usable
|
||||
? `lab not usable: ${capability.why}`
|
||||
: !binary || !existsSync(binary)
|
||||
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
||||
: !bundle || !existsSync(bundle)
|
||||
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
||||
: false;
|
||||
|
||||
const SCENARIO = "store-window";
|
||||
let instanceId = "";
|
||||
let held: HeldImage[] = [];
|
||||
|
||||
function quote(s: string): string {
|
||||
return `'${s.replaceAll("'", `'\\''`)}'`;
|
||||
}
|
||||
|
||||
async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
||||
const { stdout } = await exec(instanceId, machine, [
|
||||
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
|
||||
], timeoutMs);
|
||||
const marker = stdout.lastIndexOf("__exit=");
|
||||
if (marker < 0) return { out: stdout, ok: false };
|
||||
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
||||
}
|
||||
|
||||
async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
|
||||
const { out, ok } = await on(machine, command, timeoutMs);
|
||||
if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
|
||||
return out;
|
||||
}
|
||||
|
||||
async function mesh(command: string, timeoutMs?: number): Promise<string> {
|
||||
return must("anchor", `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
|
||||
}
|
||||
|
||||
function tokenFrom(said: string): string {
|
||||
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
|
||||
assert.ok(found, `no token in:\n${said}`);
|
||||
return found;
|
||||
}
|
||||
|
||||
async function until(what: string, within: number, check: () => Promise<boolean>, why: () => Promise<string>): Promise<void> {
|
||||
const end = Date.now() + within;
|
||||
while (Date.now() < end) {
|
||||
if (await check()) return;
|
||||
await new Promise((r) => setTimeout(r, 3000));
|
||||
}
|
||||
assert.fail(`${what} did not happen within ${Math.round(within / 1000)}s:\n${await why()}`);
|
||||
}
|
||||
|
||||
before(async () => {
|
||||
if (skip) return;
|
||||
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
||||
onProgress: (m) => console.log(`raise: ${m}`),
|
||||
});
|
||||
instanceId = raised.instanceId;
|
||||
held = raised.images;
|
||||
|
||||
await must("anchor", `cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${foundationBundle(bundle, raised.images)}\nMESHBUNDLE`);
|
||||
await must("anchor", `${HOST_PATH} apply /tmp/foundation.lock`, 600_000);
|
||||
const up = await must("anchor", `docker ps --format '{{.Names}}'`);
|
||||
for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) {
|
||||
assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`);
|
||||
}
|
||||
await mesh("node add anchor");
|
||||
const own = tokenFrom(await mesh("token issue --node anchor"));
|
||||
await must("anchor", `${HOST_PATH} enrol --token ${quote(own)}`);
|
||||
await must("anchor", `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
|
||||
}, { timeout: 1_800_000 });
|
||||
|
||||
after(async () => {
|
||||
if (process.env["MESH_LAB_KEEP"]) { console.log(`MESH_LAB_KEEP set — leaving ${instanceId} standing`); return; }
|
||||
if (instanceId) await destroy(instanceId);
|
||||
await destroyAll(`${SCENARIO}-`);
|
||||
}, { timeout: 600_000 });
|
||||
|
||||
test("a machine enrolling while the store is away joins when it comes back, with the keys it started with", {
|
||||
skip, timeout: 900_000,
|
||||
}, async () => {
|
||||
await mesh("node add laptop");
|
||||
const token = tokenFrom(await mesh("token issue --node laptop"));
|
||||
|
||||
// A report that will arrive in the gap: the anchor is pushed a step that takes ten seconds, so
|
||||
// its report lands after the store has gone.
|
||||
const sleeper = onTheMachine("alpine", held);
|
||||
await must("anchor", `printf %s '{"module":"slow","version":"1","resources":[` +
|
||||
`{"id":"step","type":"container","name":"slow-step","image":"${sleeper}","run-once":true,` +
|
||||
`"args":["sh","-c","sleep 10"]}]}' > /tmp/slow.json`);
|
||||
await must("anchor", `docker cp /tmp/slow.json mesh-controller:/slow.json`);
|
||||
await mesh("module add /slow.json");
|
||||
await mesh("assign anchor slow");
|
||||
await mesh("push anchor");
|
||||
|
||||
// The store goes away, as it does when the foundation is adopted; the broker stays, so the
|
||||
// report and the enrolment reach the control plane and the control plane cannot write.
|
||||
await must("anchor", `docker stop mesh-store`);
|
||||
await until("the anchor's report arriving in the gap and being held", 120_000,
|
||||
async () => /could not keep anchor's report .*holding it/.test((await on("anchor", `docker logs mesh-controller 2>&1`)).out),
|
||||
async () => `--- controller ---\n${(await on("anchor", `docker logs --tail 30 mesh-controller 2>&1`)).out}\n` +
|
||||
`--- anchor host ---\n${(await on("anchor", `tail -10 /var/log/mesh-host.log`)).out}`);
|
||||
|
||||
// The machine enrols into the gap. In the background: it will be told "not now" and keep asking.
|
||||
await must("laptop", `nohup sh -c ${quote(`${HOST_PATH} enrol --token ${quote(token)} > /tmp/enrol.log 2>&1; ` +
|
||||
`echo "exit=$?" >> /tmp/enrol.log`)} > /dev/null 2>&1 & sleep 1`);
|
||||
|
||||
// The control plane said "not now" at least once, while the anchor's report was held — so the
|
||||
// gap was hit, and the enrolment was answered rather than queued behind what the store owed.
|
||||
await until("the control plane asking the machine to enrol again", 90_000,
|
||||
async () => /asked "laptop" to enrol again/.test((await on("anchor", `docker logs mesh-controller 2>&1`)).out),
|
||||
async () => `--- controller ---\n${(await on("anchor", `docker logs --tail 30 mesh-controller 2>&1`)).out}\n` +
|
||||
`--- laptop enrol ---\n${(await on("laptop", `cat /tmp/enrol.log`)).out}`);
|
||||
|
||||
// The store comes back; the enrolment completes on its own, with nothing done by hand.
|
||||
await must("anchor", `docker start mesh-store`);
|
||||
await until("the enrolment completing", 150_000,
|
||||
async () => /exit=/.test((await on("laptop", `cat /tmp/enrol.log`)).out),
|
||||
async () => `--- laptop enrol ---\n${(await on("laptop", `cat /tmp/enrol.log`)).out}\n` +
|
||||
`--- controller ---\n${(await on("anchor", `docker logs --tail 30 mesh-controller 2>&1`)).out}`);
|
||||
const said = (await on("laptop", `cat /tmp/enrol.log`)).out;
|
||||
assert.match(said, /exit=0/, `the enrolment did not complete after the store came back:\n${said}`);
|
||||
assert.match(said, /enrolled as laptop/, `the machine was not enrolled as laptop:\n${said}`);
|
||||
|
||||
// The mesh holds the very keys the machine generated at the start: its identity is the live key,
|
||||
// and its sealing key is the one on its record.
|
||||
const identity = said.match(/generated this node's identity: (\S+)/)?.[1];
|
||||
const sealing = said.match(/generated this node's sealing key:\s+(\S+)/)?.[1];
|
||||
assert.ok(identity && sealing, `the enrolment did not say which keys it generated:\n${said}`);
|
||||
const nodeId = (await must("anchor", `docker exec mesh-store psql -U postgres -d inventory -qAt ` +
|
||||
`-c "select id from node where name = 'laptop'"`)).trim();
|
||||
const live = (await must("anchor", `docker exec mesh-store psql -U postgres -d identity -qAt ` +
|
||||
`-c "select encode(public, 'base64') from node_key where node = '${nodeId}' and revoked is null"`)).trim();
|
||||
assert.equal(live, identity, `the mesh's live key for laptop is not the one it generated`);
|
||||
const sealedTo = (await must("anchor", `docker exec mesh-store psql -U postgres -d inventory -qAt ` +
|
||||
`-c "select sealing_key from node where name = 'laptop'"`)).trim();
|
||||
assert.equal(sealedTo, sealing, `the mesh's sealing key for laptop is not the one it generated`);
|
||||
|
||||
// The report held through the gap was recorded once the store was back.
|
||||
await until("the anchor's held report being recorded", 60_000,
|
||||
async () => {
|
||||
const r = await on("anchor", `docker exec mesh-controller /mesh-controller status --json`);
|
||||
if (!r.ok) return false;
|
||||
const state = JSON.parse(r.out) as { reported: { node: string; outcome: string; current: boolean }[] };
|
||||
return state.reported.some((w) => w.node === "anchor" && w.outcome === "applied" && w.current);
|
||||
},
|
||||
async () => (await on("anchor", `docker logs --tail 20 mesh-controller 2>&1`)).out);
|
||||
|
||||
// And the machine is a working member: pushed something, it applies it and is heard from.
|
||||
await must("laptop", `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
|
||||
await must("anchor", `printf %s '{"module":"marker","version":"1","resources":[` +
|
||||
`{"id":"marker","type":"file","path":"/etc/store-window","content":"joined\\\\n","mode":"0644"}]}' > /tmp/marker.json`);
|
||||
await must("anchor", `docker cp /tmp/marker.json mesh-controller:/marker.json`);
|
||||
await mesh("module add /marker.json");
|
||||
await mesh("assign laptop marker");
|
||||
await mesh("push laptop");
|
||||
await until("the machine applying what it was pushed", 120_000,
|
||||
async () => (await on("laptop", `grep -q joined /etc/store-window`)).ok,
|
||||
async () => (await on("laptop", `tail -20 /var/log/mesh-host.log`)).out);
|
||||
await until("the mesh hearing the machine's report", 120_000,
|
||||
async () => {
|
||||
const r = await on("anchor", `docker exec mesh-controller /mesh-controller status --json`);
|
||||
if (!r.ok) return false;
|
||||
const state = JSON.parse(r.out) as { reported: { node: string; outcome: string; current: boolean }[] };
|
||||
return state.reported.some((w) => w.node === "laptop" && w.outcome === "applied" && w.current);
|
||||
},
|
||||
async () => (await on("anchor", `docker exec mesh-controller /mesh-controller status 2>&1`)).out);
|
||||
});
|
||||
@@ -4,7 +4,7 @@
|
||||
* whole-mesh-novox.test.ts; same harness and topology, a larger, media-heavy module set.
|
||||
*
|
||||
* Foundation (store, broker, control) rides `anchor` and nothing else; ALL of ace's services ride the
|
||||
* `ace` node. An overlay is placed so the two DB consumers (baserow, letta) reach the postgres/redis
|
||||
* `ace` node. An overlay is placed so the two DB consumers (baserow, letta) reach the postgres
|
||||
* providers co-located with them. The media stack shares the operator-owned library directories
|
||||
* under /services/media (ADR 0051 `accesses`): the mesh writes an `access` resource that CONFIRMS
|
||||
* each path exists and mounts it, but creates and chowns none of it — so before() pre-creates those
|
||||
@@ -264,7 +264,7 @@ test("the whole ace service set resolves, installs and converges on one node in
|
||||
}, async () => {
|
||||
for (const d of DROPPED) console.log(`DROPPED ${d.name}: ${d.why}`);
|
||||
|
||||
// The overlay, so a consumer's binding `at` is non-empty (baserow/letta reach postgres/redis).
|
||||
// The overlay, so a consumer's binding `at` is non-empty (baserow/letta reach postgres).
|
||||
await mesh("overlay place anchor --hub --endpoint 192.0.2.10:51820 --site lab");
|
||||
await mesh(`overlay place ${NODE} --site lab`);
|
||||
await mesh("assign anchor networking");
|
||||
|
||||
@@ -119,7 +119,7 @@ const CATALOGUE_ON_MACHINE = "/opt/mesh-catalog";
|
||||
* a third, this list is where the bed finds out, by the installer saying which manifest it could
|
||||
* not read.
|
||||
*/
|
||||
const CATALOGUE_MODULES = ["distribution", "builder"];
|
||||
const CATALOGUE_MODULES = ["distribution", "build-agent"];
|
||||
|
||||
/**
|
||||
* Where this mesh keeps its own images, as the anchor reaches it.
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
// A run in which no test passed has proven nothing, and says so (novox/hq issue 693).
|
||||
//
|
||||
// Seen on the lab: the genesis bed stopped before its subject, skipped its only test as "NOT REACHED",
|
||||
// and the suite printed "0 passed, 0 failed" and exited 0, so the lab recorded the run as passed.
|
||||
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { judge, type Receipt, endToEnd } from "../src/lastrun.ts";
|
||||
import { judgeRun, oneLine, type Skip } from "../src/outcomes.ts";
|
||||
import { counted, exitOf, reportOn } from "../src/suite.ts";
|
||||
|
||||
// The suite as a person runs it: not as a child of this test runner, which would have it report to us.
|
||||
const suiteEnv = (state: string): NodeJS.ProcessEnv => {
|
||||
const env: NodeJS.ProcessEnv = { ...process.env, XDG_STATE_HOME: state, NO_COLOR: "1" };
|
||||
delete env["NODE_TEST_CONTEXT"];
|
||||
return env;
|
||||
};
|
||||
|
||||
const lab = fileURLToPath(new URL("..", import.meta.url));
|
||||
const unreached: Skip = {
|
||||
file: "test/integration/genesis-keeps-its-bus.test.ts",
|
||||
test: "genesis adopts its bus",
|
||||
reason: "ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base, before the broker step:\n docker exited 1",
|
||||
};
|
||||
const noBed: Skip = { file: "test/integration/mesh.test.ts", test: "a builder builds", reason: "set MESH_LAB_BUILDER to a built mesh-builder" };
|
||||
|
||||
test("a run whose only test skipped did not pass", () => {
|
||||
const said = judgeRun({ passed: 0, failed: 0 }, [unreached]);
|
||||
assert.equal(said.passed, false);
|
||||
assert.match(said.why, /no test passed: 1 skipped, 1 of them as NOT REACHED/);
|
||||
});
|
||||
|
||||
test("a run that ran no test at all did not pass", () => {
|
||||
assert.deepEqual(judgeRun({ passed: 0, failed: 0 }, []), { passed: false, why: "no test passed: none ran" });
|
||||
});
|
||||
|
||||
test("a NOT REACHED skip beside tests that passed is still not passed", () => {
|
||||
const said = judgeRun({ passed: 3, failed: 0 }, [unreached]);
|
||||
assert.equal(said.passed, false);
|
||||
assert.match(said.why, /1 test\(s\) skipped as NOT REACHED/);
|
||||
});
|
||||
|
||||
// The design limit: a skip its file declares expected, beside tests that passed, leaves the run passed.
|
||||
test("an expected skip beside tests that passed leaves the run passed", () => {
|
||||
assert.deepEqual(judgeRun({ passed: 21, failed: 0 }, [noBed]), { passed: true, why: "" });
|
||||
});
|
||||
|
||||
test("the suite's exit is not 0 for a run that did not pass, and is the runner's otherwise", () => {
|
||||
assert.equal(exitOf(0, { passed: 0, failed: 0 }, [unreached]), 1);
|
||||
assert.equal(exitOf(0, { passed: 21, failed: 0 }, [noBed]), 0);
|
||||
assert.equal(exitOf(1, { passed: 20, failed: 1 }, []), 1);
|
||||
assert.equal(exitOf(0, { passed: null, failed: null }, []), 0, "unread totals are reportOn's case, not this one");
|
||||
});
|
||||
|
||||
test("the recorded line says what skipped and why, and NOT PASSED", () => {
|
||||
let kept: unknown;
|
||||
const said = reportOn({ passed: 0, failed: 0, skipped: 1 }, [unreached], (_p, _f, outcome) => {
|
||||
kept = outcome;
|
||||
return { against: { "mesh-lab": "7ed7da1" }, ran: [unreached.file] };
|
||||
});
|
||||
assert.match(said, /— 0 passed, 0 failed, 1 skipped, against mesh-lab 7ed7da1/);
|
||||
assert.match(said, /^NOT PASSED: no test passed: 1 skipped, 1 of them as NOT REACHED$/m);
|
||||
assert.match(said, /^skipped: genesis adopts its bus \(test\/integration\/genesis-keeps-its-bus\.test\.ts\) # ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base, before the broker step: docker exited 1$/m);
|
||||
assert.deepEqual(kept, {
|
||||
skipped: 1,
|
||||
skips: [{ ...unreached, reason: oneLine(unreached.reason) }],
|
||||
notPassed: "no test passed: 1 skipped, 1 of them as NOT REACHED",
|
||||
});
|
||||
});
|
||||
|
||||
// A test that prints a summary-shaped line before it skips must not stand for the runner's summary.
|
||||
test("the runner's last summary is read, not a line a test printed before it", () => {
|
||||
const output = "\u2139 pass 5\n\u2139 fail 0\n\u2139 skipped 0\n\ufe63 genesis adopts its bus (1ms) # NOT REACHED\n" +
|
||||
"\u2139 tests 1\n\u2139 pass 0\n\u2139 fail 0\n\u2139 skipped 1\n";
|
||||
const counts = counted(output);
|
||||
assert.deepEqual(counts, { passed: 0, failed: 0, skipped: 1 });
|
||||
assert.equal(exitOf(0, counts, [unreached]), 1);
|
||||
});
|
||||
|
||||
test("NOT REACHED is matched in any case", () => {
|
||||
const said = judgeRun({ passed: 2, failed: 0 }, [{ ...noBed, reason: "the broker step was not reached" }]);
|
||||
assert.equal(said.passed, false);
|
||||
});
|
||||
|
||||
test("the runner's skipped total is read", () => {
|
||||
assert.deepEqual(counted("ℹ tests 1\nℹ pass 0\nℹ fail 0\nℹ skipped 1\n"), { passed: 0, failed: 0, skipped: 1 });
|
||||
});
|
||||
|
||||
test("a receipt of a run that proved nothing is not current, and says why", () => {
|
||||
const receipt: Receipt = {
|
||||
at: "2026-10-14T00:33:11Z", passed: 0, failed: 0, against: { "mesh-lab": "aaa" }, ran: [endToEnd],
|
||||
skipped: 1, skips: [unreached], notPassed: "no test passed: 1 skipped",
|
||||
};
|
||||
const said = judge(receipt, new Date("2026-10-14T01:00:00Z"), { "mesh-lab": "aaa" });
|
||||
assert.equal(said.current, false, said.lines.join("\n"));
|
||||
const text = said.lines.join("\n");
|
||||
assert.match(text, /did not pass: no test passed: 1 skipped/);
|
||||
assert.match(text, /skipped: genesis adopts its bus — ADR 0333 §5 NOT REACHED/);
|
||||
// An older receipt, from before the fields, with nothing passed, is judged by its count.
|
||||
const older = judge({ at: receipt.at, passed: 0, failed: 0, against: receipt.against, ran: [endToEnd] },
|
||||
new Date("2026-10-14T01:00:00Z"), { "mesh-lab": "aaa" });
|
||||
assert.equal(older.current, false);
|
||||
});
|
||||
|
||||
// The real runner, end to end: the suite command against a file whose only test skips as NOT REACHED,
|
||||
// the way the genesis bed does.
|
||||
test("the suite exits non-zero for a file whose only test skipped as NOT REACHED, and records why", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mesh-lab-693-"));
|
||||
try {
|
||||
const file = join(dir, "unreached.test.ts");
|
||||
writeFileSync(file, [
|
||||
`import { test } from "node:test";`,
|
||||
`test("genesis adopts its bus", (t) => {`,
|
||||
` console.log("a line the bed printed");`,
|
||||
` t.skip("ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base:\\nNotFound: content digest");`,
|
||||
`});`,
|
||||
].join("\n"));
|
||||
const ran = spawnSync(process.execPath, ["--experimental-strip-types", join(lab, "src", "cli.ts"), "suite", "--no-build", file], {
|
||||
cwd: lab, encoding: "utf8", env: suiteEnv(dir),
|
||||
});
|
||||
assert.equal(ran.status, 1, `the suite exited ${ran.status}:\n${ran.stdout}${ran.stderr}`);
|
||||
assert.match(ran.stdout, /0 passed, 0 failed, 1 skipped/);
|
||||
assert.match(ran.stdout, /^NOT PASSED: no test passed: 1 skipped, 1 of them as NOT REACHED$/m);
|
||||
assert.match(ran.stdout, /^skipped: genesis adopts its bus \(.*unreached\.test\.ts\) # ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base: NotFound: content digest$/m);
|
||||
const receipt = JSON.parse(readFileSync(join(dir, "mesh-lab", "last-run.json"), "utf8")) as Receipt;
|
||||
assert.equal(receipt.skipped, 1);
|
||||
assert.equal(receipt.notPassed, "no test passed: 1 skipped, 1 of them as NOT REACHED");
|
||||
assert.match(receipt.skips?.[0]?.reason ?? "", /NOT REACHED/);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("the suite does not take a summary a test printed for the runner's", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mesh-lab-693-"));
|
||||
try {
|
||||
const file = join(dir, "forged.test.ts");
|
||||
writeFileSync(file, [
|
||||
`import { test } from "node:test";`,
|
||||
`test("genesis adopts its bus", (t) => {`,
|
||||
` console.log("\\u2139 pass 5\\n\\u2139 fail 0");`,
|
||||
` t.skip("ADR 0333 §5 NOT REACHED: stopped at step 16");`,
|
||||
`});`,
|
||||
].join("\n"));
|
||||
const ran = spawnSync(process.execPath, ["--experimental-strip-types", join(lab, "src", "cli.ts"), "suite", "--no-build", file], {
|
||||
cwd: lab, encoding: "utf8", env: suiteEnv(dir),
|
||||
});
|
||||
assert.equal(ran.status, 1, `the suite exited ${ran.status}:\n${ran.stdout}${ran.stderr}`);
|
||||
assert.match(ran.stdout, /0 passed, 0 failed, 1 skipped/);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("the suite still exits 0 for a file that passes a test and skips another for a declared reason", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mesh-lab-693-"));
|
||||
try {
|
||||
const file = join(dir, "declared.test.ts");
|
||||
writeFileSync(file, [
|
||||
`import { test } from "node:test";`,
|
||||
`test("a module lands", () => {});`,
|
||||
`test("a builder builds", { skip: "set MESH_LAB_BUILDER to a built mesh-builder" }, () => {});`,
|
||||
].join("\n"));
|
||||
const ran = spawnSync(process.execPath, ["--experimental-strip-types", join(lab, "src", "cli.ts"), "suite", "--no-build", file], {
|
||||
cwd: lab, encoding: "utf8", env: suiteEnv(dir),
|
||||
});
|
||||
assert.equal(ran.status, 0, `the suite exited ${ran.status}:\n${ran.stdout}${ran.stderr}`);
|
||||
assert.match(ran.stdout, /1 passed, 0 failed, 1 skipped/);
|
||||
assert.doesNotMatch(ran.stdout, /NOT PASSED/);
|
||||
assert.match(ran.stdout, /^skipped: a builder builds \(.*declared\.test\.ts\) # set MESH_LAB_BUILDER to a built mesh-builder$/m);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
+7
-6
@@ -5,7 +5,7 @@ import { counted, reportOn } from "../src/suite.ts";
|
||||
// The totals come from the runner's own summary, and from nothing else.
|
||||
test("the runner's totals are read from its summary", () => {
|
||||
const said = counted("✔ something (1ms)\nℹ tests 22\nℹ pass 22\nℹ fail 0\n");
|
||||
assert.deepEqual(said, { passed: 22, failed: 0 });
|
||||
assert.deepEqual(said, { passed: 22, failed: 0, skipped: null });
|
||||
});
|
||||
|
||||
// A test *named* like a total must not be mistaken for one. The summary is a line of its own, and
|
||||
@@ -18,7 +18,7 @@ test("a test named like a total is not a total", () => {
|
||||
// A failing run is read as a failing run.
|
||||
test("failures are read", () => {
|
||||
const said = counted("ℹ pass 21\nℹ fail 1\n");
|
||||
assert.deepEqual(said, { passed: 21, failed: 1 });
|
||||
assert.deepEqual(said, { passed: 21, failed: 1, skipped: null });
|
||||
});
|
||||
|
||||
// **No totals is not zero failures.** A run whose result could not be read is a run nobody can say
|
||||
@@ -36,7 +36,7 @@ test("output with no summary yields no totals rather than a clean bill", () => {
|
||||
// check it — 04-ISSUES/005 in miniature, inside the fix for it.
|
||||
test("a run whose result could not be read writes nothing", () => {
|
||||
let wrote = false;
|
||||
const said = reportOn({ passed: null, failed: null }, () => {
|
||||
const said = reportOn({ passed: null, failed: null }, [], () => {
|
||||
wrote = true;
|
||||
return { against: {}, ran: [] };
|
||||
});
|
||||
@@ -46,12 +46,13 @@ test("a run whose result could not be read writes nothing", () => {
|
||||
|
||||
test("a run that was read is recorded, with what it was read against", () => {
|
||||
let got: [number, number] | null = null;
|
||||
const said = reportOn({ passed: 22, failed: 0 }, (p, f) => {
|
||||
const said = reportOn({ passed: 22, failed: 0 }, [], (p, f) => {
|
||||
got = [p, f];
|
||||
return { against: { "mesh-lab": "abc1234" }, ran: ["test/integration/mesh.test.ts"] };
|
||||
});
|
||||
assert.deepEqual(got, [22, 0]);
|
||||
assert.match(said, /mesh\.test\.ts — 22 passed, 0 failed, against mesh-lab abc1234/);
|
||||
assert.match(said, /mesh\.test\.ts — 22 passed, 0 failed, 0 skipped, against mesh-lab abc1234/);
|
||||
assert.doesNotMatch(said, /NOT PASSED/);
|
||||
});
|
||||
|
||||
// The runner's real output, colours and all.
|
||||
@@ -64,5 +65,5 @@ test("the runner's totals are read from output as it actually arrives", () => {
|
||||
"\u001b[34m\u2139 pass 22\u001b[39m\n" +
|
||||
"\u001b[34m\u2139 fail 0\u001b[39m\n" +
|
||||
"\u001b[34m\u2139 duration_ms 98.9\u001b[39m\n";
|
||||
assert.deepEqual(counted(real), { passed: 22, failed: 0 });
|
||||
assert.deepEqual(counted(real), { passed: 22, failed: 0, skipped: null });
|
||||
});
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { uplinkAddress } from "../src/lifecycle/raise.ts";
|
||||
|
||||
// A workstation behind a VPN client that routes every private range leaves incus nothing to pick, so
|
||||
// the uplink's range may be named; otherwise incus picks it.
|
||||
test("the uplink's range is incus's to pick unless the host names one", () => {
|
||||
assert.equal(uplinkAddress({}), "auto");
|
||||
assert.equal(uplinkAddress({ MESH_LAB_UPLINK_V4: " 192.168.231.1/24 " }), "192.168.231.1/24");
|
||||
});
|
||||
Reference in New Issue
Block a user